Skip to content

optimize retry logic & paimon param in PaimonService #753

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jarad0628 merged 7 commits into from
Apr 23, 2026
Merged

optimize retry logic & paimon param in PaimonService #753

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
aee5224
feat: PaimonService perf Test Case optimize
lebron-hj Apr 10, 2026
67789c6
fix: optimize enableAutoCompact & log in PaimonService
lebron-hj Apr 13, 2026
5b6482b
fix: optimize retry logic in PaimonService
lebron-hj Apr 14, 2026
1cc1736
fix: optimize logic in PaimonService：After paimon is configured with …
lebron-hj Apr 17, 2026
efc3493
fix: gen single test perf project
lebron-hj Apr 21, 2026
ea45513
fix: gen single test perf project
lebron-hj Apr 21, 2026
2dd90dc
fix: delete postpone bucket support
lebron-hj Apr 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
194 changes: 194 additions & 0 deletions connectors/connector-perf-test/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,194 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<groupId>io.tapdata</groupId>
<artifactId>connector-perf-test</artifactId>
<version>1.0-SNAPSHOT</version>
<packaging>jar</packaging>

<name>Paimon Connector Performance Test</name>
<description>Paimon 写入性能参数调优测试 - 独立 Maven 工程</description>

<properties>
<!-- Java 11：DataGeneratorTest 使用了 var (JDK 10+) -->
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<java.version>11</java.version>

<paimon.version>1.3.1</paimon.version>
<hadoop.version>3.3.6</hadoop.version>
<tapdata.pdk.runner.version>2.5-SNAPSHOT</tapdata.pdk.runner.version>
<tapdata.api.version>2.0.6-SNAPSHOT</tapdata.api.version>
<aws.sdk.version>1.12.600</aws.sdk.version>
<mockito.version>4.11.0</mockito.version>
<junit.jupiter.version>5.8.1</junit.jupiter.version>
</properties>

<dependencies>
<!-- Logging -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>2.0.12</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.5.13</version>
</dependency>
<!-- paimon-connector 主工程（需先 mvn install） -->
<dependency>
<groupId>io.tapdata</groupId>
<artifactId>paimon-connector</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>

<!-- PDK Runner：提供 TapRuntime / BeanUtils 等运行时实现 -->
<dependency>
<groupId>io.tapdata</groupId>
<artifactId>tapdata-pdk-runner</artifactId>
<version>${tapdata.pdk.runner.version}</version>
</dependency>

<!-- PDK API -->
<dependency>
<groupId>io.tapdata</groupId>
<artifactId>tapdata-pdk-api</artifactId>
<version>${tapdata.api.version}</version>
</dependency>

<!-- Tapdata API（含 TapCallbackOffset 等，与 paimon-connector 一致） -->
<dependency>
<groupId>io.tapdata</groupId>
<artifactId>tapdata-api</artifactId>
<version>${tapdata.api.version}</version>
</dependency>

<dependency>
<groupId>org.apache.paimon</groupId>
<artifactId>paimon-core</artifactId>
<version>${paimon.version}</version>
</dependency>
<dependency>
<groupId>org.apache.paimon</groupId>
<artifactId>paimon-common</artifactId>
<version>${paimon.version}</version>
</dependency>
<dependency>
<groupId>org.apache.paimon</groupId>
<artifactId>paimon-format</artifactId>
<version>${paimon.version}</version>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion><groupId>log4j</groupId><artifactId>log4j</artifactId></exclusion>
<exclusion><groupId>org.slf4j</groupId><artifactId>slf4j-log4j12</artifactId></exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
</dependency>

<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>${aws.sdk.version}</version>
</dependency>

<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.12.0</version>
</dependency>

<!-- compile scope（非 test）：
PerformanceTestRunner.java:452 在 main() 内直接调用 Mockito.mock()，
exec:java 使用 compile classpath，Mockito 必须在编译期/运行期可见 -->
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>${mockito.version}</version>
</dependency>

<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<version>${junit.jupiter.version}</version>
<scope>test</scope>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.11.0</version>
<configuration>
<source>${java.version}</source>
<target>${java.version}</target>
<encoding>UTF-8</encoding>
</configuration>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.22.2</version>
<configuration>
<!-- S3FileScanDebugTest 无 @Test 注解，排除避免误触发 -->
<excludes>
<exclude>**/S3FileScanDebugTest.java</exclude>
</excludes>
<forkedProcessTimeoutInSeconds>300</forkedProcessTimeoutInSeconds>
</configuration>
</plugin>

<!-- exec:java 使用 compile classpath（源在 src/main/java，无需 test scope hack） -->
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>3.1.0</version>
<configuration>
<mainClass>io.tapdata.connector.paimon.perf.PerformanceTestRunner</mainClass>
<cleanupDaemonThreads>false</cleanupDaemonThreads>
<classpathScope>compile</classpathScope>
</configuration>
</plugin>
</plugins>
</build>

<!-- 从 connectors/pom.xml 复制的仓库配置（standalone 工程无法继承父 POM） -->
<repositories>
<repository>
<id>nexus-releases</id>
<name>nexus-maven-release</name>
<url>https://nexus.tapdata.net/repository/maven-releases/</url>
<releases><enabled>true</enabled></releases>
<snapshots><enabled>false</enabled></snapshots>
</repository>
<repository>
<id>nexus-snapshots</id>
<name>nexus-maven-snapshot</name>
<url>https://nexus.tapdata.net/repository/maven-snapshots/</url>
<releases><enabled>false</enabled></releases>
<snapshots><enabled>true</enabled></snapshots>
</repository>
<repository>
<id>tapdata-tapdata-maven</id>
<url>https://tapdata-maven.pkg.coding.net/repository/tapdata/maven/</url>
<releases><enabled>true</enabled></releases>
<snapshots><enabled>true</enabled></snapshots>
</repository>
</repositories>
</project>
32 changes: 10 additions & 22 deletions connectors/paimon-connector/run-perf-test.sh → ...tors/connector-perf-test/run-perf-test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
#!/bin/bash
################################################################################
# Paimon 写入性能参数调优测试 - 一键启动脚本
#
# Paimon 写入性能参数调优测试 - 一键启动脚本（独立工程版）
#
# 测试模式配置已统一到 TestModeConfig.java
# 新增测试模式只需修改 TestModeConfig.ALL_MODES 列表
# 新增测试模式只需修改 TestModeConfig.java，无需修改此脚本
#
# 用法: ./run-perf-test.sh [模式]
#
Expand Down Expand Up @@ -34,14 +34,6 @@ YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

# ═══════════════════════════════════════════════════════════════════════
# 测试模式配置（与 TestModeConfig.java 保持一致）
# 新增模式请修改 TestModeConfig.java，无需修改此脚本
# ═══════════════════════════════════════════════════════════════════════

# 所有可用的命令行别名（用于帮助提示）
VALID_MODES="basic all nosmallfile single bucket compaction buffer target format pkupdate parallelism auto"

echo -e "${BLUE}"
echo "════════════════════════════════════════════════════════════"
echo " Paimon 1.3.1 写入性能参数调优测试"
Expand All @@ -65,15 +57,15 @@ if [ "$1" = "--help" ] || [ "$1" = "-h" ]; then
echo " 9 format - 文件格式压缩测试(TC-60~64)"
echo " 10 pkupdate - 主键更新测试(TC-70~73)"
echo " 11 parallelism - 写入并行度测试(TC-80~83)"
echo " auto - 全自动运行(无需交互)"
echo " auto - 全自动运行(无需按回车)"
echo ""
echo "默认模式: basic (1)"
exit 0
fi

# ── 环境检查 ──────────────────────────────────────────────────────────────────
if ! command -v java &>/dev/null; then
echo -e "${RED}[ERROR] 未找到 Java，请先安装 JDK 8+${NC}"; exit 1
echo -e "${RED}[ERROR] 未找到 Java，请先安装 JDK 11+${NC}"; exit 1
fi
echo -e "${GREEN}[INFO] Java : $(java -version 2>&1 | head -1)${NC}"

Expand All @@ -86,15 +78,15 @@ echo ""
# ── 获取测试模式 ────────────────────────────────────────────────────────────
TEST_MODE="${1:-}"

# ── 编译主工程 ──────────────────────────────────────────────────────────────
# ── 编译 paimon-connector 主工程（兄弟目录） ──────────────────────────────
echo -e "${YELLOW}[1/3] 编译 paimon-connector 主工程...${NC}"
mvn clean install -DskipTests -q -f pom.xml
mvn clean install -DskipTests -q -f ../paimon-connector/pom.xml
echo -e "${GREEN} 主工程编译完成${NC}"
echo ""

# ── 编译测试代码 ────────────────────────────────────────────────────────────
# ── 编译性能测试代码（src/main/java，标准 compile goal） ────────────────────
echo -e "${YELLOW}[2/3] 编译性能测试代码...${NC}"
mvn test-compile -q -f pom-perf-test.xml
mvn compile -q
echo -e "${GREEN} 测试代码编译完成${NC}"
echo ""

Expand All @@ -108,13 +100,11 @@ JVM_OPTS="-Xmx4g -Xms512m -XX:+UseG1GC"
if [ -n "$TEST_MODE" ]; then
echo -e "${BLUE}>> 测试模式: ${TEST_MODE}${NC}"
mvn exec:java \
-f pom-perf-test.xml \
-Dexec.mainClass="io.tapdata.connector.paimon.perf.PerformanceTestRunner" \
-Dexec.cleanupDaemonThreads=false \
-Dexec.args="${TEST_MODE}" \
-Dexec.jvmArgs="${JVM_OPTS}"
else
# 交互式菜单（与 TestModeConfig.java 保持一致）
# 交互式菜单
echo -e "${BLUE}请选择测试模式 (直接回车默认 1):${NC}"
echo " 1 basic - 基础用例组(TC-01~03)"
echo " 2 all - 全量测试(所有组)"
Expand All @@ -134,8 +124,6 @@ else
echo ""
echo -e "${BLUE}>> 测试模式: ${CHOICE}${NC}"
mvn exec:java \
-f pom-perf-test.xml \
-Dexec.mainClass="io.tapdata.connector.paimon.perf.PerformanceTestRunner" \
-Dexec.cleanupDaemonThreads=false \
-Dexec.args="${CHOICE}" \
-Dexec.jvmArgs="${JVM_OPTS}"
Expand Down
0 ...connector/paimon/perf/AutoTestRunner.java → ...connector/paimon/perf/AutoTestRunner.java
View file
Open in desktop
File renamed without changes.
Loading