From 0539251573d7652057743c1f783f178ef2bbc1d0 Mon Sep 17 00:00:00 2001 From: xylaaaaa <2392805527@qq.com> Date: Fri, 22 May 2026 13:08:45 +0800 Subject: [PATCH] [fix](regression) Enable Paimon GCS regression ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: Re-enable test_paimon_gcs in the external regression pipeline, configure Paimon to use the GCS filesystem provider, and avoid generating invalid catalog properties such as gs.endpoint='null'. The external pipeline now supplies the standard GCS endpoint and region while credentials remain provided by CI secret configuration. ### Release note None ### Check List (For Author) - Test: Regression test / Manual test - Regression test: ./run-regression-test.sh --run --conf regression-test/conf/regression-conf.groovy -d external_table_p0/paimon -s test_paimon_gcs -c "jdbc:mysql://127.0.0.1:9030/?useLocalSessionState=true&allowLoadLocalInfile=true&zeroDateTimeBehavior=round" -conf enablePaimonTest=true. This is a real run, not dry-run; it failed at show databases with NoAuthWithAWSException because local GCSAk/GCSSk are not available, and verified the generated catalog no longer contains 'null' endpoint/credential values. - Regression test: ./run-regression-test.sh --run --conf regression-test/pipeline/external/conf/regression-conf.groovy -d external_table_p0/paimon -s test_paimon_gcs -c "jdbc:mysql://127.0.0.1:9030/?useLocalSessionState=true&allowLoadLocalInfile=true&zeroDateTimeBehavior=round" -ha 127.0.0.1:8030 -conf GCSAk=dummy -conf GCSSk=dummy. This is a real run, not dry-run; it failed with GCS 403 Forbidden, confirming the case reaches storage.googleapis.com with credentials. - Behavior changed: No - Does this need documentation: No --- .../external/conf/regression-conf.groovy | 3 +- .../paimon/test_paimon_gcs.groovy | 33 +++++++++++-------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/regression-test/pipeline/external/conf/regression-conf.groovy b/regression-test/pipeline/external/conf/regression-conf.groovy index 281085bce960a7..1aca8c4866f411 100644 --- a/regression-test/pipeline/external/conf/regression-conf.groovy +++ b/regression-test/pipeline/external/conf/regression-conf.groovy @@ -75,7 +75,6 @@ excludeSuites = "000_the_start_sentinel_do_not_touch," + // keep this line as th "test_profile," + "test_refresh_mtmv," + "test_spark_load," + - "test_paimon_gcs," + "test_broker_load_func," + "test_stream_stub_fault_injection," + "test_iceberg_overwrite_with_wrong_partition," + @@ -183,6 +182,8 @@ max_failure_num=50 externalEnvIp="127.0.0.1" enablePaimonTest=true +GCSEndpoint="https://storage.googleapis.com" +GCSRegion="us-east1" hdfs_port=8020 diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_gcs.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_gcs.groovy index 807ad6c8f6ec90..945bd28176a187 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_gcs.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_gcs.groovy @@ -26,15 +26,29 @@ suite("test_paimon_gcs", "p0,external") { String table_name = "hive_test_table" for (String propertyPrefix : ["gs", "s3"]) { + def catalogProperties = { boolean withRegion -> + def properties = [ + "'type' = 'paimon'", + "'warehouse' = '${gcs_warehouse}'", + "'fs.gcs.support' = 'true'", + ] + def optionalProperties = [ + [gcs_endpoint, "'${propertyPrefix}.endpoint' = '${gcs_endpoint}'"], + [gcs_ak, "'${propertyPrefix}.access_key' = '${gcs_ak}'"], + [gcs_sk, "'${propertyPrefix}.secret_key' = '${gcs_sk}'"], + ] + if (withRegion) { + optionalProperties.add([gcs_region, "'${propertyPrefix}.region' = '${gcs_region}'"]) + } + properties.addAll(optionalProperties.findAll { it[0] != null && !it[0].trim().isEmpty() } + .collect { it[1] }) + properties.join(",\n") + } def catalog_name = "test_paimon_gcs_${propertyPrefix}" sql """drop catalog if exists ${catalog_name}""" sql """ CREATE CATALOG ${catalog_name} PROPERTIES ( - 'type' = 'paimon', - 'warehouse' = '${gcs_warehouse}', - '${propertyPrefix}.endpoint' = '${gcs_endpoint}', - '${propertyPrefix}.access_key' = '${gcs_ak}', - '${propertyPrefix}.secret_key' = '${gcs_sk}' + ${catalogProperties(false)} ); """ sql """switch `${catalog_name}`""" @@ -55,12 +69,7 @@ suite("test_paimon_gcs", "p0,external") { sql """drop catalog if exists ${catalog_name}_with_region""" sql """ CREATE CATALOG ${catalog_name}_with_region PROPERTIES ( - 'type' = 'paimon', - 'warehouse' = '${gcs_warehouse}', - '${propertyPrefix}.endpoint' = '${gcs_endpoint}', - '${propertyPrefix}.access_key' = '${gcs_ak}', - '${propertyPrefix}.secret_key' = '${gcs_sk}', - '${propertyPrefix}.region' = '${gcs_region}' + ${catalogProperties(true)} ); """ sql """switch `${catalog_name}_with_region`""" @@ -80,5 +89,3 @@ suite("test_paimon_gcs", "p0,external") { } } } - -