diff --git a/.github/workflows/core-hadoop3-ci.yml b/.github/workflows/core-hadoop3-ci.yml index e93a32aa4f..004f179afe 100644 --- a/.github/workflows/core-hadoop3-ci.yml +++ b/.github/workflows/core-hadoop3-ci.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - jdk: [ '11' ] + jdk: [ '11', '17' ] spark: [ '3.3','3.4', '3.5' ] name: Build Amoro with JDK ${{ matrix.jdk }} Spark-${{ matrix.spark }} steps: diff --git a/README.md b/README.md index 23edb7c712..0d8572488f 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ Amoro contains modules as below: ## Building -Amoro is built using Maven with JDK 8, 11 and 17(required for `amoro-format-mixed/amoro-mixed-trino` module). +Amoro is built using Maven with JDK 11 and 17(required for `amoro-format-mixed/amoro-mixed-trino` module, experimental for other modules). * Build all modules without `amoro-mixed-trino`: `./mvnw clean package` * Build and skip tests: `./mvnw clean package -DskipTests` diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/flow/TestKeyedContinuousOptimizing.java b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/flow/TestKeyedContinuousOptimizing.java index a1f3c2c20e..acdacbb9a0 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/flow/TestKeyedContinuousOptimizing.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/flow/TestKeyedContinuousOptimizing.java @@ -77,92 +77,102 @@ public static Object[] parameters() { @Test public void run() throws Exception { - MixedTable table = getMixedTable(); - - int partitionCount = 2; - int primaryUpperBound = 30000; - - long writeTargetFileSize = 1024 * 12; - long selfTargetFileSize = table.format() == TableFormat.ICEBERG ? 1024 * 384 : 1024 * 128; - int minorTriggerCount = table.format() == TableFormat.ICEBERG ? 3 : 4; - int availableCore = 10; - - int cycle = 5; - int recordCountOnceWrite = 2500; - - // close full optimize - table.updateProperties().set(SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL, "-1").commit(); - // Need move file to hive scene - table.updateProperties().set(SELF_OPTIMIZING_FULL_REWRITE_ALL_FILES, "false").commit(); - - KeyedTableDataView view = - new KeyedTableDataView( - table, - tableTestHelper().primaryKeySpec().getPkSchema(), - partitionCount, - primaryUpperBound, - writeTargetFileSize, - null); - - // init checker - DataConcurrencyChecker dataConcurrencyChecker = new DataConcurrencyChecker(view); - OptimizingCountChecker optimizingCountChecker = new OptimizingCountChecker(0); - FullOptimizingWrite2HiveChecker fullOptimizingWrite2HiveChecker = - new FullOptimizingWrite2HiveChecker(view); - FullOptimizingMove2HiveChecker fullOptimizingMove2HiveChecker = - new FullOptimizingMove2HiveChecker(view); - MinorOptimizingCheck minorOptimizingCheck = new MinorOptimizingCheck(); - - CompleteOptimizingFlow.Builder builder = - CompleteOptimizingFlow.builder(table, availableCore) - .setTargetSize(selfTargetFileSize) - .setFragmentRatio(null) - .setDuplicateRatio(null) - .setMinorTriggerFileCount(minorTriggerCount) - .addChecker(dataConcurrencyChecker) - .addChecker(optimizingCountChecker) - .addChecker(minorOptimizingCheck); - - if (table.format() == TableFormat.MIXED_HIVE) { - builder - .addChecker(fullOptimizingWrite2HiveChecker) - .addChecker(fullOptimizingMove2HiveChecker); - } - - CompleteOptimizingFlow optimizingFlow = builder.build(); - - // full optimizing need move file to hive from change - view.append(recordCountOnceWrite); - mustFullCycle(table, optimizingFlow::optimize); + // Save the original value to restore later + long originalFilterEqDeleteTriggerRecordCount = + CombinedDeleteFilter.FILTER_EQ_DELETE_TRIGGER_RECORD_COUNT; + + try { + MixedTable table = getMixedTable(); + + int partitionCount = 2; + int primaryUpperBound = 30000; + + long writeTargetFileSize = 1024 * 12; + long selfTargetFileSize = table.format() == TableFormat.ICEBERG ? 1024 * 384 : 1024 * 128; + int minorTriggerCount = table.format() == TableFormat.ICEBERG ? 3 : 4; + int availableCore = 10; + + int cycle = 5; + int recordCountOnceWrite = 2500; + + // close full optimize + table.updateProperties().set(SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL, "-1").commit(); + // Need move file to hive scene + table.updateProperties().set(SELF_OPTIMIZING_FULL_REWRITE_ALL_FILES, "false").commit(); + + KeyedTableDataView view = + new KeyedTableDataView( + table, + tableTestHelper().primaryKeySpec().getPkSchema(), + partitionCount, + primaryUpperBound, + writeTargetFileSize, + null); + + // init checker + DataConcurrencyChecker dataConcurrencyChecker = new DataConcurrencyChecker(view); + OptimizingCountChecker optimizingCountChecker = new OptimizingCountChecker(0); + FullOptimizingWrite2HiveChecker fullOptimizingWrite2HiveChecker = + new FullOptimizingWrite2HiveChecker(view); + FullOptimizingMove2HiveChecker fullOptimizingMove2HiveChecker = + new FullOptimizingMove2HiveChecker(view); + MinorOptimizingCheck minorOptimizingCheck = new MinorOptimizingCheck(); + + CompleteOptimizingFlow.Builder builder = + CompleteOptimizingFlow.builder(table, availableCore) + .setTargetSize(selfTargetFileSize) + .setFragmentRatio(null) + .setDuplicateRatio(null) + .setMinorTriggerFileCount(minorTriggerCount) + .addChecker(dataConcurrencyChecker) + .addChecker(optimizingCountChecker) + .addChecker(minorOptimizingCheck); + + if (table.format() == TableFormat.MIXED_HIVE) { + builder + .addChecker(fullOptimizingWrite2HiveChecker) + .addChecker(fullOptimizingMove2HiveChecker); + } - view.append(recordCountOnceWrite); - optimizingFlow.optimize(); + CompleteOptimizingFlow optimizingFlow = builder.build(); - // full optimizing need move file to hive from change and base - view.append(recordCountOnceWrite); - mustFullCycle(table, optimizingFlow::optimize); + // full optimizing need move file to hive from change + view.append(recordCountOnceWrite); + mustFullCycle(table, optimizingFlow::optimize); - while (cycle-- > 0) { - view.onlyDelete(recordCountOnceWrite); + view.append(recordCountOnceWrite); optimizingFlow.optimize(); - view.cdc(recordCountOnceWrite); - optimizingFlow.optimize(); + // full optimizing need move file to hive from change and base + view.append(recordCountOnceWrite); + mustFullCycle(table, optimizingFlow::optimize); - view.upsert(recordCountOnceWrite); - if (cycle % 2 == 0) { - // Trigger BloomFilter - CombinedDeleteFilter.FILTER_EQ_DELETE_TRIGGER_RECORD_COUNT = 2499L; + while (cycle-- > 0) { + view.onlyDelete(recordCountOnceWrite); + optimizingFlow.optimize(); - mustFullCycle(table, optimizingFlow::optimize); - } else { + view.cdc(recordCountOnceWrite); optimizingFlow.optimize(); + + view.upsert(recordCountOnceWrite); + if (cycle % 2 == 0) { + // Trigger BloomFilter + CombinedDeleteFilter.FILTER_EQ_DELETE_TRIGGER_RECORD_COUNT = 2499L; + + mustFullCycle(table, optimizingFlow::optimize); + } else { + optimizingFlow.optimize(); + } } - } - List checkers = optimizingFlow.unTriggerChecker(); - if (checkers.size() != 0) { - throw new IllegalStateException("Some checkers are not triggered:" + checkers); + List checkers = optimizingFlow.unTriggerChecker(); + if (!checkers.isEmpty()) { + throw new IllegalStateException("Some checkers are not triggered:" + checkers); + } + } finally { + // Restore the original value to avoid polluting other tests + CombinedDeleteFilter.FILTER_EQ_DELETE_TRIGGER_RECORD_COUNT = + originalFilterEqDeleteTriggerRecordCount; } } diff --git a/amoro-common/pom.xml b/amoro-common/pom.xml index 2b21ff9997..d08d9c6877 100644 --- a/amoro-common/pom.xml +++ b/amoro-common/pom.xml @@ -194,6 +194,11 @@ + + + org.apache.maven.plugins + maven-surefire-plugin + ${basedir}/src/main/java diff --git a/amoro-common/src/test/java/org/apache/amoro/testutils/FailsOnJava17.java b/amoro-common/src/test/java/org/apache/amoro/testutils/FailsOnJava17.java new file mode 100644 index 0000000000..8e9b2ed593 --- /dev/null +++ b/amoro-common/src/test/java/org/apache/amoro/testutils/FailsOnJava17.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.testutils; + +/** + * Marker interface for tests that fail on Java 17. + * + *

Tests annotated with this interface will be excluded when running on Java 17. Use this for + * tests that depend on internal JDK APIs that have been removed or restricted in Java 17. + * + *

Usage: + * + * 

{@code
+ * @Category(FailsOnJava17.class)
+ * public class MyTest {
+ *     // test code
+ * }
+ * }
+ */ +public interface FailsOnJava17 {} diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/pom.xml b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/pom.xml index 51f31421df..a98ef5482c 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/pom.xml +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/pom.xml @@ -325,6 +325,14 @@ test + + org.apache.amoro + amoro-common + ${project.version} + test-jar + test + + org.apache.amoro amoro-mixed-hive @@ -425,7 +433,21 @@ org.apache.amoro.listener.AmoroRunListener - -verbose:class + ${surefire.excludedGroups.jdk} + -verbose:class + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.lang.invoke=ALL-UNNAMED + --add-opens=java.base/java.lang.reflect=ALL-UNNAMED + --add-opens=java.base/java.io=ALL-UNNAMED + --add-opens=java.base/java.net=ALL-UNNAMED + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED + --add-opens=java.base/sun.nio.cs=ALL-UNNAMED + --add-opens=java.base/sun.security.action=ALL-UNNAMED + --add-opens=java.base/sun.util.calendar=ALL-UNNAMED diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/TestFlinkSource.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/TestFlinkSource.java index ef5d08a68b..2af22bc1c8 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/TestFlinkSource.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/TestFlinkSource.java @@ -29,6 +29,7 @@ import org.apache.amoro.flink.table.FlinkSource; import org.apache.amoro.flink.table.MixedFormatTableLoader; import org.apache.amoro.flink.util.DataUtil; +import org.apache.amoro.testutils.FailsOnJava17; import org.apache.flink.configuration.Configuration; import org.apache.flink.core.execution.JobClient; import org.apache.flink.streaming.api.CheckpointingMode; @@ -49,6 +50,7 @@ import org.apache.iceberg.io.WriteResult; import org.junit.Assert; import org.junit.Test; +import org.junit.experimental.categories.Category; import java.io.IOException; import java.time.LocalDateTime; @@ -63,6 +65,7 @@ import java.util.Optional; import java.util.Set; +@Category(FailsOnJava17.class) public class TestFlinkSource extends FlinkTestBase { protected static final FileFormat FILE_FORMAT = diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/TestMixedFormatSource.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/TestMixedFormatSource.java index ecdf4c47e9..78d48b7702 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/TestMixedFormatSource.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/read/TestMixedFormatSource.java @@ -41,6 +41,7 @@ import org.apache.amoro.table.MixedTable; import org.apache.amoro.table.TableIdentifier; import org.apache.amoro.table.UnkeyedTable; +import org.apache.amoro.testutils.FailsOnJava17; import org.apache.amoro.utils.TableFileUtil; import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.JobStatus; @@ -86,6 +87,7 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; +import org.junit.experimental.categories.Category; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -104,6 +106,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; +@Category(FailsOnJava17.class) public class TestMixedFormatSource extends TestRowDataReaderFunction implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(TestMixedFormatSource.class); private static final long serialVersionUID = 7418812854449034756L; diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestKeyed.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestKeyed.java index 05fc24eb23..4fdee6b75c 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestKeyed.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestKeyed.java @@ -38,6 +38,7 @@ import org.apache.amoro.hive.catalog.HiveCatalogTestHelper; import org.apache.amoro.hive.catalog.HiveTableTestHelper; import org.apache.amoro.table.TableProperties; +import org.apache.amoro.testutils.FailsOnJava17; import org.apache.commons.collections.CollectionUtils; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.table.api.ApiExpression; @@ -57,6 +58,7 @@ import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; +import org.junit.experimental.categories.Category; import org.junit.rules.TemporaryFolder; import org.junit.rules.TestName; import org.junit.runner.RunWith; @@ -77,6 +79,7 @@ import java.util.Set; @RunWith(Parameterized.class) +@Category(FailsOnJava17.class) public class TestKeyed extends FlinkTestBase { public static final Logger LOG = LoggerFactory.getLogger(TestKeyed.class); diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestLookupSecondary.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestLookupSecondary.java index 659f4e955f..80b8af79d2 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestLookupSecondary.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestLookupSecondary.java @@ -26,6 +26,7 @@ import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; import org.apache.amoro.table.MixedTable; import org.apache.amoro.table.TableIdentifier; +import org.apache.amoro.testutils.FailsOnJava17; import org.apache.commons.lang3.ArrayUtils; import org.apache.flink.table.api.TableResult; import org.apache.flink.table.data.RowData; @@ -39,6 +40,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.junit.experimental.categories.Category; import java.io.IOException; import java.util.ArrayList; @@ -48,6 +50,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +@Category(FailsOnJava17.class) public class TestLookupSecondary extends CatalogITCaseBase implements FlinkTaskWriterBaseTest { private String db; diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestUnkeyed.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestUnkeyed.java index ccab05a761..80e3c4898a 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestUnkeyed.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestUnkeyed.java @@ -39,6 +39,7 @@ import org.apache.amoro.mixed.MixedFormatCatalog; import org.apache.amoro.table.MixedTable; import org.apache.amoro.table.TableIdentifier; +import org.apache.amoro.testutils.FailsOnJava17; import org.apache.flink.table.api.ApiExpression; import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.Table; @@ -57,6 +58,7 @@ import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; +import org.junit.experimental.categories.Category; import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -74,6 +76,7 @@ import java.util.Set; @RunWith(Parameterized.class) +@Category(FailsOnJava17.class) public class TestUnkeyed extends FlinkTestBase { @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestUnkeyedOverwrite.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestUnkeyedOverwrite.java index fcb092e3d6..2f762adf06 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestUnkeyedOverwrite.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/table/TestUnkeyedOverwrite.java @@ -28,6 +28,7 @@ import org.apache.amoro.hive.TestHMS; import org.apache.amoro.hive.catalog.HiveCatalogTestHelper; import org.apache.amoro.hive.catalog.HiveTableTestHelper; +import org.apache.amoro.testutils.FailsOnJava17; import org.apache.flink.table.api.ApiExpression; import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.Table; @@ -38,6 +39,7 @@ import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; +import org.junit.experimental.categories.Category; import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -49,6 +51,7 @@ import java.util.List; @RunWith(Parameterized.class) +@Category(FailsOnJava17.class) public class TestUnkeyedOverwrite extends FlinkTestBase { private static final Logger LOGGER = LoggerFactory.getLogger(TestUnkeyedOverwrite.class); diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestAdaptHiveWriter.java b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestAdaptHiveWriter.java index 057765df1c..f6cc6b0212 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestAdaptHiveWriter.java +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/write/TestAdaptHiveWriter.java @@ -36,6 +36,7 @@ import org.apache.amoro.table.LocationKind; import org.apache.amoro.table.MixedTable; import org.apache.amoro.table.WriteOperationKind; +import org.apache.amoro.testutils.FailsOnJava17; import org.apache.flink.table.data.DecimalData; import org.apache.flink.table.data.GenericRowData; import org.apache.flink.table.data.RowData; @@ -54,6 +55,7 @@ import org.junit.Assume; import org.junit.ClassRule; import org.junit.Test; +import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -69,6 +71,7 @@ import java.util.stream.Collectors; @RunWith(Parameterized.class) +@Category(FailsOnJava17.class) public class TestAdaptHiveWriter extends TableTestBase { @ClassRule public static TestHMS TEST_HMS = new TestHMS(); diff --git a/amoro-format-paimon/pom.xml b/amoro-format-paimon/pom.xml index 87b29c4282..ff98e9559f 100644 --- a/amoro-format-paimon/pom.xml +++ b/amoro-format-paimon/pom.xml @@ -78,4 +78,13 @@ + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + diff --git a/dev/deps/dependencies-hadoop-2-spark-3.3 b/dev/deps/dependencies-hadoop-2-spark-3.3 index 039fb92ac3..26c19a3b29 100644 --- a/dev/deps/dependencies-hadoop-2-spark-3.3 +++ b/dev/deps/dependencies-hadoop-2-spark-3.3 @@ -383,7 +383,7 @@ simpleclient_httpserver/0.16.0//simpleclient_httpserver-0.16.0.jar simpleclient_tracer_common/0.16.0//simpleclient_tracer_common-0.16.0.jar simpleclient_tracer_otel/0.16.0//simpleclient_tracer_otel-0.16.0.jar simpleclient_tracer_otel_agent/0.16.0//simpleclient_tracer_otel_agent-0.16.0.jar -slf4j-api/1.7.30//slf4j-api-1.7.30.jar +slf4j-api/2.0.17//slf4j-api-2.0.17.jar snakeyaml-engine/2.7//snakeyaml-engine-2.7.jar snakeyaml/2.2//snakeyaml-2.2.jar snappy-java/1.1.8.4//snappy-java-1.1.8.4.jar diff --git a/dev/deps/dependencies-hadoop-3-spark-3.5 b/dev/deps/dependencies-hadoop-3-spark-3.5 index 58fa40e409..0236e197ee 100644 --- a/dev/deps/dependencies-hadoop-3-spark-3.5 +++ b/dev/deps/dependencies-hadoop-3-spark-3.5 @@ -349,7 +349,7 @@ simpleclient_httpserver/0.16.0//simpleclient_httpserver-0.16.0.jar simpleclient_tracer_common/0.16.0//simpleclient_tracer_common-0.16.0.jar simpleclient_tracer_otel/0.16.0//simpleclient_tracer_otel-0.16.0.jar simpleclient_tracer_otel_agent/0.16.0//simpleclient_tracer_otel_agent-0.16.0.jar -slf4j-api/1.7.30//slf4j-api-1.7.30.jar +slf4j-api/2.0.17//slf4j-api-2.0.17.jar snakeyaml-engine/2.7//snakeyaml-engine-2.7.jar snakeyaml/2.2//snakeyaml-2.2.jar snappy-java/1.1.10.5//snappy-java-1.1.10.5.jar diff --git a/pom.xml b/pom.xml index 641407b88e..19e25f9670 100644 --- a/pom.xml +++ b/pom.xml @@ -112,7 +112,7 @@ 2.12.15 2.12 2.33 - 1.7.30 + 2.0.17 2.20.0 4.13.2 5.9.1 @@ -168,6 +168,26 @@ compile compile provided + + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.lang.invoke=ALL-UNNAMED + --add-opens=java.base/java.lang.reflect=ALL-UNNAMED + --add-opens=java.base/java.io=ALL-UNNAMED + --add-opens=java.base/java.net=ALL-UNNAMED + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED + --add-opens=java.base/sun.nio.cs=ALL-UNNAMED + --add-opens=java.base/sun.security.action=ALL-UNNAMED + --add-opens=java.base/sun.util.calendar=ALL-UNNAMED + + --add-exports=java.base/sun.net.util=ALL-UNNAMED + --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED + --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED + + -XX:+UseG1GC -Xms256m -XX:+IgnoreUnrecognizedVMOptions ${jvm.module.opens} @@ -1480,6 +1500,8 @@ ${maven-surefire-plugin.version} false + ${amoro.surefire.baseArgLine} + ${surefire.excludedGroups.jdk} @@ -1853,12 +1875,53 @@ java11 - [11,) + [11,17) 11 + + java17 + + 17 + + + 17 + 17 + org.apache.amoro.testutils.FailsOnJava17 + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + --add-exports=java.base/sun.net.util=ALL-UNNAMED + --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED + --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 17 + 17 + + --add-exports=java.base/sun.net.util=ALL-UNNAMED + --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED + --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED + + + + + + + spark-3.3