From 878bd71256b66bfad13ee3e4160995073a2782e1 Mon Sep 17 00:00:00 2001 From: shangeyao Date: Wed, 1 Jul 2026 11:07:43 +0800 Subject: [PATCH 1/4] [Build] Raise StreamPark compile and CI baseline to JDK 11 Set Maven release 11, update Spotless compliance, switch Docker and CI workflows to JDK 11, and align module compiler overrides. Closes #4409 (part 3/3). Ref #4410. Generated-by: Cursor Co-authored-by: Cursor --- .github/workflows/backend.yml | 2 +- .github/workflows/docker-push.yml | 2 +- .github/workflows/e2e.yml | 4 +-- .github/workflows/unit-test.yml | 2 +- docker/Dockerfile | 4 +-- pom.xml | 25 ++++++++++++++++--- streampark-e2e/pom.xml | 4 +-- .../pom.xml | 4 +-- .../pom.xml | 4 +-- .../pom.xml | 4 +-- .../streampark-spark-cli/create_app.sh | 2 +- .../spotless_streampark_formatter.xml | 6 ++--- 12 files changed, 40 insertions(+), 23 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 878e12661a..d8de19686d 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -80,7 +80,7 @@ jobs: strategy: fail-fast: false matrix: - java: [ 8 , 11 ] + java: [ 11 ] steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/docker-push.yml b/.github/workflows/docker-push.yml index c7172b25db..75f643f8eb 100644 --- a/.github/workflows/docker-push.yml +++ b/.github/workflows/docker-push.yml @@ -70,7 +70,7 @@ jobs: - name: Setup Java and Scala uses: olafurpg/setup-scala@v13 with: - java-version: adopt@1.8 + java-version: adopt@11 - uses: actions/setup-node@v3 with: diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 5e87c7ebd0..db04e73429 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -77,10 +77,10 @@ jobs: - uses: actions/checkout@v4 with: submodules: true - - name: Set up JDK 8 + - name: Set up JDK 11 uses: actions/setup-java@v4 with: - java-version: 8 + java-version: 11 distribution: 'adopt' - name: Cache local Maven repository uses: actions/cache@v4 diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 4d0a9fa64e..2cc58868de 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -80,7 +80,7 @@ jobs: strategy: fail-fast: false matrix: - java: [ 8 , 11 ] + java: [ 11 ] steps: - name: Checkout uses: actions/checkout@v4 diff --git a/docker/Dockerfile b/docker/Dockerfile index ac5fca4f4f..f3fc10083f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -19,10 +19,10 @@ FROM ubuntu:22.04 USER root ENV LANG=C.UTF-8 \ - JAVA_HOME=/usr/lib/jvm/jdk8 + JAVA_HOME=/usr/lib/jvm/jdk11 # Build arguments for version management -ARG JAVA_MAJOR_VERSION=8 +ARG JAVA_MAJOR_VERSION=11 ARG TINI_VERSION=v0.19.0 # Base system setup diff --git a/pom.xml b/pom.xml index 8f5103008b..68b6b536f9 100644 --- a/pom.xml +++ b/pom.xml @@ -82,7 +82,7 @@ - 1.8 + 11 UTF-8 UTF-8 @@ -697,8 +697,7 @@ maven-compiler-plugin ${maven-compiler-plugin.version} - ${project.build.jdk} - ${project.build.jdk} + ${project.build.jdk} UTF-8 false @@ -715,7 +714,7 @@ scala-maven-plugin ${scala-maven-plugin.version} - -target:jvm-${project.build.jdk} + -target:${project.build.jdk} ${project.build.jdk} ${project.build.jdk} @@ -1009,6 +1008,24 @@ true + + + jdk9-plus-test-opens + + [9,) + + + + + org.apache.maven.plugins + maven-surefire-plugin + + --add-opens java.base/jdk.internal.loader=ALL-UNNAMED --add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED + + + + + diff --git a/streampark-e2e/pom.xml b/streampark-e2e/pom.xml index 6e601d388b..a41604694d 100644 --- a/streampark-e2e/pom.xml +++ b/streampark-e2e/pom.xml @@ -30,8 +30,8 @@ - 8 - 8 + 11 + 11 UTF-8 5.8.1 diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/pom.xml b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/pom.xml index 23a84158a3..3c1659a503 100644 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/pom.xml +++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/pom.xml @@ -30,8 +30,8 @@ StreamPark : Flink Connector Elasticsearch5 - 8 - 8 + 11 + 11 1.14.0 diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/pom.xml b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/pom.xml index 0d944d1e62..546e52abf3 100644 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/pom.xml +++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/pom.xml @@ -30,8 +30,8 @@ StreamPark : Flink Connector Elasticsearch6 - 8 - 8 + 11 + 11 diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/pom.xml b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/pom.xml index 53bde3349c..d1c7195be0 100644 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/pom.xml +++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/pom.xml @@ -29,8 +29,8 @@ StreamPark : Flink Connector Elasticsearch7 - 8 - 8 + 11 + 11 diff --git a/streampark-spark/streampark-spark-cli/create_app.sh b/streampark-spark/streampark-spark-cli/create_app.sh index 1de938e828..998811ee5d 100644 --- a/streampark-spark/streampark-spark-cli/create_app.sh +++ b/streampark-spark/streampark-spark-cli/create_app.sh @@ -281,7 +281,7 @@ cat > $name/pom.xml <UTF-8 UTF-8 - 1.8 + 11 64m diff --git a/tools/checkstyle/spotless_streampark_formatter.xml b/tools/checkstyle/spotless_streampark_formatter.xml index f05ccaeb43..12b28b6b63 100644 --- a/tools/checkstyle/spotless_streampark_formatter.xml +++ b/tools/checkstyle/spotless_streampark_formatter.xml @@ -20,9 +20,9 @@ --> - - - + + + From 2b20fab3bc6d18fc2e6d3a05279c00560cb1b1f2 Mon Sep 17 00:00:00 2001 From: shangeyao Date: Wed, 1 Jul 2026 11:06:31 +0800 Subject: [PATCH 2/4] [Common] Fix ClassLoaderUtils dynamic classpath on JDK 9+ Walk the class hierarchy to locate the ucp field on JDK 11+ app classloaders, add JUnit coverage, and enable surefire add-opens on JDK 9+. Closes #4409 (part 1/3). Ref #4410. Generated-by: Cursor Co-authored-by: Cursor --- streampark-common/pom.xml | 20 +++++++ .../common/util/ClassLoaderUtils.scala | 17 ++++-- .../common/util/ClassLoaderUtilsTest.scala | 53 +++++++++++++++++++ 3 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 streampark-common/src/test/scala/org/apache/streampark/common/util/ClassLoaderUtilsTest.scala diff --git a/streampark-common/pom.xml b/streampark-common/pom.xml index 115477f4aa..03f4211b32 100644 --- a/streampark-common/pom.xml +++ b/streampark-common/pom.xml @@ -197,4 +197,24 @@ + + + + jdk9-plus-test-opens + + [9,) + + + + + org.apache.maven.plugins + maven-surefire-plugin + + --add-opens java.base/jdk.internal.loader=ALL-UNNAMED --add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED + + + + + + diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/ClassLoaderUtils.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/ClassLoaderUtils.scala index e052de96c3..e1664dc859 100644 --- a/streampark-common/src/main/scala/org/apache/streampark/common/util/ClassLoaderUtils.scala +++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/ClassLoaderUtils.scala @@ -146,9 +146,20 @@ object ClassLoaderUtils extends Logger { addURL.setAccessible(true) addURL.invoke(c, file.toURI.toURL) case _ => - val field = classLoader.getClass.getDeclaredField("ucp") - field.setAccessible(true) - val ucp = field.get(classLoader) + var clazz: Class[_] = classLoader.getClass + var ucpField: java.lang.reflect.Field = null + while (clazz != null && ucpField == null) { + try { + ucpField = clazz.getDeclaredField("ucp") + } catch { + case _: NoSuchFieldException => clazz = clazz.getSuperclass + } + } + require( + ucpField != null, + "[StreamPark] ClassLoaderUtils.addURL: cannot locate ucp field on classloader chain") + ucpField.setAccessible(true) + val ucp = ucpField.get(classLoader) val addURL = ucp.getClass.getDeclaredMethod("addURL", Array(classOf[URL]): _*) addURL.setAccessible(true) diff --git a/streampark-common/src/test/scala/org/apache/streampark/common/util/ClassLoaderUtilsTest.scala b/streampark-common/src/test/scala/org/apache/streampark/common/util/ClassLoaderUtilsTest.scala new file mode 100644 index 0000000000..f4193e154a --- /dev/null +++ b/streampark-common/src/test/scala/org/apache/streampark/common/util/ClassLoaderUtilsTest.scala @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util + +import org.junit.jupiter.api.{Assertions, Test} + +import java.io.{File, FileOutputStream} +import java.util.jar.{JarEntry, JarOutputStream} + +class ClassLoaderUtilsTest { + + @Test def loadJarShouldAppendJarToSystemClassloader(): Unit = { + val jarFile = File.createTempFile("streampark-classloader-test", ".jar") + jarFile.deleteOnExit() + try { + val jarOut = new JarOutputStream(new FileOutputStream(jarFile)) + try { + jarOut.putNextEntry(new JarEntry("META-INF/MANIFEST.MF")) + jarOut.write("Manifest-Version: 1.0\n".getBytes("UTF-8")) + jarOut.closeEntry() + } finally { + jarOut.close() + } + ClassLoaderUtils.loadJar(jarFile.getAbsolutePath) + } finally { + jarFile.delete() + } + } + + @Test def loadResourceShouldAppendDirectoryToSystemClassloader(): Unit = { + val dir = FileUtils.createTempDir() + try { + ClassLoaderUtils.loadResource(dir.getAbsolutePath) + } finally { + Assertions.assertTrue(dir.delete()) + } + } +} From 6c4fdd965160faa9cdf4749ebb8f0d920eb3c838 Mon Sep 17 00:00:00 2001 From: shangeyao Date: Wed, 1 Jul 2026 11:06:52 +0800 Subject: [PATCH 3/4] [Console] JDK 11 runtime readiness for StreamPark Console Enable add-opens JVM opts, enforce JDK 11+ at startup, fix classpath layout, add javax.annotation-api, and document Console vs job JDK requirements. Closes #4409 (part 2/3). Ref #4410. Generated-by: Cursor Co-authored-by: Cursor --- .../streampark-console-service/pom.xml | 11 ++- .../src/main/assembly/bin/jvm_opts.sh | 37 ++++----- .../src/main/assembly/bin/setclasspath.sh | 9 ++ .../src/main/assembly/bin/streampark.sh | 17 ++-- .../src/main/assembly/conf/streampark-env.sh | 2 + .../main/assembly/script/JDK_UPGRADE_GUIDE.md | 82 +++++++++++++++++++ 6 files changed, 126 insertions(+), 32 deletions(-) create mode 100644 streampark-console/streampark-console-service/src/main/assembly/script/JDK_UPGRADE_GUIDE.md diff --git a/streampark-console/streampark-console-service/pom.xml b/streampark-console/streampark-console-service/pom.xml index 3b94cd0ca1..929248c233 100644 --- a/streampark-console/streampark-console-service/pom.xml +++ b/streampark-console/streampark-console-service/pom.xml @@ -37,8 +37,6 @@ 3.5.3.1 1.14 streampark-console-webapp - 64m - 512m 512m 1.21 @@ -305,6 +303,13 @@ provided + + + javax.annotation + javax.annotation-api + 1.3.2 + + com.github.ben-manes.caffeine caffeine @@ -497,7 +502,7 @@ org.apache.maven.plugins maven-surefire-plugin - -Dfile.encoding=utf-8 + -Dfile.encoding=utf-8 --add-opens java.base/jdk.internal.loader=ALL-UNNAMED --add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED diff --git a/streampark-console/streampark-console-service/src/main/assembly/bin/jvm_opts.sh b/streampark-console/streampark-console-service/src/main/assembly/bin/jvm_opts.sh index 2654c17515..0cc621a8cd 100644 --- a/streampark-console/streampark-console-service/src/main/assembly/bin/jvm_opts.sh +++ b/streampark-console/streampark-console-service/src/main/assembly/bin/jvm_opts.sh @@ -1,21 +1,19 @@ #!/bin/bash # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -server -Xms1g @@ -26,13 +24,6 @@ -XX:+HeapDumpOnOutOfMemoryError -XX:+IgnoreUnrecognizedVMOptions --XX:+PrintGCDateStamps --XX:+PrintGCDetails --XX:+PrintGC - --XX:+UseGCLogFileRotation --XX:GCLogFileSize=50M --XX:NumberOfGCLogFiles=10 - -# solved jdk1.8+ dynamic loading of resources to the classpath issue, if jdk > 1.8, you can enable this parameter -#--add-opens java.base/jdk.internal.loader=ALL-UNNAMED --add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED +# Required for dynamic classpath (ClassLoaderUtils) on JDK 9+. Ignored on JDK 8. +--add-opens java.base/jdk.internal.loader=ALL-UNNAMED +--add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED diff --git a/streampark-console/streampark-console-service/src/main/assembly/bin/setclasspath.sh b/streampark-console/streampark-console-service/src/main/assembly/bin/setclasspath.sh index 6eb52560f4..e952be9968 100755 --- a/streampark-console/streampark-console-service/src/main/assembly/bin/setclasspath.sh +++ b/streampark-console/streampark-console-service/src/main/assembly/bin/setclasspath.sh @@ -104,3 +104,12 @@ fi if [[ -z "$JAVA_HOME" ]]; then echo "Warning: JAVA_HOME environment variable is not set." fi + +REQUIRED_JAVA_MAJOR=11 +# shellcheck disable=SC2006 +java_version=`"$JAVACMD" -version 2>&1 | awk -F '"' '/version/ {print $2}'` +java_major=$(echo "$java_version" | awk -F '.' '{if ($1 == 1) {print $2} else {print $1}}') +if [[ "$java_major" -lt "$REQUIRED_JAVA_MAJOR" ]]; then + echo "Error: StreamPark requires JDK ${REQUIRED_JAVA_MAJOR} or later (current: ${java_version})." >&2 + exit 1 +fi diff --git a/streampark-console/streampark-console-service/src/main/assembly/bin/streampark.sh b/streampark-console/streampark-console-service/src/main/assembly/bin/streampark.sh index 35ded9208d..f8f15df3c5 100755 --- a/streampark-console/streampark-console-service/src/main/assembly/bin/streampark.sh +++ b/streampark-console/streampark-console-service/src/main/assembly/bin/streampark.sh @@ -270,7 +270,11 @@ fi JVM_OPTS=${JVM_OPTS:-"${JVM_ARGS}"} JVM_OPTS="$JVM_OPTS -XX:HeapDumpPath=${APP_HOME}/logs/dump.hprof" -JVM_OPTS="$JVM_OPTS -Xloggc:${APP_HOME}/logs/gc.log" +JVM_OPTS="$JVM_OPTS -Xlog:gc*:file=${APP_HOME}/logs/gc.log:time,uptime,level,tags:filecount=10,filesize=50M" + +build_java_classpath_prefix() { + echo ".:${JAVA_HOME}/lib" +} # ----- Execute The Requested Command ----------------------------------------- @@ -370,13 +374,13 @@ start() { echo_w "Using HADOOP_HOME: ${HADOOP_HOME}" fi - # # classpath options: - # 1): java env (lib and jre/lib) + # 1): java lib (JDK 11+ layout) # 2): StreamPark # 3): hadoop conf # shellcheck disable=SC2091 - local APP_CLASSPATH=".:${JAVA_HOME}/lib:${JAVA_HOME}/jre/lib" + local APP_CLASSPATH + APP_CLASSPATH=$(build_java_classpath_prefix) # shellcheck disable=SC2206 # shellcheck disable=SC2010 local JARS=$(ls "$APP_LIB"/*.jar | grep -v "$APP_LIB/streampark-flink-shims_.*.jar$") @@ -437,11 +441,12 @@ start_docker() { fi # classpath options: - # 1): java env (lib and jre/lib) + # 1): java lib (JDK 11+ layout) # 2): StreamPark # 3): hadoop conf # shellcheck disable=SC2091 - local APP_CLASSPATH=".:${JAVA_HOME}/lib:${JAVA_HOME}/jre/lib" + local APP_CLASSPATH + APP_CLASSPATH=$(build_java_classpath_prefix) # shellcheck disable=SC2206 # shellcheck disable=SC2155 # shellcheck disable=SC2010 diff --git a/streampark-console/streampark-console-service/src/main/assembly/conf/streampark-env.sh b/streampark-console/streampark-console-service/src/main/assembly/conf/streampark-env.sh index 3fa607db0d..3d3ff7d2ec 100644 --- a/streampark-console/streampark-console-service/src/main/assembly/conf/streampark-env.sh +++ b/streampark-console/streampark-console-service/src/main/assembly/conf/streampark-env.sh @@ -31,6 +31,8 @@ ### # Technically, the only required environment variable is JAVA_HOME. +# StreamPark 3.0 requires JDK 11 or later for the Console process. +# Flink/Spark job JDK is configured separately via flink-env.sh / spark-env.sh. # All others are optional. However, the defaults are probably not # preferred. Many sites configure these options outside of streampark, # such as in /etc/profile.d diff --git a/streampark-console/streampark-console-service/src/main/assembly/script/JDK_UPGRADE_GUIDE.md b/streampark-console/streampark-console-service/src/main/assembly/script/JDK_UPGRADE_GUIDE.md new file mode 100644 index 0000000000..4db28e1ad7 --- /dev/null +++ b/streampark-console/streampark-console-service/src/main/assembly/script/JDK_UPGRADE_GUIDE.md @@ -0,0 +1,82 @@ +# StreamPark JDK Upgrade Guide + +StreamPark 3.0 requires **JDK 11 or later** to run the Console process. + +## Console JDK vs Job JDK + +| Component | JDK requirement | Configuration | +|-----------|-----------------|---------------| +| StreamPark Console | **JDK 11+** | `JAVA_HOME` in `streampark-env.sh` or system environment | +| Flink jobs | Depends on Flink version | `$FLINK_HOME/conf/flink-env.sh` → `JAVA_HOME` | +| Spark jobs | Depends on Spark version | `$SPARK_HOME/conf/spark-env.sh` → `JAVA_HOME` | + +Upgrading the Console to JDK 11 **does not require** upgrading Flink/Spark cluster JDK at the same time. + +## Compatibility Matrix + +| Engine | Minimum job JDK | Recommended job JDK | +|--------|-----------------|---------------------| +| Flink 1.17–1.20 | 8 | 11 | +| Flink 2.x | 11 | 11 or 17 | +| Spark 3.5+ | 11 | 11 or 17 | + +## Upgrade Steps + +### 1. Standalone deployment + +1. Install JDK 11 (e.g. OpenJDK 11, Amazon Corretto 11). +2. Set `JAVA_HOME` in `conf/streampark-env.sh`: + ```bash + export JAVA_HOME=/path/to/jdk-11 + ``` +3. Restart Console: + ```bash + ./bin/streampark.sh restart + ``` +4. Verify: + ```bash + ./bin/streampark.sh status + java -version # should show 11+ + ``` + +### 2. Docker deployment + +Official StreamPark Docker images from 3.0 onward use JDK 11 internally. Pull the latest image: + +```bash +docker pull apache/streampark:latest +``` + +### 3. Build from source + +JDK 11+ is required to build StreamPark 3.0: + +```bash +export JAVA_HOME=/path/to/jdk-11 +./build.sh +``` + +## JVM Options + +StreamPark enables the following JVM options by default (see `bin/jvm_opts.sh`): + +``` +--add-opens java.base/jdk.internal.loader=ALL-UNNAMED +--add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED +``` + +These are required for dynamic classpath loading (Flink shims, user JARs). Do not remove them unless you know the impact. + +## Troubleshooting + +| Symptom | Cause | Fix | +|---------|-------|-----| +| `StreamPark requires JDK 11 or later` on start | Console running on JDK 8 | Upgrade `JAVA_HOME` to JDK 11+ | +| `NoSuchFieldException: ucp` | Missing `--add-opens` | Ensure `jvm_opts.sh` is not overridden incorrectly | +| `javax.annotation.PostConstruct` not found | Missing annotation API | Use StreamPark 3.0+ distribution (includes dependency) | +| Hadoop/YARN connection issues on JDK 11 | Jersey classpath | Verify Hadoop 3.3.x; check Hadoop client compatibility | + +## Related Issues + +- #4409 — JDK 11 migration proposal +- #4410 — StreamPark 3.0 roadmap From 63b913234621826abd53dc67bf911df1864341e1 Mon Sep 17 00:00:00 2001 From: shangeyao Date: Wed, 1 Jul 2026 17:29:22 +0800 Subject: [PATCH 4/4] [Build][E2E] Fix JDK 11 MavenWrapper and stabilize local basic E2E Compile MavenWrapperHelper with --release 11 at runtime and stop packaging prebuilt wrapper classes so project builds work in JDK 11 containers. Update basic E2E for Docker 29, local Chrome, Git SSL, and Projects page flows verified on macOS. Generated-by: Cursor Co-authored-by: Cursor --- mvnw | 2 +- .../src/main/assembly/assembly.xml | 3 + .../src/main/assembly/bin/mvnw | 4 +- .../e2e/cases/ProjectsManagementTest.java | 15 +++-- .../e2e/pages/common/CommonFactory.java | 26 ++++++++- .../e2e/pages/resource/ProjectsPage.java | 56 +++++++++++++------ .../docker/basic/docker-compose.yaml | 2 + .../src/test/resources/docker/basic/gitconfig | 19 +++++++ .../e2e/core/StreamParkExtension.java | 42 ++++++++++---- .../src/main/resources/docker-java.properties | 20 +++++++ 10 files changed, 149 insertions(+), 40 deletions(-) create mode 100644 streampark-e2e/streampark-e2e-case/src/test/resources/docker/basic/gitconfig create mode 100644 streampark-e2e/streampark-e2e-core/src/main/resources/docker-java.properties diff --git a/mvnw b/mvnw index 7cc74242d3..b86fa4750a 100755 --- a/mvnw +++ b/mvnw @@ -201,7 +201,7 @@ if [ ! -e "$javaSource" ]; then fi log " - Compiling $javaSource starting ..." -"$JAVA_HOME/bin/javac" "$javaSource" +"$JAVA_HOME/bin/javac" --release 11 "$javaSource" if [ -r "$wrapperJarPath" ]; then log "Found $wrapperJarPath" diff --git a/streampark-console/streampark-console-service/src/main/assembly/assembly.xml b/streampark-console/streampark-console-service/src/main/assembly/assembly.xml index 37667ac70a..86a4c7cba4 100644 --- a/streampark-console/streampark-console-service/src/main/assembly/assembly.xml +++ b/streampark-console/streampark-console-service/src/main/assembly/assembly.xml @@ -49,6 +49,9 @@ ${project.build.directory}/../../../.mvn bin/.mvn 0755 + + **/*.class + ${project.build.directory}/../src/main/assembly/bin diff --git a/streampark-console/streampark-console-service/src/main/assembly/bin/mvnw b/streampark-console/streampark-console-service/src/main/assembly/bin/mvnw index 7aca23b4cd..bcec77c3af 100755 --- a/streampark-console/streampark-console-service/src/main/assembly/bin/mvnw +++ b/streampark-console/streampark-console-service/src/main/assembly/bin/mvnw @@ -202,9 +202,9 @@ if [ ! -e "$javaSource" ]; then exit 1 fi -if [ ! -e "$javaClass" ]; then +if [ ! -e "$javaClass" ] || [ "$javaSource" -nt "$javaClass" ]; then log " - Compiling $javaClass starting ..." - ("$JAVA_HOME/bin/javac" "$javaSource") + ("$JAVA_HOME/bin/javac" --release 11 "$javaSource") fi if [ -r "$wrapperJarPath" ]; then diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/ProjectsManagementTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/ProjectsManagementTest.java index 72a90b5ca4..068f25ebbe 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/ProjectsManagementTest.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/ProjectsManagementTest.java @@ -27,6 +27,7 @@ import org.junit.jupiter.api.Test; import org.openqa.selenium.WebElement; import org.openqa.selenium.remote.RemoteWebDriver; +import org.openqa.selenium.support.PageFactory; import org.testcontainers.shaded.org.awaitility.Awaitility; import java.time.Duration; @@ -36,7 +37,7 @@ @StreamPark(composeFiles = "docker/basic/docker-compose.yaml") public class ProjectsManagementTest { - private final Duration PROJECT_BUILD_TIMEOUT_MINUTES = Duration.ofMinutes(5); + private final Duration PROJECT_BUILD_TIMEOUT_MINUTES = Duration.ofMinutes(15); public static RemoteWebDriver browser; @@ -98,10 +99,14 @@ void testBuildProject() { Awaitility.await().timeout(PROJECT_BUILD_TIMEOUT_MINUTES) .untilAsserted( - () -> assertThat(projectsPage.projectList) - .as("Projects list should contain build success project") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("SUCCESSFUL"))); + () -> { + browser.navigate().refresh(); + PageFactory.initElements(projectsPage.driver, projectsPage); + assertThat(projectsPage.projectList) + .as("Projects list should contain build success project") + .extracting(WebElement::getText) + .anyMatch(it -> it.contains("SUCCESSFUL")); + }); } diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/common/CommonFactory.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/common/CommonFactory.java index 0007ae24b8..6c6995bcc3 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/common/CommonFactory.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/common/CommonFactory.java @@ -18,7 +18,10 @@ package org.apache.streampark.e2e.pages.common; import org.openqa.selenium.By; +import org.openqa.selenium.ElementClickInterceptedException; +import org.openqa.selenium.JavascriptExecutor; import org.openqa.selenium.Keys; +import org.openqa.selenium.Platform; import org.openqa.selenium.WebDriver; import org.openqa.selenium.WebElement; import org.openqa.selenium.support.ui.ExpectedConditions; @@ -33,15 +36,34 @@ public static void WebElementDeleteAndInput(WebElement element, String value) { element.sendKeys(value); } + public static void WebElementDeleteAndInput(WebDriver driver, WebElement element, String value) { + new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) + .until(ExpectedConditions.visibilityOf(element)); + new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) + .until(ExpectedConditions.elementToBeClickable(element)); + WebElementDelete(element); + element.sendKeys(value); + } + public static void WebElementDelete(WebElement element) { - element.sendKeys(Keys.CONTROL + "a"); + element.sendKeys(Keys.chord(selectAllKey(), "a")); element.sendKeys(Keys.BACK_SPACE); } public static void WebElementClick(WebDriver driver, WebElement clickableElement) { new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) .until(ExpectedConditions.elementToBeClickable(clickableElement)); - clickableElement.click(); + try { + clickableElement.click(); + } catch (ElementClickInterceptedException ex) { + JavascriptExecutor executor = (JavascriptExecutor) driver; + executor.executeScript("arguments[0].scrollIntoView({block: 'center'});", clickableElement); + executor.executeScript("arguments[0].click();", clickableElement); + } + } + + private static CharSequence selectAllKey() { + return Platform.getCurrent().is(Platform.MAC) ? Keys.COMMAND : Keys.CONTROL; } public static void WebDriverWaitForElementVisibilityAndInvisibility(WebDriver driver, String msg) { diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/resource/ProjectsPage.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/resource/ProjectsPage.java index 6f7f497a5f..3fbe135957 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/resource/ProjectsPage.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/resource/ProjectsPage.java @@ -23,7 +23,6 @@ import lombok.Getter; import lombok.SneakyThrows; import org.openqa.selenium.By; -import org.openqa.selenium.Keys; import org.openqa.selenium.WebElement; import org.openqa.selenium.remote.RemoteWebDriver; import org.openqa.selenium.support.FindBy; @@ -34,6 +33,9 @@ import java.util.List; +import static org.apache.streampark.e2e.pages.common.CommonFactory.WebElementClick; +import static org.apache.streampark.e2e.pages.common.CommonFactory.WebElementDeleteAndInput; + @Getter public class ProjectsPage extends NavBarPage implements ResourcePage.Tab { @@ -67,14 +69,12 @@ public ProjectsPage createProject(String projectName, String projectDescription) { waitForPageLoading(); - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(buttonCreateProject)); - - buttonCreateProject.click(); + WebElementClick(driver, buttonCreateProject); new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) .until(ExpectedConditions.urlContains("/project/add")); + createProjectForm = new CreateProjectForm(); createProjectForm.inputProjectName.sendKeys(projectName); createProjectForm.selectCveDropdown.click(); @@ -103,6 +103,7 @@ public ProjectsPage createProject(String projectName, createProjectForm.inputDescription.sendKeys(projectDescription); createProjectForm.buttonSubmit.click(); + waitForListPageAfterSubmit(); return this; } @@ -111,7 +112,7 @@ public ProjectsPage editProject(String projectName, String newProjectName) { waitForPageLoading(); - projectList.stream() + WebElement editButton = projectList.stream() .filter(it -> it.getText().contains(projectName)) .flatMap( it -> it.findElements( @@ -119,17 +120,17 @@ public ProjectsPage editProject(String projectName, .stream()) .filter(WebElement::isDisplayed) .findFirst() - .orElseThrow(() -> new RuntimeException("No edit button in project list")) - .click(); + .orElseThrow(() -> new RuntimeException("No edit button in project list")); + WebElementClick(driver, editButton); new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) .until(ExpectedConditions.urlContains("/project/edit")); - createProjectForm.inputProjectName.sendKeys(Keys.CONTROL + "a"); - createProjectForm.inputProjectName.sendKeys(Keys.BACK_SPACE); - createProjectForm.inputProjectName.sendKeys(newProjectName); + createProjectForm = new CreateProjectForm(); + WebElementDeleteAndInput(driver, createProjectForm.inputProjectName, newProjectName); createProjectForm.buttonSubmit.click(); + waitForListPageAfterSubmit(); return this; } @@ -137,14 +138,14 @@ public ProjectsPage editProject(String projectName, public ProjectsPage buildProject(String projectName) { waitForPageLoading(); - projectList.stream() + WebElement buildBtn = projectList.stream() .filter(it -> it.getText().contains(projectName)) .flatMap( it -> it.findElements(By.className("e2e-project-build-btn")).stream()) .filter(WebElement::isDisplayed) .findFirst() - .orElseThrow(() -> new RuntimeException("No build button in project list")) - .click(); + .orElseThrow(() -> new RuntimeException("No build button in project list")); + WebElementClick(driver, buildBtn); new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) .until(ExpectedConditions.elementToBeClickable(buildConfirmButton)); @@ -157,7 +158,7 @@ public ProjectsPage buildProject(String projectName) { @SneakyThrows public ProjectsPage deleteProject(String projectName) { waitForPageLoading(); - projectList.stream() + WebElement deleteBtn = projectList.stream() .filter(it -> it.getText().contains(projectName)) .flatMap( it -> it @@ -165,19 +166,38 @@ public ProjectsPage deleteProject(String projectName) { .stream()) .filter(WebElement::isDisplayed) .findFirst() - .orElseThrow(() -> new RuntimeException("No delete button in project list")) - .click(); + .orElseThrow(() -> new RuntimeException("No delete button in project list")); + WebElementClick(driver, deleteBtn); new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) .until(ExpectedConditions.elementToBeClickable(deleteConfirmButton)); deleteConfirmButton.click(); + PageFactory.initElements(driver, this); return this; } private void waitForPageLoading() { new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.urlContains("/resource/project")); + .until(d -> isProjectListPage(d.getCurrentUrl())); + PageFactory.initElements(driver, this); + new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) + .until(ExpectedConditions.elementToBeClickable(buttonCreateProject)); + createProjectForm = new CreateProjectForm(); + } + + private void waitForListPageAfterSubmit() { + new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) + .until(d -> isProjectListPage(d.getCurrentUrl())); + PageFactory.initElements(driver, this); + new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) + .until(ExpectedConditions.elementToBeClickable(buttonCreateProject)); + createProjectForm = new CreateProjectForm(); + } + + private boolean isProjectListPage(String url) { + return url.contains("/resource/project") + || (url.contains("/project") && !url.contains("/project/add") && !url.contains("/project/edit")); } @Getter diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/basic/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/basic/docker-compose.yaml index 30b77fe550..35c74674a6 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/basic/docker-compose.yaml +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/basic/docker-compose.yaml @@ -34,6 +34,8 @@ services: - e2e volumes: - ${HOME}/streampark_build_logs:/tmp/streampark/logs/build_logs/ + - ${HOME}/.m2:/root/.m2 + - ./gitconfig:/root/.gitconfig:ro healthcheck: test: [ "CMD", "curl", "http://localhost:10000" ] interval: 5s diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/basic/gitconfig b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/basic/gitconfig new file mode 100644 index 0000000000..3e94fc6027 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/basic/gitconfig @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +[http] + sslVerify = false diff --git a/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/StreamParkExtension.java b/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/StreamParkExtension.java index 8b5d51d677..733256d771 100644 --- a/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/StreamParkExtension.java +++ b/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/StreamParkExtension.java @@ -25,6 +25,7 @@ import org.junit.jupiter.api.extension.BeforeEachCallback; import org.junit.jupiter.api.extension.ExtensionContext; import org.openqa.selenium.WebDriver; +import org.openqa.selenium.chrome.ChromeDriver; import org.openqa.selenium.chrome.ChromeOptions; import org.openqa.selenium.remote.RemoteWebDriver; import org.testcontainers.Testcontainers; @@ -59,6 +60,8 @@ final class StreamParkExtension implements BeforeAllCallback, AfterAllCallback, private final boolean M1_CHIP_FLAG = Objects.equals(System.getProperty("m1_chip"), "true"); + private final boolean LOCAL_BROWSER = Objects.equals(System.getProperty("local_browser"), "true"); + private final int LOCAL_PORT = 10001; private final int DOCKER_PORT = 10000; @@ -87,15 +90,19 @@ public void beforeAll(ExtensionContext context) throws IOException { runInDockerContainer(context); } - setBrowserContainerByOsName(); + if (LOCAL_BROWSER) { + driver = new ChromeDriver(new ChromeOptions()); + } else { + setBrowserContainerByOsName(); - if (compose != null) { - Testcontainers.exposeHostPorts(compose.getServicePort(serviceName, DOCKER_PORT)); - browser.withAccessToHost(true); - } - browser.start(); + if (compose != null) { + Testcontainers.exposeHostPorts(compose.getServicePort(serviceName, DOCKER_PORT)); + browser.withAccessToHost(true); + } + browser.start(); - driver = new RemoteWebDriver(browser.getSeleniumAddress(), new ChromeOptions()); + driver = new RemoteWebDriver(browser.getSeleniumAddress(), new ChromeOptions()); + } driver .manage() @@ -106,7 +113,9 @@ public void beforeAll(ExtensionContext context) throws IOException { driver.get(new URL("http", address.getHost(), address.getPort(), rootPath).toString()); - browser.beforeTest(new TestDescription(context)); + if (!LOCAL_BROWSER) { + browser.beforeTest(new TestDescription(context)); + } final Class clazz = context.getRequiredTestClass(); Stream.of(clazz.getDeclaredFields()) @@ -117,7 +126,7 @@ public void beforeAll(ExtensionContext context) throws IOException { private void runInLocal() { Testcontainers.exposeHostPorts(LOCAL_PORT); - address = HostAndPort.fromParts("host.testcontainers.internal", LOCAL_PORT); + address = HostAndPort.fromParts(browserHost(), LOCAL_PORT); rootPath = "/"; } @@ -126,10 +135,14 @@ private void runInDockerContainer(ExtensionContext context) { compose.start(); address = HostAndPort.fromParts( - "host.testcontainers.internal", compose.getServicePort(serviceName, DOCKER_PORT)); + browserHost(), compose.getServicePort(serviceName, DOCKER_PORT)); rootPath = "/"; } + private String browserHost() { + return LOCAL_BROWSER ? "localhost" : "host.testcontainers.internal"; + } + private void setBrowserContainerByOsName() { DockerImageName imageName; @@ -172,8 +185,13 @@ record = Files.createTempDirectory("record-"); @Override public void afterAll(ExtensionContext context) { - browser.afterTest(new TestDescription(context), Optional.empty()); - browser.stop(); + if (driver != null) { + driver.quit(); + } + if (!LOCAL_BROWSER && browser != null) { + browser.afterTest(new TestDescription(context), Optional.empty()); + browser.stop(); + } if (compose != null) { compose.stop(); } diff --git a/streampark-e2e/streampark-e2e-core/src/main/resources/docker-java.properties b/streampark-e2e/streampark-e2e-core/src/main/resources/docker-java.properties new file mode 100644 index 0000000000..74f51e401a --- /dev/null +++ b/streampark-e2e/streampark-e2e-core/src/main/resources/docker-java.properties @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Docker Engine 29+ requires API version 1.44 or later. +# Testcontainers 1.19.x defaults to 1.32 via docker-java, which is rejected by the daemon. +api.version=1.44