diff --git a/streampark-console/streampark-console-service/pom.xml b/streampark-console/streampark-console-service/pom.xml index 3b94cd0ca1..929248c233 100644 --- a/streampark-console/streampark-console-service/pom.xml +++ b/streampark-console/streampark-console-service/pom.xml @@ -37,8 +37,6 @@ 3.5.3.1 1.14 streampark-console-webapp - 64m - 512m 512m 1.21 @@ -305,6 +303,13 @@ provided + + + javax.annotation + javax.annotation-api + 1.3.2 + + com.github.ben-manes.caffeine caffeine @@ -497,7 +502,7 @@ org.apache.maven.plugins maven-surefire-plugin - -Dfile.encoding=utf-8 + -Dfile.encoding=utf-8 --add-opens java.base/jdk.internal.loader=ALL-UNNAMED --add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED diff --git a/streampark-console/streampark-console-service/src/main/assembly/bin/jvm_opts.sh b/streampark-console/streampark-console-service/src/main/assembly/bin/jvm_opts.sh index 2654c17515..0cc621a8cd 100644 --- a/streampark-console/streampark-console-service/src/main/assembly/bin/jvm_opts.sh +++ b/streampark-console/streampark-console-service/src/main/assembly/bin/jvm_opts.sh @@ -1,21 +1,19 @@ #!/bin/bash # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -server -Xms1g @@ -26,13 +24,6 @@ -XX:+HeapDumpOnOutOfMemoryError -XX:+IgnoreUnrecognizedVMOptions --XX:+PrintGCDateStamps --XX:+PrintGCDetails --XX:+PrintGC - --XX:+UseGCLogFileRotation --XX:GCLogFileSize=50M --XX:NumberOfGCLogFiles=10 - -# solved jdk1.8+ dynamic loading of resources to the classpath issue, if jdk > 1.8, you can enable this parameter -#--add-opens java.base/jdk.internal.loader=ALL-UNNAMED --add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED +# Required for dynamic classpath (ClassLoaderUtils) on JDK 9+. Ignored on JDK 8. +--add-opens java.base/jdk.internal.loader=ALL-UNNAMED +--add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED diff --git a/streampark-console/streampark-console-service/src/main/assembly/bin/setclasspath.sh b/streampark-console/streampark-console-service/src/main/assembly/bin/setclasspath.sh index 6eb52560f4..e952be9968 100755 --- a/streampark-console/streampark-console-service/src/main/assembly/bin/setclasspath.sh +++ b/streampark-console/streampark-console-service/src/main/assembly/bin/setclasspath.sh @@ -104,3 +104,12 @@ fi if [[ -z "$JAVA_HOME" ]]; then echo "Warning: JAVA_HOME environment variable is not set." fi + +REQUIRED_JAVA_MAJOR=11 +# shellcheck disable=SC2006 +java_version=`"$JAVACMD" -version 2>&1 | awk -F '"' '/version/ {print $2}'` +java_major=$(echo "$java_version" | awk -F '.' '{if ($1 == 1) {print $2} else {print $1}}') +if [[ "$java_major" -lt "$REQUIRED_JAVA_MAJOR" ]]; then + echo "Error: StreamPark requires JDK ${REQUIRED_JAVA_MAJOR} or later (current: ${java_version})." >&2 + exit 1 +fi diff --git a/streampark-console/streampark-console-service/src/main/assembly/bin/streampark.sh b/streampark-console/streampark-console-service/src/main/assembly/bin/streampark.sh index 35ded9208d..f8f15df3c5 100755 --- a/streampark-console/streampark-console-service/src/main/assembly/bin/streampark.sh +++ b/streampark-console/streampark-console-service/src/main/assembly/bin/streampark.sh @@ -270,7 +270,11 @@ fi JVM_OPTS=${JVM_OPTS:-"${JVM_ARGS}"} JVM_OPTS="$JVM_OPTS -XX:HeapDumpPath=${APP_HOME}/logs/dump.hprof" -JVM_OPTS="$JVM_OPTS -Xloggc:${APP_HOME}/logs/gc.log" +JVM_OPTS="$JVM_OPTS -Xlog:gc*:file=${APP_HOME}/logs/gc.log:time,uptime,level,tags:filecount=10,filesize=50M" + +build_java_classpath_prefix() { + echo ".:${JAVA_HOME}/lib" +} # ----- Execute The Requested Command ----------------------------------------- @@ -370,13 +374,13 @@ start() { echo_w "Using HADOOP_HOME: ${HADOOP_HOME}" fi - # # classpath options: - # 1): java env (lib and jre/lib) + # 1): java lib (JDK 11+ layout) # 2): StreamPark # 3): hadoop conf # shellcheck disable=SC2091 - local APP_CLASSPATH=".:${JAVA_HOME}/lib:${JAVA_HOME}/jre/lib" + local APP_CLASSPATH + APP_CLASSPATH=$(build_java_classpath_prefix) # shellcheck disable=SC2206 # shellcheck disable=SC2010 local JARS=$(ls "$APP_LIB"/*.jar | grep -v "$APP_LIB/streampark-flink-shims_.*.jar$") @@ -437,11 +441,12 @@ start_docker() { fi # classpath options: - # 1): java env (lib and jre/lib) + # 1): java lib (JDK 11+ layout) # 2): StreamPark # 3): hadoop conf # shellcheck disable=SC2091 - local APP_CLASSPATH=".:${JAVA_HOME}/lib:${JAVA_HOME}/jre/lib" + local APP_CLASSPATH + APP_CLASSPATH=$(build_java_classpath_prefix) # shellcheck disable=SC2206 # shellcheck disable=SC2155 # shellcheck disable=SC2010 diff --git a/streampark-console/streampark-console-service/src/main/assembly/conf/streampark-env.sh b/streampark-console/streampark-console-service/src/main/assembly/conf/streampark-env.sh index 3fa607db0d..3d3ff7d2ec 100644 --- a/streampark-console/streampark-console-service/src/main/assembly/conf/streampark-env.sh +++ b/streampark-console/streampark-console-service/src/main/assembly/conf/streampark-env.sh @@ -31,6 +31,8 @@ ### # Technically, the only required environment variable is JAVA_HOME. +# StreamPark 3.0 requires JDK 11 or later for the Console process. +# Flink/Spark job JDK is configured separately via flink-env.sh / spark-env.sh. # All others are optional. However, the defaults are probably not # preferred. Many sites configure these options outside of streampark, # such as in /etc/profile.d diff --git a/streampark-console/streampark-console-service/src/main/assembly/script/JDK_UPGRADE_GUIDE.md b/streampark-console/streampark-console-service/src/main/assembly/script/JDK_UPGRADE_GUIDE.md new file mode 100644 index 0000000000..4db28e1ad7 --- /dev/null +++ b/streampark-console/streampark-console-service/src/main/assembly/script/JDK_UPGRADE_GUIDE.md @@ -0,0 +1,82 @@ +# StreamPark JDK Upgrade Guide + +StreamPark 3.0 requires **JDK 11 or later** to run the Console process. + +## Console JDK vs Job JDK + +| Component | JDK requirement | Configuration | +|-----------|-----------------|---------------| +| StreamPark Console | **JDK 11+** | `JAVA_HOME` in `streampark-env.sh` or system environment | +| Flink jobs | Depends on Flink version | `$FLINK_HOME/conf/flink-env.sh` → `JAVA_HOME` | +| Spark jobs | Depends on Spark version | `$SPARK_HOME/conf/spark-env.sh` → `JAVA_HOME` | + +Upgrading the Console to JDK 11 **does not require** upgrading Flink/Spark cluster JDK at the same time. + +## Compatibility Matrix + +| Engine | Minimum job JDK | Recommended job JDK | +|--------|-----------------|---------------------| +| Flink 1.17–1.20 | 8 | 11 | +| Flink 2.x | 11 | 11 or 17 | +| Spark 3.5+ | 11 | 11 or 17 | + +## Upgrade Steps + +### 1. Standalone deployment + +1. Install JDK 11 (e.g. OpenJDK 11, Amazon Corretto 11). +2. Set `JAVA_HOME` in `conf/streampark-env.sh`: + ```bash + export JAVA_HOME=/path/to/jdk-11 + ``` +3. Restart Console: + ```bash + ./bin/streampark.sh restart + ``` +4. Verify: + ```bash + ./bin/streampark.sh status + java -version # should show 11+ + ``` + +### 2. Docker deployment + +Official StreamPark Docker images from 3.0 onward use JDK 11 internally. Pull the latest image: + +```bash +docker pull apache/streampark:latest +``` + +### 3. Build from source + +JDK 11+ is required to build StreamPark 3.0: + +```bash +export JAVA_HOME=/path/to/jdk-11 +./build.sh +``` + +## JVM Options + +StreamPark enables the following JVM options by default (see `bin/jvm_opts.sh`): + +``` +--add-opens java.base/jdk.internal.loader=ALL-UNNAMED +--add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED +``` + +These are required for dynamic classpath loading (Flink shims, user JARs). Do not remove them unless you know the impact. + +## Troubleshooting + +| Symptom | Cause | Fix | +|---------|-------|-----| +| `StreamPark requires JDK 11 or later` on start | Console running on JDK 8 | Upgrade `JAVA_HOME` to JDK 11+ | +| `NoSuchFieldException: ucp` | Missing `--add-opens` | Ensure `jvm_opts.sh` is not overridden incorrectly | +| `javax.annotation.PostConstruct` not found | Missing annotation API | Use StreamPark 3.0+ distribution (includes dependency) | +| Hadoop/YARN connection issues on JDK 11 | Jersey classpath | Verify Hadoop 3.3.x; check Hadoop client compatibility | + +## Related Issues + +- #4409 — JDK 11 migration proposal +- #4410 — StreamPark 3.0 roadmap