[Build] Raise StreamPark compile and CI baseline to JDK 11

.github/workflows/backend.yml

@@ -80,7 +80,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        java: [ 8 , 11 ]
+        java: [ 11 ]
     steps:
       - name: Checkout
         uses: actions/checkout@v4

.github/workflows/docker-push.yml

@@ -70,7 +70,7 @@ jobs:
       - name: Setup Java and Scala
         uses: olafurpg/setup-scala@v13
         with:
-          java-version: adopt@1.8
+          java-version: adopt@11
 
       - uses: actions/setup-node@v3
         with:

.github/workflows/e2e.yml

@@ -77,10 +77,10 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: true
-      - name: Set up JDK 8
+      - name: Set up JDK 11
         uses: actions/setup-java@v4
         with:
-          java-version: 8
+          java-version: 11
           distribution: 'adopt'
       - name: Cache local Maven repository
         uses: actions/cache@v4

.github/workflows/unit-test.yml

@@ -80,7 +80,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        java: [ 8 , 11 ]
+        java: [ 11 ]
     steps:
       - name: Checkout
         uses: actions/checkout@v4

docker/Dockerfile

@@ -19,10 +19,10 @@ FROM ubuntu:22.04
 USER root
 
 ENV LANG=C.UTF-8 \
-    JAVA_HOME=/usr/lib/jvm/jdk8
+    JAVA_HOME=/usr/lib/jvm/jdk11
 
 # Build arguments for version management
-ARG JAVA_MAJOR_VERSION=8
+ARG JAVA_MAJOR_VERSION=11
 ARG TINI_VERSION=v0.19.0
 
 # Base system setup

pom.xml

@@ -82,7 +82,7 @@
     </issueManagement>
 
     <properties>
-        <project.build.jdk>1.8</project.build.jdk>
+        <project.build.jdk>11</project.build.jdk>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
 
@@ -697,8 +697,7 @@
                     <artifactId>maven-compiler-plugin</artifactId>
                     <version>${maven-compiler-plugin.version}</version>
                     <configuration>
-                        <source>${project.build.jdk}</source>
-                        <target>${project.build.jdk}</target>
+                        <release>${project.build.jdk}</release>
                         <encoding>UTF-8</encoding>
                         <!-- The semantics of this option are reversed, see MCOMPILER-209. -->
                         <useIncrementalCompilation>false</useIncrementalCompilation>
@@ -715,7 +714,7 @@
                     <artifactId>scala-maven-plugin</artifactId>
                     <version>${scala-maven-plugin.version}</version>
                     <configuration>
-                        <addScalacArgs>-target:jvm-${project.build.jdk}</addScalacArgs>
+                        <addScalacArgs>-target:${project.build.jdk}</addScalacArgs>
                         <target>${project.build.jdk}</target>
                         <source>${project.build.jdk}</source>
                         <args>
@@ -1009,6 +1008,24 @@
                 <spotless.skip>true</spotless.skip>
             </properties>
         </profile>
+
+        <profile>
+            <id>jdk9-plus-test-opens</id>
+            <activation>
+                <jdk>[9,)</jdk>
+            </activation>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-surefire-plugin</artifactId>
+                        <configuration>
+                            <argLine>--add-opens java.base/jdk.internal.loader=ALL-UNNAMED --add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED</argLine>
+                        </configuration>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
     </profiles>
 
 </project>

streampark-common/pom.xml

@@ -197,4 +197,24 @@
 
         </plugins>
     </build>
+
+    <profiles>
+        <profile>
+            <id>jdk9-plus-test-opens</id>
+            <activation>
+                <jdk>[9,)</jdk>
+            </activation>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-surefire-plugin</artifactId>
+                        <configuration>
+                            <argLine>--add-opens java.base/jdk.internal.loader=ALL-UNNAMED --add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED</argLine>
+                        </configuration>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
 </project>

streampark-common/src/main/scala/org/apache/streampark/common/util/ClassLoaderUtils.scala

@@ -146,9 +146,20 @@ object ClassLoaderUtils extends Logger {
         addURL.setAccessible(true)
         addURL.invoke(c, file.toURI.toURL)
       case _ =>
-        val field = classLoader.getClass.getDeclaredField("ucp")
-        field.setAccessible(true)
-        val ucp = field.get(classLoader)
+        var clazz: Class[_] = classLoader.getClass
+        var ucpField: java.lang.reflect.Field = null
+        while (clazz != null && ucpField == null) {
+          try {
+            ucpField = clazz.getDeclaredField("ucp")
+          } catch {
+            case _: NoSuchFieldException => clazz = clazz.getSuperclass
+          }
+        }
+        require(
+          ucpField != null,
+          "[StreamPark] ClassLoaderUtils.addURL: cannot locate ucp field on classloader chain")
+        ucpField.setAccessible(true)
+        val ucp = ucpField.get(classLoader)
         val addURL =
           ucp.getClass.getDeclaredMethod("addURL", Array(classOf[URL]): _*)
         addURL.setAccessible(true)

streampark-common/src/test/scala/org/apache/streampark/common/util/ClassLoaderUtilsTest.scala

@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.streampark.common.util
+
+import org.junit.jupiter.api.{Assertions, Test}
+
+import java.io.{File, FileOutputStream}
+import java.util.jar.{JarEntry, JarOutputStream}
+
+class ClassLoaderUtilsTest {
+
+  @Test def loadJarShouldAppendJarToSystemClassloader(): Unit = {
+    val jarFile = File.createTempFile("streampark-classloader-test", ".jar")
+    jarFile.deleteOnExit()
+    try {
+      val jarOut = new JarOutputStream(new FileOutputStream(jarFile))
+      try {
+        jarOut.putNextEntry(new JarEntry("META-INF/MANIFEST.MF"))
+        jarOut.write("Manifest-Version: 1.0\n".getBytes("UTF-8"))
+        jarOut.closeEntry()
+      } finally {
+        jarOut.close()
+      }
+      ClassLoaderUtils.loadJar(jarFile.getAbsolutePath)
+    } finally {
+      jarFile.delete()
+    }
+  }
+
+  @Test def loadResourceShouldAppendDirectoryToSystemClassloader(): Unit = {
+    val dir = FileUtils.createTempDir()
+    try {
+      ClassLoaderUtils.loadResource(dir.getAbsolutePath)
+    } finally {
+      Assertions.assertTrue(dir.delete())
+    }
+  }
+}

streampark-console/streampark-console-service/pom.xml

@@ -37,8 +37,6 @@
         <mybatis-plus.version>3.5.3.1</mybatis-plus.version>
         <streampark.flink.shims.version>1.14</streampark.flink.shims.version>
         <frontend.project.name>streampark-console-webapp</frontend.project.name>
-        <PermGen>64m</PermGen>
-        <MaxPermGen>512m</MaxPermGen>
         <CodeCacheSize>512m</CodeCacheSize>
 
         <commons-compress.version>1.21</commons-compress.version>
@@ -305,6 +303,13 @@
             <scope>provided</scope>
         </dependency>
 
+        <!-- JSR-250 annotations removed from JDK 11 module path -->
+        <dependency>
+            <groupId>javax.annotation</groupId>
+            <artifactId>javax.annotation-api</artifactId>
+            <version>1.3.2</version>
+        </dependency>
+
         <dependency>
             <groupId>com.github.ben-manes.caffeine</groupId>
             <artifactId>caffeine</artifactId>
@@ -497,7 +502,7 @@
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-surefire-plugin</artifactId>
                 <configuration>
-                    <argLine>-Dfile.encoding=utf-8</argLine>
+                    <argLine>-Dfile.encoding=utf-8 --add-opens java.base/jdk.internal.loader=ALL-UNNAMED --add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED</argLine>
                 </configuration>
             </plugin>
 

streampark-console/streampark-console-service/src/main/assembly/assembly.xml

@@ -49,6 +49,9 @@
             <directory>${project.build.directory}/../../../.mvn</directory>
             <outputDirectory>bin/.mvn</outputDirectory>
             <fileMode>0755</fileMode>
+            <excludes>
+                <exclude>**/*.class</exclude>
+            </excludes>
         </fileSet>
         <fileSet>
             <directory>${project.build.directory}/../src/main/assembly/bin</directory>

streampark-console/streampark-console-service/src/main/assembly/bin/jvm_opts.sh

@@ -1,21 +1,19 @@
 #!/bin/bash
 #
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
 #
 #    http://www.apache.org/licenses/LICENSE-2.0
 #
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 -server
 -Xms1g
@@ -26,13 +24,6 @@
 -XX:+HeapDumpOnOutOfMemoryError
 -XX:+IgnoreUnrecognizedVMOptions
 
--XX:+PrintGCDateStamps
--XX:+PrintGCDetails
--XX:+PrintGC
-
--XX:+UseGCLogFileRotation
--XX:GCLogFileSize=50M
--XX:NumberOfGCLogFiles=10
-
-# solved jdk1.8+ dynamic loading of resources to the classpath issue, if jdk > 1.8, you can enable this parameter
-#--add-opens java.base/jdk.internal.loader=ALL-UNNAMED --add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED
+# Required for dynamic classpath (ClassLoaderUtils) on JDK 9+. Ignored on JDK 8.
+--add-opens java.base/jdk.internal.loader=ALL-UNNAMED
+--add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED

streampark-console/streampark-console-service/src/main/assembly/bin/setclasspath.sh

@@ -104,3 +104,12 @@
 if [[ -z "$JAVA_HOME" ]]; then
   echo "Warning: JAVA_HOME environment variable is not set."
 fi
+
+REQUIRED_JAVA_MAJOR=11
+# shellcheck disable=SC2006
+java_version=`"$JAVACMD" -version 2>&1 | awk -F '"' '/version/ {print $2}'`
+java_major=$(echo "$java_version" | awk -F '.' '{if ($1 == 1) {print $2} else {print $1}}')
+if [[ "$java_major" -lt "$REQUIRED_JAVA_MAJOR" ]]; then
+  echo "Error: StreamPark requires JDK ${REQUIRED_JAVA_MAJOR} or later (current: ${java_version})." >&2
+  exit 1
+fi

streampark-console/streampark-console-service/src/main/assembly/bin/streampark.sh

@@ -270,7 +270,11 @@
 
 JVM_OPTS=${JVM_OPTS:-"${JVM_ARGS}"}
 JVM_OPTS="$JVM_OPTS -XX:HeapDumpPath=${APP_HOME}/logs/dump.hprof"
-JVM_OPTS="$JVM_OPTS -Xloggc:${APP_HOME}/logs/gc.log"
+JVM_OPTS="$JVM_OPTS -Xlog:gc*:file=${APP_HOME}/logs/gc.log:time,uptime,level,tags:filecount=10,filesize=50M"
+
+build_java_classpath_prefix() {
+  echo ".:${JAVA_HOME}/lib"
+}
 
 # ----- Execute The Requested Command -----------------------------------------
 
@@ -370,13 +374,13 @@
     echo_w "Using HADOOP_HOME:   ${HADOOP_HOME}"
   fi
 
-  #
   # classpath options:
-  # 1): java env (lib and jre/lib)
+  # 1): java lib (JDK 11+ layout)
   # 2): StreamPark
   # 3): hadoop conf
   # shellcheck disable=SC2091
-  local APP_CLASSPATH=".:${JAVA_HOME}/lib:${JAVA_HOME}/jre/lib"
+  local APP_CLASSPATH
+  APP_CLASSPATH=$(build_java_classpath_prefix)
   # shellcheck disable=SC2206
   # shellcheck disable=SC2010
   local JARS=$(ls "$APP_LIB"/*.jar | grep -v "$APP_LIB/streampark-flink-shims_.*.jar$")
@@ -437,11 +441,12 @@
   fi
 
   # classpath options:
-  # 1): java env (lib and jre/lib)
+  # 1): java lib (JDK 11+ layout)
   # 2): StreamPark
   # 3): hadoop conf
   # shellcheck disable=SC2091
-  local APP_CLASSPATH=".:${JAVA_HOME}/lib:${JAVA_HOME}/jre/lib"
+  local APP_CLASSPATH
+  APP_CLASSPATH=$(build_java_classpath_prefix)
   # shellcheck disable=SC2206
   # shellcheck disable=SC2155
   # shellcheck disable=SC2010

streampark-console/streampark-console-service/src/main/assembly/conf/streampark-env.sh

@@ -31,6 +31,8 @@
 ###
 
 # Technically, the only required environment variable is JAVA_HOME.
+# StreamPark 3.0 requires JDK 11 or later for the Console process.
+# Flink/Spark job JDK is configured separately via flink-env.sh / spark-env.sh.
 # All others are optional.  However, the defaults are probably not
 # preferred.  Many sites configure these options outside of streampark,
 # such as in /etc/profile.d