diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 878e12661a..d8de19686d 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -80,7 +80,7 @@ jobs: strategy: fail-fast: false matrix: - java: [ 8 , 11 ] + java: [ 11 ] steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/docker-push.yml b/.github/workflows/docker-push.yml index c7172b25db..75f643f8eb 100644 --- a/.github/workflows/docker-push.yml +++ b/.github/workflows/docker-push.yml @@ -70,7 +70,7 @@ jobs: - name: Setup Java and Scala uses: olafurpg/setup-scala@v13 with: - java-version: adopt@1.8 + java-version: adopt@11 - uses: actions/setup-node@v3 with: diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 5e87c7ebd0..db04e73429 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -77,10 +77,10 @@ jobs: - uses: actions/checkout@v4 with: submodules: true - - name: Set up JDK 8 + - name: Set up JDK 11 uses: actions/setup-java@v4 with: - java-version: 8 + java-version: 11 distribution: 'adopt' - name: Cache local Maven repository uses: actions/cache@v4 diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 4d0a9fa64e..2cc58868de 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -80,7 +80,7 @@ jobs: strategy: fail-fast: false matrix: - java: [ 8 , 11 ] + java: [ 11 ] steps: - name: Checkout uses: actions/checkout@v4 diff --git a/docker/Dockerfile b/docker/Dockerfile index ac5fca4f4f..f3fc10083f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -19,10 +19,10 @@ FROM ubuntu:22.04 USER root ENV LANG=C.UTF-8 \ - JAVA_HOME=/usr/lib/jvm/jdk8 + JAVA_HOME=/usr/lib/jvm/jdk11 # Build arguments for version management -ARG JAVA_MAJOR_VERSION=8 +ARG JAVA_MAJOR_VERSION=11 ARG TINI_VERSION=v0.19.0 # Base system setup diff --git a/mvnw b/mvnw index 7cc74242d3..b86fa4750a 100755 --- a/mvnw +++ b/mvnw @@ -201,7 +201,7 @@ if [ ! -e "$javaSource" ]; then fi log " - Compiling $javaSource starting ..." -"$JAVA_HOME/bin/javac" "$javaSource" +"$JAVA_HOME/bin/javac" --release 11 "$javaSource" if [ -r "$wrapperJarPath" ]; then log "Found $wrapperJarPath" diff --git a/pom.xml b/pom.xml index 8f5103008b..68b6b536f9 100644 --- a/pom.xml +++ b/pom.xml @@ -82,7 +82,7 @@ - 1.8 + 11 UTF-8 UTF-8 @@ -697,8 +697,7 @@ maven-compiler-plugin ${maven-compiler-plugin.version} - ${project.build.jdk} - ${project.build.jdk} + ${project.build.jdk} UTF-8 false @@ -715,7 +714,7 @@ scala-maven-plugin ${scala-maven-plugin.version} - -target:jvm-${project.build.jdk} + -target:${project.build.jdk} ${project.build.jdk} ${project.build.jdk} @@ -1009,6 +1008,24 @@ true + + + jdk9-plus-test-opens + + [9,) + + + + + org.apache.maven.plugins + maven-surefire-plugin + + --add-opens java.base/jdk.internal.loader=ALL-UNNAMED --add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED + + + + + diff --git a/streampark-common/pom.xml b/streampark-common/pom.xml index 115477f4aa..03f4211b32 100644 --- a/streampark-common/pom.xml +++ b/streampark-common/pom.xml @@ -197,4 +197,24 @@ + + + + jdk9-plus-test-opens + + [9,) + + + + + org.apache.maven.plugins + maven-surefire-plugin + + --add-opens java.base/jdk.internal.loader=ALL-UNNAMED --add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED + + + + + + diff --git a/streampark-common/src/main/scala/org/apache/streampark/common/util/ClassLoaderUtils.scala b/streampark-common/src/main/scala/org/apache/streampark/common/util/ClassLoaderUtils.scala index e052de96c3..e1664dc859 100644 --- a/streampark-common/src/main/scala/org/apache/streampark/common/util/ClassLoaderUtils.scala +++ b/streampark-common/src/main/scala/org/apache/streampark/common/util/ClassLoaderUtils.scala @@ -146,9 +146,20 @@ object ClassLoaderUtils extends Logger { addURL.setAccessible(true) addURL.invoke(c, file.toURI.toURL) case _ => - val field = classLoader.getClass.getDeclaredField("ucp") - field.setAccessible(true) - val ucp = field.get(classLoader) + var clazz: Class[_] = classLoader.getClass + var ucpField: java.lang.reflect.Field = null + while (clazz != null && ucpField == null) { + try { + ucpField = clazz.getDeclaredField("ucp") + } catch { + case _: NoSuchFieldException => clazz = clazz.getSuperclass + } + } + require( + ucpField != null, + "[StreamPark] ClassLoaderUtils.addURL: cannot locate ucp field on classloader chain") + ucpField.setAccessible(true) + val ucp = ucpField.get(classLoader) val addURL = ucp.getClass.getDeclaredMethod("addURL", Array(classOf[URL]): _*) addURL.setAccessible(true) diff --git a/streampark-common/src/test/scala/org/apache/streampark/common/util/ClassLoaderUtilsTest.scala b/streampark-common/src/test/scala/org/apache/streampark/common/util/ClassLoaderUtilsTest.scala new file mode 100644 index 0000000000..f4193e154a --- /dev/null +++ b/streampark-common/src/test/scala/org/apache/streampark/common/util/ClassLoaderUtilsTest.scala @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util + +import org.junit.jupiter.api.{Assertions, Test} + +import java.io.{File, FileOutputStream} +import java.util.jar.{JarEntry, JarOutputStream} + +class ClassLoaderUtilsTest { + + @Test def loadJarShouldAppendJarToSystemClassloader(): Unit = { + val jarFile = File.createTempFile("streampark-classloader-test", ".jar") + jarFile.deleteOnExit() + try { + val jarOut = new JarOutputStream(new FileOutputStream(jarFile)) + try { + jarOut.putNextEntry(new JarEntry("META-INF/MANIFEST.MF")) + jarOut.write("Manifest-Version: 1.0\n".getBytes("UTF-8")) + jarOut.closeEntry() + } finally { + jarOut.close() + } + ClassLoaderUtils.loadJar(jarFile.getAbsolutePath) + } finally { + jarFile.delete() + } + } + + @Test def loadResourceShouldAppendDirectoryToSystemClassloader(): Unit = { + val dir = FileUtils.createTempDir() + try { + ClassLoaderUtils.loadResource(dir.getAbsolutePath) + } finally { + Assertions.assertTrue(dir.delete()) + } + } +} diff --git a/streampark-console/streampark-console-service/pom.xml b/streampark-console/streampark-console-service/pom.xml index 3b94cd0ca1..929248c233 100644 --- a/streampark-console/streampark-console-service/pom.xml +++ b/streampark-console/streampark-console-service/pom.xml @@ -37,8 +37,6 @@ 3.5.3.1 1.14 streampark-console-webapp - 64m - 512m 512m 1.21 @@ -305,6 +303,13 @@ provided + + + javax.annotation + javax.annotation-api + 1.3.2 + + com.github.ben-manes.caffeine caffeine @@ -497,7 +502,7 @@ org.apache.maven.plugins maven-surefire-plugin - -Dfile.encoding=utf-8 + -Dfile.encoding=utf-8 --add-opens java.base/jdk.internal.loader=ALL-UNNAMED --add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED diff --git a/streampark-console/streampark-console-service/src/main/assembly/assembly.xml b/streampark-console/streampark-console-service/src/main/assembly/assembly.xml index 37667ac70a..86a4c7cba4 100644 --- a/streampark-console/streampark-console-service/src/main/assembly/assembly.xml +++ b/streampark-console/streampark-console-service/src/main/assembly/assembly.xml @@ -49,6 +49,9 @@ ${project.build.directory}/../../../.mvn bin/.mvn 0755 + + **/*.class + ${project.build.directory}/../src/main/assembly/bin diff --git a/streampark-console/streampark-console-service/src/main/assembly/bin/jvm_opts.sh b/streampark-console/streampark-console-service/src/main/assembly/bin/jvm_opts.sh index 2654c17515..0cc621a8cd 100644 --- a/streampark-console/streampark-console-service/src/main/assembly/bin/jvm_opts.sh +++ b/streampark-console/streampark-console-service/src/main/assembly/bin/jvm_opts.sh @@ -1,21 +1,19 @@ #!/bin/bash # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -server -Xms1g @@ -26,13 +24,6 @@ -XX:+HeapDumpOnOutOfMemoryError -XX:+IgnoreUnrecognizedVMOptions --XX:+PrintGCDateStamps --XX:+PrintGCDetails --XX:+PrintGC - --XX:+UseGCLogFileRotation --XX:GCLogFileSize=50M --XX:NumberOfGCLogFiles=10 - -# solved jdk1.8+ dynamic loading of resources to the classpath issue, if jdk > 1.8, you can enable this parameter -#--add-opens java.base/jdk.internal.loader=ALL-UNNAMED --add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED +# Required for dynamic classpath (ClassLoaderUtils) on JDK 9+. Ignored on JDK 8. +--add-opens java.base/jdk.internal.loader=ALL-UNNAMED +--add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED diff --git a/streampark-console/streampark-console-service/src/main/assembly/bin/mvnw b/streampark-console/streampark-console-service/src/main/assembly/bin/mvnw index 7aca23b4cd..bcec77c3af 100755 --- a/streampark-console/streampark-console-service/src/main/assembly/bin/mvnw +++ b/streampark-console/streampark-console-service/src/main/assembly/bin/mvnw @@ -202,9 +202,9 @@ if [ ! -e "$javaSource" ]; then exit 1 fi -if [ ! -e "$javaClass" ]; then +if [ ! -e "$javaClass" ] || [ "$javaSource" -nt "$javaClass" ]; then log " - Compiling $javaClass starting ..." - ("$JAVA_HOME/bin/javac" "$javaSource") + ("$JAVA_HOME/bin/javac" --release 11 "$javaSource") fi if [ -r "$wrapperJarPath" ]; then diff --git a/streampark-console/streampark-console-service/src/main/assembly/bin/setclasspath.sh b/streampark-console/streampark-console-service/src/main/assembly/bin/setclasspath.sh index 6eb52560f4..e952be9968 100755 --- a/streampark-console/streampark-console-service/src/main/assembly/bin/setclasspath.sh +++ b/streampark-console/streampark-console-service/src/main/assembly/bin/setclasspath.sh @@ -104,3 +104,12 @@ fi if [[ -z "$JAVA_HOME" ]]; then echo "Warning: JAVA_HOME environment variable is not set." fi + +REQUIRED_JAVA_MAJOR=11 +# shellcheck disable=SC2006 +java_version=`"$JAVACMD" -version 2>&1 | awk -F '"' '/version/ {print $2}'` +java_major=$(echo "$java_version" | awk -F '.' '{if ($1 == 1) {print $2} else {print $1}}') +if [[ "$java_major" -lt "$REQUIRED_JAVA_MAJOR" ]]; then + echo "Error: StreamPark requires JDK ${REQUIRED_JAVA_MAJOR} or later (current: ${java_version})." >&2 + exit 1 +fi diff --git a/streampark-console/streampark-console-service/src/main/assembly/bin/streampark.sh b/streampark-console/streampark-console-service/src/main/assembly/bin/streampark.sh index 35ded9208d..f8f15df3c5 100755 --- a/streampark-console/streampark-console-service/src/main/assembly/bin/streampark.sh +++ b/streampark-console/streampark-console-service/src/main/assembly/bin/streampark.sh @@ -270,7 +270,11 @@ fi JVM_OPTS=${JVM_OPTS:-"${JVM_ARGS}"} JVM_OPTS="$JVM_OPTS -XX:HeapDumpPath=${APP_HOME}/logs/dump.hprof" -JVM_OPTS="$JVM_OPTS -Xloggc:${APP_HOME}/logs/gc.log" +JVM_OPTS="$JVM_OPTS -Xlog:gc*:file=${APP_HOME}/logs/gc.log:time,uptime,level,tags:filecount=10,filesize=50M" + +build_java_classpath_prefix() { + echo ".:${JAVA_HOME}/lib" +} # ----- Execute The Requested Command ----------------------------------------- @@ -370,13 +374,13 @@ start() { echo_w "Using HADOOP_HOME: ${HADOOP_HOME}" fi - # # classpath options: - # 1): java env (lib and jre/lib) + # 1): java lib (JDK 11+ layout) # 2): StreamPark # 3): hadoop conf # shellcheck disable=SC2091 - local APP_CLASSPATH=".:${JAVA_HOME}/lib:${JAVA_HOME}/jre/lib" + local APP_CLASSPATH + APP_CLASSPATH=$(build_java_classpath_prefix) # shellcheck disable=SC2206 # shellcheck disable=SC2010 local JARS=$(ls "$APP_LIB"/*.jar | grep -v "$APP_LIB/streampark-flink-shims_.*.jar$") @@ -437,11 +441,12 @@ start_docker() { fi # classpath options: - # 1): java env (lib and jre/lib) + # 1): java lib (JDK 11+ layout) # 2): StreamPark # 3): hadoop conf # shellcheck disable=SC2091 - local APP_CLASSPATH=".:${JAVA_HOME}/lib:${JAVA_HOME}/jre/lib" + local APP_CLASSPATH + APP_CLASSPATH=$(build_java_classpath_prefix) # shellcheck disable=SC2206 # shellcheck disable=SC2155 # shellcheck disable=SC2010 diff --git a/streampark-console/streampark-console-service/src/main/assembly/conf/streampark-env.sh b/streampark-console/streampark-console-service/src/main/assembly/conf/streampark-env.sh index 3fa607db0d..3d3ff7d2ec 100644 --- a/streampark-console/streampark-console-service/src/main/assembly/conf/streampark-env.sh +++ b/streampark-console/streampark-console-service/src/main/assembly/conf/streampark-env.sh @@ -31,6 +31,8 @@ ### # Technically, the only required environment variable is JAVA_HOME. +# StreamPark 3.0 requires JDK 11 or later for the Console process. +# Flink/Spark job JDK is configured separately via flink-env.sh / spark-env.sh. # All others are optional. However, the defaults are probably not # preferred. Many sites configure these options outside of streampark, # such as in /etc/profile.d diff --git a/streampark-console/streampark-console-service/src/main/assembly/script/JDK_UPGRADE_GUIDE.md b/streampark-console/streampark-console-service/src/main/assembly/script/JDK_UPGRADE_GUIDE.md new file mode 100644 index 0000000000..4db28e1ad7 --- /dev/null +++ b/streampark-console/streampark-console-service/src/main/assembly/script/JDK_UPGRADE_GUIDE.md @@ -0,0 +1,82 @@ +# StreamPark JDK Upgrade Guide + +StreamPark 3.0 requires **JDK 11 or later** to run the Console process. + +## Console JDK vs Job JDK + +| Component | JDK requirement | Configuration | +|-----------|-----------------|---------------| +| StreamPark Console | **JDK 11+** | `JAVA_HOME` in `streampark-env.sh` or system environment | +| Flink jobs | Depends on Flink version | `$FLINK_HOME/conf/flink-env.sh` → `JAVA_HOME` | +| Spark jobs | Depends on Spark version | `$SPARK_HOME/conf/spark-env.sh` → `JAVA_HOME` | + +Upgrading the Console to JDK 11 **does not require** upgrading Flink/Spark cluster JDK at the same time. + +## Compatibility Matrix + +| Engine | Minimum job JDK | Recommended job JDK | +|--------|-----------------|---------------------| +| Flink 1.17–1.20 | 8 | 11 | +| Flink 2.x | 11 | 11 or 17 | +| Spark 3.5+ | 11 | 11 or 17 | + +## Upgrade Steps + +### 1. Standalone deployment + +1. Install JDK 11 (e.g. OpenJDK 11, Amazon Corretto 11). +2. Set `JAVA_HOME` in `conf/streampark-env.sh`: + ```bash + export JAVA_HOME=/path/to/jdk-11 + ``` +3. Restart Console: + ```bash + ./bin/streampark.sh restart + ``` +4. Verify: + ```bash + ./bin/streampark.sh status + java -version # should show 11+ + ``` + +### 2. Docker deployment + +Official StreamPark Docker images from 3.0 onward use JDK 11 internally. Pull the latest image: + +```bash +docker pull apache/streampark:latest +``` + +### 3. Build from source + +JDK 11+ is required to build StreamPark 3.0: + +```bash +export JAVA_HOME=/path/to/jdk-11 +./build.sh +``` + +## JVM Options + +StreamPark enables the following JVM options by default (see `bin/jvm_opts.sh`): + +``` +--add-opens java.base/jdk.internal.loader=ALL-UNNAMED +--add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED +``` + +These are required for dynamic classpath loading (Flink shims, user JARs). Do not remove them unless you know the impact. + +## Troubleshooting + +| Symptom | Cause | Fix | +|---------|-------|-----| +| `StreamPark requires JDK 11 or later` on start | Console running on JDK 8 | Upgrade `JAVA_HOME` to JDK 11+ | +| `NoSuchFieldException: ucp` | Missing `--add-opens` | Ensure `jvm_opts.sh` is not overridden incorrectly | +| `javax.annotation.PostConstruct` not found | Missing annotation API | Use StreamPark 3.0+ distribution (includes dependency) | +| Hadoop/YARN connection issues on JDK 11 | Jersey classpath | Verify Hadoop 3.3.x; check Hadoop client compatibility | + +## Related Issues + +- #4409 — JDK 11 migration proposal +- #4410 — StreamPark 3.0 roadmap diff --git a/streampark-e2e/pom.xml b/streampark-e2e/pom.xml index 6e601d388b..a41604694d 100644 --- a/streampark-e2e/pom.xml +++ b/streampark-e2e/pom.xml @@ -30,8 +30,8 @@ - 8 - 8 + 11 + 11 UTF-8 5.8.1 diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/ProjectsManagementTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/ProjectsManagementTest.java index 72a90b5ca4..068f25ebbe 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/ProjectsManagementTest.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/ProjectsManagementTest.java @@ -27,6 +27,7 @@ import org.junit.jupiter.api.Test; import org.openqa.selenium.WebElement; import org.openqa.selenium.remote.RemoteWebDriver; +import org.openqa.selenium.support.PageFactory; import org.testcontainers.shaded.org.awaitility.Awaitility; import java.time.Duration; @@ -36,7 +37,7 @@ @StreamPark(composeFiles = "docker/basic/docker-compose.yaml") public class ProjectsManagementTest { - private final Duration PROJECT_BUILD_TIMEOUT_MINUTES = Duration.ofMinutes(5); + private final Duration PROJECT_BUILD_TIMEOUT_MINUTES = Duration.ofMinutes(15); public static RemoteWebDriver browser; @@ -98,10 +99,14 @@ void testBuildProject() { Awaitility.await().timeout(PROJECT_BUILD_TIMEOUT_MINUTES) .untilAsserted( - () -> assertThat(projectsPage.projectList) - .as("Projects list should contain build success project") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("SUCCESSFUL"))); + () -> { + browser.navigate().refresh(); + PageFactory.initElements(projectsPage.driver, projectsPage); + assertThat(projectsPage.projectList) + .as("Projects list should contain build success project") + .extracting(WebElement::getText) + .anyMatch(it -> it.contains("SUCCESSFUL")); + }); } diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/common/CommonFactory.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/common/CommonFactory.java index 0007ae24b8..6c6995bcc3 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/common/CommonFactory.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/common/CommonFactory.java @@ -18,7 +18,10 @@ package org.apache.streampark.e2e.pages.common; import org.openqa.selenium.By; +import org.openqa.selenium.ElementClickInterceptedException; +import org.openqa.selenium.JavascriptExecutor; import org.openqa.selenium.Keys; +import org.openqa.selenium.Platform; import org.openqa.selenium.WebDriver; import org.openqa.selenium.WebElement; import org.openqa.selenium.support.ui.ExpectedConditions; @@ -33,15 +36,34 @@ public static void WebElementDeleteAndInput(WebElement element, String value) { element.sendKeys(value); } + public static void WebElementDeleteAndInput(WebDriver driver, WebElement element, String value) { + new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) + .until(ExpectedConditions.visibilityOf(element)); + new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) + .until(ExpectedConditions.elementToBeClickable(element)); + WebElementDelete(element); + element.sendKeys(value); + } + public static void WebElementDelete(WebElement element) { - element.sendKeys(Keys.CONTROL + "a"); + element.sendKeys(Keys.chord(selectAllKey(), "a")); element.sendKeys(Keys.BACK_SPACE); } public static void WebElementClick(WebDriver driver, WebElement clickableElement) { new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) .until(ExpectedConditions.elementToBeClickable(clickableElement)); - clickableElement.click(); + try { + clickableElement.click(); + } catch (ElementClickInterceptedException ex) { + JavascriptExecutor executor = (JavascriptExecutor) driver; + executor.executeScript("arguments[0].scrollIntoView({block: 'center'});", clickableElement); + executor.executeScript("arguments[0].click();", clickableElement); + } + } + + private static CharSequence selectAllKey() { + return Platform.getCurrent().is(Platform.MAC) ? Keys.COMMAND : Keys.CONTROL; } public static void WebDriverWaitForElementVisibilityAndInvisibility(WebDriver driver, String msg) { diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/resource/ProjectsPage.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/resource/ProjectsPage.java index 6f7f497a5f..3fbe135957 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/resource/ProjectsPage.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/resource/ProjectsPage.java @@ -23,7 +23,6 @@ import lombok.Getter; import lombok.SneakyThrows; import org.openqa.selenium.By; -import org.openqa.selenium.Keys; import org.openqa.selenium.WebElement; import org.openqa.selenium.remote.RemoteWebDriver; import org.openqa.selenium.support.FindBy; @@ -34,6 +33,9 @@ import java.util.List; +import static org.apache.streampark.e2e.pages.common.CommonFactory.WebElementClick; +import static org.apache.streampark.e2e.pages.common.CommonFactory.WebElementDeleteAndInput; + @Getter public class ProjectsPage extends NavBarPage implements ResourcePage.Tab { @@ -67,14 +69,12 @@ public ProjectsPage createProject(String projectName, String projectDescription) { waitForPageLoading(); - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(buttonCreateProject)); - - buttonCreateProject.click(); + WebElementClick(driver, buttonCreateProject); new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) .until(ExpectedConditions.urlContains("/project/add")); + createProjectForm = new CreateProjectForm(); createProjectForm.inputProjectName.sendKeys(projectName); createProjectForm.selectCveDropdown.click(); @@ -103,6 +103,7 @@ public ProjectsPage createProject(String projectName, createProjectForm.inputDescription.sendKeys(projectDescription); createProjectForm.buttonSubmit.click(); + waitForListPageAfterSubmit(); return this; } @@ -111,7 +112,7 @@ public ProjectsPage editProject(String projectName, String newProjectName) { waitForPageLoading(); - projectList.stream() + WebElement editButton = projectList.stream() .filter(it -> it.getText().contains(projectName)) .flatMap( it -> it.findElements( @@ -119,17 +120,17 @@ public ProjectsPage editProject(String projectName, .stream()) .filter(WebElement::isDisplayed) .findFirst() - .orElseThrow(() -> new RuntimeException("No edit button in project list")) - .click(); + .orElseThrow(() -> new RuntimeException("No edit button in project list")); + WebElementClick(driver, editButton); new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) .until(ExpectedConditions.urlContains("/project/edit")); - createProjectForm.inputProjectName.sendKeys(Keys.CONTROL + "a"); - createProjectForm.inputProjectName.sendKeys(Keys.BACK_SPACE); - createProjectForm.inputProjectName.sendKeys(newProjectName); + createProjectForm = new CreateProjectForm(); + WebElementDeleteAndInput(driver, createProjectForm.inputProjectName, newProjectName); createProjectForm.buttonSubmit.click(); + waitForListPageAfterSubmit(); return this; } @@ -137,14 +138,14 @@ public ProjectsPage editProject(String projectName, public ProjectsPage buildProject(String projectName) { waitForPageLoading(); - projectList.stream() + WebElement buildBtn = projectList.stream() .filter(it -> it.getText().contains(projectName)) .flatMap( it -> it.findElements(By.className("e2e-project-build-btn")).stream()) .filter(WebElement::isDisplayed) .findFirst() - .orElseThrow(() -> new RuntimeException("No build button in project list")) - .click(); + .orElseThrow(() -> new RuntimeException("No build button in project list")); + WebElementClick(driver, buildBtn); new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) .until(ExpectedConditions.elementToBeClickable(buildConfirmButton)); @@ -157,7 +158,7 @@ public ProjectsPage buildProject(String projectName) { @SneakyThrows public ProjectsPage deleteProject(String projectName) { waitForPageLoading(); - projectList.stream() + WebElement deleteBtn = projectList.stream() .filter(it -> it.getText().contains(projectName)) .flatMap( it -> it @@ -165,19 +166,38 @@ public ProjectsPage deleteProject(String projectName) { .stream()) .filter(WebElement::isDisplayed) .findFirst() - .orElseThrow(() -> new RuntimeException("No delete button in project list")) - .click(); + .orElseThrow(() -> new RuntimeException("No delete button in project list")); + WebElementClick(driver, deleteBtn); new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) .until(ExpectedConditions.elementToBeClickable(deleteConfirmButton)); deleteConfirmButton.click(); + PageFactory.initElements(driver, this); return this; } private void waitForPageLoading() { new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.urlContains("/resource/project")); + .until(d -> isProjectListPage(d.getCurrentUrl())); + PageFactory.initElements(driver, this); + new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) + .until(ExpectedConditions.elementToBeClickable(buttonCreateProject)); + createProjectForm = new CreateProjectForm(); + } + + private void waitForListPageAfterSubmit() { + new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) + .until(d -> isProjectListPage(d.getCurrentUrl())); + PageFactory.initElements(driver, this); + new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) + .until(ExpectedConditions.elementToBeClickable(buttonCreateProject)); + createProjectForm = new CreateProjectForm(); + } + + private boolean isProjectListPage(String url) { + return url.contains("/resource/project") + || (url.contains("/project") && !url.contains("/project/add") && !url.contains("/project/edit")); } @Getter diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/basic/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/basic/docker-compose.yaml index 30b77fe550..35c74674a6 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/basic/docker-compose.yaml +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/basic/docker-compose.yaml @@ -34,6 +34,8 @@ services: - e2e volumes: - ${HOME}/streampark_build_logs:/tmp/streampark/logs/build_logs/ + - ${HOME}/.m2:/root/.m2 + - ./gitconfig:/root/.gitconfig:ro healthcheck: test: [ "CMD", "curl", "http://localhost:10000" ] interval: 5s diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/basic/gitconfig b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/basic/gitconfig new file mode 100644 index 0000000000..3e94fc6027 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/basic/gitconfig @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +[http] + sslVerify = false diff --git a/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/StreamParkExtension.java b/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/StreamParkExtension.java index 8b5d51d677..733256d771 100644 --- a/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/StreamParkExtension.java +++ b/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/StreamParkExtension.java @@ -25,6 +25,7 @@ import org.junit.jupiter.api.extension.BeforeEachCallback; import org.junit.jupiter.api.extension.ExtensionContext; import org.openqa.selenium.WebDriver; +import org.openqa.selenium.chrome.ChromeDriver; import org.openqa.selenium.chrome.ChromeOptions; import org.openqa.selenium.remote.RemoteWebDriver; import org.testcontainers.Testcontainers; @@ -59,6 +60,8 @@ final class StreamParkExtension implements BeforeAllCallback, AfterAllCallback, private final boolean M1_CHIP_FLAG = Objects.equals(System.getProperty("m1_chip"), "true"); + private final boolean LOCAL_BROWSER = Objects.equals(System.getProperty("local_browser"), "true"); + private final int LOCAL_PORT = 10001; private final int DOCKER_PORT = 10000; @@ -87,15 +90,19 @@ public void beforeAll(ExtensionContext context) throws IOException { runInDockerContainer(context); } - setBrowserContainerByOsName(); + if (LOCAL_BROWSER) { + driver = new ChromeDriver(new ChromeOptions()); + } else { + setBrowserContainerByOsName(); - if (compose != null) { - Testcontainers.exposeHostPorts(compose.getServicePort(serviceName, DOCKER_PORT)); - browser.withAccessToHost(true); - } - browser.start(); + if (compose != null) { + Testcontainers.exposeHostPorts(compose.getServicePort(serviceName, DOCKER_PORT)); + browser.withAccessToHost(true); + } + browser.start(); - driver = new RemoteWebDriver(browser.getSeleniumAddress(), new ChromeOptions()); + driver = new RemoteWebDriver(browser.getSeleniumAddress(), new ChromeOptions()); + } driver .manage() @@ -106,7 +113,9 @@ public void beforeAll(ExtensionContext context) throws IOException { driver.get(new URL("http", address.getHost(), address.getPort(), rootPath).toString()); - browser.beforeTest(new TestDescription(context)); + if (!LOCAL_BROWSER) { + browser.beforeTest(new TestDescription(context)); + } final Class clazz = context.getRequiredTestClass(); Stream.of(clazz.getDeclaredFields()) @@ -117,7 +126,7 @@ public void beforeAll(ExtensionContext context) throws IOException { private void runInLocal() { Testcontainers.exposeHostPorts(LOCAL_PORT); - address = HostAndPort.fromParts("host.testcontainers.internal", LOCAL_PORT); + address = HostAndPort.fromParts(browserHost(), LOCAL_PORT); rootPath = "/"; } @@ -126,10 +135,14 @@ private void runInDockerContainer(ExtensionContext context) { compose.start(); address = HostAndPort.fromParts( - "host.testcontainers.internal", compose.getServicePort(serviceName, DOCKER_PORT)); + browserHost(), compose.getServicePort(serviceName, DOCKER_PORT)); rootPath = "/"; } + private String browserHost() { + return LOCAL_BROWSER ? "localhost" : "host.testcontainers.internal"; + } + private void setBrowserContainerByOsName() { DockerImageName imageName; @@ -172,8 +185,13 @@ record = Files.createTempDirectory("record-"); @Override public void afterAll(ExtensionContext context) { - browser.afterTest(new TestDescription(context), Optional.empty()); - browser.stop(); + if (driver != null) { + driver.quit(); + } + if (!LOCAL_BROWSER && browser != null) { + browser.afterTest(new TestDescription(context), Optional.empty()); + browser.stop(); + } if (compose != null) { compose.stop(); } diff --git a/streampark-e2e/streampark-e2e-core/src/main/resources/docker-java.properties b/streampark-e2e/streampark-e2e-core/src/main/resources/docker-java.properties new file mode 100644 index 0000000000..74f51e401a --- /dev/null +++ b/streampark-e2e/streampark-e2e-core/src/main/resources/docker-java.properties @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Docker Engine 29+ requires API version 1.44 or later. +# Testcontainers 1.19.x defaults to 1.32 via docker-java, which is rejected by the daemon. +api.version=1.44 diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/pom.xml b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/pom.xml index 23a84158a3..3c1659a503 100644 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/pom.xml +++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/pom.xml @@ -30,8 +30,8 @@ StreamPark : Flink Connector Elasticsearch5 - 8 - 8 + 11 + 11 1.14.0 diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/pom.xml b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/pom.xml index 0d944d1e62..546e52abf3 100644 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/pom.xml +++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/pom.xml @@ -30,8 +30,8 @@ StreamPark : Flink Connector Elasticsearch6 - 8 - 8 + 11 + 11 diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/pom.xml b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/pom.xml index 53bde3349c..d1c7195be0 100644 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/pom.xml +++ b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/pom.xml @@ -29,8 +29,8 @@ StreamPark : Flink Connector Elasticsearch7 - 8 - 8 + 11 + 11 diff --git a/streampark-spark/streampark-spark-cli/create_app.sh b/streampark-spark/streampark-spark-cli/create_app.sh index 1de938e828..998811ee5d 100644 --- a/streampark-spark/streampark-spark-cli/create_app.sh +++ b/streampark-spark/streampark-spark-cli/create_app.sh @@ -281,7 +281,7 @@ cat > $name/pom.xml <UTF-8 UTF-8 - 1.8 + 11 64m diff --git a/tools/checkstyle/spotless_streampark_formatter.xml b/tools/checkstyle/spotless_streampark_formatter.xml index f05ccaeb43..12b28b6b63 100644 --- a/tools/checkstyle/spotless_streampark_formatter.xml +++ b/tools/checkstyle/spotless_streampark_formatter.xml @@ -20,9 +20,9 @@ --> - - - + + +