diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index 9b13dbd069..0ca8a2ca62 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -737,6 +737,10 @@ jobs: spark-test-spark34: needs: build-native-lib-centos-7 runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + group: [1, 2, 3] container: apache/gluten:centos-8-jdk8 steps: - uses: actions/checkout@v4 @@ -752,7 +756,7 @@ jobs: pip3 install setuptools==77.0.3 && \ pip3 install pyspark==3.4.4 cython && \ pip3 install pandas==2.2.3 pyarrow==20.0.0 - - name: Build and Run unit test for Spark 3.4.4 (other tests) + - name: Build and Run unit test for Spark 3.4.4 (other tests, group ${{ matrix.group }}) run: | cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.12 @@ -761,21 +765,31 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH java -version export SPARK_HOME=/opt/shims/spark34/spark_home/ - ls -l $SPARK_HOME - $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Piceberg-test -Pdelta -Phudi -Ppaimon -Pspark-ut \ - -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest \ - -DargLine="-Dspark.test.home=$SPARK_HOME" + TAGS_EXCLUDE="org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest" + if [ "${{ matrix.group }}" = "1" ]; then + $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Piceberg-test -Pdelta -Phudi -Ppaimon -Pspark-ut \ + -DtagsToExclude="$TAGS_EXCLUDE" -DargLine="-Dspark.test.home=$SPARK_HOME" \ + -DwildcardSuites="org.apache.spark.sql.streaming,org.apache.spark.GlutenSortShuffleSuite,org.apache.gluten" + elif [ "${{ matrix.group }}" = "2" ]; then + $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Piceberg-test -Pdelta -Phudi -Ppaimon -Pspark-ut \ + -DtagsToExclude="$TAGS_EXCLUDE" -DargLine="-Dspark.test.home=$SPARK_HOME" \ + -DwildcardSuites="org.apache.spark.sql.execution,org.apache.spark.sql.catalyst,org.apache.spark.sql.errors,org.apache.spark.sql.extension" + else + $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Piceberg-test -Pdelta -Phudi -Ppaimon -Pspark-ut \ + -DtagsToExclude="$TAGS_EXCLUDE" -DargLine="-Dspark.test.home=$SPARK_HOME" \ + -DwildcardSuites="org.apache.spark.sql.GlutenSQL,org.apache.spark.sql.Gluten,org.apache.spark.sql.connector,org.apache.spark.sql.sources,org.apache.spark.sql.hive,org.apache.spark.sql.gluten,org.apache.spark.sql.shim" + fi - name: Upload test report if: always() uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-report + name: spark-test-spark34-group${{ matrix.group }}-report path: '**/surefire-reports/TEST-*.xml' - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-test-log + name: spark-test-spark34-group${{ matrix.group }}-test-log path: | **/target/*.log **/gluten-ut/**/hs_err_*.log @@ -784,12 +798,16 @@ jobs: if: failure() uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-golden-files + name: spark-test-spark34-group${{ matrix.group }}-golden-files path: /tmp/tpch-approved-plan/** spark-test-spark34-slow: needs: build-native-lib-centos-7 runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + suite: [extended, slow-hive] env: #TODO remove after image update LANG: C.UTF-8 LC_ALL: C.UTF-8 @@ -805,7 +823,7 @@ jobs: run: | rm -rf /opt/shims/spark34 bash .github/workflows/util/install-spark-resources.sh 3.4 - - name: Build and Run unit test for Spark 3.4.4 (slow tests) + - name: Build and Run unit test for Spark 3.4.4 (slow tests, ${{ matrix.suite }}) run: | cd $GITHUB_WORKSPACE/ yum install -y java-17-openjdk-devel @@ -813,24 +831,26 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH java -version export SPARK_HOME=/opt/shims/spark34/spark_home/ - ls -l $SPARK_HOME - $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ - -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest \ - -DargLine="-Dspark.test.home=$SPARK_HOME" - $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ - -DtagsToInclude=org.apache.spark.tags.SlowHiveTest \ - -DargLine="-Dspark.test.home=$SPARK_HOME" + if [ "${{ matrix.suite }}" = "extended" ]; then + $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ + -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest \ + -DargLine="-Dspark.test.home=$SPARK_HOME" + else + $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ + -DtagsToInclude=org.apache.spark.tags.SlowHiveTest \ + -DargLine="-Dspark.test.home=$SPARK_HOME" + fi - name: Upload test report if: always() uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-report + name: spark-test-spark34-slow-${{ matrix.suite }}-report path: '**/surefire-reports/TEST-*.xml' - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-test-log + name: spark-test-spark34-slow-${{ matrix.suite }}-test-log path: | **/target/*.log **/gluten-ut/**/hs_err_*.log @@ -839,6 +859,10 @@ jobs: spark-test-spark35: needs: build-native-lib-centos-7 runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + group: [1, 2, 3] env: SPARK_TESTING: true container: apache/gluten:centos-8-jdk8 @@ -856,7 +880,7 @@ jobs: pip3 install setuptools==77.0.3 && \ pip3 install pyspark==3.5.5 cython && \ pip3 install pandas==2.2.3 pyarrow==20.0.0 - - name: Build and Run unit test for Spark 3.5.5 (other tests) + - name: Build and Run unit test for Spark 3.5.5 (other tests, group ${{ matrix.group }}) run: | cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.12 @@ -864,20 +888,31 @@ jobs: export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ - -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \ - -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest + TAGS_EXCLUDE="org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest" + if [ "${{ matrix.group }}" = "1" ]; then + $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" -DtagsToExclude="$TAGS_EXCLUDE" \ + -DwildcardSuites="org.apache.spark.sql.streaming,org.apache.spark.GlutenSortShuffleSuite,org.apache.gluten" + elif [ "${{ matrix.group }}" = "2" ]; then + $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" -DtagsToExclude="$TAGS_EXCLUDE" \ + -DwildcardSuites="org.apache.spark.sql.execution,org.apache.spark.sql.catalyst,org.apache.spark.sql.errors,org.apache.spark.sql.extension" + else + $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" -DtagsToExclude="$TAGS_EXCLUDE" \ + -DwildcardSuites="org.apache.spark.sql.GlutenSQL,org.apache.spark.sql.Gluten,org.apache.spark.sql.connector,org.apache.spark.sql.sources,org.apache.spark.sql.hive,org.apache.spark.sql.gluten,org.apache.spark.sql.shim" + fi - name: Upload test report if: always() uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-report + name: spark-test-spark35-group${{ matrix.group }}-report path: '**/surefire-reports/TEST-*.xml' - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-test-log + name: spark-test-spark35-group${{ matrix.group }}-test-log path: | **/target/*.log **/gluten-ut/**/hs_err_*.log @@ -886,12 +921,16 @@ jobs: if: failure() uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-golden-files + name: spark-test-spark35-group${{ matrix.group }}-golden-files path: /tmp/tpch-approved-plan/** spark-test-spark35-scala213: needs: build-native-lib-centos-7 runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + group: [1, 2, 3] env: SPARK_TESTING: true container: apache/gluten:centos-8-jdk8 @@ -909,7 +948,7 @@ jobs: pip3 install setuptools==77.0.3 && \ pip3 install pyspark==3.5.5 cython && \ pip3 install pandas==2.2.3 pyarrow==20.0.0 - - name: Build and Run unit test for Spark 3.5.5 with scala-2.13 (other tests) + - name: Build and Run unit test for Spark 3.5.5 with scala-2.13 (other tests, group ${{ matrix.group }}) run: | cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.13 @@ -917,20 +956,31 @@ jobs: export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-3.5 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Piceberg \ - -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=/opt/shims/spark35-scala-2.13/spark_home/" \ - -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest + TAGS_EXCLUDE="org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest" + if [ "${{ matrix.group }}" = "1" ]; then + $MVN_CMD clean test -Pspark-3.5 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark35-scala-2.13/spark_home/" -DtagsToExclude="$TAGS_EXCLUDE" \ + -DwildcardSuites="org.apache.spark.sql.streaming,org.apache.spark.GlutenSortShuffleSuite,org.apache.gluten" + elif [ "${{ matrix.group }}" = "2" ]; then + $MVN_CMD clean test -Pspark-3.5 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark35-scala-2.13/spark_home/" -DtagsToExclude="$TAGS_EXCLUDE" \ + -DwildcardSuites="org.apache.spark.sql.execution,org.apache.spark.sql.catalyst,org.apache.spark.sql.errors,org.apache.spark.sql.extension" + else + $MVN_CMD clean test -Pspark-3.5 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark35-scala-2.13/spark_home/" -DtagsToExclude="$TAGS_EXCLUDE" \ + -DwildcardSuites="org.apache.spark.sql.GlutenSQL,org.apache.spark.sql.Gluten,org.apache.spark.sql.connector,org.apache.spark.sql.sources,org.apache.spark.sql.hive,org.apache.spark.sql.gluten,org.apache.spark.sql.shim" + fi - name: Upload test report if: always() uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-report + name: spark-test-spark35-scala213-group${{ matrix.group }}-report path: '**/surefire-reports/TEST-*.xml' - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-test-log + name: spark-test-spark35-scala213-group${{ matrix.group }}-test-log path: | **/target/*.log **/gluten-ut/**/hs_err_*.log @@ -939,6 +989,10 @@ jobs: spark-test-spark35-slow: needs: build-native-lib-centos-7 runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + suite: [extended, slow-hive] env: SPARK_TESTING: true LANG: C.UTF-8 #TODO remove after image update @@ -955,30 +1009,33 @@ jobs: run: | rm -rf /opt/shims/spark35 bash .github/workflows/util/install-spark-resources.sh 3.5 - - name: Build and Run unit test for Spark 3.5.5 (slow tests) + - name: Build and Run unit test for Spark 3.5.5 (slow tests, ${{ matrix.suite }}) run: | cd $GITHUB_WORKSPACE/ yum install -y java-17-openjdk-devel export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ - -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \ - -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest - $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ - -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \ - -DtagsToInclude=org.apache.spark.tags.SlowHiveTest + if [ "${{ matrix.suite }}" = "extended" ]; then + $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \ + -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest + else + $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \ + -DtagsToInclude=org.apache.spark.tags.SlowHiveTest + fi - name: Upload test report if: always() uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-report + name: spark-test-spark35-slow-${{ matrix.suite }}-report path: '**/surefire-reports/TEST-*.xml' - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-test-log + name: spark-test-spark35-slow-${{ matrix.suite }}-test-log path: | **/target/*.log **/gluten-ut/**/hs_err_*.log @@ -1130,6 +1187,10 @@ jobs: spark-test-spark40: needs: build-native-lib-centos-7 runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + group: [1, 2, 3] env: SPARK_TESTING: true container: apache/gluten:centos-8-jdk17 @@ -1151,7 +1212,7 @@ jobs: run: | rm -rf /opt/shims/spark40 bash .github/workflows/util/install-spark-resources.sh 4.0 - - name: Build and Run unit test for Spark 4.0.0 with scala-2.13 (other tests) + - name: Build and Run unit test for Spark 4.0.0 with scala-2.13 (other tests, group ${{ matrix.group }}) run: | cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.13 @@ -1159,20 +1220,31 @@ jobs: export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pdelta -Ppaimon \ - -Pspark-ut -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" \ - -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest + TAGS_EXCLUDE="org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest" + if [ "${{ matrix.group }}" = "1" ]; then + $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pdelta -Ppaimon -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" -DtagsToExclude="$TAGS_EXCLUDE" \ + -DwildcardSuites="org.apache.spark.sql.streaming,org.apache.spark.GlutenSortShuffleSuite,org.apache.gluten" + elif [ "${{ matrix.group }}" = "2" ]; then + $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pdelta -Ppaimon -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" -DtagsToExclude="$TAGS_EXCLUDE" \ + -DwildcardSuites="org.apache.spark.sql.execution,org.apache.spark.sql.catalyst,org.apache.spark.sql.errors,org.apache.spark.sql.extension" + else + $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pdelta -Ppaimon -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" -DtagsToExclude="$TAGS_EXCLUDE" \ + -DwildcardSuites="org.apache.spark.sql.GlutenSQL,org.apache.spark.sql.Gluten,org.apache.spark.sql.connector,org.apache.spark.sql.sources,org.apache.spark.sql.hive,org.apache.spark.sql.gluten,org.apache.spark.sql.shim" + fi - name: Upload test report if: always() uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-report + name: spark-test-spark40-group${{ matrix.group }}-report path: '**/surefire-reports/TEST-*.xml' - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-test-log + name: spark-test-spark40-group${{ matrix.group }}-test-log path: | **/target/*.log **/gluten-ut/**/hs_err_*.log @@ -1181,6 +1253,10 @@ jobs: spark-test-spark40-slow: needs: build-native-lib-centos-7 runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + suite: [extended, slow-hive] env: SPARK_TESTING: true LANG: C.UTF-8 #TODO remove after image update @@ -1197,30 +1273,33 @@ jobs: run: | rm -rf /opt/shims/spark40 bash .github/workflows/util/install-spark-resources.sh 4.0 - - name: Build and Run unit test for Spark 4.0 (slow tests) + - name: Build and Run unit test for Spark 4.0 (slow tests, ${{ matrix.suite }}) run: | cd $GITHUB_WORKSPACE/ yum install -y java-17-openjdk-devel export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pdelta -Ppaimon -Pspark-ut \ - -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" \ - -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest - $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pdelta -Pspark-ut \ - -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" \ - -DtagsToInclude=org.apache.spark.tags.SlowHiveTest + if [ "${{ matrix.suite }}" = "extended" ]; then + $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pdelta -Ppaimon -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" \ + -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest + else + $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pdelta -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" \ + -DtagsToInclude=org.apache.spark.tags.SlowHiveTest + fi - name: Upload test report if: always() uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-report + name: spark-test-spark40-slow-${{ matrix.suite }}-report path: '**/surefire-reports/TEST-*.xml' - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-test-log + name: spark-test-spark40-slow-${{ matrix.suite }}-test-log path: | **/target/*.log **/gluten-ut/**/hs_err_*.log @@ -1229,6 +1308,14 @@ jobs: spark-test-spark41: needs: build-native-lib-centos-7 runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + # Split tests into 3 groups to run in parallel and cut the ~2h wall-clock time. + # group1 – streaming tests + # group2 – execution / catalyst / errors / extension tests + # group3 – top-level sql, connector, sources, hive and remaining tests + group: [1, 2, 3] env: SPARK_TESTING: true container: apache/gluten:centos-9-jdk17 @@ -1252,7 +1339,7 @@ jobs: run: | rm -rf /opt/shims/spark41 bash .github/workflows/util/install-spark-resources.sh 4.1 - - name: Build and Run unit test for Spark 4.1.0 with scala-2.13 (other tests) + - name: Build and Run unit test for Spark 4.1.0 with scala-2.13 (other tests, group ${{ matrix.group }}) run: | cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.13 @@ -1260,20 +1347,34 @@ jobs: export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox \ - -Pspark-ut -Pdelta -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/" \ - -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest + TAGS_EXCLUDE="org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest" + if [ "${{ matrix.group }}" = "1" ]; then + # Group 1: streaming + gluten utils + top-level spark tests (~55 classes) + $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pspark-ut -Pdelta \ + -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/" -DtagsToExclude="$TAGS_EXCLUDE" \ + -DwildcardSuites="org.apache.spark.sql.streaming,org.apache.spark.GlutenSortShuffleSuite,org.apache.gluten" + elif [ "${{ matrix.group }}" = "2" ]; then + # Group 2: execution + catalyst + errors + extension tests (~140 classes) + $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pspark-ut -Pdelta \ + -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/" -DtagsToExclude="$TAGS_EXCLUDE" \ + -DwildcardSuites="org.apache.spark.sql.execution,org.apache.spark.sql.catalyst,org.apache.spark.sql.errors,org.apache.spark.sql.extension" + else + # Group 3: top-level sql, connector, sources, hive and remaining tests (~205 classes) + $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pspark-ut -Pdelta \ + -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/" -DtagsToExclude="$TAGS_EXCLUDE" \ + -DwildcardSuites="org.apache.spark.sql.GlutenSQL,org.apache.spark.sql.Gluten,org.apache.spark.sql.connector,org.apache.spark.sql.sources,org.apache.spark.sql.hive,org.apache.spark.sql.gluten,org.apache.spark.sql.shim" + fi - name: Upload test report if: always() uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-report + name: spark-test-spark41-group${{ matrix.group }}-report path: '**/surefire-reports/TEST-*.xml' - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-test-log + name: spark-test-spark41-group${{ matrix.group }}-test-log path: | **/target/*.log **/gluten-ut/**/hs_err_*.log @@ -1282,6 +1383,11 @@ jobs: spark-test-spark41-slow: needs: build-native-lib-centos-7 runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + # Run ExtendedSQLTest and SlowHiveTest in parallel instead of sequentially. + suite: [extended, slow-hive] env: SPARK_TESTING: true LANG: C.UTF-8 #TODO remove after image update @@ -1298,30 +1404,33 @@ jobs: run: | rm -rf /opt/shims/spark41 bash .github/workflows/util/install-spark-resources.sh 4.1 - - name: Build and Run unit test for Spark 4.1 (slow tests) + - name: Build and Run unit test for Spark 4.1 (slow tests, ${{ matrix.suite }}) run: | cd $GITHUB_WORKSPACE/ yum install -y java-17-openjdk-devel export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pspark-ut \ - -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/" \ - -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest - $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pspark-ut \ - -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/" \ - -DtagsToInclude=org.apache.spark.tags.SlowHiveTest + if [ "${{ matrix.suite }}" = "extended" ]; then + $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/" \ + -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest + else + $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/" \ + -DtagsToInclude=org.apache.spark.tags.SlowHiveTest + fi - name: Upload test report if: always() uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-report + name: spark-test-spark41-slow-${{ matrix.suite }}-report path: '**/surefire-reports/TEST-*.xml' - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-test-log + name: spark-test-spark41-slow-${{ matrix.suite }}-test-log path: | **/target/*.log **/gluten-ut/**/hs_err_*.log