feat: support Spark 4

.github/workflows/ci.yml

@@ -10,16 +10,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Set up JDK 17
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@v5
         with:
           distribution: "temurin"
           java-version: 17
 
       - name: Cache Maven dependencies
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
@@ -32,17 +32,17 @@ jobs:
     runs-on: ubuntu-latest
     needs: format
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
       - name: Set up Maven Central Repository
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@v5
         with:
           java-version: 17
           distribution: "adopt"
       # Also exclude vtl-prov because of vtl-sdmx scope test dependency
       - name: Test
-        run: mvn test -pl '!vtl-sdmx,!vtl-prov'
+        run: mvn install -pl vtl-parser && mvn test -pl '!vtl-sdmx,!vtl-prov'
   test:
     name: Run Trevas tests
     if: (github.repository != 'InseeFr/Trevas' &&
@@ -53,15 +53,15 @@ jobs:
     runs-on: ubuntu-latest
     needs: format
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
       - name: Set up Maven Central Repository
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@v5
         with:
           java-version: 17
           distribution: "adopt"
-      - uses: s4u/maven-settings-action@v3.0.0
+      - uses: s4u/maven-settings-action@v4.0.0
         with:
           githubServer: false
           servers: |
@@ -71,22 +71,70 @@ jobs:
                 "password": "${{ secrets.GH_PACKAGES_PASSWORD }}"
             }]
       - name: Test
-        run: mvn test
+        run: mvn install -pl vtl-parser && mvn test
+
+  spark-integration:
+    name: Spark ${{ matrix.spark-label }}
+    runs-on: ubuntu-latest
+    needs: format
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - module: vtl-spark
+            spark-label: "3.x (Scala 2.12)"
+            extra-args: ""
+          - module: vtl-spark4
+            spark-label: "4.x (Scala 2.13)"
+            extra-args: ""
+          - module: vtl-prov
+            spark-label: "vtl-prov + Spark 4 (tests)"
+            extra-args: "-Pspark4-tests"
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+      - name: Set up JDK 17
+        uses: actions/setup-java@v5
+        with:
+          java-version: 17
+          distribution: temurin
+      - name: Cache Maven dependencies
+        uses: actions/cache@v5
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+      - uses: s4u/maven-settings-action@v4.0.0
+        with:
+          githubServer: false
+          servers: |
+            [{
+                "id": "Github",
+                "username": "${{ secrets.GH_PACKAGES_USERNAME }}",
+                "password": "${{ secrets.GH_PACKAGES_PASSWORD }}"
+            }]
+      # Install shaded vtl-parser first, then exclude it from the reactor so dependent
+      # modules resolve the installed jar (JPMS fr.insee.vtl.parser.shaded), not target/classes.
+      - name: Test ${{ matrix.module }} (and upstream modules)
+        run: mvn install -pl vtl-parser -DskipTests && mvn -B test -pl ${{ matrix.module }} -am --projects '!vtl-parser' ${{ matrix.extra-args }}
+
   package:
     name: Package Trevas modules
     # Filter thanks to test job
     runs-on: ubuntu-latest
-    needs: test
+    needs: [ test, spark-integration ]
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
       - name: Set up Maven Central Repository
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@v5
         with:
           java-version: 17
           distribution: "adopt"
-      - uses: s4u/maven-settings-action@v3.0.0
+      - uses: s4u/maven-settings-action@v4.0.0
         with:
           githubServer: false
           servers: |
@@ -134,16 +182,16 @@ jobs:
     name: Publish Trevas modules on Maven
     if: startsWith(github.event.ref, 'refs/tags/v')
     # Temp update because of sonar token issue
-    needs: test
+    needs: [ test, spark-integration ]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
       - name: Import GPG Key
-        uses: crazy-max/ghaction-import-gpg@v1
+        uses: crazy-max/ghaction-import-gpg@v7
         env:
           GPG_PRIVATE_KEY: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }}
           PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
-      - uses: s4u/maven-settings-action@v3.0.0
+      - uses: s4u/maven-settings-action@v4.0.0
         with:
           githubServer: false
           servers: |

.github/workflows/docs.yml

@@ -10,8 +10,8 @@ jobs:
     if: github.repository == 'inseefr/trevas'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-node@v4
+      - uses: actions/checkout@v6
+      - uses: actions/setup-node@v5
         with:
           node-version: 24
       - name: Install dependencies
@@ -21,7 +21,7 @@ jobs:
         run: yarn build
         working-directory: docs
       - name: Deploy to GitHub Pages
-        uses: peaceiris/actions-gh-pages@v3
+        uses: peaceiris/actions-gh-pages@v4
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           # Build output to publish to the `gh-pages` branch:

.github/workflows/tck-vtl-tf.yml

@@ -12,15 +12,15 @@ jobs:
 
     steps:
       - name: Checkout main project
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           fetch-depth: 0
 
       - name: Clone vtl spec repo (branch fix/tck-2.1)
         run: git clone --branch fix/tck-2.1 https://github.com/sdmx-twg/vtl.git
 
       - name: Install Python 3
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: '3.11'
 
@@ -34,12 +34,12 @@ jobs:
           mv vtl/tck/v2.1.zip coverage/src/main/resources/
 
       - name: Set up Java
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@v5
         with:
           distribution: 'temurin'
           java-version: '17'
 
-      - uses: s4u/maven-settings-action@v3.0.0
+      - uses: s4u/maven-settings-action@v4.0.0
         with:
           githubServer: false
           servers: |
@@ -50,7 +50,7 @@ jobs:
             }]
 
       - name: Cache Maven packages
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
@@ -67,11 +67,26 @@ jobs:
         if: always()
         run: python3 coverage/scripts/prettify_tck_surefire_xml.py
 
+      # only-summary: table per XML only (no per-test expansion). max-annotations: 0 avoids PR file annotations.
       - name: Publish JUnit test results
-        uses: dorny/test-reporter@v2
+        uses: dorny/test-reporter@v3
         if: always()
         with:
           name: JUnit Test Report
           path: coverage/target/surefire-reports/*.xml
           reporter: java-junit
+          only-summary: 'true'
+          max-annotations: 0
           fail-on-error: 'false'
+
+      - name: Generate TCK scripts report
+        if: always()
+        run: python3 coverage/scripts/render_tck_job_summary.py
+
+      - name: Upload full TCK script report artifact
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tck-scripts-report
+          path: coverage/target/tck-scripts-report.md
+          if-no-files-found: ignore

.gitignore

@@ -7,4 +7,6 @@ target/
 **/.settings/
 **/pom.xml.versionsBackup
 
-*-dev.properties
+*-dev.properties
+**/__pycache__/
+**/*.pyc