diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c018e5021..3816f720a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,16 +10,16 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set up JDK 17 - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: "temurin" java-version: 17 - name: Cache Maven dependencies - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} @@ -32,17 +32,17 @@ jobs: runs-on: ubuntu-latest needs: format steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up Maven Central Repository - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: "adopt" # Also exclude vtl-prov because of vtl-sdmx scope test dependency - name: Test - run: mvn test -pl '!vtl-sdmx,!vtl-prov' + run: mvn install -pl vtl-parser && mvn test -pl '!vtl-sdmx,!vtl-prov' test: name: Run Trevas tests if: (github.repository != 'InseeFr/Trevas' && @@ -53,15 +53,15 @@ jobs: runs-on: ubuntu-latest needs: format steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up Maven Central Repository - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: "adopt" - - uses: s4u/maven-settings-action@v3.0.0 + - uses: s4u/maven-settings-action@v4.0.0 with: githubServer: false servers: | @@ -71,22 +71,70 @@ jobs: "password": "${{ secrets.GH_PACKAGES_PASSWORD }}" }] - name: Test - run: mvn test + run: mvn install -pl vtl-parser && mvn test + + spark-integration: + name: Spark ${{ matrix.spark-label }} + runs-on: ubuntu-latest + needs: format + strategy: + fail-fast: false + matrix: + include: + - module: vtl-spark + spark-label: "3.x (Scala 2.12)" + extra-args: "" + - module: vtl-spark4 + spark-label: "4.x (Scala 2.13)" + extra-args: "" + - module: vtl-prov + spark-label: "vtl-prov + Spark 4 (tests)" + extra-args: "-Pspark4-tests" + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Set up JDK 17 + uses: actions/setup-java@v5 + with: + java-version: 17 + distribution: temurin + - name: Cache Maven dependencies + uses: actions/cache@v5 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - uses: s4u/maven-settings-action@v4.0.0 + with: + githubServer: false + servers: | + [{ + "id": "Github", + "username": "${{ secrets.GH_PACKAGES_USERNAME }}", + "password": "${{ secrets.GH_PACKAGES_PASSWORD }}" + }] + # Install shaded vtl-parser first, then exclude it from the reactor so dependent + # modules resolve the installed jar (JPMS fr.insee.vtl.parser.shaded), not target/classes. + - name: Test ${{ matrix.module }} (and upstream modules) + run: mvn install -pl vtl-parser -DskipTests && mvn -B test -pl ${{ matrix.module }} -am --projects '!vtl-parser' ${{ matrix.extra-args }} + package: name: Package Trevas modules # Filter thanks to test job runs-on: ubuntu-latest - needs: test + needs: [ test, spark-integration ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up Maven Central Repository - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: "adopt" - - uses: s4u/maven-settings-action@v3.0.0 + - uses: s4u/maven-settings-action@v4.0.0 with: githubServer: false servers: | @@ -134,16 +182,16 @@ jobs: name: Publish Trevas modules on Maven if: startsWith(github.event.ref, 'refs/tags/v') # Temp update because of sonar token issue - needs: test + needs: [ test, spark-integration ] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Import GPG Key - uses: crazy-max/ghaction-import-gpg@v1 + uses: crazy-max/ghaction-import-gpg@v7 env: GPG_PRIVATE_KEY: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} - - uses: s4u/maven-settings-action@v3.0.0 + - uses: s4u/maven-settings-action@v4.0.0 with: githubServer: false servers: | diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 26b09d597..cff15e0f8 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -10,8 +10,8 @@ jobs: if: github.repository == 'inseefr/trevas' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@v6 + - uses: actions/setup-node@v5 with: node-version: 24 - name: Install dependencies @@ -21,7 +21,7 @@ jobs: run: yarn build working-directory: docs - name: Deploy to GitHub Pages - uses: peaceiris/actions-gh-pages@v3 + uses: peaceiris/actions-gh-pages@v4 with: github_token: ${{ secrets.GITHUB_TOKEN }} # Build output to publish to the `gh-pages` branch: diff --git a/.github/workflows/tck-vtl-tf.yml b/.github/workflows/tck-vtl-tf-spark3.yml similarity index 58% rename from .github/workflows/tck-vtl-tf.yml rename to .github/workflows/tck-vtl-tf-spark3.yml index c740d5a48..0bd571b06 100644 --- a/.github/workflows/tck-vtl-tf.yml +++ b/.github/workflows/tck-vtl-tf-spark3.yml @@ -1,4 +1,4 @@ -name: Run VTL TF TCK +name: Run VTL TF TCK (Spark 3) on: push: @@ -7,12 +7,15 @@ on: branches: [ master, develop ] jobs: - test: + tck-spark3: + name: TCK VTL v2.1 (Spark 3) runs-on: ubuntu-latest + env: + TCK_REPORT_TITLE: 'TCK VTL v2.1 (Spark 3)' steps: - name: Checkout main project - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 @@ -20,7 +23,7 @@ jobs: run: git clone --branch fix/tck-2.1 https://github.com/sdmx-twg/vtl.git - name: Install Python 3 - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: '3.11' @@ -34,12 +37,12 @@ jobs: mv vtl/tck/v2.1.zip coverage/src/main/resources/ - name: Set up Java - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: 'temurin' java-version: '17' - - uses: s4u/maven-settings-action@v3.0.0 + - uses: s4u/maven-settings-action@v4.0.0 with: githubServer: false servers: | @@ -50,28 +53,43 @@ jobs: }] - name: Cache Maven packages - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + key: ${{ runner.os }}-maven-tck-spark3-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- - - name: Build and run tests + - name: Build dependencies and run TCK (coverage only) continue-on-error: true - run: mvn clean test --batch-mode + run: | + mvn clean install -pl coverage -am -DskipTests --batch-mode + mvn test -pl coverage --batch-mode - # dorny/test-reporter titles come from Surefire ; rewrite TCK names so - # passed tests show "Test N" + newline + tab + path (see coverage/scripts/prettify_tck_surefire_xml.py). - name: Prettify TCK Surefire XML for GitHub if: always() run: python3 coverage/scripts/prettify_tck_surefire_xml.py - name: Publish JUnit test results - uses: dorny/test-reporter@v2 + uses: dorny/test-reporter@v3 if: always() with: - name: JUnit Test Report + name: JUnit (Spark 3) path: coverage/target/surefire-reports/*.xml reporter: java-junit + only-summary: 'true' + max-annotations: 0 fail-on-error: 'false' + fail-on-empty: 'false' + + - name: Generate TCK scripts report + if: always() + run: python3 coverage/scripts/render_tck_job_summary.py + + - name: Upload full TCK script report artifact + if: always() + uses: actions/upload-artifact@v7 + with: + name: tck-scripts-report-spark3 + path: coverage/target/tck-scripts-report.md + if-no-files-found: ignore diff --git a/.github/workflows/tck-vtl-tf-spark4.yml b/.github/workflows/tck-vtl-tf-spark4.yml new file mode 100644 index 000000000..6d188714e --- /dev/null +++ b/.github/workflows/tck-vtl-tf-spark4.yml @@ -0,0 +1,95 @@ +name: Run VTL TF TCK (Spark 4) + +on: + push: + branches: [ '**' ] + pull_request: + branches: [ master, develop ] + +jobs: + tck-spark4: + name: TCK VTL v2.1 (Spark 4) + runs-on: ubuntu-latest + env: + TCK_REPORT_TITLE: 'TCK VTL v2.1 (Spark 4)' + + steps: + - name: Checkout main project + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Clone vtl spec repo (branch fix/tck-2.1) + run: git clone --branch fix/tck-2.1 https://github.com/sdmx-twg/vtl.git + + - name: Install Python 3 + uses: actions/setup-python@v6 + with: + python-version: '3.11' + + - name: Run TCK generator script + run: | + DOC_VERSION=v2.1 python3 vtl/scripts/generate_tck_files.py + + - name: Move generated TCK zip to resources + run: | + mkdir -p coverage/src/main/resources + mv vtl/tck/v2.1.zip coverage/src/main/resources/ + + - name: Set up Java + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: '17' + + - uses: s4u/maven-settings-action@v4.0.0 + with: + githubServer: false + servers: | + [{ + "id": "Github", + "username": "${{ secrets.GH_PACKAGES_USERNAME }}", + "password": "${{ secrets.GH_PACKAGES_PASSWORD }}" + }] + + - name: Cache Maven packages + uses: actions/cache@v5 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-tck-spark4-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Build dependencies and run TCK (coverage only, Spark 4 profile) + continue-on-error: true + run: | + mvn clean install -pl coverage -am -DskipTests --batch-mode -Pspark4-tck + mvn test -pl coverage --batch-mode -Pspark4-tck + + - name: Prettify TCK Surefire XML for GitHub + if: always() + run: python3 coverage/scripts/prettify_tck_surefire_xml.py + + - name: Publish JUnit test results + uses: dorny/test-reporter@v3 + if: always() + with: + name: JUnit (Spark 4) + path: coverage/target/surefire-reports/*.xml + reporter: java-junit + only-summary: 'true' + max-annotations: 0 + fail-on-error: 'false' + fail-on-empty: 'false' + + - name: Generate TCK scripts report + if: always() + run: python3 coverage/scripts/render_tck_job_summary.py + + - name: Upload full TCK script report artifact + if: always() + uses: actions/upload-artifact@v7 + with: + name: tck-scripts-report-spark4 + path: coverage/target/tck-scripts-report.md + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index 87c2c4a52..9ee113d25 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,6 @@ target/ **/.settings/ **/pom.xml.versionsBackup -*-dev.properties \ No newline at end of file +*-dev.properties +**/__pycache__/ +**/*.pyc \ No newline at end of file diff --git a/coverage/pom.xml b/coverage/pom.xml index bc86b4215..8c56b1b5c 100644 --- a/coverage/pom.xml +++ b/coverage/pom.xml @@ -18,6 +18,9 @@ ${project.basedir}/../coverage/target/site/jacoco-aggregate/jacoco.xml + + vtl-spark + TCK VTL v2.1 (Spark 3) @@ -48,7 +51,7 @@ fr.insee.trevas - vtl-spark + ${trevas.spark.artifact} 2.4.0-SNAPSHOT @@ -70,6 +73,9 @@ 3.5.4 --add-exports java.base/sun.nio.ch=ALL-UNNAMED + + ${tck.suite.title} + + + + + spark4-tck + + vtl-spark4 + TCK VTL v2.1 (Spark 4) + + + \ No newline at end of file diff --git a/coverage/scripts/render_tck_job_summary.py b/coverage/scripts/render_tck_job_summary.py new file mode 100644 index 000000000..13cb188a9 --- /dev/null +++ b/coverage/scripts/render_tck_job_summary.py @@ -0,0 +1,501 @@ +#!/usr/bin/env python3 +""" +After `mvn test` (+ optional prettify), build a plain-text-friendly Markdown report: + + TCK scripts output + Source zip: … + Total cases: … + + Per test (same shape for pass / skip / fail): + ✅ Test N + folder » with » chevrons » ex_k + +