v2.0

.gitlab-ci.yml

@@ -1,48 +1,153 @@
-#========================== STEPS USED IN WORKFLOWS ====================================
-.step_python_setup_install: &step_python_setup_install
-  - echo -e "\e[95m===== Setup Python"
-  - python --version ; pip --version
-  - python3 -m venv venv 
-  - source venv/bin/activate
+stages:
+  - build
+  - test
+  - notify
 
-.step_install_pyrrha_test: &step_install_pyrrha_test
-  - echo -e "\e[95m===== Install Pyrrha with test extension"
-  - pip install '.[test]'
+# Paths that should trigger test and build pipelines.
+# Anchored here so test and build jobs stay in sync.
+.source_paths: &source_paths
+  - tests/**/*
+  - src/**/*
+  - pyproject.toml
+  - .gitlab-ci.yml
+  - ci/ghidra/*
 
-.step_configure_disassembler: &step_configure_disassembler
-  - if [[ ${DISASSEMBLER} == "ida" ]]; then
-    echo -e "\e[95m===== Configure IDA" &&
-    mkdir -p ~/.idapro/ &&
-    echo $KEY | base64 -d  > ~/.idapro/$KEY_NAME &&
-    echo $REG | base64 -d  > ~/.idapro/ida.reg &&
-    export IDA_LICENSE=keyfile=$KEY_NAME &&
-    idapyswitch -a ; fi;
+#======================== BUILD DOCKER IMAGE AND PUSH TO REGISTRY ======================
+build_image:
+  stage: build
+  image: docker:29-dind
+  tags:
+    - dind
+  rules:
+    # Release tags always build, regardless of which files changed.
+    - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+$/'
+    # Branch builds only when source, tests, or packaging metadata changed.
+    - if: '$CI_COMMIT_BRANCH == "main"'
+      changes: *source_paths
+    - if: '$CI_COMMIT_BRANCH == "dev"'
+      changes: *source_paths
+  parallel:
+    matrix:
+      - BACKEND: "ida"
+        VERSION: [91, 93]
+        LATEST: 93
+      - BACKEND: "ghidra"
+        LATEST: "12.1"
+        VERSION: "12.0.4"
+        GHIDRA_SHA256: "c3b458661d69e26e203d739c0c82d143cc8a4a29d9e571f099c2cf4bda62a120"
+        DATE: "20260303"
+      - BACKEND: "ghidra"
+        LATEST: "12.1"
+        VERSION: "12.1"
+        GHIDRA_SHA256: "aa5cbcbbf48f41ca185fce900e19592f1ade4cd5994eb6e0ede468dac8a6f302"
+        DATE: "20260513"
+  variables:
+    DOCKER_IMAGE_NAME: $CI_REGISTRY_IMAGE/pyrrha-$BACKEND
+    DOCKER_HOST: unix:///var/run/docker.sock
+    DOCKER_TLS_CERTDIR: ""
+  before_script:
+    - echo "$CA_CERT" > /usr/local/share/ca-certificates/local-ca.pem
+    - update-ca-certificates
+    - |
+      dockerd-entrypoint.sh --host=unix:///var/run/docker.sock &
+      for i in $(seq 1 30); do
+        if docker info >/dev/null 2>&1; then
+          echo "Docker daemon is ready"
+          break
+        fi
+        echo "Waiting for docker daemon... ($i/30)"
+        sleep 2
+      done
+    - echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY"
+    - apk add --no-cache bash
+  script:
+    # Resolve the pyrrha version component from the ref that triggered the pipeline.
+    #   - tag v1.2.3  -> "1.2.3"
+    #   - main branch -> "main"
+    #   - dev branch  -> "dev"
+    - |
+      if [[ -n "$CI_COMMIT_TAG" && "$CI_COMMIT_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+        PYRRHA_VERSION="${CI_COMMIT_TAG#v}"
+        REF_KIND="tag"
+      elif [[ "$CI_COMMIT_BRANCH" == "main" ]]; then
+        PYRRHA_VERSION="main"
+        REF_KIND="main"
+      elif [[ "$CI_COMMIT_BRANCH" == "dev" ]]; then
+        PYRRHA_VERSION="dev"
+        REF_KIND="dev"
+      else
+        echo "Unexpected ref (branch=$CI_COMMIT_BRANCH, tag=$CI_COMMIT_TAG) — aborting."
+        exit 1
+      fi
+      echo "Resolved PYRRHA_VERSION=$PYRRHA_VERSION (REF_KIND=$REF_KIND)"
+
+    - |
+      if [ "$BACKEND" = "ghidra" ]; then
+        ci/ghidra/build.sh --version "$VERSION" --date "$DATE" --sha256 "$GHIDRA_SHA256" --name "$CI_REGISTRY_IMAGE/ci/$BACKEND"
+      fi
+    - |
+      if [ "$BACKEND" = "ghidra" ]; then
+        docker push "$CI_REGISTRY_IMAGE/ci/$BACKEND:$VERSION"
+      fi
 
-.step_gen_artifacts: &step_gen_artifacts
-  - echo -e "\e[95m===== Generate artifacts"
-  - mkdir -p ${ARTIFACTS} 
-  - cp -r tests ${ARTIFACTS}
-  - (cd ${ARTIFACTS} && pyrrha $MAPPER --db ${DB} --debug tests/test_fw ${MAPPER_OPTIONS})
-  - ls ${ARTIFACTS}
+    # Primary image tag: <backend-version>-<pyrrha-version>
+    - PRIMARY_TAG="${VERSION}-${PYRRHA_VERSION}"
+    - echo "Building $DOCKER_IMAGE_NAME:$PRIMARY_TAG"
+    - |
+      docker build --pull \
+        -t "$DOCKER_IMAGE_NAME:$PRIMARY_TAG" \
+        -f ci/pyrrha/Dockerfile \
+        --build-arg DISASS_IMAGE=$CI_REGISTRY_IMAGE/ci/${BACKEND} \
+        --build-arg DISASS_IMAGE_VERSION=$VERSION \
+        .
+    - docker push "$DOCKER_IMAGE_NAME:$PRIMARY_TAG"
 
-.step_run_tests: &step_run_tests
-  - echo -e "\e[95m===== Tests"
-  - coverage run --source=${TEST_COVERAGE_SOURCE} -m pytest  --junitxml=report.xml -vvv -x ${TEST_SUP_OPTIONS} ${TEST_PATH}
-  - coverage xml
-  - coverage report
+    # Additional floating tags, only for the LATEST backend version of this matrix row.
+    #   - main  -> `latest` and `<latest-backend>`
+    #   - dev   -> `latest-dev`
+    #   - tag   -> `stable`
+    - |
+      if [[ "$VERSION" == "$LATEST" ]]; then
+        case "$REF_KIND" in
+          main)
+            for t in "latest" "$VERSION"; do
+              docker tag "$DOCKER_IMAGE_NAME:$PRIMARY_TAG" "$DOCKER_IMAGE_NAME:$t"
+              docker push "$DOCKER_IMAGE_NAME:$t"
+            done
+            ;;
+          dev)
+            docker tag "$DOCKER_IMAGE_NAME:$PRIMARY_TAG" "$DOCKER_IMAGE_NAME:latest-dev"
+            docker push "$DOCKER_IMAGE_NAME:latest-dev"
+            ;;
+          tag)
+            docker tag "$DOCKER_IMAGE_NAME:$PRIMARY_TAG" "$DOCKER_IMAGE_NAME:stable"
+            docker push "$DOCKER_IMAGE_NAME:stable"
+            ;;
+        esac
+      fi
+  after_script:
+    - docker logout $CI_REGISTRY
 
 #========================== OBJECTS TESTS ====================================
 
-test_data_structures: 
+test_data_structures:
   stage: test
-  before_script: 
-    - *step_python_setup_install
-    - *step_install_pyrrha_test
+  # Only run tests when source, tests, or packaging metadata changed.
+  # Inherited by test_fs and test_fs-cg via `extends`.
+  rules:
+    - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+$/'
+    - changes: *source_paths
+  before_script:
+    - echo -e "\e[95m===== Install Pyrrha with test extension"
+    - pip install '.[test]'
   script:
-    - *step_run_tests
+    - echo -e "\e[95m===== Tests"
+    - coverage run --source=${TEST_COVERAGE_SOURCE} -m pytest  --junitxml=report.xml -vvv -x ${TEST_SUP_OPTIONS} ${TEST_PATH}
+    - coverage xml
+    - coverage report
   image: python:latest
   variables:
-    TEST_COVERAGE_SOURCE: pyrrha_mapper.common.objects
+    TEST_COVERAGE_SOURCE: pyrrha_mapper.mappers.objects
     TEST_PATH: tests/test_filesystem_objects.py
   coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
   artifacts:
@@ -52,88 +157,132 @@ test_data_structures:
         coverage_format: cobertura
         path: coverage.xml
 
+test_decomp_objects:
+  extends:
+    - test_data_structures
+  variables:
+    TEST_COVERAGE_SOURCE: pyrrha_mapper.mappers.decomp_objects
+    TEST_PATH: tests/test_decomp_objects.py
+
 #========================== MAPPERS TESTS ====================================
-.run_pyrrha_test_artifacts:
-  stage: test
-  before_script: 
-    - *step_python_setup_install
-    - *step_install_pyrrha_test
-  script:
-    - *step_gen_artifacts
-    - *step_run_tests
-  coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
+test_fs:
+  extends:
+    - test_data_structures
   artifacts:
     name: db_$CI_JOB_NAME_SLUG
-    when: always
     paths:
-      - ${ARTIFACTS}/${DB}.srctrldb
-      - ${ARTIFACTS}/${DB}.srctrlprj
-    reports:
-      junit: report.xml
-      coverage_report:
-        coverage_format: cobertura
-        path: coverage.xml
-  variables:
-    ARTIFACTS: tmp/artifacts
-
-test_fs: 
-  extends: 
-    - .run_pyrrha_test_artifacts
-  image: python:latest
+      - test_artifacts/
+    when: always
   variables:
     DB: fs
     MAPPER: fs
-    TEST_COVERAGE_SOURCE: pyrrha_mapper.common.filesystem_mapper,pyrrha_mapper.fs
+    TEST_COVERAGE_SOURCE: pyrrha_mapper.mappers.imports_mapper
     TEST_PATH: tests/test_cli.py::TestFSMapper
-
+    PYTEST_ARTIFACTS_DIR: test_artifacts
 
-.test_fs-cg:
-  extends: 
-    - .run_pyrrha_test_artifacts
-  before_script: 
-    - !reference [.run_pyrrha_test_artifacts, before_script]
-    - *step_configure_disassembler
-  image: 
-    name: $CONTAINER_PATH/${DISASSEMBLER}:${VERSION}
+test_fs-cg:
+  extends:
+    - test_fs
+  # Pull the image built in this pipeline when on main, dev, or a release tag
+  # (tagged as <version>-main, <version>-dev, or <version>-<semver>).
+  # For any other branch (e.g. feature branches) no image is built, so we
+  # fall back to `latest` which always tracks the last successful main build.
+  image:
+    name: $CI_REGISTRY_IMAGE/pyrrha-${BACKEND}:${DISASS_IMAGE_TAG}
     docker:
       user: user
+  before_script:
+    - echo -e "\e[95m===== Install Pyrrha with test extension"
+    - pip install '.[test]'
   variables:
-    DB: ${DISASSEMBLER}_${VERSION}_${EXPORTER}
+    DB: ${BACKEND}_${VERSION}
     MAPPER: fs-cg
-    MAPPER_OPTIONS: '--disassembler ${DISASSEMBLER} --exporter ${EXPORTER}'
-    TEST_COVERAGE_SOURCE: pyrrha_mapper.common.filesystem_mapper,pyrrha_mapper.intercg
+    MAPPER_OPTIONS: '--backend ${BACKEND}'
+    TEST_COVERAGE_SOURCE: pyrrha_mapper.mappers.intercg_bin_loader,pyrrha_mapper.mappers.intercg_mapper
     TEST_PATH: tests/test_cli.py::TestFsCgMapper
     TEST_SUP_OPTIONS: ${MAPPER_OPTIONS}
-
-test_fs-cg_ghidra:
-  extends:
-    - .test_fs-cg
-  variables:
-    DISASSEMBLER: ghidra
+    HEXRAYS_LICENSE: "${IDA_LICENSE}"
+    # Default: fall back to latest (last successful main build).
+    # Overridden per-ref by the rules below.
+    DISASS_IMAGE_TAG: "latest"
+    # Point pyghidra at the JDK as proper job variables so they are present in
+    # the environment of EVERY process in the job — including the
+    # `coverage run -m pytest` process and any multiprocessing workers spawned
+    # by the mapper (which use the "spawn" start method and re-exec Python).
+    # Shell `export`s in before_script are not guaranteed to reach those
+    # re-exec'd children, but GitLab job variables always are.
+    # /opt/java/openjdk is the temurin JDK path, confirmed present in all
+    # ghidra images. pyghidra reads JAVA_HOME_OVERRIDE first (launcher.py:202),
+    # bypassing LaunchSupport entirely.
+    JAVA_HOME: /opt/java/openjdk
+    JAVA_HOME_OVERRIDE: /opt/java/openjdk
+  rules:
+    - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+$/'
+      variables:
+        DISASS_IMAGE_TAG: "${VERSION}-${CI_COMMIT_TAG#v}"  # strip the leading v, e.g. 12.0.4-1.2.3
+    - if: '$CI_COMMIT_BRANCH == "main"'
+      changes: *source_paths
+      variables:
+        DISASS_IMAGE_TAG: "${VERSION}-main"
+    - if: '$CI_COMMIT_BRANCH == "dev"'
+      changes: *source_paths
+      variables:
+        DISASS_IMAGE_TAG: "${VERSION}-dev"
+    - changes: *source_paths   # any other branch: use latest
   parallel:
     matrix:
-      - VERSION: 11.1.2
-        EXPORTER: binexport
+      - BACKEND: "ida"
+        VERSION: [91, 93]
+      - BACKEND: "ghidra"
+        VERSION: ["12.0.4"]
 
-test_fs-cg_ida:
+test_decomp:
   extends:
-    - .test_fs-cg
+    - test_fs-cg
   variables:
-    DISASSEMBLER: ida
-  parallel:
-    matrix:
-      - VERSION: 84
-        EXPORTER: [quokka, binexport]
-      - VERSION: 91
-        EXPORTER: quokka
+    DB: decomp_${BACKEND}_${VERSION}
+    MAPPER: decomp
+    MAPPER_OPTIONS: '--backend ${BACKEND}'
+    TEST_COVERAGE_SOURCE: pyrrha_mapper.mappers.decomp_mapper,pyrrha_mapper.mappers.decomp_objects
+    TEST_PATH: tests/test_cli.py::TestDecompMapper
+    TEST_SUP_OPTIONS: ${MAPPER_OPTIONS}
+
+#========================== TRIGGER INTERNAL DOC UPDATE ================================
+.trigger_docs_base:
+  stage: notify
+  trigger:
+    project: firmware-re/cartography/pyrrha-internal-documentation
+    branch: main
+    # strategy: depend  # flip on to surface downstream failure on pyrrha
+                        # CI. Off for now — a broken docs build shouldn't
+                        # block pyrrha.
+  variables:
+    UPSTREAM_PIPELINE_URL: $CI_PIPELINE_URL
+
+trigger_docs_main:
+  extends: .trigger_docs_base
+  variables:
+    UPSTREAM_REF: "main"
+    UPSTREAM_SHA: $CI_COMMIT_SHA
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main"
+      when: on_success
+
+trigger_docs_dev:
+  extends: .trigger_docs_base
+  variables:
+    UPSTREAM_REF: "dev"
+    UPSTREAM_SHA: $CI_COMMIT_SHA
+  rules:
+    - if: $CI_COMMIT_BRANCH == "dev"
+      when: on_success
+
+trigger_docs_tag:
+  extends: .trigger_docs_base
+  variables:
+    UPSTREAM_REF: $CI_COMMIT_TAG
+    UPSTREAM_TAG: $CI_COMMIT_TAG
+    UPSTREAM_SHA: $CI_COMMIT_SHA
   rules:
-    - if: $VERSION == "84"
-      variables: 
-        KEY: $IDA_KEY
-        KEY_NAME: ida.key
-        REG: $IDA84_REG
-    - if: $VERSION == "91"
-      variables: 
-        KEY: $LICENSE
-        KEY_NAME: ida_license.hexlic
-        REG: $IDA_REG
+    - if: $CI_COMMIT_TAG =~ /^v\d+\.\d+/
+      when: on_success