Merging dev branch into main

.gitignore

@@ -15,4 +15,12 @@ data/
 *.til
 *.nam
 .coverage
-*.log
+*.log
+*.run
+*.hexlic
+*.reg
+*.key
+*.i64
+*.BinExport
+*.quokka
+.gitlab-ci-local*

.gitlab-ci.yml

@@ -1,25 +1,140 @@
-image: python:latest
+#========================== STEPS USED IN WORKFLOWS ====================================
+.step_python_setup_install: &step_python_setup_install
+  - echo -e "\e[95m===== Setup Python"
+  - python --version ; pip --version
+  - python3 -m venv venv 
+  - source venv/bin/activate
 
+.step_install_pyrrha_test: &step_install_pyrrha_test
+  - echo -e "\e[95m===== Install Pyrrha with test extension"
+  - pip install '.[test]'
 
-before_script:
-  - python --version ; pip --version  # For debugging
-  - pip install virtualenv
-  - virtualenv venv
-  - source venv/bin/activate
+.step_configure_disassembler: &step_configure_disassembler
+  - if [[ ${DISASSEMBLER} == "ida" ]]; then
+    echo -e "\e[95m===== Configure IDA" &&
+    mkdir -p ~/.idapro/ &&
+    echo $KEY | base64 -d  > ~/.idapro/$KEY_NAME &&
+    echo $REG | base64 -d  > ~/.idapro/ida.reg &&
+    export IDA_LICENSE=keyfile=$KEY_NAME &&
+    idapyswitch -a ; fi;
+
+.step_gen_artifacts: &step_gen_artifacts
+  - echo -e "\e[95m===== Generate artifacts"
+  - mkdir -p ${ARTIFACTS} 
+  - cp -r tests ${ARTIFACTS}
+  - (cd ${ARTIFACTS} && pyrrha $MAPPER --db ${DB} -j $(nproc) tests/test_fw ${MAPPER_OPTIONS})
+
+.step_run_tests: &step_run_tests
+  - echo -e "\e[95m===== Tests"
+  - coverage run --source=${TEST_COVERAGE_SOURCE} -m pytest  --junitxml=report.xml -vvv -x ${TEST_SUP_OPTIONS} ${TEST_PATH}
+  - coverage xml
+  - coverage report
+
+#========================== OBJECTS TESTS ====================================
+
+test_data_structures: 
+  stage: test
+  before_script: 
+    - *step_python_setup_install
+    - *step_install_pyrrha_test
+  script:
+    - *step_run_tests
+  image: python:latest
+  variables:
+    TEST_COVERAGE_SOURCE: pyrrha_mapper.common.objects
+    TEST_PATH: tests/test_filesystem_objects.py
+  coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
+  artifacts:
+    reports:
+      junit: report.xml
+      coverage_report:
+        coverage_format: cobertura
+        path: coverage.xml
 
-test:
+#========================== MAPPERS TESTS ====================================
+.run_pyrrha_test_artifacts:
+  stage: test
+  before_script: 
+    - *step_python_setup_install
+    - *step_install_pyrrha_test
   script:
-    - pip install ".[test]"
-    - pytest  --junitxml=report.xml  --cov="pyrrha_mapper" --cov-report xml -v tests/
-    - coverage report
+    - *step_gen_artifacts
+    - *step_run_tests
   coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
   artifacts:
+    name: db_$CI_JOB_NAME_SLUG
     when: always
+    paths:
+      - ${ARTIFACTS}/${DB}.srctrldb
+      - ${ARTIFACTS}/${DB}.srctrlprj
     reports:
       junit: report.xml
       coverage_report:
         coverage_format: cobertura
         path: coverage.xml
+  variables:
+    ARTIFACTS: tmp/artifacts
+
+test_fs: 
+  extends: 
+    - .run_pyrrha_test_artifacts
+  image: python:latest
+  variables:
+    DB: fs
+    MAPPER: fs
+    TEST_COVERAGE_SOURCE: pyrrha_mapper.common.filesystem_mapper,pyrrha_mapper.fs
+    TEST_PATH: tests/test_cli.py::TestFSMapper
+
 
+.test_fs-cg:
+  extends: 
+    - .run_pyrrha_test_artifacts
+  before_script: 
+    - !reference [.run_pyrrha_test_artifacts, before_script]
+    - *step_configure_disassembler
+  image: 
+    name: $CONTAINER_PATH/${DISASSEMBLER}:${VERSION}
+    docker:
+      user: user
+  variables:
+    DB: ${DISASSEMBLER}_${VERSION}_${EXPORTER}
+    MAPPER: fs-cg
+    MAPPER_OPTIONS: '--disassembler ${DISASSEMBLER} --exporter ${EXPORTER}'
+    TEST_COVERAGE_SOURCE: pyrrha_mapper.common.filesystem_mapper,pyrrha_mapper.intercg
+    TEST_PATH: tests/test_cli.py::TestFsCgMapper
+    TEST_SUP_OPTIONS: ${MAPPER_OPTIONS}
 
+test_fs-cg_ghidra:
+  extends:
+    - .test_fs-cg
+  variables:
+    DISASSEMBLER: ghidra
+  parallel:
+    matrix:
+      - VERSION: 11.1.2
+        EXPORTER: binexport
 
+test_fs-cg_ida:
+  extends:
+    - .test_fs-cg
+  variables:
+    DISASSEMBLER: ida
+  parallel:
+    matrix:
+      - VERSION: 11.1.2
+        EXPORTER: binexport
+      - VERSION: 84
+        EXPORTER: [quokka, binexport]
+      - VERSION: 91
+        EXPORTER: quokka
+  rules:
+    - if: $VERSION == "84"
+      variables: 
+        KEY: $IDA_KEY
+        KEY_NAME: ida.key
+        REG: $IDA84_REG
+    - if: $VERSION == "91"
+      variables: 
+        KEY: $LICENSE
+        KEY_NAME: ida_license.hexlic
+        REG: $IDA_REG

ci/ghidra/Dockerfile

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+
+#  Copyright 2023-2025 Quarkslab
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+FROM openjdk:21-jdk-slim
+SHELL ["/bin/bash", "-c"]
+
+# ======================== Ghidra Installation =================================
+
+ARG GHIDRA_VERSION=11.1.2
+ARG GHIDRA_RELEASE_DATE=20240709
+ARG GHIDRA_URL=https://github.com/NationalSecurityAgency/ghidra/releases/download/Ghidra_${GHIDRA_VERSION}_build/ghidra_${GHIDRA_VERSION}_PUBLIC_${GHIDRA_RELEASE_DATE}.zip
+ENV GHIDRA_INSTALL_DIR=/opt/ghidra_${GHIDRA_VERSION}_PUBLIC
+
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \
+    ca-certificates \
+    libfreetype6 \
+    libmagic1 \
+    libpython3-dev \
+    python3-minimal \
+    python3-pip \
+    python3-venv \
+    python-is-python3 \
+    unzip \
+    wget \
+    &&  rm -rf /var/lib/apt/lists/*
+
+RUN wget $GHIDRA_URL -O ghidra.zip && unzip ghidra.zip -d /opt/ && rm ghidra.zip
+
+ENV PATH=${GHIDRA_INSTALL_DIR}:${PATH}
+
+# ======================== Plugin Installation ==============================
+
+ARG BINEXPORT_URL=https://github.com/google/binexport/archive/refs/heads/main.zip
+ARG GRADLE_VERSION=8.14.3
+ARG GRADLE_URL=https://services.gradle.org/distributions/gradle-${GRADLE_VERSION}-bin.zip
+ARG GHIDRA_PLUGIN_DIR=/root/.config/ghidra/ghidra_${GHIDRA_VERSION}_PUBLIC/Extensions
+
+RUN wget ${GRADLE_URL} -O gradle.zip  \
+    && unzip gradle.zip -d gradle \
+    && wget ${BINEXPORT_URL} -O binexport.zip \
+    && mkdir -p ${GHIDRA_PLUGIN_DIR} \
+    && unzip binexport.zip  binexport-main/java/* -d binexport \
+    && (cd binexport/binexport-main/java/ && /gradle/gradle-${GRADLE_VERSION}/bin/gradle buildExtension -PGHIDRA_INSTALL_DIR=${GHIDRA_INSTALL_DIR} && unzip dist/ghidra_${GHIDRA_VERSION}_PUBLIC_$( date +%Y%m%d)_BinExport.zip -d ${GHIDRA_PLUGIN_DIR}) \
+    && rm -rf gradle.zip gradle binexport.zip binexport \
+    && apt-get purge --yes wget unzip  && apt --yes autoremove
+
+# ======================== USER CREATION ==============================
+ARG USER_GHIDRA_PLUGIN_DIR=/home/user/.config/ghidra/ghidra_${GHIDRA_VERSION}_PUBLIC/Extensions
+
+RUN useradd --create-home -u 1000 -m user && chown -R user:user $GHIDRA_INSTALL_DIR 
+RUN mkdir -p ${USER_GHIDRA_PLUGIN_DIR} && mv ${GHIDRA_PLUGIN_DIR}/* ${USER_GHIDRA_PLUGIN_DIR} && chown -R user:user /home/user/.config
+USER user
+WORKDIR /home/user
+
+CMD ["/bin/bash"]

ci/ida/Dockerfile

@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+
+#  Copyright 2023-2025 Quarkslab
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+FROM docker.io/library/debian:testing-slim
+SHELL ["/bin/bash", "-c"]
+
+# ============= How to generate the required data ===========================
+# paths can be changed from commandline if needed
+# idapro.hexlic: license file, downloaded from your account on hex-rays website
+# ida-pro_91.run: executable file, downloaded from your account on hex-rays website
+# ida.reg: history file, to be generated manually. Keep in memory that the licence has alredy been accepted.
+#      1. Build this docker with an empty ida.reg and launch it.
+#      2. Launch idat and accept the license.
+#      3. In another terminal, get the id of the current ida docker with `docker ps`
+#      4. Run `docker cp ID:/root/.idapro/ida.reg ./` where ID is the id get at the 
+#         previous step, you know have a correct ida.reg.
+#      5. Rebuild your image with the correct ida.reg
+# ===========================================================================
+
+# ======================== IDA Installation =================================
+
+ARG IDA_VERSION=91
+ARG IDA_INSTALLER=ida-pro_${IDA_VERSION}.run
+ENV IDA_INSTALL_DIR=/opt/ida_${IDA_VERSION}
+
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \
+    ca-certificates \
+    ccache \
+    cmake \
+    g++ \
+    gcc \
+    git \
+    libpython3-dev \
+    libqt5gui5 \
+    libfontconfig1 \
+    libmagic1 \
+    libsecret-1-0 \
+    make \
+    ninja-build \
+    python3-minimal \
+    python3-pip \
+    python3-venv \
+    python-is-python3 \
+    unzip \
+    xcb-proto \
+    wget \
+    zlib1g-dev \
+    && mkdir -p $IDA_INSTALL_DIR ~/.local/share/applications/
+
+RUN --mount=type=bind,src=${IDA_INSTALLER},target=${IDA_INSTALLER} DEBIAN_FRONTEND=noninteractive apt-get install --yes --reinstall libxcb-xinerama0 && \
+    ./${IDA_INSTALLER} --mode unattended --prefix ${IDA_INSTALL_DIR}
+
+ENV PATH=${IDA_INSTALL_DIR}:${PATH}
+
+# ======================== Plugin Installation ==============================
+
+ARG QUOKKA_VERSION=v0.6.1
+ARG QUOKKA_URL=https://github.com/quarkslab/quokka/releases/download/${QUOKKA_VERSION}/${IDA_VERSION}-quokka_plugin0064.so
+ARG BINEXPORT_URL=https://github.com/google/binexport/releases/download/v12-20240417-ghidra_11.0.3/BinExport-Linux.zip
+RUN if [[ ${IDA_VERSION} -eq 84 ]]; then \
+    wget ${QUOKKA_URL} -O ${IDA_INSTALL_DIR}/plugins/quokka64.so \
+    && wget ${BINEXPORT_URL} -O binexport.zip \
+    && unzip -j binexport.zip ida/binexport12_ida.so ida/binexport12_ida64.so -d ${IDA_INSTALL_DIR}/plugins/ \
+    && rm -f binexport.zip ; \
+    else wget ${QUOKKA_URL} -O ${IDA_INSTALL_DIR}/plugins/quokka.so ; fi \
+    && apt-get purge --yes wget \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN useradd --create-home -u 1000 -m user && chown -R user:user $IDA_INSTALL_DIR
+USER user
+RUN $IDA_INSTALL_DIR/idapyswitch -a
+WORKDIR /home/user