Implementation ci dataset

.gitlab-ci.yml

@@ -3,11 +3,19 @@ stages:
   - trigger
   - test
 
+# Pipeline mode:
+# - default (CI_PIPELINE_MODE unset): normal pytest CI
+# - implementation (CI_PIPELINE_MODE=implementation): dedicated implementation pipeline
+
 generator:
   stage: generate
   image: python:3.8-alpine
   variables:
     N_TESTS_PER_YAML: 4
+  rules:
+    - if: '$CI_PIPELINE_MODE == "implementation"'
+      when: never
+    - when: on_success
   tags:
     - k8s-default
   before_script:
@@ -21,9 +29,25 @@ generator:
 
 pytests:
   stage: trigger
+  rules:
+    - if: '$CI_PIPELINE_MODE == "implementation"'
+      when: never
+    - when: on_success
   trigger:
     include:
       - local: test/pytest/ci-template.yml
       - artifact: test/pytest/pytests.yml
         job: generator
     strategy: depend
+
+implementation-pytests:
+  stage: trigger
+  rules:
+    - if: '$CI_PIPELINE_MODE == "implementation"'
+      when: on_success
+    - when: never
+  trigger:
+    include:
+      - local: test/pytest/implementation/ci-template.yml
+      - local: test/pytest/implementation/pytests.yml
+    strategy: depend

test/pytest/conftest.py

@@ -56,6 +56,7 @@ def synthesis_config():
         'run_synthesis': str_to_bool(os.getenv('RUN_SYNTHESIS', 'false')),
         'tools_version': {
             'Vivado': os.getenv('VIVADO_VERSION', '2020.1'),
+            'VivadoAccelerator': os.getenv('VIVADO_VERSION', '2020.1'),
             'Vitis': os.getenv('VITIS_VERSION', '2024.1'),
             'Quartus': os.getenv('QUARTUS_VERSION', 'latest'),
             'oneAPI': os.getenv('ONEAPI_VERSION', '2025.0.1'),
@@ -66,4 +67,18 @@ def synthesis_config():
             'Quartus': {'synth': True, 'fpgasynth': False},
             'oneAPI': {'build_type': 'report', 'run': False},
         },
+        'implementation_build_args': {
+            # Full accelerator flow for implementation dataset collection:
+            # run HLS synth, downstream Vivado synth, and bitfile generation.
+            'VivadoAccelerator': {
+                'reset': True,
+                'csim': False,
+                'synth': True,
+                'cosim': True,
+                'validation': False,
+                'export': True,
+                'vsynth': True,
+                'bitfile': True,
+            }
+        },
     }

test/pytest/generate_ci_yaml.py

@@ -25,6 +25,7 @@
 
 # Blacklisted tests will be skipped
 BLACKLIST = {'test_reduction'}
+EXCLUDED_DIRS = {'implementation'}
 
 # Long-running tests will not be bundled with other tests
 LONGLIST = {'test_hgq_layers', 'test_hgq_players', 'test_qkeras', 'test_pytorch_api'}
@@ -44,6 +45,11 @@
 }
 
 
+def include_in_default_ci(path: Path) -> bool:
+    """Exclude dedicated suites (e.g. implementation-only jobs) from the default CI matrix."""
+    return not any(part in EXCLUDED_DIRS for part in path.parts)
+
+
 def collect_test_functions_from_ast(test_file):
     """Collect all test function names using AST parsing (no imports)."""
     with open(test_file, encoding='utf-8') as f:
@@ -79,7 +85,8 @@ def generate_test_yaml(test_root='.'):
     test_paths = [
         path
         for path in test_root.glob('**/test_*.py')
-        if path.stem not in (BLACKLIST | LONGLIST | set(SPLIT_BY_TEST_CASE.keys()) | KERAS3_LIST)
+        if include_in_default_ci(path)
+        and path.stem not in (BLACKLIST | LONGLIST | set(SPLIT_BY_TEST_CASE.keys()) | KERAS3_LIST)
     ]
     need_example_models = [uses_example_model(path) for path in test_paths]
 
@@ -99,15 +106,17 @@ def generate_test_yaml(test_root='.'):
         else:
             yml.update(diff_yml)
 
-    test_paths = [path for path in test_root.glob('**/test_*.py') if path.stem in LONGLIST]
+    test_paths = [path for path in test_root.glob('**/test_*.py') if include_in_default_ci(path) and path.stem in LONGLIST]
     for path in test_paths:
         name = path.stem.replace('test_', '')
         test_file = str(path.relative_to(test_root))
         needs_examples = uses_example_model(path)
         diff_yml = yaml.safe_load(template.format(name, '.pytest', test_file, int(needs_examples)))
         yml.update(diff_yml)
 
-    test_paths = [path for path in test_root.glob('**/test_*.py') if path.stem in SPLIT_BY_TEST_CASE]
+    test_paths = [
+        path for path in test_root.glob('**/test_*.py') if include_in_default_ci(path) and path.stem in SPLIT_BY_TEST_CASE
+    ]
     for path in test_paths:
         stem = path.stem
         name_base = stem.replace('test_', '')
@@ -125,7 +134,9 @@ def generate_test_yaml(test_root='.'):
             else:
                 yml.update(diff_yml)
 
-    keras3_paths = [path for path in test_root.glob('**/test_*.py') if path.stem in KERAS3_LIST]
+    keras3_paths = [
+        path for path in test_root.glob('**/test_*.py') if include_in_default_ci(path) and path.stem in KERAS3_LIST
+    ]
     keras3_need_examples = [uses_example_model(path) for path in keras3_paths]
 
     k3_idxs = list(range(len(keras3_need_examples)))

test/pytest/implementation/README.md

@@ -0,0 +1,67 @@
+# Implementation CI Suite
+
+This directory contains manually listed implementation tests. These jobs run full backend implementation flows and collect dataset artifacts with actual tool reports and generated project files.
+
+The normal pytest CI should stay separate from this suite. Implementation tests are intended for manually triggered dataset collection, not for the regular fast test matrix.
+
+## Pipeline Layout
+
+Implementation CI uses:
+
+- `test/pytest/implementation/pytests.yml`: static job list
+- `test/pytest/implementation/ci-template.yml`: implementation-specific GitLab templates
+- `test/pytest/implementation/implementation_helpers.py`: dataset artifact helpers used by implementation tests
+
+Jobs in `pytests.yml` extend backend-specific runtime templates from `ci-template.yml`, for example:
+
+- `.pytest-implementation-vivadoaccelerator`
+- `.pytest-implementation-vivado-runtime`
+
+Each concrete job sets `PYTESTFILE` to one specific test function, so each model/backend run becomes a separate CI job and artifact set.
+
+## Dataset Artifacts
+
+Implementation tests should use `run_implementation_collection_test()` from `implementation_helpers.py`. The helper:
+
+- runs `hls_model.build(...)` with backend-specific implementation build args
+- captures full terminal output to `*_build.log`
+- writes the hls4ml report returned by the backend to `*_hls4ml_report.json`
+- writes one compact dataset record to `*_dataset.json`
+- compresses the generated project directory to `*_project.zip`
+- records the project archive path, size, and SHA256 in the dataset record
+
+The dataset record includes:
+
+- hls4ml source commit and repository URL
+- example-models source commit and model file names
+- backend, project name, board/part metadata, and build args
+- backend-specific toolchain versions
+- CI metadata for finding the run later
+- build timing and the hls4ml report
+- pointers to the log, hls4ml report JSON, bitstream files, and project archive
+
+`IMPLEMENTATION_DATASET_DIR` controls where artifacts are written. In CI it is set to:
+
+```bash
+test/pytest/implementation
+```
+
+## Test Policy
+
+Implementation tests should load models from the `example-models` submodule instead of defining models inline. This keeps dataset records tied to a model name, model file, and exact `example-models` commit.
+
+Keep test code focused on:
+
+- loading the example model
+- converting it with the backend under test
+- passing clear metadata to `run_implementation_collection_test()`
+
+Avoid duplicating dataset/report parsing in individual tests. Add backend-specific dataset behavior in `implementation_helpers.py` when needed.
+
+## Adding A Test
+
+1. Add or update a `test_*.py` file in this directory.
+2. Load a model from `example-models`.
+3. Add model metadata including name, source, source commit, and model file paths.
+4. Add a static job in `pytests.yml` with `PYTESTFILE` pointing to the specific test function.
+5. Set tool versions and backend runtime template through the job/template variables.

test/pytest/implementation/ci-template.yml

@@ -0,0 +1,64 @@
+.pytest-implementation:
+  stage: test
+  image: gitlab-registry.cern.ch/fastmachinelearning/hls4ml-testing:0.6.3.base
+  tags:
+    - k8s-default
+  variables:
+    CONDA_ENV: "hls4ml-testing"
+    EXTRA_DEPS: "[da,testing,testing-keras2,sr,optimization]"
+    IMPLEMENTATION_DATASET_DIR: "test/pytest/implementation"
+  before_script:
+    - eval "$(conda shell.bash hook)"
+    - conda activate "$CONDA_ENV"
+    - git config --global --add safe.directory /builds/fastmachinelearning/hls4ml
+    - git submodule update --init --recursive hls4ml/templates/catapult/
+    - if [ $EXAMPLEMODEL == 1 ]; then git submodule update --init example-models; fi
+    - pip install .${EXTRA_DEPS}
+  artifacts:
+    when: always
+    reports:
+      junit:
+        - test/pytest/report.xml
+    paths:
+      - test/pytest/*.tar.gz
+      - test/pytest/implementation/*_dataset.json
+      - test/pytest/implementation/*_hls4ml_report.json
+      - test/pytest/implementation/*_build.log
+      - test/pytest/implementation/*_project.zip
+
+.pytest-implementation-vivado-runtime:
+  extends: .pytest-implementation
+  script:
+    - mkdir -p cmd_vivado_${VIVADO_VERSION}
+    - echo '#!/bin/bash' > cmd_vivado_${VIVADO_VERSION}/vivado_hls
+    - echo "apptainer exec --cleanenv --env LANG=C,LC_ALL=C /cvmfs/projects.cern.ch/hls4ml/vivado/${VIVADO_VERSION} vivado_hls \"\$@\"" >> cmd_vivado_${VIVADO_VERSION}/vivado_hls
+    - chmod +x cmd_vivado_${VIVADO_VERSION}/vivado_hls
+    - echo '#!/bin/bash' > cmd_vivado_${VIVADO_VERSION}/vivado
+    - echo "apptainer exec --cleanenv --env LANG=C,LC_ALL=C /cvmfs/projects.cern.ch/hls4ml/vivado/${VIVADO_VERSION} vivado \"\$@\"" >> cmd_vivado_${VIVADO_VERSION}/vivado
+    - chmod +x cmd_vivado_${VIVADO_VERSION}/vivado
+    - export PATH=$PWD/cmd_vivado_${VIVADO_VERSION}:$PATH
+    - export IMPLEMENTATION_DATASET_DIR=$PWD/test/pytest/implementation
+    - cd test/pytest
+    - pytest $PYTESTFILE -s -rA --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed
+
+.pytest-implementation-vitis-runtime:
+  extends: .pytest-implementation
+  script:
+    - mkdir -p cmd_vivado_${VIVADO_VERSION}
+    - echo '#!/bin/bash' > cmd_vivado_${VIVADO_VERSION}/vivado
+    - echo "apptainer exec --cleanenv --env LANG=C,LC_ALL=C /cvmfs/projects.cern.ch/hls4ml/vitis/${VITIS_VERSION} vivado \"\$@\"" >> cmd_vivado_${VIVADO_VERSION}/vivado
+    - chmod +x cmd_vivado_${VIVADO_VERSION}/vivado
+    - mkdir -p cmd_vitis_${VITIS_VERSION}
+    - echo '#!/bin/bash' > cmd_vitis_${VITIS_VERSION}/vitis-run
+    - echo "apptainer exec /cvmfs/projects.cern.ch/hls4ml/vitis/${VITIS_VERSION} vitis-run \"\$@\"" >> cmd_vitis_${VITIS_VERSION}/vitis-run
+    - chmod +x cmd_vitis_${VITIS_VERSION}/vitis-run
+    - echo '#!/bin/bash' > cmd_vitis_${VITIS_VERSION}/v++
+    - echo "apptainer exec /cvmfs/projects.cern.ch/hls4ml/vitis/${VITIS_VERSION} v++ \"\$@\"" >> cmd_vitis_${VITIS_VERSION}/v++
+    - chmod +x cmd_vitis_${VITIS_VERSION}/v++
+    - echo '#!/bin/bash' > cmd_vitis_${VITIS_VERSION}/vitis
+    - echo "apptainer exec /cvmfs/projects.cern.ch/hls4ml/vitis/${VITIS_VERSION} vitis \"\$@\"" >> cmd_vitis_${VITIS_VERSION}/vitis
+    - chmod +x cmd_vitis_${VITIS_VERSION}/vitis
+    - export PATH=$PWD/cmd_vivado_${VIVADO_VERSION}:$PWD/cmd_vitis_${VITIS_VERSION}:$PATH
+    - export IMPLEMENTATION_DATASET_DIR=$PWD/test/pytest/implementation
+    - cd test/pytest
+    - pytest $PYTESTFILE -s -rA --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed