From cd45b22f02c44cfffc3274cd26a62e984e86dc4a Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Thu, 12 Mar 2026 13:34:31 -0500 Subject: [PATCH 01/27] test VivadoAccelerator synth --- test/pytest/ci-template.yml | 2 + test/pytest/conftest.py | 11 ++++ test/pytest/synthesis_helpers.py | 2 + test/pytest/test_keras_api_vivadoacc.py | 67 +++++++++++++++++++++++++ 4 files changed, 82 insertions(+) create mode 100644 test/pytest/test_keras_api_vivadoacc.py diff --git a/test/pytest/ci-template.yml b/test/pytest/ci-template.yml index ebbcd8a21e..47e52c1ce8 100644 --- a/test/pytest/ci-template.yml +++ b/test/pytest/ci-template.yml @@ -15,6 +15,8 @@ - pip install .${EXTRA_DEPS} # set up vivado_hls command + # TODO: For VivadoAccelerator full synthesis (vsynth/bitfile), export/provide the full Vivado toolchain (`vivado`), + # not only `vivado_hls`. - mkdir -p cmd_vivado_${VIVADO_VERSION} - echo '#!/bin/bash' > cmd_vivado_${VIVADO_VERSION}/vivado_hls - echo "apptainer exec --cleanenv --env LANG=C,LC_ALL=C /cvmfs/projects.cern.ch/hls4ml/vivado/${VIVADO_VERSION} vivado_hls \"\$@\"" >> cmd_vivado_${VIVADO_VERSION}/vivado_hls diff --git a/test/pytest/conftest.py b/test/pytest/conftest.py index 7841a1a989..0e1d9d1fae 100644 --- a/test/pytest/conftest.py +++ b/test/pytest/conftest.py @@ -56,12 +56,23 @@ def synthesis_config(): 'run_synthesis': str_to_bool(os.getenv('RUN_SYNTHESIS', 'false')), 'tools_version': { 'Vivado': os.getenv('VIVADO_VERSION', '2020.1'), + 'VivadoAccelerator': os.getenv('VIVADO_VERSION', '2020.1'), 'Vitis': os.getenv('VITIS_VERSION', '2024.1'), 'Quartus': os.getenv('QUARTUS_VERSION', 'latest'), 'oneAPI': os.getenv('ONEAPI_VERSION', '2025.0.1'), }, 'build_args': { 'Vivado': {'csim': False, 'synth': True, 'export': False}, + # Full accelerator flow: run C/RTL synthesis, downstream Vivado synthesis, and board project/bitfile. + 'VivadoAccelerator': { + 'csim': False, + 'synth': True, + 'cosim': False, + 'validation': False, + 'export': True, + 'vsynth': True, + 'bitfile': True, + }, 'Vitis': {'csim': False, 'synth': True, 'export': False}, 'Quartus': {'synth': True, 'fpgasynth': False}, 'oneAPI': {'build_type': 'report', 'run': False}, diff --git a/test/pytest/synthesis_helpers.py b/test/pytest/synthesis_helpers.py index 27d953b101..7a71700b2c 100644 --- a/test/pytest/synthesis_helpers.py +++ b/test/pytest/synthesis_helpers.py @@ -119,6 +119,7 @@ def compare_oneapi_backend(data, baseline): COMPARE_FUNCS = { 'Vivado': compare_vitis_backend, + 'VivadoAccelerator': compare_vitis_backend, 'Vitis': compare_vitis_backend, 'oneAPI': compare_oneapi_backend, } @@ -126,6 +127,7 @@ def compare_oneapi_backend(data, baseline): EXPECTED_REPORT_KEYS = { 'Vivado': {'CSynthesisReport'}, + 'VivadoAccelerator': {'CSynthesisReport'}, 'Vitis': {'CSynthesisReport'}, 'oneAPI': {'report'}, } diff --git a/test/pytest/test_keras_api_vivadoacc.py b/test/pytest/test_keras_api_vivadoacc.py new file mode 100644 index 0000000000..725e0e3621 --- /dev/null +++ b/test/pytest/test_keras_api_vivadoacc.py @@ -0,0 +1,67 @@ +from pathlib import Path + +import numpy as np +import pytest +import tensorflow as tf +from synthesis_helpers import run_synthesis_test +from tensorflow.keras.layers import ( + Activation, + Dense, +) + +import hls4ml + +test_root_path = Path(__file__).parent + + +@pytest.mark.parametrize('backend', ['Vivado', 'VivadoAccelerator']) +@pytest.mark.parametrize('io_type', ['io_parallel']) +def test_dense(test_case_id, backend, io_type, synthesis_config): + model = tf.keras.models.Sequential() + model.add( + Dense( + 2, + input_shape=(1,), + name='Dense', + use_bias=True, + kernel_initializer=tf.keras.initializers.RandomUniform(minval=1, maxval=10), + bias_initializer='zeros', + kernel_regularizer=None, + bias_regularizer=None, + activity_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + ) + ) + model.add(Activation(activation='elu', name='Activation')) + model.compile(optimizer='adam', loss='mse') + + X_input = np.random.rand(100, 1) + + keras_prediction = model.predict(X_input) + + config = hls4ml.utils.config_from_keras_model(model) + output_dir = str(test_root_path / test_case_id) + baseline_file_name = f'{test_case_id}.json' + + hls_model = hls4ml.converters.convert_from_keras_model( + model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type + ) + + hls_model.compile() + + hls_prediction = hls_model.predict(X_input) + + np.testing.assert_allclose(hls_prediction, keras_prediction, rtol=1e-2, atol=0.01) + + assert len(model.layers) + 1 == len(hls_model.get_layers()) + assert list(hls_model.get_layers())[0].attributes['class_name'] == 'InputLayer' + assert list(hls_model.get_layers())[1].attributes['class_name'] == model.layers[0]._name + assert list(hls_model.get_layers())[2].attributes['class_name'] == 'ELU' + assert list(hls_model.get_layers())[0].attributes['input_shape'] == list(model.layers[0].input_shape[1:]) + assert list(hls_model.get_layers())[1].attributes['n_in'] == model.layers[0].input_shape[1:][0] + assert list(hls_model.get_layers())[1].attributes['n_out'] == model.layers[0].output_shape[1:][0] + assert list(hls_model.get_layers())[2].attributes['activation'] == str(model.layers[1].activation).split()[1] + assert list(hls_model.get_layers())[1].attributes['activation'] == str(model.layers[0].activation).split()[1] + + run_synthesis_test(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) From e915de1bcebe535b56a46bdf8fce09ae963e2c1b Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Thu, 12 Mar 2026 14:36:36 -0500 Subject: [PATCH 02/27] add VivadoAccelerator test baseline --- ...t_dense_io_parallel-VivadoAccelerator.json | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 test/pytest/baselines/Vivado/2020.1/test_keras_api_vivadoacc_test_dense_io_parallel-VivadoAccelerator.json diff --git a/test/pytest/baselines/Vivado/2020.1/test_keras_api_vivadoacc_test_dense_io_parallel-VivadoAccelerator.json b/test/pytest/baselines/Vivado/2020.1/test_keras_api_vivadoacc_test_dense_io_parallel-VivadoAccelerator.json new file mode 100644 index 0000000000..fb49d15a71 --- /dev/null +++ b/test/pytest/baselines/Vivado/2020.1/test_keras_api_vivadoacc_test_dense_io_parallel-VivadoAccelerator.json @@ -0,0 +1,34 @@ +{ + "CSynthesisReport": { + "TargetClockPeriod": "5.00", + "EstimatedClockPeriod": "4.367", + "BestLatency": "24", + "WorstLatency": "24", + "IntervalMin": "25", + "IntervalMax": "25", + "BRAM_18K": "1", + "DSP": "2", + "FF": "2282", + "LUT": "2089", + "URAM": "0", + "AvailableBRAM_18K": "280", + "AvailableDSP": "220", + "AvailableFF": "106400", + "AvailableLUT": "53200", + "AvailableURAM": "0" + }, + "VivadoSynthReport": { + "LUT": "71", + "FF": "159", + "BRAM_18K": "0.5", + "DSP48E": "2" + }, + "TimingReport": { + "WNS": 1.16, + "TNS": 0.0, + "WHS": 0.012, + "THS": 0.0, + "WPWS": 3.75, + "TPWS": 0.0 + } +} From 72d34148471e59ba61f3c0f7a4a02a6c406d6365 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Thu, 12 Mar 2026 14:42:21 -0500 Subject: [PATCH 03/27] add: export vivado command in CI --- test/pytest/ci-template.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/pytest/ci-template.yml b/test/pytest/ci-template.yml index 47e52c1ce8..379860ef2b 100644 --- a/test/pytest/ci-template.yml +++ b/test/pytest/ci-template.yml @@ -14,13 +14,14 @@ - if [ $EXAMPLEMODEL == 1 ]; then git submodule update --init example-models; fi - pip install .${EXTRA_DEPS} - # set up vivado_hls command - # TODO: For VivadoAccelerator full synthesis (vsynth/bitfile), export/provide the full Vivado toolchain (`vivado`), - # not only `vivado_hls`. + # set up Vivado commands - mkdir -p cmd_vivado_${VIVADO_VERSION} - echo '#!/bin/bash' > cmd_vivado_${VIVADO_VERSION}/vivado_hls - echo "apptainer exec --cleanenv --env LANG=C,LC_ALL=C /cvmfs/projects.cern.ch/hls4ml/vivado/${VIVADO_VERSION} vivado_hls \"\$@\"" >> cmd_vivado_${VIVADO_VERSION}/vivado_hls + - echo '#!/bin/bash' > cmd_vivado_${VIVADO_VERSION}/vivado + - echo "apptainer exec --cleanenv --env LANG=C,LC_ALL=C /cvmfs/projects.cern.ch/hls4ml/vivado/${VIVADO_VERSION} vivado \"\$@\"" >> cmd_vivado_${VIVADO_VERSION}/vivado - chmod +x cmd_vivado_${VIVADO_VERSION}/vivado_hls + - chmod +x cmd_vivado_${VIVADO_VERSION}/vivado - export PATH=$PWD/cmd_vivado_${VIVADO_VERSION}:$PATH # set up vitis-run command From 1bf32364d80179f68996d1deab6a2bc304ec707f Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Thu, 12 Mar 2026 14:42:47 -0500 Subject: [PATCH 04/27] assign vivadoaccelerator test to a signle job --- test/pytest/generate_ci_yaml.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/pytest/generate_ci_yaml.py b/test/pytest/generate_ci_yaml.py index 684abc0511..80b86768c1 100644 --- a/test/pytest/generate_ci_yaml.py +++ b/test/pytest/generate_ci_yaml.py @@ -41,6 +41,7 @@ # Value = chunk size per CI job SPLIT_BY_TEST_CASE = { 'test_keras_api': 1, + 'test_keras_api_vivadoacc': 1, } From 9e35fff3a2bc6047f511d39d06c183c4b039cdcc Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Thu, 12 Mar 2026 15:16:28 -0500 Subject: [PATCH 05/27] add board and part in test_keras_api_vivadoacc --- test/pytest/test_keras_api_vivadoacc.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/test/pytest/test_keras_api_vivadoacc.py b/test/pytest/test_keras_api_vivadoacc.py index 725e0e3621..6509968c67 100644 --- a/test/pytest/test_keras_api_vivadoacc.py +++ b/test/pytest/test_keras_api_vivadoacc.py @@ -12,9 +12,11 @@ import hls4ml test_root_path = Path(__file__).parent +VIVADOACC_BOARD = 'pynq-z2' +VIVADOACC_PART = 'xc7z020clg400-1' -@pytest.mark.parametrize('backend', ['Vivado', 'VivadoAccelerator']) +@pytest.mark.parametrize('backend', ['VivadoAccelerator']) @pytest.mark.parametrize('io_type', ['io_parallel']) def test_dense(test_case_id, backend, io_type, synthesis_config): model = tf.keras.models.Sequential() @@ -45,7 +47,13 @@ def test_dense(test_case_id, backend, io_type, synthesis_config): baseline_file_name = f'{test_case_id}.json' hls_model = hls4ml.converters.convert_from_keras_model( - model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type + model, + hls_config=config, + output_dir=output_dir, + backend=backend, + io_type=io_type, + board=VIVADOACC_BOARD, + part=VIVADOACC_PART, ) hls_model.compile() @@ -64,4 +72,9 @@ def test_dense(test_case_id, backend, io_type, synthesis_config): assert list(hls_model.get_layers())[2].attributes['activation'] == str(model.layers[1].activation).split()[1] assert list(hls_model.get_layers())[1].attributes['activation'] == str(model.layers[0].activation).split()[1] - run_synthesis_test(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) + run_synthesis_test( + config=synthesis_config, + hls_model=hls_model, + baseline_file_name=baseline_file_name, + backend=backend, + ) From dee7f602117672236b022f07230b1cc910a96418 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Fri, 13 Mar 2026 16:49:41 -0500 Subject: [PATCH 06/27] fix: update vivado board and part --- test/pytest/test_keras_api_vivadoacc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/pytest/test_keras_api_vivadoacc.py b/test/pytest/test_keras_api_vivadoacc.py index 6509968c67..b259c9fc7a 100644 --- a/test/pytest/test_keras_api_vivadoacc.py +++ b/test/pytest/test_keras_api_vivadoacc.py @@ -12,8 +12,8 @@ import hls4ml test_root_path = Path(__file__).parent -VIVADOACC_BOARD = 'pynq-z2' -VIVADOACC_PART = 'xc7z020clg400-1' +VIVADOACC_BOARD = 'zcu102' +VIVADOACC_PART = 'xczu9eg-ffvb1156-2-e' @pytest.mark.parametrize('backend', ['VivadoAccelerator']) From 791d062322a7eb86b32b701dde8327e7276435e8 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Fri, 13 Mar 2026 17:00:19 -0500 Subject: [PATCH 07/27] update baselines --- ...t_dense_io_parallel-VivadoAccelerator.json | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/test/pytest/baselines/Vivado/2020.1/test_keras_api_vivadoacc_test_dense_io_parallel-VivadoAccelerator.json b/test/pytest/baselines/Vivado/2020.1/test_keras_api_vivadoacc_test_dense_io_parallel-VivadoAccelerator.json index fb49d15a71..5b7cb5c46a 100644 --- a/test/pytest/baselines/Vivado/2020.1/test_keras_api_vivadoacc_test_dense_io_parallel-VivadoAccelerator.json +++ b/test/pytest/baselines/Vivado/2020.1/test_keras_api_vivadoacc_test_dense_io_parallel-VivadoAccelerator.json @@ -1,34 +1,34 @@ { "CSynthesisReport": { "TargetClockPeriod": "5.00", - "EstimatedClockPeriod": "4.367", - "BestLatency": "24", - "WorstLatency": "24", - "IntervalMin": "25", - "IntervalMax": "25", + "EstimatedClockPeriod": "4.349", + "BestLatency": "11", + "WorstLatency": "11", + "IntervalMin": "12", + "IntervalMax": "12", "BRAM_18K": "1", "DSP": "2", - "FF": "2282", - "LUT": "2089", + "FF": "622", + "LUT": "1947", "URAM": "0", - "AvailableBRAM_18K": "280", - "AvailableDSP": "220", - "AvailableFF": "106400", - "AvailableLUT": "53200", + "AvailableBRAM_18K": "1824", + "AvailableDSP": "2520", + "AvailableFF": "548160", + "AvailableLUT": "274080", "AvailableURAM": "0" }, "VivadoSynthReport": { - "LUT": "71", - "FF": "159", + "LUT": "47", + "FF": "35", "BRAM_18K": "0.5", "DSP48E": "2" }, "TimingReport": { - "WNS": 1.16, + "WNS": 3.988, "TNS": 0.0, - "WHS": 0.012, + "WHS": 0.01, "THS": 0.0, - "WPWS": 3.75, + "WPWS": 3.5, "TPWS": 0.0 } } From 00484980ba293aafbf4de79fdfec8a0eadf1d67b Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Mon, 16 Mar 2026 10:34:15 -0500 Subject: [PATCH 08/27] move VivadoAccelerator baselines files --- ...as_api_vivadoacc_test_dense_io_parallel-VivadoAccelerator.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/pytest/baselines/{Vivado => VivadoAccelerator}/2020.1/test_keras_api_vivadoacc_test_dense_io_parallel-VivadoAccelerator.json (100%) diff --git a/test/pytest/baselines/Vivado/2020.1/test_keras_api_vivadoacc_test_dense_io_parallel-VivadoAccelerator.json b/test/pytest/baselines/VivadoAccelerator/2020.1/test_keras_api_vivadoacc_test_dense_io_parallel-VivadoAccelerator.json similarity index 100% rename from test/pytest/baselines/Vivado/2020.1/test_keras_api_vivadoacc_test_dense_io_parallel-VivadoAccelerator.json rename to test/pytest/baselines/VivadoAccelerator/2020.1/test_keras_api_vivadoacc_test_dense_io_parallel-VivadoAccelerator.json From e6301870482729c24f868759b952c1669fefd82f Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Tue, 14 Apr 2026 10:53:09 -0500 Subject: [PATCH 09/27] add new CI for implementation tests --- .gitlab-ci.yml | 26 ++++ ...t_dense_io_parallel-VivadoAccelerator.json | 34 ------ test/pytest/ci-base-template.yml | 29 +++++ test/pytest/ci-template.yml | 38 +----- test/pytest/conftest.py | 17 ++- test/pytest/generate_ci_yaml.py | 20 ++- test/pytest/implementation/README.md | 36 ++++++ test/pytest/implementation/ci-template.yml | 21 ++++ test/pytest/implementation/pytests.yml | 7 ++ .../test_keras_api_vivadoacc.py | 27 ++--- test/pytest/synthesis_helpers.py | 114 ++++++++++++++++++ 11 files changed, 271 insertions(+), 98 deletions(-) delete mode 100644 test/pytest/baselines/VivadoAccelerator/2020.1/test_keras_api_vivadoacc_test_dense_io_parallel-VivadoAccelerator.json create mode 100644 test/pytest/ci-base-template.yml create mode 100644 test/pytest/implementation/README.md create mode 100644 test/pytest/implementation/ci-template.yml create mode 100644 test/pytest/implementation/pytests.yml rename test/pytest/{ => implementation}/test_keras_api_vivadoacc.py (80%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 89535c1937..f3eec056d1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,11 +3,19 @@ stages: - trigger - test +# Pipeline mode: +# - default (CI_PIPELINE_MODE unset): normal pytest CI +# - implementation (CI_PIPELINE_MODE=implementation): dedicated implementation pipeline + generator: stage: generate image: python:3.8-alpine variables: N_TESTS_PER_YAML: 4 + rules: + - if: '$CI_PIPELINE_MODE == "implementation"' + when: never + - when: on_success tags: - k8s-default before_script: @@ -21,9 +29,27 @@ generator: pytests: stage: trigger + rules: + - if: '$CI_PIPELINE_MODE == "implementation"' + when: never + - when: on_success trigger: include: + - local: test/pytest/ci-base-template.yml - local: test/pytest/ci-template.yml - artifact: test/pytest/pytests.yml job: generator strategy: depend + +implementation-pytests: + stage: trigger + rules: + - if: '$CI_PIPELINE_MODE == "implementation"' + when: on_success + - when: never + trigger: + include: + - local: test/pytest/ci-base-template.yml + - local: test/pytest/implementation/ci-template.yml + - local: test/pytest/implementation/pytests.yml + strategy: depend diff --git a/test/pytest/baselines/VivadoAccelerator/2020.1/test_keras_api_vivadoacc_test_dense_io_parallel-VivadoAccelerator.json b/test/pytest/baselines/VivadoAccelerator/2020.1/test_keras_api_vivadoacc_test_dense_io_parallel-VivadoAccelerator.json deleted file mode 100644 index 5b7cb5c46a..0000000000 --- a/test/pytest/baselines/VivadoAccelerator/2020.1/test_keras_api_vivadoacc_test_dense_io_parallel-VivadoAccelerator.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "CSynthesisReport": { - "TargetClockPeriod": "5.00", - "EstimatedClockPeriod": "4.349", - "BestLatency": "11", - "WorstLatency": "11", - "IntervalMin": "12", - "IntervalMax": "12", - "BRAM_18K": "1", - "DSP": "2", - "FF": "622", - "LUT": "1947", - "URAM": "0", - "AvailableBRAM_18K": "1824", - "AvailableDSP": "2520", - "AvailableFF": "548160", - "AvailableLUT": "274080", - "AvailableURAM": "0" - }, - "VivadoSynthReport": { - "LUT": "47", - "FF": "35", - "BRAM_18K": "0.5", - "DSP48E": "2" - }, - "TimingReport": { - "WNS": 3.988, - "TNS": 0.0, - "WHS": 0.01, - "THS": 0.0, - "WPWS": 3.5, - "TPWS": 0.0 - } -} diff --git a/test/pytest/ci-base-template.yml b/test/pytest/ci-base-template.yml new file mode 100644 index 0000000000..9aad35380a --- /dev/null +++ b/test/pytest/ci-base-template.yml @@ -0,0 +1,29 @@ +.pytest-base: + stage: test + image: gitlab-registry.cern.ch/fastmachinelearning/hls4ml-testing:0.6.3.base + tags: + - k8s-default + variables: + CONDA_ENV: "hls4ml-testing" + EXTRA_DEPS: "[da,testing,testing-keras2,sr,optimization]" + before_script: + - eval "$(conda shell.bash hook)" + - conda activate "$CONDA_ENV" + - git config --global --add safe.directory /builds/fastmachinelearning/hls4ml + - git submodule update --init --recursive hls4ml/templates/catapult/ + - if [ $EXAMPLEMODEL == 1 ]; then git submodule update --init example-models; fi + - pip install .${EXTRA_DEPS} + + # set up Vivado HLS command used by regular synthesis tests + - mkdir -p cmd_vivado_${VIVADO_VERSION} + - echo '#!/bin/bash' > cmd_vivado_${VIVADO_VERSION}/vivado_hls + - echo "apptainer exec --cleanenv --env LANG=C,LC_ALL=C /cvmfs/projects.cern.ch/hls4ml/vivado/${VIVADO_VERSION} vivado_hls \"\$@\"" >> cmd_vivado_${VIVADO_VERSION}/vivado_hls + - chmod +x cmd_vivado_${VIVADO_VERSION}/vivado_hls + - export PATH=$PWD/cmd_vivado_${VIVADO_VERSION}:$PATH + + # set up vitis-run command + - mkdir -p cmd_vitis_${VITIS_VERSION} + - echo '#!/bin/bash' > cmd_vitis_${VITIS_VERSION}/vitis-run + - echo "apptainer exec /cvmfs/projects.cern.ch/hls4ml/vitis/${VITIS_VERSION} vitis-run \"\$@\"" >> cmd_vitis_${VITIS_VERSION}/vitis-run + - chmod +x cmd_vitis_${VITIS_VERSION}/vitis-run + - export PATH=$PWD/cmd_vitis_${VITIS_VERSION}:$PATH diff --git a/test/pytest/ci-template.yml b/test/pytest/ci-template.yml index 379860ef2b..d8fe4214a4 100644 --- a/test/pytest/ci-template.yml +++ b/test/pytest/ci-template.yml @@ -1,39 +1,9 @@ .pytest: - stage: test - image: gitlab-registry.cern.ch/fastmachinelearning/hls4ml-testing:0.6.3.base - tags: - - k8s-default - variables: - CONDA_ENV: "hls4ml-testing" - EXTRA_DEPS: "[da,testing,testing-keras2,sr,optimization]" - before_script: - - eval "$(conda shell.bash hook)" - - conda activate "$CONDA_ENV" - - git config --global --add safe.directory /builds/fastmachinelearning/hls4ml - - git submodule update --init --recursive hls4ml/templates/catapult/ - - if [ $EXAMPLEMODEL == 1 ]; then git submodule update --init example-models; fi - - pip install .${EXTRA_DEPS} - - # set up Vivado commands - - mkdir -p cmd_vivado_${VIVADO_VERSION} - - echo '#!/bin/bash' > cmd_vivado_${VIVADO_VERSION}/vivado_hls - - echo "apptainer exec --cleanenv --env LANG=C,LC_ALL=C /cvmfs/projects.cern.ch/hls4ml/vivado/${VIVADO_VERSION} vivado_hls \"\$@\"" >> cmd_vivado_${VIVADO_VERSION}/vivado_hls - - echo '#!/bin/bash' > cmd_vivado_${VIVADO_VERSION}/vivado - - echo "apptainer exec --cleanenv --env LANG=C,LC_ALL=C /cvmfs/projects.cern.ch/hls4ml/vivado/${VIVADO_VERSION} vivado \"\$@\"" >> cmd_vivado_${VIVADO_VERSION}/vivado - - chmod +x cmd_vivado_${VIVADO_VERSION}/vivado_hls - - chmod +x cmd_vivado_${VIVADO_VERSION}/vivado - - export PATH=$PWD/cmd_vivado_${VIVADO_VERSION}:$PATH - - # set up vitis-run command - - mkdir -p cmd_vitis_${VITIS_VERSION} - - echo '#!/bin/bash' > cmd_vitis_${VITIS_VERSION}/vitis-run - - echo "apptainer exec /cvmfs/projects.cern.ch/hls4ml/vitis/${VITIS_VERSION} vitis-run \"\$@\"" >> cmd_vitis_${VITIS_VERSION}/vitis-run - - chmod +x cmd_vitis_${VITIS_VERSION}/vitis-run - - export PATH=$PWD/cmd_vitis_${VITIS_VERSION}:$PATH - - # Load Intel oneAPI environment variables - - source /opt/intel/oneapi/setvars.sh --force + extends: .pytest-base script: + # Load Intel oneAPI environment variables for oneAPI-related tests in the default matrix. + - source /opt/intel/oneapi/setvars.sh --force + - cd test/pytest - pytest $PYTESTFILE -rA --cov-report xml --cov-report term --cov=hls4ml --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed artifacts: diff --git a/test/pytest/conftest.py b/test/pytest/conftest.py index 0e1d9d1fae..5f58ff8b11 100644 --- a/test/pytest/conftest.py +++ b/test/pytest/conftest.py @@ -63,18 +63,23 @@ def synthesis_config(): }, 'build_args': { 'Vivado': {'csim': False, 'synth': True, 'export': False}, - # Full accelerator flow: run C/RTL synthesis, downstream Vivado synthesis, and board project/bitfile. + # Default synthesis-style arguments used by baseline comparison tests. + 'VivadoAccelerator': {'csim': False, 'synth': True, 'export': False}, + 'Vitis': {'csim': False, 'synth': True, 'export': False}, + 'Quartus': {'synth': True, 'fpgasynth': False}, + 'oneAPI': {'build_type': 'report', 'run': False}, + }, + 'implementation_build_args': { + # Full accelerator flow for implementation dataset collection: + # run HLS synth, downstream Vivado synth, and bitfile generation. 'VivadoAccelerator': { 'csim': False, 'synth': True, - 'cosim': False, + 'cosim': True, 'validation': False, 'export': True, 'vsynth': True, 'bitfile': True, - }, - 'Vitis': {'csim': False, 'synth': True, 'export': False}, - 'Quartus': {'synth': True, 'fpgasynth': False}, - 'oneAPI': {'build_type': 'report', 'run': False}, + } }, } diff --git a/test/pytest/generate_ci_yaml.py b/test/pytest/generate_ci_yaml.py index 80b86768c1..bf9fed81e2 100644 --- a/test/pytest/generate_ci_yaml.py +++ b/test/pytest/generate_ci_yaml.py @@ -25,6 +25,7 @@ # Blacklisted tests will be skipped BLACKLIST = {'test_reduction'} +EXCLUDED_DIRS = {'implementation'} # Long-running tests will not be bundled with other tests LONGLIST = {'test_hgq_layers', 'test_hgq_players', 'test_qkeras', 'test_pytorch_api'} @@ -41,10 +42,14 @@ # Value = chunk size per CI job SPLIT_BY_TEST_CASE = { 'test_keras_api': 1, - 'test_keras_api_vivadoacc': 1, } +def include_in_default_ci(path: Path) -> bool: + """Exclude dedicated suites (e.g. implementation-only jobs) from the default CI matrix.""" + return not any(part in EXCLUDED_DIRS for part in path.parts) + + def collect_test_functions_from_ast(test_file): """Collect all test function names using AST parsing (no imports).""" with open(test_file, encoding='utf-8') as f: @@ -80,7 +85,8 @@ def generate_test_yaml(test_root='.'): test_paths = [ path for path in test_root.glob('**/test_*.py') - if path.stem not in (BLACKLIST | LONGLIST | set(SPLIT_BY_TEST_CASE.keys()) | KERAS3_LIST) + if include_in_default_ci(path) + and path.stem not in (BLACKLIST | LONGLIST | set(SPLIT_BY_TEST_CASE.keys()) | KERAS3_LIST) ] need_example_models = [uses_example_model(path) for path in test_paths] @@ -100,7 +106,7 @@ def generate_test_yaml(test_root='.'): else: yml.update(diff_yml) - test_paths = [path for path in test_root.glob('**/test_*.py') if path.stem in LONGLIST] + test_paths = [path for path in test_root.glob('**/test_*.py') if include_in_default_ci(path) and path.stem in LONGLIST] for path in test_paths: name = path.stem.replace('test_', '') test_file = str(path.relative_to(test_root)) @@ -108,7 +114,9 @@ def generate_test_yaml(test_root='.'): diff_yml = yaml.safe_load(template.format(name, '.pytest', test_file, int(needs_examples))) yml.update(diff_yml) - test_paths = [path for path in test_root.glob('**/test_*.py') if path.stem in SPLIT_BY_TEST_CASE] + test_paths = [ + path for path in test_root.glob('**/test_*.py') if include_in_default_ci(path) and path.stem in SPLIT_BY_TEST_CASE + ] for path in test_paths: stem = path.stem name_base = stem.replace('test_', '') @@ -126,7 +134,9 @@ def generate_test_yaml(test_root='.'): else: yml.update(diff_yml) - keras3_paths = [path for path in test_root.glob('**/test_*.py') if path.stem in KERAS3_LIST] + keras3_paths = [ + path for path in test_root.glob('**/test_*.py') if include_in_default_ci(path) and path.stem in KERAS3_LIST + ] keras3_need_examples = [uses_example_model(path) for path in keras3_paths] k3_idxs = list(range(len(keras3_need_examples))) diff --git a/test/pytest/implementation/README.md b/test/pytest/implementation/README.md new file mode 100644 index 0000000000..d18e314658 --- /dev/null +++ b/test/pytest/implementation/README.md @@ -0,0 +1,36 @@ +# Implementation CI Suite + +This directory contains implementation-oriented pytest cases (post-HLS reports and full bitfile flow checks). + +## Pipeline separation + +The project keeps one GitLab entrypoint (`.gitlab-ci.yml`) but has two pipeline modes: + +- default mode (`CI_PIPELINE_MODE` unset): runs normal pytest CI only +- implementation mode (`CI_PIPELINE_MODE=implementation`): runs only the implementation child pipeline + +Implementation mode is dispatched to: + +- `test/pytest/implementation/pytests.yml` (static job list) + +Both normal and implementation templates reuse the shared base setup: + +- `test/pytest/ci-base-template.yml` + +and keep their behavior-specific logic in: + +- `test/pytest/ci-template.yml` (normal matrix) +- `test/pytest/implementation/ci-template.yml` (implementation suite) + +Tool exposure policy: + +- normal template uses `vivado_hls` +- implementation template additionally exposes full `vivado` for bitfile-oriented flows +- implementation dataset output path is controlled by `IMPLEMENTATION_DATASET_DIR` + (default in CI: `test/pytest/implementation`) + +## Adding new implementation tests + +1. Add a new file in this folder matching `test_*.py`. +2. Add a matching static CI job entry in `test/pytest/implementation/pytests.yml`. +3. Keep tests collect-only: validate full flow success and emit dataset artifacts. diff --git a/test/pytest/implementation/ci-template.yml b/test/pytest/implementation/ci-template.yml new file mode 100644 index 0000000000..126b036da6 --- /dev/null +++ b/test/pytest/implementation/ci-template.yml @@ -0,0 +1,21 @@ +.pytest-implementation: + extends: .pytest-base + variables: + IMPLEMENTATION_DATASET_DIR: "test/pytest/implementation" + script: + # Expose full Vivado only for implementation pipeline jobs. + - echo '#!/bin/bash' > cmd_vivado_${VIVADO_VERSION}/vivado + - echo "apptainer exec --cleanenv --env LANG=C,LC_ALL=C /cvmfs/projects.cern.ch/hls4ml/vivado/${VIVADO_VERSION} vivado \"\$@\"" >> cmd_vivado_${VIVADO_VERSION}/vivado + - chmod +x cmd_vivado_${VIVADO_VERSION}/vivado + + - cd test/pytest + - pytest $PYTESTFILE -rA --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed + artifacts: + when: always + reports: + junit: + - test/pytest/report.xml + paths: + - test/pytest/*.tar.gz + - test/pytest/synthesis_report_*.json + - ${IMPLEMENTATION_DATASET_DIR}/implementation_dataset_*.json diff --git a/test/pytest/implementation/pytests.yml b/test/pytest/implementation/pytests.yml new file mode 100644 index 0000000000..f66b3dca8d --- /dev/null +++ b/test/pytest/implementation/pytests.yml @@ -0,0 +1,7 @@ +pytest.implementation.keras_api_vivadoacc: + extends: .pytest-implementation + variables: + PYTESTFILE: implementation/test_keras_api_vivadoacc.py + EXAMPLEMODEL: 0 + VIVADO_VERSION: "2020.1" + VITIS_VERSION: "2024.1" diff --git a/test/pytest/test_keras_api_vivadoacc.py b/test/pytest/implementation/test_keras_api_vivadoacc.py similarity index 80% rename from test/pytest/test_keras_api_vivadoacc.py rename to test/pytest/implementation/test_keras_api_vivadoacc.py index b259c9fc7a..5f8eff2d5e 100644 --- a/test/pytest/test_keras_api_vivadoacc.py +++ b/test/pytest/implementation/test_keras_api_vivadoacc.py @@ -3,11 +3,8 @@ import numpy as np import pytest import tensorflow as tf -from synthesis_helpers import run_synthesis_test -from tensorflow.keras.layers import ( - Activation, - Dense, -) +from synthesis_helpers import run_implementation_collection_test +from tensorflow.keras.layers import Activation, Dense import hls4ml @@ -28,24 +25,16 @@ def test_dense(test_case_id, backend, io_type, synthesis_config): use_bias=True, kernel_initializer=tf.keras.initializers.RandomUniform(minval=1, maxval=10), bias_initializer='zeros', - kernel_regularizer=None, - bias_regularizer=None, - activity_regularizer=None, - kernel_constraint=None, - bias_constraint=None, ) ) model.add(Activation(activation='elu', name='Activation')) model.compile(optimizer='adam', loss='mse') - X_input = np.random.rand(100, 1) - - keras_prediction = model.predict(X_input) + x_input = np.random.rand(100, 1) + keras_prediction = model.predict(x_input) config = hls4ml.utils.config_from_keras_model(model) output_dir = str(test_root_path / test_case_id) - baseline_file_name = f'{test_case_id}.json' - hls_model = hls4ml.converters.convert_from_keras_model( model, hls_config=config, @@ -57,8 +46,7 @@ def test_dense(test_case_id, backend, io_type, synthesis_config): ) hls_model.compile() - - hls_prediction = hls_model.predict(X_input) + hls_prediction = hls_model.predict(x_input) np.testing.assert_allclose(hls_prediction, keras_prediction, rtol=1e-2, atol=0.01) @@ -72,9 +60,10 @@ def test_dense(test_case_id, backend, io_type, synthesis_config): assert list(hls_model.get_layers())[2].attributes['activation'] == str(model.layers[1].activation).split()[1] assert list(hls_model.get_layers())[1].attributes['activation'] == str(model.layers[0].activation).split()[1] - run_synthesis_test( + run_implementation_collection_test( config=synthesis_config, hls_model=hls_model, - baseline_file_name=baseline_file_name, + test_case_id=test_case_id, backend=backend, + metadata={'board': VIVADOACC_BOARD, 'part': VIVADOACC_PART}, ) diff --git a/test/pytest/synthesis_helpers.py b/test/pytest/synthesis_helpers.py index 7a71700b2c..ca73b72e05 100644 --- a/test/pytest/synthesis_helpers.py +++ b/test/pytest/synthesis_helpers.py @@ -1,4 +1,7 @@ import json +import os +import subprocess +from datetime import datetime, timezone from pathlib import Path import pytest @@ -132,6 +135,60 @@ def compare_oneapi_backend(data, baseline): 'oneAPI': {'report'}, } +IMPLEMENTATION_EXPECTED_REPORT_KEYS = { + 'VivadoAccelerator': {'CSynthesisReport', 'VivadoSynthReport', 'TimingReport'}, +} + +BITFILE_REQUIRED_BACKENDS = {'VivadoAccelerator'} + +IMPLEMENTATION_REQUIRED_METADATA_FIELDS = { + 'VivadoAccelerator': {'board', 'part'}, +} + +DEFAULT_IMPLEMENTATION_DATASET_DIR = Path(__file__).parent / 'implementation' +IMPLEMENTATION_DATASET_DIR_ENV = 'IMPLEMENTATION_DATASET_DIR' + + +def _resolve_commit_sha(): + commit_sha = os.getenv('CI_COMMIT_SHA') + if commit_sha: + return commit_sha + + try: + return subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip() + except (subprocess.CalledProcessError, FileNotFoundError): + return 'unknown' + + +def _collect_bitfiles(output_dir): + output_path = Path(output_dir) + return sorted(str(path.relative_to(output_path)) for path in output_path.rglob('*.bit')) + + +def _save_implementation_dataset(data, test_case_id): + dataset_dir = Path(os.getenv(IMPLEMENTATION_DATASET_DIR_ENV, str(DEFAULT_IMPLEMENTATION_DATASET_DIR))) + dataset_dir.mkdir(parents=True, exist_ok=True) + out_path = dataset_dir / f'implementation_dataset_{test_case_id}.json' + with open(out_path, 'w') as fp: + json.dump(data, fp, indent=4, sort_keys=True) + + +def _validate_implementation_metadata(backend, metadata): + if metadata is None: + metadata = {} + if not isinstance(metadata, dict): + raise AssertionError('Implementation collection metadata must be a dictionary.') + + required_fields = IMPLEMENTATION_REQUIRED_METADATA_FIELDS.get(backend, set()) + missing_fields = sorted(field for field in required_fields if not metadata.get(field)) + if missing_fields: + raise AssertionError( + f'Missing required metadata for backend {backend}: {missing_fields}. ' + f'Provided metadata keys: {sorted(metadata.keys())}' + ) + + return metadata + def run_synthesis_test(config, hls_model, baseline_file_name, backend): """ @@ -184,3 +241,60 @@ def run_synthesis_test(config, hls_model, baseline_file_name, backend): raise AssertionError(f'No comparison function defined for backend: {backend}') compare_func(data, baseline) + + +def run_implementation_collection_test(config, hls_model, test_case_id, backend, metadata=None): + """ + Run an implementation-oriented backend build and write a dataset artifact. + + This helper is intended for implementation collection tests (not baseline comparison tests). + It runs the backend build using implementation-specific build arguments, validates that + expected report sections exist, optionally validates backend-specific output artifacts + (e.g. bitfile presence), and saves a JSON dataset artifact with report data + metadata. + + Args: + config (dict): Test configuration fixture, expected to contain tool versions and build args. + hls_model (object): hls4ml model instance to build. + test_case_id (str): Unique test identifier used in output dataset filename. + backend (str): Backend name (e.g. 'VivadoAccelerator'). + metadata (dict, optional): Backend metadata. + Required fields are backend-specific and validated by + ``IMPLEMENTATION_REQUIRED_METADATA_FIELDS``. + Example for VivadoAccelerator: ``{'board': 'zcu102', 'part': 'xczu9eg-ffvb1156-2-e'}``. + + Raises: + AssertionError: If required report keys are missing, required metadata is missing, + metadata type is invalid, or required output artifacts (e.g. bitfile) are missing. + pytest.fail: If backend build execution fails. + """ + build_args = config.get('implementation_build_args', config.get('build_args', {})) + try: + report = hls_model.build(**build_args.get(backend, {})) + except Exception as e: + pytest.fail(f'hls_model.build failed: {e}') + + expected_keys = IMPLEMENTATION_EXPECTED_REPORT_KEYS.get(backend, set()) + assert report and expected_keys.issubset(report.keys()), ( + f'Implementation collection failed: Missing expected keys in report: ' + f'expected {expected_keys}, got {set(report.keys()) if report else set()}' + ) + + bitfiles = [] + if backend in BITFILE_REQUIRED_BACKENDS: + bitfiles = _collect_bitfiles(hls_model.config.get_output_dir()) + assert bitfiles, 'Bitfile generation failed: no .bit file was found in the output directory.' + + metadata = _validate_implementation_metadata(backend, metadata) + + dataset_metadata = { + 'test_id': test_case_id, + 'backend': backend, + 'tool_version': config.get('tools_version', {}).get(backend, 'unknown'), + 'commit_sha': _resolve_commit_sha(), + 'collected_at_utc': datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace('+00:00', 'Z'), + 'bitfiles': bitfiles, + } + dataset_metadata.update(metadata) + + dataset = {'metadata': dataset_metadata, 'report': report} + _save_implementation_dataset(dataset, test_case_id) From dd8912a803e2611a1a084d62d11332d3b7d77791 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Tue, 14 Apr 2026 14:39:37 -0500 Subject: [PATCH 10/27] update ci scripts to handle different tools installation --- test/pytest/ci-base-template.yml | 14 ------ test/pytest/ci-template.yml | 13 +++++ test/pytest/implementation/README.md | 8 +++- test/pytest/implementation/ci-template.yml | 5 -- test/pytest/implementation/pytests.yml | 55 +++++++++++++++++++++- 5 files changed, 73 insertions(+), 22 deletions(-) diff --git a/test/pytest/ci-base-template.yml b/test/pytest/ci-base-template.yml index 9aad35380a..4dbd89d05d 100644 --- a/test/pytest/ci-base-template.yml +++ b/test/pytest/ci-base-template.yml @@ -13,17 +13,3 @@ - git submodule update --init --recursive hls4ml/templates/catapult/ - if [ $EXAMPLEMODEL == 1 ]; then git submodule update --init example-models; fi - pip install .${EXTRA_DEPS} - - # set up Vivado HLS command used by regular synthesis tests - - mkdir -p cmd_vivado_${VIVADO_VERSION} - - echo '#!/bin/bash' > cmd_vivado_${VIVADO_VERSION}/vivado_hls - - echo "apptainer exec --cleanenv --env LANG=C,LC_ALL=C /cvmfs/projects.cern.ch/hls4ml/vivado/${VIVADO_VERSION} vivado_hls \"\$@\"" >> cmd_vivado_${VIVADO_VERSION}/vivado_hls - - chmod +x cmd_vivado_${VIVADO_VERSION}/vivado_hls - - export PATH=$PWD/cmd_vivado_${VIVADO_VERSION}:$PATH - - # set up vitis-run command - - mkdir -p cmd_vitis_${VITIS_VERSION} - - echo '#!/bin/bash' > cmd_vitis_${VITIS_VERSION}/vitis-run - - echo "apptainer exec /cvmfs/projects.cern.ch/hls4ml/vitis/${VITIS_VERSION} vitis-run \"\$@\"" >> cmd_vitis_${VITIS_VERSION}/vitis-run - - chmod +x cmd_vitis_${VITIS_VERSION}/vitis-run - - export PATH=$PWD/cmd_vitis_${VITIS_VERSION}:$PATH diff --git a/test/pytest/ci-template.yml b/test/pytest/ci-template.yml index d8fe4214a4..d5959cc28a 100644 --- a/test/pytest/ci-template.yml +++ b/test/pytest/ci-template.yml @@ -4,6 +4,19 @@ # Load Intel oneAPI environment variables for oneAPI-related tests in the default matrix. - source /opt/intel/oneapi/setvars.sh --force + # Tool wrappers for normal pytest CI. + - mkdir -p cmd_vivado_${VIVADO_VERSION} + - echo '#!/bin/bash' > cmd_vivado_${VIVADO_VERSION}/vivado_hls + - echo "apptainer exec --cleanenv --env LANG=C,LC_ALL=C /cvmfs/projects.cern.ch/hls4ml/vivado/${VIVADO_VERSION} vivado_hls \"\$@\"" >> cmd_vivado_${VIVADO_VERSION}/vivado_hls + - chmod +x cmd_vivado_${VIVADO_VERSION}/vivado_hls + - export PATH=$PWD/cmd_vivado_${VIVADO_VERSION}:$PATH + + - mkdir -p cmd_vitis_${VITIS_VERSION} + - echo '#!/bin/bash' > cmd_vitis_${VITIS_VERSION}/vitis-run + - echo "apptainer exec /cvmfs/projects.cern.ch/hls4ml/vitis/${VITIS_VERSION} vitis-run \"\$@\"" >> cmd_vitis_${VITIS_VERSION}/vitis-run + - chmod +x cmd_vitis_${VITIS_VERSION}/vitis-run + - export PATH=$PWD/cmd_vitis_${VITIS_VERSION}:$PATH + - cd test/pytest - pytest $PYTESTFILE -rA --cov-report xml --cov-report term --cov=hls4ml --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed artifacts: diff --git a/test/pytest/implementation/README.md b/test/pytest/implementation/README.md index d18e314658..fb3e27ef14 100644 --- a/test/pytest/implementation/README.md +++ b/test/pytest/implementation/README.md @@ -25,7 +25,9 @@ and keep their behavior-specific logic in: Tool exposure policy: - normal template uses `vivado_hls` -- implementation template additionally exposes full `vivado` for bitfile-oriented flows +- implementation template exposes `vivado_hls`, `vivado`, `vitis-run`, `v++`, and `vitis` +- implementation jobs select tool-exposure mode by extending a script profile in + `implementation/pytests.yml` (`.implementation.script.tools-*`) - implementation dataset output path is controlled by `IMPLEMENTATION_DATASET_DIR` (default in CI: `test/pytest/implementation`) @@ -33,4 +35,6 @@ Tool exposure policy: 1. Add a new file in this folder matching `test_*.py`. 2. Add a matching static CI job entry in `test/pytest/implementation/pytests.yml`. -3. Keep tests collect-only: validate full flow success and emit dataset artifacts. +3. Set per-job tool versions (`VIVADO_VERSION`, `VITIS_VERSION`) in `pytests.yml`. +4. Select a per-job tool wrapper script profile in `pytests.yml`. +5. Keep tests collect-only: validate full flow success and emit dataset artifacts. diff --git a/test/pytest/implementation/ci-template.yml b/test/pytest/implementation/ci-template.yml index 126b036da6..2ca1b9dd27 100644 --- a/test/pytest/implementation/ci-template.yml +++ b/test/pytest/implementation/ci-template.yml @@ -3,11 +3,6 @@ variables: IMPLEMENTATION_DATASET_DIR: "test/pytest/implementation" script: - # Expose full Vivado only for implementation pipeline jobs. - - echo '#!/bin/bash' > cmd_vivado_${VIVADO_VERSION}/vivado - - echo "apptainer exec --cleanenv --env LANG=C,LC_ALL=C /cvmfs/projects.cern.ch/hls4ml/vivado/${VIVADO_VERSION} vivado \"\$@\"" >> cmd_vivado_${VIVADO_VERSION}/vivado - - chmod +x cmd_vivado_${VIVADO_VERSION}/vivado - - cd test/pytest - pytest $PYTESTFILE -rA --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed artifacts: diff --git a/test/pytest/implementation/pytests.yml b/test/pytest/implementation/pytests.yml index f66b3dca8d..b5ff8b19d3 100644 --- a/test/pytest/implementation/pytests.yml +++ b/test/pytest/implementation/pytests.yml @@ -1,7 +1,60 @@ -pytest.implementation.keras_api_vivadoacc: +.implementation.runtime.vivado-image: + extends: .pytest-implementation + script: + # Prepare Vivado wrapper directory. + - mkdir -p cmd_vivado_${VIVADO_VERSION} + # Export vivado_hls from Vivado apptainer. + - echo '#!/bin/bash' > cmd_vivado_${VIVADO_VERSION}/vivado_hls + - echo "apptainer exec --cleanenv --env LANG=C,LC_ALL=C /cvmfs/projects.cern.ch/hls4ml/vivado/${VIVADO_VERSION} vivado_hls \"\$@\"" >> cmd_vivado_${VIVADO_VERSION}/vivado_hls + - chmod +x cmd_vivado_${VIVADO_VERSION}/vivado_hls + # Export vivado from Vivado apptainer. + - echo '#!/bin/bash' > cmd_vivado_${VIVADO_VERSION}/vivado + - echo "apptainer exec --cleanenv --env LANG=C,LC_ALL=C /cvmfs/projects.cern.ch/hls4ml/vivado/${VIVADO_VERSION} vivado \"\$@\"" >> cmd_vivado_${VIVADO_VERSION}/vivado + - chmod +x cmd_vivado_${VIVADO_VERSION}/vivado + # Add wrapper paths. + - export PATH=$PWD/cmd_vivado_${VIVADO_VERSION}:$PWD/cmd_vitis_${VITIS_VERSION}:$PATH + # Run implementation test. + - cd test/pytest + - pytest $PYTESTFILE -rA --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed + +.implementation.runtime.vitis-image: extends: .pytest-implementation + script: + # Prepare Vivado wrapper directory. + - mkdir -p cmd_vivado_${VIVADO_VERSION} + # Export vivado from Vitis apptainer. + - echo '#!/bin/bash' > cmd_vivado_${VIVADO_VERSION}/vivado + - echo "apptainer exec --cleanenv --env LANG=C,LC_ALL=C /cvmfs/projects.cern.ch/hls4ml/vitis/${VITIS_VERSION} vivado \"\$@\"" >> cmd_vivado_${VIVADO_VERSION}/vivado + - chmod +x cmd_vivado_${VIVADO_VERSION}/vivado + # Prepare Vitis wrapper directory. + - mkdir -p cmd_vitis_${VITIS_VERSION} + # Export vitis-run from Vitis apptainer. + - echo '#!/bin/bash' > cmd_vitis_${VITIS_VERSION}/vitis-run + - echo "apptainer exec /cvmfs/projects.cern.ch/hls4ml/vitis/${VITIS_VERSION} vitis-run \"\$@\"" >> cmd_vitis_${VITIS_VERSION}/vitis-run + - chmod +x cmd_vitis_${VITIS_VERSION}/vitis-run + # Export v++ from Vitis apptainer. + - echo '#!/bin/bash' > cmd_vitis_${VITIS_VERSION}/v++ + - echo "apptainer exec /cvmfs/projects.cern.ch/hls4ml/vitis/${VITIS_VERSION} v++ \"\$@\"" >> cmd_vitis_${VITIS_VERSION}/v++ + - chmod +x cmd_vitis_${VITIS_VERSION}/v++ + # Export vitis from Vitis apptainer. + - echo '#!/bin/bash' > cmd_vitis_${VITIS_VERSION}/vitis + - echo "apptainer exec /cvmfs/projects.cern.ch/hls4ml/vitis/${VITIS_VERSION} vitis \"\$@\"" >> cmd_vitis_${VITIS_VERSION}/vitis + - chmod +x cmd_vitis_${VITIS_VERSION}/vitis + # Add wrapper paths. + - export PATH=$PWD/cmd_vivado_${VIVADO_VERSION}:$PWD/cmd_vitis_${VITIS_VERSION}:$PATH + # Run implementation test. + - cd test/pytest + - pytest $PYTESTFILE -rA --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed + +.implementation.test.keras_api_vivadoacc: variables: PYTESTFILE: implementation/test_keras_api_vivadoacc.py EXAMPLEMODEL: 0 + +pytest.implementation.keras_api_vivadoacc: + extends: + - .implementation.runtime.vivado-image + - .implementation.test.keras_api_vivadoacc + variables: VIVADO_VERSION: "2020.1" VITIS_VERSION: "2024.1" From 26512c1546d1d08d1f2a1b5f38caefb364ab916e Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Thu, 30 Apr 2026 14:08:45 -0500 Subject: [PATCH 11/27] Add implementation CI dataset collection --- .gitlab-ci.yml | 2 - test/pytest/ci-base-template.yml | 15 -- test/pytest/ci-template.yml | 24 +- test/pytest/implementation/ci-template.yml | 64 +++++- test/pytest/implementation/pytests.yml | 51 +---- .../test_keras_api_vivadoacc.py | 2 +- test/pytest/implementation_helpers.py | 207 ++++++++++++++++++ test/pytest/synthesis_helpers.py | 116 ---------- 8 files changed, 286 insertions(+), 195 deletions(-) delete mode 100644 test/pytest/ci-base-template.yml create mode 100644 test/pytest/implementation_helpers.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f3eec056d1..6e36bedef3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,7 +35,6 @@ pytests: - when: on_success trigger: include: - - local: test/pytest/ci-base-template.yml - local: test/pytest/ci-template.yml - artifact: test/pytest/pytests.yml job: generator @@ -49,7 +48,6 @@ implementation-pytests: - when: never trigger: include: - - local: test/pytest/ci-base-template.yml - local: test/pytest/implementation/ci-template.yml - local: test/pytest/implementation/pytests.yml strategy: depend diff --git a/test/pytest/ci-base-template.yml b/test/pytest/ci-base-template.yml deleted file mode 100644 index 4dbd89d05d..0000000000 --- a/test/pytest/ci-base-template.yml +++ /dev/null @@ -1,15 +0,0 @@ -.pytest-base: - stage: test - image: gitlab-registry.cern.ch/fastmachinelearning/hls4ml-testing:0.6.3.base - tags: - - k8s-default - variables: - CONDA_ENV: "hls4ml-testing" - EXTRA_DEPS: "[da,testing,testing-keras2,sr,optimization]" - before_script: - - eval "$(conda shell.bash hook)" - - conda activate "$CONDA_ENV" - - git config --global --add safe.directory /builds/fastmachinelearning/hls4ml - - git submodule update --init --recursive hls4ml/templates/catapult/ - - if [ $EXAMPLEMODEL == 1 ]; then git submodule update --init example-models; fi - - pip install .${EXTRA_DEPS} diff --git a/test/pytest/ci-template.yml b/test/pytest/ci-template.yml index d5959cc28a..ebbcd8a21e 100644 --- a/test/pytest/ci-template.yml +++ b/test/pytest/ci-template.yml @@ -1,22 +1,36 @@ .pytest: - extends: .pytest-base - script: - # Load Intel oneAPI environment variables for oneAPI-related tests in the default matrix. - - source /opt/intel/oneapi/setvars.sh --force + stage: test + image: gitlab-registry.cern.ch/fastmachinelearning/hls4ml-testing:0.6.3.base + tags: + - k8s-default + variables: + CONDA_ENV: "hls4ml-testing" + EXTRA_DEPS: "[da,testing,testing-keras2,sr,optimization]" + before_script: + - eval "$(conda shell.bash hook)" + - conda activate "$CONDA_ENV" + - git config --global --add safe.directory /builds/fastmachinelearning/hls4ml + - git submodule update --init --recursive hls4ml/templates/catapult/ + - if [ $EXAMPLEMODEL == 1 ]; then git submodule update --init example-models; fi + - pip install .${EXTRA_DEPS} - # Tool wrappers for normal pytest CI. + # set up vivado_hls command - mkdir -p cmd_vivado_${VIVADO_VERSION} - echo '#!/bin/bash' > cmd_vivado_${VIVADO_VERSION}/vivado_hls - echo "apptainer exec --cleanenv --env LANG=C,LC_ALL=C /cvmfs/projects.cern.ch/hls4ml/vivado/${VIVADO_VERSION} vivado_hls \"\$@\"" >> cmd_vivado_${VIVADO_VERSION}/vivado_hls - chmod +x cmd_vivado_${VIVADO_VERSION}/vivado_hls - export PATH=$PWD/cmd_vivado_${VIVADO_VERSION}:$PATH + # set up vitis-run command - mkdir -p cmd_vitis_${VITIS_VERSION} - echo '#!/bin/bash' > cmd_vitis_${VITIS_VERSION}/vitis-run - echo "apptainer exec /cvmfs/projects.cern.ch/hls4ml/vitis/${VITIS_VERSION} vitis-run \"\$@\"" >> cmd_vitis_${VITIS_VERSION}/vitis-run - chmod +x cmd_vitis_${VITIS_VERSION}/vitis-run - export PATH=$PWD/cmd_vitis_${VITIS_VERSION}:$PATH + # Load Intel oneAPI environment variables + - source /opt/intel/oneapi/setvars.sh --force + script: - cd test/pytest - pytest $PYTESTFILE -rA --cov-report xml --cov-report term --cov=hls4ml --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed artifacts: diff --git a/test/pytest/implementation/ci-template.yml b/test/pytest/implementation/ci-template.yml index 2ca1b9dd27..bf24916ab0 100644 --- a/test/pytest/implementation/ci-template.yml +++ b/test/pytest/implementation/ci-template.yml @@ -1,10 +1,19 @@ .pytest-implementation: - extends: .pytest-base + stage: test + image: gitlab-registry.cern.ch/fastmachinelearning/hls4ml-testing:0.6.3.base + tags: + - k8s-default variables: + CONDA_ENV: "hls4ml-testing" + EXTRA_DEPS: "[da,testing,testing-keras2,sr,optimization]" IMPLEMENTATION_DATASET_DIR: "test/pytest/implementation" - script: - - cd test/pytest - - pytest $PYTESTFILE -rA --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed + before_script: + - eval "$(conda shell.bash hook)" + - conda activate "$CONDA_ENV" + - git config --global --add safe.directory /builds/fastmachinelearning/hls4ml + - git submodule update --init --recursive hls4ml/templates/catapult/ + - if [ $EXAMPLEMODEL == 1 ]; then git submodule update --init example-models; fi + - pip install .${EXTRA_DEPS} artifacts: when: always reports: @@ -12,5 +21,48 @@ - test/pytest/report.xml paths: - test/pytest/*.tar.gz - - test/pytest/synthesis_report_*.json - - ${IMPLEMENTATION_DATASET_DIR}/implementation_dataset_*.json + - test/pytest/implementation/implementation_dataset_*.json + - test/pytest/implementation/implementation_parsed_report_*.json + - test/pytest/implementation/implementation_build_output_*.log + - test/pytest/implementation/*/ + - test/pytest/implementation/**/*.bit + - test/pytest/implementation/**/*.hwh + - test/pytest/implementation/**/*.rpt + - test/pytest/implementation/**/*.log + +.pytest-implementation-vivado-runtime: + extends: .pytest-implementation + script: + - mkdir -p cmd_vivado_${VIVADO_VERSION} + - echo '#!/bin/bash' > cmd_vivado_${VIVADO_VERSION}/vivado_hls + - echo "apptainer exec --cleanenv --env LANG=C,LC_ALL=C /cvmfs/projects.cern.ch/hls4ml/vivado/${VIVADO_VERSION} vivado_hls \"\$@\"" >> cmd_vivado_${VIVADO_VERSION}/vivado_hls + - chmod +x cmd_vivado_${VIVADO_VERSION}/vivado_hls + - echo '#!/bin/bash' > cmd_vivado_${VIVADO_VERSION}/vivado + - echo "apptainer exec --cleanenv --env LANG=C,LC_ALL=C /cvmfs/projects.cern.ch/hls4ml/vivado/${VIVADO_VERSION} vivado \"\$@\"" >> cmd_vivado_${VIVADO_VERSION}/vivado + - chmod +x cmd_vivado_${VIVADO_VERSION}/vivado + - export PATH=$PWD/cmd_vivado_${VIVADO_VERSION}:$PATH + - export IMPLEMENTATION_DATASET_DIR=$PWD/test/pytest/implementation + - cd test/pytest + - pytest $PYTESTFILE -rA --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed + +.pytest-implementation-vitis-runtime: + extends: .pytest-implementation + script: + - mkdir -p cmd_vivado_${VIVADO_VERSION} + - echo '#!/bin/bash' > cmd_vivado_${VIVADO_VERSION}/vivado + - echo "apptainer exec --cleanenv --env LANG=C,LC_ALL=C /cvmfs/projects.cern.ch/hls4ml/vitis/${VITIS_VERSION} vivado \"\$@\"" >> cmd_vivado_${VIVADO_VERSION}/vivado + - chmod +x cmd_vivado_${VIVADO_VERSION}/vivado + - mkdir -p cmd_vitis_${VITIS_VERSION} + - echo '#!/bin/bash' > cmd_vitis_${VITIS_VERSION}/vitis-run + - echo "apptainer exec /cvmfs/projects.cern.ch/hls4ml/vitis/${VITIS_VERSION} vitis-run \"\$@\"" >> cmd_vitis_${VITIS_VERSION}/vitis-run + - chmod +x cmd_vitis_${VITIS_VERSION}/vitis-run + - echo '#!/bin/bash' > cmd_vitis_${VITIS_VERSION}/v++ + - echo "apptainer exec /cvmfs/projects.cern.ch/hls4ml/vitis/${VITIS_VERSION} v++ \"\$@\"" >> cmd_vitis_${VITIS_VERSION}/v++ + - chmod +x cmd_vitis_${VITIS_VERSION}/v++ + - echo '#!/bin/bash' > cmd_vitis_${VITIS_VERSION}/vitis + - echo "apptainer exec /cvmfs/projects.cern.ch/hls4ml/vitis/${VITIS_VERSION} vitis \"\$@\"" >> cmd_vitis_${VITIS_VERSION}/vitis + - chmod +x cmd_vitis_${VITIS_VERSION}/vitis + - export PATH=$PWD/cmd_vivado_${VIVADO_VERSION}:$PWD/cmd_vitis_${VITIS_VERSION}:$PATH + - export IMPLEMENTATION_DATASET_DIR=$PWD/test/pytest/implementation + - cd test/pytest + - pytest $PYTESTFILE -rA --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed diff --git a/test/pytest/implementation/pytests.yml b/test/pytest/implementation/pytests.yml index b5ff8b19d3..c2c6a14109 100644 --- a/test/pytest/implementation/pytests.yml +++ b/test/pytest/implementation/pytests.yml @@ -1,51 +1,3 @@ -.implementation.runtime.vivado-image: - extends: .pytest-implementation - script: - # Prepare Vivado wrapper directory. - - mkdir -p cmd_vivado_${VIVADO_VERSION} - # Export vivado_hls from Vivado apptainer. - - echo '#!/bin/bash' > cmd_vivado_${VIVADO_VERSION}/vivado_hls - - echo "apptainer exec --cleanenv --env LANG=C,LC_ALL=C /cvmfs/projects.cern.ch/hls4ml/vivado/${VIVADO_VERSION} vivado_hls \"\$@\"" >> cmd_vivado_${VIVADO_VERSION}/vivado_hls - - chmod +x cmd_vivado_${VIVADO_VERSION}/vivado_hls - # Export vivado from Vivado apptainer. - - echo '#!/bin/bash' > cmd_vivado_${VIVADO_VERSION}/vivado - - echo "apptainer exec --cleanenv --env LANG=C,LC_ALL=C /cvmfs/projects.cern.ch/hls4ml/vivado/${VIVADO_VERSION} vivado \"\$@\"" >> cmd_vivado_${VIVADO_VERSION}/vivado - - chmod +x cmd_vivado_${VIVADO_VERSION}/vivado - # Add wrapper paths. - - export PATH=$PWD/cmd_vivado_${VIVADO_VERSION}:$PWD/cmd_vitis_${VITIS_VERSION}:$PATH - # Run implementation test. - - cd test/pytest - - pytest $PYTESTFILE -rA --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed - -.implementation.runtime.vitis-image: - extends: .pytest-implementation - script: - # Prepare Vivado wrapper directory. - - mkdir -p cmd_vivado_${VIVADO_VERSION} - # Export vivado from Vitis apptainer. - - echo '#!/bin/bash' > cmd_vivado_${VIVADO_VERSION}/vivado - - echo "apptainer exec --cleanenv --env LANG=C,LC_ALL=C /cvmfs/projects.cern.ch/hls4ml/vitis/${VITIS_VERSION} vivado \"\$@\"" >> cmd_vivado_${VIVADO_VERSION}/vivado - - chmod +x cmd_vivado_${VIVADO_VERSION}/vivado - # Prepare Vitis wrapper directory. - - mkdir -p cmd_vitis_${VITIS_VERSION} - # Export vitis-run from Vitis apptainer. - - echo '#!/bin/bash' > cmd_vitis_${VITIS_VERSION}/vitis-run - - echo "apptainer exec /cvmfs/projects.cern.ch/hls4ml/vitis/${VITIS_VERSION} vitis-run \"\$@\"" >> cmd_vitis_${VITIS_VERSION}/vitis-run - - chmod +x cmd_vitis_${VITIS_VERSION}/vitis-run - # Export v++ from Vitis apptainer. - - echo '#!/bin/bash' > cmd_vitis_${VITIS_VERSION}/v++ - - echo "apptainer exec /cvmfs/projects.cern.ch/hls4ml/vitis/${VITIS_VERSION} v++ \"\$@\"" >> cmd_vitis_${VITIS_VERSION}/v++ - - chmod +x cmd_vitis_${VITIS_VERSION}/v++ - # Export vitis from Vitis apptainer. - - echo '#!/bin/bash' > cmd_vitis_${VITIS_VERSION}/vitis - - echo "apptainer exec /cvmfs/projects.cern.ch/hls4ml/vitis/${VITIS_VERSION} vitis \"\$@\"" >> cmd_vitis_${VITIS_VERSION}/vitis - - chmod +x cmd_vitis_${VITIS_VERSION}/vitis - # Add wrapper paths. - - export PATH=$PWD/cmd_vivado_${VIVADO_VERSION}:$PWD/cmd_vitis_${VITIS_VERSION}:$PATH - # Run implementation test. - - cd test/pytest - - pytest $PYTESTFILE -rA --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed - .implementation.test.keras_api_vivadoacc: variables: PYTESTFILE: implementation/test_keras_api_vivadoacc.py @@ -53,8 +5,7 @@ pytest.implementation.keras_api_vivadoacc: extends: - - .implementation.runtime.vivado-image + - .pytest-implementation-vivado-runtime - .implementation.test.keras_api_vivadoacc variables: VIVADO_VERSION: "2020.1" - VITIS_VERSION: "2024.1" diff --git a/test/pytest/implementation/test_keras_api_vivadoacc.py b/test/pytest/implementation/test_keras_api_vivadoacc.py index 5f8eff2d5e..715241ce2a 100644 --- a/test/pytest/implementation/test_keras_api_vivadoacc.py +++ b/test/pytest/implementation/test_keras_api_vivadoacc.py @@ -3,7 +3,7 @@ import numpy as np import pytest import tensorflow as tf -from synthesis_helpers import run_implementation_collection_test +from implementation_helpers import run_implementation_collection_test from tensorflow.keras.layers import Activation, Dense import hls4ml diff --git a/test/pytest/implementation_helpers.py b/test/pytest/implementation_helpers.py new file mode 100644 index 0000000000..1a5233032a --- /dev/null +++ b/test/pytest/implementation_helpers.py @@ -0,0 +1,207 @@ +import json +import os +import sys +import time +from contextlib import contextmanager +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +DATASET_SCHEMA_VERSION = 'implementation-dataset/v1' +DATASET_DIR_ENV = 'IMPLEMENTATION_DATASET_DIR' +DEFAULT_DATASET_DIR = Path(__file__).parent / 'implementation' + +EXPECTED_REPORT_KEYS = { + 'VivadoAccelerator': {'CSynthesisReport'}, +} + +REQUIRED_METADATA_FIELDS = { + 'VivadoAccelerator': {'board', 'part'}, +} + +BITFILE_REQUIRED_BACKENDS = {'VivadoAccelerator'} + + +def _utc_now(): + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace('+00:00', 'Z') + + +def _project_root(): + return Path(os.getenv('CI_PROJECT_DIR', Path(__file__).parents[2])) + + +def _portable_path(path): + return os.path.relpath(path, _project_root()) + + +def _collect_files(output_dir, suffixes=None): + output_path = Path(output_dir) + files = [] + if not output_path.exists(): + return files + + for path in sorted(output_path.rglob('*')): + if not path.is_file() or (suffixes is not None and path.suffix not in suffixes): + continue + entry = { + 'path': str(path.relative_to(output_path)), + 'size_bytes': path.stat().st_size, + } + files.append(entry) + return files + + +def _dataset_dir(): + return Path(os.getenv(DATASET_DIR_ENV, str(DEFAULT_DATASET_DIR))) + + +def _artifact_path(filename): + dataset_dir = _dataset_dir() + dataset_dir.mkdir(parents=True, exist_ok=True) + return dataset_dir / filename + + +def _write_json(data, filename): + out_path = _artifact_path(filename) + with open(out_path, 'w') as fp: + json.dump(data, fp, indent=4, sort_keys=True) + return out_path + + +@contextmanager +def _capture_terminal_output(path): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + sys.stdout.flush() + sys.stderr.flush() + saved_stdout = os.dup(1) + saved_stderr = os.dup(2) + with open(path, 'w') as fp: + os.dup2(fp.fileno(), 1) + os.dup2(fp.fileno(), 2) + try: + yield path + finally: + sys.stdout.flush() + sys.stderr.flush() + os.dup2(saved_stdout, 1) + os.dup2(saved_stderr, 2) + os.close(saved_stdout) + os.close(saved_stderr) + + +def _validate_metadata(backend, metadata): + if metadata is None: + metadata = {} + if not isinstance(metadata, dict): + raise TypeError('Implementation metadata must be a dictionary.') + + missing = REQUIRED_METADATA_FIELDS.get(backend, set()) - metadata.keys() + if missing: + raise AssertionError(f'Missing implementation metadata for {backend}: {sorted(missing)}') + return metadata + + +def _build_dataset( + config, + hls_model, + test_case_id, + backend, + metadata, + report, + build_args, + build_started_at, + build_finished_at, + build_duration_seconds, + parsed_report_path, + build_output_path, +): + output_dir = hls_model.config.get_output_dir() + + return { + 'schema_version': DATASET_SCHEMA_VERSION, + 'test_id': test_case_id, + 'hls_config': { + 'backend': backend, + 'project_name': hls_model.config.get_project_name(), + 'build_args': build_args, + }, + 'metadata': metadata, + 'toolchain': { + 'version': config.get('tools_version', {}).get(backend, 'unknown'), + }, + 'ci': { + 'commit_sha': os.getenv('CI_COMMIT_SHA'), + 'commit_ref': os.getenv('CI_COMMIT_REF_NAME'), + 'commit_tag': os.getenv('CI_COMMIT_TAG'), + 'pipeline_id': os.getenv('CI_PIPELINE_ID'), + 'job_id': os.getenv('CI_JOB_ID'), + 'project_url': os.getenv('CI_PROJECT_URL'), + 'job_image': os.getenv('CI_JOB_IMAGE'), + 'runner_description': os.getenv('CI_RUNNER_DESCRIPTION'), + 'runner_tags': os.getenv('CI_RUNNER_TAGS'), + }, + 'build': { + 'status': 'success', + 'started_at_utc': build_started_at, + 'finished_at_utc': build_finished_at, + 'duration_seconds': build_duration_seconds, + }, + 'parsed_reports': report, + 'artifacts': { + 'output_dir': _portable_path(output_dir), + 'parsed_report_json': _portable_path(parsed_report_path), + 'build_output_log': _portable_path(build_output_path), + 'bitstreams': _collect_files(output_dir, {'.bit'}), + 'reports': _collect_files(output_dir, {'.rpt', '.xml'}), + 'logs': _collect_files(output_dir, {'.log'}), + 'raw_output_files': _collect_files(output_dir), + }, + } + + +def run_implementation_collection_test(config, hls_model, test_case_id, backend, metadata=None): + """ + Build an implementation target and emit a dataset record plus raw backend reports. + """ + metadata = _validate_metadata(backend, metadata) + build_args = config.get('implementation_build_args', {}).get(backend, config.get('build_args', {}).get(backend, {})) + build_output_path = _artifact_path(f'implementation_build_output_{test_case_id}.log') + + started_at = _utc_now() + started = time.monotonic() + try: + with _capture_terminal_output(build_output_path): + report = hls_model.build(**build_args) + except Exception as e: + pytest.fail(f'hls_model.build failed: {e}') + finished_at = _utc_now() + duration = round(time.monotonic() - started, 3) + + expected_keys = EXPECTED_REPORT_KEYS.get(backend, set()) + assert report and expected_keys.issubset(report.keys()), ( + f'Implementation failed: missing expected report keys: expected {expected_keys}, got {set(report.keys())}' + ) + + output_dir = hls_model.config.get_output_dir() + bitfiles = _collect_files(output_dir, {'.bit'}) + if backend in BITFILE_REQUIRED_BACKENDS: + assert bitfiles, f'Implementation failed: no bitstream was generated in {output_dir}' + + parsed_report_path = _write_json(report, f'implementation_parsed_report_{test_case_id}.json') + dataset = _build_dataset( + config=config, + hls_model=hls_model, + test_case_id=test_case_id, + backend=backend, + metadata=metadata, + report=report, + build_args=build_args, + build_started_at=started_at, + build_finished_at=finished_at, + build_duration_seconds=duration, + parsed_report_path=parsed_report_path, + build_output_path=build_output_path, + ) + _write_json(dataset, f'implementation_dataset_{test_case_id}.json') diff --git a/test/pytest/synthesis_helpers.py b/test/pytest/synthesis_helpers.py index ca73b72e05..27d953b101 100644 --- a/test/pytest/synthesis_helpers.py +++ b/test/pytest/synthesis_helpers.py @@ -1,7 +1,4 @@ import json -import os -import subprocess -from datetime import datetime, timezone from pathlib import Path import pytest @@ -122,7 +119,6 @@ def compare_oneapi_backend(data, baseline): COMPARE_FUNCS = { 'Vivado': compare_vitis_backend, - 'VivadoAccelerator': compare_vitis_backend, 'Vitis': compare_vitis_backend, 'oneAPI': compare_oneapi_backend, } @@ -130,65 +126,10 @@ def compare_oneapi_backend(data, baseline): EXPECTED_REPORT_KEYS = { 'Vivado': {'CSynthesisReport'}, - 'VivadoAccelerator': {'CSynthesisReport'}, 'Vitis': {'CSynthesisReport'}, 'oneAPI': {'report'}, } -IMPLEMENTATION_EXPECTED_REPORT_KEYS = { - 'VivadoAccelerator': {'CSynthesisReport', 'VivadoSynthReport', 'TimingReport'}, -} - -BITFILE_REQUIRED_BACKENDS = {'VivadoAccelerator'} - -IMPLEMENTATION_REQUIRED_METADATA_FIELDS = { - 'VivadoAccelerator': {'board', 'part'}, -} - -DEFAULT_IMPLEMENTATION_DATASET_DIR = Path(__file__).parent / 'implementation' -IMPLEMENTATION_DATASET_DIR_ENV = 'IMPLEMENTATION_DATASET_DIR' - - -def _resolve_commit_sha(): - commit_sha = os.getenv('CI_COMMIT_SHA') - if commit_sha: - return commit_sha - - try: - return subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip() - except (subprocess.CalledProcessError, FileNotFoundError): - return 'unknown' - - -def _collect_bitfiles(output_dir): - output_path = Path(output_dir) - return sorted(str(path.relative_to(output_path)) for path in output_path.rglob('*.bit')) - - -def _save_implementation_dataset(data, test_case_id): - dataset_dir = Path(os.getenv(IMPLEMENTATION_DATASET_DIR_ENV, str(DEFAULT_IMPLEMENTATION_DATASET_DIR))) - dataset_dir.mkdir(parents=True, exist_ok=True) - out_path = dataset_dir / f'implementation_dataset_{test_case_id}.json' - with open(out_path, 'w') as fp: - json.dump(data, fp, indent=4, sort_keys=True) - - -def _validate_implementation_metadata(backend, metadata): - if metadata is None: - metadata = {} - if not isinstance(metadata, dict): - raise AssertionError('Implementation collection metadata must be a dictionary.') - - required_fields = IMPLEMENTATION_REQUIRED_METADATA_FIELDS.get(backend, set()) - missing_fields = sorted(field for field in required_fields if not metadata.get(field)) - if missing_fields: - raise AssertionError( - f'Missing required metadata for backend {backend}: {missing_fields}. ' - f'Provided metadata keys: {sorted(metadata.keys())}' - ) - - return metadata - def run_synthesis_test(config, hls_model, baseline_file_name, backend): """ @@ -241,60 +182,3 @@ def run_synthesis_test(config, hls_model, baseline_file_name, backend): raise AssertionError(f'No comparison function defined for backend: {backend}') compare_func(data, baseline) - - -def run_implementation_collection_test(config, hls_model, test_case_id, backend, metadata=None): - """ - Run an implementation-oriented backend build and write a dataset artifact. - - This helper is intended for implementation collection tests (not baseline comparison tests). - It runs the backend build using implementation-specific build arguments, validates that - expected report sections exist, optionally validates backend-specific output artifacts - (e.g. bitfile presence), and saves a JSON dataset artifact with report data + metadata. - - Args: - config (dict): Test configuration fixture, expected to contain tool versions and build args. - hls_model (object): hls4ml model instance to build. - test_case_id (str): Unique test identifier used in output dataset filename. - backend (str): Backend name (e.g. 'VivadoAccelerator'). - metadata (dict, optional): Backend metadata. - Required fields are backend-specific and validated by - ``IMPLEMENTATION_REQUIRED_METADATA_FIELDS``. - Example for VivadoAccelerator: ``{'board': 'zcu102', 'part': 'xczu9eg-ffvb1156-2-e'}``. - - Raises: - AssertionError: If required report keys are missing, required metadata is missing, - metadata type is invalid, or required output artifacts (e.g. bitfile) are missing. - pytest.fail: If backend build execution fails. - """ - build_args = config.get('implementation_build_args', config.get('build_args', {})) - try: - report = hls_model.build(**build_args.get(backend, {})) - except Exception as e: - pytest.fail(f'hls_model.build failed: {e}') - - expected_keys = IMPLEMENTATION_EXPECTED_REPORT_KEYS.get(backend, set()) - assert report and expected_keys.issubset(report.keys()), ( - f'Implementation collection failed: Missing expected keys in report: ' - f'expected {expected_keys}, got {set(report.keys()) if report else set()}' - ) - - bitfiles = [] - if backend in BITFILE_REQUIRED_BACKENDS: - bitfiles = _collect_bitfiles(hls_model.config.get_output_dir()) - assert bitfiles, 'Bitfile generation failed: no .bit file was found in the output directory.' - - metadata = _validate_implementation_metadata(backend, metadata) - - dataset_metadata = { - 'test_id': test_case_id, - 'backend': backend, - 'tool_version': config.get('tools_version', {}).get(backend, 'unknown'), - 'commit_sha': _resolve_commit_sha(), - 'collected_at_utc': datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace('+00:00', 'Z'), - 'bitfiles': bitfiles, - } - dataset_metadata.update(metadata) - - dataset = {'metadata': dataset_metadata, 'report': report} - _save_implementation_dataset(dataset, test_case_id) From 3837bbe5bfe0a6940241dd71dad672edcf272523 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Thu, 7 May 2026 16:13:04 -0500 Subject: [PATCH 12/27] update conftest.py --- test/pytest/conftest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/pytest/conftest.py b/test/pytest/conftest.py index 5f58ff8b11..654cd3d586 100644 --- a/test/pytest/conftest.py +++ b/test/pytest/conftest.py @@ -63,8 +63,6 @@ def synthesis_config(): }, 'build_args': { 'Vivado': {'csim': False, 'synth': True, 'export': False}, - # Default synthesis-style arguments used by baseline comparison tests. - 'VivadoAccelerator': {'csim': False, 'synth': True, 'export': False}, 'Vitis': {'csim': False, 'synth': True, 'export': False}, 'Quartus': {'synth': True, 'fpgasynth': False}, 'oneAPI': {'build_type': 'report', 'run': False}, @@ -73,6 +71,7 @@ def synthesis_config(): # Full accelerator flow for implementation dataset collection: # run HLS synth, downstream Vivado synth, and bitfile generation. 'VivadoAccelerator': { + 'reset': True, 'csim': False, 'synth': True, 'cosim': True, From 5cf3b897503af032b91c837e812fbd021c571132 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Fri, 8 May 2026 11:38:52 -0500 Subject: [PATCH 13/27] update implementaion tests reports and example models usage --- test/pytest/implementation/ci-template.yml | 6 +- test/pytest/implementation/pytests.yml | 22 ++++- .../test_keras_api_vivadoacc.py | 69 -------------- .../test_vivadoaccelerator_implementation.py | 95 +++++++++++++++++++ test/pytest/implementation_helpers.py | 15 +-- 5 files changed, 123 insertions(+), 84 deletions(-) delete mode 100644 test/pytest/implementation/test_keras_api_vivadoacc.py create mode 100644 test/pytest/implementation/test_vivadoaccelerator_implementation.py diff --git a/test/pytest/implementation/ci-template.yml b/test/pytest/implementation/ci-template.yml index bf24916ab0..8b91c352c2 100644 --- a/test/pytest/implementation/ci-template.yml +++ b/test/pytest/implementation/ci-template.yml @@ -21,9 +21,9 @@ - test/pytest/report.xml paths: - test/pytest/*.tar.gz - - test/pytest/implementation/implementation_dataset_*.json - - test/pytest/implementation/implementation_parsed_report_*.json - - test/pytest/implementation/implementation_build_output_*.log + - test/pytest/implementation/*_dataset.json + - test/pytest/implementation/*_hls4ml_report.json + - test/pytest/implementation/*_build.log - test/pytest/implementation/*/ - test/pytest/implementation/**/*.bit - test/pytest/implementation/**/*.hwh diff --git a/test/pytest/implementation/pytests.yml b/test/pytest/implementation/pytests.yml index c2c6a14109..eb6b510806 100644 --- a/test/pytest/implementation/pytests.yml +++ b/test/pytest/implementation/pytests.yml @@ -1,11 +1,23 @@ -.implementation.test.keras_api_vivadoacc: +.implementation.test.keras_3layer_vivadoacc: variables: - PYTESTFILE: implementation/test_keras_api_vivadoacc.py - EXAMPLEMODEL: 0 + PYTESTFILE: implementation/test_vivadoaccelerator_implementation.py::test_keras_3layer + EXAMPLEMODEL: 1 -pytest.implementation.keras_api_vivadoacc: +pytest.implementation.keras_3layer_vivadoacc: extends: - .pytest-implementation-vivado-runtime - - .implementation.test.keras_api_vivadoacc + - .implementation.test.keras_3layer_vivadoacc + variables: + VIVADO_VERSION: "2020.1" + +.implementation.test.keras_conv1d_small_vivadoacc: + variables: + PYTESTFILE: implementation/test_vivadoaccelerator_implementation.py::test_keras_conv1d_small + EXAMPLEMODEL: 1 + +pytest.implementation.keras_conv1d_small_vivadoacc: + extends: + - .pytest-implementation-vivado-runtime + - .implementation.test.keras_conv1d_small_vivadoacc variables: VIVADO_VERSION: "2020.1" diff --git a/test/pytest/implementation/test_keras_api_vivadoacc.py b/test/pytest/implementation/test_keras_api_vivadoacc.py deleted file mode 100644 index 715241ce2a..0000000000 --- a/test/pytest/implementation/test_keras_api_vivadoacc.py +++ /dev/null @@ -1,69 +0,0 @@ -from pathlib import Path - -import numpy as np -import pytest -import tensorflow as tf -from implementation_helpers import run_implementation_collection_test -from tensorflow.keras.layers import Activation, Dense - -import hls4ml - -test_root_path = Path(__file__).parent -VIVADOACC_BOARD = 'zcu102' -VIVADOACC_PART = 'xczu9eg-ffvb1156-2-e' - - -@pytest.mark.parametrize('backend', ['VivadoAccelerator']) -@pytest.mark.parametrize('io_type', ['io_parallel']) -def test_dense(test_case_id, backend, io_type, synthesis_config): - model = tf.keras.models.Sequential() - model.add( - Dense( - 2, - input_shape=(1,), - name='Dense', - use_bias=True, - kernel_initializer=tf.keras.initializers.RandomUniform(minval=1, maxval=10), - bias_initializer='zeros', - ) - ) - model.add(Activation(activation='elu', name='Activation')) - model.compile(optimizer='adam', loss='mse') - - x_input = np.random.rand(100, 1) - keras_prediction = model.predict(x_input) - - config = hls4ml.utils.config_from_keras_model(model) - output_dir = str(test_root_path / test_case_id) - hls_model = hls4ml.converters.convert_from_keras_model( - model, - hls_config=config, - output_dir=output_dir, - backend=backend, - io_type=io_type, - board=VIVADOACC_BOARD, - part=VIVADOACC_PART, - ) - - hls_model.compile() - hls_prediction = hls_model.predict(x_input) - - np.testing.assert_allclose(hls_prediction, keras_prediction, rtol=1e-2, atol=0.01) - - assert len(model.layers) + 1 == len(hls_model.get_layers()) - assert list(hls_model.get_layers())[0].attributes['class_name'] == 'InputLayer' - assert list(hls_model.get_layers())[1].attributes['class_name'] == model.layers[0]._name - assert list(hls_model.get_layers())[2].attributes['class_name'] == 'ELU' - assert list(hls_model.get_layers())[0].attributes['input_shape'] == list(model.layers[0].input_shape[1:]) - assert list(hls_model.get_layers())[1].attributes['n_in'] == model.layers[0].input_shape[1:][0] - assert list(hls_model.get_layers())[1].attributes['n_out'] == model.layers[0].output_shape[1:][0] - assert list(hls_model.get_layers())[2].attributes['activation'] == str(model.layers[1].activation).split()[1] - assert list(hls_model.get_layers())[1].attributes['activation'] == str(model.layers[0].activation).split()[1] - - run_implementation_collection_test( - config=synthesis_config, - hls_model=hls_model, - test_case_id=test_case_id, - backend=backend, - metadata={'board': VIVADOACC_BOARD, 'part': VIVADOACC_PART}, - ) diff --git a/test/pytest/implementation/test_vivadoaccelerator_implementation.py b/test/pytest/implementation/test_vivadoaccelerator_implementation.py new file mode 100644 index 0000000000..1c3bcfb533 --- /dev/null +++ b/test/pytest/implementation/test_vivadoaccelerator_implementation.py @@ -0,0 +1,95 @@ +from pathlib import Path + +import numpy as np +from implementation_helpers import run_implementation_collection_test +from tensorflow.keras.models import model_from_json + +import hls4ml + +test_root_path = Path(__file__).parent +example_model_path = (test_root_path / '../../../example-models').resolve() + +BACKEND = 'VivadoAccelerator' +IO_TYPE = 'io_parallel' +VIVADOACC_BOARD = 'zcu102' +VIVADOACC_PART = 'xczu9eg-ffvb1156-2-e' + + +def _load_keras_example_model(model_json, weights_h5): + model_path = example_model_path / model_json + with model_path.open('r') as f: + model = model_from_json(f.read()) + model.load_weights(example_model_path / weights_h5) + return model + + +def _random_input_for_model(model, n_samples=100): + input_shape = model.input_shape + if isinstance(input_shape, list): + input_shape = input_shape[0] + input_shape = tuple(dim if dim is not None else n_samples for dim in input_shape) + return np.random.rand(*input_shape).astype('float32') + + +def _run_example_model_implementation( + *, + model_name, + model_json, + weights_h5, + test_case_id, + synthesis_config, +): + model = _load_keras_example_model(model_json, weights_h5) + x_input = _random_input_for_model(model) + keras_prediction = model.predict(x_input) + + hls_config = hls4ml.utils.config_from_keras_model(model, granularity='name', backend=BACKEND) + output_dir = str(test_root_path / test_case_id) + hls_model = hls4ml.converters.convert_from_keras_model( + model, + hls_config=hls_config, + output_dir=output_dir, + backend=BACKEND, + io_type=IO_TYPE, + board=VIVADOACC_BOARD, + part=VIVADOACC_PART, + ) + + hls_model.compile() + hls_prediction = hls_model.predict(x_input) + np.testing.assert_allclose(hls_prediction, keras_prediction, rtol=1e-2, atol=0.01) + + run_implementation_collection_test( + config=synthesis_config, + hls_model=hls_model, + test_case_id=test_case_id, + backend=BACKEND, + metadata={ + 'artifact_id': f'{model_name}_vivadoacc_{VIVADOACC_BOARD}', + 'model_name': model_name, + 'model_json': str(Path(model_json)), + 'weights_h5': str(Path(weights_h5)), + 'board': VIVADOACC_BOARD, + 'part': VIVADOACC_PART, + }, + ) + + +def test_keras_3layer(test_case_id, synthesis_config): + _run_example_model_implementation( + model_name='keras_3layer', + model_json='keras/KERAS_3layer.json', + weights_h5='keras/KERAS_3layer_weights.h5', + test_case_id=test_case_id, + synthesis_config=synthesis_config, + ) + + +def test_keras_conv1d_small(test_case_id, synthesis_config): + _run_example_model_implementation( + model_name='keras_conv1d_small', + model_json='keras/KERAS_conv1d_small.json', + weights_h5='keras/KERAS_conv1d_small_weights.h5', + test_case_id=test_case_id, + synthesis_config=synthesis_config, + ) diff --git a/test/pytest/implementation_helpers.py b/test/pytest/implementation_helpers.py index 1a5233032a..7531c21e8f 100644 --- a/test/pytest/implementation_helpers.py +++ b/test/pytest/implementation_helpers.py @@ -114,7 +114,7 @@ def _build_dataset( build_started_at, build_finished_at, build_duration_seconds, - parsed_report_path, + hls4ml_report_path, build_output_path, ): output_dir = hls_model.config.get_output_dir() @@ -148,10 +148,10 @@ def _build_dataset( 'finished_at_utc': build_finished_at, 'duration_seconds': build_duration_seconds, }, - 'parsed_reports': report, + 'hls4ml_report': report, 'artifacts': { 'output_dir': _portable_path(output_dir), - 'parsed_report_json': _portable_path(parsed_report_path), + 'hls4ml_report_json': _portable_path(hls4ml_report_path), 'build_output_log': _portable_path(build_output_path), 'bitstreams': _collect_files(output_dir, {'.bit'}), 'reports': _collect_files(output_dir, {'.rpt', '.xml'}), @@ -166,8 +166,9 @@ def run_implementation_collection_test(config, hls_model, test_case_id, backend, Build an implementation target and emit a dataset record plus raw backend reports. """ metadata = _validate_metadata(backend, metadata) + artifact_id = metadata.get('artifact_id', test_case_id) build_args = config.get('implementation_build_args', {}).get(backend, config.get('build_args', {}).get(backend, {})) - build_output_path = _artifact_path(f'implementation_build_output_{test_case_id}.log') + build_output_path = _artifact_path(f'{artifact_id}_build.log') started_at = _utc_now() started = time.monotonic() @@ -189,7 +190,7 @@ def run_implementation_collection_test(config, hls_model, test_case_id, backend, if backend in BITFILE_REQUIRED_BACKENDS: assert bitfiles, f'Implementation failed: no bitstream was generated in {output_dir}' - parsed_report_path = _write_json(report, f'implementation_parsed_report_{test_case_id}.json') + hls4ml_report_path = _write_json(report, f'{artifact_id}_hls4ml_report.json') dataset = _build_dataset( config=config, hls_model=hls_model, @@ -201,7 +202,7 @@ def run_implementation_collection_test(config, hls_model, test_case_id, backend, build_started_at=started_at, build_finished_at=finished_at, build_duration_seconds=duration, - parsed_report_path=parsed_report_path, + hls4ml_report_path=hls4ml_report_path, build_output_path=build_output_path, ) - _write_json(dataset, f'implementation_dataset_{test_case_id}.json') + _write_json(dataset, f'{artifact_id}_dataset.json') From ad45034d30f5e74495e17c72eb84406f0baee168 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Fri, 8 May 2026 11:42:35 -0500 Subject: [PATCH 14/27] update the ci templates hirerarchy --- test/pytest/implementation/pytests.yml | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/test/pytest/implementation/pytests.yml b/test/pytest/implementation/pytests.yml index eb6b510806..036f261daa 100644 --- a/test/pytest/implementation/pytests.yml +++ b/test/pytest/implementation/pytests.yml @@ -1,23 +1,18 @@ -.implementation.test.keras_3layer_vivadoacc: - variables: - PYTESTFILE: implementation/test_vivadoaccelerator_implementation.py::test_keras_3layer - EXAMPLEMODEL: 1 - -pytest.implementation.keras_3layer_vivadoacc: +.pytest-implementation-vivadoaccelerator: extends: - .pytest-implementation-vivado-runtime - - .implementation.test.keras_3layer_vivadoacc variables: VIVADO_VERSION: "2020.1" + EXAMPLEMODEL: 1 -.implementation.test.keras_conv1d_small_vivadoacc: +pytest.implementation.keras_3layer_vivadoacc: + extends: + - .pytest-implementation-vivadoaccelerator variables: - PYTESTFILE: implementation/test_vivadoaccelerator_implementation.py::test_keras_conv1d_small - EXAMPLEMODEL: 1 + PYTESTFILE: implementation/test_vivadoaccelerator_implementation.py::test_keras_3layer pytest.implementation.keras_conv1d_small_vivadoacc: extends: - - .pytest-implementation-vivado-runtime - - .implementation.test.keras_conv1d_small_vivadoacc + - .pytest-implementation-vivadoaccelerator variables: - VIVADO_VERSION: "2020.1" + PYTESTFILE: implementation/test_vivadoaccelerator_implementation.py::test_keras_conv1d_small From 1c56ae3a61bf080d6d12c6e107e70bf62e1c7583 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Fri, 8 May 2026 11:54:26 -0500 Subject: [PATCH 15/27] update commits related metadata --- .../test_vivadoaccelerator_implementation.py | 18 +++++++++++--- test/pytest/implementation_helpers.py | 24 ++++++++++++++----- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/test/pytest/implementation/test_vivadoaccelerator_implementation.py b/test/pytest/implementation/test_vivadoaccelerator_implementation.py index 1c3bcfb533..33a95bf99c 100644 --- a/test/pytest/implementation/test_vivadoaccelerator_implementation.py +++ b/test/pytest/implementation/test_vivadoaccelerator_implementation.py @@ -1,3 +1,4 @@ +import subprocess from pathlib import Path import numpy as np @@ -23,6 +24,13 @@ def _load_keras_example_model(model_json, weights_h5): return model +def _example_models_commit(): + try: + return subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=example_model_path, text=True).strip() + except (FileNotFoundError, subprocess.CalledProcessError): + return None + + def _random_input_for_model(model, n_samples=100): input_shape = model.input_shape if isinstance(input_shape, list): @@ -66,9 +74,13 @@ def _run_example_model_implementation( backend=BACKEND, metadata={ 'artifact_id': f'{model_name}_vivadoacc_{VIVADOACC_BOARD}', - 'model_name': model_name, - 'model_json': str(Path(model_json)), - 'weights_h5': str(Path(weights_h5)), + 'model': { + 'name': model_name, + 'source': 'example-models', + 'source_commit': _example_models_commit(), + 'model_json': str(Path(model_json)), + 'weights_h5': str(Path(weights_h5)), + }, 'board': VIVADOACC_BOARD, 'part': VIVADOACC_PART, }, diff --git a/test/pytest/implementation_helpers.py b/test/pytest/implementation_helpers.py index 7531c21e8f..f6bf155c51 100644 --- a/test/pytest/implementation_helpers.py +++ b/test/pytest/implementation_helpers.py @@ -1,5 +1,6 @@ import json import os +import subprocess import sys import time from contextlib import contextmanager @@ -28,7 +29,14 @@ def _utc_now(): def _project_root(): - return Path(os.getenv('CI_PROJECT_DIR', Path(__file__).parents[2])) + return Path(__file__).parents[2] + + +def _git_commit(path): + try: + return subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=path, text=True).strip() + except (FileNotFoundError, subprocess.CalledProcessError): + return None def _portable_path(path): @@ -118,27 +126,31 @@ def _build_dataset( build_output_path, ): output_dir = hls_model.config.get_output_dir() + model = metadata.get('model', {}) + run_metadata = {key: value for key, value in metadata.items() if key not in {'artifact_id', 'model'}} return { 'schema_version': DATASET_SCHEMA_VERSION, 'test_id': test_case_id, + 'source': { + 'hls4ml_commit': _git_commit(_project_root()), + 'repository_url': 'https://github.com/fastmachinelearning/hls4ml', + }, + 'model': model, 'hls_config': { 'backend': backend, 'project_name': hls_model.config.get_project_name(), 'build_args': build_args, }, - 'metadata': metadata, + 'metadata': run_metadata, 'toolchain': { 'version': config.get('tools_version', {}).get(backend, 'unknown'), }, 'ci': { - 'commit_sha': os.getenv('CI_COMMIT_SHA'), - 'commit_ref': os.getenv('CI_COMMIT_REF_NAME'), - 'commit_tag': os.getenv('CI_COMMIT_TAG'), 'pipeline_id': os.getenv('CI_PIPELINE_ID'), 'job_id': os.getenv('CI_JOB_ID'), 'project_url': os.getenv('CI_PROJECT_URL'), - 'job_image': os.getenv('CI_JOB_IMAGE'), + 'job_url': os.getenv('CI_JOB_URL'), 'runner_description': os.getenv('CI_RUNNER_DESCRIPTION'), 'runner_tags': os.getenv('CI_RUNNER_TAGS'), }, From c979c5a73bf0e1b513a168875ce78bce26abe21e Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Fri, 8 May 2026 12:01:20 -0500 Subject: [PATCH 16/27] add zip file artifact of project folder --- test/pytest/implementation/ci-template.yml | 6 +--- test/pytest/implementation_helpers.py | 32 ++++++++++++++++++++-- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/test/pytest/implementation/ci-template.yml b/test/pytest/implementation/ci-template.yml index 8b91c352c2..b9128e852f 100644 --- a/test/pytest/implementation/ci-template.yml +++ b/test/pytest/implementation/ci-template.yml @@ -24,11 +24,7 @@ - test/pytest/implementation/*_dataset.json - test/pytest/implementation/*_hls4ml_report.json - test/pytest/implementation/*_build.log - - test/pytest/implementation/*/ - - test/pytest/implementation/**/*.bit - - test/pytest/implementation/**/*.hwh - - test/pytest/implementation/**/*.rpt - - test/pytest/implementation/**/*.log + - test/pytest/implementation/*_project.zip .pytest-implementation-vivado-runtime: extends: .pytest-implementation diff --git a/test/pytest/implementation_helpers.py b/test/pytest/implementation_helpers.py index f6bf155c51..d6d741a227 100644 --- a/test/pytest/implementation_helpers.py +++ b/test/pytest/implementation_helpers.py @@ -5,7 +5,9 @@ import time from contextlib import contextmanager from datetime import datetime, timezone +from hashlib import sha256 from pathlib import Path +from zipfile import ZIP_DEFLATED, ZipFile import pytest @@ -60,6 +62,25 @@ def _collect_files(output_dir, suffixes=None): return files +def _file_sha256(path): + digest = sha256() + with open(path, 'rb') as fp: + for chunk in iter(lambda: fp.read(1024 * 1024), b''): + digest.update(chunk) + return digest.hexdigest() + + +def _zip_directory(directory, zip_path): + directory = Path(directory) + zip_path = Path(zip_path) + zip_path.parent.mkdir(parents=True, exist_ok=True) + with ZipFile(zip_path, 'w', ZIP_DEFLATED) as archive: + for path in sorted(directory.rglob('*')): + if path.is_file(): + archive.write(path, path.relative_to(directory.parent)) + return zip_path + + def _dataset_dir(): return Path(os.getenv(DATASET_DIR_ENV, str(DEFAULT_DATASET_DIR))) @@ -124,6 +145,7 @@ def _build_dataset( build_duration_seconds, hls4ml_report_path, build_output_path, + project_archive_path, ): output_dir = hls_model.config.get_output_dir() model = metadata.get('model', {}) @@ -166,9 +188,11 @@ def _build_dataset( 'hls4ml_report_json': _portable_path(hls4ml_report_path), 'build_output_log': _portable_path(build_output_path), 'bitstreams': _collect_files(output_dir, {'.bit'}), - 'reports': _collect_files(output_dir, {'.rpt', '.xml'}), - 'logs': _collect_files(output_dir, {'.log'}), - 'raw_output_files': _collect_files(output_dir), + 'project_archive': { + 'path': _portable_path(project_archive_path), + 'size_bytes': Path(project_archive_path).stat().st_size, + 'sha256': _file_sha256(project_archive_path), + }, }, } @@ -203,6 +227,7 @@ def run_implementation_collection_test(config, hls_model, test_case_id, backend, assert bitfiles, f'Implementation failed: no bitstream was generated in {output_dir}' hls4ml_report_path = _write_json(report, f'{artifact_id}_hls4ml_report.json') + project_archive_path = _zip_directory(output_dir, _artifact_path(f'{artifact_id}_project.zip')) dataset = _build_dataset( config=config, hls_model=hls_model, @@ -216,5 +241,6 @@ def run_implementation_collection_test(config, hls_model, test_case_id, backend, build_duration_seconds=duration, hls4ml_report_path=hls4ml_report_path, build_output_path=build_output_path, + project_archive_path=project_archive_path, ) _write_json(dataset, f'{artifact_id}_dataset.json') From 1b882919a4eadcb5c6e2be14e7dbfc553af447ce Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Fri, 8 May 2026 13:40:33 -0500 Subject: [PATCH 17/27] remove accuracy tests --- .../test_vivadoaccelerator_implementation.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/test/pytest/implementation/test_vivadoaccelerator_implementation.py b/test/pytest/implementation/test_vivadoaccelerator_implementation.py index 33a95bf99c..b707cbac51 100644 --- a/test/pytest/implementation/test_vivadoaccelerator_implementation.py +++ b/test/pytest/implementation/test_vivadoaccelerator_implementation.py @@ -1,7 +1,6 @@ import subprocess from pathlib import Path -import numpy as np from implementation_helpers import run_implementation_collection_test from tensorflow.keras.models import model_from_json @@ -31,14 +30,6 @@ def _example_models_commit(): return None -def _random_input_for_model(model, n_samples=100): - input_shape = model.input_shape - if isinstance(input_shape, list): - input_shape = input_shape[0] - input_shape = tuple(dim if dim is not None else n_samples for dim in input_shape) - return np.random.rand(*input_shape).astype('float32') - - def _run_example_model_implementation( *, model_name, @@ -48,8 +39,6 @@ def _run_example_model_implementation( synthesis_config, ): model = _load_keras_example_model(model_json, weights_h5) - x_input = _random_input_for_model(model) - keras_prediction = model.predict(x_input) hls_config = hls4ml.utils.config_from_keras_model(model, granularity='name', backend=BACKEND) output_dir = str(test_root_path / test_case_id) @@ -64,8 +53,6 @@ def _run_example_model_implementation( ) hls_model.compile() - hls_prediction = hls_model.predict(x_input) - np.testing.assert_allclose(hls_prediction, keras_prediction, rtol=1e-2, atol=0.01) run_implementation_collection_test( config=synthesis_config, From c2b42dfed577e3bd1054203887cca17fb22f46ea Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Mon, 11 May 2026 10:58:19 -0500 Subject: [PATCH 18/27] add onnx models support --- test/pytest/implementation/pytests.yml | 6 ++ .../test_vivadoaccelerator_implementation.py | 61 +++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/test/pytest/implementation/pytests.yml b/test/pytest/implementation/pytests.yml index 036f261daa..e53931c98a 100644 --- a/test/pytest/implementation/pytests.yml +++ b/test/pytest/implementation/pytests.yml @@ -16,3 +16,9 @@ pytest.implementation.keras_conv1d_small_vivadoacc: - .pytest-implementation-vivadoaccelerator variables: PYTESTFILE: implementation/test_vivadoaccelerator_implementation.py::test_keras_conv1d_small + +pytest.implementation.tiny_unet_ch_last_vivadoacc: + extends: + - .pytest-implementation-vivadoaccelerator + variables: + PYTESTFILE: implementation/test_vivadoaccelerator_implementation.py::test_tiny_unet_ch_last diff --git a/test/pytest/implementation/test_vivadoaccelerator_implementation.py b/test/pytest/implementation/test_vivadoaccelerator_implementation.py index b707cbac51..1776bb1d3e 100644 --- a/test/pytest/implementation/test_vivadoaccelerator_implementation.py +++ b/test/pytest/implementation/test_vivadoaccelerator_implementation.py @@ -2,6 +2,7 @@ from pathlib import Path from implementation_helpers import run_implementation_collection_test +from qonnx.core.modelwrapper import ModelWrapper from tensorflow.keras.models import model_from_json import hls4ml @@ -23,6 +24,12 @@ def _load_keras_example_model(model_json, weights_h5): return model +def _load_onnx_example_model(model_onnx): + model_path = example_model_path / model_onnx + assert model_path.is_file() + return ModelWrapper(str(model_path)) + + def _example_models_commit(): try: return subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=example_model_path, text=True).strip() @@ -74,6 +81,51 @@ def _run_example_model_implementation( ) +def _run_onnx_example_model_implementation( + *, + model_name, + model_onnx, + test_case_id, + synthesis_config, +): + model = _load_onnx_example_model(model_onnx) + + hls_config = hls4ml.utils.config.config_from_onnx_model( + model, + granularity='name', + backend=BACKEND, + default_precision='fixed<32,16>', + ) + output_dir = str(test_root_path / test_case_id) + hls_model = hls4ml.converters.convert_from_onnx_model( + model, + hls_config=hls_config, + output_dir=output_dir, + backend=BACKEND, + io_type=IO_TYPE, + board=VIVADOACC_BOARD, + part=VIVADOACC_PART, + ) + + run_implementation_collection_test( + config=synthesis_config, + hls_model=hls_model, + test_case_id=test_case_id, + backend=BACKEND, + metadata={ + 'artifact_id': f'{model_name}_vivadoacc_{VIVADOACC_BOARD}', + 'model': { + 'name': model_name, + 'source': 'example-models', + 'source_commit': _example_models_commit(), + 'model_onnx': str(Path(model_onnx)), + }, + 'board': VIVADOACC_BOARD, + 'part': VIVADOACC_PART, + }, + ) + + def test_keras_3layer(test_case_id, synthesis_config): _run_example_model_implementation( model_name='keras_3layer', @@ -84,6 +136,15 @@ def test_keras_3layer(test_case_id, synthesis_config): ) +def test_tiny_unet_ch_last(test_case_id, synthesis_config): + _run_onnx_example_model_implementation( + model_name='tiny_unet_ch_last', + model_onnx='onnx/tiny_unet_ch_last.onnx', + test_case_id=test_case_id, + synthesis_config=synthesis_config, + ) + + def test_keras_conv1d_small(test_case_id, synthesis_config): _run_example_model_implementation( model_name='keras_conv1d_small', From 1fcbdd1c3d495627d4bae6d725582d0e6f8a36b8 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Mon, 11 May 2026 11:17:05 -0500 Subject: [PATCH 19/27] fix tool versions in report --- test/pytest/implementation_helpers.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/test/pytest/implementation_helpers.py b/test/pytest/implementation_helpers.py index d6d741a227..8b3bc6a413 100644 --- a/test/pytest/implementation_helpers.py +++ b/test/pytest/implementation_helpers.py @@ -25,6 +25,14 @@ BITFILE_REQUIRED_BACKENDS = {'VivadoAccelerator'} +TOOLCHAINS_BY_BACKEND = { + 'VivadoAccelerator': {'vivado': ('VivadoAccelerator', 'Vivado')}, + 'VitisUnified': { + 'vivado': ('Vivado',), + 'vitis': ('VitisUnified', 'Vitis'), + }, +} + def _utc_now(): return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace('+00:00', 'Z') @@ -132,6 +140,18 @@ def _validate_metadata(backend, metadata): return metadata +def _toolchain_versions(config, backend): + versions = config.get('tools_version', {}) + toolchains = TOOLCHAINS_BY_BACKEND.get(backend) + if toolchains is None: + return {backend.lower(): versions.get(backend, 'unknown')} + + resolved = {} + for tool_name, version_keys in toolchains.items(): + resolved[tool_name] = next((versions[key] for key in version_keys if versions.get(key)), 'unknown') + return resolved + + def _build_dataset( config, hls_model, @@ -153,6 +173,7 @@ def _build_dataset( return { 'schema_version': DATASET_SCHEMA_VERSION, + 'created_at_utc': _utc_now(), 'test_id': test_case_id, 'source': { 'hls4ml_commit': _git_commit(_project_root()), @@ -165,9 +186,7 @@ def _build_dataset( 'build_args': build_args, }, 'metadata': run_metadata, - 'toolchain': { - 'version': config.get('tools_version', {}).get(backend, 'unknown'), - }, + 'toolchain': _toolchain_versions(config, backend), 'ci': { 'pipeline_id': os.getenv('CI_PIPELINE_ID'), 'job_id': os.getenv('CI_JOB_ID'), From 14b8ea5c78395d65b8df02b992425657ba96594d Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Mon, 11 May 2026 11:40:05 -0500 Subject: [PATCH 20/27] remove onnx tests --- test/pytest/implementation/pytests.yml | 6 -- .../test_vivadoaccelerator_implementation.py | 61 ------------------- 2 files changed, 67 deletions(-) diff --git a/test/pytest/implementation/pytests.yml b/test/pytest/implementation/pytests.yml index e53931c98a..036f261daa 100644 --- a/test/pytest/implementation/pytests.yml +++ b/test/pytest/implementation/pytests.yml @@ -16,9 +16,3 @@ pytest.implementation.keras_conv1d_small_vivadoacc: - .pytest-implementation-vivadoaccelerator variables: PYTESTFILE: implementation/test_vivadoaccelerator_implementation.py::test_keras_conv1d_small - -pytest.implementation.tiny_unet_ch_last_vivadoacc: - extends: - - .pytest-implementation-vivadoaccelerator - variables: - PYTESTFILE: implementation/test_vivadoaccelerator_implementation.py::test_tiny_unet_ch_last diff --git a/test/pytest/implementation/test_vivadoaccelerator_implementation.py b/test/pytest/implementation/test_vivadoaccelerator_implementation.py index 1776bb1d3e..b707cbac51 100644 --- a/test/pytest/implementation/test_vivadoaccelerator_implementation.py +++ b/test/pytest/implementation/test_vivadoaccelerator_implementation.py @@ -2,7 +2,6 @@ from pathlib import Path from implementation_helpers import run_implementation_collection_test -from qonnx.core.modelwrapper import ModelWrapper from tensorflow.keras.models import model_from_json import hls4ml @@ -24,12 +23,6 @@ def _load_keras_example_model(model_json, weights_h5): return model -def _load_onnx_example_model(model_onnx): - model_path = example_model_path / model_onnx - assert model_path.is_file() - return ModelWrapper(str(model_path)) - - def _example_models_commit(): try: return subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=example_model_path, text=True).strip() @@ -81,51 +74,6 @@ def _run_example_model_implementation( ) -def _run_onnx_example_model_implementation( - *, - model_name, - model_onnx, - test_case_id, - synthesis_config, -): - model = _load_onnx_example_model(model_onnx) - - hls_config = hls4ml.utils.config.config_from_onnx_model( - model, - granularity='name', - backend=BACKEND, - default_precision='fixed<32,16>', - ) - output_dir = str(test_root_path / test_case_id) - hls_model = hls4ml.converters.convert_from_onnx_model( - model, - hls_config=hls_config, - output_dir=output_dir, - backend=BACKEND, - io_type=IO_TYPE, - board=VIVADOACC_BOARD, - part=VIVADOACC_PART, - ) - - run_implementation_collection_test( - config=synthesis_config, - hls_model=hls_model, - test_case_id=test_case_id, - backend=BACKEND, - metadata={ - 'artifact_id': f'{model_name}_vivadoacc_{VIVADOACC_BOARD}', - 'model': { - 'name': model_name, - 'source': 'example-models', - 'source_commit': _example_models_commit(), - 'model_onnx': str(Path(model_onnx)), - }, - 'board': VIVADOACC_BOARD, - 'part': VIVADOACC_PART, - }, - ) - - def test_keras_3layer(test_case_id, synthesis_config): _run_example_model_implementation( model_name='keras_3layer', @@ -136,15 +84,6 @@ def test_keras_3layer(test_case_id, synthesis_config): ) -def test_tiny_unet_ch_last(test_case_id, synthesis_config): - _run_onnx_example_model_implementation( - model_name='tiny_unet_ch_last', - model_onnx='onnx/tiny_unet_ch_last.onnx', - test_case_id=test_case_id, - synthesis_config=synthesis_config, - ) - - def test_keras_conv1d_small(test_case_id, synthesis_config): _run_example_model_implementation( model_name='keras_conv1d_small', From a68edf2a2a565a32cb352a7508fd6a5eb7183629 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Mon, 11 May 2026 11:41:07 -0500 Subject: [PATCH 21/27] update vivadoacc keras test cases --- .../test_vivadoaccelerator_implementation.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/pytest/implementation/test_vivadoaccelerator_implementation.py b/test/pytest/implementation/test_vivadoaccelerator_implementation.py index b707cbac51..894ddf3b3d 100644 --- a/test/pytest/implementation/test_vivadoaccelerator_implementation.py +++ b/test/pytest/implementation/test_vivadoaccelerator_implementation.py @@ -74,11 +74,11 @@ def _run_example_model_implementation( ) -def test_keras_3layer(test_case_id, synthesis_config): +def test_keras_1layer(test_case_id, synthesis_config): _run_example_model_implementation( - model_name='keras_3layer', - model_json='keras/KERAS_3layer.json', - weights_h5='keras/KERAS_3layer_weights.h5', + model_name='keras_1layer', + model_json='keras/KERAS_1layer.json', + weights_h5='keras/KERAS_1layer_weights.h5', test_case_id=test_case_id, synthesis_config=synthesis_config, ) From 880212c91a2d3ca198593c971b6b407525aaa92a Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Mon, 11 May 2026 11:41:49 -0500 Subject: [PATCH 22/27] update implementation pytest.yml --- test/pytest/implementation/pytests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/pytest/implementation/pytests.yml b/test/pytest/implementation/pytests.yml index 036f261daa..004fabd091 100644 --- a/test/pytest/implementation/pytests.yml +++ b/test/pytest/implementation/pytests.yml @@ -5,11 +5,11 @@ VIVADO_VERSION: "2020.1" EXAMPLEMODEL: 1 -pytest.implementation.keras_3layer_vivadoacc: +pytest.implementation.keras_1layer_vivadoacc: extends: - .pytest-implementation-vivadoaccelerator variables: - PYTESTFILE: implementation/test_vivadoaccelerator_implementation.py::test_keras_3layer + PYTESTFILE: implementation/test_vivadoaccelerator_implementation.py::test_keras_1layer pytest.implementation.keras_conv1d_small_vivadoacc: extends: From 10729f0a2c2a70c298ed86fa5b56df3090418c5a Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Tue, 12 May 2026 09:30:08 -0500 Subject: [PATCH 23/27] move implementation helpers into implementation tests --- test/pytest/{ => implementation}/implementation_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename test/pytest/{ => implementation}/implementation_helpers.py (98%) diff --git a/test/pytest/implementation_helpers.py b/test/pytest/implementation/implementation_helpers.py similarity index 98% rename from test/pytest/implementation_helpers.py rename to test/pytest/implementation/implementation_helpers.py index 8b3bc6a413..112a1a6b24 100644 --- a/test/pytest/implementation_helpers.py +++ b/test/pytest/implementation/implementation_helpers.py @@ -13,7 +13,7 @@ DATASET_SCHEMA_VERSION = 'implementation-dataset/v1' DATASET_DIR_ENV = 'IMPLEMENTATION_DATASET_DIR' -DEFAULT_DATASET_DIR = Path(__file__).parent / 'implementation' +DEFAULT_DATASET_DIR = Path(__file__).parent EXPECTED_REPORT_KEYS = { 'VivadoAccelerator': {'CSynthesisReport'}, @@ -39,7 +39,7 @@ def _utc_now(): def _project_root(): - return Path(__file__).parents[2] + return Path(__file__).parents[3] def _git_commit(path): From 4585990cdb5f1afc505d1ccdf662d811fa5f983e Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Tue, 12 May 2026 09:36:25 -0500 Subject: [PATCH 24/27] update implementation README --- test/pytest/implementation/README.md | 77 +++++++++++++++++++--------- 1 file changed, 52 insertions(+), 25 deletions(-) diff --git a/test/pytest/implementation/README.md b/test/pytest/implementation/README.md index fb3e27ef14..5e2a0dc724 100644 --- a/test/pytest/implementation/README.md +++ b/test/pytest/implementation/README.md @@ -1,40 +1,67 @@ # Implementation CI Suite -This directory contains implementation-oriented pytest cases (post-HLS reports and full bitfile flow checks). +This directory contains manually listed implementation tests. These jobs run full backend implementation flows and collect dataset artifacts with actual tool reports and generated project files. -## Pipeline separation +The normal pytest CI should stay separate from this suite. Implementation tests are intended for manually triggered dataset collection, not for the regular fast test matrix. -The project keeps one GitLab entrypoint (`.gitlab-ci.yml`) but has two pipeline modes: +## Pipeline Layout -- default mode (`CI_PIPELINE_MODE` unset): runs normal pytest CI only -- implementation mode (`CI_PIPELINE_MODE=implementation`): runs only the implementation child pipeline +Implementation CI uses: -Implementation mode is dispatched to: +- `test/pytest/implementation/pytests.yml`: static job list +- `test/pytest/implementation/ci-template.yml`: implementation-specific GitLab templates +- `test/pytest/implementation/implementation_helpers.py`: dataset artifact helpers used by implementation tests -- `test/pytest/implementation/pytests.yml` (static job list) +Jobs in `pytests.yml` extend backend-specific runtime templates from `ci-template.yml`, for example: -Both normal and implementation templates reuse the shared base setup: +- `.pytest-implementation-vivadoaccelerator` +- `.pytest-implementation-vivado-runtime` -- `test/pytest/ci-base-template.yml` +Each concrete job sets `PYTESTFILE` to one specific test function, so each model/backend run becomes a separate CI job and artifact set. -and keep their behavior-specific logic in: +## Dataset Artifacts -- `test/pytest/ci-template.yml` (normal matrix) -- `test/pytest/implementation/ci-template.yml` (implementation suite) +Implementation tests should use `run_implementation_collection_test()` from `implementation_helpers.py`. The helper: -Tool exposure policy: +- runs `hls_model.build(...)` with backend-specific implementation build args +- captures full terminal output to `*_build.log` +- writes the hls4ml report returned by the backend to `*_hls4ml_report.json` +- writes one compact dataset record to `*_dataset.json` +- compresses the generated project directory to `*_project.zip` +- records the project archive path, size, and SHA256 in the dataset record -- normal template uses `vivado_hls` -- implementation template exposes `vivado_hls`, `vivado`, `vitis-run`, `v++`, and `vitis` -- implementation jobs select tool-exposure mode by extending a script profile in - `implementation/pytests.yml` (`.implementation.script.tools-*`) -- implementation dataset output path is controlled by `IMPLEMENTATION_DATASET_DIR` - (default in CI: `test/pytest/implementation`) +The dataset record includes: -## Adding new implementation tests +- hls4ml source commit and repository URL +- example-models source commit and model file names +- backend, project name, board/part metadata, and build args +- backend-specific toolchain versions +- CI metadata for finding the run later +- build timing and the hls4ml report +- pointers to the log, hls4ml report JSON, bitstream files, and project archive -1. Add a new file in this folder matching `test_*.py`. -2. Add a matching static CI job entry in `test/pytest/implementation/pytests.yml`. -3. Set per-job tool versions (`VIVADO_VERSION`, `VITIS_VERSION`) in `pytests.yml`. -4. Select a per-job tool wrapper script profile in `pytests.yml`. -5. Keep tests collect-only: validate full flow success and emit dataset artifacts. +`IMPLEMENTATION_DATASET_DIR` controls where artifacts are written. In CI it is set to: + +```bash +test/pytest/implementation +``` + +## Test Policy + +Implementation tests should load models from the `example-models` submodule instead of defining models inline. This keeps dataset records tied to a model name, model file, and exact `example-models` commit. + +Keep test code focused on: + +- loading the example model +- converting it with the backend under test +- passing clear metadata to `run_implementation_collection_test()` + +Avoid duplicating dataset/report parsing in individual tests. Add backend-specific dataset behavior in `implementation_helpers.py` when needed. + +## Adding A Test + +1. Add or update a `test_*.py` file in this directory. +2. Load a model from `example-models`. +3. Add model metadata including name, source, source commit, and model file paths. +4. Add a static job in `pytests.yml` with `PYTESTFILE` pointing to the specific test function. +5. Set tool versions and backend runtime template through the job/template variables. From 305906cc3250d52a2a0c3b28abedd5ab7d0846c3 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Thu, 14 May 2026 10:09:32 -0500 Subject: [PATCH 25/27] update vivadoaccelerator report expected keys --- test/pytest/implementation/implementation_helpers.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/pytest/implementation/implementation_helpers.py b/test/pytest/implementation/implementation_helpers.py index 112a1a6b24..eeb34ca5d7 100644 --- a/test/pytest/implementation/implementation_helpers.py +++ b/test/pytest/implementation/implementation_helpers.py @@ -16,7 +16,14 @@ DEFAULT_DATASET_DIR = Path(__file__).parent EXPECTED_REPORT_KEYS = { - 'VivadoAccelerator': {'CSynthesisReport'}, + 'VivadoAccelerator': { + 'CSynthesisReport', + 'CosimReport', + 'CosimResults', + 'ImplementationReport', + 'TimingReport', + 'VivadoSynthReport', + }, } REQUIRED_METADATA_FIELDS = { From 1449de0c5377a5f57b45bd7c62f7ca4a8af57524 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Thu, 14 May 2026 10:20:30 -0500 Subject: [PATCH 26/27] remove complile step from vivado impl test --- .../implementation/test_vivadoaccelerator_implementation.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/pytest/implementation/test_vivadoaccelerator_implementation.py b/test/pytest/implementation/test_vivadoaccelerator_implementation.py index 894ddf3b3d..e9b589d0d4 100644 --- a/test/pytest/implementation/test_vivadoaccelerator_implementation.py +++ b/test/pytest/implementation/test_vivadoaccelerator_implementation.py @@ -52,8 +52,6 @@ def _run_example_model_implementation( part=VIVADOACC_PART, ) - hls_model.compile() - run_implementation_collection_test( config=synthesis_config, hls_model=hls_model, From f8e23ac21c01a96ada0858b28e1c20f411d4de14 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Wed, 27 May 2026 09:54:16 -0500 Subject: [PATCH 27/27] add stream implementation logs while preserving artifatc --- test/pytest/implementation/ci-template.yml | 4 ++-- .../implementation/implementation_helpers.py | 19 ++++++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/test/pytest/implementation/ci-template.yml b/test/pytest/implementation/ci-template.yml index b9128e852f..4f481ccaed 100644 --- a/test/pytest/implementation/ci-template.yml +++ b/test/pytest/implementation/ci-template.yml @@ -39,7 +39,7 @@ - export PATH=$PWD/cmd_vivado_${VIVADO_VERSION}:$PATH - export IMPLEMENTATION_DATASET_DIR=$PWD/test/pytest/implementation - cd test/pytest - - pytest $PYTESTFILE -rA --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed + - pytest $PYTESTFILE -s -rA --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed .pytest-implementation-vitis-runtime: extends: .pytest-implementation @@ -61,4 +61,4 @@ - export PATH=$PWD/cmd_vivado_${VIVADO_VERSION}:$PWD/cmd_vitis_${VITIS_VERSION}:$PATH - export IMPLEMENTATION_DATASET_DIR=$PWD/test/pytest/implementation - cd test/pytest - - pytest $PYTESTFILE -rA --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed + - pytest $PYTESTFILE -s -rA --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed diff --git a/test/pytest/implementation/implementation_helpers.py b/test/pytest/implementation/implementation_helpers.py index eeb34ca5d7..c4090be81c 100644 --- a/test/pytest/implementation/implementation_helpers.py +++ b/test/pytest/implementation/implementation_helpers.py @@ -121,16 +121,21 @@ def _capture_terminal_output(path): sys.stderr.flush() saved_stdout = os.dup(1) saved_stderr = os.dup(2) - with open(path, 'w') as fp: - os.dup2(fp.fileno(), 1) - os.dup2(fp.fileno(), 2) + tee = subprocess.Popen(['tee', str(path)], stdin=subprocess.PIPE) + os.dup2(tee.stdin.fileno(), 1) + os.dup2(tee.stdin.fileno(), 2) + + try: + yield path + finally: + sys.stdout.flush() + sys.stderr.flush() try: - yield path - finally: - sys.stdout.flush() - sys.stderr.flush() os.dup2(saved_stdout, 1) os.dup2(saved_stderr, 2) + finally: + tee.stdin.close() + tee.wait() os.close(saved_stdout) os.close(saved_stderr)