From c3409c558e062e0ab711a687709b41735f805204 Mon Sep 17 00:00:00 2001 From: hannahwestra25 Date: Thu, 26 Mar 2026 12:15:00 -0400 Subject: [PATCH 1/2] add partner-integration-tests yml for pipeline --- partner-integration-tests.yml | 123 ++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 partner-integration-tests.yml diff --git a/partner-integration-tests.yml b/partner-integration-tests.yml new file mode 100644 index 000000000..569846ed9 --- /dev/null +++ b/partner-integration-tests.yml @@ -0,0 +1,123 @@ + +trigger: none # Disable automatic CI triggers + +schedules: +- cron: "0 6 * * *" # 6 AM UTC = 10 PM PST (UTC-8) / 11PM PDT (UTC-6) + displayName: Nightly Partner Integration Tests at 10 PM PST + branches: + include: + - main + always: true # Run even if there are no code changes + +jobs: +- job: PartnerIntegrationTests + displayName: "Builds the pyrit environment and runs partner integration tests" + timeoutInMinutes: 360 # Allows the job to run up to 6 hours + pool: + vmImage: ubuntu-latest + steps: + - checkout: self + fetchDepth: 1 + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.12' + addToPath: true + - bash: | + mkdir -p ~/.pyrit + displayName: "Create PyRIT configuration directory" + name: create_pyrit_dir + - task: AzureKeyVault@2 + displayName: Azure Key Vault - retrieve .env file secret + inputs: + azureSubscription: 'integration-test-service-connection' + KeyVaultName: 'pyrit-environment' + SecretsFilter: 'env-global' + RunAsPreJob: false + - bash: | + python -c " + import os; + secret = os.environ.get('PYRIT_TEST_SECRET'); + if not secret: + raise ValueError('PYRIT_TEST_SECRET is not set'); + with open(os.path.expanduser('~/.pyrit/.env'), 'w') as file: + file.write(secret)" + env: + PYRIT_TEST_SECRET: $(env-global) + name: create_env_file + - bash: | + cp build_scripts/env_local_integration_test ~/.pyrit/.env.local + displayName: "Create .env.local from example" + - script: + wget -qO- https://astral.sh/uv/install.sh | sh + name: install_uv + - bash: sudo apt-get install python3-tk + name: install_tkinter + - bash: | + set -e + # Detect Ubuntu version + UBUNTU_VERSION=$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2) + SUPPORTED_VERSIONS="18.04 20.04 22.04 24.04 24.10" + + if ! [[ "$SUPPORTED_VERSIONS" == *"$UBUNTU_VERSION"* ]]; then + echo "Ubuntu $UBUNTU_VERSION is not currently supported." + exit 1 + fi + + # Download the package to configure the Microsoft repo + curl -sSL -O https://packages.microsoft.com/config/ubuntu/$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2)/packages-microsoft-prod.deb + # Install the package + sudo dpkg -i packages-microsoft-prod.deb + # Delete the file + rm packages-microsoft-prod.deb + + # Install the driver + sudo apt-get update + sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 + + echo "Microsoft ODBC Driver 18 installed successfully." + displayName: 'Install ODBC Driver 18 for SQL Server' + - bash: uv sync --extra dev --extra all + name: install_PyRIT + - bash: df -all -h + name: disk_space_check +# This step ensures that integration tests are run outside of the PyRIT repository to test that .env files are accessed correctly. + - bash: | + PyRIT_DIR=$(pwd) + NEW_DIR="partner_integration_test_directory" + cd .. + mkdir -p $NEW_DIR/tests + cp -r $PyRIT_DIR/doc $NEW_DIR + cp -r $PyRIT_DIR/assets $NEW_DIR + cp -r $PyRIT_DIR/tests/partner_integration $NEW_DIR/tests + cd $NEW_DIR + displayName: "Create and switch to new partner integration test directory" + - task: AzureCLI@2 + displayName: "Authenticate with service principal, cache Cognitive Services access token, and run tests" + inputs: + azureSubscription: 'partner-integration-test-service-connection' + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + # Prefetch token for Cognitive Services before ID token expires (60-90 minute validity) + az account get-access-token --scope https://cognitiveservices.azure.com/.default --output none + echo "Cognitive Services access token cached successfully." + + # Prefetch token for Azure ML / Foundry model endpoints + az account get-access-token --scope https://ml.azure.com/.default --output none + echo "Azure ML/Foundry access token cached successfully." + + # Prefetch token for Azure SQL Database + az account get-access-token --scope https://database.windows.net/.default --output none + echo "Azure SQL Database access token cached successfully." + + # Run partner integration tests + make partner-integration-test + - bash: | + rm -f ~/.pyrit/.env ~/.pyrit/.env.local + name: clean_up_env_files + condition: always() + - task: PublishTestResults@2 + condition: always() + inputs: + testResultsFormat: 'JUnit' + testResultsFiles: 'junit/test-results.xml' From 7317c7c9f05b868cf64a4681df7e29ded27d7745 Mon Sep 17 00:00:00 2001 From: hannahwestra25 Date: Tue, 31 Mar 2026 18:08:45 -0400 Subject: [PATCH 2/2] move ado pipeline ymls into folder and add shared template --- .../component-governance.yml | 0 ado_pipeline_config/end-to-end-tests.yml | 21 +++ ado_pipeline_config/integration-tests.yml | 19 +++ .../partner-integration-tests.yml | 20 +++ .../test-job-template.yml | 42 +++--- end-to-end-tests.yml | 124 ------------------ partner-integration-tests.yml | 123 ----------------- 7 files changed, 83 insertions(+), 266 deletions(-) rename component-governance.yml => ado_pipeline_config/component-governance.yml (100%) create mode 100644 ado_pipeline_config/end-to-end-tests.yml create mode 100644 ado_pipeline_config/integration-tests.yml create mode 100644 ado_pipeline_config/partner-integration-tests.yml rename integration-tests.yml => ado_pipeline_config/test-job-template.yml (82%) delete mode 100644 end-to-end-tests.yml delete mode 100644 partner-integration-tests.yml diff --git a/component-governance.yml b/ado_pipeline_config/component-governance.yml similarity index 100% rename from component-governance.yml rename to ado_pipeline_config/component-governance.yml diff --git a/ado_pipeline_config/end-to-end-tests.yml b/ado_pipeline_config/end-to-end-tests.yml new file mode 100644 index 000000000..a26a0fd06 --- /dev/null +++ b/ado_pipeline_config/end-to-end-tests.yml @@ -0,0 +1,21 @@ +# Runs end-to-end scenario tests using pyrit_scan CLI + +trigger: none # Disable automatic CI triggers + +schedules: +- cron: "0 7 * * *" # 7 AM UTC = 11 PM PST (UTC-8) / Midnight PDT (UTC-7) + displayName: Nightly E2E Tests at 11 PM PST + branches: + include: + - main + always: true # Run even if there are no code changes + +jobs: +- template: test-job-template.yml + parameters: + jobName: EndToEndTests + jobDisplayName: "Run end-to-end scenario tests" + testAzureSubscription: 'integration-test-service-connection' + newDir: e2e_test_directory + testsFolder: end_to_end + makeTarget: end-to-end-test diff --git a/ado_pipeline_config/integration-tests.yml b/ado_pipeline_config/integration-tests.yml new file mode 100644 index 000000000..7ab07266c --- /dev/null +++ b/ado_pipeline_config/integration-tests.yml @@ -0,0 +1,19 @@ + +# Builds the pyrit environment and runs integration tests + +trigger: + branches: + include: + - main + +# There are additional PR triggers for this that are configurable in ADO. + +jobs: +- template: test-job-template.yml + parameters: + jobName: IntegrationTests + jobDisplayName: "Builds the pyrit environment and runs integration tests" + testAzureSubscription: 'integration-test-service-connection' + newDir: integration_test_directory + testsFolder: integration + makeTarget: integration-test diff --git a/ado_pipeline_config/partner-integration-tests.yml b/ado_pipeline_config/partner-integration-tests.yml new file mode 100644 index 000000000..b101208a7 --- /dev/null +++ b/ado_pipeline_config/partner-integration-tests.yml @@ -0,0 +1,20 @@ + +trigger: none # Disable automatic CI triggers + +schedules: +- cron: "0 6 * * *" # 6 AM UTC = 10 PM PST (UTC-8) / 11PM PDT (UTC-6) + displayName: Nightly Partner Integration Tests at 10 PM PST + branches: + include: + - main + always: true # Run even if there are no code changes + +jobs: +- template: test-job-template.yml + parameters: + jobName: PartnerIntegrationTests + jobDisplayName: "Builds the pyrit environment and runs partner integration tests" + testAzureSubscription: 'partner-integration-test-service-connection' + newDir: partner_integration_test_directory + testsFolder: partner_integration + makeTarget: partner-integration-test diff --git a/integration-tests.yml b/ado_pipeline_config/test-job-template.yml similarity index 82% rename from integration-tests.yml rename to ado_pipeline_config/test-job-template.yml index ff6fe4d0e..b4a854894 100644 --- a/integration-tests.yml +++ b/ado_pipeline_config/test-job-template.yml @@ -1,17 +1,21 @@ - -# Builds the pyrit environment and runs integration tests - -trigger: - branches: - include: - - main - -# There are additional PR triggers for this that are configurable in ADO. +parameters: + - name: jobName + type: string + - name: jobDisplayName + type: string + - name: testAzureSubscription + type: string + - name: newDir + type: string + - name: testsFolder + type: string + - name: makeTarget + type: string jobs: -- job: IntegrationTests - displayName: "Builds the pyrit environment and runs integration tests" - timeoutInMinutes: 360 # Allows the job to run up to 6 hours +- job: ${{ parameters.jobName }} + displayName: ${{ parameters.jobDisplayName }} + timeoutInMinutes: 360 pool: vmImage: ubuntu-latest steps: @@ -79,21 +83,21 @@ jobs: name: install_PyRIT - bash: df -all -h name: disk_space_check -# This step ensures that integration tests are run outside of the PyRIT repository to test that .env files are accessed correctly. +# This step ensures that tests are run outside of the PyRIT repository to test that .env files are accessed correctly. - bash: | PyRIT_DIR=$(pwd) - NEW_DIR="integration_test_directory" + NEW_DIR="${{ parameters.newDir }}" cd .. mkdir -p $NEW_DIR/tests cp -r $PyRIT_DIR/doc $NEW_DIR cp -r $PyRIT_DIR/assets $NEW_DIR - cp -r $PyRIT_DIR/tests/integration $NEW_DIR/tests + cp -r $PyRIT_DIR/tests/${{ parameters.testsFolder }} $NEW_DIR/tests cd $NEW_DIR - displayName: "Create and switch to new integration test directory" + displayName: "Create and switch to new test directory" - task: AzureCLI@2 displayName: "Authenticate with service principal, cache Cognitive Services access token, and run tests" inputs: - azureSubscription: 'integration-test-service-connection' + azureSubscription: ${{ parameters.testAzureSubscription }} scriptType: 'bash' scriptLocation: 'inlineScript' inlineScript: | @@ -109,8 +113,8 @@ jobs: az account get-access-token --scope https://database.windows.net/.default --output none echo "Azure SQL Database access token cached successfully." - # Run integration tests - make integration-test + # Run tests + make ${{ parameters.makeTarget }} - bash: | rm -f ~/.pyrit/.env ~/.pyrit/.env.local name: clean_up_env_files diff --git a/end-to-end-tests.yml b/end-to-end-tests.yml deleted file mode 100644 index f18a5151f..000000000 --- a/end-to-end-tests.yml +++ /dev/null @@ -1,124 +0,0 @@ -# Runs end-to-end scenario tests using pyrit_scan CLI - -trigger: none # Disable automatic CI triggers - -schedules: -- cron: "0 7 * * *" # 7 AM UTC = 11 PM PST (UTC-8) / Midnight PDT (UTC-7) - displayName: Nightly E2E Tests at 11 PM PST - branches: - include: - - main - always: true # Run even if there are no code changes - -jobs: -- job: EndToEndTests - displayName: "Run end-to-end scenario tests" - timeoutInMinutes: 360 # Allows the job to run up to 6 hours - pool: - vmImage: ubuntu-latest - steps: - - checkout: self - fetchDepth: 1 - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.12' - addToPath: true - - bash: | - mkdir -p ~/.pyrit - displayName: "Create PyRIT configuration directory" - name: create_pyrit_dir - - task: AzureKeyVault@2 - displayName: Azure Key Vault - retrieve .env file secret - inputs: - azureSubscription: 'integration-test-service-connection' - KeyVaultName: 'pyrit-environment' - SecretsFilter: 'env-global' - RunAsPreJob: false - - bash: | - python -c " - import os; - secret = os.environ.get('PYRIT_TEST_SECRET'); - if not secret: - raise ValueError('PYRIT_TEST_SECRET is not set'); - with open(os.path.expanduser('~/.pyrit/.env'), 'w') as file: - file.write(secret)" - env: - PYRIT_TEST_SECRET: $(env-global) - name: create_env_file - - bash: | - cp build_scripts/env_local_integration_test ~/.pyrit/.env.local - displayName: "Create .env.local from example" - - script: - wget -qO- https://astral.sh/uv/install.sh | sh - name: install_uv - - bash: sudo apt-get install python3-tk - name: install_tkinter - - bash: | - set -e - # Detect Ubuntu version - UBUNTU_VERSION=$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2) - SUPPORTED_VERSIONS="18.04 20.04 22.04 24.04 24.10" - - if ! [[ "$SUPPORTED_VERSIONS" == *"$UBUNTU_VERSION"* ]]; then - echo "Ubuntu $UBUNTU_VERSION is not currently supported." - exit 1 - fi - - # Download the package to configure the Microsoft repo - curl -sSL -O https://packages.microsoft.com/config/ubuntu/$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2)/packages-microsoft-prod.deb - # Install the package - sudo dpkg -i packages-microsoft-prod.deb - # Delete the file - rm packages-microsoft-prod.deb - - # Install the driver - sudo apt-get update - sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 - - echo "Microsoft ODBC Driver 18 installed successfully." - displayName: 'Install ODBC Driver 18 for SQL Server' - - bash: uv sync --extra dev --extra all - name: install_PyRIT - - bash: df -all -h - name: disk_space_check -# This step ensures that end-to-end tests are run outside of the PyRIT repository to test that .env files are accessed correctly. - - bash: | - PyRIT_DIR=$(pwd) - NEW_DIR="e2e_test_directory" - cd .. - mkdir -p $NEW_DIR/tests - cp -r $PyRIT_DIR/doc $NEW_DIR - cp -r $PyRIT_DIR/assets $NEW_DIR - cp -r $PyRIT_DIR/tests/end_to_end $NEW_DIR/tests - cd $NEW_DIR - displayName: "Create and switch to E2E test directory" - - task: AzureCLI@2 - displayName: "Authenticate with service principal, cache access tokens, and run E2E tests" - inputs: - azureSubscription: 'integration-test-service-connection' - scriptType: 'bash' - scriptLocation: 'inlineScript' - inlineScript: | - # Prefetch token for Cognitive Services before ID token expires (60-90 minute validity) - az account get-access-token --scope https://cognitiveservices.azure.com/.default --output none - echo "Cognitive Services access token cached successfully." - - # Prefetch token for Azure ML / Foundry model endpoints - az account get-access-token --scope https://ml.azure.com/.default --output none - echo "Azure ML/Foundry access token cached successfully." - - # Prefetch token for Azure SQL Database - az account get-access-token --scope https://database.windows.net/.default --output none - echo "Azure SQL Database access token cached successfully." - - # Run end-to-end tests - make end-to-end-test - - bash: | - rm -f ~/.pyrit/.env ~/.pyrit/.env.local - name: clean_up_env_files - condition: always() - - task: PublishTestResults@2 - condition: always() - inputs: - testResultsFormat: 'JUnit' - testResultsFiles: 'junit/test-results.xml' diff --git a/partner-integration-tests.yml b/partner-integration-tests.yml deleted file mode 100644 index 569846ed9..000000000 --- a/partner-integration-tests.yml +++ /dev/null @@ -1,123 +0,0 @@ - -trigger: none # Disable automatic CI triggers - -schedules: -- cron: "0 6 * * *" # 6 AM UTC = 10 PM PST (UTC-8) / 11PM PDT (UTC-6) - displayName: Nightly Partner Integration Tests at 10 PM PST - branches: - include: - - main - always: true # Run even if there are no code changes - -jobs: -- job: PartnerIntegrationTests - displayName: "Builds the pyrit environment and runs partner integration tests" - timeoutInMinutes: 360 # Allows the job to run up to 6 hours - pool: - vmImage: ubuntu-latest - steps: - - checkout: self - fetchDepth: 1 - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.12' - addToPath: true - - bash: | - mkdir -p ~/.pyrit - displayName: "Create PyRIT configuration directory" - name: create_pyrit_dir - - task: AzureKeyVault@2 - displayName: Azure Key Vault - retrieve .env file secret - inputs: - azureSubscription: 'integration-test-service-connection' - KeyVaultName: 'pyrit-environment' - SecretsFilter: 'env-global' - RunAsPreJob: false - - bash: | - python -c " - import os; - secret = os.environ.get('PYRIT_TEST_SECRET'); - if not secret: - raise ValueError('PYRIT_TEST_SECRET is not set'); - with open(os.path.expanduser('~/.pyrit/.env'), 'w') as file: - file.write(secret)" - env: - PYRIT_TEST_SECRET: $(env-global) - name: create_env_file - - bash: | - cp build_scripts/env_local_integration_test ~/.pyrit/.env.local - displayName: "Create .env.local from example" - - script: - wget -qO- https://astral.sh/uv/install.sh | sh - name: install_uv - - bash: sudo apt-get install python3-tk - name: install_tkinter - - bash: | - set -e - # Detect Ubuntu version - UBUNTU_VERSION=$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2) - SUPPORTED_VERSIONS="18.04 20.04 22.04 24.04 24.10" - - if ! [[ "$SUPPORTED_VERSIONS" == *"$UBUNTU_VERSION"* ]]; then - echo "Ubuntu $UBUNTU_VERSION is not currently supported." - exit 1 - fi - - # Download the package to configure the Microsoft repo - curl -sSL -O https://packages.microsoft.com/config/ubuntu/$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2)/packages-microsoft-prod.deb - # Install the package - sudo dpkg -i packages-microsoft-prod.deb - # Delete the file - rm packages-microsoft-prod.deb - - # Install the driver - sudo apt-get update - sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 - - echo "Microsoft ODBC Driver 18 installed successfully." - displayName: 'Install ODBC Driver 18 for SQL Server' - - bash: uv sync --extra dev --extra all - name: install_PyRIT - - bash: df -all -h - name: disk_space_check -# This step ensures that integration tests are run outside of the PyRIT repository to test that .env files are accessed correctly. - - bash: | - PyRIT_DIR=$(pwd) - NEW_DIR="partner_integration_test_directory" - cd .. - mkdir -p $NEW_DIR/tests - cp -r $PyRIT_DIR/doc $NEW_DIR - cp -r $PyRIT_DIR/assets $NEW_DIR - cp -r $PyRIT_DIR/tests/partner_integration $NEW_DIR/tests - cd $NEW_DIR - displayName: "Create and switch to new partner integration test directory" - - task: AzureCLI@2 - displayName: "Authenticate with service principal, cache Cognitive Services access token, and run tests" - inputs: - azureSubscription: 'partner-integration-test-service-connection' - scriptType: 'bash' - scriptLocation: 'inlineScript' - inlineScript: | - # Prefetch token for Cognitive Services before ID token expires (60-90 minute validity) - az account get-access-token --scope https://cognitiveservices.azure.com/.default --output none - echo "Cognitive Services access token cached successfully." - - # Prefetch token for Azure ML / Foundry model endpoints - az account get-access-token --scope https://ml.azure.com/.default --output none - echo "Azure ML/Foundry access token cached successfully." - - # Prefetch token for Azure SQL Database - az account get-access-token --scope https://database.windows.net/.default --output none - echo "Azure SQL Database access token cached successfully." - - # Run partner integration tests - make partner-integration-test - - bash: | - rm -f ~/.pyrit/.env ~/.pyrit/.env.local - name: clean_up_env_files - condition: always() - - task: PublishTestResults@2 - condition: always() - inputs: - testResultsFormat: 'JUnit' - testResultsFiles: 'junit/test-results.xml'