From e34b36544b7ddebeb9e718c8a1a6eff63335faf5 Mon Sep 17 00:00:00 2001 From: hannahwestra25 Date: Tue, 31 Mar 2026 12:54:35 -0400 Subject: [PATCH] add templates --- integration-tests.yml | 119 ++---------------- partner-integration-tests.yml | 20 ++++ templates/integration-test-job-template.yml | 126 ++++++++++++++++++++ 3 files changed, 154 insertions(+), 111 deletions(-) create mode 100644 partner-integration-tests.yml create mode 100644 templates/integration-test-job-template.yml diff --git a/integration-tests.yml b/integration-tests.yml index ff6fe4d0e..d2e2d4d21 100644 --- a/integration-tests.yml +++ b/integration-tests.yml @@ -9,114 +9,11 @@ trigger: # There are additional PR triggers for this that are configurable in ADO. jobs: -- job: IntegrationTests - displayName: "Builds the pyrit environment and runs integration tests" - timeoutInMinutes: 360 # Allows the job to run up to 6 hours - pool: - vmImage: ubuntu-latest - steps: - - checkout: self - fetchDepth: 1 - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.12' - addToPath: true - - bash: | - mkdir -p ~/.pyrit - displayName: "Create PyRIT configuration directory" - name: create_pyrit_dir - - task: AzureKeyVault@2 - displayName: Azure Key Vault - retrieve .env file secret - inputs: - azureSubscription: 'integration-test-service-connection' - KeyVaultName: 'pyrit-environment' - SecretsFilter: 'env-global' - RunAsPreJob: false - - bash: | - python -c " - import os; - secret = os.environ.get('PYRIT_TEST_SECRET'); - if not secret: - raise ValueError('PYRIT_TEST_SECRET is not set'); - with open(os.path.expanduser('~/.pyrit/.env'), 'w') as file: - file.write(secret)" - env: - PYRIT_TEST_SECRET: $(env-global) - name: create_env_file - - bash: | - cp build_scripts/env_local_integration_test ~/.pyrit/.env.local - displayName: "Create .env.local from example" - - script: - wget -qO- https://astral.sh/uv/install.sh | sh - name: install_uv - - bash: sudo apt-get install python3-tk - name: install_tkinter - - bash: | - set -e - # Detect Ubuntu version - UBUNTU_VERSION=$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2) - SUPPORTED_VERSIONS="18.04 20.04 22.04 24.04 24.10" - - if ! [[ "$SUPPORTED_VERSIONS" == *"$UBUNTU_VERSION"* ]]; then - echo "Ubuntu $UBUNTU_VERSION is not currently supported." - exit 1 - fi - - # Download the package to configure the Microsoft repo - curl -sSL -O https://packages.microsoft.com/config/ubuntu/$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2)/packages-microsoft-prod.deb - # Install the package - sudo dpkg -i packages-microsoft-prod.deb - # Delete the file - rm packages-microsoft-prod.deb - - # Install the driver - sudo apt-get update - sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 - - echo "Microsoft ODBC Driver 18 installed successfully." - displayName: 'Install ODBC Driver 18 for SQL Server' - - bash: uv sync --extra dev --extra all - name: install_PyRIT - - bash: df -all -h - name: disk_space_check -# This step ensures that integration tests are run outside of the PyRIT repository to test that .env files are accessed correctly. - - bash: | - PyRIT_DIR=$(pwd) - NEW_DIR="integration_test_directory" - cd .. - mkdir -p $NEW_DIR/tests - cp -r $PyRIT_DIR/doc $NEW_DIR - cp -r $PyRIT_DIR/assets $NEW_DIR - cp -r $PyRIT_DIR/tests/integration $NEW_DIR/tests - cd $NEW_DIR - displayName: "Create and switch to new integration test directory" - - task: AzureCLI@2 - displayName: "Authenticate with service principal, cache Cognitive Services access token, and run tests" - inputs: - azureSubscription: 'integration-test-service-connection' - scriptType: 'bash' - scriptLocation: 'inlineScript' - inlineScript: | - # Prefetch token for Cognitive Services before ID token expires (60-90 minute validity) - az account get-access-token --scope https://cognitiveservices.azure.com/.default --output none - echo "Cognitive Services access token cached successfully." - - # Prefetch token for Azure ML / Foundry model endpoints - az account get-access-token --scope https://ml.azure.com/.default --output none - echo "Azure ML/Foundry access token cached successfully." - - # Prefetch token for Azure SQL Database - az account get-access-token --scope https://database.windows.net/.default --output none - echo "Azure SQL Database access token cached successfully." - - # Run integration tests - make integration-test - - bash: | - rm -f ~/.pyrit/.env ~/.pyrit/.env.local - name: clean_up_env_files - condition: always() - - task: PublishTestResults@2 - condition: always() - inputs: - testResultsFormat: 'JUnit' - testResultsFiles: 'junit/test-results.xml' +- template: templates/integration-test-job-template.yml + parameters: + jobName: IntegrationTests + jobDisplayName: "Builds the pyrit environment and runs integration tests" + testAzureSubscription: 'integration-test-service-connection' + newDir: integration_test_directory + testsFolder: integration + makeTarget: integration-test diff --git a/partner-integration-tests.yml b/partner-integration-tests.yml new file mode 100644 index 000000000..42577239a --- /dev/null +++ b/partner-integration-tests.yml @@ -0,0 +1,20 @@ + +trigger: none # Disable automatic CI triggers + +schedules: +- cron: "0 6 * * *" # 6 AM UTC = 10 PM PST (UTC-8) / 11PM PDT (UTC-6) + displayName: Nightly Partner Integration Tests at 10 PM PST + branches: + include: + - main + always: true # Run even if there are no code changes + +jobs: +- template: templates/integration-test-job-template.yml + parameters: + jobName: PartnerIntegrationTests + jobDisplayName: "Builds the pyrit environment and runs partner integration tests" + testAzureSubscription: 'partner-integration-test-service-connection' + newDir: partner_integration_test_directory + testsFolder: partner_integration + makeTarget: partner-integration-test diff --git a/templates/integration-test-job-template.yml b/templates/integration-test-job-template.yml new file mode 100644 index 000000000..b4a854894 --- /dev/null +++ b/templates/integration-test-job-template.yml @@ -0,0 +1,126 @@ +parameters: + - name: jobName + type: string + - name: jobDisplayName + type: string + - name: testAzureSubscription + type: string + - name: newDir + type: string + - name: testsFolder + type: string + - name: makeTarget + type: string + +jobs: +- job: ${{ parameters.jobName }} + displayName: ${{ parameters.jobDisplayName }} + timeoutInMinutes: 360 + pool: + vmImage: ubuntu-latest + steps: + - checkout: self + fetchDepth: 1 + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.12' + addToPath: true + - bash: | + mkdir -p ~/.pyrit + displayName: "Create PyRIT configuration directory" + name: create_pyrit_dir + - task: AzureKeyVault@2 + displayName: Azure Key Vault - retrieve .env file secret + inputs: + azureSubscription: 'integration-test-service-connection' + KeyVaultName: 'pyrit-environment' + SecretsFilter: 'env-global' + RunAsPreJob: false + - bash: | + python -c " + import os; + secret = os.environ.get('PYRIT_TEST_SECRET'); + if not secret: + raise ValueError('PYRIT_TEST_SECRET is not set'); + with open(os.path.expanduser('~/.pyrit/.env'), 'w') as file: + file.write(secret)" + env: + PYRIT_TEST_SECRET: $(env-global) + name: create_env_file + - bash: | + cp build_scripts/env_local_integration_test ~/.pyrit/.env.local + displayName: "Create .env.local from example" + - script: + wget -qO- https://astral.sh/uv/install.sh | sh + name: install_uv + - bash: sudo apt-get install python3-tk + name: install_tkinter + - bash: | + set -e + # Detect Ubuntu version + UBUNTU_VERSION=$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2) + SUPPORTED_VERSIONS="18.04 20.04 22.04 24.04 24.10" + + if ! [[ "$SUPPORTED_VERSIONS" == *"$UBUNTU_VERSION"* ]]; then + echo "Ubuntu $UBUNTU_VERSION is not currently supported." + exit 1 + fi + + # Download the package to configure the Microsoft repo + curl -sSL -O https://packages.microsoft.com/config/ubuntu/$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2)/packages-microsoft-prod.deb + # Install the package + sudo dpkg -i packages-microsoft-prod.deb + # Delete the file + rm packages-microsoft-prod.deb + + # Install the driver + sudo apt-get update + sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 + + echo "Microsoft ODBC Driver 18 installed successfully." + displayName: 'Install ODBC Driver 18 for SQL Server' + - bash: uv sync --extra dev --extra all + name: install_PyRIT + - bash: df -all -h + name: disk_space_check +# This step ensures that tests are run outside of the PyRIT repository to test that .env files are accessed correctly. + - bash: | + PyRIT_DIR=$(pwd) + NEW_DIR="${{ parameters.newDir }}" + cd .. + mkdir -p $NEW_DIR/tests + cp -r $PyRIT_DIR/doc $NEW_DIR + cp -r $PyRIT_DIR/assets $NEW_DIR + cp -r $PyRIT_DIR/tests/${{ parameters.testsFolder }} $NEW_DIR/tests + cd $NEW_DIR + displayName: "Create and switch to new test directory" + - task: AzureCLI@2 + displayName: "Authenticate with service principal, cache Cognitive Services access token, and run tests" + inputs: + azureSubscription: ${{ parameters.testAzureSubscription }} + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + # Prefetch token for Cognitive Services before ID token expires (60-90 minute validity) + az account get-access-token --scope https://cognitiveservices.azure.com/.default --output none + echo "Cognitive Services access token cached successfully." + + # Prefetch token for Azure ML / Foundry model endpoints + az account get-access-token --scope https://ml.azure.com/.default --output none + echo "Azure ML/Foundry access token cached successfully." + + # Prefetch token for Azure SQL Database + az account get-access-token --scope https://database.windows.net/.default --output none + echo "Azure SQL Database access token cached successfully." + + # Run tests + make ${{ parameters.makeTarget }} + - bash: | + rm -f ~/.pyrit/.env ~/.pyrit/.env.local + name: clean_up_env_files + condition: always() + - task: PublishTestResults@2 + condition: always() + inputs: + testResultsFormat: 'JUnit' + testResultsFiles: 'junit/test-results.xml'