DRAFT add templates

integration-tests.yml

@@ -9,114 +9,11 @@ trigger:
 # There are additional PR triggers for this that are configurable in ADO.
 
 jobs:
-- job: IntegrationTests
-  displayName: "Builds the pyrit environment and runs integration tests"
-  timeoutInMinutes: 360   # Allows the job to run up to 6 hours
-  pool:
-    vmImage: ubuntu-latest
-  steps:
-  - checkout: self
-    fetchDepth: 1
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.12'
-      addToPath: true
-  - bash: |
-      mkdir -p ~/.pyrit
-    displayName: "Create PyRIT configuration directory"
-    name: create_pyrit_dir
-  - task: AzureKeyVault@2
-    displayName: Azure Key Vault - retrieve .env file secret
-    inputs:
-      azureSubscription: 'integration-test-service-connection'
-      KeyVaultName: 'pyrit-environment'
-      SecretsFilter: 'env-global'
-      RunAsPreJob: false
-  - bash: |
-      python -c "
-      import os;
-      secret = os.environ.get('PYRIT_TEST_SECRET');
-      if not secret:
-        raise ValueError('PYRIT_TEST_SECRET is not set');
-      with open(os.path.expanduser('~/.pyrit/.env'), 'w') as file:
-        file.write(secret)"
-    env:
-      PYRIT_TEST_SECRET: $(env-global)
-    name: create_env_file
-  - bash: |
-      cp build_scripts/env_local_integration_test ~/.pyrit/.env.local
-    displayName: "Create .env.local from example"
-  - script:
-      wget -qO- https://astral.sh/uv/install.sh | sh
-    name: install_uv
-  - bash: sudo apt-get install python3-tk
-    name: install_tkinter
-  - bash: |
-      set -e
-      # Detect Ubuntu version
-      UBUNTU_VERSION=$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2)
-      SUPPORTED_VERSIONS="18.04 20.04 22.04 24.04 24.10"
-
-      if ! [[ "$SUPPORTED_VERSIONS" == *"$UBUNTU_VERSION"* ]]; then
-        echo "Ubuntu $UBUNTU_VERSION is not currently supported."
-        exit 1
-      fi
-
-      # Download the package to configure the Microsoft repo
-      curl -sSL -O https://packages.microsoft.com/config/ubuntu/$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2)/packages-microsoft-prod.deb
-      # Install the package
-      sudo dpkg -i packages-microsoft-prod.deb
-      # Delete the file
-      rm packages-microsoft-prod.deb
-
-      # Install the driver
-      sudo apt-get update
-      sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18
-
-      echo "Microsoft ODBC Driver 18 installed successfully."
-    displayName: 'Install ODBC Driver 18 for SQL Server'
-  - bash: uv sync --extra dev --extra all
-    name: install_PyRIT
-  - bash: df -all -h
-    name: disk_space_check
-# This step ensures that integration tests are run outside of the PyRIT repository to test that .env files are accessed correctly.
-  - bash: |
-      PyRIT_DIR=$(pwd)
-      NEW_DIR="integration_test_directory"
-      cd ..
-      mkdir -p $NEW_DIR/tests
-      cp -r $PyRIT_DIR/doc $NEW_DIR
-      cp -r $PyRIT_DIR/assets $NEW_DIR
-      cp -r $PyRIT_DIR/tests/integration $NEW_DIR/tests
-      cd $NEW_DIR
-    displayName: "Create and switch to new integration test directory"
-  - task: AzureCLI@2
-    displayName: "Authenticate with service principal, cache Cognitive Services access token, and run tests"
-    inputs:
-      azureSubscription: 'integration-test-service-connection'
-      scriptType: 'bash'
-      scriptLocation: 'inlineScript'
-      inlineScript: |
-        # Prefetch token for Cognitive Services before ID token expires (60-90 minute validity)
-        az account get-access-token --scope https://cognitiveservices.azure.com/.default --output none
-        echo "Cognitive Services access token cached successfully."
-
-        # Prefetch token for Azure ML / Foundry model endpoints
-        az account get-access-token --scope https://ml.azure.com/.default --output none
-        echo "Azure ML/Foundry access token cached successfully."
-
-        # Prefetch token for Azure SQL Database
-        az account get-access-token --scope https://database.windows.net/.default --output none
-        echo "Azure SQL Database access token cached successfully."
-
-        # Run integration tests
-        make integration-test
-  - bash: |
-      rm -f ~/.pyrit/.env ~/.pyrit/.env.local
-    name: clean_up_env_files
-    condition: always()
-  - task: PublishTestResults@2
-    condition: always()
-    inputs:
-      testResultsFormat: 'JUnit'
-      testResultsFiles: 'junit/test-results.xml'
+- template: templates/integration-test-job-template.yml
+  parameters:
+    jobName: IntegrationTests
+    jobDisplayName: "Builds the pyrit environment and runs integration tests"
+    testAzureSubscription: 'integration-test-service-connection'
+    newDir: integration_test_directory
+    testsFolder: integration
+    makeTarget: integration-test

partner-integration-tests.yml

@@ -0,0 +1,20 @@
+
+trigger: none  # Disable automatic CI triggers
+
+schedules:
+- cron: "0 6 * * *"  # 6 AM UTC = 10 PM PST (UTC-8) / 11PM PDT (UTC-6)
+  displayName: Nightly Partner Integration Tests at 10 PM PST
+  branches:
+    include:
+    - main
+  always: true  # Run even if there are no code changes
+
+jobs:
+- template: templates/integration-test-job-template.yml
+  parameters:
+    jobName: PartnerIntegrationTests
+    jobDisplayName: "Builds the pyrit environment and runs partner integration tests"
+    testAzureSubscription: 'partner-integration-test-service-connection'
+    newDir: partner_integration_test_directory
+    testsFolder: partner_integration
+    makeTarget: partner-integration-test

templates/integration-test-job-template.yml

@@ -0,0 +1,126 @@
+parameters:
+  - name: jobName
+    type: string
+  - name: jobDisplayName
+    type: string
+  - name: testAzureSubscription
+    type: string
+  - name: newDir
+    type: string
+  - name: testsFolder
+    type: string
+  - name: makeTarget
+    type: string
+
+jobs:
+- job: ${{ parameters.jobName }}
+  displayName: ${{ parameters.jobDisplayName }}
+  timeoutInMinutes: 360
+  pool:
+    vmImage: ubuntu-latest
+  steps:
+  - checkout: self
+    fetchDepth: 1
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '3.12'
+      addToPath: true
+  - bash: |
+      mkdir -p ~/.pyrit
+    displayName: "Create PyRIT configuration directory"
+    name: create_pyrit_dir
+  - task: AzureKeyVault@2
+    displayName: Azure Key Vault - retrieve .env file secret
+    inputs:
+      azureSubscription: 'integration-test-service-connection'
+      KeyVaultName: 'pyrit-environment'
+      SecretsFilter: 'env-global'
+      RunAsPreJob: false
+  - bash: |
+      python -c "
+      import os;
+      secret = os.environ.get('PYRIT_TEST_SECRET');
+      if not secret:
+        raise ValueError('PYRIT_TEST_SECRET is not set');
+      with open(os.path.expanduser('~/.pyrit/.env'), 'w') as file:
+        file.write(secret)"
+    env:
+      PYRIT_TEST_SECRET: $(env-global)
+    name: create_env_file
+  - bash: |
+      cp build_scripts/env_local_integration_test ~/.pyrit/.env.local
+    displayName: "Create .env.local from example"
+  - script:
+      wget -qO- https://astral.sh/uv/install.sh | sh
+    name: install_uv
+  - bash: sudo apt-get install python3-tk
+    name: install_tkinter
+  - bash: |
+      set -e
+      # Detect Ubuntu version
+      UBUNTU_VERSION=$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2)
+      SUPPORTED_VERSIONS="18.04 20.04 22.04 24.04 24.10"
+
+      if ! [[ "$SUPPORTED_VERSIONS" == *"$UBUNTU_VERSION"* ]]; then
+        echo "Ubuntu $UBUNTU_VERSION is not currently supported."
+        exit 1
+      fi
+
+      # Download the package to configure the Microsoft repo
+      curl -sSL -O https://packages.microsoft.com/config/ubuntu/$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2)/packages-microsoft-prod.deb
+      # Install the package
+      sudo dpkg -i packages-microsoft-prod.deb
+      # Delete the file
+      rm packages-microsoft-prod.deb
+
+      # Install the driver
+      sudo apt-get update
+      sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18
+
+      echo "Microsoft ODBC Driver 18 installed successfully."
+    displayName: 'Install ODBC Driver 18 for SQL Server'
+  - bash: uv sync --extra dev --extra all
+    name: install_PyRIT
+  - bash: df -all -h
+    name: disk_space_check
+# This step ensures that tests are run outside of the PyRIT repository to test that .env files are accessed correctly.
+  - bash: |
+      PyRIT_DIR=$(pwd)
+      NEW_DIR="${{ parameters.newDir }}"
+      cd ..
+      mkdir -p $NEW_DIR/tests
+      cp -r $PyRIT_DIR/doc $NEW_DIR
+      cp -r $PyRIT_DIR/assets $NEW_DIR
+      cp -r $PyRIT_DIR/tests/${{ parameters.testsFolder }} $NEW_DIR/tests
+      cd $NEW_DIR
+    displayName: "Create and switch to new test directory"
+  - task: AzureCLI@2
+    displayName: "Authenticate with service principal, cache Cognitive Services access token, and run tests"
+    inputs:
+      azureSubscription: ${{ parameters.testAzureSubscription }}
+      scriptType: 'bash'
+      scriptLocation: 'inlineScript'
+      inlineScript: |
+        # Prefetch token for Cognitive Services before ID token expires (60-90 minute validity)
+        az account get-access-token --scope https://cognitiveservices.azure.com/.default --output none
+        echo "Cognitive Services access token cached successfully."
+
+        # Prefetch token for Azure ML / Foundry model endpoints
+        az account get-access-token --scope https://ml.azure.com/.default --output none
+        echo "Azure ML/Foundry access token cached successfully."
+
+        # Prefetch token for Azure SQL Database
+        az account get-access-token --scope https://database.windows.net/.default --output none
+        echo "Azure SQL Database access token cached successfully."
+
+        # Run tests
+        make ${{ parameters.makeTarget }}
+  - bash: |
+      rm -f ~/.pyrit/.env ~/.pyrit/.env.local
+    name: clean_up_env_files
+    condition: always()
+  - task: PublishTestResults@2
+    condition: always()
+    inputs:
+      testResultsFormat: 'JUnit'
+      testResultsFiles: 'junit/test-results.xml'