From c0cac7369b6ea5d8b4173d8a52ae84cda5a10f22 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Tue, 12 May 2026 17:34:03 +0200 Subject: [PATCH 01/20] Add workflow to read PerfStar pipeline data from AzDO via OIDC Uses a User-Assigned Managed Identity (msbuild-azdo-reader) with OIDC federated credentials to authenticate GitHub Actions to Azure DevOps. Supports: - Listing recent PerfStar-Scheduled (25429) and PerfStar-Branch-Trigger (25430) runs - Fetching specific run details - Listing artifacts (CrankAssets performance results) - Showing build timeline/stage status Requires secrets: AZDO_READER_CLIENT_ID, AZDO_READER_TENANT_ID, AZDO_READER_SUBSCRIPTION_ID --- .github/workflows/read-azdo-perfstar.yml | 143 +++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 .github/workflows/read-azdo-perfstar.yml diff --git a/.github/workflows/read-azdo-perfstar.yml b/.github/workflows/read-azdo-perfstar.yml new file mode 100644 index 00000000000..6a8f20704c5 --- /dev/null +++ b/.github/workflows/read-azdo-perfstar.yml @@ -0,0 +1,143 @@ +name: Read AzDO PerfStar Results + +on: + workflow_dispatch: + inputs: + pipeline: + description: 'PerfStar pipeline to query' + required: true + type: choice + options: + - 'scheduled' + - 'branch-trigger' + run_id: + description: 'Specific run ID (leave empty for latest successful)' + required: false + default: '' + count: + description: 'Number of recent runs to list (when no run_id specified)' + required: false + default: '10' + +permissions: + id-token: write + contents: read + +env: + AZDO_ORG: DevDiv + AZDO_PROJECT: DevDiv + PIPELINE_SCHEDULED: '25429' + PIPELINE_BRANCH_TRIGGER: '25430' + +jobs: + read-perfstar: + runs-on: ubuntu-latest + steps: + - name: Azure Login (OIDC) + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZDO_READER_CLIENT_ID }} + tenant-id: ${{ secrets.AZDO_READER_TENANT_ID }} + subscription-id: ${{ secrets.AZDO_READER_SUBSCRIPTION_ID }} + + - name: Get AzDO Bearer Token + id: token + run: | + AZDO_TOKEN=$(az account get-access-token \ + --resource "499b84ac-1321-427f-aa17-267ca6975798" \ + --query accessToken -o tsv) + echo "::add-mask::${AZDO_TOKEN}" + echo "azdo_token=${AZDO_TOKEN}" >> "$GITHUB_OUTPUT" + + - name: Resolve Pipeline ID + id: pipeline + run: | + if [ "${{ inputs.pipeline }}" = "scheduled" ]; then + echo "id=${PIPELINE_SCHEDULED}" >> "$GITHUB_OUTPUT" + echo "name=PerfStar-Scheduled" >> "$GITHUB_OUTPUT" + else + echo "id=${PIPELINE_BRANCH_TRIGGER}" >> "$GITHUB_OUTPUT" + echo "name=PerfStar-Branch-Trigger" >> "$GITHUB_OUTPUT" + fi + + - name: Fetch Pipeline Runs + id: runs + env: + AZDO_TOKEN: ${{ steps.token.outputs.azdo_token }} + PIPELINE_ID: ${{ steps.pipeline.outputs.id }} + RUN_ID: ${{ inputs.run_id }} + COUNT: ${{ inputs.count }} + run: | + BASE_URL="https://dev.azure.com/${AZDO_ORG}/${AZDO_PROJECT}/_apis" + + if [ -n "${RUN_ID}" ]; then + echo "## Fetching run ${RUN_ID} from ${{ steps.pipeline.outputs.name }}" >> "$GITHUB_STEP_SUMMARY" + RUN_DATA=$(curl -sf -H "Authorization: Bearer ${AZDO_TOKEN}" \ + "${BASE_URL}/pipelines/${PIPELINE_ID}/runs/${RUN_ID}?api-version=7.1") + echo "${RUN_DATA}" | jq . + echo '```json' >> "$GITHUB_STEP_SUMMARY" + echo "${RUN_DATA}" | jq '{id, name, state, result, createdDate, finishedDate}' >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + echo "target_run_id=${RUN_ID}" >> "$GITHUB_OUTPUT" + else + echo "## Latest ${COUNT} runs from ${{ steps.pipeline.outputs.name }}" >> "$GITHUB_STEP_SUMMARY" + RUNS=$(curl -sf -H "Authorization: Bearer ${AZDO_TOKEN}" \ + "${BASE_URL}/pipelines/${PIPELINE_ID}/runs?api-version=7.1") + echo '```' >> "$GITHUB_STEP_SUMMARY" + echo "${RUNS}" | jq -r ".value[:${COUNT}][] | \"\(.id) | \(.state) | \(.result) | \(.createdDate) | \(.name)\"" >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + + # Find latest successful run + LATEST_SUCCESS=$(echo "${RUNS}" | jq -r '[.value[] | select(.result == "succeeded")][0].id // empty') + if [ -n "${LATEST_SUCCESS}" ]; then + echo "Latest successful run: ${LATEST_SUCCESS}" + echo "target_run_id=${LATEST_SUCCESS}" >> "$GITHUB_OUTPUT" + else + echo "::warning::No successful runs found in recent history" + echo "target_run_id=" >> "$GITHUB_OUTPUT" + fi + fi + + - name: List Artifacts + if: steps.runs.outputs.target_run_id != '' + env: + AZDO_TOKEN: ${{ steps.token.outputs.azdo_token }} + BUILD_ID: ${{ steps.runs.outputs.target_run_id }} + run: | + BASE_URL="https://dev.azure.com/${AZDO_ORG}/${AZDO_PROJECT}/_apis" + + echo "## Artifacts for build ${BUILD_ID}" >> "$GITHUB_STEP_SUMMARY" + ARTIFACTS=$(curl -sf -H "Authorization: Bearer ${AZDO_TOKEN}" \ + "${BASE_URL}/build/builds/${BUILD_ID}/artifacts?api-version=7.1") + + echo '```' >> "$GITHUB_STEP_SUMMARY" + echo "${ARTIFACTS}" | jq -r '.value[] | "\(.name) (\(.resource.type))"' >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + + # Show Crank artifacts specifically (performance results) + CRANK_ARTIFACTS=$(echo "${ARTIFACTS}" | jq -r '.value[] | select(.name | startswith("CrankAssets")) | .name') + if [ -n "${CRANK_ARTIFACTS}" ]; then + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "### Performance result artifacts:" >> "$GITHUB_STEP_SUMMARY" + echo "${CRANK_ARTIFACTS}" | while read -r artifact_name; do + echo "- ${artifact_name}" >> "$GITHUB_STEP_SUMMARY" + done + fi + + - name: Fetch Build Timeline (Stage/Job Status) + if: steps.runs.outputs.target_run_id != '' + env: + AZDO_TOKEN: ${{ steps.token.outputs.azdo_token }} + BUILD_ID: ${{ steps.runs.outputs.target_run_id }} + run: | + BASE_URL="https://dev.azure.com/${AZDO_ORG}/${AZDO_PROJECT}/_apis" + + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "## Build Timeline Summary" >> "$GITHUB_STEP_SUMMARY" + + TIMELINE=$(curl -sf -H "Authorization: Bearer ${AZDO_TOKEN}" \ + "${BASE_URL}/build/builds/${BUILD_ID}/timeline?api-version=7.1") + + echo '```' >> "$GITHUB_STEP_SUMMARY" + echo "${TIMELINE}" | jq -r '.records[] | select(.type == "Stage") | "\(.name) | \(.state) | \(.result)"' >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" From 2f38c59adfa16a2ef538e550c186a8f4e310b1ea Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Tue, 12 May 2026 19:53:49 +0200 Subject: [PATCH 02/20] Add push trigger on feature branch to test OIDC flow --- .github/workflows/read-azdo-perfstar.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/read-azdo-perfstar.yml b/.github/workflows/read-azdo-perfstar.yml index 6a8f20704c5..7ed49069c6b 100644 --- a/.github/workflows/read-azdo-perfstar.yml +++ b/.github/workflows/read-azdo-perfstar.yml @@ -1,6 +1,9 @@ name: Read AzDO PerfStar Results on: + push: + branches: + - feature/azdo-pipeline-reader workflow_dispatch: inputs: pipeline: @@ -52,7 +55,8 @@ jobs: - name: Resolve Pipeline ID id: pipeline run: | - if [ "${{ inputs.pipeline }}" = "scheduled" ]; then + PIPELINE_INPUT="${{ inputs.pipeline || 'scheduled' }}" + if [ "${PIPELINE_INPUT}" = "scheduled" ]; then echo "id=${PIPELINE_SCHEDULED}" >> "$GITHUB_OUTPUT" echo "name=PerfStar-Scheduled" >> "$GITHUB_OUTPUT" else @@ -65,8 +69,8 @@ jobs: env: AZDO_TOKEN: ${{ steps.token.outputs.azdo_token }} PIPELINE_ID: ${{ steps.pipeline.outputs.id }} - RUN_ID: ${{ inputs.run_id }} - COUNT: ${{ inputs.count }} + RUN_ID: ${{ inputs.run_id || '' }} + COUNT: ${{ inputs.count || '10' }} run: | BASE_URL="https://dev.azure.com/${AZDO_ORG}/${AZDO_PROJECT}/_apis" From 78495fd7ce639a8352593557a5f7570f6fa844c2 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Tue, 12 May 2026 19:54:10 +0200 Subject: [PATCH 03/20] Use pull_request trigger to test OIDC flow on PR --- .github/workflows/read-azdo-perfstar.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/read-azdo-perfstar.yml b/.github/workflows/read-azdo-perfstar.yml index 7ed49069c6b..ae69f0acf27 100644 --- a/.github/workflows/read-azdo-perfstar.yml +++ b/.github/workflows/read-azdo-perfstar.yml @@ -1,9 +1,11 @@ name: Read AzDO PerfStar Results on: - push: + pull_request: branches: - - feature/azdo-pipeline-reader + - main + paths: + - '.github/workflows/read-azdo-perfstar.yml' workflow_dispatch: inputs: pipeline: From f6ba35876ea8cd4a7d1deb0989ab4b5dc65c7786 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Tue, 12 May 2026 19:56:35 +0200 Subject: [PATCH 04/20] Replace azure/login action with inline az login --federated-token The dotnet org likely restricts third-party actions. Use inline OIDC token exchange instead. --- .github/workflows/read-azdo-perfstar.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/read-azdo-perfstar.yml b/.github/workflows/read-azdo-perfstar.yml index ae69f0acf27..572c2a9f462 100644 --- a/.github/workflows/read-azdo-perfstar.yml +++ b/.github/workflows/read-azdo-perfstar.yml @@ -39,11 +39,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Azure Login (OIDC) - uses: azure/login@v2 - with: - client-id: ${{ secrets.AZDO_READER_CLIENT_ID }} - tenant-id: ${{ secrets.AZDO_READER_TENANT_ID }} - subscription-id: ${{ secrets.AZDO_READER_SUBSCRIPTION_ID }} + run: | + az login --service-principal \ + --username "${{ secrets.AZDO_READER_CLIENT_ID }}" \ + --tenant "${{ secrets.AZDO_READER_TENANT_ID }}" \ + --federated-token "$(curl -sS -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ + "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=api://AzureADTokenExchange" | jq -r '.value')" \ + --output none - name: Get AzDO Bearer Token id: token From b0693e385a4daf7196444bfcacf6c951e9d9923a Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Tue, 12 May 2026 19:58:00 +0200 Subject: [PATCH 05/20] Use pure curl OIDC flow (no azure/login, no az CLI) Match the pattern from dotnet/maui#35250 - get OIDC token from GitHub, exchange directly with Azure AD for AzDO-scoped bearer token. --- .github/workflows/read-azdo-perfstar.yml | 41 +++++++++++++++++------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/.github/workflows/read-azdo-perfstar.yml b/.github/workflows/read-azdo-perfstar.yml index 572c2a9f462..552cab79124 100644 --- a/.github/workflows/read-azdo-perfstar.yml +++ b/.github/workflows/read-azdo-perfstar.yml @@ -38,21 +38,38 @@ jobs: read-perfstar: runs-on: ubuntu-latest steps: - - name: Azure Login (OIDC) + - name: Get OIDC Token + id: oidc run: | - az login --service-principal \ - --username "${{ secrets.AZDO_READER_CLIENT_ID }}" \ - --tenant "${{ secrets.AZDO_READER_TENANT_ID }}" \ - --federated-token "$(curl -sS -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ - "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=api://AzureADTokenExchange" | jq -r '.value')" \ - --output none - - - name: Get AzDO Bearer Token + OIDC_TOKEN=$(curl -s -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" \ + "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=api://AzureADTokenExchange" \ + | jq -r '.value') + if [ -z "$OIDC_TOKEN" ] || [ "$OIDC_TOKEN" = "null" ]; then + echo "::error::Failed to get OIDC token" + exit 1 + fi + echo "::add-mask::${OIDC_TOKEN}" + echo "oidc_token=${OIDC_TOKEN}" >> "$GITHUB_OUTPUT" + + - name: Exchange for AzDO Token id: token + env: + OIDC_TOKEN: ${{ steps.oidc.outputs.oidc_token }} run: | - AZDO_TOKEN=$(az account get-access-token \ - --resource "499b84ac-1321-427f-aa17-267ca6975798" \ - --query accessToken -o tsv) + AZURE_RESPONSE=$(curl -s -X POST \ + "https://login.microsoftonline.com/${{ secrets.AZDO_READER_TENANT_ID }}/oauth2/v2.0/token" \ + -d "grant_type=client_credentials" \ + -d "client_id=${{ secrets.AZDO_READER_CLIENT_ID }}" \ + -d "client_assertion_type=urn:ietf:params:oauth:client-assertion-type:jwt-bearer" \ + -d "client_assertion=${OIDC_TOKEN}" \ + -d "scope=499b84ac-1321-427f-aa17-267ca6975798/.default") + + AZDO_TOKEN=$(echo "$AZURE_RESPONSE" | jq -r '.access_token') + if [ -z "$AZDO_TOKEN" ] || [ "$AZDO_TOKEN" = "null" ]; then + echo "::error::Failed to get Azure AD token" + echo "$AZURE_RESPONSE" | jq '{error, error_description, error_codes}' 2>/dev/null || true + exit 1 + fi echo "::add-mask::${AZDO_TOKEN}" echo "azdo_token=${AZDO_TOKEN}" >> "$GITHUB_OUTPUT" From 55ba7d682d917dc38432a7c1012869c6e428d7dc Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Tue, 12 May 2026 20:02:46 +0200 Subject: [PATCH 06/20] Remove temporary pull_request trigger used for testing --- .github/workflows/read-azdo-perfstar.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/read-azdo-perfstar.yml b/.github/workflows/read-azdo-perfstar.yml index 552cab79124..2806048bfc6 100644 --- a/.github/workflows/read-azdo-perfstar.yml +++ b/.github/workflows/read-azdo-perfstar.yml @@ -1,11 +1,6 @@ name: Read AzDO PerfStar Results on: - pull_request: - branches: - - main - paths: - - '.github/workflows/read-azdo-perfstar.yml' workflow_dispatch: inputs: pipeline: From f03e54ca91993ab72235e8e31775b1ad14debf94 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Tue, 12 May 2026 20:03:24 +0200 Subject: [PATCH 07/20] Address review feedback: validate inputs, remove verbose output, improve error reporting - Validate run_id is numeric and count is 1-100 - Remove raw 'jq .' output that could expose extra API fields - Replace curl -sf with -sS --fail-with-body for actionable diagnostics --- .github/workflows/read-azdo-perfstar.yml | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/workflows/read-azdo-perfstar.yml b/.github/workflows/read-azdo-perfstar.yml index 2806048bfc6..91b096d3e48 100644 --- a/.github/workflows/read-azdo-perfstar.yml +++ b/.github/workflows/read-azdo-perfstar.yml @@ -80,6 +80,20 @@ jobs: echo "name=PerfStar-Branch-Trigger" >> "$GITHUB_OUTPUT" fi + - name: Validate inputs + env: + RUN_ID: ${{ inputs.run_id || '' }} + COUNT: ${{ inputs.count || '10' }} + run: | + if [ -n "${RUN_ID}" ] && ! [[ "${RUN_ID}" =~ ^[0-9]+$ ]]; then + echo "::error::run_id must be a positive integer, got: '${RUN_ID}'" + exit 1 + fi + if ! [[ "${COUNT}" =~ ^[0-9]+$ ]] || [ "${COUNT}" -lt 1 ] || [ "${COUNT}" -gt 100 ]; then + echo "::error::count must be an integer between 1 and 100, got: '${COUNT}'" + exit 1 + fi + - name: Fetch Pipeline Runs id: runs env: @@ -92,16 +106,15 @@ jobs: if [ -n "${RUN_ID}" ]; then echo "## Fetching run ${RUN_ID} from ${{ steps.pipeline.outputs.name }}" >> "$GITHUB_STEP_SUMMARY" - RUN_DATA=$(curl -sf -H "Authorization: Bearer ${AZDO_TOKEN}" \ + RUN_DATA=$(curl -sS --fail-with-body -H "Authorization: Bearer ${AZDO_TOKEN}" \ "${BASE_URL}/pipelines/${PIPELINE_ID}/runs/${RUN_ID}?api-version=7.1") - echo "${RUN_DATA}" | jq . echo '```json' >> "$GITHUB_STEP_SUMMARY" echo "${RUN_DATA}" | jq '{id, name, state, result, createdDate, finishedDate}' >> "$GITHUB_STEP_SUMMARY" echo '```' >> "$GITHUB_STEP_SUMMARY" echo "target_run_id=${RUN_ID}" >> "$GITHUB_OUTPUT" else echo "## Latest ${COUNT} runs from ${{ steps.pipeline.outputs.name }}" >> "$GITHUB_STEP_SUMMARY" - RUNS=$(curl -sf -H "Authorization: Bearer ${AZDO_TOKEN}" \ + RUNS=$(curl -sS --fail-with-body -H "Authorization: Bearer ${AZDO_TOKEN}" \ "${BASE_URL}/pipelines/${PIPELINE_ID}/runs?api-version=7.1") echo '```' >> "$GITHUB_STEP_SUMMARY" echo "${RUNS}" | jq -r ".value[:${COUNT}][] | \"\(.id) | \(.state) | \(.result) | \(.createdDate) | \(.name)\"" >> "$GITHUB_STEP_SUMMARY" @@ -127,7 +140,7 @@ jobs: BASE_URL="https://dev.azure.com/${AZDO_ORG}/${AZDO_PROJECT}/_apis" echo "## Artifacts for build ${BUILD_ID}" >> "$GITHUB_STEP_SUMMARY" - ARTIFACTS=$(curl -sf -H "Authorization: Bearer ${AZDO_TOKEN}" \ + ARTIFACTS=$(curl -sS --fail-with-body -H "Authorization: Bearer ${AZDO_TOKEN}" \ "${BASE_URL}/build/builds/${BUILD_ID}/artifacts?api-version=7.1") echo '```' >> "$GITHUB_STEP_SUMMARY" @@ -155,7 +168,7 @@ jobs: echo "" >> "$GITHUB_STEP_SUMMARY" echo "## Build Timeline Summary" >> "$GITHUB_STEP_SUMMARY" - TIMELINE=$(curl -sf -H "Authorization: Bearer ${AZDO_TOKEN}" \ + TIMELINE=$(curl -sS --fail-with-body -H "Authorization: Bearer ${AZDO_TOKEN}" \ "${BASE_URL}/build/builds/${BUILD_ID}/timeline?api-version=7.1") echo '```' >> "$GITHUB_STEP_SUMMARY" From b7c2b8159582a9505ef58e31386ac347d16556e7 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Tue, 12 May 2026 20:04:07 +0200 Subject: [PATCH 08/20] Add setup documentation for AzDO PerfStar reader workflow --- .github/docs/azdo-perfstar-reader-setup.md | 122 +++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 .github/docs/azdo-perfstar-reader-setup.md diff --git a/.github/docs/azdo-perfstar-reader-setup.md b/.github/docs/azdo-perfstar-reader-setup.md new file mode 100644 index 00000000000..e8af0674ef0 --- /dev/null +++ b/.github/docs/azdo-perfstar-reader-setup.md @@ -0,0 +1,122 @@ +# Reading AzDO PerfStar Pipeline Data from GitHub Actions (No PAT) + +This document describes how the `read-azdo-perfstar.yml` workflow authenticates +to Azure DevOps and reads pipeline run data — without any PAT or stored credentials. + +## Architecture + +``` +GitHub Actions ──► GitHub OIDC Provider ──► Azure AD (federated credential) ──► AzDO REST API + (JWT id-token) (exchange for bearer token) (Read Pipelines) +``` + +1. The workflow requests an OIDC JWT from GitHub's token endpoint +2. The JWT is exchanged with Azure AD via the managed identity's federated credential +3. Azure AD returns a bearer token scoped to Azure DevOps +4. The bearer token calls the AzDO REST API to read pipeline runs, logs, and artifacts + +> **Important:** The `azure/login` GitHub Action is **blocked by org policy** +> in the `dotnet` org. The workflow uses **manual OIDC token exchange via `curl`** +> instead — no third-party action dependencies. + +## Components + +| Component | Value | +|-----------|-------| +| Managed Identity | `msbuild-azdo-reader` | +| Client ID | Stored in `AZDO_READER_CLIENT_ID` secret | +| Tenant | Microsoft (`72f988bf-86f1-41af-91ab-2d7cd011db47`) | +| Subscription | `CodeTestingAgentDev` (`bb947664-5d18-4aaa-8bbe-40dde6075462`) | +| Resource Group | `CodeTestingAgent` | +| AzDO Org/Project | `DevDiv` / `DevDiv` | +| Target Pipelines | 25429 (PerfStar-Scheduled), 25430 (PerfStar-Branch-Trigger) | +| Access Level | Read-only (View builds) | + +## Setup Steps (Already Completed) + +### 1. Create the Managed Identity + +```bash +az account set --subscription "CodeTestingAgentDev" +az identity create --name "msbuild-azdo-reader" --resource-group "CodeTestingAgent" --location "eastus" +``` + +### 2. Add OIDC Federated Credentials + +These allow GitHub Actions in `dotnet/msbuild` to authenticate as the identity: + +```bash +# Main branch +az identity federated-credential create \ + --name github-dotnet-msbuild-main \ + --identity-name "msbuild-azdo-reader" \ + --resource-group "CodeTestingAgent" \ + --issuer "https://token.actions.githubusercontent.com" \ + --subject "repo:dotnet/msbuild:ref:refs/heads/main" \ + --audiences "api://AzureADTokenExchange" + +# Pull requests +az identity federated-credential create \ + --name github-dotnet-msbuild-pr \ + --identity-name "msbuild-azdo-reader" \ + --resource-group "CodeTestingAgent" \ + --issuer "https://token.actions.githubusercontent.com" \ + --subject "repo:dotnet/msbuild:pull_request" \ + --audiences "api://AzureADTokenExchange" +``` + +> **Subject claim is case-sensitive.** The repo name in the subject must match +> exactly (e.g. `dotnet/msbuild`). + +> **Microsoft tenant restriction:** Only repos in GitHub Enterprise orgs +> (`dotnet`, `microsoft`, etc.) work — personal forks fail with `AADSTS7002381`. + +### 3. Register in AzDO + +File a Service Ticket in DevDiv (Area: `DevDiv\VSEng\DDBuild\Operations`, +type: "AzDO Administration Request") to add the MI to the DevDiv org with +read access to pipelines 25429 and 25430. + +### 4. GitHub Secrets + +| Secret | Value | +|--------|-------| +| `AZDO_READER_CLIENT_ID` | Managed identity Client ID | +| `AZDO_READER_TENANT_ID` | `72f988bf-86f1-41af-91ab-2d7cd011db47` | +| `AZDO_READER_SUBSCRIPTION_ID` | `bb947664-5d18-4aaa-8bbe-40dde6075462` | + +## How the Token Flow Works + +``` +1. Workflow declares `permissions: { id-token: write }` at workflow level +2. Step "Get OIDC Token" requests a JWT from GitHub's token endpoint + ($ACTIONS_ID_TOKEN_REQUEST_URL, audience: api://AzureADTokenExchange) +3. Step "Exchange for AzDO Token" POSTs the JWT to Azure AD as a client_assertion + (grant_type=client_credentials, scope: 499b84ac-.../.default) +4. Azure AD validates the JWT against the federated credential, returns a bearer token +5. Subsequent steps call AzDO REST APIs with the bearer token +``` + +## Usage + +Trigger via `workflow_dispatch` from the Actions tab: + +- **pipeline**: `scheduled` (25429) or `branch-trigger` (25430) +- **run_id**: Specific build ID (leave empty for latest) +- **count**: Number of recent runs to list (default: 10) + +## Troubleshooting + +| Symptom | Cause | Fix | +|---------|-------|-----| +| `startup_failure` (no logs) | Third-party Action blocked by org policy | Use manual `curl` OIDC exchange, not `azure/login` | +| `AADSTS70021: No matching federated identity record` | Subject claim mismatch | Check exact casing and event type in federated credential | +| `AADSTS7002381: enterprise claim` | Personal fork | Only enterprise org repos work with Microsoft tenant | +| `403` from AzDO API | MI not added to DevDiv org, or no "View builds" permission | File DDBuild Operations ticket | +| `Failed to get OIDC token` | Missing `id-token: write` permission | Ensure permissions block is present | + +## References + +- [Use service principals and managed identities in Azure DevOps](https://learn.microsoft.com/en-us/azure/devops/integrate/get-started/authentication/service-principal-managed-identity) +- [GitHub OIDC docs](https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/about-security-hardening-with-openid-connect) +- [AzDO Pipelines REST API](https://learn.microsoft.com/en-us/rest/api/azure/devops/pipelines/runs) From c28b183a626e46988ac30a80f76515e6fa7ceaac Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Thu, 14 May 2026 12:25:25 +0200 Subject: [PATCH 09/20] Sync PerfStar data: download artifacts, store on perf/dashboard, add schedule+chaining --- .github/docs/azdo-perfstar-reader-setup.md | 61 +++- .github/workflows/read-azdo-perfstar.yml | 306 ++++++++++++++------- 2 files changed, 259 insertions(+), 108 deletions(-) diff --git a/.github/docs/azdo-perfstar-reader-setup.md b/.github/docs/azdo-perfstar-reader-setup.md index e8af0674ef0..1aeec463e03 100644 --- a/.github/docs/azdo-perfstar-reader-setup.md +++ b/.github/docs/azdo-perfstar-reader-setup.md @@ -1,24 +1,45 @@ -# Reading AzDO PerfStar Pipeline Data from GitHub Actions (No PAT) +# Syncing PerfStar Performance Data from AzDO to GitHub This document describes how the `read-azdo-perfstar.yml` workflow authenticates -to Azure DevOps and reads pipeline run data — without any PAT or stored credentials. +to Azure DevOps, downloads PerfStar performance results, and commits them to the +`perf/dashboard` branch — without any PAT or stored credentials. ## Architecture ``` GitHub Actions ──► GitHub OIDC Provider ──► Azure AD (federated credential) ──► AzDO REST API - (JWT id-token) (exchange for bearer token) (Read Pipelines) + (JWT id-token) (exchange for bearer token) (Download artifacts) ``` 1. The workflow requests an OIDC JWT from GitHub's token endpoint 2. The JWT is exchanged with Azure AD via the managed identity's federated credential 3. Azure AD returns a bearer token scoped to Azure DevOps -4. The bearer token calls the AzDO REST API to read pipeline runs, logs, and artifacts +4. The bearer token calls the AzDO REST API to find builds and download artifacts +5. Artifacts (`CrankAssetsThinnedGOLDWIN`, `CrankAssetsThinnedGOLDLIN`) are + extracted and committed to `perf/dashboard` branch under `data/YYYY-MM-DD/` > **Important:** The `azure/login` GitHub Action is **blocked by org policy** > in the `dotnet` org. The workflow uses **manual OIDC token exchange via `curl`** > instead — no third-party action dependencies. +## Data Layout + +``` +perf/dashboard branch +└── data/ + ├── 2026-05-11/ + │ ├── GOLDWIN/ + │ │ ├── net8-console-app-rebuild-dotnet.json + │ │ └── ... + │ └── GOLDLIN/ + │ ├── net8-console-app-rebuild-dotnet.json + │ └── ... + ├── 2026-05-12/ + │ ├── GOLDWIN/ + │ └── GOLDLIN/ + └── ... +``` + ## Components | Component | Value | @@ -29,7 +50,8 @@ GitHub Actions ──► GitHub OIDC Provider ──► Azure AD (federated cred | Subscription | `CodeTestingAgentDev` (`bb947664-5d18-4aaa-8bbe-40dde6075462`) | | Resource Group | `CodeTestingAgent` | | AzDO Org/Project | `DevDiv` / `DevDiv` | -| Target Pipelines | 25429 (PerfStar-Scheduled), 25430 (PerfStar-Branch-Trigger) | +| Target Pipeline | 25429 (PerfStar-Scheduled) | +| Artifacts | `CrankAssetsThinnedGOLDWIN`, `CrankAssetsThinnedGOLDLIN` | | Access Level | Read-only (View builds) | ## Setup Steps (Already Completed) @@ -99,11 +121,32 @@ read access to pipelines 25429 and 25430. ## Usage -Trigger via `workflow_dispatch` from the Actions tab: +The workflow runs automatically on a daily schedule (6 pm UTC) and can also be +triggered manually from the Actions tab. + +### Scheduled runs + +Every day at 6 pm UTC the workflow: + +1. Looks at the `perf/dashboard` branch to find the latest `data/YYYY-MM-DD` folder +2. Processes the **next** day (latest + 1) +3. Finds the latest **scheduled** AzDO build for that date on pipeline 25429 +4. Downloads `CrankAssetsThinnedGOLDWIN` and `CrankAssetsThinnedGOLDLIN` (top-level `.json` only) +5. Commits to `perf/dashboard` under `data/YYYY-MM-DD/GOLDWIN/` and `GOLDLIN/` +6. If the processed date is not today, dispatches itself for the next date + +### Manual dispatch (`workflow_dispatch`) + +- **start_date** *(optional)*: Date to process (`YYYY-MM-DD`). Omit to auto-detect. +- **end_date** *(optional)*: Stop processing after this date. Defaults to today. + +When no `start_date` is given the workflow behaves identically to a scheduled run. + +### Build selection -- **pipeline**: `scheduled` (25429) or `branch-trigger` (25430) -- **run_id**: Specific build ID (leave empty for latest) -- **count**: Number of recent runs to list (default: 10) +When multiple AzDO builds exist for a single day, the workflow prefers the latest +**scheduled** run. If no scheduled run is found it falls back to the latest run +of any trigger type. ## Troubleshooting diff --git a/.github/workflows/read-azdo-perfstar.yml b/.github/workflows/read-azdo-perfstar.yml index 91b096d3e48..793d0b02615 100644 --- a/.github/workflows/read-azdo-perfstar.yml +++ b/.github/workflows/read-azdo-perfstar.yml @@ -1,39 +1,98 @@ -name: Read AzDO PerfStar Results +name: Sync PerfStar Data on: + schedule: + - cron: '0 18 * * *' # Daily at 6 pm UTC workflow_dispatch: inputs: - pipeline: - description: 'PerfStar pipeline to query' - required: true - type: choice - options: - - 'scheduled' - - 'branch-trigger' - run_id: - description: 'Specific run ID (leave empty for latest successful)' + start_date: + description: 'Start date (YYYY-MM-DD). Defaults to day after latest stored data.' required: false default: '' - count: - description: 'Number of recent runs to list (when no run_id specified)' + end_date: + description: 'End date (YYYY-MM-DD). Defaults to today.' required: false - default: '10' + default: '' + pull_request: # Temporary — remove after testing permissions: id-token: write - contents: read + contents: write + actions: write env: AZDO_ORG: DevDiv AZDO_PROJECT: DevDiv - PIPELINE_SCHEDULED: '25429' - PIPELINE_BRANCH_TRIGGER: '25430' + PIPELINE_ID: '25429' + DATA_BRANCH: perf/dashboard jobs: - read-perfstar: + sync-perfstar-data: runs-on: ubuntu-latest steps: + # ── 1. Determine which date to process ──────────────────────────── + - name: Determine processing date + id: date + env: + INPUT_START_DATE: ${{ inputs.start_date || '' }} + INPUT_END_DATE: ${{ inputs.end_date || '' }} + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + TODAY=$(date -u +%Y-%m-%d) + END_DATE="${INPUT_END_DATE:-$TODAY}" + + if [ -n "$INPUT_START_DATE" ]; then + TARGET_DATE="$INPUT_START_DATE" + echo "Using provided start date: $TARGET_DATE" + else + # Find the latest YYYY-MM-DD folder on the data branch + LATEST=$(gh api "repos/${{ github.repository }}/contents/data?ref=${{ env.DATA_BRANCH }}" \ + --jq '[.[].name | select(test("^[0-9]{4}-[0-9]{2}-[0-9]{2}$"))] | sort | last // empty' \ + 2>/dev/null || echo "") + + if [ -n "$LATEST" ]; then + TARGET_DATE=$(date -u -d "$LATEST + 1 day" +%Y-%m-%d) + echo "Latest stored data: $LATEST → processing next day: $TARGET_DATE" + else + TARGET_DATE=$(date -u -d "yesterday" +%Y-%m-%d) + echo "No existing data found → starting from: $TARGET_DATE" + fi + fi + + echo "target_date=${TARGET_DATE}" >> "$GITHUB_OUTPUT" + echo "end_date=${END_DATE}" >> "$GITHUB_OUTPUT" + echo "today=${TODAY}" >> "$GITHUB_OUTPUT" + + if [[ "$TARGET_DATE" > "$TODAY" ]]; then + echo "skip=true" >> "$GITHUB_OUTPUT" + echo "::notice::Target date $TARGET_DATE is in the future — nothing to do" + elif [[ "$TARGET_DATE" > "$END_DATE" ]]; then + echo "skip=true" >> "$GITHUB_OUTPUT" + echo "::notice::Target date $TARGET_DATE is past end date $END_DATE — nothing to do" + else + echo "skip=false" >> "$GITHUB_OUTPUT" + fi + + # ── 2. Skip early if data already stored ────────────────────────── + - name: Check if data already exists + if: steps.date.outputs.skip != 'true' + id: check + env: + GH_TOKEN: ${{ github.token }} + TARGET_DATE: ${{ steps.date.outputs.target_date }} + run: | + if gh api "repos/${{ github.repository }}/contents/data/${TARGET_DATE}?ref=${{ env.DATA_BRANCH }}" \ + > /dev/null 2>&1; then + echo "exists=true" >> "$GITHUB_OUTPUT" + echo "::notice::Data for ${TARGET_DATE} already exists on ${{ env.DATA_BRANCH }}" + else + echo "exists=false" >> "$GITHUB_OUTPUT" + fi + + # ── 3. Authenticate to AzDO via OIDC ────────────────────────────── - name: Get OIDC Token + if: steps.date.outputs.skip != 'true' && steps.check.outputs.exists != 'true' id: oidc run: | OIDC_TOKEN=$(curl -s -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" \ @@ -47,6 +106,7 @@ jobs: echo "oidc_token=${OIDC_TOKEN}" >> "$GITHUB_OUTPUT" - name: Exchange for AzDO Token + if: steps.date.outputs.skip != 'true' && steps.check.outputs.exists != 'true' id: token env: OIDC_TOKEN: ${{ steps.oidc.outputs.oidc_token }} @@ -68,109 +128,157 @@ jobs: echo "::add-mask::${AZDO_TOKEN}" echo "azdo_token=${AZDO_TOKEN}" >> "$GITHUB_OUTPUT" - - name: Resolve Pipeline ID - id: pipeline - run: | - PIPELINE_INPUT="${{ inputs.pipeline || 'scheduled' }}" - if [ "${PIPELINE_INPUT}" = "scheduled" ]; then - echo "id=${PIPELINE_SCHEDULED}" >> "$GITHUB_OUTPUT" - echo "name=PerfStar-Scheduled" >> "$GITHUB_OUTPUT" - else - echo "id=${PIPELINE_BRANCH_TRIGGER}" >> "$GITHUB_OUTPUT" - echo "name=PerfStar-Branch-Trigger" >> "$GITHUB_OUTPUT" - fi - - - name: Validate inputs + # ── 4. Find the right AzDO build for the target date ────────────── + - name: Find AzDO run for target date + if: steps.date.outputs.skip != 'true' && steps.check.outputs.exists != 'true' + id: find_run env: - RUN_ID: ${{ inputs.run_id || '' }} - COUNT: ${{ inputs.count || '10' }} + AZDO_TOKEN: ${{ steps.token.outputs.azdo_token }} + TARGET_DATE: ${{ steps.date.outputs.target_date }} run: | - if [ -n "${RUN_ID}" ] && ! [[ "${RUN_ID}" =~ ^[0-9]+$ ]]; then - echo "::error::run_id must be a positive integer, got: '${RUN_ID}'" - exit 1 + set -euo pipefail + BASE_URL="https://dev.azure.com/${AZDO_ORG}/${AZDO_PROJECT}/_apis" + + # Fetch recent succeeded builds from PerfStar-Scheduled (25429) + BUILDS=$(curl -sS --fail-with-body -H "Authorization: Bearer ${AZDO_TOKEN}" \ + "${BASE_URL}/build/builds?definitions=${PIPELINE_ID}&statusFilter=completed&resultFilter=succeeded&\$top=50&api-version=7.1") + + # Prefer the latest *scheduled* run whose startTime falls on TARGET_DATE + SCHEDULED_RUN=$(echo "$BUILDS" | jq -r --arg d "$TARGET_DATE" \ + '[.value[] | select(.startTime | startswith($d)) | select(.reason == "schedule")] + | sort_by(.startTime) | reverse | .[0].id // empty') + + if [ -n "$SCHEDULED_RUN" ]; then + echo "Found scheduled run $SCHEDULED_RUN for $TARGET_DATE" + echo "run_id=${SCHEDULED_RUN}" >> "$GITHUB_OUTPUT" + exit 0 fi - if ! [[ "${COUNT}" =~ ^[0-9]+$ ]] || [ "${COUNT}" -lt 1 ] || [ "${COUNT}" -gt 100 ]; then - echo "::error::count must be an integer between 1 and 100, got: '${COUNT}'" - exit 1 + + # Fall back to latest run of any trigger type on that date + ANY_RUN=$(echo "$BUILDS" | jq -r --arg d "$TARGET_DATE" \ + '[.value[] | select(.startTime | startswith($d))] + | sort_by(.startTime) | reverse | .[0].id // empty') + + if [ -n "$ANY_RUN" ]; then + echo "::warning::No scheduled run for $TARGET_DATE; using run $ANY_RUN" + echo "run_id=${ANY_RUN}" >> "$GITHUB_OUTPUT" + else + echo "::warning::No successful builds found for $TARGET_DATE on pipeline ${PIPELINE_ID}" + echo "run_id=" >> "$GITHUB_OUTPUT" fi - - name: Fetch Pipeline Runs - id: runs + # ── 5. Download & extract artifacts ──────────────────────────────── + - name: Download and extract artifacts + if: steps.find_run.outputs.run_id != '' + id: download env: AZDO_TOKEN: ${{ steps.token.outputs.azdo_token }} - PIPELINE_ID: ${{ steps.pipeline.outputs.id }} - RUN_ID: ${{ inputs.run_id || '' }} - COUNT: ${{ inputs.count || '10' }} + BUILD_ID: ${{ steps.find_run.outputs.run_id }} + TARGET_DATE: ${{ steps.date.outputs.target_date }} run: | + set -euo pipefail BASE_URL="https://dev.azure.com/${AZDO_ORG}/${AZDO_PROJECT}/_apis" - if [ -n "${RUN_ID}" ]; then - echo "## Fetching run ${RUN_ID} from ${{ steps.pipeline.outputs.name }}" >> "$GITHUB_STEP_SUMMARY" - RUN_DATA=$(curl -sS --fail-with-body -H "Authorization: Bearer ${AZDO_TOKEN}" \ - "${BASE_URL}/pipelines/${PIPELINE_ID}/runs/${RUN_ID}?api-version=7.1") - echo '```json' >> "$GITHUB_STEP_SUMMARY" - echo "${RUN_DATA}" | jq '{id, name, state, result, createdDate, finishedDate}' >> "$GITHUB_STEP_SUMMARY" - echo '```' >> "$GITHUB_STEP_SUMMARY" - echo "target_run_id=${RUN_ID}" >> "$GITHUB_OUTPUT" - else - echo "## Latest ${COUNT} runs from ${{ steps.pipeline.outputs.name }}" >> "$GITHUB_STEP_SUMMARY" - RUNS=$(curl -sS --fail-with-body -H "Authorization: Bearer ${AZDO_TOKEN}" \ - "${BASE_URL}/pipelines/${PIPELINE_ID}/runs?api-version=7.1") - echo '```' >> "$GITHUB_STEP_SUMMARY" - echo "${RUNS}" | jq -r ".value[:${COUNT}][] | \"\(.id) | \(.state) | \(.result) | \(.createdDate) | \(.name)\"" >> "$GITHUB_STEP_SUMMARY" - echo '```' >> "$GITHUB_STEP_SUMMARY" + for SUFFIX in GOLDWIN GOLDLIN; do + ARTIFACT="CrankAssetsThinned${SUFFIX}" + echo "::group::Downloading ${ARTIFACT}" - # Find latest successful run - LATEST_SUCCESS=$(echo "${RUNS}" | jq -r '[.value[] | select(.result == "succeeded")][0].id // empty') - if [ -n "${LATEST_SUCCESS}" ]; then - echo "Latest successful run: ${LATEST_SUCCESS}" - echo "target_run_id=${LATEST_SUCCESS}" >> "$GITHUB_OUTPUT" - else - echo "::warning::No successful runs found in recent history" - echo "target_run_id=" >> "$GITHUB_OUTPUT" + ARTIFACT_INFO=$(curl -sS --fail-with-body -H "Authorization: Bearer ${AZDO_TOKEN}" \ + "${BASE_URL}/build/builds/${BUILD_ID}/artifacts?artifactName=${ARTIFACT}&api-version=7.1") + + DOWNLOAD_URL=$(echo "$ARTIFACT_INFO" | jq -r '.resource.downloadUrl') + if [ -z "$DOWNLOAD_URL" ] || [ "$DOWNLOAD_URL" = "null" ]; then + echo "::error::Artifact ${ARTIFACT} not found in build ${BUILD_ID}" + exit 1 fi - fi - - name: List Artifacts - if: steps.runs.outputs.target_run_id != '' + curl -sS -L -H "Authorization: Bearer ${AZDO_TOKEN}" \ + "$DOWNLOAD_URL" -o "${ARTIFACT}.zip" + + mkdir -p "output/data/${TARGET_DATE}/${SUFFIX}" + + # Extract only top-level .json files — skip _manifest/ and subdirs + unzip -j -o "${ARTIFACT}.zip" "${ARTIFACT}/*.json" \ + -d "output/data/${TARGET_DATE}/${SUFFIX}/" + + rm -f "${ARTIFACT}.zip" + + COUNT=$(find "output/data/${TARGET_DATE}/${SUFFIX}" -maxdepth 1 -name '*.json' | wc -l) + echo "Extracted ${COUNT} JSON files for ${SUFFIX}" + echo "::endgroup::" + done + + echo "downloaded=true" >> "$GITHUB_OUTPUT" + + # Step summary + echo "## PerfStar data for ${TARGET_DATE}" >> "$GITHUB_STEP_SUMMARY" + echo "AzDO build: [${BUILD_ID}](https://dev.azure.com/${AZDO_ORG}/${AZDO_PROJECT}/_build/results?buildId=${BUILD_ID})" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + for SUFFIX in GOLDWIN GOLDLIN; do + echo "
${SUFFIX}" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + ls "output/data/${TARGET_DATE}/${SUFFIX}/" >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + echo "
" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + done + + # ── 6. Commit to the perf/dashboard branch ──────────────────────── + - name: Push to perf/dashboard branch + if: steps.download.outputs.downloaded == 'true' env: - AZDO_TOKEN: ${{ steps.token.outputs.azdo_token }} - BUILD_ID: ${{ steps.runs.outputs.target_run_id }} + TARGET_DATE: ${{ steps.date.outputs.target_date }} + GH_TOKEN: ${{ github.token }} run: | - BASE_URL="https://dev.azure.com/${AZDO_ORG}/${AZDO_PROJECT}/_apis" + set -euo pipefail - echo "## Artifacts for build ${BUILD_ID}" >> "$GITHUB_STEP_SUMMARY" - ARTIFACTS=$(curl -sS --fail-with-body -H "Authorization: Bearer ${AZDO_TOKEN}" \ - "${BASE_URL}/build/builds/${BUILD_ID}/artifacts?api-version=7.1") + git clone --depth 1 --branch "${{ env.DATA_BRANCH }}" \ + "https://x-access-token:${GH_TOKEN}@github.com/${{ github.repository }}.git" \ + perf-branch - echo '```' >> "$GITHUB_STEP_SUMMARY" - echo "${ARTIFACTS}" | jq -r '.value[] | "\(.name) (\(.resource.type))"' >> "$GITHUB_STEP_SUMMARY" - echo '```' >> "$GITHUB_STEP_SUMMARY" + cp -r "output/data/${TARGET_DATE}" "perf-branch/data/${TARGET_DATE}" - # Show Crank artifacts specifically (performance results) - CRANK_ARTIFACTS=$(echo "${ARTIFACTS}" | jq -r '.value[] | select(.name | startswith("CrankAssets")) | .name') - if [ -n "${CRANK_ARTIFACTS}" ]; then - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "### Performance result artifacts:" >> "$GITHUB_STEP_SUMMARY" - echo "${CRANK_ARTIFACTS}" | while read -r artifact_name; do - echo "- ${artifact_name}" >> "$GITHUB_STEP_SUMMARY" - done - fi + cd perf-branch + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add "data/${TARGET_DATE}" + git commit -m "perf: add ${TARGET_DATE} performance data" + git push - - name: Fetch Build Timeline (Stage/Job Status) - if: steps.runs.outputs.target_run_id != '' + # ── 7. Chain to the next date if needed ──────────────────────────── + - name: Trigger next date + if: "!failure() && !cancelled() && steps.date.outputs.skip != 'true'" env: - AZDO_TOKEN: ${{ steps.token.outputs.azdo_token }} - BUILD_ID: ${{ steps.runs.outputs.target_run_id }} + TARGET_DATE: ${{ steps.date.outputs.target_date }} + END_DATE: ${{ steps.date.outputs.end_date }} + TODAY: ${{ steps.date.outputs.today }} + GH_TOKEN: ${{ github.token }} run: | - BASE_URL="https://dev.azure.com/${AZDO_ORG}/${AZDO_PROJECT}/_apis" + # Only chain when we haven't reached today yet + if [[ "$TARGET_DATE" >= "$TODAY" ]]; then + echo "Reached today ($TODAY) — stopping chain" + exit 0 + fi - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "## Build Timeline Summary" >> "$GITHUB_STEP_SUMMARY" + NEXT_DATE=$(date -u -d "$TARGET_DATE + 1 day" +%Y-%m-%d) + + if [[ "$NEXT_DATE" > "$TODAY" ]]; then + echo "Next date $NEXT_DATE is in the future — stopping" + exit 0 + fi + + if [[ "$NEXT_DATE" > "$END_DATE" ]]; then + echo "Next date $NEXT_DATE is past end date $END_DATE — stopping" + exit 0 + fi + + echo "Dispatching next run: start_date=$NEXT_DATE end_date=$END_DATE" - TIMELINE=$(curl -sS --fail-with-body -H "Authorization: Bearer ${AZDO_TOKEN}" \ - "${BASE_URL}/build/builds/${BUILD_ID}/timeline?api-version=7.1") + REF="${{ github.event_name == 'pull_request' && github.head_ref || github.ref_name }}" - echo '```' >> "$GITHUB_STEP_SUMMARY" - echo "${TIMELINE}" | jq -r '.records[] | select(.type == "Stage") | "\(.name) | \(.state) | \(.result)"' >> "$GITHUB_STEP_SUMMARY" - echo '```' >> "$GITHUB_STEP_SUMMARY" + gh workflow run read-azdo-perfstar.yml \ + --ref "$REF" \ + -f "start_date=$NEXT_DATE" \ + -f "end_date=$END_DATE" \ + || echo "::warning::Could not dispatch next run — manual re-trigger may be needed" From 0d2e7d36c1ac81ce35c275ba50115e8c417d7d89 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Thu, 14 May 2026 14:20:17 +0200 Subject: [PATCH 10/20] Fix: use date-range query for AzDO builds, improve error handling --- .github/workflows/read-azdo-perfstar.yml | 35 ++++++++++++++++-------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/.github/workflows/read-azdo-perfstar.yml b/.github/workflows/read-azdo-perfstar.yml index 793d0b02615..3ab9d9f6b58 100644 --- a/.github/workflows/read-azdo-perfstar.yml +++ b/.github/workflows/read-azdo-perfstar.yml @@ -136,17 +136,29 @@ jobs: AZDO_TOKEN: ${{ steps.token.outputs.azdo_token }} TARGET_DATE: ${{ steps.date.outputs.target_date }} run: | - set -euo pipefail BASE_URL="https://dev.azure.com/${AZDO_ORG}/${AZDO_PROJECT}/_apis" + NEXT_DAY=$(date -u -d "$TARGET_DATE + 1 day" +%Y-%m-%d) + + # Query succeeded builds whose startTime falls on TARGET_DATE + QUERY="definitions=${PIPELINE_ID}" + QUERY+="&statusFilter=completed&resultFilter=succeeded" + QUERY+="&queryOrder=startTimeDescending" + QUERY+="&minTime=${TARGET_DATE}T00:00:00Z" + QUERY+="&maxTime=${NEXT_DAY}T00:00:00Z" + QUERY+="&api-version=7.1" - # Fetch recent succeeded builds from PerfStar-Scheduled (25429) BUILDS=$(curl -sS --fail-with-body -H "Authorization: Bearer ${AZDO_TOKEN}" \ - "${BASE_URL}/build/builds?definitions=${PIPELINE_ID}&statusFilter=completed&resultFilter=succeeded&\$top=50&api-version=7.1") + "${BASE_URL}/build/builds?${QUERY}") + + BUILD_COUNT=$(echo "$BUILDS" | jq '.count') + echo "Found $BUILD_COUNT succeeded build(s) for $TARGET_DATE" + + # Debug: list all builds found + echo "$BUILDS" | jq -r '.value[] | " id=\(.id) reason=\(.reason) start=\(.startTime) finish=\(.finishTime)"' - # Prefer the latest *scheduled* run whose startTime falls on TARGET_DATE - SCHEDULED_RUN=$(echo "$BUILDS" | jq -r --arg d "$TARGET_DATE" \ - '[.value[] | select(.startTime | startswith($d)) | select(.reason == "schedule")] - | sort_by(.startTime) | reverse | .[0].id // empty') + # Prefer the latest *scheduled* run + SCHEDULED_RUN=$(echo "$BUILDS" | jq -r \ + '[.value[] | select(.reason == "schedule")] | first | .id // empty') if [ -n "$SCHEDULED_RUN" ]; then echo "Found scheduled run $SCHEDULED_RUN for $TARGET_DATE" @@ -154,10 +166,8 @@ jobs: exit 0 fi - # Fall back to latest run of any trigger type on that date - ANY_RUN=$(echo "$BUILDS" | jq -r --arg d "$TARGET_DATE" \ - '[.value[] | select(.startTime | startswith($d))] - | sort_by(.startTime) | reverse | .[0].id // empty') + # Fall back to latest run of any trigger type + ANY_RUN=$(echo "$BUILDS" | jq -r '.value[0].id // empty') if [ -n "$ANY_RUN" ]; then echo "::warning::No scheduled run for $TARGET_DATE; using run $ANY_RUN" @@ -281,4 +291,5 @@ jobs: --ref "$REF" \ -f "start_date=$NEXT_DATE" \ -f "end_date=$END_DATE" \ - || echo "::warning::Could not dispatch next run — manual re-trigger may be needed" + && echo "Dispatched successfully" \ + || echo "::warning::Could not dispatch next run (may need manual re-trigger)" From aa24ec225a4965b2c2c3db87e56e0840219cd949 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Thu, 14 May 2026 14:21:55 +0200 Subject: [PATCH 11/20] Fix: remove unsupported API params, use larger top with client-side filtering --- .github/workflows/read-azdo-perfstar.yml | 43 ++++++++++++++---------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/.github/workflows/read-azdo-perfstar.yml b/.github/workflows/read-azdo-perfstar.yml index 3ab9d9f6b58..c273f8127bf 100644 --- a/.github/workflows/read-azdo-perfstar.yml +++ b/.github/workflows/read-azdo-perfstar.yml @@ -137,28 +137,33 @@ jobs: TARGET_DATE: ${{ steps.date.outputs.target_date }} run: | BASE_URL="https://dev.azure.com/${AZDO_ORG}/${AZDO_PROJECT}/_apis" - NEXT_DAY=$(date -u -d "$TARGET_DATE + 1 day" +%Y-%m-%d) - # Query succeeded builds whose startTime falls on TARGET_DATE - QUERY="definitions=${PIPELINE_ID}" - QUERY+="&statusFilter=completed&resultFilter=succeeded" - QUERY+="&queryOrder=startTimeDescending" - QUERY+="&minTime=${TARGET_DATE}T00:00:00Z" - QUERY+="&maxTime=${NEXT_DAY}T00:00:00Z" - QUERY+="&api-version=7.1" + # Query recent succeeded builds (large window to cover the target date) + BUILDS=$(curl -sS -H "Authorization: Bearer ${AZDO_TOKEN}" \ + "${BASE_URL}/build/builds?definitions=${PIPELINE_ID}&statusFilter=completed&resultFilter=succeeded&\$top=200&api-version=7.1") - BUILDS=$(curl -sS --fail-with-body -H "Authorization: Bearer ${AZDO_TOKEN}" \ - "${BASE_URL}/build/builds?${QUERY}") + HTTP_STATUS=$? + if [ $HTTP_STATUS -ne 0 ]; then + echo "::error::Failed to query AzDO builds API (curl exit $HTTP_STATUS)" + echo "$BUILDS" | head -20 + exit 1 + fi - BUILD_COUNT=$(echo "$BUILDS" | jq '.count') - echo "Found $BUILD_COUNT succeeded build(s) for $TARGET_DATE" + BUILD_COUNT=$(echo "$BUILDS" | jq '.count // 0') + echo "Total succeeded builds returned: $BUILD_COUNT" - # Debug: list all builds found - echo "$BUILDS" | jq -r '.value[] | " id=\(.id) reason=\(.reason) start=\(.startTime) finish=\(.finishTime)"' + # Filter to builds whose startTime falls on TARGET_DATE + MATCHING=$(echo "$BUILDS" | jq --arg d "$TARGET_DATE" \ + '[.value[] | select(.startTime | startswith($d))]') + MATCH_COUNT=$(echo "$MATCHING" | jq 'length') + echo "Builds starting on $TARGET_DATE: $MATCH_COUNT" + + # Debug: show matching builds + echo "$MATCHING" | jq -r '.[] | " id=\(.id) reason=\(.reason) start=\(.startTime)"' # Prefer the latest *scheduled* run - SCHEDULED_RUN=$(echo "$BUILDS" | jq -r \ - '[.value[] | select(.reason == "schedule")] | first | .id // empty') + SCHEDULED_RUN=$(echo "$MATCHING" | jq -r \ + '[.[] | select(.reason == "schedule")] | first | .id // empty') if [ -n "$SCHEDULED_RUN" ]; then echo "Found scheduled run $SCHEDULED_RUN for $TARGET_DATE" @@ -167,12 +172,16 @@ jobs: fi # Fall back to latest run of any trigger type - ANY_RUN=$(echo "$BUILDS" | jq -r '.value[0].id // empty') + ANY_RUN=$(echo "$MATCHING" | jq -r '.[0].id // empty') if [ -n "$ANY_RUN" ]; then echo "::warning::No scheduled run for $TARGET_DATE; using run $ANY_RUN" echo "run_id=${ANY_RUN}" >> "$GITHUB_OUTPUT" else + # Show the dates of the most recent builds for debugging + echo "::group::Recent build dates (for debugging)" + echo "$BUILDS" | jq -r '.value[:10][] | " id=\(.id) start=\(.startTime) reason=\(.reason)"' + echo "::endgroup::" echo "::warning::No successful builds found for $TARGET_DATE on pipeline ${PIPELINE_ID}" echo "run_id=" >> "$GITHUB_OUTPUT" fi From 55520231ee60bd347b49debe087b2d0bb550606c Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Thu, 14 May 2026 14:24:21 +0200 Subject: [PATCH 12/20] Debug: add step summary for build query, fix chain step fail-safety --- .github/workflows/read-azdo-perfstar.yml | 105 ++++++++++++----------- 1 file changed, 55 insertions(+), 50 deletions(-) diff --git a/.github/workflows/read-azdo-perfstar.yml b/.github/workflows/read-azdo-perfstar.yml index c273f8127bf..ad7391a72ed 100644 --- a/.github/workflows/read-azdo-perfstar.yml +++ b/.github/workflows/read-azdo-perfstar.yml @@ -138,53 +138,58 @@ jobs: run: | BASE_URL="https://dev.azure.com/${AZDO_ORG}/${AZDO_PROJECT}/_apis" - # Query recent succeeded builds (large window to cover the target date) - BUILDS=$(curl -sS -H "Authorization: Bearer ${AZDO_TOKEN}" \ - "${BASE_URL}/build/builds?definitions=${PIPELINE_ID}&statusFilter=completed&resultFilter=succeeded&\$top=200&api-version=7.1") - - HTTP_STATUS=$? - if [ $HTTP_STATUS -ne 0 ]; then - echo "::error::Failed to query AzDO builds API (curl exit $HTTP_STATUS)" - echo "$BUILDS" | head -20 - exit 1 - fi - - BUILD_COUNT=$(echo "$BUILDS" | jq '.count // 0') - echo "Total succeeded builds returned: $BUILD_COUNT" - - # Filter to builds whose startTime falls on TARGET_DATE - MATCHING=$(echo "$BUILDS" | jq --arg d "$TARGET_DATE" \ - '[.value[] | select(.startTime | startswith($d))]') - MATCH_COUNT=$(echo "$MATCHING" | jq 'length') - echo "Builds starting on $TARGET_DATE: $MATCH_COUNT" - - # Debug: show matching builds - echo "$MATCHING" | jq -r '.[] | " id=\(.id) reason=\(.reason) start=\(.startTime)"' + # Query recent builds (without result filter first, for debugging) + ALL_BUILDS=$(curl -sS -H "Authorization: Bearer ${AZDO_TOKEN}" \ + "${BASE_URL}/build/builds?definitions=${PIPELINE_ID}&statusFilter=completed&\$top=30&api-version=7.1" || true) - # Prefer the latest *scheduled* run - SCHEDULED_RUN=$(echo "$MATCHING" | jq -r \ - '[.[] | select(.reason == "schedule")] | first | .id // empty') + ALL_COUNT=$(echo "$ALL_BUILDS" | jq '.count // 0' 2>/dev/null || echo 0) - if [ -n "$SCHEDULED_RUN" ]; then - echo "Found scheduled run $SCHEDULED_RUN for $TARGET_DATE" - echo "run_id=${SCHEDULED_RUN}" >> "$GITHUB_OUTPUT" - exit 0 - fi - - # Fall back to latest run of any trigger type - ANY_RUN=$(echo "$MATCHING" | jq -r '.[0].id // empty') + # Write debug info to step summary so it's visible without login + echo "## Build Query Debug" >> "$GITHUB_STEP_SUMMARY" + echo "Target date: \`$TARGET_DATE\` Pipeline: \`${PIPELINE_ID}\`" >> "$GITHUB_STEP_SUMMARY" + echo "Total completed builds returned: $ALL_COUNT" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "### Recent builds (up to 15)" >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + echo "$ALL_BUILDS" | jq -r '.value[:15][] | "id=\(.id) result=\(.result) reason=\(.reason) queue=\(.queueTime) start=\(.startTime) finish=\(.finishTime)"' >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || echo "(failed to parse)" >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + + # Now filter for succeeded builds matching the target date + # Try matching by startTime, queueTime, and finishTime + for FIELD in startTime queueTime finishTime; do + MATCH=$(echo "$ALL_BUILDS" | jq -r --arg d "$TARGET_DATE" --arg f "$FIELD" \ + '[.value[] | select(.result == "succeeded") | select(.[$f] | startswith($d))]' 2>/dev/null || echo "[]") + MATCH_COUNT=$(echo "$MATCH" | jq 'length') + echo "Succeeded builds with $FIELD on $TARGET_DATE: $MATCH_COUNT" + + if [ "$MATCH_COUNT" -gt 0 ]; then + # Prefer scheduled runs + SCHEDULED_RUN=$(echo "$MATCH" | jq -r \ + '[.[] | select(.reason == "schedule")] | first | .id // empty') + + if [ -n "$SCHEDULED_RUN" ]; then + echo "Found scheduled run $SCHEDULED_RUN (matched by $FIELD)" + echo "run_id=${SCHEDULED_RUN}" >> "$GITHUB_OUTPUT" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "**Selected build: $SCHEDULED_RUN** (scheduled, matched by $FIELD)" >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + ANY_RUN=$(echo "$MATCH" | jq -r '.[0].id // empty') + if [ -n "$ANY_RUN" ]; then + echo "::warning::No scheduled run for $TARGET_DATE; using run $ANY_RUN (matched by $FIELD)" + echo "run_id=${ANY_RUN}" >> "$GITHUB_OUTPUT" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "**Selected build: $ANY_RUN** (non-scheduled, matched by $FIELD)" >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + fi + done - if [ -n "$ANY_RUN" ]; then - echo "::warning::No scheduled run for $TARGET_DATE; using run $ANY_RUN" - echo "run_id=${ANY_RUN}" >> "$GITHUB_OUTPUT" - else - # Show the dates of the most recent builds for debugging - echo "::group::Recent build dates (for debugging)" - echo "$BUILDS" | jq -r '.value[:10][] | " id=\(.id) start=\(.startTime) reason=\(.reason)"' - echo "::endgroup::" - echo "::warning::No successful builds found for $TARGET_DATE on pipeline ${PIPELINE_ID}" - echo "run_id=" >> "$GITHUB_OUTPUT" - fi + echo "::warning::No successful builds found for $TARGET_DATE on pipeline ${PIPELINE_ID}" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "**No matching build found for $TARGET_DATE**" >> "$GITHUB_STEP_SUMMARY" + echo "run_id=" >> "$GITHUB_OUTPUT" # ── 5. Download & extract artifacts ──────────────────────────────── - name: Download and extract artifacts @@ -296,9 +301,9 @@ jobs: REF="${{ github.event_name == 'pull_request' && github.head_ref || github.ref_name }}" - gh workflow run read-azdo-perfstar.yml \ - --ref "$REF" \ - -f "start_date=$NEXT_DATE" \ - -f "end_date=$END_DATE" \ - && echo "Dispatched successfully" \ - || echo "::warning::Could not dispatch next run (may need manual re-trigger)" + if gh workflow run read-azdo-perfstar.yml --ref "$REF" -f "start_date=$NEXT_DATE" -f "end_date=$END_DATE"; then + echo "Dispatched successfully" + else + echo "::warning::Could not dispatch next run (exit $?) — may need manual re-trigger" + fi + exit 0 From 5d00150ec4d1e09b0d106f8b43eb30fe94877cf2 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Thu, 14 May 2026 14:27:58 +0200 Subject: [PATCH 13/20] Debug: disable chain step, add exit 0 to find_run, fix jq fallbacks --- .github/workflows/read-azdo-perfstar.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/read-azdo-perfstar.yml b/.github/workflows/read-azdo-perfstar.yml index ad7391a72ed..68179165742 100644 --- a/.github/workflows/read-azdo-perfstar.yml +++ b/.github/workflows/read-azdo-perfstar.yml @@ -159,7 +159,7 @@ jobs: for FIELD in startTime queueTime finishTime; do MATCH=$(echo "$ALL_BUILDS" | jq -r --arg d "$TARGET_DATE" --arg f "$FIELD" \ '[.value[] | select(.result == "succeeded") | select(.[$f] | startswith($d))]' 2>/dev/null || echo "[]") - MATCH_COUNT=$(echo "$MATCH" | jq 'length') + MATCH_COUNT=$(echo "$MATCH" | jq 'length' 2>/dev/null || echo 0) echo "Succeeded builds with $FIELD on $TARGET_DATE: $MATCH_COUNT" if [ "$MATCH_COUNT" -gt 0 ]; then @@ -190,6 +190,7 @@ jobs: echo "" >> "$GITHUB_STEP_SUMMARY" echo "**No matching build found for $TARGET_DATE**" >> "$GITHUB_STEP_SUMMARY" echo "run_id=" >> "$GITHUB_OUTPUT" + exit 0 # ── 5. Download & extract artifacts ──────────────────────────────── - name: Download and extract artifacts @@ -272,7 +273,7 @@ jobs: # ── 7. Chain to the next date if needed ──────────────────────────── - name: Trigger next date - if: "!failure() && !cancelled() && steps.date.outputs.skip != 'true'" + if: "false" # Temporarily disabled for debugging env: TARGET_DATE: ${{ steps.date.outputs.target_date }} END_DATE: ${{ steps.date.outputs.end_date }} From 76e02edf273b424a8050297bb0853bf18ef6ebb8 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Thu, 14 May 2026 14:46:24 +0200 Subject: [PATCH 14/20] Switch to Pipelines API for build discovery (known to work from run #4) --- .github/workflows/read-azdo-perfstar.yml | 82 +++++++++++------------- 1 file changed, 39 insertions(+), 43 deletions(-) diff --git a/.github/workflows/read-azdo-perfstar.yml b/.github/workflows/read-azdo-perfstar.yml index 68179165742..0316229188d 100644 --- a/.github/workflows/read-azdo-perfstar.yml +++ b/.github/workflows/read-azdo-perfstar.yml @@ -138,58 +138,54 @@ jobs: run: | BASE_URL="https://dev.azure.com/${AZDO_ORG}/${AZDO_PROJECT}/_apis" - # Query recent builds (without result filter first, for debugging) - ALL_BUILDS=$(curl -sS -H "Authorization: Bearer ${AZDO_TOKEN}" \ - "${BASE_URL}/build/builds?definitions=${PIPELINE_ID}&statusFilter=completed&\$top=30&api-version=7.1" || true) + # Use the Pipelines API (known to work from run #4) + RUNS=$(curl -sS -H "Authorization: Bearer ${AZDO_TOKEN}" \ + "${BASE_URL}/pipelines/${PIPELINE_ID}/runs?api-version=7.1" || true) - ALL_COUNT=$(echo "$ALL_BUILDS" | jq '.count // 0' 2>/dev/null || echo 0) + RUN_COUNT=$(echo "$RUNS" | jq '.count // 0' 2>/dev/null || echo 0) - # Write debug info to step summary so it's visible without login echo "## Build Query Debug" >> "$GITHUB_STEP_SUMMARY" echo "Target date: \`$TARGET_DATE\` Pipeline: \`${PIPELINE_ID}\`" >> "$GITHUB_STEP_SUMMARY" - echo "Total completed builds returned: $ALL_COUNT" >> "$GITHUB_STEP_SUMMARY" + echo "Total pipeline runs returned: $RUN_COUNT" >> "$GITHUB_STEP_SUMMARY" echo "" >> "$GITHUB_STEP_SUMMARY" - echo "### Recent builds (up to 15)" >> "$GITHUB_STEP_SUMMARY" + echo "### Recent runs (up to 15)" >> "$GITHUB_STEP_SUMMARY" echo '```' >> "$GITHUB_STEP_SUMMARY" - echo "$ALL_BUILDS" | jq -r '.value[:15][] | "id=\(.id) result=\(.result) reason=\(.reason) queue=\(.queueTime) start=\(.startTime) finish=\(.finishTime)"' >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || echo "(failed to parse)" >> "$GITHUB_STEP_SUMMARY" + echo "$RUNS" | jq -r '.value[:15][] | "id=\(.id) state=\(.state) result=\(.result) created=\(.createdDate) finished=\(.finishedDate)"' >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || echo "(failed to parse)" >> "$GITHUB_STEP_SUMMARY" echo '```' >> "$GITHUB_STEP_SUMMARY" - # Now filter for succeeded builds matching the target date - # Try matching by startTime, queueTime, and finishTime - for FIELD in startTime queueTime finishTime; do - MATCH=$(echo "$ALL_BUILDS" | jq -r --arg d "$TARGET_DATE" --arg f "$FIELD" \ - '[.value[] | select(.result == "succeeded") | select(.[$f] | startswith($d))]' 2>/dev/null || echo "[]") - MATCH_COUNT=$(echo "$MATCH" | jq 'length' 2>/dev/null || echo 0) - echo "Succeeded builds with $FIELD on $TARGET_DATE: $MATCH_COUNT" - - if [ "$MATCH_COUNT" -gt 0 ]; then - # Prefer scheduled runs - SCHEDULED_RUN=$(echo "$MATCH" | jq -r \ - '[.[] | select(.reason == "schedule")] | first | .id // empty') - - if [ -n "$SCHEDULED_RUN" ]; then - echo "Found scheduled run $SCHEDULED_RUN (matched by $FIELD)" - echo "run_id=${SCHEDULED_RUN}" >> "$GITHUB_OUTPUT" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "**Selected build: $SCHEDULED_RUN** (scheduled, matched by $FIELD)" >> "$GITHUB_STEP_SUMMARY" - exit 0 - fi - - ANY_RUN=$(echo "$MATCH" | jq -r '.[0].id // empty') - if [ -n "$ANY_RUN" ]; then - echo "::warning::No scheduled run for $TARGET_DATE; using run $ANY_RUN (matched by $FIELD)" - echo "run_id=${ANY_RUN}" >> "$GITHUB_OUTPUT" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "**Selected build: $ANY_RUN** (non-scheduled, matched by $FIELD)" >> "$GITHUB_STEP_SUMMARY" - exit 0 - fi + # Filter for succeeded runs whose createdDate falls on TARGET_DATE + # Prefer scheduled runs (these have no triggering pipeline) + MATCHING=$(echo "$RUNS" | jq --arg d "$TARGET_DATE" \ + '[.value[] | select(.result == "succeeded") | select(.createdDate | startswith($d))]' 2>/dev/null || echo "[]") + MATCH_COUNT=$(echo "$MATCHING" | jq 'length' 2>/dev/null || echo 0) + echo "Succeeded runs on $TARGET_DATE: $MATCH_COUNT" + + if [ "$MATCH_COUNT" -gt 0 ]; then + # The Pipelines API run .id is the same as the Build API buildId + BEST_RUN=$(echo "$MATCHING" | jq -r 'last | .id // empty') + echo "Selected run: $BEST_RUN" + echo "run_id=${BEST_RUN}" >> "$GITHUB_OUTPUT" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "**Selected run: $BEST_RUN**" >> "$GITHUB_STEP_SUMMARY" + else + # Try finishedDate as well + MATCHING_FIN=$(echo "$RUNS" | jq --arg d "$TARGET_DATE" \ + '[.value[] | select(.result == "succeeded") | select(.finishedDate | startswith($d))]' 2>/dev/null || echo "[]") + FIN_COUNT=$(echo "$MATCHING_FIN" | jq 'length' 2>/dev/null || echo 0) + + if [ "$FIN_COUNT" -gt 0 ]; then + BEST_RUN=$(echo "$MATCHING_FIN" | jq -r 'last | .id // empty') + echo "Selected run (matched by finishedDate): $BEST_RUN" + echo "run_id=${BEST_RUN}" >> "$GITHUB_OUTPUT" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "**Selected run: $BEST_RUN** (matched by finishedDate)" >> "$GITHUB_STEP_SUMMARY" + else + echo "::warning::No successful builds found for $TARGET_DATE on pipeline ${PIPELINE_ID}" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "**No matching build found for $TARGET_DATE**" >> "$GITHUB_STEP_SUMMARY" + echo "run_id=" >> "$GITHUB_OUTPUT" fi - done - - echo "::warning::No successful builds found for $TARGET_DATE on pipeline ${PIPELINE_ID}" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "**No matching build found for $TARGET_DATE**" >> "$GITHUB_STEP_SUMMARY" - echo "run_id=" >> "$GITHUB_OUTPUT" + fi exit 0 # ── 5. Download & extract artifacts ──────────────────────────────── From 862869a1bf6bc386ec9370aba71f61719ed4027e Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Thu, 14 May 2026 14:48:32 +0200 Subject: [PATCH 15/20] Debug: increase to 100, add unique dates to summary --- .github/workflows/read-azdo-perfstar.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/read-azdo-perfstar.yml b/.github/workflows/read-azdo-perfstar.yml index 0316229188d..197eb21a848 100644 --- a/.github/workflows/read-azdo-perfstar.yml +++ b/.github/workflows/read-azdo-perfstar.yml @@ -139,8 +139,9 @@ jobs: BASE_URL="https://dev.azure.com/${AZDO_ORG}/${AZDO_PROJECT}/_apis" # Use the Pipelines API (known to work from run #4) + # Request more runs to cover several days of history RUNS=$(curl -sS -H "Authorization: Bearer ${AZDO_TOKEN}" \ - "${BASE_URL}/pipelines/${PIPELINE_ID}/runs?api-version=7.1" || true) + "${BASE_URL}/pipelines/${PIPELINE_ID}/runs?api-version=7.1&\$top=100" || true) RUN_COUNT=$(echo "$RUNS" | jq '.count // 0' 2>/dev/null || echo 0) @@ -148,9 +149,14 @@ jobs: echo "Target date: \`$TARGET_DATE\` Pipeline: \`${PIPELINE_ID}\`" >> "$GITHUB_STEP_SUMMARY" echo "Total pipeline runs returned: $RUN_COUNT" >> "$GITHUB_STEP_SUMMARY" echo "" >> "$GITHUB_STEP_SUMMARY" - echo "### Recent runs (up to 15)" >> "$GITHUB_STEP_SUMMARY" + echo "### Recent runs (up to 30)" >> "$GITHUB_STEP_SUMMARY" echo '```' >> "$GITHUB_STEP_SUMMARY" - echo "$RUNS" | jq -r '.value[:15][] | "id=\(.id) state=\(.state) result=\(.result) created=\(.createdDate) finished=\(.finishedDate)"' >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || echo "(failed to parse)" >> "$GITHUB_STEP_SUMMARY" + echo "$RUNS" | jq -r '.value[:30][] | "id=\(.id) state=\(.state) result=\(.result) created=\(.createdDate) finished=\(.finishedDate) name=\(.name)"' >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || echo "(failed to parse)" >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "### Unique dates in runs" >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + echo "$RUNS" | jq -r '[.value[] | .createdDate[:10]] | unique | .[]' >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || echo "(failed)" >> "$GITHUB_STEP_SUMMARY" echo '```' >> "$GITHUB_STEP_SUMMARY" # Filter for succeeded runs whose createdDate falls on TARGET_DATE From 78d42cbb071d5f04871f134c585a47a32ba612f6 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Thu, 14 May 2026 15:00:56 +0200 Subject: [PATCH 16/20] Debug: echo pipeline runs to stdout for log visibility --- .github/workflows/read-azdo-perfstar.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/read-azdo-perfstar.yml b/.github/workflows/read-azdo-perfstar.yml index 197eb21a848..4bf16882f7c 100644 --- a/.github/workflows/read-azdo-perfstar.yml +++ b/.github/workflows/read-azdo-perfstar.yml @@ -144,6 +144,7 @@ jobs: "${BASE_URL}/pipelines/${PIPELINE_ID}/runs?api-version=7.1&\$top=100" || true) RUN_COUNT=$(echo "$RUNS" | jq '.count // 0' 2>/dev/null || echo 0) + echo "Total pipeline runs returned: $RUN_COUNT" echo "## Build Query Debug" >> "$GITHUB_STEP_SUMMARY" echo "Target date: \`$TARGET_DATE\` Pipeline: \`${PIPELINE_ID}\`" >> "$GITHUB_STEP_SUMMARY" @@ -151,12 +152,13 @@ jobs: echo "" >> "$GITHUB_STEP_SUMMARY" echo "### Recent runs (up to 30)" >> "$GITHUB_STEP_SUMMARY" echo '```' >> "$GITHUB_STEP_SUMMARY" - echo "$RUNS" | jq -r '.value[:30][] | "id=\(.id) state=\(.state) result=\(.result) created=\(.createdDate) finished=\(.finishedDate) name=\(.name)"' >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || echo "(failed to parse)" >> "$GITHUB_STEP_SUMMARY" + echo "$RUNS" | jq -r '.value[:30][] | "id=\(.id) state=\(.state) result=\(.result) created=\(.createdDate) finished=\(.finishedDate) name=\(.name)"' 2>/dev/null | tee -a "$GITHUB_STEP_SUMMARY" || echo "(failed to parse)" | tee -a "$GITHUB_STEP_SUMMARY" echo '```' >> "$GITHUB_STEP_SUMMARY" echo "" >> "$GITHUB_STEP_SUMMARY" echo "### Unique dates in runs" >> "$GITHUB_STEP_SUMMARY" echo '```' >> "$GITHUB_STEP_SUMMARY" - echo "$RUNS" | jq -r '[.value[] | .createdDate[:10]] | unique | .[]' >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || echo "(failed)" >> "$GITHUB_STEP_SUMMARY" + UNIQUE_DATES=$(echo "$RUNS" | jq -r '[.value[] | .createdDate[:10]] | unique | .[]' 2>/dev/null || echo "(failed)") + echo "$UNIQUE_DATES" | tee -a "$GITHUB_STEP_SUMMARY" echo '```' >> "$GITHUB_STEP_SUMMARY" # Filter for succeeded runs whose createdDate falls on TARGET_DATE From 68c28e2de038c16c6e79ddc6b0bdc8362eb6b01b Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Thu, 14 May 2026 15:02:58 +0200 Subject: [PATCH 17/20] Accept failed runs (PerfStar produces artifacts even on failure) --- .github/workflows/read-azdo-perfstar.yml | 51 +++++++++++------------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/.github/workflows/read-azdo-perfstar.yml b/.github/workflows/read-azdo-perfstar.yml index 4bf16882f7c..42009eefc7e 100644 --- a/.github/workflows/read-azdo-perfstar.yml +++ b/.github/workflows/read-azdo-perfstar.yml @@ -161,38 +161,27 @@ jobs: echo "$UNIQUE_DATES" | tee -a "$GITHUB_STEP_SUMMARY" echo '```' >> "$GITHUB_STEP_SUMMARY" - # Filter for succeeded runs whose createdDate falls on TARGET_DATE - # Prefer scheduled runs (these have no triggering pipeline) + # Filter for completed runs whose createdDate falls on TARGET_DATE. + # PerfStar runs often report result=failed even when artifacts are produced, + # so we accept any completed run and let the artifact download step fail gracefully. MATCHING=$(echo "$RUNS" | jq --arg d "$TARGET_DATE" \ - '[.value[] | select(.result == "succeeded") | select(.createdDate | startswith($d))]' 2>/dev/null || echo "[]") + '[.value[] | select(.state == "completed") | select(.createdDate | startswith($d))]' 2>/dev/null || echo "[]") MATCH_COUNT=$(echo "$MATCHING" | jq 'length' 2>/dev/null || echo 0) - echo "Succeeded runs on $TARGET_DATE: $MATCH_COUNT" + echo "Completed runs on $TARGET_DATE: $MATCH_COUNT" if [ "$MATCH_COUNT" -gt 0 ]; then - # The Pipelines API run .id is the same as the Build API buildId + # Pick the latest completed run for the date BEST_RUN=$(echo "$MATCHING" | jq -r 'last | .id // empty') - echo "Selected run: $BEST_RUN" + BEST_RESULT=$(echo "$MATCHING" | jq -r 'last | .result // "unknown"') + echo "Selected run: $BEST_RUN (result=$BEST_RESULT)" echo "run_id=${BEST_RUN}" >> "$GITHUB_OUTPUT" echo "" >> "$GITHUB_STEP_SUMMARY" - echo "**Selected run: $BEST_RUN**" >> "$GITHUB_STEP_SUMMARY" + echo "**Selected run: $BEST_RUN** (result=$BEST_RESULT)" >> "$GITHUB_STEP_SUMMARY" else - # Try finishedDate as well - MATCHING_FIN=$(echo "$RUNS" | jq --arg d "$TARGET_DATE" \ - '[.value[] | select(.result == "succeeded") | select(.finishedDate | startswith($d))]' 2>/dev/null || echo "[]") - FIN_COUNT=$(echo "$MATCHING_FIN" | jq 'length' 2>/dev/null || echo 0) - - if [ "$FIN_COUNT" -gt 0 ]; then - BEST_RUN=$(echo "$MATCHING_FIN" | jq -r 'last | .id // empty') - echo "Selected run (matched by finishedDate): $BEST_RUN" - echo "run_id=${BEST_RUN}" >> "$GITHUB_OUTPUT" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "**Selected run: $BEST_RUN** (matched by finishedDate)" >> "$GITHUB_STEP_SUMMARY" - else - echo "::warning::No successful builds found for $TARGET_DATE on pipeline ${PIPELINE_ID}" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "**No matching build found for $TARGET_DATE**" >> "$GITHUB_STEP_SUMMARY" - echo "run_id=" >> "$GITHUB_OUTPUT" - fi + echo "::warning::No completed builds found for $TARGET_DATE on pipeline ${PIPELINE_ID}" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "**No matching build found for $TARGET_DATE**" >> "$GITHUB_STEP_SUMMARY" + echo "run_id=" >> "$GITHUB_OUTPUT" fi exit 0 @@ -208,17 +197,19 @@ jobs: set -euo pipefail BASE_URL="https://dev.azure.com/${AZDO_ORG}/${AZDO_PROJECT}/_apis" + ARTIFACT_COUNT=0 for SUFFIX in GOLDWIN GOLDLIN; do ARTIFACT="CrankAssetsThinned${SUFFIX}" echo "::group::Downloading ${ARTIFACT}" - ARTIFACT_INFO=$(curl -sS --fail-with-body -H "Authorization: Bearer ${AZDO_TOKEN}" \ + ARTIFACT_INFO=$(curl -sS -H "Authorization: Bearer ${AZDO_TOKEN}" \ "${BASE_URL}/build/builds/${BUILD_ID}/artifacts?artifactName=${ARTIFACT}&api-version=7.1") DOWNLOAD_URL=$(echo "$ARTIFACT_INFO" | jq -r '.resource.downloadUrl') if [ -z "$DOWNLOAD_URL" ] || [ "$DOWNLOAD_URL" = "null" ]; then - echo "::error::Artifact ${ARTIFACT} not found in build ${BUILD_ID}" - exit 1 + echo "::warning::Artifact ${ARTIFACT} not found in build ${BUILD_ID} — skipping" + echo "::endgroup::" + continue fi curl -sS -L -H "Authorization: Bearer ${AZDO_TOKEN}" \ @@ -234,9 +225,15 @@ jobs: COUNT=$(find "output/data/${TARGET_DATE}/${SUFFIX}" -maxdepth 1 -name '*.json' | wc -l) echo "Extracted ${COUNT} JSON files for ${SUFFIX}" + ARTIFACT_COUNT=$((ARTIFACT_COUNT + 1)) echo "::endgroup::" done + if [ "$ARTIFACT_COUNT" -eq 0 ]; then + echo "::error::No artifacts found in build ${BUILD_ID} — nothing to store" + exit 1 + fi + echo "downloaded=true" >> "$GITHUB_OUTPUT" # Step summary From d182f61a537305a65626c9b284b1698da5add407 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Thu, 14 May 2026 15:05:05 +0200 Subject: [PATCH 18/20] Clean up: remove debug output, re-enable chaining, pick latest run --- .github/workflows/read-azdo-perfstar.yml | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/.github/workflows/read-azdo-perfstar.yml b/.github/workflows/read-azdo-perfstar.yml index 42009eefc7e..4e025cefd7e 100644 --- a/.github/workflows/read-azdo-perfstar.yml +++ b/.github/workflows/read-azdo-perfstar.yml @@ -146,21 +146,6 @@ jobs: RUN_COUNT=$(echo "$RUNS" | jq '.count // 0' 2>/dev/null || echo 0) echo "Total pipeline runs returned: $RUN_COUNT" - echo "## Build Query Debug" >> "$GITHUB_STEP_SUMMARY" - echo "Target date: \`$TARGET_DATE\` Pipeline: \`${PIPELINE_ID}\`" >> "$GITHUB_STEP_SUMMARY" - echo "Total pipeline runs returned: $RUN_COUNT" >> "$GITHUB_STEP_SUMMARY" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "### Recent runs (up to 30)" >> "$GITHUB_STEP_SUMMARY" - echo '```' >> "$GITHUB_STEP_SUMMARY" - echo "$RUNS" | jq -r '.value[:30][] | "id=\(.id) state=\(.state) result=\(.result) created=\(.createdDate) finished=\(.finishedDate) name=\(.name)"' 2>/dev/null | tee -a "$GITHUB_STEP_SUMMARY" || echo "(failed to parse)" | tee -a "$GITHUB_STEP_SUMMARY" - echo '```' >> "$GITHUB_STEP_SUMMARY" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "### Unique dates in runs" >> "$GITHUB_STEP_SUMMARY" - echo '```' >> "$GITHUB_STEP_SUMMARY" - UNIQUE_DATES=$(echo "$RUNS" | jq -r '[.value[] | .createdDate[:10]] | unique | .[]' 2>/dev/null || echo "(failed)") - echo "$UNIQUE_DATES" | tee -a "$GITHUB_STEP_SUMMARY" - echo '```' >> "$GITHUB_STEP_SUMMARY" - # Filter for completed runs whose createdDate falls on TARGET_DATE. # PerfStar runs often report result=failed even when artifacts are produced, # so we accept any completed run and let the artifact download step fail gracefully. @@ -170,9 +155,9 @@ jobs: echo "Completed runs on $TARGET_DATE: $MATCH_COUNT" if [ "$MATCH_COUNT" -gt 0 ]; then - # Pick the latest completed run for the date - BEST_RUN=$(echo "$MATCHING" | jq -r 'last | .id // empty') - BEST_RESULT=$(echo "$MATCHING" | jq -r 'last | .result // "unknown"') + # Pick the latest completed run for the date (API returns newest first) + BEST_RUN=$(echo "$MATCHING" | jq -r 'first | .id // empty') + BEST_RESULT=$(echo "$MATCHING" | jq -r 'first | .result // "unknown"') echo "Selected run: $BEST_RUN (result=$BEST_RESULT)" echo "run_id=${BEST_RUN}" >> "$GITHUB_OUTPUT" echo "" >> "$GITHUB_STEP_SUMMARY" @@ -274,7 +259,7 @@ jobs: # ── 7. Chain to the next date if needed ──────────────────────────── - name: Trigger next date - if: "false" # Temporarily disabled for debugging + if: steps.download.outputs.downloaded == 'true' env: TARGET_DATE: ${{ steps.date.outputs.target_date }} END_DATE: ${{ steps.date.outputs.end_date }} From eb92b9b5e34a6f748f1b54b34d01352787a70161 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Thu, 14 May 2026 17:16:19 +0200 Subject: [PATCH 19/20] Prefer scheduled runs via Build API reason; fix chain step condition --- .github/workflows/read-azdo-perfstar.yml | 30 ++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/.github/workflows/read-azdo-perfstar.yml b/.github/workflows/read-azdo-perfstar.yml index 4e025cefd7e..0c7781f0d59 100644 --- a/.github/workflows/read-azdo-perfstar.yml +++ b/.github/workflows/read-azdo-perfstar.yml @@ -155,9 +155,24 @@ jobs: echo "Completed runs on $TARGET_DATE: $MATCH_COUNT" if [ "$MATCH_COUNT" -gt 0 ]; then - # Pick the latest completed run for the date (API returns newest first) - BEST_RUN=$(echo "$MATCHING" | jq -r 'first | .id // empty') - BEST_RESULT=$(echo "$MATCHING" | jq -r 'first | .result // "unknown"') + # When multiple runs exist, prefer scheduled runs over manual/CI triggers. + # The Pipelines API doesn't expose 'reason', so query the Build API. + IDS=$(echo "$MATCHING" | jq -r '[.[].id | tostring] | join(",")') + BUILD_DETAILS=$(curl -sS -H "Authorization: Bearer ${AZDO_TOKEN}" \ + "${BASE_URL}/build/builds?buildIds=${IDS}&api-version=7.1" 2>/dev/null || echo '{}') + + SCHEDULED_ID=$(echo "$BUILD_DETAILS" | jq -r \ + '[.value[] | select(.reason == "schedule")] | sort_by(.id) | last | .id // empty' 2>/dev/null || echo '') + + if [ -n "$SCHEDULED_ID" ]; then + BEST_RUN="$SCHEDULED_ID" + BEST_RESULT=$(echo "$MATCHING" | jq -r --argjson id "$SCHEDULED_ID" '.[] | select(.id == $id) | .result // "unknown"') + echo "Preferred scheduled run: $BEST_RUN (result=$BEST_RESULT)" + else + BEST_RUN=$(echo "$MATCHING" | jq -r 'first | .id // empty') + BEST_RESULT=$(echo "$MATCHING" | jq -r 'first | .result // "unknown"') + echo "No scheduled run found, using latest: $BEST_RUN (result=$BEST_RESULT)" + fi echo "Selected run: $BEST_RUN (result=$BEST_RESULT)" echo "run_id=${BEST_RUN}" >> "$GITHUB_OUTPUT" echo "" >> "$GITHUB_STEP_SUMMARY" @@ -259,13 +274,20 @@ jobs: # ── 7. Chain to the next date if needed ──────────────────────────── - name: Trigger next date - if: steps.download.outputs.downloaded == 'true' + if: steps.date.outputs.skip != 'true' && steps.check.outputs.exists != 'true' env: TARGET_DATE: ${{ steps.date.outputs.target_date }} END_DATE: ${{ steps.date.outputs.end_date }} TODAY: ${{ steps.date.outputs.today }} + DOWNLOADED: ${{ steps.download.outputs.downloaded }} GH_TOKEN: ${{ github.token }} run: | + # If nothing was downloaded (no builds/artifacts), don't chain + if [ "$DOWNLOADED" != "true" ]; then + echo "No data downloaded for $TARGET_DATE — not chaining" + exit 0 + fi + # Only chain when we haven't reached today yet if [[ "$TARGET_DATE" >= "$TODAY" ]]; then echo "Reached today ($TODAY) — stopping chain" From 422977c234a869be0301340e94ac494e5c8cf022 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Thu, 14 May 2026 17:18:09 +0200 Subject: [PATCH 20/20] Fix bash syntax: >= not valid in [[ ]], use ! < instead --- .github/workflows/read-azdo-perfstar.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/read-azdo-perfstar.yml b/.github/workflows/read-azdo-perfstar.yml index 0c7781f0d59..d2dd0f9cb3d 100644 --- a/.github/workflows/read-azdo-perfstar.yml +++ b/.github/workflows/read-azdo-perfstar.yml @@ -289,7 +289,7 @@ jobs: fi # Only chain when we haven't reached today yet - if [[ "$TARGET_DATE" >= "$TODAY" ]]; then + if [[ ! "$TARGET_DATE" < "$TODAY" ]]; then echo "Reached today ($TODAY) — stopping chain" exit 0 fi