Update CI to label-driven testing

.github/workflows/rocm-ci-dispatch.yml

@@ -0,0 +1,133 @@
+# Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved.
+#
+# See LICENSE for license information.
+
+name: PR Automatic CI
+
+on:
+  pull_request:
+    branches:
+      - 'dev'
+      - 'release_v2.*_rocm'
+    types: [ labeled, synchronize, reopened ]
+
+permissions:
+  contents: read
+  actions: write
+
+jobs:
+  determine_level:
+    runs-on: ubuntu-latest
+    outputs:
+      test_level: ${{ steps.set_level.outputs.test_level }}
+    steps:
+      - name: Determine CI dispatch from labels
+        id: set_level
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const parseLevelLabel = (labelName) => {
+              const label = (labelName || '').toLowerCase();
+              if (label === 'ci-level 3') return 3;
+              if (label === 'ci-level 2') return 2;
+              if (label === 'ci-level 1') return 1;
+              return 0;
+            };
+
+            const labels = (context.payload.pull_request.labels || [])
+              .map(label => parseLevelLabel(label.name));
+            const action = context.payload.action;
+            const currentLevel = labels.length ? Math.max(...labels) : 0;
+
+            // Determine if a CI level label was added, and what level it was.
+            const addedLevel = action === 'labeled' ? parseLevelLabel(context.payload.label?.name) : 0;
+            if(addedLevel > 0){
+              core.info(`Dispatch debug: action=${action}, addedLabel=${context.payload.label.name}, addedLevel=${addedLevel}, currentLevel=${currentLevel}, labels=[${labels.join(',')}]`);
+            }
+
+            let requiresDispatch = false;
+            if (action === 'labeled') {
+              // Only dispatch when the added CI-level label is now the highest level.
+              // Adding a lower-level label should not trigger a new run.
+              requiresDispatch = addedLevel > 0 && addedLevel === currentLevel;
+              core.info(`Dispatch debug: initial labeled decision requiresDispatch=${requiresDispatch}`);
+
+              // If an existing run for this PR/commit already satisfies the newly
+              // added level (same or higher), reuse it and do not dispatch again.
+              if (requiresDispatch) {
+                const prNumber = context.payload.pull_request.number;
+                const headSha = context.payload.pull_request.head.sha;
+                const owner = context.repo.owner;
+                const repo = context.repo.repo;
+
+                core.info(`Dispatch debug: checking existing runs for PR #${prNumber}, headSha=${headSha}`);
+
+                const runs = await github.paginate(github.rest.actions.listWorkflowRuns, {
+                  owner,
+                  repo,
+                  workflow_id: 'rocm-ci-dispatch.yml',
+                  per_page: 100,
+                  head_sha: headSha,
+                });
+
+                core.info(`Dispatch debug: total workflow runs fetched=${runs.length}`);
+
+                const candidateRuns = runs.filter(run => {
+                  const prMatch = (run.pull_requests || []).some(pr => pr.number === prNumber);
+                  const completedNotSuccess = run.status === 'completed' && run.conclusion !== 'success';
+                  return prMatch && !completedNotSuccess;
+                });
+
+                core.info(`Dispatch debug: candidate runs matching PR+sha=${candidateRuns.length}`);
+
+                let satisfiesAddedLevel = false;
+                for (const run of candidateRuns) {
+                  let detectedLevel = 0;
+
+                  // Inspect called-workflow job names, which include level.
+                  const jobsResp = await github.rest.actions.listJobsForWorkflowRun({
+                    owner,
+                    repo,
+                    run_id: run.id,
+                    per_page: 100,
+                  });
+
+                  for (const job of jobsResp.data.jobs || []) {
+                    const m = (job.name || '').match(/(?:CI\s+)?Level\s*(\d+)/i);
+                    if (m) {
+                      detectedLevel = Number(m[1]);
+                      core.info(`Dispatch debug: run_id=${run.id}, job_id=${job.id}, job_name=${job.name}, detectedLevel=${detectedLevel}, threshold=${addedLevel}`);
+                      break;
+                    }
+                  }
+
+                  if (detectedLevel >= addedLevel) {
+                    satisfiesAddedLevel = true;
+                    core.info(`Dispatch debug: run_id=${run.id} satisfies added level; skipping new dispatch`);
+                    break;
+                  }
+                }
+
+                if (satisfiesAddedLevel) {
+                  requiresDispatch = false;
+                }
+                core.info(`Dispatch debug: post-existing-run check requiresDispatch=${requiresDispatch}`);
+              }
+            } else if (action === 'synchronize' || action === 'reopened') {
+              requiresDispatch = currentLevel > 0;
+            }
+            core.info(`Dispatch debug: final requiresDispatch=${requiresDispatch}, output test_level=${requiresDispatch ? String(currentLevel) : ''}`);
+            core.setOutput('test_level', requiresDispatch ? String(currentLevel) : '');
+
+  dispatch:
+    # Run this job if there is a valid level to test, which requires
+    # that any of the following are true:
+    # - A ci-level label higher than any existing ci-level label(s) was added
+    # - A commit was pushed with existing ci-level label(s)
+    # - The PR was reopened with existing ci-level label(s)
+    if: ${{ needs.determine_level.outputs.test_level != '' }}
+    needs: determine_level
+    name: CI Level ${{ needs.determine_level.outputs.test_level }}
+    uses: ./.github/workflows/rocm-ci.yml
+    with:
+      test_level: ${{ needs.determine_level.outputs.test_level }}

.github/workflows/rocm-ci.yml

@@ -2,19 +2,30 @@
 #
 # See LICENSE for license information.
 
-name: TransformerEngine CI
+name: Build and Test Branch
+run-name: CI Level ${{ inputs.test_level || '1' }}
 
 on:
   push:
     branches:
       - 'dev'
-      - 'release_v1.*_rocm'
       - 'release_v2.*_rocm'
-  pull_request:
-    branches:
-      - 'dev'
-      - 'release_v1.**_rocm'
-      - 'release_v2.**_rocm'
+  workflow_call:
+    inputs:
+      test_level:
+        description: 'Test Level (1-3)'
+        required: false
+        default: '1'
+        type: string
+      docker_image_override:
+        description: 'Manual Docker Image (Leave empty to use config file value)'
+        required: false
+        type: string
+      test_config_from_source:
+        description: 'DEBUG: Use config.json from current source branch instead of dev'
+        required: false
+        default: false
+        type: boolean
   workflow_dispatch:
     inputs:
       test_level:
@@ -36,7 +47,7 @@ concurrency:
 
 jobs:
   build_and_test:
-    name: Build and Test on GPU (${{ matrix.runner }})
+    name: Build and Test on GPU (${{ matrix.runner }}) - Level ${{ inputs.test_level || '1' }}
     timeout-minutes: 720
     runs-on: ${{ matrix.runner }}
     strategy: