[evals] Adding in some vally testing for some basic azd scenarios

.github/actions/vally-eval/action.yml

@@ -51,6 +51,16 @@ runs:
       with:
         node-version: ${{ inputs.node-version }}
 
+    - name: Set up Go
+      uses: actions/setup-go@v6
+      with:
+        go-version-file: cli/azd/test/evals/go.mod
+
+    - name: Test report generator
+      shell: bash
+      working-directory: ${{ inputs.working-directory }}
+      run: go test ./...
+
     - name: Install eval dependencies
       shell: bash
       working-directory: ${{ inputs.working-directory }}
@@ -79,7 +89,12 @@ runs:
         # and export that value as 'vally-run-dir'.
         #
         run_dir=$(find vally-results -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort | tail -n1)
-        echo "vally-run-dir=$WORKING_DIRECTORY/${run_dir:-vally-results}" >> "$GITHUB_OUTPUT"
+        run_dir=${run_dir:-vally-results}
+        echo "vally-run-dir=$WORKING_DIRECTORY/$run_dir" >> "$GITHUB_OUTPUT"
+
+        if [[ -d "$run_dir" ]]; then
+          npm run report -- --output-dir "$run_dir/vally_report"
+        fi
 
     - name: Write job summary
       if: always()
@@ -97,6 +112,10 @@ runs:
           else
             echo "_No eval-results.md found -- see the vally-results artifact and raw output above._"
           fi
+          echo
+          echo "--------------------------------------------------------------------------------"
+          echo "Additional reports generated by vally_report.go are in \`$VALLY_RUN_DIR/vally_report/\` in the artifact."
+          echo "--------------------------------------------------------------------------------"
         } >> "$GITHUB_STEP_SUMMARY"
 
     - name: Upload vally results

.github/workflows/vally-ci.yml

@@ -18,13 +18,13 @@ concurrency:
   cancel-in-progress: true 
 
 jobs:
-  evals:
-    name: Run vally evals
+  qna:
+    name: Run vally evals (Q&A)
     runs-on: ubuntu-latest
     timeout-minutes: 90
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v7
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.0
         with:
           persist-credentials: false
 
@@ -33,6 +33,25 @@ jobs:
         uses: ./.github/actions/vally-eval
         with:
           working-directory: cli/azd/test/evals
-          eval-file: eval.yaml
+          eval-file: eval-azd-qna.yaml
+          artifact-name: ${{ github.job }}-vally-results
+          github-token: ${{ github.token }}
+
+  deploy:
+    name: Run vally evals (deploy)
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Run vally eval
+        id: vally
+        uses: ./.github/actions/vally-eval
+        with:
+          working-directory: cli/azd/test/evals
+          eval-file: eval-azd-deploy.yaml
           artifact-name: ${{ github.job }}-vally-results
           github-token: ${{ github.token }}

cli/azd/.vscode/cspell.yaml

@@ -441,28 +441,6 @@ overrides:
       - covcounters
       - covmeta
       - AMRD
-  - filename: test/eval/README.md
-    words:
-      - Waza
-      - waza
-      - urlopen
-  - filename: "test/eval/graders/*.py"
-    words:
-      - Waza
-      - waza
-      - hdrs
-      - mysite
-      - mydb
-  - filename: "test/eval/tasks/**/*.yaml"
-    words:
-      - authenticat
-      - idempoten
-  - filename: "test/eval/tests/human/*.test.ts"
-    words:
-      - compdef
-      - badcfg
-      - provison
-      - notacommand
   - filename: pkg/infra/provisioning/bicep/bicep_provider.go
     words:
       - BIZSPARK