[evals] Adding in some vally testing for some basic azd scenarios

.github/actions/vally-eval/action.yml

@@ -40,7 +40,7 @@ outputs:
     description: Exit code of the `vally eval` invocation.
     value: ${{ steps.run.outputs.vally-exit-code }}
   vally-run-dir:
-    description: Path to the timestamped run directory produced by the eval, relative to the repo root.
+    description: Absolute path to the timestamped run directory produced by the eval.
     value: ${{ steps.run.outputs.vally-run-dir }}
 
 runs:
@@ -51,6 +51,16 @@ runs:
       with:
         node-version: ${{ inputs.node-version }}
 
+    - name: Set up Go
+      uses: actions/setup-go@v6
+      with:
+        go-version-file: ${{ inputs.working-directory }}/go.mod
+
+    - name: Test report generator
+      shell: bash
+      working-directory: ${{ inputs.working-directory }}
+      run: go test ./...
+
     - name: Install eval dependencies
       shell: bash
       working-directory: ${{ inputs.working-directory }}
@@ -64,7 +74,6 @@ runs:
         GH_TOKEN: ${{ inputs.github-token }}
         JUNIT: ${{ inputs.junit }}
         EVAL_FILE: ${{ inputs.eval-file }}
-        WORKING_DIRECTORY: ${{ inputs.working-directory }}
       run: |
         junit_flag=""
         [[ "$JUNIT" == "true" ]] && junit_flag="--junit"
@@ -79,7 +88,19 @@ runs:
         # and export that value as 'vally-run-dir'.
         #
         run_dir=$(find vally-results -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort | tail -n1)
-        echo "vally-run-dir=$WORKING_DIRECTORY/${run_dir:-vally-results}" >> "$GITHUB_OUTPUT"
+        run_dir=${run_dir:-vally-results}
+        echo "vally-run-dir=$(pwd)/$run_dir" >> "$GITHUB_OUTPUT"
+
+    - name: Generate vally report
+      if: always() && steps.run.outputs.vally-run-dir != ''
+      shell: bash
+      working-directory: ${{ inputs.working-directory }}
+      env:
+        VALLY_RUN_DIR: ${{ steps.run.outputs.vally-run-dir }}
+      run: |
+        if [[ -d "$VALLY_RUN_DIR" ]]; then
+          npm run report -- --output-dir "$VALLY_RUN_DIR/vally_report"
+        fi
 
     - name: Write job summary
       if: always()
@@ -97,6 +118,10 @@ runs:
           else
             echo "_No eval-results.md found -- see the vally-results artifact and raw output above._"
           fi
+          echo
+          echo "--------------------------------------------------------------------------------"
+          echo "Additional reports generated by vally_report.go are in \`$VALLY_RUN_DIR/vally_report/\` in the artifact."
+          echo "--------------------------------------------------------------------------------"
         } >> "$GITHUB_STEP_SUMMARY"
 
     - name: Upload vally results
@@ -112,7 +137,6 @@ runs:
       shell: bash
       env:
         VALLY_EXIT_CODE: ${{ steps.run.outputs.vally-exit-code }}
-        VALLY_RUN_DIR: ${{ steps.run.outputs.vally-run-dir }}
       run: |
         if [[ "${VALLY_EXIT_CODE:-1}" -ne 0 ]]; then
           echo "npx vally eval itself failed (exit code ${VALLY_EXIT_CODE:-unknown}) -- see the 'Run vally eval' log above."

.github/workflows/vally-ci.yml

@@ -15,16 +15,16 @@ permissions:
 
 concurrency:
   group: vally-ci-${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true 
+  cancel-in-progress: true
 
 jobs:
-  evals:
-    name: Run vally evals
+  qna:
+    name: Run vally evals (Q&A)
     runs-on: ubuntu-latest
     timeout-minutes: 90
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v7
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
 
@@ -33,6 +33,25 @@ jobs:
         uses: ./.github/actions/vally-eval
         with:
           working-directory: cli/azd/test/evals
-          eval-file: eval.yaml
+          eval-file: eval-azd-qna.yaml
+          artifact-name: ${{ github.job }}-vally-results
+          github-token: ${{ github.token }}
+
+  deploy:
+    name: Run vally evals (deploy)
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Run vally eval
+        id: vally
+        uses: ./.github/actions/vally-eval
+        with:
+          working-directory: cli/azd/test/evals
+          eval-file: eval-azd-deploy.yaml
           artifact-name: ${{ github.job }}-vally-results
           github-token: ${{ github.token }}

cli/azd/.vscode/cspell.yaml

@@ -441,28 +441,6 @@ overrides:
       - covcounters
       - covmeta
       - AMRD
-  - filename: test/eval/README.md
-    words:
-      - Waza
-      - waza
-      - urlopen
-  - filename: "test/eval/graders/*.py"
-    words:
-      - Waza
-      - waza
-      - hdrs
-      - mysite
-      - mydb
-  - filename: "test/eval/tasks/**/*.yaml"
-    words:
-      - authenticat
-      - idempoten
-  - filename: "test/eval/tests/human/*.test.ts"
-    words:
-      - compdef
-      - badcfg
-      - provison
-      - notacommand
   - filename: pkg/infra/provisioning/bicep/bicep_provider.go
     words:
       - BIZSPARK