agent-opfor/.github/workflows/evaluator-tests.yml at master · KeyValueSoftwareSystems/agent-opfor · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
name: Evaluator Tests

on:
  push:
    branches: [master, main]
    paths:
      - "evaluators/**"
      - "core/src/evaluators/**"
      - "core/src/prompts/**"
  pull_request:
    branches: [master, main]
    paths:
      - "evaluators/**"
      - "core/src/evaluators/**"
      - "core/src/prompts/**"
  workflow_dispatch:
    inputs:
      evaluator_filter:
        description: "Filter to a specific evaluator ID (e.g. 'prompt-injection'). Leave blank for all."
        required: false
        type: string

concurrency:
  group: evaluator-tests-${{ github.ref }}
  cancel-in-progress: true

jobs:
  fixture-check:
    name: Fixture Check
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-node@v4
        with:
          node-version: 22
          cache: npm

      - run: npm ci

      - run: npm run build

      - name: Check every evaluator has a .test.yaml
        run: node --import tsx/esm core/tests/evaluators.fixture-check.ts

  judge-smoke-tests:
    name: Judge Smoke Tests
    runs-on: ubuntu-latest
    needs: fixture-check
    timeout-minutes: 30

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Detect changed evaluators
        id: changed
        run: |
          if [ -n "${{ github.event.inputs.evaluator_filter }}" ]; then
            echo "filter=${{ github.event.inputs.evaluator_filter }}" >> "$GITHUB_OUTPUT"
            echo "Using manual filter: ${{ github.event.inputs.evaluator_filter }}"
            exit 0
          fi

          BASE=${{ github.event.pull_request.base.sha || github.event.before || 'HEAD~1' }}
          FILES=$(git diff --name-only "$BASE"...HEAD -- 'evaluators/')

          IDS=""
          for f in $FILES; do
            # skip test fixtures and pattern files
            case "$f" in
              *.test.yaml|*.test.yml) continue ;;
              */patterns/*) ;;
              *) ;;
            esac

            dir=$(dirname "$f")
            # folder-based: evaluators/<surface>/<category>/<vuln>/evaluator.yaml
            # or pattern:   evaluators/<surface>/<category>/<vuln>/patterns/foo.yaml
            # flat-file:    evaluators/<surface>/<category>/foo.yaml
            if echo "$f" | grep -q '/patterns/'; then
              dir=$(dirname "$dir")
            fi
            id=$(basename "$dir")
            IDS="$IDS $id"
          done

          UNIQUE=$(echo "$IDS" | tr ' ' '\n' | sort -u | grep -v '^$' | tr '\n' ',' | sed 's/,$//')
          echo "filter=$UNIQUE" >> "$GITHUB_OUTPUT"
          if [ -z "$UNIQUE" ]; then
            echo "No evaluator changes detected — will run all"
          else
            echo "Changed evaluators: $UNIQUE"
          fi

      - uses: actions/setup-node@v4
        with:
          node-version: 22
          cache: npm

      - run: npm ci

      - run: npm run build

      - name: Run evaluator judge smoke tests
        run: node --import tsx/esm --test core/tests/evaluators.judge.test.ts
        env:
          GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
          JUDGE_PROVIDER: groq
          JUDGE_MODEL: llama-3.3-70b-versatile
          EVALUATOR_FILTER: ${{ steps.changed.outputs.filter }}