fix: make hunt work from a published install #11
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Evaluator Tests | |
| on: | |
| push: | |
| branches: [master, main] | |
| paths: | |
| - "evaluators/**" | |
| - "core/src/evaluators/**" | |
| - "core/src/prompts/**" | |
| pull_request: | |
| branches: [master, main] | |
| paths: | |
| - "evaluators/**" | |
| - "core/src/evaluators/**" | |
| - "core/src/prompts/**" | |
| workflow_dispatch: | |
| inputs: | |
| evaluator_filter: | |
| description: "Filter to a specific evaluator ID (e.g. 'prompt-injection'). Leave blank for all." | |
| required: false | |
| type: string | |
| concurrency: | |
| group: evaluator-tests-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| fixture-check: | |
| name: Fixture Check | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 22 | |
| cache: npm | |
| - run: npm ci | |
| - run: npm run build | |
| - name: Check every evaluator has a .test.yaml | |
| run: node --import tsx/esm core/tests/evaluators.fixture-check.ts | |
| judge-smoke-tests: | |
| name: Judge Smoke Tests | |
| runs-on: ubuntu-latest | |
| needs: fixture-check | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Detect changed evaluators | |
| id: changed | |
| run: | | |
| if [ -n "${{ github.event.inputs.evaluator_filter }}" ]; then | |
| echo "filter=${{ github.event.inputs.evaluator_filter }}" >> "$GITHUB_OUTPUT" | |
| echo "Using manual filter: ${{ github.event.inputs.evaluator_filter }}" | |
| exit 0 | |
| fi | |
| BASE=${{ github.event.pull_request.base.sha || github.event.before || 'HEAD~1' }} | |
| FILES=$(git diff --name-only "$BASE"...HEAD -- 'evaluators/') | |
| IDS="" | |
| for f in $FILES; do | |
| # skip test fixtures and pattern files | |
| case "$f" in | |
| *.test.yaml|*.test.yml) continue ;; | |
| */patterns/*) ;; | |
| *) ;; | |
| esac | |
| dir=$(dirname "$f") | |
| # folder-based: evaluators/<surface>/<category>/<vuln>/evaluator.yaml | |
| # or pattern: evaluators/<surface>/<category>/<vuln>/patterns/foo.yaml | |
| # flat-file: evaluators/<surface>/<category>/foo.yaml | |
| if echo "$f" | grep -q '/patterns/'; then | |
| dir=$(dirname "$dir") | |
| fi | |
| id=$(basename "$dir") | |
| IDS="$IDS $id" | |
| done | |
| UNIQUE=$(echo "$IDS" | tr ' ' '\n' | sort -u | grep -v '^$' | tr '\n' ',' | sed 's/,$//') | |
| echo "filter=$UNIQUE" >> "$GITHUB_OUTPUT" | |
| if [ -z "$UNIQUE" ]; then | |
| echo "No evaluator changes detected — will run all" | |
| else | |
| echo "Changed evaluators: $UNIQUE" | |
| fi | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 22 | |
| cache: npm | |
| - run: npm ci | |
| - run: npm run build | |
| - name: Run evaluator judge smoke tests | |
| run: node --import tsx/esm --test core/tests/evaluators.judge.test.ts | |
| env: | |
| GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} | |
| JUDGE_PROVIDER: groq | |
| JUDGE_MODEL: llama-3.3-70b-versatile | |
| EVALUATOR_FILTER: ${{ steps.changed.outputs.filter }} |