test.yaml

name: "Test Dev Container Features"

on:
  push:
    branches: [main]
    paths:
      - "src/**"
      - "test/**"
  pull_request:
    branches: [main]
    paths:
      - "src/**"
      - "test/**"
  workflow_dispatch:
    inputs:
      tier:
        description: "Which tier to run"
        type: choice
        default: full
        options:
          - full
          - daily
  schedule:
    # Daily 06:00 UTC — high-risk tier (most fragile floating installers only)
    - cron: "0 6 * * *"
    # Monday 07:00 UTC — full tier (every feature)
    - cron: "0 7 * * 1"

jobs:
  # Single source of truth for "what runs". The daily cron (and a manual
  # `tier: daily` dispatch) run only the most fragile features; everything else
  # (weekly cron, full dispatch, push, PR) runs the complete matrix.
  setup:
    runs-on: ubuntu-latest
    outputs:
      tier: ${{ steps.plan.outputs.tier }}
      autogen-matrix: ${{ steps.plan.outputs.autogen-matrix }}
      scenario-features: ${{ steps.plan.outputs.scenario-features }}
    steps:
      - id: plan
        env:
          EVENT_SCHEDULE: ${{ github.event.schedule }}
          DISPATCH_TIER: ${{ github.event.inputs.tier }}
        run: |
          if [ "$EVENT_SCHEDULE" = "0 6 * * *" ] || [ "$DISPATCH_TIER" = "daily" ]; then
            tier=daily
          else
            tier=full
          fi
          echo "tier=$tier" >> "$GITHUB_OUTPUT"

          full_autogen='{"include":[{"feature":"ai-clis","baseImage":"mcr.microsoft.com/devcontainers/javascript-node:22"},{"feature":"modern-cli-tools","baseImage":"mcr.microsoft.com/devcontainers/base:ubuntu-24.04"},{"feature":"node-dev-tools","baseImage":"mcr.microsoft.com/devcontainers/javascript-node:22"},{"feature":"rust-dev-tools","baseImage":"mcr.microsoft.com/devcontainers/rust:latest"},{"feature":"github-actions-tools","baseImage":"mcr.microsoft.com/devcontainers/base:ubuntu-24.04"},{"feature":"python-tools","baseImage":"mcr.microsoft.com/devcontainers/python:latest"}]}'
          daily_autogen='{"include":[{"feature":"ai-clis","baseImage":"mcr.microsoft.com/devcontainers/javascript-node:22"},{"feature":"modern-cli-tools","baseImage":"mcr.microsoft.com/devcontainers/base:ubuntu-24.04"}]}'
          full_scenarios='["ai-clis","modern-cli-tools","node-dev-tools","rust-dev-tools","github-actions-tools","python-tools"]'
          daily_scenarios='["ai-clis","modern-cli-tools"]'

          if [ "$tier" = "daily" ]; then
            echo "autogen-matrix=$daily_autogen" >> "$GITHUB_OUTPUT"
            echo "scenario-features=$daily_scenarios" >> "$GITHUB_OUTPUT"
          else
            echo "autogen-matrix=$full_autogen" >> "$GITHUB_OUTPUT"
            echo "scenario-features=$full_scenarios" >> "$GITHUB_OUTPUT"
          fi

      - name: Show plan
        run: |
          echo "tier=${{ steps.plan.outputs.tier }}"
          echo "autogen-matrix=${{ steps.plan.outputs.autogen-matrix }}"
          echo "scenario-features=${{ steps.plan.outputs.scenario-features }}"

  test-autogenerated:
    needs: setup
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.setup.outputs.autogen-matrix) }}
    steps:
      - uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: "22"

      - name: Install DevContainer CLI
        run: npm install -g @devcontainers/cli

      - name: Run auto-generated tests
        run: devcontainer features test --features ${{ matrix.feature }} --base-image ${{ matrix.baseImage }} .

  test-scenarios:
    needs: setup
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        feature: ${{ fromJSON(needs.setup.outputs.scenario-features) }}
    steps:
      - uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: "22"

      - name: Install DevContainer CLI
        run: npm install -g @devcontainers/cli

      - name: Run scenario tests
        run: devcontainer features test --features ${{ matrix.feature }} --skip-autogenerated .

  test-global:
    needs: setup
    # The all-features composition test is broad; skip it on the daily
    # high-risk tier and keep it on full runs (weekly / push / PR / dispatch).
    # It is a blocking job like the others: a failure marks the run red (so PR
    # authors catch composition breakage) and surfaces to notify-on-failure on
    # scheduled/dispatched runs. (It cannot be continue-on-error, or its result
    # would report as success to the notify gate and never alert.)
    if: needs.setup.outputs.tier != 'daily'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: "22"

      - name: Install DevContainer CLI
        run: npm install -g @devcontainers/cli

      - name: Run global scenario tests
        run: devcontainer features test --global-scenarios-only .

  # On scheduled (or manually dispatched) runs, surface failures as a single
  # deduplicated GitHub issue. Upstream third-party installers can break without
  # any change to this repo, so a red check on an otherwise-idle repo would
  # otherwise go unnoticed. Not gated to PRs — those already show a red check.
  notify-on-failure:
    needs: [setup, test-autogenerated, test-scenarios, test-global]
    if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
    runs-on: ubuntu-latest
    permissions:
      issues: write
    steps:
      - uses: actions/github-script@v7
        env:
          NEEDS: ${{ toJSON(needs) }}
          TIER: ${{ needs.setup.outputs.tier }}
        with:
          script: |
            const needs = JSON.parse(process.env.NEEDS);
            const failed = Object.entries(needs)
              .filter(([, v]) => v.result === 'failure')
              .map(([name]) => name);

            if (failed.length === 0) {
              core.info('No failed jobs — nothing to report.');
              return;
            }

            const tier = process.env.TIER || 'unknown';
            const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
            const label = 'installer-breakage';
            const title = 'Scheduled feature validation failed';
            const body = [
              `Scheduled feature validation (**${tier}** tier) failed.`,
              '',
              `**Failed jobs:** ${failed.join(', ')}`,
              `**Run:** ${runUrl}`,
              '',
              'This usually means an upstream third-party installer changed or broke.',
              'Open the run above to see which feature failed.',
            ].join('\n');

            const existing = await github.rest.issues.listForRepo({
              owner: context.repo.owner,
              repo: context.repo.repo,
              state: 'open',
              labels: label,
            });

            if (existing.data.length > 0) {
              const number = existing.data[0].number;
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: number,
                body,
              });
              core.info(`Commented on existing issue #${number}.`);
            } else {
              const created = await github.rest.issues.create({
                owner: context.repo.owner,
                repo: context.repo.repo,
                title,
                body,
                labels: [label],
              });
              core.info(`Opened issue #${created.data.number}.`);
            }