fix: resolve concurrency races and overhaul test diagnostics

.github/workflows/codeql.yml

@@ -0,0 +1,78 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  schedule:
+    # Run weekly on Sundays at 04:00 UTC to catch new CVEs in dependencies.
+    - cron: '0 4 * * 0'
+
+jobs:
+  analyze:
+    name: Analyze (C/C++)
+    runs-on: ubuntu-24.04
+    permissions:
+      security-events: write    # Required to upload SARIF results.
+      contents: read            # Required to checkout code.
+      actions: read             # Required for private repos.
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # CodeQL supports: cpp, csharp, go, java, javascript, python, ruby, swift.
+        language: [ 'c-cpp' ]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # ── Initialize CodeQL ────────────────────────────────────────────
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        # Use the extended query suite for deeper analysis (security + quality).
+        queries: security-extended
+
+    # ── Toolchain Setup ──────────────────────────────────────────────
+    # CodeQL for C/C++ needs to observe the build. We must set up the
+    # same LLVM 19 + Bazel toolchain used by the main CI pipeline so
+    # that the CodeQL tracer can instrument the compilation.
+    - name: Set up LLVM 19
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y wget gnupg ca-certificates
+        sudo mkdir -p /usr/share/keyrings
+        wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo gpg --dearmor -o /usr/share/keyrings/llvm-archive-keyring.gpg
+        echo "deb [signed-by=/usr/share/keyrings/llvm-archive-keyring.gpg] http://apt.llvm.org/noble/ llvm-toolchain-noble-19 main" | sudo tee /etc/apt/sources.list.d/llvm.list
+        sudo apt-get update
+        sudo apt-get install -y clang-19 lld-19 libclang-rt-19-dev
+        sudo ln -sf /usr/bin/clang-19 /usr/bin/clang
+        sudo ln -sf /usr/bin/clang++-19 /usr/bin/clang++
+        sudo ln -sf /usr/bin/lld-19 /usr/bin/lld
+
+    - name: Cache Bazel
+      uses: actions/cache@v4
+      with:
+        path: |
+          ~/.cache/bazel
+        key: ${{ runner.os }}-codeql-bazel-${{ hashFiles('**/MODULE.bazel', '**/WORKSPACE', '**/WORKSPACE.bazel') }}
+        restore-keys: |
+          ${{ runner.os }}-codeql-bazel-
+
+    # ── Build (observed by CodeQL tracer) ────────────────────────────
+    # Use the CI bazelrc for resource-constrained runners.
+    # Build all targets so CodeQL sees the full source graph.
+    - name: Bazel Build (CodeQL traced)
+      run: |
+        bazel --bazelrc=.github/workflows/ci.bazelrc build \
+          --verbose_failures \
+          //...
+
+    # ── Upload Results ───────────────────────────────────────────────
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{ matrix.language }}"

.github/workflows/linux_ci.yml

@@ -40,24 +40,70 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-bazel-
 
+    # ── Environment Verification ─────────────────────────────────────
+    # Runs the full dcodex-setup.sh --test to verify the environment is
+    # correctly configured and all sanitizer suites pass. This catches
+    # setup-level issues (missing deps, broken symlinks) that individual
+    # test steps wouldn't detect.
+    - name: "🔧 Environment Verification (dcodex-setup.sh --test)"
+      run: |
+        chmod +x ./dcodex-setup.sh
+        sudo REPO_DIR="$GITHUB_WORKSPACE" \
+             BAZEL_JOBS=4 \
+             BAZEL_MEM_MB=2048 \
+             SKIP_APT=0 \
+             ./dcodex-setup.sh --test
+      env:
+        DEBIAN_FRONTEND: noninteractive
+
+    # The dcodex-setup.sh runs as root (sudo), creating .bazel/output_base
+    # and /tmp/bazel-test-logs (via .bazelrc --test_tmpdir) owned by root.
+    # Subsequent Bazel steps run as the runner user and need read/write access.
+    - name: Fix root-owned directory permissions
+      run: |
+        sudo chown -R "$(id -u):$(id -g)" .bazel
+        sudo chown -R "$(id -u):$(id -g)" /tmp/bazel-test-logs 2>/dev/null || true
+
+    # ── Fine-grained Test Steps ──────────────────────────────────────
+    # These provide per-sanitizer visibility in the GitHub Actions UI.
+    # The dcodex-setup.sh step above is the single source of truth;
+    # these are redundant-but-visible steps for PR review ergonomics.
+
     - name: Bazel Build
       run: |
-        bazel --bazelrc=.github/workflows/ci.bazelrc build //...
+        bazel --bazelrc=.github/workflows/ci.bazelrc build \
+          --verbose_failures \
+          //...
 
     - name: Bazel Test (Standard)
       run: |
-        bazel --bazelrc=.github/workflows/ci.bazelrc test //...
+        bazel --bazelrc=.github/workflows/ci.bazelrc test \
+          --verbose_failures \
+          //...
+
+    - name: Bazel Test (ASan + UBSan)
+      run: |
+        bazel --bazelrc=.github/workflows/ci.bazelrc test \
+          --config=asan \
+          --verbose_failures \
+          --jobs=2 \
+          //src/engine:sandbox_test \
+          //src/engine:warm_worker_pool_test \
+          //src/engine:dynamic_worker_coordinator_test \
+          //src/engine:tsan_checker
 
-    - name: Bazel Test (TSan)
+    - name: Bazel Test (TSan — concurrency tests)
       run: |
         bazel --bazelrc=.github/workflows/ci.bazelrc test --config=tsan \
-          //... \
+          --verbose_failures \
           --jobs=2 \
-          --test_tag_filters=-no-sandbox-tsan
+          --test_tag_filters=-no-sandbox-tsan \
+          //...
 
-    - name: Bazel Test (TSan - Sandbox, constrained)
+    - name: Bazel Test (TSan — Sandbox, constrained)
       run: |
         bazel --bazelrc=.github/workflows/ci.bazelrc test --config=tsan \
+          --verbose_failures \
           //src/engine:sandbox_test \
           --jobs=1 \
           --runs_per_test=1 \
@@ -95,11 +141,6 @@ jobs:
           exit 1
         fi
 
-    - name: Bazel Test (ASan)
-      run: |
-        bazel --bazelrc=.github/workflows/ci.bazelrc test --config=asan //... \
-          --jobs=2
-
     # MSan is intentionally excluded from CI. It requires ALL linked libraries
     # (including libstdc++) to be compiled with MSan instrumentation. The system
     # libstdc++ on GitHub runners is not instrumented, producing false positives
@@ -111,4 +152,6 @@ jobs:
       uses: actions/upload-artifact@v4
       with:
         name: bazel-test-logs
-        path: bazel-testlogs/
+        path: |
+          bazel-testlogs/
+          /tmp/dcodex-test-*.log