Gloas lookup sync

.github/forbidden-files.txt

@@ -12,4 +12,6 @@ beacon_node/http_api/src/block_rewards.rs
 common/eth2/src/lighthouse/attestation_performance.rs
 common/eth2/src/lighthouse/block_packing_efficiency.rs
 common/eth2/src/lighthouse/block_rewards.rs
+common/test_random_derive/
 consensus/types/src/execution/state_payload_status.rs
+consensus/types/src/test_utils/test_random/

.github/workflows/local-testnet.yml

@@ -14,7 +14,7 @@ concurrency:
 
 jobs:
   dockerfile-ubuntu:
-    runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x;snapshot.key=lighthouse-ubuntu-latest-v1' || 'ubuntu-latest' }}
     steps:
       - uses: actions/checkout@v5
 
@@ -31,7 +31,7 @@ jobs:
           retention-days: 3
 
   run-local-testnet:
-    runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x;snapshot.key=lighthouse-ubuntu-latest-v1' || 'ubuntu-latest' }}
     needs: dockerfile-ubuntu
     steps:
       - uses: actions/checkout@v5
@@ -173,7 +173,7 @@ jobs:
   # Tests checkpoint syncing to a live network (current fork) and a running devnet (usually next scheduled fork)
   checkpoint-sync-test:
     name: checkpoint-sync-test-${{ matrix.network }}
-    runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x;snapshot.key=lighthouse-ubuntu-latest-v1' || 'ubuntu-latest' }}
     needs: dockerfile-ubuntu
     if: contains(github.event.pull_request.labels.*.name, 'syncing')
     continue-on-error: true
@@ -216,7 +216,7 @@ jobs:
   # Test syncing from genesis on a local testnet. Aims to cover forward syncing both short and long distances.
   genesis-sync-test:
     name: genesis-sync-test-${{ matrix.fork }}-${{ matrix.offline_secs }}s
-    runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x;snapshot.key=lighthouse-ubuntu-latest-v1' || 'ubuntu-latest' }}
     needs: dockerfile-ubuntu
     strategy:
       matrix:
@@ -259,7 +259,7 @@ jobs:
   # a PR is safe to merge. New jobs should be added here.
   local-testnet-success:
     name: local-testnet-success
-    runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x;snapshot.key=lighthouse-ubuntu-latest-v1' || 'ubuntu-latest' }}
     needs: [
       'dockerfile-ubuntu',
       'run-local-testnet',

.github/workflows/test-suite.yml

@@ -85,8 +85,8 @@ jobs:
             while IFS= read -r file || [ -n "$file" ]; do
               # Skip comments and empty lines
               [[ "$file" =~ ^#.*$ || -z "$file" ]] && continue
-              if [ -f "$file" ]; then
-                echo "::error::Forbidden file exists: $file"
+              if [ -e "$file" ]; then
+                echo "::error::Forbidden file or directory exists: $file"
                 status=1
               fi
             done < .github/forbidden-files.txt
@@ -97,22 +97,29 @@ jobs:
     name: release-tests-ubuntu
     needs: [check-labels]
     if: needs.check-labels.outputs.skip_ci != 'true'
-    runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x;snapshot.key=lighthouse-ubuntu-latest-v1' || 'ubuntu-latest' }}
     steps:
     - uses: actions/checkout@v5
     # Set Java version to 21. (required since Web3Signer 24.12.0).
-    - uses: actions/setup-java@v4
+    # On sigp/lighthouse, Java 21 is baked into the snapshot.
+    - if: github.repository != 'sigp/lighthouse'
+      uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
         java-version: '21'
-    - name: Get latest version of stable Rust
+    - if: github.repository != 'sigp/lighthouse'
+      name: Get latest version of stable Rust
       uses: moonrepo/setup-rust@v1
       with:
           channel: stable
           cache-target: release
           bins: cargo-nextest
       env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - if: github.repository == 'sigp/lighthouse'
+      uses: Swatinem/rust-cache@v2
+      with:
+          cache-provider: warpbuild
     - name: Run tests in release
       run: make test-release
     - name: Show cache stats
@@ -123,34 +130,44 @@ jobs:
     name: beacon-chain-tests
     needs: [check-labels]
     if: needs.check-labels.outputs.skip_ci != 'true'
-    runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x;snapshot.key=lighthouse-ubuntu-latest-v1' || 'ubuntu-latest' }}
     env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
     - uses: actions/checkout@v5
-    - name: Get latest version of stable Rust
+    - if: github.repository != 'sigp/lighthouse'
+      name: Get latest version of stable Rust
       uses: moonrepo/setup-rust@v1
       with:
           channel: stable
           cache-target: release
           bins: cargo-nextest
+    - if: github.repository == 'sigp/lighthouse'
+      uses: Swatinem/rust-cache@v2
+      with:
+          cache-provider: warpbuild
     - name: Run beacon_chain tests for all known forks
       run: make test-beacon-chain
   http-api-tests:
     name: http-api-tests
     needs: [check-labels]
     if: needs.check-labels.outputs.skip_ci != 'true'
-    runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x;snapshot.key=lighthouse-ubuntu-latest-v1' || 'ubuntu-latest' }}
     env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
     - uses: actions/checkout@v5
-    - name: Get latest version of stable Rust
+    - if: github.repository != 'sigp/lighthouse'
+      name: Get latest version of stable Rust
       uses: moonrepo/setup-rust@v1
       with:
           channel: stable
           cache-target: release
           bins: cargo-nextest
+    - if: github.repository == 'sigp/lighthouse'
+      uses: Swatinem/rust-cache@v2
+      with:
+          cache-provider: warpbuild
     - name: Run http_api tests for all recent forks
       run: make test-http-api
   op-pool-tests:
@@ -220,16 +237,21 @@ jobs:
     name: debug-tests-ubuntu
     needs: [check-labels]
     if: needs.check-labels.outputs.skip_ci != 'true'
-    runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x;snapshot.key=lighthouse-ubuntu-latest-v1' || 'ubuntu-latest' }}
     env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
     - uses: actions/checkout@v5
-    - name: Get latest version of stable Rust
+    - if: github.repository != 'sigp/lighthouse'
+      name: Get latest version of stable Rust
       uses: moonrepo/setup-rust@v1
       with:
           channel: stable
           bins: cargo-nextest
+    - if: github.repository == 'sigp/lighthouse'
+      uses: Swatinem/rust-cache@v2
+      with:
+          cache-provider: warpbuild
     - name: Run tests in debug
       run: make test-debug
   state-transition-vectors-ubuntu:
@@ -250,17 +272,22 @@ jobs:
     name: ef-tests-ubuntu
     needs: [check-labels]
     if: needs.check-labels.outputs.skip_ci != 'true'
-    runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x;snapshot.key=lighthouse-ubuntu-latest-v1' || 'ubuntu-latest' }}
     env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
     - uses: actions/checkout@v5
-    - name: Get latest version of stable Rust
+    - if: github.repository != 'sigp/lighthouse'
+      name: Get latest version of stable Rust
       uses: moonrepo/setup-rust@v1
       with:
           channel: stable
           cache-target: release
           bins: cargo-nextest
+    - if: github.repository == 'sigp/lighthouse'
+      uses: Swatinem/rust-cache@v2
+      with:
+          cache-provider: warpbuild
     - name: Run consensus-spec-tests with blst and fake_crypto
       run: make test-ef
   basic-simulator-ubuntu:
@@ -311,14 +338,14 @@ jobs:
     name: execution-engine-integration-ubuntu
     needs: [check-labels]
     if: needs.check-labels.outputs.skip_ci != 'true'
-    runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x;snapshot.key=lighthouse-ubuntu-latest-v1' || 'ubuntu-latest' }}
     steps:
     - uses: actions/checkout@v5
-    - name: Get latest version of stable Rust
+    - if: github.repository != 'sigp/lighthouse'
+      name: Get latest version of stable Rust
       uses: moonrepo/setup-rust@v1
       with:
           channel: stable
-          cache-target: release
           cache: false
       env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -387,10 +414,6 @@ jobs:
           cache: false
       env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    - name: Fetch libssl1.1
-      run: wget https://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb
-    - name: Install libssl1.1
-      run: sudo dpkg -i libssl1.1_1.1.1f-1ubuntu2_amd64.deb
     - name: Create Cargo config dir
       run: mkdir -p .cargo
     - name: Install custom Cargo config

.github/workflows/warpbuild-ubuntu-latest-snapshot.yml

@@ -0,0 +1,63 @@
+name: Bake warpbuild snapshot (lighthouse-ubuntu-latest)
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Every week (Sunday at 00:00 UTC)
+    - cron: "0 0 * * 0"
+  pull_request:
+    branches: [stable, unstable]
+    paths:
+      - '.github/workflows/warpbuild-ubuntu-latest-snapshot.yml'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  bake:
+    runs-on: warp-ubuntu-latest-x64-8x
+    steps:
+      - name: Install system deps
+        run: |
+          set -euxo pipefail
+          sudo apt-get update -qq
+          sudo apt-get install -y --no-install-recommends \
+            pkg-config \
+            libssl-dev \
+            build-essential \
+            cmake \
+            clang \
+            llvm-dev \
+            libclang-dev \
+            protobuf-compiler \
+            git \
+            gcc \
+            g++ \
+            make
+
+      - name: Install Rust toolchain (stable)
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          components: rustfmt,clippy
+
+      - name: Install cargo bins
+        run: |
+          cargo install --locked cargo-nextest
+          cargo install --locked cargo-audit
+          cargo install --locked cargo-deny
+          cargo install --locked cargo-sort
+          cargo install --locked cargo-hack
+
+      - name: Install Java (Temurin 21)
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '21'
+
+      - name: Save snapshot
+        uses: WarpBuilds/snapshot-save@v1
+        with:
+          alias: 'lighthouse-ubuntu-latest-v1'
+          fail-on-error: true
+          wait-timeout-minutes: 60