release: gloas

.github/workflows/buf.yaml

@@ -24,7 +24,7 @@ jobs:
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
       - name: Set up buf
-        uses: bufbuild/buf-setup-action@a47c93e0b1648769eb9a2e1f98e7b4e3e13089f0 # v1.50.0
+        uses: bufbuild/buf-setup-action@a47c93e0b1648d5651a065437926377d060baa99 # v1.50.0
 
       - name: Lint protos
         run: buf lint

deploy/local/docker-compose/vector-http-kafka.yaml

@@ -52,6 +52,7 @@ transforms:
       eth_v2_beacon_block_v2: .event.name == "BEACON_API_ETH_V2_BEACON_BLOCK_V2"
       eth_v2_beacon_block_voluntary_exit: .event.name == "BEACON_API_ETH_V2_BEACON_BLOCK_VOLUNTARY_EXIT"
       eth_v2_beacon_block_withdrawal: .event.name == "BEACON_API_ETH_V2_BEACON_BLOCK_WITHDRAWAL"
+      eth_v2_beacon_block_access_list: .event.name == "BEACON_API_ETH_V2_BEACON_BLOCK_ACCESS_LIST"
       eth_v2_beacon_block: .event.name == "BEACON_API_ETH_V2_BEACON_BLOCK"
       mempool_transaction_v2: .event.name == "MEMPOOL_TRANSACTION_V2"
       mempool_transaction: .event.name == "MEMPOOL_TRANSACTION"
@@ -763,6 +764,22 @@ sinks:
       enabled: true
     encoding:
       codec: json
+  beacon_api_eth_v2_beacon_block_access_list_kafka:
+    type: kafka
+    buffer:
+      max_events: 500000
+    batch:
+      timeout_secs: 0.5
+    inputs:
+      - xatu_server_events_router.eth_v2_beacon_block_access_list
+    bootstrap_servers: "${KAFKA_BROKERS}"
+    key_field: "event.id"
+    topic: beacon-api-eth-v2-beacon-block-access-list
+    compression: snappy
+    healthcheck:
+      enabled: true
+    encoding:
+      codec: json
   beacon_api_eth_v1_beacon_blob_sidecar_kafka:
     type: kafka
     buffer:

deploy/local/docker-compose/vector-kafka-clickhouse.yaml

@@ -424,6 +424,7 @@ transforms:
       canonical_beacon_block_voluntary_exit: .event.name == "BEACON_API_ETH_V2_BEACON_BLOCK_VOLUNTARY_EXIT"
       canonical_beacon_validators: .event.name == "BEACON_API_ETH_V1_BEACON_VALIDATORS"
       canonical_beacon_block_withdrawal: .event.name == "BEACON_API_ETH_V2_BEACON_BLOCK_WITHDRAWAL"
+      canonical_beacon_block_access_list: .event.name == "BEACON_API_ETH_V2_BEACON_BLOCK_ACCESS_LIST"
       canonical_beacon_block: .event.name == "BEACON_API_ETH_V2_BEACON_BLOCK_V2" && .meta.client.additional_data.finalized_when_requested == true
       canonical_beacon_proposer_duty: .event.name == "BEACON_API_ETH_V1_PROPOSER_DUTY" && .meta.client.additional_data.state_id == "finalized"
       canonical_beacon_committee: .event.name == "BEACON_API_ETH_V1_BEACON_COMMITTEE" && .meta.client.additional_data.state_id == "finalized"
@@ -480,6 +481,7 @@ transforms:
       - xatu_server_events_router.canonical_beacon_block_proposer_slashing
       - xatu_server_events_router.canonical_beacon_block_voluntary_exit
       - xatu_server_events_router.canonical_beacon_block_withdrawal
+      - xatu_server_events_router.canonical_beacon_block_access_list
       - xatu_server_events_router.canonical_beacon_proposer_duty
       - xatu_server_events_router.canonical_beacon_committee
       - xatu_server_events_router.eth_v1_beacon_committee
@@ -1207,6 +1209,7 @@ transforms:
       .execution_payload_base_fee_per_gas = .data.message.body.execution_payload.base_fee_per_gas
       .execution_payload_blob_gas_used = .data.message.body.execution_payload.blob_gas_used
       .execution_payload_excess_blob_gas = .data.message.body.execution_payload.excess_blob_gas
+      .execution_payload_slot_number = .data.message.body.execution_payload.slot_number
       .execution_payload_gas_limit = .data.message.body.execution_payload.gas_limit
       .execution_payload_gas_used = .data.message.body.execution_payload.gas_used
       .execution_payload_state_root = .data.message.body.execution_payload.state_root
@@ -1278,6 +1281,7 @@ transforms:
         .execution_payload_base_fee_per_gas = .data.message.body.execution_payload.base_fee_per_gas
         .execution_payload_blob_gas_used = .data.message.body.execution_payload.blob_gas_used
         .execution_payload_excess_blob_gas = .data.message.body.execution_payload.excess_blob_gas
+        .execution_payload_slot_number = .data.message.body.execution_payload.slot_number
         .execution_payload_gas_limit = .data.message.body.execution_payload.gas_limit
         .execution_payload_gas_used = .data.message.body.execution_payload.gas_used
         .execution_payload_state_root = .data.message.body.execution_payload.state_root
@@ -1557,6 +1561,41 @@ transforms:
       del(.event)
       del(.meta)
       del(.data)
+  canonical_beacon_block_access_list_formatted:
+    type: remap
+    inputs:
+      - xatu_server_events_router.canonical_beacon_block_access_list
+    source: |-
+      .slot = .meta.client.additional_data.block.slot.number
+      slot_start_date_time, err = parse_timestamp(.meta.client.additional_data.block.slot.start_date_time, format: "%+");
+      if err == null {
+        .slot_start_date_time = to_unix_timestamp(slot_start_date_time)
+      } else {
+        .error = err
+        .error_description = "failed to parse slot start date time"
+        log(., level: "error", rate_limit_secs: 60)
+      }
+      .epoch = .meta.client.additional_data.block.epoch.number
+      epoch_start_date_time, err = parse_timestamp(.meta.client.additional_data.block.epoch.start_date_time, format: "%+");
+      if err == null {
+        .epoch_start_date_time = to_unix_timestamp(epoch_start_date_time)
+      } else {
+        .error = err
+        .error_description = "failed to parse epoch start date time"
+        log(., level: "error", rate_limit_secs: 60)
+      }
+      .block_root = .meta.client.additional_data.block.root
+      .block_number = .meta.client.additional_data.blockNumber
+      .block_hash = .meta.client.additional_data.blockHash
+      .address = .data.address
+      .change_type = .data.change_type
+      .block_access_index = .data.block_access_index
+      .storage_key = .data.storage_key
+      .new_value = .data.new_value
+      .updated_date_time = to_unix_timestamp(now())
+      del(.event)
+      del(.meta)
+      del(.data)
   canonical_beacon_blob_sidecar_formatted:
     type: remap
     inputs:
@@ -2313,6 +2352,7 @@ transforms:
       .execution_payload_base_fee_per_gas = .data.message.body.execution_payload.base_fee_per_gas
       .execution_payload_blob_gas_used = .data.message.body.execution_payload.blob_gas_used
       .execution_payload_excess_blob_gas = .data.message.body.execution_payload.excess_blob_gas
+      .execution_payload_slot_number = .data.message.body.execution_payload.slot_number
       .execution_payload_gas_limit = .data.message.body.execution_payload.gas_limit
       .execution_payload_gas_used = .data.message.body.execution_payload.gas_used
       .execution_payload_transactions_count = .meta.client.additional_data.transactions_count
@@ -3508,6 +3548,26 @@ sinks:
     healthcheck:
       enabled: true
     skip_unknown_fields: false
+  canonical_beacon_block_access_list_clickhouse:
+    type: clickhouse
+    inputs:
+      - canonical_beacon_block_access_list_formatted
+    database: default
+    endpoint: "${CLICKHOUSE_ENDPOINT}"
+    table: canonical_beacon_block_access_list
+    auth:
+      strategy: basic
+      user: "${CLICKHOUSE_USER}"
+      password: "${CLICKHOUSE_PASSWORD}"
+    batch:
+      max_bytes: 52428800
+      max_events: 200000
+      timeout_secs: 1
+    buffer:
+      max_events: 200000
+    healthcheck:
+      enabled: true
+    skip_unknown_fields: false
   canonical_beacon_blob_sidecar_clickhouse:
     type: clickhouse
     inputs:

deploy/local/docker-compose/xatu-server.yaml

@@ -197,6 +197,7 @@ services:
             - BEACON_API_ETH_V2_BEACON_BLOCK_VOLUNTARY_EXIT
             - BEACON_API_ETH_V2_BEACON_BLOCK_DEPOSIT
             - BEACON_API_ETH_V2_BEACON_BLOCK_WITHDRAWAL
+            - BEACON_API_ETH_V2_BEACON_BLOCK_ACCESS_LIST
             - BEACON_API_ETH_V2_BEACON_BLOCK_ELABORATED_ATTESTATION
             - BEACON_API_ETH_V2_BEACON_BLOCK
             - BEACON_API_ETH_V2_BEACON_BLOCK_V2

deploy/migrations/clickhouse/106_gloas_bals_support.down.sql

@@ -0,0 +1,19 @@
+-- Drop canonical_beacon_block_access_list tables
+DROP TABLE IF EXISTS default.canonical_beacon_block_access_list ON CLUSTER '{cluster}';
+DROP TABLE IF EXISTS default.canonical_beacon_block_access_list_local ON CLUSTER '{cluster}';
+
+-- Remove columns from canonical_beacon_block
+ALTER TABLE default.canonical_beacon_block ON CLUSTER '{cluster}'
+    DROP COLUMN IF EXISTS execution_payload_block_access_list_root,
+    DROP COLUMN IF EXISTS execution_payload_slot_number;
+
+ALTER TABLE default.canonical_beacon_block_local ON CLUSTER '{cluster}'
+    DROP COLUMN IF EXISTS execution_payload_block_access_list_root,
+    DROP COLUMN IF EXISTS execution_payload_slot_number;
+
+-- Remove columns from beacon_api_eth_v2_beacon_block
+ALTER TABLE default.beacon_api_eth_v2_beacon_block ON CLUSTER '{cluster}'
+    DROP COLUMN IF EXISTS execution_payload_slot_number;
+
+ALTER TABLE default.beacon_api_eth_v2_beacon_block_local ON CLUSTER '{cluster}'
+    DROP COLUMN IF EXISTS execution_payload_slot_number;

deploy/migrations/clickhouse/106_gloas_bals_support.up.sql

@@ -0,0 +1,63 @@
+-- Add execution_payload_slot_number to beacon_api_eth_v2_beacon_block
+ALTER TABLE default.beacon_api_eth_v2_beacon_block_local ON CLUSTER '{cluster}'
+    ADD COLUMN IF NOT EXISTS execution_payload_slot_number Nullable(UInt64)
+        CODEC(DoubleDelta, ZSTD(1)) AFTER execution_payload_excess_blob_gas;
+
+ALTER TABLE default.beacon_api_eth_v2_beacon_block ON CLUSTER '{cluster}'
+    ADD COLUMN IF NOT EXISTS execution_payload_slot_number Nullable(UInt64)
+        CODEC(DoubleDelta, ZSTD(1)) AFTER execution_payload_excess_blob_gas;
+
+-- Add execution_payload_slot_number and execution_payload_block_access_list_root to canonical_beacon_block
+ALTER TABLE default.canonical_beacon_block_local ON CLUSTER '{cluster}'
+    ADD COLUMN IF NOT EXISTS execution_payload_slot_number Nullable(UInt64)
+        CODEC(DoubleDelta, ZSTD(1)) AFTER execution_payload_excess_blob_gas,
+    ADD COLUMN IF NOT EXISTS execution_payload_block_access_list_root Nullable(FixedString(66))
+        CODEC(ZSTD(1)) AFTER execution_payload_slot_number;
+
+ALTER TABLE default.canonical_beacon_block ON CLUSTER '{cluster}'
+    ADD COLUMN IF NOT EXISTS execution_payload_slot_number Nullable(UInt64)
+        CODEC(DoubleDelta, ZSTD(1)) AFTER execution_payload_excess_blob_gas,
+    ADD COLUMN IF NOT EXISTS execution_payload_block_access_list_root Nullable(FixedString(66))
+        CODEC(ZSTD(1)) AFTER execution_payload_slot_number;
+
+-- Create canonical_beacon_block_access_list table
+CREATE TABLE IF NOT EXISTS default.canonical_beacon_block_access_list_local ON CLUSTER '{cluster}' (
+    updated_date_time DateTime CODEC(DoubleDelta, ZSTD(1)),
+    slot UInt32 CODEC(DoubleDelta, ZSTD(1)),
+    slot_start_date_time DateTime CODEC(DoubleDelta, ZSTD(1)),
+    epoch UInt32 CODEC(DoubleDelta, ZSTD(1)),
+    epoch_start_date_time DateTime CODEC(DoubleDelta, ZSTD(1)),
+    block_root FixedString(66) CODEC(ZSTD(1)),
+    block_number UInt64 CODEC(DoubleDelta, ZSTD(1)),
+    block_hash FixedString(66) CODEC(ZSTD(1)),
+    address FixedString(42) CODEC(ZSTD(1)),
+    change_type LowCardinality(String) CODEC(ZSTD(1)),
+    block_access_index UInt16 CODEC(DoubleDelta, ZSTD(1)),
+    storage_key FixedString(66) CODEC(ZSTD(1)),
+    new_value Nullable(String) CODEC(ZSTD(1)),
+    meta_client_name LowCardinality(String) CODEC(ZSTD(1)),
+    meta_client_id String CODEC(ZSTD(1)),
+    meta_client_version LowCardinality(String) CODEC(ZSTD(1)),
+    meta_client_implementation LowCardinality(String) CODEC(ZSTD(1)),
+    meta_client_os LowCardinality(String) CODEC(ZSTD(1)),
+    meta_client_ip Nullable(IPv6) CODEC(ZSTD(1)),
+    meta_client_geo_city LowCardinality(String) CODEC(ZSTD(1)),
+    meta_client_geo_country LowCardinality(String) CODEC(ZSTD(1)),
+    meta_client_geo_country_code LowCardinality(String) CODEC(ZSTD(1)),
+    meta_client_geo_continent_code LowCardinality(String) CODEC(ZSTD(1)),
+    meta_client_geo_longitude Nullable(Float64) CODEC(ZSTD(1)),
+    meta_client_geo_latitude Nullable(Float64) CODEC(ZSTD(1)),
+    meta_client_geo_autonomous_system_number Nullable(UInt32) CODEC(ZSTD(1)),
+    meta_client_geo_autonomous_system_organization Nullable(String) CODEC(ZSTD(1)),
+    meta_network_id Int32 CODEC(DoubleDelta, ZSTD(1)),
+    meta_network_name LowCardinality(String) CODEC(ZSTD(1)),
+    meta_consensus_version LowCardinality(String) CODEC(ZSTD(1)),
+    meta_consensus_implementation LowCardinality(String) CODEC(ZSTD(1)),
+    meta_labels Map(String, String) CODEC(ZSTD(1))
+) ENGINE = ReplicatedReplacingMergeTree(updated_date_time)
+PARTITION BY toStartOfMonth(slot_start_date_time)
+ORDER BY (slot_start_date_time, meta_network_name, block_hash, address, change_type, storage_key, block_access_index);
+
+CREATE TABLE IF NOT EXISTS default.canonical_beacon_block_access_list ON CLUSTER '{cluster}'
+    AS default.canonical_beacon_block_access_list_local
+    ENGINE = Distributed('{cluster}', default, canonical_beacon_block_access_list_local, rand());

deploy/migrations/clickhouse/107_gloas_epbs_support.down.sql

@@ -0,0 +1,74 @@
+-- Reverse EIP-7732 ePBS support
+
+-- Drop new tables
+DROP TABLE IF EXISTS default.canonical_beacon_block_payload_attestation ON CLUSTER '{cluster}';
+DROP TABLE IF EXISTS default.canonical_beacon_block_payload_attestation_local ON CLUSTER '{cluster}';
+
+DROP TABLE IF EXISTS default.canonical_beacon_block_execution_payload_bid ON CLUSTER '{cluster}';
+DROP TABLE IF EXISTS default.canonical_beacon_block_execution_payload_bid_local ON CLUSTER '{cluster}';
+
+DROP TABLE IF EXISTS default.beacon_api_eth_v1_events_execution_payload ON CLUSTER '{cluster}';
+DROP TABLE IF EXISTS default.beacon_api_eth_v1_events_execution_payload_local ON CLUSTER '{cluster}';
+
+DROP TABLE IF EXISTS default.beacon_api_eth_v1_events_payload_attestation ON CLUSTER '{cluster}';
+DROP TABLE IF EXISTS default.beacon_api_eth_v1_events_payload_attestation_local ON CLUSTER '{cluster}';
+
+DROP TABLE IF EXISTS default.beacon_api_eth_v1_events_execution_payload_bid ON CLUSTER '{cluster}';
+DROP TABLE IF EXISTS default.beacon_api_eth_v1_events_execution_payload_bid_local ON CLUSTER '{cluster}';
+
+DROP TABLE IF EXISTS default.beacon_api_eth_v1_events_proposer_preferences ON CLUSTER '{cluster}';
+DROP TABLE IF EXISTS default.beacon_api_eth_v1_events_proposer_preferences_local ON CLUSTER '{cluster}';
+
+DROP TABLE IF EXISTS default.libp2p_gossipsub_execution_payload_envelope ON CLUSTER '{cluster}';
+DROP TABLE IF EXISTS default.libp2p_gossipsub_execution_payload_envelope_local ON CLUSTER '{cluster}';
+
+DROP TABLE IF EXISTS default.libp2p_gossipsub_execution_payload_bid ON CLUSTER '{cluster}';
+DROP TABLE IF EXISTS default.libp2p_gossipsub_execution_payload_bid_local ON CLUSTER '{cluster}';
+
+DROP TABLE IF EXISTS default.libp2p_gossipsub_payload_attestation_message ON CLUSTER '{cluster}';
+DROP TABLE IF EXISTS default.libp2p_gossipsub_payload_attestation_message_local ON CLUSTER '{cluster}';
+
+DROP TABLE IF EXISTS default.libp2p_gossipsub_proposer_preferences ON CLUSTER '{cluster}';
+DROP TABLE IF EXISTS default.libp2p_gossipsub_proposer_preferences_local ON CLUSTER '{cluster}';
+
+-- Remove ePBS columns from beacon block tables
+ALTER TABLE default.canonical_beacon_block ON CLUSTER '{cluster}'
+    DROP COLUMN IF EXISTS payload_present,
+    DROP COLUMN IF EXISTS execution_payment,
+    DROP COLUMN IF EXISTS bid_value,
+    DROP COLUMN IF EXISTS builder_index;
+
+ALTER TABLE default.canonical_beacon_block_local ON CLUSTER '{cluster}'
+    DROP COLUMN IF EXISTS payload_present,
+    DROP COLUMN IF EXISTS execution_payment,
+    DROP COLUMN IF EXISTS bid_value,
+    DROP COLUMN IF EXISTS builder_index;
+
+ALTER TABLE default.beacon_api_eth_v2_beacon_block ON CLUSTER '{cluster}'
+    DROP COLUMN IF EXISTS payload_present,
+    DROP COLUMN IF EXISTS execution_payment,
+    DROP COLUMN IF EXISTS bid_value,
+    DROP COLUMN IF EXISTS builder_index;
+
+ALTER TABLE default.beacon_api_eth_v2_beacon_block_local ON CLUSTER '{cluster}'
+    DROP COLUMN IF EXISTS payload_present,
+    DROP COLUMN IF EXISTS execution_payment,
+    DROP COLUMN IF EXISTS bid_value,
+    DROP COLUMN IF EXISTS builder_index;
+
+-- Remove Gloas columns from DataColumnSidecar tables
+ALTER TABLE default.beacon_api_eth_v1_events_data_column_sidecar ON CLUSTER '{cluster}'
+    DROP COLUMN IF EXISTS sidecar_beacon_block_root,
+    DROP COLUMN IF EXISTS sidecar_slot;
+
+ALTER TABLE default.beacon_api_eth_v1_events_data_column_sidecar_local ON CLUSTER '{cluster}'
+    DROP COLUMN IF EXISTS sidecar_beacon_block_root,
+    DROP COLUMN IF EXISTS sidecar_slot;
+
+ALTER TABLE default.libp2p_gossipsub_data_column_sidecar ON CLUSTER '{cluster}'
+    DROP COLUMN IF EXISTS sidecar_beacon_block_root,
+    DROP COLUMN IF EXISTS sidecar_slot;
+
+ALTER TABLE default.libp2p_gossipsub_data_column_sidecar_local ON CLUSTER '{cluster}'
+    DROP COLUMN IF EXISTS sidecar_beacon_block_root,
+    DROP COLUMN IF EXISTS sidecar_slot;