Allow LH to checkpoint sync into a non-finalized checkpoint

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -5874,7 +5874,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                     chain
                         .canonical_head
                         .fork_choice_read_lock()
-                        .get_justified_block()
+                        .get_justified_or_anchor_block()
                 },
                 "invalid_payload_fork_choice_get_justified",
             )

beacon_node/beacon_chain/src/beacon_fork_choice_store.rs

@@ -187,13 +187,8 @@ where
         if anchor_block_header.state_root.is_zero() {
             anchor_block_header.state_root = unadvanced_state_root;
         }
-        let anchor_block_root = anchor_block_header.canonical_root();
-        let anchor_epoch = anchor_state.current_epoch();
-        let justified_checkpoint = Checkpoint {
-            epoch: anchor_epoch,
-            root: anchor_block_root,
-        };
-        let finalized_checkpoint = justified_checkpoint;
+        let justified_checkpoint = anchor_state.current_justified_checkpoint();
+        let finalized_checkpoint = anchor_state.finalized_checkpoint();
         let justified_balances = JustifiedBalances::from_justified_state(&anchor_state)?;
         let justified_state_root = anchor_state.canonical_root()?;
 

beacon_node/beacon_chain/src/canonical_head.rs

@@ -115,6 +115,7 @@ pub struct CachedHead<E: EthSpec> {
     justified_hash: Option<ExecutionBlockHash>,
     /// The `execution_payload.block_hash` of the finalized block. Set to `None` before Bellatrix.
     finalized_hash: Option<ExecutionBlockHash>,
+    pub anchor_block: (Hash256, Slot),
 }
 
 impl<E: EthSpec> CachedHead<E> {
@@ -221,6 +222,10 @@ impl<E: EthSpec> CachedHead<E> {
         self.justified_checkpoint
     }
 
+    pub fn finalized_checkpoint_from_state(&self) -> Checkpoint {
+        self.snapshot.beacon_state.finalized_checkpoint()
+    }
+
     /// Returns the cached values of `ForkChoice::forkchoice_update_parameters`.
     ///
     /// Useful for supplying to the execution layer.
@@ -272,6 +277,7 @@ impl<T: BeaconChainTypes> CanonicalHead<T> {
             head_hash: forkchoice_update_params.head_hash,
             justified_hash: forkchoice_update_params.justified_hash,
             finalized_hash: forkchoice_update_params.finalized_hash,
+            anchor_block: fork_choice.get_anchor_block(),
         };
 
         Self {
@@ -323,6 +329,7 @@ impl<T: BeaconChainTypes> CanonicalHead<T> {
             head_hash: forkchoice_update_params.head_hash,
             justified_hash: forkchoice_update_params.justified_hash,
             finalized_hash: forkchoice_update_params.finalized_hash,
+            anchor_block: fork_choice.get_anchor_block(),
         };
 
         *fork_choice_write_lock = fork_choice;
@@ -608,8 +615,9 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
 
         // Check to ensure that the finalized block hasn't been marked as invalid. If it has,
         // shut down Lighthouse.
-        let finalized_proto_block = fork_choice_read_lock.get_finalized_block()?;
+        let finalized_proto_block = fork_choice_read_lock.get_finalized_or_anchor_block()?;
         check_finalized_payload_validity(self, &finalized_proto_block)?;
+        let anchor_block = fork_choice_read_lock.get_anchor_block();
 
         // Sanity check the finalized checkpoint.
         //
@@ -700,6 +708,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                 head_hash: new_forkchoice_update_parameters.head_hash,
                 justified_hash: new_forkchoice_update_parameters.justified_hash,
                 finalized_hash: new_forkchoice_update_parameters.finalized_hash,
+                anchor_block,
             };
 
             let new_head = {
@@ -727,6 +736,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                 head_hash: new_forkchoice_update_parameters.head_hash,
                 justified_hash: new_forkchoice_update_parameters.justified_hash,
                 finalized_hash: new_forkchoice_update_parameters.finalized_hash,
+                anchor_block,
             };
 
             let mut cached_head_write_lock = self.canonical_head.cached_head_write_lock();

beacon_node/beacon_chain/src/fork_revert.rs

@@ -98,59 +98,49 @@ pub fn reset_fork_choice_to_finalization<E: EthSpec, Hot: ItemStore<E>, Cold: It
     current_slot: Option<Slot>,
     spec: &ChainSpec,
 ) -> Result<ForkChoice<BeaconForkChoiceStore<E, Hot, Cold>, E>, String> {
-    // Fetch finalized block.
-    let finalized_checkpoint = head_state.finalized_checkpoint();
-    let finalized_block_root = finalized_checkpoint.root;
-    let finalized_block = store
-        .get_full_block(&finalized_block_root)
-        .map_err(|e| format!("Error loading finalized block: {:?}", e))?
+    // Fetch the store split as the most recent state internally considered finalized. If the node
+    // started with checkpoint sync and before its first finalization the split state does not equal
+    // to the last finalized state; and the last finalized state is unavailable.
+    let split = store.get_split_info();
+    let new_anchor_block_root = split.block_root;
+    let new_anchor_slot = split.slot;
+    let new_anchor_block = store
+        .get_full_block(&new_anchor_block_root)
+        .map_err(|e| format!("Error loading split block: {:?}", e))?
         .ok_or_else(|| {
             format!(
-                "Finalized block missing for revert: {:?}",
-                finalized_block_root
+                "Split block missing for revert: {:?}",
+                new_anchor_block_root
             )
         })?;
 
     // Advance finalized state to finalized epoch (to handle skipped slots).
-    let finalized_state_root = finalized_block.state_root();
+    let new_anchor_state_root = split.state_root;
     // The enshrined finalized state should be in the state cache.
-    let mut finalized_state = store
-        .get_state(&finalized_state_root, Some(finalized_block.slot()), true)
-        .map_err(|e| format!("Error loading finalized state: {:?}", e))?
+    let new_anchor_state = store
+        .get_state(&new_anchor_state_root, Some(new_anchor_slot), true)
+        .map_err(|e| format!("Error loading split state: {:?}", e))?
         .ok_or_else(|| {
             format!(
-                "Finalized block state missing from database: {:?}",
-                finalized_state_root
+                "Split block state missing from database: {:?}",
+                new_anchor_state_root
             )
         })?;
-    let finalized_slot = finalized_checkpoint.epoch.start_slot(E::slots_per_epoch());
-    complete_state_advance(
-        &mut finalized_state,
-        Some(finalized_state_root),
-        finalized_slot,
-        spec,
-    )
-    .map_err(|e| {
-        format!(
-            "Error advancing finalized state to finalized epoch: {:?}",
-            e
-        )
-    })?;
-    let finalized_snapshot = BeaconSnapshot {
-        beacon_block_root: finalized_block_root,
-        beacon_block: Arc::new(finalized_block),
-        beacon_state: finalized_state,
+    let new_anchor_snapshot = BeaconSnapshot {
+        beacon_block_root: new_anchor_block_root,
+        beacon_block: Arc::new(new_anchor_block),
+        beacon_state: new_anchor_state,
     };
 
     let fc_store =
-        BeaconForkChoiceStore::get_forkchoice_store(store.clone(), finalized_snapshot.clone())
+        BeaconForkChoiceStore::get_forkchoice_store(store.clone(), new_anchor_snapshot.clone())
             .map_err(|e| format!("Unable to reset fork choice store for revert: {e:?}"))?;
 
     let mut fork_choice = ForkChoice::from_anchor(
         fc_store,
-        finalized_block_root,
-        &finalized_snapshot.beacon_block,
-        &finalized_snapshot.beacon_state,
+        new_anchor_block_root,
+        &new_anchor_snapshot.beacon_block,
+        &new_anchor_snapshot.beacon_state,
         current_slot,
         spec,
     )
@@ -160,10 +150,10 @@ pub fn reset_fork_choice_to_finalization<E: EthSpec, Hot: ItemStore<E>, Cold: It
     // We do not replay attestations presently, relying on the absence of other blocks
     // to guarantee `head_block_root` as the head.
     let blocks = store
-        .load_blocks_to_replay(finalized_slot + 1, head_state.slot(), head_block_root)
+        .load_blocks_to_replay(new_anchor_slot + 1, head_state.slot(), head_block_root)
         .map_err(|e| format!("Error loading blocks to replay for fork choice: {:?}", e))?;
 
-    let mut state = finalized_snapshot.beacon_state;
+    let mut state = new_anchor_snapshot.beacon_state;
     for block in blocks {
         complete_state_advance(&mut state, None, block.slot(), spec)
             .map_err(|e| format!("State advance failed: {:?}", e))?;

beacon_node/client/src/notifier.rs

@@ -120,7 +120,24 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
             let cached_head = beacon_chain.canonical_head.cached_head();
             let head_slot = cached_head.head_slot();
             let head_root = cached_head.head_block_root();
-            let finalized_checkpoint = cached_head.finalized_checkpoint();
+            let finalized_checkpoint = cached_head.finalized_checkpoint_from_state();
+            let finalized_root_str = {
+                let finalized_slot = finalized_checkpoint
+                    .epoch
+                    .start_slot(T::EthSpec::slots_per_epoch());
+                if cached_head.anchor_block.1 > finalized_slot {
+                    // Anchor block ahead of finalized checkpoint, the node has a subjective concept
+                    // of finality and will reject blocks that other nodes in the network will
+                    // accept
+                    format!(
+                        "{}/anchor/{}",
+                        finalized_checkpoint.root, cached_head.anchor_block.0
+                    )
+                } else {
+                    // Regular mode
+                    format!("{}", finalized_checkpoint.root)
+                }
+            };
 
             metrics::set_gauge(&metrics::NOTIFIER_HEAD_SLOT, head_slot.as_u64() as i64);
 
@@ -179,7 +196,7 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
 
             debug!(
                 peers = peer_count_pretty(connected_peer_count),
-                finalized_root = %finalized_checkpoint.root,
+                finalized_root = finalized_root_str,
                 finalized_epoch = %finalized_checkpoint.epoch,
                 head_block = %head_root,
                 %head_slot,
@@ -253,6 +270,7 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
                         speed = sync_speed_pretty(speed),
                         est_time =
                             estimated_time_pretty(speedo.estimated_time_till_slot(current_slot)),
+                        finalized_root = finalized_root_str,
                         "Syncing"
                     );
                 } else {
@@ -261,6 +279,7 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
                         distance,
                         est_time =
                             estimated_time_pretty(speedo.estimated_time_till_slot(current_slot)),
+                        finalized_root = finalized_root_str,
                         "Syncing"
                     );
                 }
@@ -298,7 +317,7 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
                 info!(
                     peers = peer_count_pretty(connected_peer_count),
                     exec_hash = block_hash,
-                    finalized_root = %finalized_checkpoint.root,
+                    finalized_root = finalized_root_str,
                     finalized_epoch = %finalized_checkpoint.epoch,
                     epoch = %current_epoch,
                     block = block_info,
@@ -309,7 +328,7 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
                 metrics::set_gauge(&metrics::IS_SYNCED, 0);
                 info!(
                     peers = peer_count_pretty(connected_peer_count),
-                    finalized_root = %finalized_checkpoint.root,
+                    finalized_root = finalized_root_str,
                     finalized_epoch = %finalized_checkpoint.epoch,
                     %head_slot,
                     %current_slot,

beacon_node/network/src/network_beacon_processor/rpc_methods.rs

@@ -108,11 +108,7 @@ impl<T: BeaconChainTypes> NetworkBeaconProcessor<T> {
         } else {
             // Remote finalized epoch is less than ours.
             let remote_finalized_slot = start_slot(*remote.finalized_epoch());
-            if remote_finalized_slot < self.chain.store.get_oldest_block_slot() {
-                // Peer's finalized checkpoint is older than anything in our DB. We are unlikely
-                // to be able to help them sync.
-                Some("Old finality out of range".to_string())
-            } else if remote_finalized_slot < self.chain.store.get_split_slot() {
+            if remote_finalized_slot < self.chain.store.get_split_slot() {
                 // Peer's finalized slot is in range for a quick block root check in our freezer DB.
                 // If that block root check fails, reject them as they're on a different finalized
                 // chain.

beacon_node/network/src/status.rs

@@ -20,7 +20,7 @@ impl<T: BeaconChainTypes> ToStatusMessage for BeaconChain<T> {
 pub(crate) fn status_message<T: BeaconChainTypes>(beacon_chain: &BeaconChain<T>) -> StatusMessage {
     let fork_digest = beacon_chain.enr_fork_id().fork_digest;
     let cached_head = beacon_chain.canonical_head.cached_head();
-    let mut finalized_checkpoint = cached_head.finalized_checkpoint();
+    let mut finalized_checkpoint = cached_head.finalized_checkpoint_from_state();
 
     // Alias the genesis checkpoint root to `0x00`.
     let spec = &beacon_chain.spec;

beacon_node/network/src/sync/range_sync/range.rs

@@ -147,11 +147,19 @@ where
                 let target_head_slot =
                     remote_finalized_slot + (2 * T::EthSpec::slots_per_epoch()) + 1;
 
+                // If the node started with checkpoint sync, local_info.finalized_epoch might be
+                // older than the current store split.
+                let start_epoch = self
+                    .beacon_chain
+                    .store
+                    .get_split_slot()
+                    .epoch(T::EthSpec::slots_per_epoch());
+
                 // Note: We keep current head chains. These can continue syncing whilst we complete
                 // this new finalized chain.
 
                 self.chains.add_peer_or_create_chain(
-                    local_info.finalized_epoch,
+                    start_epoch,
                     remote_info.finalized_root,
                     target_head_slot,
                     peer_id,