[DO-NOT-MERGE] glamsterdam-devnet-5

execution/engineapi/engine_server.go

@@ -388,8 +388,6 @@ func (s *EngineServer) newPayload(ctx context.Context, req *engine_types.Executi
 		}
 	}
 
-	log.Debug(fmt.Sprintf("bal from header: %s", blockAccessList.DebugString()))
-
 	if (!s.config.IsCancun(header.Time) && version >= clparams.DenebVersion) ||
 		(s.config.IsCancun(header.Time) && version < clparams.DenebVersion) ||
 		(!s.config.IsPrague(header.Time) && version >= clparams.ElectraVersion) ||

execution/execmodule/exec_module_test.go

@@ -39,6 +39,7 @@ import (
 	"github.com/erigontech/erigon/db/kv"
 	"github.com/erigontech/erigon/db/kv/rawdbv3"
 	"github.com/erigontech/erigon/db/rawdb"
+	"github.com/erigontech/erigon/db/state/changeset"
 	"github.com/erigontech/erigon/execution/builder"
 	"github.com/erigontech/erigon/execution/chain"
 	"github.com/erigontech/erigon/execution/commitment/commitmentdb"
@@ -2050,3 +2051,113 @@ func TestInsertBlocksWithBatchedFCU_BadBlockRecovery_Foreground(t *testing.T) {
 func TestInsertBlocksWithBatchedFCU_BadBlockRecovery_Background(t *testing.T) {
 	runBatchedFCUBadBlockRecovery(t, true)
 }
+
+// transferGen returns a deterministic per-block tx generator: identical
+// inputs produce identical blocks, which lets tests build forks that share
+// a prefix with the canonical chain (requires a pre-Cancun config — Cancun+
+// blocks get a random ParentBeaconBlockRoot in blockgen).
+func transferGen(t *testing.T, key *ecdsa.PrivateKey, to common.Address, amount uint64) func(int, *blockgen.BlockGen) {
+	return func(i int, b *blockgen.BlockGen) {
+		tx, err := types.SignTx(
+			types.NewTransaction(uint64(i), to, uint256.NewInt(amount), 50000, uint256.NewInt(1), nil),
+			*types.LatestSignerForChainID(nil),
+			key,
+		)
+		require.NoError(t, err)
+		b.AddTx(tx)
+	}
+}
+
+// A batch longer than MaxReorgDepth must still produce changesets for the
+// last MaxReorgDepth blocks, otherwise no reorg of any depth is possible
+// after the batch.
+func TestLargeBatchExecGeneratesChangesetsForReorgWindow(t *testing.T) {
+	ctx := t.Context()
+	privKey, err := crypto.GenerateKey()
+	require.NoError(t, err)
+	senderAddr := crypto.PubkeyToAddress(privKey.PublicKey)
+	m := execmoduletester.New(t,
+		execmoduletester.WithKey(privKey),
+		execmoduletester.WithAlwaysGenerateChangesets(false),
+	)
+
+	maxReorgDepth := m.Cfg().Sync.MaxReorgDepth
+	chainLen := int(maxReorgDepth) + 14
+
+	chainPack, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, chainLen,
+		transferGen(t, privKey, senderAddr, 1_000))
+	require.NoError(t, err)
+
+	insRes, err := insertBlocks(ctx, m.ExecModule, chainPack.Blocks)
+	require.NoError(t, err)
+	require.Equal(t, execmodule.ExecutionStatusSuccess, insRes)
+	fcuRes, err := updateForkChoice(ctx, m.ExecModule, chainPack.TopBlock.Header())
+	require.NoError(t, err)
+	require.Equal(t, execmodule.ExecutionStatusSuccess, fcuRes.Status)
+
+	require.NoError(t, m.DB.ViewTemporal(ctx, func(tx kv.TemporalTx) error {
+		lowest, err := changeset.ReadLowestUnwindableBlock(tx)
+		require.NoError(t, err)
+		require.Equal(t, uint64(chainLen)-maxReorgDepth, lowest,
+			"changesets must cover the last MaxReorgDepth blocks of the batch")
+		return nil
+	}))
+}
+
+// After a single batch execution longer than MaxReorgDepth, an FCU onto a
+// fork branching a few blocks below the tip must unwind and re-execute
+// instead of failing with ReorgTooDeep.
+func TestUpdateForkChoiceShallowReorgAfterLargeBatchExec(t *testing.T) {
+	ctx := t.Context()
+	privKey, err := crypto.GenerateKey()
+	require.NoError(t, err)
+	senderAddr := crypto.PubkeyToAddress(privKey.PublicKey)
+	m := execmoduletester.New(t,
+		execmoduletester.WithKey(privKey),
+		execmoduletester.WithAlwaysGenerateChangesets(false),
+	)
+
+	maxReorgDepth := m.Cfg().Sync.MaxReorgDepth
+	chainLen := int(maxReorgDepth) + 14
+	const reorgDepth = 4
+	divergeFrom := chainLen - reorgDepth
+
+	canonical, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, chainLen,
+		transferGen(t, privKey, senderAddr, 1_000))
+	require.NoError(t, err)
+	fork, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, chainLen,
+		func(i int, b *blockgen.BlockGen) {
+			amount := uint64(1_000)
+			if i >= divergeFrom {
+				amount = 2_000
+			}
+			transferGen(t, privKey, senderAddr, amount)(i, b)
+		})
+	require.NoError(t, err)
+	require.Equal(t, canonical.Blocks[divergeFrom-1].Hash(), fork.Blocks[divergeFrom-1].Hash())
+	require.NotEqual(t, canonical.Blocks[divergeFrom].Hash(), fork.Blocks[divergeFrom].Hash())
+
+	insRes, err := insertBlocks(ctx, m.ExecModule, canonical.Blocks)
+	require.NoError(t, err)
+	require.Equal(t, execmodule.ExecutionStatusSuccess, insRes)
+	fcuRes, err := updateForkChoice(ctx, m.ExecModule, canonical.TopBlock.Header())
+	require.NoError(t, err)
+	require.Equal(t, execmodule.ExecutionStatusSuccess, fcuRes.Status)
+
+	insRes, err = insertBlocks(ctx, m.ExecModule, fork.Blocks[divergeFrom:])
+	require.NoError(t, err)
+	require.Equal(t, execmodule.ExecutionStatusSuccess, insRes)
+	fcuRes, err = updateForkChoice(ctx, m.ExecModule, fork.TopBlock.Header())
+	require.NoError(t, err)
+	require.Equal(t, execmodule.ExecutionStatusSuccess, fcuRes.Status,
+		"shallow reorg of %d blocks after a %d-block batch must succeed; status=%s validationError=%q",
+		reorgDepth, chainLen, fcuRes.Status, fcuRes.ValidationError)
+
+	require.NoError(t, m.DB.ViewTemporal(ctx, func(tx kv.TemporalTx) error {
+		execProg, err := stages.GetStageProgress(tx, stages.Execution)
+		require.NoError(t, err)
+		require.Equal(t, uint64(chainLen), execProg)
+		require.Equal(t, fork.TopBlock.Hash(), rawdb.ReadHeadBlockHash(tx))
+		return nil
+	}))
+}

execution/execmodule/execmoduletester/exec_module_tester.go

@@ -333,24 +333,35 @@ func WithFcuBackgroundCommit() Option {
 	}
 }
 
+// WithAlwaysGenerateChangesets pins --experimental.always-generate-changesets
+// regardless of the tester default: true for tests that reorg deeper than
+// MaxReorgDepth, false for tests that rely on the windowed-changesets
+// production behaviour.
+func WithAlwaysGenerateChangesets(v bool) Option {
+	return func(opts *options) {
+		opts.alwaysGenerateChangesets = &v
+	}
+}
+
 func WithFcuBackgroundPrune() Option {
 	return func(opts *options) {
 		opts.fcuBackgroundPrune = true
 	}
 }
 
 type options struct {
-	stepSize            *uint64
-	experimentalBAL     bool
-	genesis             *types.Genesis
-	chainConfig         *chain.Config
-	key                 *ecdsa.PrivateKey
-	engine              rules.Engine
-	pruneMode           *prune.Mode
-	withTxPool          bool
-	enableDomains       []kv.Domain
-	fcuBackgroundCommit bool
-	fcuBackgroundPrune  bool
+	stepSize                 *uint64
+	experimentalBAL          bool
+	genesis                  *types.Genesis
+	chainConfig              *chain.Config
+	key                      *ecdsa.PrivateKey
+	engine                   rules.Engine
+	pruneMode                *prune.Mode
+	withTxPool               bool
+	enableDomains            []kv.Domain
+	fcuBackgroundCommit      bool
+	fcuBackgroundPrune       bool
+	alwaysGenerateChangesets *bool
 }
 
 func applyOptions(opts []Option) options {
@@ -420,7 +431,9 @@ func New(tb testing.TB, opts ...Option) *ExecModuleTester {
 	cfg.Sync.BodyDownloadTimeoutSeconds = 10
 	cfg.TxPool.Disable = !withTxPool
 	cfg.Dirs = dirs
-	cfg.AlwaysGenerateChangesets = true
+	if opt.alwaysGenerateChangesets != nil {
+		cfg.AlwaysGenerateChangesets = *opt.alwaysGenerateChangesets
+	}
 	cfg.PersistReceiptsCacheV2 = true
 	cfg.ChaosMonkey = false
 	cfg.Snapshot.ChainName = gspec.Config.ChainName

execution/stagedsync/bal_create.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/erigontech/erigon/common"
 	"github.com/erigontech/erigon/common/dbg"
 	"github.com/erigontech/erigon/common/log/v3"
 	"github.com/erigontech/erigon/db/kv"
@@ -14,40 +15,31 @@ import (
 	"github.com/erigontech/erigon/execution/types"
 )
 
-func CreateBAL(blockNum uint64, txIO *state.VersionedIO, dataDir string) types.BlockAccessList {
+func CreateBAL(blockNum uint64, txIO *state.VersionedIO, dataDir string, logger log.Logger) types.BlockAccessList {
 	bal := txIO.AsBlockAccessList()
 	if dbg.TraceBlockAccessLists {
-		writeBALToFile(bal, blockNum, dataDir)
+		writeBALToFile(bal, dataDir, fmt.Sprintf("computed_bal_%d.txt", blockNum), logger)
 	}
 	return bal
 }
 
-// writeBALToFile writes the Block Access List to a text file for debugging/analysis
-func writeBALToFile(bal types.BlockAccessList, blockNum uint64, dataDir string) {
+// writeBALToFile dumps the Block Access List to <dataDir>/bal/<name> for debugging/analysis.
+func writeBALToFile(bal types.BlockAccessList, dataDir string, name string, logger log.Logger) {
 	if dataDir == "" {
 		return
 	}
-
 	balDir := filepath.Join(dataDir, "bal")
-	if err := os.MkdirAll(balDir, 0755); err != nil {
-		log.Warn("Failed to create BAL directory", "dir", balDir, "error", err)
+	if err := os.MkdirAll(balDir, 0o755); err != nil {
+		logger.Warn("failed to create BAL debug directory", "dir", balDir, "err", err)
 		return
 	}
-
-	filename := filepath.Join(balDir, fmt.Sprintf("bal_block_%d.txt", blockNum))
-
-	file, err := os.Create(filename)
-	if err != nil {
-		log.Warn("Failed to create BAL file", "blockNum", blockNum, "error", err)
-		return
+	path := filepath.Join(balDir, name)
+	if err := os.WriteFile(path, []byte(bal.DebugString()), 0o644); err != nil {
+		logger.Warn("failed to write BAL debug file", "path", path, "err", err)
 	}
-	defer file.Close()
-
-	fmt.Fprintf(file, "Block Access List for Block %d\n", blockNum)
-	bal.DebugPrint(file)
 }
 
-func ProcessBAL(tx kv.TemporalRwTx, h *types.Header, vio *state.VersionedIO, isEIP7928 bool, experimental bool, dataDir string) error {
+func ProcessBAL(tx kv.TemporalRwTx, h *types.Header, vio *state.VersionedIO, isEIP7928 bool, experimental bool, dataDir string, logger log.Logger) error {
 	if !isEIP7928 && !experimental {
 		return nil
 	}
@@ -56,97 +48,63 @@ func ProcessBAL(tx kv.TemporalRwTx, h *types.Header, vio *state.VersionedIO, isE
 	}
 	blockNum := h.Number.Uint64()
 	blockHash := h.Hash()
-	bal := CreateBAL(blockNum, vio, dataDir)
-	err := bal.Validate()
+	computedBlockBal := CreateBAL(blockNum, vio, dataDir, logger)
+	err := computedBlockBal.Validate()
 	if err != nil {
 		return fmt.Errorf("block %d: invalid computed block access list: %w", blockNum, err)
 	}
-	log.Debug("bal", "blockNum", blockNum, "hash", bal.Hash())
 	if !isEIP7928 {
 		return nil
 	}
 	// EIP-7928 size bound is only consensus-binding once Amsterdam activates.
-	if err := bal.ValidateMaxItems(h.GasLimit); err != nil {
+	if err := computedBlockBal.ValidateMaxItems(h.GasLimit); err != nil {
 		return fmt.Errorf("%w, block %d: %w", rules.ErrInvalidBlock, blockNum, err)
 	}
 	if h.BlockAccessListHash == nil {
 		return fmt.Errorf("block %d: EIP-7928 active but BlockAccessListHash is nil in header", blockNum)
 	}
-	headerBALHash := *h.BlockAccessListHash
-	dbBALBytes, err := rawdb.ReadBlockAccessListBytes(tx, blockHash, blockNum)
+	blockBalHash := *h.BlockAccessListHash
+	blockBalBytes, err := rawdb.ReadBlockAccessListBytes(tx, blockHash, blockNum)
 	if err != nil {
 		return fmt.Errorf("block %d: read stored block access list: %w", blockNum, err)
 	}
 	// A stored BAL sidecar may be absent — eth/71 backfill is best-effort and
 	// never blocks stage progress — so cross-check it only when present.
-	if dbBALBytes != nil {
-		dbBAL, err := types.DecodeBlockAccessListBytes(dbBALBytes)
+	var blockBal types.BlockAccessList
+	if blockBalBytes != nil {
+		blockBal, err = types.DecodeBlockAccessListBytes(blockBalBytes)
 		if err != nil {
 			return fmt.Errorf("block %d: read stored block access list: %w", blockNum, err)
 		}
-		if err = dbBAL.Validate(); err != nil {
+		if err = blockBal.Validate(); err != nil {
 			return fmt.Errorf("block %d: db block access list is invalid: %w", blockNum, err)
 		}
-		if err = dbBAL.ValidateMaxItems(h.GasLimit); err != nil {
+		if err = blockBal.ValidateMaxItems(h.GasLimit); err != nil {
 			return fmt.Errorf("block %d: stored block access list exceeds max items: %w", blockNum, err)
 		}
-
-		if headerBALHash != dbBAL.Hash() {
-			log.Info(fmt.Sprintf("bal from block: %s", dbBAL.DebugString()))
-			return fmt.Errorf("block %d: invalid block access list, hash mismatch: got %s expected %s", blockNum, dbBAL.Hash(), headerBALHash)
+		if blockBalHash != blockBal.Hash() {
+			reportBALMismatch(blockNum, blockHash, blockBal, blockBalHash, computedBlockBal, dataDir, logger)
+			return fmt.Errorf("block %d: invalid block access list, hash mismatch: got %s expected %s", blockNum, blockBal.Hash(), blockBalHash)
 		}
 	}
 	// Always validate computed BAL against header. The BalancePath cross-check
 	// in VersionMap.validateRead ensures deterministic parallel execution even
 	// without a stored BAL body (HasBAL=false), so the computed BAL is accurate.
-	if headerBALHash != bal.Hash() {
-		// Dump full BAL DebugStrings to stderr via the logger so container
-		// stdout captures (kurtosis/docker log collection) preserve the diff
-		// even when the on-disk dumpDir below is unreachable (e.g. ephemeral
-		// container, CI runner without artifact capture of /data).
-		// Each DebugString is one multi-line value tagged with block metadata
-		// so operators can diff computed vs stored without hunting for files.
-		computedDebug := bal.DebugString()
-		var storedDebug string
-		if dbBALBytes != nil {
-			dbBAL2, decErr := types.DecodeBlockAccessListBytes(dbBALBytes)
-			if decErr != nil {
-				log.Warn("failed to decode stored BAL for debug dump", "err", decErr)
-			} else if dbBAL2 != nil {
-				storedDebug = dbBAL2.DebugString()
-			}
-		}
-		log.Error("BAL mismatch: computed", "block", blockNum, "hash", bal.Hash(), "headerHash", headerBALHash, "bal", computedDebug)
-		if storedDebug != "" {
-			log.Error("BAL mismatch: stored (from sidecar)", "block", blockNum, "hash", headerBALHash, "bal", storedDebug)
-		}
-
-		dumpDir := ""
-		if dataDir != "" {
-			balDir := filepath.Join(dataDir, "bal")
-			if err := os.MkdirAll(balDir, 0o755); err != nil {
-				log.Warn("failed to create BAL debug directory", "dir", balDir, "err", err)
-			} else {
-				computedPath := filepath.Join(balDir, fmt.Sprintf("computed_bal_%d.txt", blockNum))
-				if err := os.WriteFile(computedPath, []byte(computedDebug), 0o644); err != nil {
-					log.Warn("failed to write computed BAL debug file", "path", computedPath, "err", err)
-				} else {
-					dumpDir = balDir
-				}
-				if storedDebug != "" {
-					storedPath := filepath.Join(balDir, fmt.Sprintf("stored_bal_%d.txt", blockNum))
-					if err := os.WriteFile(storedPath, []byte(storedDebug), 0o644); err != nil {
-						log.Warn("failed to write stored BAL debug file", "path", storedPath, "err", err)
-					}
-				}
-			}
-		}
-		if dumpDir != "" {
-			return fmt.Errorf("%w, block=%d (hash=%s): block access list mismatch: got %s expected %s; debug dumps in %s",
-				rules.ErrInvalidBlock, blockNum, blockHash, bal.Hash(), headerBALHash, dumpDir)
-		}
+	computedBlockBalHash := computedBlockBal.Hash()
+	if blockBalHash != computedBlockBalHash {
+		reportBALMismatch(blockNum, blockHash, blockBal, blockBalHash, computedBlockBal, dataDir, logger)
 		return fmt.Errorf("%w, block=%d (hash=%s): block access list mismatch: got %s expected %s",
-			rules.ErrInvalidBlock, blockNum, blockHash, bal.Hash(), headerBALHash)
+			rules.ErrInvalidBlock, blockNum, blockHash, computedBlockBalHash, blockBalHash)
 	}
 	return nil
 }
+
+// reportBALMismatch logs a BAL hash mismatch and dumps the block's BAL and the
+// computed one under <dataDir>/bal for offline diffing.
+func reportBALMismatch(blockNum uint64, blockHash common.Hash, blockBal types.BlockAccessList, blockBalHash common.Hash, computedBlockBal types.BlockAccessList, dataDir string, logger log.Logger) {
+	logger.Error("BAL mismatch", "blockNum", blockNum, "blockHash", blockHash, "blockBalHash", blockBalHash, "computedBlockBalHash", computedBlockBal.Hash())
+	writeBALToFile(computedBlockBal, dataDir, fmt.Sprintf("computed_bal_%d.txt", blockNum), logger)
+	if blockBal != nil {
+		writeBALToFile(blockBal, dataDir, fmt.Sprintf("block_bal_%d.txt", blockNum), logger)
+	}
+}