add configurable historical balance health check for finalized block …

CONFIG.md

@@ -911,6 +911,8 @@ SyncThreshold = 5 # Default
 LeaseDuration = '0s' # Default
 NodeIsSyncingEnabled = false # Default
 FinalizedBlockPollInterval = '5s' # Default
+HistoricalBalanceCheckEnabled = false # Default
+HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000000' # Default
 EnforceRepeatableRead = true # Default
 DeathDeclarationDelay = '1m' # Default
 NewHeadsPollInterval = '0s' # Default
@@ -989,6 +991,23 @@ reported based on latest block and finality depth.
 
 Set to 0 to disable.
 
+### HistoricalBalanceCheckEnabled
+```toml
+HistoricalBalanceCheckEnabled = false # Default
+```
+HistoricalBalanceCheckEnabled controls whether NodePool health polling also verifies historical state availability
+by executing `eth_getBalance` for HistoricalBalanceCheckAddress at the latest finalized block.
+Finalized block selection follows chain finality settings:
+- `FinalityTagEnabled = true`: use `finalized` tag
+- `FinalityTagEnabled = false`: use `latest - FinalityDepth`
+
+### HistoricalBalanceCheckAddress
+```toml
+HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000000' # Default
+```
+HistoricalBalanceCheckAddress is the probe account used by the historical balance health check.
+This check is only active when `HistoricalBalanceCheckEnabled = true`.
+
 ### EnforceRepeatableRead
 ```toml
 EnforceRepeatableRead = true # Default

pkg/client/helpers_test.go

@@ -95,6 +95,8 @@ type TestNodePoolConfig struct {
 	NodeLeaseDuration                 time.Duration
 	NodeIsSyncingEnabledVal           bool
 	NodeFinalizedBlockPollInterval    time.Duration
+	HistoricalBalanceCheckEnabledVal  bool
+	HistoricalBalanceCheckAddressVal  string
 	NodeErrors                        config.ClientErrors
 	EnforceRepeatableReadVal          bool
 	NodeDeathDeclarationDelay         time.Duration
@@ -118,6 +120,14 @@ func (tc TestNodePoolConfig) FinalizedBlockPollInterval() time.Duration {
 	return tc.NodeFinalizedBlockPollInterval
 }
 
+func (tc TestNodePoolConfig) HistoricalBalanceCheckEnabled() bool {
+	return tc.HistoricalBalanceCheckEnabledVal
+}
+
+func (tc TestNodePoolConfig) HistoricalBalanceCheckAddress() string {
+	return tc.HistoricalBalanceCheckAddressVal
+}
+
 func (tc TestNodePoolConfig) NewHeadsPollInterval() time.Duration {
 	return tc.NodeNewHeadsPollInterval
 }

pkg/client/rpc_client.go

@@ -105,6 +105,8 @@ type RPCClient struct {
 	finalityTagEnabled             bool
 	finalityDepth                  uint32
 	safeDepth                      uint32
+	historicalBalanceCheckEnabled  bool
+	historicalBalanceCheckAddress  common.Address
 	externalRequestMaxResponseSize uint32
 
 	ws        atomic.Pointer[rawclient]
@@ -142,6 +144,8 @@ func NewRPCClient(
 		finalityTagEnabled:             supportsFinalityTags,
 		finalityDepth:                  finalityDepth,
 		safeDepth:                      safeDepth,
+		historicalBalanceCheckEnabled:  cfg.HistoricalBalanceCheckEnabled(),
+		historicalBalanceCheckAddress:  common.HexToAddress(cfg.HistoricalBalanceCheckAddress()),
 		externalRequestMaxResponseSize: externalRequestMaxResponseSize,
 	}
 	r.cfg = cfg
@@ -180,10 +184,35 @@ func (r *RPCClient) ClientVersion(ctx context.Context) (version string, err erro
 	if err != nil {
 		return "", fmt.Errorf("fetching client version failed: %w", err)
 	}
+	if r.historicalBalanceCheckEnabled {
+		if err = r.checkHistoricalStateAtFinalized(ctx); err != nil {
+			return "", fmt.Errorf("historical balance health check failed: %w", err)
+		}
+	}
 	r.rpcLog.Debugf("client version: %s", version)
 	return version, nil
 }
 
+func (r *RPCClient) checkHistoricalStateAtFinalized(ctx context.Context) error {
+	var blockNumber *big.Int
+	if r.finalityTagEnabled {
+		blockNumber = big.NewInt(rpc.FinalizedBlockNumber.Int64())
+	} else {
+		latestBlock, err := r.BlockNumber(ctx)
+		if err != nil {
+			return fmt.Errorf("fetching latest block number failed: %w", err)
+		}
+		latest := int64(latestBlock)
+		finalizedHeight := max(int64(0), latest-int64(r.finalityDepth))
+		blockNumber = big.NewInt(finalizedHeight)
+	}
+	_, err := r.BalanceAt(ctx, r.historicalBalanceCheckAddress, blockNumber)
+	if err != nil {
+		return fmt.Errorf("fetching balance for address %s at block %s failed: %w", r.historicalBalanceCheckAddress.String(), blockNumber.String(), err)
+	}
+	return nil
+}
+
 func (r *RPCClient) Dial(callerCtx context.Context) error {
 	ctx, cancel, _ := r.AcquireQueryCtx(callerCtx, r.rpcTimeout)
 	defer cancel()

pkg/client/rpc_client_internal_test.go

@@ -436,3 +436,135 @@ func NewTestRPCClient(t *testing.T, opts RPCClientOpts) *RPCClient {
 func ptr[T any](v T) *T {
 	return &v
 }
+
+func TestRPCClient_ClientVersion_HistoricalBalanceCheck(t *testing.T) {
+	t.Parallel()
+	chainID := big.NewInt(1337)
+	probeAddress := "0x0000000000000000000000000000000000000001"
+
+	t.Run("disabled only calls web3_clientVersion", func(t *testing.T) {
+		t.Parallel()
+		methodCalls := make([]string, 0)
+		wsURL := testutils.NewWSServer(t, chainID, func(method string, _ gjson.Result) (resp testutils.JSONRPCResponse) {
+			methodCalls = append(methodCalls, method)
+			switch method {
+			case "web3_clientVersion":
+				resp.Result = `"test-client"`
+			default:
+				require.Fail(t, "unexpected method: "+method)
+			}
+			return
+		}).WSURL()
+
+		rpcClient := NewDialedTestRPCClient(t, RPCClientOpts{
+			HTTP: wsURL,
+			Cfg: &TestNodePoolConfig{
+				NodeFinalizedBlockPollInterval: 1 * time.Second,
+			},
+			FinalityTagsEnabled: true,
+			ChainID:             chainID,
+		})
+
+		version, err := rpcClient.ClientVersion(t.Context())
+		require.NoError(t, err)
+		require.Equal(t, "test-client", version)
+		require.Equal(t, []string{"web3_clientVersion"}, methodCalls)
+	})
+
+	t.Run("enabled uses finalized tag", func(t *testing.T) {
+		t.Parallel()
+		wsURL := testutils.NewWSServer(t, chainID, func(method string, params gjson.Result) (resp testutils.JSONRPCResponse) {
+			switch method {
+			case "web3_clientVersion":
+				resp.Result = `"test-client"`
+			case "eth_getBalance":
+				require.Equal(t, probeAddress, params.Array()[0].String())
+				require.Equal(t, "finalized", params.Array()[1].String())
+				resp.Result = `"0x0"`
+			default:
+				require.Fail(t, "unexpected method: "+method)
+			}
+			return
+		}).WSURL()
+
+		rpcClient := NewDialedTestRPCClient(t, RPCClientOpts{
+			HTTP: wsURL,
+			Cfg: &TestNodePoolConfig{
+				NodeFinalizedBlockPollInterval:   1 * time.Second,
+				HistoricalBalanceCheckEnabledVal: true,
+				HistoricalBalanceCheckAddressVal: probeAddress,
+			},
+			FinalityTagsEnabled: true,
+			ChainID:             chainID,
+		})
+
+		version, err := rpcClient.ClientVersion(t.Context())
+		require.NoError(t, err)
+		require.Equal(t, "test-client", version)
+	})
+
+	t.Run("enabled in depth mode uses latest-finalityDepth", func(t *testing.T) {
+		t.Parallel()
+		wsURL := testutils.NewWSServer(t, chainID, func(method string, params gjson.Result) (resp testutils.JSONRPCResponse) {
+			switch method {
+			case "web3_clientVersion":
+				resp.Result = `"test-client"`
+			case "eth_blockNumber":
+				resp.Result = `"0x14"` // 20
+			case "eth_getBalance":
+				require.Equal(t, probeAddress, params.Array()[0].String())
+				require.Equal(t, "0x10", params.Array()[1].String()) // 20 - 4
+				resp.Result = `"0x0"`
+			default:
+				require.Fail(t, "unexpected method: "+method)
+			}
+			return
+		}).WSURL()
+
+		rpcClient := NewDialedTestRPCClient(t, RPCClientOpts{
+			HTTP: wsURL,
+			Cfg: &TestNodePoolConfig{
+				NodeFinalizedBlockPollInterval:   1 * time.Second,
+				HistoricalBalanceCheckEnabledVal: true,
+				HistoricalBalanceCheckAddressVal: probeAddress,
+			},
+			FinalityTagsEnabled: false,
+			FinalityDepth:       4,
+			ChainID:             chainID,
+		})
+
+		version, err := rpcClient.ClientVersion(t.Context())
+		require.NoError(t, err)
+		require.Equal(t, "test-client", version)
+	})
+
+	t.Run("probe failure returns health check error", func(t *testing.T) {
+		t.Parallel()
+		wsURL := testutils.NewWSServer(t, chainID, func(method string, _ gjson.Result) (resp testutils.JSONRPCResponse) {
+			switch method {
+			case "web3_clientVersion":
+				resp.Result = `"test-client"`
+			case "eth_getBalance":
+				resp.Error.Message = "balance failure"
+			default:
+				require.Fail(t, "unexpected method: "+method)
+			}
+			return
+		}).WSURL()
+
+		rpcClient := NewDialedTestRPCClient(t, RPCClientOpts{
+			HTTP: wsURL,
+			Cfg: &TestNodePoolConfig{
+				NodeFinalizedBlockPollInterval:   1 * time.Second,
+				HistoricalBalanceCheckEnabledVal: true,
+				HistoricalBalanceCheckAddressVal: probeAddress,
+			},
+			FinalityTagsEnabled: true,
+			ChainID:             chainID,
+		})
+
+		_, err := rpcClient.ClientVersion(t.Context())
+		require.Error(t, err)
+		require.ErrorContains(t, err, "historical balance health check failed")
+	})
+}

pkg/config/chain_scoped_node_pool.go

@@ -3,6 +3,8 @@ package config
 import (
 	"time"
 
+	"github.com/ethereum/go-ethereum/common"
+
 	"github.com/smartcontractkit/chainlink-evm/pkg/config/toml"
 )
 
@@ -38,6 +40,20 @@ func (n *NodePoolConfig) FinalizedBlockPollInterval() time.Duration {
 	return n.C.FinalizedBlockPollInterval.Duration()
 }
 
+func (n *NodePoolConfig) HistoricalBalanceCheckEnabled() bool {
+	if n.C.HistoricalBalanceCheckEnabled == nil {
+		return false
+	}
+	return *n.C.HistoricalBalanceCheckEnabled
+}
+
+func (n *NodePoolConfig) HistoricalBalanceCheckAddress() string {
+	if n.C.HistoricalBalanceCheckAddress == nil {
+		return common.Address{}.String()
+	}
+	return n.C.HistoricalBalanceCheckAddress.String()
+}
+
 func (n *NodePoolConfig) NewHeadsPollInterval() time.Duration {
 	return n.C.NewHeadsPollInterval.Duration()
 }

pkg/config/config.go

@@ -208,6 +208,8 @@ type NodePool interface {
 	LeaseDuration() time.Duration
 	NodeIsSyncingEnabled() bool
 	FinalizedBlockPollInterval() time.Duration
+	HistoricalBalanceCheckEnabled() bool
+	HistoricalBalanceCheckAddress() string
 	Errors() ClientErrors
 	EnforceRepeatableRead() bool
 	DeathDeclarationDelay() time.Duration

pkg/config/config_test.go

@@ -369,6 +369,8 @@ func TestNodePoolConfig(t *testing.T) {
 	require.False(t, cfg.EVM().NodePool().NodeIsSyncingEnabled())
 	require.True(t, cfg.EVM().NodePool().EnforceRepeatableRead())
 	require.Equal(t, time.Minute, cfg.EVM().NodePool().DeathDeclarationDelay())
+	require.False(t, cfg.EVM().NodePool().HistoricalBalanceCheckEnabled())
+	require.Equal(t, "0x0000000000000000000000000000000000000000", cfg.EVM().NodePool().HistoricalBalanceCheckAddress())
 }
 
 func TestClientErrorsConfig(t *testing.T) {

pkg/config/toml/config.go

@@ -1085,6 +1085,8 @@ type NodePool struct {
 	LeaseDuration                  *commonconfig.Duration
 	NodeIsSyncingEnabled           *bool
 	FinalizedBlockPollInterval     *commonconfig.Duration
+	HistoricalBalanceCheckEnabled  *bool
+	HistoricalBalanceCheckAddress  *types.EIP55Address
 	Errors                         ClientErrors `toml:",omitempty"`
 	EnforceRepeatableRead          *bool
 	DeathDeclarationDelay          *commonconfig.Duration
@@ -1115,6 +1117,12 @@ func (p *NodePool) setFrom(f *NodePool) {
 	if v := f.FinalizedBlockPollInterval; v != nil {
 		p.FinalizedBlockPollInterval = v
 	}
+	if v := f.HistoricalBalanceCheckEnabled; v != nil {
+		p.HistoricalBalanceCheckEnabled = v
+	}
+	if v := f.HistoricalBalanceCheckAddress; v != nil {
+		p.HistoricalBalanceCheckAddress = v
+	}
 
 	if v := f.EnforceRepeatableRead; v != nil {
 		p.EnforceRepeatableRead = v
@@ -1150,6 +1158,9 @@ func (p *NodePool) ValidateConfig(finalityTagEnabled *bool) (err error) {
 				Msg: "must be greater than 0"})
 		}
 	}
+	if p.HistoricalBalanceCheckEnabled != nil && *p.HistoricalBalanceCheckEnabled && p.HistoricalBalanceCheckAddress == nil {
+		err = multierr.Append(err, commonconfig.ErrMissing{Name: "HistoricalBalanceCheckAddress", Msg: "required when HistoricalBalanceCheckEnabled is true"})
+	}
 	return
 }
 

pkg/config/toml/config_test.go

@@ -306,6 +306,8 @@ var fullConfig = EVMConfig{
 			LeaseDuration:                  config.MustNewDuration(0),
 			NodeIsSyncingEnabled:           ptr(true),
 			FinalizedBlockPollInterval:     config.MustNewDuration(time.Second),
+			HistoricalBalanceCheckEnabled:  ptr(true),
+			HistoricalBalanceCheckAddress:  ptr(types.MustEIP55Address("0x0000000000000000000000000000000000000001")),
 			EnforceRepeatableRead:          ptr(true),
 			DeathDeclarationDelay:          config.MustNewDuration(time.Minute),
 			VerifyChainID:                  ptr(true),

pkg/config/toml/defaults/fallback.toml

@@ -84,6 +84,8 @@ SyncThreshold = 5
 LeaseDuration = '0s'
 NodeIsSyncingEnabled = false
 FinalizedBlockPollInterval = '5s'
+HistoricalBalanceCheckEnabled = false
+HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000000'
 EnforceRepeatableRead = true
 DeathDeclarationDelay = '1m'
 NewHeadsPollInterval = '0s'

pkg/config/toml/docs.toml

@@ -452,6 +452,15 @@ NodeIsSyncingEnabled = false # Default
 #
 # Set to 0 to disable.
 FinalizedBlockPollInterval = '5s' # Default
+# HistoricalBalanceCheckEnabled controls whether NodePool health polling also verifies historical state availability
+# by executing `eth_getBalance` for HistoricalBalanceCheckAddress at the latest finalized block.
+# Finalized block selection follows chain finality settings:
+# - `FinalityTagEnabled = true`: use `finalized` tag
+# - `FinalityTagEnabled = false`: use `latest - FinalityDepth`
+HistoricalBalanceCheckEnabled = false # Default
+# HistoricalBalanceCheckAddress is the probe account used by the historical balance health check.
+# This check is only active when `HistoricalBalanceCheckEnabled = true`.
+HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000000' # Default
 # EnforceRepeatableRead defines if Core should only use RPCs whose most recently finalized block is greater or equal to
 # `highest finalized block - FinalizedBlockOffset`. In other words, exclude RPCs lagging on latest finalized
 # block.

pkg/config/toml/testdata/config-full.toml

@@ -118,6 +118,8 @@ SyncThreshold = 13
 LeaseDuration = '0s'
 NodeIsSyncingEnabled = true
 FinalizedBlockPollInterval = '1s'
+HistoricalBalanceCheckEnabled = true
+HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000001'
 EnforceRepeatableRead = true
 DeathDeclarationDelay = '1m0s'
 NewHeadsPollInterval = '0s'