Skip to content
12 changes: 12 additions & 0 deletions multinode/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ type MultiNode struct {
FinalityDepth *uint32
FinalityTagEnabled *bool
FinalizedBlockOffset *uint32

// Finalized State Availability Check
FinalizedStateCheckFailureThreshold *uint32
}

func (c *MultiNodeConfig) Enabled() bool {
Expand Down Expand Up @@ -94,6 +97,10 @@ func (c *MultiNodeConfig) FinalityTagEnabled() bool { return *c.MultiNode.Finali

func (c *MultiNodeConfig) FinalizedBlockOffset() uint32 { return *c.MultiNode.FinalizedBlockOffset }

func (c *MultiNodeConfig) FinalizedStateCheckFailureThreshold() uint32 {
return *c.MultiNode.FinalizedStateCheckFailureThreshold
}

func (c *MultiNodeConfig) SetFrom(f *MultiNodeConfig) {
if f.MultiNode.Enabled != nil {
c.MultiNode.Enabled = f.MultiNode.Enabled
Expand Down Expand Up @@ -150,4 +157,9 @@ func (c *MultiNodeConfig) SetFrom(f *MultiNodeConfig) {
if f.MultiNode.FinalizedBlockOffset != nil {
c.MultiNode.FinalizedBlockOffset = f.MultiNode.FinalizedBlockOffset
}

// Finalized State Availability Check
if f.MultiNode.FinalizedStateCheckFailureThreshold != nil {
c.MultiNode.FinalizedStateCheckFailureThreshold = f.MultiNode.FinalizedStateCheckFailureThreshold
}
}
2 changes: 1 addition & 1 deletion multinode/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/prometheus/client_model v0.6.2
github.com/smartcontractkit/chainlink-common v0.10.1-0.20260305114348-b8bbac30bfc7
github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20250717121125-2350c82883e2
github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20260310180305-3ee91a6d9ae9
github.com/stretchr/testify v1.11.1
go.uber.org/zap v1.27.1
)
Expand Down
4 changes: 2 additions & 2 deletions multinode/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,8 @@ github.com/smartcontractkit/chainlink-common v0.10.1-0.20260305114348-b8bbac30bf
github.com/smartcontractkit/chainlink-common v0.10.1-0.20260305114348-b8bbac30bfc7/go.mod h1:0ghbAr7tRO0tT5ZqBXhOyzgUO37tNNe33Yn0hskauVM=
github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.10 h1:FJAFgXS9oqASnkS03RE1HQwYQQxrO4l46O5JSzxqLgg=
github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.10/go.mod h1:oiDa54M0FwxevWwyAX773lwdWvFYYlYHHQV1LQ5HpWY=
github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20250717121125-2350c82883e2 h1:ysZjKH+BpWlQhF93kr/Lc668UlCvT9NjfcsGdZT19I8=
github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20250717121125-2350c82883e2/go.mod h1:jo+cUqNcHwN8IF7SInQNXDZ8qzBsyMpnLdYbDswviFc=
github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20260310180305-3ee91a6d9ae9 h1:GK+2aFpW/Z5ZnMGCa9NU6o7LKHQ/9xJVZx2yMAMudnc=
github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20260310180305-3ee91a6d9ae9/go.mod h1:HG/aei0MgBOpsyRLexdKGtOUO8yjSJO3iUu0Uu8KBm4=
github.com/smartcontractkit/freeport v0.1.3-0.20250716200817-cb5dfd0e369e h1:Hv9Mww35LrufCdM9wtS9yVi/rEWGI1UnjHbcKKU0nVY=
github.com/smartcontractkit/freeport v0.1.3-0.20250716200817-cb5dfd0e369e/go.mod h1:T4zH9R8R8lVWKfU7tUvYz2o2jMv1OpGCdpY2j2QZXzU=
github.com/smartcontractkit/libocr v0.0.0-20250912173940-f3ab0246e23d h1:LokA9PoCNb8mm8mDT52c3RECPMRsGz1eCQORq+J3n74=
Expand Down
68 changes: 68 additions & 0 deletions multinode/mock_node_metrics_test.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

92 changes: 92 additions & 0 deletions multinode/mock_rpc_client_test.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 4 additions & 1 deletion multinode/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ type NodeConfig interface {
DeathDeclarationDelay() time.Duration
NewHeadsPollInterval() time.Duration
VerifyChainID() bool
FinalizedStateCheckFailureThreshold() uint32
}

type ChainConfig interface {
Expand All @@ -48,13 +49,15 @@ type nodeMetrics interface {
IncrementNodeTransitionsToInvalidChainID(ctx context.Context, nodeName string)
IncrementNodeTransitionsToUnusable(ctx context.Context, nodeName string)
IncrementNodeTransitionsToSyncing(ctx context.Context, nodeName string)
IncrementNodeTransitionsToFinalizedStateNotAvailable(ctx context.Context, nodeName string)
RecordNodeClientVersion(ctx context.Context, nodeName string, version string)
SetHighestSeenBlock(ctx context.Context, nodeName string, blockNumber int64)
SetHighestFinalizedBlock(ctx context.Context, nodeName string, blockNumber int64)
IncrementSeenBlocks(ctx context.Context, nodeName string)
IncrementPolls(ctx context.Context, nodeName string)
IncrementPollsFailed(ctx context.Context, nodeName string)
IncrementPollsSuccess(ctx context.Context, nodeName string)
IncrementFinalizedStateFailed(ctx context.Context, nodeName string)
}

type Node[
Expand Down Expand Up @@ -273,7 +276,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) verifyChainID(callerCtx context.Context, lgg
// The node is already closed, and any subsequent transition is invalid.
// To make spotting such transitions a bit easier, return the invalid node state.
return nodeStateLen
case nodeStateDialed, nodeStateOutOfSync, nodeStateInvalidChainID, nodeStateSyncing:
case nodeStateDialed, nodeStateOutOfSync, nodeStateInvalidChainID, nodeStateSyncing, nodeStateFinalizedStateNotAvailable:
default:
panic(fmt.Sprintf("cannot verify node in state %v", st))
}
Expand Down
35 changes: 34 additions & 1 deletion multinode/node_fsm.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ func (n nodeState) String() string {
return "Syncing"
case nodeStateFinalizedBlockOutOfSync:
return "FinalizedBlockOutOfSync"
case nodeStateFinalizedStateNotAvailable:
return "FinalizedStateNotAvailable"
default:
return fmt.Sprintf("nodeState(%d)", n)
}
Expand Down Expand Up @@ -72,6 +74,8 @@ const (
nodeStateSyncing
// nodeStateFinalizedBlockOutOfSync - node is lagging behind on latest finalized block
nodeStateFinalizedBlockOutOfSync
// nodeStateFinalizedStateNotAvailable - node cannot serve historical state at finalized block
nodeStateFinalizedStateNotAvailable
// nodeStateLen tracks the number of states
nodeStateLen
)
Expand Down Expand Up @@ -182,7 +186,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) transitionToAlive(fn func()) {
return
}
switch n.state {
case nodeStateDialed, nodeStateInvalidChainID, nodeStateSyncing:
case nodeStateDialed, nodeStateInvalidChainID, nodeStateSyncing, nodeStateFinalizedStateNotAvailable:
n.state = nodeStateAlive
default:
panic(transitionFail(n.state, nodeStateAlive))
Expand Down Expand Up @@ -288,6 +292,8 @@ func (n *node[CHAIN_ID, HEAD, RPC]) declareState(state nodeState) {
n.declareSyncing()
case nodeStateAlive:
n.declareAlive()
case nodeStateFinalizedStateNotAvailable:
n.declareFinalizedStateNotAvailable()
default:
panic(fmt.Sprintf("%#v state declaration is not implemented", state))
}
Expand Down Expand Up @@ -351,6 +357,33 @@ func (n *node[CHAIN_ID, HEAD, RPC]) transitionToSyncing(fn func()) {
fn()
}

func (n *node[CHAIN_ID, HEAD, RPC]) declareFinalizedStateNotAvailable() {
n.transitionToFinalizedStateNotAvailable(func() {
n.lfcLog.Errorw("RPC Node cannot serve finalized state", "nodeState", n.state)
n.wg.Add(1)
go n.finalizedStateNotAvailableLoop()
})
}

func (n *node[CHAIN_ID, HEAD, RPC]) transitionToFinalizedStateNotAvailable(fn func()) {
ctx, cancel := n.stopCh.NewCtx()
defer cancel()
n.metrics.IncrementNodeTransitionsToFinalizedStateNotAvailable(ctx, n.name)
n.stateMu.Lock()
defer n.stateMu.Unlock()
if n.state == nodeStateClosed {
return
}
switch n.state {
case nodeStateAlive:
n.rpc.Close()
n.state = nodeStateFinalizedStateNotAvailable
default:
panic(transitionFail(n.state, nodeStateFinalizedStateNotAvailable))
}
fn()
}

func transitionFail(from nodeState, to nodeState) string {
return fmt.Sprintf("cannot transition from %#v to %#v", from, to)
}
2 changes: 1 addition & 1 deletion multinode/node_fsm_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ func TestUnit_Node_StateTransitions(t *testing.T) {

t.Run("transitionToAlive", func(t *testing.T) {
const destinationState = nodeStateAlive
allowedStates := []nodeState{nodeStateDialed, nodeStateInvalidChainID, nodeStateSyncing}
allowedStates := []nodeState{nodeStateDialed, nodeStateInvalidChainID, nodeStateSyncing, nodeStateFinalizedStateNotAvailable}
rpc := newMockRPCClient[ID, Head](t)
testTransition(t, rpc, testNode.transitionToAlive, destinationState, allowedStates...)
})
Expand Down
Loading
Loading