fix: allow users to optionally configure node startup even if index reconciliation fails (#12930)

* optional index start on reconciliation failure

* warning

* fix ChangeLog

* fix make gen

* fix compilation

Author: aarshkshah1992

CHANGELOG.md

@@ -1,18 +1,19 @@
 # Lotus changelog
 
-> **Historical Note**  
+> **Historical Note**
 > Previous changelog entries are archived in:
 > * [CHANGELOG_0.x.md](./documentation/changelog/CHANGELOG_0.x.md) - v0.1.0 to v0.9.1
 > * [CHANGELOG_1.0x.md](./documentation/changelog/CHANGELOG_1.0x.md) - v1.0.0 to v1.9.0
 > * [CHANGELOG_1.1x.md](./documentation/changelog/CHANGELOG_1.1x.md) - v1.10.0 to v1.19.0
 > * [CHANGELOG_1.2x.md](./documentation/changelog/CHANGELOG_1.2x.md) - v1.20.0 to v1.29.2
 
-# UNRELEASED
-
 # UNRELEASED v.1.32.0
 
 See https://github.com/filecoin-project/lotus/blob/release/v1.32.0/CHANGELOG.md
 
+# Special Zondax v1.31.2 tag:
+- Allow users to start node even if Index Reconciliation fails (https://github.com/filecoin-project/lotus/pull/12930)
+
 # Node v1.31.1 / 2025-01-27
 
 This Lotus release introduces several new features and improvements, including JSON output for tipsets in `lotus chain list` cmd, enhanced logging during network upgrade migrations, and additional Bootstrap nodes. It also includes a refactored Ethereum API implementation into smaller, more manageable modules in a new `github.com/filecoin-project/lotus/node/impl/eth` package, as well as adding network name as a tag in most metrics - making it easier to create Graphana Dashboards for multiple networks. Please review the upgrade warnings and documentation for any important changes affecting RPC providers, node operators, and storage providers.
@@ -156,7 +157,7 @@ Network Version: 24
 Actor Version: 15
 Manifest CID: bafy2bzaceakwje2hyinucrhgtsfo44p54iw4g6otbv5ghov65vajhxgntr53u
 
-Actor             CID  
+Actor             CID
 account           bafk2bzacecia5zacqt4gvd4z7275lnkhgraq75shy63cphakphhw6crf4joii
 cron              bafk2bzacecbyx7utt3tkvhqnfk64kgtlt5jlvv56o2liwczikgzfowk2cvqvk
 datacap           bafk2bzacecrypcpyzidphfl3sf3vhrjbiwzu7w3hoole45wsk2bqpverw4tni

documentation/en/default-lotus-config.toml

@@ -348,6 +348,24 @@
   # env var: LOTUS_CHAININDEXER_MAXRECONCILETIPSETS
   #MaxReconcileTipsets = 8640
 
+  # AllowIndexReconciliationFailure determines whether node startup should continue
+  # if the index reconciliation with the chain state fails.
+  # 
+  # When set to true:
+  # - If index reconciliation fails during startup, the node will log a warning but continue to start.
+  # 
+  # When set to false (default):
+  # - If index reconciliation fails during startup, the node will fail to start.
+  # - This ensures that the index is always in a consistent state with the chain before the node starts.
+  # 
+  # Default: false
+  # // WARNING: Only set to true if you are okay with an index that may be out of sync with the chain.
+  # This can lead to inaccurate or missing data in RPC responses that depend on the indexer.
+  #
+  # type: bool
+  # env var: LOTUS_CHAININDEXER_ALLOWINDEXRECONCILIATIONFAILURE
+  #AllowIndexReconciliationFailure = false
+
 
 [FaultReporter]
   # EnableConsensusFaultReporter controls whether the node will monitor and

node/builder_chain.go

@@ -330,7 +330,7 @@ func ConfigFullNode(c interface{}) Option {
 			Override(new(index.Indexer), modules.ChainIndexer(cfg.ChainIndexer)),
 			Override(new(full.ChainIndexerAPI), modules.ChainIndexHandler(cfg.ChainIndexer)),
 			If(cfg.ChainIndexer.EnableIndexer,
-				Override(InitChainIndexerKey, modules.InitChainIndexer),
+				Override(InitChainIndexerKey, modules.InitChainIndexer(cfg.ChainIndexer)),
 			),
 		),
 	)

node/config/doc_gen.go

@@ -628,6 +628,21 @@ type ChainIndexerConfig struct {
 	// Note: Setting this value too low may result in incomplete indexing, while setting it too high
 	// may increase startup time.
 	MaxReconcileTipsets uint64
+
+	// AllowIndexReconciliationFailure determines whether node startup should continue
+	// if the index reconciliation with the chain state fails.
+	//
+	// When set to true:
+	// - If index reconciliation fails during startup, the node will log a warning but continue to start.
+	//
+	// When set to false (default):
+	// - If index reconciliation fails during startup, the node will fail to start.
+	// - This ensures that the index is always in a consistent state with the chain before the node starts.
+	//
+	// Default: false
+	// // WARNING: Only set to true if you are okay with an index that may be out of sync with the chain.
+	// This can lead to inaccurate or missing data in RPC responses that depend on the indexer.
+	AllowIndexReconciliationFailure bool
 }
 
 type HarmonyDB struct {

node/config/types.go

@@ -51,60 +51,66 @@ func ChainIndexer(cfg config.ChainIndexerConfig) func(lc fx.Lifecycle, mctx help
 	}
 }
 
-func InitChainIndexer(lc fx.Lifecycle, mctx helpers.MetricsCtx, indexer index.Indexer,
+func InitChainIndexer(cfg config.ChainIndexerConfig) func(lc fx.Lifecycle, mctx helpers.MetricsCtx, indexer index.Indexer,
 	evapi EventHelperAPI, mp *messagepool.MessagePool, sm *stmgr.StateManager) {
-	ctx := helpers.LifecycleCtx(mctx, lc)
+	return func(lc fx.Lifecycle, mctx helpers.MetricsCtx, indexer index.Indexer,
+		evapi EventHelperAPI, mp *messagepool.MessagePool, sm *stmgr.StateManager) {
+		ctx := helpers.LifecycleCtx(mctx, lc)
 
-	lc.Append(fx.Hook{
-		OnStart: func(_ context.Context) error {
-			indexer.SetActorToDelegatedAddresFunc(func(ctx context.Context, emitter abi.ActorID, ts *types.TipSet) (address.Address, bool) {
-				idAddr, err := address.NewIDAddress(uint64(emitter))
+		lc.Append(fx.Hook{
+			OnStart: func(_ context.Context) error {
+				indexer.SetActorToDelegatedAddresFunc(func(ctx context.Context, emitter abi.ActorID, ts *types.TipSet) (address.Address, bool) {
+					idAddr, err := address.NewIDAddress(uint64(emitter))
+					if err != nil {
+						return address.Undef, false
+					}
+
+					actor, err := sm.LoadActor(ctx, idAddr, ts)
+					if err != nil || actor.DelegatedAddress == nil {
+						return idAddr, true
+					}
+
+					return *actor.DelegatedAddress, true
+				})
+
+				indexer.SetRecomputeTipSetStateFunc(func(ctx context.Context, ts *types.TipSet) error {
+					_, _, err := sm.RecomputeTipSetState(ctx, ts)
+					return err
+				})
+
+				ch, err := mp.Updates(ctx)
 				if err != nil {
-					return address.Undef, false
+					return err
 				}
+				go WaitForMpoolUpdates(ctx, ch, indexer)
 
-				actor, err := sm.LoadActor(ctx, idAddr, ts)
-				if err != nil || actor.DelegatedAddress == nil {
-					return idAddr, true
+				ev, err := events.NewEvents(ctx, &evapi)
+				if err != nil {
+					return err
 				}
 
-				return *actor.DelegatedAddress, true
-			})
-
-			indexer.SetRecomputeTipSetStateFunc(func(ctx context.Context, ts *types.TipSet) error {
-				_, _, err := sm.RecomputeTipSetState(ctx, ts)
-				return err
-			})
-
-			ch, err := mp.Updates(ctx)
-			if err != nil {
-				return err
-			}
-			go WaitForMpoolUpdates(ctx, ch, indexer)
-
-			ev, err := events.NewEvents(ctx, &evapi)
-			if err != nil {
-				return err
-			}
-
-			// Tipset listener
+				// Tipset listener
 
-			// `ObserveAndBlock` returns the current head and guarantees that it will call the observer with all future tipsets
-			head, unlockObserver, err := ev.ObserveAndBlock(indexer)
-			if err != nil {
-				return xerrors.Errorf("error while observing tipsets: %w", err)
-			}
-			if err := indexer.ReconcileWithChain(ctx, head); err != nil {
+				// `ObserveAndBlock` returns the current head and guarantees that it will call the observer with all future tipsets
+				head, unlockObserver, err := ev.ObserveAndBlock(indexer)
+				if err != nil {
+					return xerrors.Errorf("error while observing tipsets: %w", err)
+				}
+				if err := indexer.ReconcileWithChain(ctx, head); err != nil {
+					unlockObserver()
+					if !cfg.AllowIndexReconciliationFailure {
+						return xerrors.Errorf("error while reconciling chain index with chain state: %w", err)
+					}
+					log.Warnf("error while reconciling chain index with chain state: %s", err)
+				}
 				unlockObserver()
-				return xerrors.Errorf("error while reconciling chain index with chain state: %w", err)
-			}
-			unlockObserver()
 
-			indexer.Start()
+				indexer.Start()
 
-			return nil
-		},
-	})
+				return nil
+			},
+		})
+	}
 }
 
 func ChainIndexHandler(cfg config.ChainIndexerConfig) func(helpers.MetricsCtx, repo.LockedRepo, fx.Lifecycle, index.Indexer) (*full.ChainIndexHandler, error) {