state: node bootstrap in v0.37.x (cometbft#1514)

* node/state:bootstrap state api (cometbft#1057)

* add bootstrap state cmd

* add a missing line

* Initial API impl

* Added error message for missing appHash

* Added changelog, removed cli

* Added PR number

* Unified hex encoding with rest of the code

* Applied PR review comments

* Proper blockstore initialization in case of offline statesync

* Reverted forcing blocksync, not needed for correct operation

* Added changelog and comments

* Removed printfs, added check for empty state store

* Fixed linter

* Apply minor suggestions from code review

Co-authored-by: Thane Thomson <connect@thanethomson.com>

* Moved the appHash check up

* Apply minor suggestions from code review

Co-authored-by: Sergio Mena <sergio@informal.systems>

* Apply suggestions from code review

Co-authored-by: Sergio Mena <sergio@informal.systems>

* Fixed linter

* Do not look for VE when starting up after offline statesync

* Extracted  check for offline statesync outside load commit

* Reconstruct seen commit after offline statesync

* Call reconstructSeenCommit from reconstructLastCommit

* Reading offline statesync height only once and passing it as a parameter

* Moved up option initialization to make sure offline statesync is enabled

* Added error to panic message

* Update consensus/state.go

Co-authored-by: Sergio Mena <sergio@informal.systems>

* Apply suggestions from code review

Co-authored-by: Sergio Mena <sergio@informal.systems>

* Adjusted new lines

* Added unit test to test int conversion and fixed linter

* Apply suggestions from code review

Co-authored-by: Thane Thomson <connect@thanethomson.com>

* Replaced closing ifs with defer, added errors to error messages

* linter fix

* Adapted bootstrap code to use proper genesis file functions

* Reverted genesis doc changes

* Moved deferred closing before checking for whether the store is empty

* Moved deferred close before error check

---------

Co-authored-by: HuangYi <huang@crypto.com>
Co-authored-by: yihuang <yi.codeplayer@gmail.com>
Co-authored-by: Thane Thomson <connect@thanethomson.com>
Co-authored-by: Sergio Mena <sergio@informal.systems>

* Fixed cherry pick conflicts

* Removed go API changes in state store

* Remove breaking go API changes in the blocksync reactor

* Removed breaking changelog and fixed mocks

* [pair programming] Found a more readable way to add the methods needed for the store

* Removed duplicated code from blocksync reactor

* Apply suggestions from code review

Co-authored-by: Sergio Mena <sergio@informal.systems>

* Removed comments

* Removed unused variable from state

---------

Co-authored-by: HuangYi <huang@crypto.com>
Co-authored-by: yihuang <yi.codeplayer@gmail.com>
Co-authored-by: Thane Thomson <connect@thanethomson.com>
Co-authored-by: Sergio Mena <sergio@informal.systems>

.changelog/unreleased/features/1057-bootstrap-state-api.md

@@ -0,0 +1,2 @@
+- `[node/state]` Add Go API to bootstrap block store and state store to a height
+  ([\#1057](https://github.com/tendermint/tendermint/pull/#1057)) (@yihuang)

blocksync/reactor.go

@@ -60,21 +60,29 @@ type Reactor struct {
 	errorsCh   <-chan peerError
 }
 
-// NewReactor returns new reactor instance.
-func NewReactor(state sm.State, blockExec *sm.BlockExecutor, store *store.BlockStore,
-	blockSync bool) *Reactor {
-
-	if state.LastBlockHeight != store.Height() {
-		panic(fmt.Sprintf("state (%v) and store (%v) height mismatch", state.LastBlockHeight,
-			store.Height()))
+func NewReactorWithOfflineStateSync(state sm.State, blockExec *sm.BlockExecutor, store *store.BlockStore,
+	blockSync bool, offlineStateSyncHeight int64) *Reactor {
+
+	storeHeight := store.Height()
+	if storeHeight == 0 {
+		// If state sync was performed offline and the stores were bootstrapped to height H
+		// the state store's lastHeight will be H while blockstore's Height and Base are still 0
+		// 1. This scenario should not lead to a panic in this case, which is indicated by
+		// having a OfflineStateSyncHeight > 0
+		// 2. We need to instruct the blocksync reactor to start fetching blocks from H+1
+		// instead of 0.
+		storeHeight = offlineStateSyncHeight
+	}
+	if state.LastBlockHeight != storeHeight {
+		panic(fmt.Sprintf("state (%v) and store (%v) height mismatch, stores were left in an inconsistent state", state.LastBlockHeight,
+			storeHeight))
 	}
-
 	requestsCh := make(chan BlockRequest, maxTotalRequesters)
 
 	const capacity = 1000                      // must be bigger than peers count
 	errorsCh := make(chan peerError, capacity) // so we don't block in #Receive#pool.AddBlock
 
-	startHeight := store.Height() + 1
+	startHeight := storeHeight + 1
 	if startHeight == 1 {
 		startHeight = state.InitialHeight
 	}
@@ -93,6 +101,14 @@ func NewReactor(state sm.State, blockExec *sm.BlockExecutor, store *store.BlockS
 	return bcR
 }
 
+// NewReactor returns new reactor instance.
+func NewReactor(state sm.State, blockExec *sm.BlockExecutor, store *store.BlockStore,
+	blockSync bool) *Reactor {
+
+	return NewReactorWithOfflineStateSync(state, blockExec, store, blockSync, 0)
+
+}
+
 // SetLogger implements service.Service by setting the logger on reactor and pool.
 func (bcR *Reactor) SetLogger(l log.Logger) {
 	bcR.BaseService.Logger = l

consensus/state.go

@@ -171,14 +171,21 @@ func NewState(
 		evsw:             cmtevents.NewEventSwitch(),
 		metrics:          NopMetrics(),
 	}
-
+	for _, option := range options {
+		option(cs)
+	}
 	// set function defaults (may be overwritten before calling Start)
 	cs.decideProposal = cs.defaultDecideProposal
 	cs.doPrevote = cs.defaultDoPrevote
 	cs.setProposal = cs.defaultSetProposal
 
 	// We have no votes, so reconstruct LastCommit from SeenCommit.
 	if state.LastBlockHeight > 0 {
+		// In case of out of band performed statesync, the state store
+		// will have a state but no extended commit (as no block has been downloaded).
+		// If the height at which the vote extensions are enabled is lower
+		// than the height at which we statesync, consensus will panic because
+		// it will try to reconstruct the extended commit here.
 		cs.reconstructLastCommit(state)
 	}
 
@@ -187,9 +194,6 @@ func NewState(
 	// NOTE: we do not call scheduleRound0 yet, we do that upon Start()
 
 	cs.BaseService = *service.NewBaseService(nil, "State", cs)
-	for _, option := range options {
-		option(cs)
-	}
 
 	return cs
 }

node/node.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"os"
 	"strings"
 	"time"
 
@@ -21,12 +22,13 @@ import (
 	cs "github.com/cometbft/cometbft/consensus"
 	"github.com/cometbft/cometbft/crypto"
 	"github.com/cometbft/cometbft/evidence"
+	"github.com/cometbft/cometbft/light"
 
 	cmtjson "github.com/cometbft/cometbft/libs/json"
 	"github.com/cometbft/cometbft/libs/log"
 	cmtpubsub "github.com/cometbft/cometbft/libs/pubsub"
 	"github.com/cometbft/cometbft/libs/service"
-	"github.com/cometbft/cometbft/light"
+
 	mempl "github.com/cometbft/cometbft/mempool"
 	mempoolv0 "github.com/cometbft/cometbft/mempool/v0"
 	mempoolv1 "github.com/cometbft/cometbft/mempool/v1" //nolint:staticcheck // SA1019 Priority mempool deprecated but still supported in this release.
@@ -188,6 +190,124 @@ func StateProvider(stateProvider statesync.StateProvider) Option {
 	}
 }
 
+// BootstrapState synchronizes the stores with the application after state sync
+// has been performed offline. It is expected that the block store and state
+// store are empty at the time the function is called.
+//
+// If the block store is not empty, the function returns an error.
+func BootstrapState(ctx context.Context, config *cfg.Config, dbProvider DBProvider, height uint64, appHash []byte) (err error) {
+	logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout))
+	if ctx == nil {
+		ctx = context.Background()
+	}
+
+	if config == nil {
+		logger.Info("no config provided, using default configuration")
+		config = cfg.DefaultConfig()
+	}
+
+	if dbProvider == nil {
+		dbProvider = DefaultDBProvider
+	}
+	blockStore, stateDB, err := initDBs(config, dbProvider)
+
+	defer func() {
+		if derr := blockStore.Close(); derr != nil {
+			logger.Error("Failed to close blockstore", "err", derr)
+			// Set the return value
+			err = derr
+		}
+	}()
+
+	if err != nil {
+		return err
+	}
+
+	if !blockStore.IsEmpty() {
+		return fmt.Errorf("blockstore not empty, trying to initialize non empty state")
+	}
+
+	stateStore := sm.NewBootstrapStore(stateDB, sm.StoreOptions{
+		DiscardABCIResponses: config.Storage.DiscardABCIResponses,
+	})
+
+	defer func() {
+		if derr := stateStore.Close(); derr != nil {
+			logger.Error("Failed to close statestore", "err", derr)
+			// Set the return value
+			err = derr
+		}
+	}()
+	state, err := stateStore.Load()
+	if err != nil {
+		return err
+	}
+
+	if !state.IsEmpty() {
+		return fmt.Errorf("state not empty, trying to initialize non empty state")
+	}
+
+	genState, _, err := LoadStateFromDBOrGenesisDocProvider(stateDB, DefaultGenesisDocProviderFunc(config))
+	if err != nil {
+		return err
+	}
+
+	stateProvider, err := statesync.NewLightClientStateProvider(
+		ctx,
+		genState.ChainID, genState.Version, genState.InitialHeight,
+		config.StateSync.RPCServers, light.TrustOptions{
+			Period: config.StateSync.TrustPeriod,
+			Height: config.StateSync.TrustHeight,
+			Hash:   config.StateSync.TrustHashBytes(),
+		}, logger.With("module", "light"))
+	if err != nil {
+		return fmt.Errorf("failed to set up light client state provider: %w", err)
+	}
+
+	state, err = stateProvider.State(ctx, height)
+	if err != nil {
+		return err
+	}
+	if appHash == nil {
+		logger.Info("warning: cannot verify appHash. Verification will happen when node boots up!")
+	} else {
+		if !bytes.Equal(appHash, state.AppHash) {
+			if err := blockStore.Close(); err != nil {
+				logger.Error("failed to close blockstore: %w", err)
+			}
+			if err := stateStore.Close(); err != nil {
+				logger.Error("failed to close statestore: %w", err)
+			}
+			return fmt.Errorf("the app hash returned by the light client does not match the provided appHash, expected %X, got %X", state.AppHash, appHash)
+		}
+	}
+
+	commit, err := stateProvider.Commit(ctx, height)
+	if err != nil {
+		return err
+	}
+
+	if err = stateStore.Bootstrap(state); err != nil {
+		return err
+	}
+
+	err = blockStore.SaveSeenCommit(state.LastBlockHeight, commit)
+	if err != nil {
+		return err
+	}
+
+	// Once the stores are bootstrapped, we need to set the height at which the node has finished
+	// statesyncing. This will allow the blocksync reactor to fetch blocks at a proper height.
+	// In case this operation fails, it is equivalent to a failure in  online state sync where the operator
+	// needs to manually delete the state and blockstores and rerun the bootstrapping process.
+	err = stateStore.SetOfflineStateSyncHeight(state.LastBlockHeight)
+	if err != nil {
+		return fmt.Errorf("failed to set synced height: %w", err)
+	}
+
+	return err
+}
+
 //------------------------------------------------------------------------------
 
 // Node is the highest level interface to a full CometBFT node.
@@ -448,10 +568,11 @@ func createBlockchainReactor(config *cfg.Config,
 	blockStore *store.BlockStore,
 	blockSync bool,
 	logger log.Logger,
+	offlineStateSyncHeight int64,
 ) (bcReactor p2p.Reactor, err error) {
 	switch config.BlockSync.Version {
 	case "v0":
-		bcReactor = bc.NewReactor(state.Copy(), blockExec, blockStore, blockSync)
+		bcReactor = bc.NewReactorWithOfflineStateSync(state.Copy(), blockExec, blockStore, blockSync, offlineStateSyncHeight)
 	case "v1", "v2":
 		return nil, fmt.Errorf("block sync version %s has been deprecated. Please use v0", config.BlockSync.Version)
 	default:
@@ -718,7 +839,6 @@ func NewNode(config *cfg.Config,
 		metricsProvider, logger, options...)
 }
 
-// NewNodeWithContext is cancellable version of NewNode.
 func NewNodeWithContext(ctx context.Context,
 	config *cfg.Config,
 	privValidator types.PrivValidator,
@@ -730,12 +850,13 @@ func NewNodeWithContext(ctx context.Context,
 	logger log.Logger,
 	options ...Option,
 ) (*Node, error) {
+
 	blockStore, stateDB, err := initDBs(config, dbProvider)
 	if err != nil {
 		return nil, err
 	}
 
-	stateStore := sm.NewStore(stateDB, sm.StoreOptions{
+	stateStore := sm.NewBootstrapStore(stateDB, sm.StoreOptions{
 		DiscardABCIResponses: config.Storage.DiscardABCIResponses,
 	})
 
@@ -830,9 +951,15 @@ func NewNodeWithContext(ctx context.Context,
 		evidencePool,
 		sm.BlockExecutorWithMetrics(smMetrics),
 	)
-
+	offlineStateSyncHeight := int64(0)
+	if blockStore.Height() == 0 {
+		offlineStateSyncHeight, err = stateStore.GetOfflineStateSyncHeight()
+		if err != nil && err.Error() != "value empty" {
+			panic(fmt.Sprintf("failed to retrieve statesynced height from store %s; expected state store height to be %v", err, state.LastBlockHeight))
+		}
+	}
 	// Make BlockchainReactor. Don't start block sync if we're doing a state sync first.
-	bcReactor, err := createBlockchainReactor(config, state, blockExec, blockStore, blockSync && !stateSync, logger)
+	bcReactor, err := createBlockchainReactor(config, state, blockExec, blockStore, blockSync && !stateSync, logger, offlineStateSyncHeight)
 	if err != nil {
 		return nil, fmt.Errorf("could not create blockchain reactor: %w", err)
 	}
@@ -848,6 +975,10 @@ func NewNodeWithContext(ctx context.Context,
 		config, state, blockExec, blockStore, mempool, evidencePool,
 		privValidator, csMetrics, stateSync || blockSync, eventBus, consensusLogger,
 	)
+	err = stateStore.SetOfflineStateSyncHeight(0)
+	if err != nil {
+		panic(fmt.Sprintf("failed to reset the offline state sync height %s", err))
+	}
 
 	// Set up state sync reactor, and schedule a sync if requested.
 	// FIXME The way we do phased startups (e.g. replay -> block sync -> consensus) is very messy,

state/export_test.go

@@ -45,3 +45,11 @@ func SaveValidatorsInfo(db dbm.DB, height, lastHeightChanged int64, valSet *type
 	stateStore := dbStore{db, StoreOptions{DiscardABCIResponses: false}}
 	return stateStore.saveValidatorsInfo(height, lastHeightChanged, valSet)
 }
+
+func Int64ToBytes(val int64) []byte {
+	return int64ToBytes(val)
+}
+
+func Int64FromBytes(val []byte) int64 {
+	return int64FromBytes(val)
+}