feat: add recover node buffer list for pathdb

cmd/geth/chaincmd.go

@@ -41,9 +41,6 @@ import (
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/params"
-	"github.com/ethereum/go-ethereum/trie"
-	"github.com/ethereum/go-ethereum/triedb/pathdb"
-	"github.com/olekukonko/tablewriter"
 	"github.com/urfave/cli/v2"
 )
 
@@ -188,18 +185,6 @@ It's deprecated, please use "geth db import" instead.
 This command dumps out the state for a given block (or latest, if none provided).
 `,
 	}
-
-	dumpRootHashCommand = &cli.Command{
-		Action: dumpAllRootHashInPath,
-		Name:   "dump-roothash",
-		Usage:  "Dump all available state root hash in path mode",
-		Flags:  flags.Merge([]cli.Flag{}, utils.DatabaseFlags),
-		Description: `
-The dump-roothash command dump all available state root hash in path mode.
-If you use "dump" command in path mode, please note that it only keeps at most 129 blocks which belongs to diffLayer or diskLayer.
-Therefore, you must specify the blockNumber or blockHash that locates in diffLayer or diskLayer.
-"geth" will print all available blockNumber and related block state root hash, and you can query block hash by block number.
-`}
 )
 
 // initGenesis will initialise the given JSON format genesis file and writes it as
@@ -617,26 +602,3 @@ func hashish(x string) bool {
 	_, err := strconv.Atoi(x)
 	return err != nil
 }
-
-func dumpAllRootHashInPath(ctx *cli.Context) error {
-	stack, _ := makeConfigNode(ctx)
-	defer stack.Close()
-	db := utils.MakeChainDatabase(ctx, stack, true)
-	defer db.Close()
-	triedb := trie.NewDatabase(db, &trie.Config{PathDB: pathdb.ReadOnly})
-	defer triedb.Close()
-
-	scheme, err := rawdb.ParseStateScheme(ctx.String(utils.StateSchemeFlag.Name), db)
-	if err != nil {
-		return err
-	}
-	if scheme == rawdb.HashScheme {
-		return errors.New("incorrect state scheme, you should use it in path mode")
-	}
-
-	table := tablewriter.NewWriter(os.Stdout)
-	table.SetHeader([]string{"Block Number", "Block State Root Hash"})
-	table.AppendBulk(triedb.GetAllRooHash())
-	table.Render()
-	return nil
-}

cmd/geth/dbcmd.go

@@ -87,7 +87,6 @@ Remove blockchain and state databases`,
 			dbPruneHashTrieCmd,
 			dbTrieGetCmd,
 			dbTrieDeleteCmd,
-			ancientToDiffLayerCmd,
 		},
 	}
 	dbInspectCmd = &cli.Command{
@@ -271,13 +270,6 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "Shows metadata about the chain status.",
 	}
-	ancientToDiffLayerCmd = &cli.Command{
-		Name:        "ancient-to-dl",
-		Usage:       "Convert the data in ancientDB into diffLayer",
-		Description: "A convenient test tool to for path db diffLayer converting",
-		Action:      ancientToDiffLayer,
-		Flags:       flags.Merge(utils.DatabaseFlags),
-	}
 )
 
 func removeDB(ctx *cli.Context) error {
@@ -1121,21 +1113,6 @@ func hbss2pbss(ctx *cli.Context) error {
 	return nil
 }
 
-func ancientToDiffLayer(ctx *cli.Context) error {
-	stack, _ := makeConfigNode(ctx)
-	defer stack.Close()
-	db := utils.MakeChainDatabase(ctx, stack, true)
-	defer db.Close()
-	triedb := utils.MakeTrieDatabase(ctx, stack, db, false, true, false)
-	// triedb := trie.NewDatabase(db, &trie.Config{PathDB: nil})
-	defer triedb.Close()
-
-	if err := triedb.DiffLayerConvertTool(); err != nil {
-		log.Error("Failed to get diff layer from ancient db", "error", err)
-	}
-	return nil
-}
-
 func pruneHashTrie(ctx *cli.Context) error {
 	if ctx.NArg() != 0 {
 		return fmt.Errorf("required none argument")

cmd/geth/main.go

@@ -241,7 +241,6 @@ func init() {
 		removedbCommand,
 		dumpCommand,
 		dumpGenesisCommand,
-		dumpRootHashCommand,
 		// See accountcmd.go:
 		accountCommand,
 		walletCommand,

core/blockchain.go

@@ -164,8 +164,8 @@ type CacheConfig struct {
 	KeepProofBlockSpan   uint64                // Block span of keep proof
 	SnapshotNoBuild      bool                  // Whether the background generation is allowed
 	SnapshotWait         bool                  // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it
-	JournalFilePath      string
-	JournalFile          bool
+	JournalFilePath      string                // The file path to journal pathdb diff layers
+	JournalFile          bool                  // Whether to enable journal file
 
 	TrieCommitInterval uint64 // Define a block height interval, commit trie every TrieCommitInterval block height.
 }

core/rawdb/accessors_state.go

@@ -224,43 +224,60 @@ func ReadStateStorageHistory(db ethdb.AncientReaderOp, id uint64) []byte {
 	return blob
 }
 
+// ReadStateTrieNodesHistory retrieves the trie nodes corresponding to the specified
+// state history. Compute the position of state history in freezer by minus one
+// since the id of first state history starts from one(zero for initial state).
+func ReadStateTrieNodesHistory(db ethdb.AncientReaderOp, id uint64) []byte {
+	blob, err := db.Ancient(stateHistoryTrieNodesData, id-1)
+	if err != nil {
+		return nil
+	}
+	return blob
+}
+
 // ReadStateHistory retrieves the state history from database with provided id.
 // Compute the position of state history in freezer by minus one since the id
 // of first state history starts from one(zero for initial state).
-func ReadStateHistory(db ethdb.AncientReaderOp, id uint64) ([]byte, []byte, []byte, []byte, []byte, error) {
+func ReadStateHistory(db ethdb.AncientReaderOp, id uint64) ([]byte, []byte, []byte, []byte, []byte, []byte, error) {
 	meta, err := db.Ancient(stateHistoryMeta, id-1)
 	if err != nil {
-		return nil, nil, nil, nil, nil, err
+		return nil, nil, nil, nil, nil, nil, err
 	}
 	accountIndex, err := db.Ancient(stateHistoryAccountIndex, id-1)
 	if err != nil {
-		return nil, nil, nil, nil, nil, err
+		return nil, nil, nil, nil, nil, nil, err
 	}
 	storageIndex, err := db.Ancient(stateHistoryStorageIndex, id-1)
 	if err != nil {
-		return nil, nil, nil, nil, nil, err
+		return nil, nil, nil, nil, nil, nil, err
 	}
 	accountData, err := db.Ancient(stateHistoryAccountData, id-1)
 	if err != nil {
-		return nil, nil, nil, nil, nil, err
+		return nil, nil, nil, nil, nil, nil, err
 	}
 	storageData, err := db.Ancient(stateHistoryStorageData, id-1)
 	if err != nil {
-		return nil, nil, nil, nil, nil, err
+		return nil, nil, nil, nil, nil, nil, err
+	}
+	trieNodesData, err := db.Ancient(stateHistoryStorageData, id-1)
+	if err != nil {
+		return nil, nil, nil, nil, nil, nil, err
 	}
-	return meta, accountIndex, storageIndex, accountData, storageData, nil
+	return meta, accountIndex, storageIndex, accountData, storageData, trieNodesData, nil
 }
 
 // WriteStateHistory writes the provided state history to database. Compute the
 // position of state history in freezer by minus one since the id of first state
 // history starts from one(zero for initial state).
-func WriteStateHistory(db ethdb.AncientWriter, id uint64, meta []byte, accountIndex []byte, storageIndex []byte, accounts []byte, storages []byte) {
+func WriteStateHistory(db ethdb.AncientWriter, id uint64, meta []byte, accountIndex []byte, storageIndex []byte,
+	accounts []byte, storages []byte, trieNodes []byte) {
 	db.ModifyAncients(func(op ethdb.AncientWriteOp) error {
 		op.AppendRaw(stateHistoryMeta, id-1, meta)
 		op.AppendRaw(stateHistoryAccountIndex, id-1, accountIndex)
 		op.AppendRaw(stateHistoryStorageIndex, id-1, storageIndex)
 		op.AppendRaw(stateHistoryAccountData, id-1, accounts)
 		op.AppendRaw(stateHistoryStorageData, id-1, storages)
+		op.AppendRaw(stateHistoryTrieNodesData, id-1, trieNodes)
 		return nil
 	})
 }

core/rawdb/ancient_scheme.go

@@ -51,19 +51,21 @@ const (
 	stateHistoryTableSize = 2 * 1000 * 1000 * 1000
 
 	// stateHistoryAccountIndex indicates the name of the freezer state history table.
-	stateHistoryMeta         = "history.meta"
-	stateHistoryAccountIndex = "account.index"
-	stateHistoryStorageIndex = "storage.index"
-	stateHistoryAccountData  = "account.data"
-	stateHistoryStorageData  = "storage.data"
+	stateHistoryMeta          = "history.meta"
+	stateHistoryAccountIndex  = "account.index"
+	stateHistoryStorageIndex  = "storage.index"
+	stateHistoryAccountData   = "account.data"
+	stateHistoryStorageData   = "storage.data"
+	stateHistoryTrieNodesData = "trienodes.data"
 )
 
 var stateFreezerNoSnappy = map[string]bool{
-	stateHistoryMeta:         true,
-	stateHistoryAccountIndex: false,
-	stateHistoryStorageIndex: false,
-	stateHistoryAccountData:  false,
-	stateHistoryStorageData:  false,
+	stateHistoryMeta:          true,
+	stateHistoryAccountIndex:  false,
+	stateHistoryStorageIndex:  false,
+	stateHistoryAccountData:   false,
+	stateHistoryStorageData:   false,
+	stateHistoryTrieNodesData: false,
 }
 
 const (

eth/backend.go

@@ -132,7 +132,19 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 		log.Warn("Sanitizing invalid miner gas price", "provided", config.Miner.GasPrice, "updated", ethconfig.Defaults.Miner.GasPrice)
 		config.Miner.GasPrice = new(big.Int).Set(ethconfig.Defaults.Miner.GasPrice)
 	}
-	if config.NoPruning && config.TrieDirtyCache > 0 {
+
+	// Assemble the Ethereum object
+	chainDb, err := stack.OpenDatabaseWithFreezer(ChainData, config.DatabaseCache, config.DatabaseHandles,
+		config.DatabaseFreezer, ChainDBNamespace, false)
+	if err != nil {
+		return nil, err
+	}
+	config.StateScheme, err = rawdb.ParseStateScheme(config.StateScheme, chainDb)
+	if err != nil {
+		return nil, err
+	}
+
+	if config.StateScheme == rawdb.HashScheme && config.NoPruning && config.TrieDirtyCache > 0 {
 		if config.SnapshotCache > 0 {
 			config.TrieCleanCache += config.TrieDirtyCache * 3 / 5
 			config.SnapshotCache += config.TrieDirtyCache * 2 / 5
@@ -156,18 +168,9 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 		"trie_clean_cache", common.StorageSize(config.TrieCleanCache)*1024*1024,
 		"trie_dirty_cache", common.StorageSize(config.TrieDirtyCache)*1024*1024,
 		"snapshot_cache", common.StorageSize(config.SnapshotCache)*1024*1024)
-	// Assemble the Ethereum object
-	chainDb, err := stack.OpenDatabaseWithFreezer(ChainData, config.DatabaseCache, config.DatabaseHandles,
-		config.DatabaseFreezer, ChainDBNamespace, false)
-	if err != nil {
-		return nil, err
-	}
-	scheme, err := rawdb.ParseStateScheme(config.StateScheme, chainDb)
-	if err != nil {
-		return nil, err
-	}
+
 	// Try to recover offline state pruning only in hash-based.
-	if scheme == rawdb.HashScheme {
+	if config.StateScheme == rawdb.HashScheme {
 		if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb); err != nil {
 			log.Error("Failed to recover state", "error", err)
 		}
@@ -235,7 +238,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 			Preimages:            config.Preimages,
 			NoTries:              config.NoTries,
 			StateHistory:         config.StateHistory,
-			StateScheme:          scheme,
+			StateScheme:          config.StateScheme,
 			TrieCommitInterval:   config.TrieCommitInterval,
 			PathNodeBuffer:       config.PathNodeBuffer,
 			ProposeBlockInterval: config.ProposeBlockInterval,

trie/testutil/utils.go

@@ -20,6 +20,7 @@ import (
 	crand "crypto/rand"
 	"encoding/binary"
 	mrand "math/rand"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
@@ -59,3 +60,14 @@ func RandomNode() *trienode.Node {
 	val := RandBytes(100)
 	return trienode.New(crypto.Keccak256Hash(val), val)
 }
+
+// RandomString generates specified length random string.
+func RandomString() string {
+	mrand.New(mrand.NewSource(time.Now().UnixNano()))
+	const chars = "0123456789abcdefghijklmnopqrstuvwxyz"
+	characters := make([]byte, 10)
+	for i := range characters {
+		characters[i] = chars[mrand.Intn(len(chars))]
+	}
+	return string(characters)
+}

trie/trienode/node.go

@@ -197,8 +197,3 @@ func (set *MergedNodeSet) Flatten() map[common.Hash]map[string]*Node {
 	}
 	return nodes
 }
-
-// // ReverseFlatten reverts flattened two-dimensional map to merge node set
-// func ReverseFlatten(nodes map[common.Hash]map[string]*Node) MergedNodeSet {
-//
-// }