From a83c03ee07f21eb4c0e7b5fd2ab27d6a9c0c565a Mon Sep 17 00:00:00 2001 From: VM Date: Fri, 26 Jul 2024 15:44:36 +0800 Subject: [PATCH] feat: add recover node buffer list for pathdb --- cmd/geth/main.go | 1 - cmd/utils/flags.go | 9 ----- core/blockchain.go | 54 ++++++++++++++-------------- eth/backend.go | 35 +++++++++--------- eth/ethconfig/config.go | 13 ++++--- eth/ethconfig/gen_config.go | 6 ---- trie/triedb/pathdb/database.go | 19 +++++----- trie/triedb/pathdb/journal.go | 3 +- trie/triedb/pathdb/nodebufferlist.go | 1 - 9 files changed, 59 insertions(+), 82 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index ebf5512a84..3f19f7b260 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -101,7 +101,6 @@ var ( utils.EnableProofKeeperFlag, utils.KeepProofBlockSpanFlag, utils.JournalFileFlag, - utils.EnableRecoverNodeBufferListFlag, utils.LightServeFlag, // deprecated utils.LightIngressFlag, // deprecated utils.LightEgressFlag, // deprecated diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 90bd959ce8..70621a57d9 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -346,12 +346,6 @@ var ( Value: false, Category: flags.StateCategory, } - EnableRecoverNodeBufferListFlag = &cli.BoolFlag{ - Name: "pathdb.recovernodebufferlist", - Usage: "Enable recovering node buffer list", - Value: false, - Category: flags.StateCategory, - } StateHistoryFlag = &cli.Uint64Flag{ Name: "history.state", Usage: "Number of recent blocks to retain state history for (default = 90,000 blocks, 0 = entire chain)", @@ -1893,9 +1887,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.IsSet(JournalFileFlag.Name) { cfg.JournalFileEnabled = true } - if ctx.IsSet(EnableRecoverNodeBufferListFlag.Name) { - cfg.EnableRecoverNodeBufferList = true - } if ctx.String(GCModeFlag.Name) == "archive" && cfg.TransactionHistory != 0 { cfg.TransactionHistory = 0 diff --git a/core/blockchain.go b/core/blockchain.go index 353d584bb0..16da82b6b8 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -149,25 +149,24 @@ const ( // CacheConfig contains the configuration values for the trie database // and state snapshot these are resident in a blockchain. type CacheConfig struct { - TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory - TrieCleanNoPrefetch bool // Whether to disable heuristic state prefetching for followup blocks - TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk - TrieDirtyDisabled bool // Whether to disable trie write caching and GC altogether (archive node) - TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk - SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory - Preimages bool // Whether to store preimage of trie key to the disk - NoTries bool // Insecure settings. Do not have any tries in databases if enabled. - StateHistory uint64 // Number of blocks from head whose state histories are reserved. - StateScheme string // Scheme used to store ethereum states and merkle tree nodes on top - PathNodeBuffer pathdb.NodeBufferType // Type of trienodebuffer to cache trie nodes in disklayer - ProposeBlockInterval uint64 // Propose block to L1 block interval. - EnableProofKeeper bool // Whether to enable proof keeper - KeepProofBlockSpan uint64 // Block span of keep proof - SnapshotNoBuild bool // Whether the background generation is allowed - SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it - JournalFilePath string // The file path to journal pathdb diff layers - JournalFile bool // Whether to enable journal file - EnableRecoverNodeBufferList bool // Whether enable recover node buffer list. + TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory + TrieCleanNoPrefetch bool // Whether to disable heuristic state prefetching for followup blocks + TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk + TrieDirtyDisabled bool // Whether to disable trie write caching and GC altogether (archive node) + TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk + SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory + Preimages bool // Whether to store preimage of trie key to the disk + NoTries bool // Insecure settings. Do not have any tries in databases if enabled. + StateHistory uint64 // Number of blocks from head whose state histories are reserved. + StateScheme string // Scheme used to store ethereum states and merkle tree nodes on top + PathNodeBuffer pathdb.NodeBufferType // Type of trienodebuffer to cache trie nodes in disklayer + ProposeBlockInterval uint64 // Propose block to L1 block interval. + EnableProofKeeper bool // Whether to enable proof keeper + KeepProofBlockSpan uint64 // Block span of keep proof + SnapshotNoBuild bool // Whether the background generation is allowed + SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it + JournalFilePath string // The file path to journal pathdb diff layers + JournalFile bool // Whether to enable journal file TrieCommitInterval uint64 // Define a block height interval, commit trie every TrieCommitInterval block height. } @@ -185,15 +184,14 @@ func (c *CacheConfig) triedbConfig(keepFunc pathdb.NotifyKeepFunc) *trie.Config } if c.StateScheme == rawdb.PathScheme { config.PathDB = &pathdb.Config{ - TrieNodeBufferType: c.PathNodeBuffer, - StateHistory: c.StateHistory, - CleanCacheSize: c.TrieCleanLimit * 1024 * 1024, - DirtyCacheSize: c.TrieDirtyLimit * 1024 * 1024, - ProposeBlockInterval: c.ProposeBlockInterval, - NotifyKeep: keepFunc, - JournalFilePath: c.JournalFilePath, - JournalFile: c.JournalFile, - EnableRecoverNodeBufferList: c.EnableRecoverNodeBufferList, + TrieNodeBufferType: c.PathNodeBuffer, + StateHistory: c.StateHistory, + CleanCacheSize: c.TrieCleanLimit * 1024 * 1024, + DirtyCacheSize: c.TrieDirtyLimit * 1024 * 1024, + ProposeBlockInterval: c.ProposeBlockInterval, + NotifyKeep: keepFunc, + JournalFilePath: c.JournalFilePath, + JournalFile: c.JournalFile, } } return config diff --git a/eth/backend.go b/eth/backend.go index e9c53757af..0588d63fc5 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -228,24 +228,23 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { EnableOpcodeOptimizations: config.EnableOpcodeOptimizing, } cacheConfig = &core.CacheConfig{ - TrieCleanLimit: config.TrieCleanCache, - TrieCleanNoPrefetch: config.NoPrefetch, - TrieDirtyLimit: config.TrieDirtyCache, - TrieDirtyDisabled: config.NoPruning, - TrieTimeLimit: config.TrieTimeout, - SnapshotLimit: config.SnapshotCache, - Preimages: config.Preimages, - NoTries: config.NoTries, - StateHistory: config.StateHistory, - StateScheme: config.StateScheme, - TrieCommitInterval: config.TrieCommitInterval, - PathNodeBuffer: config.PathNodeBuffer, - ProposeBlockInterval: config.ProposeBlockInterval, - EnableProofKeeper: config.EnableProofKeeper, - KeepProofBlockSpan: config.KeepProofBlockSpan, - JournalFilePath: journalFilePath, - JournalFile: config.JournalFileEnabled, - EnableRecoverNodeBufferList: config.EnableRecoverNodeBufferList, + TrieCleanLimit: config.TrieCleanCache, + TrieCleanNoPrefetch: config.NoPrefetch, + TrieDirtyLimit: config.TrieDirtyCache, + TrieDirtyDisabled: config.NoPruning, + TrieTimeLimit: config.TrieTimeout, + SnapshotLimit: config.SnapshotCache, + Preimages: config.Preimages, + NoTries: config.NoTries, + StateHistory: config.StateHistory, + StateScheme: config.StateScheme, + TrieCommitInterval: config.TrieCommitInterval, + PathNodeBuffer: config.PathNodeBuffer, + ProposeBlockInterval: config.ProposeBlockInterval, + EnableProofKeeper: config.EnableProofKeeper, + KeepProofBlockSpan: config.KeepProofBlockSpan, + JournalFilePath: journalFilePath, + JournalFile: config.JournalFileEnabled, } ) // Override the chain config with provided settings. diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 97c4ceb3e6..51ff72c6cc 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -129,13 +129,12 @@ type Config struct { // State scheme represents the scheme used to store ethereum states and trie // nodes on top. It can be 'hash', 'path', or none which means use the scheme // consistent with persistent state. - StateScheme string `toml:",omitempty"` - PathNodeBuffer pathdb.NodeBufferType `toml:",omitempty"` // Type of trienodebuffer to cache trie nodes in disklayer - ProposeBlockInterval uint64 `toml:",omitempty"` // Keep the same with op-proposer propose block interval - EnableProofKeeper bool `toml:",omitempty"` // Whether to enable proof keeper - KeepProofBlockSpan uint64 `toml:",omitempty"` // Span block of keep proof - JournalFileEnabled bool `toml:",omitempty"` // Whether the TrieJournal is stored using journal file - EnableRecoverNodeBufferList bool `toml:",omitempty"` // Whether enable recover node buffer list. + StateScheme string `toml:",omitempty"` + PathNodeBuffer pathdb.NodeBufferType `toml:",omitempty"` // Type of trienodebuffer to cache trie nodes in disklayer + ProposeBlockInterval uint64 `toml:",omitempty"` // Keep the same with op-proposer propose block interval + EnableProofKeeper bool `toml:",omitempty"` // Whether to enable proof keeper + KeepProofBlockSpan uint64 `toml:",omitempty"` // Span block of keep proof + JournalFileEnabled bool `toml:",omitempty"` // Whether the TrieJournal is stored using journal file // RequiredBlocks is a set of block number -> hash mappings which must be in the // canonical chain of all remote peers. Setting the option makes geth verify the diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 8c02d5c16b..6300a2ee33 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -34,7 +34,6 @@ func (c Config) MarshalTOML() (interface{}, error) { EnableProofKeeper bool `toml:",omitempty"` KeepProofBlockSpan uint64 `toml:",omitempty"` JournalFileEnabled bool `toml:",omitempty"` - EnableRecoverNodeBufferList bool `toml:",omitempty"` RequiredBlocks map[uint64]common.Hash `toml:"-"` LightServ int `toml:",omitempty"` LightIngress int `toml:",omitempty"` @@ -94,7 +93,6 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.EnableProofKeeper = c.EnableProofKeeper enc.KeepProofBlockSpan = c.KeepProofBlockSpan enc.JournalFileEnabled = c.JournalFileEnabled - enc.EnableRecoverNodeBufferList = c.EnableRecoverNodeBufferList enc.RequiredBlocks = c.RequiredBlocks enc.LightServ = c.LightServ enc.LightIngress = c.LightIngress @@ -158,7 +156,6 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { EnableProofKeeper *bool `toml:",omitempty"` KeepProofBlockSpan *uint64 `toml:",omitempty"` JournalFileEnabled *bool `toml:",omitempty"` - EnableRecoverNodeBufferList *bool `toml:",omitempty"` RequiredBlocks map[uint64]common.Hash `toml:"-"` LightServ *int `toml:",omitempty"` LightIngress *int `toml:",omitempty"` @@ -253,9 +250,6 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.JournalFileEnabled != nil { c.JournalFileEnabled = *dec.JournalFileEnabled } - if dec.EnableRecoverNodeBufferList != nil { - c.EnableRecoverNodeBufferList = *dec.EnableRecoverNodeBufferList - } if dec.RequiredBlocks != nil { c.RequiredBlocks = dec.RequiredBlocks } diff --git a/trie/triedb/pathdb/database.go b/trie/triedb/pathdb/database.go index 187b3ab09e..82632d7c61 100644 --- a/trie/triedb/pathdb/database.go +++ b/trie/triedb/pathdb/database.go @@ -103,16 +103,15 @@ type layer interface { // Config contains the settings for database. type Config struct { - TrieNodeBufferType NodeBufferType // Type of trienodebuffer to cache trie nodes in disklayer - StateHistory uint64 // Number of recent blocks to maintain state history for - CleanCacheSize int // Maximum memory allowance (in bytes) for caching clean nodes - DirtyCacheSize int // Maximum memory allowance (in bytes) for caching dirty nodes - ReadOnly bool // Flag whether the database is opened in read only mode. - ProposeBlockInterval uint64 // Propose block to L1 block interval. - NotifyKeep NotifyKeepFunc // NotifyKeep is used to keep the proof which maybe queried by op-proposer. - JournalFilePath string // The journal file path - JournalFile bool // Whether to use journal file mode - EnableRecoverNodeBufferList bool // Whether enable recover node buffer list. + TrieNodeBufferType NodeBufferType // Type of trienodebuffer to cache trie nodes in disklayer + StateHistory uint64 // Number of recent blocks to maintain state history for + CleanCacheSize int // Maximum memory allowance (in bytes) for caching clean nodes + DirtyCacheSize int // Maximum memory allowance (in bytes) for caching dirty nodes + ReadOnly bool // Flag whether the database is opened in read only mode. + ProposeBlockInterval uint64 // Propose block to L1 block interval. + NotifyKeep NotifyKeepFunc // NotifyKeep is used to keep the proof which maybe queried by op-proposer. + JournalFilePath string // The journal file path + JournalFile bool // Whether to use journal file mode } // sanitize checks the provided user configurations and changes anything that's diff --git a/trie/triedb/pathdb/journal.go b/trie/triedb/pathdb/journal.go index 660d30b576..b6608fe42b 100644 --- a/trie/triedb/pathdb/journal.go +++ b/trie/triedb/pathdb/journal.go @@ -266,8 +266,7 @@ func (db *Database) loadLayers() layer { dl *diskLayer stateID = rawdb.ReadPersistentStateID(db.diskdb) ) - if (errors.Is(err, errMissJournal) || errors.Is(err, errUnmatchedJournal)) && db.config.EnableRecoverNodeBufferList && - db.config.TrieNodeBufferType == NodeBufferList { + if (errors.Is(err, errMissJournal) || errors.Is(err, errUnmatchedJournal)) && db.config.TrieNodeBufferType == NodeBufferList { start := time.Now() if db.freezer == nil { log.Crit("Use unopened freezer db to recover node buffer list") diff --git a/trie/triedb/pathdb/nodebufferlist.go b/trie/triedb/pathdb/nodebufferlist.go index f8247a0ad7..c99ff3e91f 100644 --- a/trie/triedb/pathdb/nodebufferlist.go +++ b/trie/triedb/pathdb/nodebufferlist.go @@ -283,7 +283,6 @@ func (nf *nodebufferlist) createBlockInterval(startBlock, endBlock uint64) [][]u return intervalBoundaries } -// startStateID: persistentStateID+1, endStateID: freezerLength func (nf *nodebufferlist) createStateInterval(freezer *rawdb.ResettableFreezer, startStateID, endStateID uint64, blockIntervals [][]uint64) ([][]uint64, error) { blockMap, err := readAllBlockNumbers(freezer, startStateID, endStateID)