pathdb: load diff layers from freezer db

cmd/geth/chaincmd.go

@@ -39,6 +39,9 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
+	"github.com/olekukonko/tablewriter"
 	"github.com/urfave/cli/v2"
 )
 
@@ -156,6 +159,18 @@ It's deprecated, please use "geth db import" instead.
 This command dumps out the state for a given block (or latest, if none provided).
 `,
 	}
+
+	dumpRootHashCommand = &cli.Command{
+		Action: dumpAllRootHashInPath,
+		Name:   "dump-roothash",
+		Usage:  "Dump all available state root hash in path mode",
+		Flags:  flags.Merge([]cli.Flag{}, utils.DatabaseFlags),
+		Description: `
+The dump-roothash command dump all available state root hash in path mode.
+If you use "dump" command in path mode, please note that it only keeps at most 129 blocks which belongs to diffLayer or diskLayer.
+Therefore, you must specify the blockNumber or blockHash that locates in diffLayer or diskLayer.
+"geth" will print all available blockNumber and related block state root hash, and you can query block hash by block number.
+`}
 )
 
 // initGenesis will initialise the given JSON format genesis file and writes it as
@@ -483,3 +498,26 @@ func hashish(x string) bool {
 	_, err := strconv.Atoi(x)
 	return err != nil
 }
+
+func dumpAllRootHashInPath(ctx *cli.Context) error {
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+	db := utils.MakeChainDatabase(ctx, stack, true)
+	defer db.Close()
+	triedb := trie.NewDatabase(db, &trie.Config{PathDB: pathdb.ReadOnly})
+	defer triedb.Close()
+
+	scheme, err := rawdb.ParseStateScheme(ctx.String(utils.StateSchemeFlag.Name), db)
+	if err != nil {
+		return err
+	}
+	if scheme == rawdb.HashScheme {
+		return errors.New("incorrect state scheme, you should use it in path mode")
+	}
+
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetHeader([]string{"Block Number", "Block State Root Hash"})
+	table.AppendBulk(triedb.GetAllRooHash())
+	table.Render()
+	return nil
+}

cmd/geth/dbcmd.go

@@ -76,6 +76,7 @@ Remove blockchain and state databases`,
 			dbPruneHashTrieCmd,
 			dbTrieGetCmd,
 			dbTrieDeleteCmd,
+			ancientToDiffLayerCmd,
 		},
 	}
 	dbInspectCmd = &cli.Command{
@@ -259,6 +260,13 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "Shows metadata about the chain status.",
 	}
+	ancientToDiffLayerCmd = &cli.Command{
+		Name:        "ancient-to-dl",
+		Usage:       "Convert the data in ancientDB into diffLayer",
+		Description: "A convenient test tool to for path db diffLayer converting",
+		Action:      ancientToDiffLayer,
+		Flags:       flags.Merge(utils.DatabaseFlags),
+	}
 )
 
 func removeDB(ctx *cli.Context) error {
@@ -1090,6 +1098,21 @@ func hbss2pbss(ctx *cli.Context) error {
 	return nil
 }
 
+func ancientToDiffLayer(ctx *cli.Context) error {
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+	db := utils.MakeChainDatabase(ctx, stack, true)
+	defer db.Close()
+	triedb := utils.MakeTrieDatabase(ctx, stack, db, false, true, false)
+	// triedb := trie.NewDatabase(db, &trie.Config{PathDB: nil})
+	defer triedb.Close()
+
+	if err := triedb.DiffLayerConvertTool(); err != nil {
+		log.Error("Failed to get diff layer from ancient db", "error", err)
+	}
+	return nil
+}
+
 func pruneHashTrie(ctx *cli.Context) error {
 	if ctx.NArg() != 0 {
 		return fmt.Errorf("required none argument")

cmd/geth/main.go

@@ -238,6 +238,7 @@ func init() {
 		removedbCommand,
 		dumpCommand,
 		dumpGenesisCommand,
+		dumpRootHashCommand,
 		// See accountcmd.go:
 		accountCommand,
 		walletCommand,

trie/database.go

@@ -363,3 +363,21 @@ func (db *Database) IsVerkle() bool {
 func (db *Database) Config() *Config {
 	return db.config
 }
+
+// DiffLayerConvertTool
+func (db *Database) DiffLayerConvertTool() error {
+	pdb, ok := db.backend.(*pathdb.Database)
+	if !ok {
+		return errors.New("not supported")
+	}
+	return pdb.ConvertTool1(&trieLoader{db: db})
+}
+
+func (db *Database) GetAllRooHash() [][]string {
+	pdb, ok := db.backend.(*pathdb.Database)
+	if !ok {
+		log.Error("Not supported")
+		return nil
+	}
+	return pdb.GetAllRooHash()
+}

trie/triedb/pathdb/database.go

@@ -21,6 +21,9 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"sort"
+	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -159,6 +162,7 @@ type Database struct {
 	freezer    *rawdb.ResettableFreezer // Freezer for storing trie histories, nil possible in tests
 	lock       sync.RWMutex             // Lock to prevent mutations from happening at the same time
 	capLock    sync.Mutex
+	tree2      *layerTree
 }
 
 // New attempts to load an already existing layer from a persistent key-value
@@ -580,3 +584,229 @@ func (db *Database) DeleteTrieJournal(writer ethdb.KeyValueWriter) error {
 	}
 	return nil
 }
+
+func (db *Database) ConvertTool1(loader triestate.TrieLoader) error {
+	dl := db.tree.bottom()
+	stateID := rawdb.ReadStateID(db.diskdb, dl.rootHash())
+	persistentStateID := rawdb.ReadPersistentStateID(db.diskdb)
+
+	ancient, err := db.diskdb.AncientDatadir()
+	if err != nil {
+		log.Error("Failed to get ancient datadir", "error", err)
+		return err
+	}
+	freezer, err := rawdb.NewStateFreezer(ancient, true)
+	if err != nil {
+		log.Error("Failed to new state freezer", "error", err)
+		return err
+	}
+	db.freezer = freezer
+
+	freezerLength, err := db.freezer.Ancients()
+	if err != nil {
+		log.Error("Failed to get freezer ancients", "error", err)
+		return err
+	}
+	tail, err := db.freezer.Tail()
+	if err != nil {
+		log.Error("Failed to get freezer tail", "error", err)
+		return err
+	}
+	log.Info("Print ancient db meta", "state id", *stateID, "persistent state id", persistentStateID,
+		"freezer length", freezerLength, "freezer tail", tail, "config", db.config.StateHistory,
+		"bottom stateID", dl.stateID(), "bottom root", dl.rootHash().String())
+
+	db.tree2 = newLayerTree(dl)
+	waitingRecoverNum := freezerLength - tail
+	start := time.Now()
+	historySize := 0
+	diffSize := uint64(0)
+	for i := uint64(0); i < waitingRecoverNum; i++ {
+		h, err := readHistory(db.freezer, *stateID-i)
+		if err != nil {
+			if checkError(err) {
+				log.Info("There are no more states in disk db", "state id", *stateID-i)
+				continue
+			}
+			log.Error("Failed to read history from freezer db", "error", err)
+			return err
+		}
+		historySize += h.Size()
+		log.Info("print history size", "size", common.StorageSize(h.Size()), "history root", h.meta.root.String(),
+			"history parent root", h.meta.parent.String(), "current state id", *stateID-i)
+
+		incomplete := make(map[common.Address]struct{})
+		for _, addr := range h.meta.incomplete {
+			incomplete[addr] = struct{}{}
+		}
+		states := triestate.New(h.accounts, h.storages, incomplete)
+
+		size, err := db.addDiffLayer(h.meta.root, h.meta.parent, *stateID-i, h.meta.block, nil, states)
+		if err != nil {
+			log.Error("Failed to add diff layer", "error", err)
+			return err
+		}
+		diffSize += size
+	}
+	layerTreeSize := uint64(db.tree2.len() * 32)
+	log.Info("Succeed to add diff layer", "elapsed", common.PrettyDuration(time.Since(start)),
+		"waitingRecoverNum", waitingRecoverNum, "total history size", common.StorageSize(historySize),
+		"total diff size", common.StorageSize(diffSize), "layer tree size", common.StorageSize(layerTreeSize))
+
+	return nil
+}
+
+func (db *Database) ConvertTool(loader triestate.TrieLoader) error {
+	dl := db.tree.bottom()
+	stateID := rawdb.ReadStateID(db.diskdb, dl.rootHash())
+	persistentStateID := rawdb.ReadPersistentStateID(db.diskdb)
+
+	ancient, err := db.diskdb.AncientDatadir()
+	if err != nil {
+		log.Error("Failed to get ancient datadir", "error", err)
+		return err
+	}
+	freezer, err := rawdb.NewStateFreezer(ancient, true)
+	if err != nil {
+		log.Error("Failed to new state freezer", "error", err)
+		return err
+	}
+	db.freezer = freezer
+
+	freezerLength, err := db.freezer.Ancients()
+	if err != nil {
+		log.Error("Failed to get freezer ancients", "error", err)
+		return err
+	}
+	tail, err := db.freezer.Tail()
+	if err != nil {
+		log.Error("Failed to get freezer tail", "error", err)
+		return err
+	}
+	log.Info("Print ancient db meta", "state id", *stateID, "persistent state id", persistentStateID,
+		"freezer length", freezerLength, "freezer tail", tail, "config", db.config.StateHistory,
+		"bottom stateID", dl.stateID(), "bottom root", dl.rootHash().String())
+
+	db.tree2 = newLayerTree(dl)
+	waitingRecoverNum := freezerLength - persistentStateID
+	start := time.Now()
+	var (
+		nodes *trienode.MergedNodeSet
+		count = uint64(2)
+	)
+	for {
+		h, err := readHistory(db.freezer, *stateID+count)
+		if err != nil {
+			if checkError(err) {
+				log.Info("There are no more states in disk db", "count", count)
+				break
+			}
+			log.Error("Failed to read history from freezer db", "error", err)
+			return err
+		}
+		log.Info("print history size", "size", h.Size(), "history root", h.meta.root.String(),
+			"history parent root", h.meta.parent.String(), "current state id", *stateID+count)
+
+		if count > 2 {
+			break
+		}
+		dl, nodes, err = dl.apply(dl.rootHash(), h, loader)
+		if err != nil {
+			log.Error("Failed to revert", "error", err)
+			return err
+		}
+		db.tree.reset(dl)
+
+		incomplete := make(map[common.Address]struct{})
+		for _, addr := range h.meta.incomplete {
+			incomplete[addr] = struct{}{}
+		}
+		states := triestate.New(h.accounts, h.storages, incomplete)
+
+		if _, err = db.addDiffLayer(h.meta.root, h.meta.parent, *stateID+count, h.meta.block, nodes, states); err != nil {
+			log.Error("Failed to add diff layer", "error", err)
+			return err
+		}
+		count++
+	}
+	// for i := uint64(0); i < 2; i++ {
+	// 	h, err := readHistory(db.freezer, *stateID-i)
+	// 	if err != nil {
+	// 		log.Error("Failed to read history from freezer db", "error", err)
+	// 		return err
+	// 	}
+	// 	log.Info("print history size", "size", h.Size(), "history root", h.meta.root.String(),
+	// 		"history parent root", h.meta.parent.String(), "current state id", *stateID-i)
+	//
+	// 	dl, nodes, err = dl.revert1(h, loader)
+	// 	if err != nil {
+	// 		log.Error("Failed to revert", "error", err)
+	// 		return err
+	// 	}
+	// 	db.tree.reset(dl)
+	//
+	// 	incomplete := make(map[common.Address]struct{})
+	// 	for _, addr := range h.meta.incomplete {
+	// 		incomplete[addr] = struct{}{}
+	// 	}
+	// 	states := triestate.New(h.accounts, h.storages, incomplete)
+	//
+	// 	if err = db.addDiffLayer(h.meta.root, h.meta.parent, *stateID-i, h.meta.block, nodes, states); err != nil {
+	// 		log.Error("Failed to add diff layer", "error", err)
+	// 		return err
+	// 	}
+	// }
+	log.Info("Succeed to add diff layer", "elapsed", common.PrettyDuration(time.Since(start)),
+		"waitingRecoverNum", waitingRecoverNum)
+
+	return nil
+}
+
+func (db *Database) addDiffLayer(root common.Hash, parentRoot common.Hash, stateID uint64, block uint64,
+	nodes *trienode.MergedNodeSet, states *triestate.Set) (uint64, error) {
+	// Hold the lock to prevent concurrent mutations.
+	db.lock.Lock()
+	defer db.lock.Unlock()
+
+	root, parentRoot = types.TrieRootHash(root), types.TrieRootHash(parentRoot)
+	if root == parentRoot {
+		return 0, errors.New("layer cycle")
+	}
+	// TODO: parent now is nil
+	l := newDiffLayer(nil, root, stateID, block, nil, states)
+
+	// TODO: no need to use lock now
+	// db.tree2.lock.Lock()
+	db.tree2.layers[l.rootHash()] = l
+	// db.tree2.lock.Unlock()
+
+	log.Info("done", "layer tree length", db.tree2.len(), "size", common.StorageSize(l.memory))
+	return l.memory, nil
+}
+
+func checkError(err error) bool {
+	if strings.Contains(err.Error(), "state history not found") {
+		return true
+	}
+	return false
+}
+
+func (db *Database) GetAllRooHash() [][]string {
+	db.lock.Lock()
+	defer db.lock.Unlock()
+
+	data := make([][]string, 0, len(db.tree.layers))
+	for _, v := range db.tree.layers {
+		if dl, ok := v.(*diffLayer); ok {
+			data = append(data, []string{fmt.Sprintf("%d", dl.block), dl.rootHash().String()})
+		}
+	}
+	sort.Slice(data, func(i, j int) bool {
+		block1, _ := strconv.Atoi(data[i][0])
+		block2, _ := strconv.Atoi(data[j][0])
+		return block1 > block2
+	})
+
+	data = append(data, []string{"-1", db.tree.bottom().rootHash().String()})
+	return data
+}