pathdb: load diff layers from freezer db

cmd/geth/chaincmd.go

@@ -41,6 +41,9 @@ import (
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ethereum/go-ethereum/triedb/pathdb"
+	"github.com/olekukonko/tablewriter"
 	"github.com/urfave/cli/v2"
 )
 
@@ -185,6 +188,18 @@ It's deprecated, please use "geth db import" instead.
 This command dumps out the state for a given block (or latest, if none provided).
 `,
 	}
+
+	dumpRootHashCommand = &cli.Command{
+		Action: dumpAllRootHashInPath,
+		Name:   "dump-roothash",
+		Usage:  "Dump all available state root hash in path mode",
+		Flags:  flags.Merge([]cli.Flag{}, utils.DatabaseFlags),
+		Description: `
+The dump-roothash command dump all available state root hash in path mode.
+If you use "dump" command in path mode, please note that it only keeps at most 129 blocks which belongs to diffLayer or diskLayer.
+Therefore, you must specify the blockNumber or blockHash that locates in diffLayer or diskLayer.
+"geth" will print all available blockNumber and related block state root hash, and you can query block hash by block number.
+`}
 )
 
 // initGenesis will initialise the given JSON format genesis file and writes it as
@@ -602,3 +617,26 @@ func hashish(x string) bool {
 	_, err := strconv.Atoi(x)
 	return err != nil
 }
+
+func dumpAllRootHashInPath(ctx *cli.Context) error {
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+	db := utils.MakeChainDatabase(ctx, stack, true)
+	defer db.Close()
+	triedb := trie.NewDatabase(db, &trie.Config{PathDB: pathdb.ReadOnly})
+	defer triedb.Close()
+
+	scheme, err := rawdb.ParseStateScheme(ctx.String(utils.StateSchemeFlag.Name), db)
+	if err != nil {
+		return err
+	}
+	if scheme == rawdb.HashScheme {
+		return errors.New("incorrect state scheme, you should use it in path mode")
+	}
+
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetHeader([]string{"Block Number", "Block State Root Hash"})
+	table.AppendBulk(triedb.GetAllRooHash())
+	table.Render()
+	return nil
+}

cmd/geth/dbcmd.go

@@ -87,6 +87,7 @@ Remove blockchain and state databases`,
 			dbPruneHashTrieCmd,
 			dbTrieGetCmd,
 			dbTrieDeleteCmd,
+			ancientToDiffLayerCmd,
 		},
 	}
 	dbInspectCmd = &cli.Command{
@@ -270,6 +271,13 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "Shows metadata about the chain status.",
 	}
+	ancientToDiffLayerCmd = &cli.Command{
+		Name:        "ancient-to-dl",
+		Usage:       "Convert the data in ancientDB into diffLayer",
+		Description: "A convenient test tool to for path db diffLayer converting",
+		Action:      ancientToDiffLayer,
+		Flags:       flags.Merge(utils.DatabaseFlags),
+	}
 )
 
 func removeDB(ctx *cli.Context) error {
@@ -1113,6 +1121,21 @@ func hbss2pbss(ctx *cli.Context) error {
 	return nil
 }
 
+func ancientToDiffLayer(ctx *cli.Context) error {
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+	db := utils.MakeChainDatabase(ctx, stack, true)
+	defer db.Close()
+	triedb := utils.MakeTrieDatabase(ctx, stack, db, false, true, false)
+	// triedb := trie.NewDatabase(db, &trie.Config{PathDB: nil})
+	defer triedb.Close()
+
+	if err := triedb.DiffLayerConvertTool(); err != nil {
+		log.Error("Failed to get diff layer from ancient db", "error", err)
+	}
+	return nil
+}
+
 func pruneHashTrie(ctx *cli.Context) error {
 	if ctx.NArg() != 0 {
 		return fmt.Errorf("required none argument")

cmd/geth/main.go

@@ -241,6 +241,7 @@ func init() {
 		removedbCommand,
 		dumpCommand,
 		dumpGenesisCommand,
+		dumpRootHashCommand,
 		// See accountcmd.go:
 		accountCommand,
 		walletCommand,

trie/trienode/node.go

@@ -197,3 +197,8 @@ func (set *MergedNodeSet) Flatten() map[common.Hash]map[string]*Node {
 	}
 	return nodes
 }
+
+// // ReverseFlatten reverts flattened two-dimensional map to merge node set
+// func ReverseFlatten(nodes map[common.Hash]map[string]*Node) MergedNodeSet {
+//
+// }

trie/triestate/state.go

@@ -24,6 +24,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie/trienode"
 	"golang.org/x/crypto/sha3"
@@ -142,6 +143,47 @@ func Apply(prevRoot common.Hash, postRoot common.Hash, accounts map[common.Addre
 	return ctx.nodes.Flatten(), nil
 }
 
+func ApplyForDiff(prevRoot common.Hash, postRoot common.Hash, accounts map[common.Address][]byte, storages map[common.Address]map[common.Hash][]byte,
+	loader TrieLoader) (*trienode.MergedNodeSet, error) {
+	tr, err := loader.OpenTrie(prevRoot)
+	if err != nil {
+		log.Error("Failed to open trie", "error", err)
+		return nil, err
+	}
+	ctx := &context{
+		prevRoot:    prevRoot,
+		postRoot:    postRoot,
+		accounts:    accounts,
+		storages:    storages,
+		accountTrie: tr,
+		nodes:       trienode.NewMergedNodeSet(),
+	}
+	for addr, account := range accounts {
+		var err error
+		if len(account) == 0 {
+			err = deleteAccountForRecovering(ctx, loader, addr)
+		} else {
+			err = updateAccountForRecovering(ctx, loader, addr)
+		}
+		// err = updateAccountForRecovering(ctx, loader, addr)
+		if err != nil {
+			return nil, fmt.Errorf("failed to apply state, err: %w", err)
+		}
+	}
+	root, result, err := tr.Commit(false)
+
+	if err != nil {
+		return nil, err
+	}
+	if root != postRoot {
+		return nil, fmt.Errorf("failed to revert state, want %#x, got %#x", postRoot, root)
+	}
+	if err := ctx.nodes.Merge(result); err != nil {
+		return nil, err
+	}
+	return ctx.nodes, nil
+}
+
 // updateAccount the account was present in prev-state, and may or may not
 // existent in post-state. Apply the reverse diff and verify if the storage
 // root matches the one in prev-state account.
@@ -217,6 +259,7 @@ func deleteAccount(ctx *context, loader TrieLoader, addr common.Address) error {
 	addrHash := h.hash(addr.Bytes())
 	blob, err := ctx.accountTrie.Get(addrHash.Bytes())
 	if err != nil {
+		log.Error("9")
 		return err
 	}
 	if len(blob) == 0 {
@@ -240,6 +283,7 @@ func deleteAccount(ctx *context, loader TrieLoader, addr common.Address) error {
 	}
 	root, result, err := st.Commit(false)
 	if err != nil {
+		log.Error("13")
 		return err
 	}
 	if root != types.EmptyRootHash {
@@ -256,6 +300,128 @@ func deleteAccount(ctx *context, loader TrieLoader, addr common.Address) error {
 	return ctx.accountTrie.Delete(addrHash.Bytes())
 }
 
+// updateAccountForRecovering the account is present in post-state, and may or may not
+// be existent in prev-state. Apply the diff and verify if the storage root matches the
+// one in post-state account.
+func updateAccountForRecovering(ctx *context, loader TrieLoader, addr common.Address) error {
+	// The account was present in post-state, decode it from the
+	// 'slim-rlp' format bytes.
+	h := newHasher()
+	defer h.release()
+
+	addrHash := h.hash(addr.Bytes())
+	post, err := types.FullAccount(ctx.accounts[addr])
+	if err != nil {
+		log.Error("Failed to full account for updating", "error", err, "addr", addr.String())
+		return err
+	}
+	// The account may or may not be existent in prev-state, try to
+	// load it and decode if it's found.
+	blob, err := ctx.accountTrie.Get(addrHash.Bytes())
+	if err != nil {
+		log.Error("Failed to get for updating", "error", err)
+		return err
+	}
+	prev := types.NewEmptyStateAccount()
+	if len(blob) != 0 {
+		if err = rlp.DecodeBytes(blob, &prev); err != nil {
+			log.Error("Failed to decode bytes for updating", "error", err)
+			return err
+		}
+	}
+	// Apply all storage changes into the prev-state storage trie
+	st, err := loader.OpenStorageTrie(ctx.prevRoot, addrHash, prev.Root)
+	if err != nil {
+		log.Error("Failed to open storage trie for updating", "error", err)
+		return err
+	}
+	for k, v := range ctx.storages[addr] {
+		if len(v) == 0 {
+			err = st.Delete(k.Bytes())
+		} else {
+			err = st.Update(k.Bytes(), v)
+		}
+		if err != nil {
+			log.Error("Failed to delete or update", "error", err)
+			return err
+		}
+	}
+	root, result, err := st.Commit(false)
+	if err != nil {
+		log.Error("Failed to commit for updating", "error", err)
+		return err
+	}
+	if root != post.Root {
+		return errors.New("failed to reset storage trie")
+	}
+	if result != nil {
+		if err = ctx.nodes.Merge(result); err != nil {
+			log.Error("Failed to merge for updating", "error", err)
+			return err
+		}
+	}
+	// Write the post-state account into the main trie
+	full, err := rlp.EncodeToBytes(post)
+	if err != nil {
+		log.Error("Failed to encode bytes", "error", err)
+		return err
+	}
+	return ctx.accountTrie.Update(addrHash.Bytes(), full)
+}
+
+// deleteAccountForRecovering the account is not present in post-state, and was expected
+// to be existent in prev-state. Apply the diff and verify if the account and storage
+// is wiped out correctly.
+func deleteAccountForRecovering(ctx *context, loader TrieLoader, addr common.Address) error {
+	// The account must be existent in prev-state, load the account
+	h := newHasher()
+	defer h.release()
+
+	addrHash := h.hash(addr.Bytes())
+	blob, err := ctx.accountTrie.Get(addrHash.Bytes())
+	if err != nil {
+		return err
+	}
+	if len(blob) == 0 {
+		return fmt.Errorf("account is nonexistent %#x", addrHash)
+	}
+	var prev types.StateAccount
+	if err = rlp.DecodeBytes(blob, prev); err != nil {
+		log.Error("Failed to decode bytes for deleting accounts", "error", err)
+		return err
+	}
+	st, err := loader.OpenStorageTrie(ctx.prevRoot, addrHash, prev.Root)
+	if err != nil {
+		log.Error("Failed to open storage trie for del", "error", err)
+		return err
+	}
+	for k, v := range ctx.storages[addr] {
+		if len(v) != 0 {
+			return errors.New("expect storage deletion")
+		}
+		if err = st.Delete(k.Bytes()); err != nil {
+			log.Error("Failed to delete for del", "error", err)
+			return err
+		}
+	}
+	root, result, err := st.Commit(false)
+	if err != nil {
+		log.Error("Failed to commit for del", "error", err)
+		return err
+	}
+	if root != types.EmptyRootHash {
+		return errors.New("failed to clear storage trie")
+	}
+	if result != nil {
+		if err = ctx.nodes.Merge(result); err != nil {
+			log.Error("Failed to merge for del", "error", err)
+			return err
+		}
+	}
+	// Delete the prev-state account from the main trie.
+	return ctx.accountTrie.Delete(addrHash.Bytes())
+}
+
 // hasher is used to compute the sha256 hash of the provided data.
 type hasher struct{ sha crypto.KeccakState }
 

triedb/database.go

@@ -391,3 +391,21 @@ func (db *Database) IsVerkle() bool {
 func (db *Database) Config() *Config {
 	return db.config
 }
+
+// DiffLayerConvertTool
+func (db *Database) DiffLayerConvertTool() error {
+	pdb, ok := db.backend.(*pathdb.Database)
+	if !ok {
+		return errors.New("not supported")
+	}
+	return pdb.ConvertTool1(&trieLoader{db: db})
+}
+
+func (db *Database) GetAllRooHash() [][]string {
+	pdb, ok := db.backend.(*pathdb.Database)
+	if !ok {
+		log.Error("Not supported")
+		return nil
+	}
+	return pdb.GetAllRooHash()
+}