From 3e13e83656def9830ebcdaf8b5d98b83d4b85ce7 Mon Sep 17 00:00:00 2001 From: Jordan Hrycaj Date: Thu, 26 Sep 2024 18:01:22 +0100 Subject: [PATCH 1/2] Backend agnostic read-only DB wrapper framework --- nimbus/db/edge_db.nim | 47 +++++++ nimbus/db/edge_db/db_desc.nim | 119 +++++++++++++++++ nimbus/db/edge_db/init_era1_coredb.nim | 174 +++++++++++++++++++++++++ tests/all_tests.nim | 3 +- tests/test_edgedb.nim | 20 +++ tests/test_edgedb/test_era1_coredb.nim | 87 +++++++++++++ tests/test_edgedb/test_helpers.nim | 145 +++++++++++++++++++++ 7 files changed, 594 insertions(+), 1 deletion(-) create mode 100644 nimbus/db/edge_db.nim create mode 100644 nimbus/db/edge_db/db_desc.nim create mode 100644 nimbus/db/edge_db/init_era1_coredb.nim create mode 100644 tests/test_edgedb.nim create mode 100644 tests/test_edgedb/test_era1_coredb.nim create mode 100644 tests/test_edgedb/test_helpers.nim diff --git a/nimbus/db/edge_db.nim b/nimbus/db/edge_db.nim new file mode 100644 index 0000000000..91408b2452 --- /dev/null +++ b/nimbus/db/edge_db.nim @@ -0,0 +1,47 @@ +# Nimbus +# Copyright (c) 2023-2024 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +## Policy driven read-only database wrapper +## ======================================== +## +## It lives on the edge and pretends to be agnostic of a particular +## backend implementation. +## +{.push raises: [].} + +import + pkg/eth/common, + pkg/results, + edge_db/[ + db_desc, + init_era1_coredb, + ] + +export + EdgeDbColumn, + EdgeDbError, + EdgeDbRef, + init + +proc get*( + edg: EdgeDbRef; + col: EdgeDbColumn; + key: uint64; + ): Result[Blob,EdgeDbError] = + edg.uintGetPolFn(edg, col, key) + +proc get*( + edg: EdgeDbRef; + col: EdgeDbColumn; + key: openArray[byte]; + ): Result[Blob,EdgeDbError] = + edg.blobGetPolFn(edg, col, key) + +# End diff --git a/nimbus/db/edge_db/db_desc.nim b/nimbus/db/edge_db/db_desc.nim new file mode 100644 index 0000000000..2761894c4c --- /dev/null +++ b/nimbus/db/edge_db/db_desc.nim @@ -0,0 +1,119 @@ +# Nimbus +# Copyright (c) 2023-2024 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +{.push raises: [].} + +import + pkg/eth/common, + pkg/results + +type + EdgeDbError* = enum + ## Allows more granulated failure information. + NothingSerious = 0 + EdgeKeyNotFound + EdgeColUnsupported + EdgeKeyTypeUnsupported + + EdgeDbColumn* = enum + ## Specify object type to query for + Oops = 0 + EthBlockData + EthHeaderData + EthBodyData + + EdgeDbGetRef* = ref object of RootObj + ## Descriptor common to a set of `getFn()` implementations. This basic type + ## will be interited by sprcific implementations. + + + EdgeDbUintGetFn* = + proc(dsc: EdgeDbGetRef; + col: EdgeDbColumn; + key: uint64; + ): Result[Blob,EdgeDbError] + {.gcsafe, raises: [].} + ## Particular `getFn()` instance. Will return a RLP encoded serialised + ## data result. + + EdgeDbBlobGetFn* = + proc(dsc: EdgeDbGetRef; + col: EdgeDbColumn; + key: openArray[byte]; + ): Result[Blob,EdgeDbError] + {.gcsafe, raises: [].} + ## Ditto for `Blob` like key + + + EdgeDbUintGetPolicyFn* = + proc(edg: EdgeDbRef; + col: EdgeDbColumn; + key: uint64; + ): Result[Blob,EdgeDbError] + {.gcsafe, raises: [].} + ## Implemenation of `get()`. This function employs a set of `getFn()` + ## instances (in some order) for finding a result. + + EdgeDbBlobGetPolicyFn* = + proc(edg: EdgeDbRef; + col: EdgeDbColumn; + key: openArray[byte]; + ): Result[Blob,EdgeDbError] + {.gcsafe, raises: [].} + ## Ditto for `Blob` like key + + + EdgeDbRef* = ref object + ## Visible database wrapper. + getDesc*: EdgeDbGetRef + uintGetFns*: seq[EdgeDbUintGetFn] + blobGetFns*: seq[EdgeDbBlobGetFn] + uintGetPolFn*: EdgeDbUintGetPolicyFn + blobGetPolFn*: EdgeDbBlobGetPolicyFn + +# ------------------------------------------------------------------------------ +# Public helpers, sequentially trying a list of `getFn()` instances +# ------------------------------------------------------------------------------ + +proc uintGetSeqentiallyUntilFound*( + edg: EdgeDbRef; + col: EdgeDbColumn; + key: uint64; + ): Result[Blob,EdgeDbError] = + ## Simple linear get policy. + var error = EdgeColUnsupported + + for fn in edg.uintGetFns: + let err = edg.getDesc.fn(col,key).errorOr: + return ok(value) + if err != EdgeColUnsupported: + error = EdgeKeyNotFound + + err(error) + +proc blobGetSeqentiallyUntilFound*( + edg: EdgeDbRef; + col: EdgeDbColumn; + key: openArray[byte]; + ): Result[Blob,EdgeDbError] = + ## Ditto for `Blob` like key + var error = EdgeColUnsupported + + for fn in edg.blobGetFns: + let err = edg.getDesc.fn(col,key).errorOr: + return ok(value) + if err != EdgeColUnsupported: + error = EdgeKeyNotFound + + err(error) + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/edge_db/init_era1_coredb.nim b/nimbus/db/edge_db/init_era1_coredb.nim new file mode 100644 index 0000000000..0c119377b4 --- /dev/null +++ b/nimbus/db/edge_db/init_era1_coredb.nim @@ -0,0 +1,174 @@ +# Nimbus +# Copyright (c) 2023-2024 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +import + std/sets, + pkg/eth/[common, rlp], + pkg/results, + ./db_desc, + ".."/[core_db, era1_db, storage_types] + +type + EdgeE1CdbRef = ref object of EdgeDbGetRef + era1: Era1DbRef + cdb: CoreDbRef + + EdgeE1CdbDbg = ref object of EdgeE1CdbRef + ## For debugging. Keys in the `e1xcpt` set are not found by the `Era1` + ## driver. + e1xcpt: HashSet[BlockNumber] + +# ------------------------------------------------------------------------------ +# Private helpers +# ------------------------------------------------------------------------------ + +proc getBlockHash( + w: EdgeDbGetRef; + k: uint64; + ): Result[Hash256,EdgeDbError] = + var hash: Hash256 + if EdgeE1CdbRef(w).cdb.getBlockHash(BlockNumber(k), hash): + return ok(hash) + err(EdgeKeyNotFound) + +proc getBlockHeader( + w: EdgeDbGetRef; + h: Hash256; + ): Result[BlockHeader,EdgeDbError] = + var header: BlockHeader + if EdgeE1CdbRef(w).cdb.getBlockHeader(h, header): + return ok(header) + err(EdgeKeyNotFound) + +proc getBlockBody( + w: EdgeDbGetRef; + h: Hash256; + ): Result[BlockBody,EdgeDbError] = + var body: BlockBody + if EdgeE1CdbRef(w).cdb.getBlockBody(h, body): + return ok(body) + err(EdgeKeyNotFound) + +# ------------------------------------------------------------------------------ +# Private drivers +# ------------------------------------------------------------------------------ + +proc blobGetUnsupported( + dsc: EdgeDbGetRef; + col: EdgeDbColumn; + key: openArray[byte]; + ): Result[Blob,EdgeDbError] = + err(EdgeKeyTypeUnsupported) + + +proc getEra1Obj( + dsc: EdgeDbGetRef; + col: EdgeDbColumn; + key: uint64; + ): Result[Blob,EdgeDbError] = + case col: + of EthBlockData: + let w = EdgeE1CdbRef(dsc).era1.getEthBlock(key).valueOr: + return err(EdgeKeyNotFound) + ok(rlp.encode w) + + of EthHeaderData: + let w = EdgeE1CdbRef(dsc).era1.getBlockTuple(key).valueOr: + return err(EdgeKeyNotFound) + ok(rlp.encode w.header) + + of EthBodyData: + let w = EdgeE1CdbRef(dsc).era1.getBlockTuple(key).valueOr: + return err(EdgeKeyNotFound) + ok(rlp.encode w.body) + + else: + err(EdgeColUnsupported) + + +proc getCoreDbObj( + dsc: EdgeDbGetRef; + col: EdgeDbColumn; + key: uint64; + ): Result[Blob,EdgeDbError] = + case col: + of EthBlockData: + let h = ? dsc.getBlockHash(key) + ok(rlp.encode EthBlock.init(? dsc.getBlockHeader(h), ? dsc.getBlockBody(h))) + + of EthHeaderData: + let + h = ? dsc.getBlockHash(key) + kvt = EdgeE1CdbRef(dsc).cdb.ctx.getKvt() + + # Fetching directly from the DB avoids re-encoding the header object + data = kvt.get(genericHashKey(h).toOpenArray).valueOr: + return err(EdgeKeyNotFound) + ok(data) + + of EthBodyData: + ok(rlp.encode(? dsc.getBlockBody(? dsc.getBlockHash(key)))) + + else: + err(EdgeColUnsupported) + +# ------------------------------------------------------------------------------ +# Public constructor (debuging version) +# ------------------------------------------------------------------------------ + +proc init*( + T: type EdgeDbRef; + era1: Era1DbRef; + e1xcpt: HashSet[BlockNumber]; + cdb: CoreDbRef; + ): T = + ## Initalise for trying `Era1` first, then `CoreDb`. This goes with some + ## exceptions for `Era1`. If an argument key is in `excpt` if will not be + ## found by the `Era1` driver. + ## + ## This constructor is mainly designed for debugging. + ## + proc getEra1Expt( + dsc: EdgeDbGetRef; + col: EdgeDbColumn; + key: uint64; + ): Result[Blob,EdgeDbError] = + if key in EdgeE1CdbDbg(dsc).e1xcpt: + return err(EdgeKeyNotFound) + dsc.getEra1Obj(col, key) + + T(getDesc: EdgeE1CdbDbg(era1: era1, cdb: cdb, e1xcpt: e1xcpt), + uintGetFns: @[EdgeDbUintGetFn(getEra1Expt), + EdgeDbUintGetFn(getCoreDbObj)], + blobGetFns: @[EdgeDbBlobGetFn(blobGetUnsupported)], + uintGetPolFn: uintGetSeqentiallyUntilFound, + blobGetPolFn: blobGetSeqentiallyUntilFound) + +# ------------------------------------------------------------------------------ +# Public constructor +# ------------------------------------------------------------------------------ + +proc init*( + T: type EdgeDbRef; + era1: Era1DbRef; + cdb: CoreDbRef; + ): T = + ## Initalise for trying `Era1` first, then `CoreDb`. + ## + T(getDesc: EdgeE1CdbRef(era1: era1, cdb: cdb), + uintGetFns: @[EdgeDbUintGetFn(getEra1Obj), + EdgeDbUintGetFn(getCoreDbObj)], + blobGetFns: @[EdgeDbBlobGetFn(blobGetUnsupported)], + uintGetPolFn: uintGetSeqentiallyUntilFound, + blobGetPolFn: blobGetSeqentiallyUntilFound) + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/tests/all_tests.nim b/tests/all_tests.nim index 25268f8af6..494da8b54c 100644 --- a/tests/all_tests.nim +++ b/tests/all_tests.nim @@ -41,4 +41,5 @@ cliBuilder: ./test_beacon/test_skeleton, ./test_getproof_json, ./test_aristo, - ./test_coredb + ./test_coredb, + ./test_edgedb diff --git a/tests/test_edgedb.nim b/tests/test_edgedb.nim new file mode 100644 index 0000000000..28f4bed23c --- /dev/null +++ b/tests/test_edgedb.nim @@ -0,0 +1,20 @@ +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +import + ./test_edgedb/[ + test_era1_coredb, + ] + +proc edgeDbMain*() = + testEra1CoreDbMain() + +when isMainModule: + edgeDbMain() + +# End diff --git a/tests/test_edgedb/test_era1_coredb.nim b/tests/test_edgedb/test_era1_coredb.nim new file mode 100644 index 0000000000..f952d2e4e8 --- /dev/null +++ b/tests/test_edgedb/test_era1_coredb.nim @@ -0,0 +1,87 @@ +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +import + std/[algorithm, sets], + pkg/eth/common, + pkg/[results, unittest2], + ../../nimbus/common, + ../../nimbus/core/chain, + ../../nimbus/db/[core_db, edge_db, era1_db], + ./test_helpers + +const + NotFound = Result[Blob,EdgeDbError].err(EdgeKeyNotFound) + +# ------------------------------------------------------------------------------ +# Test runner +# ------------------------------------------------------------------------------ + +proc testEra1CoreDbMain*() = + suite "EdgeDb test Era1Db vs CoreDb": + let + e1db = newEra1DbInstance() + blockNumbers = e1db.randomBlockNumbers(24, 12345) + + test "Find existing blocks": + let + cdb = newCoreDbInstance() + edb = EdgeDbRef.init(e1db, cdb) + for bn in blockNumbers: + let + blkData = edb.get(EthBlockData, bn).expect "valid block data" + hdrData = edb.get(EthHeaderData, bn).expect "valid header data" + bdyData = edb.get(EthBodyData, bn).expect "valid body data" + e1Blk = e1db.getEthBlock(bn).expect "valid eth block" + e1Tpl = e1db.getBlockTuple(bn).expect "valid block tuple" + check blkData == rlp.encode(e1Blk) + check hdrData == rlp.encode(e1Tpl.header) + check bdyData == rlp.encode(e1Tpl.body) + + test "Fail finding missing blocks": + let + cdb = newCoreDbInstance() + edb = EdgeDbRef.init(e1db, blockNumbers.toHashSet, cdb) + for bn in blockNumbers: + check edb.get(EthBlockData, bn) == NotFound + check edb.get(EthHeaderData, bn) == NotFound + check edb.get(EthBodyData, bn) == NotFound + + test "Find existing blocks in CoreDb": + when not defined(release): + setErrorLevel() # reduce logging for on-the-fly testing + let + com = newCommonInstance() + edb = EdgeDbRef.init(e1db, blockNumbers.toHashSet, com.db) + + # Import relevant blocks into CoreDb + let + maxBlk = blockNumbers.sorted[^1] + chunk = 1024 + chain = com.newChain() + for n in 1u64.countUp(maxBlk, chunk): + let blks = e1db.getBlockList(n, min(n + chunk.uint - 1, maxBlk)) + discard chain.persistBlocks(blks).expect "working import" + + for bn in blockNumbers: + let + blkData = edb.get(EthBlockData, bn).expect "valid block data" + hdrData = edb.get(EthHeaderData, bn).expect "valid header data" + bdyData = edb.get(EthBodyData, bn).expect "valid body data" + e1Blk = e1db.getEthBlock(bn).expect "valid eth block" + e1Tpl = e1db.getBlockTuple(bn).expect "valid block tuple" + check blkData == rlp.encode(e1Blk) + check hdrData == rlp.encode(e1Tpl.header) + check bdyData == rlp.encode(e1Tpl.body) + +when isMainModule: + testEra1CoreDbMain() + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/tests/test_edgedb/test_helpers.nim b/tests/test_edgedb/test_helpers.nim new file mode 100644 index 0000000000..d30abd100c --- /dev/null +++ b/tests/test_edgedb/test_helpers.nim @@ -0,0 +1,145 @@ +# Nimbus +# Copyright (c) 2023-2024 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or +# distributed except according to those terms. + +import + std/[bitops, os, strutils], + pkg/[chronicles, results], + ../../nimbus/common, + ../../nimbus/db/[core_db, era1_db] + +const + baseDir = [".", "..", ".."/"..", $DirSep] + mainDir = [".", "tests"] + subDir = ["replay", "custom-network"] + + era1BaseName = "mainnet-00000-5ec1ffb8.era1" + +type + PrngDesc = object + prng: uint32 ## random state + +# ------------------------------------------------------------------------------ +# Private helpers +# ------------------------------------------------------------------------------ + +proc findFilePathHelper( + file: string; + baseDir: openArray[string] = baseDir; + mainDir: openArray[string] = mainDir; + subDir: openArray[string] = subDir; + ): Result[string,void] = + for dir in baseDir: + if dir.dirExists: + for main in mainDir: + if (dir / main).dirExists: + for sub in subDir: + if (dir / main / sub).dirExists: + let path = dir / main / sub / file + if path.fileExists: + return ok(path) + echo "*** File not found \"", file, "\"." + err() + +# ----------------- + +proc posixPrngRand(state: var uint32): byte = + ## POSIX.1-2001 example of a rand() implementation, see manual page rand(3). + state = state * 1103515245 + 12345; + let val = (state shr 16) and 32767 # mod 2^31 + (val shr 8).byte # Extract second byte + +proc randU64(state: var uint32): uint64 = + var a: array[sizeof result,byte] + for n in 0 ..< a.len: + a[n] = state.posixPrngRand + (addr result).copyMem(unsafeAddr a, sizeof a) + +proc randU64(state: var uint32; top: uint64): uint64 = + let mask = (1 shl (64 - top.countLeadingZeroBits)) - 1 + for _ in 0 ..< 100: + let w = mask.uint64 and state.randU64 + if w < top: + return w + raiseAssert "Not here (!)" + +# ----------------- + +proc lastBlockNumber(e1db: Era1DbRef): BlockNumber = + var + minNum = BlockNumber(1) + maxNum = BlockNumber(4700013) # MainNet + middle = (maxNum + minNum) div 2 + delta = maxNum - minNum + while 1 < delta: + if e1db.getEthBlock(middle).isOk: + minNum = middle + else: + maxNum = middle + middle = (maxNum + minNum) div 2 + delta = maxNum - minNum + minNum + +# ------------------------------------------------------------------------------ +# Public functions +# ------------------------------------------------------------------------------ + +proc setTraceLevel* = + discard + when defined(chronicles_runtime_filtering) and loggingEnabled: + setLogLevel(LogLevel.TRACE) + +proc setDebugLevel* = + discard + when defined(chronicles_runtime_filtering) and loggingEnabled: + setLogLevel(LogLevel.DEBUG) + +proc setErrorLevel* = + discard + when defined(chronicles_runtime_filtering) and loggingEnabled: + setLogLevel(LogLevel.ERROR) + +# ------------------- + +proc newEra1DbInstance*(): Era1DbRef = + let + repoFile = era1BaseName.findFilePathHelper().expect "valid path" + repoDir = repoFile.splitFile.dir + network = era1BaseName.split('-')[0] + + Era1DbRef.init(repoDir, network).expect "valid repo" + +proc randomBlockNumbers*( + e1db: Era1DbRef; + nInst: int; + seed: uint32): seq[BlockNumber] = + let top = e1db.lastBlockNumber.uint64 + var state = seed + for _ in 0 ..< nInst: + result.add state.randU64 top + +proc getBlockList*(e1db: Era1DbRef; first, last: BlockNumber): seq[EthBlock] = + for bn in first .. last: + result.add e1db.getEthBlock(bn).expect "valid eth block" + +# ------------------- + +proc newCoreDbInstance*(): CoreDbRef = + AristoDbMemory.newCoreDbRef() + +proc newCommonInstance*(): CommonRef = + const networkId = MainNet + CommonRef.new( + db = AristoDbMemory.newCoreDbRef(), + networkId = networkId, + params = networkId.networkParams()) + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ From 7a37034ee7cf835a6c2b4fa3ed1427ce5c9d73ad Mon Sep 17 00:00:00 2001 From: Jordan Hrycaj Date: Fri, 27 Sep 2024 16:20:21 +0100 Subject: [PATCH 2/2] Fix copyright headers --- tests/test_edgedb.nim | 2 ++ tests/test_edgedb/test_era1_coredb.nim | 2 ++ tests/test_edgedb/test_helpers.nim | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_edgedb.nim b/tests/test_edgedb.nim index 28f4bed23c..4b1bc580bc 100644 --- a/tests/test_edgedb.nim +++ b/tests/test_edgedb.nim @@ -1,3 +1,5 @@ +# Nimbus +# Copyright (c) 2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # http://www.apache.org/licenses/LICENSE-2.0) diff --git a/tests/test_edgedb/test_era1_coredb.nim b/tests/test_edgedb/test_era1_coredb.nim index f952d2e4e8..1365715815 100644 --- a/tests/test_edgedb/test_era1_coredb.nim +++ b/tests/test_edgedb/test_era1_coredb.nim @@ -1,3 +1,5 @@ +# Nimbus +# Copyright (c) 2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # http://www.apache.org/licenses/LICENSE-2.0) diff --git a/tests/test_edgedb/test_helpers.nim b/tests/test_edgedb/test_helpers.nim index d30abd100c..1af3c35963 100644 --- a/tests/test_edgedb/test_helpers.nim +++ b/tests/test_edgedb/test_helpers.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2023-2024 Status Research & Development GmbH +# Copyright (c) 2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # http://www.apache.org/licenses/LICENSE-2.0)