Backend agnostic read-only DB wrapper framework

nimbus/db/edge_db.nim

@@ -0,0 +1,47 @@
+# Nimbus
+# Copyright (c) 2023-2024 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
+#    http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
+#    http://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed except
+# according to those terms.
+
+## Policy driven read-only database wrapper
+## ========================================
+##
+## It lives on the edge and pretends to be agnostic of a particular
+## backend implementation.
+##
+{.push raises: [].}
+
+import
+  pkg/eth/common,
+  pkg/results,
+  edge_db/[
+    db_desc,
+    init_era1_coredb,
+  ]
+
+export
+  EdgeDbColumn,
+  EdgeDbError,
+  EdgeDbRef,
+  init
+
+proc get*(
+    edg: EdgeDbRef;
+    col: EdgeDbColumn;
+    key: uint64;
+      ): Result[Blob,EdgeDbError] =
+  edg.uintGetPolFn(edg, col, key)
+
+proc get*(
+    edg: EdgeDbRef;
+    col: EdgeDbColumn;
+    key: openArray[byte];
+      ): Result[Blob,EdgeDbError] =
+  edg.blobGetPolFn(edg, col, key)
+
+# End

nimbus/db/edge_db/db_desc.nim

@@ -0,0 +1,119 @@
+# Nimbus
+# Copyright (c) 2023-2024 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
+#    http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
+#    http://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed except
+# according to those terms.
+
+{.push raises: [].}
+
+import
+  pkg/eth/common,
+  pkg/results
+
+type
+  EdgeDbError* = enum
+    ## Allows more granulated failure information.
+    NothingSerious = 0
+    EdgeKeyNotFound
+    EdgeColUnsupported
+    EdgeKeyTypeUnsupported
+
+  EdgeDbColumn* = enum
+    ## Specify object type to query for
+    Oops = 0
+    EthBlockData
+    EthHeaderData
+    EthBodyData
+
+  EdgeDbGetRef* = ref object of RootObj
+    ## Descriptor common to a set of `getFn()` implementations. This basic type
+    ## will be interited by sprcific implementations.
+
+
+  EdgeDbUintGetFn* =
+    proc(dsc: EdgeDbGetRef;
+         col: EdgeDbColumn;
+         key: uint64;
+        ): Result[Blob,EdgeDbError]
+        {.gcsafe, raises: [].}
+      ## Particular `getFn()` instance. Will return a RLP encoded serialised
+      ## data result.
+
+  EdgeDbBlobGetFn* =
+    proc(dsc: EdgeDbGetRef;
+         col: EdgeDbColumn;
+         key: openArray[byte];
+        ): Result[Blob,EdgeDbError]
+        {.gcsafe, raises: [].}
+      ## Ditto for `Blob` like key
+
+
+  EdgeDbUintGetPolicyFn* =
+    proc(edg: EdgeDbRef;
+         col: EdgeDbColumn;
+         key: uint64;
+        ): Result[Blob,EdgeDbError]
+        {.gcsafe, raises: [].}
+      ## Implemenation of `get()`. This function employs a set of `getFn()`
+      ## instances (in some order) for finding a result.
+
+  EdgeDbBlobGetPolicyFn* =
+    proc(edg: EdgeDbRef;
+         col: EdgeDbColumn;
+         key: openArray[byte];
+        ): Result[Blob,EdgeDbError]
+        {.gcsafe, raises: [].}
+      ##  Ditto for `Blob` like key
+
+
+  EdgeDbRef* = ref object
+    ## Visible database wrapper.
+    getDesc*: EdgeDbGetRef
+    uintGetFns*: seq[EdgeDbUintGetFn]
+    blobGetFns*: seq[EdgeDbBlobGetFn]
+    uintGetPolFn*: EdgeDbUintGetPolicyFn
+    blobGetPolFn*: EdgeDbBlobGetPolicyFn
+
+# ------------------------------------------------------------------------------
+# Public helpers, sequentially trying a list of `getFn()` instances
+# ------------------------------------------------------------------------------
+
+proc uintGetSeqentiallyUntilFound*(
+  edg: EdgeDbRef;
+  col: EdgeDbColumn;
+  key: uint64;
+    ): Result[Blob,EdgeDbError] =
+  ## Simple linear get policy.
+  var error = EdgeColUnsupported
+
+  for fn in edg.uintGetFns:
+    let err = edg.getDesc.fn(col,key).errorOr:
+      return ok(value)
+    if err != EdgeColUnsupported:
+      error = EdgeKeyNotFound
+
+  err(error)
+
+proc blobGetSeqentiallyUntilFound*(
+  edg: EdgeDbRef;
+  col: EdgeDbColumn;
+  key: openArray[byte];
+    ): Result[Blob,EdgeDbError] =
+  ## Ditto for `Blob` like key
+  var error = EdgeColUnsupported
+
+  for fn in edg.blobGetFns:
+    let err = edg.getDesc.fn(col,key).errorOr:
+      return ok(value)
+    if err != EdgeColUnsupported:
+      error = EdgeKeyNotFound
+
+  err(error)
+
+# ------------------------------------------------------------------------------
+# End
+# ------------------------------------------------------------------------------

nimbus/db/edge_db/init_era1_coredb.nim

@@ -0,0 +1,174 @@
+# Nimbus
+# Copyright (c) 2023-2024 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
+#    http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
+#    http://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed except
+# according to those terms.
+
+import
+  std/sets,
+  pkg/eth/[common, rlp],
+  pkg/results,
+  ./db_desc,
+  ".."/[core_db, era1_db, storage_types]
+
+type
+  EdgeE1CdbRef = ref object of EdgeDbGetRef
+    era1: Era1DbRef
+    cdb: CoreDbRef
+
+  EdgeE1CdbDbg = ref object of EdgeE1CdbRef
+    ## For debugging. Keys in the `e1xcpt` set are not found by the `Era1`
+    ## driver.
+    e1xcpt: HashSet[BlockNumber]
+
+# ------------------------------------------------------------------------------
+# Private helpers
+# ------------------------------------------------------------------------------
+
+proc getBlockHash(
+    w: EdgeDbGetRef;
+    k: uint64;
+      ): Result[Hash256,EdgeDbError] =
+  var hash: Hash256
+  if EdgeE1CdbRef(w).cdb.getBlockHash(BlockNumber(k), hash):
+    return ok(hash)
+  err(EdgeKeyNotFound)
+
+proc getBlockHeader(
+    w: EdgeDbGetRef;
+    h: Hash256;
+      ): Result[BlockHeader,EdgeDbError] =
+  var header: BlockHeader
+  if EdgeE1CdbRef(w).cdb.getBlockHeader(h, header):
+    return ok(header)
+  err(EdgeKeyNotFound)
+
+proc getBlockBody(
+    w: EdgeDbGetRef;
+    h: Hash256;
+      ): Result[BlockBody,EdgeDbError] =
+  var body: BlockBody
+  if EdgeE1CdbRef(w).cdb.getBlockBody(h, body):
+    return ok(body)
+  err(EdgeKeyNotFound)
+
+# ------------------------------------------------------------------------------
+# Private drivers
+# ------------------------------------------------------------------------------
+
+proc blobGetUnsupported(
+    dsc: EdgeDbGetRef;
+    col: EdgeDbColumn;
+    key: openArray[byte];
+      ): Result[Blob,EdgeDbError] =
+  err(EdgeKeyTypeUnsupported)
+
+
+proc getEra1Obj(
+    dsc: EdgeDbGetRef;
+    col: EdgeDbColumn;
+    key: uint64;
+      ): Result[Blob,EdgeDbError] =
+  case col:
+  of EthBlockData:
+    let w = EdgeE1CdbRef(dsc).era1.getEthBlock(key).valueOr:
+      return err(EdgeKeyNotFound)
+    ok(rlp.encode w)
+
+  of EthHeaderData:
+    let w = EdgeE1CdbRef(dsc).era1.getBlockTuple(key).valueOr:
+      return err(EdgeKeyNotFound)
+    ok(rlp.encode w.header)
+
+  of EthBodyData:
+    let w = EdgeE1CdbRef(dsc).era1.getBlockTuple(key).valueOr:
+      return err(EdgeKeyNotFound)
+    ok(rlp.encode w.body)
+
+  else:
+    err(EdgeColUnsupported)
+
+
+proc getCoreDbObj(
+    dsc: EdgeDbGetRef;
+    col: EdgeDbColumn;
+    key: uint64;
+      ): Result[Blob,EdgeDbError] =
+  case col:
+  of EthBlockData:
+    let h = ? dsc.getBlockHash(key)
+    ok(rlp.encode EthBlock.init(? dsc.getBlockHeader(h), ? dsc.getBlockBody(h)))
+
+  of EthHeaderData:
+    let
+      h = ? dsc.getBlockHash(key)
+      kvt =  EdgeE1CdbRef(dsc).cdb.ctx.getKvt()
+
+      # Fetching directly from the DB avoids re-encoding the header object
+      data = kvt.get(genericHashKey(h).toOpenArray).valueOr:
+        return err(EdgeKeyNotFound)
+    ok(data)
+
+  of EthBodyData:
+    ok(rlp.encode(? dsc.getBlockBody(? dsc.getBlockHash(key))))
+
+  else:
+    err(EdgeColUnsupported)
+
+# ------------------------------------------------------------------------------
+# Public constructor (debuging version)
+# ------------------------------------------------------------------------------
+
+proc init*(
+    T: type EdgeDbRef;
+    era1: Era1DbRef;
+    e1xcpt: HashSet[BlockNumber];
+    cdb: CoreDbRef;
+      ): T =
+  ## Initalise for trying `Era1` first, then `CoreDb`. This goes with some
+  ## exceptions for `Era1`. If an argument key is in `excpt` if will not be
+  ## found by the `Era1` driver.
+  ##
+  ## This constructor is mainly designed for debugging.
+  ##
+  proc getEra1Expt(
+      dsc: EdgeDbGetRef;
+      col: EdgeDbColumn;
+      key: uint64;
+        ): Result[Blob,EdgeDbError] =
+    if key in EdgeE1CdbDbg(dsc).e1xcpt:
+      return err(EdgeKeyNotFound)
+    dsc.getEra1Obj(col, key)
+
+  T(getDesc:      EdgeE1CdbDbg(era1: era1, cdb: cdb, e1xcpt: e1xcpt),
+    uintGetFns:   @[EdgeDbUintGetFn(getEra1Expt),
+                    EdgeDbUintGetFn(getCoreDbObj)],
+    blobGetFns:   @[EdgeDbBlobGetFn(blobGetUnsupported)],
+    uintGetPolFn: uintGetSeqentiallyUntilFound,
+    blobGetPolFn: blobGetSeqentiallyUntilFound)
+
+# ------------------------------------------------------------------------------
+# Public constructor
+# ------------------------------------------------------------------------------
+
+proc init*(
+    T: type EdgeDbRef;
+    era1: Era1DbRef;
+    cdb: CoreDbRef;
+      ): T =
+  ## Initalise for trying `Era1` first, then `CoreDb`.
+  ##
+  T(getDesc:      EdgeE1CdbRef(era1: era1, cdb: cdb),
+    uintGetFns:   @[EdgeDbUintGetFn(getEra1Obj),
+                    EdgeDbUintGetFn(getCoreDbObj)],
+    blobGetFns:   @[EdgeDbBlobGetFn(blobGetUnsupported)],
+    uintGetPolFn: uintGetSeqentiallyUntilFound,
+    blobGetPolFn: blobGetSeqentiallyUntilFound)
+
+# ------------------------------------------------------------------------------
+# End
+# ------------------------------------------------------------------------------

tests/all_tests.nim

@@ -41,4 +41,5 @@ cliBuilder:
           ./test_beacon/test_skeleton,
           ./test_getproof_json,
           ./test_aristo,
-          ./test_coredb
+          ./test_coredb,
+          ./test_edgedb

tests/test_edgedb.nim

@@ -0,0 +1,22 @@
+# Nimbus
+# Copyright (c) 2024 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
+#    http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
+#    http://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed except
+# according to those terms.
+
+import
+  ./test_edgedb/[
+    test_era1_coredb,
+  ]
+
+proc edgeDbMain*() =
+  testEra1CoreDbMain()
+
+when isMainModule:
+  edgeDbMain()
+
+# End