Merge pull request #1 from netsec-ethz/juagargi/hummingbird_sciond

Hummingbird library in scion daemon
netsec-ethz · Mar 11, 2024 · 6c34da3 · 6c34da3
2 parents 9eea093 + 299b738
commit 6c34da3
Show file tree

Hide file tree

Showing 44 changed files with 4,024 additions and 1,216 deletions.
diff --git a/daemon/BUILD.bazel b/daemon/BUILD.bazel
@@ -17,6 +17,7 @@ go_library(
         "//pkg/private/prom:go_default_library",
         "//pkg/private/serrors:go_default_library",
         "//private/env:go_default_library",
+        "//private/hummingbirddb:go_default_library",
         "//private/revcache:go_default_library",
         "//private/trust:go_default_library",
         "//private/trust/grpc:go_default_library",

diff --git a/daemon/cmd/daemon/main.go b/daemon/cmd/daemon/main.go
@@ -113,12 +113,24 @@ func realMain(ctx context.Context) error {
 	})
 	defer pathDB.Close()
 	defer revCache.Close()
+
+	hbirdDB, err := storage.NewHummingbirdStorage(globalCfg.HbirdDB)
+	if err != nil {
+		return serrors.WrapStr("initializing hummingbird storage", err)
+	}
+	defer hbirdDB.Close()
+
 	cleaner := periodic.Start(pathdb.NewCleaner(pathDB, "sd_segments"),
 		300*time.Second, 295*time.Second)
 	defer cleaner.Stop()
 	rcCleaner := periodic.Start(revcache.NewCleaner(revCache, "sd_revocation"),
 		10*time.Second, 10*time.Second)
 	defer rcCleaner.Stop()
+	defer cleaner.Stop()
+
+	// Hummingbird DB has its cleaner started already in the New method. Use that one.
+	hbirdCleaner := hbirdDB.GetCleaner()
+	defer hbirdCleaner.Stop()
 
 	dialer := &libgrpc.TCPDialer{
 		SvcResolver: func(dst addr.SVC) []resolver.Address {
@@ -271,9 +283,11 @@ func realMain(ctx context.Context) error {
 					Cfg:        globalCfg.SD,
 				},
 			),
+
 			Engine:      engine,
 			RevCache:    revCache,
 			DRKeyClient: drkeyClientEngine,
+			FlyoverDB:   hbirdDB,
 		},
 	))
 

diff --git a/daemon/config/config.go b/daemon/config/config.go
@@ -46,6 +46,7 @@ type Config struct {
 	Tracing       env.Tracing        `toml:"tracing,omitempty"`
 	TrustDB       storage.DBConfig   `toml:"trust_db,omitempty"`
 	PathDB        storage.DBConfig   `toml:"path_db,omitempty"`
+	HbirdDB       storage.DBConfig   `toml:"hbird_db,omitempty"`
 	SD            SDConfig           `toml:"sd,omitempty"`
 	TrustEngine   trustengine.Config `toml:"trustengine,omitempty"`
 	DRKeyLevel2DB storage.DBConfig   `toml:"drkey_level2_db,omitempty"`
@@ -61,6 +62,7 @@ func (cfg *Config) InitDefaults() {
 		&cfg.Tracing,
 		cfg.TrustDB.WithDefault(fmt.Sprintf(storage.DefaultTrustDBPath, "sd")),
 		cfg.PathDB.WithDefault(fmt.Sprintf(storage.DefaultPathDBPath, "sd")),
+		cfg.HbirdDB.WithDefault(fmt.Sprintf(storage.DefaultHbirdDBPath, "sd")),
 		&cfg.SD,
 		&cfg.TrustEngine,
 	)
@@ -75,6 +77,7 @@ func (cfg *Config) Validate() error {
 		&cfg.API,
 		&cfg.TrustDB,
 		&cfg.PathDB,
+		&cfg.HbirdDB,
 		&cfg.SD,
 		&cfg.TrustEngine,
 		&cfg.DRKeyLevel2DB,
@@ -103,6 +106,13 @@ func (cfg *Config) Sample(dst io.Writer, path config.Path, _ config.CtxMap) {
 			),
 			"path_db",
 		),
+		config.OverrideName(
+			config.FormatData(
+				&cfg.HbirdDB,
+				fmt.Sprintf(storage.DefaultHbirdDBPath, "sd"),
+			),
+			"hbird_db",
+		),
 		&cfg.SD,
 		&cfg.TrustEngine,
 		config.OverrideName(

diff --git a/daemon/daemon.go b/daemon/daemon.go
@@ -36,6 +36,7 @@ import (
 	"github.com/scionproto/scion/pkg/private/prom"
 	"github.com/scionproto/scion/pkg/private/serrors"
 	"github.com/scionproto/scion/private/env"
+	"github.com/scionproto/scion/private/hummingbirddb"
 	"github.com/scionproto/scion/private/revcache"
 	"github.com/scionproto/scion/private/trust"
 	trustgrpc "github.com/scionproto/scion/private/trust/grpc"
@@ -114,6 +115,8 @@ type ServerConfig struct {
 	Engine      trust.Engine
 	Topology    servers.Topology
 	DRKeyClient *drkey.ClientEngine
+
+	FlyoverDB hummingbirddb.DB
 }
 
 // NewServer constructs a daemon API server.
@@ -126,6 +129,7 @@ func NewServer(cfg ServerConfig) *servers.DaemonServer {
 		ASInspector: cfg.Engine.Inspector,
 		RevCache:    cfg.RevCache,
 		DRKeyClient: cfg.DRKeyClient,
+		FlyoverDB:   cfg.FlyoverDB,
 		Metrics: servers.Metrics{
 			PathsRequests: servers.RequestMetrics{
 				Requests: metrics.NewPromCounterFrom(prometheus.CounterOpts{

diff --git a/daemon/internal/servers/BUILD.bazel b/daemon/internal/servers/BUILD.bazel
@@ -1,9 +1,10 @@
-load("//tools/lint:go.bzl", "go_library")
+load("//tools/lint:go.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
     srcs = [
         "grpc.go",
+        "grpc_hummingbird.go",
         "metrics.go",
     ],
     importpath = "github.com/scionproto/scion/daemon/internal/servers",
@@ -13,6 +14,7 @@ go_library(
         "//daemon/fetcher:go_default_library",
         "//pkg/addr:go_default_library",
         "//pkg/drkey:go_default_library",
+        "//pkg/hummingbird:go_default_library",
         "//pkg/log:go_default_library",
         "//pkg/metrics:go_default_library",
         "//pkg/private/common:go_default_library",
@@ -24,6 +26,7 @@ go_library(
         "//pkg/proto/daemon:go_default_library",
         "//pkg/snet:go_default_library",
         "//pkg/snet/path:go_default_library",
+        "//private/hummingbirddb:go_default_library",
         "//private/revcache:go_default_library",
         "//private/topology:go_default_library",
         "//private/trust:go_default_library",
@@ -33,3 +36,23 @@ go_library(
         "@org_golang_x_sync//singleflight:go_default_library",
     ],
 )
+
+go_test(
+    name = "go_default_test",
+    srcs = ["grpc_hummingbird_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//pkg/addr:go_default_library",
+        "//pkg/hummingbird:go_default_library",
+        "//pkg/private/common:go_default_library",
+        "//pkg/private/util:go_default_library",
+        "//pkg/private/xtest:go_default_library",
+        "//pkg/proto/daemon:go_default_library",
+        "//pkg/slayers/path:go_default_library",
+        "//pkg/slayers/path/scion:go_default_library",
+        "//pkg/snet:go_default_library",
+        "//pkg/snet/path:go_default_library",
+        "//private/hummingbirddb:go_default_library",
+        "@com_github_stretchr_testify//require:go_default_library",
+    ],
+)
diff --git a/daemon/internal/servers/grpc.go b/daemon/internal/servers/grpc.go
@@ -40,6 +40,7 @@ import (
 	sdpb "github.com/scionproto/scion/pkg/proto/daemon"
 	"github.com/scionproto/scion/pkg/snet"
 	snetpath "github.com/scionproto/scion/pkg/snet/path"
+	"github.com/scionproto/scion/private/hummingbirddb"
 	"github.com/scionproto/scion/private/revcache"
 	"github.com/scionproto/scion/private/topology"
 	"github.com/scionproto/scion/private/trust"
@@ -61,6 +62,8 @@ type DaemonServer struct {
 	ASInspector trust.Inspector
 	DRKeyClient *drkey_daemon.ClientEngine
 
+	FlyoverDB hummingbirddb.DB
+
 	Metrics Metrics
 
 	foregroundPathDedupe singleflight.Group