diff --git a/README.md b/README.md index 9b0a73d..ceb55a1 100644 --- a/README.md +++ b/README.md @@ -52,10 +52,10 @@ The Blob syncer server provides eth compatible API to query historical blob * GET /eth/v1/beacon/blob_sidecars/{block_id} -| ParameterName | Type | Description | -|--------------| ------ |------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| block_id | string | Block identifier. Can be one of: , . note: "head" (canonical head in node's view), "genesis", "finalized" are not support | -| indices | array | Array of indices for blob sidecars to request for in the specified block. Returns all blob sidecars in the block if not specified | +| ParameterName | Type | Description | +|--------------|-----------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| block_id | string | Block identifier. Can be one of: slot, . note: "head" (canonical head in node's view), "genesis", "finalized" are not support | +| indices | array of string | Array of indices for blob sidecars to request for in the specified block. Returns all blob sidecars in the block if not specified | 200: Ok response diff --git a/cmd/blob-syncer/main.go b/cmd/blob-syncer/main.go index 096852d..4bfb345 100644 --- a/cmd/blob-syncer/main.go +++ b/cmd/blob-syncer/main.go @@ -2,13 +2,15 @@ package main import ( "flag" + "os" + + "github.com/spf13/pflag" + "github.com/spf13/viper" + "github.com/bnb-chain/blob-syncer/config" syncerdb "github.com/bnb-chain/blob-syncer/db" "github.com/bnb-chain/blob-syncer/logging" "github.com/bnb-chain/blob-syncer/syncer" - "github.com/spf13/pflag" - "github.com/spf13/viper" - "os" ) func initFlags() { @@ -31,7 +33,6 @@ func main() { if configFilePath == "" { configFilePath = os.Getenv(config.EnvVarConfigFilePath) } - configFilePath = "config/local/config-syncer.json" cfg = config.ParseSyncerConfigFromFile(configFilePath) if cfg == nil { panic("failed to get configuration") diff --git a/config/config.go b/config/config.go index 4c36185..78ba040 100644 --- a/config/config.go +++ b/config/config.go @@ -3,14 +3,16 @@ package config import ( "encoding/json" "fmt" - "github.com/bnb-chain/blob-syncer/cache" - syncerdb "github.com/bnb-chain/blob-syncer/db" - "gorm.io/driver/mysql" - "gorm.io/gorm" - "gorm.io/gorm/logger" "log" "os" "time" + + "gorm.io/driver/mysql" + "gorm.io/gorm" + "gorm.io/gorm/logger" + + "github.com/bnb-chain/blob-syncer/cache" + syncerdb "github.com/bnb-chain/blob-syncer/db" ) type SyncerConfig struct { diff --git a/db/bundle.go b/db/bundle.go index 3bcd14c..2d07274 100644 --- a/db/bundle.go +++ b/db/bundle.go @@ -9,9 +9,10 @@ const ( ) type Bundle struct { - Id int64 - Name string `gorm:"NOT NULL;uniqueIndex:idx_bundle_name;size:64"` - Status InnerBundleStatus `gorm:"NOT NULL"` + Id int64 + Name string `gorm:"NOT NULL;uniqueIndex:idx_bundle_name;size:64"` + Status InnerBundleStatus `gorm:"NOT NULL"` + Calibrated bool } func (*Bundle) TableName() string { diff --git a/db/dao.go b/db/dao.go index dd79225..1374c28 100644 --- a/db/dao.go +++ b/db/dao.go @@ -33,7 +33,7 @@ type BlockDB interface { func (d *BlobSvcDB) GetBlock(slot uint64) (*Block, error) { block := Block{} err := d.db.Model(Block{}).Where("slot = ?", slot).Take(&block).Error - if err != nil && err != gorm.ErrRecordNotFound { + if err != nil { return nil, err } return &block, nil @@ -111,7 +111,7 @@ type BundleDB interface { func (d *BlobSvcDB) GetLatestFinalizingBundle() (*Bundle, error) { bundle := Bundle{} - err := d.db.Model(Bundle{}).Where("status = ?", Finalizing).Order("id desc").Take(&bundle).Error + err := d.db.Model(Bundle{}).Where("status = ? and calibrated = false", Finalizing).Order("id desc").Take(&bundle).Error if err != nil { return nil, err } diff --git a/db/type.go b/db/type.go index 460af7e..bff15c1 100644 --- a/db/type.go +++ b/db/type.go @@ -1,6 +1,8 @@ package db -import "github.com/go-sql-driver/mysql" +import ( + "github.com/go-sql-driver/mysql" +) var ( ErrDuplicateEntryCode = 1062 diff --git a/restapi/configure_blob_syncer.go b/restapi/configure_blob_syncer.go index f6ee229..4c5bd76 100644 --- a/restapi/configure_blob_syncer.go +++ b/restapi/configure_blob_syncer.go @@ -4,6 +4,13 @@ package restapi import ( "crypto/tls" + "net/http" + "os" + + "github.com/go-openapi/errors" + "github.com/go-openapi/runtime" + "github.com/go-openapi/swag" + "github.com/bnb-chain/blob-syncer/cache" "github.com/bnb-chain/blob-syncer/config" syncerdb "github.com/bnb-chain/blob-syncer/db" @@ -12,11 +19,6 @@ import ( "github.com/bnb-chain/blob-syncer/restapi/operations" "github.com/bnb-chain/blob-syncer/restapi/operations/blob" "github.com/bnb-chain/blob-syncer/service" - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/swag" - "net/http" - "os" ) //go:generate swagger generate server --target ../../blob-syncer --name BlobSyncer --spec ../swagger.yaml --principal interface{} diff --git a/restapi/handlers/blob.go b/restapi/handlers/blob.go index 6ce52c7..b8d1e6a 100644 --- a/restapi/handlers/blob.go +++ b/restapi/handlers/blob.go @@ -3,13 +3,14 @@ package handlers import ( "encoding/hex" "fmt" - "github.com/bnb-chain/blob-syncer/util" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/go-openapi/runtime/middleware" "github.com/bnb-chain/blob-syncer/models" "github.com/bnb-chain/blob-syncer/restapi/operations/blob" "github.com/bnb-chain/blob-syncer/service" + "github.com/bnb-chain/blob-syncer/util" ) const rootLength = 32 diff --git a/service/blob.go b/service/blob.go index 4e4d083..9556524 100644 --- a/service/blob.go +++ b/service/blob.go @@ -2,7 +2,6 @@ package service import ( "fmt" - "github.com/bnb-chain/blob-syncer/util" "strconv" "github.com/bnb-chain/blob-syncer/cache" @@ -10,6 +9,7 @@ import ( "github.com/bnb-chain/blob-syncer/db" "github.com/bnb-chain/blob-syncer/external" "github.com/bnb-chain/blob-syncer/models" + "github.com/bnb-chain/blob-syncer/util" ) type Blob interface { diff --git a/syncer/conveter.go b/syncer/conveter.go index d20352e..98e196a 100644 --- a/syncer/conveter.go +++ b/syncer/conveter.go @@ -2,6 +2,7 @@ package syncer import ( "encoding/json" + "github.com/prysmaticlabs/prysm/v5/api/server/structs" v1 "github.com/prysmaticlabs/prysm/v5/proto/engine/v1" ethpb "github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1" diff --git a/syncer/syncer.go b/syncer/syncer.go index 60b8d44..cb1ee01 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -4,7 +4,6 @@ import ( "context" "encoding/hex" "fmt" - "github.com/bnb-chain/blob-syncer/util" "math/big" "os" "path/filepath" @@ -25,6 +24,7 @@ import ( "github.com/bnb-chain/blob-syncer/external" "github.com/bnb-chain/blob-syncer/logging" "github.com/bnb-chain/blob-syncer/types" + "github.com/bnb-chain/blob-syncer/util" ) const ( diff --git a/syncer/verifier.go b/syncer/verifier.go index 44c780d..9f48726 100644 --- a/syncer/verifier.go +++ b/syncer/verifier.go @@ -4,23 +4,25 @@ import ( "bytes" "context" "errors" - "fmt" + "os" + "path/filepath" + "time" + + "github.com/prysmaticlabs/prysm/v5/api/server/structs" + "gorm.io/gorm" + "github.com/bnb-chain/blob-syncer/db" + "github.com/bnb-chain/blob-syncer/external" "github.com/bnb-chain/blob-syncer/logging" "github.com/bnb-chain/blob-syncer/types" "github.com/bnb-chain/blob-syncer/util" - "github.com/prysmaticlabs/prysm/v5/api/server/structs" - "os" - "path/filepath" - "time" ) var ( ErrVerificationFailed = errors.New("verification failed") + ErrBundleNotSealed = errors.New("bundle not sealed yet") ) -const bundleNotSealedAlertThresh = int64(1 * time.Hour) - // Verify is used to verify the blob uploaded to bundle service is indeed in Greenfield, and integrity. func (s *BlobSyncer) verify() error { @@ -45,6 +47,9 @@ func (s *BlobSyncer) verify() error { verifyBlock, err := s.blobDao.GetBlock(verifyBlockSlot) if err != nil { + if err == gorm.ErrRecordNotFound { + return nil + } logging.Logger.Errorf("failed to get block from DB, slot=%d err=%s", verifyBlockSlot, err.Error()) return err } @@ -78,7 +83,9 @@ func (s *BlobSyncer) verify() error { err = s.verifyBlobAtSlot(verifyBlockSlot, sideCars, blobMetas, bundleName) if err != nil { - logging.Logger.Errorf("failed to verifyBlobAtSlot, slot=%d err=%s", verifyBlockSlot, err.Error()) + if err == external.ErrorBundleNotExist || err == ErrBundleNotSealed { + return nil + } if err == ErrVerificationFailed { return s.reUploadBundle(bundleName) } @@ -100,19 +107,17 @@ func (s *BlobSyncer) verifyBlobAtSlot(slot uint64, sidecars []*structs.Sidecar, return err } if bundleInfo.Status == BundleStatusFinalized || bundleInfo.Status == BundleStatusCreatedOnChain { - if time.Now().Unix()-bundleInfo.CreatedTimestamp > bundleNotSealedAlertThresh { - // todo alarm - return fmt.Errorf("the bundle is supposed to be sealed") - } - return nil - } else if bundleInfo.Status != BundleStatusSealedOnChain { - return fmt.Errorf("unexpect status, should not happen") + return ErrBundleNotSealed } for i := 0; i < len(sidecars); i++ { // get blob from bundle servic blobFromBundle, err := s.bundleClient.GetObject(s.getBucketName(), bundleName, types.GetBlobName(slot, i)) if err != nil { + if err == external.ErrorBundleObjectNotExist { + logging.Logger.Errorf("the bundle object not found in bundle service, object=%s", types.GetBlobName(slot, i)) + return ErrVerificationFailed + } return err } @@ -156,15 +161,14 @@ func (s *BlobSyncer) verifyBlobAtSlot(slot uint64, sidecars []*structs.Sidecar, } func (s *BlobSyncer) reUploadBundle(bundleName string) error { - newBundleName := bundleName + "-calibrated-" + util.Int64ToString(time.Now().Unix()) + newBundleName := bundleName + "_calibrated_" + util.Int64ToString(time.Now().Unix()) startSlot, endSlot, err := types.ParseBundleName(bundleName) if err != nil { return err } - calibratedBundleDir := fmt.Sprintf("%s/%s/", s.config.TempDir, newBundleName) - _, err = os.Stat(calibratedBundleDir) + _, err = os.Stat(s.getBundleDir(newBundleName)) if os.IsNotExist(err) { - err = os.MkdirAll(filepath.Dir(calibratedBundleDir), os.ModePerm) + err = os.MkdirAll(filepath.Dir(s.getBundleDir(newBundleName)), os.ModePerm) if err != nil { return err } @@ -199,8 +203,7 @@ func (s *BlobSyncer) reUploadBundle(bundleName string) error { logging.Logger.Infof("save calibrated block(slot=%d) and blobs(num=%d) to DB \n", slot, len(blobToSave)) } - bundleFilePath := fmt.Sprintf("%s/%s.bundle", s.config.TempDir, bundleName) - if err := s.finalizeBundle(bundleName, calibratedBundleDir, bundleFilePath); err != nil { + if err := s.finalizeBundle(newBundleName, s.getBundleDir(newBundleName), s.getBundleFilePath(newBundleName)); err != nil { return err } return nil diff --git a/util/util.go b/util/util.go index 2dd99da..2f6f859 100644 --- a/util/util.go +++ b/util/util.go @@ -2,9 +2,10 @@ package util import ( "crypto/sha256" - "github.com/ethereum/go-ethereum/common/hexutil" "strconv" "strings" + + "github.com/ethereum/go-ethereum/common/hexutil" ) // StringToUint64 converts string to uint64