enrich

README.md

@@ -52,10 +52,10 @@ The Blob syncer server provides eth compatible API to query historical blob
 
 * GET /eth/v1/beacon/blob_sidecars/{block_id}
 
-| ParameterName | Type   | Description                                                                                                                                                            |
-|--------------| ------ |------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| block_id     | string | Block identifier. Can be one of:  <slot>, <hex encoded blockRoot with 0x prefix>. note: "head" (canonical head in node's view), "genesis", "finalized" are not support |
-| indices         | array<string> | Array of indices for blob sidecars to request for in the specified block. Returns all blob sidecars in the block if not specified                                      |
+| ParameterName | Type            | Description                                                                                                                                                          |
+|--------------|-----------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| block_id     | string          | Block identifier. Can be one of: slot,  <hex encoded blockRoot with 0x prefix>. note: "head" (canonical head in node's view), "genesis", "finalized" are not support |
+| indices         | array of string | Array of indices for blob sidecars to request for in the specified block. Returns all blob sidecars in the block if not specified                                    |
 
 
 200: Ok response

cmd/blob-syncer/main.go

@@ -2,13 +2,15 @@ package main
 
 import (
 	"flag"
+	"os"
+
+	"github.com/spf13/pflag"
+	"github.com/spf13/viper"
+
 	"github.com/bnb-chain/blob-syncer/config"
 	syncerdb "github.com/bnb-chain/blob-syncer/db"
 	"github.com/bnb-chain/blob-syncer/logging"
 	"github.com/bnb-chain/blob-syncer/syncer"
-	"github.com/spf13/pflag"
-	"github.com/spf13/viper"
-	"os"
 )
 
 func initFlags() {
@@ -31,7 +33,6 @@ func main() {
 	if configFilePath == "" {
 		configFilePath = os.Getenv(config.EnvVarConfigFilePath)
 	}
-	configFilePath = "config/local/config-syncer.json"
 	cfg = config.ParseSyncerConfigFromFile(configFilePath)
 	if cfg == nil {
 		panic("failed to get configuration")

config/config.go

@@ -3,14 +3,16 @@ package config
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/bnb-chain/blob-syncer/cache"
-	syncerdb "github.com/bnb-chain/blob-syncer/db"
-	"gorm.io/driver/mysql"
-	"gorm.io/gorm"
-	"gorm.io/gorm/logger"
 	"log"
 	"os"
 	"time"
+
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+
+	"github.com/bnb-chain/blob-syncer/cache"
+	syncerdb "github.com/bnb-chain/blob-syncer/db"
 )
 
 type SyncerConfig struct {

db/bundle.go

@@ -9,9 +9,10 @@ const (
 )
 
 type Bundle struct {
-	Id     int64
-	Name   string            `gorm:"NOT NULL;uniqueIndex:idx_bundle_name;size:64"`
-	Status InnerBundleStatus `gorm:"NOT NULL"`
+	Id         int64
+	Name       string            `gorm:"NOT NULL;uniqueIndex:idx_bundle_name;size:64"`
+	Status     InnerBundleStatus `gorm:"NOT NULL"`
+	Calibrated bool
 }
 
 func (*Bundle) TableName() string {

db/dao.go

@@ -33,7 +33,7 @@ type BlockDB interface {
 func (d *BlobSvcDB) GetBlock(slot uint64) (*Block, error) {
 	block := Block{}
 	err := d.db.Model(Block{}).Where("slot = ?", slot).Take(&block).Error
-	if err != nil && err != gorm.ErrRecordNotFound {
+	if err != nil {
 		return nil, err
 	}
 	return &block, nil
@@ -111,7 +111,7 @@ type BundleDB interface {
 
 func (d *BlobSvcDB) GetLatestFinalizingBundle() (*Bundle, error) {
 	bundle := Bundle{}
-	err := d.db.Model(Bundle{}).Where("status = ?", Finalizing).Order("id desc").Take(&bundle).Error
+	err := d.db.Model(Bundle{}).Where("status = ? and calibrated = false", Finalizing).Order("id desc").Take(&bundle).Error
 	if err != nil {
 		return nil, err
 	}

db/type.go

@@ -1,6 +1,8 @@
 package db
 
-import "github.com/go-sql-driver/mysql"
+import (
+	"github.com/go-sql-driver/mysql"
+)
 
 var (
 	ErrDuplicateEntryCode = 1062

restapi/configure_blob_syncer.go

@@ -4,6 +4,13 @@ package restapi
 
 import (
 	"crypto/tls"
+	"net/http"
+	"os"
+
+	"github.com/go-openapi/errors"
+	"github.com/go-openapi/runtime"
+	"github.com/go-openapi/swag"
+
 	"github.com/bnb-chain/blob-syncer/cache"
 	"github.com/bnb-chain/blob-syncer/config"
 	syncerdb "github.com/bnb-chain/blob-syncer/db"
@@ -12,11 +19,6 @@ import (
 	"github.com/bnb-chain/blob-syncer/restapi/operations"
 	"github.com/bnb-chain/blob-syncer/restapi/operations/blob"
 	"github.com/bnb-chain/blob-syncer/service"
-	"github.com/go-openapi/errors"
-	"github.com/go-openapi/runtime"
-	"github.com/go-openapi/swag"
-	"net/http"
-	"os"
 )
 
 //go:generate swagger generate server --target ../../blob-syncer --name BlobSyncer --spec ../swagger.yaml --principal interface{}

restapi/handlers/blob.go

@@ -3,13 +3,14 @@ package handlers
 import (
 	"encoding/hex"
 	"fmt"
-	"github.com/bnb-chain/blob-syncer/util"
+
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/go-openapi/runtime/middleware"
 
 	"github.com/bnb-chain/blob-syncer/models"
 	"github.com/bnb-chain/blob-syncer/restapi/operations/blob"
 	"github.com/bnb-chain/blob-syncer/service"
+	"github.com/bnb-chain/blob-syncer/util"
 )
 
 const rootLength = 32

service/blob.go

@@ -2,14 +2,14 @@ package service
 
 import (
 	"fmt"
-	"github.com/bnb-chain/blob-syncer/util"
 	"strconv"
 
 	"github.com/bnb-chain/blob-syncer/cache"
 	"github.com/bnb-chain/blob-syncer/config"
 	"github.com/bnb-chain/blob-syncer/db"
 	"github.com/bnb-chain/blob-syncer/external"
 	"github.com/bnb-chain/blob-syncer/models"
+	"github.com/bnb-chain/blob-syncer/util"
 )
 
 type Blob interface {

syncer/conveter.go

@@ -2,6 +2,7 @@ package syncer
 
 import (
 	"encoding/json"
+
 	"github.com/prysmaticlabs/prysm/v5/api/server/structs"
 	v1 "github.com/prysmaticlabs/prysm/v5/proto/engine/v1"
 	ethpb "github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1"

syncer/syncer.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/hex"
 	"fmt"
-	"github.com/bnb-chain/blob-syncer/util"
 	"math/big"
 	"os"
 	"path/filepath"
@@ -25,6 +24,7 @@ import (
 	"github.com/bnb-chain/blob-syncer/external"
 	"github.com/bnb-chain/blob-syncer/logging"
 	"github.com/bnb-chain/blob-syncer/types"
+	"github.com/bnb-chain/blob-syncer/util"
 )
 
 const (

syncer/verifier.go

@@ -4,23 +4,25 @@ import (
 	"bytes"
 	"context"
 	"errors"
-	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/prysmaticlabs/prysm/v5/api/server/structs"
+	"gorm.io/gorm"
+
 	"github.com/bnb-chain/blob-syncer/db"
+	"github.com/bnb-chain/blob-syncer/external"
 	"github.com/bnb-chain/blob-syncer/logging"
 	"github.com/bnb-chain/blob-syncer/types"
 	"github.com/bnb-chain/blob-syncer/util"
-	"github.com/prysmaticlabs/prysm/v5/api/server/structs"
-	"os"
-	"path/filepath"
-	"time"
 )
 
 var (
 	ErrVerificationFailed = errors.New("verification failed")
+	ErrBundleNotSealed    = errors.New("bundle not sealed yet")
 )
 
-const bundleNotSealedAlertThresh = int64(1 * time.Hour)
-
 // Verify is used to verify the blob uploaded to bundle service is indeed in Greenfield, and integrity.
 func (s *BlobSyncer) verify() error {
 
@@ -45,6 +47,9 @@ func (s *BlobSyncer) verify() error {
 
 	verifyBlock, err := s.blobDao.GetBlock(verifyBlockSlot)
 	if err != nil {
+		if err == gorm.ErrRecordNotFound {
+			return nil
+		}
 		logging.Logger.Errorf("failed to get block from DB, slot=%d err=%s", verifyBlockSlot, err.Error())
 		return err
 	}
@@ -78,7 +83,9 @@ func (s *BlobSyncer) verify() error {
 
 	err = s.verifyBlobAtSlot(verifyBlockSlot, sideCars, blobMetas, bundleName)
 	if err != nil {
-		logging.Logger.Errorf("failed to verifyBlobAtSlot, slot=%d err=%s", verifyBlockSlot, err.Error())
+		if err == external.ErrorBundleNotExist || err == ErrBundleNotSealed {
+			return nil
+		}
 		if err == ErrVerificationFailed {
 			return s.reUploadBundle(bundleName)
 		}
@@ -100,19 +107,17 @@ func (s *BlobSyncer) verifyBlobAtSlot(slot uint64, sidecars []*structs.Sidecar,
 		return err
 	}
 	if bundleInfo.Status == BundleStatusFinalized || bundleInfo.Status == BundleStatusCreatedOnChain {
-		if time.Now().Unix()-bundleInfo.CreatedTimestamp > bundleNotSealedAlertThresh {
-			// todo alarm
-			return fmt.Errorf("the bundle is supposed to be sealed")
-		}
-		return nil
-	} else if bundleInfo.Status != BundleStatusSealedOnChain {
-		return fmt.Errorf("unexpect status, should not happen")
+		return ErrBundleNotSealed
 	}
 
 	for i := 0; i < len(sidecars); i++ {
 		// get blob from bundle servic
 		blobFromBundle, err := s.bundleClient.GetObject(s.getBucketName(), bundleName, types.GetBlobName(slot, i))
 		if err != nil {
+			if err == external.ErrorBundleObjectNotExist {
+				logging.Logger.Errorf("the bundle object not found in bundle service, object=%s", types.GetBlobName(slot, i))
+				return ErrVerificationFailed
+			}
 			return err
 		}
 
@@ -156,15 +161,14 @@ func (s *BlobSyncer) verifyBlobAtSlot(slot uint64, sidecars []*structs.Sidecar,
 }
 
 func (s *BlobSyncer) reUploadBundle(bundleName string) error {
-	newBundleName := bundleName + "-calibrated-" + util.Int64ToString(time.Now().Unix())
+	newBundleName := bundleName + "_calibrated_" + util.Int64ToString(time.Now().Unix())
 	startSlot, endSlot, err := types.ParseBundleName(bundleName)
 	if err != nil {
 		return err
 	}
-	calibratedBundleDir := fmt.Sprintf("%s/%s/", s.config.TempDir, newBundleName)
-	_, err = os.Stat(calibratedBundleDir)
+	_, err = os.Stat(s.getBundleDir(newBundleName))
 	if os.IsNotExist(err) {
-		err = os.MkdirAll(filepath.Dir(calibratedBundleDir), os.ModePerm)
+		err = os.MkdirAll(filepath.Dir(s.getBundleDir(newBundleName)), os.ModePerm)
 		if err != nil {
 			return err
 		}
@@ -199,8 +203,7 @@ func (s *BlobSyncer) reUploadBundle(bundleName string) error {
 		logging.Logger.Infof("save calibrated block(slot=%d) and blobs(num=%d) to DB \n", slot, len(blobToSave))
 	}
 
-	bundleFilePath := fmt.Sprintf("%s/%s.bundle", s.config.TempDir, bundleName)
-	if err := s.finalizeBundle(bundleName, calibratedBundleDir, bundleFilePath); err != nil {
+	if err := s.finalizeBundle(newBundleName, s.getBundleDir(newBundleName), s.getBundleFilePath(newBundleName)); err != nil {
 		return err
 	}
 	return nil

util/util.go

@@ -2,9 +2,10 @@ package util
 
 import (
 	"crypto/sha256"
-	"github.com/ethereum/go-ethereum/common/hexutil"
 	"strconv"
 	"strings"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
 )
 
 // StringToUint64 converts string to uint64