Sanity check. Remove mod go-eth

cmd/main.go

@@ -24,9 +24,12 @@ const (
 )
 
 var (
-	sanityEntry    uint64 = 0
-	sanityBlock    uint64 = 0
-	sanityBookmark uint64 = 0
+	initSanityEntry    bool   = false
+	initSanityBlock    bool   = false
+	initSanityBookmark bool   = false
+	sanityEntry        uint64 = 0
+	sanityBlock        uint64 = 0
+	sanityBookmark     uint64 = 0
 )
 
 // main runs a datastream server or client
@@ -465,20 +468,34 @@ func printEntryNum(e *datastreamer.FileEntry, c *datastreamer.StreamClient, s *d
 
 // checkEntryBlockSanity checks entry, bookmark, and block sequence consistency
 func checkEntryBlockSanity(e *datastreamer.FileEntry, c *datastreamer.StreamClient, s *datastreamer.StreamServer) error {
+	// Sanity check initialization
+	if !initSanityEntry {
+		initSanityEntry = true
+		if c.FromEntry > 0 {
+			sanityEntry = c.FromEntry
+		} else {
+			sanityEntry = 0
+		}
+	}
+
 	// Log work in progress
 	if e.Number%100000 == 0 {
-		log.Infof("Sanity check entry #%d...", e.Number)
+		log.Infof("Checking entry #%d...", e.Number)
 	}
 
 	// Sanity check for entry sequence
 	if sanityEntry > 0 {
 		if e.Number != sanityEntry {
-			log.Infof("SANITY CHECK failed: Entry received[%d] | Entry expected[%d]", e.Number, sanityEntry)
+			if e.Number < sanityEntry {
+				log.Warnf("(X) SANITY CHECK failed: REPEATED entries? Received[%d] | Entry expected[%d]", e.Number, sanityEntry)
+			} else {
+				log.Warnf("(X) SANITY CHECK failed: GAP entries? Received[%d] | Entry expected[%d]", e.Number, sanityEntry)
+			}
 			return errors.New("sanity check failed for entry sequence")
 		}
 	} else {
 		if e.Number != 0 {
-			log.Infof("SANITY CHECK failed: Entry received[%d] | Entry expected[0]", e.Number)
+			log.Warnf("(X) SANITY CHECK failed: Entry received[%d] | Entry expected[0]", e.Number)
 			return errors.New("sanity check failed for entry sequence")
 		}
 	}
@@ -489,13 +506,23 @@ func checkEntryBlockSanity(e *datastreamer.FileEntry, c *datastreamer.StreamClie
 		blockNum := binary.LittleEndian.Uint64(e.Data[8:16])
 		if sanityBlock > 0 {
 			if blockNum != sanityBlock {
-				log.Infof("SANITY CHECK failed: Block received[%d] | Block expected[%d]", blockNum, sanityBlock)
+				if blockNum < sanityBlock {
+					log.Infof("(X) SANITY CHECK failed (%d): REPEATED blocks? Received[%d] | Block expected[%d]", e.Number, blockNum, sanityBlock)
+				} else {
+					log.Infof("(X) SANITY CHECK failed (%d): GAP blocks? Received[%d] | Block expected[%d]", e.Number, blockNum, sanityBlock)
+				}
 				sanityBlock = blockNum
 			}
 		} else {
 			if blockNum != 0 {
-				log.Infof("SANITY CHECK failed: Block received[%d] | Block expected[0]", blockNum)
-				sanityBlock = 0
+				if initSanityBlock {
+					log.Infof("(X) SANITY CHECK failed (%d): Block received[%d] | Block expected[0]", e.Number, blockNum)
+					sanityBlock = 0
+				} else {
+					log.Infof("SANITY CHECK note (%d): First Block received[%d]", e.Number, blockNum)
+					sanityBlock = blockNum
+				}
+				initSanityBlock = true
 			}
 		}
 		sanityBlock++
@@ -506,13 +533,23 @@ func checkEntryBlockSanity(e *datastreamer.FileEntry, c *datastreamer.StreamClie
 		bookmarkNum := binary.LittleEndian.Uint64(e.Data[1:9])
 		if sanityBookmark > 0 {
 			if bookmarkNum != sanityBookmark {
-				log.Infof("SANITY CHECK failed: Bookmark received[%d] | Bookmark expected[%d]", bookmarkNum, sanityBookmark)
+				if bookmarkNum < sanityBookmark {
+					log.Infof("(X) SANITY CHECK failed (%d): REPEATED bookmarks? Received[%d] | Bookmark expected[%d]", e.Number, bookmarkNum, sanityBookmark)
+				} else {
+					log.Infof("(X) SANITY CHECK failed (%d): GAP bookmarks? Received[%d] | Bookmark expected[%d]", e.Number, bookmarkNum, sanityBookmark)
+				}
 				sanityBookmark = bookmarkNum
 			}
 		} else {
 			if bookmarkNum != 0 {
-				log.Infof("SANITY CHECK failed: Bookmark received[%d] | Bookmark expected[0]", bookmarkNum)
-				sanityBookmark = 0
+				if initSanityBookmark {
+					log.Infof("(X) SANITY CHECK failed (%d): Bookmark received[%d] | Bookmark expected[0]", e.Number, bookmarkNum)
+					sanityBookmark = 0
+				} else {
+					log.Infof("SANITY CHECK note (%d): First Bookmark received[%d]", e.Number, bookmarkNum)
+					sanityBookmark = bookmarkNum
+				}
+				initSanityBookmark = true
 			}
 		}
 		sanityBookmark++

datastreamer/datastreamer_test.go

@@ -2,6 +2,7 @@ package datastreamer_test
 
 import (
 	"encoding/binary"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"os"
@@ -10,14 +11,55 @@ import (
 
 	"github.com/0xPolygonHermez/zkevm-data-streamer/datastreamer"
 	"github.com/0xPolygonHermez/zkevm-data-streamer/log"
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/stretchr/testify/require"
 )
 
+// AUX ------------------------------------------------------------------------
+const hashLength = 32
+
+type hash [hashLength]byte
+
+func (h *hash) setBytes(b []byte) {
+	if len(b) > len(h) {
+		b = b[len(b)-hashLength:]
+	}
+
+	copy(h[hashLength-len(b):], b)
+}
+
+func bytesToHash(b []byte) hash {
+	var h hash
+	h.setBytes(b)
+	return h
+}
+
+func has0xPrefix(str string) bool {
+	return len(str) >= 2 && str[0] == '0' && (str[1] == 'x' || str[1] == 'X')
+}
+
+func hex2Bytes(str string) []byte {
+	h, _ := hex.DecodeString(str)
+	return h
+}
+
+func fromHex(s string) []byte {
+	if has0xPrefix(s) {
+		s = s[2:]
+	}
+	if len(s)%2 == 1 {
+		s = "0" + s
+	}
+	return hex2Bytes(s)
+}
+
+func hexToHash(s string) hash { return bytesToHash(fromHex(s)) }
+
+// ----------------------------------------------------------------------------
+
 type TestEntry struct {
-	FieldA uint64      // 8 bytes
-	FieldB common.Hash // 32 bytes
-	FieldC []byte      // n bytes
+	FieldA uint64 // 8 bytes
+	FieldB hash   // 32 bytes
+	FieldC []byte // n bytes
 }
 
 type TestBookmark struct {
@@ -41,7 +83,7 @@ func (t TestEntry) Encode() []byte {
 
 func (t TestEntry) Decode(bytes []byte) TestEntry {
 	t.FieldA = binary.LittleEndian.Uint64(bytes[:8])
-	t.FieldB = common.BytesToHash(bytes[8:40])
+	t.FieldB = bytesToHash(bytes[8:40])
 	t.FieldC = bytes[40:]
 	return t
 }
@@ -69,27 +111,27 @@ var (
 	testEntries = []TestEntry{
 		{
 			FieldA: 0,
-			FieldB: common.HexToHash("0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"),
+			FieldB: hexToHash("0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"),
 			FieldC: []byte("test entry 0"),
 		},
 		{
 			FieldA: 1,
-			FieldB: common.HexToHash("0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"),
+			FieldB: hexToHash("0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"),
 			FieldC: []byte("test entry 1"),
 		},
 		{
 			FieldA: 2,
-			FieldB: common.HexToHash("0x2234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"),
+			FieldB: hexToHash("0x2234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"),
 			FieldC: []byte("test entry 2"),
 		},
 		{
 			FieldA: 3,
-			FieldB: common.HexToHash("0x3234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"),
+			FieldB: hexToHash("0x3234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"),
 			FieldC: []byte("test entry 3"),
 		},
 		{
 			FieldA: 4,
-			FieldB: common.HexToHash("0x3234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"),
+			FieldB: hexToHash("0x3234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"),
 			FieldC: []byte("large test entry 4 large test entry 4 large test entry 4 large test entry 4" +
 				"large test entry 4 large test entry 4 large test entry 4 large test entry 4" +
 				"large test entry 4 large test entry 4 large test entry 4 large test entry 4" +
@@ -105,13 +147,13 @@ var (
 
 	badUpdateEntry = TestEntry{
 		FieldA: 10,
-		FieldB: common.HexToHash("0xa1cdef7890abcdef1234567890abcdef1234567890abcdef1234567890123456"),
+		FieldB: hexToHash("0xa1cdef7890abcdef1234567890abcdef1234567890abcdef1234567890123456"),
 		FieldC: []byte("test entry not updated"),
 	}
 
 	okUpdateEntry = TestEntry{
 		FieldA: 11,
-		FieldB: common.HexToHash("0xa2cdef7890abcdef1234567890abcdef1234567890abcdef1234567890123456"),
+		FieldB: hexToHash("0xa2cdef7890abcdef1234567890abcdef1234567890abcdef1234567890123456"),
 		FieldC: []byte("update entry"),
 	}
 

go.mod

@@ -3,7 +3,6 @@ module github.com/0xPolygonHermez/zkevm-data-streamer
 go 1.19
 
 require (
-	github.com/ethereum/go-ethereum v1.13.2
 	github.com/hermeznetwork/tracerr v0.3.2
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/spf13/viper v1.17.0
@@ -33,7 +32,6 @@ require (
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	go.uber.org/multierr v1.10.0 // indirect
-	golang.org/x/crypto v0.13.0 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
 	golang.org/x/sys v0.12.0 // indirect
 	golang.org/x/text v0.13.0 // indirect

go.sum

@@ -58,8 +58,6 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
 github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/ethereum/go-ethereum v1.13.2 h1:g9mCpfPWqCA1OL4e6C98PeVttb0HadfBRuKTGvMnOvw=
-github.com/ethereum/go-ethereum v1.13.2/go.mod h1:gkQ5Ygi64ZBh9M/4iXY1R8WqoNCx1Ey0CkYn2BD4/fw=
 github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
@@ -225,8 +223,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
-golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -526,8 +522,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=