diff --git a/.github/workflows/qa-clean-exit-block-downloading.yml b/.github/workflows/qa-clean-exit-block-downloading.yml new file mode 100644 index 00000000000..c378dd793bb --- /dev/null +++ b/.github/workflows/qa-clean-exit-block-downloading.yml @@ -0,0 +1,111 @@ +name: QA - Clean exit (block downloading) + +on: + push: + branches: + - 'release/2.*' + pull_request: + branches: + - 'release/2.*' + types: + - ready_for_review + workflow_dispatch: # Run manually + +jobs: + clean-exit-bd-test: + runs-on: [self-hosted, Erigon2] + env: + ERIGON_REFERENCE_DATA_DIR: /opt/erigon-versions/reference-version/datadir + ERIGON_TESTBED_DATA_DIR: /opt/erigon-testbed/datadir + ERIGON_QA_PATH: /home/qarunner/erigon-qa + WORKING_TIME_SECONDS: 600 + CHAIN: mainnet + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Clean Erigon Build Directory + run: | + make clean + + - name: Build Erigon + run: | + make erigon + working-directory: ${{ github.workspace }} + + - name: Pause the Erigon instance dedicated to db maintenance + run: | + python3 $ERIGON_QA_PATH/test_system/db-producer/pause_production.py || true + + - name: Restore Erigon Testbed Data Directory + run: | + rsync -a --delete $ERIGON_REFERENCE_DATA_DIR/ $ERIGON_TESTBED_DATA_DIR/ + + - name: Run Erigon, send ctrl-c and check for clean exiting + id: test_step + run: | + set +e # Disable exit on error + + # Run Erigon, send ctrl-c and check logs + python3 $ERIGON_QA_PATH/test_system/qa-tests/clean-exit/run_and_check_clean_exit.py ${{ github.workspace }}/build/bin $ERIGON_TESTBED_DATA_DIR $WORKING_TIME_SECONDS Erigon2 + + # Capture monitoring script exit status + test_exit_status=$? + + # Save the subsection reached status + echo "::set-output name=test_executed::true" + + # Clean up Erigon process if it's still running + if kill -0 $ERIGON_PID 2> /dev/null; then + echo "Terminating Erigon" + kill $ERIGON_PID + wait $ERIGON_PID + fi + + # Check test runner script exit status + if [ $test_exit_status -eq 0 ]; then + echo "Tests completed successfully" + echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT" + else + echo "Error detected during tests" + echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT" + fi + + - name: Delete Erigon Testbed Data Directory + if: always() + run: | + rm -rf $ERIGON_TESTBED_DATA_DIR + + - name: Resume the Erigon instance dedicated to db maintenance + run: | + python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true + + - name: Save test results + if: steps.test_step.outputs.test_executed == 'true' + env: + TEST_RESULT: ${{ steps.test_step.outputs.TEST_RESULT }} + run: | + db_version=$(python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/prod_info.py $ERIGON_REFERENCE_DATA_DIR/../production.ini production erigon_repo_commit) + if [ -z "$db_version" ]; then + db_version="no-version" + fi + + python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/upload_test_results.py --repo erigon --commit $(git rev-parse HEAD) --branch ${{ github.ref_name }} --test_name clean-exit-block-downloading --chain $CHAIN --runner ${{ runner.name }} --db_version $db_version --outcome $TEST_RESULT --result_file ${{ github.workspace }}/result.json + + - name: Upload test results + if: steps.test_step.outputs.test_executed == 'true' + uses: actions/upload-artifact@v4 + with: + name: test-results + path: ${{ github.workspace }}/result.json + + - name: Action for Success + if: steps.test_step.outputs.TEST_RESULT == 'success' + run: echo "::notice::Tests completed successfully" + + - name: Action for Not Success + if: steps.test_step.outputs.TEST_RESULT != 'success' + run: | + echo "::error::Error detected during tests" + exit 1 \ No newline at end of file diff --git a/.github/workflows/qa-clean-exit-snapshot-downloading.yml b/.github/workflows/qa-clean-exit-snapshot-downloading.yml new file mode 100644 index 00000000000..5f953a654ff --- /dev/null +++ b/.github/workflows/qa-clean-exit-snapshot-downloading.yml @@ -0,0 +1,101 @@ +name: QA - Clean exit (snapshot downloading) + +on: + push: + branches: + - 'release/2.*' + pull_request: + branches: + - 'release/2.*' + types: + - ready_for_review + workflow_dispatch: # Run manually + +jobs: + clean-exit-sd-test: + runs-on: self-hosted + env: + ERIGON_DATA_DIR: ${{ github.workspace }}/erigon_data + ERIGON_QA_PATH: /home/qarunner/erigon-qa + WORKING_TIME_SECONDS: 600 + CHAIN: mainnet + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Clean Erigon Build & Data Directories + run: | + make clean + rm -rf $ERIGON_DATA_DIR + + - name: Build Erigon + run: | + make erigon + working-directory: ${{ github.workspace }} + + - name: Pause the Erigon instance dedicated to db maintenance + run: | + python3 $ERIGON_QA_PATH/test_system/db-producer/pause_production.py || true + + - name: Run Erigon, send ctrl-c and check for clean exiting + id: test_step + run: | + set +e # Disable exit on error + + # Run Erigon, send ctrl-c and check logs + python3 $ERIGON_QA_PATH/test_system/qa-tests/clean-exit/run_and_check_clean_exit.py ${{ github.workspace }}/build/bin $ERIGON_DATA_DIR $WORKING_TIME_SECONDS Erigon2 + + # Capture monitoring script exit status + test_exit_status=$? + + # Save the subsection reached status + echo "::set-output name=test_executed::true" + + # Clean up Erigon process if it's still running + if kill -0 $ERIGON_PID 2> /dev/null; then + echo "Terminating Erigon" + kill $ERIGON_PID + wait $ERIGON_PID + fi + + # Check test runner script exit status + if [ $test_exit_status -eq 0 ]; then + echo "Tests completed successfully" + echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT" + else + echo "Error detected during tests" + echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT" + fi + + - name: Clean up Erigon data directory + if: always() + run: | + rm -rf $ERIGON_DATA_DIR + + - name: Resume the Erigon instance dedicated to db maintenance + run: | + python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true + + - name: Save test results + if: steps.test_step.outputs.test_executed == 'true' + env: + TEST_RESULT: ${{ steps.test_step.outputs.TEST_RESULT }} + run: python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/upload_test_results.py --repo erigon --commit $(git rev-parse HEAD) --branch ${{ github.ref_name }} --test_name clean-exit-snapshot-downloading --chain $CHAIN --runner ${{ runner.name }} --outcome $TEST_RESULT --result_file ${{ github.workspace }}/result.json + + - name: Upload test results + if: steps.test_step.outputs.test_executed == 'true' + uses: actions/upload-artifact@v4 + with: + name: test-results + path: ${{ github.workspace }}/result.json + + - name: Action for Success + if: steps.test_step.outputs.TEST_RESULT == 'success' + run: echo "::notice::Tests completed successfully" + + - name: Action for Not Success + if: steps.test_step.outputs.TEST_RESULT != 'success' + run: | + echo "::error::Error detected during tests" + exit 1 \ No newline at end of file diff --git a/.github/workflows/qa-snap-download.yml b/.github/workflows/qa-snap-download.yml new file mode 100644 index 00000000000..630429e14d5 --- /dev/null +++ b/.github/workflows/qa-snap-download.yml @@ -0,0 +1,102 @@ +name: QA - Snapshot Download + +on: + push: + branches: + - 'release/2.*' + pull_request: + branches: + - 'release/2.*' + types: + - ready_for_review + workflow_dispatch: # Run manually + +jobs: + snap-download-test: + runs-on: [self-hosted, Erigon2] + timeout-minutes: 800 + env: + ERIGON_DATA_DIR: ${{ github.workspace }}/erigon_data + ERIGON_QA_PATH: /home/qarunner/erigon-qa + TOTAL_TIME_SECONDS: 43200 # 12 hours + CHAIN: mainnet + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Clean Erigon Build & Data Directories + run: | + make clean + rm -rf $ERIGON_DATA_DIR + + - name: Build Erigon + run: | + make erigon + working-directory: ${{ github.workspace }} + + - name: Pause the Erigon instance dedicated to db maintenance + run: | + python3 $ERIGON_QA_PATH/test_system/db-producer/pause_production.py || true + + - name: Run Erigon and monitor snapshot downloading + id: test_step + run: | + set +e # Disable exit on error + + # Run Erigon, monitor snapshot downloading and check logs + python3 $ERIGON_QA_PATH/test_system/qa-tests/snap-download/run_and_check_snap_download.py ${{ github.workspace }}/build/bin $ERIGON_DATA_DIR $TOTAL_TIME_SECONDS Erigon2 $CHAIN + + # Capture monitoring script exit status + test_exit_status=$? + + # Save the subsection reached status + echo "::set-output name=test_executed::true" + + # Clean up Erigon process if it's still running + if kill -0 $ERIGON_PID 2> /dev/null; then + echo "Terminating Erigon" + kill $ERIGON_PID + wait $ERIGON_PID + fi + + # Check test runner script exit status + if [ $test_exit_status -eq 0 ]; then + echo "Tests completed successfully" + echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT" + else + echo "Error detected during tests" + echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT" + fi + + - name: Clean up Erigon data directory + if: always() + run: | + rm -rf $ERIGON_DATA_DIR + + - name: Resume the Erigon instance dedicated to db maintenance + run: | + python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true + + - name: Save test results + if: steps.test_step.outputs.test_executed == 'true' + env: + TEST_RESULT: ${{ steps.test_step.outputs.TEST_RESULT }} + run: python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/upload_test_results.py --repo erigon --commit $(git rev-parse HEAD) --branch ${{ github.ref_name }} --test_name snap-download --chain $CHAIN --runner ${{ runner.name }} --outcome $TEST_RESULT --result_file ${{ github.workspace }}/result-$CHAIN.json + + - name: Upload test results + if: steps.test_step.outputs.test_executed == 'true' + uses: actions/upload-artifact@v4 + with: + name: test-results + path: ${{ github.workspace }}/result-${{ env.CHAIN }}.json + + - name: Action for Success + if: steps.test_step.outputs.TEST_RESULT == 'success' + run: echo "::notice::Tests completed successfully" + + - name: Action for Not Success + if: steps.test_step.outputs.TEST_RESULT != 'success' + run: | + echo "::error::Error detected during tests" + exit 1 \ No newline at end of file diff --git a/.github/workflows/qa-tip-tracking.yml b/.github/workflows/qa-tip-tracking.yml new file mode 100644 index 00000000000..2a5299ed252 --- /dev/null +++ b/.github/workflows/qa-tip-tracking.yml @@ -0,0 +1,115 @@ +name: QA - Tip tracking + +on: + push: + branches: + - 'release/2.*' + pull_request: + branches: + - 'release/2.*' + types: + - ready_for_review + workflow_dispatch: # Run manually + +jobs: + tip-tracking-test: + runs-on: [self-hosted, Erigon2] + timeout-minutes: 600 + env: + ERIGON_REFERENCE_DATA_DIR: /opt/erigon-versions/reference-version/datadir + ERIGON_TESTBED_DATA_DIR: /opt/erigon-testbed/datadir + ERIGON_QA_PATH: /home/qarunner/erigon-qa + TRACKING_TIME_SECONDS: 14400 # 4 hours + TOTAL_TIME_SECONDS: 28800 # 8 hours + CHAIN: mainnet + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Clean Erigon Build Directory + run: | + make clean + + - name: Build Erigon + run: | + make erigon + working-directory: ${{ github.workspace }} + + - name: Pause the Erigon instance dedicated to db maintenance + run: | + python3 $ERIGON_QA_PATH/test_system/db-producer/pause_production.py || true + + - name: Restore Erigon Testbed Data Directory + run: | + rsync -a --delete $ERIGON_REFERENCE_DATA_DIR/ $ERIGON_TESTBED_DATA_DIR/ + + - name: Run Erigon, wait sync and check ability to maintain sync + id: test_step + run: | + set +e # Disable exit on error + + # 1. Launch the testbed Erigon instance + # 2. Allow time for the Erigon to achieve synchronization + # 3. Begin timing the duration that Erigon maintains synchronization + python3 $ERIGON_QA_PATH/test_system/qa-tests/tip-tracking/run_and_check_tip_tracking.py ${{ github.workspace }}/build/bin $ERIGON_TESTBED_DATA_DIR $TRACKING_TIME_SECONDS $TOTAL_TIME_SECONDS Erigon2 $CHAIN + + # Capture monitoring script exit status + test_exit_status=$? + + # Save the subsection reached status + echo "::set-output name=test_executed::true" + + # Clean up Erigon process if it's still running + if kill -0 $ERIGON_PID 2> /dev/null; then + echo "Terminating Erigon" + kill $ERIGON_PID + wait $ERIGON_PID + fi + + # Check test runner script exit status + if [ $test_exit_status -eq 0 ]; then + echo "Tests completed successfully" + echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT" + else + echo "Error detected during tests" + echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT" + fi + + - name: Delete Erigon Testbed Data Directory + if: always() + run: | + rm -rf $ERIGON_TESTBED_DATA_DIR + + - name: Resume the Erigon instance dedicated to db maintenance + run: | + python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true + + - name: Save test results + if: steps.test_step.outputs.test_executed == 'true' + env: + TEST_RESULT: ${{ steps.test_step.outputs.TEST_RESULT }} + run: | + db_version=$(python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/prod_info.py $ERIGON_REFERENCE_DATA_DIR/../production.ini production erigon_repo_commit) + if [ -z "$db_version" ]; then + db_version="no-version" + fi + + python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/upload_test_results.py --repo erigon --commit $(git rev-parse HEAD) --branch ${{ github.ref_name }} --test_name tip-tracking --chain $CHAIN --runner ${{ runner.name }} --db_version $db_version --outcome $TEST_RESULT --result_file ${{ github.workspace }}/result-$CHAIN.json + + - name: Upload test results + if: steps.test_step.outputs.test_executed == 'true' + uses: actions/upload-artifact@v4 + with: + name: test-results + path: ${{ github.workspace }}/result-${{ env.CHAIN }}.json + + - name: Action for Success + if: steps.test_step.outputs.TEST_RESULT == 'success' + run: echo "::notice::Tests completed successfully" + + - name: Action for Not Success + if: steps.test_step.outputs.TEST_RESULT != 'success' + run: | + echo "::error::Error detected during tests" + exit 1 \ No newline at end of file diff --git a/cmd/diag/_images/dbs/example_databases.png b/cmd/diag/_images/dbs/example_databases.png new file mode 100644 index 00000000000..5f74a2942bd Binary files /dev/null and b/cmd/diag/_images/dbs/example_databases.png differ diff --git a/cmd/diag/_images/dbs/example_databases_name.png b/cmd/diag/_images/dbs/example_databases_name.png new file mode 100644 index 00000000000..a1bbf5d01ca Binary files /dev/null and b/cmd/diag/_images/dbs/example_databases_name.png differ diff --git a/cmd/diag/_images/dbs/example_databases_name_populated.png b/cmd/diag/_images/dbs/example_databases_name_populated.png new file mode 100644 index 00000000000..aabef5b99c9 Binary files /dev/null and b/cmd/diag/_images/dbs/example_databases_name_populated.png differ diff --git a/cmd/diag/_images/downloader/example_downloader.png b/cmd/diag/_images/downloader/example_downloader.png new file mode 100644 index 00000000000..a74e48e1e40 Binary files /dev/null and b/cmd/diag/_images/downloader/example_downloader.png differ diff --git a/cmd/diag/_images/downloader/example_downloader_files.png b/cmd/diag/_images/downloader/example_downloader_files.png new file mode 100644 index 00000000000..5dab609515b Binary files /dev/null and b/cmd/diag/_images/downloader/example_downloader_files.png differ diff --git a/cmd/diag/_images/downloader/example_downloader_files_downloaded.png b/cmd/diag/_images/downloader/example_downloader_files_downloaded.png new file mode 100644 index 00000000000..7d120e2f714 Binary files /dev/null and b/cmd/diag/_images/downloader/example_downloader_files_downloaded.png differ diff --git a/cmd/diag/_images/downloader/example_downloader_files_filename.png b/cmd/diag/_images/downloader/example_downloader_files_filename.png new file mode 100644 index 00000000000..5ff5b5766ef Binary files /dev/null and b/cmd/diag/_images/downloader/example_downloader_files_filename.png differ diff --git a/cmd/diag/_images/local_connection.png b/cmd/diag/_images/local_connection.png new file mode 100644 index 00000000000..4b640cad181 Binary files /dev/null and b/cmd/diag/_images/local_connection.png differ diff --git a/cmd/diag/_images/stages/example_stages.png b/cmd/diag/_images/stages/example_stages.png new file mode 100644 index 00000000000..7c9891fc373 Binary files /dev/null and b/cmd/diag/_images/stages/example_stages.png differ diff --git a/cmd/diag/db/db.go b/cmd/diag/db/db.go index 65f16c3895c..a9056ecc949 100644 --- a/cmd/diag/db/db.go +++ b/cmd/diag/db/db.go @@ -1,9 +1,11 @@ package db import ( - "encoding/json" "fmt" + "os" + "github.com/jedib0t/go-pretty/v6/table" + "github.com/jedib0t/go-pretty/v6/text" "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon/cmd/diag/flags" "github.com/ledgerwatch/erigon/cmd/diag/util" @@ -11,181 +13,110 @@ import ( ) type DBInfo struct { - name string - tables []BDTableInfo - count int - size string + name string `header:"DB Name"` + tables []BDTableInfo `header:"Tables"` + count int `header:"Keys Count"` + size string `header:"Size"` } type BDTableInfo struct { - Name string - Count int - Size uint64 + Name string `header:"Table Name"` + Count int `header:"Keys Count"` + Size uint64 `header:"Size"` } +var ( + DBPopulatedFlag = cli.BoolFlag{ + Name: "db.appearance.populated", + Aliases: []string{"dbap"}, + Usage: "Print populated table content only", + Required: false, + Value: false, + } + + DBNameFlag = cli.StringFlag{ + Name: "db.name", + Aliases: []string{"dbn"}, + Usage: "DB name to print info about. If not set, all dbs will be printed.", + Required: false, + Value: "", + } +) + var Command = cli.Command{ + Action: startPrintDBsInfo, Name: "databases", Aliases: []string{"dbs"}, + Usage: "Print database tables info.", ArgsUsage: "", - Subcommands: []*cli.Command{ - { - Name: "all", - Aliases: []string{"a"}, - Action: printAllDBsInfo, - Usage: "Print database tables info.", - ArgsUsage: "", - Flags: []cli.Flag{ - &flags.DebugURLFlag, - &flags.OutputFlag, - }, - }, - { - Name: "populated", - Aliases: []string{"pop"}, - Action: printPopuplatedDBsInfo, - Usage: "Print database tables info which is not empty.", - ArgsUsage: "", - Flags: []cli.Flag{ - &flags.DebugURLFlag, - &flags.OutputFlag, - }, - }, + Flags: []cli.Flag{ + &flags.DebugURLFlag, + &flags.OutputFlag, + &DBPopulatedFlag, + &DBNameFlag, }, Description: ``, } -func printAllDBsInfo(cliCtx *cli.Context) error { - data, err := AllDBsInfo(cliCtx) +func startPrintDBsInfo(cliCtx *cli.Context) error { + data, err := DBsInfo(cliCtx) if err != nil { - return err - } - - switch cliCtx.String(flags.OutputFlag.Name) { - case "json": - bytes, err := json.Marshal(data) - if err != nil { - return err - } - - fmt.Println(string(bytes)) - - case "text": - printDBsInfo(data) + util.RenderError(err) + return nil } - return nil -} + dbToPrint := cliCtx.String(DBNameFlag.Name) -func printPopuplatedDBsInfo(cliCtx *cli.Context) error { - data, err := AllDBsInfo(cliCtx) - if err != nil { - return err - } - - // filter out empty tables - for i := 0; i < len(data); i++ { - tables := data[i].tables - for j := 0; j < len(tables); j++ { - if tables[j].Count == 0 { - tables = append(tables[:j], tables[j+1:]...) - j-- + if dbToPrint != "" { + for _, db := range data { + if db.name == dbToPrint { + printDBsInfo([]DBInfo{db}) + return nil } } - data[i].tables = tables - } - //filter out empty dbs - for i := 0; i < len(data); i++ { - if len(data[i].tables) == 0 { - data = append(data[:i], data[i+1:]...) - i-- - } + fmt.Printf("DB %s not found\n", dbToPrint) + return nil } - switch cliCtx.String(flags.OutputFlag.Name) { - case "json": - bytes, err := json.Marshal(data) - if err != nil { - return err - } - - fmt.Println(string(bytes)) - - case "text": - printDBsInfo(data) - } + printDBsInfo(data) + txt := text.Colors{text.BgGreen, text.Bold} + fmt.Println(txt.Sprint("To get detailed info about Erigon node state use 'diag ui' command.")) return nil } func printDBsInfo(data []DBInfo) { - fmt.Print("\n") - fmt.Println("------------------------DBs-------------------") - fmt.Println("DB Name Keys Count Size") - fmt.Println("----------------------------------------------") - for _, db := range data { - nLen := len(db.name) - fmt.Print(db.name) - for i := 0; i < 20-nLen; i++ { - fmt.Print(" ") - } + txt := text.Colors{text.FgBlue, text.Bold} + fmt.Println(txt.Sprint("Databases Info:")) + t := table.NewWriter() + t.SetOutputMirror(os.Stdout) + t.AppendHeader(table.Row{"DB Name", "Keys Count", "Size"}) - fmt.Printf("%d", db.count) - for i := 0; i < 22-len(fmt.Sprint(db.count)); i++ { - fmt.Print(" ") - } - - fmt.Printf("%s\n", db.size) - } - - fmt.Print("\n") - fmt.Print("\n") - - //db := data[0] for _, db := range data { + t.AppendRow(table.Row{db.name, db.count, db.size}) + } - nl := len(db.name) - - dashCount := (60 - nl) / 2 - - for i := 0; i < dashCount; i++ { - fmt.Print("-") - } - - //fmt.Printf(" %s ", db.name) - fmt.Print("\033[1m " + db.name + " \033[0m") + t.AppendSeparator() + t.Render() - for i := 0; i < dashCount; i++ { - fmt.Print("-") - } - fmt.Print("\n") + t.ResetHeaders() + t.AppendHeader(table.Row{"Table Name", "Keys Count", "Size"}) - //fmt.Println("------------------------------------------------------------") - fmt.Println("Table Name Keys Count Size") - for i := 0; i < 60; i++ { - fmt.Print("-") + for _, db := range data { + t.ResetRows() + fmt.Println(txt.Sprint("DB " + db.name + " tables:")) + for _, tbl := range db.tables { + t.AppendRow(table.Row{tbl.Name, tbl.Count, common.ByteCount(tbl.Size)}) } - fmt.Print("\n") - for _, table := range db.tables { - nLen := len(table.Name) - fmt.Printf("%s", table.Name) - for i := 0; i < 34-nLen; i++ { - fmt.Print(" ") - } - - fmt.Printf("%d", table.Count) - for i := 0; i < 22-len(fmt.Sprint(table.Count)); i++ { - fmt.Print(" ") - } - - fmt.Printf("%s\n", common.ByteCount(table.Size)) - } + t.AppendSeparator() + t.Render() fmt.Print("\n") } } -func AllDBsInfo(cliCtx *cli.Context) ([]DBInfo, error) { +func DBsInfo(cliCtx *cli.Context) ([]DBInfo, error) { data := make([]DBInfo, 0) dbsNames, err := getAllDbsNames(cliCtx) @@ -215,12 +146,35 @@ func AllDBsInfo(cliCtx *cli.Context) ([]DBInfo, error) { data = append(data, dbInfo) } + // filter out empty tables + if cliCtx.Bool(DBPopulatedFlag.Name) { + // filter out empty tables + for i := 0; i < len(data); i++ { + tables := data[i].tables + for j := 0; j < len(tables); j++ { + if tables[j].Count == 0 { + tables = append(tables[:j], tables[j+1:]...) + j-- + } + } + data[i].tables = tables + } + + //filter out empty dbs + for i := 0; i < len(data); i++ { + if len(data[i].tables) == 0 { + data = append(data[:i], data[i+1:]...) + i-- + } + } + } + return data, nil } func getAllDbsNames(cliCtx *cli.Context) ([]string, error) { var data []string - url := "http://" + cliCtx.String(flags.DebugURLFlag.Name) + "/debug/dbs" + url := "http://" + cliCtx.String(flags.DebugURLFlag.Name) + flags.ApiPath + "/dbs" err := util.MakeHttpGetCall(cliCtx.Context, url, &data) if err != nil { @@ -232,7 +186,7 @@ func getAllDbsNames(cliCtx *cli.Context) ([]string, error) { func getDb(cliCtx *cli.Context, dbName string) ([]BDTableInfo, error) { var data []BDTableInfo - url := "http://" + cliCtx.String(flags.DebugURLFlag.Name) + "/debug/dbs/" + dbName + "/tables" + url := "http://" + cliCtx.String(flags.DebugURLFlag.Name) + flags.ApiPath + "/dbs/" + dbName + "/tables" err := util.MakeHttpGetCall(cliCtx.Context, url, &data) if err != nil { diff --git a/cmd/diag/downloader/diag_downloader.go b/cmd/diag/downloader/diag_downloader.go index af3350e4b70..4988f719ad7 100644 --- a/cmd/diag/downloader/diag_downloader.go +++ b/cmd/diag/downloader/diag_downloader.go @@ -1,9 +1,11 @@ package downloader import ( - "encoding/json" "fmt" + "time" + "github.com/jedib0t/go-pretty/v6/table" + "github.com/jedib0t/go-pretty/v6/text" "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/diagnostics" "github.com/ledgerwatch/erigon/cmd/diag/flags" @@ -11,67 +13,372 @@ import ( "github.com/urfave/cli/v2" ) +var ( + FileFilterFlag = cli.StringFlag{ + Name: "downloader.file.filter", + Aliases: []string{"dff"}, + Usage: "Filter files list [all|active|inactive|downloaded|queued], dafault value is all", + Required: false, + Value: "all", + } + + FileNameFlag = cli.StringFlag{ + Name: "downloader.file.name", + Aliases: []string{"dfn"}, + Usage: "File name to print details about.", + Required: false, + Value: "", + } +) + var Command = cli.Command{ - Action: print, + Action: printDownloadStatus, Name: "downloader", Aliases: []string{"dl"}, - Usage: "print snapshot download stats", + Usage: "Print snapshot download status", ArgsUsage: "", Flags: []cli.Flag{ &flags.DebugURLFlag, &flags.OutputFlag, }, + Subcommands: []*cli.Command{ + { + Name: "files", + Aliases: []string{"fls"}, + Action: printFiles, + Usage: "Print snapshot download files status", + ArgsUsage: "", + Flags: []cli.Flag{ + &flags.DebugURLFlag, + &flags.OutputFlag, + &FileFilterFlag, + &FileNameFlag, + }, + }, + }, Description: ``, } -func print(cliCtx *cli.Context) error { - var data diagnostics.SyncStatistics - url := "http://" + cliCtx.String(flags.DebugURLFlag.Name) + "/debug/snapshot-sync" +func printDownloadStatus(cliCtx *cli.Context) error { + data, err := getData(cliCtx) - err := util.MakeHttpGetCall(cliCtx.Context, url, &data) + if err != nil { + util.RenderError(err) + return nil + } + + snapshotDownloadStatus := getSnapshotStatusRow(data.SnapshotDownload) + + switch cliCtx.String(flags.OutputFlag.Name) { + case "json": + util.RenderJson(snapshotDownloadStatus) + + case "text": + util.RenderTableWithHeader( + "Snapshot download info:", + table.Row{"Status", "Progress", "Downloaded", "Total", "Time Left", "Total Time", "Download Rate", "Upload Rate", "Peers", "Files", "Connections", "Alloc", "Sys"}, + []table.Row{snapshotDownloadStatus}, + ) + } + + return nil +} + +func printFiles(cliCtx *cli.Context) error { + if cliCtx.String(FileNameFlag.Name) != "" { + return printFile(cliCtx) + } + + data, err := getData(cliCtx) if err != nil { - return err + util.RenderError(err) + return nil } + snapshotDownloadStatus := getSnapshotStatusRow(data.SnapshotDownload) + + snapDownload := data.SnapshotDownload + + files := snapDownload.SegmentsDownloading + rows := []table.Row{} + + for _, file := range files { + rows = append(rows, getFileRow(file)) + } + + filteredRows := filterRows(rows, cliCtx.String(FileFilterFlag.Name)) + switch cliCtx.String(flags.OutputFlag.Name) { case "json": - bytes, err := json.Marshal(data.SnapshotDownload) + util.RenderJson(snapshotDownloadStatus) + util.RenderJson(filteredRows) + case "text": + //Print overall status + util.RenderTableWithHeader( + "Snapshot download info:", + table.Row{"Status", "Progress", "Downloaded", "Total", "Time Left", "Total Time", "Download Rate", "Upload Rate", "Peers", "Files", "Connections", "Alloc", "Sys"}, + []table.Row{snapshotDownloadStatus}, + ) + + //Print files status + util.RenderTableWithHeader( + "Files download info:", + table.Row{"File", "Progress", "Total", "Downloaded", "Peers", "Peers Download Rate", "Webseeds", "Webseeds Download Rate", "Time Left", "Active"}, + filteredRows, + ) + } + + return nil +} + +func printFile(cliCtx *cli.Context) error { + data, err := getData(cliCtx) + + if err != nil { + return err + } + + snapDownload := data.SnapshotDownload + + if file, ok := snapDownload.SegmentsDownloading[cliCtx.String(FileNameFlag.Name)]; ok { + + if file.DownloadedBytes >= file.TotalBytes { + fileRow := getDownloadedFileRow(file) + switch cliCtx.String(flags.OutputFlag.Name) { + case "json": + util.RenderJson(fileRow) + case "text": + //Print file status + util.RenderTableWithHeader( + "File download info:", + table.Row{"File", "Size", "Average Download Rate", "Time Took"}, + []table.Row{fileRow}, + ) + } + } else { + fileRow := getFileRow(file) + filePeers := getPeersRows(file.Peers) + fileWebseeds := getPeersRows(file.Webseeds) - if err != nil { - return err + switch cliCtx.String(flags.OutputFlag.Name) { + case "json": + util.RenderJson(fileRow) + util.RenderJson(filePeers) + util.RenderJson(fileWebseeds) + case "text": + //Print file status + util.RenderTableWithHeader( + "file download info:", + table.Row{"File", "Progress", "Total", "Downloaded", "Peers", "Peers Download Rate", "Webseeds", "Webseeds Download Rate", "Time Left", "Active"}, + []table.Row{fileRow}, + ) + + //Print peers and webseeds status + util.RenderTableWithHeader( + "", + table.Row{"Peer", "Download Rate"}, + filePeers, + ) + + util.RenderTableWithHeader( + "", + table.Row{"Webseed", "Download Rate"}, + fileWebseeds, + ) + } } + } else { + txt := text.Colors{text.FgWhite, text.BgRed} + fmt.Printf("%s %s", txt.Sprint("[ERROR]"), "File with name: "+cliCtx.String(FileNameFlag.Name)+" does not exist.") + } - fmt.Println(string(bytes)) + return nil +} - case "text": - fmt.Println("-------------------Snapshot Download-------------------") - - snapDownload := data.SnapshotDownload - var remainingBytes uint64 - percent := 50 - if snapDownload.Total > snapDownload.Downloaded { - remainingBytes = snapDownload.Total - snapDownload.Downloaded - percent = int((snapDownload.Downloaded*100)/snapDownload.Total) / 2 +func getDownloadedFileRow(file diagnostics.SegmentDownloadStatistics) table.Row { + averageDownloadRate := common.ByteCount(file.DownloadedStats.AverageRate) + "/s" + totalDownloadTimeString := time.Duration(file.DownloadedStats.TimeTook) * time.Second + + row := table.Row{ + file.Name, + common.ByteCount(file.TotalBytes), + averageDownloadRate, + totalDownloadTimeString.String(), + } + + return row +} + +func getSnapshotStatusRow(snapDownload diagnostics.SnapshotDownloadStatistics) table.Row { + status := "Downloading" + if snapDownload.DownloadFinished { + status = "Finished" + } + + downloadedPercent := getPercentDownloaded(snapDownload.Downloaded, snapDownload.Total) + + remainingBytes := snapDownload.Total - snapDownload.Downloaded + downloadTimeLeft := diagnostics.CalculateTime(remainingBytes, snapDownload.DownloadRate) + + totalDownloadTimeString := time.Duration(snapDownload.TotalTime) * time.Second + + rowObj := table.Row{ + status, // Status + downloadedPercent, // Progress + common.ByteCount(snapDownload.Downloaded), // Downloaded + common.ByteCount(snapDownload.Total), // Total + downloadTimeLeft, // Time Left + totalDownloadTimeString.String(), // Total Time + common.ByteCount(snapDownload.DownloadRate) + "/s", // Download Rate + common.ByteCount(snapDownload.UploadRate) + "/s", // Upload Rate + snapDownload.Peers, // Peers + snapDownload.Files, // Files + snapDownload.Connections, // Connections + common.ByteCount(snapDownload.Alloc), // Alloc + common.ByteCount(snapDownload.Sys), // Sys + } + + return rowObj +} + +func getFileRow(file diagnostics.SegmentDownloadStatistics) table.Row { + peersDownloadRate := getFileDownloadRate(file.Peers) + webseedsDownloadRate := getFileDownloadRate(file.Webseeds) + totalDownloadRate := peersDownloadRate + webseedsDownloadRate + downloadedPercent := getPercentDownloaded(file.DownloadedBytes, file.TotalBytes) + remainingBytes := file.TotalBytes - file.DownloadedBytes + downloadTimeLeft := diagnostics.CalculateTime(remainingBytes, totalDownloadRate) + isActive := "false" + if totalDownloadRate > 0 { + isActive = "true" + } + + row := table.Row{ + file.Name, + downloadedPercent, + common.ByteCount(file.TotalBytes), + common.ByteCount(file.DownloadedBytes), + len(file.Peers), + common.ByteCount(peersDownloadRate) + "/s", + len(file.Webseeds), + common.ByteCount(webseedsDownloadRate) + "/s", + downloadTimeLeft, + isActive, + } + + return row +} + +func getPeersRows(peers []diagnostics.SegmentPeer) []table.Row { + rows := make([]table.Row, 0) + + for _, peer := range peers { + row := table.Row{ + peer.Url, + common.ByteCount(peer.DownloadRate) + "/s", } - logstr := "[" + rows = append(rows, row) + } - for i := 1; i < 50; i++ { - if i < percent { - logstr += "#" - } else { - logstr += "." - } + return rows +} + +func getFileDownloadRate(peers []diagnostics.SegmentPeer) uint64 { + var downloadRate uint64 + + for _, peer := range peers { + downloadRate += peer.DownloadRate + } + + return downloadRate +} + +func getData(cliCtx *cli.Context) (diagnostics.SyncStatistics, error) { + var data diagnostics.SyncStatistics + url := "http://" + cliCtx.String(flags.DebugURLFlag.Name) + flags.ApiPath + "/snapshot-sync" + + err := util.MakeHttpGetCall(cliCtx.Context, url, &data) + + if err != nil { + return data, err + } + + return data, nil +} + +func filterRows(rows []table.Row, filter string) []table.Row { + switch filter { + case "all": + return rows + case "active": + return filterActive(rows) + case "inactive": + return filterInactive(rows) + case "downloaded": + return filterDownloaded(rows) + case "queued": + return filterQueued(rows) + } + + return rows +} + +func filterActive(rows []table.Row) []table.Row { + filtered := []table.Row{} + + for _, row := range rows { + if row[len(row)-1] == "true" { + filtered = append(filtered, row) } + } - logstr += "]" + return filtered +} - fmt.Println("Download:", logstr, common.ByteCount(snapDownload.Downloaded), "/", common.ByteCount(snapDownload.Total)) - downloadTimeLeft := util.CalculateTime(remainingBytes, snapDownload.DownloadRate) +func filterInactive(rows []table.Row) []table.Row { + filtered := []table.Row{} - fmt.Println("Time left:", downloadTimeLeft) + for _, row := range rows { + if row[len(row)-1] == "false" { + filtered = append(filtered, row) + } } - return nil + return filtered +} + +func filterDownloaded(rows []table.Row) []table.Row { + filtered := []table.Row{} + + for _, row := range rows { + if row[1] == "100.00%" { + filtered = append(filtered, row) + } + } + + return filtered +} + +func filterQueued(rows []table.Row) []table.Row { + filtered := []table.Row{} + + for _, row := range rows { + if row[1] == "0.00%" { + filtered = append(filtered, row) + } + } + + return filtered +} + +func getPercentDownloaded(downloaded, total uint64) string { + percent := float32(downloaded) / float32(total/100) + + if percent > 100 { + percent = 100 + } + + return fmt.Sprintf("%.2f%%", percent) } diff --git a/cmd/diag/flags/flags.go b/cmd/diag/flags/flags.go index a172bfb3f3e..d4bec07b4e5 100644 --- a/cmd/diag/flags/flags.go +++ b/cmd/diag/flags/flags.go @@ -3,6 +3,8 @@ package flags import "github.com/urfave/cli/v2" var ( + ApiPath = "/debug/diag" + DebugURLFlag = cli.StringFlag{ Name: "debug.addr", Aliases: []string{"da"}, @@ -18,4 +20,20 @@ var ( Required: false, Value: "text", } + + AutoUpdateFlag = cli.BoolFlag{ + Name: "autoupdate", + Aliases: []string{"au"}, + Usage: "Auto update the output", + Required: false, + Value: false, + } + + AutoUpdateIntervalFlag = cli.IntFlag{ + Name: "autoupdate.interval", + Aliases: []string{"aui"}, + Usage: "Auto update interval in milliseconds", + Required: false, + Value: 20000, + } ) diff --git a/cmd/diag/main.go b/cmd/diag/main.go index a6bff652ea0..63ebe6d400b 100644 --- a/cmd/diag/main.go +++ b/cmd/diag/main.go @@ -8,16 +8,17 @@ import ( "path/filepath" "syscall" - "github.com/ledgerwatch/log/v3" "github.com/urfave/cli/v2" "github.com/ledgerwatch/erigon/cmd/diag/db" "github.com/ledgerwatch/erigon/cmd/diag/downloader" "github.com/ledgerwatch/erigon/cmd/diag/stages" + "github.com/ledgerwatch/erigon/cmd/diag/ui" "github.com/ledgerwatch/erigon/cmd/snapshots/sync" "github.com/ledgerwatch/erigon/cmd/utils" "github.com/ledgerwatch/erigon/params" "github.com/ledgerwatch/erigon/turbo/logging" + "github.com/ledgerwatch/log/v3" ) func main() { @@ -33,6 +34,7 @@ func main() { &downloader.Command, &stages.Command, &db.Command, + &ui.Command, } app.Flags = []cli.Flag{} diff --git a/cmd/diag/readme.md b/cmd/diag/readme.md new file mode 100644 index 00000000000..279a0f7258b --- /dev/null +++ b/cmd/diag/readme.md @@ -0,0 +1,96 @@ +# Diagnostics commands + +## Diagnostics command architecture +Diagnostics command is implemented to retrieve information from the Erigon node on a headless machine. It makes it easier to run and get information from the node as it runs on the same machine and doesn't require a PIN-secured remote connection. This is done to make the diagnostics process quicker to set up and make the entire process less resistant. + + +### Diagnostics command local connection diagram +![overview](./_images/local_connection.png) + +[Compare with remote connection](https://github.com/ledgerwatch/diagnostics?tab=readme-ov-file#diagnostics-architecture-diagram) + +## Available commands +| | | +|--|--| +|databases|Displays information about databases. [Details](#databases)| +|downloader|Displays info about the snapshot download process| +|stages|Displays the current status of node synchronization| +|ui|Serves local UI interface to browse through all info collected by diagnostics| +||| + +### Global flags +||||| +|--|--|--|--| +|Flag|Default Value|Allowed Values|Description| +|debug.addr|"localhost:6060"|string|Address of diagnostics endpoint in Erigon node. This endpoint must match the values of `diagnostics.endpoint.addr:diagnostics.endpoint.port`. By default, it is `localhost:6060`.| +|output|`text`|`text`, `json`|Defines the output format for diagnostics data. It can be either `text` or `json`. `text` means that the output will be prettified, `json` means that the output will be in JSON format. By default, the output is in `text` format.| +|help|||Shows details about the command| +||||| + + +### Databases +`./build/bin/diag databases` +#### Available flags: +||||| +|--|--|--|--| +|Flag|Default Value|Allowed Values|Description| +|db.appearance.populated|false|boolean|Print only for populated tables content.| +|db.name|""|string|DB name to print info about. If not set, all DBs will be printed.| +|||| + +Examples: +- `./build/bin/diag databases` +![img](./_images/dbs/example_databases.png) +- `./build/bin/diag databases --db.name=caplin/indexing/beacon_indicies` +![img](./_images/dbs/example_databases_name.png) +- `./build/bin/diag databases --db.name=caplin/indexing/beacon_indicies --db.appearance.populated` +![img](./_images/dbs/example_databases_name_populated.png) + +### Downloader +`./build/bin/diag downloader` +Display Snapshot download status + +#### Available subcommands: +||| +|--|--| +|files|Displays status for each file along with overall download status| +||| + +#### Available flags: +||||| +|--|--|--|--| +|Flag|Default Value|Allowed Values|Description| +|downloader.file.filter|`all`|`all`, `active`, `inactive`, `downloaded`, `queued`|Filter files to display.| +|downloader.file.name|""|string|File name to print details about.| +|||| + +Examples: +- `./build/bin/diag downloader` +![img](./_images/downloader/example_downloader.png) +- `./build/bin/diag downloader files` +![img](./_images/downloader/example_downloader_files.png) +- `./build/bin/diag downloader files --downloader.file.filter=downloaded` +![img](./_images/downloader/example_downloader_files_downloaded.png) +- `./build/bin/diag downloader files --downloader.file.name=v1-005400-005500-transactions.seg` +![img](./_images/downloader/example_downloader_files_filename.png) + +### Stages +`./build/bin/diag stages current` +Display node synchronization status + +Example output: + +![img](./_images/stages/example_stages.png) + +### UI +`./build/bin/diag ui` +Serve diagnostics ui locally + +#### Available flags: +||||| +|--|--|--|--| +|Flag|Default Value|Allowed Values|Description| +|ui.addr|`127.0.0.1:6060`|string|URL to serve UI web application.| +|||| + +After running this command, it enables you to navigate through all available diagnostics data using a web application. You can see what is currently [available](https://github.com/ledgerwatch/diagnostics?tab=readme-ov-file#currently-implemented-diagnostics). This command allows you to skip the session setup to connect to your node as it automatically connects to a running node. \ No newline at end of file diff --git a/cmd/diag/stages/stages.go b/cmd/diag/stages/stages.go index 9837de2f041..a2b74468b09 100644 --- a/cmd/diag/stages/stages.go +++ b/cmd/diag/stages/stages.go @@ -1,9 +1,10 @@ package stages import ( - "encoding/json" - "fmt" + "sync" + "time" + "github.com/jedib0t/go-pretty/v6/table" "github.com/ledgerwatch/erigon-lib/diagnostics" "github.com/ledgerwatch/erigon/cmd/diag/flags" "github.com/ledgerwatch/erigon/cmd/diag/util" @@ -14,6 +15,13 @@ var Command = cli.Command{ Name: "stages", Aliases: []string{"st"}, ArgsUsage: "", + Action: printStages, + Flags: []cli.Flag{ + &flags.DebugURLFlag, + &flags.OutputFlag, + &flags.AutoUpdateFlag, + &flags.AutoUpdateIntervalFlag, + }, Subcommands: []*cli.Command{ { Name: "current", @@ -24,43 +32,170 @@ var Command = cli.Command{ Flags: []cli.Flag{ &flags.DebugURLFlag, &flags.OutputFlag, + &flags.AutoUpdateFlag, + &flags.AutoUpdateIntervalFlag, }, }, }, Description: ``, } +func printStages(cliCtx *cli.Context) error { + err := printSyncStages(cliCtx, false) + if err != nil { + util.RenderError(err) + } + + return nil +} + func printCurentStage(cliCtx *cli.Context) error { - var data diagnostics.SyncStatistics - url := "http://" + cliCtx.String(flags.DebugURLFlag.Name) + "/debug/snapshot-sync" + err := printSyncStages(cliCtx, true) + if err != nil { + util.RenderError(err) + } + + return nil +} + +func printSyncStages(cliCtx *cli.Context, isCurrent bool) error { + autoupdate := cliCtx.Bool(flags.AutoUpdateFlag.Name) + + syncStages, err := querySyncInfo(cliCtx) + if err != nil { + util.RenderError(err) + return nil + } else { + var stagesRows []table.Row + if isCurrent { + stagesRows = getCurrentStageRow(syncStages) + } else { + stagesRows = getStagesRows(syncStages) + } + printData(cliCtx, stagesRows) + } + + if autoupdate { + interval := time.Duration(cliCtx.Int(flags.AutoUpdateIntervalFlag.Name)) * time.Millisecond + var wg sync.WaitGroup + wg.Add(1) + + ticker := time.NewTicker(interval) + go func() { + for { + select { + case <-ticker.C: + syncStages, err := querySyncInfo(cliCtx) + if err == nil { + var stagesRows []table.Row + if isCurrent { + stagesRows = getCurrentStageRow(syncStages) + } else { + stagesRows = getStagesRows(syncStages) + } + + printData(cliCtx, stagesRows) + } else { + util.RenderError(err) + wg.Done() + return + } + + case <-cliCtx.Done(): + ticker.Stop() + wg.Done() + return + } + } + }() + + wg.Wait() + } + + return nil +} + +func querySyncInfo(cliCtx *cli.Context) ([]diagnostics.SyncStage, error) { + var data []diagnostics.SyncStage + url := "http://" + cliCtx.String(flags.DebugURLFlag.Name) + flags.ApiPath + "/sync-stages" err := util.MakeHttpGetCall(cliCtx.Context, url, &data) if err != nil { - return err + return nil, err } + return data, nil +} + +func printData(cliCtx *cli.Context, data []table.Row) { switch cliCtx.String(flags.OutputFlag.Name) { case "json": - bytes, err := json.Marshal(data.SyncStages.StagesList) - if err != nil { - return err - } - - fmt.Println(string(bytes)) + util.RenderJson(data) case "text": - fmt.Println("-------------------Stages-------------------") - - for idx, stage := range data.SyncStages.StagesList { - if idx == int(data.SyncStages.CurrentStage) { - fmt.Println("[" + stage + "]" + " - Running") - } else if idx < int(data.SyncStages.CurrentStage) { - fmt.Println("[" + stage + "]" + " - Completed") - } else { - fmt.Println("[" + stage + "]" + " - Queued") + util.RenderTableWithHeader( + "", + table.Row{"Stage", "SubStage", "Status", "Time Elapsed", "Progress"}, + data, + ) + } +} + +func getStagesRows(stages []diagnostics.SyncStage) []table.Row { + return createSyncStageRows(stages, false) +} + +func getCurrentStageRow(stages []diagnostics.SyncStage) []table.Row { + return createSyncStageRows(stages, true) +} + +func createSyncStageRows(stages []diagnostics.SyncStage, forCurrentStage bool) []table.Row { + rows := []table.Row{} + for _, stage := range stages { + + if forCurrentStage { + if stage.State != diagnostics.Running { + continue } } + + stageRow := createStageRowFromStage(stage) + rows = append(rows, stageRow) + + for _, substage := range stage.SubStages { + subStageRow := createSubStageRowFromSubstageStage(substage) + rows = append(rows, subStageRow) + } + + if len(stage.SubStages) != 0 { + rows = append(rows, table.Row{"", "", "", "", ""}) + } + + if forCurrentStage { + break + } + } - return nil + return rows +} + +func createStageRowFromStage(stage diagnostics.SyncStage) table.Row { + return table.Row{ + stage.ID, + "", + stage.State.String(), + stage.Stats.TimeElapsed, + stage.Stats.Progress, + } +} + +func createSubStageRowFromSubstageStage(substage diagnostics.SyncSubStage) table.Row { + return table.Row{ + "", + substage.ID, + substage.State.String(), + substage.Stats.TimeElapsed, + substage.Stats.Progress, + } } diff --git a/cmd/diag/ui/ui.go b/cmd/diag/ui/ui.go new file mode 100644 index 00000000000..1620747b5d9 --- /dev/null +++ b/cmd/diag/ui/ui.go @@ -0,0 +1,137 @@ +package ui + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + "sync" + "time" + + "github.com/ledgerwatch/erigonwatch" + + "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" + "github.com/go-chi/cors" + "github.com/jedib0t/go-pretty/v6/text" + "github.com/ledgerwatch/erigon/cmd/diag/flags" + "github.com/urfave/cli/v2" +) + +var ( + UIURLFlag = cli.StringFlag{ + Name: "ui.addr", + Usage: "URL to serve UI web application", + Required: false, + Value: "127.0.0.1:6060", + } +) + +var Command = cli.Command{ + Name: "ui", + Action: runUI, + Aliases: []string{"u"}, + Usage: "run local ui", + ArgsUsage: "", + Flags: []cli.Flag{ + &flags.DebugURLFlag, + &UIURLFlag, + }, + Description: ``, +} + +func runUI(cli *cli.Context) error { + supportedSubpaths := []string{ + "sentry-network", + "sentinel-network", + "downloader", + "logs", + "chain", + "data", + "debug", + "testing", + "performance", + "documentation", + "issues", + "admin", + } + + listenUrl := cli.String(UIURLFlag.Name) + + assets, _ := erigonwatch.UIFiles() + fs := http.FileServer(http.FS(assets)) + + r := chi.NewRouter() + r.Use(middleware.Logger) + r.Use(middleware.Recoverer) + r.Use(middleware.RouteHeaders(). + Route("Origin", "*", cors.Handler(cors.Options{ + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"}, + AllowedHeaders: []string{"Accept", "Content-Type", "session-id"}, + AllowCredentials: false, // <----------<<< do not allow credentials + })). + Handler) + + r.Mount("/", fs) + + for _, subpath := range supportedSubpaths { + addhandler(r, "/"+subpath, fs) + } + + // Use the file system to serve static files + url := "http://" + cli.String(flags.DebugURLFlag.Name) + addr := DiagAddress{ + Address: url, + } + + //r.Get("/diagaddr", writeDiagAdderss(addr)) + r.Handle("/data", http.StripPrefix("/data", fs)) + + r.HandleFunc("/diagaddr", func(w http.ResponseWriter, r *http.Request) { + writeDiagAdderss(w, addr) + }) + + srv := &http.Server{ + Addr: listenUrl, + Handler: r, + MaxHeaderBytes: 1 << 20, + ReadHeaderTimeout: 1 * time.Minute, + } + + var wg sync.WaitGroup + wg.Add(1) + + go func() { + defer wg.Done() // Signal that the goroutine has completed + err := srv.ListenAndServe() + + if err != nil { + log.Fatal(err) + } + }() + + uiUrl := fmt.Sprintf("http://%s", listenUrl) + fmt.Println(text.Hyperlink(uiUrl, fmt.Sprintf("UI running on %s", uiUrl))) + + wg.Wait() // Wait for the server goroutine to finish + return nil +} + +func addhandler(r *chi.Mux, path string, handler http.Handler) { + r.Handle(path, http.StripPrefix(path, handler)) +} + +type DiagAddress struct { + Address string `json:"address"` +} + +func writeDiagAdderss(w http.ResponseWriter, addr DiagAddress) { + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Content-Type", "application/json") + + if err := json.NewEncoder(w).Encode(addr); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + +} diff --git a/cmd/diag/util/util.go b/cmd/diag/util/util.go index f6c9e6184e2..43cda2761a7 100644 --- a/cmd/diag/util/util.go +++ b/cmd/diag/util/util.go @@ -6,7 +6,12 @@ import ( "fmt" "io" "net/http" + "os" + "strings" "time" + + "github.com/jedib0t/go-pretty/v6/table" + "github.com/jedib0t/go-pretty/v6/text" ) func MakeHttpGetCall(ctx context.Context, url string, data interface{}) error { @@ -21,6 +26,9 @@ func MakeHttpGetCall(ctx context.Context, url string, data interface{}) error { resp, err := client.Do(req) if err != nil { + if strings.Contains(err.Error(), "connection refused") { + return fmt.Errorf("it looks like the Erigon node is not running, is running incorrectly, or you have specified the wrong diagnostics URL. If you run the Erigon node with the '--diagnostics.endpoint.addr' or '--diagnostics.endpoint.port' flags, you must also specify the '--debug.addr' flag with the same address and port") + } return err } @@ -32,20 +40,58 @@ func MakeHttpGetCall(ctx context.Context, url string, data interface{}) error { err = json.Unmarshal(body, &data) if err != nil { + if err.Error() == "invalid character 'p' after top-level value" { + return fmt.Errorf("diagnostics was not initialized yet. Please try again in a few seconds") + } + return err } return nil } -func CalculateTime(amountLeft, rate uint64) string { - if rate == 0 { - return "999hrs:99m" +func RenderJson(data interface{}) { + bytes, err := json.Marshal(data) + + if err == nil { + fmt.Println(string(bytes)) + fmt.Print("\n") + } +} + +func RenderTableWithHeader(title string, header table.Row, rows []table.Row) { + if title != "" { + txt := text.Colors{text.FgBlue, text.Bold} + fmt.Println(txt.Sprint(title)) + + if len(rows) == 0 { + txt := text.Colors{text.FgRed, text.Bold} + fmt.Println(txt.Sprint("No data to show")) + } } - timeLeftInSeconds := amountLeft / rate - hours := timeLeftInSeconds / 3600 - minutes := (timeLeftInSeconds / 60) % 60 + if len(rows) > 0 { + t := table.NewWriter() + t.SetOutputMirror(os.Stdout) + + t.AppendHeader(header) + if len(rows) > 0 { + t.AppendRows(rows) + } + + t.AppendSeparator() + t.Render() + } + + fmt.Print("\n") +} + +func RenderUseDiagUI() { + txt := text.Colors{text.BgGreen, text.Bold} + fmt.Println(txt.Sprint("To get detailed info about Erigon node state use 'diag ui' command.")) +} - return fmt.Sprintf("%dhrs:%dm", hours, minutes) +func RenderError(err error) { + txt := text.Colors{text.FgWhite, text.BgRed} + fmt.Printf("%s %s\n", txt.Sprint("[ERROR]"), err) } diff --git a/cmd/rpcdaemon/cli/httpcfg/http_cfg.go b/cmd/rpcdaemon/cli/httpcfg/http_cfg.go index 37d15eba9c9..7ff328c543c 100644 --- a/cmd/rpcdaemon/cli/httpcfg/http_cfg.go +++ b/cmd/rpcdaemon/cli/httpcfg/http_cfg.go @@ -42,6 +42,7 @@ type HttpCfg struct { API []string Gascap uint64 + Feecap float64 MaxTraces uint64 WebsocketPort int WebsocketEnabled bool diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 1511f1c2cca..a8a6be14e9b 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -2061,7 +2061,8 @@ func SetEthConfig(ctx *cli.Context, nodeConfig *nodecfg.Config, cfg *ethconfig.C if known, ok := snapcfg.KnownWebseeds[chain]; ok { webseedsList = append(webseedsList, known...) } - cfg.Downloader, err = downloadercfg2.New(cfg.Dirs, version, lvl, downloadRate, uploadRate, ctx.Int(TorrentPortFlag.Name), ctx.Int(TorrentConnsPerFileFlag.Name), ctx.Int(TorrentDownloadSlotsFlag.Name), ctx.StringSlice(TorrentDownloadSlotsFlag.Name), webseedsList, chain, true) + cfg.Downloader, err = downloadercfg2.New(cfg.Dirs, version, lvl, downloadRate, uploadRate, ctx.Int(TorrentPortFlag.Name), ctx.Int(TorrentConnsPerFileFlag.Name), ctx.Int(TorrentDownloadSlotsFlag.Name), libcommon.CliString2Array(ctx.String(TorrentStaticPeersFlag.Name)), webseedsList, chain, true) + if err != nil { panic(err) } diff --git a/core/state_transition.go b/core/state_transition.go index 2ec0d3e59f2..c2c63ff9343 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -322,7 +322,8 @@ func (st *StateTransition) preCheck(gasBailout bool) error { // Make sure the transaction gasFeeCap is greater than the block's baseFee. if st.evm.ChainRules().IsLondon { // Skip the checks if gas fields are zero and baseFee was explicitly disabled (eth_call) - if !st.evm.Config().NoBaseFee || !st.gasFeeCap.IsZero() || !st.tip.IsZero() { + skipCheck := st.evm.Config().NoBaseFee && st.gasFeeCap.IsZero() && st.tip.IsZero() + if !skipCheck { if err := CheckEip1559TxGasFeeCap(st.msg.From(), st.gasFeeCap, st.tip, st.evm.Context.BaseFee, st.msg.IsFree()); err != nil { return err } @@ -337,7 +338,7 @@ func (st *StateTransition) preCheck(gasBailout bool) error { return err } maxFeePerBlobGas := st.msg.MaxFeePerBlobGas() - if blobGasPrice.Cmp(maxFeePerBlobGas) > 0 { + if !st.evm.Config().NoBaseFee && blobGasPrice.Cmp(maxFeePerBlobGas) > 0 { return fmt.Errorf("%w: address %v, maxFeePerBlobGas: %v blobGasPrice: %v, excessBlobGas: %v", ErrMaxFeePerBlobGas, st.msg.From().Hex(), st.msg.MaxFeePerBlobGas(), blobGasPrice, st.evm.Context.ExcessBlobGas) diff --git a/core/vm/evm.go b/core/vm/evm.go index f1a392ba1b6..f95006dad5e 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -101,6 +101,12 @@ type EVM struct { // NewEVM returns a new EVM. The returned EVM is not thread safe and should // only ever be used *once*. func NewEVM(blockCtx evmtypes.BlockContext, txCtx evmtypes.TxContext, state evmtypes.IntraBlockState, chainConfig *chain.Config, vmConfig Config) *EVM { + if vmConfig.NoBaseFee { + if txCtx.GasPrice.IsZero() { + blockCtx.BaseFee = new(uint256.Int) + } + } + evm := &EVM{ Context: blockCtx, TxContext: txCtx, diff --git a/diagnostics/mem.go b/diagnostics/mem.go index 1ad34adea56..592b7fdbdcc 100644 --- a/diagnostics/mem.go +++ b/diagnostics/mem.go @@ -20,8 +20,8 @@ func SetupMemAccess(metricsMux *http.ServeMux) { } func writeMem(w http.ResponseWriter) { - memStats, err := mem.ReadVirtualMemStats() - if err != nil { + memStats, err := mem.ReadVirtualMemStats() //nolint + if err != nil { //nolint http.Error(w, err.Error(), http.StatusInternalServerError) return } diff --git a/diagnostics/setup.go b/diagnostics/setup.go index 6534ecac245..7eb9363a3c6 100644 --- a/diagnostics/setup.go +++ b/diagnostics/setup.go @@ -50,11 +50,13 @@ func Setup(ctx *cli.Context, node *node.ErigonNode, metricsMux *http.ServeMux, p } speedTest := ctx.Bool(diagnoticsSpeedTestFlag) - - diagnostic := diaglib.NewDiagnosticClient(diagMux, node.Backend().DataDir(), speedTest) - diagnostic.Setup() - - SetupEndpoints(ctx, node, diagMux, diagnostic) + diagnostic, err := diaglib.NewDiagnosticClient(ctx.Context, diagMux, node.Backend().DataDir(), speedTest) + if err == nil { + diagnostic.Setup() + SetupEndpoints(ctx, node, diagMux, diagnostic) + } else { + log.Error("[Diagnostics] Failure in setting up diagnostics", "err", err) + } } func SetupDiagnosticsEndpoint(metricsMux *http.ServeMux, addres string) *http.ServeMux { diff --git a/diagnostics/snapshot_sync.go b/diagnostics/snapshot_sync.go index 9100977ab5b..265f8496124 100644 --- a/diagnostics/snapshot_sync.go +++ b/diagnostics/snapshot_sync.go @@ -41,6 +41,12 @@ func SetupStagesAccess(metricsMux *http.ServeMux, diag *diaglib.DiagnosticClient w.Header().Set("Content-Type", "application/json") writeNetworkSpeed(w, diag) }) + + metricsMux.HandleFunc("/sync-stages", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Content-Type", "application/json") + writeSyncStages(w, diag) + }) } func writeNetworkSpeed(w http.ResponseWriter, diag *diaglib.DiagnosticClient) { @@ -62,3 +68,7 @@ func writeFilesList(w http.ResponseWriter, diag *diaglib.DiagnosticClient) { func writeHardwareInfo(w http.ResponseWriter, diag *diaglib.DiagnosticClient) { json.NewEncoder(w).Encode(diag.HardwareInfo()) } + +func writeSyncStages(w http.ResponseWriter, diag *diaglib.DiagnosticClient) { + json.NewEncoder(w).Encode(diag.GetSyncStages()) +} diff --git a/erigon-lib/common/dir/rw_dir.go b/erigon-lib/common/dir/rw_dir.go index c9d0a1b6c16..6e07f6665b1 100644 --- a/erigon-lib/common/dir/rw_dir.go +++ b/erigon-lib/common/dir/rw_dir.go @@ -19,6 +19,7 @@ package dir import ( "os" "path/filepath" + "strings" "golang.org/x/sync/errgroup" ) @@ -133,6 +134,9 @@ func ListFiles(dir string, extensions ...string) (paths []string, err error) { if f.IsDir() && !f.Type().IsRegular() { continue } + if strings.HasPrefix(f.Name(), ".") { + continue + } match := false if len(extensions) == 0 { match = true diff --git a/erigon-lib/diagnostics/block_execution.go b/erigon-lib/diagnostics/block_execution.go index 0944ddb2ec8..fcb448376a8 100644 --- a/erigon-lib/diagnostics/block_execution.go +++ b/erigon-lib/diagnostics/block_execution.go @@ -25,7 +25,7 @@ func (d *DiagnosticClient) runBlockExecutionListener(rootCtx context.Context) { d.syncStats.BlockExecution = info d.mu.Unlock() - if int(d.syncStats.SyncStages.CurrentStage) >= len(d.syncStats.SyncStages.StagesList) { + if d.syncStats.SyncFinished { return } } diff --git a/erigon-lib/diagnostics/client.go b/erigon-lib/diagnostics/client.go index df5b04a76fc..ecadc345126 100644 --- a/erigon-lib/diagnostics/client.go +++ b/erigon-lib/diagnostics/client.go @@ -1,17 +1,28 @@ package diagnostics import ( + "context" "net/http" + "path/filepath" "sync" + "github.com/c2h5oh/datasize" + "golang.org/x/sync/semaphore" + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/mdbx" + "github.com/ledgerwatch/log/v3" ) type DiagnosticClient struct { + ctx context.Context + db kv.RwDB metricsMux *http.ServeMux dataDirPath string speedTest bool + syncStages []SyncStage syncStats SyncStatistics snapshotFileList SnapshoFilesList mu sync.Mutex @@ -27,20 +38,54 @@ type DiagnosticClient struct { networkSpeedMutex sync.Mutex } -func NewDiagnosticClient(metricsMux *http.ServeMux, dataDirPath string, speedTest bool) *DiagnosticClient { +func NewDiagnosticClient(ctx context.Context, metricsMux *http.ServeMux, dataDirPath string, speedTest bool) (*DiagnosticClient, error) { + dirPath := filepath.Join(dataDirPath, "diagnostics") + db, err := createDb(ctx, dirPath) + if err != nil { + return nil, err + } + + hInfo := ReadSysInfo(db) + ss := ReadSyncStages(db) + snpdwl := ReadSnapshotDownloadInfo(db) + snpidx := ReadSnapshotIndexingInfo(db) + return &DiagnosticClient{ - metricsMux: metricsMux, - dataDirPath: dataDirPath, - speedTest: speedTest, - syncStats: SyncStatistics{}, - hardwareInfo: HardwareInfo{}, + ctx: ctx, + db: db, + metricsMux: metricsMux, + dataDirPath: dataDirPath, + speedTest: speedTest, + syncStages: ss, + syncStats: SyncStatistics{ + SnapshotDownload: snpdwl, + SnapshotIndexing: snpidx, + }, + hardwareInfo: hInfo, snapshotFileList: SnapshoFilesList{}, bodies: BodiesInfo{}, resourcesUsage: ResourcesUsage{ MemoryUsage: []MemoryStats{}, }, peersStats: NewPeerStats(1000), // 1000 is the limit of peers; TODO: make it configurable through a flag + }, nil +} + +func createDb(ctx context.Context, dbDir string) (db kv.RwDB, err error) { + db, err = mdbx.NewMDBX(log.New()). + Label(kv.DiagnosticsDB). + WithTableCfg(func(defaultBuckets kv.TableCfg) kv.TableCfg { return kv.DiagnosticsTablesCfg }). + GrowthStep(4 * datasize.MB). + MapSize(16 * datasize.GB). + PageSize(uint64(4 * datasize.KB)). + RoTxsLimiter(semaphore.NewWeighted(9_000)). + Path(dbDir). + Open(ctx) + if err != nil { + return nil, err } + + return db, nil } func (d *DiagnosticClient) Setup() { diff --git a/erigon-lib/diagnostics/entities.go b/erigon-lib/diagnostics/entities.go index e7e75c91b12..5eaea58f081 100644 --- a/erigon-lib/diagnostics/entities.go +++ b/erigon-lib/diagnostics/entities.go @@ -20,9 +20,22 @@ import ( "time" ) -type PeerStatisticsGetter interface { - GetPeersStatistics() map[string]*PeerStatistics -} +type SyncStageType string + +const ( + Snapshots SyncStageType = "Snapshots" + BlockHashes SyncStageType = "BlockHashes" + Senders SyncStageType = "Senders" + Execution SyncStageType = "Execution" + HashState SyncStageType = "HashState" + IntermediateHashes SyncStageType = "IntermediateHashes" + CallTraces SyncStageType = "CallTraces" + AccountHistoryIndex SyncStageType = "AccountHistoryIndex" + StorageHistoryIndex SyncStageType = "StorageHistoryIndex" + LogIndex SyncStageType = "LogIndex" + TxLookup SyncStageType = "TxLookup" + Finish SyncStageType = "Finish" +) type PeerStatistics struct { PeerType string @@ -54,10 +67,10 @@ type PeerStatisticMsgUpdate struct { } type SyncStatistics struct { - SyncStages SyncStages `json:"syncStages"` SnapshotDownload SnapshotDownloadStatistics `json:"snapshotDownload"` SnapshotIndexing SnapshotIndexingStatistics `json:"snapshotIndexing"` BlockExecution BlockExecutionStatistics `json:"blockExecution"` + SyncFinished bool `json:"syncFinished"` } type SnapshotDownloadStatistics struct { @@ -77,11 +90,23 @@ type SnapshotDownloadStatistics struct { } type SegmentDownloadStatistics struct { - Name string `json:"name"` - TotalBytes uint64 `json:"totalBytes"` - DownloadedBytes uint64 `json:"downloadedBytes"` - Webseeds []SegmentPeer `json:"webseeds"` - Peers []SegmentPeer `json:"peers"` + Name string `json:"name"` + TotalBytes uint64 `json:"totalBytes"` + DownloadedBytes uint64 `json:"downloadedBytes"` + Webseeds []SegmentPeer `json:"webseeds"` + Peers []SegmentPeer `json:"peers"` + DownloadedStats FileDownloadedStatistics `json:"downloadedStats"` +} + +type FileDownloadedStatistics struct { + TimeTook float64 `json:"timeTook"` + AverageRate uint64 `json:"averageRate"` +} + +type FileDownloadedStatisticsUpdate struct { + FileName string `json:"fileName"` + TimeTook float64 `json:"timeTook"` + AverageRate uint64 `json:"averageRate"` } type SegmentPeer struct { @@ -105,19 +130,6 @@ type SnapshotSegmentIndexingFinishedUpdate struct { SegmentName string `json:"segmentName"` } -type SyncStagesList struct { - Stages []string `json:"stages"` -} - -type CurrentSyncStage struct { - Stage uint `json:"stage"` -} - -type SyncStages struct { - StagesList []string `json:"stagesList"` - CurrentStage uint `json:"currentStage"` -} - type BlockExecutionStatistics struct { From uint64 `json:"from"` To uint64 `json:"to"` @@ -237,14 +249,19 @@ type MemoryStats struct { Alloc uint64 `json:"alloc"` Sys uint64 `json:"sys"` OtherFields []interface{} - Timestamp time.Time `json:"timestamp"` - StageIndex int `json:"stageIndex"` + Timestamp time.Time `json:"timestamp"` + StageIndex CurrentSyncStagesIdxs `json:"stageIndex"` } type NetworkSpeedTestResult struct { Latency time.Duration `json:"latency"` DownloadSpeed float64 `json:"downloadSpeed"` UploadSpeed float64 `json:"uploadSpeed"` + PacketLoss float64 `json:"packetLoss"` +} + +func (ti FileDownloadedStatisticsUpdate) Type() Type { + return TypeOf(ti) } func (ti MemoryStats) Type() Type { @@ -295,14 +312,6 @@ func (ti SnapshotSegmentIndexingFinishedUpdate) Type() Type { return TypeOf(ti) } -func (ti SyncStagesList) Type() Type { - return TypeOf(ti) -} - -func (ti CurrentSyncStage) Type() Type { - return TypeOf(ti) -} - func (ti PeerStatisticMsgUpdate) Type() Type { return TypeOf(ti) } diff --git a/erigon-lib/diagnostics/provider.go b/erigon-lib/diagnostics/provider.go index cfbc362cd1d..fe27c22f5d4 100644 --- a/erigon-lib/diagnostics/provider.go +++ b/erigon-lib/diagnostics/provider.go @@ -80,30 +80,6 @@ type registry struct { var providers = map[Type]*registry{} var providerMutex sync.RWMutex -func RegisterProvider(provider Provider, infoType Type, logger log.Logger) { - providerMutex.Lock() - defer providerMutex.Unlock() - - reg := providers[infoType] - - if reg != nil { - for _, p := range reg.providers { - if p == provider { - return - } - } - } else { - reg = ®istry{} - providers[infoType] = reg - } - - reg.providers = append(reg.providers, provider) - - if reg.context != nil { - go startProvider(reg.context, infoType, provider, logger) - } -} - func StartProviders(ctx context.Context, infoType Type, logger log.Logger) { providerMutex.Lock() diff --git a/erigon-lib/diagnostics/provider_test.go b/erigon-lib/diagnostics/provider_test.go index b5f2fefc7f4..5329e7de4ea 100644 --- a/erigon-lib/diagnostics/provider_test.go +++ b/erigon-lib/diagnostics/provider_test.go @@ -17,10 +17,7 @@ func (ti testInfo) Type() diagnostics.Type { return diagnostics.TypeOf(ti) } -type testProvider struct { -} - -func (t *testProvider) StartDiagnostics(ctx context.Context) error { +func StartDiagnostics(ctx context.Context) error { timer := time.NewTicker(1 * time.Second) defer timer.Stop() @@ -39,64 +36,11 @@ func (t *testProvider) StartDiagnostics(ctx context.Context) error { func TestProviderRegistration(t *testing.T) { - // diagnostics provider - provider := &testProvider{} - diagnostics.RegisterProvider(provider, diagnostics.TypeOf(testInfo{}), log.Root()) - - // diagnostics receiver - ctx, ch, cancel := diagnostics.Context[testInfo](context.Background(), 1) - diagnostics.StartProviders(ctx, diagnostics.TypeOf(testInfo{}), log.Root()) - - for info := range ch { - if info.count == 3 { - cancel() - } - } -} - -func TestDelayedProviderRegistration(t *testing.T) { - - time.AfterFunc(1*time.Second, func() { - // diagnostics provider - provider := &testProvider{} - diagnostics.RegisterProvider(provider, diagnostics.TypeOf(testInfo{}), log.Root()) - }) - // diagnostics receiver ctx, ch, cancel := diagnostics.Context[testInfo](context.Background(), 1) diagnostics.StartProviders(ctx, diagnostics.TypeOf(testInfo{}), log.Root()) - for info := range ch { - if info.count == 3 { - cancel() - } - } -} - -func TestProviderFuncRegistration(t *testing.T) { - - // diagnostics provider - diagnostics.RegisterProvider(diagnostics.ProviderFunc(func(ctx context.Context) error { - timer := time.NewTicker(1 * time.Second) - defer timer.Stop() - - var count int - - for { - select { - case <-ctx.Done(): - return nil - case <-timer.C: - diagnostics.Send(testInfo{count}) - count++ - } - } - }), diagnostics.TypeOf(testInfo{}), log.Root()) - - // diagnostics receiver - ctx, ch, cancel := diagnostics.Context[testInfo](context.Background(), 1) - - diagnostics.StartProviders(ctx, diagnostics.TypeOf(testInfo{}), log.Root()) + go StartDiagnostics(ctx) for info := range ch { if info.count == 3 { diff --git a/erigon-lib/diagnostics/resources_usage.go b/erigon-lib/diagnostics/resources_usage.go index 9700f728e67..36064ce44b8 100644 --- a/erigon-lib/diagnostics/resources_usage.go +++ b/erigon-lib/diagnostics/resources_usage.go @@ -31,7 +31,7 @@ func (d *DiagnosticClient) runMemoryStatsListener(rootCtx context.Context) { return case info := <-ch: d.resourcesUsageMutex.Lock() - info.StageIndex = int(d.syncStats.SyncStages.CurrentStage) + info.StageIndex = d.getCurrentSyncIdxs() d.resourcesUsage.MemoryUsage = append(d.resourcesUsage.MemoryUsage, info) d.resourcesUsageMutex.Unlock() } diff --git a/erigon-lib/diagnostics/snapshots.go b/erigon-lib/diagnostics/snapshots.go index 25f636c8d29..f042510c6d4 100644 --- a/erigon-lib/diagnostics/snapshots.go +++ b/erigon-lib/diagnostics/snapshots.go @@ -2,16 +2,26 @@ package diagnostics import ( "context" + "encoding/json" + "fmt" + "time" + "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/log/v3" ) +var ( + SnapshotDownloadStatisticsKey = []byte("diagSnapshotDownloadStatistics") + SnapshotIndexingStatisticsKey = []byte("diagSnapshotIndexingStatistics") +) + func (d *DiagnosticClient) setupSnapshotDiagnostics(rootCtx context.Context) { d.runSnapshotListener(rootCtx) d.runSegmentDownloadingListener(rootCtx) d.runSegmentIndexingListener(rootCtx) d.runSegmentIndexingFinishedListener(rootCtx) d.runSnapshotFilesListListener(rootCtx) + d.runFileDownloadedListener(rootCtx) } func (d *DiagnosticClient) runSnapshotListener(rootCtx context.Context) { @@ -25,6 +35,7 @@ func (d *DiagnosticClient) runSnapshotListener(rootCtx context.Context) { case <-rootCtx.Done(): return case info := <-ch: + d.mu.Lock() d.syncStats.SnapshotDownload.Downloaded = info.Downloaded d.syncStats.SnapshotDownload.Total = info.Total @@ -38,17 +49,63 @@ func (d *DiagnosticClient) runSnapshotListener(rootCtx context.Context) { d.syncStats.SnapshotDownload.Sys = info.Sys d.syncStats.SnapshotDownload.DownloadFinished = info.DownloadFinished d.syncStats.SnapshotDownload.TorrentMetadataReady = info.TorrentMetadataReady + + downloadedPercent := getPercentDownloaded(info.Downloaded, info.Total) + remainingBytes := info.Total - info.Downloaded + downloadTimeLeft := CalculateTime(remainingBytes, info.DownloadRate) + totalDownloadTimeString := time.Duration(info.TotalTime) * time.Second + + d.updateSnapshotStageStats(SyncStageStats{ + TimeElapsed: totalDownloadTimeString.String(), + TimeLeft: downloadTimeLeft, + Progress: downloadedPercent, + }, "Downloading snapshots") + + if err := d.db.Update(d.ctx, SnapshotDownloadUpdater(d.syncStats.SnapshotDownload)); err != nil { + log.Error("[Diagnostics] Failed to update snapshot download info", "err", err) + } + + d.saveSyncStagesToDB() + d.mu.Unlock() - if info.DownloadFinished { + if d.snapshotStageFinished() { return } } } - }() } +func getPercentDownloaded(downloaded, total uint64) string { + percent := float32(downloaded) / float32(total/100) + + if percent > 100 { + percent = 100 + } + + return fmt.Sprintf("%.2f%%", percent) +} + +func (d *DiagnosticClient) updateSnapshotStageStats(stats SyncStageStats, subStageInfo string) { + idxs := d.getCurrentSyncIdxs() + if idxs.Stage == -1 || idxs.SubStage == -1 { + log.Warn("[Diagnostics] Can't find running stage or substage while updating Snapshots stage stats.", "stages:", d.syncStages, "stats:", stats, "subStageInfo:", subStageInfo) + return + } + + d.syncStages[idxs.Stage].SubStages[idxs.SubStage].Stats = stats +} + +func (d *DiagnosticClient) snapshotStageFinished() bool { + idx := d.getCurrentSyncIdxs() + if idx.Stage > 0 { + return true + } else { + return false + } +} + func (d *DiagnosticClient) runSegmentDownloadingListener(rootCtx context.Context) { go func() { ctx, ch, closeChannel := Context[SegmentDownloadStatistics](rootCtx, 1) @@ -65,7 +122,21 @@ func (d *DiagnosticClient) runSegmentDownloadingListener(rootCtx context.Context d.syncStats.SnapshotDownload.SegmentsDownloading = map[string]SegmentDownloadStatistics{} } - d.syncStats.SnapshotDownload.SegmentsDownloading[info.Name] = info + if val, ok := d.syncStats.SnapshotDownload.SegmentsDownloading[info.Name]; ok { + val.TotalBytes = info.TotalBytes + val.DownloadedBytes = info.DownloadedBytes + val.Webseeds = info.Webseeds + val.Peers = info.Peers + + d.syncStats.SnapshotDownload.SegmentsDownloading[info.Name] = val + } else { + d.syncStats.SnapshotDownload.SegmentsDownloading[info.Name] = info + } + + if err := d.db.Update(d.ctx, SnapshotDownloadUpdater(d.syncStats.SnapshotDownload)); err != nil { + log.Error("[Diagnostics] Failed to update snapshot download info", "err", err) + } + d.mu.Unlock() } } @@ -84,6 +155,9 @@ func (d *DiagnosticClient) runSegmentIndexingListener(rootCtx context.Context) { return case info := <-ch: d.addOrUpdateSegmentIndexingState(info) + if err := d.db.Update(d.ctx, SnapshotIndexingUpdater(d.syncStats.SnapshotIndexing)); err != nil { + log.Error("[Diagnostics] Failed to update snapshot indexing info", "err", err) + } } } }() @@ -117,6 +191,11 @@ func (d *DiagnosticClient) runSegmentIndexingFinishedListener(rootCtx context.Co Sys: 0, }) } + + if err := d.db.Update(d.ctx, SnapshotIndexingUpdater(d.syncStats.SnapshotIndexing)); err != nil { + log.Error("[Diagnostics] Failed to update snapshot indexing info", "err", err) + } + d.mu.Unlock() } } @@ -148,6 +227,19 @@ func (d *DiagnosticClient) addOrUpdateSegmentIndexingState(upd SnapshotIndexingS } d.syncStats.SnapshotIndexing.TimeElapsed = upd.TimeElapsed + + totalProgress := 0 + for _, seg := range d.syncStats.SnapshotIndexing.Segments { + totalProgress += seg.Percent + } + + d.updateSnapshotStageStats(SyncStageStats{ + TimeElapsed: SecondsToHHMMString(uint64(upd.TimeElapsed)), + TimeLeft: "unknown", + Progress: fmt.Sprintf("%d%%", totalProgress/len(d.syncStats.SnapshotIndexing.Segments)), + }, "Indexing snapshots") + + d.saveSyncStagesToDB() } func (d *DiagnosticClient) runSnapshotFilesListListener(rootCtx context.Context) { @@ -173,6 +265,88 @@ func (d *DiagnosticClient) runSnapshotFilesListListener(rootCtx context.Context) }() } +func (d *DiagnosticClient) runFileDownloadedListener(rootCtx context.Context) { + go func() { + ctx, ch, closeChannel := Context[FileDownloadedStatisticsUpdate](rootCtx, 1) + defer closeChannel() + + StartProviders(ctx, TypeOf(FileDownloadedStatisticsUpdate{}), log.Root()) + for { + select { + case <-rootCtx.Done(): + return + case info := <-ch: + d.mu.Lock() + + if d.syncStats.SnapshotDownload.SegmentsDownloading == nil { + d.syncStats.SnapshotDownload.SegmentsDownloading = map[string]SegmentDownloadStatistics{} + } + + if val, ok := d.syncStats.SnapshotDownload.SegmentsDownloading[info.FileName]; ok { + val.DownloadedStats = FileDownloadedStatistics{ + TimeTook: info.TimeTook, + AverageRate: info.AverageRate, + } + + d.syncStats.SnapshotDownload.SegmentsDownloading[info.FileName] = val + } else { + d.syncStats.SnapshotDownload.SegmentsDownloading[info.FileName] = SegmentDownloadStatistics{ + Name: info.FileName, + TotalBytes: 0, + DownloadedBytes: 0, + Webseeds: nil, + Peers: nil, + DownloadedStats: FileDownloadedStatistics{ + TimeTook: info.TimeTook, + AverageRate: info.AverageRate, + }, + } + } + + d.mu.Unlock() + } + } + }() +} + +func (d *DiagnosticClient) UpdateFileDownloadedStatistics(downloadedInfo *FileDownloadedStatisticsUpdate, downloadingInfo *SegmentDownloadStatistics) { + if d.syncStats.SnapshotDownload.SegmentsDownloading == nil { + d.syncStats.SnapshotDownload.SegmentsDownloading = map[string]SegmentDownloadStatistics{} + } + + if downloadedInfo != nil { + dwStats := FileDownloadedStatistics{ + TimeTook: downloadedInfo.TimeTook, + AverageRate: downloadedInfo.AverageRate, + } + if val, ok := d.syncStats.SnapshotDownload.SegmentsDownloading[downloadedInfo.FileName]; ok { + val.DownloadedStats = dwStats + + d.syncStats.SnapshotDownload.SegmentsDownloading[downloadedInfo.FileName] = val + } else { + d.syncStats.SnapshotDownload.SegmentsDownloading[downloadedInfo.FileName] = SegmentDownloadStatistics{ + Name: downloadedInfo.FileName, + TotalBytes: 0, + DownloadedBytes: 0, + Webseeds: make([]SegmentPeer, 0), + Peers: make([]SegmentPeer, 0), + DownloadedStats: dwStats, + } + } + } else { + if val, ok := d.syncStats.SnapshotDownload.SegmentsDownloading[downloadingInfo.Name]; ok { + val.TotalBytes = downloadingInfo.TotalBytes + val.DownloadedBytes = downloadingInfo.DownloadedBytes + val.Webseeds = downloadingInfo.Webseeds + val.Peers = downloadingInfo.Peers + + d.syncStats.SnapshotDownload.SegmentsDownloading[downloadingInfo.Name] = val + } else { + d.syncStats.SnapshotDownload.SegmentsDownloading[downloadingInfo.Name] = *downloadingInfo + } + } +} + func (d *DiagnosticClient) SyncStatistics() SyncStatistics { return d.syncStats } @@ -180,3 +354,45 @@ func (d *DiagnosticClient) SyncStatistics() SyncStatistics { func (d *DiagnosticClient) SnapshotFilesList() SnapshoFilesList { return d.snapshotFileList } + +func ReadSnapshotDownloadInfo(db kv.RoDB) (info SnapshotDownloadStatistics) { + data := ReadDataFromTable(db, kv.DiagSyncStages, SnapshotDownloadStatisticsKey) + + if len(data) == 0 { + return SnapshotDownloadStatistics{} + } + + err := json.Unmarshal(data, &info) + + if err != nil { + log.Error("[Diagnostics] Failed to read snapshot download info", "err", err) + return SnapshotDownloadStatistics{} + } else { + return info + } +} + +func ReadSnapshotIndexingInfo(db kv.RoDB) (info SnapshotIndexingStatistics) { + data := ReadDataFromTable(db, kv.DiagSyncStages, SnapshotIndexingStatisticsKey) + + if len(data) == 0 { + return SnapshotIndexingStatistics{} + } + + err := json.Unmarshal(data, &info) + + if err != nil { + log.Error("[Diagnostics] Failed to read snapshot indexing info", "err", err) + return SnapshotIndexingStatistics{} + } else { + return info + } +} + +func SnapshotDownloadUpdater(info SnapshotDownloadStatistics) func(tx kv.RwTx) error { + return PutDataToTable(kv.DiagSyncStages, SnapshotDownloadStatisticsKey, info) +} + +func SnapshotIndexingUpdater(info SnapshotIndexingStatistics) func(tx kv.RwTx) error { + return PutDataToTable(kv.DiagSyncStages, SnapshotIndexingStatisticsKey, info) +} diff --git a/erigon-lib/diagnostics/snapshots_test.go b/erigon-lib/diagnostics/snapshots_test.go new file mode 100644 index 00000000000..3c8fef60d4c --- /dev/null +++ b/erigon-lib/diagnostics/snapshots_test.go @@ -0,0 +1,57 @@ +package diagnostics_test + +import ( + "testing" + + "github.com/ledgerwatch/erigon-lib/diagnostics" + "github.com/stretchr/testify/require" +) + +func NewTestDiagnosticClient() (*diagnostics.DiagnosticClient, error) { + return &diagnostics.DiagnosticClient{}, nil +} + +func TestUpdateFileDownloadingStats(t *testing.T) { + d, err := NewTestDiagnosticClient() + + require.NoError(t, err) + + d.UpdateFileDownloadedStatistics(nil, &segmentDownloadStatsMock) + + sd := d.SyncStatistics().SnapshotDownload.SegmentsDownloading + require.NotNil(t, sd) + require.NotEqual(t, len(sd), 0) + + require.Equal(t, sd["test"], segmentDownloadStatsMock) + + d.UpdateFileDownloadedStatistics(&fileDownloadedUpdMock, nil) + + require.Equal(t, sd["test"], diagnostics.SegmentDownloadStatistics{ + Name: "test", + TotalBytes: 1, + DownloadedBytes: 1, + Webseeds: make([]diagnostics.SegmentPeer, 0), + Peers: make([]diagnostics.SegmentPeer, 0), + DownloadedStats: diagnostics.FileDownloadedStatistics{ + TimeTook: 1.0, + AverageRate: 1, + }, + }) +} + +var ( + fileDownloadedUpdMock = diagnostics.FileDownloadedStatisticsUpdate{ + FileName: "test", + TimeTook: 1.0, + AverageRate: 1, + } + + segmentDownloadStatsMock = diagnostics.SegmentDownloadStatistics{ + Name: "test", + TotalBytes: 1, + DownloadedBytes: 1, + Webseeds: make([]diagnostics.SegmentPeer, 0), + Peers: make([]diagnostics.SegmentPeer, 0), + DownloadedStats: diagnostics.FileDownloadedStatistics{}, + } +) diff --git a/erigon-lib/diagnostics/speedtest.go b/erigon-lib/diagnostics/speedtest.go index 77da08ac7a6..54fad6f161d 100644 --- a/erigon-lib/diagnostics/speedtest.go +++ b/erigon-lib/diagnostics/speedtest.go @@ -5,6 +5,7 @@ import ( "time" "github.com/showwin/speedtest-go/speedtest" + "github.com/showwin/speedtest-go/speedtest/transport" ) func (d *DiagnosticClient) setupSpeedtestDiagnostics(rootCtx context.Context) { @@ -17,37 +18,55 @@ func (d *DiagnosticClient) setupSpeedtestDiagnostics(rootCtx context.Context) { }() } +var cacheServerList speedtest.Servers + func (d *DiagnosticClient) runSpeedTest(rootCtx context.Context) NetworkSpeedTestResult { var speedtestClient = speedtest.New() - serverList, _ := speedtestClient.FetchServers() - targets, _ := serverList.FindServer([]int{}) + + serverList, err := speedtestClient.FetchServers() + // Ensure that the server list can rolled back to the previous cache. + if err == nil { + cacheServerList = serverList + } + targets, _ := cacheServerList.FindServer([]int{}) latency := time.Duration(0) downloadSpeed := float64(0) uploadSpeed := float64(0) + packetLoss := float64(-1) + + analyzer := speedtest.NewPacketLossAnalyzer(nil) if len(targets) > 0 { s := targets[0] - err := s.PingTestContext(rootCtx, nil) + err = s.PingTestContext(rootCtx, nil) if err == nil { latency = s.Latency } err = s.DownloadTestContext(rootCtx) if err == nil { - downloadSpeed = s.DLSpeed + downloadSpeed = s.DLSpeed.Mbps() } err = s.UploadTestContext(rootCtx) if err == nil { - uploadSpeed = s.ULSpeed + uploadSpeed = s.ULSpeed.Mbps() } + + ctx, cancel := context.WithTimeout(rootCtx, time.Second*15) + + defer cancel() + _ = analyzer.RunWithContext(ctx, s.Host, func(pl *transport.PLoss) { + packetLoss = pl.Loss() + }) } return NetworkSpeedTestResult{ Latency: latency, DownloadSpeed: downloadSpeed, UploadSpeed: uploadSpeed, + PacketLoss: packetLoss, } } diff --git a/erigon-lib/diagnostics/stages.go b/erigon-lib/diagnostics/stages.go index 7687dff135b..bc7670609e2 100644 --- a/erigon-lib/diagnostics/stages.go +++ b/erigon-lib/diagnostics/stages.go @@ -2,30 +2,109 @@ package diagnostics import ( "context" + "encoding/json" + "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/log/v3" ) +var ( + StagesListKey = []byte("diagStagesList") + CurrentStageKey = []byte("diagCurrentStage") +) + +type CurrentSyncStagesIdxs struct { + Stage int `json:"currentStage"` + SubStage int `json:"currentSubStage"` +} + +type SyncStage struct { + ID string `json:"id"` + State StageState `json:"state"` + SubStages []SyncSubStage `json:"subStages"` + Stats SyncStageStats `json:"stats"` +} + +type SyncSubStage struct { + ID string `json:"id"` + State StageState `json:"state"` + Stats SyncStageStats `json:"stats"` +} + +type SyncStageStats struct { + TimeElapsed string `json:"timeElapsed"` + TimeLeft string `json:"timeLeft"` + Progress string `json:"progress"` +} + +type SetSyncSubStageList struct { + Stage string + List []SyncSubStage +} + +func (ti SetSyncSubStageList) Type() Type { + return TypeOf(ti) +} + +type SyncStageList struct { + StagesList []SyncStage `json:"stages"` +} + +func (ti SyncStageList) Type() Type { + return TypeOf(ti) +} + +type StageState int + +const ( + Queued StageState = iota + Running + Completed +) + +func (s StageState) String() string { + return [...]string{"Queued", "Running", "Completed"}[s] +} + +type CurrentSyncStage struct { + Stage string `json:"stage"` +} + +func (ti CurrentSyncStage) Type() Type { + return TypeOf(ti) +} + +type CurrentSyncSubStage struct { + SubStage string `json:"subStage"` +} + +func (ti CurrentSyncSubStage) Type() Type { + return TypeOf(ti) +} + func (d *DiagnosticClient) setupStagesDiagnostics(rootCtx context.Context) { - d.runCurrentSyncStageListener(rootCtx) d.runSyncStagesListListener(rootCtx) + d.runCurrentSyncStageListener(rootCtx) + d.runCurrentSyncSubStageListener(rootCtx) + d.runSubStageListener(rootCtx) } func (d *DiagnosticClient) runSyncStagesListListener(rootCtx context.Context) { go func() { - ctx, ch, closeChannel := Context[SyncStagesList](rootCtx, 1) + ctx, ch, closeChannel := Context[SyncStageList](rootCtx, 1) defer closeChannel() - StartProviders(ctx, TypeOf(SyncStagesList{}), log.Root()) + StartProviders(ctx, TypeOf(SyncStageList{}), log.Root()) for { select { case <-rootCtx.Done(): return case info := <-ch: d.mu.Lock() - d.syncStats.SyncStages.StagesList = info.Stages + d.SetStagesList(info.StagesList) d.mu.Unlock() - return + + d.saveSyncStagesToDB() } } }() @@ -43,12 +122,173 @@ func (d *DiagnosticClient) runCurrentSyncStageListener(rootCtx context.Context) return case info := <-ch: d.mu.Lock() - d.syncStats.SyncStages.CurrentStage = info.Stage - if int(d.syncStats.SyncStages.CurrentStage) >= len(d.syncStats.SyncStages.StagesList) { - return - } + d.SetCurrentSyncStage(info) + d.mu.Unlock() + + d.saveSyncStagesToDB() + } + } + }() +} + +func (d *DiagnosticClient) runCurrentSyncSubStageListener(rootCtx context.Context) { + go func() { + ctx, ch, closeChannel := Context[CurrentSyncSubStage](rootCtx, 1) + defer closeChannel() + + StartProviders(ctx, TypeOf(CurrentSyncSubStage{}), log.Root()) + for { + select { + case <-rootCtx.Done(): + return + case info := <-ch: + d.mu.Lock() + d.SetCurrentSyncSubStage(info) + d.mu.Unlock() + + d.saveSyncStagesToDB() + } + } + }() +} + +func (d *DiagnosticClient) runSubStageListener(rootCtx context.Context) { + go func() { + ctx, ch, closeChannel := Context[SetSyncSubStageList](rootCtx, 1) + defer closeChannel() + + StartProviders(ctx, TypeOf(SetSyncSubStageList{}), log.Root()) + for { + select { + case <-rootCtx.Done(): + return + case info := <-ch: + d.mu.Lock() + d.SetSubStagesList(info.Stage, info.List) d.mu.Unlock() + + d.saveSyncStagesToDB() } } }() } + +func (d *DiagnosticClient) saveSyncStagesToDB() { + if err := d.db.Update(d.ctx, StagesListUpdater(d.syncStages)); err != nil { + log.Error("[Diagnostics] Failed to update stages list", "err", err) + } +} + +func (d *DiagnosticClient) getCurrentSyncIdxs() CurrentSyncStagesIdxs { + currentIdxs := CurrentSyncStagesIdxs{ + Stage: -1, + SubStage: -1, + } + + for sIdx, stage := range d.syncStages { + if stage.State == Running { + currentIdxs.Stage = sIdx + + for subIdx, subStage := range stage.SubStages { + if subStage.State == Running { + currentIdxs.SubStage = subIdx + } + } + break + } + } + + return currentIdxs +} + +func (d *DiagnosticClient) SetStagesList(stages []SyncStage) { + if len(d.syncStages) != len(stages) { + d.syncStages = stages + } +} + +func (d *DiagnosticClient) SetSubStagesList(stageId string, subStages []SyncSubStage) { + for idx, stage := range d.syncStages { + if stage.ID == stageId { + if len(d.syncStages[idx].SubStages) != len(subStages) { + d.syncStages[idx].SubStages = subStages + break + } + } + } +} + +func (d *DiagnosticClient) SetCurrentSyncStage(css CurrentSyncStage) { + isSet := false + for idx, stage := range d.syncStages { + if !isSet { + if stage.ID == css.Stage { + d.syncStages[idx].State = Running + isSet = true + } else { + d.setStagesState(idx, Completed) + } + } else { + d.setStagesState(idx, Queued) + } + } +} + +func (d *DiagnosticClient) setStagesState(stadeIdx int, state StageState) { + d.syncStages[stadeIdx].State = state + d.setSubStagesState(stadeIdx, state) +} + +func (d *DiagnosticClient) setSubStagesState(stadeIdx int, state StageState) { + for subIdx := range d.syncStages[stadeIdx].SubStages { + d.syncStages[stadeIdx].SubStages[subIdx].State = state + } +} + +func (d *DiagnosticClient) SetCurrentSyncSubStage(css CurrentSyncSubStage) { + for idx, stage := range d.syncStages { + if stage.State == Running { + for subIdx, subStage := range stage.SubStages { + if subStage.ID == css.SubStage { + if d.syncStages[idx].SubStages[subIdx].State == Completed { + return + } + + if subIdx > 0 { + d.syncStages[idx].SubStages[subIdx-1].State = Completed + } + + d.syncStages[idx].SubStages[subIdx].State = Running + } + } + + break + } + } +} + +func ReadSyncStages(db kv.RoDB) []SyncStage { + data := ReadDataFromTable(db, kv.DiagSyncStages, StagesListKey) + + if len(data) == 0 { + return []SyncStage{} + } + + var info []SyncStage + err := json.Unmarshal(data, &info) + + if err != nil { + log.Error("[Diagnostics] Failed to read stages list", "err", err) + return []SyncStage{} + } else { + return info + } +} + +func StagesListUpdater(info []SyncStage) func(tx kv.RwTx) error { + return PutDataToTable(kv.DiagSyncStages, StagesListKey, info) +} + +func (d *DiagnosticClient) GetSyncStages() []SyncStage { + return d.syncStages +} diff --git a/erigon-lib/diagnostics/stages_test.go b/erigon-lib/diagnostics/stages_test.go new file mode 100644 index 00000000000..2f49816a5fd --- /dev/null +++ b/erigon-lib/diagnostics/stages_test.go @@ -0,0 +1,109 @@ +package diagnostics_test + +import ( + "testing" + + "github.com/ledgerwatch/erigon-lib/diagnostics" + "github.com/stretchr/testify/require" +) + +func TestInitSyncStages(t *testing.T) { + d, err := NewTestDiagnosticClient() + require.NoError(t, err) + + stages := diagnostics.InitStagesFromList(nodeStages) + d.SetStagesList(stages) + require.Equal(t, d.GetSyncStages(), stagesListMock) + + subStages := diagnostics.InitSubStagesFromList(snapshotsSubStages) + require.Equal(t, subStages, subStagesListMock) + d.SetSubStagesList("Snapshots", subStages) + + require.Equal(t, d.GetSyncStages(), stagesListWithSnapshotsSubStagesMock) +} + +func TestSetCurrentSyncStage(t *testing.T) { + d, err := NewTestDiagnosticClient() + require.NoError(t, err) + + stages := diagnostics.InitStagesFromList(nodeStages) + d.SetStagesList(stages) + subStages := diagnostics.InitSubStagesFromList(snapshotsSubStages) + d.SetSubStagesList("Snapshots", subStages) + + d.SetCurrentSyncStage(diagnostics.CurrentSyncStage{Stage: "Snapshots"}) + require.Equal(t, d.GetSyncStages()[0].State, diagnostics.Running) + + d.SetCurrentSyncStage(diagnostics.CurrentSyncStage{Stage: "BlockHashes"}) + require.Equal(t, d.GetSyncStages()[0].State, diagnostics.Completed) + require.Equal(t, d.GetSyncStages()[1].State, diagnostics.Running) + + d.SetCurrentSyncStage(diagnostics.CurrentSyncStage{Stage: "Snapshots"}) + require.Equal(t, d.GetSyncStages()[0].State, diagnostics.Running) + require.Equal(t, d.GetSyncStages()[1].State, diagnostics.Queued) + require.Equal(t, d.GetSyncStages()[2].State, diagnostics.Queued) +} + +func TestSetCurrentSyncSubStage(t *testing.T) { + d, err := NewTestDiagnosticClient() + require.NoError(t, err) + + stages := diagnostics.InitStagesFromList(nodeStages) + d.SetStagesList(stages) + subStages := diagnostics.InitSubStagesFromList(snapshotsSubStages) + d.SetSubStagesList("Snapshots", subStages) + + d.SetCurrentSyncStage(diagnostics.CurrentSyncStage{Stage: "Snapshots"}) + d.SetCurrentSyncSubStage(diagnostics.CurrentSyncSubStage{SubStage: "Download header-chain"}) + require.Equal(t, d.GetSyncStages()[0].SubStages[0].State, diagnostics.Running) + + d.SetCurrentSyncSubStage(diagnostics.CurrentSyncSubStage{SubStage: "Download snapshots"}) + require.Equal(t, d.GetSyncStages()[0].SubStages[0].State, diagnostics.Completed) + require.Equal(t, d.GetSyncStages()[0].SubStages[1].State, diagnostics.Running) + + d.SetCurrentSyncSubStage(diagnostics.CurrentSyncSubStage{SubStage: "Download header-chain"}) + require.Equal(t, d.GetSyncStages()[0].SubStages[0].State, diagnostics.Completed) + require.Equal(t, d.GetSyncStages()[0].SubStages[1].State, diagnostics.Running) + require.Equal(t, d.GetSyncStages()[0].SubStages[2].State, diagnostics.Queued) +} + +var ( + nodeStages = []string{"Snapshots", "BlockHashes", "Senders"} + snapshotsSubStages = []string{"Download header-chain", "Download snapshots", "Indexing", "Fill DB"} + + stagesListMock = []diagnostics.SyncStage{ + {ID: "Snapshots", State: diagnostics.Queued, SubStages: []diagnostics.SyncSubStage{}}, + {ID: "BlockHashes", State: diagnostics.Queued, SubStages: []diagnostics.SyncSubStage{}}, + {ID: "Senders", State: diagnostics.Queued, SubStages: []diagnostics.SyncSubStage{}}, + } + + subStagesListMock = []diagnostics.SyncSubStage{ + { + ID: "Download header-chain", + State: diagnostics.Queued, + }, + { + ID: "Download snapshots", + State: diagnostics.Queued, + }, + { + ID: "Indexing", + State: diagnostics.Queued, + }, + { + ID: "Fill DB", + State: diagnostics.Queued, + }, + } + + stagesListWithSnapshotsSubStagesMock = []diagnostics.SyncStage{ + {ID: "Snapshots", State: diagnostics.Queued, SubStages: []diagnostics.SyncSubStage{ + {ID: "Download header-chain", State: diagnostics.Queued}, + {ID: "Download snapshots", State: diagnostics.Queued}, + {ID: "Indexing", State: diagnostics.Queued}, + {ID: "Fill DB", State: diagnostics.Queued}, + }}, + {ID: "BlockHashes", State: diagnostics.Queued, SubStages: []diagnostics.SyncSubStage{}}, + {ID: "Senders", State: diagnostics.Queued, SubStages: []diagnostics.SyncSubStage{}}, + } +) diff --git a/erigon-lib/diagnostics/sys_info.go b/erigon-lib/diagnostics/sys_info.go index 6b12ce1b80b..b870e649324 100644 --- a/erigon-lib/diagnostics/sys_info.go +++ b/erigon-lib/diagnostics/sys_info.go @@ -1,14 +1,36 @@ package diagnostics import ( - "github.com/ledgerwatch/erigon-lib/diskutils" + "encoding/json" + "github.com/shirou/gopsutil/v3/cpu" "github.com/shirou/gopsutil/v3/disk" "github.com/shirou/gopsutil/v3/mem" + + "github.com/ledgerwatch/erigon-lib/diskutils" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/log/v3" +) + +var ( + SystemRamInfoKey = []byte("diagSystemRamInfo") + SystemCpuInfoKey = []byte("diagSystemCpuInfo") + SystemDiskInfoKey = []byte("diagSystemDiskInfo") ) func (d *DiagnosticClient) setupSysInfoDiagnostics() { sysInfo := GetSysInfo(d.dataDirPath) + if err := d.db.Update(d.ctx, RAMInfoUpdater(sysInfo.RAM)); err != nil { + log.Error("[Diagnostics] Failed to update RAM info", "err", err) + } + + if err := d.db.Update(d.ctx, CPUInfoUpdater(sysInfo.CPU)); err != nil { + log.Error("[Diagnostics] Failed to update CPU info", "err", err) + } + + if err := d.db.Update(d.ctx, DiskInfoUpdater(sysInfo.Disk)); err != nil { + log.Error("[Diagnostics] Failed to update Disk info", "err", err) + } d.mu.Lock() d.hardwareInfo = sysInfo @@ -106,3 +128,81 @@ func GetCPUInfo() CPUInfo { Mhz: mhz, } } + +func ReadSysInfo(db kv.RoDB) (info HardwareInfo) { + ram := ReadRAMInfo(db) + cpu := ReadCPUInfo(db) + disk := ReadDickInfo(db) + + return HardwareInfo{ + RAM: ram, + CPU: cpu, + Disk: disk, + } +} + +func ReadRAMInfo(db kv.RoDB) RAMInfo { + data := ReadDataFromTable(db, kv.DiagSystemInfo, SystemRamInfoKey) + + if len(data) == 0 { + return RAMInfo{} + } + + var info RAMInfo + err := json.Unmarshal(data, &info) + + if err != nil { + log.Error("[Diagnostics] Failed to read RAM info", "err", err) + return RAMInfo{} + } else { + return info + } +} + +func ReadCPUInfo(db kv.RoDB) CPUInfo { + data := ReadDataFromTable(db, kv.DiagSystemInfo, SystemCpuInfoKey) + + if len(data) == 0 { + return CPUInfo{} + } + + var info CPUInfo + err := json.Unmarshal(data, &info) + + if err != nil { + log.Error("[Diagnostics] Failed to read CPU info", "err", err) + return CPUInfo{} + } else { + return info + } +} + +func ReadDickInfo(db kv.RoDB) DiskInfo { + data := ReadDataFromTable(db, kv.DiagSystemInfo, SystemDiskInfoKey) + + if len(data) == 0 { + return DiskInfo{} + } + + var info DiskInfo + err := json.Unmarshal(data, &info) + + if err != nil { + log.Error("[Diagnostics] Failed to read Disk info", "err", err) + return DiskInfo{} + } else { + return info + } +} + +func RAMInfoUpdater(info RAMInfo) func(tx kv.RwTx) error { + return PutDataToTable(kv.DiagSystemInfo, SystemRamInfoKey, info) +} + +func CPUInfoUpdater(info CPUInfo) func(tx kv.RwTx) error { + return PutDataToTable(kv.DiagSystemInfo, SystemCpuInfoKey, info) +} + +func DiskInfoUpdater(info DiskInfo) func(tx kv.RwTx) error { + return PutDataToTable(kv.DiagSystemInfo, SystemDiskInfoKey, info) +} diff --git a/erigon-lib/diagnostics/utils.go b/erigon-lib/diagnostics/utils.go new file mode 100644 index 00000000000..055bcb9ee71 --- /dev/null +++ b/erigon-lib/diagnostics/utils.go @@ -0,0 +1,91 @@ +package diagnostics + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/ledgerwatch/erigon-lib/kv" +) + +func ReadDataFromTable(db kv.RoDB, table string, key []byte) (data []byte) { + if err := db.View(context.Background(), func(tx kv.Tx) error { + bytes, err := tx.GetOne(table, key) + + if err != nil { + return err + } + + data = bytes + + return nil + }); err != nil { + return []byte{} + } + return data +} + +func PutDataToTable(table string, key []byte, info any) func(tx kv.RwTx) error { + return func(tx kv.RwTx) error { + infoBytes, err := json.Marshal(info) + + if err != nil { + return err + } + + return tx.Put(table, key, infoBytes) + } +} + +func InitStagesFromList(list []string) []SyncStage { + stages := make([]SyncStage, 0, len(list)) + + for _, stage := range list { + stages = append(stages, SyncStage{ + ID: stage, + State: Queued, + SubStages: []SyncSubStage{}, + Stats: SyncStageStats{}, + }) + } + + return stages +} + +func InitSubStagesFromList(list []string) []SyncSubStage { + subStages := make([]SyncSubStage, 0, len(list)) + + for _, subStage := range list { + subStages = append(subStages, + SyncSubStage{ + ID: subStage, + State: Queued, + }, + ) + } + + return subStages +} + +func CalculateTime(amountLeft, rate uint64) string { + if rate == 0 { + return "999hrs:99m" + } + timeLeftInSeconds := amountLeft / rate + + hours := timeLeftInSeconds / 3600 + minutes := (timeLeftInSeconds / 60) % 60 + + return fmt.Sprintf("%dhrs:%dm", hours, minutes) +} + +func SecondsToHHMMString(seconds uint64) string { + if seconds == 0 { + return "0hrs:0m" + } + + hours := seconds / 3600 + minutes := (seconds / 60) % 60 + + return fmt.Sprintf("%dhrs:%dm", hours, minutes) +} diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index 10653fbeed4..fb44565afe0 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -330,8 +330,6 @@ func New(ctx context.Context, cfg *downloadercfg.Cfg, logger log.Logger, verbosi d.ctx, d.stopMainLoop = context.WithCancel(ctx) if cfg.AddTorrentsFromDisk { - var downloadMismatches []string - for _, download := range lock.Downloads { if info, err := d.torrentInfo(download.Name); err == nil { if info.Completed != nil { @@ -357,29 +355,15 @@ func New(ctx context.Context, cfg *downloadercfg.Cfg, logger log.Logger, verbosi fileHash := hex.EncodeToString(fileHashBytes) if fileHash != download.Hash && fileHash != hash { - d.logger.Error("[snapshots] download db mismatch", "file", download.Name, "lock", download.Hash, "db", hash, "disk", fileHash, "downloaded", *info.Completed) - downloadMismatches = append(downloadMismatches, download.Name) + d.logger.Debug("[snapshots] download db mismatch", "file", download.Name, "lock", download.Hash, "db", hash, "disk", fileHash, "downloaded", *info.Completed) } else { - d.logger.Warn("[snapshots] lock hash does not match completed download", "file", download.Name, "lock", hash, "download", download.Hash, "downloaded", *info.Completed) + d.logger.Debug("[snapshots] lock hash does not match completed download", "file", download.Name, "lock", hash, "download", download.Hash, "downloaded", *info.Completed) } } } } } - if len(downloadMismatches) > 0 { - return nil, fmt.Errorf("downloaded files have mismatched hashes: %s", strings.Join(downloadMismatches, ",")) - } - - //TODO: why do we need it if we have `addTorrentFilesFromDisk`? what if they are conflict? - //TODO: why it's before `BuildTorrentFilesIfNeed`? what if they are conflict? - //TODO: even if hash is saved in "snapshots-lock.json" - it still must preserve `prohibit_new_downloads.lock` and don't download new files ("user restart" must be fast, "erigon3 has .kv files which never-ending merge and delete small files") - //for _, it := range lock.Downloads { - // if err := d.AddMagnetLink(ctx, snaptype.Hex2InfoHash(it.Hash), it.Name); err != nil { - // return nil, err - // } - //} - if err := d.BuildTorrentFilesIfNeed(d.ctx, lock.Chain, lock.Downloads); err != nil { return nil, err } @@ -2591,10 +2575,17 @@ func openClient(ctx context.Context, dbDir, snapDir string, cfg *torrent.ClientC //}) cfg.DefaultStorage = m + dnsResolver := &downloadercfg.DnsCacheResolver{RefreshTimeout: 24 * time.Hour} + cfg.TrackerDialContext = dnsResolver.DialContext + torrentClient, err = torrent.NewClient(cfg) if err != nil { return nil, nil, nil, nil, fmt.Errorf("torrent.NewClient: %w", err) } + go func() { + dnsResolver.Run(ctx) + }() + return db, c, m, torrentClient, nil } diff --git a/erigon-lib/downloader/downloadercfg/dns_cache_resolver.go b/erigon-lib/downloader/downloadercfg/dns_cache_resolver.go new file mode 100644 index 00000000000..1aab7c41f7d --- /dev/null +++ b/erigon-lib/downloader/downloadercfg/dns_cache_resolver.go @@ -0,0 +1,49 @@ +package downloadercfg + +import ( + "context" + "net" + "time" + + "github.com/rs/dnscache" +) + +// DnsCacheResolver resolves DNS requests for an HTTP client using an in-memory cache. +type DnsCacheResolver struct { + RefreshTimeout time.Duration + + resolver dnscache.Resolver +} + +func (r *DnsCacheResolver) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + host, port, err := net.SplitHostPort(address) + if err != nil { + return nil, err + } + ips, err := r.resolver.LookupHost(ctx, host) + if err != nil { + return nil, err + } + var conn net.Conn + for _, ip := range ips { + var dialer net.Dialer + conn, err = dialer.DialContext(ctx, network, net.JoinHostPort(ip, port)) + if err == nil { + break + } + } + return conn, err +} + +func (r *DnsCacheResolver) Run(ctx context.Context) { + ticker := time.NewTicker(r.RefreshTimeout) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + r.resolver.Refresh(true) + } + } +} diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index 111881b7cd2..a915b934619 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -4,10 +4,11 @@ go 1.21 require ( github.com/erigontech/mdbx-go v0.27.24 - github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240509031412-4975b6ec206f + github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240619030755-1e16b374e6cb github.com/ledgerwatch/interfaces v0.0.0-20240320062914-b57f05746087 github.com/ledgerwatch/log/v3 v3.9.0 github.com/ledgerwatch/secp256k1 v1.0.0 + github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 ) replace github.com/ledgerwatch/erigon => ../../erigon @@ -127,9 +128,8 @@ require ( github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect - github.com/showwin/speedtest-go v1.6.12 + github.com/showwin/speedtest-go v1.7.7 github.com/sirupsen/logrus v1.9.3 // indirect github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/tklauser/numcpus v0.8.0 // indirect diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 65bae6932cc..98bea8f466e 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -267,8 +267,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240509031412-4975b6ec206f h1:Z5gGUN+LkQVp0zjPgSfuqZkIrbp2lItO6yRG9yt7ODQ= -github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240509031412-4975b6ec206f/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= +github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240619030755-1e16b374e6cb h1:aGbaiFpYmDfj+D0qJF+wJ9MbkftG0MbGNUwf6XClnFY= +github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240619030755-1e16b374e6cb/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/interfaces v0.0.0-20240320062914-b57f05746087 h1:Y59HUAT/+02Qbm6g7MuY7i8E0kUihPe7+ftDnR8oQzQ= github.com/ledgerwatch/interfaces v0.0.0-20240320062914-b57f05746087/go.mod h1:ugQv1QllJzBny3cKZKxUrSnykkjkBgm27eQM6dnGAcc= github.com/ledgerwatch/log/v3 v3.9.0 h1:iDwrXe0PVwBC68Dd94YSsHbMgQ3ufsgjzXtFNFVZFRk= @@ -415,8 +415,8 @@ github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFt github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= -github.com/showwin/speedtest-go v1.6.12 h1:q+hWNn2cM35KkqtXGGbSmuJgd67gTP8+VlneY2hq9vU= -github.com/showwin/speedtest-go v1.6.12/go.mod h1:uLgdWCNarXxlYsL2E5TOZpCIwpgSWnEANZp7gfHXHu0= +github.com/showwin/speedtest-go v1.7.7 h1:VmK75SZOTKiuWjIVrs+mo7ZoKEw0utiGCvpnurS0olU= +github.com/showwin/speedtest-go v1.7.7/go.mod h1:uLgdWCNarXxlYsL2E5TOZpCIwpgSWnEANZp7gfHXHu0= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= diff --git a/erigon-lib/kv/kv_interface.go b/erigon-lib/kv/kv_interface.go index e7a76b8c40b..5a02dc3cd49 100644 --- a/erigon-lib/kv/kv_interface.go +++ b/erigon-lib/kv/kv_interface.go @@ -149,12 +149,13 @@ type DBVerbosityLvl int8 type Label uint8 const ( - ChainDB Label = 0 - TxPoolDB Label = 1 - SentryDB Label = 2 - ConsensusDB Label = 3 - DownloaderDB Label = 4 - InMem Label = 5 + ChainDB Label = 0 + TxPoolDB Label = 1 + SentryDB Label = 2 + ConsensusDB Label = 3 + DownloaderDB Label = 4 + InMem Label = 5 + DiagnosticsDB Label = 6 ) func (l Label) String() string { @@ -171,6 +172,8 @@ func (l Label) String() string { return "downloader" case InMem: return "inMem" + case DiagnosticsDB: + return "diagnostics" default: return "unknown" } @@ -189,6 +192,8 @@ func UnmarshalLabel(s string) Label { return DownloaderDB case "inMem": return InMem + case "diagnostics": + return DiagnosticsDB default: panic(fmt.Sprintf("unexpected label: %s", s)) } diff --git a/erigon-lib/kv/tables.go b/erigon-lib/kv/tables.go index 9098f09cb47..3b737aa7a6f 100644 --- a/erigon-lib/kv/tables.go +++ b/erigon-lib/kv/tables.go @@ -523,6 +523,9 @@ const ( INTERMEDIATE_TX_STATEROOTS = "hermez_intermediate_tx_stateRoots" // l2blockno -> stateRoot BATCH_WITNESSES = "hermez_batch_witnesses" // batch number -> witness BATCH_COUNTERS = "hermez_batch_counters" + //Diagnostics tables + DiagSystemInfo = "DiagSystemInfo" + DiagSyncStages = "DiagSyncStages" ) // Keys @@ -773,6 +776,11 @@ var ChaindataDeprecatedTables = []string{ TransitionBlockKey, } +var DiagnosticsTables = []string{ + DiagSystemInfo, + DiagSyncStages, +} + type CmpFunc func(k1, k2, v1, v2 []byte) int type TableCfg map[string]TableCfgItem @@ -870,6 +878,7 @@ var BorTablesCfg = TableCfg{ var TxpoolTablesCfg = TableCfg{} var SentryTablesCfg = TableCfg{} var DownloaderTablesCfg = TableCfg{} +var DiagnosticsTablesCfg = TableCfg{} var ReconTablesCfg = TableCfg{ PlainStateD: {Flags: DupSort}, CodeD: {Flags: DupSort}, @@ -886,6 +895,8 @@ func TablesCfgByLabel(label Label) TableCfg { return SentryTablesCfg case DownloaderDB: return DownloaderTablesCfg + case DiagnosticsDB: + return DiagnosticsTablesCfg default: panic(fmt.Sprintf("unexpected label: %s", label)) } @@ -948,6 +959,13 @@ func reinit() { ReconTablesCfg[name] = TableCfgItem{} } } + + for _, name := range DiagnosticsTables { + _, ok := DiagnosticsTablesCfg[name] + if !ok { + DiagnosticsTablesCfg[name] = TableCfgItem{} + } + } } // Temporal diff --git a/erigon-lib/txpool/pool.go b/erigon-lib/txpool/pool.go index 846a6b3e6be..d9ecdc0d051 100644 --- a/erigon-lib/txpool/pool.go +++ b/erigon-lib/txpool/pool.go @@ -835,10 +835,8 @@ func toBlobs(_blobs [][]byte) []gokzg4844.Blob { func (p *TxPool) validateTx(txn *types.TxSlot, isLocal bool, stateCache kvcache.CacheView) txpoolcfg.DiscardReason { isShanghai := p.isShanghai() || p.isAgra() - if isShanghai { - if txn.DataLen > fixedgas.MaxInitCodeSize { - return txpoolcfg.InitCodeTooLarge - } + if isShanghai && txn.Creation && txn.DataLen > fixedgas.MaxInitCodeSize { + return txpoolcfg.InitCodeTooLarge // EIP-3860 } if txn.Type == types.BlobTxType { if !p.isCancun() { diff --git a/erigon-lib/txpool/pool_test.go b/erigon-lib/txpool/pool_test.go index 93878af5c87..6030ee6e03f 100644 --- a/erigon-lib/txpool/pool_test.go +++ b/erigon-lib/txpool/pool_test.go @@ -645,26 +645,43 @@ func TestShanghaiValidateTx(t *testing.T) { expected txpoolcfg.DiscardReason dataLen int isShanghai bool + creation bool }{ "no shanghai": { expected: txpoolcfg.Success, dataLen: 32, isShanghai: false, + creation: true, }, "shanghai within bounds": { expected: txpoolcfg.Success, dataLen: 32, isShanghai: true, + creation: true, }, - "shanghai exactly on bound": { + "shanghai exactly on bound - create tx": { expected: txpoolcfg.Success, dataLen: fixedgas.MaxInitCodeSize, isShanghai: true, + creation: true, }, - "shanghai one over bound": { + "shanghai one over bound - create tx": { expected: txpoolcfg.InitCodeTooLarge, dataLen: fixedgas.MaxInitCodeSize + 1, isShanghai: true, + creation: true, + }, + "shanghai exactly on bound - calldata tx": { + expected: txpoolcfg.Success, + dataLen: fixedgas.MaxInitCodeSize, + isShanghai: true, + creation: false, + }, + "shanghai one over bound - calldata tx": { + expected: txpoolcfg.Success, + dataLen: fixedgas.MaxInitCodeSize + 1, + isShanghai: true, + creation: false, }, } @@ -700,7 +717,7 @@ func TestShanghaiValidateTx(t *testing.T) { FeeCap: *uint256.NewInt(21000), Gas: 500000, SenderID: 0, - Creation: true, + Creation: test.creation, } txns := types.TxSlots{ diff --git a/eth/backend.go b/eth/backend.go index c987fda1fce..8023aa12c66 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -36,6 +36,7 @@ import ( "github.com/ledgerwatch/erigon-lib/common/dir" "github.com/ledgerwatch/erigon-lib/common/disk" "github.com/ledgerwatch/erigon-lib/common/mem" + "github.com/ledgerwatch/erigon-lib/diagnostics" "github.com/0xPolygonHermez/zkevm-data-streamer/datastreamer" "github.com/ledgerwatch/erigon/zk/sequencer" @@ -1775,7 +1776,10 @@ func (s *Ethereum) Start() error { return currentTD } + nodeStages := s.stagedSync.StagesIdsList() + if params.IsChainPoS(s.chainConfig, currentTDProvider) { + nodeStages = s.pipelineStagedSync.StagesIdsList() s.waitForStageLoopStop = nil // TODO: Ethereum.Stop should wait for execution_server shutdown go s.eth1ExecutionServer.Start(s.sentryCtx) } else if s.config.PolygonSync { @@ -1801,6 +1805,9 @@ func (s *Ethereum) Start() error { go stages2.StageLoop(s.sentryCtx, s.chainDB, s.stagedSync, s.sentriesClient.Hd, s.waitForStageLoopStop, s.config.Sync.LoopThrottle, s.logger, s.blockReader, hook, s.config.ForcePartialCommit) } + stages := diagnostics.InitStagesFromList(nodeStages) + diagnostics.Send(diagnostics.SyncStageList{StagesList: stages}) + if s.chainConfig.Bor != nil { s.engine.(*bor.Bor).Start(s.chainDB) } diff --git a/eth/stagedsync/stage_snapshots.go b/eth/stagedsync/stage_snapshots.go index 2081faf3258..bbdbaa78ec7 100644 --- a/eth/stagedsync/stage_snapshots.go +++ b/eth/stagedsync/stage_snapshots.go @@ -29,6 +29,7 @@ import ( "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/common/dbg" "github.com/ledgerwatch/erigon-lib/common/dir" + "github.com/ledgerwatch/erigon-lib/diagnostics" "github.com/ledgerwatch/erigon-lib/downloader" "github.com/ledgerwatch/erigon-lib/downloader/snaptype" "github.com/ledgerwatch/erigon-lib/etl" @@ -193,12 +194,21 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R if !cfg.blockReader.FreezingCfg().Enabled { return nil } + + diagnostics.Send(diagnostics.CurrentSyncStage{Stage: string(stages.Snapshots)}) + cstate := snapshotsync.NoCaplin if cfg.caplin { //TODO(Giulio2002): uncomment cstate = snapshotsync.AlsoCaplin } if cfg.snapshotUploader != nil { + subStages := diagnostics.InitSubStagesFromList([]string{"Indexing", "Fill DB"}) + diagnostics.Send(diagnostics.SetSyncSubStageList{ + Stage: string(stages.Snapshots), + List: subStages, + }) + u := cfg.snapshotUploader u.init(ctx, logger) @@ -236,6 +246,13 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R cfg.notifier.Events.OnNewSnapshot() } } else { + subStages := diagnostics.InitSubStagesFromList([]string{"Download snapshots", "Indexing", "Fill DB"}) + diagnostics.Send(diagnostics.SetSyncSubStageList{ + Stage: string(stages.Snapshots), + List: subStages, + }) + + diagnostics.Send(diagnostics.CurrentSyncSubStage{SubStage: "Download snapshots"}) if err := snapshotsync.WaitForDownloader(ctx, s.LogPrefix(), cfg.historyV3, cfg.blobs, cstate, cfg.agg, tx, cfg.blockReader, &cfg.chainConfig, cfg.snapshotDownloader, s.state.StagesIdsList()); err != nil { return err } @@ -252,6 +269,7 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R return histBlockNumProgress }) + diagnostics.Send(diagnostics.CurrentSyncSubStage{SubStage: "Indexing"}) if err := cfg.blockRetire.BuildMissedIndicesIfNeed(ctx, s.LogPrefix(), cfg.notifier.Events, &cfg.chainConfig); err != nil { return err } @@ -282,6 +300,7 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R s.BlockNumber = frozenBlocks } + diagnostics.Send(diagnostics.CurrentSyncSubStage{SubStage: "Fill DB"}) if err := FillDBFromSnapshots(s.LogPrefix(), ctx, tx, cfg.dirs, cfg.blockReader, cfg.agg, logger); err != nil { return err } diff --git a/eth/stagedsync/sync.go b/eth/stagedsync/sync.go index 9455acadb0a..c2fd2fbdd8f 100644 --- a/eth/stagedsync/sync.go +++ b/eth/stagedsync/sync.go @@ -93,9 +93,8 @@ func (s *Sync) NextStage() { return } s.currentStage++ - isDiagEnabled := diagnostics.TypeOf(diagnostics.CurrentSyncStage{}).Enabled() - if isDiagEnabled { - diagnostics.Send(diagnostics.CurrentSyncStage{Stage: s.currentStage}) + if s.currentStage < uint(len(s.stages)) { + diagnostics.Send(diagnostics.CurrentSyncStage{Stage: string(s.stages[s.currentStage].ID)}) } } @@ -166,10 +165,8 @@ func (s *Sync) SetCurrentStage(id stages.SyncStage) error { for i, stage := range s.stages { if stage.ID == id { s.currentStage = uint(i) - isDiagEnabled := diagnostics.TypeOf(diagnostics.CurrentSyncStage{}).Enabled() - if isDiagEnabled { - diagnostics.Send(diagnostics.CurrentSyncStage{Stage: s.currentStage}) - } + + diagnostics.Send(diagnostics.CurrentSyncStage{Stage: string(id)}) return nil } diff --git a/go.mod b/go.mod index 7d7deec10bd..6130540029e 100644 --- a/go.mod +++ b/go.mod @@ -68,10 +68,12 @@ require ( github.com/iden3/go-iden3-crypto v0.0.15 github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c github.com/jackpal/go-nat-pmp v1.0.2 + github.com/jedib0t/go-pretty/v6 v6.5.9 github.com/json-iterator/go v1.1.12 github.com/julienschmidt/httprouter v1.3.0 github.com/klauspost/compress v1.17.3 github.com/ledgerwatch/erigon-lib v1.0.0 + github.com/ledgerwatch/erigonwatch v0.1.2 github.com/libp2p/go-libp2p v0.31.0 github.com/libp2p/go-libp2p-mplex v0.9.0 github.com/libp2p/go-libp2p-pubsub v0.9.3 @@ -181,8 +183,6 @@ require ( github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/mock v1.6.0 // indirect - github.com/google/go-cmp v0.6.0 // indirect - github.com/golang/protobuf v1.5.4 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd // indirect github.com/hermeznetwork/tracerr v0.3.2 // indirect @@ -204,7 +204,7 @@ require ( github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240509031412-4975b6ec206f // indirect + github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240619030755-1e16b374e6cb // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect @@ -220,6 +220,7 @@ require ( github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect github.com/miekg/dns v1.1.55 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect @@ -273,13 +274,14 @@ require ( github.com/quic-go/webtransport-go v0.5.3 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/rivo/uniseg v0.2.0 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/shirou/gopsutil/v3 v3.24.3 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/shopspring/decimal v1.2.0 // indirect - github.com/showwin/speedtest-go v1.6.12 // indirect + github.com/showwin/speedtest-go v1.7.7 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/sosodev/duration v1.1.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect diff --git a/go.sum b/go.sum index 44025740a3f..35e27776bd1 100644 --- a/go.sum +++ b/go.sum @@ -567,6 +567,8 @@ github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7Bd github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk= github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= +github.com/jedib0t/go-pretty/v6 v6.5.9 h1:ACteMBRrrmm1gMsXe9PSTOClQ63IXDUt03H5U+UV8OU= +github.com/jedib0t/go-pretty/v6 v6.5.9/go.mod h1:zbn98qrYlh95FIhwwsbIip0LYpwSG8SUOScs+v9/t0E= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -608,8 +610,10 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240509031412-4975b6ec206f h1:Z5gGUN+LkQVp0zjPgSfuqZkIrbp2lItO6yRG9yt7ODQ= -github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240509031412-4975b6ec206f/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= +github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240619030755-1e16b374e6cb h1:aGbaiFpYmDfj+D0qJF+wJ9MbkftG0MbGNUwf6XClnFY= +github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240619030755-1e16b374e6cb/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= +github.com/ledgerwatch/erigonwatch v0.1.2 h1:/jq0r3oFh61pYk65Rw10aeCJj2Mzs1E2AG6TXG668og= +github.com/ledgerwatch/erigonwatch v0.1.2/go.mod h1:5K2cWaom0/rURye4dUUEQg2UyCH2A5zHVp86TDDMaA4= github.com/ledgerwatch/log/v3 v3.9.0 h1:iDwrXe0PVwBC68Dd94YSsHbMgQ3ufsgjzXtFNFVZFRk= github.com/ledgerwatch/log/v3 v3.9.0/go.mod h1:EiAY6upmI/6LkNhOVxb4eVsmsP11HZCnZ3PlJMjYiqE= github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ= @@ -669,6 +673,8 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -875,6 +881,8 @@ github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtD github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= @@ -908,8 +916,8 @@ github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnj github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/showwin/speedtest-go v1.6.12 h1:q+hWNn2cM35KkqtXGGbSmuJgd67gTP8+VlneY2hq9vU= -github.com/showwin/speedtest-go v1.6.12/go.mod h1:uLgdWCNarXxlYsL2E5TOZpCIwpgSWnEANZp7gfHXHu0= +github.com/showwin/speedtest-go v1.7.7 h1:VmK75SZOTKiuWjIVrs+mo7ZoKEw0utiGCvpnurS0olU= +github.com/showwin/speedtest-go v1.7.7/go.mod h1:uLgdWCNarXxlYsL2E5TOZpCIwpgSWnEANZp7gfHXHu0= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= diff --git a/p2p/discover/v4_udp.go b/p2p/discover/v4_udp.go index 5491f541744..e3f2a300206 100644 --- a/p2p/discover/v4_udp.go +++ b/p2p/discover/v4_udp.go @@ -605,14 +605,15 @@ func (t *UDPv4) loop() { return case p := <-t.addReplyMatcher: - func() { - mutex.Lock() - defer mutex.Unlock() - p.deadline = time.Now().Add(t.replyTimeout) - listUpdate <- plist.PushBack(p) - }() + mutex.Lock() + p.deadline = time.Now().Add(t.replyTimeout) + back := plist.PushBack(p) + mutex.Unlock() + listUpdate <- back case r := <-t.gotreply: + var removals []*list.Element + func() { mutex.Lock() defer mutex.Unlock() @@ -628,7 +629,7 @@ func (t *UDPv4) loop() { if requestDone { p.errc <- nil plist.Remove(el) - listUpdate <- el + removals = append(removals, el) } // Reset the continuous timeout counter (time drift detection) contTimeouts = 0 @@ -637,6 +638,10 @@ func (t *UDPv4) loop() { r.matched <- matched }() + for _, el := range removals { + listUpdate <- el + } + case key := <-t.gotkey: go func() { if key, err := v4wire.DecodePubkey(crypto.S256(), key); err == nil { diff --git a/turbo/adapter/ethapi/api.go b/turbo/adapter/ethapi/api.go index 534bd70d6cd..b3b1b39d340 100644 --- a/turbo/adapter/ethapi/api.go +++ b/turbo/adapter/ethapi/api.go @@ -112,7 +112,7 @@ func (args *CallArgs) ToMessage(globalGasCap uint64, baseFee *uint256.Int) (type gasFeeCap, gasTipCap = gasPrice, gasPrice } else { // User specified 1559 gas fields (or none), use those - gasFeeCap = uint256.MustFromBig(baseFee.ToBig()) + gasFeeCap = new(uint256.Int) if args.MaxFeePerGas != nil { overflow := gasFeeCap.SetFromBig(args.MaxFeePerGas.ToInt()) if overflow { diff --git a/turbo/cli/flags.go b/turbo/cli/flags.go index f9cb542d1df..a6eb93b9a6b 100644 --- a/turbo/cli/flags.go +++ b/turbo/cli/flags.go @@ -488,6 +488,7 @@ func setEmbeddedRpcDaemon(ctx *cli.Context, cfg *nodecfg.Config, logger log.Logg DBReadConcurrency: ctx.Int(utils.DBReadConcurrencyFlag.Name), RpcAllowListFilePath: ctx.String(utils.RpcAccessListFlag.Name), Gascap: ctx.Uint64(utils.RpcGasCapFlag.Name), + Feecap: ctx.Float64(utils.RPCGlobalTxFeeCapFlag.Name), MaxTraces: ctx.Uint64(utils.TraceMaxtracesFlag.Name), TraceCompatibility: ctx.Bool(utils.RpcTraceCompatFlag.Name), BatchLimit: ctx.Int(utils.RpcBatchLimit.Name), diff --git a/turbo/engineapi/engine_server.go b/turbo/engineapi/engine_server.go index 25d8f75c2d8..8bf7f3d16f0 100644 --- a/turbo/engineapi/engine_server.go +++ b/turbo/engineapi/engine_server.go @@ -91,7 +91,7 @@ func (e *EngineServer) Start( ) { base := jsonrpc.NewBaseApi(filters, stateCache, blockReader, agg, httpConfig.WithDatadir, httpConfig.EvmCallTimeout, engineReader, httpConfig.Dirs) - ethImpl := jsonrpc.NewEthAPI(base, db, eth, txPool, mining, httpConfig.Gascap, httpConfig.ReturnDataLimit, ðconfig.Defaults, httpConfig.AllowUnprotectedTxs, httpConfig.MaxGetProofRewindBlockCount, httpConfig.WebsocketSubscribeLogsChannelSize, e.logger) + ethImpl := jsonrpc.NewEthAPI(base, db, eth, txPool, mining, httpConfig.Gascap, httpConfig.Feecap, httpConfig.ReturnDataLimit, ðconfig.Defaults, httpConfig.AllowUnprotectedTxs, httpConfig.MaxGetProofRewindBlockCount, httpConfig.WebsocketSubscribeLogsChannelSize, e.logger) // engineImpl := NewEngineAPI(base, db, engineBackend) // e.startEngineMessageHandler() @@ -770,7 +770,18 @@ func (e *EngineServer) HandleNewPayload( if !success { return &engine_types.PayloadStatus{Status: engine_types.SyncingStatus}, nil } - return &engine_types.PayloadStatus{Status: engine_types.ValidStatus, LatestValidHash: &headerHash}, nil + + status, _, latestValidHash, err := e.chainRW.ValidateChain(ctx, headerHash, headerNumber) + if err != nil { + return nil, err + } + + if status == execution.ExecutionStatus_Busy || status == execution.ExecutionStatus_TooFarAway { + e.logger.Debug(fmt.Sprintf("[%s] New payload: Client is still syncing", logPrefix)) + return &engine_types.PayloadStatus{Status: engine_types.SyncingStatus}, nil + } else { + return &engine_types.PayloadStatus{Status: engine_types.ValidStatus, LatestValidHash: &latestValidHash}, nil + } } else { return &engine_types.PayloadStatus{Status: engine_types.SyncingStatus}, nil } diff --git a/turbo/jsonrpc/corner_cases_support_test.go b/turbo/jsonrpc/corner_cases_support_test.go index 6aca590a868..faf58a71475 100644 --- a/turbo/jsonrpc/corner_cases_support_test.go +++ b/turbo/jsonrpc/corner_cases_support_test.go @@ -7,12 +7,14 @@ import ( "context" "testing" - "github.com/ledgerwatch/erigon-lib/common" "github.com/stretchr/testify/require" + "github.com/ledgerwatch/erigon-lib/common" + + "github.com/ledgerwatch/log/v3" + "github.com/ledgerwatch/erigon/cmd/rpcdaemon/rpcdaemontest" "github.com/ledgerwatch/erigon/rpc" - "github.com/ledgerwatch/log/v3" ) // TestNotFoundMustReturnNil - next methods - when record not found in db - must return nil instead of error @@ -21,7 +23,7 @@ func TestNotFoundMustReturnNil(t *testing.T) { require := require.New(t) m, _, _ := rpcdaemontest.CreateTestSentry(t) api := NewEthAPI(newBaseApiForTest(m), - m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) ctx := context.Background() a, err := api.GetTransactionByBlockNumberAndIndex(ctx, 10_000, 1) diff --git a/turbo/jsonrpc/daemon.go b/turbo/jsonrpc/daemon.go index 57ea816af14..953a65adb98 100644 --- a/turbo/jsonrpc/daemon.go +++ b/turbo/jsonrpc/daemon.go @@ -1,7 +1,10 @@ package jsonrpc import ( + "github.com/ledgerwatch/log/v3" + "github.com/ledgerwatch/erigon-lib/gointerfaces/txpool" + "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/kvcache" libstate "github.com/ledgerwatch/erigon-lib/state" @@ -16,7 +19,6 @@ import ( "github.com/ledgerwatch/erigon/turbo/services" "github.com/ledgerwatch/erigon/zk/sequencer" "github.com/ledgerwatch/erigon/zk/syncer" - "github.com/ledgerwatch/log/v3" txpool2 "github.com/ledgerwatch/erigon/zk/txpool" ) @@ -35,7 +37,7 @@ func APIList(db kv.RoDB, eth rpchelper.ApiBackend, txPool txpool.TxpoolClient, r base := NewBaseApi(filters, stateCache, blockReader, agg, cfg.WithDatadir, cfg.EvmCallTimeout, engine, cfg.Dirs) base.SetL2RpcUrl(ethCfg.Zk.L2RpcUrl) - ethImpl := NewEthAPI(base, db, eth, txPool, mining, cfg.Gascap, cfg.ReturnDataLimit, ethCfg, cfg.AllowUnprotectedTxs, cfg.MaxGetProofRewindBlockCount, cfg.WebsocketSubscribeLogsChannelSize, logger) + ethImpl := NewEthAPI(base, db, eth, txPool, mining, cfg.Gascap, cfg.Feecap, cfg.ReturnDataLimit, ethCfg, cfg.AllowUnprotectedTxs, cfg.MaxGetProofRewindBlockCount, cfg.WebsocketSubscribeLogsChannelSize, logger) erigonImpl := NewErigonAPI(base, db, eth) txpoolImpl := NewTxPoolAPI(base, db, txPool, rawPool, rpcUrl) netImpl := NewNetAPIImpl(eth) diff --git a/turbo/jsonrpc/debug_api_test.go b/turbo/jsonrpc/debug_api_test.go index 15d79187c8f..02e8f5fb899 100644 --- a/turbo/jsonrpc/debug_api_test.go +++ b/turbo/jsonrpc/debug_api_test.go @@ -11,6 +11,9 @@ import ( "github.com/davecgh/go-spew/spew" jsoniter "github.com/json-iterator/go" + "github.com/ledgerwatch/log/v3" + "github.com/stretchr/testify/require" + "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/iter" @@ -22,8 +25,6 @@ import ( "github.com/ledgerwatch/erigon/rpc" "github.com/ledgerwatch/erigon/rpc/rpccfg" "github.com/ledgerwatch/erigon/turbo/adapter/ethapi" - "github.com/ledgerwatch/log/v3" - "github.com/stretchr/testify/require" ) var dumper = spew.ConfigState{Indent: " "} @@ -55,7 +56,7 @@ func TestTraceBlockByNumber(t *testing.T) { agg := m.HistoryV3Components() stateCache := kvcache.New(kvcache.DefaultCoherentConfig) baseApi := NewBaseApi(nil, stateCache, m.BlockReader, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs) - ethApi := NewEthAPI(baseApi, m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + ethApi := NewEthAPI(baseApi, m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) api := NewPrivateDebugAPI(baseApi, m.DB, 0) for _, tt := range debugTraceTransactionTests { var buf bytes.Buffer @@ -100,7 +101,7 @@ func TestTraceBlockByNumber(t *testing.T) { func TestTraceBlockByHash(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - ethApi := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + ethApi := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) api := NewPrivateDebugAPI(newBaseApiForTest(m), m.DB, 0) for _, tt := range debugTraceTransactionTests { var buf bytes.Buffer diff --git a/turbo/jsonrpc/erigon_receipts_test.go b/turbo/jsonrpc/erigon_receipts_test.go index 740d646039a..24c512bcca3 100644 --- a/turbo/jsonrpc/erigon_receipts_test.go +++ b/turbo/jsonrpc/erigon_receipts_test.go @@ -10,11 +10,14 @@ import ( "testing" "github.com/holiman/uint256" - libcommon "github.com/ledgerwatch/erigon-lib/common" - "github.com/ledgerwatch/erigon-lib/kv" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/kv" + + "github.com/ledgerwatch/log/v3" + "github.com/ledgerwatch/erigon/cmd/rpcdaemon/rpcdaemontest" "github.com/ledgerwatch/erigon/common" "github.com/ledgerwatch/erigon/core" @@ -25,14 +28,13 @@ import ( "github.com/ledgerwatch/erigon/params" "github.com/ledgerwatch/erigon/rpc" "github.com/ledgerwatch/erigon/turbo/stages/mock" - "github.com/ledgerwatch/log/v3" ) func TestGetLogs(t *testing.T) { assert := assert.New(t) m, _, _ := rpcdaemontest.CreateTestSentry(t) { - ethApi := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + ethApi := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) logs, err := ethApi.GetLogs(context.Background(), filters.FilterCriteria{FromBlock: big.NewInt(0), ToBlock: big.NewInt(10)}) assert.NoError(err) diff --git a/turbo/jsonrpc/eth_api.go b/turbo/jsonrpc/eth_api.go index d6df3d316d8..fb590acfac5 100644 --- a/turbo/jsonrpc/eth_api.go +++ b/turbo/jsonrpc/eth_api.go @@ -94,7 +94,7 @@ type EthAPI interface { // Sending related (see ./eth_call.go) Call(ctx context.Context, args ethapi2.CallArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *ethapi2.StateOverrides) (hexutility.Bytes, error) - EstimateGas(ctx context.Context, argsOrNil *ethapi2.CallArgs) (hexutil.Uint64, error) + EstimateGas(ctx context.Context, argsOrNil *ethapi2.CallArgs, blockNrOrHash *rpc.BlockNumberOrHash) (hexutil.Uint64, error) SendRawTransaction(ctx context.Context, encodedTx hexutility.Bytes) (common.Hash, error) SendTransaction(_ context.Context, txObject interface{}) (common.Hash, error) Sign(ctx context.Context, _ common.Address, _ hexutility.Bytes) (hexutility.Bytes, error) @@ -360,6 +360,7 @@ type APIImpl struct { gasCache *GasPriceCache db kv.RoDB GasCap uint64 + FeeCap float64 ReturnDataLimit int ZkRpcUrl string PoolManagerUrl string @@ -377,7 +378,7 @@ type APIImpl struct { } // NewEthAPI returns APIImpl instance -func NewEthAPI(base *BaseAPI, db kv.RoDB, eth rpchelper.ApiBackend, txPool txpool.TxpoolClient, mining txpool.MiningClient, gascap uint64, returnDataLimit int, ethCfg *ethconfig.Config, allowUnprotectedTxs bool, maxGetProofRewindBlockCount int, subscribeLogsChannelSize int, logger log.Logger) *APIImpl { +func NewEthAPI(base *BaseAPI, db kv.RoDB, eth rpchelper.ApiBackend, txPool txpool.TxpoolClient, mining txpool.MiningClient, gascap uint64, feecap float64, returnDataLimit int, ethCfg *ethconfig.Config, allowUnprotectedTxs bool, maxGetProofRewindBlockCount int, subscribeLogsChannelSize int, logger log.Logger) *APIImpl { if gascap == 0 { gascap = uint64(math.MaxUint64 / 2) } @@ -390,12 +391,13 @@ func NewEthAPI(base *BaseAPI, db kv.RoDB, eth rpchelper.ApiBackend, txPool txpoo mining: mining, gasCache: NewGasPriceCache(), GasCap: gascap, + FeeCap: feecap, + AllowUnprotectedTxs: allowUnprotectedTxs, ReturnDataLimit: returnDataLimit, ZkRpcUrl: ethCfg.L2RpcUrl, PoolManagerUrl: ethCfg.PoolManagerUrl, AllowFreeTransactions: ethCfg.AllowFreeTransactions, AllowPreEIP155Transactions: ethCfg.AllowPreEIP155Transactions, - AllowUnprotectedTxs: allowUnprotectedTxs, MaxGetProofRewindBlockCount: maxGetProofRewindBlockCount, L1RpcUrl: ethCfg.L1RpcUrl, DefaultGasPrice: ethCfg.DefaultGasPrice, diff --git a/turbo/jsonrpc/eth_api_test.go b/turbo/jsonrpc/eth_api_test.go index 00997b6e08d..e2bca25995e 100644 --- a/turbo/jsonrpc/eth_api_test.go +++ b/turbo/jsonrpc/eth_api_test.go @@ -11,9 +11,10 @@ import ( "github.com/ledgerwatch/erigon-lib/common/hexutil" "github.com/holiman/uint256" - "github.com/ledgerwatch/erigon-lib/common" "github.com/stretchr/testify/assert" + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/kv/kvcache" "github.com/ledgerwatch/erigon/core" "github.com/ledgerwatch/erigon/rpc" @@ -21,8 +22,9 @@ import ( "github.com/ledgerwatch/erigon/turbo/adapter/ethapi" "github.com/ledgerwatch/erigon/turbo/stages/mock" - "github.com/ledgerwatch/erigon/cmd/rpcdaemon/rpcdaemontest" "github.com/ledgerwatch/log/v3" + + "github.com/ledgerwatch/erigon/cmd/rpcdaemon/rpcdaemontest" ) func newBaseApiForTest(m *mock.MockSentry) *BaseAPI { @@ -58,7 +60,7 @@ func TestGetTransactionReceipt(t *testing.T) { db := m.DB agg := m.HistoryV3Components() stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, m.BlockReader, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs), db, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + api := NewEthAPI(NewBaseApi(nil, stateCache, m.BlockReader, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs), db, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) // Call GetTransactionReceipt for transaction which is not in the database if _, err := api.GetTransactionReceipt(context.Background(), common.Hash{}); err != nil { t.Errorf("calling GetTransactionReceipt with empty hash: %v", err) @@ -67,7 +69,7 @@ func TestGetTransactionReceipt(t *testing.T) { func TestGetTransactionReceiptUnprotected(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) // Call GetTransactionReceipt for un-protected transaction if _, err := api.GetTransactionReceipt(context.Background(), common.HexToHash("0x3f3cb8a0e13ed2481f97f53f7095b9cbc78b6ffb779f2d3e565146371a8830ea")); err != nil { t.Errorf("calling GetTransactionReceipt for unprotected tx: %v", err) @@ -79,7 +81,7 @@ func TestGetTransactionReceiptUnprotected(t *testing.T) { func TestGetStorageAt_ByBlockNumber_WithRequireCanonicalDefault(t *testing.T) { assert := assert.New(t) m, _, _ := rpcdaemontest.CreateTestSentry(t) - api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") result, err := api.GetStorageAt(context.Background(), addr, "0x0", rpc.BlockNumberOrHashWithNumber(0)) @@ -93,7 +95,7 @@ func TestGetStorageAt_ByBlockNumber_WithRequireCanonicalDefault(t *testing.T) { func TestGetStorageAt_ByBlockHash_WithRequireCanonicalDefault(t *testing.T) { assert := assert.New(t) m, _, _ := rpcdaemontest.CreateTestSentry(t) - api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") result, err := api.GetStorageAt(context.Background(), addr, "0x0", rpc.BlockNumberOrHashWithHash(m.Genesis.Hash(), false)) @@ -107,7 +109,7 @@ func TestGetStorageAt_ByBlockHash_WithRequireCanonicalDefault(t *testing.T) { func TestGetStorageAt_ByBlockHash_WithRequireCanonicalTrue(t *testing.T) { assert := assert.New(t) m, _, _ := rpcdaemontest.CreateTestSentry(t) - api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") result, err := api.GetStorageAt(context.Background(), addr, "0x0", rpc.BlockNumberOrHashWithHash(m.Genesis.Hash(), true)) @@ -120,7 +122,7 @@ func TestGetStorageAt_ByBlockHash_WithRequireCanonicalTrue(t *testing.T) { func TestGetStorageAt_ByBlockHash_WithRequireCanonicalDefault_BlockNotFoundError(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") offChain, err := core.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 1, func(i int, block *core.BlockGen) { @@ -141,7 +143,7 @@ func TestGetStorageAt_ByBlockHash_WithRequireCanonicalDefault_BlockNotFoundError func TestGetStorageAt_ByBlockHash_WithRequireCanonicalTrue_BlockNotFoundError(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") offChain, err := core.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 1, func(i int, block *core.BlockGen) { @@ -163,7 +165,7 @@ func TestGetStorageAt_ByBlockHash_WithRequireCanonicalTrue_BlockNotFoundError(t func TestGetStorageAt_ByBlockHash_WithRequireCanonicalDefault_NonCanonicalBlock(t *testing.T) { assert := assert.New(t) m, _, orphanedChain := rpcdaemontest.CreateTestSentry(t) - api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") orphanedBlock := orphanedChain[0].Blocks[0] @@ -182,7 +184,7 @@ func TestGetStorageAt_ByBlockHash_WithRequireCanonicalDefault_NonCanonicalBlock( func TestGetStorageAt_ByBlockHash_WithRequireCanonicalTrue_NonCanonicalBlock(t *testing.T) { m, _, orphanedChain := rpcdaemontest.CreateTestSentry(t) - api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") orphanedBlock := orphanedChain[0].Blocks[0] @@ -198,7 +200,7 @@ func TestGetStorageAt_ByBlockHash_WithRequireCanonicalTrue_NonCanonicalBlock(t * func TestCall_ByBlockHash_WithRequireCanonicalDefault_NonCanonicalBlock(t *testing.T) { m, _, orphanedChain := rpcdaemontest.CreateTestSentry(t) - api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) from := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") to := common.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e") @@ -221,7 +223,7 @@ func TestCall_ByBlockHash_WithRequireCanonicalDefault_NonCanonicalBlock(t *testi func TestCall_ByBlockHash_WithRequireCanonicalTrue_NonCanonicalBlock(t *testing.T) { m, _, orphanedChain := rpcdaemontest.CreateTestSentry(t) - api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) from := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") to := common.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e") diff --git a/turbo/jsonrpc/eth_block_test.go b/turbo/jsonrpc/eth_block_test.go index db5a2d154ff..699475d6ae6 100644 --- a/turbo/jsonrpc/eth_block_test.go +++ b/turbo/jsonrpc/eth_block_test.go @@ -10,10 +10,13 @@ import ( "github.com/ledgerwatch/erigon-lib/common/hexutil" + "github.com/stretchr/testify/assert" + "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/gointerfaces/txpool" "github.com/ledgerwatch/erigon-lib/kv/kvcache" - "github.com/stretchr/testify/assert" + + "github.com/ledgerwatch/log/v3" "github.com/ledgerwatch/erigon/cmd/rpcdaemon/rpcdaemontest" "github.com/ledgerwatch/erigon/core/rawdb" @@ -23,13 +26,12 @@ import ( "github.com/ledgerwatch/erigon/rpc/rpccfg" "github.com/ledgerwatch/erigon/turbo/rpchelper" "github.com/ledgerwatch/erigon/turbo/stages/mock" - "github.com/ledgerwatch/log/v3" ) // Gets the latest block number with the latest tag func TestGetBlockByNumberWithLatestTag(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) b, err := api.GetBlockByNumber(context.Background(), rpc.LatestBlockNumber, false) expected := common.HexToHash("0x5883164d4100b95e1d8e931b8b9574586a1dea7507941e6ad3c1e3a2591485fd") if err != nil { @@ -59,7 +61,7 @@ func TestGetBlockByNumberWithLatestTag_WithHeadHashInDb(t *testing.T) { } tx.Commit() - api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) block, err := api.GetBlockByNumber(ctx, rpc.LatestBlockNumber, false) if err != nil { t.Errorf("error retrieving block by number: %s", err) @@ -90,7 +92,7 @@ func TestGetBlockByNumberWithPendingTag(t *testing.T) { RplBlock: rlpBlock, }) - api := NewEthAPI(NewBaseApi(ff, stateCache, m.BlockReader, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + api := NewEthAPI(NewBaseApi(ff, stateCache, m.BlockReader, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) b, err := api.GetBlockByNumber(context.Background(), rpc.PendingBlockNumber, false) if err != nil { t.Errorf("error getting block number with pending tag: %s", err) @@ -101,7 +103,7 @@ func TestGetBlockByNumberWithPendingTag(t *testing.T) { func TestGetBlockByNumber_WithFinalizedTag_NoFinalizedBlockInDb(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) ctx := context.Background() - api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) if _, err := api.GetBlockByNumber(ctx, rpc.FinalizedBlockNumber, false); err != nil { assert.ErrorIs(t, rpchelper.UnknownBlockError, err) } @@ -128,7 +130,7 @@ func TestGetBlockByNumber_WithFinalizedTag_WithFinalizedBlockInDb(t *testing.T) } tx.Commit() - api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) block, err := api.GetBlockByNumber(ctx, rpc.FinalizedBlockNumber, false) if err != nil { t.Errorf("error retrieving block by number: %s", err) @@ -140,7 +142,7 @@ func TestGetBlockByNumber_WithFinalizedTag_WithFinalizedBlockInDb(t *testing.T) func TestGetBlockByNumber_WithSafeTag_NoSafeBlockInDb(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) ctx := context.Background() - api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) if _, err := api.GetBlockByNumber(ctx, rpc.SafeBlockNumber, false); err != nil { assert.ErrorIs(t, rpchelper.UnknownBlockError, err) } @@ -167,7 +169,7 @@ func TestGetBlockByNumber_WithSafeTag_WithSafeBlockInDb(t *testing.T) { } tx.Commit() - api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) block, err := api.GetBlockByNumber(ctx, rpc.SafeBlockNumber, false) if err != nil { t.Errorf("error retrieving block by number: %s", err) @@ -180,7 +182,7 @@ func TestGetBlockTransactionCountByHash(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) ctx := context.Background() - api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) blockHash := common.HexToHash("0x6804117de2f3e6ee32953e78ced1db7b20214e0d8c745a03b8fecf7cc8ee76ef") tx, err := m.DB.BeginRw(ctx) @@ -212,7 +214,7 @@ func TestGetBlockTransactionCountByHash(t *testing.T) { func TestGetBlockTransactionCountByHash_ZeroTx(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) ctx := context.Background() - api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) blockHash := common.HexToHash("0x5883164d4100b95e1d8e931b8b9574586a1dea7507941e6ad3c1e3a2591485fd") tx, err := m.DB.BeginRw(ctx) @@ -244,7 +246,7 @@ func TestGetBlockTransactionCountByHash_ZeroTx(t *testing.T) { func TestGetBlockTransactionCountByNumber(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) ctx := context.Background() - api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) blockHash := common.HexToHash("0x6804117de2f3e6ee32953e78ced1db7b20214e0d8c745a03b8fecf7cc8ee76ef") tx, err := m.DB.BeginRw(ctx) @@ -276,7 +278,7 @@ func TestGetBlockTransactionCountByNumber(t *testing.T) { func TestGetBlockTransactionCountByNumber_ZeroTx(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) ctx := context.Background() - api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) blockHash := common.HexToHash("0x5883164d4100b95e1d8e931b8b9574586a1dea7507941e6ad3c1e3a2591485fd") diff --git a/turbo/jsonrpc/eth_call.go b/turbo/jsonrpc/eth_call.go index 7f4d7f621a3..061ececf1ff 100644 --- a/turbo/jsonrpc/eth_call.go +++ b/turbo/jsonrpc/eth_call.go @@ -114,7 +114,7 @@ func headerByNumberOrHash(ctx context.Context, tx kv.Tx, blockNrOrHash rpc.Block } // EstimateGas implements eth_estimateGas. Returns an estimate of how much gas is necessary to allow the transaction to complete. The transaction will not be added to the blockchain. -func (api *APIImpl) EstimateGas(ctx context.Context, argsOrNil *ethapi2.CallArgs) (hexutil.Uint64, error) { +func (api *APIImpl) EstimateGas(ctx context.Context, argsOrNil *ethapi2.CallArgs, blockNrOrHash *rpc.BlockNumberOrHash) (hexutil.Uint64, error) { var args ethapi2.CallArgs // if we actually get CallArgs here, we use them if argsOrNil != nil { @@ -138,39 +138,36 @@ func (api *APIImpl) EstimateGas(ctx context.Context, argsOrNil *ethapi2.CallArgs args.From = new(libcommon.Address) } - chainConfig, err := api.chainConfig(ctx, dbtx) - if err != nil { - return 0, err - } - - latestCanBlockNumber, latestCanHash, isLatest, err := rpchelper.GetCanonicalBlockNumber(latestNumOrHash, dbtx, api.filters) // DoCall cannot be executed on non-canonical blocks - if err != nil { - return 0, err - } - - // try and get the block from the lru cache first then try DB before failing - block := api.tryBlockFromLru(latestCanHash) - if block == nil { - block, err = api.blockWithSenders(ctx, dbtx, latestCanHash, latestCanBlockNumber) - if err != nil { - return 0, err - } - } - if block == nil { - return 0, fmt.Errorf("could not find latest block in cache or db") - } - - stateReader, err := rpchelper.CreateStateReaderFromBlockNumber(ctx, dbtx, latestCanBlockNumber, isLatest, 0, api.stateCache, api.historyV3(dbtx), chainConfig.ChainName) - if err != nil { - return 0, err + bNrOrHash := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) + if blockNrOrHash != nil { + bNrOrHash = *blockNrOrHash } - header := block.HeaderNoCopy() // Determine the highest gas limit can be used during the estimation. if args.Gas != nil && uint64(*args.Gas) >= params.TxGas { hi = uint64(*args.Gas) } else { - hi = header.GasLimit + // Retrieve the block to act as the gas ceiling + h, err := headerByNumberOrHash(ctx, dbtx, bNrOrHash, api) + if err != nil { + return 0, err + } + if h == nil { + // if a block number was supplied and there is no header return 0 + if blockNrOrHash != nil { + return 0, nil + } + + // block number not supplied, so we haven't found a pending block, read the latest block instead + h, err = headerByNumberOrHash(ctx, dbtx, latestNumOrHash, api) + if err != nil { + return 0, err + } + if h == nil { + return 0, nil + } + } + hi = h.GasLimit } var feeCap *big.Int @@ -180,8 +177,6 @@ func (api *APIImpl) EstimateGas(ctx context.Context, argsOrNil *ethapi2.CallArgs feeCap = args.GasPrice.ToInt() } else if args.MaxFeePerGas != nil { feeCap = args.MaxFeePerGas.ToInt() - } else if header.BaseFee != nil { - feeCap = new(big.Int).Set(header.BaseFee) } else { feeCap = libcommon.Big0 } @@ -226,8 +221,35 @@ func (api *APIImpl) EstimateGas(ctx context.Context, argsOrNil *ethapi2.CallArgs } gasCap = hi + chainConfig, err := api.chainConfig(ctx, dbtx) + if err != nil { + return 0, err + } engine := api.engine() + latestCanBlockNumber, latestCanHash, isLatest, err := rpchelper.GetCanonicalBlockNumber(latestNumOrHash, dbtx, api.filters) // DoCall cannot be executed on non-canonical blocks + if err != nil { + return 0, err + } + + // try and get the block from the lru cache first then try DB before failing + block := api.tryBlockFromLru(latestCanHash) + if block == nil { + block, err = api.blockWithSenders(ctx, dbtx, latestCanHash, latestCanBlockNumber) + if err != nil { + return 0, err + } + } + if block == nil { + return 0, fmt.Errorf("could not find latest block in cache or db") + } + + stateReader, err := rpchelper.CreateStateReaderFromBlockNumber(ctx, dbtx, latestCanBlockNumber, isLatest, 0, api.stateCache, api.historyV3(dbtx), chainConfig.ChainName) + if err != nil { + return 0, err + } + header := block.HeaderNoCopy() + caller, err := transactions.NewReusableCaller(engine, stateReader, nil, header, args, api.GasCap, latestNumOrHash, dbtx, api._blockReader, chainConfig, api.evmCallTimeout) if err != nil { return 0, err diff --git a/turbo/jsonrpc/eth_callMany_test.go b/turbo/jsonrpc/eth_callMany_test.go index e72589ae79b..c93ebef8208 100644 --- a/turbo/jsonrpc/eth_callMany_test.go +++ b/turbo/jsonrpc/eth_callMany_test.go @@ -14,6 +14,8 @@ import ( "github.com/ledgerwatch/erigon-lib/common/hexutility" "github.com/ledgerwatch/erigon-lib/kv/kvcache" + "github.com/ledgerwatch/log/v3" + "github.com/ledgerwatch/erigon/accounts/abi/bind" "github.com/ledgerwatch/erigon/accounts/abi/bind/backends" "github.com/ledgerwatch/erigon/core/types" @@ -24,7 +26,6 @@ import ( "github.com/ledgerwatch/erigon/rpc/rpccfg" "github.com/ledgerwatch/erigon/turbo/adapter/ethapi" "github.com/ledgerwatch/erigon/turbo/jsonrpc/contracts" - "github.com/ledgerwatch/log/v3" ) // block 1 contains 3 Transactions @@ -86,7 +87,7 @@ func TestCallMany(t *testing.T) { db := contractBackend.DB() engine := contractBackend.Engine() api := NewEthAPI(NewBaseApi(nil, stateCache, contractBackend.BlockReader(), contractBackend.Agg(), false, rpccfg.DefaultEvmCallTimeout, engine, - datadir.New(t.TempDir())), db, nil, nil, nil, 5000000, 100_000, ðconfig.Defaults, false, 100_000, 128, log.New()) + datadir.New(t.TempDir())), db, nil, nil, nil, 5000000, 1e18, 100_000, ðconfig.Defaults, false, 100_000, 128, log.New()) callArgAddr1 := ethapi.CallArgs{From: &address, To: &tokenAddr, Nonce: &nonce, MaxPriorityFeePerGas: (*hexutil.Big)(big.NewInt(1e9)), diff --git a/turbo/jsonrpc/eth_call_test.go b/turbo/jsonrpc/eth_call_test.go index e0b41b57239..565e86373c4 100644 --- a/turbo/jsonrpc/eth_call_test.go +++ b/turbo/jsonrpc/eth_call_test.go @@ -22,6 +22,8 @@ import ( "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/kvcache" + "github.com/ledgerwatch/log/v3" + "github.com/ledgerwatch/erigon/cmd/rpcdaemon/rpcdaemontest" "github.com/ledgerwatch/erigon/common/math" "github.com/ledgerwatch/erigon/core" @@ -36,7 +38,6 @@ import ( "github.com/ledgerwatch/erigon/turbo/rpchelper" "github.com/ledgerwatch/erigon/turbo/stages/mock" "github.com/ledgerwatch/erigon/turbo/trie" - "github.com/ledgerwatch/log/v3" ) func TestEstimateGas(t *testing.T) { @@ -46,13 +47,13 @@ func TestEstimateGas(t *testing.T) { ctx, conn := rpcdaemontest.CreateTestGrpcConn(t, mock.Mock(t)) mining := txpool.NewMiningClient(conn) ff := rpchelper.New(ctx, nil, nil, mining, func() {}, m.Log) - api := NewEthAPI(NewBaseApi(ff, stateCache, m.BlockReader, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, log.New()) + api := NewEthAPI(NewBaseApi(ff, stateCache, m.BlockReader, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) var from = libcommon.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") var to = libcommon.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e") if _, err := api.EstimateGas(context.Background(), ðapi.CallArgs{ From: &from, To: &to, - }); err != nil { + }, nil); err != nil { t.Errorf("calling EstimateGas: %v", err) } } @@ -61,7 +62,7 @@ func TestEthCallNonCanonical(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) agg := m.HistoryV3Components() stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, m.BlockReader, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, log.New()) + api := NewEthAPI(NewBaseApi(nil, stateCache, m.BlockReader, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) var from = libcommon.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") var to = libcommon.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e") if _, err := api.Call(context.Background(), ethapi.CallArgs{ @@ -80,7 +81,7 @@ func TestEthCallToPrunedBlock(t *testing.T) { m, bankAddress, contractAddress := chainWithDeployedContract(t) doPrune(t, m.DB, pruneTo) - api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) callData := hexutil.MustDecode("0x2e64cec1") callDataBytes := hexutility.Bytes(callData) @@ -101,7 +102,7 @@ func TestGetProof(t *testing.T) { if m.HistoryV3 { t.Skip("not supported by Erigon3") } - api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, false, maxGetProofRewindBlockCount, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, maxGetProofRewindBlockCount, 128, log.New()) key := func(b byte) libcommon.Hash { result := libcommon.Hash{} diff --git a/turbo/jsonrpc/eth_filters_test.go b/turbo/jsonrpc/eth_filters_test.go index 2f975ee3c6d..523d0893049 100644 --- a/turbo/jsonrpc/eth_filters_test.go +++ b/turbo/jsonrpc/eth_filters_test.go @@ -15,14 +15,18 @@ import ( "github.com/ledgerwatch/erigon/rpc/rpccfg" + "github.com/stretchr/testify/assert" + "github.com/ledgerwatch/erigon-lib/gointerfaces/txpool" + "github.com/ledgerwatch/erigon-lib/kv/kvcache" - "github.com/stretchr/testify/assert" + + "github.com/ledgerwatch/log/v3" "github.com/ledgerwatch/erigon/cmd/rpcdaemon/rpcdaemontest" "github.com/ledgerwatch/erigon/eth/filters" "github.com/ledgerwatch/erigon/turbo/rpchelper" - "github.com/ledgerwatch/erigon/turbo/snapshotsync/freezeblocks" + "github.com/ledgerwatch/erigon/turbo/stages/mock" ) func TestNewFilters(t *testing.T) { @@ -34,7 +38,7 @@ func TestNewFilters(t *testing.T) { ctx, conn := rpcdaemontest.CreateTestGrpcConn(t, m) mining := txpool.NewMiningClient(conn) ff := rpchelper.New(ctx, nil, nil, mining, func() {}, m.Log) - api := NewEthAPI(NewBaseApi(ff, stateCache, m.BlockReader, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + api := NewEthAPI(NewBaseApi(ff, stateCache, m.BlockReader, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) ptf, err := api.NewPendingTransactionFilter(ctx) assert.Nil(err) diff --git a/turbo/jsonrpc/eth_mining_test.go b/turbo/jsonrpc/eth_mining_test.go index 95b37de796a..58e6734567e 100644 --- a/turbo/jsonrpc/eth_mining_test.go +++ b/turbo/jsonrpc/eth_mining_test.go @@ -9,15 +9,17 @@ import ( "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/rpc/rpccfg" + "github.com/ledgerwatch/log/v3" + "github.com/stretchr/testify/require" + "github.com/ledgerwatch/erigon-lib/gointerfaces/txpool" + "github.com/ledgerwatch/erigon-lib/kv/kvcache" "github.com/ledgerwatch/erigon/cmd/rpcdaemon/rpcdaemontest" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/rlp" "github.com/ledgerwatch/erigon/turbo/rpchelper" "github.com/ledgerwatch/erigon/turbo/stages/mock" - "github.com/ledgerwatch/log/v3" - "github.com/stretchr/testify/require" ) func TestPendingBlock(t *testing.T) { @@ -28,7 +30,7 @@ func TestPendingBlock(t *testing.T) { stateCache := kvcache.New(kvcache.DefaultCoherentConfig) engine := ethash.NewFaker() api := NewEthAPI(NewBaseApi(ff, stateCache, m.BlockReader, nil, false, rpccfg.DefaultEvmCallTimeout, engine, - m.Dirs), nil, nil, nil, mining, 5000000, 100_000, ðconfig.Defaults, false, 100_000, 128, log.New()) + m.Dirs), nil, nil, nil, mining, 5000000, 1e18, 100_000, ðconfig.Defaults, false, 100_000, 128, log.New()) expect := uint64(12345) b, err := rlp.EncodeToBytes(types.NewBlockWithHeader(&types.Header{Number: big.NewInt(int64(expect))})) require.NoError(t, err) diff --git a/turbo/jsonrpc/eth_system_test.go b/turbo/jsonrpc/eth_system_test.go index dc4863c0b2c..67981438993 100644 --- a/turbo/jsonrpc/eth_system_test.go +++ b/turbo/jsonrpc/eth_system_test.go @@ -10,14 +10,16 @@ import ( "testing" "github.com/holiman/uint256" + libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/log/v3" + "github.com/ledgerwatch/erigon/core" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/crypto" "github.com/ledgerwatch/erigon/params" "github.com/ledgerwatch/erigon/turbo/stages/mock" - "github.com/ledgerwatch/log/v3" ) func TestGasPrice(t *testing.T) { @@ -43,7 +45,7 @@ func TestGasPrice(t *testing.T) { t.Run(testCase.description, func(t *testing.T) { m := createGasPriceTestKV(t, testCase.chainSize) defer m.DB.Close() - eth := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, false, 100_000, 128, log.New()) + eth := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 1e18, 100_000, false, 100_000, 128, log.New()) ctx := context.Background() result, err := eth.GasPrice(ctx) diff --git a/turbo/jsonrpc/send_transaction.go b/turbo/jsonrpc/send_transaction.go index 7abe99bb386..1ce8293f0dd 100644 --- a/turbo/jsonrpc/send_transaction.go +++ b/turbo/jsonrpc/send_transaction.go @@ -14,7 +14,6 @@ import ( "github.com/ledgerwatch/erigon/core/rawdb" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/crypto" - "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/params" ) @@ -48,7 +47,7 @@ func (api *APIImpl) SendRawTransaction(ctx context.Context, encodedTx hexutility // If the transaction fee cap is already specified, ensure the // fee of the given transaction is _reasonable_. - if err := checkTxFee(txn.GetPrice().ToBig(), txn.GetGas(), ethconfig.Defaults.RPCTxFeeCap); err != nil { + if err := checkTxFee(txn.GetPrice().ToBig(), txn.GetGas(), api.FeeCap); err != nil { return common.Hash{}, err } if !api.AllowPreEIP155Transactions && !txn.Protected() && !api.AllowUnprotectedTxs { diff --git a/turbo/jsonrpc/send_transaction_test.go b/turbo/jsonrpc/send_transaction_test.go index 1563547ec87..2f8e8707abe 100644 --- a/turbo/jsonrpc/send_transaction_test.go +++ b/turbo/jsonrpc/send_transaction_test.go @@ -90,7 +90,7 @@ func TestSendRawTransaction(t *testing.T) { ctx, conn := rpcdaemontest.CreateTestGrpcConn(t, mockSentry) txPool := txpool.NewTxpoolClient(conn) ff := rpchelper.New(ctx, nil, txPool, txpool.NewMiningClient(conn), func() {}, mockSentry.Log) - api := jsonrpc.NewEthAPI(newBaseApiForTest(mockSentry), mockSentry.DB, nil, txPool, nil, 5000000, 100_000, ðconfig.Defaults, false, 100_000, 128, logger) + api := jsonrpc.NewEthAPI(newBaseApiForTest(mockSentry), mockSentry.DB, nil, txPool, nil, 5000000, 1e18, 100_000, ðconfig.Defaults, false, 100_000, 128, logger) buf := bytes.NewBuffer(nil) err = txn.MarshalBinary(buf) @@ -142,7 +142,7 @@ func TestSendRawTransactionUnprotected(t *testing.T) { ctx, conn := rpcdaemontest.CreateTestGrpcConn(t, mockSentry) txPool := txpool.NewTxpoolClient(conn) ff := rpchelper.New(ctx, nil, txPool, txpool.NewMiningClient(conn), func() {}, mockSentry.Log) - api := jsonrpc.NewEthAPI(newBaseApiForTest(mockSentry), mockSentry.DB, nil, txPool, nil, 5000000, 100_000, ðconfig.Defaults, false, 100_000, 128, logger) + api := jsonrpc.NewEthAPI(newBaseApiForTest(mockSentry), mockSentry.DB, nil, txPool, nil, 5000000, 1e18, 100_000, ðconfig.Defaults, false, 100_000, 128, logger) // Enable unproteced txs flag api.AllowUnprotectedTxs = true diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index efbcefc74d9..f56a85b89de 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -249,7 +249,6 @@ func WaitForDownloader(ctx context.Context, logPrefix string, histV3, blobs bool func logStats(ctx context.Context, stats *proto_downloader.StatsReply, startTime time.Time, stagesIdsList []string, logPrefix string, logReason string) { var m runtime.MemStats - diagnostics.Send(diagnostics.SyncStagesList{Stages: stagesIdsList}) diagnostics.Send(diagnostics.SnapshotDownloadStatistics{ Downloaded: stats.BytesCompleted, Total: stats.BytesTotal, diff --git a/turbo/transactions/tracing.go b/turbo/transactions/tracing.go index 98e8306565d..7fe5f4be459 100644 --- a/turbo/transactions/tracing.go +++ b/turbo/transactions/tracing.go @@ -328,7 +328,7 @@ func ExecuteTraceTx( execCb func(evm *vm.EVM, refunds bool) (*core.ExecutionResult, error), ) error { // Run the transaction with tracing enabled. - evm := vm.NewEVM(blockCtx, txCtx, ibs, chainConfig, vm.Config{Debug: true, Tracer: tracer}) + evm := vm.NewEVM(blockCtx, txCtx, ibs, chainConfig, vm.Config{Debug: true, Tracer: tracer, NoBaseFee: true}) var refunds = true if config != nil && config.NoRefunds != nil && *config.NoRefunds {