From 3dcb2a191db82b38b9c73e2e2bd56ef4e2d6f5da Mon Sep 17 00:00:00 2001 From: Ethen Date: Wed, 25 Oct 2023 14:12:31 -0700 Subject: [PATCH] [issue-160] Fix Slack Incompatibility Error (#173) --- .github/workflows/test.yml | 2 +- .golangci.yml | 4 +- Makefile | 4 +- docs/architecture/alerting.markdown | 18 +++++- e2e/alerting_test.go | 48 ++++++++++------ e2e/heuristic_test.go | 85 ++++++++++++++--------------- e2e/mock_servers.go | 19 +++++-- e2e/setup.go | 41 ++++---------- internal/client/pagerduty.go | 2 +- internal/client/slack.go | 32 ++++++++--- internal/core/alert.go | 2 +- internal/etl/pipeline/manager.go | 12 +++- 12 files changed, 156 insertions(+), 113 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e28c52a8..ca601106 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -66,7 +66,7 @@ jobs: # await script({github, context, core}) e2e-test: - runs-on: ubuntu-latest + runs-on: larger-runner steps: - uses: actions/checkout@v3 diff --git a/.golangci.yml b/.golangci.yml index 28ad9d8f..e61c1b6e 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,7 +1,7 @@ run: # Analysis timeout, e.g. 30s, 5m. # Default: 1m - timeout: 3m + timeout: 5m # https://github.com/golangci/golangci-lint/blob/master/.golangci.reference.yml linters-settings: @@ -220,4 +220,4 @@ issues: - noctx - wrapcheck - lll - - whitespace \ No newline at end of file + - whitespace diff --git a/Makefile b/Makefile index 23cb1007..6379d90f 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ GREEN = \033[0;32m BLUE = \033[0;34m COLOR_END = \033[0;39m -TEST_LIMIT = 120s +TEST_LIMIT = 500s build-app: @echo "$(BLUE)ยป building application binary... $(COLOR_END)" @@ -68,4 +68,4 @@ metrics-docs: build-app devnet-allocs: @echo "$(GREEN) Generating devnet allocs...$(COLOR_END)" - @./scripts/devnet-allocs.sh \ No newline at end of file + @./scripts/devnet-allocs.sh diff --git a/docs/architecture/alerting.markdown b/docs/architecture/alerting.markdown index dc1b856a..188dc9b6 100644 --- a/docs/architecture/alerting.markdown +++ b/docs/architecture/alerting.markdown @@ -57,7 +57,23 @@ An alert destination is a configurable destination that an alert can be sent to. #### Slack -The Slack alert destination is a configurable destination that allows alerts to be sent to a specific Slack channel. The Slack alert destination will be configured with a Slack webhook URL. The Slack alert destination will then use this URL to send alerts to the specified Slack channel. +The Slack alert destination is a configurable destination that allows alerts to be sent to a specific Slack channel. The Slack alert destination will be configured with a Slack webhook URL. The Slack alert destination will then use this URL to send alerts to the specified Slack channel. + +#### Setting up Slack +1. Add the [Incoming WebHooks](https://test-2kg5313.slack.com/apps/A0F7XDUAZ-incoming-webhooks?utm_source=in-prod&utm_medium=inprod-btn_app_install-index-click&tab=more_info) app to your Slack workspace. +2. Using the app, add a new webhook to your workspace for some specific channel. +3. Copy the webhook URL into an [alert routing](../alert-routing.md) entry for some severity level. This should look something like: +```yml +alertRoutes: + low: + slack: + low_oncall: + url: "https://hooks.slack.com/services/{API_KEY}" + channel: "#make-onchain-less-boring" + +``` + +Done! You should now see any generated alerts being forwarded to your specified Slack channel. #### PagerDuty diff --git a/e2e/alerting_test.go b/e2e/alerting_test.go index 20389475..27047394 100644 --- a/e2e/alerting_test.go +++ b/e2e/alerting_test.go @@ -10,9 +10,11 @@ import ( "github.com/base-org/pessimism/internal/api/models" "github.com/base-org/pessimism/internal/core" "github.com/ethereum-optimism/optimism/op-bindings/bindings" + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/wait" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/core/types" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // TestMultiDirectiveRouting ... Tests the E2E flow of a contract event heuristic with high priority alerts all @@ -24,7 +26,7 @@ func TestMultiDirectiveRouting(t *testing.T) { updateSig := "ConfigUpdate(uint256,uint8,bytes)" alertMsg := "System config gas config updated" - _, err := ts.App.BootStrap([]*models.SessionRequestParams{{ + ids, err := ts.App.BootStrap([]*models.SessionRequestParams{{ Network: core.Layer1.String(), PType: core.Live.String(), HeuristicType: core.ContractEvent.String(), @@ -32,7 +34,7 @@ func TestMultiDirectiveRouting(t *testing.T) { EndHeight: nil, AlertingParams: &core.AlertPolicy{ Msg: alertMsg, - Sev: core.HIGH.String(), + Sev: core.HIGH.String(), // The use of HIGH priority should trigger all alert destinations }, SessionParams: map[string]interface{}{ "address": ts.Cfg.L1Deployments.SystemConfigProxy.String(), @@ -40,33 +42,45 @@ func TestMultiDirectiveRouting(t *testing.T) { }, }}) - assert.NoError(t, err, "Error bootstrapping heuristic session") + require.Len(t, ids, 1, "Incorrect number of heuristic sessions created") + require.NoError(t, err, "Error bootstrapping heuristic session") sysCfg, err := bindings.NewSystemConfig(ts.Cfg.L1Deployments.SystemConfigProxy, ts.L1Client) - assert.NoError(t, err, "Error getting system config") + require.NoError(t, err, "Error getting system config") opts, err := bind.NewKeyedTransactorWithChainID(ts.Cfg.Secrets.SysCfgOwner, ts.Cfg.L1ChainIDBig()) - assert.NoError(t, err, "Error getting system config owner pk") + require.NoError(t, err, "Error getting system config owner pk") overhead := big.NewInt(10000) scalar := big.NewInt(1) tx, err := sysCfg.SetGasConfig(opts, overhead, scalar) - assert.NoError(t, err, "Error setting gas config") + require.NoError(t, err, "Error setting gas config") + + receipt, err := wait.ForReceipt(context.Background(), ts.L1Client, tx.Hash(), types.ReceiptStatusSuccessful) - txTimeoutDuration := 10 * time.Duration(ts.Cfg.DeployConfig.L1BlockTime) * time.Second - receipt, err := e2e.WaitForTransaction(tx.Hash(), ts.L1Client, txTimeoutDuration) + require.NoError(t, err, "Error waiting for transaction") + require.Equal(t, receipt.Status, types.ReceiptStatusSuccessful, "transaction failed") - assert.NoError(t, err, "Error waiting for transaction") - assert.Equal(t, receipt.Status, types.ReceiptStatusSuccessful, "transaction failed") + // Wait for Pessimism to process the newly emitted event and send a notification to the mocked Slack + // and PagerDuty servers. + require.NoError(t, wait.For(context.Background(), 500*time.Millisecond, func() (bool, error) { + pUUID := ids[0].PUUID + height, err := ts.Subsystems.PipelineHeight(pUUID) + if err != nil { + return false, err + } + + return height.Uint64() > receipt.BlockNumber.Uint64(), nil + })) - time.Sleep(3 * time.Second) slackPosts := ts.TestSlackSvr.SlackAlerts() pdPosts := ts.TestPagerDutyServer.PagerDutyAlerts() // Expect 2 alerts to each destination as alert-routing-cfg.yaml has two slack and two pagerduty destinations - assert.Equal(t, 2, len(slackPosts), "Incorrect Number of slack posts sent") - assert.Equal(t, 2, len(pdPosts), "Incorrect Number of pagerduty posts sent") + require.Equal(t, 2, len(slackPosts), "Incorrect Number of slack posts sent") + require.Equal(t, 2, len(pdPosts), "Incorrect Number of pagerduty posts sent") + assert.Contains(t, slackPosts[0].Text, "contract_event", "System contract event alert was not sent") assert.Contains(t, slackPosts[1].Text, "contract_event", "System contract event alert was not sent") assert.Contains(t, pdPosts[0].Payload.Summary, "contract_event", "System contract event alert was not sent") @@ -103,11 +117,11 @@ func TestCoolDown(t *testing.T) { }, }}) - assert.NoError(t, err, "Failed to bootstrap balance enforcement heuristic session") + require.NoError(t, err, "Failed to bootstrap balance enforcement heuristic session") // Get Alice's balance. aliceAmt, err := ts.L2Geth.L2Client.BalanceAt(context.Background(), alice, nil) - assert.NoError(t, err, "Failed to get Alice's balance") + require.NoError(t, err, "Failed to get Alice's balance") // Determine the gas cost of the transaction. gasAmt := 1_000_001 @@ -133,7 +147,7 @@ func TestCoolDown(t *testing.T) { // Send the transaction to drain Alice's account of almost all ETH. _, err = ts.L2Geth.AddL2Block(context.Background(), drainAliceTx) - assert.NoError(t, err, "Failed to create L2 block with transaction") + require.NoError(t, err, "Failed to create L2 block with transaction") // Wait for Pessimism to process the balance change and send a notification to the mocked Slack server. time.Sleep(2 * time.Second) @@ -141,7 +155,7 @@ func TestCoolDown(t *testing.T) { // Check that the balance enforcement was triggered using the mocked server cache. posts := ts.TestSlackSvr.SlackAlerts() - assert.Equal(t, 1, len(posts), "No balance enforcement alert was sent") + require.Equal(t, 1, len(posts), "No balance enforcement alert was sent") assert.Contains(t, posts[0].Text, "balance_enforcement", "Balance enforcement alert was not sent") assert.Contains(t, posts[0].Text, alertMsg) diff --git a/e2e/heuristic_test.go b/e2e/heuristic_test.go index 61612262..5729fd99 100644 --- a/e2e/heuristic_test.go +++ b/e2e/heuristic_test.go @@ -48,11 +48,11 @@ func TestBalanceEnforcement(t *testing.T) { }, }}) - assert.NoError(t, err, "Failed to bootstrap balance enforcement heuristic session") + require.NoError(t, err, "Failed to bootstrap balance enforcement heuristic session") // Get Alice's balance. aliceAmt, err := ts.L2Geth.L2Client.BalanceAt(context.Background(), alice, nil) - assert.NoError(t, err, "Failed to get Alice's balance") + require.NoError(t, err, "Failed to get Alice's balance") // Determine the gas cost of the transaction. gasAmt := 1_000_001 @@ -76,11 +76,11 @@ func TestBalanceEnforcement(t *testing.T) { Data: nil, }) - assert.Equal(t, len(ts.TestPagerDutyServer.PagerDutyAlerts()), 0, "No alerts should be sent before the transaction is sent") + require.Equal(t, len(ts.TestPagerDutyServer.PagerDutyAlerts()), 0, "No alerts should be sent before the transaction is sent") // Send the transaction to drain Alice's account of almost all ETH. _, err = ts.L2Geth.AddL2Block(context.Background(), drainAliceTx) - assert.NoError(t, err, "Failed to create L2 block with transaction") + require.NoError(t, err, "Failed to create L2 block with transaction") // Wait for Pessimism to process the balance change and send a notification to the mocked Slack server. time.Sleep(1 * time.Second) @@ -94,7 +94,7 @@ func TestBalanceEnforcement(t *testing.T) { // Get Bobs's balance. bobAmt, err := ts.L2Geth.L2Client.BalanceAt(context.Background(), bob, nil) - assert.NoError(t, err, "Failed to get Alice's balance") + require.NoError(t, err, "Failed to get Alice's balance") // Create a transaction to send the ETH back to Alice. drainBobTx := types.MustSignNewTx(ts.L2Cfg.Secrets.Bob, signer, &types.DynamicFeeTx{ @@ -110,7 +110,7 @@ func TestBalanceEnforcement(t *testing.T) { // Send the transaction to re-disperse the ETH from Bob back to Alice. _, err = ts.L2Geth.AddL2Block(context.Background(), drainBobTx) - assert.NoError(t, err, "Failed to create L2 block with transaction") + require.NoError(t, err, "Failed to create L2 block with transaction") // Wait for Pessimism to process the balance change. time.Sleep(1 * time.Second) @@ -118,7 +118,7 @@ func TestBalanceEnforcement(t *testing.T) { // Empty the mocked PagerDuty server cache. ts.TestPagerDutyServer.ClearAlerts() - // Wait to ensure that no new alerts are sent. + // Wait to ensure that no new alerts are generated. time.Sleep(1 * time.Second) // Ensure that no new alerts were sent. @@ -152,15 +152,15 @@ func TestContractEvent(t *testing.T) { "args": []interface{}{updateSig}, }, }}) - assert.NoError(t, err, "Error bootstrapping heuristic session") + require.NoError(t, err, "Error bootstrapping heuristic session") // Get bindings for the L1 system config contract. sysCfg, err := bindings.NewSystemConfig(ts.Cfg.L1Deployments.SystemConfigProxy, ts.L1Client) - assert.NoError(t, err, "Error getting system config") + require.NoError(t, err, "Error getting system config") // Obtain our signer. opts, err := bind.NewKeyedTransactorWithChainID(ts.Cfg.Secrets.SysCfgOwner, ts.Cfg.L1ChainIDBig()) - assert.NoError(t, err, "Error getting system config owner pk") + require.NoError(t, err, "Error getting system config owner pk") // Assign arbitrary gas config values. overhead := big.NewInt(10000) @@ -168,14 +168,13 @@ func TestContractEvent(t *testing.T) { // Call setGasConfig method on the L1 system config contract. tx, err := sysCfg.SetGasConfig(opts, overhead, scalar) - assert.NoError(t, err, "Error setting gas config") + require.NoError(t, err, "Error setting gas config") - // Wait for the transaction to be canonicalized. - txTimeoutDuration := 10 * time.Duration(ts.Cfg.DeployConfig.L1BlockTime) * time.Second - receipt, err := e2e.WaitForTransaction(tx.Hash(), ts.L1Client, txTimeoutDuration) + // Wait for the L1 transaction to be executed. + receipt, err := wait.ForReceipt(context.Background(), ts.L1Client, tx.Hash(), types.ReceiptStatusSuccessful) - assert.NoError(t, err, "Error waiting for transaction") - assert.Equal(t, receipt.Status, types.ReceiptStatusSuccessful, "transaction failed") + require.NoError(t, err, "Error waiting for transaction") + require.Equal(t, receipt.Status, types.ReceiptStatusSuccessful, "transaction failed") // Wait for Pessimism to process the newly emitted event and send a notification to the mocked Slack server. require.NoError(t, wait.For(context.Background(), 500*time.Millisecond, func() (bool, error) { @@ -190,7 +189,7 @@ func TestContractEvent(t *testing.T) { msgs := ts.TestSlackSvr.SlackAlerts() - assert.Equal(t, len(msgs), 1, "No system contract event alert was sent") + require.Equal(t, len(msgs), 1, "No system contract event alert was sent") assert.Contains(t, msgs[0].Text, "contract_event", "System contract event alert was not sent") assert.Contains(t, msgs[0].Text, alertMsg, "System contract event message was not propagated") } @@ -208,16 +207,16 @@ func TestWithdrawalEnforcement(t *testing.T) { defer ts.Close() opts, err := bind.NewKeyedTransactorWithChainID(ts.Cfg.Secrets.Alice, ts.Cfg.L2ChainIDBig()) - assert.NoError(t, err, "Error getting system config owner pk") + require.NoError(t, err, "Error getting system config owner pk") alertMsg := "disrupting centralized finance" // Deploy a dummy L2ToL1 message passer for testing. fakeAddr, tx, _, err := bindings.DeployL2ToL1MessagePasser(opts, ts.L2Client) - assert.NoError(t, err, "error deploying dummy message passer on L2") + require.NoError(t, err, "error deploying dummy message passer on L2") - _, err = e2e.WaitForTransaction(tx.Hash(), ts.L2Client, 10*time.Second) - assert.NoError(t, err, "error waiting for transaction") + _, err = wait.ForReceipt(context.Background(), ts.L2Client, tx.Hash(), types.ReceiptStatusSuccessful) + require.NoError(t, err, "error waiting for transaction") // Setup Pessimism to listen for fraudulent withdrawals // We use two heuristics here; one configured with a dummy L1 message passer @@ -242,40 +241,40 @@ func TestWithdrawalEnforcement(t *testing.T) { }, }, }) - assert.NoError(t, err, "Error bootstrapping heuristic session") + require.NoError(t, err, "Error bootstrapping heuristic session") optimismPortal, err := bindings.NewOptimismPortal(ts.Cfg.L1Deployments.OptimismPortalProxy, ts.L1Client) - assert.NoError(t, err) + require.NoError(t, err) l2ToL1MessagePasser, err := bindings.NewL2ToL1MessagePasser(predeploys.L2ToL1MessagePasserAddr, ts.L2Client) - assert.NoError(t, err) + require.NoError(t, err) aliceAddr := ts.Cfg.Secrets.Addresses().Alice // attach 1 ETH to the withdrawal and random calldata calldata := []byte{byte(1), byte(2), byte(3)} l2Opts, err := bind.NewKeyedTransactorWithChainID(ts.Cfg.Secrets.Alice, ts.Cfg.L2ChainIDBig()) - assert.NoError(t, err) + require.NoError(t, err) l2Opts.Value = big.NewInt(params.Ether) // Ensure L1 has enough funds for the withdrawal by depositing an equal amount into the OptimismPortal l1Opts, err := bind.NewKeyedTransactorWithChainID(ts.Cfg.Secrets.Alice, ts.Cfg.L1ChainIDBig()) - assert.NoError(t, err) + require.NoError(t, err) l1Opts.Value = l2Opts.Value depositTx, err := optimismPortal.Receive(l1Opts) - assert.NoError(t, err) + require.NoError(t, err) _, err = wait.ForReceiptOK(context.Background(), ts.L1Client, depositTx.Hash()) - assert.NoError(t, err) + require.NoError(t, err) // Initiate and prove a withdrawal withdrawTx, err := l2ToL1MessagePasser.InitiateWithdrawal(l2Opts, aliceAddr, big.NewInt(100_000), calldata) - assert.NoError(t, err) + require.NoError(t, err) withdrawReceipt, err := wait.ForReceiptOK(context.Background(), ts.L2Client, withdrawTx.Hash()) - assert.NoError(t, err) + require.NoError(t, err) _, proveReceipt := op_e2e.ProveWithdrawal(t, *ts.Cfg, ts.L1Client, ts.Sys.EthInstances["sequencer"], ts.Cfg.Secrets.Alice, withdrawReceipt) // Wait for Pessimism to process the withdrawal and send a notification to the mocked Slack server. - assert.NoError(t, wait.For(context.Background(), 500*time.Millisecond, func() (bool, error) { + require.NoError(t, wait.For(context.Background(), 500*time.Millisecond, func() (bool, error) { pUUID := ids[0].PUUID height, err := ts.Subsystems.PipelineHeight(pUUID) if err != nil { @@ -289,7 +288,7 @@ func TestWithdrawalEnforcement(t *testing.T) { // Ensure Pessimism has detected what it considers a "faulty" withdrawal alerts := ts.TestSlackSvr.SlackAlerts() - assert.Equal(t, 1, len(alerts), "expected 1 alert") + require.Equal(t, 1, len(alerts), "expected 1 alert") assert.Contains(t, alerts[0].Text, "withdrawal_enforcement", "expected alert to be for withdrawal_enforcement") assert.Contains(t, alerts[0].Text, fakeAddr.String(), "expected alert to be for dummy L2ToL1MessagePasser") assert.Contains(t, alerts[0].Text, alertMsg, "expected alert to have alert message") @@ -301,18 +300,16 @@ func TestFaultDetector(t *testing.T) { ts := e2e.CreateSysTestSuite(t) defer ts.Close() - txTimeoutDuration := 10 * time.Duration(ts.Cfg.DeployConfig.L1BlockTime) * time.Second - // Generate transactor opts l1Opts, err := bind.NewKeyedTransactorWithChainID(ts.Cfg.Secrets.Proposer, ts.Cfg.L1ChainIDBig()) - assert.Nil(t, err) + require.Nil(t, err) // Generate output oracle bindings outputOracle, err := bindings.NewL2OutputOracleTransactor(ts.Cfg.L1Deployments.L2OutputOracleProxy, ts.L1Client) - assert.Nil(t, err) + require.Nil(t, err) reader, err := bindings.NewL2OutputOracleCaller(ts.Cfg.L1Deployments.L2OutputOracleProxy, ts.L1Client) - assert.Nil(t, err) + require.Nil(t, err) alertMsg := "the fault, dear Brutus, is not in our stars, but in ourselves" @@ -333,8 +330,8 @@ func TestFaultDetector(t *testing.T) { }, }}) - assert.Nil(t, err) - assert.Len(t, ids, 1) + require.Nil(t, err) + require.Len(t, ids, 1) // Propose a forged L2 output root. @@ -342,13 +339,13 @@ func TestFaultDetector(t *testing.T) { l1Hash := [32]byte{0} latestNum, err := reader.NextBlockNumber(&bind.CallOpts{}) - assert.Nil(t, err) + require.Nil(t, err) tx, err := outputOracle.ProposeL2Output(l1Opts, dummyRoot, latestNum, l1Hash, big.NewInt(0)) - assert.Nil(t, err) + require.Nil(t, err) - receipt, err := e2e.WaitForTransaction(tx.Hash(), ts.L1Client, txTimeoutDuration) - assert.Nil(t, err) + receipt, err := wait.ForReceipt(context.Background(), ts.L1Client, tx.Hash(), types.ReceiptStatusSuccessful) + require.Nil(t, err) require.NoError(t, wait.For(context.Background(), 500*time.Millisecond, func() (bool, error) { pUUID := ids[0].PUUID @@ -361,7 +358,7 @@ func TestFaultDetector(t *testing.T) { })) alerts := ts.TestSlackSvr.SlackAlerts() - assert.Equal(t, 1, len(alerts), "expected 1 alert") + require.Equal(t, 1, len(alerts), "expected 1 alert") assert.Contains(t, alerts[0].Text, "fault_detector", "expected alert to be for fault_detector") assert.Contains(t, alerts[0].Text, alertMsg, "expected alert to have alert message") } diff --git a/e2e/mock_servers.go b/e2e/mock_servers.go index d25d6d58..c211cb63 100644 --- a/e2e/mock_servers.go +++ b/e2e/mock_servers.go @@ -73,7 +73,6 @@ func (svr *TestPagerDutyServer) mockPagerDutyPost(w http.ResponseWriter, r *http svr.Payloads = append(svr.Payloads, alert) - w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(`{"status":"success", "message":""}`)) } @@ -91,9 +90,10 @@ func (svr *TestPagerDutyServer) ClearAlerts() { // TestSlackServer ... Mock server for testing slack alerts type TestSlackServer struct { - Server *httptest.Server - Payloads []*client.SlackPayload - Port int + Server *httptest.Server + Payloads []*client.SlackPayload + Port int + Unstructured bool } // NewTestSlackServer ... Creates a new mock slack server @@ -147,9 +147,16 @@ func (svr *TestSlackServer) mockSlackPost(w http.ResponseWriter, r *http.Request } svr.Payloads = append(svr.Payloads, alert) - + // Randomly return different API payload responses + // This ensures that the client implementation can handle different + // slack workspace types w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(`{"message":"ok", "error":""}`)) + if svr.Unstructured { + _, _ = w.Write([]byte(`ok`)) + } else { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"message":"ok", "error":""}`)) + } } // SlackAlerts ... Returns the slack alerts diff --git a/e2e/setup.go b/e2e/setup.go index c9bdfbff..db8488bc 100644 --- a/e2e/setup.go +++ b/e2e/setup.go @@ -2,11 +2,9 @@ package e2e import ( "context" - "errors" "fmt" "os" "testing" - "time" "github.com/base-org/pessimism/internal/alert" "github.com/base-org/pessimism/internal/api/server" @@ -21,9 +19,6 @@ import ( "github.com/base-org/pessimism/internal/subsystem" op_e2e "github.com/ethereum-optimism/optimism/op-e2e" - "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" ) @@ -77,6 +72,14 @@ func CreateL2TestSuite(t *testing.T) *L2TestSuite { t.Fatal(err) } + if len(os.Getenv("ENABLE_ROLLUP_LOGS")) == 0 { + t.Log("set env 'ENABLE_ROLLUP_LOGS' to show rollup logs") + for name, logger := range nodeCfg.Loggers { + t.Logf("discarding logs for %s", name) + logger.SetHandler(log.DiscardHandler()) + } + } + ss := state.NewMemState() bundle := &client.Bundle{ @@ -165,7 +168,9 @@ func CreateSysTestSuite(t *testing.T) *SysTestSuite { appCfg := DefaultTestConfig() + // Use unstructured slack server responses for testing E2E system flows slackServer := NewTestSlackServer("127.0.0.1", 0) + slackServer.Unstructured = true pagerdutyServer := NewTestPagerDutyServer("127.0.0.1", 0) @@ -184,6 +189,7 @@ func CreateSysTestSuite(t *testing.T) *SysTestSuite { t.Fatal(err) } + t.Parallel() go pess.ListenForShutdown(kill) return &SysTestSuite{ @@ -288,28 +294,3 @@ func DefaultRoutingParams(slackURL core.StringFromEnv) *core.AlertRoutingParams }, } } - -// WaitForTransaction ... Waits for a transaction receipt to be generated or times out -func WaitForTransaction(hash common.Hash, client *ethclient.Client, timeout time.Duration) (*types.Receipt, error) { - timeoutCh := time.After(timeout) - ms100 := 100 - - ticker := time.NewTicker(time.Duration(ms100) * time.Millisecond) - defer ticker.Stop() - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - for { - receipt, err := client.TransactionReceipt(ctx, hash) - if receipt != nil && err == nil { - return receipt, nil - } else if err != nil && !errors.Is(err, ethereum.NotFound) { - return nil, err - } - - select { - case <-timeoutCh: - return nil, errors.New("timeout") - case <-ticker.C: - } - } -} diff --git a/internal/client/pagerduty.go b/internal/client/pagerduty.go index f8d25646..3770bab2 100644 --- a/internal/client/pagerduty.go +++ b/internal/client/pagerduty.go @@ -154,7 +154,7 @@ func (pdc *pagerdutyClient) PostEvent(ctx context.Context, event *AlertEventTrig resp, err := pdc.client.Do(req) defer func() { if err = resp.Body.Close(); err != nil { - logging.WithContext(ctx).Warn("Could not close pagerduty response body", + logging.WithContext(ctx).Warn("Could not close PagerDuty response body", zap.Error(err)) } }() diff --git a/internal/client/slack.go b/internal/client/slack.go index 65dd8d12..249ecd32 100644 --- a/internal/client/slack.go +++ b/internal/client/slack.go @@ -9,6 +9,7 @@ import ( "bytes" "context" "encoding/json" + "fmt" "io" "net/http" @@ -18,14 +19,17 @@ import ( "github.com/base-org/pessimism/internal/logging" ) +const ( + msgOK = "ok" +) + type SlackClient interface { AlertClient } type SlackConfig struct { - Channel string - URL string - Priority string + Channel string + URL string } // slackClient ... Slack client @@ -82,7 +86,7 @@ type SlackAPIResponse struct { // ToAlertResponse ... Converts a slack API response to an alert API response func (a *SlackAPIResponse) ToAlertResponse() *AlertAPIResponse { status := core.SuccessStatus - if a.Message != "ok" { + if a.Message != msgOK { status = core.FailureStatus } @@ -120,15 +124,29 @@ func (sc slackClient) PostEvent(ctx context.Context, event *AlertEventTrigger) ( } }() - // 3. read and unmarshal response + // 3.a. read and validate response bytes, err := io.ReadAll(resp.Body) if err != nil { return nil, err } + // 3.b. validate status code + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("slack API returned bad status code %d", resp.StatusCode) + } + + // 3.c. validate response body + if string(bytes) == msgOK { + return &AlertAPIResponse{ + Status: core.SuccessStatus, + Message: msgOK, + }, nil + } + + // 4 convert response to alert response var apiResp *SlackAPIResponse - if err := json.Unmarshal(bytes, &apiResp); err != nil { - return nil, err + if err = json.Unmarshal(bytes, &apiResp); err != nil { + return nil, fmt.Errorf("could not unmarshal slack response: %w", err) } return apiResp.ToAlertResponse(), nil diff --git a/internal/core/alert.go b/internal/core/alert.go index df0ba115..a7f04125 100644 --- a/internal/core/alert.go +++ b/internal/core/alert.go @@ -4,7 +4,7 @@ import ( "time" ) -// PagerDutySeverity represents the severity of an event +// PagerDutySeverity ... represents the severity of an event type PagerDutySeverity string const ( diff --git a/internal/etl/pipeline/manager.go b/internal/etl/pipeline/manager.go index e2a1b2a1..1e88ab75 100644 --- a/internal/etl/pipeline/manager.go +++ b/internal/etl/pipeline/manager.go @@ -287,5 +287,15 @@ func (em *etlManager) GetPipelineHeight(id core.PUUID) (*big.Int, error) { return nil, err } - return pipeline.BlockHeight() + height, err := pipeline.BlockHeight() + if err != nil { + return nil, err + } + + // If the pipeline is not running, return 0 + if height == nil { + return big.NewInt(0), nil + } + + return height, nil }