From 1d0d2b04b87b3bf771699fc3bd4196150f4e4858 Mon Sep 17 00:00:00 2001 From: Kirill Maksimov Date: Sun, 11 Oct 2020 16:22:46 +0300 Subject: [PATCH 1/6] Improved stress test --- test/e2e/harness.go | 21 ++++++++++++++++++--- test/e2e/network_stress_test.go | 2 +- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/test/e2e/harness.go b/test/e2e/harness.go index afd63a16e..013c6ad7d 100644 --- a/test/e2e/harness.go +++ b/test/e2e/harness.go @@ -99,6 +99,21 @@ func (h *Harness) SendTransaction(senderPublicKey []byte, senderPrivateKey []byt return nil, txId, err } out, err := h.client.SendTransaction(payload) + if err != nil { + return nil, txId, err + } + return out.TransactionResponse, txId, err +} + +func (h *Harness) SendTransactionAsync(senderPublicKey []byte, senderPrivateKey []byte, contractName string, methodName string, args ...interface{}) (*codec.TransactionResponse, string, error) { + payload, txId, err := h.client.CreateTransaction(senderPublicKey, senderPrivateKey, contractName, methodName, args...) + if err != nil { + return nil, txId, err + } + out, err := h.client.SendTransactionAsync(payload) + if err != nil { + return nil, txId, err + } return out.TransactionResponse, txId, err } @@ -163,7 +178,7 @@ func (h *Harness) GetBlockHeight() primitives.BlockHeight { if blockHeight, found := metricReader.GetAsInt(blockstorage.MetricBlockHeight); !found { return 0 - } else { + } else { return primitives.BlockHeight(blockHeight) } } @@ -176,7 +191,7 @@ func (h *Harness) GetTransactionCount() int64 { if txCount, found := metricReader.GetAsInt(transactionpool.MetricCommittedPoolTransactions); !found { return 0 - } else { + } else { return txCount } } @@ -204,7 +219,7 @@ func (h *Harness) WaitUntilTransactionPoolIsReady(t *testing.T) { if lastCommittedTimestamp, found := metricReader.GetAsInt(transactionpool.MetricLastCommittedTime); !found { return false - } else { + } else { diff := lastCommittedTimestamp - time.Now().Add(recentBlockTimeDiff*-1).UnixNano() return diff >= 0 } diff --git a/test/e2e/network_stress_test.go b/test/e2e/network_stress_test.go index 2be1547d6..83df5bd47 100644 --- a/test/e2e/network_stress_test.go +++ b/test/e2e/network_stress_test.go @@ -50,7 +50,7 @@ func TestE2EStress(t *testing.T) { target, _ := orbsClient.CreateAccount() amount := uint64(ctrlRand.Intn(10)) - response, _, err2 := h.SendTransaction(OwnerOfAllSupply.PublicKey(), OwnerOfAllSupply.PrivateKey(), "BenchmarkToken", "transfer", uint64(amount), target.AddressAsBytes()) + response, _, err2 := h.SendTransactionAsync(OwnerOfAllSupply.PublicKey(), OwnerOfAllSupply.PrivateKey(), "BenchmarkToken", "transfer", uint64(amount), target.AddressAsBytes()) if err2 != nil { fmt.Println("Encountered an error sending a transaction while stress testing", err2) From 3602e806952958e05a7e6b1133e1b54033b14fe0 Mon Sep 17 00:00:00 2001 From: Kirill Maksimov Date: Sun, 11 Oct 2020 16:41:58 +0300 Subject: [PATCH 2/6] Add rate limit for stress test --- test/e2e/network_stress_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/network_stress_test.go b/test/e2e/network_stress_test.go index 83df5bd47..417b33b6a 100644 --- a/test/e2e/network_stress_test.go +++ b/test/e2e/network_stress_test.go @@ -34,7 +34,7 @@ func TestE2EStress(t *testing.T) { var wg sync.WaitGroup - limiter := rate.NewLimiter(1000, 50) + limiter := rate.NewLimiter(rate.Limit(config.targetTPS), 50) var mutex sync.Mutex var errors []error From e4ce142b8ea7e3e03f0f182287cdca89f0e19acf Mon Sep 17 00:00:00 2001 From: Kirill Maksimov Date: Sun, 11 Oct 2020 16:46:13 +0300 Subject: [PATCH 3/6] Support async stress test --- test/e2e/harness.go | 3 +++ test/e2e/network_stress_test.go | 10 +++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/test/e2e/harness.go b/test/e2e/harness.go index 013c6ad7d..9fb74ac82 100644 --- a/test/e2e/harness.go +++ b/test/e2e/harness.go @@ -37,6 +37,7 @@ type E2EConfig struct { type StressTestConfig struct { enabled bool + async bool numberOfTransactions int64 acceptableFailureRate int64 targetTPS float64 @@ -274,6 +275,7 @@ func GetConfig() E2EConfig { stressTestNumberOfTransactions := int64(10000) stressTestFailureRate := int64(2) stressTestTargetTPS := float64(700) + stressTestAsync := os.Getenv("STRESS_TEST_ASYNC") == "true" ethereumEndpoint := "http://127.0.0.1:8545" @@ -296,6 +298,7 @@ func GetConfig() E2EConfig { AppChainUrl: appChainUrl, StressTest: StressTestConfig{ enabled: stressTestEnabled, + async: stressTestAsync, numberOfTransactions: stressTestNumberOfTransactions, acceptableFailureRate: stressTestFailureRate, targetTPS: stressTestTargetTPS, diff --git a/test/e2e/network_stress_test.go b/test/e2e/network_stress_test.go index 417b33b6a..dd512b82c 100644 --- a/test/e2e/network_stress_test.go +++ b/test/e2e/network_stress_test.go @@ -9,6 +9,7 @@ package e2e import ( "context" "fmt" + "github.com/orbs-network/orbs-client-sdk-go/codec" "sync" "testing" "time" @@ -50,7 +51,14 @@ func TestE2EStress(t *testing.T) { target, _ := orbsClient.CreateAccount() amount := uint64(ctrlRand.Intn(10)) - response, _, err2 := h.SendTransactionAsync(OwnerOfAllSupply.PublicKey(), OwnerOfAllSupply.PrivateKey(), "BenchmarkToken", "transfer", uint64(amount), target.AddressAsBytes()) + var response *codec.TransactionResponse + var err2 error + + if config.async { + response, _, err2 = h.SendTransactionAsync(OwnerOfAllSupply.PublicKey(), OwnerOfAllSupply.PrivateKey(), "BenchmarkToken", "transfer", uint64(amount), target.AddressAsBytes()) + } else { + response, _, err2 = h.SendTransaction(OwnerOfAllSupply.PublicKey(), OwnerOfAllSupply.PrivateKey(), "BenchmarkToken", "transfer", uint64(amount), target.AddressAsBytes()) + } if err2 != nil { fmt.Println("Encountered an error sending a transaction while stress testing", err2) From 908955fe54e294671b58ad0b3d7f72c9326f2c1a Mon Sep 17 00:00:00 2001 From: Kirill Maksimov Date: Sun, 11 Oct 2020 16:48:01 +0300 Subject: [PATCH 4/6] Add progress indication to stress test --- test/e2e/network_stress_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/e2e/network_stress_test.go b/test/e2e/network_stress_test.go index dd512b82c..51a39ded3 100644 --- a/test/e2e/network_stress_test.go +++ b/test/e2e/network_stress_test.go @@ -70,6 +70,10 @@ func TestE2EStress(t *testing.T) { errorTransactionStatuses = append(errorTransactionStatuses, string(response.TransactionStatus)) } } + + if i%100 == 0 { + fmt.Println(fmt.Sprintf("processed transactions: %d/%d", i, config.numberOfTransactions)) + } }() } else { mutex.Lock() From 2388954327679b3825171f11d318e3b04a921fed Mon Sep 17 00:00:00 2001 From: Kirill Maksimov Date: Sun, 11 Oct 2020 20:28:01 +0300 Subject: [PATCH 5/6] Stress test uses multiple endpoints --- go.mod | 2 +- test/e2e/harness.go | 23 +++++++++++++++++++---- test/e2e/network_stress_test.go | 27 ++++++++++++++++++--------- 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 588eecb7e..dce5a30c6 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/orbs-network/healthcheck v1.1.0 github.com/orbs-network/lean-helix-go v0.5.0 github.com/orbs-network/membuffers v0.4.0 - github.com/orbs-network/orbs-client-sdk-go v0.18.0 + github.com/orbs-network/orbs-client-sdk-go v0.19.0 github.com/orbs-network/orbs-contract-sdk v1.8.0 github.com/orbs-network/orbs-spec v0.0.0-20200715083427-d937ef1ec8ef github.com/orbs-network/scribe v0.2.3 diff --git a/test/e2e/harness.go b/test/e2e/harness.go index 9fb74ac82..e4b5da07c 100644 --- a/test/e2e/harness.go +++ b/test/e2e/harness.go @@ -41,6 +41,7 @@ type StressTestConfig struct { numberOfTransactions int64 acceptableFailureRate int64 targetTPS float64 + apiEndpoints []string } const START_HTTP_PORT = 8090 @@ -95,11 +96,15 @@ func (h *Harness) DeployNativeContract(from *keys.Ed25519KeyPair, contractName s } func (h *Harness) SendTransaction(senderPublicKey []byte, senderPrivateKey []byte, contractName string, methodName string, args ...interface{}) (*codec.TransactionResponse, string, error) { - payload, txId, err := h.client.CreateTransaction(senderPublicKey, senderPrivateKey, contractName, methodName, args...) + return h.SendTransactionWithClient(h.client, senderPublicKey, senderPrivateKey, contractName, methodName, args...) +} + +func (h *Harness) SendTransactionWithClient(client *orbsClient.OrbsClient, senderPublicKey []byte, senderPrivateKey []byte, contractName string, methodName string, args ...interface{}) (*codec.TransactionResponse, string, error) { + payload, txId, err := client.CreateTransaction(senderPublicKey, senderPrivateKey, contractName, methodName, args...) if err != nil { return nil, txId, err } - out, err := h.client.SendTransaction(payload) + out, err := client.SendTransaction(payload) if err != nil { return nil, txId, err } @@ -107,11 +112,15 @@ func (h *Harness) SendTransaction(senderPublicKey []byte, senderPrivateKey []byt } func (h *Harness) SendTransactionAsync(senderPublicKey []byte, senderPrivateKey []byte, contractName string, methodName string, args ...interface{}) (*codec.TransactionResponse, string, error) { - payload, txId, err := h.client.CreateTransaction(senderPublicKey, senderPrivateKey, contractName, methodName, args...) + return h.SendTransactionAsyncWithClient(h.client, senderPublicKey, senderPrivateKey, contractName, methodName, args...) +} + +func (h *Harness) SendTransactionAsyncWithClient(client *orbsClient.OrbsClient, senderPublicKey []byte, senderPrivateKey []byte, contractName string, methodName string, args ...interface{}) (*codec.TransactionResponse, string, error) { + payload, txId, err := client.CreateTransaction(senderPublicKey, senderPrivateKey, contractName, methodName, args...) if err != nil { return nil, txId, err } - out, err := h.client.SendTransactionAsync(payload) + out, err := client.SendTransactionAsync(payload) if err != nil { return nil, txId, err } @@ -285,10 +294,15 @@ func GetConfig() E2EConfig { ethereumEndpoint = os.Getenv("ETHEREUM_ENDPOINT") } + var stressTestAPIEndpoints = []string{os.Getenv("API_ENDPOINT")} if stressTestEnabled { stressTestNumberOfTransactions, _ = strconv.ParseInt(os.Getenv("STRESS_TEST_NUMBER_OF_TRANSACTIONS"), 10, 0) stressTestFailureRate, _ = strconv.ParseInt(os.Getenv("STRESS_TEST_FAILURE_RATE"), 10, 0) stressTestTargetTPS, _ = strconv.ParseFloat(os.Getenv("STRESS_TEST_TARGET_TPS"), 0) + stressTestAPIEndpointsOverride := strings.Split(os.Getenv("STRESS_TEST_API_ENDPOINTS"), ",") + if len(stressTestAPIEndpointsOverride) > 0 { + stressTestAPIEndpoints = stressTestAPIEndpointsOverride + } } return E2EConfig{ @@ -302,6 +316,7 @@ func GetConfig() E2EConfig { numberOfTransactions: stressTestNumberOfTransactions, acceptableFailureRate: stressTestFailureRate, targetTPS: stressTestTargetTPS, + apiEndpoints: stressTestAPIEndpoints, }, EthereumEndpoint: ethereumEndpoint, } diff --git a/test/e2e/network_stress_test.go b/test/e2e/network_stress_test.go index 51a39ded3..dbfb4fcb2 100644 --- a/test/e2e/network_stress_test.go +++ b/test/e2e/network_stress_test.go @@ -25,7 +25,8 @@ func TestE2EStress(t *testing.T) { h := NewAppHarness() ctrlRand := rand.NewControlledRand(t) - config := GetConfig().StressTest + generalConfig := GetConfig() + config := generalConfig.StressTest if !config.enabled { t.Skip("Skipping stress test") @@ -41,40 +42,47 @@ func TestE2EStress(t *testing.T) { var errors []error var errorTransactionStatuses []string + var clients []*orbsClient.OrbsClient + for _, apiEndpoint := range config.apiEndpoints { + clients = append(clients, orbsClient.NewClient(apiEndpoint, uint32(generalConfig.AppVcid), codec.NETWORK_TYPE_TEST_NET)) + } + for i := int64(0); i < config.numberOfTransactions; i++ { if err := limiter.Wait(context.Background()); err == nil { wg.Add(1) - go func() { + go func(i int64) { defer wg.Done() target, _ := orbsClient.CreateAccount() amount := uint64(ctrlRand.Intn(10)) + client := clients[i%int64(len(clients))] // select one of the clients + var response *codec.TransactionResponse var err2 error if config.async { - response, _, err2 = h.SendTransactionAsync(OwnerOfAllSupply.PublicKey(), OwnerOfAllSupply.PrivateKey(), "BenchmarkToken", "transfer", uint64(amount), target.AddressAsBytes()) + response, _, err2 = h.SendTransactionAsyncWithClient(client, OwnerOfAllSupply.PublicKey(), OwnerOfAllSupply.PrivateKey(), "BenchmarkToken", "transfer", uint64(amount), target.AddressAsBytes()) } else { - response, _, err2 = h.SendTransaction(OwnerOfAllSupply.PublicKey(), OwnerOfAllSupply.PrivateKey(), "BenchmarkToken", "transfer", uint64(amount), target.AddressAsBytes()) + response, _, err2 = h.SendTransactionWithClient(client, OwnerOfAllSupply.PublicKey(), OwnerOfAllSupply.PrivateKey(), "BenchmarkToken", "transfer", uint64(amount), target.AddressAsBytes()) } if err2 != nil { - fmt.Println("Encountered an error sending a transaction while stress testing", err2) + fmt.Println("Encountered an error sending a transaction while stress testing", client.Endpoint, err2) mutex.Lock() defer mutex.Unlock() fmt.Println("") errors = append(errors, err2) if response != nil { - errorTransactionStatuses = append(errorTransactionStatuses, string(response.TransactionStatus)) + errorTransactionStatuses = append(errorTransactionStatuses, string(response.TransactionStatus), "endpoint", client.Endpoint) } } - if i%100 == 0 { - fmt.Println(fmt.Sprintf("processed transactions: %d/%d", i, config.numberOfTransactions)) + if i+1%100 == 0 { + fmt.Println(fmt.Sprintf("processed transactions: %d/%d", i+1, config.numberOfTransactions)) } - }() + }(i) } else { mutex.Lock() defer mutex.Unlock() @@ -84,6 +92,7 @@ func TestE2EStress(t *testing.T) { wg.Wait() + // very bad and unreliable metric, does not take into account multiple endpoints yet txCount := float64(getTransactionCount(t, h) - baseTxCount) expectedNumberOfTx := float64(100-config.acceptableFailureRate) / 100 * float64(config.numberOfTransactions) From 4ab4bb951ad402fc4d2627204ed990a7539acb5f Mon Sep 17 00:00:00 2001 From: Kirill Maksimov Date: Wed, 14 Oct 2020 14:44:19 +0300 Subject: [PATCH 6/6] Stress test can now update a single key instead of introducing a key per tx --- test/e2e/harness.go | 3 +++ test/e2e/network_stress_test.go | 12 +++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/test/e2e/harness.go b/test/e2e/harness.go index e4b5da07c..a51543e2a 100644 --- a/test/e2e/harness.go +++ b/test/e2e/harness.go @@ -38,6 +38,7 @@ type E2EConfig struct { type StressTestConfig struct { enabled bool async bool + skipState bool numberOfTransactions int64 acceptableFailureRate int64 targetTPS float64 @@ -285,6 +286,7 @@ func GetConfig() E2EConfig { stressTestFailureRate := int64(2) stressTestTargetTPS := float64(700) stressTestAsync := os.Getenv("STRESS_TEST_ASYNC") == "true" + stressTestSkipState := os.Getenv("STRESS_TEST_SKIP_STATE") == "true" ethereumEndpoint := "http://127.0.0.1:8545" @@ -317,6 +319,7 @@ func GetConfig() E2EConfig { acceptableFailureRate: stressTestFailureRate, targetTPS: stressTestTargetTPS, apiEndpoints: stressTestAPIEndpoints, + skipState: stressTestSkipState, }, EthereumEndpoint: ethereumEndpoint, } diff --git a/test/e2e/network_stress_test.go b/test/e2e/network_stress_test.go index dbfb4fcb2..7977dd7e7 100644 --- a/test/e2e/network_stress_test.go +++ b/test/e2e/network_stress_test.go @@ -47,6 +47,8 @@ func TestE2EStress(t *testing.T) { clients = append(clients, orbsClient.NewClient(apiEndpoint, uint32(generalConfig.AppVcid), codec.NETWORK_TYPE_TEST_NET)) } + defaultTarget, _ := orbsClient.CreateAccount() + for i := int64(0); i < config.numberOfTransactions; i++ { if err := limiter.Wait(context.Background()); err == nil { wg.Add(1) @@ -54,7 +56,11 @@ func TestE2EStress(t *testing.T) { go func(i int64) { defer wg.Done() - target, _ := orbsClient.CreateAccount() + target := defaultTarget + if !config.skipState { + target, _ = orbsClient.CreateAccount() + } + amount := uint64(ctrlRand.Intn(10)) client := clients[i%int64(len(clients))] // select one of the clients @@ -63,9 +69,9 @@ func TestE2EStress(t *testing.T) { var err2 error if config.async { - response, _, err2 = h.SendTransactionAsyncWithClient(client, OwnerOfAllSupply.PublicKey(), OwnerOfAllSupply.PrivateKey(), "BenchmarkToken", "transfer", uint64(amount), target.AddressAsBytes()) + response, _, err2 = h.SendTransactionAsyncWithClient(client, OwnerOfAllSupply.PublicKey(), OwnerOfAllSupply.PrivateKey(), "BenchmarkToken", "transfer", amount, target.AddressAsBytes()) } else { - response, _, err2 = h.SendTransactionWithClient(client, OwnerOfAllSupply.PublicKey(), OwnerOfAllSupply.PrivateKey(), "BenchmarkToken", "transfer", uint64(amount), target.AddressAsBytes()) + response, _, err2 = h.SendTransactionWithClient(client, OwnerOfAllSupply.PublicKey(), OwnerOfAllSupply.PrivateKey(), "BenchmarkToken", "transfer", amount, target.AddressAsBytes()) } if err2 != nil {