From 29a08ce160ce1256663e9aa1e38644ef5030552c Mon Sep 17 00:00:00 2001 From: Moritz Date: Fri, 20 Dec 2024 11:14:00 +0100 Subject: [PATCH 01/11] Add basic `benchmark_mpc_sign` --- benchmarks/synth-bm/src/contract.rs | 146 ++++++++++++++++++++++++++++ benchmarks/synth-bm/src/main.rs | 6 ++ 2 files changed, 152 insertions(+) create mode 100644 benchmarks/synth-bm/src/contract.rs diff --git a/benchmarks/synth-bm/src/contract.rs b/benchmarks/synth-bm/src/contract.rs new file mode 100644 index 00000000000..d694a30c865 --- /dev/null +++ b/benchmarks/synth-bm/src/contract.rs @@ -0,0 +1,146 @@ +use std::path::PathBuf; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use crate::account::accounts_from_dir; +use crate::block_service::BlockService; +use crate::rpc::{ResponseCheckSeverity, RpcResponseHandler}; +use clap::Args; +use log::info; +use near_jsonrpc_client::methods::send_tx::RpcSendTransactionRequest; +use near_jsonrpc_client::JsonRpcClient; +use near_primitives::transaction::SignedTransaction; +use near_primitives::types::AccountId; +use near_primitives::views::TxExecutionStatus; +use serde_json::json; +use tokio::sync::mpsc; +use tokio::time; + +#[derive(Args, Debug)] +pub struct BenchmarkMpcSignArgs { + /// TODO try to have single arg for all commands + #[arg(long)] + pub rpc_url: String, + #[arg(long)] + pub user_data_dir: PathBuf, + #[arg(long)] + pub transactions_per_second: u64, + #[arg(long)] + pub num_transfers: u64, + #[arg(long)] + pub receiver_id: AccountId, + /// The `key_version` passed as argument to `sign`. + #[arg(long)] + pub key_version: u32, + #[arg(long)] + pub channel_buffer_size: usize, + #[arg(long)] + pub gas: u64, + #[arg(long)] + pub deposit: u128, +} + +pub async fn benchmark_mpc_sign(args: &BenchmarkMpcSignArgs) -> anyhow::Result<()> { + let mut accounts = accounts_from_dir(&args.user_data_dir)?; + assert!( + accounts.len() > 0, + "at least one account required in {:?} to send transactions", + args.user_data_dir + ); + + // Pick interval to achieve desired TPS. + let mut interval = + time::interval(Duration::from_micros(1_000_000 / args.transactions_per_second)); + + let client = JsonRpcClient::connect(&args.rpc_url); + let block_service = Arc::new(BlockService::new(client.clone()).await); + block_service.clone().start().await; + + // Before a request is made, a permit to send into the channel is awaited. Hence buffer size + // limits the number of outstanding requests. This helps to avoid congestion. + let (channel_tx, channel_rx) = mpsc::channel(args.channel_buffer_size); + + // Current network capacity for MPC `sign` calls is known to be around 100 TPS. At that + // rate, neither the network nor the RPC should be a bottleneck. + // Hence `wait_until: EXECUTED_OPTIMISTIC` as it provides most insights. + let wait_until = TxExecutionStatus::None; + let wait_until_channel = wait_until.clone(); + let num_expected_responses = args.num_transfers; + let response_handler_task = tokio::task::spawn(async move { + let mut rpc_response_handler = RpcResponseHandler::new( + channel_rx, + wait_until_channel, + ResponseCheckSeverity::Log, + num_expected_responses, + ); + rpc_response_handler.handle_all_responses().await; + }); + + info!("Setup complete, starting to send transactions"); + let timer = Instant::now(); + for i in 0..args.num_transfers { + let sender_idx = usize::try_from(i).unwrap() / accounts.len(); + let sender = &accounts[sender_idx]; + + // TODO randomize + let payload: [u8; 32] = [0; 32]; + let path = "/foo/bar"; + let fn_call_args = json!({ + "payload": payload, + "path": path, + "key_version": args.key_version, + }); + + let transaction = SignedTransaction::call( + sender.nonce, + sender.id.clone(), + args.receiver_id.clone(), + &sender.as_signer(), + args.deposit, + "sign".to_string(), + fn_call_args.to_string().into_bytes(), + args.gas, + block_service.get_block_hash(), + ); + let request = RpcSendTransactionRequest { + signed_transaction: transaction, + wait_until: wait_until.clone(), + }; + + // Let time pass to meet TPS target. + interval.tick().await; + + // Await permit before sending the request to make channel buffer size a limit for the + // number of outstanding requests. + let permit = channel_tx.clone().reserve_owned().await.unwrap(); + let client = client.clone(); + tokio::spawn(async move { + let res = client.call(request).await; + permit.send(res); + }); + + if i > 0 && i % 200 == 0 { + info!("sent {i} transactions in {:.2} seconds", timer.elapsed().as_secs_f64()); + } + + let sender = accounts.get_mut(sender_idx).unwrap(); + sender.nonce += 1; + } + + info!( + "Done sending {} transactions in {:.2} seconds", + args.num_transfers, + timer.elapsed().as_secs_f64() + ); + + info!("Awaiting RPC responses"); + response_handler_task.await.expect("response handler tasks should succeed"); + info!("Received all RPC responses after {:.2} seconds", timer.elapsed().as_secs_f64()); + + info!("Writing updated nonces to {:?}", args.user_data_dir); + for account in accounts.iter() { + account.write_to_dir(&args.user_data_dir)?; + } + + Ok(()) +} diff --git a/benchmarks/synth-bm/src/main.rs b/benchmarks/synth-bm/src/main.rs index 039b56302ec..ba608b8adca 100644 --- a/benchmarks/synth-bm/src/main.rs +++ b/benchmarks/synth-bm/src/main.rs @@ -3,6 +3,8 @@ use clap::{Parser, Subcommand}; mod account; use account::{create_sub_accounts, CreateSubAccountsArgs}; mod block_service; +mod contract; +use contract::BenchmarkMpcSignArgs; mod native_transfer; mod rpc; @@ -19,6 +21,7 @@ enum Commands { /// Creates sub accounts for the signer. CreateSubAccounts(CreateSubAccountsArgs), BenchmarkNativeTransfers(native_transfer::BenchmarkArgs), + BenchmarkMpcSign(BenchmarkMpcSignArgs), } #[tokio::main] @@ -35,6 +38,9 @@ async fn main() -> anyhow::Result<()> { Commands::BenchmarkNativeTransfers(args) => { native_transfer::benchmark(args).await?; } + Commands::BenchmarkMpcSign(args) => { + contract::benchmark_mpc_sign(args).await?; + } } Ok(()) } From fc56f2c9bd527ad171c2ba8c1e2313283616f55c Mon Sep 17 00:00:00 2001 From: Moritz Date: Fri, 20 Dec 2024 12:54:48 +0100 Subject: [PATCH 02/11] Randomize function args --- benchmarks/synth-bm/src/contract.rs | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/benchmarks/synth-bm/src/contract.rs b/benchmarks/synth-bm/src/contract.rs index d694a30c865..a237be314fa 100644 --- a/benchmarks/synth-bm/src/contract.rs +++ b/benchmarks/synth-bm/src/contract.rs @@ -12,13 +12,15 @@ use near_jsonrpc_client::JsonRpcClient; use near_primitives::transaction::SignedTransaction; use near_primitives::types::AccountId; use near_primitives::views::TxExecutionStatus; +use rand::distributions::{Alphanumeric, DistString}; +use rand::rngs::ThreadRng; +use rand::{thread_rng, Rng}; use serde_json::json; use tokio::sync::mpsc; use tokio::time; #[derive(Args, Debug)] pub struct BenchmarkMpcSignArgs { - /// TODO try to have single arg for all commands #[arg(long)] pub rpc_url: String, #[arg(long)] @@ -55,6 +57,7 @@ pub async fn benchmark_mpc_sign(args: &BenchmarkMpcSignArgs) -> anyhow::Result<( let client = JsonRpcClient::connect(&args.rpc_url); let block_service = Arc::new(BlockService::new(client.clone()).await); block_service.clone().start().await; + let mut rng = thread_rng(); // Before a request is made, a permit to send into the channel is awaited. Hence buffer size // limits the number of outstanding requests. This helps to avoid congestion. @@ -82,15 +85,6 @@ pub async fn benchmark_mpc_sign(args: &BenchmarkMpcSignArgs) -> anyhow::Result<( let sender_idx = usize::try_from(i).unwrap() / accounts.len(); let sender = &accounts[sender_idx]; - // TODO randomize - let payload: [u8; 32] = [0; 32]; - let path = "/foo/bar"; - let fn_call_args = json!({ - "payload": payload, - "path": path, - "key_version": args.key_version, - }); - let transaction = SignedTransaction::call( sender.nonce, sender.id.clone(), @@ -98,7 +92,7 @@ pub async fn benchmark_mpc_sign(args: &BenchmarkMpcSignArgs) -> anyhow::Result<( &sender.as_signer(), args.deposit, "sign".to_string(), - fn_call_args.to_string().into_bytes(), + new_random_mpc_sign_args(&mut rng, args.key_version).to_string().into_bytes(), args.gas, block_service.get_block_hash(), ); @@ -144,3 +138,14 @@ pub async fn benchmark_mpc_sign(args: &BenchmarkMpcSignArgs) -> anyhow::Result<( Ok(()) } + +fn new_random_mpc_sign_args(rng: &mut ThreadRng, key_version: u32) -> serde_json::Value { + let mut payload: [u8; 32] = [0; 32]; + rng.fill(&mut payload); + + json!({ + "payload": payload, + "path": Alphanumeric.sample_string(rng, 16), + "key_version": key_version, + }) +} From 304ee0cd57835d0237fabdd28f5c5d6789ecdc4a Mon Sep 17 00:00:00 2001 From: Moritz Date: Fri, 20 Dec 2024 12:58:19 +0100 Subject: [PATCH 03/11] Add docs --- .../workflows/benchmarking_synthetic_workloads.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/practices/workflows/benchmarking_synthetic_workloads.md b/docs/practices/workflows/benchmarking_synthetic_workloads.md index 0f76f18f58e..f9e83659f40 100644 --- a/docs/practices/workflows/benchmarking_synthetic_workloads.md +++ b/docs/practices/workflows/benchmarking_synthetic_workloads.md @@ -60,6 +60,18 @@ Automatic calculation of transactions per second (TPS) when RPC requests are sen http localhost:3030/metrics | grep transaction_processed ``` +### Benchmark calls to the `sign` method of an MPC contract + +Assumes the accounts that send the transactions invoking `sign` have been created as described above. Transactions can be sent to a RPC of a network that on which an instance of the [`mpc/chain-signatures`](https://github.com/near/mpc/tree/79ec50759146221e7ad8bb04520f13333b75ca07/chain-signatures/contract) is deployed. + +Transactions are sent to the RPC with `wait_until: EXECUTED_OPTIMISTIC` as the throughput for `sign` is at a level at which neither the network nor the RPC are expected to be a bottleneck. + +All options of the command can be shown with: + +```command +cargo run -- benchmark-mpc-sign --help +``` + ## Network setup and `neard` configuration Details of bringing up and configuring a network are out of scope for this document. Instead we just give a brief overview of the setup regularly used to benchmark TPS of common workloads in a single-node with a single-shard setup. From ed6e0c5593263130f2150c8ad76d080e9b238297 Mon Sep 17 00:00:00 2001 From: Moritz Date: Fri, 20 Dec 2024 18:14:16 +0100 Subject: [PATCH 04/11] Add justfile recipe --- benchmarks/synth-bm/justfile | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/benchmarks/synth-bm/justfile b/benchmarks/synth-bm/justfile index 4972dbeee85..1ab784e2d6b 100644 --- a/benchmarks/synth-bm/justfile +++ b/benchmarks/synth-bm/justfile @@ -30,3 +30,16 @@ benchmark_native_transfers: --channel-buffer-size 30000 \ --interval-duration-micros 550 \ --amount 1 + +benchmark_mpc_sign: + RUST_LOG=info \ + cargo run --release -- benchmark-mpc-sign \ + --rpc-url {{rpc_url}} \ + --user-data-dir user-data/ \ + --num-transfers 500 \ + --transactions-per-second 100 \ + --receiver-id 'v1.signer-dev.testnet' \ + --key-version 0 \ + --channel-buffer-size 500 \ + --gas 100000000000000 \ + --deposit 1 From 642ba64ccf8eb9c19384e3d870f811097c1f147c Mon Sep 17 00:00:00 2001 From: Moritz Date: Wed, 8 Jan 2025 17:22:07 +0100 Subject: [PATCH 05/11] Fix sender_idx assignment --- benchmarks/synth-bm/src/contract.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/synth-bm/src/contract.rs b/benchmarks/synth-bm/src/contract.rs index a237be314fa..f23f1563449 100644 --- a/benchmarks/synth-bm/src/contract.rs +++ b/benchmarks/synth-bm/src/contract.rs @@ -82,7 +82,7 @@ pub async fn benchmark_mpc_sign(args: &BenchmarkMpcSignArgs) -> anyhow::Result<( info!("Setup complete, starting to send transactions"); let timer = Instant::now(); for i in 0..args.num_transfers { - let sender_idx = usize::try_from(i).unwrap() / accounts.len(); + let sender_idx = usize::try_from(i).unwrap() % accounts.len(); let sender = &accounts[sender_idx]; let transaction = SignedTransaction::call( From fba97f5e2149c60f777e6a2936bedbee1872d036 Mon Sep 17 00:00:00 2001 From: Moritz Date: Wed, 8 Jan 2025 17:37:48 +0100 Subject: [PATCH 06/11] Fix parameter construction --- benchmarks/synth-bm/src/contract.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/benchmarks/synth-bm/src/contract.rs b/benchmarks/synth-bm/src/contract.rs index f23f1563449..687d15270e4 100644 --- a/benchmarks/synth-bm/src/contract.rs +++ b/benchmarks/synth-bm/src/contract.rs @@ -15,6 +15,7 @@ use near_primitives::views::TxExecutionStatus; use rand::distributions::{Alphanumeric, DistString}; use rand::rngs::ThreadRng; use rand::{thread_rng, Rng}; +use serde::Serialize; use serde_json::json; use tokio::sync::mpsc; use tokio::time; @@ -139,13 +140,21 @@ pub async fn benchmark_mpc_sign(args: &BenchmarkMpcSignArgs) -> anyhow::Result<( Ok(()) } +/// Constructs the parameters according to +/// https://github.com/near/mpc/blob/79ec50759146221e7ad8bb04520f13333b75ca07/chain-signatures/contract/src/lib.rs#L127 fn new_random_mpc_sign_args(rng: &mut ThreadRng, key_version: u32) -> serde_json::Value { + #[derive(Serialize)] + struct SignRequest { + pub payload: [u8; 32], + pub path: String, + pub key_version: u32, + } + let mut payload: [u8; 32] = [0; 32]; rng.fill(&mut payload); + let path = Alphanumeric.sample_string(rng, 16); json!({ - "payload": payload, - "path": Alphanumeric.sample_string(rng, 16), - "key_version": key_version, + "request": SignRequest { payload, path, key_version }, }) } From b7c1941f79b08695e5fcbd7827570621a94a532d Mon Sep 17 00:00:00 2001 From: Moritz Date: Wed, 8 Jan 2025 17:39:00 +0100 Subject: [PATCH 07/11] Increase gas and deposit --- benchmarks/synth-bm/justfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/synth-bm/justfile b/benchmarks/synth-bm/justfile index 1ab784e2d6b..7cec282d5f5 100644 --- a/benchmarks/synth-bm/justfile +++ b/benchmarks/synth-bm/justfile @@ -41,5 +41,5 @@ benchmark_mpc_sign: --receiver-id 'v1.signer-dev.testnet' \ --key-version 0 \ --channel-buffer-size 500 \ - --gas 100000000000000 \ - --deposit 1 + --gas 300000000000000 \ + --deposit 100000000000000000000000 From 719c47fe2fa6d93a997942c3f2070e1f2d7951f8 Mon Sep 17 00:00:00 2001 From: Moritz Date: Thu, 9 Jan 2025 10:57:01 +0100 Subject: [PATCH 08/11] Fix `TxExecutionStatus` --- benchmarks/synth-bm/src/contract.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/synth-bm/src/contract.rs b/benchmarks/synth-bm/src/contract.rs index 687d15270e4..bf0cd4b4b3c 100644 --- a/benchmarks/synth-bm/src/contract.rs +++ b/benchmarks/synth-bm/src/contract.rs @@ -67,7 +67,7 @@ pub async fn benchmark_mpc_sign(args: &BenchmarkMpcSignArgs) -> anyhow::Result<( // Current network capacity for MPC `sign` calls is known to be around 100 TPS. At that // rate, neither the network nor the RPC should be a bottleneck. // Hence `wait_until: EXECUTED_OPTIMISTIC` as it provides most insights. - let wait_until = TxExecutionStatus::None; + let wait_until = TxExecutionStatus::ExecutedOptimistic; let wait_until_channel = wait_until.clone(); let num_expected_responses = args.num_transfers; let response_handler_task = tokio::task::spawn(async move { From c9ed66e41040b36b3f49b7de972c26df66ea0530 Mon Sep 17 00:00:00 2001 From: Moritz Date: Wed, 15 Jan 2025 16:23:47 +0100 Subject: [PATCH 09/11] Comment command arguments --- benchmarks/synth-bm/src/contract.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/benchmarks/synth-bm/src/contract.rs b/benchmarks/synth-bm/src/contract.rs index bf0cd4b4b3c..6f1014f76ac 100644 --- a/benchmarks/synth-bm/src/contract.rs +++ b/benchmarks/synth-bm/src/contract.rs @@ -22,23 +22,32 @@ use tokio::time; #[derive(Args, Debug)] pub struct BenchmarkMpcSignArgs { + /// RPC node to which transactions are sent. #[arg(long)] pub rpc_url: String, + /// Directory containing data of the users that sign transactions. #[arg(long)] pub user_data_dir: PathBuf, + /// The number of transactions to send per second. May be lower when reaching hardware limits or + /// network congestion. #[arg(long)] pub transactions_per_second: u64, + /// The total number of transactions to send. #[arg(long)] pub num_transfers: u64, + /// The id of the account to which the MPC contract has been deployed. #[arg(long)] pub receiver_id: AccountId, /// The `key_version` passed as argument to `sign`. #[arg(long)] pub key_version: u32, + /// Acts as upper bound on the number of concurrently open RPC requests. #[arg(long)] pub channel_buffer_size: usize, + /// The gas (in yoctoNEAR) attached to each `sign` function call transaction. #[arg(long)] pub gas: u64, + /// The deposit (in yoctoNEAR) attached to each `sign` function call transaction. #[arg(long)] pub deposit: u128, } From 6fadee5b1733a67319980457c114df3f9a0b999d Mon Sep 17 00:00:00 2001 From: Moritz Date: Wed, 15 Jan 2025 16:25:50 +0100 Subject: [PATCH 10/11] Rename arg --- benchmarks/synth-bm/justfile | 2 +- benchmarks/synth-bm/src/contract.rs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/benchmarks/synth-bm/justfile b/benchmarks/synth-bm/justfile index 7cec282d5f5..7645b04974f 100644 --- a/benchmarks/synth-bm/justfile +++ b/benchmarks/synth-bm/justfile @@ -36,7 +36,7 @@ benchmark_mpc_sign: cargo run --release -- benchmark-mpc-sign \ --rpc-url {{rpc_url}} \ --user-data-dir user-data/ \ - --num-transfers 500 \ + --num-transactions 500 \ --transactions-per-second 100 \ --receiver-id 'v1.signer-dev.testnet' \ --key-version 0 \ diff --git a/benchmarks/synth-bm/src/contract.rs b/benchmarks/synth-bm/src/contract.rs index 6f1014f76ac..7f462bbfa47 100644 --- a/benchmarks/synth-bm/src/contract.rs +++ b/benchmarks/synth-bm/src/contract.rs @@ -34,7 +34,7 @@ pub struct BenchmarkMpcSignArgs { pub transactions_per_second: u64, /// The total number of transactions to send. #[arg(long)] - pub num_transfers: u64, + pub num_transactions: u64, /// The id of the account to which the MPC contract has been deployed. #[arg(long)] pub receiver_id: AccountId, @@ -78,7 +78,7 @@ pub async fn benchmark_mpc_sign(args: &BenchmarkMpcSignArgs) -> anyhow::Result<( // Hence `wait_until: EXECUTED_OPTIMISTIC` as it provides most insights. let wait_until = TxExecutionStatus::ExecutedOptimistic; let wait_until_channel = wait_until.clone(); - let num_expected_responses = args.num_transfers; + let num_expected_responses = args.num_transactions; let response_handler_task = tokio::task::spawn(async move { let mut rpc_response_handler = RpcResponseHandler::new( channel_rx, @@ -91,7 +91,7 @@ pub async fn benchmark_mpc_sign(args: &BenchmarkMpcSignArgs) -> anyhow::Result<( info!("Setup complete, starting to send transactions"); let timer = Instant::now(); - for i in 0..args.num_transfers { + for i in 0..args.num_transactions { let sender_idx = usize::try_from(i).unwrap() % accounts.len(); let sender = &accounts[sender_idx]; @@ -133,7 +133,7 @@ pub async fn benchmark_mpc_sign(args: &BenchmarkMpcSignArgs) -> anyhow::Result<( info!( "Done sending {} transactions in {:.2} seconds", - args.num_transfers, + args.num_transactions, timer.elapsed().as_secs_f64() ); From 1b874a9461dc40c8412cb1ca01d36874b86d19c0 Mon Sep 17 00:00:00 2001 From: Moritz Date: Wed, 15 Jan 2025 16:29:07 +0100 Subject: [PATCH 11/11] Fix typo --- docs/practices/workflows/benchmarking_synthetic_workloads.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/practices/workflows/benchmarking_synthetic_workloads.md b/docs/practices/workflows/benchmarking_synthetic_workloads.md index 1bea59af968..121dc01b549 100644 --- a/docs/practices/workflows/benchmarking_synthetic_workloads.md +++ b/docs/practices/workflows/benchmarking_synthetic_workloads.md @@ -62,7 +62,7 @@ http localhost:3030/metrics | grep transaction_processed ### Benchmark calls to the `sign` method of an MPC contract -Assumes the accounts that send the transactions invoking `sign` have been created as described above. Transactions can be sent to a RPC of a network that on which an instance of the [`mpc/chain-signatures`](https://github.com/near/mpc/tree/79ec50759146221e7ad8bb04520f13333b75ca07/chain-signatures/contract) is deployed. +Assumes the accounts that send the transactions invoking `sign` have been created as described above. Transactions can be sent to a RPC of a network on which an instance of the [`mpc/chain-signatures`](https://github.com/near/mpc/tree/79ec50759146221e7ad8bb04520f13333b75ca07/chain-signatures/contract) is deployed. Transactions are sent to the RPC with `wait_until: EXECUTED_OPTIMISTIC` as the throughput for `sign` is at a level at which neither the network nor the RPC are expected to be a bottleneck.