From 7dc7f8f5d2bd89a8fe8314d5756e1c45a2cb399a Mon Sep 17 00:00:00 2001 From: Luke Parker Date: Sun, 26 Nov 2023 10:27:30 -0500 Subject: [PATCH 1/3] Replace rustls with boring-ssl This removes all re-attempts present in monero-serai's RPC and is an attempt to narrow down the sporadic failures. Inspired by https://github.com/hyperium/hyper/issues/3427 --- Cargo.lock | 268 ++++++++++++++++-- Cargo.toml | 2 + coins/bitcoin/src/rpc.rs | 9 +- coins/monero/src/rpc/http.rs | 65 ++--- coins/monero/src/rpc/mod.rs | 6 +- common/request/Cargo.toml | 11 +- common/request/src/lib.rs | 63 ++-- common/request/src/request.rs | 10 +- common/request/src/response.rs | 17 +- .../Dockerfile.parts/Dockerfile.serai.build | 2 +- orchestration/coordinator/Dockerfile | 2 +- orchestration/message-queue/Dockerfile | 2 +- orchestration/processor/bitcoin/Dockerfile | 2 +- orchestration/processor/monero/Dockerfile | 2 +- orchestration/serai/Dockerfile | 2 +- substrate/client/src/serai/mod.rs | 2 +- 16 files changed, 342 insertions(+), 123 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d4236e699..935381996 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -171,6 +171,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "antidote" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34fde25430d87a9388dadbe6e34d7f72a462c8b43ac8d309b42b0a8505d7e2a5" + [[package]] name = "anyhow" version = "1.0.75" @@ -332,7 +338,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d9a9bf8b79a749ee0b911b91b671cc2b6c670bdbc7e3dfd537576ddc94bb2a2" dependencies = [ - "http", + "http 0.2.11", "log", "url", ] @@ -455,6 +461,26 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "bindgen" +version = "0.68.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "726e4313eb6ec35d2730258ad4e15b547ee75d6afaa1361a922e78e59b7d8078" +dependencies = [ + "bitflags 2.4.1", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "peeking_take_while", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.39", +] + [[package]] name = "bitcoin" version = "0.31.0" @@ -632,8 +658,8 @@ dependencies = [ "futures-core", "futures-util", "hex", - "http", - "hyper", + "http 0.2.11", + "hyper 0.14.27", "hyperlocal", "log", "pin-project-lite 0.2.13", @@ -660,6 +686,29 @@ dependencies = [ "serde_with", ] +[[package]] +name = "boring" +version = "5.0.0" +source = "git+https://github.com/cloudflare/boring?rev=423c260d87b69a926594ded0dd693b5cf1220452#423c260d87b69a926594ded0dd693b5cf1220452" +dependencies = [ + "bitflags 2.4.1", + "boring-sys", + "foreign-types", + "libc", + "once_cell", +] + +[[package]] +name = "boring-sys" +version = "5.0.0" +source = "git+https://github.com/cloudflare/boring?rev=423c260d87b69a926594ded0dd693b5cf1220452#423c260d87b69a926594ded0dd693b5cf1220452" +dependencies = [ + "bindgen 0.68.1", + "cmake", + "fs_extra", + "fslock", +] + [[package]] name = "borsh" version = "1.2.0" @@ -977,6 +1026,15 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" +[[package]] +name = "cmake" +version = "0.1.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +dependencies = [ + "cc", +] + [[package]] name = "codespan-reporting" version = "0.11.1" @@ -2003,7 +2061,7 @@ dependencies = [ "futures-timer", "futures-util", "hashers", - "http", + "http 0.2.11", "instant", "jsonwebtoken", "once_cell", @@ -2217,6 +2275,33 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + [[package]] name = "fork-tree" version = "3.0.0" @@ -2440,6 +2525,22 @@ dependencies = [ "winapi", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "fslock" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04412b8935272e3a9bae6f48c7bfff74c2911f60525404edfdd28e49884c3bfb" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "funty" version = "2.0.0" @@ -2733,7 +2834,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.11", "indexmap 2.1.0", "slab", "tokio", @@ -2884,6 +2985,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.5" @@ -2891,7 +3003,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes", - "http", + "http 0.2.11", + "pin-project-lite 0.2.13", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.0.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cb79eb393015dadd30fc252023adb0b2400a0caee0fa2a077e6e21a551e840" +dependencies = [ + "bytes", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", "pin-project-lite 0.2.13", ] @@ -2930,8 +3065,8 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.5", "httparse", "httpdate", "itoa", @@ -2943,6 +3078,42 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "403f9214f3e703236b221f1a9cd88ec8b4adfa5296de01ab96216361f4692f56" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "httparse", + "itoa", + "pin-project-lite 0.2.13", + "tokio", + "want", +] + +[[package]] +name = "hyper-boring" +version = "5.0.0" +source = "git+https://github.com/cloudflare/boring?rev=423c260d87b69a926594ded0dd693b5cf1220452#423c260d87b69a926594ded0dd693b5cf1220452" +dependencies = [ + "antidote", + "boring", + "http 1.0.0", + "hyper 1.0.1", + "hyper-util", + "linked_hash_set", + "once_cell", + "tokio", + "tokio-boring", + "tower-layer", + "tower-service", +] + [[package]] name = "hyper-rustls" version = "0.24.2" @@ -2950,8 +3121,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http", - "hyper", + "http 0.2.11", + "hyper 0.14.27", "log", "rustls", "rustls-native-certs", @@ -2959,6 +3130,26 @@ dependencies = [ "tokio-rustls", ] +[[package]] +name = "hyper-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca339002caeb0d159cc6e023dff48e199f081e42fa039895c7c6f38b37f2e9d" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "hyper 1.0.1", + "pin-project-lite 0.2.13", + "socket2 0.5.5", + "tokio", + "tower", + "tower-service", + "tracing", +] + [[package]] name = "hyperlocal" version = "0.8.0" @@ -2967,7 +3158,7 @@ checksum = "0fafdf7b2b2de7c9784f76e02c0935e65a8117ec3b768644379983ab333ac98c" dependencies = [ "futures-util", "hex", - "hyper", + "hyper 0.14.27", "pin-project", "tokio", ] @@ -3055,8 +3246,8 @@ dependencies = [ "attohttpc", "bytes", "futures", - "http", - "hyper", + "http 0.2.11", + "hyper 0.14.27", "log", "rand", "tokio", @@ -3252,7 +3443,7 @@ dependencies = [ "futures-channel", "futures-util", "globset", - "hyper", + "hyper 0.14.27", "jsonrpsee-types", "parking_lot 0.12.1", "rand", @@ -3286,8 +3477,8 @@ checksum = "cf4d945a6008c9b03db3354fb3c83ee02d2faa9f2e755ec1dfb69c3551b8f4ba" dependencies = [ "futures-channel", "futures-util", - "http", - "hyper", + "http 0.2.11", + "hyper 0.14.27", "jsonrpsee-core", "jsonrpsee-types", "serde", @@ -3899,7 +4090,7 @@ version = "0.11.0+8.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3386f101bcb4bd252d8e9d2fb41ec3b0862a15a62b478c355b2982efa469e3e" dependencies = [ - "bindgen", + "bindgen 0.65.1", "bzip2-sys", "cc", "glob", @@ -4718,7 +4909,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c11e44798ad209ccdd91fc192f0526a369a01234f7373e1b141c96d7cee4f0e" dependencies = [ - "proc-macro-crate 1.3.1", + "proc-macro-crate 2.0.0", "proc-macro2", "quote", "syn 2.0.39", @@ -5841,9 +6032,9 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", - "http-body", - "hyper", + "http 0.2.11", + "http-body 0.4.5", + "hyper 0.14.27", "ipnet", "js-sys", "log", @@ -6719,7 +6910,7 @@ dependencies = [ "fnv", "futures", "futures-timer", - "hyper", + "hyper 0.14.27", "hyper-rustls", "libp2p", "log", @@ -6805,7 +6996,7 @@ name = "sc-rpc-server" version = "4.0.0-dev" source = "git+https://github.com/serai-dex/substrate#49b7d20ef96b6ad42ea0266ea27f128e0ef3214d" dependencies = [ - "http", + "http 0.2.11", "jsonrpsee", "log", "serde_json", @@ -7928,9 +8119,12 @@ name = "simple-request" version = "0.1.0" dependencies = [ "base64ct", - "hyper", - "hyper-rustls", + "http-body-util", + "hyper 1.0.1", + "hyper-boring", + "hyper-util", "tokio", + "tower-service", "zeroize", ] @@ -8025,7 +8219,7 @@ dependencies = [ "base64 0.13.1", "bytes", "futures", - "http", + "http 0.2.11", "httparse", "log", "rand", @@ -8827,7 +9021,7 @@ name = "substrate-prometheus-endpoint" version = "0.10.0-dev" source = "git+https://github.com/serai-dex/substrate#49b7d20ef96b6ad42ea0266ea27f128e0ef3214d" dependencies = [ - "hyper", + "hyper 0.14.27", "log", "prometheus", "thiserror", @@ -9118,6 +9312,17 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "tokio-boring" +version = "5.0.0" +source = "git+https://github.com/cloudflare/boring?rev=423c260d87b69a926594ded0dd693b5cf1220452#423c260d87b69a926594ded0dd693b5cf1220452" +dependencies = [ + "boring", + "boring-sys", + "once_cell", + "tokio", +] + [[package]] name = "tokio-macros" version = "2.2.0" @@ -9226,6 +9431,11 @@ version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite 0.2.13", + "tokio", "tower-layer", "tower-service", "tracing", @@ -9241,8 +9451,8 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.5", "http-range-header", "pin-project-lite 0.2.13", "tower-layer", diff --git a/Cargo.toml b/Cargo.toml index 14add4b98..906368b4c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -93,3 +93,5 @@ lazy_static = { git = "https://github.com/rust-lang-nursery/lazy-static.rs", rev # subxt *can* pull these off crates.io yet there's no benefit to this sp-core-hashing = { git = "https://github.com/serai-dex/substrate" } sp-std = { git = "https://github.com/serai-dex/substrate" } + +hyper-boring = { git = "https://github.com/cloudflare/boring", rev = "423c260d87b69a926594ded0dd693b5cf1220452" } diff --git a/coins/bitcoin/src/rpc.rs b/coins/bitcoin/src/rpc.rs index 73a215e04..3f32d0971 100644 --- a/coins/bitcoin/src/rpc.rs +++ b/coins/bitcoin/src/rpc.rs @@ -6,7 +6,7 @@ use thiserror::Error; use serde::{Deserialize, de::DeserializeOwned}; use serde_json::json; -use simple_request::{hyper, Request, Client}; +use simple_request::{hyper, Full, Request, Client}; use bitcoin::{ hashes::{Hash, hex::FromHex}, @@ -62,7 +62,8 @@ impl Rpc { /// provided to this library, if the RPC has an incompatible argument layout. That is not checked /// at time of RPC creation. pub async fn new(url: String) -> Result { - let rpc = Rpc { client: Client::with_connection_pool(), url }; + let rpc = + Rpc { client: Client::with_connection_pool().map_err(|_| RpcError::ConnectionError)?, url }; // Make an RPC request to verify the node is reachable and sane let res: String = rpc.rpc_call("help", json!([])).await?; @@ -110,11 +111,11 @@ impl Rpc { let mut request = Request::from( hyper::Request::post(&self.url) .header("Content-Type", "application/json") - .body( + .body(Full::new( serde_json::to_vec(&json!({ "jsonrpc": "2.0", "method": method, "params": params })) .unwrap() .into(), - ) + )) .unwrap(), ); request.with_basic_auth(); diff --git a/coins/monero/src/rpc/http.rs b/coins/monero/src/rpc/http.rs index 270886380..9ea0b632f 100644 --- a/coins/monero/src/rpc/http.rs +++ b/coins/monero/src/rpc/http.rs @@ -90,7 +90,7 @@ impl HttpRpc { &client .request( Request::post(url.clone()) - .body(vec![].into()) + .body("".into()) .map_err(|e| RpcError::ConnectionError(format!("couldn't make request: {e:?}")))?, ) .await @@ -102,7 +102,9 @@ impl HttpRpc { connection: Arc::new(Mutex::new((challenge, client))), } } else { - Authentication::Unauthenticated(Client::with_connection_pool()) + Authentication::Unauthenticated( + Client::with_connection_pool().map_err(|e| RpcError::ConnectionError(format!("{e:?}")))?, + ) }; Ok(Rpc(HttpRpc { authentication, url })) @@ -117,7 +119,7 @@ impl HttpRpc { .map_err(|e| RpcError::ConnectionError(format!("couldn't make request: {e:?}"))) }; - async fn body_from_response(response: Response<'_>) -> Result, RpcError> { + async fn body_from_response(response: Response) -> Result, RpcError> { /* let length = usize::try_from( response @@ -210,50 +212,37 @@ impl HttpRpc { .1 .request(request) .await - .map_err(|e| RpcError::ConnectionError(format!("{e:?}"))); + .map_err(|e| RpcError::ConnectionError(format!("{e:?}")))?; - let (error, is_stale) = match &response { - Err(e) => (Some(e.clone()), false), - Ok(response) => ( - None, - if response.status() == StatusCode::UNAUTHORIZED { - if let Some(header) = response.headers().get("www-authenticate") { - header - .to_str() - .map_err(|_| { - RpcError::InvalidNode("www-authenticate header wasn't a string".to_string()) - })? - .contains("stale") - } else { - false - } - } else { - false - }, - ), - }; + let mut is_stale = false; + if response.status() == StatusCode::UNAUTHORIZED { + if let Some(header) = response.headers().get("www-authenticate") { + if header + .to_str() + .map_err(|_| { + RpcError::InvalidNode("www-authenticate header wasn't a string".to_string()) + })? + .contains("stale") + { + is_stale = true; + } + } + } - // If the connection entered an error state, drop the cached challenge as challenges are - // per-connection - // We don't need to create a new connection as simple-request will for us - if error.is_some() || is_stale { + // If the authentication is stale, drop the cached challenge and retry + if is_stale { connection_lock.0 = None; // If we're not already on our second attempt, move to the next loop iteration // (retrying all of this once) if attempt == 0 { continue; } - if let Some(e) = error { - Err(e)? - } else { - debug_assert!(is_stale); - Err(RpcError::InvalidNode( - "node claimed fresh connection had stale authentication".to_string(), - ))? - } - } else { - body_from_response(response.unwrap()).await? + Err(RpcError::InvalidNode( + "node claimed fresh connection had stale authentication".to_string(), + ))? } + + body_from_response(response).await? } }); } diff --git a/coins/monero/src/rpc/mod.rs b/coins/monero/src/rpc/mod.rs index a5bf8c81e..0480d5757 100644 --- a/coins/monero/src/rpc/mod.rs +++ b/coins/monero/src/rpc/mod.rs @@ -135,11 +135,7 @@ impl Rpc { .0 .post( route, - if let Some(params) = params { - serde_json::to_string(¶ms).unwrap().into_bytes() - } else { - vec![] - }, + if let Some(params) = params { serde_json::to_vec(¶ms).unwrap() } else { vec![] }, ) .await?; let res_str = std_shims::str::from_utf8(&res) diff --git a/common/request/Cargo.toml b/common/request/Cargo.toml index 7fd694253..528aff107 100644 --- a/common/request/Cargo.toml +++ b/common/request/Cargo.toml @@ -14,16 +14,19 @@ all-features = true rustdoc-args = ["--cfg", "docsrs"] [dependencies] -# Deprecated here means to enable deprecated warnings, not to restore deprecated APIs -hyper = { version = "0.14", default-features = false, features = ["http1", "tcp", "client", "runtime", "backports", "deprecated"] } +hyper = { version = "1", default-features = false, features = ["http1", "client"] } +hyper-util = { version = "0.1", default-features = false, features = ["http1", "client", "tokio"] } +http-body-util = { version = "0.1", default-features = false } + tokio = { version = "1", default-features = false } -hyper-rustls = { version = "0.24", default-features = false, features = ["http1", "native-tokio"], optional = true } +tower-service = { version = "0.3", default-features = false, optional = true } +hyper-boring = { version = "5", default-features = false, optional = true } zeroize = { version = "1", optional = true } base64ct = { version = "1", features = ["alloc"], optional = true } [features] -tls = ["hyper-rustls"] +tls = ["hyper-boring"] basic-auth = ["zeroize", "base64ct"] default = ["tls"] diff --git a/common/request/src/lib.rs b/common/request/src/lib.rs index 4c738e2ef..27e3df9ae 100644 --- a/common/request/src/lib.rs +++ b/common/request/src/lib.rs @@ -6,13 +6,13 @@ use std::sync::Arc; use tokio::sync::Mutex; #[cfg(feature = "tls")] -use hyper_rustls::{HttpsConnectorBuilder, HttpsConnector}; -use hyper::{ - Uri, - header::HeaderValue, - body::Body, - service::Service, - client::{HttpConnector, conn::http1::SendRequest}, +use tower_service::Service as TowerService; +#[cfg(feature = "tls")] +use hyper_boring::HttpsConnector; +use hyper::{Uri, header::HeaderValue, body::Bytes, client::conn::http1::SendRequest}; +use hyper_util::{ + rt::tokio::TokioExecutor, + client::legacy::{Client as HyperClient, connect::HttpConnector}, }; pub use hyper; @@ -29,6 +29,7 @@ pub enum Error { InconsistentHost, ConnectionError(Box), Hyper(hyper::Error), + HyperUtil(hyper_util::client::legacy::Error), } #[cfg(not(feature = "tls"))] @@ -36,10 +37,19 @@ type Connector = HttpConnector; #[cfg(feature = "tls")] type Connector = HttpsConnector; -#[derive(Clone, Debug)] +#[derive(Clone)] enum Connection { - ConnectionPool(hyper::Client), - Connection { connector: Connector, host: Uri, connection: Arc>>> }, + ConnectionPool(HyperClient>), + Connection { + connector: Connector, + host: Uri, + connection: Arc>>>>, + }, +} +impl core::fmt::Debug for Connection { + fn fmt(&self, fmt: &mut core::fmt::Formatter<'_>) -> Result<(), core::fmt::Error> { + fmt.debug_struct("Connection").finish_non_exhaustive() + } } #[derive(Clone, Debug)] @@ -48,25 +58,26 @@ pub struct Client { } impl Client { - fn connector() -> Connector { + fn connector() -> Result { #[cfg(feature = "tls")] - let res = - HttpsConnectorBuilder::new().with_native_roots().https_or_http().enable_http1().build(); + let res = HttpsConnector::new().map_err(|e| Error::ConnectionError(format!("{e:?}").into()))?; #[cfg(not(feature = "tls"))] let res = HttpConnector::new(); - res + Ok(res) } - pub fn with_connection_pool() -> Client { - Client { - connection: Connection::ConnectionPool(hyper::Client::builder().build(Self::connector())), - } + pub fn with_connection_pool() -> Result { + Ok(Client { + connection: Connection::ConnectionPool( + HyperClient::builder(TokioExecutor::new()).build(Self::connector()?), + ), + }) } pub fn without_connection_pool(host: String) -> Result { Ok(Client { connection: Connection::Connection { - connector: Self::connector(), + connector: Self::connector()?, host: { let uri: Uri = host.parse().map_err(|_| Error::InvalidUri)?; if uri.host().is_none() { @@ -79,7 +90,7 @@ impl Client { }) } - pub async fn request>(&self, request: R) -> Result, Error> { + pub async fn request>(&self, request: R) -> Result { let request: Request = request.into(); let mut request = request.0; if let Some(header_host) = request.headers().get(hyper::header::HOST) { @@ -111,8 +122,10 @@ impl Client { .insert(hyper::header::HOST, HeaderValue::from_str(&host).map_err(|_| Error::InvalidUri)?); } - let response = match &self.connection { - Connection::ConnectionPool(client) => client.request(request).await.map_err(Error::Hyper)?, + Ok(Response(match &self.connection { + Connection::ConnectionPool(client) => { + client.request(request).await.map_err(Error::HyperUtil)? + } Connection::Connection { connector, host, connection } => { let mut connection_lock = connection.lock().await; @@ -137,7 +150,7 @@ impl Client { // Send the request let res = connection.send_request(request).await; if let Ok(res) = res { - return Ok(Response(res, self)); + return Ok(Response(res)); } err = res.err(); } @@ -145,8 +158,6 @@ impl Client { *connection_lock = None; Err(Error::Hyper(err.unwrap()))? } - }; - - Ok(Response(response, self)) + })) } } diff --git a/common/request/src/request.rs b/common/request/src/request.rs index 1117e9fd6..c396eeb31 100644 --- a/common/request/src/request.rs +++ b/common/request/src/request.rs @@ -1,12 +1,12 @@ -use hyper::body::Body; #[cfg(feature = "basic-auth")] -use hyper::header::HeaderValue; +use hyper::{body::Bytes, header::HeaderValue}; +pub use http_body_util::Full; #[cfg(feature = "basic-auth")] use crate::Error; #[derive(Debug)] -pub struct Request(pub(crate) hyper::Request); +pub struct Request(pub(crate) hyper::Request>); impl Request { #[cfg(feature = "basic-auth")] fn username_password_from_uri(&self) -> Result<(String, String), Error> { @@ -59,8 +59,8 @@ impl Request { let _ = self.basic_auth_from_uri(); } } -impl From> for Request { - fn from(request: hyper::Request) -> Request { +impl From>> for Request { + fn from(request: hyper::Request>) -> Request { Request(request) } } diff --git a/common/request/src/response.rs b/common/request/src/response.rs index 04c8472b8..dfde5c3ec 100644 --- a/common/request/src/response.rs +++ b/common/request/src/response.rs @@ -1,15 +1,16 @@ use hyper::{ StatusCode, header::{HeaderValue, HeaderMap}, - body::{Buf, Body}, + body::{Buf, Incoming}, }; +use http_body_util::BodyExt; -use crate::{Client, Error}; +use crate::Error; // Borrows the client so its async task lives as long as this response exists. #[derive(Debug)] -pub struct Response<'a>(pub(crate) hyper::Response, pub(crate) &'a Client); -impl<'a> Response<'a> { +pub struct Response(pub(crate) hyper::Response); +impl Response { pub fn status(&self) -> StatusCode { self.0.status() } @@ -17,6 +18,12 @@ impl<'a> Response<'a> { self.0.headers() } pub async fn body(self) -> Result { - hyper::body::aggregate(self.0.into_body()).await.map(Buf::reader).map_err(Error::Hyper) + self + .0 + .into_body() + .collect() + .await + .map_err(Error::Hyper) + .map(|collected| Buf::reader(collected.aggregate())) } } diff --git a/orchestration/Dockerfile.parts/Dockerfile.serai.build b/orchestration/Dockerfile.parts/Dockerfile.serai.build index be3af4233..6dfa6f88e 100644 --- a/orchestration/Dockerfile.parts/Dockerfile.serai.build +++ b/orchestration/Dockerfile.parts/Dockerfile.serai.build @@ -6,7 +6,7 @@ RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload RUN apt update && apt upgrade -y && apt autoremove -y && apt clean # Add dev dependencies -RUN apt install -y pkg-config clang +RUN apt install -y pkg-config clang cmake git # Dependencies for the Serai node RUN apt install -y make protobuf-compiler diff --git a/orchestration/coordinator/Dockerfile b/orchestration/coordinator/Dockerfile index 44be31160..ea7a3a1e6 100644 --- a/orchestration/coordinator/Dockerfile +++ b/orchestration/coordinator/Dockerfile @@ -16,7 +16,7 @@ RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload RUN apt update && apt upgrade -y && apt autoremove -y && apt clean # Add dev dependencies -RUN apt install -y pkg-config clang +RUN apt install -y pkg-config clang cmake git # Dependencies for the Serai node RUN apt install -y make protobuf-compiler diff --git a/orchestration/message-queue/Dockerfile b/orchestration/message-queue/Dockerfile index c5ea8a6d9..5a88066fb 100644 --- a/orchestration/message-queue/Dockerfile +++ b/orchestration/message-queue/Dockerfile @@ -16,7 +16,7 @@ RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload RUN apt update && apt upgrade -y && apt autoremove -y && apt clean # Add dev dependencies -RUN apt install -y pkg-config clang +RUN apt install -y pkg-config clang cmake git # Dependencies for the Serai node RUN apt install -y make protobuf-compiler diff --git a/orchestration/processor/bitcoin/Dockerfile b/orchestration/processor/bitcoin/Dockerfile index f7d1c9c10..34f4969da 100644 --- a/orchestration/processor/bitcoin/Dockerfile +++ b/orchestration/processor/bitcoin/Dockerfile @@ -16,7 +16,7 @@ RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload RUN apt update && apt upgrade -y && apt autoremove -y && apt clean # Add dev dependencies -RUN apt install -y pkg-config clang +RUN apt install -y pkg-config clang cmake git # Dependencies for the Serai node RUN apt install -y make protobuf-compiler diff --git a/orchestration/processor/monero/Dockerfile b/orchestration/processor/monero/Dockerfile index cf531aa95..b11742923 100644 --- a/orchestration/processor/monero/Dockerfile +++ b/orchestration/processor/monero/Dockerfile @@ -16,7 +16,7 @@ RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload RUN apt update && apt upgrade -y && apt autoremove -y && apt clean # Add dev dependencies -RUN apt install -y pkg-config clang +RUN apt install -y pkg-config clang cmake git # Dependencies for the Serai node RUN apt install -y make protobuf-compiler diff --git a/orchestration/serai/Dockerfile b/orchestration/serai/Dockerfile index 0fa6f9b30..b58a95598 100644 --- a/orchestration/serai/Dockerfile +++ b/orchestration/serai/Dockerfile @@ -16,7 +16,7 @@ RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload RUN apt update && apt upgrade -y && apt autoremove -y && apt clean # Add dev dependencies -RUN apt install -y pkg-config clang +RUN apt install -y pkg-config clang cmake git # Dependencies for the Serai node RUN apt install -y make protobuf-compiler diff --git a/substrate/client/src/serai/mod.rs b/substrate/client/src/serai/mod.rs index b725c02a1..91f2969b0 100644 --- a/substrate/client/src/serai/mod.rs +++ b/substrate/client/src/serai/mod.rs @@ -150,7 +150,7 @@ impl Serai { } pub async fn new(url: String) -> Result { - let client = Client::with_connection_pool(); + let client = Client::with_connection_pool().map_err(|_| SeraiError::ConnectionError)?; let mut res = Serai { url, client, genesis: [0xfe; 32] }; res.genesis = res.block_hash(0).await?.ok_or_else(|| { SeraiError::InvalidNode("node didn't have the first block's hash".to_string()) From 7434a6e0bdda0c6322ff6d3762457532b7021f82 Mon Sep 17 00:00:00 2001 From: Luke Parker Date: Wed, 29 Nov 2023 05:45:49 -0500 Subject: [PATCH 2/3] Correct optional features --- common/request/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/request/Cargo.toml b/common/request/Cargo.toml index 528aff107..51fb7da4b 100644 --- a/common/request/Cargo.toml +++ b/common/request/Cargo.toml @@ -27,6 +27,6 @@ zeroize = { version = "1", optional = true } base64ct = { version = "1", features = ["alloc"], optional = true } [features] -tls = ["hyper-boring"] +tls = ["tower-service", "hyper-boring"] basic-auth = ["zeroize", "base64ct"] default = ["tls"] From f077d758e5938a86db09bc2a0388b01bebfc4dfb Mon Sep 17 00:00:00 2001 From: Luke Parker Date: Wed, 29 Nov 2023 22:25:25 -0500 Subject: [PATCH 3/3] Correct feature flagging I did compile all feature combinations to ensure it's proper now. --- common/request/Cargo.toml | 6 +++--- common/request/src/lib.rs | 1 - common/request/src/request.rs | 3 ++- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/common/request/Cargo.toml b/common/request/Cargo.toml index 51fb7da4b..c5d22a6d9 100644 --- a/common/request/Cargo.toml +++ b/common/request/Cargo.toml @@ -15,18 +15,18 @@ rustdoc-args = ["--cfg", "docsrs"] [dependencies] hyper = { version = "1", default-features = false, features = ["http1", "client"] } -hyper-util = { version = "0.1", default-features = false, features = ["http1", "client", "tokio"] } +hyper-util = { version = "0.1", default-features = false, features = ["http1", "client-legacy", "tokio"] } http-body-util = { version = "0.1", default-features = false } tokio = { version = "1", default-features = false } -tower-service = { version = "0.3", default-features = false, optional = true } +tower-service = { version = "0.3", default-features = false } hyper-boring = { version = "5", default-features = false, optional = true } zeroize = { version = "1", optional = true } base64ct = { version = "1", features = ["alloc"], optional = true } [features] -tls = ["tower-service", "hyper-boring"] +tls = ["hyper-boring"] basic-auth = ["zeroize", "base64ct"] default = ["tls"] diff --git a/common/request/src/lib.rs b/common/request/src/lib.rs index 27e3df9ae..b9bfb39bf 100644 --- a/common/request/src/lib.rs +++ b/common/request/src/lib.rs @@ -5,7 +5,6 @@ use std::sync::Arc; use tokio::sync::Mutex; -#[cfg(feature = "tls")] use tower_service::Service as TowerService; #[cfg(feature = "tls")] use hyper_boring::HttpsConnector; diff --git a/common/request/src/request.rs b/common/request/src/request.rs index c396eeb31..ab6cb7a94 100644 --- a/common/request/src/request.rs +++ b/common/request/src/request.rs @@ -1,5 +1,6 @@ +use hyper::body::Bytes; #[cfg(feature = "basic-auth")] -use hyper::{body::Bytes, header::HeaderValue}; +use hyper::header::HeaderValue; pub use http_body_util::Full; #[cfg(feature = "basic-auth")]