introduce vet kd payload

rs/bitcoin/consensus/src/payload_builder/parse.rs

@@ -1,11 +1,13 @@
 use ic_btc_replica_types::BitcoinAdapterResponse;
-use ic_interfaces::batch_payload::{iterator_to_bytes, slice_to_messages, PastPayload};
+use ic_interfaces::batch_payload::PastPayload;
 use ic_logger::{error, warn, ReplicaLogger};
 use ic_protobuf::{
     bitcoin::v1::BitcoinAdapterResponse as PbBitcoinAdapterResponse, proxy::ProxyDecodeError,
 };
 use ic_types::{
-    batch::{SelfValidatingPayload, MAX_BITCOIN_PAYLOAD_IN_BYTES},
+    batch::{
+        iterator_to_bytes, slice_to_messages, SelfValidatingPayload, MAX_BITCOIN_PAYLOAD_IN_BYTES,
+    },
     NumBytes,
 };
 use prost::Message;

rs/https_outcalls/consensus/src/payload_builder/parse.rs

@@ -1,11 +1,15 @@
-use ic_interfaces::batch_payload::{iterator_to_bytes, slice_to_messages, PastPayload};
+use ic_interfaces::batch_payload::PastPayload;
 use ic_logger::{error, ReplicaLogger};
 use ic_protobuf::{
     proxy::ProxyDecodeError,
     types::v1 as pb,
     types::v1::{canister_http_response_message::MessageType, CanisterHttpResponseMessage},
 };
-use ic_types::{batch::CanisterHttpPayload, messages::CallbackId, NumBytes};
+use ic_types::{
+    batch::{iterator_to_bytes, slice_to_messages, CanisterHttpPayload},
+    messages::CallbackId,
+    NumBytes,
+};
 use std::collections::HashSet;
 
 pub(crate) fn bytes_to_payload(data: &[u8]) -> Result<CanisterHttpPayload, ProxyDecodeError> {

rs/interfaces/src/batch_payload.rs

@@ -3,7 +3,6 @@ use ic_base_types::NumBytes;
 use ic_types::{
     batch::ValidationContext, consensus::BlockPayload, crypto::CryptoHashOf, Height, NodeId, Time,
 };
-use prost::{bytes::BufMut, DecodeError, Message};
 
 /// A list of [`PastPayload`] will be passed to invocation of
 ///  [`BatchPayloadBuilder::build_payload`].
@@ -102,41 +101,3 @@ pub trait IntoMessages<M> {
     /// returns `Ok(())` on the same payload.
     fn into_messages(payload: &[u8]) -> M;
 }
-
-/// Given an iterator of [`Message`]s, this function will deserialize the messages
-/// into a byte vector.
-///
-/// The function is given a `max_size` limit, and guarantees that the buffer will be
-/// smaller or equal than the byte limit.
-/// It may drop messages from the iterator, if they don't fit.
-pub fn iterator_to_bytes<I, M>(iter: I, max_size: NumBytes) -> Vec<u8>
-where
-    M: Message,
-    I: Iterator<Item = M>,
-{
-    let mut buffer = vec![].limit(max_size.get() as usize);
-
-    for val in iter {
-        // NOTE: This call may fail due to the encoding hitting the
-        // byte limit. We continue trying the rest of the messages
-        // nonetheless, to give smaller messages a chance as well
-        let _ = val.encode_length_delimited(&mut buffer);
-    }
-
-    buffer.into_inner()
-}
-
-/// Parse a slice filled with protobuf encoded [`Message`]s into a vector
-pub fn slice_to_messages<M>(mut data: &[u8]) -> Result<Vec<M>, DecodeError>
-where
-    M: Message + Default,
-{
-    let mut msgs = vec![];
-
-    while !data.is_empty() {
-        let msg = M::decode_length_delimited(&mut data)?;
-        msgs.push(msg)
-    }
-
-    Ok(msgs)
-}

rs/interfaces/src/consensus.rs

@@ -15,6 +15,7 @@ use crate::{
         SelfValidatingPayloadValidationFailure,
     },
     validation::{ValidationError, ValidationResult},
+    vet_kd::{InvalidVetKdPayloadReason, VetKdPayloadValidationFailure},
 };
 use ic_base_types::{NumBytes, SubnetId};
 use ic_types::{
@@ -62,6 +63,7 @@ pub enum InvalidPayloadReason {
     InvalidSelfValidatingPayload(InvalidSelfValidatingPayloadReason),
     InvalidCanisterHttpPayload(InvalidCanisterHttpPayloadReason),
     InvalidQueryStatsPayload(InvalidQueryStatsPayloadReason),
+    InvalidVetKdPayload(InvalidVetKdPayloadReason),
     /// The overall block size is too large, even though the individual payloads are valid
     PayloadTooBig {
         expected: NumBytes,
@@ -76,6 +78,7 @@ pub enum PayloadValidationFailure {
     SelfValidatingPayloadValidationFailed(SelfValidatingPayloadValidationFailure),
     CanisterHttpPayloadValidationFailed(CanisterHttpPayloadValidationFailure),
     QueryStatsPayloadValidationFailed(QueryStatsPayloadValidationFailure),
+    VetKdPayloadValidationFailed(VetKdPayloadValidationFailure),
     RegistryUnavailable(RegistryClientError),
     SubnetNotFound(SubnetId),
 }

rs/interfaces/src/lib.rs

@@ -19,6 +19,7 @@ pub mod query_stats;
 pub mod self_validating_payload;
 pub mod time_source;
 pub mod validation;
+pub mod vet_kd;
 
 // Note [Associated Types in Interfaces]
 // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

rs/interfaces/src/vet_kd.rs

@@ -0,0 +1,36 @@
+use ic_interfaces_state_manager::StateManagerError;
+use ic_protobuf::proxy::ProxyDecodeError;
+use ic_types::{
+    batch::VetKdAgreement, messages::CallbackId, registry::RegistryClientError, Height,
+};
+
+#[derive(Debug)]
+pub enum InvalidVetKdPayloadReason {
+    /// The feature is not enabled
+    Disabled,
+    /// The payload could not be deserialized
+    DeserializationFailed(ProxyDecodeError),
+    /// The payload contained a response that was already delivered
+    DuplicateResponse(CallbackId),
+    /// The payload contained a response that wasn't requested
+    MissingContext(CallbackId),
+    /// The payload contained a response for an IDkg context
+    IDkgContext(CallbackId),
+    /// A response was rejected for the wrong reason
+    MismatchedReject {
+        expected: Option<VetKdAgreement>,
+        received: Option<VetKdAgreement>,
+    },
+    /// A success response couldn't be decoded
+    DecodingError(String),
+}
+
+#[derive(Debug)]
+pub enum VetKdPayloadValidationFailure {
+    /// The state was not available for a height
+    StateUnavailable(StateManagerError),
+    /// The registry version was not available for a height
+    RegistryVersionUnavailable(Height),
+    /// The registry client returned an error
+    RegistryClientError(RegistryClientError),
+}

rs/protobuf/def/types/v1/consensus.proto

@@ -67,6 +67,7 @@ message Block {
   reserved 14;
   bytes canister_http_payload_bytes = 15;
   bytes query_stats_payload_bytes = 16;
+  bytes vet_kd_payload_bytes = 17;
   bytes payload_hash = 11;
 }
 
@@ -198,6 +199,20 @@ message CanisterQueryStats {
   uint64 egress_payload_size = 5;
 }
 
+enum VetKdErrorCode {
+  VET_KD_ERROR_CODE_UNSPECIFIED = 0;
+  VET_KD_ERROR_CODE_TIMED_OUT = 1;
+  VET_KD_ERROR_CODE_INVALID_KEY = 2;
+}
+
+message VetKdAgreement {
+  uint64 callback_id = 1;
+  oneof agreement {
+    bytes data = 2;
+    VetKdErrorCode reject = 3;
+  }
+}
+
 message IngressIdOffset {
   uint64 expiry = 1;
   bytes message_id = 2;

rs/protobuf/src/gen/types/types.v1.rs

@@ -1305,6 +1305,8 @@ pub struct Block {
     pub canister_http_payload_bytes: ::prost::alloc::vec::Vec<u8>,
     #[prost(bytes = "vec", tag = "16")]
     pub query_stats_payload_bytes: ::prost::alloc::vec::Vec<u8>,
+    #[prost(bytes = "vec", tag = "17")]
+    pub vet_kd_payload_bytes: ::prost::alloc::vec::Vec<u8>,
     #[prost(bytes = "vec", tag = "11")]
     pub payload_hash: ::prost::alloc::vec::Vec<u8>,
 }
@@ -1519,6 +1521,23 @@ pub struct CanisterQueryStats {
     pub egress_payload_size: u64,
 }
 #[derive(Clone, PartialEq, ::prost::Message)]
+pub struct VetKdAgreement {
+    #[prost(uint64, tag = "1")]
+    pub callback_id: u64,
+    #[prost(oneof = "vet_kd_agreement::Agreement", tags = "2, 3")]
+    pub agreement: ::core::option::Option<vet_kd_agreement::Agreement>,
+}
+/// Nested message and enum types in `VetKdAgreement`.
+pub mod vet_kd_agreement {
+    #[derive(Clone, PartialEq, ::prost::Oneof)]
+    pub enum Agreement {
+        #[prost(bytes, tag = "2")]
+        Data(::prost::alloc::vec::Vec<u8>),
+        #[prost(enumeration = "super::VetKdErrorCode", tag = "3")]
+        Reject(i32),
+    }
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
 pub struct IngressIdOffset {
     #[prost(uint64, tag = "1")]
     pub expiry: u64,
@@ -1583,3 +1602,32 @@ pub struct StrippedConsensusMessageId {
     #[prost(message, optional, tag = "1")]
     pub unstripped_id: ::core::option::Option<ConsensusMessageId>,
 }
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
+#[repr(i32)]
+pub enum VetKdErrorCode {
+    Unspecified = 0,
+    TimedOut = 1,
+    InvalidKey = 2,
+}
+impl VetKdErrorCode {
+    /// String value of the enum field names used in the ProtoBuf definition.
+    ///
+    /// The values are not transformed in any way and thus are considered stable
+    /// (if the ProtoBuf definition does not change) and safe for programmatic use.
+    pub fn as_str_name(&self) -> &'static str {
+        match self {
+            Self::Unspecified => "VET_KD_ERROR_CODE_UNSPECIFIED",
+            Self::TimedOut => "VET_KD_ERROR_CODE_TIMED_OUT",
+            Self::InvalidKey => "VET_KD_ERROR_CODE_INVALID_KEY",
+        }
+    }
+    /// Creates an enum from field names used in the ProtoBuf definition.
+    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+        match value {
+            "VET_KD_ERROR_CODE_UNSPECIFIED" => Some(Self::Unspecified),
+            "VET_KD_ERROR_CODE_TIMED_OUT" => Some(Self::TimedOut),
+            "VET_KD_ERROR_CODE_INVALID_KEY" => Some(Self::InvalidKey),
+            _ => None,
+        }
+    }
+}

rs/test_utilities/consensus/src/batch.rs

@@ -1,10 +1,13 @@
 use ic_base_types::NumBytes;
 use ic_interfaces::{
-    batch_payload::{iterator_to_bytes, BatchPayloadBuilder, PastPayload, ProposalContext},
+    batch_payload::{BatchPayloadBuilder, PastPayload, ProposalContext},
     consensus::PayloadValidationError,
     validation::ValidationResult,
 };
-use ic_types::{batch::ValidationContext, Height};
+use ic_types::{
+    batch::{iterator_to_bytes, ValidationContext},
+    Height,
+};
 use mockall::*;
 
 mock! {

rs/types/types/src/batch.rs

@@ -5,6 +5,7 @@ mod canister_http;
 mod execution_environment;
 mod ingress;
 mod self_validating;
+mod vet_kd;
 mod xnet;
 
 pub use self::{
@@ -15,6 +16,10 @@ pub use self::{
     },
     ingress::{IngressPayload, IngressPayloadError},
     self_validating::{SelfValidatingPayload, MAX_BITCOIN_PAYLOAD_IN_BYTES},
+    vet_kd::{
+        bytes_to_vet_kd_payload, vet_kd_payload_to_bytes, VetKdAgreement, VetKdErrorCode,
+        VetKdPayload,
+    },
     xnet::XNetPayload,
 };
 use crate::{
@@ -24,12 +29,13 @@ use crate::{
     xnet::CertifiedStreamSlice,
     Height, Randomness, RegistryVersion, ReplicaVersion, SubnetId, Time,
 };
-use ic_base_types::NodeId;
+use ic_base_types::{NodeId, NumBytes};
 use ic_btc_replica_types::BitcoinAdapterResponse;
 #[cfg(test)]
 use ic_exhaustive_derive::ExhaustiveSet;
 use ic_management_canister_types::MasterPublicKeyId;
 use ic_protobuf::{proxy::ProxyDecodeError, types::v1 as pb};
+use prost::{bytes::BufMut, DecodeError, Message};
 use serde::{Deserialize, Serialize};
 use std::{
     collections::{BTreeMap, BTreeSet},
@@ -187,6 +193,44 @@ impl BlockmakerMetrics {
     }
 }
 
+/// Given an iterator of [`Message`]s, this function will deserialize the messages
+/// into a byte vector.
+///
+/// The function is given a `max_size` limit, and guarantees that the buffer will be
+/// smaller or equal than the byte limit.
+/// It may drop messages from the iterator, if they don't fit.
+pub fn iterator_to_bytes<I, M>(iter: I, max_size: NumBytes) -> Vec<u8>
+where
+    M: Message,
+    I: Iterator<Item = M>,
+{
+    let mut buffer = vec![].limit(max_size.get() as usize);
+
+    for val in iter {
+        // NOTE: This call may fail due to the encoding hitting the
+        // byte limit. We continue trying the rest of the messages
+        // nonetheless, to give smaller messages a chance as well
+        let _ = val.encode_length_delimited(&mut buffer);
+    }
+
+    buffer.into_inner()
+}
+
+/// Parse a slice filled with protobuf encoded [`Message`]s into a vector
+pub fn slice_to_messages<M>(mut data: &[u8]) -> Result<Vec<M>, DecodeError>
+where
+    M: Message + Default,
+{
+    let mut msgs = vec![];
+
+    while !data.is_empty() {
+        let msg = M::decode_length_delimited(&mut data)?;
+        msgs.push(msg)
+    }
+
+    Ok(msgs)
+}
+
 /// Response to a subnet call that requires Consensus' involvement.
 ///
 /// Only holds the payload and callback ID, Execution populates other fields