This commit contains:-

* Apollo test cases runs for EdDSA sign/verify.
* Tidy-check run.

.github/workflows/build_and_test_clang_debug.yml

@@ -59,9 +59,6 @@ jobs:
                               -DUSE_OPENTRACING=ON \
                               -DOMIT_TEST_OUTPUT=OFF\
                               -DKEEP_APOLLO_LOGS=TRUE\
-                              -DUSE_CRYPTOPP_HASH=TRUE\
-                              -DUSE_OPENSSL_SHA_256=FALSE\
-                              -DUSE_OPENSSL_SHA3_256=FALSE\
                               -DUSE_FAKE_CLOCK_IN_TIME_SERVICE=TRUE\" "\
               && script -q -e -c "make test"
         - name: Prepare artifacts

.github/workflows/build_and_test_clang_release.yml

@@ -60,9 +60,6 @@ jobs:
                               -DUSE_OPENTRACING=ON \
                               -DOMIT_TEST_OUTPUT=OFF\
                               -DKEEP_APOLLO_LOGS=TRUE\
-                              -DUSE_CRYPTOPP_HASH=TRUE\
-                              -DUSE_OPENSSL_SHA_256=FALSE\
-                              -DUSE_OPENSSL_SHA3_256=FALSE\
                               -DUSE_FAKE_CLOCK_IN_TIME_SERVICE=TRUE\" "\
               && script -q -e -c "make test"
         - name: Prepare artifacts

.github/workflows/build_and_test_gcc_debug.yml

@@ -59,9 +59,6 @@ jobs:
                               -DOMIT_TEST_OUTPUT=OFF\
                               -DKEEP_APOLLO_LOGS=TRUE\
                               -DRUN_APOLLO_TESTS=FALSE\
-                              -DUSE_CRYPTOPP_HASH=TRUE\
-                              -DUSE_OPENSSL_SHA_256=FALSE\
-                              -DUSE_OPENSSL_SHA3_256=FALSE\
                               -DUSE_FAKE_CLOCK_IN_TIME_SERVICE=TRUE\" "\
               && script -q -e -c "make test"
         - name: Prepare artifacts

.github/workflows/build_and_test_gcc_release.yml

@@ -59,9 +59,6 @@ jobs:
                               -DOMIT_TEST_OUTPUT=OFF\
                               -DKEEP_APOLLO_LOGS=TRUE\
                               -DRUN_APOLLO_TESTS=FALSE\
-                              -DUSE_CRYPTOPP_HASH=TRUE\
-                              -DUSE_OPENSSL_SHA_256=FALSE\
-                              -DUSE_OPENSSL_SHA3_256=FALSE\
                               -DUSE_FAKE_CLOCK_IN_TIME_SERVICE=TRUE\" "\
               && script -q -e -c "make test"
         - name: Prepare artifacts

.github/workflows/clang-tidy.yml

@@ -37,9 +37,6 @@ jobs:
                             -DUSE_S3_OBJECT_STORE=TRUE \
                             -DUSE_OPENTRACING=ON \
                             -DOMIT_TEST_OUTPUT=OFF\
-                            -DUSE_CRYPTOPP_HASH=TRUE\
-                            -DUSE_OPENSSL_SHA_256=FALSE\
-                            -DUSE_OPENSSL_SHA3_256=FALSE\
                             -DUSE_FAKE_CLOCK_IN_TIME_SERVICE=TRUE\" "\
         - name: Print failure info
           if: failure()

CMakeLists.txt

@@ -114,19 +114,6 @@ if(CODECOVERAGE)
     message( "-- Building with llvm Code Coverage Tools")
 endif()
 
-if(USE_CRYPTOPP_HASH)
-    message("-- USE_CRYPTOPP_HASH Enabled")
-    string(APPEND CMAKE_CXX_FLAGS " -DUSE_CRYPTOPP_HASH")
-elseif(USE_OPENSSL_SHA_256)
-    message("-- USE_OPENSSL_SHA_256 Enabled")
-    string(APPEND CMAKE_CXX_FLAGS " -DUSE_OPENSSL_SHA_256")
-elseif(USE_OPENSSL_SHA3_256)
-    message("-- USE_OPENSSL_SHA3_256 Enabled")
-    string(APPEND CMAKE_CXX_FLAGS " -DUSE_OPENSSL_SHA3_256")
-else()
-    message(FATAL_ERROR "None of the cryptographic hashing libraries are enabled.")
-endif()
-
 if(USE_S3_OBJECT_STORE)
     add_compile_definitions(USE_S3_OBJECT_STORE=1)
 endif()

Makefile

@@ -1,6 +1,6 @@
 CONCORD_BFT_DOCKER_REPO?=concordbft/
 CONCORD_BFT_DOCKER_IMAGE?=concord-bft
-CONCORD_BFT_DOCKER_IMAGE_VERSION?=0.42
+CONCORD_BFT_DOCKER_IMAGE_VERSION?=0.43
 CONCORD_BFT_DOCKER_CONTAINER?=concord-bft
 
 CONCORD_BFT_DOCKERFILE?=Dockerfile
@@ -57,9 +57,6 @@ CONCORD_BFT_CMAKE_TSAN?=FALSE
 CONCORD_BFT_CMAKE_CODECOVERAGE?=FALSE
 CONCORD_BFT_CMAKE_USE_FAKE_CLOCK_IN_TIME_SERVICE?=FALSE
 ENABLE_RESTART_RECOVERY_TESTS?=FALSE
-CONCORD_BFT_CMAKE_USE_CRYPTOPP_HASH?=TRUE
-CONCORD_BFT_CMAKE_USE_OPENSSL_SHA_256?=FALSE
-CONCORD_BFT_CMAKE_USE_OPENSSL_SHA3_256?=FALSE
 
 ifeq (${CONCORD_BFT_CMAKE_ASAN},TRUE)
 	CONCORD_BFT_CMAKE_CXX_FLAGS_RELEASE='-O0 -g'

bftengine/src/bftengine/KeyExchangeManager.cpp

@@ -24,6 +24,8 @@
 
 namespace bftEngine::impl {
 
+using concord::util::crypto::KeyFormat;
+
 KeyExchangeManager::KeyExchangeManager(InitData* id)
     : repID_{ReplicaConfig::instance().getreplicaId()},
       clusterSize_{ReplicaConfig::instance().getnumReplicas()},
@@ -157,9 +159,8 @@ void KeyExchangeManager::exchangeTlsKeys(const std::string& type, const SeqNum&
   const std::string base_path =
       bftEngine::ReplicaConfig::instance().certificatesRootPath + "/" + std::to_string(repID_);
   std::string root_path = (use_unified_certs) ? base_path : base_path + "/" + type;
-
   std::string cert_path = (use_unified_certs) ? root_path + "/node.cert" : root_path + "/" + type + ".cert";
-  std::string prev_key_pem = concord::util::crypto::Crypto::instance()
+  std::string prev_key_pem = concord::util::cryptopp_utils::Crypto::instance()
                                  .RsaHexToPem(std::make_pair(SigManager::instance()->getSelfPrivKey(), ""))
                                  .first;
   auto cert =
@@ -299,9 +300,7 @@ void KeyExchangeManager::onPublishClientsKeys(const std::string& keys, std::opti
   if (save) saveClientsPublicKeys(keys);
 }
 
-void KeyExchangeManager::onClientPublicKeyExchange(const std::string& key,
-                                                   concord::util::crypto::KeyFormat fmt,
-                                                   NodeIdType clientId) {
+void KeyExchangeManager::onClientPublicKeyExchange(const std::string& key, KeyFormat fmt, NodeIdType clientId) {
   LOG_INFO(KEY_EX_LOG, "key: " << key << " fmt: " << (uint16_t)fmt << " client: " << clientId);
   // persist a new key
   clientPublicKeyStore_->setClientPublicKey(clientId, key, fmt);
@@ -310,7 +309,7 @@ void KeyExchangeManager::onClientPublicKeyExchange(const std::string& key,
 }
 
 void KeyExchangeManager::loadClientPublicKey(const std::string& key,
-                                             concord::util::crypto::KeyFormat fmt,
+                                             KeyFormat fmt,
                                              NodeIdType clientId,
                                              bool saveToReservedPages) {
   LOG_INFO(KEY_EX_LOG, "key: " << key << " fmt: " << (uint16_t)fmt << " client: " << clientId);

bftengine/src/bftengine/SigManager.cpp

@@ -20,14 +20,20 @@
 #include "hex_tools.h"
 
 using namespace std;
-// using concord::util::cryptopp_utils::RSASigner;
-// using concord::util::cryptopp_utils::RSAVerifier;
-using concord::util::openssl_utils::EdDSA_Signer;
-using concord::util::openssl_utils::EdDSA_Verifier;
 
 namespace bftEngine {
 namespace impl {
 
+#define RSA_Algo false
+
+#if RSA_Algo
+using concord::util::cryptopp_utils::RSASigner;
+using concord::util::cryptopp_utils::RSAVerifier;
+#else
+using concord::util::openssl_utils::EdDSA_Signer;
+using concord::util::openssl_utils::EdDSA_Verifier;
+#endif
+
 concord::messages::keys_and_signatures::ClientsPublicKeys clientsPublicKeys_;
 
 std::string SigManager::getClientsPublicKeys() {
@@ -140,17 +146,25 @@ SigManager::SigManager(PrincipalId myId,
   size_t numPublickeys = publickeys.size();
 
   ConcordAssert(publicKeysMapping.size() >= numPublickeys);
-  if (!mySigPrivateKey.first.empty())
-    mySigner_.reset(new concord::util::crypto::RSASigner(mySigPrivateKey.first.c_str(), mySigPrivateKey.second));
+  if (!mySigPrivateKey.first.empty()) {
+#if RSA_Algo
+    mySigner_.reset(new RSASigner(mySigPrivateKey.first.c_str(), mySigPrivateKey.second));
+#else
+    mySigner_.reset(new EdDSA_Signer(mySigPrivateKey.first, mySigPrivateKey.second));
+#endif
+  }
   for (const auto& p : publicKeysMapping) {
     ConcordAssert(verifiers_.count(p.first) == 0);
     ConcordAssert(p.second < numPublickeys);
 
     auto iter = publicKeyIndexToVerifier.find(p.second);
     const auto& [key, format] = publickeys[p.second];
     if (iter == publicKeyIndexToVerifier.end()) {
+#if RSA_Algo
+      verifiers_[p.first] = std::make_shared<RSAVerifier>(key.c_str(), format);
+#else
       verifiers_[p.first] = std::make_shared<EdDSA_Verifier>(key, format);
-      // verifiers_[p.first] = std::make_shared<RSAVerifier>(key.c_str(), format);
+#endif
       publicKeyIndexToVerifier[p.second] = verifiers_[p.first];
     } else {
       verifiers_[p.first] = iter->second;
@@ -246,8 +260,7 @@ bool SigManager::verifySig(
 
 void SigManager::sign(const char* data, size_t dataLength, char* outSig, uint16_t outSigLength) const {
   std::string str_data(data, dataLength);
-  std::string sig;
-  sig = mySigner_->sign(str_data);
+  std::string sig = mySigner_->sign(str_data);
   outSigLength = sig.size();
   std::memcpy(outSig, sig.c_str(), outSigLength);
 }
@@ -259,8 +272,11 @@ void SigManager::setClientPublicKey(const std::string& key, PrincipalId id, conc
   if (replicasInfo_.isIdOfExternalClient(id) || replicasInfo_.isIdOfClientService(id)) {
     try {
       std::unique_lock lock(mutex_);
-      // verifiers_.insert_or_assign(id, std::make_shared<RSAVerifier>(key.c_str(), format));
+#if RSA_Algo
+      verifiers_.insert_or_assign(id, std::make_shared<RSAVerifier>(key.c_str(), format));
+#else
       verifiers_.insert_or_assign(id, std::make_shared<EdDSA_Verifier>(key, format));
+#endif
     } catch (const std::exception& e) {
       LOG_ERROR(KEY_EX_LOG, "failed to add a key for client: " << id << " reason: " << e.what());
       throw;

bftengine/src/preprocessor/tests/preprocessor_test.cpp

@@ -155,19 +155,20 @@ class DummyPreProcessor : public PreProcessor {
 };
 
 // clang-format off
-const vector<string> eddsaPrivateKey = { //use different keys pair.
-  {"09a30490ebf6f6685556046f2497fd9c7df4a552998c9a9b6ebec742e8183174"},
-  {"09a30490ebf6f6685556046f2497fd9c7df4a552998c9a9b6ebec742e8183174"},
-  {"09a30490ebf6f6685556046f2497fd9c7df4a552998c9a9b6ebec742e8183174"},
-  {"09a30490ebf6f6685556046f2497fd9c7df4a552998c9a9b6ebec742e8183174"},
-  {"09a30490ebf6f6685556046f2497fd9c7df4a552998c9a9b6ebec742e8183174"}
+const vector<string> eddsaPrivateKey = {
+  {"61498efe1764b89357a02e2887d224154006ceacf26269f8695a4af561453eef"},
+  {"247a74ab3620ec6b9f5feab9ee1f86521da3fa2804ad45bb5bf2c5b21ef105bc"},
+  {"fb539bc3d66deda55524d903da26dbec1f4b6abf41ec5db521e617c64eb2c341"},
+  {"55ea66e855b83ec4a02bd8fcce6bb4426ad3db2a842fa2a2a6777f13e40a4717"},
+  {"f2f3d43da68329bfe31419636072e27cfd1a8fff259be4bfada667080eb00556"}
 };
+
 const vector<string> eddsaPublicKey = {
-  {"7363bc5ab96d7f85e71a5ffe0b284405ae38e2e0f032fb3ffe805d9f0e2d117b"},
-  {"7363bc5ab96d7f85e71a5ffe0b284405ae38e2e0f032fb3ffe805d9f0e2d117b"},
-  {"7363bc5ab96d7f85e71a5ffe0b284405ae38e2e0f032fb3ffe805d9f0e2d117b"},
-  {"7363bc5ab96d7f85e71a5ffe0b284405ae38e2e0f032fb3ffe805d9f0e2d117b"},
-  {"7363bc5ab96d7f85e71a5ffe0b284405ae38e2e0f032fb3ffe805d9f0e2d117b"}
+  {"386f4fb049a5d8bb0706d3793096c8f91842ce380dfc342a2001d50dfbc901f4"},
+  {"3f9e7dbde90477c24c1bacf14e073a356c1eca482d352d9cc0b16560a4e7e469"},
+  {"2311c6013ff657844669d8b803b2e1ed33fe06eed445f966a800a8fbb8d790e8"},
+  {"1ba7449655784fc9ce193a7887de1e4d3d35f7c82b802440c4f28bf678a34b34"},
+  {"c426c524c92ad9d0b740f68ee312abf0298051a7e0364a867b940e9693ae6095"}
 };
 
 const vector<string> privateKeys = {

bftengine/tests/SigManager/SigManager_test.cpp

@@ -84,7 +84,7 @@ void corrupt(char* data, size_t len) {
   }
 }
 
-TEST(RsaSignerAndRsaVerifierTest, LoadSignVerifyFromPemfiles) {
+TEST(SignerAndVerifierTest, LoadSignVerifyFromPemfiles) {
   string publicKeyFullPath({string(KEYS_BASE_PATH) + string("/1/") + PUB_KEY_NAME});
   string privateKeyFullPath({string(KEYS_BASE_PATH) + string("/1/") + PRIV_KEY_NAME});
 
@@ -103,7 +103,7 @@ TEST(RsaSignerAndRsaVerifierTest, LoadSignVerifyFromPemfiles) {
   auto signer_ = unique_ptr<EdDSA_Signer>(new EdDSA_Signer(privKey, KeyFormat::PemFormat));
 #endif
 
-  // sign with RSASigner
+  // sign with RSASigner/EdDSA_Signer
   size_t expectedSignerSigLen = signer_->signatureLength();
   sig.reserve(expectedSignerSigLen);
   size_t lenRetData;
@@ -112,7 +112,7 @@ TEST(RsaSignerAndRsaVerifierTest, LoadSignVerifyFromPemfiles) {
   lenRetData = sig.size();
   ASSERT_EQ(lenRetData, expectedSignerSigLen);
 
-  // validate with RSAVerifier
+  // validate with RSAVerifier/EdDSA_Verifier
   ASSERT_TRUE(verifier_->verify(str_data, sig));
 
   // change data randomally, expect failure
@@ -182,7 +182,7 @@ TEST(SigManagerTest, ReplicasOnlyCheckVerify) {
 
     if (i == myId) continue;
 
-    // sign with RSASigner (other replicas, mock)
+    // sign with RSASigner/EdDSA_Signer (other replicas, mock)
     expectedSignerSigLen = signer->signatureLength();
     sig.reserve(expectedSignerSigLen);
     generateRandomData(data, RANDOM_DATA_SIZE);

bftengine/tests/bcstatetransfer/bcstatetransfer_tests.cpp

@@ -43,6 +43,7 @@
 #include "hex_tools.h"  //leave for debug
 #include "RVBManager.hpp"
 #include "RangeValidationTree.hpp"
+#include "digest.hpp"
 
 #ifdef USE_ROCKSDB
 #include "rocksdb/client.h"

bftengine/tests/clientsManager/ClientsManager_test.cpp

@@ -25,11 +25,7 @@ using bftEngine::impl::SigManager;
 using bftEngine::ReplicaConfig;
 using bftEngine::ReservedPagesClientBase;
 using bftEngine::test::ReservedPagesMock;
-// using concord::util::cryptopp_utils::Crypto;
-using concord::util::openssl_utils::Crypto;
 using concord::util::crypto::KeyFormat;
-// using concord::util::cryptopp_utils::RSASigner;
-using concord::util::openssl_utils::EdDSA_Signer;
 using concord::secretsmanager::ISecretsManagerImpl;
 using concordUtil::Timers;
 using std::chrono::milliseconds;
@@ -46,6 +42,16 @@ using std::this_thread::sleep_for;
 using std::unique_ptr;
 using std::vector;
 
+#define RSA_Algo false
+
+#if RSA_Algo
+using concord::util::cryptopp_utils::RSASigner;
+using concord::util::cryptopp_utils::Crypto;
+#else
+using concord::util::openssl_utils::EdDSA_Signer;
+using concord::util::openssl_utils::Crypto;
+#endif
+
 // Testing values to be used for certain Concord-BFT configuration that ClientsManager and/or its dependencies may
 // reference.
 const ReplicaId kReplicaIdForTesting = 0;
@@ -228,8 +234,12 @@ static bool verifyClientPublicKeyLoadedToKEM(NodeIdType client_id, const pair<st
   if (!(SigManager::instance()->hasVerifier(client_id))) {
     return false;
   }
-  // RSASigner signer(expected_key.first, kKeyFormatForTesting);
+
+#if RSA_Algo
+  RSASigner signer(expected_key.first, kKeyFormatForTesting);
+#else
   EdDSA_Signer signer(expected_key.first, kKeyFormatForTesting);
+#endif
   string signature = signer.sign(kArbitraryMessageForTestingKeyAgreement);
   return SigManager::instance()->verifySig(client_id,
                                            kArbitraryMessageForTestingKeyAgreement.data(),

client/bftclient/src/bft_client.cpp

@@ -23,8 +23,14 @@ using namespace concord::secretsmanager;
 using namespace bftEngine;
 using namespace bftEngine::impl;
 using concord::util::crypto::KeyFormat;
+
+#define RSA_Algo false
+
+#if RSA_Algo
+using concord::util::cryptopp_utils::RSASigner;
+#else
 using concord::util::openssl_utils::EdDSA_Signer;
-// using concord::util::cryptopp_utils::RSASigner;
+#endif
 
 namespace bft::client {
 

client/bftclient/test/bft_client_api_tests.cpp

@@ -182,7 +182,7 @@ TEST_P(ClientApiTestParametrizedFixture, print_received_messages_and_timeout) {
       test_config_.secrets_manager_config = sd;
     }
 
-    // initialize the test's RSAVerifier
+    // initialize the test's RSAVerifier/EdDSA_Verifier
     string public_key_full_path({keypair_path + PUB_KEY_NAME});
     std::ifstream file(public_key_full_path);
     std::stringstream stream;

client/reconfiguration/src/default_handlers.cpp

@@ -12,8 +12,9 @@
 #include "client/reconfiguration/default_handlers.hpp"
 #include "bftclient/StateControl.hpp"
 #include "concord.cmf.hpp"
-#include "crypto_utils.hpp"
-#include "kvstream.h"
+#include "cryptopp_utils.hpp"
+#include "openssl_utils.hpp"
+#include "ReplicaConfig.hpp"
 
 #include <variant>
 #include <experimental/filesystem>
@@ -70,8 +71,8 @@ void ClientTlsKeyExchangeHandler::exchangeTlsKeys(const std::string& pkey_path,
   std::string master_key = sm_->decryptFile(master_key_path_).value_or(std::string());
   if (master_key.empty()) master_key = psm_.decryptFile(master_key_path_).value_or(std::string());
   if (master_key.empty()) LOG_FATAL(getLogger(), "unable to read the node master key");
-  auto cert =
-      concord::util::crypto::CertificateUtils::generateSelfSignedCert(cert_path, new_cert_keys.second, master_key);
+  auto cert = concord::util::openssl_utils::CertificateUtils::generateSelfSignedCert(
+      cert_path, new_cert_keys.second, master_key);
 
   sm_->encryptFile(pkey_path, new_cert_keys.first);
   psm_.encryptFile(cert_path, cert);

communication/src/AsyncTlsConnection.cpp

@@ -430,7 +430,7 @@ std::pair<bool, NodeNum> AsyncTlsConnection::checkCertificate(X509* received_cer
   uint32_t peerId = UINT32_MAX;
   std::string conn_type;
   // (1) First, try to verify the certificate against the latest saved certificate
-  bool res = concord::util::crypto::CertificateUtils::verifyCertificate(
+  bool res = concord::util::openssl_utils::CertificateUtils::verifyCertificate(
       received_cert, config_.certificatesRootPath_, peerId, conn_type, config_.useUnifiedCertificates_);
   if (expected_peer_id.has_value() && peerId != expected_peer_id.value()) return std::make_pair(false, peerId);
   if (res) return std::make_pair(res, peerId);

install_deps.sh

@@ -67,7 +67,7 @@ apt-get ${APT_GET_FLAGS} install \
     libz-dev \
     libzstd-dev
 
-pip3 install --upgrade wheel && pip3 install --upgrade trio
+pip3 install --upgrade wheel && pip3 install --upgrade trio && pip3 install --upgrade pip
 pip3 install \
     eliot eliot-tree \
     tatsu==4.4.0 \
@@ -76,7 +76,8 @@ pip3 install \
     ecdsa \
     protobuf==3.15.8 \
     grpcio==1.37.1 \
-    grpcio-tools==1.37.1
+    grpcio-tools==1.37.1 \
+    cryptography==3.3.2
 
 # Build 3rd parties
 wget ${WGET_FLAGS} -O cmake-linux.sh \