From db9891094895bb98fff4811ba89e5368fb9ef2c8 Mon Sep 17 00:00:00 2001 From: shgandhi Date: Wed, 6 Jul 2022 19:02:53 -0700 Subject: [PATCH] gRPC upgrade version This commit upgrades gRPC from 1.37.x to the latest release version 1.48.0. Over the past ten versions of gRPC multiple optimizations and bug fixes have been introduced. Please see https://github.com/grpc/grpc/releases for the detailed breakdown of updates from 1.38.0 to 1.48.0. This upgrade would also allow for batching of updates natively via setting the buffer hint flag for improved performance. This change updates client, TRS and UTT CMakeLists files to account for the upgrade. --- client/clientservice/CMakeLists.txt | 6 ++ client/proto/CMakeLists.txt | 115 +++++++++++++++++++---- install_deps.sh | 50 ++++------ kvbc/proto/CMakeLists.txt | 27 +++++- thin-replica-server/proto/CMakeLists.txt | 66 +++++++++---- utt/wallet-cli/proto/CMakeLists.txt | 35 +++++-- 6 files changed, 217 insertions(+), 82 deletions(-) diff --git a/client/clientservice/CMakeLists.txt b/client/clientservice/CMakeLists.txt index 01314f3357..7fbce07515 100644 --- a/client/clientservice/CMakeLists.txt +++ b/client/clientservice/CMakeLists.txt @@ -1,4 +1,9 @@ +cmake_minimum_required(VERSION 3.5) +set(CMAKE_CXX_STANDARD 17) + find_package(GRPC REQUIRED) +find_package(absl CONFIG REQUIRED) +find_package(Protobuf CONFIG REQUIRED) find_package(Boost ${MIN_BOOST_VERSION} COMPONENTS program_options REQUIRED) find_package(jaegertracing REQUIRED) @@ -13,6 +18,7 @@ target_link_libraries(clientservice-lib PUBLIC concordclient gRPC::grpc++ gRPC::grpc++_reflection + protobuf::libprotobuf logging yaml-cpp secret_retriever diff --git a/client/proto/CMakeLists.txt b/client/proto/CMakeLists.txt index 87cdb6a424..a8c5f4e81c 100644 --- a/client/proto/CMakeLists.txt +++ b/client/proto/CMakeLists.txt @@ -1,21 +1,102 @@ +cmake_minimum_required(VERSION 3.5) +set(CMAKE_CXX_STANDARD 17) + +project(concord) + find_package(Protobuf REQUIRED) +find_package(absl CONFIG REQUIRED) find_package(GRPC REQUIRED) -include_directories(${GRPC_INCLUDE_DIR}) - -protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS ${CMAKE_CURRENT_BINARY_DIR} - request/v1/request.proto - event/v1/event.proto - state_snapshot/v1/state_snapshot.proto - ../concordclient/proto/concord_client_request/v1/concord_client_request.proto -) -grpc_generate_cpp(GRPC_SRCS GRPC_HDRS ${CMAKE_CURRENT_BINARY_DIR} - request/v1/request.proto - event/v1/event.proto - state_snapshot/v1/state_snapshot.proto - ../concordclient/proto/concord_client_request/v1/concord_client_request.proto -) - -add_library(clientservice-proto STATIC ${PROTO_SRCS} ${GRPC_SRCS}) -target_link_libraries(clientservice-proto PRIVATE protobuf::libprotobuf gRPC::grpc++) +# Proto file +get_filename_component(rs_proto "request/v1/request.proto" ABSOLUTE) +get_filename_component(es_proto "event/v1/event.proto" ABSOLUTE) +get_filename_component(ss_proto "state_snapshot/v1/state_snapshot.proto" ABSOLUTE) +get_filename_component(ccr_proto "../concordclient/proto/concord_client_request/v1/concord_client_request.proto" ABSOLUTE) +get_filename_component(rs_proto_path "${rs_proto}" PATH) +get_filename_component(es_proto_path "${es_proto}" PATH) +get_filename_component(ss_proto_path "${ss_proto}" PATH) +get_filename_component(ccr_proto_path "${ccr_proto}" PATH) + +get_target_property(grpc_cpp_plugin_location gRPC::grpc_cpp_plugin LOCATION) + +# Generated sources +set(rs_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/request.pb.cc") +set(rs_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/request.pb.h") +set(rs_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/request.grpc.pb.cc") +set(rs_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/request.grpc.pb.h") +add_custom_command( + OUTPUT "${rs_proto_srcs}" "${rs_proto_hdrs}" "${rs_grpc_srcs}" "${rs_grpc_hdrs}" + COMMAND protoc + ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}" + --cpp_out "${CMAKE_CURRENT_BINARY_DIR}" + -I "${rs_proto_path}" + --plugin=protoc-gen-grpc="${grpc_cpp_plugin_location}" + "${rs_proto}" + DEPENDS "${rs_proto}") + +set(es_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/event.pb.cc") +set(es_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/event.pb.h") +set(es_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/event.grpc.pb.cc") +set(es_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/event.grpc.pb.h") +add_custom_command( + OUTPUT "${es_proto_srcs}" "${es_proto_hdrs}" "${es_grpc_srcs}" "${es_grpc_hdrs}" + COMMAND protoc + ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}" + --cpp_out "${CMAKE_CURRENT_BINARY_DIR}" + -I "${es_proto_path}" + --plugin=protoc-gen-grpc="${grpc_cpp_plugin_location}" + "${es_proto}" + DEPENDS "${es_proto}") + +set(ss_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/state_snapshot.pb.cc") +set(ss_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/state_snapshot.pb.h") +set(ss_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/state_snapshot.grpc.pb.cc") +set(ss_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/state_snapshot.grpc.pb.h") +add_custom_command( + OUTPUT "${ss_proto_srcs}" "${ss_proto_hdrs}" "${ss_grpc_srcs}" "${ss_grpc_hdrs}" + COMMAND protoc + ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}" + --cpp_out "${CMAKE_CURRENT_BINARY_DIR}" + -I "${ss_proto_path}" + --plugin=protoc-gen-grpc="${grpc_cpp_plugin_location}" + "${ss_proto}" + DEPENDS "${ss_proto}") + +set(ccr_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/concord_client_request.pb.cc") +set(ccr_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/concord_client_request.pb.h") +set(ccr_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/concord_client_request.grpc.pb.cc") +set(ccr_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/concord_client_request.grpc.pb.h") +add_custom_command( + OUTPUT "${ccr_proto_srcs}" "${ccr_proto_hdrs}" "${ccr_grpc_srcs}" "${ccr_grpc_hdrs}" + COMMAND protoc + ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}" + --cpp_out "${CMAKE_CURRENT_BINARY_DIR}" + -I "${ccr_proto_path}" + --plugin=protoc-gen-grpc="${grpc_cpp_plugin_location}" + "${ccr_proto}" + DEPENDS "${ccr_proto}") + + +# Include generated *.pb.h files +include_directories("${CMAKE_CURRENT_BINARY_DIR}") + +# clientservice-proto +add_library(clientservice-proto STATIC + ${rs_grpc_srcs} + ${rs_grpc_hdrs} + ${rs_proto_srcs} + ${rs_proto_hdrs} + ${es_grpc_srcs} + ${es_grpc_hdrs} + ${es_proto_srcs} + ${es_proto_hdrs} + ${ss_grpc_srcs} + ${ss_grpc_hdrs} + ${ss_proto_srcs} + ${ss_proto_hdrs} + ${ccr_grpc_srcs} + ${ccr_grpc_hdrs} + ${ccr_proto_srcs} + ${ccr_proto_hdrs}) +target_link_libraries(clientservice-proto PRIVATE protobuf::libprotobuf gRPC::grpc++ absl::synchronization) target_include_directories(clientservice-proto PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/install_deps.sh b/install_deps.sh index 1219a99dbd..fc722bb5e4 100755 --- a/install_deps.sh +++ b/install_deps.sh @@ -76,9 +76,9 @@ install_third_party_libraries() { pytest \ pycryptodome \ ecdsa \ - protobuf==3.15.8 \ - grpcio==1.37.1 \ - grpcio-tools==1.37.1 + protobuf==3.21.5 \ + grpcio==1.48.0 \ + grpcio-tools==1.48.0 } @@ -300,36 +300,20 @@ install_openssl() { # https://github.com/grpc/grpc/blob/master/test/distrib/cpp/run_distrib_test_cmake.sh install_grpc() { cd ${HOME} - git clone -b v1.37.1 --depth 1 --recurse-submodules https://github.com/grpc/grpc && \ - cd grpc && \ - mkdir -p ${HOME}/grpc/third_party/abseil-cpp/cmake/build && \ - cd ${HOME}/grpc/third_party/abseil-cpp/cmake/build && \ - cmake -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE \ - -DCMAKE_INSTALL_PREFIX=/usr/local \ - ../.. && \ - make -j$(nproc) install && \ - mkdir -p ${HOME}/grpc/third_party/protobuf/cmake/build && \ - cd ${HOME}/grpc/third_party/protobuf/cmake/build && \ - cmake -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/usr/local \ - .. && \ - make -j$(nproc) install && \ - mkdir -p ${HOME}/grpc/cmake/build && \ - cd ${HOME}/grpc/cmake/build && \ - cmake -DgRPC_INSTALL=ON \ - -DgRPC_ABSL_PROVIDER=package \ - -DgRPC_PROTOBUF_PROVIDER=package \ - -DgRPC_SSL_PROVIDER=package \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/usr/local \ - ../.. && \ - make -j$(nproc) install && - cd ${HOME} && \ - rm -r ${HOME}/grpc - + git clone -b v1.48.x --depth 1 --recurse-submodules https://github.com/grpc/grpc && \ + mkdir -p ${HOME}/grpc/cmake/build && \ + cd ${HOME}/grpc/cmake/build && \ + cmake -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ + -DgRPC_INSTALL=ON \ + -DgRPC_BUILD_TESTS=OFF \ + -DgRPC_SSL_PROVIDER=package \ + -DCMAKE_INSTALL_PREFIX=/opt/grpc \ + ../.. && \ + make -j$(nproc) install && \ + cd ${HOME} && \ + rm -r ${HOME}/grpc } install_prometheus() { diff --git a/kvbc/proto/CMakeLists.txt b/kvbc/proto/CMakeLists.txt index f221baec4d..13843a34a9 100644 --- a/kvbc/proto/CMakeLists.txt +++ b/kvbc/proto/CMakeLists.txt @@ -1,10 +1,27 @@ -find_package(Protobuf REQUIRED) +cmake_minimum_required(VERSION 3.5) +set(CMAKE_CXX_STANDARD 17) + +find_package(absl CONFIG REQUIRED) +find_package(GRPC REQUIRED) +find_package(Protobuf CONFIG REQUIRED) + +# Proto file +get_filename_component(concord_kvbc_proto "concord_kvbc.proto" ABSOLUTE) +get_filename_component(concord_kvbc_proto_path "${concord_kvbc_proto}" PATH) + +set(ck_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/concord_kvbc.pb.cc") +set(ck_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/concord_kvbc.pb.h") + +add_custom_command( + OUTPUT "${ck_proto_srcs}" "${ck_proto_hdrs}" + COMMAND protoc + ARGS --proto_path="${concord_kvbc_proto_path}" + --cpp_out "${CMAKE_CURRENT_BINARY_DIR}" + "${concord_kvbc_proto}" + DEPENDS "${concord_kvbc_proto}") -protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS ${CMAKE_CURRENT_BINARY_DIR} - concord_kvbc.proto -) message(STATUS "Concord KVBC protobuf generated - see " ${CMAKE_CURRENT_BINARY_DIR}) -add_library(concord-kvbc-proto STATIC ${PROTO_SRCS}) +add_library(concord-kvbc-proto STATIC ${ck_proto_srcs}) target_link_libraries(concord-kvbc-proto protobuf::libprotobuf) target_include_directories(concord-kvbc-proto PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/thin-replica-server/proto/CMakeLists.txt b/thin-replica-server/proto/CMakeLists.txt index c30488bcb3..f56ce0a663 100644 --- a/thin-replica-server/proto/CMakeLists.txt +++ b/thin-replica-server/proto/CMakeLists.txt @@ -1,30 +1,58 @@ -find_package(Protobuf REQUIRED) +cmake_minimum_required(VERSION 3.5) +set(CMAKE_CXX_STANDARD 17) + find_package(GRPC REQUIRED) +find_package(absl CONFIG REQUIRED) +find_package(Protobuf CONFIG REQUIRED) include_directories(${GRPC_INCLUDE_DIR}) -protobuf_generate_cpp(THIN_REPLICA_PROTO_SRCS THIN_REPLICA_PROTO_HDRS ${CMAKE_CURRENT_BINARY_DIR} - thin_replica.proto -) -grpc_generate_cpp(THIN_REPLICA_GRPC_SRCS THIN_REPLICA_GRPC_HDRS ${CMAKE_CURRENT_BINARY_DIR} - thin_replica.proto -) +get_filename_component(tr_proto "thin_replica.proto" ABSOLUTE) +get_filename_component(tr_proto_path "${tr_proto}" PATH) + +get_target_property(grpc_cpp_plugin_location gRPC::grpc_cpp_plugin LOCATION) + +set(tr_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/thin_replica.pb.cc") +set(tr_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/thin_replica.pb.h") +set(tr_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/thin_replica.grpc.pb.cc") +set(tr_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/thin_replica.grpc.pb.h") +add_custom_command( + OUTPUT "${tr_proto_srcs}" "${tr_proto_hdrs}" "${tr_grpc_srcs}" "${tr_grpc_hdrs}" + COMMAND protoc + ARGS --grpc_out=generate_mock_code=true:"${CMAKE_CURRENT_BINARY_DIR}" + --cpp_out "${CMAKE_CURRENT_BINARY_DIR}" + -I "${tr_proto_path}" + --plugin=protoc-gen-grpc="${grpc_cpp_plugin_location}" + "${tr_proto}" + DEPENDS "${tr_proto}") + message(STATUS "Thin replica gRPC/protobuf generated - see " ${CMAKE_CURRENT_BINARY_DIR}) -protobuf_generate_cpp(REPLICA_STATE_SNAPSHOT_PROTO_SRCS REPLICA_STATE_SNAPSHOT_PROTO_HDRS ${CMAKE_CURRENT_BINARY_DIR} - replica_state_snapshot.proto -) -grpc_generate_cpp(REPLICA_STATE_SNAPSHOT_GRPC_SRCS REPLICA_STATE_SNAPSHOT_GRPC_HDRS ${CMAKE_CURRENT_BINARY_DIR} - replica_state_snapshot.proto -) +get_filename_component(ss_proto "replica_state_snapshot.proto" ABSOLUTE) +get_filename_component(ss_proto_path "${ss_proto}" PATH) + +set(ss_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/replica_state_snapshot.pb.cc") +set(ss_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/replica_state_snapshot.pb.h") +set(ss_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/replica_state_snapshot.grpc.pb.cc") +set(ss_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/replica_state_snapshot.grpc.pb.h") +add_custom_command( + OUTPUT "${ss_proto_srcs}" "${ss_proto_hdrs}" "${ss_grpc_srcs}" "${ss_grpc_hdrs}" + COMMAND protoc + ARGS --grpc_out=generate_mock_code=true:"${CMAKE_CURRENT_BINARY_DIR}" + --cpp_out "${CMAKE_CURRENT_BINARY_DIR}" + -I "${ss_proto_path}" + --plugin=protoc-gen-grpc="${grpc_cpp_plugin_location}" + "${ss_proto}" + DEPENDS "${ss_proto}") + message(STATUS "State snapshot gRPC/protobuf generated - see " ${CMAKE_CURRENT_BINARY_DIR}) -add_library(thin-replica-proto STATIC ${THIN_REPLICA_PROTO_SRCS} ${THIN_REPLICA_PROTO_HDRS} - ${THIN_REPLICA_GRPC_SRCS} ${THIN_REPLICA_GRPC_HDRS}) -target_link_libraries(thin-replica-proto protobuf::libprotobuf gRPC::grpc++) +add_library(thin-replica-proto STATIC ${tr_proto_srcs} ${tr_proto_hdrs} + ${tr_grpc_srcs} ${tr_grpc_hdrs}) +target_link_libraries(thin-replica-proto protobuf::libprotobuf gRPC::grpc++ gRPC::grpc++_reflection) target_include_directories(thin-replica-proto PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) -add_library(replica-state-snapshot-proto STATIC ${REPLICA_STATE_SNAPSHOT_PROTO_SRCS} ${REPLICA_STATE_SNAPSHOT_PROTO_HDRS} - ${REPLICA_STATE_SNAPSHOT_GRPC_SRCS} ${REPLICA_STATE_SNAPSHOT_GRPC_HDRS}) -target_link_libraries(replica-state-snapshot-proto protobuf::libprotobuf gRPC::grpc++) +add_library(replica-state-snapshot-proto STATIC ${ss_proto_srcs} ${ss_proto_hdrs} + ${ss_grpc_srcs} ${ss_grpc_hdrs}) +target_link_libraries(replica-state-snapshot-proto protobuf::libprotobuf gRPC::grpc++ gRPC::grpc++_reflection) target_include_directories(replica-state-snapshot-proto PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/utt/wallet-cli/proto/CMakeLists.txt b/utt/wallet-cli/proto/CMakeLists.txt index fc9eeaa69f..c33c080e70 100644 --- a/utt/wallet-cli/proto/CMakeLists.txt +++ b/utt/wallet-cli/proto/CMakeLists.txt @@ -1,15 +1,34 @@ +cmake_minimum_required(VERSION 3.5) +set(CMAKE_CXX_STANDARD 17) + +project(concord) + find_package(Protobuf REQUIRED) find_package(GRPC REQUIRED) include_directories(${GRPC_INCLUDE_DIR}) -protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS ${CMAKE_CURRENT_BINARY_DIR} - api/v1/api.proto -) -grpc_generate_cpp(GRPC_SRCS GRPC_HDRS ${CMAKE_CURRENT_BINARY_DIR} - api/v1/api.proto -) +# Proto file +get_filename_component(utt_wallet_api_proto "api/v1/api.proto" ABSOLUTE) +get_filename_component(utt_wallet_api_proto_path "${utt_wallet_api_proto}" PATH) + +get_target_property(grpc_cpp_plugin_location gRPC::grpc_cpp_plugin LOCATION) + +set(utt_wallet_api_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/api.pb.cc") +set(utt_wallet_api_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/api.pb.h") +set(utt_wallet_api_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/api.grpc.pb.cc") +set(utt_wallet_api_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/api.grpc.pb.h") + +add_custom_command( + OUTPUT "${utt_wallet_api_proto_srcs}" "${utt_wallet_api_proto_hdrs}" "${utt_wallet_api_grpc_srcs}" "${utt_wallet_api_grpc_hdrs}" + COMMAND protoc + ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}" + --cpp_out "${CMAKE_CURRENT_BINARY_DIR}" + -I "${utt_wallet_api_proto_path}" + --plugin=protoc-gen-grpc="${grpc_cpp_plugin_location}" + "${utt_wallet_api_proto}" + DEPENDS "${utt_wallet_api_proto}") -add_library(utt-wallet-api-proto STATIC ${PROTO_SRCS} ${GRPC_SRCS}) +add_library(utt-wallet-api-proto STATIC ${utt_wallet_api_proto_srcs} ${utt_wallet_api_grpc_srcs}) target_link_libraries(utt-wallet-api-proto PRIVATE protobuf::libprotobuf gRPC::grpc++) -target_include_directories(utt-wallet-api-proto PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) \ No newline at end of file +target_include_directories(utt-wallet-api-proto PUBLIC ${CMAKE_CURRENT_BINARY_DIR})