diff --git a/.gitignore b/.gitignore index caf123ee..a9113a82 100644 --- a/.gitignore +++ b/.gitignore @@ -11,8 +11,10 @@ /.yardoc /_yardoc/ /cmake-build-*/ +/build-*/ /coverage/ /doc/ +/ext/cache/ /pkg/ /test/reports/ /tmp/ diff --git a/Rakefile b/Rakefile index a9d9e5da..168c3f2a 100644 --- a/Rakefile +++ b/Rakefile @@ -26,7 +26,7 @@ desc "Compile binary extension" task :compile do require 'tempfile' Dir.chdir(Dir.tmpdir) do - sh "ruby #{File.join(__dir__, 'ext', 'extconf.rb')}" + sh "ruby '#{File.join(__dir__, 'ext', 'extconf.rb')}'" end end @@ -56,15 +56,206 @@ task :render_git_revision do `git fetch --tags >/dev/null 2>&1` `git describe --long --always HEAD`.strip end - File.write(File.join(__dir__, 'ext', 'revisions.rb'), <<~REVISIONS) - cmake_flags << "-DEXT_GIT_REVISION=#{library_revision}" - cmake_flags << "-DCOUCHBASE_CXX_CLIENT_GIT_REVISION=#{core_revision}" - cmake_flags << "-DCOUCHBASE_CXX_CLIENT_GIT_DESCRIBE=#{core_describe}" - REVISIONS + File.open(File.join(__dir__, "ext", "cache", "extconf_include.rb"), "a+") do |io| + io.puts(<<~REVISIONS) + cmake_flags << "-DEXT_GIT_REVISION=#{library_revision}" + cmake_flags << "-DCOUCHBASE_CXX_CLIENT_GIT_REVISION=#{core_revision}" + cmake_flags << "-DCOUCHBASE_CXX_CLIENT_GIT_DESCRIBE=#{core_describe}" + REVISIONS + end +end + +def which(name, extra_locations = []) + ENV.fetch("PATH", "") + .split(File::PATH_SEPARATOR) + .prepend(*extra_locations) + .select { |path| File.directory?(path) } + .map { |path| [path, name].join(File::SEPARATOR) + RbConfig::CONFIG["EXEEXT"] } + .find { |file| File.executable?(file) } +end + +desc "Download and cache dependencies of C++ core" +task :cache_cxx_dependencies do + require "tempfile" + require "rubygems/package" + + output_dir = Dir.mktmpdir("cxx_output_") + output_tarball = File.join(output_dir, "cache.tar") + cpm_cache_dir = Dir.mktmpdir("cxx_cache_") + cxx_core_build_dir = Dir.mktmpdir("cxx_build_") + cxx_core_source_dir = File.join(__dir__, "ext", "couchbase") + cc = ENV.fetch("CB_CC", nil) + cxx = ENV.fetch("CB_CXX", nil) + ar = ENV.fetch("CB_AR", nil) + + cmake_extra_locations = [] + if RUBY_PLATFORM.match?(/mswin|mingw/) + cmake_extra_locations = [ + 'C:\Program Files\CMake\bin', + 'C:\Program Files\Microsoft Visual Studio\2022\Professional\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin', + 'C:\Program Files\Microsoft Visual Studio\2019\Professional\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin', + ] + local_app_data = ENV.fetch("LOCALAPPDATA", "#{Dir.home}\\AppData\\Local") + cmake_extra_locations.unshift("#{local_app_data}\\CMake\\bin") if File.directory?(local_app_data) + cc = RbConfig::CONFIG["CC"] + cxx = RbConfig::CONFIG["CXX"] + end + cmake = which("cmake", cmake_extra_locations) || which("cmake3", cmake_extra_locations) + cmake_flags = [ + "-S#{cxx_core_source_dir}", + "-B#{cxx_core_build_dir}", + "-DCOUCHBASE_CXX_CLIENT_BUILD_TESTS=OFF", + "-DCOUCHBASE_CXX_CLIENT_BUILD_TOOLS=OFF", + "-DCOUCHBASE_CXX_CLIENT_BUILD_DOCS=OFF", + "-DCOUCHBASE_CXX_CLIENT_STATIC_BORINGSSL=ON", + "-DCPM_DOWNLOAD_ALL=ON", + "-DCPM_USE_NAMED_CACHE_DIRECTORIES=ON", + "-DCPM_USE_LOCAL_PACKAGES=OFF", + "-DCPM_SOURCE_CACHE=#{cpm_cache_dir}", + ] + cmake_flags << "-DCMAKE_C_COMPILER=#{cc}" if cc + cmake_flags << "-DCMAKE_CXX_COMPILER=#{cxx}" if cxx + cmake_flags << "-DCMAKE_AR=#{ar}" if ar + + puts("-----> run cmake to dowload all depenencies (#{cmake})") + sh(cmake, *cmake_flags, verbose: true) + + puts("-----> create archive with whitelisted sources: #{output_tarball}") + File.open(output_tarball, "w+b") do |file| + Gem::Package::TarWriter.new(file) do |writer| + chdir(cxx_core_build_dir, verbose: true) do + ["mozilla-ca-bundle.sha256", "mozilla-ca-bundle.crt"].each do |path| + writer.add_file(path, 0o660) { |io| io.write(File.binread(path)) } + end + end + chdir(cpm_cache_dir, verbose: true) do + third_party_sources = Dir[ + "cpm/*.cmake", + "asio/*/LICENSE*", + "asio/*/asio/COPYING", + "asio/*/asio/asio/include/*.hpp", + "asio/*/asio/asio/include/asio/**/*.[hi]pp", + "boringssl/*/boringssl/**/*.{cc,h,c,asm,S}", + "boringssl/*/boringssl/**/CMakeLists.txt", + "boringssl/*/boringssl/LICENSE", + "fmt/*/fmt/CMakeLists.txt", + "fmt/*/fmt/ChangeLog.rst", + "fmt/*/fmt/LICENSE.rst", + "fmt/*/fmt/README.rst", + "fmt/*/fmt/include/**/*", + "fmt/*/fmt/src/**/*", + "fmt/*/fmt/support/cmake/**/*", + "gsl/*/gsl/CMakeLists.txt", + "gsl/*/gsl/GSL.natvis", + "gsl/*/gsl/LICENSE*", + "gsl/*/gsl/ThirdPartyNotices.txt", + "gsl/*/gsl/cmake/*", + "gsl/*/gsl/include/**/*", + "hdr_histogram/*/hdr_histogram/*.pc.in", + "hdr_histogram/*/hdr_histogram/CMakeLists.txt", + "hdr_histogram/*/hdr_histogram/COPYING.txt", + "hdr_histogram/*/hdr_histogram/LICENSE.txt", + "hdr_histogram/*/hdr_histogram/cmake/*", + "hdr_histogram/*/hdr_histogram/config.cmake.in", + "hdr_histogram/*/hdr_histogram/include/**/*", + "hdr_histogram/*/hdr_histogram/src/**/*", + "json/*/json/CMakeLists.txt", + "json/*/json/LICENSE*", + "json/*/json/external/PEGTL/.cmake/*", + "json/*/json/external/PEGTL/CMakeLists.txt", + "json/*/json/external/PEGTL/LICENSE*", + "json/*/json/external/PEGTL/include/**/*", + "json/*/json/include/**/*", + "llhttp/*/llhttp/*.pc.in", + "llhttp/*/llhttp/CMakeLists.txt", + "llhttp/*/llhttp/LICENSE*", + "llhttp/*/llhttp/include/*.h", + "llhttp/*/llhttp/src/*.c", + "snappy/*/snappy/CMakeLists.txt", + "snappy/*/snappy/COPYING", + "snappy/*/snappy/cmake/*", + "snappy/*/snappy/snappy-c.{h,cc}", + "snappy/*/snappy/snappy-internal.h", + "snappy/*/snappy/snappy-sinksource.{h,cc}", + "snappy/*/snappy/snappy-stubs-internal.{h,cc}", + "snappy/*/snappy/snappy-stubs-public.h.in", + "snappy/*/snappy/snappy.{h,cc}", + "spdlog/*/spdlog/CMakeLists.txt", + "spdlog/*/spdlog/LICENSE", + "spdlog/*/spdlog/cmake/*", + "spdlog/*/spdlog/include/**/*", + "spdlog/*/spdlog/src/**/*", + ].grep_v(/crypto_test_data.cc/) + + # we don't want to fail if git is not available + cpm_cmake_path = third_party_sources.grep(/cpm.*\.cmake$/).first + File.write(cpm_cmake_path, File.read(cpm_cmake_path).gsub("Git REQUIRED", "Git")) + + third_party_sources + .select { |path| File.file?(path) } + .each { |path| writer.add_file(path, 0o660) { |io| io.write(File.binread(path)) } } + end + end + end + + rm_rf(cxx_core_build_dir, verbose: true) + rm_rf(cpm_cache_dir, verbose: true) + + untar = ["tar", "-x"] + untar << "--force-local" unless RUBY_PLATFORM.match?(/darwin/) + + puts("-----> verify that tarball works as a cache for CPM") + cxx_core_build_dir = Dir.mktmpdir("cxx_build_") + cpm_cache_dir = Dir.mktmpdir("cxx_cache_") + chdir(cpm_cache_dir, verbose: true) do + sh(*untar, "-f", output_tarball, verbose: true) + end + + cmake_flags = [ + "-S#{cxx_core_source_dir}", + "-B#{cxx_core_build_dir}", + "-DCOUCHBASE_CXX_CLIENT_BUILD_TESTS=OFF", + "-DCOUCHBASE_CXX_CLIENT_BUILD_TOOLS=OFF", + "-DCOUCHBASE_CXX_CLIENT_BUILD_DOCS=OFF", + "-DCOUCHBASE_CXX_CLIENT_STATIC_BORINGSSL=ON", + "-DCPM_DOWNLOAD_ALL=OFF", + "-DCPM_USE_NAMED_CACHE_DIRECTORIES=ON", + "-DCPM_USE_LOCAL_PACKAGES=OFF", + "-DCPM_SOURCE_CACHE=#{cpm_cache_dir}", + "-DCOUCHBASE_CXX_CLIENT_EMBED_MOZILLA_CA_BUNDLE_ROOT=#{cpm_cache_dir}", + ] + cmake_flags << "-DCMAKE_C_COMPILER=#{cc}" if cc + cmake_flags << "-DCMAKE_CXX_COMPILER=#{cxx}" if cxx + cmake_flags << "-DCMAKE_AR=#{ar}" if ar + + sh(cmake, *cmake_flags, verbose: true) + + rm_rf(cxx_core_build_dir, verbose: true) + rm_rf(cpm_cache_dir, verbose: true) + + cache_dir = File.join(__dir__, "ext", "cache") + rm_rf(cache_dir, verbose: true) + abort("unable to remove #{cache_dir}") if File.directory?(cache_dir) + mkdir_p(cache_dir, verbose: true) + chdir(cache_dir, verbose: true) do + sh(*untar, "-f", output_tarball, verbose: true) + end + rm_rf(output_dir, verbose: true) + + extconf_include = File.join(cache_dir, "extconf_include.rb") + File.open(extconf_include, "w+") do |io| + io.puts(<<~CACHE_FLAGS) + cmake_flags << "-DCPM_DOWNLOAD_ALL=OFF" + cmake_flags << "-DCPM_USE_NAMED_CACHE_DIRECTORIES=ON" + cmake_flags << "-DCPM_USE_LOCAL_PACKAGES=OFF" + cmake_flags << "-DCPM_SOURCE_CACHE=\#{File.expand_path('cache', __dir__)}" + cmake_flags << "-DCOUCHBASE_CXX_CLIENT_EMBED_MOZILLA_CA_BUNDLE_ROOT=\#{File.expand_path('cache', __dir__)}" + CACHE_FLAGS + end end desc "Build the package" -task :build => :render_git_revision +task :build => [:cache_cxx_dependencies, :render_git_revision] RuboCop::RakeTask.new diff --git a/bin/check-clang-format b/bin/check-clang-format index 5aef6295..c77f3a91 100755 --- a/bin/check-clang-format +++ b/bin/check-clang-format @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -PROJECT_ROOT="$( cd "$(dirname "$0"/..)" >/dev/null 2>&1 ; pwd -P )" +PROJECT_ROOT="$( cd "$(dirname "$0")/.." >/dev/null 2>&1 ; pwd -P )" CB_GIT_CLANG_FORMAT=${CB_GIT_CLANG_FORMAT:-git-clang-format} CB_CLANG_FORMAT=${CB_CLANG_FORMAT:-$(which clang-format)} diff --git a/bin/check-clang-static-analyzer b/bin/check-clang-static-analyzer index bc02e031..2975e9c8 100755 --- a/bin/check-clang-static-analyzer +++ b/bin/check-clang-static-analyzer @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -PROJECT_ROOT="$( cd "$(dirname "$0"/..)" >/dev/null 2>&1 ; pwd -P )" +PROJECT_ROOT="$( cd "$(dirname "$0")/.." >/dev/null 2>&1 ; pwd -P )" CB_CMAKE=${CB_CMAKE:-$(which cmake)} CB_CC=${CB_CC:-$(which clang)} diff --git a/bin/jenkins/build-environment-for-alpine.sh b/bin/jenkins/build-environment-for-alpine.sh new file mode 100644 index 00000000..d5d9be27 --- /dev/null +++ b/bin/jenkins/build-environment-for-alpine.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Copyright 2020-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source /usr/local/share/chruby/chruby.sh diff --git a/bin/jenkins/build-environment-for-amazon.sh b/bin/jenkins/build-environment-for-amazon.sh new file mode 100644 index 00000000..bbccdb62 --- /dev/null +++ b/bin/jenkins/build-environment-for-amazon.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Copyright 2020-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source /usr/local/share/chruby/chruby.sh + +export CB_CC="gcc10-cc" +export CB_CXX="gcc10-c++" diff --git a/bin/jenkins/build-environment-for-centos.sh b/bin/jenkins/build-environment-for-centos.sh new file mode 100644 index 00000000..4b6187c2 --- /dev/null +++ b/bin/jenkins/build-environment-for-centos.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Copyright 2020-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source /usr/local/share/chruby/chruby.sh + +CC_PREFIX=/opt/rh/devtoolset-11/root +export CB_CC="${CC_PREFIX}/bin/gcc" +export CB_CXX="${CC_PREFIX}/bin/g++" diff --git a/bin/jenkins/build-environment-for-macos.sh b/bin/jenkins/build-environment-for-macos.sh new file mode 100644 index 00000000..943337ca --- /dev/null +++ b/bin/jenkins/build-environment-for-macos.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Copyright 2020-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source $(brew --prefix chruby)/share/chruby/chruby.sh + +# MacOS/ARM has issues building BoringSSL using GCC compiler +if [ "x$(uname -m)" != "xarm64" -a "x$(uname -m)" != "xx86_64" ] +then + for v in 12 11 + do + CC_PREFIX=$(brew --prefix gcc@${v}) + if [ -d ${CC_PREFIX} ] + then + export CB_CC="${CC_PREFIX}/bin/gcc-${v}" + export CB_CXX="${CC_PREFIX}/bin/g++-${v}" + export CB_AR="${CC_PREFIX}/bin/gcc-ar-${v}" + break + fi + done +fi diff --git a/bin/jenkins/build-environment-for-ubuntu.sh b/bin/jenkins/build-environment-for-ubuntu.sh new file mode 100644 index 00000000..d5d9be27 --- /dev/null +++ b/bin/jenkins/build-environment-for-ubuntu.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Copyright 2020-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source /usr/local/share/chruby/chruby.sh diff --git a/bin/jenkins/build-environment.sh b/bin/jenkins/build-environment.sh new file mode 100644 index 00000000..40190cb8 --- /dev/null +++ b/bin/jenkins/build-environment.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Copyright 2020-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ -f /etc/os-release ] +then + OS_ID=$(grep -o '^ID=.*' /etc/os-release | sed 's/ID=\|"//g') + OS_VERSION=$(grep -o '^VERSION_ID=.*' /etc/os-release | sed 's/VERSION_ID=\|"//g') +elif [ "$(uname -s)" = "Darwin" ] +then + OS_ID="macos" + OS_VERSION=$(uname -r) +fi + +case ${OS_ID} in + amzn) + source ${PROJECT_ROOT}/bin/jenkins/build-environment-for-amazon.sh + ;; + + centos) + source ${PROJECT_ROOT}/bin/jenkins/build-environment-for-centos.sh + ;; + + alpine) + source ${PROJECT_ROOT}/bin/jenkins/build-environment-for-alpine.sh + ;; + + ubuntu|pop) + source ${PROJECT_ROOT}/bin/jenkins/build-environment-for-ubuntu.sh + ;; + + macos) + source ${PROJECT_ROOT}/bin/jenkins/build-environment-for-macos.sh + ;; + + *) +esac diff --git a/bin/jenkins/build-extension b/bin/jenkins/build-extension deleted file mode 100755 index 0159d6af..00000000 --- a/bin/jenkins/build-extension +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2020-2021 Couchbase, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -PROJECT_ROOT="$( cd "$(dirname "$0"/../..)" >/dev/null 2>&1 ; pwd -P )" - -echo "HOSTNAME=${HOSTNAME}" -echo "NODE_NAME=${NODE_NAME}" -echo "CONTAINER_TAG=${CONTAINER_TAG}" -echo "JENKINS_SLAVE_LABELS=${JENKINS_SLAVE_LABELS}" -echo "NODE_LABELS=${NODE_LABELS}" - -set -x -set -e - -CB_RUBY_VERSION=${CB_RUBY_VERSION:-3.0} - -if [ "$(uname -s)" = "Linux" ] -then - OS_ID=unknown - OS_VERSION=unknown - if [ -f /etc/os-release ] - then - OS_ID=$(grep -o '^ID=.*' /etc/os-release | sed 's/ID=\|"//g') - OS_VERSION=$(grep -o '^VERSION_ID=.*' /etc/os-release | sed 's/VERSION_ID=\|"//g') - fi - - if [ -f /etc/alpine-release ] - then - export CB_CC="/usr/bin/gcc" - export CB_CXX="/usr/bin/g++" - elif [ ! "${OS_ID}" = "ubuntu" ] - then - export PATH="$(realpath ~/.cmake-3.21.2/bin):$PATH" - if [ "${OS_ID}" = "amzn" ] - then - export CB_CC="gcc10-cc" - export CB_CXX="gcc10-c++" - else - sudo yum install -y libstdc++-static - cc_prefix=/opt/rh/devtoolset-9/root - export CB_CC="${cc_prefix}/bin/gcc" - export CB_CXX="${cc_prefix}/bin/g++" - fi - fi - source /usr/local/share/chruby/chruby.sh -fi -if [ "$(uname -s)" = "Darwin" ] -then - if [ "x${CB_RUBY_VERSION}" = "xbrew" ] - then - ruby_prefix="$(brew --prefix ruby)" - ruby_abi_version=$(${ruby_prefix}/bin/ruby -rrbconfig -e'puts RbConfig::CONFIG["ruby_version"]') - export PATH="${ruby_prefix}/bin:${HOME}/.gem/ruby/${ruby_abi_version}/bin:$PATH" - export LDFLAGS="-L${ruby_prefix}/lib" - export CPPFLAGS="-L${ruby_prefix}/include" - export PKG_CONFIG_PATH="${ruby_prefix}/lib/pkgconfig" - else - source $(brew --prefix chruby)/share/chruby/chruby.sh - fi - cc_prefix=$(brew --prefix gcc@11) - export CB_CC="${cc_prefix}/bin/gcc-11" - export CB_CXX="${cc_prefix}/bin/g++-11" -fi - -CB_GEM_INSTALL_OPTIONS= -if [ "x${CB_RUBY_VERSION}" = "xbrew" ] -then - CB_GEM_INSTALL_OPTIONS="--user-install" -else - chruby ruby-${CB_RUBY_VERSION} -fi - -which ruby gem bundle -ruby --version -bundle --version - -gem install ${CB_GEM_INSTALL_OPTIONS} bundler gem-compiler - -bundle config set --local path ${PROJECT_ROOT}/vendor/bundle - -bundle install --verbose -export CB_EXT_BUILD_DIR="${PROJECT_ROOT}/build-${RUBY_VERSION}-${BUILD_NUMBER}" -${PROJECT_ROOT}/bin/jenkins/patch-version ${BUILD_NUMBER} - -bundle exec rake build -full_version=$(ruby -r${PROJECT_ROOT}/lib/couchbase/version.rb -e "puts Couchbase::VERSION[:sdk]") -source="${PROJECT_ROOT}/pkg/couchbase-${full_version}.gem" -precompiled="${PROJECT_ROOT}/pkg/binary" -mkdir -p ${precompiled} - -CB_GEM_COMPILE="gem compile" -if [ "x${CB_RUBY_VERSION}" = "xbrew" ] -then - gem_compiler_plugin=$(ruby -rrubygems -e "puts Gem.find_files('rubygems_plugin.rb').grep(/gem-compiler/)") - CB_GEM_COMPILE="ruby -rrubygems -r${gem_compiler_plugin} -S gem compile" -fi -if [ "$(uname -s)" = "Linux" ] -then - strip --version - if [ "x${IS_RELEASE}" = "xtrue" ] - then - CB_GEM_COMPILE="${CB_GEM_COMPILE} --strip" - fi -fi - -export CB_REMOVE_EXT_DIRECTORY=1 -if [ ! -f /etc/alpine-release ] -then - export CB_STATIC_STDLIB=1 -fi -${CB_GEM_COMPILE} --prune --verbose --output ${precompiled} ${source} -ruby -rrbconfig -rfileutils \ - -e 'FileUtils.mv(ARGV[0], ARGV[0].gsub(/\.gem$/, "-#{RbConfig::CONFIG["ruby_version"]}.gem"))' \ - ${precompiled}/*.gem -ls -lh ${precompiled}/*.gem diff --git a/bin/jenkins/build-gem.ps1 b/bin/jenkins/build-gem.ps1 new file mode 100644 index 00000000..c127fa80 --- /dev/null +++ b/bin/jenkins/build-gem.ps1 @@ -0,0 +1,136 @@ +# Copyright 2020-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +$VerbosePreference = 'Continue' + +Write-Host "PowerShell Version: $($PSVersionTable.PSVersion)" + +# Workaround for "Filename too long" errors during git clone +git config --global core.longpaths true + +$projectRoot = Resolve-Path -Path "$PSScriptRoot\..\.." +Write-Host "--- Project root: ${projectRoot}" + +$rubyVersions = "3.3.0", "3.2.2", "3.1.4", "3.0.6" +$defaultRubyVersion = $rubyVersions[0] + +$originalPath = $env:PATH + +foreach ($rubyVersion in $rubyVersions) { + Remove-Item -Path "${projectRoot}\Gemfile.lock" -ErrorAction SilentlyContinue + + $majorMinorVersion = "${rubyVersion}" -replace '(\d+)\.(\d+)\.(\d+)', '$1$2' + $installPath = "C:\Ruby${majorMinorVersion}-x64" + + $env:PATH = "$installPath\bin;$installPath\msys64\mingw64\bin;$installPath\msys64\usr\bin;$originalPath" + + $rubyBinaryPath = "${installPath}\bin\ruby.exe" + Write-Host "---" (& "${rubyBinaryPath}" --version) "(${rubyBinaryPath})" + $rubyAbiVersion = (& "${rubyBinaryPath}" -rrbconfig -e "print RbConfig::CONFIG['ruby_version']") + Write-Host "--- ABI: ${rubyAbiVersion}" + + $bundleScriptPath = "${installPath}\bin\bundle.bat" + if (-not (Test-Path "${bundleScriptPath}")) { + $bundleScriptPath = "${installPath}\bin\bundle.cmd" + } + Write-Host "---" (& "$bundleScriptPath" --version) "(${bundleScriptPath})" + + $gemScriptPath = "${installPath}\bin\gem.bat" + if (-not (Test-Path "${gemScriptPath}")) { + $gemScriptPath = "${installPath}\bin\gem.cmd" + } + Write-Host "--- rubygems" (& "${gemScriptPath}" --version) "(${gemScriptPath})" + + $bundlePath = "$env:LOCALAPPDATA\vendor\bundle\${rubyVersion}" + Write-Host "--- Install dependencies in ${bundlePath}" + $arguments = @( + "config", + "set", + "--local", + "path", + "`"${bundlePath}`"" + ) + $process = Start-Process -Wait -NoNewWindow -PassThru -FilePath $bundleScriptPath -ArgumentList $arguments + if (-not $process.ExitCode -eq 0) { + Write-Host "Unable to set bundler path. Exit code $($process.ExitCode)" + exit 1 + } + $process = Start-Process -Wait -NoNewWindow -PassThru -FilePath $bundleScriptPath -ArgumentList "install" + if (-not $process.ExitCode -eq 0) { + Write-Host "Unable to install dependencies using bundler. Exit code $($process.ExitCode)" + exit 1 + } + + $gemVersion = (& ruby -r"${projectRoot}\lib\couchbase\version.rb" -e "puts Couchbase::VERSION[:sdk]") + $sourceGemPath = "${projectRoot}\pkg\couchbase-${gemVersion}.gem" + if (Test-Path "${sourceGemPath}") { + Write-Host "--- Found source package for ${gemVersion} (${sourceGemPath})" + } else { + $buildNumber = if ($env:BUILD_NUMBER) { $env:BUILD_NUMBER } else { 0 } + Write-Host "--- Inject build number ${buildNumber} into version ${gemVersion}" + $arguments = @( + "`"${projectRoot}\bin\jenkins\patch-version.rb`"", + $buildNumber + ) + $process = Start-Process -Wait -NoNewWindow -PassThru -FilePath $rubyBinaryPath -ArgumentList $arguments + if (-not $process.ExitCode -eq 0) { + Write-Host "Unable to inject build number. Exit code $($process.ExitCode)" + exit 1 + } + $gemVersion = (& ruby -r"${projectRoot}\lib\couchbase\version.rb" -e "puts Couchbase::VERSION[:sdk]") + $sourceGemPath = "${projectRoot}\pkg\couchbase-${gemVersion}.gem" + Write-Host "--- Build source package for ${gemVersion}" + $process = Start-Process -Wait -NoNewWindow -PassThru -FilePath $bundleScriptPath -ArgumentList "exec", "rake", "build" + if (-not $process.ExitCode -eq 0) { + Write-Host "Unable to build source package. Exit code $($process.ExitCode)" + exit 1 + } + } + + $precompiledPath = "${projectRoot}\pkg\binary\${rubyAbiVersion}" + New-Item -ItemType Directory -Path $precompiledPath -Force + Write-Host "--- Build binary package for ${gemVersion} using ruby ${rubyVersion} (${precompiledPath})" + + $env:CB_STATIC_BORINGSSL = 1 + $env:CB_STATIC_STDLIB = 1 + $env:CB_REMOVE_EXT_DIRECTORY = 1 + $arguments = @( + "exec", + "gem", + "compile", + "--prune", + "--output", + "`"${precompiledPath}`"", + "`"${sourceGemPath}`"" + ) + $process = Start-Process -Wait -NoNewWindow -PassThru -FilePath $bundleScriptPath -ArgumentList $arguments + if (-not $process.ExitCode -eq 0) { + Write-Host "Unable to build binary package. Exit code $($process.ExitCode)" + exit 1 + } +} + +$majorMinorVersion = "${defaultRubyVersion}" -replace '(\d+)\.(\d+)\.(\d+)', '$1$2' +$defaultRubyPath = "C:\Ruby${majorMinorVersion}-x64" +$defaultRubyBinaryPath = "${defaultRubyPath}\bin\ruby.exe" +$env:PATH = "$defaultRubyPath\bin;$defaultRubyPath\msys64\mingw64\bin;$defaultRubyPath\msys64\usr\bin;$originalPath" + +Write-Host "--- Repackage binary gems into single `"fat`" gem" +$process = Start-Process -Wait -NoNewWindow -PassThru -FilePath $defaultRubyBinaryPath -ArgumentList "`"${projectRoot}\bin\jenkins\repackage-extension.rb`"" +if (-not $process.ExitCode -eq 0) { + Write-Host "Unable to repackage binaries into `"fat`" package. Exit code $($process.ExitCode)" + exit 1 +} + +# vim: et ts=4 sw=4 sts=4 diff --git a/bin/jenkins/build-gem.sh b/bin/jenkins/build-gem.sh new file mode 100755 index 00000000..5a12552f --- /dev/null +++ b/bin/jenkins/build-gem.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +# Copyright 2020-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +PROJECT_ROOT="$( cd "$(dirname "$0")/../.." >/dev/null 2>&1 ; pwd -P )" + +echo "HOSTNAME=${HOSTNAME}" +echo "NODE_NAME=${NODE_NAME}" +echo "CONTAINER_TAG=${CONTAINER_TAG}" +echo "JENKINS_SLAVE_LABELS=${JENKINS_SLAVE_LABELS}" +echo "NODE_LABELS=${NODE_LABELS}" + +SUPPORTED_RUBY_VERSIONS="3.3 3.2 3.1 3.0" +CMAKE_VERSION=3.28.1 +if [ -d "${HOME}/.cmake-${CMAKE_VERSION}/bin" ] +then + export PATH="${HOME}/.cmake-${CMAKE_VERSION}/bin:${PATH}" +fi + +set -x +set -e + +source ${PROJECT_ROOT}/bin/jenkins/build-environment.sh + +for RUBY_VERSION in ${SUPPORTED_RUBY_VERSIONS} +do + if ! chruby ruby-${RUBY_VERSION} + then + continue + fi + which ruby gem bundle + ruby --version + bundle --version + + bundle config set --local path ${PROJECT_ROOT}/vendor/bundle/${RUBY_VERSION} + bundle install + + GEM_VERSION=$(ruby -r${PROJECT_ROOT}/lib/couchbase/version.rb -e "puts Couchbase::VERSION[:sdk]") + echo "--- Configured build environment for ${GEM_VERSION} (ruby ${RUBY_VERSION})" + SOURCE_GEM_PATH="${PROJECT_ROOT}/pkg/couchbase-${GEM_VERSION}.gem" + if [ -f "${SOURCE_GEM_PATH}" ] + then + echo "--- Found source package ${GEM_VERSION} (${SOURCE_GEM_PATH})" + else + echo "--- Inject build number ${BUILD_NUMBER} into version ${GEM_VERSION}" + ruby ${PROJECT_ROOT}/bin/jenkins/patch-version.rb ${BUILD_NUMBER} + GEM_VERSION=$(ruby -r${PROJECT_ROOT}/lib/couchbase/version.rb -e "puts Couchbase::VERSION[:sdk]") + SOURCE_GEM_PATH="${PROJECT_ROOT}/pkg/couchbase-${GEM_VERSION}.gem" + echo "--- Build source package ${GEM_VERSION}" + bundle exec rake build + fi + + PRECOMPILED_PATH="${PROJECT_ROOT}/pkg/binary/$(ruby -rrbconfig -e 'print RbConfig::CONFIG["ruby_version"]')" + mkdir -p ${PRECOMPILED_PATH} + echo "--- Build binary package for ${GEM_VERSION} using ruby ${RUBY_VERSION} (${PRECOMPILED_PATH})" + + export CB_STATIC_BORINGSSL=1 + export CB_STATIC_STDLIB=1 + export CB_REMOVE_EXT_DIRECTORY=1 + bundle install + bundle exec gem compile --prune --output ${PRECOMPILED_PATH} ${SOURCE_GEM_PATH} +done + +echo "--- Repackage binary gems into single \"fat\" gem" +ruby ${PROJECT_ROOT}/bin/jenkins/repackage-extension.rb diff --git a/bin/jenkins/build-repos b/bin/jenkins/build-repos index d7fd90dc..7033e710 100755 --- a/bin/jenkins/build-repos +++ b/bin/jenkins/build-repos @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -PROJECT_ROOT="$( cd "$(dirname "$0"/../..)" >/dev/null 2>&1 ; pwd -P )" +PROJECT_ROOT="$( cd "$(dirname "$0")/../.." >/dev/null 2>&1 ; pwd -P )" CB_RUBY_VERSION=${CB_RUBY_VERSION:-3.0} diff --git a/bin/jenkins/check-clang-format b/bin/jenkins/check-clang-format index 90033b50..22987b55 100755 --- a/bin/jenkins/check-clang-format +++ b/bin/jenkins/check-clang-format @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -PROJECT_ROOT="$( cd "$(dirname "$0"/../..)" >/dev/null 2>&1 ; pwd -P )" +PROJECT_ROOT="$( cd "$(dirname "$0")/../.." >/dev/null 2>&1 ; pwd -P )" echo "HOSTNAME=${HOSTNAME}" echo "NODE_NAME=${NODE_NAME}" diff --git a/bin/jenkins/check-clang-static-analyzer b/bin/jenkins/check-clang-static-analyzer index 6f7a27c6..d9a321ab 100755 --- a/bin/jenkins/check-clang-static-analyzer +++ b/bin/jenkins/check-clang-static-analyzer @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -PROJECT_ROOT="$( cd "$(dirname "$0"/../..)" >/dev/null 2>&1 ; pwd -P )" +PROJECT_ROOT="$( cd "$(dirname "$0")/../.." >/dev/null 2>&1 ; pwd -P )" echo "HOSTNAME=${HOSTNAME}" echo "NODE_NAME=${NODE_NAME}" diff --git a/bin/jenkins/check-rubocop b/bin/jenkins/check-rubocop index eeb4e09f..a1182efb 100755 --- a/bin/jenkins/check-rubocop +++ b/bin/jenkins/check-rubocop @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -PROJECT_ROOT="$( cd "$(dirname "$0"/../..)" >/dev/null 2>&1 ; pwd -P )" +PROJECT_ROOT="$( cd "$(dirname "$0")/../.." >/dev/null 2>&1 ; pwd -P )" echo "HOSTNAME=${HOSTNAME}" echo "NODE_NAME=${NODE_NAME}" diff --git a/bin/jenkins/install-dependencies b/bin/jenkins/install-dependencies deleted file mode 100755 index 640e2ec7..00000000 --- a/bin/jenkins/install-dependencies +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2020-2021 Couchbase, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -PROJECT_ROOT="$( cd "$(dirname "$0"/../..)" >/dev/null 2>&1 ; pwd -P )" - -echo "HOSTNAME=${HOSTNAME}" -echo "NODE_NAME=${NODE_NAME}" -echo "CONTAINER_TAG=${CONTAINER_TAG}" -echo "JENKINS_SLAVE_LABELS=${JENKINS_SLAVE_LABELS}" -echo "NODE_LABELS=${NODE_LABELS}" - -set -x -set -e - -CB_RUBY_VERSION=${CB_RUBY_VERSION:-3.0} - -if [ "$(uname -s)" = "Linux" ] -then - if [ -f /etc/alpine-release ] - then - cat /etc/alpine-release - sudo /sbin/apk add openssl libcrypto1.1 openssl-dev clang clang-extra-tools curl make bash gcc g++ xz cmake zlib-dev linux-headers readline-dev yaml-dev - fi - - if [ ! -x /usr/local/bin/ruby-install ] - then - curl -L -o postmodern-ruby-install-f59dd9c.tar.gz https://github.com/postmodern/ruby-install/tarball/f59dd9c - tar -xzvf postmodern-ruby-install-f59dd9c.tar.gz - cd postmodern-ruby-install-f59dd9c/ - sudo make install - fi - - if [ ! -e /usr/local/share/chruby/chruby.sh ] - then - curl -L -o chruby-0.3.9.tar.gz https://github.com/postmodern/chruby/archive/v0.3.9.tar.gz - tar -xzvf chruby-0.3.9.tar.gz - cd chruby-0.3.9/ - sudo make install - fi - - OS_ID=unknown - OS_VERSION=unknown - if [ -f /etc/os-release ] - then - OS_ID=$(grep -o '^ID=.*' /etc/os-release | sed 's/ID=\|"//g') - OS_VERSION=$(grep -o '^VERSION_ID=.*' /etc/os-release | sed 's/VERSION_ID=\|"//g') - fi - - if [ "${OS_ID}" = "amzn" -a "${OS_VERSION}" = "2" ] - then - sudo yum erase -y openssl-devel || true - sudo yum install -y openssl11-static openssl11-devel gcc10 gcc10-c++ libyaml-devel readline-devel - CB_EXTRAS="CC=gcc10-cc CXX=gcc10-c++" - fi - if [ "${OS_ID}" = "centos" -a "${OS_VERSION}" = "7" ] - then - sudo yum install -y openssl11-static openssl11-devel - cc_prefix=/opt/rh/devtoolset-9/root - CB_EXTRAS="CC=${cc_prefix}/bin/gcc CXX=${cc_prefix}/bin/g++" - fi - if [ "${OS_ID}" = "ubuntu" -a "${OS_VERSION}" = "20.04" ] - then - sudo apt-get update -y - sudo apt-get install -y libssl-dev \ - clang clang-format clang-tidy clang-tools libc++-dev libc++abi-dev || true - fi - ruby-install --version - until ruby-install --update --debug - do - if [ $? != 0 ] - then - sudo rm -rf $HOME/.cache/ruby-install/ruby/*.part - d=$(( RANDOM % 10 + 1 )) - echo "sleep for $d seconds" - sleep $d - fi - done - until ruby-install --debug --jobs=6 --no-reinstall ruby ${CB_RUBY_VERSION} -- ${CB_EXTRAS} || \ - ruby-install --debug --jobs=6 --no-reinstall --no-install-deps ruby ${CB_RUBY_VERSION} -- ${CB_EXTRAS} - do - if [ $? != 0 ] - then - sudo rm -rf $HOME/.cache/ruby-install/ruby/*.part - d=$(( RANDOM % 10 + 1 )) - echo "sleep for $d seconds" - sleep $d - fi - done -fi -if [ "$(uname -s)" = "Darwin" ] -then - sw_vers - system_profiler SPSoftwareDataType SPHardwareDataType - xcode-select --print-path - - brew doctor || true - brew list --versions || true - if [ "x${CB_RUBY_VERSION}" = "xbrew" ] - then - brew install automake bison@2.7 openssl@1.1 readline libyaml gdbm libffi - brew install ruby-install chruby - brew install cmake gcc@11 - fi - cc_prefix=$(brew --prefix gcc@11) - ${cc_prefix}/bin/gcc-11 --version - ${cc_prefix}/bin/g++-11 --version - if [ "x${CB_RUBY_VERSION}" = "xbrew" ] - then - brew install ruby - $(brew --prefix ruby)/bin/ruby --version - else - ruby-install --version - ruby-install --update || true - export PATH="$(brew --prefix bison@2.7)/bin:$PATH" - bison --version - PATCH= - if [ "x${CB_RUBY_VERSION}" = "x2.7" -o "x${CB_RUBY_VERSION}" = "x3.0" ] - then - PATCH="--patch https://gist.githubusercontent.com/avsej/3cf6e57557ac13100f01cacebec10873/raw/d83398e9ef3c63081d3af1b6cc1c77a68f3bfb65/0001-Revert-merge-revision-s-e033c9d7db02a4e8d2973364ecb4.patch" - fi - ruby-install --jobs=6 --no-reinstall --no-install-deps ${PATCH} ruby ${CB_RUBY_VERSION} -- CC="${cc_prefix}/bin/gcc-11" CXX="${cc_prefix}/bin/g++-11" - fi -fi - -if [ "$(uname -s)" = "Linux" ] -then - if [ -f /etc/alpine-release ] - then - cmake --version - else - if [ ! -d ${HOME}/.cmake-3.21.2 ] - then - machine=$(uname -m) - curl -L -o cmake-3.21.2-linux-${machine}.tar.gz https://github.com/Kitware/CMake/releases/download/v3.21.2/cmake-3.21.2-linux-${machine}.tar.gz - tar xvf cmake-3.21.2-linux-${machine}.tar.gz - mv cmake-3.21.2-linux-${machine} ${HOME}/.cmake-3.21.2 - rm cmake-3.21.2-linux-${machine}.tar.gz - fi - ${HOME}/.cmake-3.21.2/bin/cmake --version - fi -fi diff --git a/bin/jenkins/install-dependencies-for-alpine.sh b/bin/jenkins/install-dependencies-for-alpine.sh new file mode 100755 index 00000000..f0a98ac9 --- /dev/null +++ b/bin/jenkins/install-dependencies-for-alpine.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env sh +# Copyright 2020-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +PROJECT_ROOT="$( cd "$(dirname "$0")/../.." >/dev/null 2>&1 ; pwd -P )" + +[ "$(id -u)" -ne 0 ] && SUDO="sudo " || SUDO="" + +set -x +set -e + +if ! command -v ruby &> /dev/null +then + ${SUDO} /sbin/apk add ruby +fi + +${SUDO} /sbin/apk add \ + bash \ + clang \ + clang-extra-tools \ + cmake \ + curl \ + g++ \ + gcc \ + git \ + linux-headers \ + make \ + openssl \ + openssl-dev \ + readline-dev \ + xz \ + yaml-dev\ + zlib-dev \ + +# NOTE: we don't want to build 3.0 on Alpine, as it will require +# switching OpenSSL to 1.1 +export SUPPORTED_RUBY_VERSIONS="3.1 3.2 3.3" + +exec bash ${PROJECT_ROOT}/bin/jenkins/install-rubies.sh diff --git a/bin/jenkins/install-dependencies-for-amazon.sh b/bin/jenkins/install-dependencies-for-amazon.sh new file mode 100755 index 00000000..24a8841e --- /dev/null +++ b/bin/jenkins/install-dependencies-for-amazon.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Copyright 2020-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +PROJECT_ROOT="$( cd "$(dirname "$0")/../.." >/dev/null 2>&1 ; pwd -P )" + +set -x +set -e + +[ "$(id -u)" -ne 0 ] && SUDO="sudo " || SUDO="" + +if ! command -v ruby &> /dev/null +then + ${SUDO} yum install -y ruby +fi + +${SUDO} yum erase -y openssl-devel || true +${SUDO} yum install -y \ + binutils \ + gcc10 \ + gcc10-c++ \ + git \ + gzip \ + libyaml-devel \ + make \ + ninja-build \ + openssl11-devel \ + readline-devel \ + tar \ + which \ + xz \ + +CMAKE_VERSION=3.28.1 +if [ ! -d "${HOME}/.cmake-${CMAKE_VERSION}" ] +then + MACHINE=$(uname -m) + curl -L -o cmake-${CMAKE_VERSION}-linux-${MACHINE}.tar.gz https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${MACHINE}.tar.gz + tar xvf cmake-${CMAKE_VERSION}-linux-${MACHINE}.tar.gz + mv cmake-${CMAKE_VERSION}-linux-${MACHINE} ${HOME}/.cmake-${CMAKE_VERSION} + rm cmake-${CMAKE_VERSION}-linux-${MACHINE}.tar.gz +fi + +export RUBY_CONFIGURE_FLAGS="CC=gcc10-cc CXX=gcc10-c++" + +exec ${PROJECT_ROOT}/bin/jenkins/install-rubies.sh diff --git a/bin/jenkins/install-dependencies-for-centos.sh b/bin/jenkins/install-dependencies-for-centos.sh new file mode 100755 index 00000000..b0aecc54 --- /dev/null +++ b/bin/jenkins/install-dependencies-for-centos.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Copyright 2020-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +PROJECT_ROOT="$( cd "$(dirname "$0")/../.." >/dev/null 2>&1 ; pwd -P )" + +set -x +set -e + +[ "$(id -u)" -ne 0 ] && SUDO="sudo " || SUDO="" + +if ! command -v ruby &> /dev/null +then + ${SUDO} yum install -y ruby +fi + +if [ ! -d /opt/rh/devtoolset-11/root ] +then + ${SUDO} yum install -y epel-release centos-release-scl + ${SUDO} yum install -y devtoolset-11 +fi + +${SUDO} yum erase -y openssl-devel || true +${SUDO} yum install -y \ + binutils \ + cmake3 \ + git \ + gzip \ + libyaml-devel \ + make \ + ninja-build \ + openssl11 \ + openssl11-devel \ + readline-devel \ + tar \ + which \ + xz \ + +CC_PREFIX=/opt/rh/devtoolset-11/root +export RUBY_CONFIGURE_FLAGS="--with-openssl-include=/usr/include/openssl11 --with-openssl-lib=/usr/lib64/openssl11 CC=${CC_PREFIX}/bin/gcc CXX=${CC_PREFIX}/bin/g++" + +exec ${PROJECT_ROOT}/bin/jenkins/install-rubies.sh diff --git a/bin/jenkins/install-dependencies-for-macos.sh b/bin/jenkins/install-dependencies-for-macos.sh new file mode 100755 index 00000000..15f63d1f --- /dev/null +++ b/bin/jenkins/install-dependencies-for-macos.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Copyright 2020-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +PROJECT_ROOT="$( cd "$(dirname "$0")/../.." >/dev/null 2>&1 ; pwd -P )" + +set -x +set -e + +sw_vers +system_profiler SPSoftwareDataType SPHardwareDataType +xcode-select --print-path + +brew doctor || true +brew list --versions || true +brew install \ + automake \ + bison \ + chruby \ + cmake \ + gcc@11 \ + gcc@12 \ + gdbm \ + libffi \ + libyaml \ + openssl@3 \ + readline \ + ruby \ + ruby-install \ + +if [ "x$(uname -m)" != "xarm64" ] +then + GCC_VER=12 + CC_PREFIX=$(brew --prefix gcc@${GCC_VER}) + ${CC_PREFIX}/bin/gcc-${GCC_VER} --version + ${CC_PREFIX}/bin/g++-${GCC_VER} --version + export RUBY_CONFIGURE_FLAGS="CC=${CC_PREFIX}/bin/gcc-${GCC_VER} CXX=${CC_PREFIX}/bin/g++-${GCC_VER}" +fi + +if [ ! -z "${BUILD_NUMBER}" ] +then + # ensure the builder will not pick up anything globally + rm -rf ${HOME}/.gem/ruby +fi +# NOTE: Exclude Ruby 3.0 to avoid openssl@1.1 +export SUPPORTED_RUBY_VERSIONS="3.1 3.2 3.3" +exec ${PROJECT_ROOT}/bin/jenkins/install-rubies.sh diff --git a/bin/jenkins/install-dependencies-for-ubuntu.sh b/bin/jenkins/install-dependencies-for-ubuntu.sh new file mode 100755 index 00000000..c2f4aa0e --- /dev/null +++ b/bin/jenkins/install-dependencies-for-ubuntu.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Copyright 2020-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +PROJECT_ROOT="$( cd "$(dirname "$0")/../.." >/dev/null 2>&1 ; pwd -P )" + +set -x +set -e + +[ "$(id -u)" -ne 0 ] && SUDO="sudo " || SUDO="" + +${SUDO} apt-get update -y + +if ! command -v ruby &> /dev/null +then + ${SUDO} apt install -y ruby +fi + +${SUDO} apt-get install -y \ + clang \ + clang-format \ + clang-tidy \ + clang-tools \ + cmake \ + curl \ + g++ \ + gcc \ + git \ + libc++-dev \ + libc++abi-dev \ + libssl-dev \ + libyaml-dev \ + make \ + ninja-build \ + xz-utils \ + +# NOTE: Ubuntu 22+ removed OpenSSL 1.1 compatibility, +# so Ruby 3.0.x will not work without extra patches +export SUPPORTED_RUBY_VERSIONS="3.1 3.2 3.3" + +rm -rf ${HOME}/.rubies +exec ${PROJECT_ROOT}/bin/jenkins/install-rubies.sh diff --git a/bin/jenkins/install-dependencies.ps1 b/bin/jenkins/install-dependencies.ps1 new file mode 100644 index 00000000..519ac325 --- /dev/null +++ b/bin/jenkins/install-dependencies.ps1 @@ -0,0 +1,105 @@ +# Copyright 2020-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +$VerbosePreference = 'Continue' + +Write-Host "--- COMPUTERNAME=$env:COMPUTERNAME" +Write-Host "--- HOSTNAME=$env:HOSTNAME" +Write-Host "--- NODE_NAME=$env:NODE_NAME" +Write-Host "--- CONTAINER_TAG=$env:CONTAINER_TAG" +Write-Host "--- JENKINS_SLAVE_LABELS=$env:JENKINS_SLAVE_LABELS" +Write-Host "--- NODE_LABELS=$env:NODE_LABELS" + +Get-Volume | Select-Object DriveLetter, FileSystemLabel, SizeRemaining, Size | Format-Table -AutoSize +Write-Host "--- PowerShell Version: $($PSVersionTable.PSVersion)" + +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12, [Net.SecurityProtocolType]::Tls11, [Net.SecurityProtocolType]::Tls + +$installPath = "$env:LOCALAPPDATA\CMake" +if (Test-Path "${installPath}\bin\cmake.exe") { + Write-Host "--- Installed CMake version (${installPath}):" (& "${installPath}\bin\cmake.exe" --version) +} else { + $cmakeVersion = "3.28.1" + $url = "https://github.com/Kitware/CMake/releases/download/v${cmakeVersion}/cmake-${cmakeVersion}-windows-x86_64.zip" + Write-Host "--- Downloading CMake version ${cmakeVersion} from ${url}" + $outputFilePath = "CMake${cmakeVersion}.zip" + Invoke-WebRequest -Uri $url -OutFile $outputFilePath + Write-Host "--- Download complete to ${outputFilePath}" + + Write-Host "--- Extract ${outputFilePath} to ${installPath}" + Expand-Archive -Path ${outputFilePath} -DestinationPath ${installPath} -Force + Get-ChildItem "${installPath}\cmake-${cmakeVersion}-windows-x86_64" | Move-Item -Destination "${installPath}" -Force + + Write-Host "--- Installation complete for CMake version ${cmakeVersion}" + Remove-Item $outputFilePath + Write-Host "--- Installed CMake version (${installPath}):" (& "${installPath}\bin\cmake.exe" --version) +} + +$rubyVersions = "3.3.0", "3.2.2", "3.1.4", "3.0.6" + +Get-ChildItem -Path 'C:\' -Directory -Filter 'Ruby*' + +foreach ($rubyVersion in $rubyVersions) { + $majorMinorVersion = $rubyVersion -replace '(\d+)\.(\d+)\.(\d+)', '$1$2' + $installPath = "C:\Ruby${majorMinorVersion}-x64" + + if (Test-Path "${installPath}\bin\ruby.exe") { + Write-Host "--- Ruby ${rubyVersion} is already installed in ${installPath}. Skipping download and installation." + Write-Host "--- Installed Ruby version:" (& "${installPath}\bin\ruby.exe" --version) + continue + } + + $url = "https://github.com/oneclick/rubyinstaller2/releases/download/RubyInstaller-${rubyVersion}-1/rubyinstaller-devkit-${rubyVersion}-1-x64.exe" + Write-Host "--- Downloading RubyInstaller version ${rubyVersion} from ${url}" + + $outputFilePath = "RubyInstaller-${rubyVersion}-1-x64.exe" + Invoke-WebRequest -Uri $url -OutFile $outputFilePath + $outputFilePath = Resolve-Path -Path $outputFilePath + + Write-Host "--- Download complete to ${outputFilePath}. Installing Ruby version ${rubyVersion}" + $maxAttempts = 2 + $attempt = 1 + while ($attempt -le $maxAttempts) { + $process = Start-Process -Wait -NoNewWindow -PassThru -FilePath $outputFilePath -ArgumentList "/NORESTART", "/VERYSILENT", "/CURRENTUSER", "/LOG=install-${rubyVersion}.log" + if ($process.ExitCode -eq 0) { + Write-Host "--- Installation complete for Ruby version ${rubyVersion}" + } else { + Write-Host "--- Installation failed with exit code $($process.ExitCode)" + exit 1 + } + + try { + Write-Host "--- Installed Ruby version:" (& "${installPath}\bin\ruby.exe" --version) + break + } catch { + Write-Host "--- Error: $_. Attempt to uninstall and reinstall" + if (Test-Path "${installPath}\unins000.exe") { + $process = Start-Process "${installPath}\unins000.exe" -ArgumentList "/VERYSILENT" -Wait -NoNewWindow + if ($process.ExitCode -eq 0) { + Write-Host "--- Uninstalled Ruby ${rubyVersion}. Retrying..." + $attempt += 1 + continue + } else { + Write-Host "--- Failed to uninstall Ruby ${rubyVersion} with exit code: $($process.ExitCode)." + exit 1 + } + } + throw + } + } + + Remove-Item $outputFilePath +} + +# vim: et ts=4 sw=4 sts=4 diff --git a/bin/jenkins/install-dependencies.sh b/bin/jenkins/install-dependencies.sh new file mode 100755 index 00000000..eae49170 --- /dev/null +++ b/bin/jenkins/install-dependencies.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env sh +# Copyright 2020-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "HOSTNAME=${HOSTNAME}" +echo "NODE_NAME=${NODE_NAME}" +echo "CONTAINER_TAG=${CONTAINER_TAG}" +echo "JENKINS_SLAVE_LABELS=${JENKINS_SLAVE_LABELS}" +echo "NODE_LABELS=${NODE_LABELS}" + +PROJECT_ROOT="$( cd "$(dirname "$0")/../.." >/dev/null 2>&1 ; pwd -P )" + +set -x +set -e + +if [ -f /etc/os-release ] +then + OS_ID=$(grep -o '^ID=.*' /etc/os-release | sed 's/ID=\|"//g') + OS_VERSION=$(grep -o '^VERSION_ID=.*' /etc/os-release | sed 's/VERSION_ID=\|"//g') +elif [ "$(uname -s)" = "Darwin" ] +then + OS_ID="macos" + OS_VERSION=$(uname -r) +fi + +case ${OS_ID} in + amzn) + exec ${PROJECT_ROOT}/bin/jenkins/install-dependencies-for-amazon.sh + ;; + + centos) + exec ${PROJECT_ROOT}/bin/jenkins/install-dependencies-for-centos.sh + ;; + + alpine) + exec ${PROJECT_ROOT}/bin/jenkins/install-dependencies-for-alpine.sh + ;; + + ubuntu|pop) + exec ${PROJECT_ROOT}/bin/jenkins/install-dependencies-for-ubuntu.sh + ;; + + macos) + exec ${PROJECT_ROOT}/bin/jenkins/install-dependencies-for-macos.sh + ;; + + *) + echo "unknown system: OS_ID=${OS_ID}, OS_VERSION=${OS_VERSION}" + exit 1 +esac diff --git a/bin/jenkins/install-gem b/bin/jenkins/install-gem index 0bac8cc9..e692c8e8 100755 --- a/bin/jenkins/install-gem +++ b/bin/jenkins/install-gem @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -PROJECT_ROOT="$( cd "$(dirname "$0"/../..)" >/dev/null 2>&1 ; pwd -P )" +PROJECT_ROOT="$( cd "$(dirname "$0")/../.." >/dev/null 2>&1 ; pwd -P )" echo "HOSTNAME=${HOSTNAME}" echo "NODE_NAME=${NODE_NAME}" diff --git a/bin/jenkins/install-rubies.sh b/bin/jenkins/install-rubies.sh new file mode 100755 index 00000000..344409b0 --- /dev/null +++ b/bin/jenkins/install-rubies.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# Copyright 2020-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +PROJECT_ROOT="$( cd "$(dirname "$0")/../.." >/dev/null 2>&1 ; pwd -P )" + +set -x +set -e + +[ "$(id -u)" -ne 0 ] && SUDO="sudo " || SUDO="" + +SUPPORTED_RUBY_VERSIONS=${SUPPORTED_RUBY_VERSIONS:-"3.1 3.2 3.3"} +RUBY_CONFIGURE_FLAGS=${RUBY_CONFIGURE_FLAGS:-} + +RUBY_INSTALL_VERSION=0.9.3 +CHRUBY_VERSION=0.3.9 + + +RUBY_INSTALL=/usr/local/bin/ruby-install +NEED_TO_INSTALL_RUBY_INSTALL=yes +if [ -x ${RUBY_INSTALL} -a "x$(${RUBY_INSTALL} --version | grep -o '[0-9]\.[0-9]\.[0-9]')" = "x${RUBY_INSTALL_VERSION}" ] +then + NEED_TO_INSTALL_RUBY_INSTALL=no +fi +if [ "x${NEED_TO_INSTALL_RUBY_INSTALL}" = "xyes" ] +then + RUBY_INSTALL_URL=https://github.com/postmodern/ruby-install/releases/download/v${RUBY_INSTALL_VERSION}/ruby-install-${RUBY_INSTALL_VERSION}.tar.gz + curl -L -o ruby-install-${RUBY_INSTALL_VERSION}.tar.gz ${RUBY_INSTALL_URL} + tar -xzf ruby-install-${RUBY_INSTALL_VERSION}.tar.gz + ( + cd ruby-install-${RUBY_INSTALL_VERSION}/ + ${SUDO} make install + ) + rm -rf ruby-install-${RUBY_INSTALL_VERSION}* +fi + +CHRUBY=/usr/local/bin/chruby-exec +NEED_TO_INSTALL_CHRUBY=yes +if [ -x ${CHRUBY} -a "x$(${CHRUBY} --version | grep -o '[0-9]\.[0-9]\.[0-9]')" = "x${CHRUBY_VERSION}" ] +then + NEED_TO_INSTALL_CHRUBY=no +fi +if [ "x${NEED_TO_INSTALL_CHRUBY}" = "xyes" ] +then + CHRUBY_URL=https://github.com/postmodern/chruby/releases/download/v${CHRUBY_VERSION}/chruby-${CHRUBY_VERSION}.tar.gz + curl -L -o chruby-${CHRUBY_VERSION}.tar.gz ${CHRUBY_URL} + tar -xzf chruby-${CHRUBY_VERSION}.tar.gz + ( + cd chruby-${CHRUBY_VERSION}/ + ${SUDO} make install + ) + rm -rf chruby-${CHRUBY_VERSION}* +fi + +until ${RUBY_INSTALL} --update +do + if [ $? != 0 ] + then + ${SUDO} rm -rf ${HOME}/.cache/ruby-install/ruby/*.part + d=$(( RANDOM % 10 + 1 )) + echo "sleep for $d seconds" + sleep $d + fi +done + +for RUBY_VERSION in ${SUPPORTED_RUBY_VERSIONS} +do + until ${RUBY_INSTALL} --jobs=6 --no-reinstall --no-install-deps ruby ${RUBY_VERSION} -- --disable-install-doc ${RUBY_CONFIGURE_FLAGS} + do + if [ $? != 0 ] + then + ${SUDO} rm -rf ${HOME}/.cache/ruby-install/ruby/*.part + d=$(( RANDOM % 10 + 1 )) + echo "sleep for $d seconds" + sleep $d + fi + done +done diff --git a/bin/jenkins/patch-version b/bin/jenkins/patch-version.rb similarity index 100% rename from bin/jenkins/patch-version rename to bin/jenkins/patch-version.rb diff --git a/bin/jenkins/repackage-extension.rb b/bin/jenkins/repackage-extension.rb new file mode 100755 index 00000000..a3cf22fd --- /dev/null +++ b/bin/jenkins/repackage-extension.rb @@ -0,0 +1,111 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "fileutils" +require "open3" +require "rbconfig" +require "tmpdir" +require "rubygems/installer" +require "rubygems/package" + +module Gem + class Collector + def initialize(gemfiles) + @gemfiles = gemfiles + end + + def bundle(target_dir) + installers = [] + + first_gemspec = nil + known_abis = [] + @gemfiles.each do |gemfile| + installer = prepare_installer(gemfile) + first_gemspec ||= installer.spec.dup # collect everything to first gemspec + installers << installer + + puts "Unpacking gem in temporary directory '#{installer.spec.full_gem_path}'..." + installer.package.extract_files(installer.spec.full_gem_path) + + # determine build artifacts from require_paths + dlext = RbConfig::CONFIG["DLEXT"] + lib_dirs = installer.spec.require_paths.join(",") + abi = installer.spec.required_ruby_version.requirements.flatten.last.segments[0, 2].join(".") + known_abis |= [abi] + Dir.glob("#{installer.spec.full_gem_path}/{#{lib_dirs}}/**/*.#{dlext}").map do |path| + dirname, basename = File.split(path) + new_dirname = File.join(dirname, abi) + new_path = File.join(new_dirname, basename).gsub(installer.spec.full_gem_path, first_gemspec.full_gem_path) + FileUtils.mkdir_p(File.dirname(new_path)) + FileUtils.mv(path, new_path) + system("strip", "--strip-all", new_path) || system("strip", new_path) + file = new_path.sub("#{first_gemspec.full_gem_path}/", "") + puts "Adding '#{file}' to gemspec" + first_gemspec.files.push file + end + end + + if first_gemspec + Dir.chdir(first_gemspec.full_gem_path) do + File.write("lib/couchbase/libcouchbase.rb", <<-RUBY) + begin + require_relative "\#{RUBY_VERSION[/(\\d+\\.\\d+)/]}/libcouchbase" + rescue LoadError + raise LoadError, "unable to load couchbase extension for Ruby \#{RUBY_VERSION}. Only available for #{known_abis.join(', ')}. " \\ + "Try to install couchbase from sources with 'gem install --platform ruby couchbase'" + end + RUBY + first_gemspec.files.push "lib/couchbase/libcouchbase.rb" + end + # remove any non-existing files + first_gemspec.files.select! { |f| File.exist?(File.join(first_gemspec.full_gem_path, f)) } + + result = repackage(first_gemspec) + output = File.join(target_dir, File.basename(result)) + FileUtils.mv(result, output) + puts "Repackaged library as: #{output}" + else + puts "No gems found" + end + + installers.each do |installer| + FileUtils.rm_rf File.dirname(installer.spec.full_gem_path) + end + end + + def prepare_installer(gemfile) + basename = File.basename(gemfile, ".gem") + installer = Gem::Installer.at(gemfile, {unpack: true}) + installer.spec.full_gem_path = File.join(Dir.glob(Dir.mktmpdir).first, basename) + installer.spec.extension_dir = File.join(installer.spec.full_gem_path, "lib") + installer + end + + def repackage(gemspec) + # clear out extensions from gemspec + gemspec.extensions.clear + + # adjust platform + gemspec.platform = Gem::Platform::CURRENT + gemspec.required_ruby_version = "> 3.0" + + # build new gem + output_gem = nil + + Dir.chdir gemspec.full_gem_path do + output_gem = Gem::Package.build(gemspec) + end + + abort "There was a problem building the gem." unless output_gem + File.join(gemspec.full_gem_path, output_gem) + end + end +end + +pkg_dir = File.expand_path("../../pkg", __dir__) +gemfiles = Dir[File.join(pkg_dir, "binary/**/*.gem")] +collector = Gem::Collector.new(gemfiles) +output_dir = File.join(pkg_dir, "fat") +FileUtils.rm_rf(output_dir) +FileUtils.mkdir_p(output_dir) +collector.bundle(output_dir) diff --git a/bin/jenkins/test-with-cbdyncluster b/bin/jenkins/test-with-cbdyncluster index 695787fc..074cf9f2 100755 --- a/bin/jenkins/test-with-cbdyncluster +++ b/bin/jenkins/test-with-cbdyncluster @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -PROJECT_ROOT="$( cd "$(dirname "$0"/../..)" >/dev/null 2>&1 ; pwd -P )" +PROJECT_ROOT="$( cd "$(dirname "$0")/../.." >/dev/null 2>&1 ; pwd -P )" echo "HOSTNAME=${HOSTNAME}" echo "NODE_NAME=${NODE_NAME}" diff --git a/bin/setup b/bin/setup index 8869c668..8b6cb96a 100755 --- a/bin/setup +++ b/bin/setup @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -PROJECT_ROOT="$( cd "$(dirname "$0"/..)" >/dev/null 2>&1 ; pwd -P )" +PROJECT_ROOT="$( cd "$(dirname "$0")/.." >/dev/null 2>&1 ; pwd -P )" set -euo pipefail IFS=$'\n\t' diff --git a/couchbase.gemspec b/couchbase.gemspec index d655f20b..3f9b66eb 100644 --- a/couchbase.gemspec +++ b/couchbase.gemspec @@ -38,69 +38,25 @@ Gem::Specification.new do |spec| "rubygems_mfa_required" => "true", } - spec.files = Dir[ - "LICENSE.txt", - "README.md", - "ext/*.cxx", - "ext/*.hxx.in", - "ext/*.rb", - "ext/CMakeLists.txt", - "ext/couchbase/CMakeLists.txt", - "ext/couchbase/LICENSE.txt", - "ext/couchbase/cmake/*", - "ext/couchbase/core/**/*", - "ext/couchbase/couchbase/**/*", - "ext/couchbase/test/*", - "ext/couchbase/third_party/asio/COPYING", - "ext/couchbase/third_party/asio/LICENSE*", - "ext/couchbase/third_party/asio/asio/include/*.hpp", - "ext/couchbase/third_party/asio/asio/include/asio/**/*.[hi]pp", - "ext/couchbase/third_party/cxx_function/cxx_function.hpp", - "ext/couchbase/third_party/expected/COPYING", - "ext/couchbase/third_party/expected/include/**/*", - "ext/couchbase/third_party/fmt/CMakeLists.txt", - "ext/couchbase/third_party/fmt/ChangeLog.rst", - "ext/couchbase/third_party/fmt/LICENSE.rst", - "ext/couchbase/third_party/fmt/README.rst", - "ext/couchbase/third_party/fmt/include/**/*", - "ext/couchbase/third_party/fmt/src/**/*", - "ext/couchbase/third_party/fmt/support/cmake/**/*", - "ext/couchbase/third_party/gsl/CMakeLists.txt", - "ext/couchbase/third_party/gsl/LICENSE*", - "ext/couchbase/third_party/gsl/ThirdPartyNotices.txt", - "ext/couchbase/third_party/gsl/cmake/*", - "ext/couchbase/third_party/gsl/include/**/*", - "ext/couchbase/third_party/hdr_histogram_c/CMakeLists.txt", - "ext/couchbase/third_party/hdr_histogram_c/COPYING.txt", - "ext/couchbase/third_party/hdr_histogram_c/LICENSE.txt", - "ext/couchbase/third_party/hdr_histogram_c/config.cmake.in", - "ext/couchbase/third_party/hdr_histogram_c/src/**/*", - "ext/couchbase/third_party/http_parser/LICENSE*", - "ext/couchbase/third_party/http_parser/http_parser.{c,h}", - "ext/couchbase/third_party/json/CMakeLists.txt", - "ext/couchbase/third_party/json/LICENSE*", - "ext/couchbase/third_party/json/external/PEGTL/.cmake/*", - "ext/couchbase/third_party/json/external/PEGTL/CMakeLists.txt", - "ext/couchbase/third_party/json/external/PEGTL/LICENSE*", - "ext/couchbase/third_party/json/external/PEGTL/include/**/*", - "ext/couchbase/third_party/json/include/**/*", - "ext/couchbase/third_party/jsonsl/*", - "ext/couchbase/third_party/snappy/CMakeLists.txt", - "ext/couchbase/third_party/snappy/COPYING", - "ext/couchbase/third_party/snappy/cmake/*", - "ext/couchbase/third_party/snappy/snappy-c.{h,cc}", - "ext/couchbase/third_party/snappy/snappy-internal.h", - "ext/couchbase/third_party/snappy/snappy-sinksource.{h,cc}", - "ext/couchbase/third_party/snappy/snappy-stubs-internal.{h,cc}", - "ext/couchbase/third_party/snappy/snappy-stubs-public.h.in", - "ext/couchbase/third_party/snappy/snappy.{h,cc}", - "ext/couchbase/third_party/spdlog/CMakeLists.txt", - "ext/couchbase/third_party/spdlog/LICENSE", - "ext/couchbase/third_party/spdlog/cmake/*", - "ext/couchbase/third_party/spdlog/include/**/*", - "ext/couchbase/third_party/spdlog/src/**/*", - "lib/**/*.rb", - ].select { |path| File.file?(path) } + spec.files = Dir.glob([ + "LICENSE.txt", + "README.md", + "ext/*.cxx", + "ext/*.hxx.in", + "ext/*.rb", + "ext/CMakeLists.txt", + "ext/cache/**/*", + "ext/couchbase/CMakeLists.txt", + "ext/couchbase/LICENSE.txt", + "ext/couchbase/cmake/*", + "ext/couchbase/core/**/*", + "ext/couchbase/couchbase/**/*", + "ext/couchbase/third_party/cxx_function/cxx_function.hpp", + "ext/couchbase/third_party/expected/COPYING", + "ext/couchbase/third_party/expected/include/**/*", + "ext/couchbase/third_party/jsonsl/*", + "lib/**/*.rb", + ], File::FNM_DOTMATCH).select { |path| File.file?(path) } spec.bindir = "exe" spec.executables = spec.files.grep(/^exe\//) { |f| File.basename(f) } spec.require_paths = ["lib"] diff --git a/ext/.idea/ext.iml b/ext/.idea/ext.iml index f08604bb..ca93eb12 100644 --- a/ext/.idea/ext.iml +++ b/ext/.idea/ext.iml @@ -1,2 +1,6 @@ - \ No newline at end of file + + + + \ No newline at end of file diff --git a/ext/.idea/vcs.xml b/ext/.idea/vcs.xml index a435eaf5..c10aafdc 100644 --- a/ext/.idea/vcs.xml +++ b/ext/.idea/vcs.xml @@ -3,18 +3,8 @@ - - - - - - - - - - \ No newline at end of file diff --git a/ext/CMakeLists.txt b/ext/CMakeLists.txt index a3cf7c30..fb7a3d41 100644 --- a/ext/CMakeLists.txt +++ b/ext/CMakeLists.txt @@ -1,14 +1,10 @@ cmake_minimum_required(VERSION 3.20) project(ruby_client LANGUAGES CXX) -if(WIN32) - add_definitions(-D_WIN32_WINNT=0x0601) -endif() - if(RUBY_HDR_DIR) set(RUBY_INCLUDE_DIR ${RUBY_HDR_DIR} ${RUBY_ARCH_HDR_DIR}) else() - find_package(Ruby 2.6.0 REQUIRED) + find_package(Ruby 3.0.0 REQUIRED) message(STATUS "RUBY_VERSION: ${RUBY_VERSION}") message(STATUS "RUBY_EXECUTABLE: ${RUBY_EXECUTABLE}") endif() diff --git a/ext/couchbase b/ext/couchbase index 23f08096..4747045c 160000 --- a/ext/couchbase +++ b/ext/couchbase @@ -1 +1 @@ -Subproject commit 23f080966d787b9a43b2899ca14e258a0dac8f1a +Subproject commit 4747045c2dca2cd855d9bbc4ba593c3a6ad19ab2 diff --git a/ext/couchbase.cxx b/ext/couchbase.cxx index 425d68cc..90059319 100644 --- a/ext/couchbase.cxx +++ b/ext/couchbase.cxx @@ -275,6 +275,13 @@ init_versions(VALUE mCouchbase) VALUE cb_BuildInfo = rb_hash_new(); rb_const_set(mCouchbase, rb_intern("BUILD_INFO"), cb_BuildInfo); +#if defined(HAVE_RUBY_VERSION_H) + rb_hash_aset( + cb_BuildInfo, + rb_id2sym(rb_intern("ruby_abi")), + rb_str_freeze(cb_str_new(fmt::format("{}.{}.{}", RUBY_API_VERSION_MAJOR, RUBY_API_VERSION_MINOR, RUBY_API_VERSION_TEENY)))); +#endif + rb_hash_aset(cb_BuildInfo, rb_id2sym(rb_intern("revision")), rb_str_freeze(rb_str_new_cstr(EXT_GIT_REVISION))); rb_hash_aset(cb_BuildInfo, rb_id2sym(rb_intern("ruby_librubyarg")), rb_str_freeze(rb_str_new_cstr(RUBY_LIBRUBYARG))); rb_hash_aset(cb_BuildInfo, rb_id2sym(rb_intern("ruby_include_dir")), rb_str_freeze(rb_str_new_cstr(RUBY_INCLUDE_DIR))); rb_hash_aset(cb_BuildInfo, rb_id2sym(rb_intern("ruby_library_dir")), rb_str_freeze(rb_str_new_cstr(RUBY_LIBRARY_DIR))); @@ -283,7 +290,8 @@ init_versions(VALUE mCouchbase) if (name == "version_major" || name == "version_minor" || name == "version_patch" || name == "version_build" || name == "__cplusplus" || name == "_MSC_VER" || name == "mozilla_ca_bundle_size") { rb_hash_aset(cb_CoreInfo, rb_id2sym(rb_intern(name.c_str())), INT2FIX(std::stoi(value))); - } else if (name == "snapshot" || name == "static_stdlib" || name == "static_openssl" || name == "mozilla_ca_bundle_embedded") { + } else if (name == "snapshot" || name == "static_stdlib" || name == "static_openssl" || name == "static_boringssl" || + name == "mozilla_ca_bundle_embedded") { rb_hash_aset(cb_CoreInfo, rb_id2sym(rb_intern(name.c_str())), value == "true" ? Qtrue : Qfalse); } else { rb_hash_aset(cb_CoreInfo, rb_id2sym(rb_intern(name.c_str())), rb_str_freeze(rb_str_new_cstr(value.c_str()))); diff --git a/ext/extconf.rb b/ext/extconf.rb index 2a448a7d..0a7a5ab2 100644 --- a/ext/extconf.rb +++ b/ext/extconf.rb @@ -21,8 +21,30 @@ SDK_VERSION = Couchbase::VERSION[:sdk] -def check_version(name) - executable = find_executable(name) +case RbConfig::CONFIG["target_os"] +when /mingw/ + require "ruby_installer/runtime" + # ridk install 1 + # ridk install 3 + # ridk exec pacman --sync --noconfirm mingw-w64-ucrt-x86_64-ninja mingw-w64-ucrt-x86_64-cmake mingw-w64-ucrt-x86_64-toolchain mingw-w64-ucrt-x86_64-go mingw-w64-ucrt-x86_64-nasm mingw-w64-ucrt-x86_64-ccache + ENV["CB_STATIC_BORINGSSL"] = "true" + ENV["CB_STATIC_STDLIB"] = "true" + ENV["GOROOT"] = File.join(RubyInstaller::Runtime.msys2_installation.msys_path, "ucrt64/lib/go") +else +end + +def which(name, extra_locations = []) + ENV.fetch("PATH", "") + .split(File::PATH_SEPARATOR) + .prepend(*extra_locations) + .select { |path| File.directory?(path) } + .map { |path| [path, name].join(File::SEPARATOR) + RbConfig::CONFIG["EXEEXT"] } + .find { |file| File.executable?(file) } +end + +def check_version(name, extra_locations = []) + executable = which(name, extra_locations) + puts "-- check #{name} (#{executable})" version = nil IO.popen([executable, "--version"]) do |io| version = io.read.split("\n").first @@ -30,7 +52,19 @@ def check_version(name) [executable, version] end -cmake, version = check_version("cmake") +cmake_extra_locations = [] +if RUBY_PLATFORM =~ /mswin|mingw/ + cmake_extra_locations = [ + 'C:\Program Files\CMake\bin', + 'C:\Program Files\Microsoft Visual Studio\2022\Professional\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin', + 'C:\Program Files\Microsoft Visual Studio\2019\Professional\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin', + ] + local_app_data = ENV.fetch("LOCALAPPDATA", "#{ENV.fetch("HOME")}\\AppData\\Local") + if File.directory?(local_app_data) + cmake_extra_locations.unshift("#{local_app_data}\\CMake\\bin") + end +end +cmake, version = check_version("cmake", cmake_extra_locations) if version[/cmake version (\d+\.\d+)/, 1].to_f < 3.15 cmake, version = check_version("cmake3") end @@ -40,7 +74,7 @@ def check_version(name) def sys(*cmd) puts "-- #{Dir.pwd}" puts "-- #{cmd.join(' ')}" - system(*cmd) + system(*cmd) || abort("failed to execute command: #{$?.inspect}\n#{cmd.join(' ')}") end build_type = ENV["DEBUG"] ? "Debug" : "Release" @@ -56,11 +90,32 @@ def sys(*cmd) "-DCOUCHBASE_CXX_CLIENT_BUILD_EXAMPLES=OFF", ] -revisions_path = File.join(__dir__, "revisions.rb") -eval(File.read(revisions_path)) if File.exist?(revisions_path) # rubocop:disable Security/Eval +extconf_include = File.expand_path("cache/extconf_include.rb", __dir__) +if File.exist?(extconf_include) + puts "-- include extra cmake options from #{extconf_include}" + eval(File.read(extconf_include)) # rubocop:disable Security/Eval +end + +if ENV["CB_STATIC"] || ENV["CB_STATIC_BORINGSSL"] + cmake_flags << "-DCOUCHBASE_CXX_CLIENT_STATIC_BORINGSSL=ON" +elsif ENV["CB_STATIC_OPENSSL"] + cmake_flags << "-DCOUCHBASE_CXX_CLIENT_STATIC_OPENSSL=ON" +end + +unless cmake_flags.grep(/BORINGSSL/) + case RbConfig::CONFIG["target_os"] + when /darwin/ + openssl_root = `brew --prefix openssl@1.1 2> /dev/null`.strip + openssl_root = `brew --prefix openssl@3 2> /dev/null`.strip if openssl_root.empty? + openssl_root = `brew --prefix openssl 2> /dev/null`.strip if openssl_root.empty? + cmake_flags << "-DOPENSSL_ROOT_DIR=#{openssl_root}" unless openssl_root.empty? + when /linux/ + openssl_root = ["/usr/lib64/openssl11", "/usr/include/openssl11"] + cmake_flags << "-DOPENSSL_ROOT_DIR=#{openssl_root.join(';')}" if openssl_root.all? { |path| File.directory?(path) } + end +end cmake_flags << "-DCOUCHBASE_CXX_CLIENT_STATIC_STDLIB=ON" if ENV["CB_STATIC"] || ENV["CB_STATIC_STDLIB"] -cmake_flags << "-DCOUCHBASE_CXX_CLIENT_STATIC_OPENSSL=ON" if ENV["CB_STATIC"] || ENV["CB_STATIC_OPENSSL"] cmake_flags << "-DENABLE_SANITIZER_ADDRESS=ON" if ENV["CB_ASAN"] cmake_flags << "-DENABLE_SANITIZER_LEAK=ON" if ENV["CB_LSAN"] cmake_flags << "-DENABLE_SANITIZER_MEMORY=ON" if ENV["CB_MSAN"] @@ -69,31 +124,21 @@ def sys(*cmd) cc = ENV["CB_CC"] cxx = ENV["CB_CXX"] +ar = ENV["CB_AR"] -case RbConfig::CONFIG["target_os"] -when /darwin/ - openssl_root = `brew --prefix openssl@1.1 2> /dev/null`.strip - openssl_root = `brew --prefix openssl@3 2> /dev/null`.strip if openssl_root.empty? - openssl_root = `brew --prefix openssl 2> /dev/null`.strip if openssl_root.empty? - cmake_flags << "-DOPENSSL_ROOT_DIR=#{openssl_root}" unless openssl_root.empty? -when /linux/ - openssl_root = ["/usr/lib64/openssl11", "/usr/include/openssl11"] - cmake_flags << "-DOPENSSL_ROOT_DIR=#{openssl_root.join(';')}" if openssl_root.all? { |path| File.directory?(path) } -when /mingw/ +if RbConfig::CONFIG["target_os"] =~ /mingw/ require "ruby_installer/runtime" RubyInstaller::Runtime.enable_dll_search_paths RubyInstaller::Runtime.enable_msys_apps cc = RbConfig::CONFIG["CC"] cxx = RbConfig::CONFIG["CXX"] - cmake_flags << "-G MSYS Makefiles" + cmake_flags << "-G Ninja" cmake_flags << "-DRUBY_LIBRUBY=#{File.basename(RbConfig::CONFIG["LIBRUBY_SO"], ".#{RbConfig::CONFIG["SOEXT"]}")}" - # ridk install 1 - # ridk install 3 - # ridk exec pacman -S --noconfirm mingw-w64-ucrt-x86_64-openssl end cmake_flags << "-DCMAKE_C_COMPILER=#{cc}" if cc cmake_flags << "-DCMAKE_CXX_COMPILER=#{cxx}" if cxx +cmake_flags << "-DCMAKE_AR=#{ar}" if ar project_path = File.expand_path(File.join(__dir__)) build_dir = ENV['CB_EXT_BUILD_DIR'] ||