Merge pull request #3 from nasa/cibuildwheel

Build python wheels with cibuildwheel. Will wait to publish to PyPI until pulling in python unit tests (separate ticket). For now, results are saved as GHA artifacts.
nasa · Jun 30, 2023 · 23d662c · 23d662c
2 parents 1a48470 + 808964d
commit 23d662c
Show file tree

Hide file tree

Showing 19 changed files with 426 additions and 377 deletions.
diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml
@@ -0,0 +1,120 @@
+name: Build
+
+on:
+  push:
+  workflow_dispatch:
+
+jobs:
+  build_wheels:
+    name: ${{ matrix.os }}-${{ github.workflow }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-latest
+            # tag '2023.04.15'
+            vcpkgCommitId: '501db0f17ef6df184fcdbfbe0f87cde2313b6ab1'
+            triplet: 'x64-linux-dynamic'
+          - os: macos-latest
+            # tag '2023.04.15'
+            vcpkgCommitId: '501db0f17ef6df184fcdbfbe0f87cde2313b6ab1'
+            triplet: 'x64-osx-dynamic'
+          - os: windows-latest
+            # tag '2023.04.15'
+            vcpkgCommitId: '501db0f17ef6df184fcdbfbe0f87cde2313b6ab1'
+            triplet: 'x64-windows'
+
+    env:
+      VCPKG_DEFAULT_TRIPLET: ${{ matrix.triplet }}
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      # Special case: cibuildwheel uses docker container for
+      # manylinux builds, so cache our vcpkg data on the
+      # host filesystem (mounted to /host by default)
+      - name: Point VCPKG_BINARY_SOURCES to host filesystem (Linux only)
+        if: runner.os == 'Linux'
+        run: |
+          echo "VCPKG_BINARY_SOURCES=/host/${RUNNER_WORKSPACE}/b/vcpkg_cache" >> $GITHUB_ENV
+        shell: bash
+
+      - name: run vcpkg
+        uses: lukka/run-vcpkg@main
+        id: run_vcpkg
+        with:
+          vcpkgDirectory: '${{ runner.workspace }}/b/vcpkg'
+          vcpkgGitCommitId: '${{ matrix.vcpkgCommitId }}'
+          # 'runVcpkgInstall=false' is especially important in the case of Linux -
+          # we need the cibuildwheel Docker container to actually build/install
+          # the dependencies, even if they are stored on the host filesystem
+          # (exposed via the /host mount point) and cached by the host runner.
+          runVcpkgInstall: false
+          vcpkgJsonGlob: '**/build-scripts/vcpkg.json'
+
+      - name: List $RUNNER_WORKSPACE before build
+        run: find $RUNNER_WORKSPACE
+        shell: bash
+
+      - name: Normalize paths (Windows only)
+        if: runner.os == 'Windows'
+        run: |
+          echo "UNIX_RUNNER_WORKSPACE=${RUNNER_WORKSPACE//\\//}" >> $GITHUB_ENV
+          echo "UNIX_GITHUB_WORKSPACE=${GITHUB_WORKSPACE//\\//}" >> $GITHUB_ENV
+        shell: bash
+
+      - name: Build wheels
+        uses: pypa/[email protected]
+        env:
+          CIBW_ENVIRONMENT_WINDOWS: >-
+            SKBUILD_CONFIGURE_OPTIONS='
+            -DCMAKE_TOOLCHAIN_FILE=${{ env.UNIX_RUNNER_WORKSPACE }}/b/vcpkg/scripts/buildsystems/vcpkg.cmake
+            -DVCPKG_TARGET_TRIPLET=${{ matrix.triplet }}
+            -DVCPKG_INSTALLED_DIR=${{ env.UNIX_RUNNER_WORKSPACE }}/b/vcpkg_installed
+            -DVCPKG_MANIFEST_DIR=${{ env.UNIX_GITHUB_WORKSPACE }}/build-scripts
+            -DUPSP_BUILD_TESTING=OFF -DUPSP_BUILD_APPLICATIONS=OFF -DUPSP_BUILD_PYBIND11=ON'
+          CIBW_ENVIRONMENT_MACOS: >-
+            SKBUILD_CONFIGURE_OPTIONS='
+            -DCMAKE_TOOLCHAIN_FILE=${{ runner.workspace }}/b/vcpkg/scripts/buildsystems/vcpkg.cmake
+            -DVCPKG_TARGET_TRIPLET=${{ matrix.triplet }}
+            -DVCPKG_INSTALLED_DIR=${{ runner.workspace }}/b/vcpkg_installed
+            -DVCPKG_MANIFEST_DIR=${{ github.workspace }}/build-scripts
+            -DUPSP_BUILD_TESTING=OFF -DUPSP_BUILD_APPLICATIONS=OFF -DUPSP_BUILD_PYBIND11=ON'
+          CIBW_ENVIRONMENT_LINUX: >-
+            SKBUILD_CONFIGURE_OPTIONS='
+            -DCMAKE_TOOLCHAIN_FILE=/host/${{ runner.workspace }}/b/vcpkg/scripts/buildsystems/vcpkg.cmake
+            -DVCPKG_TARGET_TRIPLET=${{ matrix.triplet }}
+            -DVCPKG_INSTALLED_DIR=/host/${{ runner.workspace }}/b/vcpkg_installed
+            -DVCPKG_MANIFEST_DIR=/host/${{ github.workspace }}/build-scripts
+            -DUPSP_BUILD_TESTING=OFF -DUPSP_BUILD_APPLICATIONS=OFF -DUPSP_BUILD_PYBIND11=ON'
+          CIBW_BEFORE_ALL_LINUX: "yum install -y perl-IPC-Cmd zip ninja-build"
+          CIBW_BEFORE_BUILD_WINDOWS: "pip install cmake"
+          CIBW_BUILD: "cp39-win_amd64 cp39-macosx_x86_64 cp39-manylinux_x86_64"
+          CIBW_SKIP: "*-win32 *-manylinux_i686"
+          CIBW_BUILD_VERBOSITY: 1
+        with:
+          package-dir: .
+          output-dir: wheelhouse
+          config-file: "{package}/pyproject.toml"
+
+      - uses: actions/upload-artifact@v3
+        with:
+          path: ./wheelhouse/*.whl
+
+  build_sdist:
+    name: Build SDist
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0  # for setuptools_scm
+
+    - name: Build SDist
+      run: pipx run build --sdist
+
+    - uses: actions/upload-artifact@v3
+      with:
+        path: dist/*.tar.gz
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -5,6 +5,12 @@ cmake_policy(SET CMP0074 NEW)
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
+
+if (MSVC)
+  # For MSVC to get things like "or", "and", ...
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /permissive-")
+endif()
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/Modules")
 
@@ -21,170 +27,20 @@ if (DEFINED SKBUILD)
 
   # TODO this is ugly but it works for now.
   #
-  # When the project is built by scikit-build for deployment as a
-  # Python package, libraries are dropped in the lib/ folder and there's
-  # multiple consumers of those libraries that need to resolve them at runtime:
-  #
-  # - Executables, installed to bin/, need to look in $ORIGIN/../lib
-  # - Python extension modules, installed to lib/pythonX/site-packages/upsp,
-  #   need to look in $ORIGIN/../../..
-  #
-  # Lastly, there's some build-tree-only artifacts (eg gtest unit tests),
-  # those need to look in their own build directory ($ORIGIN).
-  #
-  # We could do this on a per-target basis but I'd rather just leave
-  # the hack in one spot for now instead of being peppered all over
-  # this build file.
-  #
-  # A "better/cleaner" example of this setup can be found here (although
-  # I don't think it handles all the use cases I listed above, it's
-  # a small sample project):
-  #
-  # https://github.com/scikit-build/scikit-build-sample-projects/blob/master/projects/hello-cmake-package/CMakeLists.txt#L92
+  # Hard-code the appropriate library RPATHs, handling all possible "install" use cases.
+  # - Installing libs for use in "standard" C/C++ <prefix>/lib install
+  # - Installing libs for use in Python extensions
   #
+  # Exact rpath syntax depends on target arch.
   set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
-  set(CMAKE_INSTALL_RPATH "\$ORIGIN:\$ORIGIN/../lib:\$ORIGIN/../../../")
+  if (APPLE)
+    set(CMAKE_INSTALL_RPATH "@loader_path/;@loader_path/../../../")
+  elseif (LINUX)
+    set(CMAKE_INSTALL_RPATH "\$ORIGIN:\$ORIGIN/../lib:\$ORIGIN/../../../")
+  endif()
 endif()
 
-find_package(Eigen3 CONFIG REQUIRED)
-find_package(Boost REQUIRED)
-find_package(OpenCV CONFIG REQUIRED COMPONENTS core imgproc imgcodecs calib3d videoio tracking)
-find_package(OpenMP REQUIRED)
-find_package(MPI REQUIRED)
-find_package(Python3 COMPONENTS Interpreter Development)
-find_package(pybind11 REQUIRED)
-find_library(LIBRT rt)
-find_package(hdf5 CONFIG REQUIRED)
-find_package(IlmBase CONFIG REQUIRED)
-find_package(PythonExtensions)
-
-include_directories(cpp/include)
-
-pybind11_add_module(cine cpp/pybind11/cine.cpp)
-target_link_libraries(cine PRIVATE upsp_video)
-
-pybind11_add_module(raycast cpp/pybind11/raycast.cpp)
-target_link_libraries(raycast PRIVATE upsp_kdtree)
-
-add_library(
-    upsp_video
-    SHARED
-    cpp/lib/PSPVideo.cpp
-    cpp/lib/MrawReader.cpp
-    cpp/lib/CineReader.cpp
-)
-target_link_libraries(upsp_video PRIVATE opencv_core)
-target_link_libraries(upsp_video PRIVATE Eigen3::Eigen)
-
-add_library(
-    upsp_utils
-    SHARED
-    cpp/utils/asyncIO.c
-    cpp/utils/cv_extras.cpp
-    cpp/utils/file_io.cpp
-    cpp/utils/file_readers.cpp
-    cpp/utils/file_writers.cpp
-    cpp/utils/general_utils.cpp
-)
-target_link_libraries(upsp_utils PRIVATE opencv_core)
-target_link_libraries(upsp_utils PRIVATE ${LIBRT})
-target_link_libraries(upsp_utils PRIVATE upsp_video)
-target_link_libraries(upsp_utils PRIVATE Eigen3::Eigen)
-
-add_library(
-    upsp
-    SHARED
-    cpp/lib/cart3d.cpp
-    cpp/lib/image_processing.cpp
-    cpp/lib/logging.cpp
-    cpp/lib/kulites.cpp
-    cpp/lib/non_cv_upsp.cpp
-    cpp/lib/patches.cpp
-    cpp/lib/plot3d.cpp
-    cpp/lib/projection.cpp
-    cpp/lib/registration.cpp
-    cpp/lib/upsp_inputs.cpp
-    cpp/lib/CameraCal.cpp
-    cpp/lib/PSPHDF5.cpp
-    cpp/lib/P3DModel.cpp
-)
-target_link_libraries(upsp PUBLIC opencv_core opencv_imgproc opencv_tracking opencv_calib3d)
-target_link_libraries(upsp PRIVATE hdf5::hdf5_cpp-shared hdf5::hdf5_hl_cpp-shared)
-target_link_libraries(upsp PRIVATE Eigen3::Eigen)
-
-add_library(
-    upsp_kdtree
-    SHARED
-    cpp/utils/pspKdtree.c
-    cpp/utils/pspRT.cpp
-    cpp/utils/pspRTmem.cpp
-)
-target_link_libraries(upsp_kdtree IlmBase::Imath)
-
-function(upsp_add_executable NAME)
-    add_executable(${NAME} ${ARGN})
-    target_link_libraries(${NAME} upsp)
-    target_link_libraries(${NAME} upsp_video)
-    target_link_libraries(${NAME} upsp_utils)
-    target_link_libraries(${NAME} upsp_kdtree)
-    target_link_libraries(${NAME} opencv_core)
-    target_link_libraries(${NAME} opencv_videoio)
-    target_link_libraries(${NAME} opencv_imgproc)
-    target_link_libraries(${NAME} OpenMP::OpenMP_CXX)
-    target_link_libraries(${NAME} hdf5::hdf5_cpp-shared hdf5::hdf5_hl_cpp-shared)
-    target_link_libraries(${NAME} Eigen3::Eigen)
-endfunction()
-
-upsp_add_executable(psp_process cpp/exec/psp_process.cpp)
-target_link_libraries(psp_process MPI::MPI_CXX)
-target_link_libraries(psp_process OpenMP::OpenMP_CXX)
-target_link_libraries(psp_process hdf5::hdf5_cpp-shared hdf5::hdf5_hl_cpp-shared)
-
-add_executable(xyz_scalar_to_tbl cpp/exec/xyz_scalar_to_tbl.cpp)
-add_executable(xyz_scalar_to_tbl_delta cpp/exec/xyz_scalar_to_tbl_delta.cpp)
-
-upsp_add_executable(upsp-extract-frames cpp/exec/upsp_extract_frames.cpp)
-
-add_executable(add_field cpp/exec/add_field.cpp cpp/utils/asyncIO.c)
-target_link_libraries(add_field ${LIBRT})
-target_link_libraries(add_field hdf5::hdf5_cpp-shared hdf5::hdf5_hl_cpp-shared)
-
-add_executable(upsp_matrix_transpose cpp/exec/upsp_matrix_transpose.cpp)
-target_link_libraries(upsp_matrix_transpose MPI::MPI_CXX)
-target_link_libraries(upsp_matrix_transpose OpenMP::OpenMP_CXX)
-
-find_package(GTest CONFIG REQUIRED)
-upsp_add_executable(
-    run_tests
-    cpp/test/run_tests.cpp
-    cpp/test/test_filtering.cpp
-    cpp/test/test_general_utils.cpp
-    cpp/test/test_grid_utils.cpp
-    cpp/test/test_grids.cpp
-    cpp/test/test_models.cpp
-    cpp/test/test_mraw.cpp
-    cpp/test/test_p3dmodel.cpp
-    cpp/test/test_plot3d.cpp
-    cpp/test/test_psphdf5.cpp
-    cpp/test/test_trimodel.cpp
-)
-target_link_libraries(run_tests GTest::gtest GTest::gtest_main)
-target_link_libraries(run_tests OpenMP::OpenMP_CXX)
-target_link_libraries(run_tests hdf5::hdf5_cpp-shared hdf5::hdf5_hl_cpp-shared)
-
-install(
-    TARGETS
-    add_field
-    psp_process
-    upsp
-    upsp-extract-frames
-    upsp_kdtree
-    upsp_utils
-    upsp_video
-    xyz_scalar_to_tbl
-    LIBRARY DESTINATION lib
-    RUNTIME DESTINATION bin
-)
+add_subdirectory(cpp)
 
 install(
     PROGRAMS
@@ -196,16 +52,3 @@ install(
     DESTINATION
     bin
 )
-
-# TODO I think this works in a fairly brittle fashion right now.
-# It just so happens that the cmake_install_dir staging directory
-# created by scikit-build has a "python/upsp" folder mimicking the
-# source tree layout... so these extensions are just "dropped" into the
-# right spot. There's probably a more robust way to specify their
-# install location, probably some CMAKE_ variable I'm ignorant of.
-install(
-    TARGETS
-    cine
-    raycast
-    LIBRARY DESTINATION python/upsp
-)
diff --git a/build-scripts/vcpkg-setup-linux.sh b/build-scripts/vcpkg-setup-linux.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+VCPKG_ROOT=/opt/vcpkg
+VCPKG_TRIPLET="x64-linux-dynamic"
+
+echo "Settings up vcpkg in '$VCPKG_ROOT' ..."
+
+yum install -y \
+    perl-IPC-Cmd \
+    zip
+
+if [ ! -e "$VCPKG_ROOT" ]; then
+    git clone \
+        --depth 1 \
+        --branch "2023.04.15" \
+        https://github.com/Microsoft/vcpkg.git \
+        $VCPKG_ROOT
+
+    pushd .
+    cd $VCPKG_ROOT
+    ./bootstrap-vcpkg.sh -disableMetrics
+    popd
+fi
+
+pushd .
+cd $VCPKG_ROOT
+./vcpkg install --triplet=$VCPKG_TRIPLET \
+  imath[core] \
+  pybind11[core]
+
+popd
diff --git a/build-scripts/vcpkg-setup-macos.sh b/build-scripts/vcpkg-setup-macos.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+VCPKG_ROOT=$PWD/vcpkg
+VCPKG_TRIPLET="x64-osx-dynamic"
+
+echo "Settings up vcpkg in '$VCPKG_ROOT' ..."
+
+if [ ! -e "$VCPKG_ROOT" ]; then
+    git clone \
+        --depth 1 \
+        --branch "2023.04.15" \
+        https://github.com/Microsoft/vcpkg.git \
+        $VCPKG_ROOT
+
+    pushd .
+    cd $VCPKG_ROOT
+    ./bootstrap-vcpkg.sh -disableMetrics
+    popd
+fi
+
+pushd .
+cd $VCPKG_ROOT
+./vcpkg install --triplet=$VCPKG_TRIPLET \
+  imath[core] \
+  pybind11[core]
+
+popd
diff --git a/build-scripts/vcpkg.json b/build-scripts/vcpkg.json
@@ -0,0 +1,7 @@
+{
+  "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg.schema.json",
+  "dependencies": [
+    "imath",
+    "pybind11"
+  ]
+}