Build fiona 1.10a2 (#41)

* Build fiona 1.10a2 * Ensure pyparsing and shapely are installed for testing wheels * It's liblerc, not iliblerc. Also add some cirrus debugging * More debugging * More debugging * Even more debugging * More and more debugging * Local port for proj, more debugging * Add --depending flag to delocate and remove giflib * Remove brotli, rtmpdump brew packages Disable pcre2, postgresql, odbc for GDAL, idn2 for curl
sgillies · Apr 5, 2024 · 03c8427 · 03c8427
1 parent 09fc3d7
commit 03c8427
Show file tree

Hide file tree

Showing 19 changed files with 587 additions and 18 deletions.
diff --git a/.cirrus.yml b/.cirrus.yml
@@ -5,12 +5,12 @@ wheel_build_task:
   env:
     BUILD_PREFIX: /private/tmp/local
     REPO_DIR: Fiona
-    BUILD_COMMIT: master
+    BUILD_COMMIT: 1.10a2
     CIRRUS_CLONE_SUBMODULES: true
     PATH: /opt/homebrew/opt/[email protected]/bin:$PATH
     PLAT: "arm64"
     BUILD_DEPENDS: "setuptools cython~=3.0.2 wheel"
-    TEST_DEPENDS: "attrs pytest click mock boto3 packaging hypothesis wheel pytz fsspec aiohttp requests"
+    TEST_DEPENDS: "attrs pytest click mock boto3 packaging hypothesis wheel pytz fsspec aiohttp requests pyparsing shapely"
     MACOSX_DEPLOYMENT_TARGET: "11.0"
     matrix:
       - MB_PYTHON_VERSION: "3.12"
@@ -20,9 +20,9 @@ wheel_build_task:
       - MB_PYTHON_VERSION: "3.8"
   install_pre_requirements_script:
     - brew install pkg-config [email protected]
-    - ln -s python3 /opt/homebrew/opt/[email protected]/bin/python
+    - ln -s /opt/homebrew/opt/[email protected]/bin/python3.10 /opt/homebrew/opt/[email protected]/bin/python
   brew_remove_pkgs_script:
-    - brew remove -f --ignore-dependencies iliblerc webp zstd libtiff libxcb libxdcmp curl php lcms2 xmlto ghostscript lz4 openjpeg
+    - brew remove -f --ignore-dependencies liblerc webp zstd libtiff libxcb libxdcmp curl php lcms2 xmlto ghostscript lz4 openjpeg xz gettext giflib rtmpdump brotli
   build_wheel_script:
     - mkdir -p $BUILD_PREFIX/include
     - mkdir -p $BUILD_PREFIX/lib/pkgconfig
@@ -32,6 +32,11 @@ wheel_build_task:
     - source multibuild/common_utils.sh
     - source multibuild/travis_steps.sh
     - python3 -m pip install virtualenv
+    - which python3
+    - echo $PATH
+    - ls /opt/homebrew/opt/[email protected]/bin
+    - ls /private/tmp/local/bin
+    - which python
     - before_install
     - clean_code $REPO_DIR $BUILD_COMMIT
     - MACOSX_DEPLOYMENT_TARGET=11.0

diff --git a/.github/workflows/build.sh b/.github/workflows/build.sh
@@ -3,8 +3,9 @@ if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
   # webp, zstd, xz, libtiff cause a conflict with building webp and libtiff
   # curl from brew requires zstd, use system curl
   # if php is installed, brew tries to reinstall these after installing openblas
-  brew remove -f --ignore-dependencies iliblerc webp zstd libtiff libxcb libxdcmp curl php lcms2 xmlto ghostscript lz4 openjpeg
+  brew remove -f --ignore-dependencies liblerc webp zstd libtiff libxcb libxdcmp curl php lcms2 xmlto ghostscript lz4 openjpeg xz giflib rtmpdump brotli
   brew install pkg-config
+  brew deps --tree --installed
 fi
 
 if [[ "$MB_PYTHON_VERSION" == pypy3* ]]; then

diff --git a/.github/workflows/wheels.yaml b/.github/workflows/wheels.yaml
@@ -27,11 +27,11 @@ jobs:
           - os: "ubuntu-20.04"
             os-name: "focal"
     env:
-      BUILD_COMMIT: master
+      BUILD_COMMIT: 1.10a2
       PLAT: "${{ matrix.platform }}"
       MB_PYTHON_VERSION: "${{ matrix.python }}"
       MB_ML_VER: "2014"
-      TEST_DEPENDS: "attrs pytest click mock boto3 packaging hypothesis wheel pytz fsspec aiohttp requests"
+      TEST_DEPENDS: "attrs pytest click mock boto3 packaging hypothesis wheel pytz fsspec aiohttp requests pyparsing shapely"
       TRAVIS_OS_NAME: ${{ matrix.os-name }}
       MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }}
     steps:

diff --git a/.github/workflows/win-wheels.yaml b/.github/workflows/win-wheels.yaml
@@ -5,7 +5,7 @@ name: Windows wheels
 on: [push, pull_request, workflow_dispatch]
 
 env:
-  REF: master
+  REF: 1.4a2
 
 # cancel running jobs on new commit to PR
 concurrency:
@@ -48,7 +48,7 @@ jobs:
         env:
           VCPKG_DEFAULT_TRIPLET: x64-windows
         run: |
-          vcpkg install --feature-flags="versions,manifests" --x-manifest-root=${GITHUB_WORKSPACE} --x-install-root=$VCPKG_INSTALLATION_ROOT/installed
+          vcpkg install --feature-flags="versions,manifests" --x-manifest-root=${GITHUB_WORKSPACE} --x-install-root=$VCPKG_INSTALLATION_ROOT/installed --overlay-ports=ports
         shell: bash
 
       - name: Upload vcpkg build logs
@@ -130,8 +130,7 @@ jobs:
       - name: Install dependencies and Fiona wheel
         shell: bash
         run: |
-          python -m pip install attrs pytest click mock boto3 packaging hypothesis pytz fsspec aiohttp requests
-          python -m pip install "shapely ; python_version < '3.12'"
+          python -m pip install attrs pytest click mock boto3 packaging hypothesis pytz fsspec aiohttp requests pyparsing shapely
           python -m pip install --pre --no-deps --find-links wheelhouse/artifact fiona
           python -m pip install fiona
           python -m pip list

diff --git a/config.sh b/config.sh
@@ -110,7 +110,7 @@ function build_tiff {
     if [ -e tiff-stamp ]; then return; fi
     build_zlib
     build_jpeg
-    HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 ensure_xz
+    build_xz
     fetch_unpack https://download.osgeo.org/libtiff/tiff-${TIFF_VERSION}.tar.gz
     (cd tiff-${TIFF_VERSION} \
         && mv VERSION VERSION.txt \
@@ -177,7 +177,7 @@ function build_curl {
     CXXFLAGS="$CXXFLAGS -g -O2"
     build_openssl
     build_nghttp2
-    local flags="--prefix=$BUILD_PREFIX --with-nghttp2=$BUILD_PREFIX --with-libz --with-ssl"
+    local flags="--prefix=$BUILD_PREFIX --with-nghttp2=$BUILD_PREFIX --with-libz --with-ssl  --without-libidn2"
     #    fetch_unpack https://curl.haxx.se/download/curl-${CURL_VERSION}.tar.gz
     (cd curl-${CURL_VERSION} \
         && LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$BUILD_PREFIX/lib:$BUILD_PREFIX/lib64 ./configure $flags \
@@ -245,6 +245,9 @@ function build_gdal {
         -DGDAL_USE_SFCGAL=OFF \
         -DGDAL_USE_XERCESC=OFF \
         -DGDAL_USE_LIBXML2=OFF \
+        -DGDAL_USE_PCRE2=OFF \
+        -DGDAL_USE_POSTGRESQL=OFF \
+        -DGDAL_USE_ODBC=OFF \
         && $cmake --build . -j4 \
         && $cmake --install .)
     if [ -n "$IS_OSX" ]; then
@@ -313,7 +316,7 @@ function run_tests {
         apt-get install -y ca-certificates
     fi
     cp -R ../Fiona/tests ./tests
-    python -m pip install "shapely;python_version<'3.12'" $TEST_DEPENDS
+    python -m pip install $TEST_DEPENDS
     GDAL_ENABLE_DEPRECATED_DRIVER_GTM=YES python -m pytest -vv tests -k "not test_collection_zip_http and not test_mask_polygon_triangle and not test_show_versions and not test_append_or_driver_error and not [PCIDSK] and not cannot_append[FlatGeobuf]"
     fio --version
     fio env --formats
@@ -334,9 +337,10 @@ function build_wheel_cmd {
     if [ -n "$BUILD_DEPENDS" ]; then
         pip install $(pip_opts) $BUILD_DEPENDS
     fi
-    (cd $repo_dir && GDAL_VERSION=3.6.4 $cmd $wheelhouse)
+    (cd $repo_dir && GDAL_VERSION=3.8.4 $cmd $wheelhouse)
     if [ -n "$IS_OSX" ]; then
-	:
+        pip install delocate
+        delocate-listdeps --all --depending $wheelhouse/*.whl
     else  # manylinux
         pip install -I "git+https://github.com/sgillies/auditwheel.git#egg=auditwheel"
     fi

diff --git a/env_vars.sh b/env_vars.sh
@@ -18,4 +18,4 @@ export GDAL_CONFIG=/usr/local/bin/gdal-config
 export PACKAGE_DATA=1
 export PROJ_DATA=/usr/local/share/proj
 export AUDITWHEEL_EXTRA_LIB_NAME_TAG=fiona
-export TEST_DEPENDS="attrs pytest click mock boto3 packaging hypothesis pytz fsspec aiohttp requests"
+export TEST_DEPENDS="attrs pytest click mock boto3 packaging hypothesis pytz fsspec aiohttp requests pyparsing shapely"
diff --git a/ports/proj/fix-proj4-targets-cmake.patch b/ports/proj/fix-proj4-targets-cmake.patch
@@ -0,0 +1,48 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 7d736d129..2be8f8be9 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -129,7 +129,22 @@ if(NOT EXE_SQLITE3)
+   message(SEND_ERROR "sqlite3 binary not found!")
+ endif()
+
+-find_package(Sqlite3 REQUIRED)
++find_package(unofficial-sqlite3 CONFIG REQUIRED)
++if(unofficial-sqlite3_FOUND)
++  set(SQLITE3_FOUND true)
++  get_target_property(SQLITE3_INCLUDE_DIR unofficial::sqlite3::sqlite3 INTERFACE_INCLUDE_DIRECTORIES)
++  set(SQLITE3_LIBRARY unofficial::sqlite3::sqlite3)
++  # Extract version information from the header file
++  if(SQLITE3_INCLUDE_DIR)
++    file(STRINGS ${SQLITE3_INCLUDE_DIR}/sqlite3.h _ver_line
++         REGEX "^#define SQLITE_VERSION  *\"[0-9]+\\.[0-9]+\\.[0-9]+\""
++         LIMIT_COUNT 1)
++    string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]+"
++           SQLITE3_VERSION "${_ver_line}")
++    unset(_ver_line)
++  endif()
++endif()
++
+ if(NOT SQLITE3_FOUND)
+   message(SEND_ERROR "sqlite3 dependency not found!")
+ endif()
+@@ -151,6 +165,7 @@ if(ENABLE_TIFF)
+   find_package(TIFF REQUIRED)
+   if(TIFF_FOUND)
+     set(TIFF_ENABLED TRUE)
++    set(TIFF_LIBRARY TIFF::TIFF)
+   else()
+     message(SEND_ERROR
+       "libtiff dependency not found! Use ENABLE_TIFF=OFF to force it off")
+diff --git a/cmake/project-config.cmake.in b/cmake/project-config.cmake.in
+index 7d5579ae..0cf96252 100644
+--- a/cmake/project-config.cmake.in
++++ b/cmake/project-config.cmake.in
+@@ -27,6 +27,7 @@ if(@PROJECT_VARIANT_NAME@ STREQUAL "PROJ4")
+ endif()
+ cmake_policy(POP)
+
++find_dependency(unofficial-sqlite3 CONFIG)
+ if(DEFINED PROJ_CONFIG_FIND_TIFF_DEP)
+     find_dependency(TIFF)
+ endif()
diff --git a/ports/proj/fix-win-output-name.patch b/ports/proj/fix-win-output-name.patch
@@ -0,0 +1,13 @@
+diff --git a/cmake/ProjUtilities.cmake b/cmake/ProjUtilities.cmake
+index 6f43edb5c..1ed89d597 100644
+--- a/cmake/ProjUtilities.cmake
++++ b/cmake/ProjUtilities.cmake
+@@ -51,7 +51,7 @@ function(proj_target_output_name TARGET_NAME OUTPUT_NAME)
+
+   # On Windows, ABI version is specified using binary file name suffix.
+   # On Unix, suffix is empty and SOVERSION is used instead.
+-  if(WIN32)
++  if(0)
+     string(LENGTH "${${PROJECT_NAME}_ABI_VERSION}" abilen)
+     if(abilen GREATER 0)
+       set(SUFFIX "_${${PROJECT_NAME}_ABI_VERSION}")
diff --git a/ports/proj/portfile.cmake b/ports/proj/portfile.cmake
@@ -0,0 +1,72 @@
+vcpkg_from_github(
+    OUT_SOURCE_PATH SOURCE_PATH
+    REPO OSGeo/PROJ
+    REF "${VERSION}"
+    SHA512 45775e2b2a6b5bc490743c562155521a2ef48c5a8834cc96f88784aea785df10688f8962ae22fcac64d3b2f85378539ef1d3a082243cdc0ca3695ed8b9efa18b
+    HEAD_REF master
+    PATCHES
+        fix-win-output-name.patch
+        fix-proj4-targets-cmake.patch
+        remove_toolset_restriction.patch
+)
+
+vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS
+    FEATURES
+        net   ENABLE_CURL
+        tiff  ENABLE_TIFF
+        tools BUILD_APPS
+)
+
+vcpkg_list(SET TOOL_NAMES cct cs2cs geod gie invgeod invproj proj projinfo projsync)
+if("tools" IN_LIST FEATURES AND NOT "net" IN_LIST FEATURES)
+    set(BUILD_PROJSYNC OFF)
+    vcpkg_list(APPEND FEATURE_OPTIONS -DBUILD_PROJSYNC=${BUILD_PROJSYNC})
+    vcpkg_list(REMOVE_ITEM TOOL_NAMES projsync)
+endif()
+
+find_program(EXE_SQLITE3 NAMES "sqlite3" PATHS "${CURRENT_HOST_INSTALLED_DIR}/tools" NO_DEFAULT_PATH REQUIRED)
+
+vcpkg_cmake_configure(
+    SOURCE_PATH "${SOURCE_PATH}"
+    OPTIONS
+        ${FEATURE_OPTIONS}
+        -DNLOHMANN_JSON=external
+        -DBUILD_TESTING=OFF
+        "-DEXE_SQLITE3=${EXE_SQLITE3}"
+    OPTIONS_DEBUG
+        -DBUILD_APPS=OFF
+)
+
+vcpkg_cmake_install()
+
+if(VCPKG_LIBRARY_LINKAGE STREQUAL "static")
+    # Enforce consistency with src/lib_proj.cmake build time configuration.
+    vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/include/proj.h"
+        "#ifndef PROJ_DLL"
+        "#ifndef PROJ_DLL\n#  define PROJ_DLL\n#elif 0"
+    )
+endif()
+
+vcpkg_cmake_config_fixup(PACKAGE_NAME proj4 CONFIG_PATH lib/cmake/proj4 DO_NOT_DELETE_PARENT_CONFIG_PATH)
+vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake/proj)
+
+if ("tools" IN_LIST FEATURES)
+    vcpkg_copy_tools(TOOL_NAMES ${TOOL_NAMES} AUTO_CLEAN)
+endif ()
+
+file(REMOVE_RECURSE
+    "${CURRENT_PACKAGES_DIR}/debug/include"
+    "${CURRENT_PACKAGES_DIR}/debug/share"
+    "${CURRENT_PACKAGES_DIR}/share/doc"
+    "${CURRENT_PACKAGES_DIR}/share/man"
+)
+
+vcpkg_copy_pdbs()
+
+vcpkg_fixup_pkgconfig()
+if(NOT DEFINED VCPKG_BUILD_TYPE AND VCPKG_TARGET_IS_WINDOWS AND NOT VCPKG_TARGET_IS_MINGW)
+    vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/debug/lib/pkgconfig/proj.pc" " -lproj" " -lproj_d")
+endif()
+
+file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}")
+file(INSTALL "${SOURCE_PATH}/COPYING" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright)
diff --git a/ports/proj/remove_toolset_restriction.patch b/ports/proj/remove_toolset_restriction.patch
@@ -0,0 +1,19 @@
+diff --git a/cmake/project-config-version.cmake.in b/cmake/project-config-version.cmake.in
+index d9807b2c5..7ffe39364 100644
+--- a/cmake/project-config-version.cmake.in
++++ b/cmake/project-config-version.cmake.in
+@@ -28,14 +28,6 @@ elseif (NOT (APPLE OR (NOT DEFINED CMAKE_SIZEOF_VOID_P) OR
+   # since a multi-architecture library is built for that platform).
+   set (REASON "sizeof(*void) = @CMAKE_SIZEOF_VOID_P@")
+   set (PACKAGE_VERSION_UNSUITABLE TRUE)
+-elseif (MSVC AND NOT (
+-    # toolset version must be at least as great as @PROJECT_NAME@'s
+-    MSVC_TOOLSET_VERSION GREATER_EQUAL @MSVC_TOOLSET_VERSION@
+-    # and major versions must match
+-    AND MSVC_TOOLSET_MAJOR EQUAL @MSVC_TOOLSET_MAJOR@ ))
+-  # Reject if there's a mismatch in MSVC compiler versions
+-  set (REASON "MSVC_TOOLSET_VERSION = @MSVC_TOOLSET_VERSION@")
+-  set (PACKAGE_VERSION_UNSUITABLE TRUE)
+ elseif (PACKAGE_FIND_VERSION)
+   if (PACKAGE_FIND_VERSION VERSION_EQUAL PACKAGE_VERSION)
+     set (PACKAGE_VERSION_EXACT TRUE)
diff --git a/ports/proj/usage b/ports/proj/usage
@@ -0,0 +1,4 @@
+proj provides CMake targets:
+
+    find_package(PROJ CONFIG REQUIRED)
+    target_link_libraries(main PRIVATE PROJ::proj)
diff --git a/ports/proj/vcpkg.json b/ports/proj/vcpkg.json
@@ -0,0 +1,57 @@
+{
+  "name": "proj",
+  "version": "9.3.1",
+  "description": "PROJ library for cartographic projections",
+  "homepage": "https://proj.org/",
+  "license": "MIT",
+  "dependencies": [
+    "nlohmann-json",
+    {
+      "name": "sqlite3",
+      "default-features": false
+    },
+    {
+      "name": "sqlite3",
+      "host": true,
+      "default-features": false,
+      "features": [
+        "tool"
+      ]
+    },
+    {
+      "name": "vcpkg-cmake",
+      "host": true
+    },
+    {
+      "name": "vcpkg-cmake-config",
+      "host": true
+    }
+  ],
+  "default-features": [
+    "net",
+    "tiff"
+  ],
+  "features": {
+    "net": {
+      "description": "Enable network support",
+      "dependencies": [
+        "curl"
+      ]
+    },
+    "tiff": {
+      "description": "Enable TIFF support to read some grids",
+      "dependencies": [
+        {
+          "name": "tiff",
+          "default-features": false,
+          "features": [
+            "zip"
+          ]
+        }
+      ]
+    },
+    "tools": {
+      "description": "Build tools"
+    }
+  }
+}