From 497424a4bb7e0f43de204af7e9acd8e8ed722210 Mon Sep 17 00:00:00 2001 From: mattip Date: Wed, 7 Aug 2024 23:25:19 +0300 Subject: [PATCH 1/3] backport drop-quadmath-symbol PR --- .github/workflows/windows.yml | 5 ++- patches-windows/openblas-make-libs.patch | 15 +++++++ tools/build_openblas.sh | 50 ++++++++++++++++++------ 3 files changed, 56 insertions(+), 14 deletions(-) create mode 100644 patches-windows/openblas-make-libs.patch diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 88051fb..08ef972 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -45,13 +45,14 @@ jobs: echo "PLAT=i686" >> $env:GITHUB_ENV echo "WHEEL_PLAT=win32" >> $env:GITHUB_ENV echo "MSYSTEM=MINGW32" >> $env:GITHUB_ENV - echo "LDFLAGS=-static -static-libgcc" >> $env:GITHUB_ENV + # No ucrt on 32-bits, so use _snprintf_c instead + echo "LDFLAGS=-static -static-libgcc -Wl,--defsym,_quadmath_snprintf=__snprintf_c" >> $env:GITHUB_ENV echo "BUILD_BITS=32" >> $env:GITHUB_ENV } else { echo "PLAT=x86_64" >> $env:GITHUB_ENV echo "WHEEL_PLAT=win_amd64" >> $env:GITHUB_ENV echo "MSYSTEM=UCRT64" >> $env:GITHUB_ENV - echo "LDFLAGS=-lucrt -static -static-libgcc" >> $env:GITHUB_ENV + echo "LDFLAGS=-lucrt -static -static-libgcc -Wl,--defsym,quadmath_snprintf=snprintf" >> $env:GITHUB_ENV echo "BUILD_BITS=64" >> $env:GITHUB_ENV } if ( ${{ matrix.INTERFACE64 }} -eq "1" ) { diff --git a/patches-windows/openblas-make-libs.patch b/patches-windows/openblas-make-libs.patch new file mode 100644 index 0000000..1e4d4ae --- /dev/null +++ b/patches-windows/openblas-make-libs.patch @@ -0,0 +1,15 @@ +diff --git a/exports/Makefile b/exports/Makefile +index 668a4866e..956b51bf4 100644 +--- a/exports/Makefile ++++ b/exports/Makefile +@@ -126,8 +126,9 @@ dll : ../$(LIBDLLNAME) + ../$(LIBDLLNAME) : ../$(LIBNAME) $(LIBPREFIX).def dllinit.$(SUFFIX) + $(RANLIB) ../$(LIBNAME) + $(CC) $(CFLAGS) $(LDFLAGS) $(LIBPREFIX).def dllinit.$(SUFFIX) \ +- -shared -o ../$(LIBDLLNAME) -Wl,--out-implib,../$(IMPLIBNAME) \ ++ -shared -o ../$(LIBDLLNAME) -Wl,-gc-sections -Wl,-s -Wl,-Map,output.map \ + -Wl,--whole-archive ../$(LIBNAME) -Wl,--no-whole-archive $(FEXTRALIB) $(EXTRALIB) ++ dlltool -D $(LIBDLLNAME) -d $(LIBPREFIX).def -l ../$(LIBDLLNAME).a + + $(LIBPREFIX).def : $(GENSYM) + ./$(GENSYM) win2k $(ARCH) dummy $(EXPRECISION) $(NO_CBLAS) $(NO_LAPACK) $(NO_LAPACKE) $(NEED2UNDERSCORES) $(ONLY_CBLAS) "$(SYMBOLPREFIX)" "$(SYMBOLSUFFIX)" $(BUILD_LAPACK_DEPRECATED) $(BUILD_BFLOAT16) $(BUILD_SINGLE) $(BUILD_DOUBLE) $(BUILD_COMPLEX) $(BUILD_COMPLEX16) > $(@F) diff --git a/tools/build_openblas.sh b/tools/build_openblas.sh index 253c636..f999299 100644 --- a/tools/build_openblas.sh +++ b/tools/build_openblas.sh @@ -76,7 +76,7 @@ fflags="$fextra $cflags -frecursive -ffpe-summary=invalid,zero" # Set suffixed-ILP64 flags if [ "$if_bits" == "64" ]; then - SYMBOLSUFFIX="64_" + LIBNAMESUFFIX="64_" interface_flags="INTERFACE64=1 SYMBOLSUFFIX=64_ LIBNAMESUFFIX=64_" # We override FCOMMON_OPT, so we need to set default integer manually fflags="$fflags -fdefault-integer-8" @@ -90,8 +90,12 @@ interface_flags="$interface_flags SYMBOLPREFIX=scipy_ LIBNAMEPREFIX=scipy_ FIXED # Build name for output library from gcc version and OpenBLAS commit. GCC_TAG="gcc_$(gcc -dumpversion | tr .- _)" OPENBLAS_VERSION=$(git describe --tags --abbrev=8) + +# Patch OpenBLAS build to resolve all symbols and avoid linking +# with libquadmath +patch -p1 < ../patches-windows/openblas-make-libs.patch + # Build OpenBLAS -# Variable used in creating output libraries make BINARY=$build_bits DYNAMIC_ARCH=1 USE_THREAD=1 USE_OPENMP=0 \ NUM_THREADS=24 NO_WARMUP=1 NO_AFFINITY=1 CONSISTENT_FPCSR=1 \ BUILD_LAPACK_DEPRECATED=1 TARGET=PRESCOTT BUFFERSIZE=20\ @@ -100,8 +104,38 @@ make BINARY=$build_bits DYNAMIC_ARCH=1 USE_THREAD=1 USE_OPENMP=0 \ FCOMMON_OPT="$fflags" \ MAX_STACK_ALLOC=2048 \ $interface_flags + +# Make sure quadmath library is not statically linked in to the DLL by checking +# the output.map generated by the linker when using `-Wl,-gc-sections -Wl,-s` +# the map will have libname(o-filename) for each function pulled out of the +# library libname +# The file itself appears in the map, so look for "libquadmath.a(". Use '-A 3' +# to show a bit of context if any symbols appear (which should not happen) +set +e +grep -A 2 "libquadmath.a(" exports/output.map +case $? in + 0) + echo "link uses libquadmath.a when it should not" + exit -1 + ;; + 1) + if [ -f exports/output.map ]; then + echo "Good, verified no 'libquadmath' used when linking" + else + echo "error occurred" + exit -1 + fi + ;; + *) + echo "grep returned $?, error occurred" + exit -1 + ;; +esac +set -e + make PREFIX=$openblas_root/$build_bits $interface_flags install -DLL_BASENAME=libscipy_openblas${SYMBOLSUFFIX}${LIBNAMESUFFIX} +DLL_BASENAME=libscipy_openblas${LIBNAMESUFFIX} +cp -f *.dll.a $openblas_root/$build_bits/lib/${DLL_BASENAME}.dll.a # OpenBLAS does not build a symbol-suffixed static library on Windows: # do it ourselves. On 32-bit builds, the objcopy.def names need a '_' prefix @@ -143,16 +177,8 @@ else sed -e "s/^Cflags.*/\0 -DBLAS_SYMBOL_PREFIX=scipy_/" -i pkgconfig/scipy-openblas.pc fi popd -# Build template site.cfg for using this build -cat > ${build_bits}/site.cfg.template << EOF -[openblas${SYMBOLSUFFIX}] -libraries = $DLL_BASENAME -library_dirs = {openblas_root}\\${build_bits}\\lib -include_dirs = {openblas_root}\\${build_bits}\\include -EOF - ls $openblas_root/$build_bits/lib -zip_name="openblas${SYMBOLSUFFIX}-${OPENBLAS_VERSION}-${plat_tag}-${GCC_TAG}.zip" +zip_name="openblas${LIBNAMESUFFIX}-${OPENBLAS_VERSION}-${plat_tag}-${GCC_TAG}.zip" zip -r $zip_name $build_bits cp $zip_name ${builds_dir} From 193a199acd1db13574eb6fb649e5d03ccf159cbb Mon Sep 17 00:00:00 2001 From: mattip Date: Fri, 9 Aug 2024 08:53:14 +0300 Subject: [PATCH 2/3] update to 0.3.28 and fixup licenses --- .github/workflows/posix.yml | 2 +- .github/workflows/windows.yml | 2 +- .travis.yml | 2 +- pyproject.toml | 4 ++-- tools/LICENSE_linux.txt | 4 ++-- tools/LICENSE_osx.txt | 4 ++-- tools/LICENSE_win32.txt | 28 +++------------------------- 7 files changed, 12 insertions(+), 34 deletions(-) diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index 90776c8..3b73d23 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -11,7 +11,7 @@ on: env: REPO_DIR: OpenBLAS - OPENBLAS_COMMIT: "d11e7340" + OPENBLAS_COMMIT: "v0.3.28" MACOSX_DEPLOYMENT_TARGET: 10.9 jobs: diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 08ef972..713ea6d 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -8,7 +8,7 @@ on: workflow_dispatch: null env: - OPENBLAS_COMMIT: "d11e7340" + OPENBLAS_COMMIT: "v0.3.28" OPENBLAS_ROOT: "c:\\opt" # Preserve working directory for calls into bash # Without this, invoking bash will cd to the home directory diff --git a/.travis.yml b/.travis.yml index e7ebaf4..fca0be2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ env: global: # The archive that gets built has name from ``git describe`` on this # commit. - - OPENBLAS_COMMIT: "d11e7340" + - OPENBLAS_COMMIT: "v0.3.28" - REPO_DIR=OpenBLAS sudo: required diff --git a/pyproject.toml b/pyproject.toml index 5a3a84b..223eaa3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,8 +8,8 @@ build-backend = "setuptools.build_meta" [project] name = "scipy-openblas64" -# v0.3.27-350-gd11e7340 -version = "0.3.27.350.0" +# v0.3.28 +version = "0.3.28.0.0" requires-python = ">=3.7" description = "Provides OpenBLAS for python packaging" readme = "README.md" diff --git a/tools/LICENSE_linux.txt b/tools/LICENSE_linux.txt index 97596fe..a7fa072 100644 --- a/tools/LICENSE_linux.txt +++ b/tools/LICENSE_linux.txt @@ -5,7 +5,7 @@ This binary distribution of NumPy also bundles the following software: Name: OpenBLAS -Files: numpy.libs/libopenblas*.so +Files: numpy.libs/libscipy_openblas*.so Description: bundled as a dynamically linked library Availability: https://github.com/OpenMathLib/OpenBLAS/ License: BSD-3-Clause-Attribution @@ -41,7 +41,7 @@ License: BSD-3-Clause-Attribution Name: LAPACK -Files: numpy.libs/libopenblas*.so +Files: numpy.libs/libscipy_openblas*.so Description: bundled in OpenBLAS Availability: https://github.com/OpenMathLib/OpenBLAS/ License: BSD-3-Clause-Attribution diff --git a/tools/LICENSE_osx.txt b/tools/LICENSE_osx.txt index dd221a3..7e8796c 100644 --- a/tools/LICENSE_osx.txt +++ b/tools/LICENSE_osx.txt @@ -4,7 +4,7 @@ This binary distribution of NumPy also bundles the following software: Name: OpenBLAS -Files: numpy/.dylibs/libopenblas*.so +Files: numpy/.dylibs/libscipy_openblas*.so Description: bundled as a dynamically linked library Availability: https://github.com/OpenMathLib/OpenBLAS/ License: BSD-3-Clause-Attribution @@ -40,7 +40,7 @@ License: BSD-3-Clause-Attribution Name: LAPACK -Files: numpy/.dylibs/libopenblas*.so +Files: numpy/.dylibs/libscipy_openblas*.so Description: bundled in OpenBLAS Availability: https://github.com/OpenMathLib/OpenBLAS/ License: BSD-3-Clause-Attribution diff --git a/tools/LICENSE_win32.txt b/tools/LICENSE_win32.txt index ef954a6..6861141 100644 --- a/tools/LICENSE_win32.txt +++ b/tools/LICENSE_win32.txt @@ -5,7 +5,7 @@ This binary distribution of NumPy also bundles the following software: Name: OpenBLAS -Files: numpy.libs\libopenblas*.dll +Files: libs/libscipy_openblas*.dll Description: bundled as a dynamically linked library Availability: https://github.com/OpenMathLib/OpenBLAS/ License: BSD-3-Clause-Attribution @@ -41,7 +41,7 @@ License: BSD-3-Clause-Attribution Name: LAPACK -Files: numpy.libs\libopenblas*.dll +Files: libs/libscipy_openblas*.dll Description: bundled in OpenBLAS Availability: https://github.com/OpenMathLib/OpenBLAS/ License: BSD-3-Clause-Attribution @@ -96,7 +96,7 @@ License: BSD-3-Clause-Attribution Name: GCC runtime library -Files: numpy.libs\libopenblas*.dll +Files: libs/libscipy_openblas*.dll Description: statically linked to files compiled with gcc Availability: https://gcc.gnu.org/git/?p=gcc.git;a=tree;f=libgfortran License: GPL-3.0-with-GCC-exception @@ -878,25 +878,3 @@ may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read . - -Name: libquadmath -Files: numpy.libs\libopenb*.dll -Description: statically linked to files compiled with gcc -Availability: https://gcc.gnu.org/git/?p=gcc.git;a=tree;f=libquadmath -License: LGPL-2.1-or-later - - GCC Quad-Precision Math Library - Copyright (C) 2010-2019 Free Software Foundation, Inc. - Written by Francois-Xavier Coudert - - This file is part of the libquadmath library. - Libquadmath is free software; you can redistribute it and/or - modify it under the terms of the GNU Library General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - Libquadmath is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - https://www.gnu.org/licenses/old-licenses/lgpl-2.1.html From 547f146e4144ce4a897b0375f0d8dcf71ffd30df Mon Sep 17 00:00:00 2001 From: mattip Date: Fri, 9 Aug 2024 14:27:29 +0300 Subject: [PATCH 3/3] add '-fno-ident' to CFLAGS --- tools/build_openblas.sh | 1 + tools/build_steps.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/build_openblas.sh b/tools/build_openblas.sh index f999299..6ce4ac3 100644 --- a/tools/build_openblas.sh +++ b/tools/build_openblas.sh @@ -96,6 +96,7 @@ OPENBLAS_VERSION=$(git describe --tags --abbrev=8) patch -p1 < ../patches-windows/openblas-make-libs.patch # Build OpenBLAS +CFLAGS="$CFLAGS -fvisibility=protected -fno-ident" \ make BINARY=$build_bits DYNAMIC_ARCH=1 USE_THREAD=1 USE_OPENMP=0 \ NUM_THREADS=24 NO_WARMUP=1 NO_AFFINITY=1 CONSISTENT_FPCSR=1 \ BUILD_LAPACK_DEPRECATED=1 TARGET=PRESCOTT BUFFERSIZE=20\ diff --git a/tools/build_steps.sh b/tools/build_steps.sh index bd738d4..c04ee2a 100644 --- a/tools/build_steps.sh +++ b/tools/build_steps.sh @@ -174,13 +174,13 @@ function do_build_lib { patch_source echo start building if [ -v dynamic_list ]; then - CFLAGS="$CFLAGS -fvisibility=protected -Wno-uninitialized" \ + CFLAGS="$CFLAGS -fvisibility=protected -Wno-uninitialized -fno-ident" \ make BUFFERSIZE=20 DYNAMIC_ARCH=1 \ USE_OPENMP=0 NUM_THREADS=64 \ DYNAMIC_LIST="$dynamic_list" \ BINARY=$bitness $interface_flags $target_flags shared 2>&1 1>/dev/null else - CFLAGS="$CFLAGS -fvisibility=protected -Wno-uninitialized" \ + CFLAGS="$CFLAGS -fvisibility=protected -Wno-uninitialized -fno-ident" \ make BUFFERSIZE=20 DYNAMIC_ARCH=1 \ USE_OPENMP=0 NUM_THREADS=64 \ BINARY=$bitness $interface_flags $target_flags shared 2>&1 1>/dev/null