From d85b9a18c49ad8f5cd05c1fb410a12b05c985170 Mon Sep 17 00:00:00 2001 From: Karl Marrett Date: Mon, 17 May 2021 10:45:39 -0700 Subject: [PATCH 01/11] feat: nix flake packaged --- default.nix | 4 ++++ flake.lock | 27 +++++++++++++++++++++++++++ flake.nix | 13 +++++++++++++ openvdb.nix | 37 +++++++++++++++++++++++++++++++++++++ shell.nix | 4 ++++ 5 files changed, 85 insertions(+) create mode 100644 default.nix create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 openvdb.nix create mode 100644 shell.nix diff --git a/default.nix b/default.nix new file mode 100644 index 0000000000..1ef8a22a59 --- /dev/null +++ b/default.nix @@ -0,0 +1,4 @@ +(import (fetchTarball + "https://github.com/edolstra/flake-compat/archive/master.tar.gz") { + src = builtins.fetchGit ./.; + }).defaultNix diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000000..c3a019fda8 --- /dev/null +++ b/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1609254719, + "narHash": "sha256-XB3391z8r81Bt8NfQzB+tYdTWoraY0QVdXM21JhfW+Y=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "733e537a8ad76fd355b6f501127f7d0eb8861775", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "733e537a8ad76fd355b6f501127f7d0eb8861775", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000000..e8b0165b17 --- /dev/null +++ b/flake.nix @@ -0,0 +1,13 @@ +{ + description = "openvdb"; + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/733e537a8ad76fd355b6f501127f7d0eb8861775"; + }; + + outputs = inputs: { + + defaultPackage.x86_64-linux = import ./openvdb.nix { + nixpkgs = inputs.nixpkgs; + }; + }; +} diff --git a/openvdb.nix b/openvdb.nix new file mode 100644 index 0000000000..ed628c3a3e --- /dev/null +++ b/openvdb.nix @@ -0,0 +1,37 @@ +{ nixpkgs, ... }: +let + pkgs = import nixpkgs { system = "x86_64-linux"; }; +in + with pkgs; + stdenv.mkDerivation { + name = "openvdb"; + version = "8.0.1"; + + # https://nixos.org/nix/manual/#builtin-filterSource + src = builtins.filterSource + (path: type: lib.cleanSourceFilter path type + && baseNameOf path != "doc/*" + && baseNameOf path != "openvdb_houdini/*" + && baseNameOf path != "openvdb_maya/*" + && baseNameOf path != "pendingchanges/*" + && baseNameOf path != "tsc/*") ./.; + + cmakeFlags =["-DOPENVDB_BUILD_VDB_VIEW=ON -DOPENVDB_BUILD_VDB_RENDER=ON -DOPENVDB_BUILD_VDB_LOD=ON"]; + + enableParallelBuilding = true; + nativeBuildInputs = [ cmake pkg-config ]; + + # required dependencies for downstream development + propagatedBuildInputs = [ + openexr + tbb + c-blosc + ]; + + buildInputs = [ + unzip boost jemalloc ilmbase + # for the optional VDB_VIEW binary opengl related dependencies: + libGL glfw3 x11 libGLU xorg.libXdmcp + ]; + + } diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000000..9a25fabdb8 --- /dev/null +++ b/shell.nix @@ -0,0 +1,4 @@ +(import (fetchTarball + "https://github.com/edolstra/flake-compat/archive/master.tar.gz") { + src = builtins.fetchGit ./.; + }).shellNix From dfa86c65e37a726891cef5634c40c8bff56f053e Mon Sep 17 00:00:00 2001 From: Karl Marrett Date: Sun, 7 Nov 2021 10:15:14 -0800 Subject: [PATCH 02/11] ci: upgrade boost version to 1.75 --- openvdb.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openvdb.nix b/openvdb.nix index ed628c3a3e..84c98cd083 100644 --- a/openvdb.nix +++ b/openvdb.nix @@ -29,7 +29,7 @@ in ]; buildInputs = [ - unzip boost jemalloc ilmbase + unzip boost175 jemalloc ilmbase # for the optional VDB_VIEW binary opengl related dependencies: libGL glfw3 x11 libGLU xorg.libXdmcp ]; From ff011bddf9b6802c19e17a7a4a1f50c26caec887 Mon Sep 17 00:00:00 2001 From: Karl Marrett Date: Sun, 7 Nov 2021 11:14:03 -0800 Subject: [PATCH 03/11] ci: upgrade tbb version to 2021.4 --- openvdb.nix | 5 +++- overlay.nix | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 overlay.nix diff --git a/openvdb.nix b/openvdb.nix index 84c98cd083..0f89cc1f15 100644 --- a/openvdb.nix +++ b/openvdb.nix @@ -1,6 +1,9 @@ { nixpkgs, ... }: let - pkgs = import nixpkgs { system = "x86_64-linux"; }; + pkgs = import nixpkgs { + system = "x86_64-linux"; + overlays = [ (import ./overlay.nix) ]; + }; in with pkgs; stdenv.mkDerivation { diff --git a/overlay.nix b/overlay.nix new file mode 100644 index 0000000000..956b7c502f --- /dev/null +++ b/overlay.nix @@ -0,0 +1,80 @@ +final: prev: { + tbb = prev.stdenv.mkDerivation { + name="tbb"; + version = "2021.4"; + + src = prev.fetchFromGitHub { + #owner = "AcademySoftwareFoundation"; + owner = "oneapi-src"; + repo = "oneTBB"; + rev = "v${version}"; + sha256 = "e82d250338d3253b862c96715637870e7cfe408f"; + }; + + patches = [ + # Fixes build with Musl. + (fetchurl { + url = "https://github.com/openembedded/meta-openembedded/raw/39185eb1d1615e919e3ae14ae63b8ed7d3e5d83f/meta-oe/recipes-support/tbb/tbb/GLIBC-PREREQ-is-not-defined-on-musl.patch"; + sha256 = "gUfXQ9OZQ82qD6brgauBCsKdjLvyHafMc18B+KxZoYs="; + }) + + # Fixes build with Musl. + (fetchurl { + url = "https://github.com/openembedded/meta-openembedded/raw/39185eb1d1615e919e3ae14ae63b8ed7d3e5d83f/meta-oe/recipes-support/tbb/tbb/0001-mallinfo-is-glibc-specific-API-mark-it-so.patch"; + sha256 = "fhorfqO1hHKZ61uq+yTR7eQ8KYdyLwpM3K7WpwJpV74="; + }) + ]; + + nativeBuildInputs = lib.optionals stdenv.isDarwin [ + fixDarwinDylibNames + ]; + + makeFlags = lib.optionals stdenv.cc.isClang [ + "compiler=clang" + ]; + + enableParallelBuilding = true; + + installPhase = '' + runHook preInstall + mkdir -p $out/lib + cp "build/"*release*"/"*${stdenv.hostPlatform.extensions.sharedLibrary}* $out/lib/ + mv include $out/ + rm $out/include/index.html + runHook postInstall + ''; + + postInstall = let + pcTemplate = fetchurl { + url = "https://github.com/oneapi-src/oneTBB/raw/master/integration/pkg-config/tbb.pc.in"; + sha256 = "2pCad9txSpNbzac0vp/VY3x7HNySaYkbH3Rx8LK53pI="; + }; + in '' + # Generate pkg-config file based on upstream template. + # It should not be necessary with tbb after 2021.2. + mkdir -p "$out/lib/pkgconfig" + substitute "${pcTemplate}" "$out/lib/pkgconfig/tbb.pc" \ + --subst-var-by CMAKE_INSTALL_PREFIX "$out" \ + --subst-var-by CMAKE_INSTALL_LIBDIR "lib" \ + --subst-var-by CMAKE_INSTALL_INCLUDEDIR "include" \ + --subst-var-by TBB_VERSION "${version}" \ + --subst-var-by TBB_LIB_NAME "tbb" + ''; + + meta = with lib; { + description = "Intel Thread Building Blocks C++ Library"; + homepage = "http://threadingbuildingblocks.org/"; + license = licenses.asl20; + longDescription = '' + Intel Threading Building Blocks offers a rich and complete approach to + expressing parallelism in a C++ program. It is a library that helps you + take advantage of multi-core processor performance without having to be a + threading expert. Intel TBB is not just a threads-replacement library. It + represents a higher-level, task-based parallelism that abstracts platform + details and threading mechanisms for scalability and performance. + ''; + platforms = platforms.unix; + maintainers = with maintainers; [ thoughtpolice dizfer ]; + }; +} +} From cd4a38c1b7c091e8ea76949821a8e1bf25adbb5f Mon Sep 17 00:00:00 2001 From: Karl Marrett Date: Tue, 9 Nov 2021 05:06:22 -0800 Subject: [PATCH 04/11] ci: overlay tbb with bumped version taking previous nixpkgs specification --- openvdb.nix | 2 +- overlay.nix | 65 +++++++++++++++++++++-------------------------------- 2 files changed, 27 insertions(+), 40 deletions(-) diff --git a/openvdb.nix b/openvdb.nix index 0f89cc1f15..7aa3b2c73b 100644 --- a/openvdb.nix +++ b/openvdb.nix @@ -31,7 +31,7 @@ in c-blosc ]; - buildInputs = [ + buildInputs = [ unzip boost175 jemalloc ilmbase # for the optional VDB_VIEW binary opengl related dependencies: libGL glfw3 x11 libGLU xorg.libXdmcp diff --git a/overlay.nix b/overlay.nix index 956b7c502f..e7079a611c 100644 --- a/overlay.nix +++ b/overlay.nix @@ -4,51 +4,38 @@ final: prev: { version = "2021.4"; src = prev.fetchFromGitHub { - #owner = "AcademySoftwareFoundation"; owner = "oneapi-src"; repo = "oneTBB"; - rev = "v${version}"; - sha256 = "e82d250338d3253b862c96715637870e7cfe408f"; + rev = "v2021.4.0"; + sha256 = "eJ/NQ1XkWWlioBu05zbtZ/EwVxCAQzz5pkkKgN4RB0Y="; }; - patches = [ - # Fixes build with Musl. - (fetchurl { - url = "https://github.com/openembedded/meta-openembedded/raw/39185eb1d1615e919e3ae14ae63b8ed7d3e5d83f/meta-oe/recipes-support/tbb/tbb/GLIBC-PREREQ-is-not-defined-on-musl.patch"; - sha256 = "gUfXQ9OZQ82qD6brgauBCsKdjLvyHafMc18B+KxZoYs="; - }) + nativeBuildInputs = prev.lib.optionals prev.stdenv.isDarwin [ + prev.fixDarwinDylibNames + ]; - # Fixes build with Musl. - (fetchurl { - url = "https://github.com/openembedded/meta-openembedded/raw/39185eb1d1615e919e3ae14ae63b8ed7d3e5d83f/meta-oe/recipes-support/tbb/tbb/0001-mallinfo-is-glibc-specific-API-mark-it-so.patch"; - sha256 = "fhorfqO1hHKZ61uq+yTR7eQ8KYdyLwpM3K7WpwJpV74="; - }) - ]; + buildInputs = [ ]; - nativeBuildInputs = lib.optionals stdenv.isDarwin [ - fixDarwinDylibNames - ]; + makeFlags = prev.lib.optionals prev.stdenv.cc.isClang [ + "compiler=clang" + ]; - makeFlags = lib.optionals stdenv.cc.isClang [ - "compiler=clang" - ]; + enableParallelBuilding = true; - enableParallelBuilding = true; - - installPhase = '' - runHook preInstall - mkdir -p $out/lib - cp "build/"*release*"/"*${stdenv.hostPlatform.extensions.sharedLibrary}* $out/lib/ - mv include $out/ - rm $out/include/index.html - runHook postInstall - ''; + installPhase = '' + runHook preInstall + mkdir -p $out/lib + cp "build/"*release*"/"*${prev.stdenv.hostPlatform.extensions.sharedLibrary}* $out/lib/ + mv include $out/ + rm $out/include/index.html + runHook postInstall + ''; - postInstall = let - pcTemplate = fetchurl { - url = "https://github.com/oneapi-src/oneTBB/raw/master/integration/pkg-config/tbb.pc.in"; - sha256 = "2pCad9txSpNbzac0vp/VY3x7HNySaYkbH3Rx8LK53pI="; - }; + postInstall = let + pcTemplate = prev.fetchurl { + url = "https://github.com/oneapi-src/oneTBB/raw/master/integration/pkg-config/tbb.pc.in"; + sha256 = "2pCad9txSpNbzac0vp/VY3x7HNySaYkbH3Rx8LK53pI="; + }; in '' # Generate pkg-config file based on upstream template. # It should not be necessary with tbb after 2021.2. @@ -57,11 +44,11 @@ final: prev: { --subst-var-by CMAKE_INSTALL_PREFIX "$out" \ --subst-var-by CMAKE_INSTALL_LIBDIR "lib" \ --subst-var-by CMAKE_INSTALL_INCLUDEDIR "include" \ - --subst-var-by TBB_VERSION "${version}" \ + --subst-var-by TBB_VERSION 2021.4 \ --subst-var-by TBB_LIB_NAME "tbb" ''; - meta = with lib; { + meta = with prev.lib; { description = "Intel Thread Building Blocks C++ Library"; homepage = "http://threadingbuildingblocks.org/"; license = licenses.asl20; @@ -76,5 +63,5 @@ final: prev: { platforms = platforms.unix; maintainers = with maintainers; [ thoughtpolice dizfer ]; }; -} +}; } From f110ffbf9e56f36d23ae1fcd3234683180d6de4c Mon Sep 17 00:00:00 2001 From: Karl Marrett Date: Tue, 9 Nov 2021 05:27:50 -0800 Subject: [PATCH 05/11] ci: remove vdb_lod and vdb_render builds --- openvdb.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openvdb.nix b/openvdb.nix index 7aa3b2c73b..213ef97877 100644 --- a/openvdb.nix +++ b/openvdb.nix @@ -19,7 +19,7 @@ in && baseNameOf path != "pendingchanges/*" && baseNameOf path != "tsc/*") ./.; - cmakeFlags =["-DOPENVDB_BUILD_VDB_VIEW=ON -DOPENVDB_BUILD_VDB_RENDER=ON -DOPENVDB_BUILD_VDB_LOD=ON"]; + cmakeFlags =["-DOPENVDB_BUILD_VDB_VIEW=ON"]; enableParallelBuilding = true; nativeBuildInputs = [ cmake pkg-config ]; From e130a2ab3d213909045b2739007759fe1e94ef7e Mon Sep 17 00:00:00 2001 From: Karl Marrett Date: Tue, 9 Nov 2021 05:31:13 -0800 Subject: [PATCH 06/11] ci: overlay tbb modern nix cmake pkg-config definition --- overlay.nix | 64 +++++++++++++++-------------------------------------- 1 file changed, 18 insertions(+), 46 deletions(-) diff --git a/overlay.nix b/overlay.nix index e7079a611c..be6a9bea9d 100644 --- a/overlay.nix +++ b/overlay.nix @@ -10,58 +10,30 @@ final: prev: { sha256 = "eJ/NQ1XkWWlioBu05zbtZ/EwVxCAQzz5pkkKgN4RB0Y="; }; - nativeBuildInputs = prev.lib.optionals prev.stdenv.isDarwin [ - prev.fixDarwinDylibNames - ]; - - buildInputs = [ ]; + nativeBuildInputs = [ prev.pkgs.cmake prev.pkgs.pkg-config ]; makeFlags = prev.lib.optionals prev.stdenv.cc.isClang [ "compiler=clang" ]; - enableParallelBuilding = true; - - installPhase = '' - runHook preInstall - mkdir -p $out/lib - cp "build/"*release*"/"*${prev.stdenv.hostPlatform.extensions.sharedLibrary}* $out/lib/ - mv include $out/ - rm $out/include/index.html - runHook postInstall - ''; + cmakeFlags =["-DTBB_TEST=OFF"]; - postInstall = let - pcTemplate = prev.fetchurl { - url = "https://github.com/oneapi-src/oneTBB/raw/master/integration/pkg-config/tbb.pc.in"; - sha256 = "2pCad9txSpNbzac0vp/VY3x7HNySaYkbH3Rx8LK53pI="; - }; - in '' - # Generate pkg-config file based on upstream template. - # It should not be necessary with tbb after 2021.2. - mkdir -p "$out/lib/pkgconfig" - substitute "${pcTemplate}" "$out/lib/pkgconfig/tbb.pc" \ - --subst-var-by CMAKE_INSTALL_PREFIX "$out" \ - --subst-var-by CMAKE_INSTALL_LIBDIR "lib" \ - --subst-var-by CMAKE_INSTALL_INCLUDEDIR "include" \ - --subst-var-by TBB_VERSION 2021.4 \ - --subst-var-by TBB_LIB_NAME "tbb" - ''; + enableParallelBuilding = true; - meta = with prev.lib; { - description = "Intel Thread Building Blocks C++ Library"; - homepage = "http://threadingbuildingblocks.org/"; - license = licenses.asl20; - longDescription = '' - Intel Threading Building Blocks offers a rich and complete approach to - expressing parallelism in a C++ program. It is a library that helps you - take advantage of multi-core processor performance without having to be a - threading expert. Intel TBB is not just a threads-replacement library. It - represents a higher-level, task-based parallelism that abstracts platform - details and threading mechanisms for scalability and performance. - ''; - platforms = platforms.unix; - maintainers = with maintainers; [ thoughtpolice dizfer ]; + meta = with prev.lib; { + description = "Intel Thread Building Blocks C++ Library"; + homepage = "http://threadingbuildingblocks.org/"; + license = licenses.asl20; + longDescription = '' + Intel Threading Building Blocks offers a rich and complete approach to + expressing parallelism in a C++ program. It is a library that helps you + take advantage of multi-core processor performance without having to be a + threading expert. Intel TBB is not just a threads-replacement library. It + represents a higher-level, task-based parallelism that abstracts platform + details and threading mechanisms for scalability and performance. + ''; + platforms = platforms.unix; + maintainers = with maintainers; [ thoughtpolice dizfer ]; + }; }; -}; } From 9354d56306570c9a1ba67ce7abc365d423fae04a Mon Sep 17 00:00:00 2001 From: Karl Marrett Date: Tue, 9 Nov 2021 05:46:30 -0800 Subject: [PATCH 07/11] ci: upgrade openvdb reported version to 9.0.0 --- openvdb.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openvdb.nix b/openvdb.nix index 213ef97877..7b251c4766 100644 --- a/openvdb.nix +++ b/openvdb.nix @@ -8,7 +8,7 @@ in with pkgs; stdenv.mkDerivation { name = "openvdb"; - version = "8.0.1"; + version = "9.0.0"; # https://nixos.org/nix/manual/#builtin-filterSource src = builtins.filterSource From 470cc4c183fefcba035e1a8534bd9732c64f7f0b Mon Sep 17 00:00:00 2001 From: Karl Marrett Date: Tue, 9 Nov 2021 09:20:35 -0800 Subject: [PATCH 08/11] ci: propagate boost175 --- openvdb.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openvdb.nix b/openvdb.nix index 7b251c4766..457c8d7e1a 100644 --- a/openvdb.nix +++ b/openvdb.nix @@ -29,10 +29,11 @@ in openexr tbb c-blosc + boost175 ]; - buildInputs = [ - unzip boost175 jemalloc ilmbase + buildInputs = [ + unzip jemalloc ilmbase # for the optional VDB_VIEW binary opengl related dependencies: libGL glfw3 x11 libGLU xorg.libXdmcp ]; From 0b7340d36b095ddafde96b51d270de75739a2a90 Mon Sep 17 00:00:00 2001 From: Karl Marrett Date: Tue, 28 Dec 2021 13:44:41 -0800 Subject: [PATCH 09/11] ci: pin nixpkgs to a newer commit on stable --- flake.lock | 8 ++++---- flake.nix | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index c3a019fda8..8436f4cdfe 100644 --- a/flake.lock +++ b/flake.lock @@ -2,17 +2,17 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1609254719, - "narHash": "sha256-XB3391z8r81Bt8NfQzB+tYdTWoraY0QVdXM21JhfW+Y=", + "lastModified": 1640718479, + "narHash": "sha256-hm5yC3ynZE9bH564NmFdWv07RtCKr/fgT8Veh594j4M=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "733e537a8ad76fd355b6f501127f7d0eb8861775", + "rev": "d53978239b265066804a45b7607b010b9cb4c50c", "type": "github" }, "original": { "owner": "NixOS", "repo": "nixpkgs", - "rev": "733e537a8ad76fd355b6f501127f7d0eb8861775", + "rev": "d53978239b265066804a45b7607b010b9cb4c50c", "type": "github" } }, diff --git a/flake.nix b/flake.nix index e8b0165b17..007d3c18d3 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,8 @@ { description = "openvdb"; inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/733e537a8ad76fd355b6f501127f7d0eb8861775"; + # pick latest commit from stable branch and test it so no suprises + nixpkgs.url = "github:NixOS/nixpkgs/d53978239b265066804a45b7607b010b9cb4c50c"; }; outputs = inputs: { From c9e5310a50e05854694097f3e60e55181129ba05 Mon Sep 17 00:00:00 2001 From: Karl Marrett Date: Thu, 3 Aug 2023 16:12:46 -0700 Subject: [PATCH 10/11] fix: limit build parallelism --- openvdb.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openvdb.nix b/openvdb.nix index 457c8d7e1a..809136e82f 100644 --- a/openvdb.nix +++ b/openvdb.nix @@ -21,7 +21,8 @@ in cmakeFlags =["-DOPENVDB_BUILD_VDB_VIEW=ON"]; - enableParallelBuilding = true; + # easily maxes out RAM on github actions or systems <64 GB + enableParallelBuilding = false; nativeBuildInputs = [ cmake pkg-config ]; # required dependencies for downstream development From 9cafff887726cdee2ab04b525f12646693032985 Mon Sep 17 00:00:00 2001 From: Karl Marrett Date: Tue, 29 Aug 2023 08:48:14 -0700 Subject: [PATCH 11/11] fix: re-enable parallel nix compile, which requires the VM has >32GB of memory --- openvdb.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openvdb.nix b/openvdb.nix index 809136e82f..e51028cebb 100644 --- a/openvdb.nix +++ b/openvdb.nix @@ -22,7 +22,7 @@ in cmakeFlags =["-DOPENVDB_BUILD_VDB_VIEW=ON"]; # easily maxes out RAM on github actions or systems <64 GB - enableParallelBuilding = false; + enableParallelBuilding = true; nativeBuildInputs = [ cmake pkg-config ]; # required dependencies for downstream development