From 1bc9cb3d3782a66db07a837f4af3f5943dcc46e4 Mon Sep 17 00:00:00 2001 From: Gabriel Gerlero Date: Tue, 26 Dec 2023 06:09:09 -0300 Subject: [PATCH] Update dependency bundling --- .github/workflows/build-test.yml | 68 ++++------------------------ Makefile | 76 +++++++++++--------------------- README.md | 2 +- bundle_deps.py | 62 ++++++++++++++++++++++++++ macho.py | 15 +++++++ relativize_install_names.py | 8 ++-- 6 files changed, 116 insertions(+), 115 deletions(-) create mode 100755 bundle_deps.py create mode 100644 macho.py diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index f71f4bc..245ebf2 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -78,64 +78,21 @@ env: OPENFOAM: ${{ inputs.openfoam-version || inputs.openfoam-git-branch }} jobs: - deps: + build: runs-on: ${{ inputs.build-os || 'macos-12' }} env: BUILD_OS: ${{ inputs.build-os || 'macos-12' }} - outputs: - deps-restore-key: ${{ steps.caching.outputs.DEPS_RESTORE_KEY }} - build-restore-key: ${{ steps.caching.outputs.BUILD_RESTORE_KEY }} steps: - name: Checkout uses: actions/checkout@v4 - name: Get Make recipes for caching run: | - make deps --dry-run ${{ env.MAKE_VARS }} > make_deps.txt make build --dry-run ${{ env.MAKE_VARS }} > make_build.txt - name: Generate cache restore keys id: caching run: | - DEPS_RESTORE_KEY="deps-${{ env.OPENFOAM }}-${{ env.BUILD_OS }}-${{ hashFiles('make_deps.txt', 'Brewfile') }}-" - BUILD_RESTORE_KEY="build-${{ env.OPENFOAM }}-${{ env.BUILD_OS }}-${{ hashFiles('make_build.txt', 'Brewfile', format('OpenFOAM-v${0}.tgz.sha256', inputs.openfoam-version), 'configure.sh') }}-" - echo "DEPS_RESTORE_KEY=$DEPS_RESTORE_KEY" >> "$GITHUB_OUTPUT" + BUILD_RESTORE_KEY="build-${{ env.OPENFOAM }}-${{ env.BUILD_OS }}-${{ hashFiles('make_build.txt', 'Brewfile', 'bundle_deps.py', format('OpenFOAM-v${0}.tgz.sha256', inputs.openfoam-version), 'configure.sh') }}-" echo "BUILD_RESTORE_KEY=$BUILD_RESTORE_KEY" >> "$GITHUB_OUTPUT" - - name: Look up cached build - if: inputs.use-cached - id: cache_build - uses: actions/cache/restore@v3 - with: - path: build/*-build.sparsebundle - key: ignore - restore-keys: - ${{ steps.caching.outputs.BUILD_RESTORE_KEY }} - lookup-only: true - - name: Look up cached deps - if: inputs.use-cached && steps.cache_build.outputs.cache-matched-key == '' - id: cache_deps - uses: actions/cache/restore@v3 - with: - path: build/*-deps.sparsebundle - key: ignore - restore-keys: - ${{ steps.caching.outputs.DEPS_RESTORE_KEY }} - lookup-only: true - - name: Make deps - if: steps.cache_build.outputs.cache-matched-key == '' && steps.cache_deps.outputs.cache-matched-key == '' - run: | - make deps ${{ env.MAKE_VARS }} - - name: Save deps to cache - if: steps.cache_build.outputs.cache-matched-key == '' && steps.cache_deps.outputs.cache-matched-key == '' - uses: actions/cache/save@v3 - with: - path: build/*-deps.sparsebundle - key: ${{ steps.caching.outputs.DEPS_RESTORE_KEY }}${{ github.run_id }} - - build: - needs: deps - runs-on: ${{ inputs.build-os || 'macos-12' }} - steps: - - name: Checkout - uses: actions/checkout@v4 - name: Restore cached build if available if: inputs.use-cached id: cache_build @@ -144,20 +101,10 @@ jobs: path: build/*-build.sparsebundle key: ignore restore-keys: - ${{ needs.deps.outputs.build-restore-key }} - - name: Restore cached deps - if: steps.cache_build.outputs.cache-matched-key == '' - id: cache_deps - uses: actions/cache/restore@v3 - with: - path: build/*-deps.sparsebundle - key: ignore - restore-keys: - ${{ needs.deps.outputs.deps-restore-key }} - fail-on-cache-miss: true - - name: Reuse cached build or deps + ${{ steps.caching.outputs.build-restore-key }} + - name: Reuse cached build + if: steps.cache_build.outputs.cache-matched-key != '' run: | - touch -c build/*-deps.sparsebundle touch -c build/*-build.sparsebundle - name: Build if: steps.cache_build.outputs.cache-matched-key == '' @@ -168,7 +115,7 @@ jobs: uses: actions/cache/save@v3 with: path: build/*-build.sparsebundle - key: ${{ needs.deps.outputs.build-restore-key }}${{ github.run_id }} + key: ${{ steps.caching.outputs.build-restore-key }}${{ github.run_id }} - name: Make app run: | make zip ${{ env.MAKE_VARS }} @@ -198,6 +145,9 @@ jobs: run: | unzip *-app-*.zip working-directory: build + - name: Uninstall all Homebrew formulae + run: | + brew uninstall $(brew list --formulae) - name: Test run: | make test ${{ env.MAKE_VARS }} diff --git a/Makefile b/Makefile index 9480b1a..6a4b0d0 100644 --- a/Makefile +++ b/Makefile @@ -38,18 +38,9 @@ endif # Build targets app: build/$(APP_NAME).app build: build/$(APP_NAME)-build.sparsebundle -deps: build/$(APP_NAME)-deps.sparsebundle +deps: Brewfile.lock.json fetch-source: $(OPENFOAM_TARBALL) - -ifeq ($(DEPENDENCIES_KIND),both) -zip: - $(MAKE) zip DEPENDENCIES_KIND=standalone - $(MAKE) clean-app - $(MAKE) zip DEPENDENCIES_KIND=homebrew - $(MAKE) clean-app -else zip: build/$(DIST_NAME).zip -endif install: $(INSTALL_DIR)/$(APP_NAME).app @@ -123,20 +114,11 @@ build/$(APP_NAME).app/Contents/Resources/$(APP_NAME).dmg: build/$(APP_NAME)-buil SetFile -a C $(VOLUME) uuidgen > $(VOLUME_ID_FILE) cat $(VOLUME_ID_FILE) - rm -rf $(VOLUME)/homebrew [ ! -L $(VOLUME)/usr ] || rm $(VOLUME)/usr - rm -rf $(VOLUME)/build - rm -rf -- $(VOLUME)/**/.git(N) rm -f -- $(VOLUME)/**/.DS_Store(N) -ifeq ($(DEPENDENCIES_KIND),standalone) - rm $(VOLUME)/usr/bin/brew - rm $(VOLUME)/Brewfile - rm $(VOLUME)/Brewfile.lock.json -else ifeq ($(DEPENDENCIES_KIND),homebrew) +ifeq ($(DEPENDENCIES_KIND),homebrew) rm -rf $(VOLUME)/usr ln -s $(shell brew --prefix) $(VOLUME)/usr -else - $(error Invalid value for DEPENDENCIES_KIND) endif rm -rf $(VOLUME)/.fseventsd || true mkdir -p build/$(APP_NAME).app/Contents/Resources @@ -150,10 +132,26 @@ endif hdiutil detach $(VOLUME) rm build/$(APP_NAME)-build.sparsebundle.shadow -build/$(APP_NAME)-build.sparsebundle: build/$(APP_NAME)-deps.sparsebundle $(OPENFOAM_TARBALL) configure.sh +build/$(APP_NAME)-build.sparsebundle: Brewfile Brewfile.lock.json $(if $(filter standalone,$(DEPENDENCIES_KIND)),bundle_deps.py) $(OPENFOAM_TARBALL) configure.sh [ ! -d $(VOLUME) ] || hdiutil detach $(VOLUME) - mv build/$(APP_NAME)-deps.sparsebundle build/$(APP_NAME)-build.sparsebundle - hdiutil attach build/$(APP_NAME)-build.sparsebundle + mkdir -p build + hdiutil create \ + -size 50g \ + -fs $(VOLUME_FILESYSTEM) \ + -volname $(APP_NAME) \ + build/$(APP_NAME)-build.sparsebundle \ + -ov -attach + brew bundle check --verbose --no-upgrade +ifeq ($(DEPENDENCIES_KIND),standalone) + cd $(VOLUME) \ + && HOMEBREW_BUNDLE_FILE="$(CURDIR)/Brewfile" "$(CURDIR)/bundle_deps.py" +else ifeq ($(DEPENDENCIES_KIND),homebrew) + cp Brewfile $(VOLUME)/ + cp Brewfile.lock.json $(VOLUME)/ + ln -s $(shell brew --prefix) $(VOLUME)/usr +else + $(error Invalid value for DEPENDENCIES_KIND) +endif ifdef OPENFOAM_TARBALL tar -xzf $(OPENFOAM_TARBALL) --strip-components 1 -C $(VOLUME) else ifdef OPENFOAM_GIT_BRANCH @@ -168,32 +166,8 @@ endif && foamSystemCheck \ && ( ./Allwmake -j $(WMAKE_NJOBS) -s -q -k || true ) \ && ./Allwmake -j $(WMAKE_NJOBS) -s - hdiutil detach $(VOLUME) - -build/$(APP_NAME)-deps.sparsebundle: Brewfile $(if $(filter homebrew,$(DEPENDENCIES_KIND)),Brewfile.lock.json) - [ ! -d $(VOLUME) ] || hdiutil detach $(VOLUME) - mkdir -p build - hdiutil create \ - -size 50g \ - -fs $(VOLUME_FILESYSTEM) \ - -volname $(APP_NAME) \ - build/$(APP_NAME)-deps.sparsebundle \ - -ov -attach - cp Brewfile $(VOLUME)/ -ifeq ($(DEPENDENCIES_KIND),standalone) - git clone https://github.com/Homebrew/brew $(VOLUME)/homebrew - mkdir -p $(VOLUME)/usr/bin - ln -s ../../homebrew/bin/brew $(VOLUME)/usr/bin/ - HOMEBREW_RELOCATABLE_INSTALL_NAMES=1 $(VOLUME)/usr/bin/brew bundle --file $(VOLUME)/Brewfile --verbose - $(VOLUME)/usr/bin/brew autoremove - $(VOLUME)/usr/bin/brew list --versions -else ifeq ($(DEPENDENCIES_KIND),homebrew) - brew bundle check --verbose --no-upgrade - cp Brewfile.lock.json $(VOLUME)/ - ln -s $(shell brew --prefix) $(VOLUME)/usr -else - $(error Invalid value for DEPENDENCIES_KIND) -endif + rm -rf $(VOLUME)/build + rm -rf -- $(VOLUME)/**/.git(N) hdiutil detach $(VOLUME) $(OPENFOAM_TARBALL): $(or $(wildcard $(OPENFOAM_TARBALL).sha256), \ @@ -258,7 +232,7 @@ clean-app: clean-build: clean-app rm -f build/$(DIST_NAME).zip - rm -rf build/$(APP_NAME)-build.sparsebundle build/$(APP_NAME)-deps.sparsebundle $(TEST_DIR)/test-openfoam $(TEST_DIR)/test-bash $(TEST_DIR)/test-zsh $(TEST_DIR)/test-dmg + rm -rf build/$(APP_NAME)-build.sparsebundle $(TEST_DIR)/test-openfoam $(TEST_DIR)/test-bash $(TEST_DIR)/test-zsh $(TEST_DIR)/test-dmg rmdir $(TEST_DIR) || true rmdir build || true @@ -272,5 +246,5 @@ uninstall: # Set special targets .PHONY: app build deps fetch-source zip install test test-openfoam test-bash test-zsh test-dmg clean-app clean-build clean uninstall .PRECIOUS: build/$(APP_NAME)-build.sparsebundle -.SECONDARY: $(OPENFOAM_TARBALL) Brewfile.lock.json build/$(APP_NAME)-deps.sparsebundle build/$(APP_NAME)-build.sparsebundle +.SECONDARY: $(OPENFOAM_TARBALL) Brewfile.lock.json build/$(APP_NAME)-build.sparsebundle .DELETE_ON_ERROR: diff --git a/README.md b/README.md index 59fd2fa..49fa6c2 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,7 @@ cd openfoam-app make ``` -The Xcode Command Line Tools are required. See the available configuration variables and alternative targets for `make` in the [`Makefile`](Makefile). Note that the compilation of OpenFOAM and the necessary dependencies from source may take a while. +[Homebrew](https://brew.sh) is required. See the available configuration variables and alternative targets for `make` in the [`Makefile`](Makefile). Note that the compilation of OpenFOAM from source may take a while. ## 📄 Legal notices diff --git a/bundle_deps.py b/bundle_deps.py new file mode 100755 index 0000000..e4043e3 --- /dev/null +++ b/bundle_deps.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +""" +Bundle Homebrew dependencies from a installed with Homebrew Bundle +""" + +import subprocess +import shutil +import os + +from pathlib import Path + +import macho + +SRC_PREFIX = Path(subprocess.run(["brew", "--prefix"], stdout=subprocess.PIPE, check=True).stdout.decode().strip()) +DST_PREFIX = Path("usr") + +def change_lib_id(lib, *, id): + subprocess.run(["install_name_tool", "-id", id, lib], check=True) + subprocess.run(["codesign", "--force", "--preserve-metadata=entitlements,requirements,flags,runtime", "--sign", "-", lib], check=True) + +def copy_installed_formula(formula): + print(f"Bundling {formula}") + + src_prefix = SRC_PREFIX / "opt" / formula.name + dst_prefix = DST_PREFIX / "opt" / formula.name + + src_cellar = SRC_PREFIX / "Cellar" / formula.name + dst_cellar = DST_PREFIX / "Cellar" / formula.name + + shutil.copytree(src_cellar, dst_cellar) + + dst_prefix.parent.mkdir(exist_ok=True) + shutil.copy(src_prefix, dst_prefix, follow_symlinks=False) + + # Replace library IDs and references to other libraries (install_names) + for file in dst_cellar.rglob("*"): + if not file.is_file(): + continue + if (file.suffix == ".dylib" or file.suffix == ".so"): + macho.change_lib_id(file, id=dst_prefix.absolute() / Path(*file.relative_to(dst_cellar).parts[1:])) + if (file.suffix == "" or file.suffix == ".bin" or file.suffix == ".dylib" or file.suffix == ".so"): + for install_name in macho.get_install_names(file): + if install_name.is_absolute() and install_name.is_relative_to(SRC_PREFIX): + if install_name.is_relative_to(SRC_PREFIX): + new_install_name = DST_PREFIX.absolute() / install_name.relative_to(SRC_PREFIX) + relative_install_name = Path("@loader_path") / os.path.relpath(new_install_name, start=file.parent) + macho.change_install_name(file, install_name, relative_install_name) + +def get_deps(*, recursive=True): + if not recursive: + return {Path(formula) for formula in subprocess.run(["brew", "bundle", "list"], stdout=subprocess.PIPE, check=True).stdout.decode().splitlines()} + + deps = get_deps(recursive=False) + for dep in list(deps): + recursive_deps = {Path(formula) for formula in subprocess.run(["brew", "deps", dep], stdout=subprocess.PIPE, check=True).stdout.decode().splitlines()} + deps.update(recursive_deps) + + return deps + + +for formula in get_deps(): + copy_installed_formula(formula) diff --git a/macho.py b/macho.py new file mode 100644 index 0000000..396aafa --- /dev/null +++ b/macho.py @@ -0,0 +1,15 @@ +import subprocess + +from pathlib import Path + +def change_lib_id(lib, *, id): + subprocess.run(["install_name_tool", "-id", id, lib], check=True) + subprocess.run(["codesign", "--force", "--preserve-metadata=entitlements,requirements,flags,runtime", "--sign", "-", lib], check=True) + +def get_install_names(file): + otool_stdout = subprocess.run(["otool", "-L", file], stdout=subprocess.PIPE, check=True).stdout.decode() + install_names = [Path(line.split(" (compatibility version ")[0].strip()) for line in otool_stdout.splitlines()[1:]] + return install_names + +def change_install_name(file, old_install_name, new_install_name): + subprocess.run(["install_name_tool", "-change", old_install_name, new_install_name, file], check=True) diff --git a/relativize_install_names.py b/relativize_install_names.py index 95a8796..91511d8 100755 --- a/relativize_install_names.py +++ b/relativize_install_names.py @@ -5,17 +5,17 @@ from pathlib import Path +import macho + def relativize_install_names(file, lib_dirs): - otool_stdout = subprocess.run(["otool", "-L", file], stdout=subprocess.PIPE, check=True).stdout.decode() - install_names = [Path(line.split(" (compatibility version ")[0].strip()) for line in otool_stdout.splitlines()[1:]] - for install_name in install_names: + for install_name in macho.get_install_names(file): if install_name.is_absolute(): for lib_dir,new_lib_dir in lib_dirs.items(): lib_dir = lib_dir.absolute() if install_name.is_relative_to(lib_dir): new_install_name = new_lib_dir.absolute() / install_name.relative_to(lib_dir) relative_install_name = Path("@loader_path") / os.path.relpath(new_install_name, start=file.parent) - subprocess.run(["install_name_tool", "-change", install_name, relative_install_name, file]) + macho.change_install_name(file, install_name, relative_install_name) break # Replace references to dependencies