Skip to content

Commit

Permalink
Update dependency bundling
Browse files Browse the repository at this point in the history
  • Loading branch information
gerlero committed Dec 27, 2023
1 parent b77358a commit 1bc9cb3
Show file tree
Hide file tree
Showing 6 changed files with 116 additions and 115 deletions.
68 changes: 9 additions & 59 deletions .github/workflows/build-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -78,64 +78,21 @@ env:
OPENFOAM: ${{ inputs.openfoam-version || inputs.openfoam-git-branch }}

jobs:
deps:
build:
runs-on: ${{ inputs.build-os || 'macos-12' }}
env:
BUILD_OS: ${{ inputs.build-os || 'macos-12' }}
outputs:
deps-restore-key: ${{ steps.caching.outputs.DEPS_RESTORE_KEY }}
build-restore-key: ${{ steps.caching.outputs.BUILD_RESTORE_KEY }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Get Make recipes for caching
run: |
make deps --dry-run ${{ env.MAKE_VARS }} > make_deps.txt
make build --dry-run ${{ env.MAKE_VARS }} > make_build.txt
- name: Generate cache restore keys
id: caching
run: |
DEPS_RESTORE_KEY="deps-${{ env.OPENFOAM }}-${{ env.BUILD_OS }}-${{ hashFiles('make_deps.txt', 'Brewfile') }}-"
BUILD_RESTORE_KEY="build-${{ env.OPENFOAM }}-${{ env.BUILD_OS }}-${{ hashFiles('make_build.txt', 'Brewfile', format('OpenFOAM-v${0}.tgz.sha256', inputs.openfoam-version), 'configure.sh') }}-"
echo "DEPS_RESTORE_KEY=$DEPS_RESTORE_KEY" >> "$GITHUB_OUTPUT"
BUILD_RESTORE_KEY="build-${{ env.OPENFOAM }}-${{ env.BUILD_OS }}-${{ hashFiles('make_build.txt', 'Brewfile', 'bundle_deps.py', format('OpenFOAM-v${0}.tgz.sha256', inputs.openfoam-version), 'configure.sh') }}-"
echo "BUILD_RESTORE_KEY=$BUILD_RESTORE_KEY" >> "$GITHUB_OUTPUT"
- name: Look up cached build
if: inputs.use-cached
id: cache_build
uses: actions/cache/restore@v3
with:
path: build/*-build.sparsebundle
key: ignore
restore-keys:
${{ steps.caching.outputs.BUILD_RESTORE_KEY }}
lookup-only: true
- name: Look up cached deps
if: inputs.use-cached && steps.cache_build.outputs.cache-matched-key == ''
id: cache_deps
uses: actions/cache/restore@v3
with:
path: build/*-deps.sparsebundle
key: ignore
restore-keys:
${{ steps.caching.outputs.DEPS_RESTORE_KEY }}
lookup-only: true
- name: Make deps
if: steps.cache_build.outputs.cache-matched-key == '' && steps.cache_deps.outputs.cache-matched-key == ''
run: |
make deps ${{ env.MAKE_VARS }}
- name: Save deps to cache
if: steps.cache_build.outputs.cache-matched-key == '' && steps.cache_deps.outputs.cache-matched-key == ''
uses: actions/cache/save@v3
with:
path: build/*-deps.sparsebundle
key: ${{ steps.caching.outputs.DEPS_RESTORE_KEY }}${{ github.run_id }}

build:
needs: deps
runs-on: ${{ inputs.build-os || 'macos-12' }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Restore cached build if available
if: inputs.use-cached
id: cache_build
Expand All @@ -144,20 +101,10 @@ jobs:
path: build/*-build.sparsebundle
key: ignore
restore-keys:
${{ needs.deps.outputs.build-restore-key }}
- name: Restore cached deps
if: steps.cache_build.outputs.cache-matched-key == ''
id: cache_deps
uses: actions/cache/restore@v3
with:
path: build/*-deps.sparsebundle
key: ignore
restore-keys:
${{ needs.deps.outputs.deps-restore-key }}
fail-on-cache-miss: true
- name: Reuse cached build or deps
${{ steps.caching.outputs.build-restore-key }}
- name: Reuse cached build
if: steps.cache_build.outputs.cache-matched-key != ''
run: |
touch -c build/*-deps.sparsebundle
touch -c build/*-build.sparsebundle
- name: Build
if: steps.cache_build.outputs.cache-matched-key == ''
Expand All @@ -168,7 +115,7 @@ jobs:
uses: actions/cache/save@v3
with:
path: build/*-build.sparsebundle
key: ${{ needs.deps.outputs.build-restore-key }}${{ github.run_id }}
key: ${{ steps.caching.outputs.build-restore-key }}${{ github.run_id }}
- name: Make app
run: |
make zip ${{ env.MAKE_VARS }}
Expand Down Expand Up @@ -198,6 +145,9 @@ jobs:
run: |
unzip *-app-*.zip
working-directory: build
- name: Uninstall all Homebrew formulae
run: |
brew uninstall $(brew list --formulae)
- name: Test
run: |
make test ${{ env.MAKE_VARS }}
Expand Down
76 changes: 25 additions & 51 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -38,18 +38,9 @@ endif
# Build targets
app: build/$(APP_NAME).app
build: build/$(APP_NAME)-build.sparsebundle
deps: build/$(APP_NAME)-deps.sparsebundle
deps: Brewfile.lock.json
fetch-source: $(OPENFOAM_TARBALL)

ifeq ($(DEPENDENCIES_KIND),both)
zip:
$(MAKE) zip DEPENDENCIES_KIND=standalone
$(MAKE) clean-app
$(MAKE) zip DEPENDENCIES_KIND=homebrew
$(MAKE) clean-app
else
zip: build/$(DIST_NAME).zip
endif

install: $(INSTALL_DIR)/$(APP_NAME).app

Expand Down Expand Up @@ -123,20 +114,11 @@ build/$(APP_NAME).app/Contents/Resources/$(APP_NAME).dmg: build/$(APP_NAME)-buil
SetFile -a C $(VOLUME)
uuidgen > $(VOLUME_ID_FILE)
cat $(VOLUME_ID_FILE)
rm -rf $(VOLUME)/homebrew
[ ! -L $(VOLUME)/usr ] || rm $(VOLUME)/usr
rm -rf $(VOLUME)/build
rm -rf -- $(VOLUME)/**/.git(N)
rm -f -- $(VOLUME)/**/.DS_Store(N)
ifeq ($(DEPENDENCIES_KIND),standalone)
rm $(VOLUME)/usr/bin/brew
rm $(VOLUME)/Brewfile
rm $(VOLUME)/Brewfile.lock.json
else ifeq ($(DEPENDENCIES_KIND),homebrew)
ifeq ($(DEPENDENCIES_KIND),homebrew)
rm -rf $(VOLUME)/usr
ln -s $(shell brew --prefix) $(VOLUME)/usr
else
$(error Invalid value for DEPENDENCIES_KIND)
endif
rm -rf $(VOLUME)/.fseventsd || true
mkdir -p build/$(APP_NAME).app/Contents/Resources
Expand All @@ -150,10 +132,26 @@ endif
hdiutil detach $(VOLUME)
rm build/$(APP_NAME)-build.sparsebundle.shadow

build/$(APP_NAME)-build.sparsebundle: build/$(APP_NAME)-deps.sparsebundle $(OPENFOAM_TARBALL) configure.sh
build/$(APP_NAME)-build.sparsebundle: Brewfile Brewfile.lock.json $(if $(filter standalone,$(DEPENDENCIES_KIND)),bundle_deps.py) $(OPENFOAM_TARBALL) configure.sh
[ ! -d $(VOLUME) ] || hdiutil detach $(VOLUME)
mv build/$(APP_NAME)-deps.sparsebundle build/$(APP_NAME)-build.sparsebundle
hdiutil attach build/$(APP_NAME)-build.sparsebundle
mkdir -p build
hdiutil create \
-size 50g \
-fs $(VOLUME_FILESYSTEM) \
-volname $(APP_NAME) \
build/$(APP_NAME)-build.sparsebundle \
-ov -attach
brew bundle check --verbose --no-upgrade
ifeq ($(DEPENDENCIES_KIND),standalone)
cd $(VOLUME) \
&& HOMEBREW_BUNDLE_FILE="$(CURDIR)/Brewfile" "$(CURDIR)/bundle_deps.py"
else ifeq ($(DEPENDENCIES_KIND),homebrew)
cp Brewfile $(VOLUME)/
cp Brewfile.lock.json $(VOLUME)/
ln -s $(shell brew --prefix) $(VOLUME)/usr
else
$(error Invalid value for DEPENDENCIES_KIND)
endif
ifdef OPENFOAM_TARBALL
tar -xzf $(OPENFOAM_TARBALL) --strip-components 1 -C $(VOLUME)
else ifdef OPENFOAM_GIT_BRANCH
Expand All @@ -168,32 +166,8 @@ endif
&& foamSystemCheck \
&& ( ./Allwmake -j $(WMAKE_NJOBS) -s -q -k || true ) \
&& ./Allwmake -j $(WMAKE_NJOBS) -s
hdiutil detach $(VOLUME)

build/$(APP_NAME)-deps.sparsebundle: Brewfile $(if $(filter homebrew,$(DEPENDENCIES_KIND)),Brewfile.lock.json)
[ ! -d $(VOLUME) ] || hdiutil detach $(VOLUME)
mkdir -p build
hdiutil create \
-size 50g \
-fs $(VOLUME_FILESYSTEM) \
-volname $(APP_NAME) \
build/$(APP_NAME)-deps.sparsebundle \
-ov -attach
cp Brewfile $(VOLUME)/
ifeq ($(DEPENDENCIES_KIND),standalone)
git clone https://github.com/Homebrew/brew $(VOLUME)/homebrew
mkdir -p $(VOLUME)/usr/bin
ln -s ../../homebrew/bin/brew $(VOLUME)/usr/bin/
HOMEBREW_RELOCATABLE_INSTALL_NAMES=1 $(VOLUME)/usr/bin/brew bundle --file $(VOLUME)/Brewfile --verbose
$(VOLUME)/usr/bin/brew autoremove
$(VOLUME)/usr/bin/brew list --versions
else ifeq ($(DEPENDENCIES_KIND),homebrew)
brew bundle check --verbose --no-upgrade
cp Brewfile.lock.json $(VOLUME)/
ln -s $(shell brew --prefix) $(VOLUME)/usr
else
$(error Invalid value for DEPENDENCIES_KIND)
endif
rm -rf $(VOLUME)/build
rm -rf -- $(VOLUME)/**/.git(N)
hdiutil detach $(VOLUME)

$(OPENFOAM_TARBALL): $(or $(wildcard $(OPENFOAM_TARBALL).sha256), \
Expand Down Expand Up @@ -258,7 +232,7 @@ clean-app:

clean-build: clean-app
rm -f build/$(DIST_NAME).zip
rm -rf build/$(APP_NAME)-build.sparsebundle build/$(APP_NAME)-deps.sparsebundle $(TEST_DIR)/test-openfoam $(TEST_DIR)/test-bash $(TEST_DIR)/test-zsh $(TEST_DIR)/test-dmg
rm -rf build/$(APP_NAME)-build.sparsebundle $(TEST_DIR)/test-openfoam $(TEST_DIR)/test-bash $(TEST_DIR)/test-zsh $(TEST_DIR)/test-dmg
rmdir $(TEST_DIR) || true
rmdir build || true

Expand All @@ -272,5 +246,5 @@ uninstall:
# Set special targets
.PHONY: app build deps fetch-source zip install test test-openfoam test-bash test-zsh test-dmg clean-app clean-build clean uninstall
.PRECIOUS: build/$(APP_NAME)-build.sparsebundle
.SECONDARY: $(OPENFOAM_TARBALL) Brewfile.lock.json build/$(APP_NAME)-deps.sparsebundle build/$(APP_NAME)-build.sparsebundle
.SECONDARY: $(OPENFOAM_TARBALL) Brewfile.lock.json build/$(APP_NAME)-build.sparsebundle
.DELETE_ON_ERROR:
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ cd openfoam-app
make
```
The Xcode Command Line Tools are required. See the available configuration variables and alternative targets for `make` in the [`Makefile`](Makefile). Note that the compilation of OpenFOAM and the necessary dependencies from source may take a while.
[Homebrew](https://brew.sh) is required. See the available configuration variables and alternative targets for `make` in the [`Makefile`](Makefile). Note that the compilation of OpenFOAM from source may take a while.
## 📄 Legal notices
Expand Down
62 changes: 62 additions & 0 deletions bundle_deps.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
#!/usr/bin/env python3
"""
Bundle Homebrew dependencies from a installed with Homebrew Bundle
"""

import subprocess
import shutil
import os

from pathlib import Path

import macho

SRC_PREFIX = Path(subprocess.run(["brew", "--prefix"], stdout=subprocess.PIPE, check=True).stdout.decode().strip())
DST_PREFIX = Path("usr")

def change_lib_id(lib, *, id):
subprocess.run(["install_name_tool", "-id", id, lib], check=True)
subprocess.run(["codesign", "--force", "--preserve-metadata=entitlements,requirements,flags,runtime", "--sign", "-", lib], check=True)

def copy_installed_formula(formula):
print(f"Bundling {formula}")

src_prefix = SRC_PREFIX / "opt" / formula.name
dst_prefix = DST_PREFIX / "opt" / formula.name

src_cellar = SRC_PREFIX / "Cellar" / formula.name
dst_cellar = DST_PREFIX / "Cellar" / formula.name

shutil.copytree(src_cellar, dst_cellar)

dst_prefix.parent.mkdir(exist_ok=True)
shutil.copy(src_prefix, dst_prefix, follow_symlinks=False)

# Replace library IDs and references to other libraries (install_names)
for file in dst_cellar.rglob("*"):
if not file.is_file():
continue
if (file.suffix == ".dylib" or file.suffix == ".so"):
macho.change_lib_id(file, id=dst_prefix.absolute() / Path(*file.relative_to(dst_cellar).parts[1:]))
if (file.suffix == "" or file.suffix == ".bin" or file.suffix == ".dylib" or file.suffix == ".so"):
for install_name in macho.get_install_names(file):
if install_name.is_absolute() and install_name.is_relative_to(SRC_PREFIX):
if install_name.is_relative_to(SRC_PREFIX):
new_install_name = DST_PREFIX.absolute() / install_name.relative_to(SRC_PREFIX)
relative_install_name = Path("@loader_path") / os.path.relpath(new_install_name, start=file.parent)
macho.change_install_name(file, install_name, relative_install_name)

def get_deps(*, recursive=True):
if not recursive:
return {Path(formula) for formula in subprocess.run(["brew", "bundle", "list"], stdout=subprocess.PIPE, check=True).stdout.decode().splitlines()}

deps = get_deps(recursive=False)
for dep in list(deps):
recursive_deps = {Path(formula) for formula in subprocess.run(["brew", "deps", dep], stdout=subprocess.PIPE, check=True).stdout.decode().splitlines()}
deps.update(recursive_deps)

return deps


for formula in get_deps():
copy_installed_formula(formula)
15 changes: 15 additions & 0 deletions macho.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import subprocess

from pathlib import Path

def change_lib_id(lib, *, id):
subprocess.run(["install_name_tool", "-id", id, lib], check=True)
subprocess.run(["codesign", "--force", "--preserve-metadata=entitlements,requirements,flags,runtime", "--sign", "-", lib], check=True)

def get_install_names(file):
otool_stdout = subprocess.run(["otool", "-L", file], stdout=subprocess.PIPE, check=True).stdout.decode()
install_names = [Path(line.split(" (compatibility version ")[0].strip()) for line in otool_stdout.splitlines()[1:]]
return install_names

def change_install_name(file, old_install_name, new_install_name):
subprocess.run(["install_name_tool", "-change", old_install_name, new_install_name, file], check=True)
8 changes: 4 additions & 4 deletions relativize_install_names.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,17 @@

from pathlib import Path

import macho

def relativize_install_names(file, lib_dirs):
otool_stdout = subprocess.run(["otool", "-L", file], stdout=subprocess.PIPE, check=True).stdout.decode()
install_names = [Path(line.split(" (compatibility version ")[0].strip()) for line in otool_stdout.splitlines()[1:]]
for install_name in install_names:
for install_name in macho.get_install_names(file):
if install_name.is_absolute():
for lib_dir,new_lib_dir in lib_dirs.items():
lib_dir = lib_dir.absolute()
if install_name.is_relative_to(lib_dir):
new_install_name = new_lib_dir.absolute() / install_name.relative_to(lib_dir)
relative_install_name = Path("@loader_path") / os.path.relpath(new_install_name, start=file.parent)
subprocess.run(["install_name_tool", "-change", install_name, relative_install_name, file])
macho.change_install_name(file, install_name, relative_install_name)
break

# Replace references to dependencies
Expand Down

0 comments on commit 1bc9cb3

Please sign in to comment.