Update: test run new CI for package python #92
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Package Python Library | |
| on: | |
| push: | |
| branches: [ feat/python-package-ci ] | |
| workflow_dispatch: | |
| inputs: | |
| model_dir: | |
| description: "Path to model directory in janhq/models repo" | |
| required: true | |
| model_name: | |
| description: "name of model to be release" | |
| required: true | |
| repo_name: | |
| description: "name of repo to be checked out" | |
| required: true | |
| branch_name: | |
| description: "name of branch to be checked out" | |
| required: true | |
| default: main | |
| hf_repo: | |
| description: "name of huggingface repo to be pushed" | |
| required: true | |
| hf_prefix_branch: | |
| description: "prefix of hf branch" | |
| required: false | |
| env: | |
| MODEL_DIR: models/ichigo-wrapper # ${{ inputs.model_dir }} | |
| MODEL_NAME: ichigo # ${{ inputs.model_name }} | |
| REPO_NAME: janhq/models # ${{ inputs.model_name }} | |
| BRANCH_NAME: feat/ci-python-models # ${{ inputs.model_name }} | |
| HF_REPO: cortexso/test # ${{ inputs.hf_repo }} | |
| HF_PREFIX_BRANCH: fp16 # ${{ inputs.hf_prefix_branch }} | |
| jobs: | |
| build-and-test: | |
| runs-on: ${{ matrix.runs-on }} | |
| timeout-minutes: 3600 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - os: "linux" | |
| name: "amd64" | |
| runs-on: "ubuntu-20-04-cuda-12-0" | |
| # - os: "mac" | |
| # name: "amd64" | |
| # runs-on: "macos-selfhosted-12" | |
| # - os: "mac" | |
| # name: "arm64" | |
| # runs-on: "macos-selfhosted-12-arm64" | |
| # - os: "windows" | |
| # name: "amd64" | |
| # runs-on: "windows-cuda-12-0" | |
| steps: | |
| - name: Clone | |
| id: checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| repository: ${{env.REPO_NAME}} | |
| ref: ${{env.BRANCH_NAME}} | |
| - uses: conda-incubator/setup-miniconda@v3 | |
| if: runner.os != 'windows' | |
| with: | |
| auto-update-conda: true | |
| python-version: 3.11 | |
| - name: use python | |
| if : runner.os == 'windows' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Get Cer for code signing | |
| if: runner.os == 'macOS' | |
| run: base64 -d <<< "$CODE_SIGN_P12_BASE64" > /tmp/codesign.p12 | |
| shell: bash | |
| env: | |
| CODE_SIGN_P12_BASE64: ${{ secrets.CODE_SIGN_P12_BASE64 }} | |
| - uses: apple-actions/import-codesign-certs@v2 | |
| continue-on-error: true | |
| if: runner.os == 'macOS' | |
| with: | |
| p12-file-base64: ${{ secrets.CODE_SIGN_P12_BASE64 }} | |
| p12-password: ${{ secrets.CODE_SIGN_P12_PASSWORD }} | |
| - name: Get Cer for code signing | |
| if: runner.os == 'macOS' | |
| run: base64 -d <<< "$NOTARIZE_P8_BASE64" > /tmp/notary-key.p8 | |
| shell: bash | |
| env: | |
| NOTARIZE_P8_BASE64: ${{ secrets.NOTARIZE_P8_BASE64 }} | |
| - name: Install dependencies Windows | |
| if: runner.os == 'windows' | |
| shell: pwsh | |
| run: | | |
| python3 -m pip install fastapi | |
| python3 -m pip freeze | % { python3 -m pip uninstall -y $_ } | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install -I -r ${{env.MODEL_DIR}}/requirements.cuda.txt | |
| python3 -m pip install python-dotenv | |
| - name: Install dependencies Linux | |
| if: runner.os == 'linux' | |
| run: | | |
| conda create -y -n ${{env.MODEL_NAME}} python=3.11 | |
| source $HOME/miniconda3/bin/activate base | |
| conda init | |
| conda activate ${{env.MODEL_NAME}} | |
| python -m pip install fastapi | |
| python -m pip freeze | xargs python -m pip uninstall -y | |
| python -m pip install --upgrade pip | |
| python -m pip install -r ${{env.MODEL_DIR}}/requirements.cuda.txt | |
| python -m pip install python-dotenv | |
| - name: Install dependencies Mac | |
| if: runner.os == 'macOS' | |
| run: | | |
| conda create -y -n ${{env.MODEL_NAME}} python=3.11 | |
| source $HOME/miniconda3/bin/activate base | |
| conda init | |
| conda activate ${{env.MODEL_NAME}} | |
| python -m pip install fastapi | |
| python -m pip freeze | xargs python -m pip uninstall -y | |
| python -m pip install --upgrade pip | |
| python -m pip install -r ${{env.MODEL_DIR}}/requirements.txt | |
| python -m pip install python-dotenv | |
| - name: prepare python package windows | |
| if : runner.os == 'windows' | |
| shell: pwsh | |
| run: | | |
| $pythonPath = where.exe python | |
| echo "Python path (where.exe): $pythonPath" | |
| $pythonFolder = Split-Path -Path "$pythonPath" -Parent | |
| echo "PYTHON_FOLDER=$pythonFolder" >> $env:GITHUB_ENV | |
| copy "$pythonFolder\python*.*" "$pythonFolder\Scripts\" | |
| - name: prepare python package macos | |
| if : runner.os == 'macOs' | |
| run: | | |
| source $HOME/miniconda3/bin/activate base | |
| conda init | |
| conda activate ${{env.MODEL_NAME}} | |
| PYTHON_PATH=$(which python) | |
| echo $PYTHON_PATH | |
| PYTHON_FOLDER=$(dirname $(dirname "$PYTHON_PATH")) | |
| echo "PYTHON_FOLDER=$PYTHON_FOLDER" >> $GITHUB_ENV | |
| echo "github end PYTHON_FOLDER: ${{env.PYTHON_FOLDER}}" | |
| - name: prepare python package linux | |
| if : runner.os == 'linux' | |
| run: | | |
| source $HOME/miniconda3/bin/activate base | |
| conda init | |
| conda activate ${{env.MODEL_NAME}} | |
| PYTHON_PATH=$(which python) | |
| echo $PYTHON_PATH | |
| PYTHON_FOLDER=$(dirname $(dirname "$PYTHON_PATH")) | |
| rm -rf $PYTHON_FOLDER/lib/python3.1 | |
| echo "PYTHON_FOLDER=$PYTHON_FOLDER" >> $GITHUB_ENV | |
| echo "github end PYTHON_FOLDER: ${{env.PYTHON_FOLDER}}" | |
| - name: create plist file | |
| if: runner.os == 'macOS' | |
| run: | | |
| cat << EOF > /tmp/entitlements.plist | |
| <?xml version="1.0" encoding="UTF-8"?> | |
| <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> | |
| <plist version="1.0"> | |
| <dict> | |
| <!-- These are required for binaries built by PyInstaller --> | |
| <key>com.apple.security.cs.allow-jit</key> | |
| <true/> | |
| <key>com.apple.security.cs.allow-unsigned-executable-memory</key> | |
| <true/> | |
| <!-- Add these for additional permissions --> | |
| <key>com.apple.security.app-sandbox</key> | |
| <false/> | |
| <key>com.apple.security.network.client</key> | |
| <true/> | |
| <key>com.apple.security.network.server</key> | |
| <true/> | |
| <key>com.apple.security.device.audio-input</key> | |
| <true/> | |
| <key>com.apple.security.device.microphone</key> | |
| <true/> | |
| <key>com.apple.security.device.camera</key> | |
| <true/> | |
| <key>com.apple.security.files.user-selected.read-write</key> | |
| <true/> | |
| <key>com.apple.security.cs.disable-library-validation</key> | |
| <true/> | |
| <key>com.apple.security.cs.allow-dyld-environment-variables</key> | |
| <true/> | |
| <key>com.apple.security.cs.allow-executable-memory</key> | |
| <true/> | |
| </dict> | |
| </plist> | |
| EOF | |
| - name: Notary macOS Binary | |
| if: runner.os == 'macOS' | |
| run: | | |
| codesign --force --entitlements="/tmp/entitlements.plist" -s "${{ secrets.DEVELOPER_ID }}" --options=runtime ${{env.PYTHON_FOLDER}}/bin/python | |
| codesign --force --entitlements="/tmp/entitlements.plist" -s "${{ secrets.DEVELOPER_ID }}" --options=runtime ${{env.PYTHON_FOLDER}}/bin/python3 | |
| # Code sign all .so files and .dylib files | |
| find ${{env.PYTHON_FOLDER}} -type f \( -name "*.so" -o -name "*.dylib" \) -exec codesign --force --entitlements="/tmp/entitlements.plist" -s "${{ secrets.DEVELOPER_ID }}" --options=runtime {} \; | |
| curl -sSfL https://raw.githubusercontent.com/anchore/quill/main/install.sh | sudo sh -s -- -b /usr/local/bin | |
| # Notarize the binary | |
| quill notarize ${{env.PYTHON_FOLDER}}/bin/python | |
| quill notarize ${{env.PYTHON_FOLDER}}/bin/python3 | |
| find ${{env.PYTHON_FOLDER}} -type f \( -name "*.so" -o -name "*.dylib" \) -exec quill notarize {} \; | |
| env: | |
| QUILL_NOTARY_KEY_ID: ${{ secrets.NOTARY_KEY_ID }} | |
| QUILL_NOTARY_ISSUER: ${{ secrets.NOTARY_ISSUER }} | |
| QUILL_NOTARY_KEY: "/tmp/notary-key.p8" | |
| # - name: Upload Artifact | |
| # #if : runner.os == 'windows' || runner.os == 'linux' | |
| # uses: actions/upload-artifact@v4 | |
| # with: | |
| # name: ${{env.MODEL_NAME}}-${{ matrix.os }}-${{ matrix.name }} | |
| # path: ${{env.PYTHON_FOLDER}} | |
| # include-hidden-files: true | |
| # compression-level: 9 | |
| - name: Upload Artifact MacOS | |
| if : runner.os == 'macOS' | |
| run: | | |
| brew install zip | |
| cd ${{env.PYTHON_FOLDER}} && zip -r venv.zip * | |
| conda create -y -n hf-upload python=3.11 | |
| source $HOME/miniconda3/bin/activate base | |
| conda init | |
| conda activate hf-upload | |
| python -m pip install hf-transfer huggingface_hub | |
| huggingface-cli login --token ${{ secrets.HUGGINGFACE_TOKEN_WRITE }} --add-to-git-credential | |
| huggingface-cli upload ${{env.HF_REPO}} venv.zip . --revision ${{env.HF_PREFIX_BRANCH}}-${{ matrix.os }}-${{ matrix.name }}" | |
| rm -rf venv.zip | |
| huggingface-cli logout | |
| - name: Upload Artifact Linux | |
| if : runner.os == 'linux' | |
| run: | | |
| sudo apt-get install -y zip | |
| cd ${{env.PYTHON_FOLDER}} && zip -r venv.zip * | |
| conda create -y -n hf-upload python=3.11 | |
| source $HOME/miniconda3/bin/activate base | |
| conda init | |
| conda activate hf-upload | |
| python -m pip install hf-transfer huggingface_hub | |
| huggingface-cli login --token ${{ secrets.HUGGINGFACE_TOKEN_WRITE }} --add-to-git-credential | |
| huggingface-cli upload ${{env.HF_REPO}} venv.zip . --revision ${{env.HF_PREFIX_BRANCH}}-${{ matrix.os }}-${{ matrix.name }}" | |
| rm -rf venv.zip | |
| huggingface-cli logout | |
| - name: Upload Artifact Windows | |
| if : runner.os == 'windows' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{env.MODEL_NAME}}-${{ matrix.os }}-${{ matrix.name }} | |
| path: ${{env.PYTHON_FOLDER}} | |
| include-hidden-files: true | |
| compression-level: 9 | |
| - name: Post Upload windows | |
| if : runner.os == 'windows' | |
| run: | | |
| rm ${{env.PYTHON_FOLDER}}/Scripts/python*.* | |
| - name: Remove Keychain | |
| continue-on-error: true | |
| if: always() && runner.os == 'macOS' | |
| run: | | |
| security delete-keychain signing_temp.keychain |