testing.yml

name: Tests

on:
  workflow_dispatch: null
  push:
    branches:
    - main
    paths-ignore:
    - 'docs/arch/**'
  pull_request:
    paths-ignore:
    - 'docs/arch/**'

jobs:
  build:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version:
        - "3.12"
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Get pip cache
      id: pip-cache
      run: |
        python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)"
    - name: pip cache
      uses: actions/cache@v3
      with:
        path: ${{ steps.pip-cache.outputs.dir }}
        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.cfg') }}
        restore-keys: |
          ${{ runner.os }}-pip-
    - name: Install dev dependencies
      run: |
        pip install -U pip setuptools wheel build
        pip install -U sbom4python
        pip install -U https://github.com/scitt-community/scitt-api-emulator/archive/e89a60584fa717382f279ae24b8a1a93d458bb4d.zip
        pip install -e .[dev]
        python -m pip freeze
    - name: Build
      run: |
        python -m build .
    - name: Generate SBOM
      id: generate-sbom
      uses: anthonyharrison/sbom4python@5b458354df89357bf0253e62ea4567b1807120e2
      with:
        python-version: ${{ matrix.python-version }}
        module-name: dffml
        output-directory: sbom
    - name: in-toto attestation for cyclonedx SBOM
      id: in-toto-cyclonedx
      env:
        MODULE_NAME: dffml
      run: |
        echo "attestation<<GITHUB_OUTPUT_EOF" >> $GITHUB_OUTPUT
        (python -m json.tool --sort-keys | tee -a $GITHUB_OUTPUT) <<EOF
        {
          "_type": "https://in-toto.io/Statement/v0.1",
          "subject": [
            {
              "name": "$(cd dist/ && echo *.tar.gz)",
              "digest": {"sha256": "$(cd dist/ && sha256sum $(echo *.tar.gz) | awk '{print $1}')"}
            },
            {
              "name": "$(cd dist/ && echo *.whl)",
              "digest": {"sha256": "$(cd dist/ && sha256sum $(echo *.whl) | awk '{print $1}')"}
            }
          ],
          "predicateType": "https://cyclonedx.org/bom/v1.4",
          "predicate": $(cat "${MODULE_NAME}-py${{ matrix.python-version }}.json")
        }
        EOF
        echo "GITHUB_OUTPUT_EOF" >> $GITHUB_OUTPUT
    - name: Checkout public-keys branch
      uses: actions/checkout@v4
      with:
        ref: public-keys
        path: public-keys
    - name: Generate keypair to sign SCITT statement
      id: scitt-gen-keypair
      run: |
        ssh-keygen -q -f ssh-private -t ecdsa -b 384 -N '' -C "$(head -n 100 /dev/urandom | sha384sum | awk '{print $1}')" -I "$(date -Iseconds)" <<<y
        cat ssh-private | python -c 'import sys; from cryptography.hazmat.primitives import serialization; print(serialization.load_ssh_private_key(sys.stdin.buffer.read(), password=None).private_bytes(encoding=serialization.Encoding.PEM, format=serialization.PrivateFormat.PKCS8, encryption_algorithm=serialization.NoEncryption()).decode().rstrip())' > private-key.pem
        # cat ssh-private.pub | tee -a public-keys/x509
    - name: SSH key as x509
      shell: python -u {0}
      run: |
        from cryptography import x509
        from cryptography.x509.oid import NameOID
        from cryptography.hazmat.primitives import hashes
        from cryptography.hazmat.backends import default_backend
        from cryptography.hazmat.primitives.asymmetric import rsa
        from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat
        from jwcrypto import jwk, jws
        import datetime
        import pathlib

        key = jwk.JWK.from_pem(pathlib.Path("private-key.pem").read_bytes())
        rsa_public_key = jwk.JWK.from_json(key.export_public())

        # Convert the JWK to a public key
        public_key = rsa_public_key.get_op_key('verify')

        # Create a builder for the X.509 certificate
        subject = issuer = x509.Name([
            x509.NameAttribute(NameOID.COUNTRY_NAME, "US"),
            x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, "Oregon"),
            x509.NameAttribute(NameOID.LOCALITY_NAME, "Portland"),
            x509.NameAttribute(NameOID.ORGANIZATION_NAME, "SCITT Emulator"),
            x509.NameAttribute(NameOID.COMMON_NAME, "example.com"),
        ])

        cert_builder = x509.CertificateBuilder(
            subject_name=subject,
            issuer_name=issuer,
            public_key=public_key,
            serial_number=x509.random_serial_number(),
            not_valid_before=datetime.datetime.utcnow(),
            not_valid_after=datetime.datetime.utcnow() + datetime.timedelta(days=1),  # Certificate valid for 1 day
            extensions=[]
        )

        # Self-sign the certificate with the private key
        private_key_op = key.get_op_key('sign')
        cert = cert_builder.sign(private_key=private_key_op, algorithm=hashes.SHA256(), backend=default_backend())

        # Serialize the certificate
        cert_pem = cert.public_bytes(encoding=Encoding.PEM)

        # Save the PEM encoded certificate
        pathlib.Path("public-keys", "x509").write_bytes(cert_pem)
    - name: Remove ssh private
      run: |
        rm -v ssh-private
    - name: Push new public key
      env:
        GH_TOKEN: ${{ github.token }}
      run: |
        set -xe
        cd public-keys
        gh auth setup-git
        git config --global --add safe.directory "${PWD}"
        git config --global user.email "actions@github.com"
        git config --global user.name "GitHub Actions"
        git add -A
        # If no delta clean exit
        git commit -sm "Snapshot" || exit 0
        git push -uf origin "HEAD:public-keys"
        # Wait for propagation
        # TODO x509 style polling?
        sleep 10s
    - name: Submit SBOM to SCITT
      id: scitt-submit-sbom
      uses: scitt-community/scitt-api-emulator@f1f5c16630a28511e970b6903fbc4c0db6c07654
      with:
        issuer: did:web:raw.githubusercontent.com:intel:dffml:public-keys:x509
        subject: pkg:github/${{ github.repository }}@${{ github.sha }}
        payload: ${{ steps.in-toto-cyclonedx.outputs.attestation }}
        private-key-pem: private-key.pem
        scitt-url: https://scitt.unstable.chadig.com
    - name: Create Pull Request
      if: ${{ steps.generate-sbom.outputs.changed }}
      uses: peter-evans/create-pull-request@v5.0.2
      with:
        commit-message: "chore: update SBOM for Python ${{ matrix.python-version }}"
        title: "chore: update SBOM for Python ${{ matrix.python-version }}"
        branch: chore-sbom-py${{ matrix.python-version }}
        delete-branch: true
        author: GitHub Actions <actions@github.com>
        add-paths: sbom
    - uses: openvex/generate-vex@159b7ee4845fb48f1991395ce8501d6263407360
      name: Run vexctl
      id: vexctl
      with:
        product: pkg:github/${{ github.repository }}@${{ github.sha }}
    - name: Submit OpenVEX to SCITT
      id: scitt-submit-openvex
      uses: scitt-community/scitt-api-emulator@f1f5c16630a28511e970b6903fbc4c0db6c07654
      with:
        issuer: did:web:raw.githubusercontent.com:intel:dffml:public-keys:x509
        subject: pkg:github/${{ github.repository }}@${{ github.sha }}
        payload: ${{ steps.vexctl.outputs.openvex }}
        private-key-pem: private-key.pem
        scitt-url: https://scitt.unstable.chadig.com
    - name: Remove private key used in keypair to sign SCITT statement
      run: |
        rm -v private-key.pem

  lint:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      max-parallel: 40
      matrix:
        check: [changelog, whitespace, commit, lines]
        python-version: [3.7]
        node-version: [12.x]

    steps:
    - uses: actions/checkout@v4
    - name: Checkout full upstream repo
      run: |
        git remote set-url origin https://github.com/intel/dffml
        git fetch --prune --unshallow
        git fetch --depth=1 origin +refs/tags/*:refs/tags/*
        git config --global user.email "dffml@intel.com"
        git config --global user.name "DFFML CI/CD"
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Use Node.js ${{ matrix.node-version }}
      uses: actions/setup-node@v4
      with:
        node-version: ${{ matrix.node-version }}
    - name: Get pip cache
      id: pip-cache
      run: |
        python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)"
    - name: pip cache
      uses: actions/cache@v3
      with:
        path: ${{ steps.pip-cache.outputs.dir }}
        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
        restore-keys: |
          ${{ runner.os }}-pip-
    - name: Install dependencies
      run: |
        set -x
        ./.ci/deps.sh ${{ matrix.check }}
    - name: Run check
      run: |
        export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
        SSH_DFFML_GH_PAGES=${{ secrets.SSH_DFFML_GH_PAGES }} ./.ci/run.sh ${{ matrix.check }}

  container:
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v4
    - name: Install dependencies
      run: |
        set -x
        curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
        sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
        sudo apt-get update && sudo apt-get install -y docker-ce docker-ce-cli containerd.io
    - name: Run check
      run: |
        ./.ci/run.sh container

  test:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      max-parallel: 100
      matrix:
        plugin:
        - model/daal4py
        - model/tensorflow
        - model/scratch
        - model/scikit
        - model/vowpalWabbit
        - model/autosklearn
        - model/xgboost
        - operations/binsec
        - operations/data
        - operations/deploy
        - operations/image
        - operations/nlp
        - source/mysql
        - feature/git
        - feature/auth
        - service/http
        - configloader/yaml
        - configloader/image
        - source/mongodb
        - entities/alice
        python-version:
        - "3.12"

    steps:
    - uses: actions/checkout@v4
    - name: Checkout full upstream repo
      run: |
        git remote set-url origin https://github.com/intel/dffml
        git fetch --prune --unshallow
        git fetch --depth=1 origin +refs/tags/*:refs/tags/*
        git config --global user.email "dffml@intel.com"
        git config --global user.name "DFFML CI/CD"
    - name: Remove unused software
      run: |
        sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Get pip cache
      id: pip-cache
      run: |
        python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)"
    - name: pip cache
      uses: actions/cache@v3
      with:
        path: ${{ steps.pip-cache.outputs.dir }}
        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
        restore-keys: |
          ${{ runner.os }}-pip-
    - name: shouldi test binary cache
      uses: actions/cache@v3
      with:
        path: examples/shouldi/tests/downloads
        key: ${{ hashFiles('examples/shouldi/tests/binaries.py') }}
        restore-keys: |
          ${{ runner.os }}-shouldi-test-binaries-
    - name: Install dependencies
      run: |
        set -x
        ./.ci/deps.sh ${{ matrix.plugin }}
    - name: Test
      run: |
        set -x
        export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
        export PYTHON=python${{ matrix.python-version }}
        export PLUGIN=${{ matrix.plugin }}
        export LOGGING=debug
        export PATH="${HOME}/.local/bin:${PATH}"
        export PYPI_TOKENS=$(mktemp)
        cat > ${PYPI_TOKENS} <<EOF
        examples/shouldi=${{ secrets.PYPI_SHOULDI }}
        model/daal4py=${{ secrets.PYPI_MODEL_DAAL4PY }}
        model/tensorflow=${{ secrets.PYPI_MODEL_TENSORFLOW }}
        model/tensorflow_hub=${{ secrets.PYPI_MODEL_TENSORFLOW_HUB }}
        model/scratch=${{ secrets.PYPI_MODEL_SCRATCH }}
        model/scikit=${{ secrets.PYPI_MODEL_SCIKIT }}
        model/spacy=${{ secrets.PYPI_MODEL_SPACY }}
        model/vowpalWabbit=${{ secrets.PYPI_MODEL_VOWPALWABBIT }}
        model/autosklearn=${{ secrets.PYPI_MODEL_AUTOSKLEARN }}
        model/pytorch=${{ secrets.PYPI_MODEL_PYTORCH }}
        model/xgboost=${{ secrets.PYPI_MODEL_XGBOOST }}
        source/mysql=${{ secrets.PYPI_SOURCE_MYSQL }}
        source/mongodb=${{ secrets.PYPI_SOURCE_MONGODB }}
        feature/git=${{ secrets.PYPI_FEATURE_GIT }}
        feature/auth=${{ secrets.PYPI_FEATURE_AUTH }}
        operations/binsec=${{ secrets.PYPI_OPERATIONS_BINSEC }}
        operations/data=${{ secrets.PYPI_OPERATIONS_DATA }}
        operations/deploy=${{ secrets.PYPI_OPERATIONS_DEPLOY }}
        operations/image=${{ secrets.PYPI_OPERATIONS_IMAGE }}
        operations/nlp=${{ secrets.PYPI_OPERATIONS_NLP }}
        service/http=${{ secrets.PYPI_SERVICE_HTTP }}
        configloader/yaml=${{ secrets.PYPI_CONFIG_YAML }}
        configloader/image=${{ secrets.PYPI_CONFIG_IMAGE }}
        EOF
        export TWINE_USERNAME=__token__
        if [ "x${PLUGIN}" = "x." ]; then
          export TWINE_PASSWORD=${{ secrets.PYPI_DFFML }}
        else
          export TWINE_PASSWORD=$(grep "${PLUGIN}=" "${PYPI_TOKENS}" | sed 's/^[^=]*=//g')
        fi
        rm ${PYPI_TOKENS}
        ./.ci/run.sh "${PLUGIN}"
        if [ "x${PLUGIN}" = "x." ]; then
          CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} codecov
        fi

  tutorials:
    runs-on: ubuntu-latest
    if: false
    strategy:
      fail-fast: false
      max-parallel: 100
      matrix:
        python-version: [3.7]
        docs:
        - docs/cli.rst
        - docs/examples/dataflows.rst
        - docs/examples/integration.rst
        - docs/examples/or_covid_data_by_county.rst
        - docs/examples/shouldi.rst
        - docs/examples/innersource/swportal.rst
        - docs/examples/innersource/microservice.rst
        - docs/examples/innersource/kubernetes.rst
        - docs/examples/icecream_sales.rst
        - docs/examples/data_cleanup/data_cleanup.rst
        - docs/examples/data_cleanup/data_cleanup_classfication.rst
        - docs/installation.rst
        - docs/troubleshooting.rst
        - docs/tutorials/accuracy/mse.rst
        - docs/tutorials/dataflows/io.rst
        - docs/tutorials/dataflows/nlp.rst
        - docs/tutorials/models/iris.rst
        - docs/tutorials/models/package.rst
        - docs/tutorials/models/docs.rst
        - docs/tutorials/models/slr.rst
        - docs/tutorials/sources/complex.rst
        - docs/tutorials/sources/file.rst

    steps:
    - uses: actions/checkout@v4
    - name: Checkout full upstream repo
      run: |
        git remote set-url origin https://github.com/intel/dffml
        git fetch --prune --unshallow
        git fetch --depth=1 origin +refs/tags/*:refs/tags/*
        git config --global user.email "dffml@intel.com"
        git config --global user.name "DFFML CI/CD"
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Get pip cache
      id: pip-cache
      run: |
        python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)"
    - name: pip cache
      uses: actions/cache@v3
      with:
        path: ${{ steps.pip-cache.outputs.dir }}
        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
        restore-keys: |
          ${{ runner.os }}-pip-
    - name: Install dependencies
      run: |
        set -x
        ./.ci/deps.sh .
    - name: Test
      run: |
        set -x
        export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
        export PYTHON=python${{ matrix.python-version }}
        export PLUGIN=${{ matrix.plugin }}
        export LOGGING=debug
        export PATH="${HOME}/.local/bin:${PATH}"
        ./.ci/run.sh consoletest ${{ matrix.docs }}

  macos:
    runs-on: macos-latest
    if: false
    strategy:
      fail-fast: false
      matrix:
        python-version: [3.7]
    steps:
    - uses: actions/checkout@v4
    - name: Checkout full upstream repo
      run: |
        git remote set-url origin https://github.com/intel/dffml
        git fetch --prune --unshallow
        git config --global user.email "dffml@intel.com"
        git config --global user.name "DFFML CI/CD"
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Get pip cache
      id: pip-cache
      run: |
        python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)"
    - name: pip cache
      uses: actions/cache@v3
      with:
        path: ${{ steps.pip-cache.outputs.dir }}
        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
        restore-keys: |
          ${{ runner.os }}-pip-
    - name: Setup DFFML
      run: |
        pip install -U pip setuptools wheel
        pip install -e .[dev]
        dffml service dev install -skip model/daal4py
        # XGBoost requires libomp on OSX
        brew install libomp
    - name: Test
      run: |
        python -m unittest discover -v
        python -m pip freeze