Merge branch 'develop' into 'master'

support python3.12 and update CUDA base image

See merge request pitchtrack/pitch-detectors!59

.github/workflows/ci.yml

@@ -3,36 +3,36 @@ name: ci
 on: push
 
 jobs:
-  test:
-    runs-on: [self-hosted, gpu]
-    steps:
-      - uses: actions/checkout@v3
+  # test:
+  #   runs-on: [self-hosted, gpu]
+  #   steps:
+  #     - uses: actions/checkout@v4
 
-      - name: test gpu is available
-        run: nvidia-smi
+  #     - name: test gpu is available
+  #       run: nvidia-smi
 
-      - name: build image
-        run: make build
+  #     - name: build image
+  #       run: make build
 
-      - name: test-no-docker
-        run: make test-no-docker
+  #     - name: test-no-docker
+  #       run: make test-no-docker
 
-      - name: test
-        run: make test
+  #     - name: test
+  #       run: make test
 
-      - name: test-no-gpu
-        run: make test-no-gpu
+  #     - name: test-no-gpu
+  #       run: make test-no-gpu
 
   publish-to-pypi-and-github-release:
     if: "startsWith(github.ref, 'refs/tags/')"
     runs-on: ubuntu-latest
-    needs: test
+    # needs: test
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
+          python-version: '3.12'
 
       - name: Install pypa/build
         run: python -m pip install --upgrade setuptools build twine

.gitlab-ci.yml

@@ -0,0 +1,47 @@
+stages:
+  # - build
+  - test
+
+variables:
+  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+  PRE_COMMIT_HOME: "$CI_PROJECT_DIR/.cache/pre-commit"
+  RUFF_CACHE_DIR: "$CI_PROJECT_DIR/.cache/ruff_cache"
+  MYPY_CACHE_DIR: "$CI_PROJECT_DIR/.cache/mypy_cache"
+
+cache:
+  key: $CI_PROJECT_NAME
+  paths:
+    - .cache/
+
+
+test:
+  stage: test
+  needs: []
+  tags:
+    - u60-docker-gpu
+  image: tandav/pitch-detectors:12.4.1-cudnn-devel-ubuntu22.0-python3.12
+  variables:
+    PITCH_DETECTORS_SPICE_MODEL_PATH: /models/spice_model
+    PITCH_DETECTORS_PENN_CHECKPOINT_PATH: /models/fcnf0++.pt
+  script:
+    - export $(grep -v '^#' $S3_ENV | xargs) && python scripts/download_models.py
+    - pytest --cov pitch_detectors --cov-report term --cov-report xml --junitxml report.xml
+  coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
+  artifacts:
+    when: always
+    expire_in: 1 week
+    reports:
+      coverage_report:
+        coverage_format: cobertura
+        path: coverage.xml
+      junit: report.xml
+
+lint:
+  stage: test
+  needs: []
+  image: python:3.12@sha256:fce9bc7648ef917a5ab67176cf1c7eb41b110452e259736144bc22f32f3aa622
+  variables:
+    PIP_INDEX_URL: https://pypi.tandav.me/index/
+  script:
+    - pip install .[dev]
+    - pre-commit run --all-files

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.6.0
     hooks:
       - id: check-added-large-files
       - id: check-yaml
@@ -18,47 +18,46 @@ repos:
       - id: detect-private-key
       - id: double-quote-string-fixer
       - id: name-tests-test
-      - id: requirements-txt-fixer
 
   - repo: https://github.com/asottile/add-trailing-comma
-    rev: v2.3.0
+    rev: v3.1.0
     hooks:
       - id: add-trailing-comma
 
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.1.0
+    rev: v3.16.0
     hooks:
       - id: pyupgrade
 
-  - repo: https://github.com/pre-commit/mirrors-autopep8
-    rev: v1.7.0
+  - repo: https://github.com/hhatto/autopep8
+    rev: v2.3.1
     hooks:
       - id: autopep8
 
   - repo: https://github.com/PyCQA/autoflake
-    rev: v1.7.6
+    rev: v2.3.1
     hooks:
       - id: autoflake
 
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.0.254
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.5.0
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
 
   - repo: https://github.com/PyCQA/pylint
-    rev: v2.17.0
+    rev: v3.2.4
     hooks:
       - id: pylint
         additional_dependencies: ["pylint-per-file-ignores"]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.982
+    rev: v1.10.1
     hooks:
     - id: mypy
       additional_dependencies: [types-redis, types-tabulate, pydantic]
 
-  - repo: https://github.com/RobertCraigie/pyright-python
-    rev: v1.1.302
-    hooks:
-    - id: pyright
+  # - repo: https://github.com/RobertCraigie/pyright-python
+  #   rev: v1.1.369
+  #   hooks:
+  #   - id: pyright

README.md

@@ -1,3 +1,5 @@
+[![pipeline status](https://gitlab.tandav.me/pitchtrack/pitch-detectors/badges/master/pipeline.svg)](https://gitlab.tandav.me/pitchtrack/pitch-detectors/-/commits/master)
+
 # pitch-detectors
 collection of pitch (f0, fundamental frequency) detection algorithms with unified interface
 
@@ -50,3 +52,8 @@ plt.show()
 ## additional features
 - [ ] robust (vote + median) ensemble algorithm using all models
 - [ ] json import/export
+
+## notes:
+Tests are running in subprocess (using `scripts/run_algorithm.py`) to avoid pytorch cuda import caching.
+It's difficult to disable gpu after it has been initialized. (https://github.com/pytorch/pytorch/issues/9158)
+It is also difficult to set correct PATH and LD_LIBRARY_PATH without a subprocess.

Taskfile.yml

@@ -0,0 +1,51 @@
+version: '3'
+dotenv: ['.env']
+vars:
+  BASE_IMAGE: tandav/pitch-detectors-base:12.4.1-cudnn-devel-ubuntu22.0-python3.12
+  IMAGE: tandav/pitch-detectors:12.4.1-cudnn-devel-ubuntu22.0-python3.12
+
+tasks:
+  build-base:
+    cmd: docker build --tag {{.BASE_IMAGE}} --file docker/base.dockerfile .
+
+  push-base:
+    cmd: docker push {{.BASE_IMAGE}}
+
+  build:
+    cmd: docker build --build-arg="BASE_IMAGE={{.BASE_IMAGE}}" --tag {{.IMAGE}} --file docker/pitch-detectors.dockerfile .
+
+  push:
+    cmd: docker push {{.IMAGE}}
+
+  test:
+    deps: [build]
+    cmd: >
+      docker run --rm -t --gpus all
+      -v /media/tandav/sg8tb1/downloads-archive/libmv-data/spice_model:/spice_model:ro
+      -v /media/tandav/sg8tb1/downloads-archive/libmv-data/fcnf0++.pt:/fcnf0++.pt:ro
+      {{.IMAGE}}
+      pytest -v
+
+  test-no-docker:
+    cmd: pytest -v
+
+  freeze:
+    cmd: docker run --rm -t --gpus all {{.IMAGE}} /venv/bin/pip freeze > freeze.txt
+
+  bumpver:
+    desc: 'Bump version. Pass --<part>. Usage example: task bumpver -- --minor'
+    cmds:
+      - bumpver update --no-fetch {{.CLI_ARGS}}
+
+  evaluation:
+    deps: [build]
+    cmd: >
+      docker run --rm -t --gpus all
+      -e PITCH_DETECTORS_GPU=true
+      -e REDIS_URL={{.REDIS_URL}}
+      -v /media/tandav/sg8tb1/downloads-archive/f0-datasets:/app/f0-datasets:ro
+      {{.IMAGE}}
+      python -m pitch_detectors.evaluation
+
+  table:
+    cmd: python -m pitch_detectors.evaluation.table

docker/base.dockerfile

@@ -0,0 +1,21 @@
+FROM nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04@sha256:0a1cb6e7bd047a1067efe14efdf0276352d5ca643dfd77963dab1a4f05a003a4
+
+# https://github.com/NVIDIA/nvidia-docker/wiki/Usage
+# https://github.com/NVIDIA/nvidia-docker/issues/531
+ENV NVIDIA_DRIVER_CAPABILITIES compute,video,utility
+ENV DEBIAN_FRONTEND=noninteractive
+
+ARG PYTHON_VERSION=3.12
+
+RUN apt-get update && \
+    apt-get install -y software-properties-common && \
+    add-apt-repository -y ppa:deadsnakes/ppa && \
+    apt-get install -y python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv libsndfile-dev libasound-dev portaudio19-dev
+
+# this is only need for crepe @ git+https://github.com/tandav/crepe
+RUN apt-get install -y git
+
+# https://pythonspeed.com/articles/activate-virtualenv-dockerfile/
+ENV VIRTUAL_ENV=/venv
+RUN python${PYTHON_VERSION} -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"

docker/pitch-detectors.dockerfile

@@ -0,0 +1,18 @@
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+
+WORKDIR /app
+COPY pyproject.toml .
+ENV PIP_INDEX_URL=https://pypi.tandav.me/index/
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install --upgrade pip setuptools wheel && \
+    pip install .[dev]
+
+COPY pitch_detectors /app/pitch_detectors
+
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install --no-deps .
+
+COPY tests /app/tests
+COPY scripts/ /app/scripts
+COPY data /app/data