From dbd6ef6bfd1eff16bc4c550971f1deb078504aeb Mon Sep 17 00:00:00 2001 From: "Michael F." Date: Tue, 3 Dec 2024 17:35:58 +0100 Subject: [PATCH] fix docker, fix openapi gen --- .github/workflows/test.yml | 2 +- .openapi-generator-ignore | 2 +- .pre-commit-config.yaml | 28 ++++++++++----- Dockerfile | 69 ++++++++++++++++++------------------ Makefile | 20 +++++++---- openapi_spec/graphsense.yaml | 2 +- setup.py | 5 +-- tox.ini | 11 +++--- 8 files changed, 79 insertions(+), 60 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f359aeb2..0123e52f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,8 +15,8 @@ jobs: strategy: matrix: python-version: - - '3.9' - '3.10' + - '3.11' steps: - name: Checkout code diff --git a/.openapi-generator-ignore b/.openapi-generator-ignore index ad00540c..e55817c0 100644 --- a/.openapi-generator-ignore +++ b/.openapi-generator-ignore @@ -27,5 +27,5 @@ test-requirements.txt requirements.txt README.md tests/__init__.py -tests/conftests.py +tests/conftest.py openapi_server/__init__.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 311cd98e..91cff40c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,11 +32,23 @@ repos: id: debug-statements exclude: postmortemdebug.py - id: check-docstring-first -# - repo: local -# hooks: -# - id: pytest-check -# name: pytest-check -# entry: pytest -v -m "not slow" -# language: system -# pass_filenames: false -# always_run: true +- repo: local + hooks: + # - id: pytest-check + # name: pytest-check + # entry: pytest -x -vv + # language: system + # pass_filenames: false + # always_run: true + # - id: generate-openapi-server + # name: Generate OpenAPI Server + # entry: make generate-openapi-server + # language: system + # pass_filenames: false + # always_run: true + - id: update-openapi-version + name: Update OpenAPI Version + entry: make update-openapi-version + pass_filenames: false + language: system + always_run: true diff --git a/Dockerfile b/Dockerfile index 5baceb3c..266532ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.15 +FROM python:3.11-alpine3.20 LABEL org.opencontainers.image.title="graphsense-rest" LABEL org.opencontainers.image.maintainer="contact@ikna.io" LABEL org.opencontainers.image.url="https://www.ikna.io/" @@ -13,50 +13,49 @@ ENV CONFIG_FILE=./instance/config.yaml RUN mkdir -p /srv/graphsense-rest/ COPY requirements.txt /srv/graphsense-rest/ - -RUN apk --no-cache --update add \ - bash \ - python3 \ - python3-dev \ - py3-gunicorn \ - shadow \ - g++ \ - postgresql-dev && \ - useradd -r -m -u 10000 dockeruser && \ - apk --no-cache --update --virtual build-dependendencies add \ - gcc \ - linux-headers \ - musl-dev \ - pcre-dev \ - libpq-dev \ - python3-dev && \ - python3 -m ensurepip && \ - rm -r /usr/lib/python*/ensurepip && \ - pip3 install --upgrade pip setuptools && \ - pip3 install -r /srv/graphsense-rest/requirements.txt && \ - apk del build-dependendencies && \ - rm -rf /root/.cache - -COPY conf/gunicorn-conf.py /home/dockeruser/gunicorn-conf.py +COPY conf/gunicorn-conf.py /srv/graphsense-rest/gunicorn-conf.py COPY setup.py /srv/graphsense-rest/ COPY README.md /srv/graphsense-rest/ - -RUN mkdir /var/lib/graphsense-rest && \ - chown dockeruser /var/lib/graphsense-rest && \ - pip3 install /srv/graphsense-rest/ - COPY gsrest /srv/graphsense-rest/gsrest COPY openapi_server /srv/graphsense-rest/openapi_server -# COPY instance /srv/graphsense-rest/instance +# https://stackoverflow.com/questions/77490435/attributeerror-cython-sources +RUN echo "cython<3" > /tmp/constraint.txt +RUN apk --no-cache --update add \ + bash \ + shadow \ + git \ + postgresql-dev +RUN useradd -r -m -u 10000 dockeruser +RUN apk --no-cache --update --virtual build-dependendencies add \ + gcc \ + g++ \ + linux-headers \ + musl-dev \ + pcre-dev \ + libpq-dev \ + python3-dev + + +RUN PIP_CONSTRAINT=/tmp/constraint.txt pip3 install --upgrade pip setuptools +RUN apk del build-dependendencies +RUN chown dockeruser /srv/graphsense-rest USER dockeruser - WORKDIR /srv/graphsense-rest +ENV VIRTUAL_ENV=/srv/graphsense-rest/venv +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +RUN python3 -m venv $VIRTUAL_ENV && \ + PIP_CONSTRAINT=/tmp/constraint.txt pip3 install gunicorn && \ + PIP_CONSTRAINT=/tmp/constraint.txt pip3 install -r /srv/graphsense-rest/requirements.txt && \ + PIP_CONSTRAINT=/tmp/constraint.txt pip3 install /srv/graphsense-rest/ + + +# COPY instance /srv/graphsense-rest/instance RUN find gsrest/plugins -name requirements.txt -exec pip install -r {} \; -CMD find gsrest/plugins -name requirements.txt -exec pip install -r {} \; && /usr/bin/gunicorn \ - -c /home/dockeruser/gunicorn-conf.py \ +CMD find gsrest/plugins -name requirements.txt -exec pip install -r {} \; && gunicorn \ + -c /srv/graphsense-rest/gunicorn-conf.py \ "openapi_server:main('${CONFIG_FILE}')" \ --worker-class aiohttp.GunicornWebWorker diff --git a/Makefile b/Makefile index 41fffab6..b6de93a2 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,9 @@ all: format lint -include .env +-include .env -GS_REST_SERVICE_VERSION ?= "24.11.0" +GS_REST_SERVICE_VERSION ?= "1.9.3-ts2-dev1" +GS_REST_SERVICE_VERSIONM ?= "1.9.3-ts2-dev1" GS_REST_DEV_PORT ?= 9000 test: @@ -36,12 +37,9 @@ serve: build-docker: docker build -t graphsense-rest . -serve-docker: build-docker +serve-docker: docker run -it --network='host' -v "${PWD}/instance/config.yaml:/config.yaml:Z" -e CONFIG_FILE=/config.yaml localhost/graphsense-rest:latest -drop-integration-db: - docker stop cassandra_mock; docker stop tagstore_mock; rm ./tests/.runcass ./tests/.runts - generate-openapi-server: docker run --rm \ -v "${PWD}:/build:Z" \ @@ -59,5 +57,13 @@ run-designer: docker run -d -p 8080:8080 swaggerapi/swagger-editor echo 'Designer UI is running on port 8080' +update-openapi-version: + sed -i '/^info:/,/^ version:/s/^\(\s*version:\s*\).*/\1$(GS_REST_SERVICE_VERSIONM)/' openapi_spec/graphsense.yaml + +tag-version: + -git diff --exit-code && git diff --staged --exit-code && git tag -a v$(GS_REST_SERVICE_VERSIONM) -m 'Release v$(GS_REST_SERVICE_VERSION)' || (echo "Repo is dirty please commit first" && exit 1) + git diff --exit-code && git diff --staged --exit-code && git tag -a v$(GS_REST_SERVICE_VERSION) -m 'Release v$(GS_REST_SERVICE_VERSION)' || (echo "Repo is dirty please commit first" && exit 1) + + -.PHONY: format lint test test-all-env serve drop-integration-db generate-openapi-server get-openapi-spec-from-upstream serve-docker pre-commit install-dev +.PHONY: format lint test test-all-env serve generate-openapi-server get-openapi-spec-from-upstream serve-docker pre-commit install-dev update-openapi-version tag-version diff --git a/openapi_spec/graphsense.yaml b/openapi_spec/graphsense.yaml index f924d6ea..150790da 100644 --- a/openapi_spec/graphsense.yaml +++ b/openapi_spec/graphsense.yaml @@ -5,7 +5,7 @@ info: contact: name: Iknaio Cryptoasset Analytics GmbH email: contact@ikna.io - version: "1.8.0" + version: "1.9.3-ts2-dev1" servers: - url: 'https://api.ikna.io' paths: diff --git a/setup.py b/setup.py index 568b1922..d33ace29 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,10 @@ # coding: utf-8 +import sys from setuptools import setup, find_packages NAME = "openapi_server" -VERSION = "24.11.0" +VERSION = "1.9.3-ts2-dev1" # To install the library, run the following # @@ -27,7 +28,7 @@ description="GraphSense API", author_email="contact@ikna.io", keywords=["OpenAPI", "GraphSense API"], - python_requires='>=3.8', + python_requires='>=3.10', install_requires=REQUIRES, packages=find_packages(), package_data={'': ['openapi/openapi.yaml']}, diff --git a/tox.ini b/tox.ini index a0333e78..8bc2a656 100644 --- a/tox.ini +++ b/tox.ini @@ -1,21 +1,22 @@ [tox] skipsdist=True -envlist = py310,py311 +envlist = py39,py310 ;requires=tox-venv [gh-actions] python = + 3.9: py39 3.10: py310 - 3.11: py311 [pytest] asyncio_mode=auto -asyncio_default_fixture_loop_scope="function" [testenv] -passenv = * +allowlist_externals=bash deps=-r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt {toxinidir} + commands= - pytest -x --cov=openapi_server -vv + bash -c "cd {toxinidir}/tests/ && make ORGANIZATION={posargs:graphsense} .runcass .runts" + pytest -x --cov=openapi_server