From 6de1eeb1c27a90fba508efe7443b6599ed91b6db Mon Sep 17 00:00:00 2001 From: eskerda Date: Tue, 17 Sep 2024 14:57:35 +0200 Subject: [PATCH] add a test that only tests changes with 'master' --- .github/workflows/test_changes.yml | 41 +++++++++++++++ .github/workflows/test_python2.yml | 2 +- Makefile | 6 ++- requirements.txt | 1 + tests/conftest.py | 3 ++ tests/test_changes.py | 80 ++++++++++++++++++++++++++++++ utils/github-summary.tpl.md | 4 ++ utils/report.tpl.md | 4 ++ 8 files changed, 139 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/test_changes.yml create mode 100644 tests/test_changes.py diff --git a/.github/workflows/test_changes.yml b/.github/workflows/test_changes.yml new file mode 100644 index 000000000..3d9d6ea5d --- /dev/null +++ b/.github/workflows/test_changes.yml @@ -0,0 +1,41 @@ +name: networks that changed on this branch +run-name: Check changes between branch and master + +env: + PYBIKES_CYCLOCITY: ${{ secrets.PYBIKES_CYCLOCITY }} + PYBIKES_BYSYKKEL: ${{ secrets.PYBIKES_BYSYKKEL }} + PYBIKES_WEELO_CLIENT_ID: ${{ secrets.PYBIKES_WEELO_CLIENT_ID }} + PYBIKES_WEELO_CLIENT_SECRET: ${{ secrets.PYBIKES_WEELO_CLIENT_SECRET }} + PYBIKES_DEUTSCHEBAHN_CLIENT_ID: ${{ secrets.PYBIKES_DEUTSCHEBAHN_CLIENT_ID }} + PYBIKES_DEUTSCHEBAHN_CLIENT_SECRET: ${{ secrets.PYBIKES_DEUTSCHEBAHN_CLIENT_SECRET }} + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: make install + - name: Test instance changes on this branch + run: make test-changes T_FLAGS+='-n 10 --json-report --json-report-file=report/report.json' + # Only run summary once + - name: summary + if: (success() || failure()) && matrix.python-version == '3.8' + run: | + make github-summary >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/test_python2.yml b/.github/workflows/test_python2.yml index dec1f900c..1f0d0228b 100644 --- a/.github/workflows/test_python2.yml +++ b/.github/workflows/test_python2.yml @@ -1,7 +1,7 @@ # There are still parts of the project using python2.7 so it's a good idea # to check if it works at all -name: pybikes +name: pybikes (2.7) env: PYBIKES_CYCLOCITY: ${{ secrets.PYBIKES_CYCLOCITY }} diff --git a/Makefile b/Makefile index 57c58f6df..9115379fe 100644 --- a/Makefile +++ b/Makefile @@ -7,12 +7,16 @@ install: .PHONY: test test: install - pytest tests -m 'not update' $(T_FLAGS) + pytest tests -m 'not update and not changes' $(T_FLAGS) .PHONY: test-update test-update: install pytest tests -m update $(T_FLAGS) +.PHONY: test-changes +test-changes: install + pytest tests -m changes $(T_FLAGS) + .PHONY: lint lint: flake8 pybikes tests --count --select=E9,F63,F7,F82 --show-source --statistics diff --git a/requirements.txt b/requirements.txt index e9287a5bd..169730b00 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ pytest pytest-xdist mock; python_version < '3.3' +gitpython # Lint flake8 diff --git a/tests/conftest.py b/tests/conftest.py index 2616823a8..a2db9a94c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,3 +2,6 @@ def pytest_configure(config): config.addinivalue_line( "markers", "update: mark a test that uses network and might fail" ) + config.addinivalue_line( + "markers", "changes: mark a test as a gen test about changes" + ) diff --git a/tests/test_changes.py b/tests/test_changes.py new file mode 100644 index 000000000..169d876be --- /dev/null +++ b/tests/test_changes.py @@ -0,0 +1,80 @@ +""" Very hacky script that runs tests on bike share systems that have +changed on a specific branch. Checks for changes on data files and +classes containing bike share systems. + +This is meant for red explicit fails on changes in CI +""" + +import re +import os +import git +import inspect + +from pytest import mark +from warnings import warn + +from pybikes import BikeShareSystem +from pybikes.data import get_instances +from pybikes.compat import resources + +from tests.test_instances import get_test_cls + + +def is_system(mod, obj): + if not inspect.isclass(obj): + return False + + # Only declared in module + if obj.__module__ != mod.__name__: + return False + + if not issubclass(obj, BikeShareSystem): + return False + + if obj == BikeShareSystem: + return False + + return True + + +def generate_tests_from_changes(branch): + + # this will fail in python < 3.8 + from importlib.util import spec_from_file_location, module_from_spec + + # this might fail if branch not in git dir + g = git.cmd.Git(os.getcwd()) + changed_files = g.diff('--name-only', branch).splitlines() + clss = set() + + for file in changed_files: + if re.match(r'pybikes/data/.*\.json', file): + # Extract classes from json file + match = re.search(r'pybikes/data/(.*)\.json', file) + schema = match.group(1) + [clss.add(cls) for cls, _ in get_instances(schema)] + elif re.match(r'pybikes/.*\.py', file): + # Extract bike share classes from file + spec = spec_from_file_location('some.mod', file) + mod = module_from_spec(spec) + spec.loader.exec_module(mod) + systems = filter(lambda m: is_system(mod, m[1]), inspect.getmembers(mod)) + [clss.add(cls) for cls, _ in systems] + + for cls in sorted(clss): + test_cls = get_test_cls(cls) + # decorate with pytest mark 'changes' + test_cls = mark.changes(test_cls) + globals()[test_cls.__name__] = test_cls + + +# non-optimal not-required +try: + generate_tests_from_changes('origin/master') +except Exception as e: + warn("Failed generating tests from branch changes: " + str(e)) + +# Force pytest to succeed when no tests are marked with 'changes' +@mark.changes +def test_dummy(): + assert True diff --git a/utils/github-summary.tpl.md b/utils/github-summary.tpl.md index 0723fdcd0..3369766bc 100644 --- a/utils/github-summary.tpl.md +++ b/utils/github-summary.tpl.md @@ -1,5 +1,6 @@ # systems status report (Python {{ version }}) +{% if systems %} | overall success rate | {{ int((health.passed / health.total) * 100) }}% | |-|-| | passed | {{ health.passed }} | @@ -55,3 +56,6 @@ {% endfor %} {% endfor %} {% endfor %} +{% else %} +No systems +{% endif %} diff --git a/utils/report.tpl.md b/utils/report.tpl.md index d87ac7616..c3c6fb3fc 100644 --- a/utils/report.tpl.md +++ b/utils/report.tpl.md @@ -1,4 +1,5 @@ # systems status report (Python {{ version }}) +{% if systems %} | overall success rate | {{ int((health.passed / health.total) * 100) }}% | |-|-| @@ -54,3 +55,6 @@ {% endfor %} {% endfor %} {% endfor %} +{% else %} +No systems +{% endif %}