check changes

.github/workflows/test.yml

@@ -15,10 +15,11 @@ on:
     branches: [ "master" ]
 
 jobs:
-  test:
+  unit:
+    name: Unit tests
     runs-on: ubuntu-latest
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
     steps:
@@ -33,11 +34,68 @@ jobs:
       run: make lint
     - name: test
       run: make test
-    - name: test update
+  instance-changes:
+    name: Instance tests (only changes)
+    runs-on: ubuntu-latest
+    needs: [unit]
+    strategy:
+      fail-fast: false
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.8'
+    - name: Install dependencies
+      run: make install
+    - name: Test instance changes on this branch
+      run: make test-changes T_FLAGS+='-n 10 --json-report --json-report-file=report/report.json'
+    # Only run summary once
+    - name: all summary
+      if: (success() || failure())
       run: |
-        make report T_FLAGS+='-n 10' || true
+          make github-summary >> $GITHUB_STEP_SUMMARY
+  instance-tests:
+    name: Instance tests (all)
+    runs-on: ubuntu-latest
+    needs: [unit]
+    strategy:
+      fail-fast: false
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.8'
+    - name: Install dependencies
+      run: make install
+    - name: Test all instances
+      run: |
+        make report! T_FLAGS+='-n 10' || true
     # Only run summary once
     - name: summary
-      if: (success() || failure()) && matrix.python-version == '3.8'
+      if: (success() || failure())
       run: |
           make github-summary >> $GITHUB_STEP_SUMMARY
+
+  # This is more like a watchdog to know if the library still works with
+  # python 2 or not. Not that it really matters
+  python2:
+    name: Unit tests (2.7)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 2.7
+      uses: MatteoH2O1999/setup-python@v4
+      with:
+        python-version: '2.7'
+        allow-build: info
+        cache-build: true
+    - name: Install dependencies
+      run: make install
+    - name: lint
+      run: make lint
+    - name: test
+      run: make test

Makefile

@@ -1,4 +1,5 @@
 T_FLAGS =
+R_FILE ?= report.json
 
 .PHONY: install
 install:
@@ -7,38 +8,45 @@ install:
 
 .PHONY: test
 test: install
-	pytest tests -m 'not update' $(T_FLAGS)
+	pytest tests -m 'not update and not changes' $(T_FLAGS)
 
 .PHONY: test-update
 test-update: install
 	pytest tests -m update $(T_FLAGS)
 
+.PHONY: test-changes
+test-changes: install
+	pytest tests -m changes $(T_FLAGS)
+
 .PHONY: lint
 lint:
 	flake8 pybikes tests --count --select=E9,F63,F7,F82 --show-source --statistics
 	flake8 pybikes tests --count --exit-zero --max-complexity=10 --max-line-length=88 --statistics
 
-report/report.json:
-	pytest tests -m update --json-report --json-report-file=report/report.json $(T_FLAGS)
+report/$(R_FILE):
+	pytest tests -m update $(T_FLAGS) --json-report --json-report-file=report/$(R_FILE)
 
 .PHONY: report
-report: report/report.json
+report: report/$(R_FILE)
+
+.PHONY: report!
+report!: clean report
 
 .PHONY: summary
-summary: report/report.json
-	@./utils/report.py report/report.json
+summary: report/$(R_FILE)
+	@./utils/report.py report/$(R_FILE)
 
 .PHONY: map
-map: report/report.json
-	@./utils/report.py report/report.json --template utils/map.tpl.html > report/map.html
+map: report/$(R_FILE)
+	@./utils/report.py report/$(R_FILE) --template utils/map.tpl.html > report/map.html
 	@open report/map.html || xdg-open report/map.html
 
 .PHONY: map!
 map!: clean map
 
 .PHONY: github-summary
-github-summary: report/report.json
-	@./utils/report.py report/report.json --template utils/github-summary.tpl.md
+github-summary: report/$(R_FILE)
+	@./utils/report.py report/$(R_FILE) --template utils/github-summary.tpl.md
 
 .PHONY: clean
 clean: clean-report

requirements.txt

@@ -4,6 +4,7 @@
 pytest
 pytest-xdist
 mock; python_version < '3.3'
+gitpython
 
 # Lint
 flake8

tests/conftest.py

@@ -2,3 +2,6 @@ def pytest_configure(config):
     config.addinivalue_line(
         "markers", "update: mark a test that uses network and might fail"
     )
+    config.addinivalue_line(
+        "markers", "changes: mark a test as a gen test about changes"
+    )

tests/test_changes.py

@@ -0,0 +1,80 @@
+""" Very hacky script that runs tests on bike share systems that have
+changed on a specific branch. Checks for changes on data files and
+classes containing bike share systems.
+
+This is meant for red explicit fails on changes in CI
+"""
+
+import re
+import os
+import git
+import inspect
+
+from pytest import mark
+from warnings import warn
+
+from pybikes import BikeShareSystem
+from pybikes.data import get_instances
+from pybikes.compat import resources
+
+from tests.test_instances import get_test_cls
+
+
+def is_system(mod, obj):
+    if not inspect.isclass(obj):
+        return False
+
+    # Only declared in module
+    if obj.__module__ != mod.__name__:
+        return False
+
+    if not issubclass(obj, BikeShareSystem):
+        return False
+
+    if obj == BikeShareSystem:
+        return False
+
+    return True
+
+
+def generate_tests_from_changes(branch):
+
+    # this will fail in python < 3.8
+    from importlib.util import spec_from_file_location, module_from_spec
+
+    # this might fail if branch not in git dir
+    g = git.cmd.Git(os.getcwd())
+    changed_files = g.diff('--name-only', branch).splitlines()
+    clss = set()
+
+    for file in changed_files:
+        if re.match(r'pybikes/data/.*\.json', file):
+            # Extract classes from json file
+            match = re.search(r'pybikes/data/(.*)\.json', file)
+            schema = match.group(1)
+            [clss.add(cls) for cls, _ in get_instances(schema)]
+        elif re.match(r'pybikes/.*\.py', file):
+            # Extract bike share classes from file
+            spec = spec_from_file_location('some.mod', file)
+            mod = module_from_spec(spec)
+            spec.loader.exec_module(mod)
+            systems = filter(lambda m: is_system(mod, m[1]), inspect.getmembers(mod))
+            [clss.add(cls) for cls, _ in systems]
+
+    for cls in sorted(clss):
+        test_cls = get_test_cls(cls)
+        # decorate with pytest mark 'changes'
+        test_cls = mark.changes(test_cls)
+        globals()[test_cls.__name__] = test_cls
+
+
+# non-optimal not-required
+try:
+    generate_tests_from_changes('origin/master')
+except Exception as e:
+    warn("Failed generating tests from branch changes: " + str(e))
+
+# Force pytest to succeed when no tests are marked with 'changes'
+@mark.changes
+def test_dummy():
+    assert True

utils/github-summary.tpl.md

@@ -1,5 +1,6 @@
 # systems status report (Python {{ version }})
 
+{% if systems %}
 | overall success rate | {{ int((health.passed / health.total) * 100) }}% |
 |-|-|
 | passed | {{ health.passed }} |
@@ -55,3 +56,6 @@
         {% endfor %}
     {% endfor %}
 {% endfor %}
+{% else %}
+No systems
+{% endif %}

utils/report.tpl.md

@@ -1,4 +1,5 @@
 # systems status report (Python {{ version }})
+{% if systems %}
 
 | overall success rate | {{ int((health.passed / health.total) * 100) }}% |
 |-|-|
@@ -54,3 +55,6 @@
         {% endfor %}
     {% endfor %}
 {% endfor %}
+{% else %}
+No systems
+{% endif %}