Merge branch 'main' into feature/pnast/mic-5679-reusable-pipeline

.github/workflows/build.yml

@@ -11,11 +11,25 @@ name: build
 on: [push, pull_request, workflow_dispatch]
 
 jobs:
+  get-python-versions:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Install jq
+        run: sudo apt-get install jq
+      - name: Get Python versions
+        id: set-matrix
+        run: |
+          echo "MATRIX_RESULT=$(jq -c . python_versions.json)" >> $GITHUB_ENV
+    outputs:
+      matrix: ${{ env.MATRIX_RESULT }}
   build:
+    needs: get-python-versions
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.11", "3.12"]
+        python-version: ${{ fromJSON(needs.get-python-versions.outputs.matrix) }}
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}

.github/workflows/update_readme.yml

@@ -0,0 +1,33 @@
+# -----------------------------------------------------------------------------
+#   - invoked on push to any branch
+# -----------------------------------------------------------------------------
+    name: update README
+
+    on: push
+
+    jobs:
+      update-readme:
+        runs-on: ubuntu-latest
+        steps:
+          - name: Checkout code
+            uses: actions/checkout@v2
+          - name: Set up Python
+            uses: actions/setup-python@v2
+            with:
+              python-version: 3.11
+          - name: Update README
+            run: |
+              pip install packaging
+              python update_readme.py
+          - name: Commit and push changes
+            run: |
+              git config --local user.email "[email protected]"
+              git config --local user.name "github-actions"
+              git diff --quiet && git diff --staged --quiet || (
+                git add README.rst
+                git commit -am "update README with supported Python versions"
+                git pull --rebase origin ${{ github.ref_name }}
+                git push origin ${{ github.ref_name }}
+              )
+            env:
+              GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

CHANGELOG.rst

@@ -1,3 +1,9 @@
+**0.1.3 - 1/7/25**
+
+ - Validate currently-installed python version during setup
+ - Automatically update README when supported python versions change
+ - Automatically extract github actions supported python version test matrix
+
 **0.1.2 - 12/16/24**
 
  - Add optional arg to pass allowable schemas to the Config constructor

Makefile

@@ -63,31 +63,25 @@ install: # Install setuptools, install this package in editable mode
 format: setup.py pyproject.toml $(MAKE_SOURCES) # Run the code formatter and import sorter
 	-black $(LOCATIONS)
 	-isort $(LOCATIONS)
-	@echo "Ignore, Created by Makefile, `date`" > $@
 
 lint: .flake8 .bandit $(MAKE_SOURCES) # Run the code linter and package security vulnerability checker
 	-flake8 $(LOCATIONS)
 	-safety check
-	@echo "Ignore, Created by Makefile, `date`" > $@
 
 typecheck: pytype.cfg $(MAKE_SOURCES) # Run the type checker
 	-pytype --config=pytype.cfg $(LOCATIONS)
-	@echo "Ignore, Created by Makefile, `date`" > $@
 
 e2e: $(MAKE_SOURCES) # Run the e2e tests
 	export COVERAGE_FILE=./output/.coverage.e2e
 	pytest -vvv --runslow --cov --cov-report term --cov-report html:./output/htmlcov_e2e tests/e2e/
-	@echo "Ignore, Created by Makefile, `date`" > $@
 
 integration: $(MAKE_SOURCES) # Run unit tests
 	export COVERAGE_FILE=./output/.coverage.integration
 	pytest -vvv --runslow --cov --cov-report term --cov-report html:./output/htmlcov_integration tests/integration/
-	@echo "Ignore, Created by Makefile, `date`" > $@
 
 unit: $(MAKE_SOURCES) # Run unit tests
 	export COVERAGE_FILE=./output/.coverage.unit
 	pytest -vvv --runslow --cov --cov-report term --cov-report html:./output/htmlcov_unit tests/unit/
-	@echo "Ignore, Created by Makefile, `date`" > $@
 
 build-package: $(MAKE_SOURCES) # Build the package as a pip wheel
 	pip install build
@@ -96,7 +90,6 @@ build-package: $(MAKE_SOURCES) # Build the package as a pip wheel
 
 build-doc: $(MAKE_SOURCES) # Build the Sphinx docs
 	$(MAKE) -C docs/ html
-	@echo "Ignore, Created by Makefile, `date`" > $@
 
 clean: # Delete build artifacts and do any custom cleanup such as spinning down services
 	@rm -rf format lint typecheck build-doc build-package unit e2e integration .pytest_cache .pytype

README.rst

@@ -7,7 +7,7 @@ entity resolution (ER) pipelines.
 
 .. _python_support:
 
-Supported Python versions: 3.11, 3.12
+**Supported Python versions: 3.11, 3.12**
 
 .. _end_python_support:
 
@@ -22,21 +22,20 @@ There are a few things to install in order to use this package:
   likely need to request it from your system admin. 
   Refer to https://docs.sylabs.io/guides/4.1/admin-guide/installation.html
 
-.. highlight:: console
 
 - Install graphviz via:
-    
-    :: 
+
+  .. code-block:: console
 
     $ conda install graphviz
 
 - Install EasyLink.
 
-    Option 1 - Install from PyPI with pip::
+  Option 1 - Install from PyPI with pip::
 
     $ pip install easylink
 
-    Option 2 - Build from source with pip::
+  Option 2 - Build from source with pip::
 
     $ git clone [email protected]:ihmeuw/easylink.git  # or git clone https://github.com/ihmeuw/easylink.git
     $ cd easylink

docs/nitpick-exceptions

@@ -0,0 +1,3 @@
+py:class LayeredConfigTree
+py:class layered_config_tree.LayeredConfigTree
+py:class layered_config_tree.main.LayeredConfigTree

docs/source/api_reference/configuration.rst

@@ -0,0 +1 @@
+.. automodule:: easylink.configuration

docs/source/api_reference/graph_components.rst

@@ -0,0 +1 @@
+.. automodule:: easylink.graph_components

docs/source/api_reference/implementation.rst

@@ -0,0 +1 @@
+.. automodule:: easylink.implementation

docs/source/api_reference/index.rst

@@ -1,2 +1,11 @@
 API Reference
 =============
+
+.. automodule:: easylink
+
+.. toctree::
+    :maxdepth: 1
+    :glob:
+
+    *
+    */index

docs/source/api_reference/pipeline.rst

@@ -0,0 +1 @@
+.. automodule:: easylink.pipeline

docs/source/api_reference/pipeline_graph.rst

@@ -0,0 +1 @@
+.. automodule:: easylink.pipeline_graph

docs/source/api_reference/pipeline_schema.rst

@@ -0,0 +1 @@
+.. automodule:: easylink.pipeline_schema

docs/source/api_reference/pipeline_schema_constants/development.rst

@@ -0,0 +1 @@
+.. automodule:: easylink.pipeline_schema_constants.development

docs/source/api_reference/pipeline_schema_constants/index.rst

@@ -0,0 +1,10 @@
+PipelineSchema Constants
+========================
+
+.. automodule:: easylink.pipeline_schema_constants
+
+.. toctree::
+    :maxdepth: 1
+    :glob:
+
+    *

docs/source/api_reference/pipeline_schema_constants/tests.rst

@@ -0,0 +1 @@
+.. automodule:: easylink.pipeline_schema_constants.tests

docs/source/api_reference/rule.rst

@@ -0,0 +1 @@
+.. automodule:: easylink.rule

docs/source/api_reference/runner.rst

@@ -0,0 +1 @@
+.. automodule:: easylink.runner

docs/source/api_reference/step.rst

@@ -0,0 +1 @@
+.. automodule:: easylink.step

docs/source/api_reference/utilities/data_utils.rst

@@ -0,0 +1,2 @@
+.. automodule:: easylink.utilities.data_utils
+

docs/source/api_reference/utilities/general_utils.rst

@@ -0,0 +1,2 @@
+.. automodule:: easylink.utilities.general_utils
+

docs/source/api_reference/utilities/index.rst

@@ -0,0 +1,10 @@
+Utilities
+=========
+
+.. automodule:: easylink.utilities
+
+.. toctree::
+    :maxdepth: 1
+    :glob:
+
+    *

docs/source/api_reference/utilities/paths.rst

@@ -0,0 +1,2 @@
+.. automodule:: easylink.utilities.paths
+

docs/source/api_reference/utilities/validation_utils.rst

@@ -0,0 +1,2 @@
+.. automodule:: easylink.utilities.validation_utils
+

docs/source/conf.py

@@ -198,6 +198,7 @@
     "numpy": ("https://numpy.org/doc/stable/", None),
     "networkx": ("https://networkx.org/documentation/stable/", None),
     "layered_config_tree": ("https://layered-config-tree.readthedocs.io/en/latest/", None),
+    "loguru": ("https://loguru.readthedocs.io/en/stable/", None),
 }
 
 
@@ -213,10 +214,16 @@
     # Generate docs even if an item has no docstring.
     "undoc-members": True,
     # Don't document things with a leading underscore.
-    "private-members": False,
+    "private-members": True,
+    # Show class inheritance.
+    "show-inheritance": True,
 }
 # Display type hints in the description instead of the signature.
 autodoc_typehints = "description"
+# Mock problematic imports
+autodoc_mock_imports = [
+    "networkx",
+]
 
 
 # -- nitpicky mode --------------------------------------------------------

docs/source/user_guide/cli.rst

@@ -4,8 +4,7 @@
 Command Line Interface
 ======================
 
-.. automodule:: easylink.cli
-
 .. click:: easylink.cli:easylink
    :prog: easylink
-   :show-nested:
+   :nested: full
+   :commands: run, generate-dag

pyproject.toml

@@ -1,3 +1,6 @@
+[build-system]
+requires = ["packaging", "setuptools"]
+
 [tool.black]
 line_length = 94
 

python_versions.json

@@ -0,0 +1 @@
+["3.11", "3.12"]

setup.py

@@ -1,8 +1,36 @@
 #!/usr/bin/env python
+import json
 import os
+import sys
 
+from packaging.version import parse
 from setuptools import find_packages, setup
 
+with open("python_versions.json", "r") as f:
+    supported_python_versions = json.load(f)
+
+python_versions = [parse(v) for v in supported_python_versions]
+min_version = min(python_versions)
+max_version = max(python_versions)
+if not (
+    min_version <= parse(".".join([str(v) for v in sys.version_info[:2]])) <= max_version
+):
+    py_version = ".".join([str(v) for v in sys.version_info[:3]])
+    # NOTE: Python 3.5 does not support f-strings
+    error = (
+        "\n--------------------------------------------\n"
+        "Error: EasyLink runs under python {min_version}-{max_version}.\n"
+        "You are running python {py_version}.\n".format(
+            min_version=min_version.base_version,
+            max_version=max_version.base_version,
+            py_version=py_version,
+        )
+        + "--------------------------------------------\n"
+    )
+    print(error, file=sys.stderr)
+    sys.exit(1)
+
+
 if __name__ == "__main__":
     base_dir = os.path.dirname(__file__)
     src_dir = os.path.join(base_dir, "src")
@@ -39,10 +67,11 @@
         "pytest-mock",
     ]
     doc_requirements = [
-        "sphinx>=4.0,<8.0.0",
-        "sphinx-rtd-theme>=0.6",
+        "sphinx",
+        "sphinx-rtd-theme",
         "sphinx-autodoc-typehints",
         "sphinx-click",
+        "typing_extensions",
     ]
     lint_requirements = [
         "black==22.3.0",
@@ -54,6 +83,7 @@
         name=about["__title__"],
         description=about["__summary__"],
         long_description=long_description,
+        long_description_content_type="text/x-rst",
         license=about["__license__"],
         url=about["__uri__"],
         author=about["__author__"],

src/easylink/__init__.py

@@ -3,7 +3,7 @@
 EasyLink
 ========
 
-Research repository for the EasyLink ER ecosystem project.
+Research repository for the EasyLink entity resolution (ER) ecosystem project.
 
 """