CLI reconfiguration, stream FK and PK errors to console and log file. (…

…#11) * autoupdate hooks Signed-off-by: Caleb Grant <[email protected]> * split ci/cd process into smaller chunks Signed-off-by: Caleb Grant <[email protected]> * change log message level to warning from debug for PK and FK error failing rows. this change will stream FK and PK failing rows to the log file and console without the debug (-d) flag set. closes #6 Signed-off-by: Caleb Grant <[email protected]> * modify cli so the only positional arguments are the list of tables to upsert. closes #7 Signed-off-by: Caleb Grant <[email protected]> --------- Signed-off-by: Caleb Grant <[email protected]>
geocoug · Oct 29, 2024 · 0dc4bfa · 0dc4bfa
1 parent 1dd3d5c
commit 0dc4bfa
Show file tree

Hide file tree

Showing 3 changed files with 125 additions and 74 deletions.
diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml
@@ -3,41 +3,53 @@ name: ci/cd
 
 on:
   push:
-  pull_request:
+    branches:
+      - main
   workflow_dispatch:
 
-jobs:
+env:
+  PROJECT_NAME: pg-upsert
+  POSTGRES_HOST: localhost
+  POSTGRES_PORT: 5432
+  POSTGRES_USER: postgres
+  POSTGRES_PASSWORD: postgres
+  POSTGRES_DB: dev
 
-  # https://docs.github.com/en/actions/using-containerized-services/creating-postgresql-service-containers#running-jobs-directly-on-the-runner-machine
-  python-tests:
-    runs-on: ubuntu-latest
+jobs:
+  tests:
     strategy:
       matrix:
-        python-version: ["3.10", "3.11", "3.12"]
+        os:
+          - ubuntu-latest
+        python-version:
+          - "3.10"
+          - "3.11"
+          - "3.12"
     services:
       postgres:
         image: postgis/postgis:16-3.4-alpine
         env:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: dev
+          POSTGRES_USER: ${{ env.POSTGRES_USER }}
+          POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }}
+          POSTGRES_DB: ${{ env.POSTGRES_DB }}
         options: >-
           --health-cmd pg_isready
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
         ports:
           - 5432:5432
+    runs-on: ${{ matrix.os }}
     steps:
       - name: Check out repository code
         uses: actions/checkout@v4
       - name: Import PostgreSQL data
-        run: |
-          psql -h localhost -U postgres -d dev -f tests/data.sql
+        run: >-
+          psql -h ${{ env.POSTGRES_HOST }} -U ${{ env.POSTGRES_USER }} -d ${{ env.POSTGRES_DB }} -f tests/data.sql
         env:
-          PGPASSWORD: postgres
+          PGPASSWORD: ${{ env.POSTGRES_PASSWORD }}
       - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
@@ -49,15 +61,56 @@ jobs:
           python -m pip install pytest
           python -m pytest
         env:
-          POSTGRES_HOST: localhost
-          POSTGRES_PORT: 5432
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: dev
+          POSTGRES_HOST: ${{ env.POSTGRES_HOST }}
+          POSTGRES_PORT: ${{ env.POSTGRES_PORT }}
+          POSTGRES_USER: ${{ env.POSTGRES_USER }}
+          POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }}
+          POSTGRES_DB: ${{ env.POSTGRES_DB }}
 
+  build:
+    name: Build distribution 📦
+    runs-on: ubuntu-latest
+    needs: [tests]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Build binary distribution package
+        run: |
+          python -m pip install --upgrade pip
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          python -m pip install --upgrade build
+          python -m build --sdist --wheel --outdir dist/
+      - name: Store the distribution package
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.PROJECT_NAME }}
+          path: dist/
+
+  publish:
+    name: PyPI Publish 🚀
+    needs: [build]
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes
+    environment:
+      name: pypi
+      url: https://pypi.org/p/${{ env.PROJECT_NAME }}
+    permissions:
+      id-token: write
+    steps:
+      - name: Download the dist
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ env.PROJECT_NAME }}
+          path: dist/
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
 
   docker-build:
     name: Docker Build+Push
+    needs: [build]
     uses: geocoug/github-actions-templates/.github/workflows/docker-build.yml@main
     permissions:
       contents: read
@@ -74,24 +127,3 @@ jobs:
         type=semver,pattern={{version}}
         type=raw,value=gha-${{ github.run_id }}
       platforms: linux/amd64,linux/arm64
-
-
-  pypi-publish:
-    name: PyPI Publish
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.12'
-      - name: Build and publish
-        env:
-          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-        run: |
-          python -m pip install --upgrade pip
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-          python -m pip install --upgrade build twine
-          python -m build --sdist --wheel --outdir dist/
-          python -m twine upload --skip-existing dist/*
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -6,12 +6,12 @@ repos:
       - id: check-hooks-apply
   # Protect secrets using Gitleaks
   - repo: https://github.com/zricethezav/gitleaks
-    rev: v8.18.4
+    rev: v8.21.2
     hooks:
       - id: gitleaks
   # Supported base hooks
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: check-added-large-files
         args: [--maxkb=10000]
@@ -24,7 +24,7 @@ repos:
       - id: trailing-whitespace
   # Markdown lint
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.41.0
+    rev: v0.42.0
     hooks:
       - id: markdownlint
         types: [file]
@@ -39,7 +39,7 @@ repos:
   # Replaces Black, Flake8, isort, pydocstyle, pyupgrade, bandit, and autoflake
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.5.3
+    rev: v0.7.1
     hooks:
       # Run the Ruff linter.
       - id: ruff

diff --git a/pg_upsert/pg_upsert.py b/pg_upsert/pg_upsert.py
@@ -1295,10 +1295,10 @@ def qa_one_pk(self: PgUpsert, table: str) -> PgUpsert:
             tot_errs = next(iter(tot_errs))
             err_msg = f"{tot_errs['errcount']} duplicate keys ({tot_errs['total_rows']} rows) in table {self.stg_schema}.{table}"  # noqa: E501
             pk_errors.append(err_msg)
-            logger.debug("")
+            logger.warning("")
             err_sql = SQL("select * from ups_pk_check;")
-            logger.debug(f"\n{self._show(err_sql)}")
-            logger.debug("")
+            logger.warning(f"{self._show(err_sql)}")
+            logger.warning("")
             if self.interactive:
                 btn, return_value = TableUI(
                     "Duplicate key error",
@@ -1553,9 +1553,9 @@ def qa_one_fk(self: PgUpsert, table: str) -> PgUpsert:
                     uq_table=Identifier(const_rows["uq_table"]),
                     su_join=SQL(fk_rows["su_join"]),
                 )
-                query += SQL(" where u.{uq_column} is null").format(
-                    uq_column=Identifier(const_rows["uq_column"]),
-                )
+            query += SQL(" where u.{uq_column} is null").format(
+                uq_column=Identifier(const_rows["uq_column"]),
+            )
             if su_exists:
                 query += SQL(" and su.{uq_column} is null").format(
                     uq_column=Identifier(const_rows["uq_column"]),
@@ -1577,9 +1577,9 @@ def qa_one_fk(self: PgUpsert, table: str) -> PgUpsert:
                 logger.warning(
                     f"    Foreign key error referencing {const_rows['uq_schema']}.{const_rows['uq_table']}",
                 )
-                logger.debug("")
-                logger.debug(f"\n{self._show(check_sql)}")
-                logger.debug("")
+                logger.warning("")
+                logger.warning(f"{self._show(check_sql)}")
+                logger.warning("")
                 if self.interactive:
                     btn, return_value = TableUI(
                         "Foreign key error",
@@ -2645,16 +2645,21 @@ def ellapsed_time(start_time: datetime):
 def clparser() -> argparse.ArgumentParser:
     """Command line interface for the upsert function."""
     parser = argparse.ArgumentParser(
+        add_help=False,
         description=__description__,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
+    parser.add_argument(
+        "--help",
+        action="help",
+        help="show this help message and exit",
+    )
     parser.add_argument(
         "--version",
         action="version",
         version=f"%(prog)s {__version__}",
     )
     parser.add_argument(
-        "-d",
         "--debug",
         action="store_true",
         help="display debug output",
@@ -2669,24 +2674,25 @@ def clparser() -> argparse.ArgumentParser:
         "-l",
         "--log",
         type=Path,
-        metavar="LOGFILE",
         help="write log to LOGFILE",
     )
     parser.add_argument(
         "-e",
-        "--exclude",
-        metavar="EXCLUDE_COLUMNS",
+        "--exclude-columns",
+        dest="exclude",
+        type=str,
         help="comma-separated list of columns to exclude from null checks",
     )
     parser.add_argument(
         "-n",
-        "--null",
-        metavar="NULL_COLUMNS",
+        "--null-columns",
+        dest="null",
+        type=str,
         help="comma-separated list of columns to exclude from null checks",
     )
     parser.add_argument(
         "-c",
-        "--do-commit",
+        "--commit",
         action="store_true",
         help="commit changes to database",
     )
@@ -2699,45 +2705,58 @@ def clparser() -> argparse.ArgumentParser:
     parser.add_argument(
         "-m",
         "--upsert-method",
-        metavar="UPSERT_METHOD",
         default="upsert",
         choices=["upsert", "update", "insert"],
         help="method to use for upsert",
     )
     parser.add_argument(
-        "host",
-        metavar="HOST",
+        "-h",
+        "--host",
+        required=True,
+        type=str,
         help="database host",
     )
     parser.add_argument(
-        "port",
-        metavar="PORT",
+        "-p",
+        "--port",
+        required=True,
         type=int,
         help="database port",
     )
     parser.add_argument(
-        "database",
-        metavar="DATABASE",
+        "-d",
+        "--database",
+        required=True,
+        type=str,
         help="database name",
     )
     parser.add_argument(
-        "user",
-        metavar="USER",
+        "-u",
+        "--user",
+        required=True,
+        type=str,
         help="database user",
     )
     parser.add_argument(
-        "stg_schema",
-        metavar="STAGING_SCHEMA",
+        "-s",
+        "--staging-schema",
+        default="staging",
+        dest="stg_schema",
+        required=True,
+        type=str,
         help="staging schema name",
     )
     parser.add_argument(
-        "base_schema",
-        metavar="BASE_SCHEMA",
+        "-b",
+        "--base-schema",
+        default="public",
+        dest="base_schema",
+        required=True,
+        type=str,
         help="base schema name",
     )
     parser.add_argument(
         "tables",
-        metavar="TABLE",
         nargs="+",
         help="table name(s)",
     )
@@ -2775,7 +2794,7 @@ def main() -> None:
             tables=args.tables,
             stg_schema=args.stg_schema,
             base_schema=args.base_schema,
-            do_commit=args.do_commit,
+            do_commit=args.commit,
             upsert_method=args.upsert_method,
             interactive=args.interactive,
             exclude_cols=args.exclude.split(",") if args.exclude else None,