continue refactoring, + docs

Makefile

@@ -73,7 +73,6 @@ build-dist: $(VENV)/bin/activate ## Generate distrubition packages
 
 build-docs: ## Generate documentation
 	@printf "Building documentation\n"
-	@$(SPHINXAPIDOC) -f -o "$(SOURCEDIR)" pg_upsert
 	@$(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS)
 
 publish: $(VENV)/bin/activate ## Publish to PyPI

README.md

@@ -398,3 +398,22 @@ POSTGRES_PASSWORD=
 ```
 
 Now you can run the tests using `make test`.
+
+## Notes
+
+- The user can modify the control table to set interactive specific to each table
+- The user can modify the control table to set the upsert method specific to each table
+- The user can modify the control table to set the exclude columns specific to each table
+- The user can modify the control table to set the exclude null check columns specific to each table
+- In upsert_one check that the table has a primary key before proceeding
+- Replace all sys.exit() with a graceful exit that closes db connection and rolls back changes
+
+upsert_all():
+
+- What would happen if the user runs this method without running the QA checks first?
+- What would happen if the user modified the qa_passed attribute to True without running the QA checks?
+- What would happen if the user modified the control table to indicate that QA checks passed when they did not?
+
+TODO:
+- Modify the show() funciton to acutally show a query either via GUI or console
+- Fix upsert_one to return self

docs/conf.py

@@ -42,7 +42,7 @@
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-html_theme = "sphinx_book_theme"
+html_theme = "sphinx_rtd_theme"
 
 html_static_path = ["_static"]
 

docs/index.rst

@@ -6,26 +6,12 @@
 pg_upsert documentation
 =======================
 
-Release v\ |version|.
-
 **pg_upsert** is a Python package that runs not-NULL, Primary Key, Foreign Key, and Check Constraint checks on PostgreSQL staging tables then update and insert (upsert) data from staging tables to base tables.
 
-The API Documentation / Guide
------------------------------
-
-If you are looking for information on a specific function, class, or method,
-this part of the documentation is for you.
+Module Contents
+---------------
 
 .. toctree::
-   :maxdepth: 2
-   :caption: Contents:
-
-   modules
-
-
-Indices and tables
-==================
+   :maxdepth: 4
 
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
+   pg_upsert

docs/pg_upsert.rst

@@ -1,19 +1,10 @@
-pg\_upsert package
-==================
+.. _pg_upsert:
 
-Submodules
-----------
+pg\_upsert
+==========
 
-pg\_upsert.pg\_upsert module
-----------------------------
-
-.. automodule:: pg_upsert.pg_upsert
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-Module contents
----------------
+All of pg_upsert's functionality can be accessed by the :class:`PgUpsert` object, which
+includes all the methods and attributes mentioned in the sections below.
 
 .. automodule:: pg_upsert
    :members:

example.py

@@ -0,0 +1,103 @@
+import logging
+
+from pg_upsert import PgUpsert
+
+logger = logging.getLogger("pg_upsert")
+logger.setLevel(logging.INFO)
+handlers = [
+    logging.FileHandler("pg_upsert.log"),
+    logging.StreamHandler(),
+]
+if logger.level == logging.DEBUG:
+    formatter = logging.Formatter(
+        "%(asctime)s - %(name)s - %(levelname)s - %(lineno)d: %(message)s"
+    )
+else:
+    formatter = logging.Formatter("%(message)s")
+for handler in handlers:
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+
+# Full example of instantiating the class and running the upserts
+upsert = PgUpsert(
+    host="localhost",
+    port=5432,
+    database="dev",
+    user="docker",
+    passwd="docker",
+    tables=("genres", "books", "authors", "book_authors"),
+    stg_schema="staging",
+    base_schema="public",
+    do_commit=False,
+    upsert_method="insert",
+    interactive=False,
+    exclude_cols=None,
+    exclude_null_check_columns=None,
+    control_table="ups_control",
+).run()
+
+
+# Minimal example of instantiating the class and running the upserts
+# upsert = PgUpsert(
+#     host="localhost",
+#     port=5432,
+#     database="dev",
+#     user="docker",
+#     passwd="docker",
+#     tables=("genres", "books", "authors", "book_authors"),
+#     stg_schema="staging",
+#     base_schema="public",
+# )
+
+# upsert.run()
+
+# # Modify the control table then run run upsert on the one table
+# upsert.db.execute(
+#     f"update {upsert.control_table} set exclude_cols = 'first_name,last_name', interactive=true where table_name = 'authors';"
+# )
+# upsert.upsert_one(table="authors").commit()
+
+
+# # Run upsert on one table
+# upsert.upsert_one(table="authors").commit()
+
+
+# # Run a specific set of qa checks on one table
+# # Null checks
+# upsert.qa_one_null("authors")
+# # Primary key checks
+# upsert.qa_one_pk("authors")
+# # Foreign key checks
+# upsert.qa_one_fk("authors")
+# # Check constraint checks
+# upsert.qa_one_ck("authors")
+
+
+# # Run everything with defaults
+# upsert.run()
+
+
+# # Run only QA checks
+# upsert.qa_all()
+
+
+# # Run only upserts and commit changes (if do_commit=True)
+# upsert.upsert_all().commit()
+
+
+# # Run only upserts and do not commit changes
+# upsert.upsert_all()
+
+
+# # Modify the control table on a table-by-table basis
+# # The control table is initialized when the class is instantiated
+# logger.info(upsert.show(f"select * from {upsert.control_table}"))
+# logger.info("")
+# # Modify the exclude_cols column for the authors table and set interactive to true. The exclude_cols and exclude_null_checks values should be a comma-separated string.
+# upsert.db.execute(
+#     f"update {upsert.control_table} set exclude_cols = 'first_name,last_name', interactive=true where table_name = 'authors';"
+# )
+# logger.info(upsert.show(f"select * from {upsert.control_table}"))
+
+
+del upsert

pg_upsert/__init__.py

@@ -1,4 +1,12 @@
-from ._version import __version__
-from .pg_upsert import PgUpsert, PostgresDB
+from ._version import (
+    __author__,
+    __author_email__,
+    __description__,
+    __license__,
+    __title__,
+    __url__,
+    __version__,
+)
+from .pg_upsert import PgUpsert
 
-__all__ = ["PgUpsert", "PostgresDB"]
+__all__ = ["PgUpsert"]

pg_upsert/_version.py

@@ -1 +1,7 @@
+__title__ = "pg_upsert"
+__author__ = "Caleb Grant"
+__url__ = "https://github.com/geocoug/pg_upsert"
+__author_email__ = "[email protected]"
+__license__ = "GNU GPLv3"
 __version__ = "1.1.4"
+__description__ = "Run not-NULL, Primary Key, Foreign Key, and Check Constraint checks on staging tables then update and insert (upsert) data from staging tables to base tables."  # noqa: E501