From e371b9ca5f89e5f92e650b51bcc47478fa6a9eb3 Mon Sep 17 00:00:00 2001 From: CBroz1 Date: Wed, 15 Nov 2023 17:19:46 -0600 Subject: [PATCH] WIP: database management doc --- docs/mkdocs.yml | 14 ++-- docs/src/api/make_pages.py | 23 +++++-- docs/src/misc/database_management.md | 89 +++++++++++++++++++++++++ docs/src/misc/merge_tables.md | 2 +- notebooks/00_Setup.ipynb | 10 ++- notebooks/py_scripts/00_Setup.py | 12 ++-- src/spyglass/common/common_filter.py | 8 ++- src/spyglass/settings.py | 9 +-- src/spyglass/utils/database_settings.py | 3 +- 9 files changed, 142 insertions(+), 28 deletions(-) create mode 100644 docs/src/misc/database_management.md diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 587eeb78c..74eb24e99 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -51,12 +51,14 @@ nav: - Session Groups: misc/session_groups.md - Insert Data: misc/insert_data.md - Merge Tables: misc/merge_tables.md + - Database Management: misc/database_management.md - Tutorials: - Overview: notebooks/README.md - General: - Setup: notebooks/00_Setup.ipynb - Insert Data: notebooks/01_Insert_Data.ipynb - Data Sync: notebooks/02_Data_Sync.ipynb + - Merge Tables: notebooks/03_Merge_Tables.ipynb - Ephys: - Spike Sorting: notebooks/10_Spike_Sorting.ipynb - Curation: notebooks/11_Curation.ipynb @@ -101,8 +103,6 @@ plugins: group_by_category: false line_length: 80 docstring_style: numpy - # watch: - # - src/spyglass/ - literate-nav: nav_file: navigation.md - exclude-search: @@ -111,11 +111,11 @@ plugins: - gen-files: scripts: - ./src/api/make_pages.py - - mkdocs-jupyter: # Comment this block during dev to reduce build time - execute: False # Very slow, needs gh-action edit to work/link to db - include_source: False - ignore_h1_titles: True - ignore: ["*make_pages.py", "**checkpoints**", "*/py_scripts/*"] + # - mkdocs-jupyter: # Comment this block during dev to reduce build time + # execute: False # Very slow, needs gh-action edit to work/link to db + # include_source: False + # ignore_h1_titles: True + # ignore: ["*make_pages.py", "**checkpoints**", "*/py_scripts/*"] markdown_extensions: - attr_list diff --git a/docs/src/api/make_pages.py b/docs/src/api/make_pages.py index 8dcd37c1d..b345e8404 100644 --- a/docs/src/api/make_pages.py +++ b/docs/src/api/make_pages.py @@ -4,15 +4,30 @@ from pathlib import Path import mkdocs_gen_files +from mkdocs.utils import log + +ignored_stems = ["__init__", "_version"] + +added = 0 +add_limit = None nav = mkdocs_gen_files.Nav() -for path in sorted(Path("src").glob("**/*.py")): - if path.stem == "__init__" or "cython" in path.stem: +for path in sorted(Path("src/spyglass/").glob("**/*.py")): + if path.stem in ignored_stems or "cython" in path.stem: continue - with mkdocs_gen_files.open(f"api/{path.with_suffix('')}.md", "w") as f: + rel_path = path.relative_to("src/spyglass") + with mkdocs_gen_files.open(f"api/{rel_path.with_suffix('')}.md", "w") as f: module_path = ".".join([p for p in path.with_suffix("").parts]) print(f"::: {module_path}", file=f) - nav[path.parts] = f"{path.with_suffix('')}.md" + nav[rel_path.parts] = f"{rel_path.with_suffix('')}.md" + + if add_limit is not None: + if added < add_limit: + log.warning(f"Generated {rel_path.with_suffix('')}.md") + added += 1 + else: + break + with mkdocs_gen_files.open("api/navigation.md", "w") as nav_file: nav_file.writelines(nav.build_literate_nav()) diff --git a/docs/src/misc/database_management.md b/docs/src/misc/database_management.md new file mode 100644 index 000000000..950ef27a6 --- /dev/null +++ b/docs/src/misc/database_management.md @@ -0,0 +1,89 @@ +# Database Management + +While Spyglass can help you organize your data, there are a number of things +you'll need to do to manage users, database backups, and file cleanup. + +Some these tasks should be set to run regularly. [Cron jobs](https://www.hostinger.com/tutorials/cron-job) +can help with automation. + +## MySQL Version + +The Frank Lab's database is running MySQL 8.0 with a number of custom +configurations set by our system admin to reflect UCSF's IT security +requirements. + +DataJoint's default docker container for MySQL is version 5.7. As the Spyglass +team has hit select compatibility issues, we've worked with the DataJoint team +to update the open source package to support MySQL 8.0. + +While the Spyglass team won't be able to support earlier versions, if you run +into any issues declaring Spyglass tables with an 8.0 instance, please let us +know. + +## User Management + +The [DatabaseSettings](../api/utils/database_settings.md) class provides a +number of methods to help you manage users. By default, it will write out a +temporary `.sql` file and execute it on the database. + +### Privileges + +DataJoint schemas correspond to MySQL databases. Priveleges are managed by +schema/database prefix. + +- `SELECT` privileges allow users to read, write, and delete data. +- `ALL` privileges allow users to create, alter, or drop tables and schemas in + addition to operations above. + +In practice, DataJoint only permits alerations of secondary keys on existing +tables, and more derstructive operations would require using DataJoint to +execeute MySQL commands. + +Shared schema prefixes are those defined in the Spyglass package (e.g., +`common`, `lfp`, etc.). A 'user schema' is any schema with the username as +prefix. User types differ in the privileges they are granted on +these prifixes. + +### Users types + +- `collab_user`: `ALL` on user schema, `SELECT` on all other schemas. +- `dj_guest`: `SELECT` on all schemas. +- `dj_user`: `ALL` on shared and user schema, `SELECT` on all other schemas. + +### Setting Passwords + +New users are generted with the password `temppass`. In order to change this, +we recommend downloading DataJoint `0.14.2` (currently pre-release). + +```console +git clone https://github.com/datajoint/datajoint-python/ +pip install ./datajoint-python +``` + +Then, you the user can reset within Python: + +```python +import datajoint as dj +dj.set_password() +``` + +## Database Backups + +Coming soon... + +## File Cleanup + +Spyglass is designed to hold metadata for analyses that reference NWB files on +disk. There are several tables that retain lists of files that have been generated +during analyses. If someone deletes analysis entries, files will still be on disk. + +To remove orphaned files, we run the following commands in our cron jobs: + +```python +from spyglass.common import AnalysisNwbfile +from spyglass.spikesorting import SpikeSorting + +def main(): + AnalysisNwbfile().nightly_cleanup() + SpikeSorting().nightly_cleanup() +``` diff --git a/docs/src/misc/merge_tables.md b/docs/src/misc/merge_tables.md index dc10e2dfc..c5419d350 100644 --- a/docs/src/misc/merge_tables.md +++ b/docs/src/misc/merge_tables.md @@ -70,7 +70,7 @@ The Merge class in Spyglass's utils is a subclass of DataJoint's [Manual Table](https://datajoint.com/docs/core/design/tables/tiers/#data-entry-lookup-and-manual) and adds functions to make the awkwardness of part tables more manageable. These functions are described in the -[API section](../../api/src/spyglass/utils/dj_merge_tables.md), under +[API section](../api/utils/dj_merge_tables.md), under `utils.dj_merge_tables`. ### Restricting diff --git a/notebooks/00_Setup.ipynb b/notebooks/00_Setup.ipynb index ecf012f27..08691e907 100644 --- a/notebooks/00_Setup.ipynb +++ b/notebooks/00_Setup.ipynb @@ -219,12 +219,16 @@ "- Add yourself to the\n", " [`docker` group](https://docs.docker.com/engine/install/linux-postinstall/) so\n", " that you don't have to be sudo to run docker.\n", - "- Download the docker image for datajoint/mysql\n", + "- Download the docker image for `datajoint/mysql:8.0`.\n", "\n", " ```bash\n", - " docker pull datajoint/mysql\n", + " docker pull datajoint/mysql:8.0\n", " ```\n", "\n", + "_Note_: For demonstation, MySQL version won't matter. Some\n", + " [database management](https://lorenfranklab.github.io/spyglass/latest/misc/database_management/#mysql-version)\n", + " features of Spyglass, however, expect MySQL >= 8.\n", + "\n", "- When run, this is referred to as a 'Docker container'\n", "- Next start the container with a couple additional pieces of info...\n", "\n", @@ -233,7 +237,7 @@ " - Port mapping. Here, we map 3306 across the local machine and container.\n", "\n", " ```bash\n", - " docker run --name spyglass-db -p 3306:3306 -e MYSQL_ROOT_PASSWORD=tutorial datajoint/mysql\n", + " docker run --name spyglass-db -p 3306:3306 -e MYSQL_ROOT_PASSWORD=tutorial datajoint/mysql:8.0\n", " ```\n", "\n", "- For data to persist after terminating the container,\n", diff --git a/notebooks/py_scripts/00_Setup.py b/notebooks/py_scripts/00_Setup.py index bdc46d164..ebe28291f 100644 --- a/notebooks/py_scripts/00_Setup.py +++ b/notebooks/py_scripts/00_Setup.py @@ -5,7 +5,7 @@ # extension: .py # format_name: light # format_version: '1.5' -# jupytext_version: 1.14.5 +# jupytext_version: 1.15.2 # kernelspec: # display_name: Python 3 (ipykernel) # language: python @@ -185,12 +185,16 @@ # - Add yourself to the # [`docker` group](https://docs.docker.com/engine/install/linux-postinstall/) so # that you don't have to be sudo to run docker. -# - Download the docker image for datajoint/mysql +# - Download the docker image for `datajoint/mysql:8.0`. # # ```bash -# docker pull datajoint/mysql +# docker pull datajoint/mysql:8.0 # ``` # +# _Note_: For demonstation, MySQL version won't matter. Some +# [database management](https://lorenfranklab.github.io/spyglass/latest/misc/database_management/#mysql-version) +# features of Spyglass, however, expect MySQL >= 8. +# # - When run, this is referred to as a 'Docker container' # - Next start the container with a couple additional pieces of info... # @@ -199,7 +203,7 @@ # - Port mapping. Here, we map 3306 across the local machine and container. # # ```bash -# docker run --name spyglass-db -p 3306:3306 -e MYSQL_ROOT_PASSWORD=tutorial datajoint/mysql +# docker run --name spyglass-db -p 3306:3306 -e MYSQL_ROOT_PASSWORD=tutorial datajoint/mysql:8.0 # ``` # # - For data to persist after terminating the container, diff --git a/src/spyglass/common/common_filter.py b/src/spyglass/common/common_filter.py index b137b197d..2f3e12fac 100644 --- a/src/spyglass/common/common_filter.py +++ b/src/spyglass/common/common_filter.py @@ -225,7 +225,7 @@ def filter_data_nwb( electrode_ids: list, decimation: int, description: str = "filtered data", - type: Union[None, str] = None, + data_type: Union[None, str] = None, ): """ Filter data from an NWB electrical series using the ghostipy package, @@ -256,6 +256,10 @@ def filter_data_nwb( The NWB object ID of the filtered data and a list containing the first and last timestamps. """ + # Note: type -> data_type to avoid conflict with builtin type + # All existing refs to this func use positional args, so no need to + # adjust elsewhere, but low probability of issues with custom scripts + MEM_USE_LIMIT = 0.9 # % of RAM use permitted gsp = _import_ghostipy() @@ -323,7 +327,7 @@ def filter_data_nwb( timestamps=np.empty(output_shape_list[time_axis]), description=description, ) - if type == "LFP": + if data_type == "LFP": ecephys_module = nwbf.create_processing_module( name="ecephys", description=description ) diff --git a/src/spyglass/settings.py b/src/spyglass/settings.py index 7f1ff9572..d1e39ff27 100644 --- a/src/spyglass/settings.py +++ b/src/spyglass/settings.py @@ -18,17 +18,14 @@ class SpyglassConfig: facilitate testing. """ - def __init__(self, base_dir=None, **kwargs): + def __init__(self, base_dir: str = None, **kwargs): """ Initializes a new instance of the class. Parameters ---------- - base_dir (str): The base directory. - - Returns - ------- - None + base_dir (str) + The base directory. """ self.supplied_base_dir = base_dir self._config = dict() diff --git a/src/spyglass/utils/database_settings.py b/src/spyglass/utils/database_settings.py index 3a29b3834..8d4f299bd 100644 --- a/src/spyglass/utils/database_settings.py +++ b/src/spyglass/utils/database_settings.py @@ -65,9 +65,10 @@ def add_collab_user(self): @property def _add_dj_guest_sql(self): + # Note: changing to temppass for uniformity return [ # Create the user (if not already created) and set the password - f"{CREATE_USR}'{self.user}'@'%' IDENTIFIED BY 'Data_$haring';\n", + f"{CREATE_USR}'{self.user}'@'%'{TEMP_PASS}\n", # Grant privileges f"{GRANT_SEL}`%`.* TO '{self.user}'@'%';\n", ]