Generate utils from config/ scripts (LorenFrankLab#662)

* Contrib doc edits

* LorenFrankLab#531

* LorenFrankLab#532: PR Template

* Fix numbering

* Typo

* LorenFrankLab#658 config. Start to phase out config dir

* Update changelog

* Add docstrings. Separate sql content from funcs

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Change Log
 
+## [Unreleased]
+
+- Migrate `config` helper scripts to Spyglass codebase. #662
+- Revise contribution guidelines. #655
+- Minor bug fixes. #656, #657, #659, #651
+
 ## [0.4.2] (October 10, 2023)
 
 ### Infrastructure / Support

config/add_dj_collaborator.py

@@ -1,31 +1,12 @@
 #!/usr/bin/env python
-import os
 import sys
-import tempfile
-
-
-def add_collab_user(user_name):
-    # create a temporary file for the command
-    file = tempfile.NamedTemporaryFile(mode="w")
-
-    # Create the user (if not already created) and set the password
-    file.write(
-        f"CREATE USER IF NOT EXISTS '{user_name}'@'%' IDENTIFIED BY 'temppass';\n"
-    )
-
-    # Grant privileges to databases matching the user_name pattern
-    file.write(
-        f"GRANT ALL PRIVILEGES ON `{user_name}\_%`.* TO '{user_name}'@'%';\n"
-    )
-
-    # Grant SELECT privileges on all databases
-    file.write(f"GRANT SELECT ON `%`.* TO '{user_name}'@'%';\n")
-
-    file.flush()
-
-    # run those commands in sql
-    os.system(f"mysql -p -h lmf-db.cin.ucsf.edu < {file.name}")
+from warnings import warn
 
+from spyglass.utils.database_settings import DatabaseSettings
 
 if __name__ == "__main__":
-    add_collab_user(sys.argv[1])
+    warn(
+        "This script is deprecated. "
+        + "Use spyglass.utils.database_settings.DatabaseSettings instead."
+    )
+    DatabaseSettings(user_name=sys.argv[1]).add_collab_user()

config/add_dj_guest.py

@@ -1,36 +1,12 @@
 #!/usr/bin/env python
-import os
 import sys
-import tempfile
-
-shared_modules = [
-    "common\_%",
-    "spikesorting\_%",
-    "decoding\_%",
-    "position\_%",
-    "position_linearization\_%",
-    "ripple\_%",
-    "lfp\_%",
-]
-
-
-def add_user(user_name):
-    # create a temporary file for the command
-    file = tempfile.NamedTemporaryFile(mode="w")
-
-    # Create the user (if not already created) and set password
-    file.write(
-        f"CREATE USER IF NOT EXISTS '{user_name}'@'%' IDENTIFIED BY 'Data_$haring';\n"
-    )
-
-    # Grant privileges
-    file.write(f"GRANT SELECT ON `%`.* TO '{user_name}'@'%';\n")
-
-    file.flush()
-
-    # run those commands in sql
-    os.system(f"mysql -p -h lmf-db.cin.ucsf.edu < {file.name}")
+from warnings import warn
 
+from spyglass.utils.database_settings import DatabaseSettings
 
 if __name__ == "__main__":
-    add_user(sys.argv[1])
+    warn(
+        "This script is deprecated. "
+        + "Use spyglass.utils.database_settings.DatabaseSettings instead."
+    )
+    DatabaseSettings(user_name=sys.argv[1]).add_dj_guest()

config/add_dj_module.py

@@ -1,40 +1,12 @@
 #!/usr/bin/env python
-import grp
-import os
 import sys
-import tempfile
-
-TARGET_GROUP = "kachery-users"
-
-
-def add_module(module_name):
-    print(f"Granting everyone permissions to module {module_name}")
-
-    # create a tempoary file for the command
-    file = tempfile.NamedTemporaryFile(mode="w")
-
-    # find the kachery-users group
-    groups = grp.getgrall()
-    group_found = False  # initialize the flag as False
-    for group in groups:
-        if group.gr_name == TARGET_GROUP:
-            group_found = True  # set the flag to True when the group is found
-            break
-
-    # Check if the group was found
-    if not group_found:
-        sys.exit(f"Error: The target group {TARGET_GROUP} was not found.")
-
-    # get a list of usernames
-    for user in group.gr_mem:
-        file.write(
-            f"GRANT ALL PRIVILEGES ON `{module_name}\_%`.* TO `{user}`@'%';\n"
-        )
-    file.flush()
-
-    # run those commands in sql
-    os.system(f"mysql -p -h lmf-db.cin.ucsf.edu < {file.name}")
+from warnings import warn
 
+from spyglass.utils.database_settings import DatabaseSettings
 
 if __name__ == "__main__":
-    add_module(sys.argv[1])
+    warn(
+        "This script is deprecated. "
+        + "Use spyglass.utils.database_settings.DatabaseSettings instead."
+    )
+    DatabaseSettings().add_module(sys.argv[1])

config/add_dj_user.py

@@ -1,44 +1,12 @@
 #!/usr/bin/env python
-import os
 import sys
-import tempfile
-
-shared_modules = [
-    "common\_%",
-    "spikesorting\_%",
-    "decoding\_%",
-    "position\_%",
-    "position_linearization\_%",
-    "ripple\_%",
-    "lfp\_%",
-]
-
-
-def add_user(user_name):
-    if os.path.isdir(f"/home/{user_name}"):
-        print("Creating database user ", user_name)
-    else:
-        sys.exit(f"Error: user_name {user_name} does not exist in /home.")
-
-    # create a tempoary file for the command
-    file = tempfile.NamedTemporaryFile(mode="w")
-    create_user_query = f"CREATE USER IF NOT EXISTS '{user_name}'@'%' IDENTIFIED BY 'temppass';\n"
-    grant_privileges_query = (
-        f"GRANT ALL PRIVILEGES ON `{user_name}\_%`.* TO '{user_name}'@'%';"
-    )
-
-    file.write(create_user_query + "\n")
-    file.write(grant_privileges_query + "\n")
-    for module in shared_modules:
-        file.write(
-            f"GRANT ALL PRIVILEGES ON `{module}`.* TO '{user_name}'@'%';\n"
-        )
-    file.write(f"GRANT SELECT ON `%`.* TO '{user_name}'@'%';\n")
-    file.flush()
-
-    # run those commands in sql
-    os.system(f"mysql -p -h lmf-db.cin.ucsf.edu < {file.name}")
+from warnings import warn
 
+from spyglass.utils.database_settings import DatabaseSettings
 
 if __name__ == "__main__":
-    add_user(sys.argv[1])
+    warn(
+        "This script is deprecated. "
+        + "Use spyglass.utils.database_settings.DatabaseSettings instead."
+    )
+    DatabaseSettings(user_name=sys.argv[1]).add_dj_user()

config/dj_config.py

@@ -2,110 +2,30 @@
 
 import os
 import sys
-import tempfile
 
-import datajoint as dj
-import yaml
-import json
-import warnings
 
-from pymysql.err import OperationalError
-
-
-def generate_config(filename: str = None, **kwargs):
-    """Generate a datajoint configuration file.
-
-    Parameters
-    ----------
-    filename : str
-        The name of the file to generate. Must be either yaml or json
-    **kwargs: list of parameters names and values that can include
-        base_dir : SPYGLASS_BASE_DIR
-        database_user : user name of system running mysql
-        database_host : mysql host name (default lmf-db.cin.ucsf.edu)
-        database_port : port number for mysql server (default 3306)
-        database_use_tls : Default True. Use TLS encryption.
-    """
-    # TODO: merge with existing spyglass.settings.py
-
-    base_dir = os.environ.get("SPYGLASS_BASE_DIR") or kwargs.get("base_dir")
-    if not base_dir:
-        raise ValueError(
-            "Please set base directory environment variable SPYGLASS_BASE_DIR"
-        )
-
-    base_dir = os.path.abspath(base_dir)
-    if not os.path.exists(base_dir):
-        warnings.warn(f"Base dir does not exist on this machine: {base_dir}")
-
-    raw_dir = os.path.join(base_dir, "raw")
-    analysis_dir = os.path.join(base_dir, "analysis")
-
-    config = {
-        "database.host": kwargs.get("database_host", "lmf-db.cin.ucsf.edu"),
-        "database.user": kwargs.get("database_user"),
-        "database.port": kwargs.get("database_port", 3306),
-        "database.use_tls": kwargs.get("database_use_tls", True),
-        "filepath_checksum_size_limit": 1 * 1024**3,
-        "enable_python_native_blobs": True,
-        "stores": {
-            "raw": {
-                "protocol": "file",
-                "location": raw_dir,
-                "stage": raw_dir,
-            },
-            "analysis": {
-                "protocol": "file",
-                "location": analysis_dir,
-                "stage": analysis_dir,
-            },
-        },
-        "custom": {"spyglass_dirs": {"base": base_dir}},
-    }
-    if not kwargs.get("database_user"):
-        # Adding then removing if empty retains order to make easier to read
-        config.pop("database.user")
-
-    if not filename:
-        filename = "dj_local_config.json"
-    if os.path(filename).exists():
-        warnings.warn(f"File already exists: {filename}")
-    else:
-        with open(filename, "w") as outfile:
-            if filename.endswith("json"):
-                json.dump(config, outfile, indent=2)
-            else:
-                yaml.dump(config, outfile, default_flow_style=False)
-
-    return config
-
-
-def set_configuration(config: dict):
-    """Sets the dj.config parameters.
-
-    Parameters
-    ----------
-    config : dict
-        Datajoint config as dictionary
-    """
-    # copy the elements of config to dj.config
-    for key, value in config.items():
-        dj.config[key] = value
+def main(*args):
+    database_user, base_dir, filename = args + (None,) * (3 - len(args))
 
-    dj.set_password()  # set the users password
-    dj.config.save_global()  # save these settings
+    os.environ["SPYGLASS_BASE_DIR"] = base_dir  # need to set for import to work
 
+    from spyglass.settings import SpyglassConfig  # noqa F401
 
-def main(*args):
-    user_name, base_dir, outfile = args + (None,) * (3 - len(args))
+    config = SpyglassConfig(base_dir=base_dir)
+    save_method = (
+        "local"
+        if filename == "dj_local_conf.json"
+        else "global"
+        if filename is None
+        else "custom"
+    )
 
-    config = generate_config(
-        outfile, database_user=user_name, base_dir=base_dir
+    config.save_dj_config(
+        save_method=save_method,
+        filename=filename,
+        base_dir=base_dir,
+        database_user=database_user,
     )
-    try:
-        set_configuration(config)
-    except OperationalError as e:
-        warnings.warn(f"Database connections issues: {e}")
 
 
 if __name__ == "__main__":

dj_local_conf_example.json

@@ -1,7 +1,7 @@
 {
-  "database.host": "localhost",
-  "database.password": "tutorial",
-  "database.user": "root",
+  "database.host": "localhost or lmf-db.cin.ucsf.edu",
+  "database.password": "Delete this line for shared machines",
+  "database.user": "Your username",
   "database.port": 3306,
   "database.reconnect": true,
   "connection.init_function": null,
@@ -28,7 +28,6 @@
     }
   },
   "custom": {
-    "database.prefix": "username_",
     "spyglass_dirs": {
       "base": "/your/base/path"
     },

docs/src/installation.md

@@ -47,6 +47,8 @@ additional details, see the
 
 ### Config
 
+#### Via File (Recommended)
+
 A `dj_local_conf.json` file in your Spyglass directory (or wherever python is
 launched) can hold all the specifics needed to connect to a database. This can
 include different directories for different pipelines. If only the `base` is
@@ -69,24 +71,37 @@ specified, the subfolder names below are included as defaults.
 }
 ```
 
-For those who prefer environment variables, the following can pasted into a
-file like `~/.bashrc`.
+`dj_local_conf_example.json` can be copied and saved as `dj_local_conf.json`
+to set the configuration for a given folder. Alternatively, it can be saved as
+`.datajoint_config.json` in a user's home directory to be accessed globally.
+See
+[DataJoint docs](https://datajoint.com/docs/core/datajoint-python/0.14/quick-start/#connection)
+for more details.
+
+#### Via Environment Variables
+
+Older versions of Spyglass relied exclusively on environment for config. If
+`spyglass_dirs` is not found in the config file, Spyglass will look for
+environment variables. These can be set either once in a terminal session, or
+permanently in a `.bashrc` file.
 
 ```bash
 export SPYGLASS_BASE_DIR="/stelmo/nwb"
 export SPYGLASS_RECORDING_DIR="$SPYGLASS_BASE_DIR/recording"
 export SPYGLASS_SORTING_DIR="$SPYGLASS_BASE_DIR/sorting"
 export SPYGLASS_VIDEO_DIR="$SPYGLASS_BASE_DIR/video"
 export SPYGLASS_WAVEFORMS_DIR="$SPYGLASS_BASE_DIR/waveforms"
-export SPYGLASS_TEMP_DIR="$SPYGLASS_BASE_DIR/tmp/spyglass"
+export SPYGLASS_TEMP_DIR="$SPYGLASS_BASE_DIR/tmp"
 export DJ_SUPPORT_FILEPATH_MANAGEMENT="TRUE"
 ```
 
-And then loaded with `source ~/.bashrc`.
+To load variables from a `.bashrc` file, run `source ~/.bashrc` in a terminal.
+
+#### Temporary directory
 
-Note that a local `SPYGLASS_TEMP_DIR` (e.g., one on your machine) will speed
-up spike sorting, but make sure it has enough free space (ideally at least
-500GB)
+A temporary directory will speed up spike sorting. If unspecified by either
+method above, it will be assumed as a `tmp` subfolder relative to the base
+path. Be sure it has enough free space (ideally at least 500GB).
 
 ## File manager