LorenFrankLab#719, LorenFrankLab#804, LorenFrankLab#212

CHANGELOG.md

@@ -22,6 +22,7 @@
 - Add overview of Spyglass to docs. #779
 - Update linting for Black 24. #808
 - Steamline dependency management. #822
+- Add catch errorst during `populate_all_common`, log in `common_usage`. #XXX
 
 ### Pipelines
 

src/spyglass/common/common_behav.py

@@ -42,6 +42,20 @@ class SpatialSeries(SpyglassMixin, dj.Part):
         name=null: varchar(32)       # name of spatial series
         """
 
+    def populate(self, key=None):
+        """Insert position source data from NWB file.
+
+        WARNING: populate method on Manual table is not protected by transaction
+                protections like other DataJoint tables.
+        """
+        nwb_file_name = key.get("nwb_file_name")
+        if not nwb_file_name:
+            raise ValueError(
+                "PositionSource.populate is an alias for a non-computed table "
+                + "and must be passed a key with nwb_file_name"
+            )
+        self.insert_from_nwbfile(nwb_file_name)
+
     @classmethod
     def insert_from_nwbfile(cls, nwb_file_name):
         """Add intervals to ItervalList and PositionSource.

src/spyglass/common/common_usage.py

@@ -1,6 +1,7 @@
 """A schema to store the usage of advanced Spyglass features.
 
-Records show usage of features such as table chains, which will be used to
+Records show usage of features such as cautious delete and fault-permitting
+insert, which will be used to
 determine which features are used, how often, and by whom. This will help
 plan future development of Spyglass.
 """
@@ -21,3 +22,17 @@ class CautiousDelete(dj.Manual):
     restriction: varchar(255)
     merge_deletes = null: blob
     """
+
+
+@schema
+class InsertError(dj.Manual):
+    definition = """
+    id: int auto_increment
+    ---
+    dj_user: varchar(64)
+    connection_id: int     # MySQL CONNECTION_ID()
+    table: varchar(64)
+    error_type: varchar(64)
+    error_message: varchar(255)
+    error_raw = null: blob
+    """

src/spyglass/common/populate_all_common.py

@@ -1,3 +1,5 @@
+import datajoint as dj
+
 from spyglass.common.common_behav import (
     PositionSource,
     RawPosition,
@@ -14,51 +16,54 @@
 from spyglass.common.common_nwbfile import Nwbfile
 from spyglass.common.common_session import Session
 from spyglass.common.common_task import TaskEpoch
+from spyglass.common.common_usage import InsertError
+from spyglass.spikesorting.imported import ImportedSpikeSorting
 from spyglass.utils import logger
 
 
 def populate_all_common(nwb_file_name):
-    # Insert session one by one
-    fp = [(Nwbfile & {"nwb_file_name": nwb_file_name}).proj()]
-    logger.info("Populate Session...")
-    Session.populate(fp)
-
-    # If we use Kachery for data sharing we can uncomment the following two lines. TBD
-    # logger.info('Populate NwbfileKachery...')
-    # NwbfileKachery.populate()
-
-    logger.info("Populate ElectrodeGroup...")
-    ElectrodeGroup.populate(fp)
-
-    logger.info("Populate Electrode...")
-    Electrode.populate(fp)
-
-    logger.info("Populate Raw...")
-    Raw.populate(fp)
-
-    logger.info("Populate SampleCount...")
-    SampleCount.populate(fp)
-
-    logger.info("Populate DIOEvents...")
-    DIOEvents.populate(fp)
-
-    # sensor data (from analog ProcessingModule) is temporarily removed from NWBFile
-    # to reduce file size while it is not being used. add it back in by commenting out
-    # the removal code in spyglass/data_import/insert_sessions.py when ready
-    # logger.info('Populate SensorData')
-    # SensorData.populate(fp)
-
-    logger.info("Populate TaskEpochs")
-    TaskEpoch.populate(fp)
-    logger.info("Populate StateScriptFile")
-    StateScriptFile.populate(fp)
-    logger.info("Populate VideoFile")
-    VideoFile.populate(fp)
-    logger.info("RawPosition...")
-    PositionSource.insert_from_nwbfile(nwb_file_name)
-    RawPosition.populate(fp)
-
-    logger.info("Populate ImportedSpikeSorting...")
-    from spyglass.spikesorting.imported import ImportedSpikeSorting
+    """Insert all common tables for a given NWB file."""
+    key = [(Nwbfile & f"nwb_file_name LIKE '{nwb_file_name}'").proj()]
+    tables = [
+        Session,
+        # NwbfileKachery, # Not used by default
+        ElectrodeGroup,
+        Electrode,
+        Raw,
+        SampleCount,
+        DIOEvents,
+        # SensorData, # Not used by default. Generates large files
+        RawPosition,
+        TaskEpoch,
+        StateScriptFile,
+        VideoFile,
+        PositionSource,
+        RawPosition,
+        ImportedSpikeSorting,
+    ]
+    error_constants = dict(
+        dj_user=dj.config["database.user"],
+        connection_id=dj.conn().connection_id,
+    )
 
-    ImportedSpikeSorting.populate(fp)
+    for table in tables:
+        logger.info(f"Populating {table.__name__}...")
+        try:
+            table.populate(key)
+        except Exception as e:
+            InsertError.insert1(
+                dict(
+                    **error_constants,
+                    table=table.__name__,
+                    error_type=type(e).__name__,
+                    error_message=str(e),
+                    error_raw=str(e),
+                )
+            )
+    query = InsertError & error_constants
+    if query:
+        logger.error(
+            f"Errors occurred during population:\n{query}\n"
+            + "See common_useage.InsertError for more details"
+        )
+        return query.fetch("KEY")

src/spyglass/utils/dj_mixin.py

@@ -215,8 +215,8 @@ def delete_downstream_merge(
         if not merge_join_dict and not disable_warning:
             logger.warning(
                 f"No merge deletes found w/ {self.table_name} & "
-                + f"{restriction}.\n\tIf this is unexpected, try running with "
-                + "`reload_cache`."
+                + f"{restriction}.\n\tIf this is unexpected, try importing "
+                + " Merge table(s) and running with `reload_cache`."
             )
 
         if dry_run: