ingest-neuprint-snapshot: Adding some extra error checks to ensure th…

…e job exits with an error code if the import failed.

Related to #20

flyem_snapshot/outputs/neuprint/scripts/ingest-neuprint-snapshot-using-apptainer.sh

@@ -79,6 +79,15 @@ else
     then
         exit $?
     fi
+
+    # In theory, the apptainer command above ought to have failed already if there was a failure in the log.
+    # But I'm not sure if that works as it's supposed to, so here's an extra check
+    if grep -i 'import failed' ${WORKSPACE_DIR}/logs/import.*.log > /dev/null;
+    then
+        echo "ERROR: Apptainer exited cleanly, but the neo4j log indicates that the import failed!" 2>&1
+        exit 1
+    if
+
     # Now copy the database files from /scratch to the snapshot directory
     echo "$(date '+%Y-%m-%d %H:%M:%S') Copying database to ${SNAPSHOT_DIR}"
     cp -R ${WORKSPACE_DIR} ${SNAPSHOT_DIR}/

flyem_snapshot/outputs/neuprint/scripts/ingest-neuprint-snapshot-within-neo4j-container.sh

@@ -133,8 +133,15 @@ echo "[$(date)] Ingesting nodes and relationships"
 # (Error: Argument list too long)
 # Luckily, we can supply the arguments via a file!
 # https://github.com/neo4j/neo4j/issues/7333#issuecomment-1746238765
-/var/lib/neo4j/bin/neo4j-admin import @ingestion-args.txt | tee /logs/import.log
+/var/lib/neo4j/bin/neo4j-admin import @ingestion-args.txt > >(tee /logs/import.out.log) 2> >(tee /logs/import.err.log)
 end=$(date +%s)
+
+if grep -i 'import failed' /logs/import.*.log > /dev/null;
+then
+    echo "[$(date)] Node/relationship ingest FAILED. See /logs/import.*.log"
+    exit 1
+fi
+
 echo "[$(date)] Node/relationship ingest completed."
 echo "Duration: $(date -d@$((end-start)) -u +%H:%M:%S)"
 
@@ -163,7 +170,13 @@ grep -q 'Started\.' <(tail -n1 -f /logs/neo4j.log)
     -d data \
     --format verbose \
     -f /snapshot/create-indexes.cypher \
-    | tee /logs/create-indexes.log
+    >(tee /logs/create-indexes.out.log) 2> >(tee /logs/create-indexes.err.log)
 ##
 
+if grep -i 'database is unavailable' /logs/create-indexes.*.log > /dev/null || [ ! -s /logs/create-indexes.out.log ];
+then
+    echo "[$(date)] Index generation FAILED. See /logs/create-indexes.*.log"
+    exit 1
+fi
+
 end=$(date +%s)