diff --git a/flyem_snapshot/outputs/neuprint/scripts/ingest-neuprint-snapshot-using-apptainer.sh b/flyem_snapshot/outputs/neuprint/scripts/ingest-neuprint-snapshot-using-apptainer.sh index f341831..2ef84c0 100755 --- a/flyem_snapshot/outputs/neuprint/scripts/ingest-neuprint-snapshot-using-apptainer.sh +++ b/flyem_snapshot/outputs/neuprint/scripts/ingest-neuprint-snapshot-using-apptainer.sh @@ -79,6 +79,15 @@ else then exit $? fi + + # In theory, the apptainer command above ought to have failed already if there was a failure in the log. + # But I'm not sure if that works as it's supposed to, so here's an extra check + if grep -i 'import failed' ${WORKSPACE_DIR}/logs/import.*.log > /dev/null; + then + echo "ERROR: Apptainer exited cleanly, but the neo4j log indicates that the import failed!" 2>&1 + exit 1 + if + # Now copy the database files from /scratch to the snapshot directory echo "$(date '+%Y-%m-%d %H:%M:%S') Copying database to ${SNAPSHOT_DIR}" cp -R ${WORKSPACE_DIR} ${SNAPSHOT_DIR}/ diff --git a/flyem_snapshot/outputs/neuprint/scripts/ingest-neuprint-snapshot-within-neo4j-container.sh b/flyem_snapshot/outputs/neuprint/scripts/ingest-neuprint-snapshot-within-neo4j-container.sh index 84c9ce2..250f0ca 100755 --- a/flyem_snapshot/outputs/neuprint/scripts/ingest-neuprint-snapshot-within-neo4j-container.sh +++ b/flyem_snapshot/outputs/neuprint/scripts/ingest-neuprint-snapshot-within-neo4j-container.sh @@ -133,8 +133,15 @@ echo "[$(date)] Ingesting nodes and relationships" # (Error: Argument list too long) # Luckily, we can supply the arguments via a file! # https://github.com/neo4j/neo4j/issues/7333#issuecomment-1746238765 -/var/lib/neo4j/bin/neo4j-admin import @ingestion-args.txt | tee /logs/import.log +/var/lib/neo4j/bin/neo4j-admin import @ingestion-args.txt > >(tee /logs/import.out.log) 2> >(tee /logs/import.err.log) end=$(date +%s) + +if grep -i 'import failed' /logs/import.*.log > /dev/null; +then + echo "[$(date)] Node/relationship ingest FAILED. See /logs/import.*.log" + exit 1 +fi + echo "[$(date)] Node/relationship ingest completed." echo "Duration: $(date -d@$((end-start)) -u +%H:%M:%S)" @@ -163,7 +170,13 @@ grep -q 'Started\.' <(tail -n1 -f /logs/neo4j.log) -d data \ --format verbose \ -f /snapshot/create-indexes.cypher \ - | tee /logs/create-indexes.log + >(tee /logs/create-indexes.out.log) 2> >(tee /logs/create-indexes.err.log) ## +if grep -i 'database is unavailable' /logs/create-indexes.*.log > /dev/null || [ ! -s /logs/create-indexes.out.log ]; +then + echo "[$(date)] Index generation FAILED. See /logs/create-indexes.*.log" + exit 1 +fi + end=$(date +%s)