Merge pull request #472 from European-XFEL/feat/validation-file-access

Validation: clearer info on filesystem errors

extra_data/tests/test_validation.py

@@ -6,7 +6,7 @@
 from pytest import fixture, raises
 from tempfile import TemporaryDirectory
 
-from extra_data.validation import FileAccess, FileValidator, RunValidator, ValidationError
+from extra_data.validation import FileAccess, FileValidator, RunValidator, ValidationError, main
 from . import make_examples
 
 
@@ -39,7 +39,7 @@ def test_file_error(mock_fxe_raw_run):
 
     problems = RunValidator(mock_fxe_raw_run).run_checks()
     assert len(problems) == 1
-    assert problems[0]['msg'] == 'Could not open file'
+    assert problems[0]['msg'] == 'Could not access file'
     assert problems[0]['file'] == str(not_readable)
 
 
@@ -170,3 +170,12 @@ def test_control_data_timestamps(data_aggregator_file):
     assert problem['msg'] == 'Timestamp is decreasing, e.g. at 10 (5 < 10)'
     assert problem['dataset'] == 'CONTROL/SA1_XTD2_XGM/DOOCS/MAIN/pulseEnergy/photonFlux/timestamp'
     assert 'RAW-R0450-DA01-S00001.h5' in problem['file']
+
+
+def test_main_file_non_h5(tmp_path, capsys):
+    not_h5 = tmp_path / 'notHDF5.h5'
+    not_h5.write_text("Accessible file, not HDF5")
+
+    status = main([str(not_h5)])
+    assert status == 1
+    assert 'Could not open HDF5 file' in capsys.readouterr().out

extra_data/validation.py

@@ -211,18 +211,30 @@ def check_index_contiguous(firsts, counts, record):
         ))
 
 
-def _check_file(args):
-    runpath, filename = args
-    filepath = osp.join(runpath, filename)
-    problems = []
+def _open_file(filepath):
     try:
         fa = FileAccess(filepath)
     except Exception as e:
-        problems.append(
-            dict(msg="Could not open file", file=filepath, error=e)
-        )
-        return filename, None, problems
+        try:
+            with open(filepath, "rb") as f:
+                f.read(16)
+        except OSError as e2:
+            # Filesystem issue, e.g. dCache node down. HDF5 errors can be
+            # confusing, so record the OS error instead.
+            pb = dict(msg="Could not access file", file=filepath, error=e2)
+        else:
+            # HDF5 file corrupted or missing expected information
+            pb = dict(msg="Could not open HDF5 file", file=filepath, error=e)
+        return None, [pb]
     else:
+        return fa, []
+
+
+def _check_file(args):
+    runpath, filename = args
+    filepath = osp.join(runpath, filename)
+    fa, problems = _open_file(filepath)
+    if fa is not None:
         fv = FileValidator(fa)
         problems.extend(fv.run_checks())
         fa.close()
@@ -328,7 +340,12 @@ def main(argv=None):
         validator = RunValidator(path, term_progress=True)
     else:
         print("Checking file:", path)
-        validator = FileValidator(H5File(path).files[0])
+        fa, problems = _open_file(path)
+        if problems:
+            print(str(ValidationError(problems)))
+            return 1
+
+        validator = FileValidator(fa)
 
     try:
         validator.run_checks()