Merge pull request #542 from OpenCOMPES/more-broken-file-fixes

add further exceptions for completely empty files, and exceptions

src/sed/loader/mirrorutil.py

@@ -368,7 +368,10 @@ def mycopy(source: str, dest: str, gid: int, mode: int, replace: bool = False):
     if replace:
         if os.path.exists(dest):
             os.remove(dest)
-    shutil.copy2(source, dest)
+    try:
+        shutil.copy2(source, dest)
+    except OSError:
+        shutil.copy(source, dest)
     # fix permissions and group ownership:
     os.chown(dest, -1, gid)
     os.chmod(dest, mode)

src/sed/loader/mpes/loader.py

@@ -159,7 +159,12 @@ def hdf5_to_dataframe(
     for name, channel in channels.items():
         if channel["format"] == "per_file":
             if channel["dataset_key"] in test_proc.attrs:
-                values = [float(get_attribute(h5py.File(f), channel["dataset_key"])) for f in files]
+                values = []
+                for f in files:
+                    try:
+                        values.append(float(get_attribute(h5py.File(f), channel["dataset_key"])))
+                    except OSError:
+                        pass
                 delayeds = [
                     add_value(partition, name, value)
                     for partition, value in zip(dataframe.partitions, values)
@@ -274,7 +279,12 @@ def hdf5_to_timed_dataframe(
     for name, channel in channels.items():
         if channel["format"] == "per_file":
             if channel["dataset_key"] in test_proc.attrs:
-                values = [float(get_attribute(h5py.File(f), channel["dataset_key"])) for f in files]
+                values = []
+                for f in files:
+                    try:
+                        values.append(float(get_attribute(h5py.File(f), channel["dataset_key"])))
+                    except OSError:
+                        pass
                 delayeds = [
                     add_value(partition, name, value)
                     for partition, value in zip(dataframe.partitions, values)
@@ -843,11 +853,23 @@ def get_start_and_end_time(self) -> tuple[float, float]:
         )
         ts_from = timestamps[-1][1]
         h5filename = self.files[-1]
-        timestamps = hdf5_to_array(
-            h5filename=h5filename,
-            channels=channels,
-            time_stamps=True,
-        )
+        try:
+            timestamps = hdf5_to_array(
+                h5filename=h5filename,
+                channels=channels,
+                time_stamps=True,
+            )
+        except OSError:
+            try:
+                h5filename = self.files[-2]
+                timestamps = hdf5_to_array(
+                    h5filename=h5filename,
+                    channels=channels,
+                    time_stamps=True,
+                )
+            except OSError:
+                ts_to = ts_from
+                logger.warning("Could not read end time, using start time as end time!")
         ts_to = timestamps[-1][-1]
         return (ts_from, ts_to)