parallel cleanup

ipitio · Oct 19, 2024 · 42ceb38 · 42ceb38
1 parent 127aebc
commit 42ceb38
Showing 1 changed file with 6 additions and 2 deletions.
diff --git a/src/main.py b/src/main.py
@@ -8,6 +8,7 @@
 from pathlib import Path
 
 import pymupdf
+from joblib import Parallel, delayed
 from natsort import natsorted, ns
 from PIL import Image
 
@@ -113,8 +114,11 @@ def merge(base: Path, root: str, files: list[str]) -> None:
                 ),
             )
 
-    for root, _, files in os.walk(pdfs / "todo"):
-        cleanup(root, files)
+    # this can be parallel
+    Parallel(n_jobs=-1)(
+        delayed(cleanup)(root, files) for root, _, files in os.walk(pdfs / "todo")
+    )
 
+    # but the order here matters
     for root, _, files in os.walk(pdfs / "done"):
         merge(pdfs, root, files)