Add data module and prefetched videos in Git LFS.

rtlnl · Oct 4, 2022 · ad4bd11 · ad4bd11
1 parent 879e657
commit ad4bd11
Show file tree

Hide file tree

Showing 14 changed files with 63 additions and 0 deletions.
diff --git a/.DS_Store b/.DS_Store
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1 @@
+data/*.index filter=lfs diff=lfs merge=lfs -text
diff --git a/.gitignore b/.gitignore
@@ -130,3 +130,6 @@ dmypy.json
 
 # Gradio
 gradio_cached_examples/
+
+# MacOS
+.DS_Store
diff --git a/data.py b/data.py
@@ -0,0 +1,29 @@
+import os
+import json
+import shutil
+
+from videohash import filepath_from_url
+
+with open('apb2022.json') as filein:
+    urls, videos, url2video, video2url = [], [], {}, {}
+    for item in json.load(filein):
+        urls.append(item['url'])
+        videos.append(item['mp4'])
+        url2video[item['url']] = item['mp4']
+        video2url[item['mp4']] = item['url']
+
+for url in videos:
+    filepath = filepath_from_url(url) + '.index'
+    datapath = os.path.join('data', os.path.basename(filepath))
+    if not os.path.exists(filepath) and os.path.exists(datapath):
+        shutil.copyfile(datapath, filepath)
+
+
+if __name__ == "__main__":
+    from videomatch import get_video_index
+
+    for url in videos:
+        get_video_index(url)
+        filepath = filepath_from_url(url) + '.index'
+        datapath = os.path.join('data', os.path.basename(filepath))
+        shutil.copyfile(filepath, datapath)
diff --git a/data/0fd24e056447f6cd068f40ba2e41828c.index b/data/0fd24e056447f6cd068f40ba2e41828c.index
diff --git a/data/1de5e559d3d190ca09ac3fa6b40a0138.index b/data/1de5e559d3d190ca09ac3fa6b40a0138.index
diff --git a/data/265732a300bbe18c4d39df108fe0c121.index b/data/265732a300bbe18c4d39df108fe0c121.index
diff --git a/data/2909574cb590231c5d1028032bd849a8.index b/data/2909574cb590231c5d1028032bd849a8.index
diff --git a/data/2f71b6c1d5c91c4b73eef2ebd05bd848.index b/data/2f71b6c1d5c91c4b73eef2ebd05bd848.index
diff --git a/data/75617545ba72efdc9a7b2b140184a9c6.index b/data/75617545ba72efdc9a7b2b140184a9c6.index
diff --git a/data/9c0a24aa59358ee014ded54c88eaa6c1.index b/data/9c0a24aa59358ee014ded54c88eaa6c1.index
diff --git a/data/a3efa8f2e01e0964d7987e6b1e4b3b44.index b/data/a3efa8f2e01e0964d7987e6b1e4b3b44.index
diff --git a/data/b3cf94671de7aadc22970483150aafd5.index b/data/b3cf94671de7aadc22970483150aafd5.index
diff --git a/data/d0369414718aa9573ff9d009f4f61cc0.index b/data/d0369414718aa9573ff9d009f4f61cc0.index
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		data/*.index filter=lfs diff=lfs merge=lfs -text