Omics fixes

api/python/Makefile

@@ -1,5 +1,5 @@
 
-test:
+test: install-local
 	pytest --disable-warnings
 
 install-local:

api/python/TESTS_README

@@ -1 +1,3 @@
 Use pytest during normal development.
+
+You may need to first `make install-local`.

api/python/quilt3/packages.py

@@ -34,6 +34,7 @@
     legacy_calculate_checksum,
     legacy_calculate_checksum_bytes,
     list_object_versions,
+    list_objects,
     list_url,
     put_bytes,
 )
@@ -910,11 +911,19 @@ def set_dir(self, lkey, path=None, meta=None, update_policy="incoming"):
             src_path = src.path
             if src.basename() != '':
                 src_path += '/'
-            objects, _ = list_object_versions(src.bucket, src_path)
+            try:
+                objects, _ = list_object_versions(src.bucket, src_path)
+            except botocore.exceptions.ClientError as e:
+                # use list_objects instead
+                print(f"list_object_versions not available; using list_objects:\n{e}")
+                objects = list_objects(src.bucket, src_path, recursive=True)
+                for obj in objects:
+                    obj["IsLatest"] = True
+
             for obj in objects:
                 if not obj['IsLatest']:
                     continue
-                # Skip S3 pseduo directory files and Keys that end in /
+                # Skip S3 pseudo-directory files and Keys that end in /
                 if obj['Key'].endswith('/'):
                     if obj['Size'] != 0:
                         warnings.warn(f'Logical keys cannot end in "/", skipping: {obj["Key"]}')

api/python/tests/integration/test_access.py

@@ -0,0 +1,59 @@
+import os
+from datetime import UTC, datetime
+
+import boto3
+import pytest
+
+import quilt3 as q3
+from quilt3.data_transfer import list_object_versions
+
+NOW = datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S")
+BKT = "850787717197-1867753-fepwgrx9iujr5b9pkjudkhpgxwbuhuse1b-s3alias"
+SOURCE = f"s3://{BKT}"
+DBKT = "quilt-sales-staging"
+DEST = f"s3://{DBKT}"
+FOLDER = "850787717197/sequenceStore/1867753048/readSet/5447294294"
+FILE = "U0a_CGATGT_L001_R1_004.fastq.gz"
+KEY = f"{FOLDER}/{FILE}"
+
+
+@pytest.fixture(autouse=True)
+def client():
+    os.environ["AWS_PROFILE"] = "sales"
+    session = boto3.Session(profile_name="sales")
+    return session.client("s3")
+
+
+def test_boto3_access(client):
+    head_object = client.head_object(Bucket=BKT, Key=KEY)
+    assert head_object
+    print(f"head_object: {head_object}")
+    get_object = client.get_object(Bucket=BKT, Key=KEY)
+    print(f"get_object: {get_object}")
+    list_bucket = client.list_objects(Bucket=BKT, Prefix=FOLDER)
+    print(f"list_bucket: {list_bucket}")
+
+
+def test_package(client):
+    dir = f"{SOURCE}/{FOLDER}"
+    split = FOLDER.split("/")
+    pkg_name = f"{split[-2]}/{split[-1]}"
+    msg = f"Today's Date: {NOW}"
+
+    pkg = q3.Package()
+    try:
+        pkg = pkg.browse(pkg_name, registry=DEST)
+    except Exception as e:
+        print(f"Error browsing package: {e}")
+    print(f"Package: {pkg}")
+
+    print(f"S3 URI: {dir} @ {msg}")
+    pkg.set_dir("/", dir, meta={"timestamp": NOW})
+    print(f"Package.set_dir: {pkg}")
+    assert pkg
+
+    PKG_URI = f"quilt+{DEST}#package={pkg_name}"
+    print(f"Pushing {pkg_name} to {DEST}: {PKG_URI}")
+    check = client.list_objects(Bucket=DBKT, Prefix=pkg_name)
+    assert check.get("Prefix") == pkg_name
+    pkg.push(pkg_name, registry=DEST, message=msg, force=True)

docs/sha2-256-chunked.md

@@ -0,0 +1,43 @@
+# sha2-256-chunked
+
+## DRAFT multihash codec 0xb510
+
+> Hash of concatenated SHA2-256 digests of 8*2^n MiB source chunks
+> where n = ceil(log2(source_size / (10^4 * 8MiB))
+
+This variant of sha2-256 is designed to enable large files
+to be efficiently uploaded and hashed in parallel using fixed size chunks
+(8 MiB to start with), with the final result being a "top hash"
+that doesn't depend on the upload order.
+
+The algorithm has an upper limit of 10,000 chunks.
+If the file is larger than 80,000 MiB,
+it will double the chunk size until the number of chunks
+is under that limit.
+
+```pseudocode
+n = ceil(log2(source_size / (10_000 * 8 MiB)))
+chunk_size = 8 MiB * 2^n
+```
+
+## Inspiration
+
+This algorithm is inspired by Amazon's 
+[S3 Checksums](https://aws.amazon.com/blogs/aws/new-additional-checksum-algorithms-for-amazon-s3/)
+implementation.
+
+The key differences are:
+
+* Fixing the chunk size as (starting from) 8 MiB.
+* Always hashing the result (even if source_size < 8 MiB).
+
+It can reuse hashes generated by
+[create_multipart_upload](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/create_multipart_upload.html)
+as of at least [boto3 1.34.44](https://pypi.org/project/boto3/1.34.44/) 
+(2024-02-16), simply by rehashing the value if source_size < 8 MiB.
+
+## Status 2024-02-21
+
+It has been submitted for draft
+[multiformats registration](https://github.com/multiformats/multiformats/blob/master/contributing.md#multiformats-registrations)
+under the name `sha2-256-chunked` using the prefix `0xb510`.