fix: 'Inappropriate file type or format' in Archive Utility and bsdcpio

This reverts commit 1130fdc, "feat: no need
for local zip64 extra when streaming"

It was thought that there was no need for the ZIP_64 "extra" section in the
local header. Tests ran fine without it, and there was an argument that having
it made the ZIP file invalid at
libarchive/libarchive#1834. However, it was found in
#27 that it caused an error on
OSX's Archive Utility:

> Error 79 - Inappropriate file type or format

and a very similar error in libarchive's bsdcpio

> cpio: Inconsistent uncompressed size: 12 in central directory, 4294967295 in
> local header: Inappropriate file type or format

or in some situations

> cpio: ZIP uncompressed data is wrong size (read 12, expected 0)

.github/workflows/tests.yml

@@ -38,7 +38,10 @@ jobs:
         with:
           name: reporter
           path: ./reporter
-      - name: "Install dependencies"
+      - name: "Install bsdcpio"
+        run: |
+          ./install-libarachive.sh
+      - name: "Install python dependencies"
         run: |
           pip install -r requirements-dev.txt
           chmod +x ./reporter/cc-test-reporter

.gitignore

@@ -127,3 +127,5 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+libarchive-*

CONTRIBUTING.md

@@ -121,6 +121,7 @@ Changes are then submitted via a Pull Request (PR). To do this:
 3. Make sure you can run existing tests locally
 
     ```bash
+    ./install-libarachive.sh             # Only needed once
     pip install -r requirements-dev.txt  # Only needed once
     pytest
     ```

install-libarachive.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -e
+
+# This version is the one that comes with macOS, and behaves most similarly to its Archive Utility
+curl --output libarchive-3.5.3.tar.gz https://www.libarchive.org/downloads/libarchive-3.5.3.tar.gz
+echo "72788e5f58d16febddfa262a5215e05fc9c79f2670f641ac039e6df44330ef51 libarchive-3.5.3.tar.gz" | sha256sum --check
+tar -zxf libarchive-3.5.3.tar.gz
+(
+    cd libarchive-3.5.3
+    ./configure
+    make
+)

stream_zip.py

@@ -80,6 +80,12 @@ def _zip_64_local_header_and_data(name_encoded, mod_at_encoded, external_attr, c
 
             _raise_if_beyond(file_offset, maximum=0xffffffffffffffff, exception_class=OffsetOverflowError)
 
+            extra = zip_64_local_extra_struct.pack(
+                zip_64_extra_signature,
+                16,  # Size of extra
+                0,   # Uncompressed size - since data descriptor
+                0,   # Compressed size - since data descriptor
+            )
             yield from _(local_header_signature)
             yield from _(local_header_struct.pack(
                 45,           # Version
@@ -90,9 +96,11 @@ def _zip_64_local_header_and_data(name_encoded, mod_at_encoded, external_attr, c
                 0xffffffff,   # Compressed size - since zip64
                 0xffffffff,   # Uncompressed size - since zip64
                 len(name_encoded),
-                0,            # Size of extra
+                len(extra),
             ))
             yield from _(name_encoded)
+            yield from _(extra)
+
             uncompressed_size, compressed_size, crc_32 = yield from _zip_data(
                 chunks,
                 max_uncompressed_size=0xffffffffffffffff,

test_stream_zip.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 from io import BytesIO
+import contextlib
 import os
 import subprocess
 from tempfile import TemporaryDirectory
@@ -561,3 +562,49 @@ def get_sizes():
     sizes = list(get_sizes())
     assert set(sizes[:-1]) == {65536}
     assert sizes[-1] <= 65536
+
+
+@pytest.mark.parametrize(
+    "method",
+    [
+        ZIP_32,
+        ZIP_64,
+    ],
+)
+def test_bsdcpio(method):
+    assert method in (ZIP_32, ZIP_64)  # Paranoia check that parameterisation works
+
+    now = datetime.fromisoformat('2021-01-01 21:01:12')
+    perms = 0o600
+    zip_bytes = b''.join(stream_zip((
+        ('file-1', now, perms, method, (b'contents',)),
+    )))
+
+    @contextlib.contextmanager
+    def cwd(new_dir):
+        old_dir = os.getcwd()
+        os.chdir(new_dir)
+        try:
+            yield
+        finally:
+            os.chdir(old_dir)
+
+    def read(path):
+        with open(path, 'rb') as f:
+            return f.read()
+
+    bsdcpio = os.getcwd() + '/libarchive-3.5.3/bsdcpio'
+    with \
+            TemporaryDirectory() as d, \
+            cwd(d), \
+            subprocess.Popen(
+                [bsdcpio, '-i'],
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            ) as p:
+
+        a = p.communicate(input=zip_bytes)
+        assert a == (b'', b'1 block\n')
+        assert p.returncode == 0
+        assert read('file-1') == b'contents'