🐛 FIX: Archive creation after packing repository (#5570)

`disk_objectstore.PackedObjectReader` does not implement the ability to `SEEK_END`,
required to determine the size of a stream.
The size of the stream is required to determine if the ZIP64 extension should be used,
so in this case, we now defer to `force_zip64=True`.

aiida/tools/archive/implementations/sqlite_zip/writer.py

@@ -167,12 +167,17 @@ def _stream_binary(
             kwargs['level'] = compression
 
         # compute the file size of the handle
-        position = handle.tell()
-        handle.seek(0, os.SEEK_END)
-        file_size = handle.tell()
-        handle.seek(position)
+        try:
+            position = handle.tell()
+            handle.seek(0, os.SEEK_END)
+            kwargs['file_size'] = handle.tell()
+            handle.seek(position)
+        except NotImplementedError:
+            # the disk-objectstore PackedObjectReader handler, does not support SEEK_END,
+            # so for these objects we always use ZIP64 to be safe
+            kwargs['force_zip64'] = True
 
-        with self._zip_path.joinpath(name).open(mode='wb', file_size=file_size, **kwargs) as zip_handle:
+        with self._zip_path.joinpath(name).open(mode='wb', **kwargs) as zip_handle:
             if buffer_size is None:
                 shutil.copyfileobj(handle, zip_handle)
             else:

environment.yml

@@ -7,7 +7,7 @@ channels:
 dependencies:
 - python~=3.8
 - alembic~=1.2
-- archive-path~=0.4.0
+- archive-path~=0.4.1
 - aio-pika~=6.6
 - circus~=0.17.1
 - click-config-file~=0.6.0

pyproject.toml

@@ -25,7 +25,7 @@ keywords = ["aiida", "workflows"]
 requires-python = ">=3.8"
 dependencies = [
         "alembic~=1.2",
-        "archive-path~=0.4.0",
+        "archive-path~=0.4.1",
         "aio-pika~=6.6",
         "circus~=0.17.1",
         "click-config-file~=0.6.0",

requirements/requirements-py-3.10.txt

@@ -4,7 +4,7 @@ aiormq==3.3.1
 alabaster==0.7.12
 alembic==1.7.5
 aniso8601==9.0.1
-archive-path==0.4.0
+archive-path==0.4.1
 argon2-cffi==21.1.0
 ase==3.22.0
 asn1crypto==1.4.0

requirements/requirements-py-3.8.txt

@@ -4,7 +4,7 @@ aiormq==3.3.1
 alabaster==0.7.12
 alembic==1.7.5
 aniso8601==9.0.1
-archive-path==0.4.0
+archive-path==0.4.1
 argon2-cffi==21.1.0
 ase==3.22.0
 asn1crypto==1.4.0

requirements/requirements-py-3.9.txt

@@ -4,7 +4,7 @@ aiormq==3.3.1
 alabaster==0.7.12
 alembic==1.7.5
 aniso8601==9.0.1
-archive-path==0.4.0
+archive-path==0.4.1
 argon2-cffi==21.1.0
 ase==3.22.0
 asn1crypto==1.4.0

tests/tools/archive/test_repository.py

@@ -13,6 +13,7 @@
 import pytest
 
 from aiida import orm
+from aiida.manage import get_manager
 from aiida.tools.archive import create_archive, import_archive
 
 
@@ -36,3 +37,29 @@ def test_export_repository(aiida_profile, tmp_path):
     assert loaded.base.repository.metadata == repository_metadata
     assert loaded.base.repository.get_object_content('file_a', mode='rb') == b'file_a'
     assert loaded.base.repository.get_object_content('relative/file_b', mode='rb') == b'file_b'
+
+
+@pytest.mark.usefixtures('aiida_profile_clean')
+def test_export_repository_after_maintain(aiida_profile, tmp_path):
+    """Test exporting a node with files in the repository after maintenance.
+
+    Maintenance for the disk-objectstore repository will pack the files into binary files.
+    """
+    node = orm.Data()
+    node.base.repository.put_object_from_bytes(b'file_a', 'file_a')
+    node.base.repository.put_object_from_bytes(b'file_b', 'relative/file_b')
+    node.store()
+    node_uuid = node.uuid
+
+    repository = get_manager().get_profile_storage().get_repository()
+    repository.maintain(live=False)
+
+    filepath = os.path.join(tmp_path / 'export.aiida')
+    create_archive([node], filename=filepath)
+
+    aiida_profile.clear_profile()
+    import_archive(filepath)
+
+    loaded = orm.load_node(uuid=node_uuid)
+    assert loaded.base.repository.get_object_content('file_a', mode='rb') == b'file_a'
+    assert loaded.base.repository.get_object_content('relative/file_b', mode='rb') == b'file_b'