TST: ensure old tests pass

tiled/_tests/test_custom_format.py

@@ -33,7 +33,7 @@ async def test_xdi_round_trip(tmpdir):
                     "readable_storage": [tmpdir / "files"],
                     "init_if_not_exists": True,
                     "adapters_by_mimetype": {
-                        "application/x-xdi": "tiled.examples.xdi:read_xdi"
+                        "application/x-xdi": "tiled.examples.xdi:XDIAdapter"
                     },
                 },
             }
@@ -46,7 +46,7 @@ async def test_xdi_round_trip(tmpdir):
         await register(
             client,
             tmpdir / "files",
-            adapters_by_mimetype={"application/x-xdi": "tiled.examples.xdi:read_xdi"},
+            adapters_by_mimetype={"application/x-xdi": "tiled.examples.xdi:XDIAdapter"},
             mimetypes_by_file_ext={".xdi": "application/x-xdi"},
         )
         client["example"].export(str(tmpdir / "exported.xdi"))

tiled/_tests/test_directory_walker.py

@@ -203,17 +203,35 @@ async def test_image_file_with_sidecar_metadata_file(tmpdir):
     with open(metadata_filepath, "w") as file:
         yaml.dump(metadata, file)
 
-    def read_tiff_with_yaml_metadata(image_uri, metadata_uri, metadata=None, **kwargs):
-        with open(path_from_uri(metadata_uri)) as file:
-            metadata = yaml.safe_load(file)
-        return TiffAdapter(image_uri, metadata=metadata, **kwargs)
+    class TiffAdapterWithSidecar(TiffAdapter):
+        def __init__(self, image_uri, metadata_uri, metadata=None, **kwargs):
+            with open(path_from_uri(metadata_uri)) as file:
+                metadata = yaml.safe_load(file)
+
+            super().__init__(image_uri, metadata=metadata, **kwargs)
+
+        @classmethod
+        def from_assets(
+            cls,
+            assets: list[Asset],
+            structure: ArrayStructure,
+            metadata=None,
+            specs=None,
+            **kwargs,
+        ):
+            for ast in assets:
+                if ast.parameter == "image_uri":
+                    image_uri = ast.data_uri
+                if ast.parameter == "metadata_uri":
+                    metadata_uri = ast.data_uri
+            return cls(image_uri, metadata_uri, structure=structure, specs=specs)
 
     catalog = in_memory(
         writable_storage=tmpdir,
-        adapters_by_mimetype={MIMETYPE: read_tiff_with_yaml_metadata},
+        adapters_by_mimetype={MIMETYPE: TiffAdapterWithSidecar},
     )
     with Context.from_app(build_app(catalog)) as context:
-        adapter = read_tiff_with_yaml_metadata(
+        adapter = TiffAdapterWithSidecar(
             ensure_uri(image_filepath), ensure_uri(metadata_filepath)
         )
         client = from_context(context)
@@ -295,7 +313,7 @@ async def test_hdf5_virtual_datasets(tmpdir):
     )
     catalog = in_memory(writable_storage=tmpdir)
     with Context.from_app(build_app(catalog)) as context:
-        adapter = HDF5Adapter.from_uri(ensure_uri(filepath))
+        adapter = HDF5Adapter.from_uris(ensure_uri(filepath))
         client = from_context(context)
         client.new(
             key="VDS",

tiled/adapters/awkward_buffers.py

@@ -79,6 +79,4 @@ def from_assets(
         specs: Optional[List[Spec]] = None,
         **kwargs: Optional[Union[str, List[str], Dict[str, str]]],
     ) -> "AwkwardBuffersAdapter":
-        return cls.from_directory(
-            assets[0].data_uri, structure, metadata, specs
-        )
+        return cls.from_directory(assets[0].data_uri, structure, metadata, specs)

tiled/adapters/csv.py

@@ -70,7 +70,7 @@ def from_assets(
     def from_uris(
         cls,
         data_uris: Union[str, List[str]],
-        **kwargs: Optional[Union[str, List[str], Dict[str, str]]],
+        **kwargs: Optional[Any],
     ) -> "CSVAdapter":
         if isinstance(data_uris, str):
             data_uris = [data_uris]

tiled/adapters/excel.py

@@ -60,7 +60,9 @@ def from_file(cls, file: Any, **kwargs: Any) -> "ExcelAdapter":
         return cls(mapping, **kwargs)
 
     @classmethod
-    def from_uri(cls, data_uri: str, **kwargs: Any) -> "ExcelAdapter":
+    def from_uris(
+        cls, data_uri: Union[str, list[str]], **kwargs: Any
+    ) -> "ExcelAdapter":
         """
         Read the sheets in an Excel file.
 
@@ -83,6 +85,8 @@ def from_uri(cls, data_uri: str, **kwargs: Any) -> "ExcelAdapter":
         -------
 
         """
+        if not isinstance(data_uri, str):
+            data_uri = data_uri[0]
         file = pandas.ExcelFile(data_uri)
         return cls.from_file(file)
 
@@ -96,7 +100,7 @@ def from_assets(
         **kwargs: Optional[Union[str, List[str], Dict[str, str]]],
     ) -> "ExcelAdapter":
         data_uri = assets[0].data_uri
-        return cls.from_uri(
+        return cls.from_uris(
             data_uri,
             structure=structure,
             metadata=metadata,

tiled/adapters/hdf5.py

@@ -14,7 +14,6 @@
 from ..structures.array import ArrayStructure
 from ..structures.core import Spec, StructureFamily
 from ..structures.data_source import Asset
-from ..structures.table import TableStructure
 from ..type_aliases import JSON
 from ..utils import node_repr, path_from_uri
 from .array import ArrayAdapter
@@ -109,8 +108,15 @@ def from_assets(
         specs: Optional[List[Spec]] = None,
         **kwargs: Optional[Union[str, List[str], Dict[str, str]]],
     ) -> "HDF5Adapter":
+        if len(assets) == 1:
+            data_uri = assets[0].data_uri
+        else:
+            for ast in assets:
+                if ast.parameter == "data_uri":
+                    data_uri = ast.data_uri
+                    break
         return hdf5_lookup(
-            data_uri=assets[0].data_uri,
+            data_uri=data_uri,
             structure=structure,
             metadata=metadata,
             specs=specs,
@@ -146,6 +152,8 @@ def from_uris(
         -------
 
         """
+        if not isinstance(data_uri, str):
+            data_uri = data_uri[0]
         filepath = path_from_uri(data_uri)
         cache_key = (h5py.File, filepath, "r", swmr, libver)
         file = with_resource_cache(
@@ -383,7 +391,7 @@ def hdf5_lookup(
         raise ValueError("dataset and path kwargs should not both be set!")
 
     dataset = dataset or path or []
-    adapter = HDF5Adapter.from_uri(
+    adapter = HDF5Adapter.from_uris(
         data_uri,
         structure=structure,
         metadata=metadata,

tiled/adapters/parquet.py

@@ -48,9 +48,7 @@ def from_assets(
         specs: Optional[List[Spec]] = None,
         **kwargs: Optional[Union[str, List[str], Dict[str, str]]],
     ) -> "ParquetDatasetAdapter":
-        return cls(
-            [ast.data_uri for ast in assets], structure, metadata, specs
-        )
+        return cls([ast.data_uri for ast in assets], structure, metadata, specs)
 
     def metadata(self) -> JSON:
         """

tiled/adapters/table.py

@@ -50,9 +50,7 @@ def from_pandas(
         ddf = dask.dataframe.from_pandas(*args, npartitions=npartitions, **kwargs)
         if specs is None:
             specs = [Spec("dataframe")]
-        return cls.from_dask_dataframe(
-            ddf, metadata=metadata, specs=specs
-        )
+        return cls.from_dask_dataframe(ddf, metadata=metadata, specs=specs)
 
     @classmethod
     def from_dict(
@@ -80,9 +78,7 @@ def from_dict(
         ddf = dask.dataframe.from_dict(*args, npartitions=npartitions, **kwargs)
         if specs is None:
             specs = [Spec("dataframe")]
-        return cls.from_dask_dataframe(
-            ddf, metadata=metadata, specs=specs
-        )
+        return cls.from_dask_dataframe(ddf, metadata=metadata, specs=specs)
 
     @classmethod
     def from_dask_dataframe(

tiled/adapters/tiff.py

@@ -81,6 +81,13 @@ def from_assets(
             specs=specs,
         )
 
+    @classmethod
+    def from_uris(cls, data_uri: str, **kwargs: Optional[Any]) -> "TiffAdapter":
+        if not isinstance(data_uri, str):
+            data_uri = data_uri[0]
+
+        return cls(data_uri)
+
     def metadata(self) -> JSON:
         """
 

tiled/adapters/xarray.py

@@ -63,9 +63,7 @@ def __init__(
             raise TypeError(
                 "Use DatasetAdapter.from_dataset(...), not DatasetAdapter(...)."
             )
-        super().__init__(
-            mapping, *args, specs=specs, **kwargs
-        )
+        super().__init__(mapping, *args, specs=specs, **kwargs)
 
     def inlined_contents_enabled(self, depth: int) -> bool:
         """

tiled/adapters/zarr.py

@@ -467,3 +467,16 @@ def from_assets(
                 specs=specs,
                 **kwargs,
             )
+
+    @classmethod
+    def from_uris(
+        cls, data_uri: Union[str, list[str]], **kwargs: Optional[Any]
+    ) -> Union[ZarrArrayAdapter, ZarrGroupAdapter]:
+        if isinstance(data_uri, list):
+            data_uri = data_uri[0]
+        zarr_obj = zarr.open(path_from_uri(data_uri))  # Group or Array
+        if isinstance(zarr_obj, zarr.hierarchy.Group):
+            return ZarrGroupAdapter(zarr_obj, **kwargs)
+        else:
+            structure = ArrayStructure.from_array(zarr_obj)
+            return ZarrArrayAdapter(zarr_obj, structure=structure, **kwargs)

tiled/examples/xdi.py

@@ -8,14 +8,136 @@
 import io
 import pathlib
 import re
+from typing import Any, Dict, List, Optional, Union
 
+import dask.dataframe
 import pandas as pd
 
 from tiled.adapters.dataframe import DataFrameAdapter
-from tiled.structures.core import Spec
+from tiled.adapters.table import TableAdapter
+from tiled.structures.core import Spec, StructureFamily
+from tiled.structures.data_source import Asset
+from tiled.structures.table import TableStructure
+from tiled.type_aliases import JSON
 from tiled.utils import path_from_uri
 
 
+class XDIAdapter(TableAdapter):
+    structure_family = StructureFamily.table
+
+    def __init__(
+        self,
+        data_uri: str,
+        structure: Optional[TableStructure] = None,
+        metadata: Optional[JSON] = None,
+        specs: Optional[list[Spec]] = None,
+        **kwargs: Optional[Any],
+    ) -> None:
+        """Adapter for XDI data"""
+
+        filepath = path_from_uri(data_uri)
+        with open(filepath, "r") as file:
+            metadata = {}
+            fields = collections.defaultdict(dict)
+
+            # if isinstance(f, pathlib.PosixPath):
+            #    line = f.read_text().split('\n')[0]
+            # else:
+            line = file.readline()
+            m = re.match(r"#\s*XDI/(\S*)\s*(\S*)?", line)
+            if not m:
+                raise ValueError(
+                    f"not an XDI file, no XDI versioning information in first line\n{line}"
+                )
+
+            metadata["xdi_version"] = m[1]
+            metadata["extra_version"] = m[2]
+
+            field_end_re = re.compile(r"#\s*/{3,}")
+            header_end_re = re.compile(r"#\s*-{3,}")
+
+            has_comments = False
+
+            # read header
+            for line in file:
+                if line[0] != "#":
+                    raise ValueError(f"reached invalid line in header\n{line}")
+                if re.match(field_end_re, line):
+                    has_comments = True
+                    break
+                elif re.match(header_end_re, line):
+                    break
+
+                try:
+                    key, val = line[1:].strip().split(":", 1)
+                    val = val.strip()
+                    namespace, tag = key.split(".")
+                    # TODO coerce to lower case?
+                except ValueError:
+                    print(f"error processing line\n{line}")
+                    raise
+
+                fields[namespace][tag] = val
+
+            if has_comments:
+                comments = ""
+                for line in file:
+                    if re.match(header_end_re, line):
+                        break
+                    comments += line
+
+                metadata["comments"] = comments
+
+            metadata["fields"] = fields
+
+            line = file.readline()
+            if line[0] != "#":
+                raise ValueError(f"expected column labels. got\n{line}")
+            col_labels = line[1:].split()
+
+            # TODO validate
+
+            df = pd.read_table(file, sep=r"\s+", names=col_labels)
+
+        ddf = dask.dataframe.from_pandas(df, npartitions=1)
+        structure = TableStructure.from_dask_dataframe(ddf)
+
+        super().__init__(
+            partitions=ddf.partitions,
+            structure=structure,
+            metadata=metadata,
+            specs=(specs or []) + [Spec("xdi", version="1.0")],
+        )
+
+    @classmethod
+    def from_assets(
+        cls,
+        assets: List[Asset],
+        structure: TableStructure,
+        metadata: Optional[JSON] = None,
+        specs: Optional[List[Spec]] = None,
+        **kwargs: Optional[Union[str, List[str], Dict[str, str]]],
+    ) -> "XDIAdapter":
+        return cls(
+            assets[0].data_uri,
+            structure,
+            metadata,
+            specs,
+            **kwargs,
+        )
+
+    @classmethod
+    def from_uris(
+        cls,
+        data_uri: Union[str, List[str]],
+        **kwargs: Optional[Any],
+    ) -> "XDIAdapter":
+        if not isinstance(data_uri, str):
+            data_uri = data_uri[0]
+
+        return cls(data_uri, **kwargs)
+
+
 def read_xdi(data_uri, structure=None, metadata=None, specs=None, access_policy=None):
     "Read XDI-formatted file."
     filepath = path_from_uri(data_uri)

tiled/mimetypes.py

@@ -13,9 +13,9 @@
 AWKWARD_BUFFERS_MIMETYPE = "application/x-awkward-buffers"
 DEFAULT_ADAPTERS_BY_MIMETYPE = OneShotCachedMap(
     {
-        # "image/tiff": lambda: importlib.import_module(
-        #     "..adapters.tiff", __name__
-        # ).TiffAdapter,
+        "image/tiff": lambda: importlib.import_module(
+            "..adapters.tiff", __name__
+        ).TiffAdapter,
         "multipart/related;type=image/tiff": lambda: importlib.import_module(
             "..adapters.tiff", __name__
         ).TiffSequenceAdapter,