Added detection of "*" to enable opening of multi-file datasets.

intake_esm/source.py

@@ -47,18 +47,25 @@ def _open_dataset(
 ):
 
     _can_be_local = fsspec.utils.can_be_local(urlpath)
-    storage_options = xarray_open_kwargs['backend_kwargs'].get('storage_options', {})
+    storage_options = xarray_open_kwargs.get('backend_kwargs', {}).get('storage_options', {})
     if xarray_open_kwargs['engine'] == 'zarr':
         url = urlpath
     elif _can_be_local:
         url = fsspec.open_local(urlpath, **storage_options)
     else:
         url = fsspec.open(urlpath, **storage_options).open()
 
-    ds = xr.open_dataset(url, **xarray_open_kwargs)
+    # Handle multi-file datasets with `xr.open_mfdataset()`
+    if '*' in url or isinstance(url, list):
+        # How should we handle concat_dim, and other xr.open_mfdataset kwargs?
+        xarray_open_kwargs.update(preprocess=preprocess)
+        xarray_open_kwargs.update(parallel=True)
+        ds = xr.open_mfdataset(url, **xarray_open_kwargs)
+    else:
+        ds = xr.open_dataset(url, **xarray_open_kwargs)
+        if preprocess is not None:
+            ds = preprocess(ds)
 
-    if preprocess is not None:
-        ds = preprocess(ds)
     if varname and isinstance(varname, str):
         varname = [varname]
     if requested_variables:

tests/test_source.py

@@ -0,0 +1,42 @@
+import os
+
+import xarray
+
+from intake_esm.source import _get_xarray_open_kwargs, _open_dataset
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+f1 = os.path.join(
+    here,
+    'sample_data/cmip/cmip5/output1/NIMR-KMA/HadGEM2-AO/rcp85/mon/atmos/Amon/r1i1p1/v20130815/tasmax/tasmax_Amon_HadGEM2-AO_rcp85_r1i1p1_200511-200512.nc',
+)
+f2 = os.path.join(
+    here,
+    'sample_data/cmip/cmip5/output1/NIMR-KMA/HadGEM2-AO/rcp85/mon/atmos/Amon/r1i1p1/v20130815/tasmax/tasmax_Amon_HadGEM2-AO_rcp85_r1i1p1_200601-210012.nc',
+)
+
+multi_path = os.path.dirname(f1) + '/*.nc'
+
+
+def _common_open(fpath, varname='tasmax'):
+    _xarray_open_kwargs = _get_xarray_open_kwargs('netcdf')
+    return _open_dataset(fpath, varname, xarray_open_kwargs=_xarray_open_kwargs).compute()
+
+
+def test_open_dataset_single():
+    ds1 = _common_open(f1)
+    ds2 = _common_open(f2)
+
+    assert isinstance(ds1, xarray.Dataset)
+    assert ds1.time.values[0].isoformat() == '2005-11-16T00:00:00'
+    assert ds2.time.values[-1].isoformat() == '2006-02-16T00:00:00'
+
+
+def test_open_dataset_multi():
+    ds = _common_open(multi_path)
+
+    assert isinstance(ds, xarray.Dataset)
+    assert len(ds.time) == 4
+    assert ds.time.values[0].isoformat() == '2005-11-16T00:00:00'
+    assert ds.time.values[-1].isoformat() == '2006-02-16T00:00:00'