From 2948a6f5aac96a0496cc1cac93f8c0fe75e7001f Mon Sep 17 00:00:00 2001 From: Stephan Finkensieper Date: Wed, 13 Dec 2023 09:41:24 +0000 Subject: [PATCH 01/12] Add test triggering the error --- satpy/tests/reader_tests/test_satpy_cf_nc.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/satpy/tests/reader_tests/test_satpy_cf_nc.py b/satpy/tests/reader_tests/test_satpy_cf_nc.py index 0c22f5b3f1..6528a80723 100644 --- a/satpy/tests/reader_tests/test_satpy_cf_nc.py +++ b/satpy/tests/reader_tests/test_satpy_cf_nc.py @@ -147,6 +147,10 @@ def cf_scene(): "nadir_longitude": 1, "nadir_latitude": 1, "only_in_1": False + }, + "time_parameters": { + "nominal_start_time": tstart, + "nominal_end_time": tend } }) @@ -388,18 +392,17 @@ def test_read_prefixed_channels_by_user_no_prefix(self, cf_scene, nc_filename): np.testing.assert_array_equal(scn_["1"].data, cf_scene["1"].data) np.testing.assert_array_equal(scn_["1"].coords["lon"], cf_scene["lon"].data) # lon loded as coord - def test_orbital_parameters(self, cf_scene, nc_filename): - """Test that the orbital parameters in attributes are handled correctly.""" + def test_decoding_of_dict_type_attributes(self, cf_scene, nc_filename): + """Test decoding of dict type attributes.""" cf_scene.save_datasets(writer="cf", filename=nc_filename) scn_ = Scene(reader="satpy_cf_nc", filenames=[nc_filename]) scn_.load(["image0"]) - orig_attrs = cf_scene["image0"].attrs["orbital_parameters"] - new_attrs = scn_["image0"].attrs["orbital_parameters"] - assert isinstance(new_attrs, dict) - for key in orig_attrs: - assert orig_attrs[key] == new_attrs[key] + for attr_name in ["orbital_parameters", "time_parameters"]: + orig_attrs = cf_scene["image0"].attrs[attr_name] + new_attrs = scn_["image0"].attrs[attr_name] + assert new_attrs == orig_attrs def test_write_and_read_from_two_files(self, nc_filename, nc_filename_i): """Save two datasets with different resolution and read the solar_zenith_angle again.""" From 6ee324821f85d93f11ade113a987b743c0eef011 Mon Sep 17 00:00:00 2001 From: Stephan Finkensieper Date: Wed, 13 Dec 2023 10:04:10 +0000 Subject: [PATCH 02/12] Decode time parameters to datetime --- satpy/readers/satpy_cf_nc.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/satpy/readers/satpy_cf_nc.py b/satpy/readers/satpy_cf_nc.py index 7a26ead72b..bf8908e604 100644 --- a/satpy/readers/satpy_cf_nc.py +++ b/satpy/readers/satpy_cf_nc.py @@ -313,7 +313,12 @@ def get_dataset(self, ds_id, ds_info): data.attrs.update(nc.attrs) # For now add global attributes to all datasets if "orbital_parameters" in data.attrs: data.attrs["orbital_parameters"] = _str2dict(data.attrs["orbital_parameters"]) - + if "time_parameters" in data.attrs: + time_params = _str2dict(data.attrs["time_parameters"]) + from dateutil.parser import isoparse + for key, val in time_params.items(): + time_params[key] = isoparse(val) + data.attrs["time_parameters"] = time_params return data def get_area_def(self, dataset_id): From cd4743ccddf1b943b540de9d792af9437526df1f Mon Sep 17 00:00:00 2001 From: Stephan Finkensieper Date: Wed, 13 Dec 2023 10:21:09 +0000 Subject: [PATCH 03/12] Refactor dict type decoding --- satpy/readers/satpy_cf_nc.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/satpy/readers/satpy_cf_nc.py b/satpy/readers/satpy_cf_nc.py index bf8908e604..2e85a166f8 100644 --- a/satpy/readers/satpy_cf_nc.py +++ b/satpy/readers/satpy_cf_nc.py @@ -311,16 +311,13 @@ def get_dataset(self, ds_id, ds_info): if name != ds_id["name"]: data = data.rename(ds_id["name"]) data.attrs.update(nc.attrs) # For now add global attributes to all datasets - if "orbital_parameters" in data.attrs: - data.attrs["orbital_parameters"] = _str2dict(data.attrs["orbital_parameters"]) - if "time_parameters" in data.attrs: - time_params = _str2dict(data.attrs["time_parameters"]) - from dateutil.parser import isoparse - for key, val in time_params.items(): - time_params[key] = isoparse(val) - data.attrs["time_parameters"] = time_params + self._decode_dict_type_attrs(data) return data + def _decode_dict_type_attrs(self, data): + for key in ["orbital_parameters", "time_parameters"]: + data.attrs[key] = _str2dict(data.attrs[key]) + def get_area_def(self, dataset_id): """Get area definition from CF complient netcdf.""" try: @@ -334,8 +331,18 @@ def get_area_def(self, dataset_id): raise NotImplementedError +def _datetime_parser(json_dict): + import dateutil.parser + for key, value in json_dict.items(): + try: + json_dict[key] = dateutil.parser.parse(value) + except (TypeError, ValueError): + pass + return json_dict + + def _str2dict(val): """Convert string to dictionary.""" if isinstance(val, str): - val = json.loads(val) + val = json.loads(val, object_hook=_datetime_parser) return val From 8b29b8fb45848b8c96228385a8181dfee42ff4e6 Mon Sep 17 00:00:00 2001 From: Stephan Finkensieper Date: Wed, 13 Dec 2023 10:57:04 +0000 Subject: [PATCH 04/12] Only decode attributes if available --- satpy/readers/satpy_cf_nc.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/satpy/readers/satpy_cf_nc.py b/satpy/readers/satpy_cf_nc.py index 2e85a166f8..b9c932b852 100644 --- a/satpy/readers/satpy_cf_nc.py +++ b/satpy/readers/satpy_cf_nc.py @@ -316,7 +316,10 @@ def get_dataset(self, ds_id, ds_info): def _decode_dict_type_attrs(self, data): for key in ["orbital_parameters", "time_parameters"]: - data.attrs[key] = _str2dict(data.attrs[key]) + try: + data.attrs[key] = _str2dict(data.attrs[key]) + except KeyError: + continue def get_area_def(self, dataset_id): """Get area definition from CF complient netcdf.""" From d9ec74428ca2cdbc31ee87ae65be7f6110070b9c Mon Sep 17 00:00:00 2001 From: Stephan Finkensieper Date: Mon, 18 Dec 2023 10:09:10 +0000 Subject: [PATCH 05/12] Convert any attribute starting with "{" to string --- satpy/readers/satpy_cf_nc.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/satpy/readers/satpy_cf_nc.py b/satpy/readers/satpy_cf_nc.py index b9c932b852..24c5ef438f 100644 --- a/satpy/readers/satpy_cf_nc.py +++ b/satpy/readers/satpy_cf_nc.py @@ -315,11 +315,8 @@ def get_dataset(self, ds_id, ds_info): return data def _decode_dict_type_attrs(self, data): - for key in ["orbital_parameters", "time_parameters"]: - try: - data.attrs[key] = _str2dict(data.attrs[key]) - except KeyError: - continue + for key, val in data.attrs.items(): + data.attrs[key] = _str2dict(val) def get_area_def(self, dataset_id): """Get area definition from CF complient netcdf.""" @@ -346,6 +343,6 @@ def _datetime_parser(json_dict): def _str2dict(val): """Convert string to dictionary.""" - if isinstance(val, str): + if isinstance(val, str) and val.startswith("{"): val = json.loads(val, object_hook=_datetime_parser) return val From f01cbe117dd948d9da07458fb92f649d06c90443 Mon Sep 17 00:00:00 2001 From: Stephan Finkensieper Date: Mon, 18 Dec 2023 10:34:47 +0000 Subject: [PATCH 06/12] Refactor test scene setup (vis006) --- satpy/tests/reader_tests/test_satpy_cf_nc.py | 122 +++++++++++-------- 1 file changed, 69 insertions(+), 53 deletions(-) diff --git a/satpy/tests/reader_tests/test_satpy_cf_nc.py b/satpy/tests/reader_tests/test_satpy_cf_nc.py index 6528a80723..20ab15ad0b 100644 --- a/satpy/tests/reader_tests/test_satpy_cf_nc.py +++ b/satpy/tests/reader_tests/test_satpy_cf_nc.py @@ -88,10 +88,76 @@ def _create_test_netcdf(filename, resolution=742): @pytest.fixture(scope="session") -def cf_scene(): +def area(): + """Get fake area definition.""" + area_extent = (339045.5577, 4365586.6063, 1068143.527, 4803645.4685) + proj_dict = {"a": 6378169.0, "b": 6356583.8, "h": 35785831.0, + "lon_0": 0.0, "proj": "geos", "units": "m"} + area = AreaDefinition("test", + "test", + "test", + proj_dict, + 2, + 2, + area_extent) + return area + + +@pytest.fixture(scope="session") +def common_attrs(area): + """Get common dataset attributes.""" + return { + "start_time": datetime(2019, 4, 1, 12, 0), + "end_time": datetime(2019, 4, 1, 12, 15), + "platform_name": "tirosn", + "orbit_number": 99999, + "area": area + } + + +@pytest.fixture(scope="session") +def vis006(area, common_attrs): + """Get fake VIS006 dataset.""" + x, y = area.get_proj_coords() + y_visir = y[:, 0] + x_visir = x[0, :] + attrs = { + "name": "image0", + "id_tag": "ch_r06", + "coordinates": "lat lon", + "resolution": 1000, + "calibration": "reflectance", + "wavelength": WavelengthRange(min=0.58, central=0.63, max=0.68, unit="µm"), + "orbital_parameters": { + "projection_longitude": 1, + "projection_latitude": 1, + "projection_altitude": 1, + "satellite_nominal_longitude": 1, + "satellite_nominal_latitude": 1, + "satellite_actual_longitude": 1, + "satellite_actual_latitude": 1, + "satellite_actual_altitude": 1, + "nadir_longitude": 1, + "nadir_latitude": 1, + "only_in_1": False + }, + "time_parameters": { + "nominal_start_time": common_attrs["start_time"], + "nominal_end_time": common_attrs["end_time"] + } + } + attrs.update(common_attrs) + coords = {"y": y_visir, "x": x_visir, "acq_time": ("y", [1, 2])} + vis006 = xr.DataArray(np.array([[1, 2], [3, 4]]), + dims=("y", "x"), + coords=coords, + attrs=attrs) + return vis006 + + +@pytest.fixture(scope="session") +def cf_scene(vis006, common_attrs, area): """Create a cf scene.""" - tstart = datetime(2019, 4, 1, 12, 0) - tend = datetime(2019, 4, 1, 12, 15) data_visir = np.array([[1, 2], [3, 4]]) z_visir = [1, 2, 3, 4, 5, 6, 7] qual_data = [[1, 2, 3, 4, 5, 6, 7], @@ -100,60 +166,10 @@ def cf_scene(): lat = 33.0 * np.array([[1, 2], [3, 4]]) lon = -13.0 * np.array([[1, 2], [3, 4]]) - proj_dict = { - "a": 6378169.0, "b": 6356583.8, "h": 35785831.0, - "lon_0": 0.0, "proj": "geos", "units": "m" - } - x_size, y_size = data_visir.shape - area_extent = (339045.5577, 4365586.6063, 1068143.527, 4803645.4685) - area = AreaDefinition( - "test", - "test", - "test", - proj_dict, - x_size, - y_size, - area_extent, - ) - x, y = area.get_proj_coords() y_visir = y[:, 0] x_visir = x[0, :] - common_attrs = { - "start_time": tstart, - "end_time": tend, - "platform_name": "tirosn", - "orbit_number": 99999, - "area": area - } - - vis006 = xr.DataArray(data_visir, - dims=("y", "x"), - coords={"y": y_visir, "x": x_visir, "acq_time": ("y", time_vis006)}, - attrs={ - "name": "image0", "id_tag": "ch_r06", - "coordinates": "lat lon", "resolution": 1000, "calibration": "reflectance", - "wavelength": WavelengthRange(min=0.58, central=0.63, max=0.68, unit="µm"), - "orbital_parameters": { - "projection_longitude": 1, - "projection_latitude": 1, - "projection_altitude": 1, - "satellite_nominal_longitude": 1, - "satellite_nominal_latitude": 1, - "satellite_actual_longitude": 1, - "satellite_actual_latitude": 1, - "satellite_actual_altitude": 1, - "nadir_longitude": 1, - "nadir_latitude": 1, - "only_in_1": False - }, - "time_parameters": { - "nominal_start_time": tstart, - "nominal_end_time": tend - } - }) - ir_108 = xr.DataArray(data_visir, dims=("y", "x"), coords={"y": y_visir, "x": x_visir, "acq_time": ("y", time_vis006)}, From 6e0fdf57b21f94c74639a936bffc4065e73f3802 Mon Sep 17 00:00:00 2001 From: Stephan Finkensieper Date: Mon, 18 Dec 2023 10:44:57 +0000 Subject: [PATCH 07/12] Refactor test scene setup (ir_108) --- satpy/tests/reader_tests/test_satpy_cf_nc.py | 67 +++++++++++++------- 1 file changed, 44 insertions(+), 23 deletions(-) diff --git a/satpy/tests/reader_tests/test_satpy_cf_nc.py b/satpy/tests/reader_tests/test_satpy_cf_nc.py index 20ab15ad0b..674f895b22 100644 --- a/satpy/tests/reader_tests/test_satpy_cf_nc.py +++ b/satpy/tests/reader_tests/test_satpy_cf_nc.py @@ -89,7 +89,7 @@ def _create_test_netcdf(filename, resolution=742): @pytest.fixture(scope="session") def area(): - """Get fake area definition.""" + """Get area definition.""" area_extent = (339045.5577, 4365586.6063, 1068143.527, 4803645.4685) proj_dict = {"a": 6378169.0, "b": 6356583.8, "h": 35785831.0, "lon_0": 0.0, "proj": "geos", "units": "m"} @@ -116,11 +116,18 @@ def common_attrs(area): @pytest.fixture(scope="session") -def vis006(area, common_attrs): - """Get fake VIS006 dataset.""" +def xy_coords(area): + """Get projection coordinates.""" x, y = area.get_proj_coords() - y_visir = y[:, 0] - x_visir = x[0, :] + y = y[:, 0] + x = x[0, :] + return x, y + + +@pytest.fixture(scope="session") +def vis006(xy_coords, common_attrs): + """Get VIS006 dataset.""" + x, y = xy_coords attrs = { "name": "image0", "id_tag": "ch_r06", @@ -147,7 +154,7 @@ def vis006(area, common_attrs): } } attrs.update(common_attrs) - coords = {"y": y_visir, "x": x_visir, "acq_time": ("y", [1, 2])} + coords = {"y": y, "x": x, "acq_time": ("y", [1, 2])} vis006 = xr.DataArray(np.array([[1, 2], [3, 4]]), dims=("y", "x"), coords=coords, @@ -156,13 +163,38 @@ def vis006(area, common_attrs): @pytest.fixture(scope="session") -def cf_scene(vis006, common_attrs, area): - """Create a cf scene.""" - data_visir = np.array([[1, 2], [3, 4]]) - z_visir = [1, 2, 3, 4, 5, 6, 7] +def ir_108(xy_coords): + """Get IR_108 dataset.""" + x, y = xy_coords + coords = {"y": y, "x": x, "acq_time": ("y", [1, 2])} + attrs = {"name": "image1", "id_tag": "ch_tb11", "coordinates": "lat lon"} + ir_108 = xr.DataArray(np.array([[1, 2], [3, 4]]), + dims=("y", "x"), + coords=coords, + attrs=attrs) + return ir_108 + + +@pytest.fixture(scope="session") +def qual_flags(xy_coords): + """Get quality flags.""" qual_data = [[1, 2, 3, 4, 5, 6, 7], [1, 2, 3, 4, 5, 6, 7]] - time_vis006 = [1, 2] + x, y = xy_coords + z = [1, 2, 3, 4, 5, 6, 7] + coords = {"y": y, "z": z, "acq_time": ("y", [1, 2])} + qual_f = xr.DataArray(qual_data, + dims=("y", "z"), + coords=coords, + attrs={"name": "qual_flags", + "id_tag": "qual_flags"}) + return qual_f + + +@pytest.fixture(scope="session") +def cf_scene(vis006, ir_108, qual_flags, common_attrs, area): + """Create a cf scene.""" + data_visir = np.array([[1, 2], [3, 4]]) lat = 33.0 * np.array([[1, 2], [3, 4]]) lon = -13.0 * np.array([[1, 2], [3, 4]]) @@ -170,17 +202,6 @@ def cf_scene(vis006, common_attrs, area): y_visir = y[:, 0] x_visir = x[0, :] - ir_108 = xr.DataArray(data_visir, - dims=("y", "x"), - coords={"y": y_visir, "x": x_visir, "acq_time": ("y", time_vis006)}, - attrs={"name": "image1", "id_tag": "ch_tb11", "coordinates": "lat lon"}) - qual_f = xr.DataArray(qual_data, - dims=("y", "z"), - coords={"y": y_visir, "z": z_visir, "acq_time": ("y", time_vis006)}, - attrs={ - "name": "qual_flags", - "id_tag": "qual_flags" - }) lat = xr.DataArray(lat, dims=("y", "x"), coords={"y": y_visir, "x": x_visir}, @@ -223,7 +244,7 @@ def cf_scene(vis006, common_attrs, area): "1": prefix_data, "lat": lat, "lon": lon, - "qual_flags": qual_f + "qual_flags": qual_flags } for key in scene_dict: From 168ced2040cb088d95929598ee8e1d80b2db215d Mon Sep 17 00:00:00 2001 From: Stephan Finkensieper Date: Mon, 18 Dec 2023 10:59:58 +0000 Subject: [PATCH 08/12] Refactor test scene setup (rest) --- satpy/tests/reader_tests/test_satpy_cf_nc.py | 99 ++++++++++---------- 1 file changed, 52 insertions(+), 47 deletions(-) diff --git a/satpy/tests/reader_tests/test_satpy_cf_nc.py b/satpy/tests/reader_tests/test_satpy_cf_nc.py index 674f895b22..0710aae57c 100644 --- a/satpy/tests/reader_tests/test_satpy_cf_nc.py +++ b/satpy/tests/reader_tests/test_satpy_cf_nc.py @@ -153,7 +153,6 @@ def vis006(xy_coords, common_attrs): "nominal_end_time": common_attrs["end_time"] } } - attrs.update(common_attrs) coords = {"y": y, "x": x, "acq_time": ("y", [1, 2])} vis006 = xr.DataArray(np.array([[1, 2], [3, 4]]), dims=("y", "x"), @@ -192,63 +191,69 @@ def qual_flags(xy_coords): @pytest.fixture(scope="session") -def cf_scene(vis006, ir_108, qual_flags, common_attrs, area): - """Create a cf scene.""" - data_visir = np.array([[1, 2], [3, 4]]) +def lonlats(xy_coords): + """Get longitudes and latitudes.""" + x, y = xy_coords lat = 33.0 * np.array([[1, 2], [3, 4]]) lon = -13.0 * np.array([[1, 2], [3, 4]]) + attrs = {"name": "lat", + "standard_name": "latitude", + "modifiers": np.array([])} + dims = ("y", "x") + coords = {"y": y, "x": x} + lat = xr.DataArray(lat, dims=dims, coords=coords, attrs=attrs) + lon = xr.DataArray(lon, dims=dims, coords=coords, attrs=attrs) + return lon, lat - x, y = area.get_proj_coords() - y_visir = y[:, 0] - x_visir = x[0, :] - lat = xr.DataArray(lat, - dims=("y", "x"), - coords={"y": y_visir, "x": x_visir}, - attrs={ - "name": "lat", - "standard_name": "latitude", - "modifiers": np.array([]) - }) - lon = xr.DataArray(lon, - dims=("y", "x"), - coords={"y": y_visir, "x": x_visir}, - attrs={ - "name": "lon", - "standard_name": "longitude", - "modifiers": np.array([]) - }) - - # for prefix testing - prefix_data = xr.DataArray(data_visir, +@pytest.fixture(scope="session") +def prefix_data(xy_coords, area): + """Get dataset whose name should be prefixed.""" + x, y = xy_coords + attrs = {"name": "1", + "id_tag": "ch_r06", + "coordinates": "lat lon", + "resolution": 1000, + "calibration": "reflectance", + "wavelength": WavelengthRange(min=0.58, central=0.63, max=0.68, unit="µm"), + "area": area} + prefix_data = xr.DataArray(np.array([[1, 2], [3, 4]]), dims=("y", "x"), - coords={"y": y_visir, "x": x_visir}, - attrs={ - "name": "1", "id_tag": "ch_r06", - "coordinates": "lat lon", "resolution": 1000, "calibration": "reflectance", - "wavelength": WavelengthRange(min=0.58, central=0.63, max=0.68, unit="µm"), - "area": area - }) - - # for swath testing + coords={"y": y, "x": x}, + attrs=attrs) + return prefix_data + + +@pytest.fixture(scope="session") +def swath_data(prefix_data, lonlats): + """Get swath data.""" + lon, lat = lonlats area = SwathDefinition(lons=lon, lats=lat) swath_data = prefix_data.copy() swath_data.attrs.update({"name": "swath_data", "area": area}) + return swath_data + +@pytest.fixture(scope="session") +def datasets(vis006, ir_108, qual_flags, lonlats, prefix_data, swath_data): + """Get datasets belonging to the scene.""" + lon, lat = lonlats + return {"image0": vis006, + "image1": ir_108, + "swath_data": swath_data, + "1": prefix_data, + "lat": lat, + "lon": lon, + "qual_flags": qual_flags} + + +@pytest.fixture(scope="session") +def cf_scene(datasets, common_attrs): + """Create a cf scene.""" scene = Scene() scene.attrs["sensor"] = ["avhrr-1", "avhrr-2", "avhrr-3"] - scene_dict = { - "image0": vis006, - "image1": ir_108, - "swath_data": swath_data, - "1": prefix_data, - "lat": lat, - "lon": lon, - "qual_flags": qual_flags - } - - for key in scene_dict: - scene[key] = scene_dict[key] + for key in datasets: + scene[key] = datasets[key] if key != "swath_data": scene[key].attrs.update(common_attrs) return scene From a9b876a2cb5f1bc21cb6b7eb10f7c4756e1de1ca Mon Sep 17 00:00:00 2001 From: Stephan Finkensieper Date: Tue, 19 Dec 2023 11:41:30 +0000 Subject: [PATCH 09/12] Factorize attribute decoding --- satpy/readers/satpy_cf_nc.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/satpy/readers/satpy_cf_nc.py b/satpy/readers/satpy_cf_nc.py index 24c5ef438f..4a814dd586 100644 --- a/satpy/readers/satpy_cf_nc.py +++ b/satpy/readers/satpy_cf_nc.py @@ -311,13 +311,10 @@ def get_dataset(self, ds_id, ds_info): if name != ds_id["name"]: data = data.rename(ds_id["name"]) data.attrs.update(nc.attrs) # For now add global attributes to all datasets - self._decode_dict_type_attrs(data) + decoder = DatasetAttributeDecoder() + decoder.decode_attrs(data) return data - def _decode_dict_type_attrs(self, data): - for key, val in data.attrs.items(): - data.attrs[key] = _str2dict(val) - def get_area_def(self, dataset_id): """Get area definition from CF complient netcdf.""" try: @@ -331,6 +328,24 @@ def get_area_def(self, dataset_id): raise NotImplementedError +class DatasetAttributeDecoder: + """Decode attributes from cf-compatible to Python object.""" + + def decode_attrs(self, dataset): + """Decode dataset attributes.""" + self._decode_dict_type_attrs(dataset) + + def _decode_dict_type_attrs(self, data): + for key, val in data.attrs.items(): + data.attrs[key] = self._str2dict(val) + + def _str2dict(self, val): + """Convert string to dictionary.""" + if isinstance(val, str) and val.startswith("{"): + val = json.loads(val, object_hook=_datetime_parser) + return val + + def _datetime_parser(json_dict): import dateutil.parser for key, value in json_dict.items(): @@ -339,10 +354,3 @@ def _datetime_parser(json_dict): except (TypeError, ValueError): pass return json_dict - - -def _str2dict(val): - """Convert string to dictionary.""" - if isinstance(val, str) and val.startswith("{"): - val = json.loads(val, object_hook=_datetime_parser) - return val From edf933a0a1811b443d1dc35cb4ae4ab2317b29fe Mon Sep 17 00:00:00 2001 From: Stephan Finkensieper Date: Tue, 19 Dec 2023 12:01:07 +0000 Subject: [PATCH 10/12] Use datetime to parse timestamps --- satpy/readers/satpy_cf_nc.py | 4 ++-- satpy/tests/reader_tests/test_satpy_cf_nc.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/satpy/readers/satpy_cf_nc.py b/satpy/readers/satpy_cf_nc.py index 4a814dd586..c8188da77f 100644 --- a/satpy/readers/satpy_cf_nc.py +++ b/satpy/readers/satpy_cf_nc.py @@ -179,6 +179,7 @@ import itertools import json import logging +from datetime import datetime import xarray as xr from pyresample import AreaDefinition @@ -347,10 +348,9 @@ def _str2dict(self, val): def _datetime_parser(json_dict): - import dateutil.parser for key, value in json_dict.items(): try: - json_dict[key] = dateutil.parser.parse(value) + json_dict[key] = datetime.fromisoformat(value) except (TypeError, ValueError): pass return json_dict diff --git a/satpy/tests/reader_tests/test_satpy_cf_nc.py b/satpy/tests/reader_tests/test_satpy_cf_nc.py index 0710aae57c..e3540a4df7 100644 --- a/satpy/tests/reader_tests/test_satpy_cf_nc.py +++ b/satpy/tests/reader_tests/test_satpy_cf_nc.py @@ -107,7 +107,7 @@ def area(): def common_attrs(area): """Get common dataset attributes.""" return { - "start_time": datetime(2019, 4, 1, 12, 0), + "start_time": datetime(2019, 4, 1, 12, 0, 0, 123456), "end_time": datetime(2019, 4, 1, 12, 15), "platform_name": "tirosn", "orbit_number": 99999, From 414f2d6ff182aa2b9cd8a9006cb3dacd96ab9059 Mon Sep 17 00:00:00 2001 From: Stephan Finkensieper Date: Tue, 19 Dec 2023 12:37:40 +0000 Subject: [PATCH 11/12] Decode all timestamps to datetime --- satpy/readers/satpy_cf_nc.py | 33 ++++++++++++++------ satpy/tests/reader_tests/test_satpy_cf_nc.py | 11 ++++++- 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/satpy/readers/satpy_cf_nc.py b/satpy/readers/satpy_cf_nc.py index c8188da77f..73c26fccdd 100644 --- a/satpy/readers/satpy_cf_nc.py +++ b/satpy/readers/satpy_cf_nc.py @@ -334,23 +334,36 @@ class DatasetAttributeDecoder: def decode_attrs(self, dataset): """Decode dataset attributes.""" - self._decode_dict_type_attrs(dataset) + self._decode_dict_type_attrs(dataset.attrs) + self._decode_timestamps(dataset.attrs) - def _decode_dict_type_attrs(self, data): - for key, val in data.attrs.items(): - data.attrs[key] = self._str2dict(val) + def _decode_dict_type_attrs(self, attrs): + for key, val in attrs.items(): + attrs[key] = self._str2dict(val) def _str2dict(self, val): """Convert string to dictionary.""" if isinstance(val, str) and val.startswith("{"): - val = json.loads(val, object_hook=_datetime_parser) + val = json.loads(val, object_hook=_datetime_parser_json) return val + def _decode_timestamps(self, attrs): + for key, value in attrs.items(): + timestamp = _str2datetime(value) + if timestamp: + attrs[key] = timestamp -def _datetime_parser(json_dict): + +def _datetime_parser_json(json_dict): for key, value in json_dict.items(): - try: - json_dict[key] = datetime.fromisoformat(value) - except (TypeError, ValueError): - pass + timestamp = _str2datetime(value) + if timestamp: + json_dict[key] = timestamp return json_dict + + +def _str2datetime(string): + try: + return datetime.fromisoformat(string) + except (TypeError, ValueError): + return None diff --git a/satpy/tests/reader_tests/test_satpy_cf_nc.py b/satpy/tests/reader_tests/test_satpy_cf_nc.py index e3540a4df7..d2c50ee908 100644 --- a/satpy/tests/reader_tests/test_satpy_cf_nc.py +++ b/satpy/tests/reader_tests/test_satpy_cf_nc.py @@ -111,7 +111,8 @@ def common_attrs(area): "end_time": datetime(2019, 4, 1, 12, 15), "platform_name": "tirosn", "orbit_number": 99999, - "area": area + "area": area, + "my_timestamp": datetime(2000, 1, 1) } @@ -446,6 +447,14 @@ def test_decoding_of_dict_type_attributes(self, cf_scene, nc_filename): new_attrs = scn_["image0"].attrs[attr_name] assert new_attrs == orig_attrs + def test_decoding_of_timestamps(self, cf_scene, nc_filename): + """Test decoding of timestamps.""" + cf_scene.save_datasets(writer="cf", filename=nc_filename) + scn = Scene(reader="satpy_cf_nc", filenames=[nc_filename]) + scn.load(["image0"]) + expected = cf_scene["image0"].attrs["my_timestamp"] + assert scn["image0"].attrs["my_timestamp"] == expected + def test_write_and_read_from_two_files(self, nc_filename, nc_filename_i): """Save two datasets with different resolution and read the solar_zenith_angle again.""" _create_test_netcdf(nc_filename, resolution=742) From 37853ad8595feea8aded3cce9154ae38fd41b31e Mon Sep 17 00:00:00 2001 From: Stephan Finkensieper Date: Wed, 20 Dec 2023 16:59:38 +0000 Subject: [PATCH 12/12] Move encoding to separate module --- satpy/cf/decoding.py | 74 +++++++++++++++++++++++++++ satpy/readers/satpy_cf_nc.py | 46 +---------------- satpy/tests/cf_tests/test_decoding.py | 64 +++++++++++++++++++++++ 3 files changed, 140 insertions(+), 44 deletions(-) create mode 100644 satpy/cf/decoding.py create mode 100644 satpy/tests/cf_tests/test_decoding.py diff --git a/satpy/cf/decoding.py b/satpy/cf/decoding.py new file mode 100644 index 0000000000..0d7a9d22be --- /dev/null +++ b/satpy/cf/decoding.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2023 Satpy developers +# +# This file is part of satpy. +# +# satpy is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# satpy is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# satpy. If not, see . +"""CF decoding.""" +import copy +import json +from datetime import datetime + + +def decode_attrs(attrs): + """Decode CF-encoded attributes to Python object. + + Converts timestamps to datetime and strings starting with "{" to + dictionary. + + Args: + attrs (dict): Attributes to be decoded + + Returns (dict): Decoded attributes + """ + attrs = copy.deepcopy(attrs) + _decode_dict_type_attrs(attrs) + _decode_timestamps(attrs) + return attrs + + +def _decode_dict_type_attrs(attrs): + for key, val in attrs.items(): + attrs[key] = _str2dict(val) + + +def _str2dict(val): + """Convert string to dictionary.""" + if isinstance(val, str) and val.startswith("{"): + val = json.loads(val, object_hook=_datetime_parser_json) + return val + + +def _decode_timestamps(attrs): + for key, value in attrs.items(): + timestamp = _str2datetime(value) + if timestamp: + attrs[key] = timestamp + + +def _datetime_parser_json(json_dict): + """Traverse JSON dictionary and parse timestamps.""" + for key, value in json_dict.items(): + timestamp = _str2datetime(value) + if timestamp: + json_dict[key] = timestamp + return json_dict + + +def _str2datetime(string): + """Convert string to datetime object.""" + try: + return datetime.fromisoformat(string) + except (TypeError, ValueError): + return None diff --git a/satpy/readers/satpy_cf_nc.py b/satpy/readers/satpy_cf_nc.py index 73c26fccdd..5fab6e6235 100644 --- a/satpy/readers/satpy_cf_nc.py +++ b/satpy/readers/satpy_cf_nc.py @@ -177,13 +177,12 @@ """ import itertools -import json import logging -from datetime import datetime import xarray as xr from pyresample import AreaDefinition +import satpy.cf.decoding from satpy.dataset.dataid import WavelengthRange from satpy.readers.file_handlers import BaseFileHandler from satpy.utils import get_legacy_chunk_size @@ -312,8 +311,7 @@ def get_dataset(self, ds_id, ds_info): if name != ds_id["name"]: data = data.rename(ds_id["name"]) data.attrs.update(nc.attrs) # For now add global attributes to all datasets - decoder = DatasetAttributeDecoder() - decoder.decode_attrs(data) + data.attrs = satpy.cf.decoding.decode_attrs(data.attrs) return data def get_area_def(self, dataset_id): @@ -327,43 +325,3 @@ def get_area_def(self, dataset_id): # with the yaml_reader NotImplementedError is raised. logger.debug("No AreaDefinition to load from nc file. Falling back to SwathDefinition.") raise NotImplementedError - - -class DatasetAttributeDecoder: - """Decode attributes from cf-compatible to Python object.""" - - def decode_attrs(self, dataset): - """Decode dataset attributes.""" - self._decode_dict_type_attrs(dataset.attrs) - self._decode_timestamps(dataset.attrs) - - def _decode_dict_type_attrs(self, attrs): - for key, val in attrs.items(): - attrs[key] = self._str2dict(val) - - def _str2dict(self, val): - """Convert string to dictionary.""" - if isinstance(val, str) and val.startswith("{"): - val = json.loads(val, object_hook=_datetime_parser_json) - return val - - def _decode_timestamps(self, attrs): - for key, value in attrs.items(): - timestamp = _str2datetime(value) - if timestamp: - attrs[key] = timestamp - - -def _datetime_parser_json(json_dict): - for key, value in json_dict.items(): - timestamp = _str2datetime(value) - if timestamp: - json_dict[key] = timestamp - return json_dict - - -def _str2datetime(string): - try: - return datetime.fromisoformat(string) - except (TypeError, ValueError): - return None diff --git a/satpy/tests/cf_tests/test_decoding.py b/satpy/tests/cf_tests/test_decoding.py new file mode 100644 index 0000000000..c20cddf6da --- /dev/null +++ b/satpy/tests/cf_tests/test_decoding.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2023 Satpy developers +# +# This file is part of satpy. +# +# satpy is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# satpy is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# satpy. If not, see . +"""Tests for CF decoding.""" +from datetime import datetime + +import pytest + +import satpy.cf.decoding + + +class TestDecodeAttrs: + """Test decoding of CF-encoded attributes.""" + + @pytest.fixture() + def attrs(self): + """Get CF-encoded attributes.""" + return { + "my_integer": 0, + "my_float": 0.0, + "my_list": [1, 2, 3], + "my_timestamp1": "2000-01-01", + "my_timestamp2": "2000-01-01 12:15:33", + "my_timestamp3": "2000-01-01 12:15:33.123456", + "my_dict": '{"a": {"b": [1, 2, 3]}, "c": {"d": "2000-01-01 12:15:33.123456"}}' + } + + @pytest.fixture() + def expected(self): + """Get expected decoded results.""" + return { + "my_integer": 0, + "my_float": 0.0, + "my_list": [1, 2, 3], + "my_timestamp1": datetime(2000, 1, 1), + "my_timestamp2": datetime(2000, 1, 1, 12, 15, 33), + "my_timestamp3": datetime(2000, 1, 1, 12, 15, 33, 123456), + "my_dict": {"a": {"b": [1, 2, 3]}, + "c": {"d": datetime(2000, 1, 1, 12, 15, 33, 123456)}} + } + + def test_decoding(self, attrs, expected): + """Test decoding of CF-encoded attributes.""" + res = satpy.cf.decoding.decode_attrs(attrs) + assert res == expected + + def test_decoding_doesnt_modify_original(self, attrs): + """Test that decoding doesn't modify the original attributes.""" + satpy.cf.decoding.decode_attrs(attrs) + assert isinstance(attrs["my_dict"], str)