Skip to content

Commit

Permalink
Merge pull request #2191 from BENR0/fix_doc_reader_table
Browse files Browse the repository at this point in the history
Fix automatic reader table not listing readers with missing dependencies
  • Loading branch information
djhoese authored Aug 25, 2022
2 parents a5573c1 + 3406026 commit a7cff40
Show file tree
Hide file tree
Showing 11 changed files with 59 additions and 55 deletions.
3 changes: 2 additions & 1 deletion doc/source/_static/main.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
$(document).ready( function () {
$('table.datatable').DataTable( {
"paging": false
"paging": false,
"dom": 'lfitp'
} );
} );
4 changes: 3 additions & 1 deletion doc/source/reader_table.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
# along with satpy. If not, see <http://www.gnu.org/licenses/>.
"""Module for autogenerating reader table from config files."""

from yaml import BaseLoader

from satpy.readers import available_readers


Expand Down Expand Up @@ -72,7 +74,7 @@ def generate_reader_table():
table = [rst_table_header("Satpy Readers", header=["Description", "Reader name", "Status", "fsspec support"],
widths=[45, 25, 30, 30])]

reader_configs = available_readers(as_dict=True)
reader_configs = available_readers(as_dict=True, yaml_loader=BaseLoader)
for rc in reader_configs:
table.append(rst_table_row([rc.get("long_name", "").rstrip("\n"), rc.get("name", ""),
rc.get("status", ""), rc.get("supports_fsspec", "false")]))
Expand Down
6 changes: 1 addition & 5 deletions satpy/composites/config_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,7 @@
from typing import Callable, Iterable

import yaml

try:
from yaml import UnsafeLoader
except ImportError:
from yaml import Loader as UnsafeLoader # type: ignore
from yaml import UnsafeLoader

import satpy
from satpy import DataID, DataQuery
Expand Down
6 changes: 1 addition & 5 deletions satpy/plugin_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,7 @@
import logging

import yaml

try:
from yaml import UnsafeLoader
except ImportError:
from yaml import Loader as UnsafeLoader # type: ignore
from yaml import UnsafeLoader

from satpy._config import config_search_paths
from satpy.utils import recursive_dict_update
Expand Down
30 changes: 14 additions & 16 deletions satpy/readers/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,7 @@
from functools import total_ordering

import yaml

try:
from yaml import UnsafeLoader
except ImportError:
from yaml import Loader as UnsafeLoader # type: ignore
from yaml import UnsafeLoader

from satpy._config import config_search_paths, get_entry_points_config_dirs, glob_config

Expand Down Expand Up @@ -370,22 +366,24 @@ def get_valid_reader_names(reader):
return new_readers


def available_readers(as_dict=False):
def available_readers(as_dict=False, yaml_loader=UnsafeLoader):
"""Available readers based on current configuration.
Args:
as_dict (bool): Optionally return reader information as a dictionary.
Default: False
Default: False.
yaml_loader (Optional[Union[yaml.BaseLoader, yaml.FullLoader, yaml.UnsafeLoader]]):
The yaml loader type. Default: ``yaml.UnsafeLoader``.
Returns: List of available reader names. If `as_dict` is `True` then
a list of dictionaries including additionally reader information
is returned.
Returns:
Union[list[str], list[dict]]: List of available reader names. If `as_dict` is `True` then
a list of dictionaries including additionally reader information is returned.
"""
readers = []
for reader_configs in configs_for_reader():
try:
reader_info = read_reader_config(reader_configs)
reader_info = read_reader_config(reader_configs, loader=yaml_loader)
except (KeyError, IOError, yaml.YAMLError):
LOG.debug("Could not import reader config from: %s", reader_configs)
LOG.debug("Error loading YAML", exc_info=True)
Expand Down Expand Up @@ -454,12 +452,12 @@ def find_files_and_readers(start_time=None, end_time=None, base_dir=None,
missing_ok (bool): If False (default), raise ValueError if no files
are found. If True, return empty dictionary if no
files are found.
fs (FileSystem): Optional, instance of implementation of
fsspec.spec.AbstractFileSystem (strictly speaking,
any object of a class implementing ``.glob`` is
enough). Defaults to searching the local filesystem.
fs (:class:`fsspec.spec.AbstractFileSystem`): Optional, instance of implementation of
:class:`fsspec.spec.AbstractFileSystem` (strictly speaking, any object of a class implementing
``.glob`` is enough). Defaults to searching the local filesystem.
Returns: Dictionary mapping reader name string to list of filenames
Returns:
dict: Dictionary mapping reader name string to list of filenames
"""
reader_files = {}
Expand Down
10 changes: 2 additions & 8 deletions satpy/readers/yaml_reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,22 +26,16 @@
from collections import OrderedDict, deque
from contextlib import suppress
from fnmatch import fnmatch
from functools import cached_property
from weakref import WeakValueDictionary

import numpy as np
import xarray as xr
import yaml

try:
from yaml import UnsafeLoader
except ImportError:
from yaml import Loader as UnsafeLoader # type: ignore

from functools import cached_property

from pyresample.boundary import AreaDefBoundary, Boundary
from pyresample.geometry import AreaDefinition, StackedAreaDefinition, SwathDefinition
from trollsift.parser import globify, parse
from yaml import UnsafeLoader

from satpy import DatasetDict
from satpy.aux_download import DataDownloadMixin
Expand Down
32 changes: 32 additions & 0 deletions satpy/tests/test_readers.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@
# satpy. If not, see <http://www.gnu.org/licenses/>.
"""Test classes and functions in the readers/__init__.py module."""

import builtins
import os
import sys
import unittest
from contextlib import suppress
from unittest import mock
Expand Down Expand Up @@ -55,6 +57,8 @@
},
}

real_import = builtins.__import__


def make_dataid(**items):
"""Make a data id."""
Expand Down Expand Up @@ -625,6 +629,11 @@ def test_old_reader_name_mapping(self):
class TestYAMLFiles(unittest.TestCase):
"""Test and analyze the reader configuration files."""

def setUp(self):
"""Set up monkeypatch."""
from _pytest.monkeypatch import MonkeyPatch
self.monkeypatch = MonkeyPatch()

def test_filename_matches_reader_name(self):
"""Test that every reader filename matches the name in the YAML."""
import yaml
Expand Down Expand Up @@ -662,6 +671,29 @@ def test_available_readers(self):
self.assertIn('name', reader_info)
self.assertEqual(reader_infos, sorted(reader_infos, key=lambda reader_info: reader_info['name']))

def test_available_readers_base_loader(self):
"""Test the 'available_readers' function for yaml loader type BaseLoader."""
import yaml

from satpy import available_readers
from satpy._config import glob_config

def patched_import_error(name, globals=None, locals=None, fromlist=(), level=0):
if name in ('netcdf4', ):
raise ImportError(f"Mocked import error {name}")
return real_import(name, globals=globals, locals=locals, fromlist=fromlist, level=level)

self.monkeypatch.delitem(sys.modules, 'netcdf4', raising=False)
self.monkeypatch.setattr(builtins, '__import__', patched_import_error)

with pytest.raises(ImportError):
import netcdf4 # noqa: F401

reader_names = available_readers(yaml_loader=yaml.BaseLoader)
self.assertIn('abi_l1b', reader_names) # needs netcdf4
self.assertIn('viirs_l1b', reader_names)
self.assertEqual(len(reader_names), len(list(glob_config('readers/*.yaml'))))


class TestGroupFiles(unittest.TestCase):
"""Test the 'group_files' utility function."""
Expand Down
7 changes: 1 addition & 6 deletions satpy/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,10 @@
import numpy as np
import xarray as xr
import yaml
from yaml import BaseLoader
from yaml import BaseLoader, UnsafeLoader

from satpy import CHUNK_SIZE

try:
from yaml import UnsafeLoader
except ImportError:
from yaml import Loader as UnsafeLoader # type: ignore

_is_logging_on = False
TRACE_LEVEL = 5

Expand Down
7 changes: 1 addition & 6 deletions satpy/writers/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,9 @@
import numpy as np
import xarray as xr
import yaml

try:
from yaml import UnsafeLoader
except ImportError:
from yaml import Loader as UnsafeLoader # type: ignore

from trollimage.xrimage import XRImage
from trollsift import parser
from yaml import UnsafeLoader

from satpy import CHUNK_SIZE
from satpy._config import config_search_paths, get_entry_points_config_dirs, glob_config
Expand Down
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
pass

requires = ['numpy >=1.13', 'pillow', 'pyresample >=1.24.0', 'trollsift',
'trollimage >1.10.1', 'pykdtree', 'pyyaml', 'xarray >=0.10.1, !=0.13.0',
'trollimage >1.10.1', 'pykdtree', 'pyyaml >=5.1', 'xarray >=0.10.1, !=0.13.0',
'dask[array] >=0.17.1', 'pyproj>=2.2', 'zarr', 'donfig', 'appdirs',
'pooch', 'pyorbital']

Expand Down
7 changes: 1 addition & 6 deletions utils/convert_to_ninjotiff.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,17 +30,12 @@
import os

import yaml
from yaml import UnsafeLoader

from satpy import Scene
from satpy.pyresample import get_area_def
from satpy.utils import debug_on

try:
from yaml import UnsafeLoader
except ImportError:
from yaml import Loader as UnsafeLoader # type: ignore


debug_on()

parser = argparse.ArgumentParser(description='Turn an image into a NinjoTiff.')
Expand Down

0 comments on commit a7cff40

Please sign in to comment.