Skip to content

Commit

Permalink
split namespace rejection test in staging/dev
Browse files Browse the repository at this point in the history
  • Loading branch information
@charlienegri
charlienegri committed Oct 18, 2024
1 parent c48c41d commit aa83811
Show file tree
Hide file tree
Showing 2 changed files with 217 additions and 15 deletions.
156 changes: 156 additions & 0 deletions tests/files/api/dev.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
<mmd:mmd xmlns:mmd="http://www.met.no/schema/mmd" xmlns:gml="http://www.opengis.net/gml">
<mmd:metadata_identifier>test.no.dev:a1ddaf0f-cae0-4a15-9b37-3468e9cb1a2b</mmd:metadata_identifier>
<mmd:title xml:lang="en">Direct Broadcast data processed in satellite swath to L1C</mmd:title>
<mmd:title xml:lang="nor">Direktesendte satellittdata prosessert i satellittsveip til L1C.</mmd:title>
<mmd:abstract xml:lang="en">Direct Broadcast data received at MET NORWAY Oslo. Processed by standard processing software to geolocated and calibrated values in satellite swath in received instrument resolution.</mmd:abstract>
<mmd:abstract xml:lang="nor">Direktesendte satellittdata mottatt ved Meteorologisk Institutt Oslo. Prosessert med standard prosesseringssoftware til geolokaliserte og kalibrerte verdier i satellitsveip i mottatt instrument oppløsning.</mmd:abstract>
<mmd:metadata_status>Active</mmd:metadata_status>
<mmd:dataset_production_status>Complete</mmd:dataset_production_status>
<mmd:collection>METNCS</mmd:collection>
<mmd:last_metadata_update>
<mmd:update>
<mmd:datetime>2021-04-29T00:46:05Z</mmd:datetime>
<mmd:type>Created</mmd:type>
</mmd:update>
</mmd:last_metadata_update>
<mmd:temporal_extent>
<mmd:start_date>2021-04-29T00:28:44.977627Z</mmd:start_date>
<mmd:end_date>2021-04-29T00:39:55.000000Z</mmd:end_date>
</mmd:temporal_extent>
<mmd:iso_topic_category>climatologyMeteorologyAtmosphere</mmd:iso_topic_category>
<mmd:iso_topic_category>environment</mmd:iso_topic_category>
<mmd:iso_topic_category>oceans</mmd:iso_topic_category>
<mmd:keywords vocabulary="GCMD">
<mmd:keyword>Earth Science &gt; Atmosphere &gt; Atmospheric radiation</mmd:keyword>
<mmd:resource>https://gcmdservices.gsfc.nasa.gov/static/kms/</mmd:resource>
<mmd:separator></mmd:separator>
</mmd:keywords>
<mmd:keywords vocabulary="GEMET">
<mmd:keyword>Meteorological geographical features</mmd:keyword>
<mmd:keyword>Atmospheric conditions</mmd:keyword>
<mmd:keyword>Oceanographic geographical features</mmd:keyword>
<mmd:resource>http://inspire.ec.europa.eu/theme</mmd:resource>
<mmd:separator></mmd:separator>
</mmd:keywords>
<mmd:keywords vocabulary="Norwegian thematic categories">
<mmd:keyword>Weather and climate</mmd:keyword>
<mmd:resource>https://register.geonorge.no/subregister/metadata-kodelister/kartverket/nasjonal-temainndeling</mmd:resource>
<mmd:separator></mmd:separator>
</mmd:keywords>
<mmd:geographic_extent>
<mmd:rectangle srsName="EPSG:4326">
<mmd:north>80.49233</mmd:north>
<mmd:south>36.540688</mmd:south>
<mmd:east>79.40124</mmd:east>
<mmd:west>1.5549301</mmd:west>
</mmd:rectangle>
</mmd:geographic_extent>
<mmd:dataset_language>en</mmd:dataset_language>
<mmd:operational_status>Operational</mmd:operational_status>
<mmd:use_constraint>
<mmd:identifier>CC-BY-4.0</mmd:identifier>
<mmd:resource>http://spdx.org/licenses/CC-BY-4.0</mmd:resource>
</mmd:use_constraint>
<mmd:personnel>
<mmd:role>Technical contact</mmd:role>
<mmd:name>DIVISION FOR OBSERVATION QUALITY AND DATA PROCESSING</mmd:name>
<mmd:email>post@met.no</mmd:email>
<mmd:organisation>MET NORWAY</mmd:organisation>
</mmd:personnel>
<mmd:personnel>
<mmd:role>Metadata author</mmd:role>
<mmd:name>DIVISION FOR OBSERVATION QUALITY AND DATA PROCESSING</mmd:name>
<mmd:email>post@met.no</mmd:email>
<mmd:organisation>unknown</mmd:organisation>
</mmd:personnel>
<mmd:data_center>
<mmd:data_center_name>
<mmd:short_name>MET NORWAY</mmd:short_name>
<mmd:long_name>MET NORWAY</mmd:long_name>
</mmd:data_center_name>
<mmd:data_center_url>met.no</mmd:data_center_url>
</mmd:data_center>
<mmd:data_access>
<mmd:type>OPeNDAP</mmd:type>
<mmd:description>Open-source Project for a Network Data Access Protocol</mmd:description>
<mmd:resource>https://thredds.met.no/thredds/dodsC/remotesensingsatellite/polar-swath/2021/04/29/aqua-modis-1km-20210429002844-20210429003955.nc</mmd:resource>
</mmd:data_access>
<mmd:data_access>
<mmd:type>OGC WMS</mmd:type>
<mmd:description>OGC Web Mapping Service, URI to GetCapabilities Document.</mmd:description>
<mmd:resource>https://thredds.met.no/thredds/wms/remotesensingsatellite/polar-swath/2021/04/29/aqua-modis-1km-20210429002844-20210429003955.nc?service=WMS&amp;version=1.3.0&amp;request=GetCapabilities</mmd:resource>
<mmd:wms_layers>
<mmd:wms_layer>toa_bidirectional_reflectance</mmd:wms_layer>
<mmd:wms_layer>toa_bidirectional_reflectance</mmd:wms_layer>
<mmd:wms_layer>toa_bidirectional_reflectance</mmd:wms_layer>
<mmd:wms_layer>toa_bidirectional_reflectance</mmd:wms_layer>
<mmd:wms_layer>toa_bidirectional_reflectance</mmd:wms_layer>
<mmd:wms_layer>toa_bidirectional_reflectance</mmd:wms_layer>
<mmd:wms_layer>toa_bidirectional_reflectance</mmd:wms_layer>
<mmd:wms_layer>toa_bidirectional_reflectance</mmd:wms_layer>
<mmd:wms_layer>toa_bidirectional_reflectance</mmd:wms_layer>
<mmd:wms_layer>toa_bidirectional_reflectance</mmd:wms_layer>
<mmd:wms_layer>toa_bidirectional_reflectance</mmd:wms_layer>
<mmd:wms_layer>toa_bidirectional_reflectance</mmd:wms_layer>
<mmd:wms_layer>toa_bidirectional_reflectance</mmd:wms_layer>
<mmd:wms_layer>toa_bidirectional_reflectance</mmd:wms_layer>
<mmd:wms_layer>toa_brightness_temperature</mmd:wms_layer>
<mmd:wms_layer>toa_brightness_temperature</mmd:wms_layer>
<mmd:wms_layer>toa_brightness_temperature</mmd:wms_layer>
<mmd:wms_layer>toa_brightness_temperature</mmd:wms_layer>
<mmd:wms_layer>toa_brightness_temperature</mmd:wms_layer>
<mmd:wms_layer>toa_brightness_temperature</mmd:wms_layer>
<mmd:wms_layer>toa_bidirectional_reflectance</mmd:wms_layer>
<mmd:wms_layer>toa_brightness_temperature</mmd:wms_layer>
<mmd:wms_layer>toa_brightness_temperature</mmd:wms_layer>
<mmd:wms_layer>toa_brightness_temperature</mmd:wms_layer>
<mmd:wms_layer>toa_bidirectional_reflectance</mmd:wms_layer>
<mmd:wms_layer>toa_brightness_temperature</mmd:wms_layer>
<mmd:wms_layer>toa_brightness_temperature</mmd:wms_layer>
<mmd:wms_layer>toa_brightness_temperature</mmd:wms_layer>
<mmd:wms_layer>toa_brightness_temperature</mmd:wms_layer>
<mmd:wms_layer>toa_brightness_temperature</mmd:wms_layer>
<mmd:wms_layer>toa_brightness_temperature</mmd:wms_layer>
<mmd:wms_layer>toa_brightness_temperature</mmd:wms_layer>
<mmd:wms_layer>toa_bidirectional_reflectance</mmd:wms_layer>
<mmd:wms_layer>toa_bidirectional_reflectance</mmd:wms_layer>
<mmd:wms_layer>toa_bidirectional_reflectance</mmd:wms_layer>
<mmd:wms_layer>toa_bidirectional_reflectance</mmd:wms_layer>
<mmd:wms_layer>toa_bidirectional_reflectance</mmd:wms_layer>
<mmd:wms_layer>toa_bidirectional_reflectance</mmd:wms_layer>
</mmd:wms_layers>
</mmd:data_access>
<mmd:data_access>
<mmd:type>HTTP</mmd:type>
<mmd:description>Direct download of file</mmd:description>
<mmd:resource>https://thredds.met.no/thredds/fileServer/remotesensingsatellite/polar-swath/2021/04/29/aqua-modis-1km-20210429002844-20210429003955.nc</mmd:resource>
</mmd:data_access>
<mmd:related_dataset relation_type="parent">test.no.dev:64db6102-14ce-41e9-b93b-61dbb2cb8b4e</mmd:related_dataset>
<mmd:storage_information>
<mmd:file_name>aqua-modis-1km-20210429002844-20210429003955.nc</mmd:file_name>
<mmd:file_location>/lustre/storeB/immutable/archive/projects/remotesensing/satellite-thredds/polar-swath/2021/04/29/aqua-modis-1km-20210429002844-20210429003955.nc</mmd:file_location>
<mmd:file_format>NetCDF-CF</mmd:file_format>
<mmd:file_size unit="MB">1862.00</mmd:file_size>
<mmd:checksum type="md5sum">4e1833610272ee63228f575d1c875fbe</mmd:checksum>
</mmd:storage_information>
<mmd:project>
<mmd:short_name>Govermental core service</mmd:short_name>
<mmd:long_name>Govermental core service</mmd:long_name>
</mmd:project>
<mmd:platform>
<mmd:short_name>Aqua</mmd:short_name>
<mmd:long_name>Aqua</mmd:long_name>
<mmd:resource>https://www.wmo-sat.info/oscar/satellites/view/aqua</mmd:resource>
<mmd:instrument>
<mmd:short_name>MODIS</mmd:short_name>
<mmd:long_name>MODIS</mmd:long_name>
<mmd:resource>https://www.wmo-sat.info/oscar/instruments/view/modis</mmd:resource>
</mmd:instrument>
</mmd:platform>
<mmd:activity_type>Space Borne Instrument</mmd:activity_type>
<mmd:dataset_citation>
<mmd:author>DIVISION FOR OBSERVATION QUALITY AND DATA PROCESSING</mmd:author>
<mmd:publication_date>2021-04-29</mmd:publication_date>
<mmd:title>Direct Broadcast data processed in satellite swath to L1C</mmd:title>
</mmd:dataset_citation>
</mmd:mmd>
76 changes: 61 additions & 15 deletions tests/test_api/test_worker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,14 +124,20 @@ def testApiWorker_Validator(monkeypatch, filesDir):

# Invalid data format
passData = readFile(passFile)
assert passWorker.validate(passData) == (False, "Input must be bytes type", passData)
assert passWorker.validate(passData) == (
False,
"Input must be bytes type",
passData,
)

# Parsing error
parseFailData = bytes(readFile(parseFailFile), "utf-8")
assert parseFailWorker.validate(parseFailData) == (
False,
"Opening and ending tag mismatch: short_name line 141 and platform,"
" line 149, column 18 (<string>, line 149)", parseFailData)
" line 149, column 18 (<string>, line 149)",
parseFailData,
)

# Valid data format
with monkeypatch.context() as mp:
Expand Down Expand Up @@ -171,8 +177,11 @@ def testApiWorker_Validator(monkeypatch, filesDir):

# _check_information_content fails
with monkeypatch.context() as mp:
mp.setattr(Worker, "_check_information_content",
lambda *a: (False, "_check_information_content failed"))
mp.setattr(
Worker,
"_check_information_content",
lambda *a: (False, "_check_information_content failed"),
)

passData = bytes(readFile(passFile), "utf-8")
valid, msg, passData = passWorker.validate(passData)
Expand Down Expand Up @@ -211,31 +220,63 @@ def testApiWorker_NamespaceReplacement(filesDir):

# END Test testApiWorker_NamespaceReplacement


@pytest.mark.api
def testApiWorker_NamespaceRejectedIfWrongEnv(filesDir):
"""Test rejection if namespace contains a .dev/.staging not matching the env."""
def testApiWorker_NamespaceRejectedIfWrongEnvStaging(filesDir):
"""Test rejection if namespace contains a .staging not matching the env."""

xsdFile = os.path.join(filesDir, "mmd", "mmd.xsd")
passFile = os.path.join(filesDir, "api", "staging.xml")

xsdObj = lxml.etree.XMLSchema(lxml.etree.parse(xsdFile))
passWorker = Worker("none", passFile, xsdObj)

# Test namespace "no.met.dev" - datasets with namespaces
# "no.met.dev" and "no.met" should pass
passWorker._conf.env_string = "no.met.dev"
passWorker._conf.env_string = ""

# Valid XML
passData = bytes(readFile(passFile), "utf-8")
valid, msg, passData = passWorker.validate(passData)
assert valid is False
assert msg == "Namespace test.no.staging does not match the env "

# Test namespace "no.met" - only datasets with namespace "no.met"
# should pass
passWorker._conf.env_string = "no.met"
passWorker._conf.env_string = "staging"

# Valid XML
passData = bytes(readFile(passFile), "utf-8")
valid, msg, passData = passWorker.validate(passData)
assert valid is True


# END Test testApiWorker_NamespaceRejectedIfWrongEnvStaging


@pytest.mark.api
def testApiWorker_NamespaceRejectedIfWrongEnvDev(filesDir):
"""Test rejection if namespace contains a .dev not matching the env."""

xsdFile = os.path.join(filesDir, "mmd", "mmd.xsd")
passFile = os.path.join(filesDir, "api", "dev.xml")

xsdObj = lxml.etree.XMLSchema(lxml.etree.parse(xsdFile))
passWorker = Worker("none", passFile, xsdObj)

passWorker._conf.env_string = ""

# Valid XML
passData = bytes(readFile(passFile), "utf-8")
valid, msg, passData = passWorker.validate(passData)
assert valid is False
assert msg == "Namespace test.no.dev does not match the env "

passWorker._conf.env_string = "dev"

# END Test testApiWorker_NamespaceRejectedIfWrongEnv
# Valid XML
passData = bytes(readFile(passFile), "utf-8")
valid, msg, passData = passWorker.validate(passData)
assert valid is True


# END Test testApiWorker_NamespaceRejectedIfWrongEnvDev


@pytest.mark.api
Expand All @@ -257,7 +298,8 @@ def testApiWorker_ParentNamespaceReplacement(filesDir):
assert valid is True

match_parent_id = re.search(
b'<mmd:related_dataset relation_type="parent">(.+?)</mmd:related_dataset>', passData
b'<mmd:related_dataset relation_type="parent">(.+?)</mmd:related_dataset>',
passData,
)
parent_id = match_parent_id.group(1)
namespace = parent_id.split(b":")[0].decode()
Expand All @@ -269,7 +311,11 @@ def testApiWorker_ParentNamespaceReplacement(filesDir):
badparentData = bytes(readFile(badparentidFile), "utf-8")
valid, msg, data = badparentWorker.validate(badparentData)
assert valid is False
assert msg == "Malformed parent dataset identifier [b'64db6102-14ce-41e9-b93b-61dbb2cb8b4e']"
assert (
msg
== "Malformed parent dataset identifier [b'64db6102-14ce-41e9-b93b-61dbb2cb8b4e']"
)


# END Test testApiWorker_NamespaceReplacement

Expand Down

0 comments on commit aa83811

Please sign in to comment.