From e021dd2a500b5bfbc2ae99364bf6f35b677f2190 Mon Sep 17 00:00:00 2001 From: msweier Date: Mon, 29 Jul 2024 15:51:39 -0500 Subject: [PATCH 1/9] add aprfc-qte-01h.py --- .../processors/aprfc-qte-01h.py | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 src/cumulus_geoproc/processors/aprfc-qte-01h.py diff --git a/src/cumulus_geoproc/processors/aprfc-qte-01h.py b/src/cumulus_geoproc/processors/aprfc-qte-01h.py new file mode 100644 index 0000000..b95b150 --- /dev/null +++ b/src/cumulus_geoproc/processors/aprfc-qte-01h.py @@ -0,0 +1,98 @@ +""" +# Alaska Pacific River Forecast Center + +## QPE 06 hour total precipitation +""" + + +import os +import sys +import pyplugs + +from cumulus_geoproc import logger +from cumulus_geoproc.utils import cgdal + + +@pyplugs.register +def process(*, src: str, dst: str = None, acquirable: str = None): + """ + # Grid processor + + __Requires keyword only arguments (*)__ + + Parameters + ---------- + src : str + path to input file for processing + dst : str, optional + path to temporary directory + acquirable: str, optional + acquirable slug + + Returns + ------- + List[dict] + ``` + { + "filetype": str, Matching database acquirable + "file": str, Converted file + "datetime": str, Valid Time, ISO format with timezone + "version": str Reference Time (forecast), ISO format with timezone + } + ``` + """ + + try: + attr = {"GRIB_COMMENT" "Temperature [C]"} + # determine the path and open the file in gdal + ds, src_path, dst_path = cgdal.openfileGDAL(src, dst, GDALAccess="read_only") + + # Grad the grid from the band + if (band_number := cgdal.find_band(ds, attr)) is None: + raise Exception("Band number not found for attributes: {attr}") + + logger.debug(f"Band number '{band_number}' found for attributes {attr}") + + raster = ds.GetRasterBand(band_number) + + # Get Datetime from String Like "1599008400 sec UTC" + dt_valid = cgdal.getDate(raster, src_path, "GRIB_VALID_TIME", None, None) + + cgdal.gdal_translate_w_options( + tif := os.path.join( + dst, f'{acquirable}.{dt_valid.strftime("%Y%m%d_%H%M")}.tif' + ), + ds, + bandList=[band_number], + ) + + # validate COG + if (validate := cgdal.validate_cog("-q", tif)) == 0: + logger.debug(f"Validate COG = {validate}\t{tif} is a COG") + + outfile_list = [ + { + "filetype": acquirable, + "file": tif, + "datetime": dt_valid.isoformat(), + "version": None, + }, + ] + + except (RuntimeError, KeyError, Exception) as ex: + exc_type, exc_value, exc_traceback = sys.exc_info() + traceback_details = { + "filename": os.path.basename(exc_traceback.tb_frame.f_code.co_filename), + "line number": exc_traceback.tb_lineno, + "method": exc_traceback.tb_frame.f_code.co_name, + "type": exc_type.__name__, + "message": exc_value, + } + for k, v in traceback_details.items(): + logger.error(f"{k}: {v}") + + finally: + ds = None + raster = None + + return outfile_list From 08366b66600e1076418c10ffd1becc0d5e87b030 Mon Sep 17 00:00:00 2001 From: msweier Date: Mon, 29 Jul 2024 16:02:56 -0500 Subject: [PATCH 2/9] add aprfc-qte-01h.py --- src/cumulus_geoproc/processors/aprfc-qte-01h.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cumulus_geoproc/processors/aprfc-qte-01h.py b/src/cumulus_geoproc/processors/aprfc-qte-01h.py index b95b150..3343c4b 100644 --- a/src/cumulus_geoproc/processors/aprfc-qte-01h.py +++ b/src/cumulus_geoproc/processors/aprfc-qte-01h.py @@ -43,7 +43,7 @@ def process(*, src: str, dst: str = None, acquirable: str = None): """ try: - attr = {"GRIB_COMMENT" "Temperature [C]"} + attr = {"GRIB_COMMENT":"Temperature [C]"} # determine the path and open the file in gdal ds, src_path, dst_path = cgdal.openfileGDAL(src, dst, GDALAccess="read_only") From 066f9d405b7e6b33f74a494e16909f85a4daed8d Mon Sep 17 00:00:00 2001 From: msweier Date: Wed, 14 Aug 2024 14:41:10 -0500 Subject: [PATCH 3/9] testing aprfc-qte-01h --- .../processors/aprfc-qte-01h.py | 106 +++++++++--------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/src/cumulus_geoproc/processors/aprfc-qte-01h.py b/src/cumulus_geoproc/processors/aprfc-qte-01h.py index 3343c4b..2aad7c6 100644 --- a/src/cumulus_geoproc/processors/aprfc-qte-01h.py +++ b/src/cumulus_geoproc/processors/aprfc-qte-01h.py @@ -1,7 +1,7 @@ """ # Alaska Pacific River Forecast Center -## QPE 06 hour total precipitation +## QTE 01 hour total precipitation """ @@ -42,57 +42,57 @@ def process(*, src: str, dst: str = None, acquirable: str = None): ``` """ - try: - attr = {"GRIB_COMMENT":"Temperature [C]"} - # determine the path and open the file in gdal - ds, src_path, dst_path = cgdal.openfileGDAL(src, dst, GDALAccess="read_only") - - # Grad the grid from the band - if (band_number := cgdal.find_band(ds, attr)) is None: - raise Exception("Band number not found for attributes: {attr}") - - logger.debug(f"Band number '{band_number}' found for attributes {attr}") - - raster = ds.GetRasterBand(band_number) - - # Get Datetime from String Like "1599008400 sec UTC" - dt_valid = cgdal.getDate(raster, src_path, "GRIB_VALID_TIME", None, None) - - cgdal.gdal_translate_w_options( - tif := os.path.join( - dst, f'{acquirable}.{dt_valid.strftime("%Y%m%d_%H%M")}.tif' - ), - ds, - bandList=[band_number], - ) - - # validate COG - if (validate := cgdal.validate_cog("-q", tif)) == 0: - logger.debug(f"Validate COG = {validate}\t{tif} is a COG") - - outfile_list = [ - { - "filetype": acquirable, - "file": tif, - "datetime": dt_valid.isoformat(), - "version": None, - }, - ] - - except (RuntimeError, KeyError, Exception) as ex: - exc_type, exc_value, exc_traceback = sys.exc_info() - traceback_details = { - "filename": os.path.basename(exc_traceback.tb_frame.f_code.co_filename), - "line number": exc_traceback.tb_lineno, - "method": exc_traceback.tb_frame.f_code.co_name, - "type": exc_type.__name__, - "message": exc_value, - } - for k, v in traceback_details.items(): - logger.error(f"{k}: {v}") - - finally: - ds = None - raster = None + # try: + attr = {"GRIB_COMMENT":"Temperature [C]"} + # determine the path and open the file in gdal + ds, src_path, dst_path = cgdal.openfileGDAL(src, dst, GDALAccess="read_only") + + # Grad the grid from the band + if (band_number := cgdal.find_band(ds, attr)) is None: + raise Exception("Band number not found for attributes: {attr}") + + logger.debug(f"Band number '{band_number}' found for attributes {attr}") + + raster = ds.GetRasterBand(band_number) + + # Get Datetime from String Like "1599008400 sec UTC" + dt_valid = cgdal.getDate(raster, src_path, "GRIB_VALID_TIME", None, None) + + cgdal.gdal_translate_w_options( + tif := os.path.join( + dst, f'{acquirable}.{dt_valid.strftime("%Y%m%d_%H%M")}.tif' + ), + ds, + bandList=[band_number], + ) + + # validate COG + if (validate := cgdal.validate_cog("-q", tif)) == 0: + logger.debug(f"Validate COG = {validate}\t{tif} is a COG") + + outfile_list = [ + { + "filetype": acquirable, + "file": tif, + "datetime": dt_valid.isoformat(), + "version": None, + }, + ] + + # except (RuntimeError, KeyError, Exception) as ex: + # exc_type, exc_value, exc_traceback = sys.exc_info() + # traceback_details = { + # "filename": os.path.basename(exc_traceback.tb_frame.f_code.co_filename), + # "line number": exc_traceback.tb_lineno, + # "method": exc_traceback.tb_frame.f_code.co_name, + # "type": exc_type.__name__, + # "message": exc_value, + # } + # for k, v in traceback_details.items(): + # logger.error(f"{k}: {v}") + + # finally: + # ds = None + # raster = None return outfile_list From 5eed610980268520e2e8086e713c155235e13dc7 Mon Sep 17 00:00:00 2001 From: msweier Date: Wed, 14 Aug 2024 15:01:36 -0500 Subject: [PATCH 4/9] add logger statement --- src/cumulus_geoproc/processors/aprfc-qte-01h.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cumulus_geoproc/processors/aprfc-qte-01h.py b/src/cumulus_geoproc/processors/aprfc-qte-01h.py index 2aad7c6..c33ad62 100644 --- a/src/cumulus_geoproc/processors/aprfc-qte-01h.py +++ b/src/cumulus_geoproc/processors/aprfc-qte-01h.py @@ -47,6 +47,8 @@ def process(*, src: str, dst: str = None, acquirable: str = None): # determine the path and open the file in gdal ds, src_path, dst_path = cgdal.openfileGDAL(src, dst, GDALAccess="read_only") + logger.debug(f"ds = {ds}, src_path = {src_path}, dst_path = {dst_path}") + # Grad the grid from the band if (band_number := cgdal.find_band(ds, attr)) is None: raise Exception("Band number not found for attributes: {attr}") From 577c04c342cf907de7632ab6bea9cbb0043ba860 Mon Sep 17 00:00:00 2001 From: msweier Date: Mon, 25 Nov 2024 10:32:47 -0600 Subject: [PATCH 5/9] update aprfc qte --- Dockerfile | 2 +- .../processors/aorc-csu-precip.py | 101 ++++++++++++++++++ .../integration/fixtures/test_products.json | 7 ++ 3 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 src/cumulus_geoproc/processors/aorc-csu-precip.py diff --git a/Dockerfile b/Dockerfile index 53c8404..af638fe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ ENV PYTHONUNBUFFERED=1 ENV PYTEST_ADDOPTS="--color=yes" # env var for test data version to use, which should always be the most up to date -ENV TEST_DATA_TAG=2023-11-08 +ENV TEST_DATA_TAG=2024-08-06 RUN apt-get update -y \ && apt-get install -y python3-pip python3-venv curl \ diff --git a/src/cumulus_geoproc/processors/aorc-csu-precip.py b/src/cumulus_geoproc/processors/aorc-csu-precip.py new file mode 100644 index 0000000..5843d2e --- /dev/null +++ b/src/cumulus_geoproc/processors/aorc-csu-precip.py @@ -0,0 +1,101 @@ +""" +# Analysis Of Record for Calibration (AORC) + +## CSU archive - 800m hourly precipitation +""" + +import os +import sys +import pyplugs + +from cumulus_geoproc import logger +from cumulus_geoproc.utils import cgdal +from osgeo.gdalconst import GDT_Float32 + + +@pyplugs.register +def process(*, src: str, dst: str = None, acquirable: str = None): + """ + # Grid processor + + __Requires keyword only arguments (*)__ + + Parameters + ---------- + src : str + path to input file for processing + dst : str, optional + path to temporary directory + acquirable: str, optional + acquirable slug + + Returns + ------- + List[dict] + ``` + { + "filetype": str, Matching database acquirable + "file": str, Converted file + "datetime": str, Valid Time, ISO format with timezone + "version": str Reference Time (forecast), ISO format with timezone + } + ``` + """ + + try: + attr = {"NETCDF_VARNAME": "APCP_surface"} + # determine the path and open the file in gdal + ds, src_path, _ = cgdal.openfileGDAL(src, dst, GDALAccess="read_only") + + # Grab the grid from the band + if (band_number := cgdal.find_band(ds, attr)) is None: + raise cgdal.BandNotFoundError("Band number not found for attributes: {attr}") + + logger.debug("Band number '%s' found for attributes %s", band_number, attr) + + raster = ds.GetRasterBand(band_number) + + # Get Datetime from String Like "1599008400 sec UTC" + dt_valid = cgdal.getDate(raster, src_path, "NETCDF_DIM_time", None, None) + + cgdal.gdal_translate_w_options( + tif := os.path.join( + dst, f'{acquirable}.{dt_valid.strftime("%Y%m%d_%H%M")}.tif' + ), + ds, + bandList=[band_number], + outputSRS="EPSG:4326", + unscale=True, + outputType=GDT_Float32 + ) + + # validate COG + if (validate := cgdal.validate_cog("-q", tif)) == 0: + logger.debug("Validate COG = %s\t%s is a COG", validate, tif) + + outfile_list = [ + { + "filetype": acquirable, + "file": tif, + "datetime": dt_valid.isoformat(), + "version": None + }, + ] + + except (RuntimeError, KeyError, cgdal.BandNotFoundError): + exc_type, exc_value, exc_traceback = sys.exc_info() + traceback_details = { + "filename": os.path.basename(exc_traceback.tb_frame.f_code.co_filename), + "line number": exc_traceback.tb_lineno, + "method": exc_traceback.tb_frame.f_code.co_name, + "type": exc_type.__name__, + "message": exc_value, + } + for k, v in traceback_details.items(): + logger.error("%s: %s", k, v) + + finally: + ds = None + raster = None + + return outfile_list \ No newline at end of file diff --git a/src/tests/integration/fixtures/test_products.json b/src/tests/integration/fixtures/test_products.json index 873432b..fcbaf23 100644 --- a/src/tests/integration/fixtures/test_products.json +++ b/src/tests/integration/fixtures/test_products.json @@ -27,6 +27,13 @@ "versioned": true, "name_pattern": "qpf06f_has_96f_%Y%m%d_%H_awips_%Y%m%d%H%M.grb.gz" }, + { + "plugin": "aprfc-qte-01h", + "url": "", + "local_source": "cumulus-geoproc-test-data/fixtures/aprfc-qte-06h/20240731_akurma.t11z.2dvaranl_ndfd_3p0.grb2.gz", + "versioned": false, + "name_pattern": "%Y%m%d_akurma.t%Hz.2dvaranl_ndfd_3p0.grb2" + }, { "plugin": "cbrfc-mpe", "url": "", From 859bfd204552fc42de32a73977ed943c0cade542 Mon Sep 17 00:00:00 2001 From: msweier Date: Wed, 14 Aug 2024 14:41:10 -0500 Subject: [PATCH 6/9] aprfc-qte-01h add back try block --- .../processors/aprfc-qte-01h.py | 106 +++++++++--------- 1 file changed, 52 insertions(+), 54 deletions(-) diff --git a/src/cumulus_geoproc/processors/aprfc-qte-01h.py b/src/cumulus_geoproc/processors/aprfc-qte-01h.py index c33ad62..1c92728 100644 --- a/src/cumulus_geoproc/processors/aprfc-qte-01h.py +++ b/src/cumulus_geoproc/processors/aprfc-qte-01h.py @@ -42,59 +42,57 @@ def process(*, src: str, dst: str = None, acquirable: str = None): ``` """ - # try: - attr = {"GRIB_COMMENT":"Temperature [C]"} - # determine the path and open the file in gdal - ds, src_path, dst_path = cgdal.openfileGDAL(src, dst, GDALAccess="read_only") - - logger.debug(f"ds = {ds}, src_path = {src_path}, dst_path = {dst_path}") - - # Grad the grid from the band - if (band_number := cgdal.find_band(ds, attr)) is None: - raise Exception("Band number not found for attributes: {attr}") - - logger.debug(f"Band number '{band_number}' found for attributes {attr}") - - raster = ds.GetRasterBand(band_number) - - # Get Datetime from String Like "1599008400 sec UTC" - dt_valid = cgdal.getDate(raster, src_path, "GRIB_VALID_TIME", None, None) - - cgdal.gdal_translate_w_options( - tif := os.path.join( - dst, f'{acquirable}.{dt_valid.strftime("%Y%m%d_%H%M")}.tif' - ), - ds, - bandList=[band_number], - ) - - # validate COG - if (validate := cgdal.validate_cog("-q", tif)) == 0: - logger.debug(f"Validate COG = {validate}\t{tif} is a COG") - - outfile_list = [ - { - "filetype": acquirable, - "file": tif, - "datetime": dt_valid.isoformat(), - "version": None, - }, - ] - - # except (RuntimeError, KeyError, Exception) as ex: - # exc_type, exc_value, exc_traceback = sys.exc_info() - # traceback_details = { - # "filename": os.path.basename(exc_traceback.tb_frame.f_code.co_filename), - # "line number": exc_traceback.tb_lineno, - # "method": exc_traceback.tb_frame.f_code.co_name, - # "type": exc_type.__name__, - # "message": exc_value, - # } - # for k, v in traceback_details.items(): - # logger.error(f"{k}: {v}") - - # finally: - # ds = None - # raster = None + try: + attr = {"GRIB_COMMENT":"Temperature [C]"} + # determine the path and open the file in gdal + ds, src_path, dst_path = cgdal.openfileGDAL(src, dst, GDALAccess="read_only") + + # Grad the grid from the band + if (band_number := cgdal.find_band(ds, attr)) is None: + raise Exception("Band number not found for attributes: {attr}") + + logger.debug(f"Band number '{band_number}' found for attributes {attr}") + + raster = ds.GetRasterBand(band_number) + + # Get Datetime from String Like "1599008400 sec UTC" + dt_valid = cgdal.getDate(raster, src_path, "GRIB_VALID_TIME", None, None) + + cgdal.gdal_translate_w_options( + tif := os.path.join( + dst, f'{acquirable}.{dt_valid.strftime("%Y%m%d_%H%M")}.tif' + ), + ds, + bandList=[band_number], + ) + + # validate COG + if (validate := cgdal.validate_cog("-q", tif)) == 0: + logger.debug(f"Validate COG = {validate}\t{tif} is a COG") + + outfile_list = [ + { + "filetype": acquirable, + "file": tif, + "datetime": dt_valid.isoformat(), + "version": None, + }, + ] + + except (RuntimeError, KeyError, Exception) as ex: + exc_type, exc_value, exc_traceback = sys.exc_info() + traceback_details = { + "filename": os.path.basename(exc_traceback.tb_frame.f_code.co_filename), + "line number": exc_traceback.tb_lineno, + "method": exc_traceback.tb_frame.f_code.co_name, + "type": exc_type.__name__, + "message": exc_value, + } + for k, v in traceback_details.items(): + logger.error(f"{k}: {v}") + + finally: + ds = None + raster = None return outfile_list From f98f819593dd62b29f2a14df0eac7ee9071d71e5 Mon Sep 17 00:00:00 2001 From: msweier Date: Thu, 12 Dec 2024 15:58:05 -0600 Subject: [PATCH 7/9] remove aorc-csu-precip file that was in test dataset --- .../processors/aorc-csu-precip.py | 101 ------------------ 1 file changed, 101 deletions(-) delete mode 100644 src/cumulus_geoproc/processors/aorc-csu-precip.py diff --git a/src/cumulus_geoproc/processors/aorc-csu-precip.py b/src/cumulus_geoproc/processors/aorc-csu-precip.py deleted file mode 100644 index 5843d2e..0000000 --- a/src/cumulus_geoproc/processors/aorc-csu-precip.py +++ /dev/null @@ -1,101 +0,0 @@ -""" -# Analysis Of Record for Calibration (AORC) - -## CSU archive - 800m hourly precipitation -""" - -import os -import sys -import pyplugs - -from cumulus_geoproc import logger -from cumulus_geoproc.utils import cgdal -from osgeo.gdalconst import GDT_Float32 - - -@pyplugs.register -def process(*, src: str, dst: str = None, acquirable: str = None): - """ - # Grid processor - - __Requires keyword only arguments (*)__ - - Parameters - ---------- - src : str - path to input file for processing - dst : str, optional - path to temporary directory - acquirable: str, optional - acquirable slug - - Returns - ------- - List[dict] - ``` - { - "filetype": str, Matching database acquirable - "file": str, Converted file - "datetime": str, Valid Time, ISO format with timezone - "version": str Reference Time (forecast), ISO format with timezone - } - ``` - """ - - try: - attr = {"NETCDF_VARNAME": "APCP_surface"} - # determine the path and open the file in gdal - ds, src_path, _ = cgdal.openfileGDAL(src, dst, GDALAccess="read_only") - - # Grab the grid from the band - if (band_number := cgdal.find_band(ds, attr)) is None: - raise cgdal.BandNotFoundError("Band number not found for attributes: {attr}") - - logger.debug("Band number '%s' found for attributes %s", band_number, attr) - - raster = ds.GetRasterBand(band_number) - - # Get Datetime from String Like "1599008400 sec UTC" - dt_valid = cgdal.getDate(raster, src_path, "NETCDF_DIM_time", None, None) - - cgdal.gdal_translate_w_options( - tif := os.path.join( - dst, f'{acquirable}.{dt_valid.strftime("%Y%m%d_%H%M")}.tif' - ), - ds, - bandList=[band_number], - outputSRS="EPSG:4326", - unscale=True, - outputType=GDT_Float32 - ) - - # validate COG - if (validate := cgdal.validate_cog("-q", tif)) == 0: - logger.debug("Validate COG = %s\t%s is a COG", validate, tif) - - outfile_list = [ - { - "filetype": acquirable, - "file": tif, - "datetime": dt_valid.isoformat(), - "version": None - }, - ] - - except (RuntimeError, KeyError, cgdal.BandNotFoundError): - exc_type, exc_value, exc_traceback = sys.exc_info() - traceback_details = { - "filename": os.path.basename(exc_traceback.tb_frame.f_code.co_filename), - "line number": exc_traceback.tb_lineno, - "method": exc_traceback.tb_frame.f_code.co_name, - "type": exc_type.__name__, - "message": exc_value, - } - for k, v in traceback_details.items(): - logger.error("%s: %s", k, v) - - finally: - ds = None - raster = None - - return outfile_list \ No newline at end of file From 48f5747162edc214743560612df00f44fbcba8f9 Mon Sep 17 00:00:00 2001 From: msweier Date: Fri, 13 Dec 2024 09:38:42 -0600 Subject: [PATCH 8/9] Update test data to 2024-12-13 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index af638fe..88ce6db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ ENV PYTHONUNBUFFERED=1 ENV PYTEST_ADDOPTS="--color=yes" # env var for test data version to use, which should always be the most up to date -ENV TEST_DATA_TAG=2024-08-06 +ENV TEST_DATA_TAG=2024-12-13 RUN apt-get update -y \ && apt-get install -y python3-pip python3-venv curl \ From b050383207d6ee37d8ef4b509a4c70e0449bea29 Mon Sep 17 00:00:00 2001 From: msweier Date: Fri, 13 Dec 2024 11:47:40 -0600 Subject: [PATCH 9/9] include update to netCDF requirment --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e501510..d990e19 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ boto3==1.34.138 botocore==1.34.138 celery==5.4.0 pyplugs==0.4.0 -netCDF4==1.6.5 +netCDF4==1.7.2 requests==2.32.3 GDAL[numpy]==3.9.1 psycopg2-binary==2.9.9