From ea1b0dedc7215e66c8753e5518c9873c79ad4b76 Mon Sep 17 00:00:00 2001 From: agrouaze Date: Wed, 28 Feb 2024 14:10:13 +0100 Subject: [PATCH 1/5] test copy pasted from slcwindinversion to be adapted --- tests/test_output_name.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 tests/test_output_name.py diff --git a/tests/test_output_name.py b/tests/test_output_name.py new file mode 100644 index 0000000..d4d2ea1 --- /dev/null +++ b/tests/test_output_name.py @@ -0,0 +1,29 @@ +import pytest +from slcwindinversion.utils import get_l2_filepath + +inputs_l1c = [ + "/tmp/data/products/tests/iw/slc/l1c/4.0.0/S1B_IW_XSP__1SDV_20210420T094117_20210420T094144_026549_032B99_2058.SAFE/s1b-iw1-xsp-vv-20210420t094118-20210420t094144-026549-032b99-004_L1B_xspec_IFR_4.0.0.nc", + "/tmp/data/products/tests/iw/slc/l1b/2021/110/S1B_IW_XSP__1SDV_20210420T094117_20210420T094144_026549_032B99_2058_B03.SAFE/s1b-iw1-xsp-vv-20210420t094118-20210420t094144-026549-032b99-004_b03.nc", + "/tmp/data/products/tests/iw/slc/l1b/2021/110/S1B_IW_XSP__1SDV_20210420T094117_20210420T094144_026549_032B99_2058_B03.SAFE/l1c-s1b-iw1-xsp-vv-20210420t094118-20210420t094144-026549-032b99-004_b03.nc", +] +expected_l2 = [ + '/tmp/2021/110/S1B_IW_WSP__1SDV_20210420T094117_20210420T094144_026549_032B99_2058_C02.SAFE/l2-s1b-iw1-wsp-dv-20210420t094118-20210420t094144-026549-032b99-c02.nc' +] + + +@pytest.mark.parametrize( + ["l1c_fullpath", "expected_l2"], + ( + pytest.param(inputs_l1c[0], expected_l2[0]), + pytest.param(inputs_l1c[1], expected_l2[0]), + pytest.param(inputs_l1c[2], expected_l2[0]), + ), +) +def test_outputfile_path(l1c_fullpath, expected_l2): + version = "C02" + outputdir = "/tmp/" + actual_l2_path = get_l2_filepath( + l1c_fullpath, version=version, outputdir=outputdir) + + print(actual_l2_path) + assert actual_l2_path == expected_l2 From 7a3dcbbc45ac86be17a58e28e08f8c0d0d64cb4e Mon Sep 17 00:00:00 2001 From: agrouaze Date: Wed, 28 Feb 2024 15:47:55 +0100 Subject: [PATCH 2/5] add a unit test for output filenames --- tests/test_output_name.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/tests/test_output_name.py b/tests/test_output_name.py index d4d2ea1..372f4a9 100644 --- a/tests/test_output_name.py +++ b/tests/test_output_name.py @@ -1,29 +1,33 @@ import pytest -from slcwindinversion.utils import get_l2_filepath +from l2awinddirection.utils import get_l2_filepath -inputs_l1c = [ - "/tmp/data/products/tests/iw/slc/l1c/4.0.0/S1B_IW_XSP__1SDV_20210420T094117_20210420T094144_026549_032B99_2058.SAFE/s1b-iw1-xsp-vv-20210420t094118-20210420t094144-026549-032b99-004_L1B_xspec_IFR_4.0.0.nc", - "/tmp/data/products/tests/iw/slc/l1b/2021/110/S1B_IW_XSP__1SDV_20210420T094117_20210420T094144_026549_032B99_2058_B03.SAFE/s1b-iw1-xsp-vv-20210420t094118-20210420t094144-026549-032b99-004_b03.nc", - "/tmp/data/products/tests/iw/slc/l1b/2021/110/S1B_IW_XSP__1SDV_20210420T094117_20210420T094144_026549_032B99_2058_B03.SAFE/l1c-s1b-iw1-xsp-vv-20210420t094118-20210420t094144-026549-032b99-004_b03.nc", +inputs_tiles_files = [ + "/tests/iw/slc/l2a/winddirection/3.7.6/tiles/S1A_IW_XSP__1SDV_20231128T035702_20231128T035729_051412_063451_3781.SAFE/tiles_s1a-iw2-slc-vv-20231128t035702-20231128t035727-051412-063451-002_L1B_xspec_IFR_3.7.6_wind.nc", + "/tests/iw/slc/l2a/winddirection/3.7.6/tiles/S1A_IW_XSP__1SDV_20231128T035702_20231128T035729_051412_063451_3781.SAFE/s1a-iw2-slc-vv-20231128t035702-20231128t035727-051412-063451-002_L1B_xspec_IFR_3.7.6_wind.nc", + "/tests/iw/slc/l2a/winddirection/3.7.6/tiles/S1A_IW_XSP__1SDV_20231128T035702_20231128T035729_051412_063451_3781.SAFE/l1b-s1a-iw2-slc-vv-20231128t035702-20231128t035727-051412-063451-002-c02.nc", + "/tests/iw/slc/l2a/winddirection/3.7.6/tiles/S1A_IW_XSP__1SDV_20231128T035702_20231128T035729_051412_063451_3781.SAFE/s1a-iw2-slc-vv-20231128t035702-20231128t035727-051412-063451-002_L1B_xspec_IFR_3.8-wind.nc", + # "/tmp/data/products/tests/iw/slc/l1b/2021/110/S1B_IW_XSP__1SDV_20210420T094117_20210420T094144_026549_032B99_2058_B03.SAFE/s1b-iw1-xsp-vv-20210420t094118-20210420t094144-026549-032b99-004_b03.nc", + # "/tmp/data/products/tests/iw/slc/l1b/2021/110/S1B_IW_XSP__1SDV_20210420T094117_20210420T094144_026549_032B99_2058_B03.SAFE/l1c-s1b-iw1-xsp-vv-20210420t094118-20210420t094144-026549-032b99-004_b03.nc", ] expected_l2 = [ - '/tmp/2021/110/S1B_IW_WSP__1SDV_20210420T094117_20210420T094144_026549_032B99_2058_C02.SAFE/l2-s1b-iw1-wsp-dv-20210420t094118-20210420t094144-026549-032b99-c02.nc' + '/tmp/2023/332/S1A_IW_WDR__1SDV_20231128T035702_20231128T035729_051412_063451_3781_D02.SAFE/l2-s1a-iw2-wdr-vv-20231128t035702-20231128t035727-051412-063451-002-d02.nc' ] @pytest.mark.parametrize( - ["l1c_fullpath", "expected_l2"], + ["vignette_input", "expected_l2"], ( - pytest.param(inputs_l1c[0], expected_l2[0]), - pytest.param(inputs_l1c[1], expected_l2[0]), - pytest.param(inputs_l1c[2], expected_l2[0]), + pytest.param(inputs_tiles_files[0], expected_l2[0]), + pytest.param(inputs_tiles_files[1], expected_l2[0]), + pytest.param(inputs_tiles_files[2], expected_l2[0]), + pytest.param(inputs_tiles_files[3], expected_l2[0]), ), ) -def test_outputfile_path(l1c_fullpath, expected_l2): - version = "C02" +def test_outputfile_path(vignette_input, expected_l2): + version = "D02" outputdir = "/tmp/" actual_l2_path = get_l2_filepath( - l1c_fullpath, version=version, outputdir=outputdir) + vignette_input, version=version, outputdir=outputdir) print(actual_l2_path) assert actual_l2_path == expected_l2 From 3a34d3aa96c1b04340892d9d7ea2747393240cbf Mon Sep 17 00:00:00 2001 From: agrouaze Date: Wed, 28 Feb 2024 15:48:20 +0100 Subject: [PATCH 3/5] add a method to get the output file path --- l2awinddirection/utils.py | 71 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/l2awinddirection/utils.py b/l2awinddirection/utils.py index 816b15a..0143ba7 100644 --- a/l2awinddirection/utils.py +++ b/l2awinddirection/utils.py @@ -4,7 +4,7 @@ import logging import os from yaml import CLoader as Loader - +import datetime def get_conf(): """ @@ -22,3 +22,72 @@ def get_conf(): stream = open(config_path, 'r') conf = load(stream, Loader=Loader) return conf + + +def get_l2_filepath(vignette_fullpath, version, outputdir)->str: + """ + + Args: + vignette_fullpath: str .nc full path of the sigma0 extracted on the level-1b tiles + version : str (eg. 1.2 or D01) + outputdir: str directory where the l2 wdr will be stored + Returns: + l2_full_path: str l2 wdr path + """ + safe_file = os.path.basename(os.path.dirname(vignette_fullpath)) + pathout_root = outputdir + + safe_start_date = datetime.datetime.strptime(safe_file.split('_')[5],'%Y%m%dT%H%M%S') + pathout = os.path.join(pathout_root, safe_start_date.strftime('%Y'),safe_start_date.strftime('%j')) + + # SAFE part + safe_file = safe_file.replace("L1B", "L2").replace('XSP', 'WDR') + if ( + len(safe_file.split("_")) == 10 + ): # classical ESA SLC naming #:TODO once xsarslc will be updated this case could be removed + safe_file = safe_file.replace(".SAFE", "_" + version.upper() + ".SAFE") + else: # there is already a product ID in the L1B SAFE name + lastpart = safe_file.split("_")[-1] + safe_file = safe_file.replace(lastpart, version.upper() + ".SAFE") + # if '1SDV' in safe_file: + # pola_str = 'dv' + # elif '1SDH' in safe_file: + # pola_str = 'dh' + # elif '1SSV' in safe_file: + # pola_str = 'sv' + # elif '1SSH' in safe_file: + # pola_str = 'sh' + # else: + # raise Exception('safe file polarization is not defined as expected') + + # measurement part + base_measu = os.path.basename(vignette_fullpath) + + + # lastpiece = base_measu.split("-")[-1] + if 'IFR_' in base_measu: # there is the old string _L1B_xspec_IFR_3.7.6_wind.nc + start_IFR = base_measu.index('_L1B_xspec_IFR') + piece_to_remove = base_measu[start_IFR:] + base_measu = base_measu.replace(piece_to_remove,'.nc') + if base_measu.split('-')[-1][0] == 'c': #there is already a version product ID + base_measu = base_measu.replace(base_measu.split('-')[-1],version.lower() + ".nc") + else: + base_measu = base_measu.replace('.nc','-'+version.lower() + ".nc") + + # handle the begining of the measurement basename + if base_measu[0:3]=='l1b': + base_measu = base_measu.replace('l1b-','l2-') + elif base_measu[0:3]=='til': + base_measu = base_measu.replace('tiles_', 'l2-') + else: + base_measu = 'l2-'+base_measu + # base_measu = base_measu.replace(base_measu.split('-')[4], pola_str) # replace -vv- by -dv- or -sv- depending on SAFE information + base_measu = base_measu.replace('-xsp-','-wdr-') + base_measu = base_measu.replace('-slc-', '-wdr-') + + + l2_full_path = os.path.join(pathout, safe_file,base_measu) + logging.debug("File out: %s ", l2_full_path) + if not os.path.exists(os.path.dirname(l2_full_path)): + os.makedirs(os.path.dirname(l2_full_path), 0o0775) + return l2_full_path From 169b8ffc4343b58dc2890d8813c6486495582f35 Mon Sep 17 00:00:00 2001 From: agrouaze Date: Wed, 28 Feb 2024 15:50:31 +0100 Subject: [PATCH 4/5] used method to get output path, fix moves with workspace and clean lines to remove tf logs --- l2awinddirection/mainl2awinddirection.py | 94 ++++++++++-------------- 1 file changed, 38 insertions(+), 56 deletions(-) diff --git a/l2awinddirection/mainl2awinddirection.py b/l2awinddirection/mainl2awinddirection.py index bf39cc9..e60134d 100644 --- a/l2awinddirection/mainl2awinddirection.py +++ b/l2awinddirection/mainl2awinddirection.py @@ -6,6 +6,7 @@ """ import glob import os +os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' import pdb import shutil import xarray as xr @@ -13,28 +14,6 @@ import logging import time import numpy as np -import warnings - -warnings.filterwarnings("ignore") -# logging.getLogger("tensorflow").setLevel(logging.FATAL) -import tensorflow as tf - -tf.get_logger().setLevel("ERROR") -logging.getLogger("tensorflow").setLevel(logging.ERROR) -# tf.debugging.set_log_device_placement(True) -# logging.getLogger("tensorflow").setLevel(logging.WARNING) -# tf.logging.set_verbosity(tf.logging.ERROR) -# tf.keras.utils.disable_interactive_logging() -# import keras -# from keras.backend.tensorflow_backend import tf - -# tf.keras.utils.disable_interactive_logging() -# import absl.logging -# logging.root.removeHandler(absl.logging._absl_handler) -# absl.logging._warn_preinit_stderr = False - -# logger = tf.get_logger() -# logger.setLevel(logging.FATAL) import l2awinddirection @@ -49,10 +28,9 @@ from l2awinddirection.generate_L2A_winddir_regression_product import ( generate_wind_product, ) -from l2awinddirection.utils import get_conf +from l2awinddirection.utils import get_conf,get_l2_filepath from l2awinddirection.M64RN4 import M64RN4_distribution, M64RN4_regression -os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" # disable logging tensorflow conf = get_conf() @@ -98,7 +76,7 @@ def main(): required=True, help="Level-2A wind direction SAFE where are stored the extracted DN tiles", ) - # for now the output files will be in the same directory than the input SAFE + # for now the output files will be in the same directory as the input SAFE parser.add_argument( "--outputdir", required=True, @@ -121,16 +99,21 @@ def main(): ) parser.add_argument( "--skipmoves", - help="skip copy of the tiles in a workspace and move of the L2A winddir files from workspace to outputdir [default=False]", + help="True-> skip copy of the tiles in a workspace and move of the L2A winddir files from workspace to outputdir [default=False -> a workspace is used]", action="store_true", default=False, ) + parser.add_argument( + "--version", + help="product ID of the level-2 wdr to be generated (eg: D03)", + required=True, + ) parser.add_argument( "--remove-tiles", - help="remove the tiles files in the workspace directory [default=True]", + help="remove the tiles files in the workspace directory [default=False -> tiles kept in workspace]", required=False, - default=True, - action="store_false", + default=False, + action="store_true", ) args = parser.parse_args() fmt = "%(asctime)s %(levelname)s %(filename)s(%(lineno)d) %(message)s" @@ -166,9 +149,6 @@ def main(): model_m64rn4.model.load_weights(path_model_pdf) elif args.mode == "regression": model_m64rn4 = [] - # path_best_models = glob.glob( - # ".../analysis/s1_data_analysis/project_rmarquar/wsat/trained_models/iw/*.hdf5" - # ) path_best_models = glob.glob(os.path.join(dirmodelsreg, "*.hdf5")) assert len(path_best_models) > 1 for path in path_best_models: @@ -181,34 +161,36 @@ def main(): model_m64rn4.append(m64rn4_reg) else: raise Exception("not handled case") - # files = glob.glob("/raid/localscratch/agrouaze/tiles_iw_4_wdir/3.1/*SAFE/*.nc") + if args.l2awindirtilessafe.endswith("/"): l2awindirtilessafe = args.l2awindirtilessafe.rstrip("/") else: l2awindirtilessafe = args.l2awindirtilessafe if args.skipmoves: - workspace = os.path.dirname(l2awindirtilessafe) + workspace_input = os.path.dirname(l2awindirtilessafe) logging.info( "workspace is the place where the tiles are already stored: %s (no copies)", - workspace, + workspace_input, ) + workspace_output = args.outputdir else: if args.workspace is None: - workspace = conf["workspace_prediction"] + workspace_input = conf["workspace_prediction"] else: - workspace = args.workspace + workspace_input = args.workspace logging.info( "workspace where the tiles will be temporarily moved is: %s", - workspace, + workspace_input, ) - safefile = os.path.join(workspace, os.path.basename(l2awindirtilessafe)) - if not os.path.exists(safefile) and args.skipmoves is True: - logging.info(" step 1: move %s -> %s", l2awindirtilessafe, safefile) - shutil.copytree(l2awindirtilessafe, safefile) + workspace_output = workspace_input + input_safe_full_path = os.path.join(workspace_input, os.path.basename(l2awindirtilessafe)) + if not os.path.exists(input_safe_full_path) and args.skipmoves is True: + logging.info(" step 1: move %s -> %s", l2awindirtilessafe, input_safe_full_path) + shutil.copytree(l2awindirtilessafe, input_safe_full_path) polarization_usable = "vv" files = sorted( - glob.glob(os.path.join(safefile, "*" + polarization_usable + "*.nc")) + glob.glob(os.path.join(input_safe_full_path, "*" + polarization_usable + "*.nc")) ) logging.info("Number of files to process: %d" % len(files)) if not os.path.exists(args.outputdir): @@ -227,20 +209,15 @@ def main(): res = generate_wind_distribution_product( tiles, model_m64rn4, nb_classes=36, shape=(44, 44, 1) ) - outputfile = file.replace( - "_wind.nc", "_winddirection_pdf.nc" - ) # TODO: change to _tiles.nc -> _winddirection.nc + outputfile = get_l2_filepath(vignette_fullpath=file, version=args.version, outputdir=workspace_output) if args.skipmoves: - outputfile = outputfile.replace(workspace, args.outputdir) + # outputfile = outputfile.replace(workspace_output, args.outputdir) if not os.path.exists(os.path.dirname(outputfile)): os.makedirs(os.path.dirname(outputfile)) elif args.mode == "regression": - - outputfile = file.replace( - "_wind.nc", "_winddirection_regression.nc" - ) # TODO: change to _tiles.nc -> _winddirection.nc + outputfile = get_l2_filepath(vignette_fullpath=file, version=args.version, outputdir=workspace_output) if args.skipmoves: - outputfile = outputfile.replace(workspace, args.outputdir) + # outputfile = outputfile.replace(workspace_output, args.outputdir) if not os.path.exists(os.path.dirname(outputfile)): os.makedirs(os.path.dirname(outputfile)) @@ -262,10 +239,15 @@ def main(): if args.remove_tiles and args.skipmoves is False: logging.info("remove temporary tiles file in the workspace: %s", file) os.remove(file) - final_safe_path = os.path.join(args.outputdir, os.path.basename(l2awindirtilessafe)) - if args.skipmoves is False: - logging.info("step 3: move %s -> %s", safefile, final_safe_path) - shutil.move(safefile, final_safe_path) + + # final_safe_path = os.path.dirname(outputfile) + if args.skipmoves is False: # case for which a workspace has been used + # final_safe_path = os.path.join(args.outputdir, os.path.basename(l2awindirtilessafe)) + final_safe_path = os.path.dirname(outputfile).replace(workspace_output,args.outputdir) + logging.info("step 3: move %s -> %s", os.path.dirname(outputfile), final_safe_path) + shutil.move(os.path.dirname(outputfile), final_safe_path) + else: + final_safe_path = os.path.dirname(outputfile) logging.info("Ifremer Level-2A wind direction SAFE path: %s", final_safe_path) logging.info("successful SAFE processing") logging.info("peak memory usage: %s ", get_memory_usage()) From d89e30894f4a23fd91027d12c658985074f61af6 Mon Sep 17 00:00:00 2001 From: agrouaze Date: Wed, 28 Feb 2024 15:57:17 +0100 Subject: [PATCH 5/5] update gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index b6e4761..f48455e 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,5 @@ dmypy.json # Pyre type checker .pyre/ + +localconfig.yaml