Merge pull request #16 from umr-lops/update_naming_convention

Update naming convention

.gitignore

@@ -127,3 +127,5 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+localconfig.yaml

l2awinddirection/mainl2awinddirection.py

@@ -6,35 +6,14 @@
 """
 import glob
 import os
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
 import pdb
 import shutil
 import xarray as xr
 from tqdm import tqdm
 import logging
 import time
 import numpy as np
-import warnings
-
-warnings.filterwarnings("ignore")
-# logging.getLogger("tensorflow").setLevel(logging.FATAL)
-import tensorflow as tf
-
-tf.get_logger().setLevel("ERROR")
-logging.getLogger("tensorflow").setLevel(logging.ERROR)
-# tf.debugging.set_log_device_placement(True)
-# logging.getLogger("tensorflow").setLevel(logging.WARNING)
-# tf.logging.set_verbosity(tf.logging.ERROR)
-# tf.keras.utils.disable_interactive_logging()
-# import keras
-# from keras.backend.tensorflow_backend import tf
-
-# tf.keras.utils.disable_interactive_logging()
-# import absl.logging
-# logging.root.removeHandler(absl.logging._absl_handler)
-# absl.logging._warn_preinit_stderr = False
-
-# logger = tf.get_logger()
-# logger.setLevel(logging.FATAL)
 
 import l2awinddirection
 
@@ -49,10 +28,9 @@
 from l2awinddirection.generate_L2A_winddir_regression_product import (
     generate_wind_product,
 )
-from l2awinddirection.utils import get_conf
+from l2awinddirection.utils import get_conf,get_l2_filepath
 from l2awinddirection.M64RN4 import M64RN4_distribution, M64RN4_regression
 
-os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"  # disable logging tensorflow
 conf = get_conf()
 
 
@@ -98,7 +76,7 @@ def main():
         required=True,
         help="Level-2A wind direction SAFE where are stored the extracted DN tiles",
     )
-    # for now the output files will be in the same directory than the input SAFE
+    # for now the output files will be in the same directory as the input SAFE
     parser.add_argument(
         "--outputdir",
         required=True,
@@ -121,16 +99,21 @@ def main():
     )
     parser.add_argument(
         "--skipmoves",
-        help="skip copy of the tiles in a workspace and move of the L2A winddir files from workspace to outputdir [default=False]",
+        help="True-> skip copy of the tiles in a workspace and move of the L2A winddir files from workspace to outputdir [default=False -> a workspace is used]",
         action="store_true",
         default=False,
     )
+    parser.add_argument(
+        "--version",
+        help="product ID of the level-2 wdr to be generated (eg: D03)",
+        required=True,
+    )
     parser.add_argument(
         "--remove-tiles",
-        help="remove the tiles files in the workspace directory [default=True]",
+        help="remove the tiles files in the workspace directory [default=False -> tiles kept in workspace]",
         required=False,
-        default=True,
-        action="store_false",
+        default=False,
+        action="store_true",
     )
     args = parser.parse_args()
     fmt = "%(asctime)s %(levelname)s %(filename)s(%(lineno)d) %(message)s"
@@ -166,9 +149,6 @@ def main():
         model_m64rn4.model.load_weights(path_model_pdf)
     elif args.mode == "regression":
         model_m64rn4 = []
-        # path_best_models = glob.glob(
-        #     ".../analysis/s1_data_analysis/project_rmarquar/wsat/trained_models/iw/*.hdf5"
-        # )
         path_best_models = glob.glob(os.path.join(dirmodelsreg, "*.hdf5"))
         assert len(path_best_models) > 1
         for path in path_best_models:
@@ -181,34 +161,36 @@ def main():
             model_m64rn4.append(m64rn4_reg)
     else:
         raise Exception("not handled case")
-    # files = glob.glob("/raid/localscratch/agrouaze/tiles_iw_4_wdir/3.1/*SAFE/*.nc")
+
     if args.l2awindirtilessafe.endswith("/"):
 
         l2awindirtilessafe = args.l2awindirtilessafe.rstrip("/")
     else:
         l2awindirtilessafe = args.l2awindirtilessafe
     if args.skipmoves:
-        workspace = os.path.dirname(l2awindirtilessafe)
+        workspace_input = os.path.dirname(l2awindirtilessafe)
         logging.info(
             "workspace is the place where the tiles are already stored: %s (no copies)",
-            workspace,
+            workspace_input,
         )
+        workspace_output = args.outputdir
     else:
         if args.workspace is None:
-            workspace = conf["workspace_prediction"]
+            workspace_input = conf["workspace_prediction"]
         else:
-            workspace = args.workspace
+            workspace_input = args.workspace
         logging.info(
             "workspace where the tiles will be temporarily moved is: %s",
-            workspace,
+            workspace_input,
         )
-    safefile = os.path.join(workspace, os.path.basename(l2awindirtilessafe))
-    if not os.path.exists(safefile) and args.skipmoves is True:
-        logging.info(" step 1: move %s -> %s", l2awindirtilessafe, safefile)
-        shutil.copytree(l2awindirtilessafe, safefile)
+        workspace_output = workspace_input
+    input_safe_full_path = os.path.join(workspace_input, os.path.basename(l2awindirtilessafe))
+    if not os.path.exists(input_safe_full_path) and args.skipmoves is True:
+        logging.info(" step 1: move %s -> %s", l2awindirtilessafe, input_safe_full_path)
+        shutil.copytree(l2awindirtilessafe, input_safe_full_path)
     polarization_usable = "vv"
     files = sorted(
-        glob.glob(os.path.join(safefile, "*" + polarization_usable + "*.nc"))
+        glob.glob(os.path.join(input_safe_full_path, "*" + polarization_usable + "*.nc"))
     )
     logging.info("Number of files to process: %d" % len(files))
     if not os.path.exists(args.outputdir):
@@ -227,20 +209,15 @@ def main():
             res = generate_wind_distribution_product(
                 tiles, model_m64rn4, nb_classes=36, shape=(44, 44, 1)
             )
-            outputfile = file.replace(
-                "_wind.nc", "_winddirection_pdf.nc"
-            )  # TODO: change to _tiles.nc -> _winddirection.nc
+            outputfile = get_l2_filepath(vignette_fullpath=file, version=args.version, outputdir=workspace_output)
             if args.skipmoves:
-                outputfile = outputfile.replace(workspace, args.outputdir)
+                # outputfile = outputfile.replace(workspace_output, args.outputdir)
                 if not os.path.exists(os.path.dirname(outputfile)):
                     os.makedirs(os.path.dirname(outputfile))
         elif args.mode == "regression":
-
-            outputfile = file.replace(
-                "_wind.nc", "_winddirection_regression.nc"
-            )  # TODO: change to _tiles.nc -> _winddirection.nc
+            outputfile = get_l2_filepath(vignette_fullpath=file, version=args.version, outputdir=workspace_output)
             if args.skipmoves:
-                outputfile = outputfile.replace(workspace, args.outputdir)
+                # outputfile = outputfile.replace(workspace_output, args.outputdir)
                 if not os.path.exists(os.path.dirname(outputfile)):
                     os.makedirs(os.path.dirname(outputfile))
 
@@ -262,10 +239,15 @@ def main():
         if args.remove_tiles and args.skipmoves is False:
             logging.info("remove temporary tiles file in the workspace: %s", file)
             os.remove(file)
-    final_safe_path = os.path.join(args.outputdir, os.path.basename(l2awindirtilessafe))
-    if args.skipmoves is False:
-        logging.info("step 3: move %s -> %s", safefile, final_safe_path)
-        shutil.move(safefile, final_safe_path)
+
+    # final_safe_path = os.path.dirname(outputfile)
+    if args.skipmoves is False: # case for which a workspace has been used
+        # final_safe_path = os.path.join(args.outputdir, os.path.basename(l2awindirtilessafe))
+        final_safe_path = os.path.dirname(outputfile).replace(workspace_output,args.outputdir)
+        logging.info("step 3: move %s -> %s", os.path.dirname(outputfile), final_safe_path)
+        shutil.move(os.path.dirname(outputfile), final_safe_path)
+    else:
+        final_safe_path = os.path.dirname(outputfile)
     logging.info("Ifremer Level-2A wind direction SAFE path: %s", final_safe_path)
     logging.info("successful SAFE processing")
     logging.info("peak memory usage: %s ", get_memory_usage())

l2awinddirection/utils.py

@@ -4,7 +4,7 @@
 import logging
 import os
 from yaml import CLoader as Loader
-
+import datetime
 def get_conf():
     """
 
@@ -22,3 +22,72 @@ def get_conf():
     stream = open(config_path, 'r')
     conf = load(stream, Loader=Loader)
     return conf
+
+
+def get_l2_filepath(vignette_fullpath, version, outputdir)->str:
+    """
+
+    Args:
+        vignette_fullpath: str .nc full path of the sigma0 extracted on the level-1b tiles
+        version : str (eg. 1.2 or D01)
+        outputdir: str directory where the l2 wdr will be stored
+    Returns:
+        l2_full_path: str l2 wdr path
+    """
+    safe_file = os.path.basename(os.path.dirname(vignette_fullpath))
+    pathout_root = outputdir
+
+    safe_start_date = datetime.datetime.strptime(safe_file.split('_')[5],'%Y%m%dT%H%M%S')
+    pathout = os.path.join(pathout_root, safe_start_date.strftime('%Y'),safe_start_date.strftime('%j'))
+
+    # SAFE part
+    safe_file = safe_file.replace("L1B", "L2").replace('XSP', 'WDR')
+    if (
+        len(safe_file.split("_")) == 10
+    ):  # classical ESA SLC naming #:TODO once xsarslc will be updated this case could be removed
+        safe_file = safe_file.replace(".SAFE", "_" + version.upper() + ".SAFE")
+    else:  # there is already a product ID in the L1B SAFE name
+        lastpart = safe_file.split("_")[-1]
+        safe_file = safe_file.replace(lastpart, version.upper() + ".SAFE")
+    # if '1SDV' in safe_file:
+    #     pola_str = 'dv'
+    # elif '1SDH' in safe_file:
+    #     pola_str = 'dh'
+    # elif '1SSV' in safe_file:
+    #     pola_str = 'sv'
+    # elif '1SSH' in safe_file:
+    #     pola_str = 'sh'
+    # else:
+    #     raise Exception('safe file polarization is not defined as expected')
+
+    # measurement part
+    base_measu = os.path.basename(vignette_fullpath)
+
+
+    # lastpiece = base_measu.split("-")[-1]
+    if 'IFR_' in base_measu: # there is the old string _L1B_xspec_IFR_3.7.6_wind.nc
+        start_IFR = base_measu.index('_L1B_xspec_IFR')
+        piece_to_remove = base_measu[start_IFR:]
+        base_measu = base_measu.replace(piece_to_remove,'.nc')
+    if base_measu.split('-')[-1][0] == 'c': #there is already a version product ID
+        base_measu = base_measu.replace(base_measu.split('-')[-1],version.lower() + ".nc")
+    else:
+        base_measu = base_measu.replace('.nc','-'+version.lower() + ".nc")
+
+    # handle the begining of the measurement basename
+    if base_measu[0:3]=='l1b':
+        base_measu = base_measu.replace('l1b-','l2-')
+    elif base_measu[0:3]=='til':
+        base_measu = base_measu.replace('tiles_', 'l2-')
+    else:
+        base_measu = 'l2-'+base_measu
+    # base_measu = base_measu.replace(base_measu.split('-')[4], pola_str) # replace -vv- by -dv- or -sv- depending on SAFE information
+    base_measu = base_measu.replace('-xsp-','-wdr-')
+    base_measu = base_measu.replace('-slc-', '-wdr-')
+
+
+    l2_full_path = os.path.join(pathout, safe_file,base_measu)
+    logging.debug("File out: %s ", l2_full_path)
+    if not os.path.exists(os.path.dirname(l2_full_path)):
+        os.makedirs(os.path.dirname(l2_full_path), 0o0775)
+    return l2_full_path

tests/test_output_name.py

@@ -0,0 +1,33 @@
+import pytest
+from l2awinddirection.utils import get_l2_filepath
+
+inputs_tiles_files = [
+    "/tests/iw/slc/l2a/winddirection/3.7.6/tiles/S1A_IW_XSP__1SDV_20231128T035702_20231128T035729_051412_063451_3781.SAFE/tiles_s1a-iw2-slc-vv-20231128t035702-20231128t035727-051412-063451-002_L1B_xspec_IFR_3.7.6_wind.nc",
+   "/tests/iw/slc/l2a/winddirection/3.7.6/tiles/S1A_IW_XSP__1SDV_20231128T035702_20231128T035729_051412_063451_3781.SAFE/s1a-iw2-slc-vv-20231128t035702-20231128t035727-051412-063451-002_L1B_xspec_IFR_3.7.6_wind.nc",
+   "/tests/iw/slc/l2a/winddirection/3.7.6/tiles/S1A_IW_XSP__1SDV_20231128T035702_20231128T035729_051412_063451_3781.SAFE/l1b-s1a-iw2-slc-vv-20231128t035702-20231128t035727-051412-063451-002-c02.nc",
+   "/tests/iw/slc/l2a/winddirection/3.7.6/tiles/S1A_IW_XSP__1SDV_20231128T035702_20231128T035729_051412_063451_3781.SAFE/s1a-iw2-slc-vv-20231128t035702-20231128t035727-051412-063451-002_L1B_xspec_IFR_3.8-wind.nc",
+    # "/tmp/data/products/tests/iw/slc/l1b/2021/110/S1B_IW_XSP__1SDV_20210420T094117_20210420T094144_026549_032B99_2058_B03.SAFE/s1b-iw1-xsp-vv-20210420t094118-20210420t094144-026549-032b99-004_b03.nc",
+    # "/tmp/data/products/tests/iw/slc/l1b/2021/110/S1B_IW_XSP__1SDV_20210420T094117_20210420T094144_026549_032B99_2058_B03.SAFE/l1c-s1b-iw1-xsp-vv-20210420t094118-20210420t094144-026549-032b99-004_b03.nc",
+]
+expected_l2 = [
+    '/tmp/2023/332/S1A_IW_WDR__1SDV_20231128T035702_20231128T035729_051412_063451_3781_D02.SAFE/l2-s1a-iw2-wdr-vv-20231128t035702-20231128t035727-051412-063451-002-d02.nc'
+]
+
+
+@pytest.mark.parametrize(
+    ["vignette_input", "expected_l2"],
+    (
+        pytest.param(inputs_tiles_files[0], expected_l2[0]),
+        pytest.param(inputs_tiles_files[1], expected_l2[0]),
+        pytest.param(inputs_tiles_files[2], expected_l2[0]),
+        pytest.param(inputs_tiles_files[3], expected_l2[0]),
+    ),
+)
+def test_outputfile_path(vignette_input, expected_l2):
+    version = "D02"
+    outputdir = "/tmp/"
+    actual_l2_path = get_l2_filepath(
+        vignette_input, version=version, outputdir=outputdir)
+
+    print(actual_l2_path)
+    assert actual_l2_path == expected_l2