From cccfcf02135f8700bada2479d50f98114a64448e Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Fri, 23 Oct 2020 14:49:36 +0200 Subject: [PATCH 1/7] Add mailmap --- .mailmap | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .mailmap diff --git a/.mailmap b/.mailmap new file mode 100644 index 00000000..77772174 --- /dev/null +++ b/.mailmap @@ -0,0 +1,9 @@ +Michele Peresano Michele Peresano +Michele Peresano Michele Peresano +Julien Lefaucheur +Thierry Stolarczyk +Gaia Verna gaia-verna <51763341+gaia-verna@users.noreply.github.com> +David Landriu +Karl Kosack +Thomas Vuillaume vuillaut +Alice Donini From 7b782f79e4e6abd04b1b26e896b3c8bf56ccb37f Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Fri, 23 Oct 2020 16:03:20 +0200 Subject: [PATCH 2/7] update release drafter config --- .github/release-drafter.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml index db42a343..7fb61733 100644 --- a/.github/release-drafter.yml +++ b/.github/release-drafter.yml @@ -13,6 +13,7 @@ categories: labels: - 'bug' - 'wrong behaviour' + - 'fix' - title: '🧰 Maintenance' labels: - 'maintenance' From 3325e70fa83414dbcb2fbd418ee74833ebffb0ff Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Fri, 23 Oct 2020 16:18:50 +0200 Subject: [PATCH 3/7] Add some nice icons --- .github/release-drafter.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml index 7fb61733..4716259a 100644 --- a/.github/release-drafter.yml +++ b/.github/release-drafter.yml @@ -2,11 +2,11 @@ categories: - title: '🚀 General features' labels: - 'enhancement' - - title: 'Pipeline applications' + - title: '🔭Pipeline applications' labels: - 'pipeline applications' - 'divergent pointing' - - title: 'GRID support' + - title: '🌐GRID support' labels: - 'grid' - title: '🐛 Bug Fixes' From 7ef5360dc8d835f8069c9f5212623ac391176069 Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Fri, 23 Oct 2020 17:58:07 +0200 Subject: [PATCH 4/7] Update mailmap --- .mailmap | 1 + 1 file changed, 1 insertion(+) diff --git a/.mailmap b/.mailmap index 77772174..7922639e 100644 --- a/.mailmap +++ b/.mailmap @@ -3,6 +3,7 @@ Michele Peresano Michele Peresano Thierry Stolarczyk Gaia Verna gaia-verna <51763341+gaia-verna@users.noreply.github.com> +Gaia Verna gaia-verna David Landriu Karl Kosack Thomas Vuillaume vuillaut From 2df9eb4a88dee33a1c608c2c30d4f122dae6da57 Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Fri, 23 Oct 2020 17:59:38 +0200 Subject: [PATCH 5/7] Small format changes --- .github/release-drafter.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml index 4716259a..3083a278 100644 --- a/.github/release-drafter.yml +++ b/.github/release-drafter.yml @@ -2,11 +2,11 @@ categories: - title: '🚀 General features' labels: - 'enhancement' - - title: '🔭Pipeline applications' + - title: '🔭 Pipeline applications' labels: - 'pipeline applications' - 'divergent pointing' - - title: '🌐GRID support' + - title: '🌐 GRID support' labels: - 'grid' - title: '🐛 Bug Fixes' From f9dcc0577d9cdb6bb03f3056f98a3693da5e1e08 Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Thu, 29 Oct 2020 20:14:37 +0100 Subject: [PATCH 6/7] Update config files --- .../aux/example_config_files/grid/grid.yaml | 85 +++++++++++-------- .../protopipe/analysis.yaml | 2 +- .../protopipe/classifier.yaml | 9 +- .../protopipe/performance.yaml | 7 +- .../protopipe/regressor.yaml | 7 +- 5 files changed, 60 insertions(+), 50 deletions(-) diff --git a/protopipe/aux/example_config_files/grid/grid.yaml b/protopipe/aux/example_config_files/grid/grid.yaml index 1f681852..6c8143d1 100644 --- a/protopipe/aux/example_config_files/grid/grid.yaml +++ b/protopipe/aux/example_config_files/grid/grid.yaml @@ -1,44 +1,52 @@ General: - # Path of protopipe configuration file (will be uploaded on the GRID) - # Please refer to directory structure shown at Lugano - config_path: $PATH_CONTAINING_ANALYSIS_YAML_CONFIG_FILE + # Path to the analysis configuration file + # You only need to edit the name of the analysis + config_path: '/home/vagrant/shared_folder/analyses/$ANALYSIS_NAME/configs/' # Name of the configuration file for the protopipe analysis config_file: 'analysis.yaml' - # Type of cleaning (wave - for "wavelets" - or tail - for 'tailcut' - ) + # Type of cleaning + # wave = "wavelets" (not available at the moment) + # tail = "tailcut" mode: 'tail' # Type of the particle to process (gamma, proton or electron) - NOT a list! particle: 'gamma' - # If 'True' estimate energy (need regressor file) - estimate_energy: True + # If 'True' estimate energy + # In this case a regressor file is needed! + estimate_energy: False - # Force tailcut cleaning for energy/score estimation + # Force reconstruction cleaning also for energy/score estimation # possible only if using 'wave' mode - if 'tail', leave 'False' - force_tailcut_for_extended_cleaning: False # only evt level + force_tailcut_for_extended_cleaning: False GRID: # Username on GRID - user_name: '$USERNAME' + user_name: '' # Home on GRID - home_grid: '/vo.cta.in2p3.fr/user/$INITIAL/$USERNAME/' + # 'X' = 'user_name' initial + # 'Y' = 'user_name' + home_grid: '/vo.cta.in2p3.fr/user/X/Y/' - # Output directories on the GRID home_grid/outdir - outdir: 'cta/ana/' + # Output directory from GRID's home + # i.e. "home_grid/outdir" + outdir: '' - # Directory for DL1 - dl1_dir_energy: 'dl1_energy' + # The following four directories will live inside 'outdir' - # Directory for DL1 - dl1_dir_discrimination: 'dl1_discrimination' + # Directory for training data used for energy estimation + training_dir_energy: 'data/TRAINING/for_energy_estimation' - # home_grid/outdir/models + # Directory for training data used for particle classification + training_dir_discrimination: 'data/TRAINING/for_particle_classification' + + # Directory for storage of the two estimators model_dir: 'estimators' - # Directory for DL2 - dl2_dir: 'dl2' + # Directory for DL2 data + dl2_dir: 'data/DL2' # Number of file per job n_file_per_job: 10 @@ -46,28 +54,33 @@ GRID: # Maximum number of jobs (-1 is no limit) n_jobs_max: -1 - # List of banned site + # List of banned sites + # you can edit this if + # - something doesn't work at a certain site (or storage) + # - if you want to filter jobs due to technical problems + # Some sites that host data are e.g. + # LCG.GRIF.fr, LCG.DESY-ZEUTHEN.de, LCG.IN2P3-CC.fr banned_sites: ['LCG.CPPM.fr'] -# The following DL0 paths refer to directory structure shown at Lugano -# you will find it together with the rest of the files - -# Better use absolute paths to be sure! +# The following directory paths refer to directory structure of your analysis +# Here you need to edit only +# - the name of the analysis +# - the names of the lists of simtel files (those shown here are descriptive) EnergyRegressor: - # This list is used to build an energy regressor, if output_type=DL1 and - # if estimate_energy is False - gamma_list: '../../data/DL0/gamma_energy.list' + # Events used to train an energy regressor + # Used when output_type = TRAINING and estimate_energy is False + gamma_list: '/home/vagrant/shared_folder/analyses/$ANALYSIS_NAME/data/simtel/gamma_energy.list' GammaHadronClassifier: - # Those lists are used to build a g/h classifier, if output_type=DL1 and - # if estimate_energy is True - gamma_list: '../../data/DL0/gamma_classification.list' - proton_list: '../../data/DL0/proton_classification.list' + # Events used to train a gamma-hadron classifier + # Used when output_type = TRAINING and estimate_energy is True + gamma_list: '/home/vagrant/shared_folder/analyses/$ANALYSIS_NAME/data/simtel/gamma_classification.list' + proton_list: '/home/vagrant/shared_folder/analyses/$ANALYSIS_NAME/data/simtel/proton_classification.list' Performance: - # Those lists are used to build event lists for performance estimation, - # if output_type=DL2 - gamma_list: '../../data/DL0/gamma_perf.list' - proton_list: '../../data/DL0/proton_perf.list' - electron_list: '../../data/DL0/electron_perf.list' + # Events processed directly up to DL2 for DL3 production + # Used when output_type = DL2 + gamma_list: '/home/vagrant/shared_folder/analyses/$ANALYSIS_NAME/data/simtel/gamma_perf.list' + proton_list: '/home/vagrant/shared_folder/analyses/$ANALYSIS_NAME/data/simtel/proton_perf.list' + electron_list: '/home/vagrant/shared_folder/analyses/$ANALYSIS_NAME/data/simtel/electron_perf.list' diff --git a/protopipe/aux/example_config_files/protopipe/analysis.yaml b/protopipe/aux/example_config_files/protopipe/analysis.yaml index aa9014da..77492344 100644 --- a/protopipe/aux/example_config_files/protopipe/analysis.yaml +++ b/protopipe/aux/example_config_files/protopipe/analysis.yaml @@ -9,7 +9,7 @@ General: # WARNING: for simulations containing multiple copies of the telescopes, # only 'full_array' or custom list are supported options! array: full_array - cam_id_list : ['LSTCam'] # List of camera IDs to be used + cam_id_list : ['LSTCam', 'NectarCam'] # List of camera IDs to be used # Cleaning for reconstruction ImageCleaning: diff --git a/protopipe/aux/example_config_files/protopipe/classifier.yaml b/protopipe/aux/example_config_files/protopipe/classifier.yaml index 9494397c..c39d2163 100644 --- a/protopipe/aux/example_config_files/protopipe/classifier.yaml +++ b/protopipe/aux/example_config_files/protopipe/classifier.yaml @@ -1,10 +1,9 @@ General: model_type: 'classifier' - # [...] = your analysis local path - # Please, refer to directory structure shown at Lugano - data_dir: '[...]/data/DL1/for_classification/' - data_sig_file: 'dl1_tail_gamma_merged.h5' - data_bkg_file: 'dl1_tail_proton_merged.h5' + # [...] = your analysis local full path OUTSIDE the Vagrant box + data_dir: '[...]/data/TRAINING/for_particle_classification/' + data_sig_file: 'TRAINING_classification_tail_gamma_merged.h5' + data_bkg_file: 'TRAINING_classification_tail_proton_merged.h5' cam_id_list: ['LSTCam', 'NectarCam'] table_name_template: 'feature_events_' # Will be completed with cam_ids outdir: '[...]/estimators/gamma_hadron_classifier' diff --git a/protopipe/aux/example_config_files/protopipe/performance.yaml b/protopipe/aux/example_config_files/protopipe/performance.yaml index 65aefd20..f5ffd84c 100644 --- a/protopipe/aux/example_config_files/protopipe/performance.yaml +++ b/protopipe/aux/example_config_files/protopipe/performance.yaml @@ -1,12 +1,11 @@ general: # Directory with input data file - # [...] = your analysis local path - # Please refer to directory structure shown at Lugano + # [...] = your analysis local full path OUTSIDE the Vagrant box indir: '[...]/data/DL2' # Template name for input file - template_input_file: 'dl2_{}_{}_merged.h5' # filled with mode and particle type + template_input_file: 'DL2_{}_{}_merged.h5' # filled with mode and particle type # Directory for output files - outdir: '[...]/performance' + outdir: '[...]/data/DL3' # Output table name output_table_name: 'table_best_cutoff' diff --git a/protopipe/aux/example_config_files/protopipe/regressor.yaml b/protopipe/aux/example_config_files/protopipe/regressor.yaml index d2340b0e..03768a11 100644 --- a/protopipe/aux/example_config_files/protopipe/regressor.yaml +++ b/protopipe/aux/example_config_files/protopipe/regressor.yaml @@ -1,9 +1,8 @@ General: model_type: 'regressor' - # [...] = your analysis local path - # Please, refer to directory structure shown at Lugano - data_dir: '[...]/data/DL1/for_energy_estimation' - data_file: 'dl1_{}_gamma_merged.h5' + # [...] = your analysis local full path OUTSIDE the Vagrant box + data_dir: '[...]/data/TRAINING/for_energy_estimation' + data_file: 'TRAINING_energy_{}_gamma_merged.h5' outdir: '[...]/estimators/energy_regressor' cam_id_list: ['LSTCam', 'NectarCam'] table_name_template: 'feature_events_' From baf4f961242ceb1544f0e79d4824bcf512703bfd Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Thu, 29 Oct 2020 20:32:37 +0100 Subject: [PATCH 7/7] Remove grid config file from protopipe; read from the interface --- .../aux/example_config_files/grid/grid.yaml | 86 ------------------- 1 file changed, 86 deletions(-) delete mode 100644 protopipe/aux/example_config_files/grid/grid.yaml diff --git a/protopipe/aux/example_config_files/grid/grid.yaml b/protopipe/aux/example_config_files/grid/grid.yaml deleted file mode 100644 index 6c8143d1..00000000 --- a/protopipe/aux/example_config_files/grid/grid.yaml +++ /dev/null @@ -1,86 +0,0 @@ -General: - # Path to the analysis configuration file - # You only need to edit the name of the analysis - config_path: '/home/vagrant/shared_folder/analyses/$ANALYSIS_NAME/configs/' - # Name of the configuration file for the protopipe analysis - config_file: 'analysis.yaml' - - # Type of cleaning - # wave = "wavelets" (not available at the moment) - # tail = "tailcut" - mode: 'tail' - - # Type of the particle to process (gamma, proton or electron) - NOT a list! - particle: 'gamma' - - # If 'True' estimate energy - # In this case a regressor file is needed! - estimate_energy: False - - # Force reconstruction cleaning also for energy/score estimation - # possible only if using 'wave' mode - if 'tail', leave 'False' - force_tailcut_for_extended_cleaning: False - -GRID: - # Username on GRID - user_name: '' - - # Home on GRID - # 'X' = 'user_name' initial - # 'Y' = 'user_name' - home_grid: '/vo.cta.in2p3.fr/user/X/Y/' - - # Output directory from GRID's home - # i.e. "home_grid/outdir" - outdir: '' - - # The following four directories will live inside 'outdir' - - # Directory for training data used for energy estimation - training_dir_energy: 'data/TRAINING/for_energy_estimation' - - # Directory for training data used for particle classification - training_dir_discrimination: 'data/TRAINING/for_particle_classification' - - # Directory for storage of the two estimators - model_dir: 'estimators' - - # Directory for DL2 data - dl2_dir: 'data/DL2' - - # Number of file per job - n_file_per_job: 10 - - # Maximum number of jobs (-1 is no limit) - n_jobs_max: -1 - - # List of banned sites - # you can edit this if - # - something doesn't work at a certain site (or storage) - # - if you want to filter jobs due to technical problems - # Some sites that host data are e.g. - # LCG.GRIF.fr, LCG.DESY-ZEUTHEN.de, LCG.IN2P3-CC.fr - banned_sites: ['LCG.CPPM.fr'] - -# The following directory paths refer to directory structure of your analysis -# Here you need to edit only -# - the name of the analysis -# - the names of the lists of simtel files (those shown here are descriptive) - -EnergyRegressor: - # Events used to train an energy regressor - # Used when output_type = TRAINING and estimate_energy is False - gamma_list: '/home/vagrant/shared_folder/analyses/$ANALYSIS_NAME/data/simtel/gamma_energy.list' - -GammaHadronClassifier: - # Events used to train a gamma-hadron classifier - # Used when output_type = TRAINING and estimate_energy is True - gamma_list: '/home/vagrant/shared_folder/analyses/$ANALYSIS_NAME/data/simtel/gamma_classification.list' - proton_list: '/home/vagrant/shared_folder/analyses/$ANALYSIS_NAME/data/simtel/proton_classification.list' - -Performance: - # Events processed directly up to DL2 for DL3 production - # Used when output_type = DL2 - gamma_list: '/home/vagrant/shared_folder/analyses/$ANALYSIS_NAME/data/simtel/gamma_perf.list' - proton_list: '/home/vagrant/shared_folder/analyses/$ANALYSIS_NAME/data/simtel/proton_perf.list' - electron_list: '/home/vagrant/shared_folder/analyses/$ANALYSIS_NAME/data/simtel/electron_perf.list'