From f8d8e6d648b86b818ca3ef3167170c841d689384 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Wed, 19 Jul 2023 08:15:02 +0000 Subject: [PATCH 1/9] add estimation of the cleaning threshold with CatB pedestals --- lstchain/calib/camera/pixel_threshold_estimation.py | 1 + lstchain/io/io.py | 3 +++ lstchain/scripts/lstchain_dl1ab.py | 11 +++++++++++ 3 files changed, 15 insertions(+) diff --git a/lstchain/calib/camera/pixel_threshold_estimation.py b/lstchain/calib/camera/pixel_threshold_estimation.py index c8cc98ad79..df4334df38 100644 --- a/lstchain/calib/camera/pixel_threshold_estimation.py +++ b/lstchain/calib/camera/pixel_threshold_estimation.py @@ -75,6 +75,7 @@ def get_threshold_from_dl1_file(dl1_path, sigma_clean): # return pedestal interleaved threshold from data run for high gain return threshold_clean_pe[pedestal_id, HIGH_GAIN, :] + def get_unusable_pixels(dl1_path, pedestal_id): with tables.open_file(dl1_path) as f: calibration_id = f.root.dl1.event.telescope.monitoring.pedestal.col('calibration_id') diff --git a/lstchain/io/io.py b/lstchain/io/io.py index eb6c6abe99..054d93669e 100644 --- a/lstchain/io/io.py +++ b/lstchain/io/io.py @@ -77,6 +77,9 @@ dl1_params_tel_mon_ped_key = "/dl1/event/telescope/monitoring/pedestal" dl1_params_tel_mon_cal_key = "/dl1/event/telescope/monitoring/calibration" dl1_params_tel_mon_flat_key = "/dl1/event/telescope/monitoring/flatfield" +dl1_params_tel_mon_catB_ped_key = "/dl1/event/telescope/monitoring/CatB/pedestal" +dl1_params_tel_mon_catB_cal_key = "/dl1/event/telescope/monitoring/CatB/calibration" +dl1_params_tel_mon_catB_flat_key = "/dl1/event/telescope/monitoring/CatB/flatfield" dl1_params_lstcam_key = "/dl1/event/telescope/parameters/LST_LSTCam" dl1_images_lstcam_key = "/dl1/event/telescope/image/LST_LSTCam" dl2_params_lstcam_key = "/dl2/event/telescope/parameters/LST_LSTCam" diff --git a/lstchain/scripts/lstchain_dl1ab.py b/lstchain/scripts/lstchain_dl1ab.py index ebbe93d539..409c7d8cdf 100644 --- a/lstchain/scripts/lstchain_dl1ab.py +++ b/lstchain/scripts/lstchain_dl1ab.py @@ -126,7 +126,10 @@ def main(): catB_calib_time = np.array(catB_calib["time_min"]) catB_dc_to_pe = np.array(catB_calib["dc_to_pe"]) + catB_pedestal_per_sample = np.array(catB_calib["pedestal_per_sample"]) + catB_pedestal_mean = np.array(catB_pedestal["charge_mean"]) + catB_pedestal_std= np.array(catB_pedestal["charge_std"]) catB_time_correction = np.array(catB_calib["time_correction"]) catB_unusable_pixels = np.array(catB_calib["unusable_pixels"]) @@ -325,6 +328,14 @@ def main(): image_table['image'][ii] = image image_table['peak_time'][ii] = peak_time + # use CatB pedestals to estimate the picture threshold + if args.pedestal_cleaning: + ped_charge_mean_pe = catB_pedestal_mean[calib_idx][selected_gain, pixel_index] + ped_charge_std_pe = catB_pedestal_std[calib_idx][selected_gain, pixel_index] + threshold_clean_pe = ped_charge_mean_pe + sigma * ped_charge_std_pe + threshold_clean_pe[unusable_pixels] = pic_th + picture_th = np.clip(pedestal_thresh, pic_th, None) + if increase_nsb: # Add noise in pixels, to adjust MC to data noise levels. # TO BE DONE: in case of "pedestal cleaning" (not used now From 7cd69a99755032fda20ea6df6b19b7b2a05ce7cd Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Fri, 21 Jul 2023 11:58:42 +0000 Subject: [PATCH 2/9] correct calibration keys --- lstchain/io/io.py | 6 +++--- lstchain/scripts/lstchain_dl1ab.py | 9 ++++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/lstchain/io/io.py b/lstchain/io/io.py index 054d93669e..848ce262ff 100644 --- a/lstchain/io/io.py +++ b/lstchain/io/io.py @@ -77,9 +77,9 @@ dl1_params_tel_mon_ped_key = "/dl1/event/telescope/monitoring/pedestal" dl1_params_tel_mon_cal_key = "/dl1/event/telescope/monitoring/calibration" dl1_params_tel_mon_flat_key = "/dl1/event/telescope/monitoring/flatfield" -dl1_params_tel_mon_catB_ped_key = "/dl1/event/telescope/monitoring/CatB/pedestal" -dl1_params_tel_mon_catB_cal_key = "/dl1/event/telescope/monitoring/CatB/calibration" -dl1_params_tel_mon_catB_flat_key = "/dl1/event/telescope/monitoring/CatB/flatfield" +dl1_params_mon_tel_catB_ped_key = "/dl1/monitoring/telescope/CatB/pedestal" +dl1_params_mon_tel_catB_cal_key = "/dl1/monitoring/telescope/CatB/calibration" +dl1_params_mon_tel_catB_flat_key = "/dl1/monitoring/telescope/CatB/flatfield" dl1_params_lstcam_key = "/dl1/event/telescope/parameters/LST_LSTCam" dl1_images_lstcam_key = "/dl1/event/telescope/image/LST_LSTCam" dl2_params_lstcam_key = "/dl2/event/telescope/parameters/LST_LSTCam" diff --git a/lstchain/scripts/lstchain_dl1ab.py b/lstchain/scripts/lstchain_dl1ab.py index 409c7d8cdf..0f151284d1 100644 --- a/lstchain/scripts/lstchain_dl1ab.py +++ b/lstchain/scripts/lstchain_dl1ab.py @@ -44,6 +44,9 @@ dl1_params_lstcam_key, global_metadata, write_metadata, + dl1_params_mon_tel_catB_ped_key, + dl1_params_mon_tel_catB_ff_key, + dl1_params_mon_tel_catB_cal_key ) from lstchain.io.lstcontainers import DL1ParametersContainer from lstchain.reco.disp import disp @@ -442,9 +445,9 @@ def main(): # write a cat-B calibrations in DL1b if catB_calib: - write_table(catB_calib, outfile, "/dl1/event/telescope/monitoring/catB/calibration") - write_table(catB_pedestal, outfile, "/dl1/event/telescope/monitoring/catB/pedestal") - write_table(catB_flatfield, outfile, "/dl1/event/telescope/monitoring/catB/flatfield") + write_table(catB_calib, outfile, dl1_params_mon_tel_catB_cal_key) + write_table(catB_pedestal, outfile, dl1_params_mon_tel_catB_ped_key) + write_table(catB_flatfield, outfile,dl1_params_mon_tel_catB_flat_key) write_metadata(metadata, args.output_file) From 55f62d393d844e4a55afb29a971ca2b17de68bc1 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Wed, 26 Jul 2023 10:09:40 +0000 Subject: [PATCH 3/9] correct typo --- lstchain/scripts/lstchain_dl1ab.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lstchain/scripts/lstchain_dl1ab.py b/lstchain/scripts/lstchain_dl1ab.py index 0f151284d1..91acf012f8 100644 --- a/lstchain/scripts/lstchain_dl1ab.py +++ b/lstchain/scripts/lstchain_dl1ab.py @@ -45,7 +45,7 @@ global_metadata, write_metadata, dl1_params_mon_tel_catB_ped_key, - dl1_params_mon_tel_catB_ff_key, + dl1_params_mon_tel_catB_flat_key, dl1_params_mon_tel_catB_cal_key ) from lstchain.io.lstcontainers import DL1ParametersContainer From 6b8c3b7822fb03ad1516b26ee7465d261c06c06f Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Fri, 28 Jul 2023 08:50:14 +0000 Subject: [PATCH 4/9] correct value to clip --- lstchain/calib/camera/pixel_threshold_estimation.py | 1 - lstchain/scripts/lstchain_dl1ab.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lstchain/calib/camera/pixel_threshold_estimation.py b/lstchain/calib/camera/pixel_threshold_estimation.py index df4334df38..c8cc98ad79 100644 --- a/lstchain/calib/camera/pixel_threshold_estimation.py +++ b/lstchain/calib/camera/pixel_threshold_estimation.py @@ -75,7 +75,6 @@ def get_threshold_from_dl1_file(dl1_path, sigma_clean): # return pedestal interleaved threshold from data run for high gain return threshold_clean_pe[pedestal_id, HIGH_GAIN, :] - def get_unusable_pixels(dl1_path, pedestal_id): with tables.open_file(dl1_path) as f: calibration_id = f.root.dl1.event.telescope.monitoring.pedestal.col('calibration_id') diff --git a/lstchain/scripts/lstchain_dl1ab.py b/lstchain/scripts/lstchain_dl1ab.py index 91acf012f8..bcf8f1294c 100644 --- a/lstchain/scripts/lstchain_dl1ab.py +++ b/lstchain/scripts/lstchain_dl1ab.py @@ -337,7 +337,7 @@ def main(): ped_charge_std_pe = catB_pedestal_std[calib_idx][selected_gain, pixel_index] threshold_clean_pe = ped_charge_mean_pe + sigma * ped_charge_std_pe threshold_clean_pe[unusable_pixels] = pic_th - picture_th = np.clip(pedestal_thresh, pic_th, None) + picture_th = np.clip(threshold_clean_pe, pic_th, None) if increase_nsb: # Add noise in pixels, to adjust MC to data noise levels. From 08901cc4cb3eab077f8121f8a60db6d6b24f80ae Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Fri, 28 Jul 2023 12:31:12 +0000 Subject: [PATCH 5/9] optimized threshold calculation --- lstchain/scripts/lstchain_dl1ab.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/lstchain/scripts/lstchain_dl1ab.py b/lstchain/scripts/lstchain_dl1ab.py index bcf8f1294c..38d9d3c584 100644 --- a/lstchain/scripts/lstchain_dl1ab.py +++ b/lstchain/scripts/lstchain_dl1ab.py @@ -131,8 +131,6 @@ def main(): catB_dc_to_pe = np.array(catB_calib["dc_to_pe"]) catB_pedestal_per_sample = np.array(catB_calib["pedestal_per_sample"]) - catB_pedestal_mean = np.array(catB_pedestal["charge_mean"]) - catB_pedestal_std= np.array(catB_pedestal["charge_std"]) catB_time_correction = np.array(catB_calib["time_correction"]) catB_unusable_pixels = np.array(catB_calib["unusable_pixels"]) @@ -174,11 +172,18 @@ def main(): pedestal_thresh = get_threshold_from_dl1_file(args.input_file, sigma) cleaning_params = get_cleaning_parameters(config, clean_method_name) pic_th, boundary_th, isolated_pixels, min_n_neighbors = cleaning_params - log.info(f"Fraction of pixel cleaning thresholds above picture thr.:" + log.info(f"Fraction of Cat_A pixel cleaning thresholds above Cat_A picture thr.:" f"{np.sum(pedestal_thresh > pic_th) / len(pedestal_thresh):.3f}") picture_th = np.clip(pedestal_thresh, pic_th, None) log.info(f"Tailcut clean with pedestal threshold config used:" f"{config['tailcuts_clean_with_pedestal_threshold']}") + + if args.catB_calibration_file is not None: + catB_pedestal_mean = np.array(catB_pedestal["charge_mean"]) + catB_pedestal_std= np.array(catB_pedestal["charge_std"]) + catB_threshold_clean_pe = catB_pedestal_mean + sigma * catB_pedestal_std + + else: clean_method_name = 'tailcut' cleaning_params = get_cleaning_parameters(config, clean_method_name) @@ -333,9 +338,7 @@ def main(): # use CatB pedestals to estimate the picture threshold if args.pedestal_cleaning: - ped_charge_mean_pe = catB_pedestal_mean[calib_idx][selected_gain, pixel_index] - ped_charge_std_pe = catB_pedestal_std[calib_idx][selected_gain, pixel_index] - threshold_clean_pe = ped_charge_mean_pe + sigma * ped_charge_std_pe + threshold_clean_pe = catB_threshold_clean_pe[calib_idx][selected_gain, pixel_index] threshold_clean_pe[unusable_pixels] = pic_th picture_th = np.clip(threshold_clean_pe, pic_th, None) From da57710c057c6d872e87d5b093c2fc62269c1ecc Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Fri, 3 Nov 2023 08:48:28 +0000 Subject: [PATCH 6/9] change Cat-B keys name --- lstchain/io/io.py | 6 +++--- lstchain/scripts/lstchain_dl1ab.py | 15 ++++++++------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/lstchain/io/io.py b/lstchain/io/io.py index 848ce262ff..9616c0ab0d 100644 --- a/lstchain/io/io.py +++ b/lstchain/io/io.py @@ -77,9 +77,9 @@ dl1_params_tel_mon_ped_key = "/dl1/event/telescope/monitoring/pedestal" dl1_params_tel_mon_cal_key = "/dl1/event/telescope/monitoring/calibration" dl1_params_tel_mon_flat_key = "/dl1/event/telescope/monitoring/flatfield" -dl1_params_mon_tel_catB_ped_key = "/dl1/monitoring/telescope/CatB/pedestal" -dl1_params_mon_tel_catB_cal_key = "/dl1/monitoring/telescope/CatB/calibration" -dl1_params_mon_tel_catB_flat_key = "/dl1/monitoring/telescope/CatB/flatfield" +dl1_mon_tel_catB_ped_key = "/dl1/monitoring/telescope/CatB/pedestal" +dl1_mon_tel_catB_cal_key = "/dl1/monitoring/telescope/CatB/calibration" +dl1_mon_tel_catB_flat_key = "/dl1/monitoring/telescope/CatB/flatfield" dl1_params_lstcam_key = "/dl1/event/telescope/parameters/LST_LSTCam" dl1_images_lstcam_key = "/dl1/event/telescope/image/LST_LSTCam" dl2_params_lstcam_key = "/dl2/event/telescope/parameters/LST_LSTCam" diff --git a/lstchain/scripts/lstchain_dl1ab.py b/lstchain/scripts/lstchain_dl1ab.py index 38d9d3c584..b3da0d6657 100644 --- a/lstchain/scripts/lstchain_dl1ab.py +++ b/lstchain/scripts/lstchain_dl1ab.py @@ -44,9 +44,9 @@ dl1_params_lstcam_key, global_metadata, write_metadata, - dl1_params_mon_tel_catB_ped_key, - dl1_params_mon_tel_catB_flat_key, - dl1_params_mon_tel_catB_cal_key + dl1_mon_tel_catB_ped_key, + dl1_mon_tel_catB_flat_key, + dl1_mon_tel_catB_cal_key ) from lstchain.io.lstcontainers import DL1ParametersContainer from lstchain.reco.disp import disp @@ -336,7 +336,8 @@ def main(): image_table['image'][ii] = image image_table['peak_time'][ii] = peak_time - # use CatB pedestals to estimate the picture threshold + # use CatB pedestals to estimate the picture threshold + # as defined in the config file if args.pedestal_cleaning: threshold_clean_pe = catB_threshold_clean_pe[calib_idx][selected_gain, pixel_index] threshold_clean_pe[unusable_pixels] = pic_th @@ -448,9 +449,9 @@ def main(): # write a cat-B calibrations in DL1b if catB_calib: - write_table(catB_calib, outfile, dl1_params_mon_tel_catB_cal_key) - write_table(catB_pedestal, outfile, dl1_params_mon_tel_catB_ped_key) - write_table(catB_flatfield, outfile,dl1_params_mon_tel_catB_flat_key) + write_table(catB_calib, outfile, dl1_mon_tel_catB_cal_key) + write_table(catB_pedestal, outfile, dl1_mon_tel_catB_ped_key) + write_table(catB_flatfield, outfile,dl1_mon_tel_catB_flat_key) write_metadata(metadata, args.output_file) From a7601158f297c6159edd98d8bb17ee129c1dbf2b Mon Sep 17 00:00:00 2001 From: Ruben Lopez Coto Date: Thu, 30 Nov 2023 12:40:35 +0100 Subject: [PATCH 7/9] lowercase c --- lstchain/io/io.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lstchain/io/io.py b/lstchain/io/io.py index 9d0174c559..6bc6818a12 100644 --- a/lstchain/io/io.py +++ b/lstchain/io/io.py @@ -82,9 +82,9 @@ dl1_params_tel_mon_ped_key = "/dl1/event/telescope/monitoring/pedestal" dl1_params_tel_mon_cal_key = "/dl1/event/telescope/monitoring/calibration" dl1_params_tel_mon_flat_key = "/dl1/event/telescope/monitoring/flatfield" -dl1_mon_tel_CatB_ped_key = "/dl1/monitoring/telescope/catB/pedestal" -dl1_mon_tel_CatB_cal_key = "/dl1/monitoring/telescope/catB/calibration" -dl1_mon_tel_CatB_flat_key = "/dl1/monitoring/telescope/catB/flatfield" +dl1_mon_tel_catB_ped_key = "/dl1/monitoring/telescope/catB/pedestal" +dl1_mon_tel_catB_cal_key = "/dl1/monitoring/telescope/catB/calibration" +dl1_mon_tel_catB_flat_key = "/dl1/monitoring/telescope/catB/flatfield" dl1_params_lstcam_key = "/dl1/event/telescope/parameters/LST_LSTCam" dl1_images_lstcam_key = "/dl1/event/telescope/image/LST_LSTCam" dl2_params_lstcam_key = "/dl2/event/telescope/parameters/LST_LSTCam" From 10f845607e903e37a057039314dcc3f8d69edcd6 Mon Sep 17 00:00:00 2001 From: Ruben Lopez Coto Date: Thu, 30 Nov 2023 12:42:04 +0100 Subject: [PATCH 8/9] lowercase c --- lstchain/scripts/lstchain_dl1ab.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lstchain/scripts/lstchain_dl1ab.py b/lstchain/scripts/lstchain_dl1ab.py index 98eaaa2463..03f069f695 100644 --- a/lstchain/scripts/lstchain_dl1ab.py +++ b/lstchain/scripts/lstchain_dl1ab.py @@ -44,9 +44,6 @@ from lstchain.io.io import ( dl1_images_lstcam_key, dl1_params_lstcam_key, - dl1_mon_tel_CatB_cal_key, - dl1_mon_tel_CatB_ped_key, - dl1_mon_tel_CatB_flat_key, global_metadata, write_metadata, dl1_mon_tel_catB_ped_key, @@ -472,9 +469,9 @@ def main(): # write a cat-B calibrations in DL1b if catB_calib: - write_table(catB_calib, outfile, dl1_mon_tel_CatB_cal_key) - write_table(catB_pedestal, outfile, dl1_mon_tel_CatB_ped_key) - write_table(catB_flatfield, outfile, dl1_mon_tel_CatB_flat_key) + write_table(catB_calib, outfile, dl1_mon_tel_catB_cal_key) + write_table(catB_pedestal, outfile, dl1_mon_tel_catB_ped_key) + write_table(catB_flatfield, outfile, dl1_mon_tel_catB_flat_key) write_metadata(metadata, args.output_file) From 3512db443978b03e4525d56ab3c8af4a66604837 Mon Sep 17 00:00:00 2001 From: Ruben Lopez Coto Date: Thu, 30 Nov 2023 12:48:34 +0100 Subject: [PATCH 9/9] lowercase c --- lstchain/scripts/lstchain_merge_hdf5_files.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lstchain/scripts/lstchain_merge_hdf5_files.py b/lstchain/scripts/lstchain_merge_hdf5_files.py index 5fad0acba0..d53e49ab61 100644 --- a/lstchain/scripts/lstchain_merge_hdf5_files.py +++ b/lstchain/scripts/lstchain_merge_hdf5_files.py @@ -26,17 +26,17 @@ dl1_params_tel_mon_ped_key, dl1_params_tel_mon_cal_key, dl1_params_tel_mon_flat_key, - dl1_mon_tel_CatB_cal_key, - dl1_mon_tel_CatB_ped_key, - dl1_mon_tel_CatB_flat_key + dl1_mon_tel_catB_cal_key, + dl1_mon_tel_catB_ped_key, + dl1_mon_tel_catB_flat_key ) default_keys_to_copy = [dl1_params_tel_mon_ped_key, dl1_params_tel_mon_cal_key, dl1_params_tel_mon_flat_key, - dl1_mon_tel_CatB_cal_key, - dl1_mon_tel_CatB_ped_key, - dl1_mon_tel_CatB_flat_key + dl1_mon_tel_catB_cal_key, + dl1_mon_tel_catB_ped_key, + dl1_mon_tel_catB_flat_key ] parser = argparse.ArgumentParser(description='Merge HDF5 files')