From f3207b904f1c693e803c513fb2d54eea86049328 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Tue, 1 Feb 2022 13:43:37 +0100 Subject: [PATCH 1/4] Do not crash if not time-control file is found and take care of zero orbit numbering in modis files Signed-off-by: Adam.Dybbroe --- nwcsafpps_runner/utils.py | 54 ++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/nwcsafpps_runner/utils.py b/nwcsafpps_runner/utils.py index ab4b81c..56c2b9a 100644 --- a/nwcsafpps_runner/utils.py +++ b/nwcsafpps_runner/utils.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2018 - 2021 Pytroll Developers +# Copyright (c) 2018 - 2022 Pytroll Developers # Author(s): @@ -595,17 +595,25 @@ def create_xml_timestat_from_ascii(scene, pps_control_path): infiles = get_time_control_ascii_filename_candidates(pps_control_path, scene) LOG.info("Time control ascii file candidates: " + str(infiles)) - if len(infiles) == 1: - infile = str(infiles[0]) - LOG.info("Time control ascii file: " + str(infile)) - LOG.info("Read time control ascii file and generate XML") - ppstime_con = PPSTimeControl(infile) - ppstime_con.sum_up_processing_times() - try: - ppstime_con.write_xml() - except Exception as e: # TypeError as e: - LOG.warning('Not able to write time control xml file') - LOG.warning(e) + if len(infiles) == 0: + LOG.error("No time control ascii file candidate found!") + return [] + + if len(infiles) > 1: + msg = "More than one time control ascii file candidate found - unresolved ambiguity!" + LOG.error(msg) + return [] + + infile = str(infiles[0]) + LOG.info("Time control ascii file: " + str(infile)) + LOG.info("Read time control ascii file and generate XML") + ppstime_con = PPSTimeControl(infile) + ppstime_con.sum_up_processing_times() + try: + ppstime_con.write_xml() + except Exception as e: # TypeError as e: + LOG.warning('Not able to write time control xml file') + LOG.warning(e) # There should always be only one xml file for each ascii file found above! xmlfile = infile.replace('.txt', '.xml') @@ -634,15 +642,21 @@ def get_time_control_ascii_filename_candidates(pps_control_path, scene): st_times.append(atime.strftime("%Y%m%dT%H%M")) atime = atime + timedelta(seconds=60) + # PPSv2018 MODIS files have the orbit number set to "00000"! + norbit_candidates = [scene['orbit_number']] + if sensors in ['modis', ]: + norbit_candidates.append(0) + infiles = [] - for st_time in st_times: - txt_time_file = (os.path.join(pps_control_path, 'S_NWC_timectrl_') + - str(METOP_NAME_LETTER.get(platform_id, platform_id)) + - '_' + '%.5d' % scene['orbit_number'] + '_' + - st_time + - '*.txt') - LOG.info("glob string = " + str(txt_time_file)) - infiles = infiles + glob(txt_time_file) + for norbit in norbit_candidates: + for st_time in st_times: + txt_time_file = (os.path.join(pps_control_path, 'S_NWC_timectrl_') + + str(METOP_NAME_LETTER.get(platform_id, platform_id)) + + '_' + '%.5d' % norbit + '_' + + st_time + + '*.txt') + LOG.info("glob string = " + str(txt_time_file)) + infiles = infiles + glob(txt_time_file) return infiles From 17cd08d025036ea35b8ffa8a39f9505b95424379 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Wed, 2 Feb 2022 14:29:15 +0100 Subject: [PATCH 2/4] Add for static 99999 PPS orbit numbering used for VIIRS files Signed-off-by: Adam.Dybbroe --- nwcsafpps_runner/utils.py | 2 ++ setup.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/nwcsafpps_runner/utils.py b/nwcsafpps_runner/utils.py index 56c2b9a..7c024d6 100644 --- a/nwcsafpps_runner/utils.py +++ b/nwcsafpps_runner/utils.py @@ -646,6 +646,8 @@ def get_time_control_ascii_filename_candidates(pps_control_path, scene): norbit_candidates = [scene['orbit_number']] if sensors in ['modis', ]: norbit_candidates.append(0) + elif sensors in ['viirs', ]: + norbit_candidates.append(99999) infiles = [] for norbit in norbit_candidates: diff --git a/setup.py b/setup.py index 710deae..5b3411c 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2013 - 2021 Pytroll Community +# Copyright (c) 2013 - 2022 Pytroll Community # Author(s): From 31ebb11c9d765c00f3039762b15d26dc6260edd1 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Thu, 3 Feb 2022 09:11:55 +0100 Subject: [PATCH 3/4] Fix for VIIRS orbit numbers from PPS - they are often right, but sometimes off by 1/-1 Signed-off-by: Adam.Dybbroe --- nwcsafpps_runner/utils.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nwcsafpps_runner/utils.py b/nwcsafpps_runner/utils.py index 7c024d6..860250e 100644 --- a/nwcsafpps_runner/utils.py +++ b/nwcsafpps_runner/utils.py @@ -642,12 +642,14 @@ def get_time_control_ascii_filename_candidates(pps_control_path, scene): st_times.append(atime.strftime("%Y%m%dT%H%M")) atime = atime + timedelta(seconds=60) - # PPSv2018 MODIS files have the orbit number set to "00000"! + # For VIIRS we often see a orbit number difference of 1: norbit_candidates = [scene['orbit_number']] + for idx in [1, -1]: + norbit_candidates.append(int(scene['orbit_number']) + idx) + + # PPSv2018 MODIS files have the orbit number set to "00000"! if sensors in ['modis', ]: norbit_candidates.append(0) - elif sensors in ['viirs', ]: - norbit_candidates.append(99999) infiles = [] for norbit in norbit_candidates: From 25f9b8239b1d82bcde135d33a73555eb7308eb19 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Thu, 10 Feb 2022 14:01:31 +0100 Subject: [PATCH 4/4] Add more unit test coverage finding time controll ascii files Signed-off-by: Adam.Dybbroe --- nwcsafpps_runner/tests/test_utils.py | 97 +++++++++++++++++++++++++--- nwcsafpps_runner/utils.py | 38 ++++++----- 2 files changed, 112 insertions(+), 23 deletions(-) diff --git a/nwcsafpps_runner/tests/test_utils.py b/nwcsafpps_runner/tests/test_utils.py index 9b1e199..5575c4b 100644 --- a/nwcsafpps_runner/tests/test_utils.py +++ b/nwcsafpps_runner/tests/test_utils.py @@ -1,11 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -# Copyright (c) 2020, 2021 pps2018_runner developers +# Copyright (c) 2020 - 2022 pps2018_runner developers # -# Author(s): -# -# Erik Johansson # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,10 +21,13 @@ import os import tempfile import unittest +import pytest from datetime import datetime from nwcsafpps_runner.utils import get_outputfiles from nwcsafpps_runner.utils import get_time_control_ascii_filename_candidates +from nwcsafpps_runner.utils import get_time_control_ascii_filename +from nwcsafpps_runner.utils import FindTimeControlFileError from nwcsafpps_runner.utils import get_product_statistics_files @@ -104,16 +104,26 @@ class TestTimeControlFiles(unittest.TestCase): """Testing the handling and generation of time control XML files.""" def setUp(self): - self.testscene = {'platform_name': 'Metop-B', 'orbit_number': 46878, 'satday': - '20210930', 'sathour': '0946', + self.testscene = {'platform_name': 'Metop-B', 'orbit_number': 46878, + 'satday': '20210930', 'sathour': '0946', 'starttime': datetime(2021, 9, 30, 9, 46, 24), 'endtime': datetime(2021, 9, 30, 10, 1, 43), 'sensor': ['avhrr/3', 'mhs', 'amsu-a'], 'file4pps': '/data/metop01_20210930_0946_46878/hrpt_metop01_20210930_0946_46878.l1b'} self.filename1 = 'S_NWC_timectrl_metopb_46878_20210930T0946289Z_20210930T1001458Z.txt' self.filename2 = 'S_NWC_timectrl_metopb_46878_20210930T0949289Z_20210930T1001459Z.txt' + self.filename3 = 'S_NWC_timectrl_metopb_46878_20210930T0945289Z_20210930T1001459Z.txt' + self.filename_timeoff = 'S_NWC_timectrl_metopb_46878_20210930T0950000Z_20210930T1001459Z.txt' + + self.modis_scene = {'platform_name': 'EOS-Aqua', 'orbit_number': 5161, + 'satday': '20220209', 'sathour': '2210', + 'starttime': datetime(2022, 2, 9, 22, 10, 11), + 'endtime': datetime(2022, 2, 9, 22, 14, 41), + 'sensor': 'modis', + 'file4pps': '/data/eos/lvl1/MYD021km_A22040_221011_2022040221451.hdf'} + self.filename1_modis = 'S_NWC_timectrl_eos2_00000_20220209T2210110Z_20220209T2212284Z.txt' - def test_get_time_control_ascii_filename_candidates(self): + def test_get_time_control_ascii_filename_ok(self): with tempfile.TemporaryDirectory() as tmpdirname: @@ -124,12 +134,83 @@ def test_get_time_control_ascii_filename_candidates(self): with open(self.file2, 'w') as _: pass - ascii_files = get_time_control_ascii_filename_candidates(tmpdirname, self.testscene) + ascii_file = get_time_control_ascii_filename(self.testscene, tmpdirname) + + self.assertEqual(os.path.basename(ascii_file), + 'S_NWC_timectrl_metopb_46878_20210930T0946289Z_20210930T1001458Z.txt') + + def test_get_time_control_ascii_filename_more_than_one_file(self): + + with tempfile.TemporaryDirectory() as tmpdirname: + + self.file1 = os.path.join(tmpdirname, self.filename1) + with open(self.file1, 'w') as _: + pass + self.file2 = os.path.join(tmpdirname, self.filename3) + with open(self.file2, 'w') as _: + pass + + with pytest.raises(FindTimeControlFileError) as exec_info: + ascii_file = get_time_control_ascii_filename(self.testscene, tmpdirname) + + expected = 'More than one time control ascii file candidate found - unresolved ambiguity!' + assert str(exec_info.value) == expected + + def test_get_time_control_ascii_filename_no_files(self): + + with tempfile.TemporaryDirectory() as tmpdirname: + + self.file1 = os.path.join(tmpdirname, self.filename2) + with open(self.file1, 'w') as _: + pass + + with pytest.raises(FindTimeControlFileError) as exec_info: + ascii_file = get_time_control_ascii_filename(self.testscene, tmpdirname) + + expected = 'No time control ascii file candidate found!' + assert str(exec_info.value) == expected + + def test_get_time_control_ascii_filename_candidates_orbit_ok_time_off(self): + + with tempfile.TemporaryDirectory() as tmpdirname: + + self.file1 = os.path.join(tmpdirname, self.filename_timeoff) + with open(self.file1, 'w') as _: + pass + + ascii_files = get_time_control_ascii_filename_candidates(self.testscene, tmpdirname) + + self.assertEqual(len(ascii_files), 0) + + def test_get_time_control_ascii_filename_candidates_orbit_off_by1(self): + + with tempfile.TemporaryDirectory() as tmpdirname: + + self.file1 = os.path.join(tmpdirname, self.filename1) + with open(self.file1, 'w') as _: + pass + + self.testscene['orbit_number'] = self.testscene['orbit_number'] + 1 + ascii_files = get_time_control_ascii_filename_candidates(self.testscene, tmpdirname) self.assertEqual(len(ascii_files), 1) self.assertTrue(os.path.basename(ascii_files[0]) == 'S_NWC_timectrl_metopb_46878_20210930T0946289Z_20210930T1001458Z.txt') + def test_get_time_control_ascii_filename_candidates_modis_zero_orbit(self): + + with tempfile.TemporaryDirectory() as tmpdirname: + + self.file1 = os.path.join(tmpdirname, self.filename1_modis) + with open(self.file1, 'w') as _: + pass + + ascii_files = get_time_control_ascii_filename_candidates(self.modis_scene, tmpdirname) + + self.assertEqual(len(ascii_files), 1) + self.assertTrue(os.path.basename(ascii_files[0]) == + 'S_NWC_timectrl_eos2_00000_20220209T2210110Z_20220209T2212284Z.txt') + class TestProductStatisticsFiles(unittest.TestCase): """Testing locating the product statistics XML files.""" diff --git a/nwcsafpps_runner/utils.py b/nwcsafpps_runner/utils.py index 860250e..90852ee 100644 --- a/nwcsafpps_runner/utils.py +++ b/nwcsafpps_runner/utils.py @@ -48,6 +48,10 @@ class NwpPrepareError(Exception): pass +class FindTimeControlFileError(Exception): + pass + + PPS_OUT_PATTERN = ("S_NWC_{segment}_{orig_platform_name}_{orbit_number:05d}_" + "{start_time:%Y%m%dT%H%M%S%f}Z_{end_time:%Y%m%dT%H%M%S%f}Z.{extention}") PPS_OUT_PATTERN_MULTIPLE = ("S_NWC_{segment1}_{segment2}_{orig_platform_name}_{orbit_number:05d}_" + @@ -584,27 +588,19 @@ def get_xml_outputfiles(path, platform_name, orb, st_time=''): def create_xml_timestat_from_ascii(scene, pps_control_path): - """From ascii files with PPS time statistics create XML files and return a file list.""" - + """From ascii file(s) with PPS time statistics create XML file(s) and return a file list.""" try: from pps_time_control import PPSTimeControl except ImportError: LOG.warning("Failed to import the PPSTimeControl from pps") return [] - infiles = get_time_control_ascii_filename_candidates(pps_control_path, scene) - LOG.info("Time control ascii file candidates: " + str(infiles)) - - if len(infiles) == 0: - LOG.error("No time control ascii file candidate found!") - return [] - - if len(infiles) > 1: - msg = "More than one time control ascii file candidate found - unresolved ambiguity!" - LOG.error(msg) + try: + infile = get_time_control_ascii_filename(scene, pps_control_path) + except FindTimeControlFileError: + LOG.exception('No XML Time statistics file created!') return [] - infile = str(infiles[0]) LOG.info("Time control ascii file: " + str(infile)) LOG.info("Read time control ascii file and generate XML") ppstime_con = PPSTimeControl(infile) @@ -620,9 +616,21 @@ def create_xml_timestat_from_ascii(scene, pps_control_path): return filter4oldfiles([xmlfile], 90.) -def get_time_control_ascii_filename_candidates(pps_control_path, scene): - """From directory path, sensor and platform name get possible time-control filenames.""" +def get_time_control_ascii_filename(scene, pps_control_path): + """From the scene object and a file path get the time-control-ascii-filename (with path).""" + infiles = get_time_control_ascii_filename_candidates(scene, pps_control_path) + LOG.info("Time control ascii file candidates: " + str(infiles)) + if len(infiles) == 0: + raise FindTimeControlFileError("No time control ascii file candidate found!") + elif len(infiles) > 1: + msg = "More than one time control ascii file candidate found - unresolved ambiguity!" + raise FindTimeControlFileError(msg) + + return infiles[0] + +def get_time_control_ascii_filename_candidates(scene, pps_control_path): + """From directory path, sensor and platform name get possible time-control filenames.""" sensors = SENSOR_LIST.get(scene['platform_name'], scene['platform_name']) platform_id = SATELLITE_NAME.get(scene['platform_name'], scene['platform_name']) LOG.info("pps platform_id = %s", str(platform_id))