Merge pull request #49 from adybbroe/bugfix-xml-timestat-from-ascii

Bugfix xml timestat from ascii

nwcsafpps_runner/tests/test_utils.py

@@ -1,11 +1,8 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
-# Copyright (c) 2020, 2021 pps2018_runner developers
+# Copyright (c) 2020 - 2022 pps2018_runner developers
 #
-# Author(s):
-#
-#   Erik Johansson <erik.johansson@smhi.se>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -24,10 +21,13 @@
 import os
 import tempfile
 import unittest
+import pytest
 from datetime import datetime
 
 from nwcsafpps_runner.utils import get_outputfiles
 from nwcsafpps_runner.utils import get_time_control_ascii_filename_candidates
+from nwcsafpps_runner.utils import get_time_control_ascii_filename
+from nwcsafpps_runner.utils import FindTimeControlFileError
 from nwcsafpps_runner.utils import get_product_statistics_files
 
 
@@ -104,16 +104,26 @@ class TestTimeControlFiles(unittest.TestCase):
     """Testing the handling and generation of time control XML files."""
 
     def setUp(self):
-        self.testscene = {'platform_name': 'Metop-B', 'orbit_number': 46878, 'satday':
-                          '20210930', 'sathour': '0946',
+        self.testscene = {'platform_name': 'Metop-B', 'orbit_number': 46878,
+                          'satday': '20210930', 'sathour': '0946',
                           'starttime': datetime(2021, 9, 30, 9, 46, 24),
                           'endtime': datetime(2021, 9, 30, 10, 1, 43),
                           'sensor': ['avhrr/3', 'mhs', 'amsu-a'],
                           'file4pps': '/data/metop01_20210930_0946_46878/hrpt_metop01_20210930_0946_46878.l1b'}
         self.filename1 = 'S_NWC_timectrl_metopb_46878_20210930T0946289Z_20210930T1001458Z.txt'
         self.filename2 = 'S_NWC_timectrl_metopb_46878_20210930T0949289Z_20210930T1001459Z.txt'
+        self.filename3 = 'S_NWC_timectrl_metopb_46878_20210930T0945289Z_20210930T1001459Z.txt'
+        self.filename_timeoff = 'S_NWC_timectrl_metopb_46878_20210930T0950000Z_20210930T1001459Z.txt'
+
+        self.modis_scene = {'platform_name': 'EOS-Aqua', 'orbit_number': 5161,
+                            'satday': '20220209', 'sathour': '2210',
+                            'starttime': datetime(2022, 2, 9, 22, 10, 11),
+                            'endtime': datetime(2022, 2, 9, 22, 14, 41),
+                            'sensor': 'modis',
+                            'file4pps': '/data/eos/lvl1/MYD021km_A22040_221011_2022040221451.hdf'}
+        self.filename1_modis = 'S_NWC_timectrl_eos2_00000_20220209T2210110Z_20220209T2212284Z.txt'
 
-    def test_get_time_control_ascii_filename_candidates(self):
+    def test_get_time_control_ascii_filename_ok(self):
 
         with tempfile.TemporaryDirectory() as tmpdirname:
 
@@ -124,12 +134,83 @@ def test_get_time_control_ascii_filename_candidates(self):
             with open(self.file2, 'w') as _:
                 pass
 
-            ascii_files = get_time_control_ascii_filename_candidates(tmpdirname, self.testscene)
+            ascii_file = get_time_control_ascii_filename(self.testscene, tmpdirname)
+
+        self.assertEqual(os.path.basename(ascii_file),
+                         'S_NWC_timectrl_metopb_46878_20210930T0946289Z_20210930T1001458Z.txt')
+
+    def test_get_time_control_ascii_filename_more_than_one_file(self):
+
+        with tempfile.TemporaryDirectory() as tmpdirname:
+
+            self.file1 = os.path.join(tmpdirname, self.filename1)
+            with open(self.file1, 'w') as _:
+                pass
+            self.file2 = os.path.join(tmpdirname, self.filename3)
+            with open(self.file2, 'w') as _:
+                pass
+
+            with pytest.raises(FindTimeControlFileError) as exec_info:
+                ascii_file = get_time_control_ascii_filename(self.testscene, tmpdirname)
+
+            expected = 'More than one time control ascii file candidate found - unresolved ambiguity!'
+            assert str(exec_info.value) == expected
+
+    def test_get_time_control_ascii_filename_no_files(self):
+
+        with tempfile.TemporaryDirectory() as tmpdirname:
+
+            self.file1 = os.path.join(tmpdirname, self.filename2)
+            with open(self.file1, 'w') as _:
+                pass
+
+            with pytest.raises(FindTimeControlFileError) as exec_info:
+                ascii_file = get_time_control_ascii_filename(self.testscene, tmpdirname)
+
+            expected = 'No time control ascii file candidate found!'
+            assert str(exec_info.value) == expected
+
+    def test_get_time_control_ascii_filename_candidates_orbit_ok_time_off(self):
+
+        with tempfile.TemporaryDirectory() as tmpdirname:
+
+            self.file1 = os.path.join(tmpdirname, self.filename_timeoff)
+            with open(self.file1, 'w') as _:
+                pass
+
+            ascii_files = get_time_control_ascii_filename_candidates(self.testscene, tmpdirname)
+
+        self.assertEqual(len(ascii_files), 0)
+
+    def test_get_time_control_ascii_filename_candidates_orbit_off_by1(self):
+
+        with tempfile.TemporaryDirectory() as tmpdirname:
+
+            self.file1 = os.path.join(tmpdirname, self.filename1)
+            with open(self.file1, 'w') as _:
+                pass
+
+            self.testscene['orbit_number'] = self.testscene['orbit_number'] + 1
+            ascii_files = get_time_control_ascii_filename_candidates(self.testscene, tmpdirname)
 
         self.assertEqual(len(ascii_files), 1)
         self.assertTrue(os.path.basename(ascii_files[0]) ==
                         'S_NWC_timectrl_metopb_46878_20210930T0946289Z_20210930T1001458Z.txt')
 
+    def test_get_time_control_ascii_filename_candidates_modis_zero_orbit(self):
+
+        with tempfile.TemporaryDirectory() as tmpdirname:
+
+            self.file1 = os.path.join(tmpdirname, self.filename1_modis)
+            with open(self.file1, 'w') as _:
+                pass
+
+            ascii_files = get_time_control_ascii_filename_candidates(self.modis_scene, tmpdirname)
+
+        self.assertEqual(len(ascii_files), 1)
+        self.assertTrue(os.path.basename(ascii_files[0]) ==
+                        'S_NWC_timectrl_eos2_00000_20220209T2210110Z_20220209T2212284Z.txt')
+
 
 class TestProductStatisticsFiles(unittest.TestCase):
     """Testing locating the product statistics XML files."""

nwcsafpps_runner/utils.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2018 - 2021 Pytroll Developers
+# Copyright (c) 2018 - 2022 Pytroll Developers
 
 # Author(s):
 
@@ -48,6 +48,10 @@ class NwpPrepareError(Exception):
     pass
 
 
+class FindTimeControlFileError(Exception):
+    pass
+
+
 PPS_OUT_PATTERN = ("S_NWC_{segment}_{orig_platform_name}_{orbit_number:05d}_" +
                    "{start_time:%Y%m%dT%H%M%S%f}Z_{end_time:%Y%m%dT%H%M%S%f}Z.{extention}")
 PPS_OUT_PATTERN_MULTIPLE = ("S_NWC_{segment1}_{segment2}_{orig_platform_name}_{orbit_number:05d}_" +
@@ -584,37 +588,49 @@ def get_xml_outputfiles(path, platform_name, orb, st_time=''):
 
 
 def create_xml_timestat_from_ascii(scene, pps_control_path):
-    """From ascii files with PPS time statistics create XML files and return a file list."""
-
+    """From ascii file(s) with PPS time statistics create XML file(s) and return a file list."""
     try:
         from pps_time_control import PPSTimeControl
     except ImportError:
         LOG.warning("Failed to import the PPSTimeControl from pps")
         return []
 
-    infiles = get_time_control_ascii_filename_candidates(pps_control_path, scene)
-    LOG.info("Time control ascii file candidates: " + str(infiles))
+    try:
+        infile = get_time_control_ascii_filename(scene, pps_control_path)
+    except FindTimeControlFileError:
+        LOG.exception('No XML Time statistics file created!')
+        return []
 
-    if len(infiles) == 1:
-        infile = str(infiles[0])
-        LOG.info("Time control ascii file: " + str(infile))
-        LOG.info("Read time control ascii file and generate XML")
-        ppstime_con = PPSTimeControl(infile)
-        ppstime_con.sum_up_processing_times()
-        try:
-            ppstime_con.write_xml()
-        except Exception as e:  # TypeError as e:
-            LOG.warning('Not able to write time control xml file')
-            LOG.warning(e)
+    LOG.info("Time control ascii file: " + str(infile))
+    LOG.info("Read time control ascii file and generate XML")
+    ppstime_con = PPSTimeControl(infile)
+    ppstime_con.sum_up_processing_times()
+    try:
+        ppstime_con.write_xml()
+    except Exception as e:  # TypeError as e:
+        LOG.warning('Not able to write time control xml file')
+        LOG.warning(e)
 
     # There should always be only one xml file for each ascii file found above!
     xmlfile = infile.replace('.txt', '.xml')
     return filter4oldfiles([xmlfile], 90.)
 
 
-def get_time_control_ascii_filename_candidates(pps_control_path, scene):
-    """From directory path, sensor and platform name get possible time-control filenames."""
+def get_time_control_ascii_filename(scene, pps_control_path):
+    """From the scene object and a file path get the time-control-ascii-filename (with path)."""
+    infiles = get_time_control_ascii_filename_candidates(scene, pps_control_path)
+    LOG.info("Time control ascii file candidates: " + str(infiles))
+    if len(infiles) == 0:
+        raise FindTimeControlFileError("No time control ascii file candidate found!")
+    elif len(infiles) > 1:
+        msg = "More than one time control ascii file candidate found - unresolved ambiguity!"
+        raise FindTimeControlFileError(msg)
+
+    return infiles[0]
 
+
+def get_time_control_ascii_filename_candidates(scene, pps_control_path):
+    """From directory path, sensor and platform name get possible time-control filenames."""
     sensors = SENSOR_LIST.get(scene['platform_name'], scene['platform_name'])
     platform_id = SATELLITE_NAME.get(scene['platform_name'], scene['platform_name'])
     LOG.info("pps platform_id = %s", str(platform_id))
@@ -634,15 +650,25 @@ def get_time_control_ascii_filename_candidates(pps_control_path, scene):
             st_times.append(atime.strftime("%Y%m%dT%H%M"))
             atime = atime + timedelta(seconds=60)
 
+    # For VIIRS we often see a orbit number difference of 1:
+    norbit_candidates = [scene['orbit_number']]
+    for idx in [1, -1]:
+        norbit_candidates.append(int(scene['orbit_number']) + idx)
+
+    # PPSv2018 MODIS files have the orbit number set to "00000"!
+    if sensors in ['modis', ]:
+        norbit_candidates.append(0)
+
     infiles = []
-    for st_time in st_times:
-        txt_time_file = (os.path.join(pps_control_path, 'S_NWC_timectrl_') +
-                         str(METOP_NAME_LETTER.get(platform_id, platform_id)) +
-                         '_' + '%.5d' % scene['orbit_number'] + '_' +
-                         st_time +
-                         '*.txt')
-        LOG.info("glob string = " + str(txt_time_file))
-        infiles = infiles + glob(txt_time_file)
+    for norbit in norbit_candidates:
+        for st_time in st_times:
+            txt_time_file = (os.path.join(pps_control_path, 'S_NWC_timectrl_') +
+                             str(METOP_NAME_LETTER.get(platform_id, platform_id)) +
+                             '_' + '%.5d' % norbit + '_' +
+                             st_time +
+                             '*.txt')
+            LOG.info("glob string = " + str(txt_time_file))
+            infiles = infiles + glob(txt_time_file)
 
     return infiles
 

setup.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2013 - 2021 Pytroll Community
+# Copyright (c) 2013 - 2022 Pytroll Community
 
 # Author(s):