Revive tests

- Fix the tests
- Put common code in a common place
- Move tests related files in `tests`

Framework/script/RepoCleaner/tests/test_1_per_hour.py

@@ -1,26 +1,12 @@
 import logging
 import time
 import unittest
-from datetime import timedelta, date, datetime
-
-from Ccdb import Ccdb, ObjectVersion
-from rules import last_only
-import os
-import sys
-import importlib
-
-def import_path(path):  # needed because o2-qc-repo-cleaner has no suffix
-    module_name = os.path.basename(path).replace('-', '_')
-    spec = importlib.util.spec_from_loader(
-        module_name,
-        importlib.machinery.SourceFileLoader(module_name, path)
-    )
-    module = importlib.util.module_from_spec(spec)
-    spec.loader.exec_module(module)
-    sys.modules[module_name] = module
-    return module
-
-one_per_hour = import_path("../qcrepocleaner/rules/1_per_hour.py")
+from importlib import import_module
+
+from qcrepocleaner.Ccdb import Ccdb
+from tests import test_utils
+
+one_per_hour =  import_module(".1_per_hour", "qcrepocleaner.rules") # file names should not start with a number...
 
 class Test1PerHour(unittest.TestCase):
     """
@@ -35,86 +21,64 @@ class Test1PerHour(unittest.TestCase):
     one_minute = 60000
 
     def setUp(self):
-        self.ccdb = Ccdb('http://ccdb-test.cern.ch:8080')
+        self.ccdb = Ccdb('http://128.142.249.62:8080') # ccdb-test but please use IP to avoid DNS alerts
         self.path = "qc/TST/MO/repo/test"
         self.run = 124321
         self.extra = {}
 
 
     def test_1_per_hour(self):
         """
-        60 versions, 2 minutes apart
+        120 versions
         grace period of 15 minutes
-        First version is preserved (always). 7 are preserved during the grace period at the end.
-        One more is preserved after 1 hour. --> 9 preserved
+        First version is preserved (always). 14 are preserved during the grace period at the end.
+        One more is preserved after 1 hour. --> 16 preserved
         """
         logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s',
                             datefmt='%d-%b-%y %H:%M:%S')
         logging.getLogger().setLevel(int(10))
 
         # Prepare data
         test_path = self.path + "/test_1_per_hour"
-        self.prepare_data(test_path, 60, 2)
+        test_utils.clean_data(self.ccdb, test_path)
+        test_utils.prepare_data(self.ccdb, test_path, [120], [0], 123)
 
         stats = one_per_hour.process(self.ccdb, test_path, 15, 1, self.in_ten_years, self.extra)
-        self.assertEqual(stats["deleted"], 51)
-        self.assertEqual(stats["preserved"], 9)
+        logging.info(stats)
+        self.assertEqual(stats["deleted"], 104)
+        self.assertEqual(stats["preserved"], 16)
 
         objects_versions = self.ccdb.getVersionsList(test_path)
-        self.assertEqual(len(objects_versions), 9)
+        self.assertEqual(len(objects_versions), 16)
 
 
     def test_1_per_hour_period(self):
         """
-        60 versions, 2 minutes apart
+        120 versions
         no grace period
         period of acceptance: 1 hour in the middle
-        We have therefore 30 versions in the acceptance period.
+        We have therefore 60 versions in the acceptance period.
         Only 1 of them, the one 1 hour after the first version in the set, will be preserved, the others are deleted.
-        Thus we have 29 deletion. Everything outside the acceptance period is kept.
+        Thus we have 59 deletion. Everything outside the acceptance period is kept.
         """
         logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s',
                             datefmt='%d-%b-%y %H:%M:%S')
         logging.getLogger().setLevel(int(10))
 
         # Prepare data
         test_path = self.path + "/test_1_per_hour_period"
-        self.prepare_data(test_path, 60, 2)
+        test_utils.clean_data(self.ccdb, test_path)
+        test_utils.prepare_data(self.ccdb, test_path, [120], [0], 123)
         current_timestamp = int(time.time() * 1000)
-        logging.debug(f"{current_timestamp} - {datetime.today()}")
-
-        objects_versions = self.ccdb.getVersionsList(test_path)
-        created = len(objects_versions)
 
         stats = one_per_hour.process(self.ccdb, test_path, 15, current_timestamp-90*60*1000,
                                      current_timestamp-30*60*1000, self.extra)
-        self.assertEqual(stats["deleted"], 29)
-        self.assertEqual(stats["preserved"], 31)
+        logging.info(stats)
+        self.assertEqual(stats["deleted"], 59)
+        self.assertEqual(stats["preserved"], 61)
 
         objects_versions = self.ccdb.getVersionsList(test_path)
-        self.assertEqual(len(objects_versions), 31)
-
-
-    def prepare_data(self, path, number_versions, minutes_between):
-        """
-        Prepare a data set starting `since_minutes` in the past.
-        1 version per minute
-        """
-
-        current_timestamp = int(time.time() * 1000)
-        data = {'part': 'part'}
-        run = 1234
-        counter = 0
-
-        for x in range(number_versions+1):
-            counter = counter + 1
-            from_ts = current_timestamp - minutes_between * x * 60 * 1000
-            to_ts = current_timestamp
-            metadata = {'RunNumber': str(run)}
-            version_info = ObjectVersion(path=path, validFrom=from_ts, validTo=to_ts, metadata=metadata)
-            self.ccdb.putVersion(version=version_info, data=data)
-
-        logging.debug(f"counter : {counter}")
+        self.assertEqual(len(objects_versions), 61)
 
 
 if __name__ == '__main__':

Framework/script/RepoCleaner/tests/test_1_per_run.py

@@ -1,29 +1,13 @@
 import logging
+import os
 import time
 import unittest
-from datetime import timedelta, date, datetime
-
-from Ccdb import Ccdb, ObjectVersion
-from rules import last_only
-import os
-import sys
-import importlib
-
+from importlib import import_module
 
-def import_path(path):  # needed because o2-qc-repo-cleaner has no suffix
-    module_name = os.path.basename(path).replace('-', '_')
-    spec = importlib.util.spec_from_loader(
-        module_name,
-        importlib.machinery.SourceFileLoader(module_name, path)
-    )
-    module = importlib.util.module_from_spec(spec)
-    spec.loader.exec_module(module)
-    sys.modules[module_name] = module
-    return module
-
-
-one_per_run = import_path("../qcrepocleaner/rules/1_per_run.py")
+from qcrepocleaner.Ccdb import Ccdb
+from tests import test_utils
 
+one_per_run =  import_module(".1_per_run", "qcrepocleaner.rules") # file names should not start with a number...
 
 class Test1PerRun(unittest.TestCase):
     """
@@ -38,15 +22,14 @@ class Test1PerRun(unittest.TestCase):
     one_minute = 60000
 
     def setUp(self):
-        self.ccdb = Ccdb('http://ccdb-test.cern.ch:8080')
+        self.ccdb = Ccdb('http://128.142.249.62:8080')
         self.path = "qc/TST/MO/repo/test"
         self.run = 124321
         self.extra = {}
 
     def test_1_per_run(self):
         """
-        60 versions, 1 minute apart
-        6 runs
+        6 runs of 10 versions, versions 1 minute apart
         grace period of 15 minutes
         Preserved: 14 at the end (grace period), 6 for the runs, but 2 are in both sets --> 14+6-2=18 preserved
         """
@@ -56,7 +39,8 @@ def test_1_per_run(self):
 
         # Prepare data
         test_path = self.path + "/test_1_per_run"
-        self.prepare_data(test_path, 60)
+        test_utils.clean_data(self.ccdb, test_path)
+        test_utils.prepare_data(self.ccdb, test_path, [10, 10, 10, 10, 10, 10], [0, 0, 0, 0, 0, 0], 123)
 
         objects_versions = self.ccdb.getVersionsList(test_path)
         created = len(objects_versions)
@@ -71,8 +55,7 @@ def test_1_per_run(self):
 
     def test_1_per_run_period(self):
         """
-        60 versions 1 minute apart
-        6 runs
+        6 runs of 10 versions each, versions 1 minute apart
         no grace period
         acceptance period is only the 38 minutes in the middle
         preserved: 6 runs + 11 first and 11 last, with an overlap of 2 --> 26
@@ -83,7 +66,8 @@ def test_1_per_run_period(self):
 
         # Prepare data
         test_path = self.path + "/test_1_per_run_period"
-        self.prepare_data(test_path, 60)
+        test_utils.clean_data(self.ccdb, test_path)
+        test_utils.prepare_data(self.ccdb, test_path, [10, 10, 10, 10, 10, 10], [0, 0, 0, 0, 0, 0], 123)
         current_timestamp = int(time.time() * 1000)
 
         stats = one_per_run.process(self.ccdb, test_path, 0, current_timestamp - 49 * 60 * 1000,
@@ -94,29 +78,5 @@ def test_1_per_run_period(self):
         objects_versions = self.ccdb.getVersionsList(test_path)
         self.assertEqual(len(objects_versions), 26)
 
-    def prepare_data(self, path, since_minutes):
-        """
-        Prepare a data set starting `since_minutes` in the past.
-        1 version per minute, 1 run every 10 versions
-        """
-
-        current_timestamp = int(time.time() * 1000)
-        data = {'part': 'part'}
-        run = 1234
-        counter = 0
-
-        for x in range(since_minutes + 1):
-            counter = counter + 1
-            from_ts = current_timestamp - x * 60 * 1000
-            to_ts = current_timestamp
-            metadata = {'RunNumber': str(run)}
-            version_info = ObjectVersion(path=path, validFrom=from_ts, validTo=to_ts, metadata=metadata)
-            self.ccdb.putVersion(version=version_info, data=data)
-            if x % 10 == 0:
-                run = run + 1
-
-        logging.debug(f"counter : {counter}")
-
-
 if __name__ == '__main__':
     unittest.main()