Update the tests

Framework/script/RepoCleaner/README.md

@@ -34,9 +34,16 @@ There can be any number of these rules. The order is important as we use the fir
 The configuration for ccdb-test is described [here](../../../doc/DevelopersTips.md). 
 
 ## Unit Tests
-`cd QualityControl/Framework/script/RepoCleaner ; python3 -m unittest discover`
 
-and to test only one of them: `python3 -m unittest tests/test_NewProduction.py -k test_2_runs`
+```
+cd QualityControl/Framework/script/RepoCleaner
+source env/bin/activate
+
+# Run a test: 
+python -m unittest tests.test_Ccdb.TestCcdb.test_getObjectsList
+```
+
+`cd QualityControl/Framework/script/RepoCleaner ; python3 -m unittest discover`
 
 In particular there is a test for the `production` rule that is pretty extensive. It hits the ccdb though and it needs the following path to be truncated: 
 `

Framework/script/RepoCleaner/qcrepocleaner/Ccdb.py

@@ -26,6 +26,7 @@ def __init__(self, path: str, validFrom, validTo, createdAt, uuid=None, metadata
         :param uuid: unique id of the object
         :param validFrom: validity range smaller limit (in ms)
         :param validTo: validity range bigger limit (in ms)
+        :param createdAt: creation timestamp of the object
         '''
         self.path = path
         self.uuid = uuid
@@ -72,7 +73,8 @@ def getObjectsList(self, added_since: int = 0, path: str = "", no_wildcard: bool
         :return A list of strings, each containing a path to an object in the CCDB.
         '''
         url_for_all_obj = self.url + '/latest/' + path
-        url_for_all_obj += '/' if no_wildcard else '/.*'
+        url_for_all_obj += '/' if path else ''
+        url_for_all_obj += '' if no_wildcard else '.*'
         logger.debug(f"Ccdb::getObjectsList -> {url_for_all_obj}")
         headers = {'Accept': 'application/json', 'If-Not-Before':str(added_since)}
         r = requests.get(url_for_all_obj, headers=headers)

Framework/script/RepoCleaner/tests/test_Ccdb.py

@@ -1,31 +1,31 @@
 import logging
 import unittest
-import requests
 import responses
 
-from Ccdb import Ccdb, ObjectVersion
-from rules import production
+from qcrepocleaner.Ccdb import Ccdb, ObjectVersion, logger
+from typing import List
 
 class TestCcdb(unittest.TestCase):
 
     def setUp(self):
-        with open('../qcrepocleaner/objectsList.json') as f:   # will close() when we leave this block
+        with open('objectsList.json') as f:   # will close() when we leave this block
             self.content_objectslist = f.read()
-        with open('../versionsList.json') as f:   # will close() when we leave this block
+        with open('versionsList.json') as f:   # will close() when we leave this block
             self.content_versionslist = f.read()
         self.ccdb = Ccdb('http://ccdb-test.cern.ch:8080')
+        logging.getLogger().setLevel(logging.DEBUG)
 
     @responses.activate
     def test_getObjectsList(self):
         # Prepare mock response
         responses.add(responses.GET, 'http://ccdb-test.cern.ch:8080/latest/.*',
                       self.content_objectslist, status=200)
         # get list of objects
-        objectsList = self.ccdb.getObjectsList()
-        print(f"{objectsList}")
-        self.assertEqual(len(objectsList), 3)
-        self.assertEqual(objectsList[0], 'Test')
-        self.assertEqual(objectsList[1], 'ITSQcTask/ChipStaveCheck')
+        objects_list = self.ccdb.getObjectsList()
+        print(f"{objects_list}")
+        self.assertEqual(len(objects_list), 3)
+        self.assertEqual(objects_list[0], 'Test')
+        self.assertEqual(objects_list[1], 'ITSQcTask/ChipStaveCheck')
 
     @responses.activate
     def test_getVersionsList(self):
@@ -34,12 +34,12 @@ def test_getVersionsList(self):
         responses.add(responses.GET, 'http://ccdb-test.cern.ch:8080/browse/'+object_path,
                       self.content_versionslist, status=200)
         # get versions for object
-        versionsList: List[ObjectVersion] = self.ccdb.getVersionsList(object_path)
-        print(f"{versionsList}")
-        self.assertEqual(len(versionsList), 2)
-        self.assertEqual(versionsList[0].path, object_path)
-        self.assertEqual(versionsList[1].path, object_path)
-        self.assertEqual(versionsList[1].metadata["custom"], "34")
+        versions_list: List[ObjectVersion] = self.ccdb.getVersionsList(object_path)
+        print(f"{versions_list}")
+        self.assertEqual(len(versions_list), 2)
+        self.assertEqual(versions_list[0].path, object_path)
+        self.assertEqual(versions_list[1].path, object_path)
+        self.assertEqual(versions_list[1].metadata["custom"], "34")
 
 if __name__ == '__main__':
     unittest.main()

Framework/script/RepoCleaner/tests/test_last_only.py

@@ -1,12 +1,9 @@
 import logging
 import time
 import unittest
-from datetime import timedelta, date, datetime
-
-from Ccdb import Ccdb, ObjectVersion
-from rules import last_only
-
 
+from qcrepocleaner.Ccdb import Ccdb, ObjectVersion
+from qcrepocleaner.rules import last_only
 
 
 class TestLastOnly(unittest.TestCase):
@@ -28,6 +25,12 @@ def setUp(self):
         self.run = 124321
 
 
+    def clean_data(self, path):
+        versions = self.ccdb.getVersionsList(path)
+        for v in versions:
+            self.ccdb.deleteVersion(v)
+
+
     def test_last_only(self):
         """
         59 versions
@@ -39,6 +42,7 @@ def test_last_only(self):
 
         # Prepare data
         test_path = self.path + "/test_last_only"
+        self.clean_data(test_path)
         self.prepare_data(test_path, 60)
 
         stats = last_only.process(self.ccdb, test_path, 30, 1, self.in_ten_years, self.extra)
@@ -61,15 +65,16 @@ def test_last_only_period(self):
 
         # Prepare data
         test_path = self.path + "/test_last_only_period"
+        self.clean_data(test_path)
         self.prepare_data(test_path, 60)
         current_timestamp = int(time.time() * 1000)
 
-        stats = last_only.process(self.ccdb, test_path, 0, current_timestamp-41*60*1000, current_timestamp-19*60*1000, self.extra)
-        self.assertEqual(stats["deleted"], 19)
-        self.assertEqual(stats["preserved"], 40)
+        stats = last_only.process(self.ccdb, test_path, 0, current_timestamp-40*60*1000, current_timestamp-20*60*1000, self.extra)
+        self.assertEqual(stats["deleted"], 20)
+        self.assertEqual(stats["preserved"], 39)
 
         objects_versions = self.ccdb.getVersionsList(test_path)
-        self.assertEqual(len(objects_versions), 40)
+        self.assertEqual(len(objects_versions), 39)
 
 
     def prepare_data(self, path, since_minutes):
@@ -90,7 +95,7 @@ def prepare_data(self, path, since_minutes):
             to_ts = current_timestamp
             metadata = {'RunNumber': str(run)}
             run = run + 1
-            version_info = ObjectVersion(path=path, validFrom=from_ts, validTo=to_ts, metadata=metadata)
+            version_info = ObjectVersion(path=path, validFrom=from_ts, validTo=to_ts, createdAt=from_ts, metadata=metadata)
             self.ccdb.putVersion(version=version_info, data=data)
 
         logging.debug(f"counter : {counter}" )

Framework/script/RepoCleaner/versionsList.json

@@ -3,6 +3,7 @@
     {
       "id": "0c576bb0-7304-11e9-8d02-200114580202",
       "validFrom": "1557479683554",
+      "Created": "1557479683554",
       "validUntil": "1872839683554",
       "initialValidity": "1872839683554",
       "createTime": "1557479683563",
@@ -20,6 +21,7 @@
     {
       "id": "06fb1e80-72f7-11e9-8d02-200114580202",
       "validFrom": "1557474091106",
+      "Created": "1557474091106",
       "validUntil": "1557479683553",
       "initialValidity": "1872834091106",
       "createTime": "1557474091112",