From a63cde26526aea2f2ba8c3e3339f2a8fb13e23b3 Mon Sep 17 00:00:00 2001 From: Barthelemy Date: Fri, 1 Nov 2024 15:17:33 +0100 Subject: [PATCH] Update the tests --- Framework/script/RepoCleaner/README.md | 11 +++++-- .../{qcrepocleaner => }/objectsList.json | 0 .../script/RepoCleaner/qcrepocleaner/Ccdb.py | 4 ++- .../script/RepoCleaner/tests/test_Ccdb.py | 32 +++++++++---------- .../RepoCleaner/tests/test_last_only.py | 25 +++++++++------ .../script/RepoCleaner/versionsList.json | 2 ++ 6 files changed, 45 insertions(+), 29 deletions(-) rename Framework/script/RepoCleaner/{qcrepocleaner => }/objectsList.json (100%) diff --git a/Framework/script/RepoCleaner/README.md b/Framework/script/RepoCleaner/README.md index 748166e829..e460d0efbb 100644 --- a/Framework/script/RepoCleaner/README.md +++ b/Framework/script/RepoCleaner/README.md @@ -34,9 +34,16 @@ There can be any number of these rules. The order is important as we use the fir The configuration for ccdb-test is described [here](../../../doc/DevelopersTips.md). ## Unit Tests -`cd QualityControl/Framework/script/RepoCleaner ; python3 -m unittest discover` -and to test only one of them: `python3 -m unittest tests/test_NewProduction.py -k test_2_runs` +``` +cd QualityControl/Framework/script/RepoCleaner +source env/bin/activate + +# Run a test: +python -m unittest tests.test_Ccdb.TestCcdb.test_getObjectsList +``` + +`cd QualityControl/Framework/script/RepoCleaner ; python3 -m unittest discover` In particular there is a test for the `production` rule that is pretty extensive. It hits the ccdb though and it needs the following path to be truncated: ` diff --git a/Framework/script/RepoCleaner/qcrepocleaner/objectsList.json b/Framework/script/RepoCleaner/objectsList.json similarity index 100% rename from Framework/script/RepoCleaner/qcrepocleaner/objectsList.json rename to Framework/script/RepoCleaner/objectsList.json diff --git a/Framework/script/RepoCleaner/qcrepocleaner/Ccdb.py b/Framework/script/RepoCleaner/qcrepocleaner/Ccdb.py index ead669180a..407cec3416 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/Ccdb.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/Ccdb.py @@ -26,6 +26,7 @@ def __init__(self, path: str, validFrom, validTo, createdAt, uuid=None, metadata :param uuid: unique id of the object :param validFrom: validity range smaller limit (in ms) :param validTo: validity range bigger limit (in ms) + :param createdAt: creation timestamp of the object ''' self.path = path self.uuid = uuid @@ -72,7 +73,8 @@ def getObjectsList(self, added_since: int = 0, path: str = "", no_wildcard: bool :return A list of strings, each containing a path to an object in the CCDB. ''' url_for_all_obj = self.url + '/latest/' + path - url_for_all_obj += '/' if no_wildcard else '/.*' + url_for_all_obj += '/' if path else '' + url_for_all_obj += '' if no_wildcard else '.*' logger.debug(f"Ccdb::getObjectsList -> {url_for_all_obj}") headers = {'Accept': 'application/json', 'If-Not-Before':str(added_since)} r = requests.get(url_for_all_obj, headers=headers) diff --git a/Framework/script/RepoCleaner/tests/test_Ccdb.py b/Framework/script/RepoCleaner/tests/test_Ccdb.py index 7f04f27902..d2a83013dc 100644 --- a/Framework/script/RepoCleaner/tests/test_Ccdb.py +++ b/Framework/script/RepoCleaner/tests/test_Ccdb.py @@ -1,19 +1,19 @@ import logging import unittest -import requests import responses -from Ccdb import Ccdb, ObjectVersion -from rules import production +from qcrepocleaner.Ccdb import Ccdb, ObjectVersion, logger +from typing import List class TestCcdb(unittest.TestCase): def setUp(self): - with open('../qcrepocleaner/objectsList.json') as f: # will close() when we leave this block + with open('objectsList.json') as f: # will close() when we leave this block self.content_objectslist = f.read() - with open('../versionsList.json') as f: # will close() when we leave this block + with open('versionsList.json') as f: # will close() when we leave this block self.content_versionslist = f.read() self.ccdb = Ccdb('http://ccdb-test.cern.ch:8080') + logging.getLogger().setLevel(logging.DEBUG) @responses.activate def test_getObjectsList(self): @@ -21,11 +21,11 @@ def test_getObjectsList(self): responses.add(responses.GET, 'http://ccdb-test.cern.ch:8080/latest/.*', self.content_objectslist, status=200) # get list of objects - objectsList = self.ccdb.getObjectsList() - print(f"{objectsList}") - self.assertEqual(len(objectsList), 3) - self.assertEqual(objectsList[0], 'Test') - self.assertEqual(objectsList[1], 'ITSQcTask/ChipStaveCheck') + objects_list = self.ccdb.getObjectsList() + print(f"{objects_list}") + self.assertEqual(len(objects_list), 3) + self.assertEqual(objects_list[0], 'Test') + self.assertEqual(objects_list[1], 'ITSQcTask/ChipStaveCheck') @responses.activate def test_getVersionsList(self): @@ -34,12 +34,12 @@ def test_getVersionsList(self): responses.add(responses.GET, 'http://ccdb-test.cern.ch:8080/browse/'+object_path, self.content_versionslist, status=200) # get versions for object - versionsList: List[ObjectVersion] = self.ccdb.getVersionsList(object_path) - print(f"{versionsList}") - self.assertEqual(len(versionsList), 2) - self.assertEqual(versionsList[0].path, object_path) - self.assertEqual(versionsList[1].path, object_path) - self.assertEqual(versionsList[1].metadata["custom"], "34") + versions_list: List[ObjectVersion] = self.ccdb.getVersionsList(object_path) + print(f"{versions_list}") + self.assertEqual(len(versions_list), 2) + self.assertEqual(versions_list[0].path, object_path) + self.assertEqual(versions_list[1].path, object_path) + self.assertEqual(versions_list[1].metadata["custom"], "34") if __name__ == '__main__': unittest.main() diff --git a/Framework/script/RepoCleaner/tests/test_last_only.py b/Framework/script/RepoCleaner/tests/test_last_only.py index a8e1228789..022eacb38a 100644 --- a/Framework/script/RepoCleaner/tests/test_last_only.py +++ b/Framework/script/RepoCleaner/tests/test_last_only.py @@ -1,12 +1,9 @@ import logging import time import unittest -from datetime import timedelta, date, datetime - -from Ccdb import Ccdb, ObjectVersion -from rules import last_only - +from qcrepocleaner.Ccdb import Ccdb, ObjectVersion +from qcrepocleaner.rules import last_only class TestLastOnly(unittest.TestCase): @@ -28,6 +25,12 @@ def setUp(self): self.run = 124321 + def clean_data(self, path): + versions = self.ccdb.getVersionsList(path) + for v in versions: + self.ccdb.deleteVersion(v) + + def test_last_only(self): """ 59 versions @@ -39,6 +42,7 @@ def test_last_only(self): # Prepare data test_path = self.path + "/test_last_only" + self.clean_data(test_path) self.prepare_data(test_path, 60) stats = last_only.process(self.ccdb, test_path, 30, 1, self.in_ten_years, self.extra) @@ -61,15 +65,16 @@ def test_last_only_period(self): # Prepare data test_path = self.path + "/test_last_only_period" + self.clean_data(test_path) self.prepare_data(test_path, 60) current_timestamp = int(time.time() * 1000) - stats = last_only.process(self.ccdb, test_path, 0, current_timestamp-41*60*1000, current_timestamp-19*60*1000, self.extra) - self.assertEqual(stats["deleted"], 19) - self.assertEqual(stats["preserved"], 40) + stats = last_only.process(self.ccdb, test_path, 0, current_timestamp-40*60*1000, current_timestamp-20*60*1000, self.extra) + self.assertEqual(stats["deleted"], 20) + self.assertEqual(stats["preserved"], 39) objects_versions = self.ccdb.getVersionsList(test_path) - self.assertEqual(len(objects_versions), 40) + self.assertEqual(len(objects_versions), 39) def prepare_data(self, path, since_minutes): @@ -90,7 +95,7 @@ def prepare_data(self, path, since_minutes): to_ts = current_timestamp metadata = {'RunNumber': str(run)} run = run + 1 - version_info = ObjectVersion(path=path, validFrom=from_ts, validTo=to_ts, metadata=metadata) + version_info = ObjectVersion(path=path, validFrom=from_ts, validTo=to_ts, createdAt=from_ts, metadata=metadata) self.ccdb.putVersion(version=version_info, data=data) logging.debug(f"counter : {counter}" ) diff --git a/Framework/script/RepoCleaner/versionsList.json b/Framework/script/RepoCleaner/versionsList.json index ae3e9a2aae..f5f0a8887a 100644 --- a/Framework/script/RepoCleaner/versionsList.json +++ b/Framework/script/RepoCleaner/versionsList.json @@ -3,6 +3,7 @@ { "id": "0c576bb0-7304-11e9-8d02-200114580202", "validFrom": "1557479683554", + "Created": "1557479683554", "validUntil": "1872839683554", "initialValidity": "1872839683554", "createTime": "1557479683563", @@ -20,6 +21,7 @@ { "id": "06fb1e80-72f7-11e9-8d02-200114580202", "validFrom": "1557474091106", + "Created": "1557474091106", "validUntil": "1557479683553", "initialValidity": "1872834091106", "createTime": "1557474091112",