From e440fe3659da11fef5fd5b6cbc24a07bc89ea407 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Wed, 24 Mar 2021 09:24:33 +0100 Subject: [PATCH 1/8] FTS3: submit FTS3Job with lowercase checksum --- DataManagementSystem/Client/FTS3Job.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/DataManagementSystem/Client/FTS3Job.py b/DataManagementSystem/Client/FTS3Job.py index 6a8eba69b9d..6fd1fb6ff08 100644 --- a/DataManagementSystem/Client/FTS3Job.py +++ b/DataManagementSystem/Client/FTS3Job.py @@ -380,9 +380,15 @@ def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=N transfers.append(stageTrans) trans_metadata = {'desc': 'Transfer %s' % ftsFileID, 'fileID': ftsFileID} + + # because of an xroot bug (https://github.com/xrootd/xrootd/issues/1433) + # the checksum needs to be lowercase. It does not impact the other + # protocol, so it's fine to put it here. + # I only add it in this transfer and not the "staging" one above because it + # impacts only root -> root transfers trans = fts3.new_transfer(sourceSURL, targetSURL, - checksum='ADLER32:%s' % ftsFile.checksum, + checksum='ADLER32:%s' % ftsFile.checksum.lower(), filesize=ftsFile.size, metadata=trans_metadata, activity=self.activity) From 6ce96e6021b4eda6b6f869afd38a5e6c264b3e99 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Mon, 29 Mar 2021 14:46:18 +0200 Subject: [PATCH 2/8] Avoid pointless sleeping in Test_MySQLDB.py --- Core/Utilities/MySQL.py | 3 ++- tests/Integration/Core/Test_MySQLDB.py | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Core/Utilities/MySQL.py b/Core/Utilities/MySQL.py index 14f53238f7a..0d09c0bd136 100755 --- a/Core/Utilities/MySQL.py +++ b/Core/Utilities/MySQL.py @@ -170,6 +170,7 @@ __RCSID__ = "$Id$" MAXCONNECTRETRY = 10 +RETRY_SLEEP_DURATION = 5 def _checkFields(inFields, inValues): @@ -257,7 +258,7 @@ def get(self, dbName, retries=10): return self.__getWithRetry(dbName, retries, retries) def __getWithRetry(self, dbName, totalRetries, retriesLeft): - sleepTime = 5 * (totalRetries - retriesLeft) + sleepTime = RETRY_SLEEP_DURATION * (totalRetries - retriesLeft) if sleepTime > 0: time.sleep(sleepTime) try: diff --git a/tests/Integration/Core/Test_MySQLDB.py b/tests/Integration/Core/Test_MySQLDB.py index eecf4ccdada..db6858b2128 100644 --- a/tests/Integration/Core/Test_MySQLDB.py +++ b/tests/Integration/Core/Test_MySQLDB.py @@ -7,6 +7,7 @@ import time import pytest +import DIRAC from DIRAC import gLogger, gConfig from DIRAC.Core.Utilities import Time from DIRAC.Core.Utilities.MySQL import MySQL @@ -105,9 +106,13 @@ def genVal2(): ('mysql', 'Dirac', 'Dirac', 'AccountingDB', 3306, True), ('fake', 'fake', 'fake', 'FakeDB', 0000, False), ]) -def test_connection(host, user, password, dbName, port, expected): +def test_connection(host, user, password, dbName, port, expected, monkeypatch): """ Try to connect to a DB """ + # Avoid having many retries which sleep for long durations + monkeypatch.setattr(DIRAC.Core.Utilities.MySQL, "MAXCONNECTRETRY", 1) + monkeypatch.setattr(DIRAC.Core.Utilities.MySQL, "RETRY_SLEEP_DURATION", 0.5) + mysqlDB = getDB(host, user, password, dbName, port) result = mysqlDB._connect() assert result['OK'] is expected From da473de6fe978cfb1071eff7b9a0e04f8d8301d4 Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Wed, 31 Mar 2021 12:27:48 +0200 Subject: [PATCH 3/8] Fix for the case where no subprocess32 module found --- Resources/Computing/BatchSystems/Host.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Resources/Computing/BatchSystems/Host.py b/Resources/Computing/BatchSystems/Host.py index b51c30baf68..4493aeeb158 100644 --- a/Resources/Computing/BatchSystems/Host.py +++ b/Resources/Computing/BatchSystems/Host.py @@ -14,7 +14,10 @@ import glob import shutil import signal -import subprocess32 as subprocess +try: + import subprocess32 as subprocess +except: + import subprocess import stat import json import multiprocessing From 0b08882c3b661683363464ad81f2e5cec7dfc7b3 Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Wed, 31 Mar 2021 12:52:23 +0200 Subject: [PATCH 4/8] use ImportError exception explicitly --- Resources/Computing/BatchSystems/Host.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resources/Computing/BatchSystems/Host.py b/Resources/Computing/BatchSystems/Host.py index 4493aeeb158..ec14372b0e3 100644 --- a/Resources/Computing/BatchSystems/Host.py +++ b/Resources/Computing/BatchSystems/Host.py @@ -16,7 +16,7 @@ import signal try: import subprocess32 as subprocess -except: +except ImportError: import subprocess import stat import json From 9326e23a9a79855981101ff7dbf0ef4100bbe56f Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Mon, 29 Mar 2021 15:25:10 +0200 Subject: [PATCH 5/8] Interface: correctly strip arguments --- Interfaces/API/Dirac.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Interfaces/API/Dirac.py b/Interfaces/API/Dirac.py index 97e3618ee5e..df5111347cc 100755 --- a/Interfaces/API/Dirac.py +++ b/Interfaces/API/Dirac.py @@ -63,7 +63,7 @@ def parseArguments(args): argList = [] for arg in args: - argList += arg.split(',') + argList += [a.strip() for a in arg.split(',') if a.strip()] return argList From b424bab885d5fe967f3d526c65fafdc3129f071e Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Wed, 31 Mar 2021 18:33:32 +0200 Subject: [PATCH 6/8] ReqDB and FTS3DB compatible with SqlAlchemy 1.4 --- DataManagementSystem/DB/FTS3DB.py | 32 +++++++++++++++++++++---- RequestManagementSystem/DB/RequestDB.py | 3 ++- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/DataManagementSystem/DB/FTS3DB.py b/DataManagementSystem/DB/FTS3DB.py index 9e995156604..17c6b394a0c 100644 --- a/DataManagementSystem/DB/FTS3DB.py +++ b/DataManagementSystem/DB/FTS3DB.py @@ -137,6 +137,18 @@ ) +# About synchronize_session: +# The FTS3DB class uses SQLAlchemy in a mixed mode: +# both the ORM and the Core style. +# Up to 1.3, the `session.execute` statements had no ORM functionality, +# meaning that the session cache were not updated when using `update` or `delete` +# Although it could be an issue, it was not really one in our case, since we always close +# the session +# As of 1.4, `session.execute` supports ORM functionality, and thus needs more info to know +# how to manage `update` or `delete`. Hense the `synchronize_session` option. +# We set it to `False` simply because we do not rely on the session cache. +# Please see https://github.com/sqlalchemy/sqlalchemy/discussions/6159 for detailed discussion + ######################################################################## class FTS3DB(object): """ @@ -309,6 +321,7 @@ def getActiveJobs(self, limit=20, lastMonitor=None, jobAssignmentTag="Assigned") .where(FTS3Job.jobID.in_(jobIds) ) .values({'assignment': jobAssignmentTag}) + .execution_options(synchronize_session=False) # see comment about synchronize_session ) session.commit() @@ -381,7 +394,8 @@ def updateFileStatus(self, fileStatusDict, ftsGUID=None): updateQuery = update(FTS3File)\ .where(and_(*whereConditions) )\ - .values(updateDict) + .values(updateDict)\ + .execution_options(synchronize_session=False) # see comment about synchronize_session session.execute(updateQuery) @@ -432,6 +446,7 @@ def updateJobStatus(self, jobStatusDict): ) ) .values(updateDict) + .execution_options(synchronize_session=False) # see comment about synchronize_session ) session.commit() @@ -477,7 +492,8 @@ def cancelNonExistingJob(self, operationID, ftsGUID): .where(and_(FTS3File.operationID == FTS3Job.operationID, FTS3File.ftsGUID == FTS3Job.ftsGUID, FTS3Job.operationID == operationID, - FTS3Job.ftsGUID == ftsGUID)) + FTS3Job.ftsGUID == ftsGUID))\ + .execution_options(synchronize_session=False) # see comment about synchronize_session session.execute(updStmt) session.commit() @@ -538,6 +554,7 @@ def getNonFinishedOperations(self, limit=20, operationAssignmentTag="Assigned"): .where(FTS3Operation.operationID.in_(operationIDs) ) .values({'assignment': operationAssignmentTag}) + .execution_options(synchronize_session=False) # see comment about synchronize_session ) session.commit() @@ -585,7 +602,9 @@ def kickStuckOperations(self, limit=20, kickDelay=2): 'INTERVAL %d HOUR' % kickDelay)))) .values( { - 'assignment': None})) + 'assignment': None}) + .execution_options(synchronize_session=False) # see comment about synchronize_session + ) rowCount = result.rowcount session.commit() @@ -633,7 +652,9 @@ def kickStuckJobs(self, limit=20, kickDelay=2): 'INTERVAL %d HOUR' % kickDelay)))) .values( { - 'assignment': None})) + 'assignment': None}) + .execution_options(synchronize_session=False) # see comment about synchronize_session + ) rowCount = result.rowcount session.commit() @@ -674,7 +695,8 @@ def deleteFinalOperations(self, limit=20, deleteDelay=180): rowCount = 0 if opIDs: result = session.execute(delete(FTS3Operation) - .where(FTS3Operation.operationID.in_(opIDs))) + .where(FTS3Operation.operationID.in_(opIDs)) + .execution_options(synchronize_session=False)) rowCount = result.rowcount session.commit() diff --git a/RequestManagementSystem/DB/RequestDB.py b/RequestManagementSystem/DB/RequestDB.py index a1e1acaf70b..07412ea73e1 100644 --- a/RequestManagementSystem/DB/RequestDB.py +++ b/RequestManagementSystem/DB/RequestDB.py @@ -201,7 +201,8 @@ def cancelRequest(self, requestID): .where(Request.RequestID == requestID) .values({Request._Status: 'Canceled', Request._LastUpdate: datetime.datetime.utcnow() - .strftime(Request._datetimeFormat)})) + .strftime(Request._datetimeFormat)}) + .execution_options(synchronize_session=False)) # See FTS3DB for synchronize_session session.commit() # No row was changed From 763f10bb6f5e87281fa299dbc77fd0ff0e0d0e34 Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Wed, 31 Mar 2021 21:17:41 +0200 Subject: [PATCH 7/8] v7r0p53 notes and tags --- __init__.py | 2 +- release.notes | 5 +++++ setup.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/__init__.py b/__init__.py index 9a35362a29b..7c781a752ed 100755 --- a/__init__.py +++ b/__init__.py @@ -95,7 +95,7 @@ else: majorVersion = 7 minorVersion = 0 - patchLevel = 52 + patchLevel = 53 preVersion = 0 version = "v%sr%s" % (majorVersion, minorVersion) diff --git a/release.notes b/release.notes index 5b684dddfe7..c2929af5f0c 100644 --- a/release.notes +++ b/release.notes @@ -1,3 +1,8 @@ +[v7r0p53] + +*Interface +FIX: (#5069) parseArguments strip arguments correctly + [v7r0p52] *Core diff --git a/setup.py b/setup.py index d9c9f1bae0b..1a56f343ca1 100755 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ setup( name="DIRAC", - version="7.0.52", + version="7.0.53", url="https://github.com/DIRACGRID/DIRAC", license="GPLv3", package_dir=package_dir, From 4e269d4b2f18186af2e5703546ad5556e928eb2e Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Wed, 31 Mar 2021 21:47:06 +0200 Subject: [PATCH 8/8] v7r1p36 notes and tags --- __init__.py | 2 +- release.notes | 12 ++++++++++++ setup.py | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/__init__.py b/__init__.py index 50be2093726..dd24f6719bc 100755 --- a/__init__.py +++ b/__init__.py @@ -95,7 +95,7 @@ else: majorVersion = 7 minorVersion = 1 - patchLevel = 35 + patchLevel = 36 preVersion = 0 version = "v%sr%s" % (majorVersion, minorVersion) diff --git a/release.notes b/release.notes index 73ce836d21b..baa93a819d6 100644 --- a/release.notes +++ b/release.notes @@ -1,3 +1,15 @@ +[v7r1p36] + +*DMS +FIX: (#5080) SQLAlchemy 1.4 support for FTS3DB +FIX: (#5054) Submit FTS3 jobs with lowercase checksum + +*RMS +FIX: (#5080) SQLAlchemy 1.4 support for ReqDB + +*Resources +FIX: (#5077) Host.py - Fix for the case where no subprocess32 module found + [v7r1p35] FIX: Fixes from v7r0p52 diff --git a/setup.py b/setup.py index 54e80657114..4ff9018f0d7 100755 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ setup( name="DIRAC", - version="7.1.35", + version="7.1.36", url="https://github.com/DIRACGRID/DIRAC", license="GPLv3", package_dir=package_dir,