From 13906fd785a831487bb2ec7a6e6a8391abd0b887 Mon Sep 17 00:00:00 2001 From: fstagni Date: Fri, 16 Apr 2021 16:45:02 +0200 Subject: [PATCH 1/9] use _update instead of _transaction --- WorkloadManagementSystem/DB/JobDB.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WorkloadManagementSystem/DB/JobDB.py b/WorkloadManagementSystem/DB/JobDB.py index e097ab55ddd..36418e4820e 100755 --- a/WorkloadManagementSystem/DB/JobDB.py +++ b/WorkloadManagementSystem/DB/JobDB.py @@ -699,7 +699,7 @@ def setJobAttributes(self, jobID, attrNames, attrValues, update=False, myDate=No if myDate: cmd += ' AND LastUpdateTime < %s' % myDate - return self._transaction([cmd]) + return self._update(cmd) ############################################################################# def setJobStatus(self, jobID, status='', minorStatus='', applicationStatus='', minor=None, application=None): From c242d1dc6bf6435e36e72d68b44e266fc170388e Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Sun, 18 Apr 2021 13:32:52 +0200 Subject: [PATCH 2/9] Use | for the union of sets --- WorkloadManagementSystem/Agent/SiteDirector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WorkloadManagementSystem/Agent/SiteDirector.py b/WorkloadManagementSystem/Agent/SiteDirector.py index 6f892073c8b..0af1e8778fc 100644 --- a/WorkloadManagementSystem/Agent/SiteDirector.py +++ b/WorkloadManagementSystem/Agent/SiteDirector.py @@ -353,7 +353,7 @@ def getQueues(self, resourceDict): self.queueDict[queueName]['ParametersDict'][tagFieldName] = queueTags if ceTags: if queueTags: - allTags = list(set(ceTags) + set(queueTags)) + allTags = list(set(ceTags) | set(queueTags)) self.queueDict[queueName]['ParametersDict'][tagFieldName] = allTags else: self.queueDict[queueName]['ParametersDict'][tagFieldName] = ceTags From a8ffd415ad66239b9f94648650d84cd4ed08b115 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Mon, 19 Apr 2021 11:03:58 +0200 Subject: [PATCH 3/9] Freeze SQLALchemy to 1.3 until problems with 1.4 are solved --- docs/requirements.txt | 3 ++- docs/requirements_py3.txt | 3 ++- environment.yml | 3 ++- requirements.txt | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 1f31354e7a5..397440010ff 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -19,6 +19,7 @@ pyasn1_modules pyparsing pytz recommonmark -sqlalchemy==1.3.23 +# Freeze until all problems with 1.4 are solved +sqlalchemy==1.3.* subprocess32 suds diff --git a/docs/requirements_py3.txt b/docs/requirements_py3.txt index 2bcdd1038ac..a2c9075a5b3 100644 --- a/docs/requirements_py3.txt +++ b/docs/requirements_py3.txt @@ -16,6 +16,7 @@ pyasn1_modules pyparsing pytz recommonmark -sqlalchemy +# Freeze until all problems with 1.4 are solved +sqlalchemy==1.3.* subprocess32 suds-community # replacing suds-jurko diff --git a/environment.yml b/environment.yml index a28d0f0236a..d5df47ad7df 100644 --- a/environment.yml +++ b/environment.yml @@ -33,7 +33,8 @@ dependencies: - recommonmark - requests >=2.9.1 - six >=1.10 - - sqlalchemy >=1.0.9 + # Freeze until all problems with 1.4 are solved + - sqlalchemy ==1.3.* - subprocess32 - stomp.py =4.1.23 - suds-jurko >=0.6 diff --git a/requirements.txt b/requirements.txt index 1bbb5f8e9e2..f134560bafb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,7 +41,8 @@ recommonmark requests>=2.9.1 simplejson>=3.8.1 six>=1.10 -sqlalchemy>=1.0.9 +# Freeze until all problems with 1.4 are solved +sqlalchemy==1.3.* xmltodict # more recent version are python 3 only stomp.py==4.1.23 From bc07dafd027608a06cc5b2393e5fae0ea23a8280 Mon Sep 17 00:00:00 2001 From: fstagni Date: Tue, 20 Apr 2021 12:12:09 +0200 Subject: [PATCH 4/9] delete the local output sandbox tarfile in case it's still there --- .../JobWrapper/JobWrapper.py | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/WorkloadManagementSystem/JobWrapper/JobWrapper.py b/WorkloadManagementSystem/JobWrapper/JobWrapper.py index b6640b9ae64..3d1d1dba608 100755 --- a/WorkloadManagementSystem/JobWrapper/JobWrapper.py +++ b/WorkloadManagementSystem/JobWrapper/JobWrapper.py @@ -783,11 +783,11 @@ def processJobOutputs(self): self.outputSandboxSize = getGlobbedTotalSize(fileList) self.log.info('Attempting to upload Sandbox with limit:', self.sandboxSizeLimit) sandboxClient = SandboxStoreClient() - result = sandboxClient.uploadFilesAsSandboxForJob(fileList, self.jobID, - 'Output', self.sandboxSizeLimit) # 1024*1024*10 - if not result['OK']: - self.log.error('Output sandbox upload failed with message', result['Message']) - outputSandboxData = result.get('SandboxFileName') + result_sbUpload = sandboxClient.uploadFilesAsSandboxForJob( + fileList, self.jobID, 'Output', self.sandboxSizeLimit) # 1024*1024*10 + if not result_sbUpload['OK']: + self.log.error('Output sandbox upload failed with message', result_sbUpload['Message']) + outputSandboxData = result_sbUpload.get('SandboxFileName') if outputSandboxData: self.log.info('Attempting to upload %s as output data' % (outputSandboxData)) @@ -822,9 +822,21 @@ def processJobOutputs(self): if not outputSE and not self.defaultFailoverSE: return S_ERROR('No output SEs defined in VO configuration') - result = self.__transferOutputDataFiles(outputData, outputSE, outputPath) - if not result['OK']: - return result + result_transferODF = self.__transferOutputDataFiles(outputData, outputSE, outputPath) + + # now that we (tried to) transfer the output files, + # including possibly oversized Output Sandboxes, + # we delete the local output sandbox tarfile in case it's still there. + if not result_sbUpload['OK']: + outputSandboxData = result_sbUpload.get('SandboxFileName') + if outputSandboxData: + try: + os.unlink(outputSandboxData) + except OSError: + pass + + if not result_transferODF['OK']: + return result_transferODF return S_OK('Job outputs processed') From 13ccdb97c5acc29e510b233d95797adf3195ae42 Mon Sep 17 00:00:00 2001 From: Andre Sailer Date: Tue, 20 Apr 2021 13:40:40 +0200 Subject: [PATCH 5/9] DRA: don't run over transformation without jobs, prevents exception down the line --- TransformationSystem/Agent/DataRecoveryAgent.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/TransformationSystem/Agent/DataRecoveryAgent.py b/TransformationSystem/Agent/DataRecoveryAgent.py index d45b082bde2..a1bf1f06b2d 100644 --- a/TransformationSystem/Agent/DataRecoveryAgent.py +++ b/TransformationSystem/Agent/DataRecoveryAgent.py @@ -333,6 +333,10 @@ def treatTransformation(self, transID, transInfoDict): self.tClient, self.fcClient, self.jobMon) jobs, nDone, nFailed = tInfo.getJobs(statusList=self.jobStatus) + if not jobs: + self.log.notice('Skipping. No jobs for transformation', str(transID)) + return + if self.jobCache[transID][0] == nDone and self.jobCache[transID][1] == nFailed: self.log.notice('Skipping transformation %s because nothing changed' % transID) return From 8ce51c2dc38443375fb95e04e56552a141364728 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Tue, 20 Apr 2021 19:50:56 +0200 Subject: [PATCH 6/9] Refactor Test_Mail to use pytest --- Core/Utilities/test/Test_Mail.py | 130 +++++++++++++------------------ 1 file changed, 55 insertions(+), 75 deletions(-) diff --git a/Core/Utilities/test/Test_Mail.py b/Core/Utilities/test/Test_Mail.py index c633c16bdb9..9d62f578e7b 100644 --- a/Core/Utilities/test/Test_Mail.py +++ b/Core/Utilities/test/Test_Mail.py @@ -3,83 +3,63 @@ Test cases for DIRAC.Core.Utilities.DAG module. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function -#pylint: disable=protected-access,invalid-name,missing-docstring - -import unittest - -# sut -from DIRAC.Core.Utilities.Mail import Mail +from DIRAC.Core.Utilities.Mail import Mail __RCSID__ = "$Id $" -######################################################################## -class MailTestCase(unittest.TestCase): - """ Test case for DIRAC.Core.Utilities.Mail module - """ - pass - -class MailEQ(MailTestCase): - - - def test_createEmail(self): - """ test _create - """ - m = Mail() - res = m._create('address@dirac.org') - self.assertFalse(res['OK']) - - m._subject = 'subject' - m._fromAddress = 'from@dirac.org' - m._mailAddress = 'address@dirac.org' - m._message = 'This is a message' - res = m._create('address@dirac.org') - self.assertTrue(res['OK']) - self.assertEqual(res['Value'].__dict__['_headers'], - [('Content-Type', 'multipart/mixed'), - ('MIME-Version', '1.0'), - ('Subject', 'subject'), - ('From', 'from@dirac.org'), - ('To', 'address@dirac.org')]) - - def test_compareEmails(self): - """ test comparing of Email objects (also for insertion in sets) - """ - m1 = Mail() - m2 = Mail() - self.assertEqual(m1, m2) - - m1 = Mail() - m1._subject = 'subject' - m1._fromAddress = 'from@dirac.org' - m1._mailAddress = 'address@dirac.org' - m1._message = 'This is a message' - m2 = Mail() - m2._subject = 'subject' - m2._fromAddress = 'from@dirac.org' - m2._mailAddress = 'address@dirac.org' - m2._message = 'This is a message' - self.assertEqual(m1, m2) - m3 = Mail() - m3._subject = 'subject' - m3._fromAddress = 'from@dirac.org' - m3._mailAddress = 'address@dirac.org' - m3._message = 'This is a message a bit different' - self.assertNotEqual(m1, m3) - - s = set() - s.add(m1) - s.add(m2) - self.assertTrue(len(s) == 1) - s.add(m2) - self.assertTrue(len(s) == 1) - s.add(m3) - self.assertTrue(len(s) == 2) - s.add(m3) - self.assertTrue(len(s) == 2) - -if __name__ == '__main__': - suite = unittest.defaultTestLoader.loadTestsFromTestCase(MailTestCase) - suite.addTest( unittest.defaultTestLoader.loadTestsFromTestCase(MailEQ)) - testResult = unittest.TextTestRunner( verbosity = 2 ).run( suite ) +def test_createEmail(): + m = Mail() + res = m._create('address@dirac.org') + assert not res['OK'] + + m._subject = 'subject' + m._fromAddress = 'from@dirac.org' + m._mailAddress = 'address@dirac.org' + m._message = 'This is a message' + res = m._create('address@dirac.org') + assert res['OK'] + assert res['Value'].__dict__['_headers'] == [ + ('Content-Type', 'multipart/mixed'), + ('MIME-Version', '1.0'), + ('Subject', 'subject'), + ('From', 'from@dirac.org'), + ('To', 'address@dirac.org') + ] + + +def test_compareEmails(): + m1 = Mail() + m2 = Mail() + assert m1 == m2, (m1.__dict__, m2.__dict__) + + m1 = Mail() + m1._subject = 'subject' + m1._fromAddress = 'from@dirac.org' + m1._mailAddress = 'address@dirac.org' + m1._message = 'This is a message' + m2 = Mail() + m2._subject = 'subject' + m2._fromAddress = 'from@dirac.org' + m2._mailAddress = 'address@dirac.org' + m2._message = 'This is a message' + assert m1 == m2 + m3 = Mail() + m3._subject = 'subject' + m3._fromAddress = 'from@dirac.org' + m3._mailAddress = 'address@dirac.org' + m3._message = 'This is a message a bit different' + assert m1 != m3 + + s = {m1, m2} + assert len(s) == 1 + s.add(m2) + assert len(s) == 1 + s.add(m3) + assert len(s) == 2 + s.add(m3) + assert len(s) == 2 From c6275f309cb1fdc6521934bd84e4ccd568fc7c5c Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Tue, 20 Apr 2021 19:53:16 +0200 Subject: [PATCH 7/9] Monkeypatch socket.getfqdn in test_compareEmails to avoid random failures due to the hostname changing during the test --- Core/Utilities/test/Test_Mail.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Core/Utilities/test/Test_Mail.py b/Core/Utilities/test/Test_Mail.py index 9d62f578e7b..69372497de1 100644 --- a/Core/Utilities/test/Test_Mail.py +++ b/Core/Utilities/test/Test_Mail.py @@ -32,7 +32,10 @@ def test_createEmail(): ] -def test_compareEmails(): +def test_compareEmails(monkeypatch): + # The hostname on GitHub actions can change randomly so mock it + monkeypatch.setattr("socket.getfqdn", lambda: "localhost.example") + m1 = Mail() m2 = Mail() assert m1 == m2, (m1.__dict__, m2.__dict__) From 1664d0c65773096312488c4662b6f685594b5523 Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Wed, 21 Apr 2021 21:16:11 +0200 Subject: [PATCH 8/9] v7r0p55 notes and tags --- __init__.py | 2 +- release.notes | 6 ++++++ setup.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/__init__.py b/__init__.py index e25c40f0dfb..e4478be06af 100755 --- a/__init__.py +++ b/__init__.py @@ -95,7 +95,7 @@ else: majorVersion = 7 minorVersion = 0 - patchLevel = 54 + patchLevel = 55 preVersion = 0 version = "v%sr%s" % (majorVersion, minorVersion) diff --git a/release.notes b/release.notes index 09f71ae6c78..25b6d42df33 100644 --- a/release.notes +++ b/release.notes @@ -1,3 +1,9 @@ +[v7r0p55] + +*TS +FIX: (#5109) DataRecoveryAgent: immediately skip transformations without jobs, prevents + exception later on + [v7r0p54] *Core diff --git a/setup.py b/setup.py index 1eb55e85a03..e1e1c02b514 100755 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ setup( name="DIRAC", - version="7.0.54", + version="7.0.55", url="https://github.com/DIRACGRID/DIRAC", license="GPLv3", package_dir=package_dir, From 06c6aefd72308e0c87b771bb21d7a093b0c404a2 Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Wed, 21 Apr 2021 21:24:02 +0200 Subject: [PATCH 9/9] v7r1p38 notes and tags --- __init__.py | 2 +- release.notes | 7 +++++++ setup.py | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/__init__.py b/__init__.py index 5248976e5f5..f22de1a5ed9 100755 --- a/__init__.py +++ b/__init__.py @@ -95,7 +95,7 @@ else: majorVersion = 7 minorVersion = 1 - patchLevel = 37 + patchLevel = 38 preVersion = 0 version = "v%sr%s" % (majorVersion, minorVersion) diff --git a/release.notes b/release.notes index bed5798225b..b1609e5a739 100644 --- a/release.notes +++ b/release.notes @@ -1,3 +1,10 @@ +[v7r1p38] + +*WMS +FIX: (#5108) delete the local output sandbox tarfile in case it's still there +FIX: (#5106) SiteDirector - use | operand for the union of tests +CHANGE: (#5104) JobDB.setJobAttributes: use _update instead of _transaction (as not needed) + [v7r1p37] *DataManagement diff --git a/setup.py b/setup.py index 18dc20d04e7..ffe819e3e71 100755 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ setup( name="DIRAC", - version="7.1.37", + version="7.1.38", url="https://github.com/DIRACGRID/DIRAC", license="GPLv3", package_dir=package_dir,