diff --git a/environment.yml b/environment.yml index f5f892e7e14..131b8bf0a1a 100644 --- a/environment.yml +++ b/environment.yml @@ -33,7 +33,8 @@ dependencies: - recommonmark - requests >=2.9.1 - six >=1.10 - - sqlalchemy >=1.0.9 + # Freeze until all problems with 1.4 are solved + - sqlalchemy ==1.3.* - subprocess32 - stomp.py =4.1.23 - suds-jurko >=0.6 diff --git a/release.notes b/release.notes index d64b7d3fcf6..7fbdbb347ca 100644 --- a/release.notes +++ b/release.notes @@ -169,6 +169,15 @@ NEW: (#4910) --runslow option on unit tests to allow faster local tests NEW: (#4938) added a helloworld test for the (yet to be implemented) cloud testing in certification CHANGE: (#4968) Change the defaults for tests (to MySQL 8 and ES 7) +[v7r1p38] + +FIX: fixes from v7r0p55 + +*WMS +FIX: (#5108) delete the local output sandbox tarfile in case it's still there +FIX: (#5106) SiteDirector - use | operand for the union of tests +CHANGE: (#5104) JobDB.setJobAttributes: use _update instead of _transaction (as not needed) + [v7r1p37] *DataManagement @@ -789,6 +798,12 @@ FIX: (#4551) align ProxyDB test to current changes NEW: (#4289) Document how to run integration tests in docker NEW: (#4551) add DNProperties description to Registry/Users subsection +[v7r0p55] + +*TS +FIX: (#5109) DataRecoveryAgent: immediately skip transformations without jobs, prevents + exception later on + [v7r0p54] *Core diff --git a/requirements.txt b/requirements.txt index 934286c952e..c910c22a06a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,7 +45,8 @@ recommonmark requests>=2.9.1 simplejson>=3.8.1 six>=1.10 -sqlalchemy>=1.0.9 +# Freeze until all problems with 1.4 are solved +sqlalchemy==1.3.* xmltodict # more recent version are python 3 only stomp.py==4.1.23 diff --git a/src/DIRAC/Core/Utilities/test/Test_Mail.py b/src/DIRAC/Core/Utilities/test/Test_Mail.py index 9cb971f1c25..69372497de1 100644 --- a/src/DIRAC/Core/Utilities/test/Test_Mail.py +++ b/src/DIRAC/Core/Utilities/test/Test_Mail.py @@ -7,82 +7,62 @@ from __future__ import division from __future__ import print_function -#pylint: disable=protected-access,invalid-name,missing-docstring - -import unittest - -# sut -from DIRAC.Core.Utilities.Mail import Mail +from DIRAC.Core.Utilities.Mail import Mail __RCSID__ = "$Id $" -######################################################################## -class MailTestCase(unittest.TestCase): - """ Test case for DIRAC.Core.Utilities.Mail module - """ - pass - -class MailEQ(MailTestCase): - - - def test_createEmail(self): - """ test _create - """ - m = Mail() - res = m._create('address@dirac.org') - self.assertFalse(res['OK']) - - m._subject = 'subject' - m._fromAddress = 'from@dirac.org' - m._mailAddress = 'address@dirac.org' - m._message = 'This is a message' - res = m._create('address@dirac.org') - self.assertTrue(res['OK']) - self.assertEqual(res['Value'].__dict__['_headers'], - [('Content-Type', 'multipart/mixed'), - ('MIME-Version', '1.0'), - ('Subject', 'subject'), - ('From', 'from@dirac.org'), - ('To', 'address@dirac.org')]) - - def test_compareEmails(self): - """ test comparing of Email objects (also for insertion in sets) - """ - m1 = Mail() - m2 = Mail() - self.assertEqual(m1, m2) - - m1 = Mail() - m1._subject = 'subject' - m1._fromAddress = 'from@dirac.org' - m1._mailAddress = 'address@dirac.org' - m1._message = 'This is a message' - m2 = Mail() - m2._subject = 'subject' - m2._fromAddress = 'from@dirac.org' - m2._mailAddress = 'address@dirac.org' - m2._message = 'This is a message' - self.assertEqual(m1, m2) - m3 = Mail() - m3._subject = 'subject' - m3._fromAddress = 'from@dirac.org' - m3._mailAddress = 'address@dirac.org' - m3._message = 'This is a message a bit different' - self.assertNotEqual(m1, m3) - - s = set() - s.add(m1) - s.add(m2) - self.assertTrue(len(s) == 1) - s.add(m2) - self.assertTrue(len(s) == 1) - s.add(m3) - self.assertTrue(len(s) == 2) - s.add(m3) - self.assertTrue(len(s) == 2) - -if __name__ == '__main__': - suite = unittest.defaultTestLoader.loadTestsFromTestCase(MailTestCase) - suite.addTest( unittest.defaultTestLoader.loadTestsFromTestCase(MailEQ)) - testResult = unittest.TextTestRunner( verbosity = 2 ).run( suite ) +def test_createEmail(): + m = Mail() + res = m._create('address@dirac.org') + assert not res['OK'] + + m._subject = 'subject' + m._fromAddress = 'from@dirac.org' + m._mailAddress = 'address@dirac.org' + m._message = 'This is a message' + res = m._create('address@dirac.org') + assert res['OK'] + assert res['Value'].__dict__['_headers'] == [ + ('Content-Type', 'multipart/mixed'), + ('MIME-Version', '1.0'), + ('Subject', 'subject'), + ('From', 'from@dirac.org'), + ('To', 'address@dirac.org') + ] + + +def test_compareEmails(monkeypatch): + # The hostname on GitHub actions can change randomly so mock it + monkeypatch.setattr("socket.getfqdn", lambda: "localhost.example") + + m1 = Mail() + m2 = Mail() + assert m1 == m2, (m1.__dict__, m2.__dict__) + + m1 = Mail() + m1._subject = 'subject' + m1._fromAddress = 'from@dirac.org' + m1._mailAddress = 'address@dirac.org' + m1._message = 'This is a message' + m2 = Mail() + m2._subject = 'subject' + m2._fromAddress = 'from@dirac.org' + m2._mailAddress = 'address@dirac.org' + m2._message = 'This is a message' + assert m1 == m2 + m3 = Mail() + m3._subject = 'subject' + m3._fromAddress = 'from@dirac.org' + m3._mailAddress = 'address@dirac.org' + m3._message = 'This is a message a bit different' + assert m1 != m3 + + s = {m1, m2} + assert len(s) == 1 + s.add(m2) + assert len(s) == 1 + s.add(m3) + assert len(s) == 2 + s.add(m3) + assert len(s) == 2 diff --git a/src/DIRAC/TransformationSystem/Agent/DataRecoveryAgent.py b/src/DIRAC/TransformationSystem/Agent/DataRecoveryAgent.py index 90a18406c6b..0899a54b1e4 100644 --- a/src/DIRAC/TransformationSystem/Agent/DataRecoveryAgent.py +++ b/src/DIRAC/TransformationSystem/Agent/DataRecoveryAgent.py @@ -337,6 +337,10 @@ def treatTransformation(self, transID, transInfoDict): self.tClient, self.fcClient, self.jobMon) jobs, nDone, nFailed = tInfo.getJobs(statusList=self.jobStatus) + if not jobs: + self.log.notice('Skipping. No jobs for transformation', str(transID)) + return + if self.jobCache[transID][0] == nDone and self.jobCache[transID][1] == nFailed: self.log.notice('Skipping transformation %s because nothing changed' % transID) return diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py b/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py index 2b2ff6d077b..6f40099dcbe 100644 --- a/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py @@ -328,7 +328,7 @@ def getQueues(self, resourceDict): self.queueDict[queueName]['ParametersDict'][tagFieldName] = queueTags if ceTags: if queueTags: - allTags = list(set(ceTags) + set(queueTags)) + allTags = list(set(ceTags) | set(queueTags)) self.queueDict[queueName]['ParametersDict'][tagFieldName] = allTags else: self.queueDict[queueName]['ParametersDict'][tagFieldName] = ceTags diff --git a/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py b/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py index 860f9cd2619..c96db651a6a 100755 --- a/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py +++ b/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py @@ -676,7 +676,7 @@ def setJobAttributes(self, jobID, attrNames, attrValues, update=False, myDate=No if myDate: cmd += ' AND LastUpdateTime < %s' % myDate - return self._transaction([cmd]) + return self._update(cmd) ############################################################################# def setJobStatus(self, jobID, status='', minorStatus='', applicationStatus='', minor=None, application=None): diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py index 067e0bd02e2..27bb6011133 100755 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py @@ -788,11 +788,11 @@ def processJobOutputs(self): self.outputSandboxSize = getGlobbedTotalSize(fileList) self.log.info('Attempting to upload Sandbox with limit:', self.sandboxSizeLimit) sandboxClient = SandboxStoreClient() - result = sandboxClient.uploadFilesAsSandboxForJob(fileList, self.jobID, - 'Output', self.sandboxSizeLimit) # 1024*1024*10 - if not result['OK']: - self.log.error('Output sandbox upload failed with message', result['Message']) - outputSandboxData = result.get('SandboxFileName') + result_sbUpload = sandboxClient.uploadFilesAsSandboxForJob( + fileList, self.jobID, 'Output', self.sandboxSizeLimit) # 1024*1024*10 + if not result_sbUpload['OK']: + self.log.error('Output sandbox upload failed with message', result_sbUpload['Message']) + outputSandboxData = result_sbUpload.get('SandboxFileName') if outputSandboxData: self.log.info('Attempting to upload %s as output data' % (outputSandboxData)) @@ -829,9 +829,21 @@ def processJobOutputs(self): if not outputSE and not self.defaultFailoverSE: return S_ERROR('No output SEs defined in VO configuration') - result = self.__transferOutputDataFiles(outputData, outputSE, outputPath) - if not result['OK']: - return result + result_transferODF = self.__transferOutputDataFiles(outputData, outputSE, outputPath) + + # now that we (tried to) transfer the output files, + # including possibly oversized Output Sandboxes, + # we delete the local output sandbox tarfile in case it's still there. + if not result_sbUpload['OK']: + outputSandboxData = result_sbUpload.get('SandboxFileName') + if outputSandboxData: + try: + os.unlink(outputSandboxData) + except OSError: + pass + + if not result_transferODF['OK']: + return result_transferODF return S_OK('Job outputs processed')