Conflicts resolved with rel-v7r1

environment.yml

@@ -33,7 +33,8 @@ dependencies:
   - recommonmark
   - requests >=2.9.1
   - six >=1.10
-  - sqlalchemy >=1.0.9
+  # Freeze until all problems with 1.4 are solved
+  - sqlalchemy ==1.3.*
   - subprocess32
   - stomp.py =4.1.23
   - suds-jurko >=0.6

release.notes

@@ -169,6 +169,15 @@ NEW: (#4910) --runslow option on unit tests to allow faster local tests
 NEW: (#4938) added a helloworld test for the (yet to be implemented) cloud testing in certification
 CHANGE: (#4968) Change the defaults for tests (to MySQL 8 and ES 7)
 
+[v7r1p38]
+
+FIX: fixes from v7r0p55
+
+*WMS
+FIX: (#5108) delete the local output sandbox tarfile in case it's still there
+FIX: (#5106) SiteDirector - use | operand for the union of tests
+CHANGE: (#5104) JobDB.setJobAttributes: use _update instead of _transaction (as not needed)
+
 [v7r1p37]
 
 *DataManagement
@@ -789,6 +798,12 @@ FIX: (#4551) align ProxyDB test to current changes
 NEW: (#4289) Document how to run integration tests in docker
 NEW: (#4551) add DNProperties description to Registry/Users subsection
 
+[v7r0p55]
+
+*TS
+FIX: (#5109) DataRecoveryAgent: immediately skip transformations without jobs, prevents
+     exception later on
+
 [v7r0p54]
 
 *Core

requirements.txt

@@ -45,7 +45,8 @@ recommonmark
 requests>=2.9.1
 simplejson>=3.8.1
 six>=1.10
-sqlalchemy>=1.0.9
+# Freeze until all problems with 1.4 are solved
+sqlalchemy==1.3.*
 xmltodict
 # more recent version are python 3 only
 stomp.py==4.1.23

src/DIRAC/Core/Utilities/test/Test_Mail.py

@@ -7,82 +7,62 @@
 from __future__ import division
 from __future__ import print_function
 
-#pylint: disable=protected-access,invalid-name,missing-docstring
-
-import unittest
-
-# sut
-from  DIRAC.Core.Utilities.Mail import Mail
+from DIRAC.Core.Utilities.Mail import Mail
 
 __RCSID__ = "$Id $"
 
-########################################################################
-class MailTestCase(unittest.TestCase):
-  """ Test case for DIRAC.Core.Utilities.Mail module
-  """
-  pass
-
-class MailEQ(MailTestCase):
-
-
-  def test_createEmail(self):
-    """ test _create
-    """
-    m = Mail()
-    res = m._create('address@dirac.org')
-    self.assertFalse(res['OK'])
-
-    m._subject = 'subject'
-    m._fromAddress = 'from@dirac.org'
-    m._mailAddress = 'address@dirac.org'
-    m._message = 'This is a message'
-    res = m._create('address@dirac.org')
-    self.assertTrue(res['OK'])
-    self.assertEqual(res['Value'].__dict__['_headers'],
-                     [('Content-Type', 'multipart/mixed'),
-                      ('MIME-Version', '1.0'),
-                      ('Subject', 'subject'),
-                      ('From', 'from@dirac.org'),
-                      ('To', 'address@dirac.org')])
-
-  def test_compareEmails(self):
-    """ test comparing of Email objects (also for insertion in sets)
-    """
-    m1 = Mail()
-    m2 = Mail()
-    self.assertEqual(m1, m2)
-
-    m1 = Mail()
-    m1._subject = 'subject'
-    m1._fromAddress = 'from@dirac.org'
-    m1._mailAddress = 'address@dirac.org'
-    m1._message = 'This is a message'
-    m2 = Mail()
-    m2._subject = 'subject'
-    m2._fromAddress = 'from@dirac.org'
-    m2._mailAddress = 'address@dirac.org'
-    m2._message = 'This is a message'
-    self.assertEqual(m1, m2)
-    m3 = Mail()
-    m3._subject = 'subject'
-    m3._fromAddress = 'from@dirac.org'
-    m3._mailAddress = 'address@dirac.org'
-    m3._message = 'This is a message a bit different'
-    self.assertNotEqual(m1, m3)
-
-    s = set()
-    s.add(m1)
-    s.add(m2)
-    self.assertTrue(len(s) == 1)
-    s.add(m2)
-    self.assertTrue(len(s) == 1)
-    s.add(m3)
-    self.assertTrue(len(s) == 2)
-    s.add(m3)
-    self.assertTrue(len(s) == 2)
-
 
-if __name__ == '__main__':
-  suite = unittest.defaultTestLoader.loadTestsFromTestCase(MailTestCase)
-  suite.addTest( unittest.defaultTestLoader.loadTestsFromTestCase(MailEQ))
-  testResult = unittest.TextTestRunner( verbosity = 2 ).run( suite )
+def test_createEmail():
+  m = Mail()
+  res = m._create('address@dirac.org')
+  assert not res['OK']
+
+  m._subject = 'subject'
+  m._fromAddress = 'from@dirac.org'
+  m._mailAddress = 'address@dirac.org'
+  m._message = 'This is a message'
+  res = m._create('address@dirac.org')
+  assert res['OK']
+  assert res['Value'].__dict__['_headers'] == [
+      ('Content-Type', 'multipart/mixed'),
+      ('MIME-Version', '1.0'),
+      ('Subject', 'subject'),
+      ('From', 'from@dirac.org'),
+      ('To', 'address@dirac.org')
+  ]
+
+
+def test_compareEmails(monkeypatch):
+  # The hostname on GitHub actions can change randomly so mock it
+  monkeypatch.setattr("socket.getfqdn", lambda: "localhost.example")
+
+  m1 = Mail()
+  m2 = Mail()
+  assert m1 == m2, (m1.__dict__, m2.__dict__)
+
+  m1 = Mail()
+  m1._subject = 'subject'
+  m1._fromAddress = 'from@dirac.org'
+  m1._mailAddress = 'address@dirac.org'
+  m1._message = 'This is a message'
+  m2 = Mail()
+  m2._subject = 'subject'
+  m2._fromAddress = 'from@dirac.org'
+  m2._mailAddress = 'address@dirac.org'
+  m2._message = 'This is a message'
+  assert m1 == m2
+  m3 = Mail()
+  m3._subject = 'subject'
+  m3._fromAddress = 'from@dirac.org'
+  m3._mailAddress = 'address@dirac.org'
+  m3._message = 'This is a message a bit different'
+  assert m1 != m3
+
+  s = {m1, m2}
+  assert len(s) == 1
+  s.add(m2)
+  assert len(s) == 1
+  s.add(m3)
+  assert len(s) == 2
+  s.add(m3)
+  assert len(s) == 2

src/DIRAC/TransformationSystem/Agent/DataRecoveryAgent.py

@@ -337,6 +337,10 @@ def treatTransformation(self, transID, transInfoDict):
                                self.tClient, self.fcClient, self.jobMon)
     jobs, nDone, nFailed = tInfo.getJobs(statusList=self.jobStatus)
 
+    if not jobs:
+      self.log.notice('Skipping. No jobs for transformation', str(transID))
+      return
+
     if self.jobCache[transID][0] == nDone and self.jobCache[transID][1] == nFailed:
       self.log.notice('Skipping transformation %s because nothing changed' % transID)
       return

src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py

@@ -328,7 +328,7 @@ def getQueues(self, resourceDict):
               self.queueDict[queueName]['ParametersDict'][tagFieldName] = queueTags
             if ceTags:
               if queueTags:
-                allTags = list(set(ceTags) + set(queueTags))
+                allTags = list(set(ceTags) | set(queueTags))
                 self.queueDict[queueName]['ParametersDict'][tagFieldName] = allTags
               else:
                 self.queueDict[queueName]['ParametersDict'][tagFieldName] = ceTags

src/DIRAC/WorkloadManagementSystem/DB/JobDB.py

@@ -676,7 +676,7 @@ def setJobAttributes(self, jobID, attrNames, attrValues, update=False, myDate=No
     if myDate:
       cmd += ' AND LastUpdateTime < %s' % myDate
 
-    return self._transaction([cmd])
+    return self._update(cmd)
 
 #############################################################################
   def setJobStatus(self, jobID, status='', minorStatus='', applicationStatus='', minor=None, application=None):

src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py

@@ -788,11 +788,11 @@ def processJobOutputs(self):
       self.outputSandboxSize = getGlobbedTotalSize(fileList)
       self.log.info('Attempting to upload Sandbox with limit:', self.sandboxSizeLimit)
       sandboxClient = SandboxStoreClient()
-      result = sandboxClient.uploadFilesAsSandboxForJob(fileList, self.jobID,
-                                                        'Output', self.sandboxSizeLimit)  # 1024*1024*10
-      if not result['OK']:
-        self.log.error('Output sandbox upload failed with message', result['Message'])
-        outputSandboxData = result.get('SandboxFileName')
+      result_sbUpload = sandboxClient.uploadFilesAsSandboxForJob(
+          fileList, self.jobID, 'Output', self.sandboxSizeLimit)  # 1024*1024*10
+      if not result_sbUpload['OK']:
+        self.log.error('Output sandbox upload failed with message', result_sbUpload['Message'])
+        outputSandboxData = result_sbUpload.get('SandboxFileName')
         if outputSandboxData:
 
           self.log.info('Attempting to upload %s as output data' % (outputSandboxData))
@@ -829,9 +829,21 @@ def processJobOutputs(self):
       if not outputSE and not self.defaultFailoverSE:
         return S_ERROR('No output SEs defined in VO configuration')
 
-      result = self.__transferOutputDataFiles(outputData, outputSE, outputPath)
-      if not result['OK']:
-        return result
+      result_transferODF = self.__transferOutputDataFiles(outputData, outputSE, outputPath)
+
+      # now that we (tried to) transfer the output files,
+      # including possibly oversized Output Sandboxes,
+      # we delete the local output sandbox tarfile in case it's still there.
+      if not result_sbUpload['OK']:
+        outputSandboxData = result_sbUpload.get('SandboxFileName')
+        if outputSandboxData:
+          try:
+            os.unlink(outputSandboxData)
+          except OSError:
+            pass
+
+      if not result_transferODF['OK']:
+        return result_transferODF
 
     return S_OK('Job outputs processed')