Conflicts resolved with rel-v7r1

docs/source/AdministratorGuide/ServerInstallations/environment_variable_configuration.rst

@@ -35,6 +35,12 @@ DIRAC_USE_M2CRYPTO
 DIRAC_M2CRYPTO_SPLIT_HANDSHAKE
   If ``true`` or ``yes`` the SSL handshake is done in a new thread (default Yes)
 
+DIRAC_M2CRYPTO_SSL_CIPHERS
+  If set, overwrites the default SSL ciphers accepted. It should be a column separated list. See :py:mod:`DIRAC.Core.DISET`
+
+DIRAC_M2CRYPTO_SSL_METHODS
+  If set, overwrites the default SSL methods accepted. It should be a column separated list. See :py:mod:`DIRAC.Core.DISET`
+
 DIRAC_NO_CFG
   If set to anything, cfg files on the command line must be passed to the command using the --cfg option.
 

docs/source/AdministratorGuide/Tutorials/managingIdentities.rst

@@ -63,7 +63,7 @@ Testing the ``ProxyManager``
 
 The simplest way to test it is to upload your user proxy::
 
-  [diracuser@dirac-tuto ~]$ dirac-proxy-init -U
+  [diracuser@dirac-tuto ~]$ dirac-proxy-init
   Generating proxy...
   Uploading proxy for dirac_user...
   Proxy generated:

release.notes

@@ -96,6 +96,25 @@ NEW: (#4910) --runslow option on unit tests to allow faster local tests
 NEW: (#4938) added a helloworld test for the (yet to be implemented) cloud testing in certification
 CHANGE: (#4968) Change the defaults for tests (to MySQL 8 and ES 7)
 
+[v7r1p29]
+
+FIX: Fixes from v7r0p49 patch release
+
+*Core
+CHANGE: (#4954) using argparse for a few scripts instead of getopt
+NEW: (#4957) ObjectLoader: do not fail if one module fails importing in recursive loading
+
+*Framework
+CHANGE: (#4964) dirac-proxy-init always uploads the proxy unless --no-upload is specified
+FIX: (#4964) ProxyManagerClient checks for group and groupless proxies in its cache
+
+*WMS
+FIX: (#4954) PilotCStoJSONSynchronizer - bug fixed in getting CE information
+CHANGE: (#4956) JobAgent - resorting to calculate time left with info in possession
+
+*Workflow
+FIX: (#4953) ModuleBase - remove deprecated function that does not do anything
+
 [v7r1p28]
 
 CHANGE: Revert #4919 - absolute imports in several modules
@@ -608,6 +627,12 @@ FIX: (#4551) align ProxyDB test to current changes
 NEW: (#4289) Document how to run integration tests in docker
 NEW: (#4551) add DNProperties description to Registry/Users subsection
 
+[v7r0p49]
+
+*Core
+NEW: (#4967) introduce DIRAC_M2CRYPTO_SSL_METHODS to configure accepted protocols
+NEW: (#4967) introduce DIRAC_M2CRYPTO_SSL_CIPHERS to configure accepted ciphers
+
 [v7r0p48]
 
 CHANGE: (#4949) change default top-level bdii to cclcgtopbdii01.in2p3.fr (was lcg-bdii.cern.ch)

src/DIRAC/Core/DISET/__init__.py

@@ -10,3 +10,18 @@
 DEFAULT_RPC_TIMEOUT = 600
 #: Default timeout to establish a connection
 DEFAULT_CONNECTION_TIMEOUT = 10
+
+#: Default SSL Ciher accepted. Current default is for pyGSI/M2crypto compatibility
+#: Can be changed with DIRAC_M2CRYPTO_SSL_CIPHERS
+#: Recommandation (incompatible with pyGSI)
+# pylint: disable=line-too-long
+#: AES256-GCM-SHA384:AES256-SHA256:AES128-GCM-SHA256:AES128-SHA256:HIGH:MEDIUM:RSA:!3DES:!RC4:!aNULL:!eNULL:!MD5:!SEED:!IDEA:!SHA # noqa
+# Cipher line should be as readable as possible, sorry pylint
+# pylint: disable=line-too-long
+DEFAULT_SSL_CIPHERS = 'AES256-GCM-SHA384:AES256-SHA256:AES256-SHA:CAMELLIA256-SHA:AES128-GCM-SHA256:AES128-SHA256:AES128-SHA:HIGH:MEDIUM:RSA:!3DES:!RC4:!aNULL:!eNULL:!MD5:!SEED:!IDEA'  # noqa
+
+#: Default SSL methods accepted. Current default accepts TLSv1 for pyGSI/M2crypto compatibility
+#: Can be changed with DIRAC_M2CRYPTO_SSL_METHODS
+#: Recommandation (incompatible with pyGSI)
+# TLSv2:TLSv3
+DEFAULT_SSL_METHODS = 'TLSv1:TLSv2:TLSv3'

src/DIRAC/Core/DISET/private/Transports/M2SSLTransport.py

@@ -80,6 +80,15 @@ def __init__(self, *args, **kwargs):
 
     self.__locked = False  # We don't support locking, so this is always false.
 
+    # If not specified in the arguments (never is in DIRAC code...)
+    # and we are setting up a server listing connection, set the accepted
+    # ssl methods and ciphers
+    if kwargs.get('bServerMode'):
+      if 'sslMethods' not in kwargs:
+        kwargs['sslMethods'] = os.environ.get('DIRAC_M2CRYPTO_SSL_METHODS')
+      if 'sslCiphers' not in kwargs:
+        kwargs['sslCiphers'] = os.environ.get('DIRAC_M2CRYPTO_SSL_CIPHERS')
+
     self.__ctx = kwargs.pop('ctx', None)
     if not self.__ctx:
       self.__ctx = getM2SSLContext(**kwargs)

src/DIRAC/Core/DISET/private/Transports/SSL/M2Utils.py

@@ -9,13 +9,10 @@
 import os
 from M2Crypto import SSL, m2, X509
 
+from DIRAC.Core.DISET import DEFAULT_SSL_CIPHERS, DEFAULT_SSL_METHODS
 from DIRAC.Core.Security import Locations
 from DIRAC.Core.Security.m2crypto.X509Chain import X509Chain
 
-# Default ciphers to use if unspecified
-# Cipher line should be as readable as possible, sorry pylint
-# pylint: disable=line-too-long
-DEFAULT_SSL_CIPHERS = "AES256-GCM-SHA384:AES256-SHA256:AES256-SHA:CAMELLIA256-SHA:AES128-GCM-SHA256:AES128-SHA256:AES128-SHA:HIGH:MEDIUM:RSA:!3DES:!RC4:!aNULL:!eNULL:!MD5:!SEED:!IDEA"  # noqa
 # Verify depth of peer certs
 VERIFY_DEPTH = 50
 
@@ -66,7 +63,7 @@ def getM2SSLContext(ctx=None, **kwargs):
         - proxyLocation: String, Path to file to use as proxy, defaults to
                                  usual location(s) if not set.
         - skipCACheck: Boolean, if True, don't verify peer certificates.
-        - sslMethod: String, List of SSL algorithms to enable in OpenSSL style
+        - sslMethods: String, List of SSL algorithms to enable in OpenSSL style
                               cipher format, e.g. "SSLv3:TLSv1".
         - sslCiphers: String, OpenSSL style cipher string of ciphers to allow
                               on this connection.
@@ -129,15 +126,15 @@ def getM2SSLContext(ctx=None, **kwargs):
     ctx.get_cert_store().set_flags(X509.verify_allow_proxy_certs)  # pylint: disable=no-member
 
   # Other parameters
-  sslMethod = kwargs.get('sslMethod', None)
-  if sslMethod:
+  sslMethods = kwargs.get('sslMethods', DEFAULT_SSL_METHODS)
+  if sslMethods:
     # Pylint can't see the m2 constants due to the way the library is loaded
     # We just have to disable that warning for the next bit...
     # pylint: disable=no-member
     methods = [('SSLv2', m2.SSL_OP_NO_SSLv2),
                ('SSLv3', m2.SSL_OP_NO_SSLv3),
                ('TLSv1', m2.SSL_OP_NO_TLSv1)]
-    allowed_methods = sslMethod.split(':')
+    allowed_methods = sslMethods.split(':')
     # If a method isn't explicitly allowed, set the flag to disable it...
     for method, method_flag in methods:
       if method not in allowed_methods:

src/DIRAC/Core/Utilities/ObjectLoader.py

@@ -152,14 +152,16 @@ def loadObject(self, importString, objName=False, hideExceptions=False):
     except AttributeError:
       return S_ERROR(DErrno.EIMPERR, "%s does not contain a %s object" % (importString, objName))
 
-  def getObjects(self, modulePath, reFilter=None, parentClass=None, recurse=False):
+  def getObjects(self, modulePath, reFilter=None, parentClass=None, recurse=False, continueOnError=False):
     """ Search for modules under a certain path
 
         modulePath is the import string needed to access the parent module.
         Root modules will be included automatically (like DIRAC). For instance "ConfigurationSystem.Service"
 
         reFilter is a regular expression to filter what to load. For instance ".*Handler"
         parentClass is a class object from which the loaded modules have to import from. For instance RequestHandler
+
+        :param continueOnError: if True, continue loading further module even if one fails
     """
 
     if 'OrderedDict' in dir(collections):
@@ -204,6 +206,9 @@ def getObjects(self, modulePath, reFilter=None, parentClass=None, recurse=False)
         fullName = "%s.%s" % (impPath, modName)
         result = self.__recurseImport(modName, parentModule=parentModule, fullName=fullName)
         if not result['OK']:
+          if continueOnError:
+            gLogger.error("Error loading module but continueOnError is true", "module %s error %s" % (fullName, result))
+            continue
           return result
         if not result['Value']:
           continue

src/DIRAC/Core/scripts/dirac_install_extension.py

@@ -9,16 +9,13 @@
 
 """
 
-from __future__ import unicode_literals
-
+from __future__ import print_function
 from __future__ import absolute_import
 from __future__ import division
-from __future__ import print_function
 
 import os
 import sys
 import six
-import time
 import getopt
 import importlib
 

src/DIRAC/Core/scripts/dirac_version.py

@@ -19,7 +19,6 @@
 
 __RCSID__ = "$Id$"
 
-import sys
 import argparse
 
 parser = argparse.ArgumentParser(

src/DIRAC/FrameworkSystem/Client/ProxyManagerClient.py

@@ -98,15 +98,27 @@ def userHasProxy(self, userDN, userGroup, validSeconds=0):
 
         :return: S_OK()/S_ERROR()
     """
-    cacheKey = (userDN, userGroup)
-    if self.__usersCache.exists(cacheKey, validSeconds):
-      return S_OK(True)
+
+    # For backward compatibility reasons with versions prior to v7r1
+    # we need to check for proxy with a group
+    # AND for groupless proxy even if not specified
+
+    cacheKeys = ((userDN, userGroup), (userDN, ''))
+    for cacheKey in cacheKeys:
+      if self.__usersCache.exists(cacheKey, validSeconds):
+        return S_OK(True)
+
     # Get list of users from the DB with proxys at least 300 seconds
     gLogger.verbose("Updating list of users in proxy management")
     retVal = self.__refreshUserCache(validSeconds)
     if not retVal['OK']:
       return retVal
-    return S_OK(self.__usersCache.exists(cacheKey, validSeconds))
+
+    for cacheKey in cacheKeys:
+      if self.__usersCache.exists(cacheKey, validSeconds):
+        return S_OK(True)
+
+    return S_OK(False)
 
   @gUsersSync
   def getUserPersistence(self, userDN, userGroup, validSeconds=0):

src/DIRAC/FrameworkSystem/DB/ProxyDB.py

@@ -1362,11 +1362,6 @@ def _notifyProxyAboutToExpire(self, userDN, lTime):
 
   DN:    %s
 
-  If you plan on keep using this credentials please upload a newer proxy to
-  DIRAC by executing:
-
-  $ dirac-proxy-init --upload
-
   If you have been issued different certificate, please make sure you have a
   proxy uploaded with that certificate.
 

src/DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py

@@ -33,23 +33,32 @@
 __RCSID__ = "$Id$"
 
 
+
 class Params(ProxyGeneration.CLIParams):
 
   addVOMSExt = False
-  uploadProxy = False
+  uploadProxy = True
   uploadPilot = False
 
   def setVOMSExt(self, _arg):
     self.addVOMSExt = True
     return S_OK()
 
-  def setUploadProxy(self, _arg):
-    self.uploadProxy = True
+  def disableProxyUpload(self, _arg):
+    self.uploadProxy = False
     return S_OK()
 
   def registerCLISwitches(self):
     ProxyGeneration.CLIParams.registerCLISwitches(self)
-    Script.registerSwitch("U", "upload", "Upload a long lived proxy to the ProxyManager", self.setUploadProxy)
+    Script.registerSwitch(
+        "U",
+        "upload",
+        "Upload a long lived proxy to the ProxyManager (deprecated, see --no-upload)")
+    Script.registerSwitch(
+        "N",
+        "no-upload",
+        "Do not upload a long lived proxy to the ProxyManager",
+        self.disableProxyUpload)
     Script.registerSwitch("M", "VOMS", "Add voms extension", self.setVOMSExt)
 
 
@@ -151,7 +160,7 @@ def uploadProxy(self):
     resultProxyUpload = ProxyUpload.uploadProxy(upParams)
     if not resultProxyUpload['OK']:
       gLogger.error(resultProxyUpload['Message'])
-      sys.exit(1)
+      return resultProxyUpload
     self.__uploadedInfo = resultProxyUpload['Value']
     gLogger.info("Proxy uploaded")
     return S_OK()

src/DIRAC/Resources/Computing/BatchSystems/TimeLeft/SLURMResourceUsage.py

@@ -7,8 +7,6 @@
 
 __RCSID__ = "$Id$"
 
-import os
-
 from DIRAC import S_OK, S_ERROR
 from DIRAC.Resources.Computing.BatchSystems.TimeLeft.TimeLeft import runCommand
 from DIRAC.Resources.Computing.BatchSystems.TimeLeft.ResourceUsage import ResourceUsage
@@ -56,7 +54,7 @@ def getResourceUsage(self):
       cpu = float(cpu)
 
     consumed = {'CPU': wallClock,
-                'CPULimit': wallClockLimit,
+                'CPULimit': cpuLimit,
                 'WallClock': wallClock,
                 'WallClockLimit': wallClockLimit}
 

src/DIRAC/Workflow/Modules/ModuleBase.py

@@ -226,6 +226,7 @@ def _finalize(self, status=''):
   def finalize(self):
     """ Just finalizing the module execution by flushing the logs. This will be done always.
     """
+
     self.log.info('===== Terminating ' + str(self.__class__) + ' ===== ')
 
   #############################################################################

src/DIRAC/WorkloadManagementSystem/Agent/JobAgent.py

@@ -77,7 +77,6 @@ def __init__(self, agentName, loadName, baseAgentName=False, properties=None):
     self.initTimes = os.times()
     self.timeLeft = 0.0
     self.timeLeftUtil = None
-    self.timeLeftError = ''
     self.pilotInfoReportedFlag = False
 
   #############################################################################
@@ -144,9 +143,6 @@ def execute(self):
       # Only call timeLeft utility after a job has been picked up
       self.log.info('Attempting to check CPU time left for filling mode')
       if self.fillingMode:
-        if self.timeLeftError:
-          self.log.warn("Disabling filling mode as errors calculating time left", self.timeLeftError)
-          return self.__finish(self.timeLeftError)
         self.log.info('normalized CPU units remaining in slot', self.timeLeft)
         if self.timeLeft <= self.minimumTimeLeft:
           return self.__finish('No more time left')
@@ -184,9 +180,8 @@ def execute(self):
       if runningJobs:
         self.log.info('No available slots', '%d running jobs' % runningJobs)
         return S_OK('Job Agent cycle complete with %d running jobs' % runningJobs)
-      else:
-        self.log.info('CE is not available')
-        return self.__finish('CE Not Available')
+      self.log.info('CE is not available (and there are no running jobs)')
+      return self.__finish('CE Not Available')
 
     result = self.computingElement.getDescription()
     if not result['OK']:
@@ -395,11 +390,11 @@ def execute(self):
     if result['OK']:
       self.timeLeft = result['Value']
     else:
-      if result['Message'] != 'Current batch system is not supported':
-        self.timeLeftError = result['Message']
-      else:
-        # if the batch system is not defined, use the process time and the CPU normalization defined locally
-        self.timeLeft = self._getCPUTimeLeft()
+      self.log.warn(
+          "There were errors calculating time left using the Timeleft utility",
+          result['Message'])
+      self.log.warn("The time left will be calculated using os.times() and the info in our possession")
+      self.timeLeft = self._getCPUTimeLeft()
 
     return S_OK('Job Agent cycle complete')
 
@@ -417,7 +412,9 @@ def __saveJobJDLRequest(self, jobID, jobJDL):
 
   #############################################################################
   def _getCPUTimeLeft(self):
-    """Return the TimeLeft as estimated by DIRAC using the Normalization Factor in the Local Config.
+    """
+    Return the TimeLeft as estimated by DIRAC using the process time
+    and the CPU normalization defined locally in the Local Config.
     """
     cpuTime = sum(os.times()[:-1])
     self.log.info('Current raw CPU time consumed is %s' % cpuTime)
@@ -647,6 +644,7 @@ def __finish(self, message, stop=True):
                     'with message "%s", execution complete.' % message)
       self.am_stopExecution()
       return S_ERROR(message)
+
     return S_OK(message)
 
   #############################################################################

src/DIRAC/WorkloadManagementSystem/Utilities/PilotCStoJSONSynchronizer.py

@@ -143,12 +143,17 @@ def getCSDict(self):
           if localCEType is not None:
             pilotDict['CEs'][ce].setdefault('LocalCEType', localCEType)
 
-          queueList = gConfig.getSections('/Resources/Sites/' + grid + '/' + site + '/CEs/' + ce + '/Queues/')
+          res = gConfig.getSections('/Resources/Sites/' + grid + '/' + site + '/CEs/' + ce + '/Queues/')
+          if not res['OK']:
+            # Skip but log it
+            self.log.error("No queues found for CE", ce + ': ' + res['Message'])
+            continue
+          queueList = res['Value']
           for queue in queueList:
             localCEType = gConfig.getValue(
-                '/Resources/Sites/' + grid + '/' + site + '/CEs/' + ce + '/' + queue + '/LocalCEType')
+                '/Resources/Sites/' + grid + '/' + site + '/CEs/' + ce + '/Queues/' + queue + '/LocalCEType')
             if localCEType is not None:
-              pilotDict['CEs'][ce][queue].setdefault('LocalCEType', localCEType)
+              pilotDict['CEs'][ce].setdefault(queue, {'LocalCEType': localCEType})
 
     defaultSetup = gConfig.getValue('/DIRAC/DefaultSetup')
     if defaultSetup: