From 2844ab957d57f82a2679b7f012bb89cf7b00a485 Mon Sep 17 00:00:00 2001 From: Marko Petric Date: Wed, 28 Apr 2021 15:58:37 +0200 Subject: [PATCH 1/7] Read X509_CERT_DIR from central place --- FrameworkSystem/scripts/dirac-proxy-init.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/FrameworkSystem/scripts/dirac-proxy-init.py b/FrameworkSystem/scripts/dirac-proxy-init.py index 6adaa990b2d..5f17ca9ff5a 100755 --- a/FrameworkSystem/scripts/dirac-proxy-init.py +++ b/FrameworkSystem/scripts/dirac-proxy-init.py @@ -27,6 +27,7 @@ from DIRAC.Core.Base import Script from DIRAC.FrameworkSystem.Client import ProxyGeneration, ProxyUpload from DIRAC.Core.Security import X509Chain, ProxyInfo, Properties, VOMS +from DIRAC.Core.Security.Locations import getCAsLocation from DIRAC.ConfigurationSystem.Client.Helpers import Registry from DIRAC.FrameworkSystem.Client.BundleDeliveryClient import BundleDeliveryClient @@ -188,10 +189,10 @@ def printInfo(self): self.__uploadedInfo[userDN][group].strftime("%Y/%m/%d %H:%M"))) def checkCAs(self): - if "X509_CERT_DIR" not in os.environ: - gLogger.warn("X509_CERT_DIR is unset. Abort check of CAs") + caDir = getCAsLocation() + if not caDir: + gLogger.warn("No valid CA dir found.") return - caDir = os.environ["X509_CERT_DIR"] # In globus standards .r0 files are CRLs. They have the same names of the CAs but diffent file extension searchExp = os.path.join(caDir, "*.r0") crlList = glob.glob(searchExp) From 53be301b0b7085637e53a52b1c181f1ddf0ccf83 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Tue, 11 May 2021 19:04:52 +0200 Subject: [PATCH 2/7] Make tarfile error message clearer --- WorkloadManagementSystem/Utilities/PilotWrapper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/WorkloadManagementSystem/Utilities/PilotWrapper.py b/WorkloadManagementSystem/Utilities/PilotWrapper.py index 153c2f58282..fdafcfe6d97 100644 --- a/WorkloadManagementSystem/Utilities/PilotWrapper.py +++ b/WorkloadManagementSystem/Utilities/PilotWrapper.py @@ -199,8 +199,8 @@ def pilotWrapperScript(pilotFilesCompressedEncodedDict=None, pt.extractall() pt.close() except Exception as x: - print("tarfile failed with message %%s" %% repr(x), file=sys.stderr) - logger.error("tarfile failed with message %%s" %% repr(x)) + print("tarfile failed with message (this is normal!) %%s" %% repr(x), file=sys.stderr) + logger.error("tarfile failed with message (this is normal!) %%s" %% repr(x)) logger.warn("Trying tar command (tar -xvf pilot.tar)") res = os.system("tar -xvf pilot.tar") if res: From 65e1e06e3d5e2bef39a5a1c3d3554a2b9450f817 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Tue, 11 May 2021 20:58:28 +0200 Subject: [PATCH 3/7] Further clarifications in pilot logs --- WorkloadManagementSystem/Utilities/PilotWrapper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/WorkloadManagementSystem/Utilities/PilotWrapper.py b/WorkloadManagementSystem/Utilities/PilotWrapper.py index fdafcfe6d97..318d187857a 100644 --- a/WorkloadManagementSystem/Utilities/PilotWrapper.py +++ b/WorkloadManagementSystem/Utilities/PilotWrapper.py @@ -210,8 +210,8 @@ def pilotWrapperScript(pilotFilesCompressedEncodedDict=None, # if we get here we break out of the loop of locations break except (url_library_URLError, Exception) as e: - print('%%s unreacheable' %% loc, file=sys.stderr) - logger.error('%%s unreacheable' %% loc) + print('%%s unreacheable (this is normal!)' %% loc, file=sys.stderr) + logger.error('%%s unreacheable (this is normal!)' %% loc) logger.exception(e) else: From 00e78c21c1046dca39de1c4f84fee973867dd73a Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 12 May 2021 08:18:59 +0200 Subject: [PATCH 4/7] Add more "(this is normal!)" annotations to pilot logs --- WorkloadManagementSystem/Utilities/PilotWrapper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/WorkloadManagementSystem/Utilities/PilotWrapper.py b/WorkloadManagementSystem/Utilities/PilotWrapper.py index 318d187857a..981995234fc 100644 --- a/WorkloadManagementSystem/Utilities/PilotWrapper.py +++ b/WorkloadManagementSystem/Utilities/PilotWrapper.py @@ -204,8 +204,8 @@ def pilotWrapperScript(pilotFilesCompressedEncodedDict=None, logger.warn("Trying tar command (tar -xvf pilot.tar)") res = os.system("tar -xvf pilot.tar") if res: - logger.error("tar failed with exit code %%d, giving up" %% int(res)) - print("tar failed with exit code %%d, giving up" %% int(res), file=sys.stderr) + logger.error("tar failed with exit code %%d, giving up (this is normal!)" %% int(res)) + print("tar failed with exit code %%d, giving up (this is normal!)" %% int(res), file=sys.stderr) raise # if we get here we break out of the loop of locations break From c55b26c4e4429a98a1eba88e31c51fe5ddf3c8a8 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Wed, 12 May 2021 16:03:33 +0200 Subject: [PATCH 5/7] DFC FileManagerPS: getReplicas speed improvment --- .../FileManager/FileManagerPs.py | 32 ++++++------ .../DB/FileCatalogWithFkAndPsDB.sql | 49 ++++++++++++++++++- 2 files changed, 64 insertions(+), 17 deletions(-) diff --git a/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py b/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py index 4467d0a5b9a..c8a7d84b699 100755 --- a/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py +++ b/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py @@ -775,51 +775,51 @@ def _setFileParameter(self, fileID, paramName, paramValue, connection=False): # _getFileReplicas related methods # - def _getFileReplicas(self, fileIDs, fields_input=['PFN'], allStatus=False, connection=False): + def _getFileReplicas(self, fileIDs, fields_input=None, allStatus=False, connection=False): """ Get replicas for the given list of files specified by their fileIDs :param fileIDs : list of file ids - :param fields_input : metadata of the Replicas we are interested in + :param fields_input : metadata of the Replicas we are interested in (default to PFN) :param allStatus : if True, all the Replica statuses will be considered, otherwise, only the db.visibleReplicaStatus :returns S_OK with a dict { fileID : { SE name : dict of metadata } } """ - connection = self._getConnection(connection) + if fields_input is None: + fields_input = ['PFN'] fields = list(fields_input) + # always add Status in the list of required fields if 'Status' not in fields: fields.append('Status') - replicas = {} + # We initialize the dictionary with empty dict + # as default value, because this is what we want for + # non existing replicas + replicas = {fileID: {} for fileID in fileIDs} # Format the status to be used in a IN clause in the stored procedure fStatus = stringListToString(self.db.visibleReplicaStatus) fieldNames = ["FileID", "SE", "Status", "RepType", "CreationDate", "ModificationDate", "PFN"] - for fileID in fileIDs: - - result = self.db.executeStoredProcedureWithCursor('ps_get_all_info_of_replicas', - (fileID, allStatus, fStatus)) + for chunks in breakListIntoChunks(fileIDs, 1000): + # Format the FileIDs to be used in a IN clause in the stored procedure + formatedFileIds = intListToString(chunks) + result = self.db.executeStoredProcedureWithCursor('ps_get_all_info_of_replicas_bulk', + (formatedFileIds, allStatus, fStatus)) if not result['OK']: return result rows = result['Value'] - if not rows: - replicas[fileID] = {} - for row in rows: - rowDict = dict(zip(fieldNames, row)) - - # Returns only the required metadata se = rowDict["SE"] - repForFile = replicas.setdefault(fileID, {}) - repForFile[se] = dict((key, rowDict.get(key, "Unknown metadata field")) for key in fields) + fileID = rowDict['FileID'] + replicas[fileID][se] = dict((key, rowDict.get(key, "Unknown metadata field")) for key in fields) return S_OK(replicas) diff --git a/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql b/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql index e9c38b75357..284dbcbc796 100755 --- a/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql +++ b/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql @@ -1322,6 +1322,7 @@ DELIMITER ; +-- TO BE DEPRECATED IN FAVOR OF THE BULK METHOD -- ps_get_all_info_of_replicas : get the info of all replicas of a given file -- -- file_id : id of the file @@ -1353,7 +1354,7 @@ BEGIN JOIN FC_StorageElements se on r.SEID = se.SEID JOIN FC_Statuses st on r.Status = st.StatusID WHERE FileID =',file_id, - ' and st.Status in (',visibleReplicaStatus,') '); + ' AND st.Status IN (', visibleReplicaStatus, ')'); PREPARE stmt FROM @sql; EXECUTE stmt; @@ -1364,6 +1365,52 @@ BEGIN END // DELIMITER ; +-- ps_get_all_info_of_replicas_bulk : get the info of all replicas for a list of file ids +-- +-- file_ids : list of file IDs +-- allStatus : if False, consider visibleReplicaStatus +-- visibleReplicaStatus : list of status we are interested in +-- +-- output : FileID, se.SEName, st.Status, RepType, CreationDate, ModificationDate, PFN + +DROP PROCEDURE IF EXISTS ps_get_all_info_of_replicas_bulk; +DELIMITER // +CREATE PROCEDURE ps_get_all_info_of_replicas_bulk +(IN file_ids TEXT, IN allStatus BOOLEAN, IN visibleReplicaStatus TEXT) +BEGIN + + + IF allStatus THEN + + SET @sql = CONCAT('SELECT SQL_NO_CACHE FileID, se.SEName, st.Status, RepType, CreationDate, ModificationDate, PFN + FROM FC_Replicas r + JOIN FC_StorageElements se on r.SEID = se.SEID + JOIN FC_Statuses st on r.Status = st.StatusID + WHERE FileID IN (', file_ids, ')'); + PREPARE stmt FROM @sql; + EXECUTE stmt; + DEALLOCATE PREPARE stmt; + + ELSE + + SET @sql = CONCAT( + 'SELECT SQL_NO_CACHE FileID, se.SEName, st.Status, RepType, CreationDate, ModificationDate, PFN + FROM FC_Replicas r + JOIN FC_StorageElements se on r.SEID = se.SEID + JOIN FC_Statuses st on r.Status = st.StatusID + WHERE FileID IN (', file_ids, ') ', + 'AND st.Status IN (', visibleReplicaStatus, ')'); + + PREPARE stmt FROM @sql; + EXECUTE stmt; + DEALLOCATE PREPARE stmt; + + END IF; + +END // +DELIMITER ; + + -- ps_get_all_directory_info : get all the info of a given directory -- -- dir_name : name of the directory From ae7e45b05b3f1a93f881c7c512026113afcff3c9 Mon Sep 17 00:00:00 2001 From: Andre Sailer Date: Mon, 17 May 2021 14:18:34 +0200 Subject: [PATCH 6/7] SiteDirector: fix wrong return type in function --- WorkloadManagementSystem/Agent/SiteDirector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WorkloadManagementSystem/Agent/SiteDirector.py b/WorkloadManagementSystem/Agent/SiteDirector.py index ea83ca5ce13..6e7485f0b95 100644 --- a/WorkloadManagementSystem/Agent/SiteDirector.py +++ b/WorkloadManagementSystem/Agent/SiteDirector.py @@ -823,7 +823,7 @@ def _getPilotsWeMayWantToSubmit(self, ceDict): result = self.matcherClient.getMatchingTaskQueues(ceDict) if not result['OK']: self.log.error('Could not retrieve TaskQueues from TaskQueueDB', result['Message']) - return result + return 0, {} taskQueueDict = result['Value'] if not taskQueueDict: self.log.verbose('No matching TQs found', From 9d619185fe70dd1f2d9a1158e658180d74b58b17 Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Wed, 19 May 2021 15:55:34 +0200 Subject: [PATCH 7/7] v7r1p40 notes and tags --- __init__.py | 2 +- release.notes | 16 ++++++++++++++++ setup.py | 2 +- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/__init__.py b/__init__.py index d99c3488c5c..993a9d1224a 100755 --- a/__init__.py +++ b/__init__.py @@ -95,7 +95,7 @@ else: majorVersion = 7 minorVersion = 1 - patchLevel = 39 + patchLevel = 40 preVersion = 0 version = "v%sr%s" % (majorVersion, minorVersion) diff --git a/release.notes b/release.notes index 5f86c37c0b7..3b7fd3461d5 100644 --- a/release.notes +++ b/release.notes @@ -1,3 +1,19 @@ +[v7r1p40] + +*Core +CHANGE: (#5123) Read X509_CERT_DIR from central place + +*WorkloadManagement +CHANGE: (#5143) Clarify that "tarfile failed with message" can be normal +FIX: (#5146) SiteDirector._getPilotsWeMayWantToSubmit: fix return type in case of error + +*DataManagement +CHANGE: (#5144) FileManagerPS has bulk method for getReplicas + +*Resources +CHANGE: (#5137) HTCondorCE submits jobs with -spool option when a local schedd is used to + spool the pilot wrappers that can be deleted afterwards + [v7r1p39] *WMS diff --git a/setup.py b/setup.py index 973a1de3ce2..e180dfb0cd4 100755 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ setup( name="DIRAC", - version="7.1.39", + version="7.1.40", url="https://github.com/DIRACGRID/DIRAC", license="GPLv3", package_dir=package_dir,