From 175f9870c7c4b342f75ddbc242222c9bb2d1123d Mon Sep 17 00:00:00 2001 From: fstagni Date: Thu, 18 Jan 2024 16:52:58 +0100 Subject: [PATCH 1/2] feat: remove not used JobDB.Jobs fields --- src/DIRAC/Interfaces/API/Dirac.py | 2 +- .../Interfaces/scripts/dirac_wms_job_attributes.py | 14 ++------------ src/DIRAC/WorkloadManagementSystem/DB/JobDB.py | 2 -- src/DIRAC/WorkloadManagementSystem/DB/JobDB.sql | 9 --------- 4 files changed, 3 insertions(+), 24 deletions(-) diff --git a/src/DIRAC/Interfaces/API/Dirac.py b/src/DIRAC/Interfaces/API/Dirac.py index f119fe84788..0d3b936ba6f 100755 --- a/src/DIRAC/Interfaces/API/Dirac.py +++ b/src/DIRAC/Interfaces/API/Dirac.py @@ -2033,7 +2033,7 @@ def getJobAttributes(self, jobID, printOutput=False): Example Usage: >>> print(dirac.getJobAttributes(79241)) - {'AccountedFlag': 'False','ApplicationNumStatus': '0', + {'AccountedFlag': 'False', 'ApplicationStatus': 'Job Finished Successfully', 'CPUTime': '0.0'} diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_attributes.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_attributes.py index 4ddc4b78e10..44e564086bf 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_job_attributes.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_attributes.py @@ -9,37 +9,27 @@ Example: $ dirac-wms-job-attributes 1 {'AccountedFlag': 'False', - 'ApplicationNumStatus': '0', 'ApplicationStatus': 'Unknown', 'CPUTime': '0.0', - 'DeletedFlag': 'False', 'EndExecTime': '2011-02-14 11:28:01', - 'FailedFlag': 'False', 'HeartBeatTime': '2011-02-14 11:28:01', - 'ISandboxReadyFlag': 'False', 'JobGroup': 'NoGroup', 'JobID': '1', 'JobName': 'DIRAC_vhamar_602138', 'JobSplitType': 'Single', 'JobType': 'normal', - 'KilledFlag': 'False', 'LastUpdateTime': '2011-02-14 11:28:11', - 'MasterJobID': '0', 'MinorStatus': 'Execution Complete', - 'OSandboxReadyFlag': 'False', 'Owner': 'vhamar', 'OwnerGroup': 'eela_user', + 'VO': 'dteam', 'RescheduleCounter': '0', 'RescheduleTime': 'None', - 'RetrievedFlag': 'False', - 'RunNumber': '0', 'Site': 'EELA.UTFSM.cl', 'StartExecTime': '2011-02-14 11:27:48', 'Status': 'Done', 'SubmissionTime': '2011-02-14 10:12:40', - 'SystemPriority': '0', - 'UserPriority': '1', - 'VerifiedFlag': 'True'} + 'UserPriority': '1'} """ from DIRAC import exit as dExit from DIRAC import gLogger diff --git a/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py b/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py index cc819b329b7..2880f6fc53e 100755 --- a/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py +++ b/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py @@ -1187,8 +1187,6 @@ def rescheduleJob(self, jobID): jobAttrs["ApplicationStatus"] = "Unknown" - jobAttrs["ApplicationNumStatus"] = 0 - jobAttrs["LastUpdateTime"] = str(datetime.datetime.utcnow()) jobAttrs["RescheduleTime"] = str(datetime.datetime.utcnow()) diff --git a/src/DIRAC/WorkloadManagementSystem/DB/JobDB.sql b/src/DIRAC/WorkloadManagementSystem/DB/JobDB.sql index f2123dfd618..ffd13fb8877 100755 --- a/src/DIRAC/WorkloadManagementSystem/DB/JobDB.sql +++ b/src/DIRAC/WorkloadManagementSystem/DB/JobDB.sql @@ -37,7 +37,6 @@ CREATE TABLE `Jobs` ( `JobType` VARCHAR(32) NOT NULL DEFAULT 'user', `JobGroup` VARCHAR(32) NOT NULL DEFAULT '00000000', `JobSplitType` ENUM('Single','Master','Subjob','DAGNode') NOT NULL DEFAULT 'Single', - `MasterJobID` INT(11) UNSIGNED NOT NULL DEFAULT 0, `Site` VARCHAR(100) NOT NULL DEFAULT 'ANY', `JobName` VARCHAR(128) NOT NULL DEFAULT 'Unknown', `Owner` VARCHAR(64) NOT NULL DEFAULT 'Unknown', @@ -52,17 +51,9 @@ CREATE TABLE `Jobs` ( `Status` VARCHAR(32) NOT NULL DEFAULT 'Received', `MinorStatus` VARCHAR(128) NOT NULL DEFAULT 'Unknown', `ApplicationStatus` VARCHAR(255) DEFAULT 'Unknown', - `ApplicationNumStatus` INT(11) NOT NULL DEFAULT 0, `UserPriority` INT(11) NOT NULL DEFAULT 0, - `SystemPriority` INT(11) NOT NULL DEFAULT 0, `RescheduleCounter` INT(11) NOT NULL DEFAULT 0, `VerifiedFlag` ENUM('True','False') NOT NULL DEFAULT 'False', - `DeletedFlag` ENUM('True','False') NOT NULL DEFAULT 'False', - `KilledFlag` ENUM('True','False') NOT NULL DEFAULT 'False', - `FailedFlag` ENUM('True','False') NOT NULL DEFAULT 'False', - `ISandboxReadyFlag` ENUM('True','False') NOT NULL DEFAULT 'False', - `OSandboxReadyFlag` ENUM('True','False') NOT NULL DEFAULT 'False', - `RetrievedFlag` ENUM('True','False') NOT NULL DEFAULT 'False', `AccountedFlag` ENUM('True','False','Failed') NOT NULL DEFAULT 'False', PRIMARY KEY (`JobID`), FOREIGN KEY (`JobID`) REFERENCES `JobJDLs`(`JobID`), From 4f9ee7aa71133df61b10d0dd47aee159243605a0 Mon Sep 17 00:00:00 2001 From: fstagni Date: Thu, 18 Jan 2024 17:02:12 +0100 Subject: [PATCH 2/2] feat: removed unused JobDB.Jobs.JobSplitType column --- src/DIRAC/Interfaces/scripts/dirac_wms_job_attributes.py | 5 ----- src/DIRAC/WorkloadManagementSystem/Agent/OptimizerModule.py | 5 +---- src/DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py | 2 +- src/DIRAC/WorkloadManagementSystem/DB/JobDB.py | 2 +- src/DIRAC/WorkloadManagementSystem/DB/JobDB.sql | 2 -- src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py | 4 +--- 6 files changed, 4 insertions(+), 16 deletions(-) diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_attributes.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_attributes.py index 44e564086bf..4a09a39038f 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_job_attributes.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_attributes.py @@ -1,8 +1,4 @@ #!/usr/bin/env python -######################################################################## -# File : dirac-wms-job-attributes -# Author : Stuart Paterson -######################################################################## """ Retrieve attributes associated with the given DIRAC job @@ -16,7 +12,6 @@ 'JobGroup': 'NoGroup', 'JobID': '1', 'JobName': 'DIRAC_vhamar_602138', - 'JobSplitType': 'Single', 'JobType': 'normal', 'LastUpdateTime': '2011-02-14 11:28:11', 'MinorStatus': 'Execution Complete', diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/OptimizerModule.py b/src/DIRAC/WorkloadManagementSystem/Agent/OptimizerModule.py index 119619e3767..9e2b477dcbc 100755 --- a/src/DIRAC/WorkloadManagementSystem/Agent/OptimizerModule.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/OptimizerModule.py @@ -277,9 +277,6 @@ def sendAccountingRecord(self, job, msg, classAdJob): userGroup = classAdJob.getAttributeString("OwnerGroup") jobGroup = classAdJob.getAttributeString("JobGroup") jobType = classAdJob.getAttributeString("JobType") - jobClass = "unknown" - if classAdJob.lookupAttribute("JobSplitType"): - jobClass = classAdJob.getAttributeString("JobSplitType") inputData = [] processingType = "unknown" if classAdJob.lookupAttribute("ProcessingType"): @@ -297,7 +294,7 @@ def sendAccountingRecord(self, job, msg, classAdJob): "UserGroup": userGroup, "JobGroup": jobGroup, "JobType": jobType, - "JobClass": jobClass, + "JobClass": "unknown", "ProcessingType": processingType, "FinalMajorStatus": self.failedStatus, "FinalMinorStatus": msg, diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py b/src/DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py index 66005dab159..e9137468d6d 100755 --- a/src/DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py @@ -422,7 +422,7 @@ def _sendAccounting(self, jobID): "UserGroup": jobDict["OwnerGroup"], "JobGroup": jobDict["JobGroup"], "JobType": jobDict["JobType"], - "JobClass": jobDict["JobSplitType"], + "JobClass": "unknown", "ProcessingType": processingType, "FinalMajorStatus": JobStatus.FAILED, "FinalMinorStatus": JobMinorStatus.STALLED_PILOT_NOT_RUNNING, diff --git a/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py b/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py index 2880f6fc53e..72a3612fb2e 100755 --- a/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py +++ b/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py @@ -1828,7 +1828,7 @@ def setJobCommandStatus(self, jobID, command, status): def getSummarySnapshot(self, requestedFields=False): """Get the summary snapshot for a given combination""" if not requestedFields: - requestedFields = ["Status", "MinorStatus", "Site", "Owner", "OwnerGroup", "JobGroup", "JobSplitType"] + requestedFields = ["Status", "MinorStatus", "Site", "Owner", "OwnerGroup", "JobGroup"] valueFields = ["COUNT(JobID)", "SUM(RescheduleCounter)"] defString = ", ".join(requestedFields) valueString = ", ".join(valueFields) diff --git a/src/DIRAC/WorkloadManagementSystem/DB/JobDB.sql b/src/DIRAC/WorkloadManagementSystem/DB/JobDB.sql index ffd13fb8877..6f5ee8f7f90 100755 --- a/src/DIRAC/WorkloadManagementSystem/DB/JobDB.sql +++ b/src/DIRAC/WorkloadManagementSystem/DB/JobDB.sql @@ -36,7 +36,6 @@ CREATE TABLE `Jobs` ( `JobID` INT(11) UNSIGNED NOT NULL DEFAULT 0, `JobType` VARCHAR(32) NOT NULL DEFAULT 'user', `JobGroup` VARCHAR(32) NOT NULL DEFAULT '00000000', - `JobSplitType` ENUM('Single','Master','Subjob','DAGNode') NOT NULL DEFAULT 'Single', `Site` VARCHAR(100) NOT NULL DEFAULT 'ANY', `JobName` VARCHAR(128) NOT NULL DEFAULT 'Unknown', `Owner` VARCHAR(64) NOT NULL DEFAULT 'Unknown', @@ -59,7 +58,6 @@ CREATE TABLE `Jobs` ( FOREIGN KEY (`JobID`) REFERENCES `JobJDLs`(`JobID`), KEY `JobType` (`JobType`), KEY `JobGroup` (`JobGroup`), - KEY `JobSplitType` (`JobSplitType`), KEY `Site` (`Site`), KEY `Owner` (`Owner`), KEY `OwnerGroup` (`OwnerGroup`), diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py index 96db53e9c5c..0e77bc1e321 100755 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py @@ -161,7 +161,6 @@ def __init__(self, jobID=None, jobReport=None): self.jobType = "unknown" self.processingType = "unknown" self.userGroup = "unknown" - self.jobClass = "unknown" self.inputDataFiles = 0 self.outputDataFiles = 0 self.inputDataSize = 0 @@ -201,7 +200,6 @@ def initialize(self, arguments): self.outputDataFiles = len(dataParam) self.processingType = self.jobArgs.get("ProcessingType", self.processingType) self.userGroup = self.jobArgs.get("OwnerGroup", self.userGroup) - self.jobClass = self.jobArgs.get("JobSplitType", self.jobClass) if not self.cpuNormalizationFactor: self.cpuNormalizationFactor = float(self.ceArgs.get("CPUNormalizationFactor", self.cpuNormalizationFactor)) @@ -1259,7 +1257,7 @@ def sendJobAccounting(self, status="", minorStatus=""): "UserGroup": self.userGroup, "JobGroup": self.jobGroup, "JobType": self.jobType, - "JobClass": self.jobClass, + "JobClass": "unknown", "ProcessingType": self.processingType, "Site": self.siteName, "FinalMajorStatus": self.wmsMajorStatus,