diff --git a/config/config.json b/config/config.json index c1fa35c8..3c07eb5b 100644 --- a/config/config.json +++ b/config/config.json @@ -40,7 +40,6 @@ "hostname_mode": "hostname", "pcp_log_dir": "/data/pcp-logs/my_cluster_name", "script_dir": "/data/jobscripts/my_cluster_name", - "job_uniq_mode": "local_job_id", // fast_index uses an alternative method of indexing job-level pcp archives which can significantly speed // up the indexarchives.py script. The tradeoff is that the indexed archive end time is not found and the diff --git a/src/supremm/xdmodaccount.py b/src/supremm/xdmodaccount.py index 25ae40b1..1cd00a84 100644 --- a/src/supremm/xdmodaccount.py +++ b/src/supremm/xdmodaccount.py @@ -13,13 +13,6 @@ class XDMoDAcct(Accounting): def __init__(self, resource_id, config): super(XDMoDAcct, self).__init__(resource_id, config) - job_uniq_mode = 'local_job_id' - for _, resconfig in config.resourceconfigs(): - if resconfig['resource_id'] == resource_id: - if 'job_unique_mode' in resconfig: - job_uniq_mode = resconfig['job_unique_mode'] - break - self.dbsettings = config.getsection("datawarehouse") xdmod_schema_version = self.detectXdmodSchema() @@ -31,14 +24,14 @@ def __init__(self, resource_id, config): SELECT jf.`job_id` AS `job_id`, jf.`resource_id` AS `resource_id`, - COALESCE(jf.`local_job_id_raw`, jf.`local_jobid`) AS `local_job_id`, - """ - if job_uniq_mode == 'local_job_id': - self._query += "COALESCE(jf.`local_job_id_raw`, jf.`local_jobid`) as `job_uniq_id`, " - else: - self._query += "IF(jf.`local_job_array_index` = -1, jf.`local_jobid`, CONCAT(jf.`local_jobid`, '_', jf.`local_job_array_index`)) as job_uniq_id, " - - self._query += """ + CASE + WHEN jf.`local_job_id_raw` IS NULL THEN IF(jf.`local_job_array_index` = -1, jf.`local_jobid`, CONCAT(jf.`local_jobid`, '_', jf.`local_job_array_index`)) + WHEN sj.`source_format` = 'slurm' THEN jf.`local_job_id_raw` + ELSE IF(jf.`local_job_array_index` = -1, jf.`local_jobid`, CONCAT(jf.`local_jobid`, '_', jf.`local_job_array_index`)) + END AS job_uniq_id, + jf.`local_jobid` AS `local_job_id`, + jf.`local_job_array_index` AS `local_job_array_index`, + jf.`local_job_id_raw` AS `local_job_id_raw`, jf.`start_time_ts` AS `start_time`, jf.`end_time_ts` AS `end_time`, jf.`submit_time_ts` AS `submit`, @@ -84,14 +77,14 @@ def __init__(self, resource_id, config): SELECT jf.`job_id` as `job_id`, jf.`resource_id` as `resource_id`, - COALESCE(jf.`local_job_id_raw`, jf.`local_jobid`) as `local_job_id`, - """ - if job_uniq_mode == 'local_job_id': - self._query += "COALESCE(jf.`local_job_id_raw`, jf.`local_jobid`) as `job_uniq_id`, " - else: - self._query += "IF(jf.`local_job_array_index` = -1, jf.`local_jobid`, CONCAT(jf.`local_jobid`, '_', jf.`local_job_array_index`)) as job_uniq_id, " - - self._query += """ + CASE + WHEN jf.`local_job_id_raw` IS NULL THEN IF(jf.`local_job_array_index` = -1, jf.`local_jobid`, CONCAT(jf.`local_jobid`, '_', jf.`local_job_array_index`)) + WHEN sj.`source_format` = 'slurm' THEN jf.`local_job_id_raw` + ELSE IF(jf.`local_job_array_index` = -1, jf.`local_jobid`, CONCAT(jf.`local_jobid`, '_', jf.`local_job_array_index`)) + END AS job_uniq_index, + jf.`local_jobid` AS `local_job_id`, + jf.`local_job_array_index` AS `local_job_array_index`, + jf.`local_job_id_raw` AS `local_job_id_raw`, jf.`start_time_ts` as `start_time`, jf.`end_time_ts` as `end_time`, jf.`submit_time_ts` as `submit`,