Skip to content

Commit

Permalink
Update as per review comments.
Browse files Browse the repository at this point in the history
  • Loading branch information
@jpwhite4
jpwhite4 committed May 26, 2023
1 parent 84f0178 commit 4215b2c
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 24 deletions.
1 change: 0 additions & 1 deletion config/config.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@
"hostname_mode": "hostname",
"pcp_log_dir": "/data/pcp-logs/my_cluster_name",
"script_dir": "/data/jobscripts/my_cluster_name",
"job_uniq_mode": "local_job_id",

// fast_index uses an alternative method of indexing job-level pcp archives which can significantly speed
// up the indexarchives.py script. The tradeoff is that the indexed archive end time is not found and the
Expand Down
39 changes: 16 additions & 23 deletions src/supremm/xdmodaccount.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,6 @@ class XDMoDAcct(Accounting):
def __init__(self, resource_id, config):
super(XDMoDAcct, self).__init__(resource_id, config)

job_uniq_mode = 'local_job_id'
for _, resconfig in config.resourceconfigs():
if resconfig['resource_id'] == resource_id:
if 'job_unique_mode' in resconfig:
job_uniq_mode = resconfig['job_unique_mode']
break

self.dbsettings = config.getsection("datawarehouse")

xdmod_schema_version = self.detectXdmodSchema()
Expand All @@ -31,14 +24,14 @@ def __init__(self, resource_id, config):
SELECT
jf.`job_id` AS `job_id`,
jf.`resource_id` AS `resource_id`,
COALESCE(jf.`local_job_id_raw`, jf.`local_jobid`) AS `local_job_id`,
"""
if job_uniq_mode == 'local_job_id':
self._query += "COALESCE(jf.`local_job_id_raw`, jf.`local_jobid`) as `job_uniq_id`, "
else:
self._query += "IF(jf.`local_job_array_index` = -1, jf.`local_jobid`, CONCAT(jf.`local_jobid`, '_', jf.`local_job_array_index`)) as job_uniq_id, "

self._query += """
CASE
WHEN jf.`local_job_id_raw` IS NULL THEN IF(jf.`local_job_array_index` = -1, jf.`local_jobid`, CONCAT(jf.`local_jobid`, '_', jf.`local_job_array_index`))
WHEN sj.`source_format` = 'slurm' THEN jf.`local_job_id_raw`
ELSE IF(jf.`local_job_array_index` = -1, jf.`local_jobid`, CONCAT(jf.`local_jobid`, '_', jf.`local_job_array_index`))
END AS job_uniq_id,
jf.`local_jobid` AS `local_job_id`,
jf.`local_job_array_index` AS `local_job_array_index`,
jf.`local_job_id_raw` AS `local_job_id_raw`,
jf.`start_time_ts` AS `start_time`,
jf.`end_time_ts` AS `end_time`,
jf.`submit_time_ts` AS `submit`,
Expand Down Expand Up @@ -84,14 +77,14 @@ def __init__(self, resource_id, config):
SELECT
jf.`job_id` as `job_id`,
jf.`resource_id` as `resource_id`,
COALESCE(jf.`local_job_id_raw`, jf.`local_jobid`) as `local_job_id`,
"""
if job_uniq_mode == 'local_job_id':
self._query += "COALESCE(jf.`local_job_id_raw`, jf.`local_jobid`) as `job_uniq_id`, "
else:
self._query += "IF(jf.`local_job_array_index` = -1, jf.`local_jobid`, CONCAT(jf.`local_jobid`, '_', jf.`local_job_array_index`)) as job_uniq_id, "

self._query += """
CASE
WHEN jf.`local_job_id_raw` IS NULL THEN IF(jf.`local_job_array_index` = -1, jf.`local_jobid`, CONCAT(jf.`local_jobid`, '_', jf.`local_job_array_index`))
WHEN sj.`source_format` = 'slurm' THEN jf.`local_job_id_raw`
ELSE IF(jf.`local_job_array_index` = -1, jf.`local_jobid`, CONCAT(jf.`local_jobid`, '_', jf.`local_job_array_index`))
END AS job_uniq_index,
jf.`local_jobid` AS `local_job_id`,
jf.`local_job_array_index` AS `local_job_array_index`,
jf.`local_job_id_raw` AS `local_job_id_raw`,
jf.`start_time_ts` as `start_time`,
jf.`end_time_ts` as `end_time`,
jf.`submit_time_ts` as `submit`,
Expand Down

0 comments on commit 4215b2c

Please sign in to comment.