From e8017fbb0ffdee27a56829f90ed08b0da09e9dea Mon Sep 17 00:00:00 2001 From: Sophie Reed Date: Mon, 25 Nov 2024 19:10:22 -0800 Subject: [PATCH] Classify --- python/lsst/production/tools/htmlUtils.py | 208 +++++++++++---------- python/lsst/production/tools/tractTable.py | 107 ++++++----- templates/metrics/tracts.html | 14 +- 3 files changed, 173 insertions(+), 156 deletions(-) diff --git a/python/lsst/production/tools/htmlUtils.py b/python/lsst/production/tools/htmlUtils.py index 2790064..2eb60e9 100644 --- a/python/lsst/production/tools/htmlUtils.py +++ b/python/lsst/production/tools/htmlUtils.py @@ -23,6 +23,26 @@ import numpy as np +class cell_entry: + + def __init__(self): + + self.num_bad = 0 + self.link = "noInfo" + self.debug_group = None + self.val_str = None + self.sig_str = None + + +class cell_contents: + + def __init__(self): + self.num_fails = 0 + self.text = "" + self.link = "noInfo" + self.debug_group = None + + def mk_table_value(row, metric_defs, val_col_name, sig_col_name): """Turn the values from the given columns into formatted cell contents. @@ -59,54 +79,56 @@ def mk_table_value(row, metric_defs, val_col_name, sig_col_name): isn't in the table. """ + cell = cell_entry() # Make a string of the val and sig columns if val_col_name in row.columns and sig_col_name in row.columns: val = row[val_col_name] - val_str = f"{val:.3g}" + cell.val_str = f"{val:.3g}" sig = row[sig_col_name] - sig_str = f"{sig:.3g}" + cell.sig_str = f"{sig:.3g}" elif val_col_name in row.columns and sig_col_name not in row.columns: val = row[val_col_name] - val_str = f"{val:.3g}" + cell.val_str = f"{val:.3g}" sig = None - sig_str = "-" + cell.sig_str = "-" elif val_col_name not in row.columns and sig_col_name in row.columns: - val_str = "-" + cell.val_str = "-" val = None sig = row[sig_col_name] - sig_str = f"{sig:.3g}" + cell.sig_str = f"{sig:.3g}" else: - return None, None, None, None, None + return cell bad_val = 0 - link = "metrics.report_page" + cell.link = "metrics.report_page" if val_col_name in metric_defs: - debug_group = metric_defs[val_col_name]["debugGroup"] + cell.debug_group = metric_defs[val_col_name]["debugGroup"] else: - debug_group = None + cell.debug_group = None # Add formatting to the string if there are thresholds # for the metric if np.isnan(val): - val_str = f"{val:.3g} " + cell.val_str = f"{val:.3g} " if np.isnan(sig): - sig_str = f"{sig:.3g}\n" + cell.sig_str = f"{sig:.3g}\n" if val_col_name in metric_defs: high_val = metric_defs[val_col_name]["highThreshold"] low_val = metric_defs[val_col_name]["lowThreshold"] - debug_group = metric_defs[val_col_name]["debugGroup"] + cell.debug_group = metric_defs[val_col_name]["debugGroup"] if val < low_val or val > high_val: - val_str = f"{val:.3g}" + cell.val_str = f"{val:.3g}" bad_val += 1 if sig_col_name in metric_defs: high_sig = metric_defs[sig_col_name]["highThreshold"] low_sig = metric_defs[sig_col_name]["lowThreshold"] if sig < low_sig or sig > high_sig: - sig_str = f"{sig:.3g}\n" + cell.sig_str = f"{sig:.3g}\n" bad_val += 1 - return val_str, sig_str, bad_val, link, debug_group + cell.num_bad = bad_val + return cell def mk_table_headers(t, col_dict): @@ -181,9 +203,10 @@ def mk_tract_cell(tract): tract_str : `str` The link that the tract should go to """ - tract_str = "metrics.single_tract" + cell = cell_contents() + cell.text = "metrics.single_tract" - return (tract_str,) + return cell def mk_summary_plot_cell(tract): @@ -202,8 +225,9 @@ def mk_summary_plot_cell(tract): This needs updating with a plot navigator link """ # plot_str = "" - plot_str = "nav link needed" - return (plot_str,) + cell = cell_contents() + cell.text = "nav link needed" + return cell def mk_patch_num_cell(t, n, bands): @@ -223,26 +247,26 @@ def mk_patch_num_cell(t, n, bands): patch_str : `str` The formatted number of patches in each band """ + cell = cell_contents() patch_str = "" for band in ["u", "g", "r", "i", "z", "y"]: if band in bands: patch_col = "coaddPatchCount_" + band + "_patchCount" patch_str += "" + band + ": " + str(int(t[patch_col][n])) + "
\n" - return (patch_str,) + cell.text = patch_str + return cell -def mk_shape_cols(t, metric_defs, n, bands, cols): +def mk_shape_cols(row, metric_defs, bands, cols): """Make shape column cell contents Parameters ---------- - t : `astropy.table.Table` - Table of metrics + row : `astropy.table.Table` + A row of the metrics table metric_defs : `dict` A dictionary of metrics and their thresholds - n : `n` - The row number bands : `list` A list of the bands the metrics are in cols : `list` @@ -253,28 +277,26 @@ def mk_shape_cols(t, metric_defs, n, bands, cols): shape_strs : `tuple` of `str` A tuple of the formatted strings for the shape columns """ - shape_strs = [] + shape_cells = [] for col in cols: for sn in ["highSNStars", "lowSNStars"]: - shape_str = "" - num_bad = 0 debug_group_all = None + full_cell = cell_contents() for band in ["u", "g", "r", "i", "z", "y"]: if band in bands: val_col_name = col + "_" + band + "_" + sn + "_median" sig_col_name = col + "_" + band + "_" + sn + "_sigmaMad" - val_str, sig_str, bad_val, link, debug_group = mk_table_value( - t[n], metric_defs, val_col_name, sig_col_name + cell_entry = mk_table_value( + row, metric_defs, val_col_name, sig_col_name ) - num_bad += bad_val - shape_str += ( - "" + band + f": " + val_str + " σ: " - ) - shape_str += sig_str + "
\n" - if debug_group is not None: - debug_group_all = debug_group - shape_strs.append((shape_str, num_bad, link, debug_group_all)) - return shape_strs + full_cell.num_fails += cell_entry.num_bad + full_cell.text += f"{band}: {cell_entry.val_str} " + full_cell.text += f"σ: {cell_entry.sig_str}
\n" + if cell_entry.debug_group is not None: + full_cell.debug_group = cell_entry.debug_group + full_cell.link = cell_entry.link + shape_cells.append(full_cell) + return shape_cells def mk_stellar_locus_cols(t, metric_defs, n, cols): @@ -296,37 +318,34 @@ def mk_stellar_locus_cols(t, metric_defs, n, cols): row_strs : `tuple` of `str` A tuple of the formatted strings for the stellar locus columns """ - row_strs = [] + row_cells = [] for col in cols: - row_str = "" - num_bad = 0 - debug_group_all = None for flux, flux1 in zip(["psfFlux", "cModelFlux"], ["PSF", "CModel"]): if (col[0] == "w" or col[0] == "x") and flux1 == "PSF": flux1 += "P" + full_cell = cell_contents() val_col_name = col + flux1 + "_" + col + "_" + flux + "_median" sig_col_name = col + flux1 + "_" + col + "_" + flux + "_sigmaMAD" - val_str, sig_str, bad_val, link, debug_group = mk_table_value( - t[n], metric_defs, val_col_name, sig_col_name - ) - if val_str is None: + cell_entry = mk_table_value(t[n], metric_defs, val_col_name, sig_col_name) + if cell_entry.val_str is None: continue - row_str += ( + full_cell.text += ( "" + flux + "
Med: " - + val_str + + cell_entry.val_str + " σ: " - + sig_str + + cell_entry.sig_str + "
" ) - row_str += "
\n" - num_bad += bad_val - if debug_group is not None: - debug_group_all = debug_group - row_strs.append((row_str, num_bad, link, debug_group_all)) + full_cell.text += "
\n" + full_cell.num_fails += cell_entry.num_bad + if cell_entry.debug_group is not None: + full_cell.debug_group = cell_entry.debug_group + full_cell.link = cell_entry.link + row_cells.append(full_cell) - return row_strs + return row_cells def mk_num_inputs_cell(t, metric_defs, n, bands): @@ -350,18 +369,21 @@ def mk_num_inputs_cell(t, metric_defs, n, bands): """ row_str = "" + full_cell = cell_contents() for band in ["u", "g", "r", "i", "z", "y"]: if band in bands: val_col_name = "coaddInputCount_" + band + "_inputCount_median" sig_col_name = "coaddInputCount_" + band + "_inputCount_sigmaMad" - val_str, sig_str, _, _, _ = mk_table_value( - t[n], metric_defs, val_col_name, sig_col_name + cell_entry = mk_table_value(t[n], metric_defs, val_col_name, sig_col_name) + full_cell.text += ( + "" + band + ":" + cell_entry.val_str + " σ " ) - row_str += "" + band + ":" + val_str + " σ " - row_str += sig_str + "
\n" + full_cell.text += cell_entry.sig_str + "
\n" + if cell_entry.debug_group is not None: + full_cell.debug_group = cell_entry.debug_group - return (row_str,) + return full_cell def mk_photom_cols(t, metric_defs, n, bands, cols): @@ -385,32 +407,26 @@ def mk_photom_cols(t, metric_defs, n, bands, cols): row_strs : `tuple` of `str` A tuple of the formatted strings for the photometry columns """ - row_strs = [] + row_cells = [] for col in cols: - row_str = "" - num_bad = 0 - debug_group_all = None + full_cell = cell_contents() for band in ["u", "g", "r", "i", "z", "y"]: if band in bands: for part in ["psf_cModel_diff"]: val_col_name = col + "_" + band + "_" + part + "_median" sig_col_name = col + "_" + band + "_" + part + "_sigmaMad" - val_str, sig_str, bad_val, link, debug_group = mk_table_value( + cell_entry = mk_table_value( t[n], metric_defs, val_col_name, sig_col_name ) - else: - val_str = None - sig_str = None - if val_str is None: - continue - row_str += f"{band}: {val_str} σ: {sig_str}
\n" - num_bad += bad_val - if debug_group is not None: - debug_group_all = debug_group - row_strs.append((row_str, num_bad, link, debug_group_all)) + full_cell.text += f"{band}: {cell_entry.val_str} " + full_cell.text += f"σ: {cell_entry.sig_str}
\n" + full_cell.num_fails += cell_entry.num_bad + full_cell.debug_group = cell_entry.debug_group + full_cell.link = cell_entry.link + row_cells.append(full_cell) - return row_strs + return row_cells def mk_sky_cols(t, metric_defs, n, bands, cols): @@ -434,36 +450,22 @@ def mk_sky_cols(t, metric_defs, n, bands, cols): row_strs : `tuple` of `str` A tuple of the formatted strings for the sky columns """ - row_strs = [] + row_cells = [] for col in cols: for stat, dev in [("mean", "stdev"), ("median", "sigmaMAD")]: - row_str = "" - num_bad = 0 - debug_group_all = None + full_cell = cell_contents() for band in ["u", "g", "r", "i", "z", "y"]: if band in bands: val_col_name = col + "_" + band + "_" + stat + "Sky" sig_col_name = col + "_" + band + "_" + dev + "Sky" - val_str, sig_str, bad_val, link, debug_group = mk_table_value( + cell_entry = mk_table_value( t[n], metric_defs, val_col_name, sig_col_name ) - else: - val_str = None - - if val_str is None: - continue - row_str += ( - "" - + band - + ": " - + val_str - + " σ: " - + sig_str - + "
\n" - ) - num_bad += bad_val - if debug_group is not None: - debug_group_all = debug_group - row_strs.append((row_str, num_bad, link, debug_group_all)) - - return row_strs + full_cell.text += f"{band}: {cell_entry.val_str} " + full_cell.text += f"σ: {cell_entry.sig_str}
\n" + full_cell.num_fails += cell_entry.num_bad + if full_cell.debug_group is not None: + full_cell.debug_group = cell_entry.debug_group + row_cells.append(full_cell) + + return row_cells diff --git a/python/lsst/production/tools/tractTable.py b/python/lsst/production/tools/tractTable.py index b306c0a..039bc57 100644 --- a/python/lsst/production/tools/tractTable.py +++ b/python/lsst/production/tools/tractTable.py @@ -45,13 +45,13 @@ def shorten_repo(repo_name): """ Return the repo name without any '/repo/' prefix """ - return repo_name.split('/')[-1] + return repo_name.split("/")[-1] @bp.route("/") def index(): - #collection_names = ["u/sr525/metricsPlotsPDR2_wholeSky"] + # collection_names = ["u/sr525/metricsPlotsPDR2_wholeSky"] official_collection_entries = [] user_collection_entries = [] @@ -65,40 +65,55 @@ def index(): while is_truncated: response = s3_client.list_objects( - Bucket="rubin-plot-navigator", - Marker=marker + Bucket="rubin-plot-navigator", Marker=marker ) - is_truncated = response['IsTruncated'] - print(f"Number of responses {len(response['Contents'])} truncated {is_truncated}") - for entry in response['Contents']: - marker = entry['Key'] + is_truncated = response["IsTruncated"] + print( + f"Number of responses {len(response['Contents'])} truncated {is_truncated}" + ) + for entry in response["Contents"]: + marker = entry["Key"] for repo in REPO_NAMES: repo_encoded = urllib.parse.quote_plus(repo) - if entry['Key'].startswith(repo_encoded): - collection_enc = entry['Key'].replace(repo_encoded + "/collection_", "", 1).replace(".json.gz", "") + if entry["Key"].startswith(repo_encoded): + collection_enc = ( + entry["Key"] + .replace(repo_encoded + "/collection_", "", 1) + .replace(".json.gz", "") + ) collection = urllib.parse.unquote(collection_enc) - if collection.startswith('u'): - user_collection_entries.append({"name": collection, - "updated": entry['LastModified'], - "repo": shorten_repo(repo), "url": - urllib.parse.quote(collection, safe="")}) + if collection.startswith("u"): + user_collection_entries.append( + { + "name": collection, + "updated": entry["LastModified"], + "repo": shorten_repo(repo), + "url": urllib.parse.quote(collection, safe=""), + } + ) else: - official_collection_entries.append({"name": collection, - "updated": entry['LastModified'], - "repo": shorten_repo(repo), "url": - urllib.parse.quote(collection, safe="")}) - + official_collection_entries.append( + { + "name": collection, + "updated": entry["LastModified"], + "repo": shorten_repo(repo), + "url": urllib.parse.quote(collection, safe=""), + } + ) except botocore.exceptions.ClientError as e: print(e) - official_collection_entries.sort(key=lambda x: x['updated'], reverse=True) - user_collection_entries.sort(key=lambda x: x['updated'], reverse=True) + official_collection_entries.sort(key=lambda x: x["updated"], reverse=True) + user_collection_entries.sort(key=lambda x: x["updated"], reverse=True) - return render_template("metrics/index.html", collection_entries=official_collection_entries, - user_collection_entries=user_collection_entries) + return render_template( + "metrics/index.html", + collection_entries=official_collection_entries, + user_collection_entries=user_collection_entries, + ) @bp.route("/collection//") @@ -125,14 +140,14 @@ def collection(repo, collection): expanded_repo_name = repo else: for test_name in REPO_NAMES: - if(repo == test_name.split('/')[-1]): + if repo == test_name.split("/")[-1]: expanded_repo_name = f"/repo/{repo}" if not expanded_repo_name: return {"error": f"Invalid repo name {repo}, collection {collection}"}, 404 butler = Butler(expanded_repo_name) - if(expanded_repo_name == "/repo/main"): + if expanded_repo_name == "/repo/main": dataId = {"skymap": "hsc_rings_v1", "instrument": "HSC"} else: dataId = {"skymap": "lsst_cells_v1", "instrument": "LSSTComCam"} @@ -179,39 +194,39 @@ def collection(repo, collection): num_bad = 0 cell_vals = [] # Get the number of failed values and prep cell contents - for cell_val, bad_val, link, debug_group in mk_shape_cols( - t, metric_defs, n, bands, col_dict["shape_cols"] + for shape_cell in mk_shape_cols( + t[n], metric_defs, bands, col_dict["shape_cols"] ): - cell_vals.append((cell_val, link, debug_group)) - if bad_val is not None: - num_bad += bad_val + cell_vals.append(shape_cell) + if shape_cell.num_fails is not None: + num_bad += shape_cell.num_fails # Make the cell details for the stellar locus columns - for cell_val, bad_val, link, debug_group in mk_stellar_locus_cols( + for sl_cell in mk_stellar_locus_cols( t, metric_defs, n, col_dict["stellar_locus_cols"] ): - cell_vals.append((cell_val, link, debug_group)) - if bad_val is not None: - num_bad += bad_val + cell_vals.append(sl_cell) + if sl_cell.num_fails is not None: + num_bad += sl_cell.num_fails # Make the cell contents for the photometry columns - for cell_val, bad_val, link, debug_group in mk_photom_cols( + for photom_cell in mk_photom_cols( t, metric_defs, n, bands, col_dict["photom_cols"] ): - cell_vals.append((cell_val, link, debug_group)) - if bad_val is not None: - num_bad += bad_val + cell_vals.append(photom_cell) + if photom_cell.num_fails is not None: + num_bad += photom_cell.num_fails # Make the cell contents for the sky columns - for cell_val, bad_val, link, debug_group in mk_sky_cols( - t, metric_defs, n, bands, col_dict["sky_cols"] - ): - cell_vals.append((cell_val, link, debug_group)) - if bad_val is not None: - num_bad += bad_val + for sky_cell in mk_sky_cols(t, metric_defs, n, bands, col_dict["sky_cols"]): + cell_vals.append(sky_cell) + if sky_cell.num_fails is not None: + num_bad += sky_cell.num_fails # Add a nan/bad summary cell next but need to calculate these numbers first - row_list.append((num_bad,)) + bad_cell = cell_contents() + bad_cell.text = str(num_bad) + row_list.append(bad_cell) for val in cell_vals: row_list.append(val) diff --git a/templates/metrics/tracts.html b/templates/metrics/tracts.html index e928419..769d334 100644 --- a/templates/metrics/tracts.html +++ b/templates/metrics/tracts.html @@ -26,23 +26,23 @@ {% for tract in content_dict %} - + {{tract}} {% for cell in content_dict[tract][1:] %} - {% if cell|length == 1 %} - {{cell[0]|safe}} + {% if cell.text|length == 1 %} + {{cell.text|safe}} {% else %} - {% if 'badValue' in cell[0] %} + {% if 'badValue' in cell.text %} - - {{cell[0]|safe}} + {{cell.text|safe}} {% else %} - {{cell[0]|safe}} + {{cell.text|safe}} {% endif %} {% endif %} {% endfor %}