From 81ee2f2160a372dc56349fae22e8e8e2abc1cf36 Mon Sep 17 00:00:00 2001 From: Sophie Reed Date: Wed, 30 Oct 2024 12:04:28 -0700 Subject: [PATCH 1/8] Switch to snake case and tidy up --- python/lsst/production/tools/htmlUtils.py | 135 ++++++++++ python/lsst/production/tools/tractTable.py | 278 ++++++++------------- templates/metrics/tracts.html | 8 +- 3 files changed, 243 insertions(+), 178 deletions(-) create mode 100644 python/lsst/production/tools/htmlUtils.py diff --git a/python/lsst/production/tools/htmlUtils.py b/python/lsst/production/tools/htmlUtils.py new file mode 100644 index 0000000..0c114ea --- /dev/null +++ b/python/lsst/production/tools/htmlUtils.py @@ -0,0 +1,135 @@ +import numpy as np + +def mk_table_value(t, metric_defs, val_col_name, sig_col_name, n): + if val_col_name in t.columns and sig_col_name in t.columns: + val = t[val_col_name][n] + val_str = f"{val:.3g}" + sig = t[sig_col_name][n] + sig_str = f"{sig:.3g}" + elif val_col_name in t.columns and sig_col_name not in t.columns: + val = t[val_col_name][n] + val_str = f"{val:.3g}" + sig_str = "-" + elif val_col_name not in t.columns and sig_col_name in t.columns: + val_str = "_" + sig = t[sig_col_name][n] + sig_str = f"{sig:.3g}" + else: + return None, None + + if np.isnan(val): + val_str = f"{val:.3g} " + if np.isnan(sig): + sig_str = f"{sig:.3g}\n" + if val_col_name in metric_defs: + high_val = metric_defs[val_col_name]["highThreshold"] + low_val = metric_defs[val_col_name]["lowThreshold"] + link = metric_defs[val_col_name]["debugGroup"] + "ReportPage.html" + if val < low_val or val > high_val: + #valStr = "{val:.3g} " + val_str = f"{val:.3g}" + if sig_col_name in metric_defs: + high_sig = metric_defs[sig_col_name]["highThreshold"] + low_sig = metric_defs[sig_col_name]["lowThreshold"] + if sig < low_sig or sig > high_sig: + #sigStr = "{sig:.3g}\n" + sig_str = f"{sig:.3g}\n" + + return val_str, sig_str + + +def mk_table_headers(t, col_dict): + + col_first = [] + bands = [] + for col in t.columns: + col_sections = col.split("_") + for section in col_sections: + if len(section) == 1: + bands.append(section) + col_first.append(col.split("_")[0]) + + bands = list(set(bands)) + + table_headers = col_dict["table_cols"] + for shape_col in col_dict["shape_cols"]: + table_headers.append(shape_col + "
High SN Stars") + table_headers.append(shape_col + "
Low SN Stars") + for col in col_dict["stellar_locus_cols"]: + table_headers.append(col) + + header_list = [] + link_list = [] + header_dict = {} + for header in table_headers: + if header == "failed metrics": + # Needs to link to metric fail page + header_dict[header] = "metrics.index" + elif header == "corners": + # Needs to link to metric summary page + header_dict[header] = "metrics.index" + else: + # Needs to link to the correct point on the histogram page + #headerList.append("")[0] + f">{header}") + header_dict[header] = "metrics.index" + + return header_dict, bands + + +def mk_tract_cell(tract): + #tractStr = "" + #tractStr += str(tract) + "" + tract_str = str(tract) + + return tract_str + + +def mk_summary_plot_cell(tract): + plot_str = "" + return plot_str + + +def mk_patch_num_cell(t, n, bands): + patch_str = "" + for band in ["g", "r", "i", "z", "y"]: + if band in bands: + patch_col = "coaddPatchCount_" + band + "_patchCount" + patch_str += "" + band + ": " + str(int(t[patch_col][n])) + "
\n" + + return patch_str + + +def mk_shape_cols(t, metric_defs, n, bands, col_dict): + shape_strs = [] + for col in col_dict["shape_cols"]: + for sn in ["highSNStars", "lowSNStars"]: + shape_str = "" + for band in ["g", "r", "i", "z", "y"]: + if band in bands: + val_col_name = col + "_" + band + "_" + sn + "_median" + sig_col_name = col + "_" + band + "_" + sn + "_sigmaMad" + val_str, sig_str = mk_table_value(t, metric_defs, val_col_name, sig_col_name, n) + shape_str += "" + band + f": " + val_str + " σ: " + shape_str += sig_str + "
\n" + shape_strs.append(shape_str) + return shape_strs + + +def mk_stellar_locus_cols(t, metric_defs, n, col_dict): + row_strs = [] + for col in col_dict["stellar_locus_cols"]: + for (flux, flux1) in zip(["psfFlux", "CModel"], ["PSF", "CModel"]): + if col[0] == "w" or col[0] == "x": + flux1 += "P" + val_col_name = col + flux1 + "_" + col + "_" + flux + "_median" + sig_col_name = col + flux1 + "_" + col + "_" + flux + "_sigmaMAD" + val_str, sig_str = mk_table_value(t, metric_defs, val_col_name, sig_col_name, n) + if val_str is None: + continue + row_str = "Med: " + val_str + " σ: " + sig_str + "
\n" + row_strs.append(row_str) + + return row_strs + + + diff --git a/python/lsst/production/tools/tractTable.py b/python/lsst/production/tools/tractTable.py index 251309c..13aaf15 100644 --- a/python/lsst/production/tools/tractTable.py +++ b/python/lsst/production/tools/tractTable.py @@ -19,222 +19,152 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from lsst.daf.butler import Butler - -from flask import Blueprint, Flask, render_template, url_for -import numpy as np +import os import urllib.parse -import yaml + import boto3 import botocore -import os +import numpy as np +import yaml +from flask import Blueprint, Flask, render_template, url_for +from lsst.daf.butler import Butler + +from .htmlUtils import * -bp = Blueprint("metrics", __name__, url_prefix="/plot-navigator/metrics", static_folder="../../../../static") +bp = Blueprint( + "metrics", + __name__, + url_prefix="/plot-navigator/metrics", + static_folder="../../../../static", +) NO_BUTLER = True + @bp.route("/") def index(): collection_names = ["u/sr525/metricsPlotsPDR2_wholeSky"] - collection_entries = [{"name": name, "url": urllib.parse.quote(name, safe='')} for name in collection_names] - - return render_template("metrics/index.html", - collection_entries=collection_entries) - - -def mkTableValue(t, metricDefs, valColName, sigColName, n): - if valColName in t.columns and sigColName in t.columns: - val = t[valColName][n] - valStr = f"{val:.3g}" - sig = t[sigColName][n] - sigStr = f"{sig:.3g}" - elif valColName in t.columns and sigColName not in t.columns: - val = t[valColName][n] - valStr = f"{val:.3g}" - sigStr = "-" - elif valColName not in t.columns and sigColName in t.columns: - valStr = "_" - sig = t[sigColName][n] - sigStr = f"{sig:.3g}" - else: - return None, None - - if np.isnan(val): - valStr = f"{val:.3g} " - if np.isnan(sig): - sigStr = f"{sig:.3g}\n" - if valColName in metricDefs: - highVal = metricDefs[valColName]["highThreshold"] - lowVal = metricDefs[valColName]["lowThreshold"] - link = metricDefs[valColName]["debugGroup"] + "ReportPage.html" - if val < lowVal or val > highVal: - #valStr = "{val:.3g} " - valStr = f"{val:.3g}" - if sigColName in metricDefs: - highSig = metricDefs[sigColName]["highThreshold"] - lowSig = metricDefs[sigColName]["lowThreshold"] - if sig < lowSig or sig > highSig: - #sigStr = "{sig:.3g}\n" - sigStr = f"{sig:.3g}\n" - - return valStr, sigStr - - -def mkTableHeaders(t): - - colFirst = [] - bands = [] - for col in t.columns: - colSections = col.split("_") - for section in colSections: - if len(section) == 1: - bands.append(section) - colFirst.append(col.split("_")[0]) - - bands = list(set(bands)) - - tableCols = ["tract", "failed metrics", "corners", "nPatches"] - shapeCols = ["shapeSizeFractionalDiff", "e1Diff", "e2Diff"] - photomCols = ["psfCModelScatter"] - stellarLocusCols = ["yPerpPSF", "yPerpCModel", "wPerpCModel", "wPerpPSFP", "xPerpPSFP", "xPerpCModel"] - stellarLocusCols = ["yPerp", "wPerp", "xPerp"] - skyCols = ["skyFluxStatisticMetric"] - - tableHeaders = tableCols - for shapeCol in shapeCols: - tableHeaders.append(shapeCol + "
High SN Stars") - tableHeaders.append(shapeCol + "
Low SN Stars") - for col in stellarLocusCols: - tableHeaders.append(col) - - headerList = [] - linkList = [] - headerDict = {} - for header in tableHeaders: - if header == "failed metrics": - # Needs to link to metric fail page - headerDict[header] = "metrics.index" - elif header == "corners": - # Needs to link to metric summary page - headerDict[header] = "metrics.index" - else: - # Needs to link to the correct point on the histogram page - #headerList.append("")[0] + f">{header}") - headerDict[header] = "metrics.index" - - return headerDict, bands - -def mkTractCell(tract): - #tractStr = "" - #tractStr += str(tract) + "" - tractStr = str(tract) - - return tractStr - -def mkSummaryPlotCell(tract): - plotStr = "" - return plotStr - -def mkPatchNumCell(t, n, bands): - patchStr = "" - for band in ["g", "r", "i", "z", "y"]: - if band in bands: - patchCol = "coaddPatchCount_" + band + "_patchCount" - patchStr += "" + band + ": " + str(int(t[patchCol][n])) + "
\n" - - return patchStr - - -def mkShapeCols(t, metricDefs, n, bands): - shapeCols = ["shapeSizeFractionalDiff", "e1Diff", "e2Diff"] - shapeStrs = [] - for col in shapeCols: - for sn in ["highSNStars", "lowSNStars"]: - shapeStr = "" - for band in ["g", "r", "i", "z", "y"]: - if band in bands: - valColName = col + "_" + band + "_" + sn + "_median" - sigColName = col + "_" + band + "_" + sn + "_sigmaMad" - valStr, sigStr = mkTableValue(t, metricDefs, valColName, sigColName, n) - shapeStr += "" + band + f": " + valStr + " σ: " + sigStr + "
\n" - shapeStrs.append(shapeStr) - return shapeStrs - - -def mkStellarLocusCols(t, metricDefs, n): - rowStrs = [] - stellarLocusCols = ["yPerp", "wPerp", "xPerp"] - for col in stellarLocusCols: - for (flux, flux1) in zip(["psfFlux", "CModel"], ["PSF", "CModel"]): - if col[0] == "w" or col[0] == "x": - flux1 += "P" - valColName = col + flux1 + "_" + col + "_" + flux + "_median" - sigColName = col + flux1 + "_" + col + "_" + flux + "_sigmaMAD" - valStr, sigStr = mkTableValue(t, metricDefs, valColName, sigColName, n) - if valStr is None: - continue - rowStr = "Med: " + valStr + " σ: " + sigStr + "
\n" - rowStrs.append(rowStr) - - return rowStrs + collection_entries = [ + {"name": name, "url": urllib.parse.quote(name, safe="")} + for name in collection_names + ] + + return render_template("metrics/index.html", collection_entries=collection_entries) @bp.route("/collection/") def collection(collection_urlencoded): + """Make all the information needed to supply to the template + for the tract summary table. - collection = urllib.parse.unquote(collection_urlencoded) + Currently set up to take the standard columns from the + analyzeObjectTableTract portion of the + coaddObjectCore pipeline metrics. - bands = ['g','r','i','z','y'] + The thresholds for the metrics are in the metricDefs + yaml file. + """ - # tracts = [{"number": 9812}, {"number": 1234}, {"number": 9000}] + collection = urllib.parse.unquote(collection_urlencoded) butler = Butler("/repo/main") dataId = {"skymap": "hsc_rings_v1", "instrument": "HSC"} - t = butler.get("objectTableCore_metricsTable", collections=collection, dataId=dataId) + t = butler.get( + "objectTableCore_metricsTable", collections=collection, dataId=dataId + ) - # with open("../../metricThresholds/metricInformation.yaml", "r") as filename: - # metricDefs = yaml.safe_load(filename) + col_dict = { + "table_cols": ["tract", "corners", "nPatches", "nInputs", "failed metrics"], + "shape_cols": ["shapeSizeFractionalDiff", "e1Diff", "e2Diff"], + "photom_cols": ["psfCModelScatter"], + "stellar_locus_cols": ["yPerp", "wPerp", "xPerp"], + "sky_cols": ["skyFluxStatisticMetric"], + } - session = boto3.Session(profile_name='rubin-plot-navigator') - s3_client = session.client('s3', endpoint_url=os.getenv("S3_ENDPOINT_URL")) + session = boto3.Session(profile_name="rubin-plot-navigator") + s3_client = session.client("s3", endpoint_url=os.getenv("S3_ENDPOINT_URL")) metric_threshold_filename = "metricInformation.yaml" try: - response = s3_client.get_object(Bucket='rubin-plot-navigator', Key=metric_threshold_filename) - metricDefs = yaml.safe_load(response["Body"]) + response = s3_client.get_object( + Bucket="rubin-plot-navigator", Key=metric_threshold_filename + ) + metric_defs = yaml.safe_load(response["Body"]) except botocore.exceptions.ClientError as e: print(e) + # Make the headers for the table + header_dict, bands = mk_table_headers(t, col_dict) - headerDict, bands = mkTableHeaders(t) - contentDict = {} - for (n, tract) in enumerate(t["tract"]): - rowList = [] - rowList.append(mkTractCell(tract)) + # Make the content for the table + content_dict = {} + for n, tract in enumerate(t["tract"]): + row_list = [] + row_list.append(mk_tract_cell(tract)) # Add a nan/bad summary cell next but need to calculate these numbers first - rowList.append("0") + row_list.append("0") # Add a summary plot in the i band of the tract - rowList.append(mkSummaryPlotCell(tract)) - rowList.append(mkPatchNumCell(t, n, bands)) - for cellStr in mkShapeCols(t, metricDefs, n, bands): - rowList.append(cellStr) - for cellStr in mkStellarLocusCols(t, metricDefs, n): - rowList.append(cellStr) - - contentDict[tract] = rowList + row_list.append(mk_summary_plot_cell(tract)) + row_list.append(mk_patch_num_cell(t, n, bands)) + # Add the number of inputs + row_list.append(mk_num_inputs_cell(t, metric_defs, n, bands)) + + num_bad = 0 + cell_vals = [] + # Get the number of failed values and prep cell contents + for cell_val, bad_val, link, debug_group in mk_shape_cols( + t, metric_defs, n, bands, col_dict + ): + cell_vals.append((cell_val, link, debug_group)) + if bad_val is not None: + num_bad += bad_val + + # Make the cell details for the stellar locus columns + for cell_val, bad_val, link, debug_group in mk_stellar_locus_cols( + t, metric_defs, n, col_dict + ): + cell_vals.append((cell_val, link, debug_group)) + if bad_val is not None: + num_bad += bad_val + + # Make the cell contents for the photometry columns + for cell_val, bad_val, link, debug_group in mk_photom_cols( + t, metric_defs, n, bands, col_dict + ): + cell_vals.append((cell_val, link, debug_group)) + if bad_val is not None: + num_bad += bad_val + + # Make the cell contents for the sky columns + for cell_val, bad_val, link, debug_group in mk_sky_cols( + t, metric_defs, n, bands, col_dict + ): + cell_vals.append((cell_val, link, debug_group)) + if bad_val is not None: + num_bad += bad_val + # Add a nan/bad summary cell next but need to calculate these numbers first + row_list.append((num_bad,)) + for val in cell_vals: + row_list.append(val) + content_dict[tract] = row_list + return render_template( + "metrics/tracts.html", + header_dict=header_dict, + content_dict=content_dict, + collection_urlencoded=collection_urlencoded, + ) - return render_template("metrics/tracts.html", headerDict=headerDict, contentDict=contentDict) @bp.route("/histograms/") def histograms(collection_name): - bands = ['g','r','i','z','y'] + bands = ["g", "r", "i", "z", "y"] return render_template("metrics/histograms.html") diff --git a/templates/metrics/tracts.html b/templates/metrics/tracts.html index ee87655..291d2f8 100644 --- a/templates/metrics/tracts.html +++ b/templates/metrics/tracts.html @@ -10,14 +10,14 @@ - {% for header in headerDict %} - + {% for header in header_dict %} + {% endfor %} - {% for tract in contentDict %} + {% for tract in content_dict %} - {% for cell in contentDict[tract] %} + {% for cell in content_dict[tract] %} {% endfor %} From 92d938e918cd6fb0617069e6c8b32986729f8d71 Mon Sep 17 00:00:00 2001 From: Sophie Reed Date: Wed, 30 Oct 2024 17:47:12 -0700 Subject: [PATCH 2/8] Fix link colors --- static/custom.css | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/static/custom.css b/static/custom.css index 03dc6f9..078475e 100644 --- a/static/custom.css +++ b/static/custom.css @@ -36,16 +36,27 @@ a:link { text-decoration: none; } -a:visited{ +a:visited { color: #000000; } +a.tableHeader:link, a.tableHeader:visited { + color: #FFFFFF; + text-decoration: none; +} + +a.tableLink:link, a.tableLink:visited { + color: #909B9C; + text-decoration: none; +} + + .nanValue { color: #CC5500; } .badValue { - color: #CD1717; + color: #000000; background: #CC5500; padding: 1pt; font-weight: bold; From b4c6d2aee862a2d190a322b1403ca9d8848d1912 Mon Sep 17 00:00:00 2001 From: Sophie Reed Date: Wed, 30 Oct 2024 17:47:43 -0700 Subject: [PATCH 3/8] Add a metric report page template --- templates/metrics/report_page.html | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 templates/metrics/report_page.html diff --git a/templates/metrics/report_page.html b/templates/metrics/report_page.html new file mode 100644 index 0000000..746fe91 --- /dev/null +++ b/templates/metrics/report_page.html @@ -0,0 +1,14 @@ +{% extends 'base.html' %} +{% set active_page="metrics" %} + +{% block title %}Report for {% endblock %} + + +{% block header %}Full Collection Metric Summary{% endblock %} + +{% block content %} + +
{{header|safe}}{{header|safe}}
{{cell|safe}}
+
+ +{% endblock %} From 40b08b7872b45fd6e1d90f0c64bbab671ae69e24 Mon Sep 17 00:00:00 2001 From: Sophie Reed Date: Wed, 30 Oct 2024 17:49:12 -0700 Subject: [PATCH 4/8] Make single tract page show nothing and stop crashing --- templates/metrics/single_tract.html | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/templates/metrics/single_tract.html b/templates/metrics/single_tract.html index ee36756..50fb083 100644 --- a/templates/metrics/single_tract.html +++ b/templates/metrics/single_tract.html @@ -9,23 +9,6 @@ {% block content %} - - - - - - - {% for tract in tracts %} - - - - - {% endfor %} -
TractnPatches
{{tract['number']}} - {% for band in bands %} - {{band}}: 3:
- {% endfor %} -
{% endblock %} From b48dfeafc0033673dfee31e502aa7d5e58a0d941 Mon Sep 17 00:00:00 2001 From: Sophie Reed Date: Wed, 30 Oct 2024 18:19:47 -0700 Subject: [PATCH 5/8] Add htmlUtils to __init__ --- python/lsst/production/tools/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lsst/production/tools/__init__.py b/python/lsst/production/tools/__init__.py index f40d540..5b320e0 100644 --- a/python/lsst/production/tools/__init__.py +++ b/python/lsst/production/tools/__init__.py @@ -23,7 +23,7 @@ import urllib.parse from werkzeug.routing import BaseConverter -from . import tractTable, logs, bokeh, cache, images +from . import tractTable, logs, bokeh, cache, images, htmlUtils # This works like the built-in 'path' converter, but From 3df00abe682d0d854940d6b7fb04cec593cc0a90 Mon Sep 17 00:00:00 2001 From: Sophie Reed Date: Wed, 30 Oct 2024 18:20:49 -0700 Subject: [PATCH 6/8] Update the tracts template to fix links --- templates/metrics/tracts.html | 36 ++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/templates/metrics/tracts.html b/templates/metrics/tracts.html index 291d2f8..85e9ce8 100644 --- a/templates/metrics/tracts.html +++ b/templates/metrics/tracts.html @@ -8,22 +8,44 @@ {% block content %} + {% for header in header_dict %} - + {% endfor %} + {% for tract in content_dict %} - - {% for cell in content_dict[tract] %} - + + + {% for cell in content_dict[tract][1:] %} + {% if cell|length == 1 %} + + {% else %} + {% if 'badValue' in cell[0] %} + + {% else %} + + {% endif %} + {% endif %} {% endfor %} - + {% endfor %}
{{header|safe}} + + {{header|safe}} + +
{{cell|safe}}
+ + {{tract}} + + {{cell[0]|safe}} + + {{cell[0]|safe}} + + {{cell[0]|safe}}
- - {% endblock %} From a9edd36c1340fa7c016ac1b581f54367acb2f878 Mon Sep 17 00:00:00 2001 From: Sophie Reed Date: Wed, 30 Oct 2024 18:21:42 -0700 Subject: [PATCH 7/8] Fix links from tract table page --- python/lsst/production/tools/htmlUtils.py | 71 +++++++++++++++------- python/lsst/production/tools/tractTable.py | 7 ++- 2 files changed, 53 insertions(+), 25 deletions(-) diff --git a/python/lsst/production/tools/htmlUtils.py b/python/lsst/production/tools/htmlUtils.py index 0c114ea..e38acdd 100644 --- a/python/lsst/production/tools/htmlUtils.py +++ b/python/lsst/production/tools/htmlUtils.py @@ -15,7 +15,15 @@ def mk_table_value(t, metric_defs, val_col_name, sig_col_name, n): sig = t[sig_col_name][n] sig_str = f"{sig:.3g}" else: - return None, None + return None, None, None, None, None + + bad_val = 0 + + link = "metrics.report_page" + if val_col_name in metric_defs: + debug_group = metric_defs[val_col_name]["debugGroup"] + else: + debug_group = None if np.isnan(val): val_str = f"{val:.3g} " @@ -24,18 +32,18 @@ def mk_table_value(t, metric_defs, val_col_name, sig_col_name, n): if val_col_name in metric_defs: high_val = metric_defs[val_col_name]["highThreshold"] low_val = metric_defs[val_col_name]["lowThreshold"] - link = metric_defs[val_col_name]["debugGroup"] + "ReportPage.html" + debug_group = metric_defs[val_col_name]["debugGroup"] if val < low_val or val > high_val: - #valStr = "{val:.3g} " val_str = f"{val:.3g}" + bad_val += 1 if sig_col_name in metric_defs: high_sig = metric_defs[sig_col_name]["highThreshold"] low_sig = metric_defs[sig_col_name]["lowThreshold"] if sig < low_sig or sig > high_sig: - #sigStr = "{sig:.3g}\n" sig_str = f"{sig:.3g}\n" + bad_val += 1 - return val_str, sig_str + return val_str, sig_str, bad_val, link, debug_group def mk_table_headers(t, col_dict): @@ -64,29 +72,28 @@ def mk_table_headers(t, col_dict): for header in table_headers: if header == "failed metrics": # Needs to link to metric fail page - header_dict[header] = "metrics.index" + header_dict[header] = "metrics.histograms" elif header == "corners": # Needs to link to metric summary page - header_dict[header] = "metrics.index" + header_dict[header] = "metrics.histograms" else: # Needs to link to the correct point on the histogram page #headerList.append("")[0] + f">{header}") - header_dict[header] = "metrics.index" + header_dict[header] = "metrics.histograms" return header_dict, bands def mk_tract_cell(tract): - #tractStr = "" - #tractStr += str(tract) + "" - tract_str = str(tract) + tract_str = "metrics.single_tract" - return tract_str + return (tract_str, ) def mk_summary_plot_cell(tract): - plot_str = "" - return plot_str + #plot_str = "" + plot_str = "nav link needed" + return (plot_str, ) def mk_patch_num_cell(t, n, bands): @@ -96,7 +103,7 @@ def mk_patch_num_cell(t, n, bands): patch_col = "coaddPatchCount_" + band + "_patchCount" patch_str += "" + band + ": " + str(int(t[patch_col][n])) + "
\n" - return patch_str + return (patch_str, ) def mk_shape_cols(t, metric_defs, n, bands, col_dict): @@ -104,30 +111,50 @@ def mk_shape_cols(t, metric_defs, n, bands, col_dict): for col in col_dict["shape_cols"]: for sn in ["highSNStars", "lowSNStars"]: shape_str = "" + num_bad = 0 + debug_group_all = None for band in ["g", "r", "i", "z", "y"]: if band in bands: val_col_name = col + "_" + band + "_" + sn + "_median" sig_col_name = col + "_" + band + "_" + sn + "_sigmaMad" - val_str, sig_str = mk_table_value(t, metric_defs, val_col_name, sig_col_name, n) + val_str, sig_str, bad_val, link, debug_group = mk_table_value(t, + metric_defs, + val_col_name, + sig_col_name, + n) + num_bad += bad_val shape_str += "" + band + f": " + val_str + " σ: " shape_str += sig_str + "
\n" - shape_strs.append(shape_str) + if debug_group is not None: + debug_group_all = debug_group + shape_strs.append((shape_str, num_bad, link, debug_group_all)) return shape_strs def mk_stellar_locus_cols(t, metric_defs, n, col_dict): row_strs = [] for col in col_dict["stellar_locus_cols"]: - for (flux, flux1) in zip(["psfFlux", "CModel"], ["PSF", "CModel"]): - if col[0] == "w" or col[0] == "x": + row_str = "" + num_bad = 0 + debug_group_all = None + for (flux, flux1) in zip(["psfFlux", "cModelFlux"], ["PSF", "CModel"]): + if (col[0] == "w" or col[0] == "x") and flux1 == "PSF": flux1 += "P" val_col_name = col + flux1 + "_" + col + "_" + flux + "_median" sig_col_name = col + flux1 + "_" + col + "_" + flux + "_sigmaMAD" - val_str, sig_str = mk_table_value(t, metric_defs, val_col_name, sig_col_name, n) + val_str, sig_str, bad_val, link, debug_group = mk_table_value(t, + metric_defs, + val_col_name, + sig_col_name, + n) if val_str is None: continue - row_str = "Med: " + val_str + " σ: " + sig_str + "
\n" - row_strs.append(row_str) + row_str += "" + flux + "
Med: " + val_str + " σ: " + sig_str + "
" + row_str += "
\n" + num_bad += bad_val + if debug_group is not None: + debug_group_all = debug_group + row_strs.append((row_str, num_bad, link, debug_group_all)) return row_strs diff --git a/python/lsst/production/tools/tractTable.py b/python/lsst/production/tools/tractTable.py index 13aaf15..1243636 100644 --- a/python/lsst/production/tools/tractTable.py +++ b/python/lsst/production/tools/tractTable.py @@ -82,7 +82,6 @@ def collection(collection_urlencoded): "stellar_locus_cols": ["yPerp", "wPerp", "xPerp"], "sky_cols": ["skyFluxStatisticMetric"], } - session = boto3.Session(profile_name="rubin-plot-navigator") s3_client = session.client("s3", endpoint_url=os.getenv("S3_ENDPOINT_URL")) @@ -104,10 +103,9 @@ def collection(collection_urlencoded): row_list = [] row_list.append(mk_tract_cell(tract)) - # Add a nan/bad summary cell next but need to calculate these numbers first - row_list.append("0") # Add a summary plot in the i band of the tract row_list.append(mk_summary_plot_cell(tract)) + # Add the number of patches row_list.append(mk_patch_num_cell(t, n, bands)) # Add the number of inputs row_list.append(mk_num_inputs_cell(t, metric_defs, n, bands)) @@ -177,4 +175,7 @@ def single_tract(collection_name, tract): return render_template("metrics/single_tract.html", tract=tract) +@bp.route("/report//") +def report_page(collection_name, metric): + return render_template("metrics/report_page.html") From e2e871216ca0deb0e265c45176c8d38d49d4c7e3 Mon Sep 17 00:00:00 2001 From: Sophie Reed Date: Thu, 7 Nov 2024 17:15:53 -0800 Subject: [PATCH 8/8] Add doc strings to htmlUtils --- python/lsst/production/tools/htmlUtils.py | 373 +++++++++++++++++++-- python/lsst/production/tools/tractTable.py | 10 +- 2 files changed, 344 insertions(+), 39 deletions(-) diff --git a/python/lsst/production/tools/htmlUtils.py b/python/lsst/production/tools/htmlUtils.py index e38acdd..2790064 100644 --- a/python/lsst/production/tools/htmlUtils.py +++ b/python/lsst/production/tools/htmlUtils.py @@ -1,18 +1,79 @@ +# This file is part of production-tools. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + import numpy as np -def mk_table_value(t, metric_defs, val_col_name, sig_col_name, n): - if val_col_name in t.columns and sig_col_name in t.columns: - val = t[val_col_name][n] + +def mk_table_value(row, metric_defs, val_col_name, sig_col_name): + """Turn the values from the given columns into + formatted cell contents. + + Parameters + ---------- + t : `astropy.table.Table` + Table of metrics + metric_defs : `dict` + A dictionary of metrics and their thresholds + val_col_name : `str` + The name of the metric column + sig_col_name : `str` + The associated sigma column + n : `n` + The row number + + Returns + ------- + val_str : `str` + The formatted string for the metric value + sig_str : `str` + The formatted string for the sigma value + bad_val : `int` + Number of metrics outside the threshold + link : `str` + The page that the bad value should link to + debug_group : `str` + The group of metrics that this metric belongs to + + Notes + ----- + Returns None for everything if the given metric name + isn't in the table. + """ + + # Make a string of the val and sig columns + if val_col_name in row.columns and sig_col_name in row.columns: + val = row[val_col_name] val_str = f"{val:.3g}" - sig = t[sig_col_name][n] + sig = row[sig_col_name] sig_str = f"{sig:.3g}" - elif val_col_name in t.columns and sig_col_name not in t.columns: - val = t[val_col_name][n] + elif val_col_name in row.columns and sig_col_name not in row.columns: + val = row[val_col_name] val_str = f"{val:.3g}" + sig = None sig_str = "-" - elif val_col_name not in t.columns and sig_col_name in t.columns: - val_str = "_" - sig = t[sig_col_name][n] + elif val_col_name not in row.columns and sig_col_name in row.columns: + val_str = "-" + val = None + sig = row[sig_col_name] sig_str = f"{sig:.3g}" else: return None, None, None, None, None @@ -25,6 +86,8 @@ def mk_table_value(t, metric_defs, val_col_name, sig_col_name, n): else: debug_group = None + # Add formatting to the string if there are thresholds + # for the metric if np.isnan(val): val_str = f"{val:.3g} " if np.isnan(sig): @@ -47,6 +110,22 @@ def mk_table_value(t, metric_defs, val_col_name, sig_col_name, n): def mk_table_headers(t, col_dict): + """Make column headers + + Parameters + ---------- + t : `astropy.table.Table` + Table of metrics + col_dict : `dict` + Dictionary of the grouped column headers + + Returns + ------- + header_dict : `dict` + A dictionary of the headers and what pages they should link to + bands : `list` + The bands that the metrics cover + """ col_first = [] bands = [] @@ -65,6 +144,11 @@ def mk_table_headers(t, col_dict): table_headers.append(shape_col + "
Low SN Stars") for col in col_dict["stellar_locus_cols"]: table_headers.append(col) + for col in col_dict["photom_cols"]: + table_headers.append(col) + for col in col_dict["sky_cols"]: + table_headers.append(col + "
mean, stdev") + table_headers.append(col + "
median, sigmaMAD") header_list = [] link_list = [] @@ -78,52 +162,114 @@ def mk_table_headers(t, col_dict): header_dict[header] = "metrics.histograms" else: # Needs to link to the correct point on the histogram page - #headerList.append("")[0] + f">{header}") + # headerList.append("")[0] + f">{header}") header_dict[header] = "metrics.histograms" return header_dict, bands def mk_tract_cell(tract): + """Make the content of the tract cell + + Parameters + ---------- + tract : `float` + The tract number + + Returns + ------- + tract_str : `str` + The link that the tract should go to + """ tract_str = "metrics.single_tract" - return (tract_str, ) + return (tract_str,) def mk_summary_plot_cell(tract): - #plot_str = "" + """Make the contents of the summary plot cell + + Parameters + ---------- + tract : `float` + + Returns + ------- + plot_str : `str` + + Notes + ----- + This needs updating with a plot navigator link + """ + # plot_str = "" plot_str = "nav link needed" - return (plot_str, ) + return (plot_str,) def mk_patch_num_cell(t, n, bands): + """Make patch number cell content + + Parameters + ---------- + t : `astropy.table.Table` + Table of metrics + n : `n` + The row number + bands : `list` + A list of the bands the metrics are in + + Returns + ------- + patch_str : `str` + The formatted number of patches in each band + """ patch_str = "" - for band in ["g", "r", "i", "z", "y"]: + for band in ["u", "g", "r", "i", "z", "y"]: if band in bands: patch_col = "coaddPatchCount_" + band + "_patchCount" patch_str += "" + band + ": " + str(int(t[patch_col][n])) + "
\n" - return (patch_str, ) + return (patch_str,) + + +def mk_shape_cols(t, metric_defs, n, bands, cols): + """Make shape column cell contents + Parameters + ---------- + t : `astropy.table.Table` + Table of metrics + metric_defs : `dict` + A dictionary of metrics and their thresholds + n : `n` + The row number + bands : `list` + A list of the bands the metrics are in + cols : `list` + A list of columns to format -def mk_shape_cols(t, metric_defs, n, bands, col_dict): + Returns + ------- + shape_strs : `tuple` of `str` + A tuple of the formatted strings for the shape columns + """ shape_strs = [] - for col in col_dict["shape_cols"]: + for col in cols: for sn in ["highSNStars", "lowSNStars"]: shape_str = "" num_bad = 0 debug_group_all = None - for band in ["g", "r", "i", "z", "y"]: + for band in ["u", "g", "r", "i", "z", "y"]: if band in bands: val_col_name = col + "_" + band + "_" + sn + "_median" sig_col_name = col + "_" + band + "_" + sn + "_sigmaMad" - val_str, sig_str, bad_val, link, debug_group = mk_table_value(t, - metric_defs, - val_col_name, - sig_col_name, - n) + val_str, sig_str, bad_val, link, debug_group = mk_table_value( + t[n], metric_defs, val_col_name, sig_col_name + ) num_bad += bad_val - shape_str += "" + band + f": " + val_str + " σ: " + shape_str += ( + "" + band + f": " + val_str + " σ: " + ) shape_str += sig_str + "
\n" if debug_group is not None: debug_group_all = debug_group @@ -131,25 +277,49 @@ def mk_shape_cols(t, metric_defs, n, bands, col_dict): return shape_strs -def mk_stellar_locus_cols(t, metric_defs, n, col_dict): +def mk_stellar_locus_cols(t, metric_defs, n, cols): + """Make stellar locus column cell contents + + Parameters + ---------- + t : `astropy.table.Table` + Table of metrics + metric_defs : `dict` + A dictionary of metrics and their thresholds + n : `n` + The row number + cols : `list` + A list of the columns to format + + Returns + ------- + row_strs : `tuple` of `str` + A tuple of the formatted strings for the stellar locus columns + """ row_strs = [] - for col in col_dict["stellar_locus_cols"]: + for col in cols: row_str = "" num_bad = 0 debug_group_all = None - for (flux, flux1) in zip(["psfFlux", "cModelFlux"], ["PSF", "CModel"]): + for flux, flux1 in zip(["psfFlux", "cModelFlux"], ["PSF", "CModel"]): if (col[0] == "w" or col[0] == "x") and flux1 == "PSF": flux1 += "P" val_col_name = col + flux1 + "_" + col + "_" + flux + "_median" sig_col_name = col + flux1 + "_" + col + "_" + flux + "_sigmaMAD" - val_str, sig_str, bad_val, link, debug_group = mk_table_value(t, - metric_defs, - val_col_name, - sig_col_name, - n) + val_str, sig_str, bad_val, link, debug_group = mk_table_value( + t[n], metric_defs, val_col_name, sig_col_name + ) if val_str is None: continue - row_str += "" + flux + "
Med: " + val_str + " σ: " + sig_str + "
" + row_str += ( + "" + + flux + + "
Med: " + + val_str + + " σ: " + + sig_str + + "
" + ) row_str += "
\n" num_bad += bad_val if debug_group is not None: @@ -159,4 +329,141 @@ def mk_stellar_locus_cols(t, metric_defs, n, col_dict): return row_strs +def mk_num_inputs_cell(t, metric_defs, n, bands): + """Format the number of inputs cell + + Parameters + ---------- + t : `astropy.table.Table` + Table of metrics + metric_defs : `dict` + A dictionary of metrics and their thresholds + n : `n` + The row number + bands : `list` + A list of the bands the metrics are in + + Returns + ------- + row_str : `str` + A string of the formatted cell contents + """ + + row_str = "" + for band in ["u", "g", "r", "i", "z", "y"]: + if band in bands: + val_col_name = "coaddInputCount_" + band + "_inputCount_median" + sig_col_name = "coaddInputCount_" + band + "_inputCount_sigmaMad" + + val_str, sig_str, _, _, _ = mk_table_value( + t[n], metric_defs, val_col_name, sig_col_name + ) + row_str += "" + band + ":" + val_str + " σ " + row_str += sig_str + "
\n" + + return (row_str,) + + +def mk_photom_cols(t, metric_defs, n, bands, cols): + """Make photometry column cell contents + + Parameters + ---------- + t : `astropy.table.Table` + Table of metrics + metric_defs : `dict` + A dictionary of metrics and their thresholds + n : `n` + The row number + bands : `list` + A list of the bands the metrics are in + cols : `list` + A list of the columns to format + + Returns + ------- + row_strs : `tuple` of `str` + A tuple of the formatted strings for the photometry columns + """ + row_strs = [] + for col in cols: + row_str = "" + num_bad = 0 + debug_group_all = None + for band in ["u", "g", "r", "i", "z", "y"]: + if band in bands: + for part in ["psf_cModel_diff"]: + val_col_name = col + "_" + band + "_" + part + "_median" + sig_col_name = col + "_" + band + "_" + part + "_sigmaMad" + val_str, sig_str, bad_val, link, debug_group = mk_table_value( + t[n], metric_defs, val_col_name, sig_col_name + ) + else: + val_str = None + sig_str = None + + if val_str is None: + continue + row_str += f"{band}: {val_str} σ: {sig_str}
\n" + num_bad += bad_val + if debug_group is not None: + debug_group_all = debug_group + row_strs.append((row_str, num_bad, link, debug_group_all)) + + return row_strs + + +def mk_sky_cols(t, metric_defs, n, bands, cols): + """Make sky column cell content + + Parameters + ---------- + t : `astropy.table.Table` + Table of metrics + metric_defs : `dict` + A dictionary of metrics and their thresholds + n : `n` + The row number + bands : `list` + A list of the bands the metrics are in + cols : `list` + A list of the columns to format + + Returns + ------- + row_strs : `tuple` of `str` + A tuple of the formatted strings for the sky columns + """ + row_strs = [] + for col in cols: + for stat, dev in [("mean", "stdev"), ("median", "sigmaMAD")]: + row_str = "" + num_bad = 0 + debug_group_all = None + for band in ["u", "g", "r", "i", "z", "y"]: + if band in bands: + val_col_name = col + "_" + band + "_" + stat + "Sky" + sig_col_name = col + "_" + band + "_" + dev + "Sky" + val_str, sig_str, bad_val, link, debug_group = mk_table_value( + t[n], metric_defs, val_col_name, sig_col_name + ) + else: + val_str = None + + if val_str is None: + continue + row_str += ( + "" + + band + + ": " + + val_str + + " σ: " + + sig_str + + "
\n" + ) + num_bad += bad_val + if debug_group is not None: + debug_group_all = debug_group + row_strs.append((row_str, num_bad, link, debug_group_all)) + return row_strs diff --git a/python/lsst/production/tools/tractTable.py b/python/lsst/production/tools/tractTable.py index 1243636..686132b 100644 --- a/python/lsst/production/tools/tractTable.py +++ b/python/lsst/production/tools/tractTable.py @@ -38,8 +38,6 @@ static_folder="../../../../static", ) -NO_BUTLER = True - @bp.route("/") def index(): @@ -114,7 +112,7 @@ def collection(collection_urlencoded): cell_vals = [] # Get the number of failed values and prep cell contents for cell_val, bad_val, link, debug_group in mk_shape_cols( - t, metric_defs, n, bands, col_dict + t, metric_defs, n, bands, col_dict["shape_cols"] ): cell_vals.append((cell_val, link, debug_group)) if bad_val is not None: @@ -122,7 +120,7 @@ def collection(collection_urlencoded): # Make the cell details for the stellar locus columns for cell_val, bad_val, link, debug_group in mk_stellar_locus_cols( - t, metric_defs, n, col_dict + t, metric_defs, n, col_dict["stellar_locus_cols"] ): cell_vals.append((cell_val, link, debug_group)) if bad_val is not None: @@ -130,7 +128,7 @@ def collection(collection_urlencoded): # Make the cell contents for the photometry columns for cell_val, bad_val, link, debug_group in mk_photom_cols( - t, metric_defs, n, bands, col_dict + t, metric_defs, n, bands, col_dict["photom_cols"] ): cell_vals.append((cell_val, link, debug_group)) if bad_val is not None: @@ -138,7 +136,7 @@ def collection(collection_urlencoded): # Make the cell contents for the sky columns for cell_val, bad_val, link, debug_group in mk_sky_cols( - t, metric_defs, n, bands, col_dict + t, metric_defs, n, bands, col_dict["sky_cols"] ): cell_vals.append((cell_val, link, debug_group)) if bad_val is not None: