From 802e75f75b0f7807aef2ed5e328e026547b8c83a Mon Sep 17 00:00:00 2001 From: Colin Slater Date: Tue, 20 Feb 2024 13:18:03 -0800 Subject: [PATCH] Add selections for detector and global plots. --- src/bootstrap_override.css | 2 +- src/dashboard_gen3.py | 165 +++++++++++++++++++++++++++---------- 2 files changed, 121 insertions(+), 46 deletions(-) diff --git a/src/bootstrap_override.css b/src/bootstrap_override.css index ae97ed3..6b88919 100644 --- a/src/bootstrap_override.css +++ b/src/bootstrap_override.css @@ -25,6 +25,6 @@ font-size: 1.7em; } -select { +select.bk-input { overflow-x: auto; } diff --git a/src/dashboard_gen3.py b/src/dashboard_gen3.py index 3930c12..7c915c5 100644 --- a/src/dashboard_gen3.py +++ b/src/dashboard_gen3.py @@ -6,6 +6,7 @@ import os import sys import logging +import time # Configure logging log = logging.getLogger(__name__) @@ -27,15 +28,17 @@ repo_config_string = os.environ.get("BUTLER_URI", None) default_repo = os.environ.get("BUTLER_DEFAULT_REPO", None) +default_collection = os.environ.get("BUTLER_DEFAULT_COLLECTION", None) if(len(dafButler.Butler.get_known_repos()) == 0 and repo_config_string is None): - print("No butler repo aliases configured, must set environment variable BUTLER_URI with butler config path.") + log.error("No butler repo aliases configured, must set environment variable BUTLER_URI with butler config path.") sys.exit(0) config = None butler = None registry = None +plot_types = None collections = [] plot_paths = {} @@ -44,20 +47,6 @@ pn.extension() - -def initialize(): - global config, butler, registry, collections - - global plot_paths - - config = None - butler = None - registry = None - collections = [] - - plot_paths = {} - - def get_tracts(refs, skymap): tracts = list( set( @@ -95,6 +84,8 @@ def get_visits(refs, instrument): tract_select = pn.widgets.MultiSelect(name="Tract", options=[], size=8) visit_select = pn.widgets.MultiSelect(name="Visit", options=[], size=8) plot_filter = pn.widgets.TextInput(name="Plot name filter", value="") +detector_select = pn.widgets.IntInput( + name="Detector Number", start=0, end=210, step=1, value=0) plot_select = pn.widgets.MultiSelect(name="Plots", options=[], size=12) @@ -110,33 +101,58 @@ def get_visits(refs, instrument): plots = pn.GridBox(["Plots will show up here when selected."], ncols=2) +def find_types(registry, storageClassName="Plot"): + func_start = time.perf_counter() + types = [] + for t in registry.queryDatasetTypes(): + try: + if t.storageClass.name == storageClassName: + types.append(t) + except KeyError: + # If we don't have the code for a given StorageClass in the environment, + # this test will throw a key error. + pass + + log.info("find_types duration: {:f}s".format(time.perf_counter() - func_start) ) + return types + def update_butler(event): global config global butler global registry + global plot_types try: butler = dafButler.Butler(config=repo_select.value) registry = butler.registry + func_start = time.perf_counter() if(all_collections.value): collections = list(registry.queryCollections()) collections.sort() else: chains = set(butler.registry.queryCollections(collectionTypes={dafButler.CollectionType.CHAINED})) - rest = {r for r in butler.registry.queryCollections(collectionTypes={dafButler.CollectionType.RUN, dafButler.CollectionType.TAGGED}) if not any(r.startswith(c) for c in chains)} + # rest = {r for r in butler.registry.queryCollections(collectionTypes={dafButler.CollectionType.RUN, dafButler.CollectionType.TAGGED}) if not any(r.startswith(c) for c in chains)} + rest = [] collections = list(chains) + list(rest) collections.sort() + log.info("Finished getting collections {:f}s".format(time.perf_counter() - func_start)) + + plot_types = find_types(registry) debug_text.value = "Successfully loaded butler." except Exception as e: debug_text.value = f"Failed to load Butler: {str(e)}" - log.error(f"{str(e)}") + log.error(f"Failed to load butler: {str(e)}") return collection_select.options = collections - collection_select.value = collections[0] + + if default_collection is not None and default_collection in collections: + collection_select.value = default_collection + else: + collection_select.value = collections[0] skymap_select.options = [x.name for x in registry.queryDimensionRecords("skymap")] instrument_select.options = [x.name for x in registry.queryDimensionRecords("instrument")] @@ -146,25 +162,22 @@ def update_butler(event): update_butler(None) -def find_types(registry, storageClassName="Plot"): - types = [] - for t in registry.queryDatasetTypes(): - try: - if t.storageClass.name == storageClassName: - types.append(t) - except KeyError: - # If we don't have the code for a given StorageClass in the environment, - # this test will throw a key error. - pass - - return types - -def find_refs(collection, types): +def find_refs(collection, types, required_dimension=None): + func_start = time.perf_counter() if(len(collection) == 0): return [] else: - return list(registry.queryDatasets(types, collections=collection, findFirst=True)) + collection_summary = registry.getCollectionSummary(collection) + if required_dimension is not None: + available_plot_types = [x for x in list(collection_summary.dataset_types) if x.storageClass_name == "Plot" and required_dimension in x.dimensions] + else: + available_plot_types = [x for x in list(collection_summary.dataset_types) if x.storageClass_name == "Plot"] + + log.debug("Avaialble plots: {:d}".format(len(available_plot_types))) + datasets = registry.queryDatasets(available_plot_types, collections=collection, findFirst=True) + log.info("END find_refs {:f}s".format(time.perf_counter() - func_start)) + return list(datasets) def update_tract_select(event): @@ -173,8 +186,7 @@ def update_tract_select(event): if registry is None or collection_select.value is None: return - types = find_types(registry) - refs = find_refs(collection_select.value, types) + refs = find_refs(collection_select.value, [t for t in plot_types if 'tract' in t.dimensions], required_dimension="tract") tract_select.options = get_tracts(refs, skymap_select.value) tract_select.value = [] @@ -186,10 +198,9 @@ def update_visit_select(event): if registry is None or collection_select.value is None: return - types = find_types(registry) if registry is not None: - refs = find_refs(collection_select.value, types) + refs = find_refs(collection_select.value, [t for t in plot_types if 'visit' in t.dimensions], required_dimension="visit") visit_select.options = get_visits(refs, instrument_select.value) visit_select.value = [] @@ -206,18 +217,18 @@ def update_visit_select(event): def update_plot_names(event): - global plot_paths, plot_datasettypes + global plot_paths, plot_datasettypes, plot_select plot_names = [] plot_refs = [] - types = find_types(registry) if visit_tract_tabs.active == 0: + func_start = time.perf_counter() for tract in tract_select.value: refs = [ ref for ref in registry.queryDatasets( - types, + [t for t in plot_types if 'tract' in t.dimensions], collections=collection_select.value, findFirst=True, dataId={"skymap": skymap_select.value, "tract": tract}, @@ -231,12 +242,16 @@ def update_plot_names(event): plot_refs.extend(refs) plot_names.extend(names) - else: + log.info("END update_plot_names tracts {:f}s".format(time.perf_counter() - func_start)) + + elif visit_tract_tabs.active == 1: + + func_start = time.perf_counter() for visit in visit_select.value: refs = list( registry.queryDatasets( - types, + [t for t in plot_types if 'visit' in t.dimensions], collections=collection_select.value, findFirst=True, dataId={"instrument": instrument_select.value, "visit": visit}, @@ -254,9 +269,64 @@ def update_plot_names(event): plot_refs.extend(refs) plot_names.extend(names) + log.info("END update_plot_names visits {:f}s".format(time.perf_counter() - func_start)) + + elif visit_tract_tabs.active == 2: + # Detector + func_start = time.perf_counter() + refs = list( + registry.queryDatasets( + [t for t in plot_types if 'detector' in t.dimensions and 'visit' not in t.dimensions and 'tract' not in t.dimensions], + collections=collection_select.value, + findFirst=True, + dataId={"instrument": instrument_select.value, "detector": detector_select.value}, + ) + ) + + if len(plot_filter.value) > 0: + refs = filter( + lambda ref: re.search(plot_filter.value, ref.datasetType.name), refs + ) + + names = [ + (f"{p.datasetType.name}", p.datasetType.name) for p in refs + ] + + plot_refs.extend(refs) + plot_names.extend(names) + log.info("END update_plot_names detector {:f}s".format(time.perf_counter() - func_start)) + + elif visit_tract_tabs.active == 3: + + # Instrument/Global plots + # If we had collections with plots from multiple instruments we would + # need to have a selector for that, but in practice that very uncommon. + func_start = time.perf_counter() + + inst_ds_types = [t for t in plot_types if 'detector' not in t.dimensions and 'visit' not in t.dimensions and 'tract' not in t.dimensions] + refs = list( + registry.queryDatasets( + inst_ds_types, + collections=collection_select.value, + findFirst=True + ) + ) + + if len(plot_filter.value) > 0: + refs = filter( + lambda ref: re.search(plot_filter.value, ref.datasetType.name), refs + ) + + names = [ + (f"{p.datasetType.name}", p.datasetType.name) for p in refs + ] + + plot_refs.extend(refs) + plot_names.extend(names) + log.info("END update_plot_names instrument {:f}s".format(time.perf_counter() - func_start)) plot_paths = { - name[0]: butler.getURI(ref, collections=collection_select.value) + name[0]: butler.getURI(ref) for name, ref in zip(plot_names, plot_refs) } @@ -270,6 +340,8 @@ def update_plot_names(event): plot_filter.param.watch(update_plot_names, "value") tract_select.param.watch(update_plot_names, "value") visit_select.param.watch(update_plot_names, "value") +detector_select.param.watch(update_plot_names, "value") +instrument_select.param.watch(update_plot_names, "value") def get_png(name): @@ -404,9 +476,12 @@ def increment_tract(increment=1): prev_tract_button.on_click(prev_tract) visit_tract_tabs = pn.Tabs( - ("Tracts", pn.Column(skymap_select, tract_select, next_prev_tract_row)), - ("Visits", pn.Column(instrument_select, visit_select, next_prev_visit_row)), + ("Tract", pn.Column(skymap_select, tract_select, next_prev_tract_row)), + ("Visit", pn.Column(instrument_select, visit_select, next_prev_visit_row)), + ("Detector", pn.Column(instrument_select, detector_select)), + ("Global", pn.Column()), ) +visit_tract_tabs.dynamic = True visit_tract_tabs.param.watch(update_plot_names, "active") bootstrap.sidebar.append(repo_select)