From 698fcf6ebbad7f9f3eba0b0d742fb4d826f7708d Mon Sep 17 00:00:00 2001
From: Colin Slater <ctslater@uw.edu>
Date: Tue, 19 Nov 2024 10:42:41 -0800
Subject: [PATCH] Add repo and collection paths to tract table URLs.

---
 python/lsst/production/tools/__init__.py   |  1 -
 python/lsst/production/tools/tractTable.py | 92 +++++++++++++++++++---
 templates/metrics/index.html               | 14 +++-
 templates/metrics/tracts.html              |  2 +
 4 files changed, 95 insertions(+), 14 deletions(-)

diff --git a/python/lsst/production/tools/__init__.py b/python/lsst/production/tools/__init__.py
index 5b320e0..252391b 100644
--- a/python/lsst/production/tools/__init__.py
+++ b/python/lsst/production/tools/__init__.py
@@ -34,7 +34,6 @@ class UrlConverter(BaseConverter):
     regex = ".*?"
 
     def to_python(self, value):
-        print(value)
         return urllib.parse.unquote(value)
 
     def to_url(self, value):
diff --git a/python/lsst/production/tools/tractTable.py b/python/lsst/production/tools/tractTable.py
index 686132b..b306c0a 100644
--- a/python/lsst/production/tools/tractTable.py
+++ b/python/lsst/production/tools/tractTable.py
@@ -38,22 +38,71 @@
     static_folder="../../../../static",
 )
 
+REPO_NAMES = os.getenv("BUTLER_REPO_NAMES").split(",")
+
+
+def shorten_repo(repo_name):
+    """
+    Return the repo name without any '/repo/' prefix
+    """
+    return repo_name.split('/')[-1]
+
 
 @bp.route("/")
 def index():
 
-    collection_names = ["u/sr525/metricsPlotsPDR2_wholeSky"]
+    #collection_names = ["u/sr525/metricsPlotsPDR2_wholeSky"]
+    official_collection_entries = []
+    user_collection_entries = []
 
-    collection_entries = [
-        {"name": name, "url": urllib.parse.quote(name, safe="")}
-        for name in collection_names
-    ]
+    session = boto3.Session(profile_name="rubin-plot-navigator")
+    s3_client = session.client("s3", endpoint_url=os.getenv("S3_ENDPOINT_URL"))
 
-    return render_template("metrics/index.html", collection_entries=collection_entries)
+    try:
 
+        is_truncated = True
+        marker = ""
+
+        while is_truncated:
+            response = s3_client.list_objects(
+                Bucket="rubin-plot-navigator",
+                Marker=marker
+            )
+            is_truncated = response['IsTruncated']
+            print(f"Number of responses {len(response['Contents'])} truncated {is_truncated}")
+            for entry in response['Contents']:
+                marker = entry['Key']
+                for repo in REPO_NAMES:
+                    repo_encoded = urllib.parse.quote_plus(repo)
+
+                    if entry['Key'].startswith(repo_encoded):
+                        collection_enc = entry['Key'].replace(repo_encoded + "/collection_", "", 1).replace(".json.gz", "")
+                        collection = urllib.parse.unquote(collection_enc)
+
+                        if collection.startswith('u'):
+                            user_collection_entries.append({"name": collection,
+                                                            "updated": entry['LastModified'],
+                                                            "repo": shorten_repo(repo), "url":
+                                                            urllib.parse.quote(collection, safe="")})
+                        else:
+                            official_collection_entries.append({"name": collection,
+                                                                "updated": entry['LastModified'],
+                                                                "repo": shorten_repo(repo), "url":
+                                                                urllib.parse.quote(collection, safe="")})
 
-@bp.route("/collection/<collection_urlencoded>")
-def collection(collection_urlencoded):
+
+    except botocore.exceptions.ClientError as e:
+        print(e)
+
+    official_collection_entries.sort(key=lambda x: x['updated'], reverse=True)
+    user_collection_entries.sort(key=lambda x: x['updated'], reverse=True)
+
+    return render_template("metrics/index.html", collection_entries=official_collection_entries,
+                           user_collection_entries=user_collection_entries)
+
+
+@bp.route("/collection/<repo>/<url:collection>")
+def collection(repo, collection):
     """Make all the information needed to supply to the template
     for the tract summary table.
 
@@ -63,12 +112,31 @@ def collection(collection_urlencoded):
 
     The thresholds for the metrics are in the metricDefs
     yaml file.
+
+    The repo name provided must not contain slashes, so the supplied
+    repo name is matched against possible values in the REPO_NAMES
+    environment variable after removing any '/repo/' prefix from
+    those names.
     """
 
-    collection = urllib.parse.unquote(collection_urlencoded)
+    expanded_repo_name = None
+
+    if repo in REPO_NAMES:
+        expanded_repo_name = repo
+    else:
+        for test_name in REPO_NAMES:
+            if(repo == test_name.split('/')[-1]):
+                expanded_repo_name = f"/repo/{repo}"
+
+    if not expanded_repo_name:
+        return {"error": f"Invalid repo name {repo}, collection {collection}"}, 404
+
+    butler = Butler(expanded_repo_name)
+    if(expanded_repo_name == "/repo/main"):
+        dataId = {"skymap": "hsc_rings_v1", "instrument": "HSC"}
+    else:
+        dataId = {"skymap": "lsst_cells_v1", "instrument": "LSSTComCam"}
 
-    butler = Butler("/repo/main")
-    dataId = {"skymap": "hsc_rings_v1", "instrument": "HSC"}
     t = butler.get(
         "objectTableCore_metricsTable", collections=collection, dataId=dataId
     )
@@ -153,7 +221,7 @@ def collection(collection_urlencoded):
         "metrics/tracts.html",
         header_dict=header_dict,
         content_dict=content_dict,
-        collection_urlencoded=collection_urlencoded,
+        collection=collection,
     )
 
 
diff --git a/templates/metrics/index.html b/templates/metrics/index.html
index 04533f3..4842a27 100644
--- a/templates/metrics/index.html
+++ b/templates/metrics/index.html
@@ -8,9 +8,21 @@
 
 {% block content %}
 
+<H3>Official Collections</H3>
 <ul>
     {% for entry in collection_entries %}
-    <li><a href="{{ url_for("metrics.collection", collection_urlencoded=entry["url"]) }}">
+    <li><a href="{{ url_for("metrics.collection", repo=entry["repo"], collection=entry["url"]) }}">
+            {{entry["name"]}}
+        </a>
+    </li>
+    {% endfor %}
+</ul>
+
+
+<H3>User Collections</H3>
+<ul>
+    {% for entry in user_collection_entries %}
+    <li><a href="{{ url_for("metrics.collection", repo=entry["repo"], collection=entry["url"]) }}">
             {{entry["name"]}}
         </a>
     </li>
diff --git a/templates/metrics/tracts.html b/templates/metrics/tracts.html
index 85e9ce8..e928419 100644
--- a/templates/metrics/tracts.html
+++ b/templates/metrics/tracts.html
@@ -7,6 +7,8 @@
 {% block header %}Tract {{tract}} Summary{% endblock %}
 
 {% block content %}
+
+<a href={{url_for(".index")}}>&lt;-- Back to collections</a>
 <table>
 
     <thead>