diff --git a/Dockerfile b/Dockerfile index 1c1045e..f2ef304 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM continuumio/miniconda3:24.7.1-0 RUN apt-get --allow-releaseinfo-change update && \ apt-get install -y \ - build-essential libpq-dev netbase \ + build-essential libpq-dev netbase curl \ && rm -rf /var/lib/apt/lists/* RUN conda install -y -q pip wheel diff --git a/python/lsst/production/tools/__init__.py b/python/lsst/production/tools/__init__.py index 94d523b..3907f2d 100644 --- a/python/lsst/production/tools/__init__.py +++ b/python/lsst/production/tools/__init__.py @@ -21,7 +21,7 @@ from flask import Flask, render_template -from . import tractTable, logs, bokeh +from . import tractTable, logs, bokeh, cache def create_app(): @@ -32,6 +32,7 @@ def create_app(): app.register_blueprint(logs.bp) app.register_blueprint(tractTable.bp) app.register_blueprint(bokeh.bp) + app.register_blueprint(cache.bp) @app.route("/") def index(): @@ -40,4 +41,4 @@ def index(): return app -__all__ = [tractTable, logs, bokeh, create_app] +__all__ = [tractTable, logs, bokeh, create_app, cache] diff --git a/python/lsst/production/tools/cache.py b/python/lsst/production/tools/cache.py new file mode 100644 index 0000000..496a067 --- /dev/null +++ b/python/lsst/production/tools/cache.py @@ -0,0 +1,72 @@ +# This file is part of production-tools. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import sys + +import lsst.daf.butler as dafButler +from flask import Blueprint, Flask, jsonify, request, abort +import arq +import time + +bp = Blueprint("cache", __name__, url_prefix="/plot-navigator/cache", static_folder="../../../../static") + + +# PUT /cache/ {repo: "", collection: ""}, return {jobId: ""} +# GET /cache/job/, return {status: ""} + +@bp.route("/", methods=["PUT"]) +async def index(): + redis_settings = arq.connections.RedisSettings(host=os.getenv("REDIS_HOST"), + port=os.getenv("REDIS_PORT")) + + redis = await arq.create_pool(redis_settings) + print(f"cache.index() received request: {request}") + if request.method == 'PUT': + + data = request.get_json() + arq_job = await redis.enqueue_job("cache_plots", data['repo'], data['collection']) + print(arq_job) + return jsonify({"jobId": arq_job.job_id}) + + else: + abort(400, description=f"Invalid HTTP Method {request.method}") + +@bp.route("/job/") +async def job(job_id): + + redis_settings = arq.connections.RedisSettings(host=os.getenv("REDIS_HOST"), + port=os.getenv("REDIS_PORT")) + + redis = await arq.create_pool(redis_settings) + arq_job = arq.jobs.Job(job_id=job_id, redis=redis) + + return jsonify({"status": str(await arq_job.status())}) + +async def cache_plots(ctx, repo, collection): + print(f"cache_plots start {repo} {collection}") + time.sleep(60) + print(f"cache_plots finished {repo} {collection}") + +class Worker: + functions = [cache_plots] + redis_settings = redis_settings + diff --git a/requirements.txt b/requirements.txt index ba53ade..5c6ffda 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -flask>2 +flask[async] lsst-daf-butler lsst-efd-client lsst-pex-config @@ -9,4 +9,5 @@ botocore eventlet bokeh numpy>=1.26.0 +arq