Skip to content

Commit

Permalink
Move profile to enqueue.
Browse files Browse the repository at this point in the history
  • Loading branch information
ktlim committed Jun 19, 2024
1 parent 1b850f4 commit e942a40
Show file tree
Hide file tree
Showing 2 changed files with 5 additions and 8 deletions.
5 changes: 4 additions & 1 deletion src/enqueue.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@
r = setup_redis()
notification_secret = os.environ["NOTIFICATION_SECRET"]
regexp = re.compile(os.environ.get("DATASET_REGEXP", r"fits$"))
profile = os.environ.get("PROFILE", "")
if profile != "":
profile += "@"


def enqueue_objects(objects):
Expand Down Expand Up @@ -119,7 +122,7 @@ def notify():
logger.info("Unrecognized secret %s", r["opaqueData"])
continue
object_names.append(
r["s3"]["bucket"]["name"] + "/" + urllib.parse.unquote_plus(r["s3"]["object"]["key"])
profile + r["s3"]["bucket"]["name"] + "/" + urllib.parse.unquote_plus(r["s3"]["object"]["key"])
)
info_list = enqueue_objects(object_names)
update_stats(info_list)
Expand Down
8 changes: 1 addition & 7 deletions src/ingest.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,12 +46,6 @@
logger = setup_logging(__name__)
r = setup_redis()
bucket = os.environ["BUCKET"]
if (pos := bucket.find("@")) >= 0:
pos += 1
profile = bucket[:pos]
bucket = bucket[pos:]
else:
profile = ""
if bucket.startswith("rubin:"):
os.environ["LSST_DISABLE_BUCKET_VALIDATION"] = "1"
redis_queue = f"QUEUE:{bucket}"
Expand Down Expand Up @@ -205,7 +199,7 @@ def main():
# Process any entries on the worker queue.
if r.llen(worker_queue) > 0:
blobs = r.lrange(worker_queue, 0, -1)
resources = [ResourcePath(f"s3://{profile}{b.decode()}") for b in blobs]
resources = [ResourcePath(f"s3://{b.decode()}") for b in blobs]

# Ingest if we have resources
if resources:
Expand Down

0 comments on commit e942a40

Please sign in to comment.