Skip to content

Commit

Permalink
Add ability to ingest with profile.
Browse files Browse the repository at this point in the history
  • Loading branch information
ktlim committed Jun 14, 2024
1 parent 41deacf commit a61b0bc
Showing 1 changed file with 8 additions and 1 deletion.
9 changes: 8 additions & 1 deletion src/ingest.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,13 @@
logger = setup_logging(__name__)
r = setup_redis()
bucket = os.environ["BUCKET"]
profile = ""
if (pos := bucket.find("@")) >= 0:
pos += 1
profile = bucket[:pos]
bucket = bucket[pos:]
else:
profile = ""
if bucket.startswith("rubin:"):
os.environ["LSST_DISABLE_BUCKET_VALIDATION"] = "1"
redis_queue = f"QUEUE:{bucket}"
Expand Down Expand Up @@ -199,7 +206,7 @@ def main():
# Process any entries on the worker queue.
if r.llen(worker_queue) > 0:
blobs = r.lrange(worker_queue, 0, -1)
resources = [ResourcePath(f"s3://{b.decode()}") for b in blobs]
resources = [ResourcePath(f"s3://{profile}{b.decode()}") for b in blobs]

# Ingest if we have resources
if resources:
Expand Down

0 comments on commit a61b0bc

Please sign in to comment.