Skip to content

Commit

Permalink
In 1104 deposit command (#68)
Browse files Browse the repository at this point in the history
* Add deposit CLI command

Why these changes are being introduced:
* A CLI command is needed for running the workflows's deposit functionality

How this addresses that need:
* Rename BaseWorkflow > Workflow across repo
* Add s3-bucket, output-queue, and email-recipients CLI params
* Add deposit CLI command
* Add DemoWorkflow for testing purposes
* Refactor Workflow class to use init defaults instead of some class attributes
* Refactor Workflow.load method to optionally override class attributes with CLI params
* Add _get_subclasses method for collecting all subclasses
* Update _build_bitstream_dict method to ignore non-file values and the metadata.csv file
* Add deposit and reconcile_discrepancies_logged CLI tests
* Add metadata mapping for DemoWorkflow

Side effects of this change:
* None

Relevant ticket(s):
* https://mitlibraries.atlassian.net/browse/IN-1104

* Update Pipfile.lock

* Updates based on discussion in PR #68

* Update click option docstrings
* Make email-recipients required and update type hint to indicate it is a comma-delimited string
* Refactor Workflow.run method to include try/except block and return a dict summarizing the results of the submission
* Add new unit tests for Workflow.run method to account for code changes

* Further updates based on discussion in PR #68

* Refactor Workflow __init__  methods to better utilize defaults and named args
* Refactor Workflow.run method's return object to provide a summary of the submission results
* Remove Workflow.load method as it is now unnecessary and update CLI to call Workflow.get_workflow method
* Update type hinting and processing for email-recipients param
  • Loading branch information
ehanson8 authored Jan 14, 2025
1 parent 791c615 commit b015028
Show file tree
Hide file tree
Showing 10 changed files with 399 additions and 132 deletions.
128 changes: 64 additions & 64 deletions Pipfile.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

48 changes: 44 additions & 4 deletions dsc/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import click

from dsc.config import Config
from dsc.workflows.base import BaseWorkflow
from dsc.workflows.base import Workflow

logger = logging.getLogger(__name__)
CONFIG = Config()
Expand All @@ -29,9 +29,28 @@
@click.option(
"-b",
"--batch-id",
help="The S3 prefix for the batch of DSpace submissions",
help="The S3 prefix for the batch of DSpace submission files",
required=True,
)
@click.option(
"-e",
"--email-recipients",
help="The recipients of the submission results email as a comma-delimited string, "
"this will override the workflow class's default value for this attribute",
required=True,
)
@click.option(
"-s",
"--s3-bucket",
help="The S3 bucket containing the DSpace submission files, "
"this will override the workflow class's default value for this attribute",
)
@click.option(
"-o",
"--output-queue",
help="The SQS output queue for the DSS result messages, "
"this will override the workflow class's default value for this attribute",
)
@click.option(
"-v", "--verbose", is_flag=True, help="Pass to log at debug level instead of info"
)
Expand All @@ -40,12 +59,22 @@ def main(
workflow_name: str,
collection_handle: str,
batch_id: str,
email_recipients: str,
s3_bucket: str | None,
output_queue: str | None,
verbose: bool, # noqa: FBT001
) -> None:
ctx.ensure_object(dict)
ctx.obj["start_time"] = perf_counter()

ctx.obj["workflow"] = BaseWorkflow.load(workflow_name, collection_handle, batch_id)
workflow_class = Workflow.get_workflow(workflow_name)
workflow = workflow_class(
collection_handle=collection_handle,
batch_id=batch_id,
email_recipients=tuple(email_recipients.split(",")),
s3_bucket=s3_bucket,
output_queue=output_queue,
)
ctx.obj["workflow"] = workflow

stream = StringIO()
root_logger = logging.getLogger()
Expand Down Expand Up @@ -86,3 +115,14 @@ def reconcile(ctx: click.Context) -> None:
logger.error(
f"No item identifiers found for these bitstreams: {no_item_identifiers}"
)
else:
logger.info("All item identifiers and bitstreams successfully matched")


@main.command()
@click.pass_context
def deposit(ctx: click.Context) -> None:
"""Send a batch of item submissions to the DSpace Submission Service (DSS)."""
workflow = ctx.obj["workflow"]
logger.debug(f"Beginning submission of batch ID: {workflow.batch_id}")
workflow.run()
8 changes: 4 additions & 4 deletions dsc/workflows/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
All primary functions used by CLI are importable from here.
"""

from dsc.workflows.base import BaseWorkflow
from dsc.workflows.base import Workflow
from dsc.workflows.base.simple_csv import SimpleCSV
from dsc.workflows.demo import DemoWorkflow

__all__ = [
"BaseWorkflow",
]
__all__ = ["DemoWorkflow", "SimpleCSV", "Workflow"]
Loading

0 comments on commit b015028

Please sign in to comment.