From 962fe367a67ac16ca3a048cec54cc75091a8852b Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Tue, 3 Oct 2023 18:34:26 -0700 Subject: [PATCH] Don't we just love docstrings --- src/lsst/cmservice/cli/commands.py | 155 ++++++++++++++++++++++++-- src/lsst/cmservice/cli/options.py | 17 +-- src/lsst/cmservice/main.py | 62 ++++++++++- src/lsst/cmservice/routers/loaders.py | 2 +- src/lsst/cmservice/routers/queries.py | 14 +-- 5 files changed, 222 insertions(+), 28 deletions(-) diff --git a/src/lsst/cmservice/cli/commands.py b/src/lsst/cmservice/cli/commands.py index c09c54f2d..d3c6670f6 100644 --- a/src/lsst/cmservice/cli/commands.py +++ b/src/lsst/cmservice/cli/commands.py @@ -95,6 +95,7 @@ def productions( client: CMClient, output: options.OutputEnum | None, ) -> None: + """List the existing productions""" result = client.get_productions() _output_pydantic_list(result, output) @@ -110,6 +111,12 @@ def campaigns( parent_name: str | None, output: options.OutputEnum | None, ) -> None: + """List the existing campaigns + + Specifying either parent-name or parent-id + will limit the results to only those + campaigns in the associated production + """ result = client.get_campaigns(parent_id, parent_name) _output_pydantic_list(result, output) @@ -125,6 +132,12 @@ def steps( parent_name: str | None, output: options.OutputEnum | None, ) -> None: + """List the existing steps + + Specifying either parent-name or parent-id + will limit the results to only those + steps in the associated campaign + """ result = client.get_steps(parent_id, parent_name) _output_pydantic_list(result, output) @@ -140,6 +153,12 @@ def groups( parent_name: str | None, output: options.OutputEnum | None, ) -> None: + """List the existing groups + + Specifying either parent-name or parent-id + will limit the results to only those + groups in the associated step + """ result = client.get_groups(parent_id, parent_name) _output_pydantic_list(result, output) @@ -153,6 +172,7 @@ def element( fullname: str, output: options.OutputEnum | None, ) -> None: + """Get a particular element""" result = client.get_element(fullname) _output_pydantic_object(result, output) @@ -166,6 +186,7 @@ def script( fullname: str, output: options.OutputEnum | None, ) -> None: + """Get a particular script""" result = client.get_script(fullname) _output_pydantic_object(result, output) @@ -179,6 +200,7 @@ def job( fullname: str, output: options.OutputEnum | None, ) -> None: + """Get a particular job""" result = client.get_job(fullname) _output_pydantic_object(result, output) @@ -188,12 +210,17 @@ def job( @options.fullname() @options.table_type() @options.output() -def spec_block( +def obj_spec_block( client: CMClient, fullname: str, table_type: TableTypeEnum, output: options.OutputEnum | None, ) -> None: + """Get the SpecBlock corresponding to a partiuclar object + + By default this selects elements, but + table-type can be set to 'script' or 'job' + """ result = client.get_spec_block(fullname, table_type) _output_pydantic_object(result, output) @@ -203,12 +230,17 @@ def spec_block( @options.fullname() @options.table_type() @options.output() -def specification( +def obj_specification( client: CMClient, fullname: str, table_type: TableTypeEnum, output: options.OutputEnum | None, ) -> None: + """Get the Specification corresponding to a partiuclar object + + By default this selects elements, but + table-type can be set to 'script' or 'job' + """ result = client.get_specification(fullname, table_type) _output_pydantic_object(result, output) @@ -218,12 +250,17 @@ def specification( @options.fullname() @options.table_type() @options.output() -def collections( +def obj_collections( client: CMClient, fullname: str, table_type: TableTypeEnum, output: options.OutputEnum | None, ) -> None: + """Get the collection parameters for a partiuclar object + + By default this selects elements, but + table-type can be set to 'script' or 'job' + """ result = client.get_collections(fullname, table_type) _output_dict(result, output) @@ -233,12 +270,17 @@ def collections( @options.fullname() @options.table_type() @options.output() -def child_config( +def obj_child_config( client: CMClient, fullname: str, table_type: TableTypeEnum, output: options.OutputEnum | None, ) -> None: + """Get the child_config parameters for a partiuclar object + + By default this selects elements, but + table-type can be set to 'script' or 'job' + """ result = client.get_child_config(fullname, table_type) _output_dict(result, output) @@ -248,12 +290,17 @@ def child_config( @options.fullname() @options.table_type() @options.output() -def data_dict( +def obj_data_dict( client: CMClient, fullname: str, table_type: TableTypeEnum, output: options.OutputEnum | None, ) -> None: + """Get the data_dict parameters for a partiuclar object + + By default this selects elements, but + table-type can be set to 'script' or 'job' + """ result = client.get_data_dict(fullname, table_type) _output_dict(result, output) @@ -263,12 +310,17 @@ def data_dict( @options.fullname() @options.table_type() @options.output() -def prerequisites( +def check_prerequisites( client: CMClient, fullname: str, table_type: TableTypeEnum, output: options.OutputEnum | None, ) -> None: + """Check if prerequisites are done for a partiuclar object + + By default this selects elements, but + table-type can be set to 'script' or 'job' + """ value = client.get_prerequisites(fullname, table_type) _output_dict({"value": value}, output) @@ -278,12 +330,13 @@ def prerequisites( @options.fullname() @options.script_name() @options.output() -def scripts( +def element_scripts( client: CMClient, fullname: str, script_name: str, output: options.OutputEnum | None, ) -> None: + """Get the Scripts used by a partiuclar element""" result = client.get_scripts(fullname, script_name) _output_pydantic_list(result, output) @@ -292,11 +345,12 @@ def scripts( @options.cmclient() @options.fullname() @options.output() -def jobs( +def element_jobs( client: CMClient, fullname: str, output: options.OutputEnum | None, ) -> None: + """Get the Jobs used by a partiuclar element""" result = client.get_jobs(fullname) _output_pydantic_list(result, output) @@ -310,6 +364,7 @@ def job_task_sets( fullname: str, output: options.OutputEnum | None, ) -> None: + """Get the TaskSets for a particular Job""" result = client.get_job_task_sets(fullname) _output_pydantic_list(result, output) @@ -323,6 +378,7 @@ def job_product_sets( fullname: str, output: options.OutputEnum | None, ) -> None: + """Get the ProductSets for a particular Job""" result = client.get_job_product_sets(fullname) _output_pydantic_list(result, output) @@ -336,6 +392,7 @@ def job_errors( fullname: str, output: options.OutputEnum | None, ) -> None: + """Get the ErrorInstances for a particular Job""" result = client.get_job_errors(fullname) _output_pydantic_list(result, output) @@ -358,6 +415,11 @@ def status_( table_type: TableTypeEnum, status: StatusEnum, ) -> None: + """Update the status of a particular Object + + By default this selects elements, but + table-type can be set to 'script' or 'job' + """ status = client.update_status( fullname=fullname, table_type=table_type, @@ -372,13 +434,58 @@ def status_( @options.table_type() @options.output() @options.update_dict() -def update_collections( +def collections( client: CMClient, output: options.OutputEnum | None, **kwargs: Any, ) -> None: - status = client.update_collections(**kwargs) - _output_dict({"status": status}, output) + """Update collections configuration of particular Object + + By default this selects elements, but + table-type can be set to 'script' or 'job' + """ + result = client.update_collections(**kwargs) + _output_dict(result, output) + + +@update.command() +@options.cmclient() +@options.fullname() +@options.table_type() +@options.output() +@options.update_dict() +def child_config( + client: CMClient, + output: options.OutputEnum | None, + **kwargs: Any, +) -> None: + """Update child_config configuration of particular Object + + By default this selects elements, but + table-type can be set to 'script' or 'job' + """ + result = client.update_child_config(**kwargs) + _output_dict(result, output) + + +@update.command() +@options.cmclient() +@options.fullname() +@options.table_type() +@options.output() +@options.update_dict() +def data_dict( + client: CMClient, + output: options.OutputEnum | None, + **kwargs: Any, +) -> None: + """Update data_dict configuration of particular Object + + By default this selects elements, but + table-type can be set to 'script' or 'job' + """ + result = client.update_data_dict(**kwargs) + _output_dict(result, output) @main.group() @@ -397,6 +504,7 @@ def groups_( child_configs: dict, output: options.OutputEnum | None, ) -> None: + """Add Groups to a Step""" result = client.add_groups( fullname=fullname, child_configs=child_configs, @@ -415,6 +523,7 @@ def steps_( child_configs: dict, output: options.OutputEnum | None, ) -> None: + """Add Steps to a Campaign""" result = client.add_steps( fullname=fullname, child_configs=child_configs, @@ -433,6 +542,7 @@ def campaign( child_configs: dict, output: options.OutputEnum | None, ) -> None: + """Add a Campaign""" result = client.add_campaign( fullname=fullname, **child_configs, @@ -455,6 +565,7 @@ def load_specification( output: options.OutputEnum | None, **kwargs: Any, ) -> None: + """Load a Specification from a yaml file""" result = client.load_specification(**kwargs) _output_pydantic_object(result, output) @@ -466,6 +577,7 @@ def load_campaign( client: CMClient, output: options.OutputEnum | None, ) -> None: + """Load a Specification from a yaml file and make a Campaign""" result = client.load_campaign() _output_pydantic_object(result, output) @@ -479,6 +591,7 @@ def error_types( output: options.OutputEnum | None, **kwargs: Any, ) -> None: + """Load a ErrorTypes from a yaml file""" result = client.load_error_types(**kwargs) _output_pydantic_list(result, output) @@ -493,6 +606,7 @@ def error_instances( output: options.OutputEnum | None, **kwargs: Any, ) -> None: + """Load a ErrorInstances from a yaml file""" result = client.load_error_instances(**kwargs) _output_pydantic_list(result, output) @@ -513,6 +627,11 @@ def process( table_type: options.PartialOption, output: options.OutputEnum | None, ) -> None: + """Process an object + + By default this selects elements, but + table-type can be set to 'script' or 'job' + """ status = client.process( fullname=fullname, table_type=table_type, @@ -531,6 +650,11 @@ def retry_script( script_name: options.PartialOption, output: options.OutputEnum | None, ) -> None: + """Create a new version of a script to retry it + + This will mark the current version as superseded. + This can only be run on failed/rejected scripts. + """ result = client.retry_script( fullname=fullname, script_name=script_name, @@ -549,6 +673,10 @@ def rescue_script( script_name: options.PartialOption, output: options.OutputEnum | None, ) -> None: + """Create a new version of a script to rescue it + + This can only be run on rescuable scripts. + """ result = client.rescue_script( fullname=fullname, script_name=script_name, @@ -567,6 +695,11 @@ def mark_script_rescued( script_name: options.PartialOption, output: options.OutputEnum | None, ) -> None: + """Mark a script as rescued + + This is usually done automatically when + the script is accepted + """ result = client.mark_script_rescued( fullname=fullname, script_name=script_name, diff --git a/src/lsst/cmservice/cli/options.py b/src/lsst/cmservice/cli/options.py index fb3e948a2..9ffb49563 100644 --- a/src/lsst/cmservice/cli/options.py +++ b/src/lsst/cmservice/cli/options.py @@ -53,7 +53,11 @@ class OutputEnum(Enum): json = auto() -child_configs = PartialOption("--child_configs", type=dict, help="Configuration for children") +child_configs = PartialOption( + "--child_configs", + type=dict, + help="Configuration to use for creating new Elements.", +) output = PartialOption( "--output", @@ -66,22 +70,21 @@ class OutputEnum(Enum): fullname = PartialOption( "--fullname", type=str, - help="Full name of object in DB", + help="Full name of object in DB.", ) parent_name = PartialOption( "--parent_name", type=str, default=None, - help="Full name of parent object in DB", + help="Full name of parent object in DB.", ) - parent_id = PartialOption( "--parent_id", type=int, default=None, - help="ID of parent object in DB", + help="ID of parent object in DB.", ) table_type = PartialOption( @@ -94,13 +97,13 @@ class OutputEnum(Enum): status = PartialOption( "--status", type=EnumChoice(StatusEnum), - help="Status of Element", + help="Status to set for Element", ) script_name = PartialOption( "--script_name", type=str, - help="Name of the script", + help="Used to distinguish scripts within an Element", ) spec_name = PartialOption( diff --git a/src/lsst/cmservice/main.py b/src/lsst/cmservice/main.py index 1da30707a..cf7eb7be0 100644 --- a/src/lsst/cmservice/main.py +++ b/src/lsst/cmservice/main.py @@ -1,5 +1,4 @@ from importlib.metadata import metadata, version -from typing import Any, Dict, List from fastapi import FastAPI from safir.dependencies.arq import arq_dependency @@ -32,7 +31,66 @@ configure_uvicorn_logging(config.log_level) -tags_metadata: List[Dict[str, Any]] | None = [] +tags_metadata = [ + { + "name": "Loaders", + "description": "Operations that load Objects in to the DB.", + }, + { + "name": "Query", + "description": "Operations query exsiting Objects in to the DB.", + }, + { + "name": "Actions", + "description": "Operations perform actions on existing Objects in to the DB." + "In many cases this will result in the creating of new objects in the DB.", + }, + { + "name": "Adders", + "description": "Operations explicitly add new Objects in to the DB." + "These are typically used when we need to do something unexpected", + }, + { + "name": "Updates", + "description": "Operations update Objects in to the DB." + "These are typically used when we need to do something unexpected", + }, + { + "name": "Productions", + "description": "Operations with `production`s. A `production` is a container for `campaign`s. " + "`production`s must be uniquely named.", + }, + { + "name": "Campaigns", + "description": "Operations with `campaign`s. A `campaign` consists of several processing `step`s " + "which are run sequentially. A `campaign` also holds configuration such as a URL for a butler repo " + "and a production area. `campaign`s must be uniquely named withing a given `production`.", + }, + { + "name": "Steps", + "description": "Operations with `step`s. A `step` consists of several processing `group`s which " + "may be run in parallel. `step`s must be uniquely named within a give `campaign`.", + }, + { + "name": "Groups", + "description": "Operations with `groups`. A `group` can be processed in a single `workflow`, " + "but we also need to account for possible failures. `group`s must be uniquely named within a " + "given `step`.", + }, + { + "name": "Scripts", + "description": "Operations with `scripts`. A `script` does a single operation, either something" + "that is done asynchronously, such as making new collections in the Butler, or creating" + "new objects in the DB, such as new `steps` and `groups`.", + }, + { + "name": "Jobs", + "description": "Operations with `jobs`. A `job` runs a single `workflow`: keeps a count" + "of the results data products and keeps track of associated errors.", + }, + {"name": "ErrorTypes", "description": "Operations with `error_types`."}, +] + app = FastAPI( title="cm-service", diff --git a/src/lsst/cmservice/routers/loaders.py b/src/lsst/cmservice/routers/loaders.py index 6b17e0127..78a0ad93b 100644 --- a/src/lsst/cmservice/routers/loaders.py +++ b/src/lsst/cmservice/routers/loaders.py @@ -10,7 +10,7 @@ router = APIRouter( prefix="/load", - tags=["Load"], + tags=["Loaders"], ) diff --git a/src/lsst/cmservice/routers/queries.py b/src/lsst/cmservice/routers/queries.py index cff67f6df..37a722d1f 100644 --- a/src/lsst/cmservice/routers/queries.py +++ b/src/lsst/cmservice/routers/queries.py @@ -18,7 +18,7 @@ @router.get( "/element", response_model=models.Element, - summary="Get an element", + summary="Get an element, i.e., a Campaign, Step or Group", ) async def get_element( fullname: str, @@ -70,7 +70,7 @@ async def get_job( @router.get( "/spec_block", response_model=models.SpecBlock, - summary="Get a SpecBlock associated to a processable", + summary="Get a SpecBlock associated to an Object", ) async def get_spec_block( fullname: str, @@ -88,7 +88,7 @@ async def get_spec_block( @router.get( "/specification", response_model=models.Specification, - summary="Get a Specficiation associated to a processable", + summary="Get a Specficiation associated to an object", ) async def get_specification( fullname: str, @@ -106,7 +106,7 @@ async def get_specification( @router.get( "/collections", response_model=dict, - summary="Get collections field associated to a processable", + summary="Get collections field associated to an object", ) async def get_collections( fullname: str, @@ -124,7 +124,7 @@ async def get_collections( @router.get( "/child_config", response_model=dict, - summary="Get child_config field associated to a processable", + summary="Get child_config field associated to an object", ) async def get_child_config( fullname: str, @@ -143,7 +143,7 @@ async def get_child_config( @router.get( "/data_dict", response_model=dict, - summary="Get data_dict field associated to a processable", + summary="Get data_dict field associated to an object", ) async def get_data_dict( fullname: str, @@ -161,7 +161,7 @@ async def get_data_dict( @router.get( "/prerequisites", response_model=bool, - summary="Check prerequisites associated to a processable", + summary="Check prerequisites associated to an object", ) async def get_prerequisites( fullname: str,