a whole campaign, WOW

examples/example_config.yaml

@@ -51,7 +51,7 @@
       handler: lsst.cmservice.handlers.elements.RunStepsScriptHandler
 - SpecBlock:
       name: job
-      handler: lsst.cmservice.handlers.jobs.PandaJobHandler
+      handler: lsst.cmservice.handlers.db.job_handler.JobHandler
       collections:
           job_run: "{root}/{campaign}/{step}/{group}/{job}"
       scripts:
@@ -78,6 +78,7 @@
           rescue: false
 - SpecBlock:
       name: group
+      handler: lsst.cmservice.handlers.db.element_handler.ElementHandler
       collections:
           group_output: "{root}/{campaign}/{step}/{group}"
           group_validation: "{root}/{campaign}/{step}/{group}/validate"
@@ -245,6 +246,7 @@
           split_method: no_split
 - SpecBlock:
       name: campaign
+      handler: lsst.cmservice.handlers.db.element_handler.ElementHandler
       collections:
           root: 'cm/p1'
           campaign_source: /prod/raw/all

requirements/dev.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
 #    pip-compile --generate-hashes --output-file=requirements/dev.txt requirements/dev.in
@@ -93,10 +93,83 @@ distlib==0.3.7 \
     --hash=sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057 \
     --hash=sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8
     # via virtualenv
+exceptiongroup==1.1.3 \
+    --hash=sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9 \
+    --hash=sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3
+    # via
+    #   -c requirements/main.txt
+    #   anyio
+    #   pytest
 filelock==3.12.4 \
     --hash=sha256:08c21d87ded6e2b9da6728c3dff51baf1dcecf973b768ef35bcbc3447edb9ad4 \
     --hash=sha256:2e6f249f1f3654291606e046b09f1fd5eac39b360664c27f5aad072012f8bcbd
     # via virtualenv
+greenlet==3.0.0 \
+    --hash=sha256:02a807b2a58d5cdebb07050efe3d7deaf915468d112dfcf5e426d0564aa3aa4a \
+    --hash=sha256:0b72b802496cccbd9b31acea72b6f87e7771ccfd7f7927437d592e5c92ed703c \
+    --hash=sha256:0d3f83ffb18dc57243e0151331e3c383b05e5b6c5029ac29f754745c800f8ed9 \
+    --hash=sha256:10b5582744abd9858947d163843d323d0b67be9432db50f8bf83031032bc218d \
+    --hash=sha256:123910c58234a8d40eaab595bc56a5ae49bdd90122dde5bdc012c20595a94c14 \
+    --hash=sha256:1482fba7fbed96ea7842b5a7fc11d61727e8be75a077e603e8ab49d24e234383 \
+    --hash=sha256:19834e3f91f485442adc1ee440171ec5d9a4840a1f7bd5ed97833544719ce10b \
+    --hash=sha256:1d363666acc21d2c204dd8705c0e0457d7b2ee7a76cb16ffc099d6799744ac99 \
+    --hash=sha256:211ef8d174601b80e01436f4e6905aca341b15a566f35a10dd8d1e93f5dbb3b7 \
+    --hash=sha256:269d06fa0f9624455ce08ae0179430eea61085e3cf6457f05982b37fd2cefe17 \
+    --hash=sha256:2e7dcdfad252f2ca83c685b0fa9fba00e4d8f243b73839229d56ee3d9d219314 \
+    --hash=sha256:334ef6ed8337bd0b58bb0ae4f7f2dcc84c9f116e474bb4ec250a8bb9bd797a66 \
+    --hash=sha256:343675e0da2f3c69d3fb1e894ba0a1acf58f481f3b9372ce1eb465ef93cf6fed \
+    --hash=sha256:37f60b3a42d8b5499be910d1267b24355c495064f271cfe74bf28b17b099133c \
+    --hash=sha256:38ad562a104cd41e9d4644f46ea37167b93190c6d5e4048fcc4b80d34ecb278f \
+    --hash=sha256:3c0d36f5adc6e6100aedbc976d7428a9f7194ea79911aa4bf471f44ee13a9464 \
+    --hash=sha256:3fd2b18432e7298fcbec3d39e1a0aa91ae9ea1c93356ec089421fabc3651572b \
+    --hash=sha256:4a1a6244ff96343e9994e37e5b4839f09a0207d35ef6134dce5c20d260d0302c \
+    --hash=sha256:4cd83fb8d8e17633ad534d9ac93719ef8937568d730ef07ac3a98cb520fd93e4 \
+    --hash=sha256:527cd90ba3d8d7ae7dceb06fda619895768a46a1b4e423bdb24c1969823b8362 \
+    --hash=sha256:56867a3b3cf26dc8a0beecdb4459c59f4c47cdd5424618c08515f682e1d46692 \
+    --hash=sha256:621fcb346141ae08cb95424ebfc5b014361621b8132c48e538e34c3c93ac7365 \
+    --hash=sha256:63acdc34c9cde42a6534518e32ce55c30f932b473c62c235a466469a710bfbf9 \
+    --hash=sha256:6512592cc49b2c6d9b19fbaa0312124cd4c4c8a90d28473f86f92685cc5fef8e \
+    --hash=sha256:6672fdde0fd1a60b44fb1751a7779c6db487e42b0cc65e7caa6aa686874e79fb \
+    --hash=sha256:6a5b2d4cdaf1c71057ff823a19d850ed5c6c2d3686cb71f73ae4d6382aaa7a06 \
+    --hash=sha256:6a68d670c8f89ff65c82b936275369e532772eebc027c3be68c6b87ad05ca695 \
+    --hash=sha256:6bb36985f606a7c49916eff74ab99399cdfd09241c375d5a820bb855dfb4af9f \
+    --hash=sha256:73b2f1922a39d5d59cc0e597987300df3396b148a9bd10b76a058a2f2772fc04 \
+    --hash=sha256:7709fd7bb02b31908dc8fd35bfd0a29fc24681d5cc9ac1d64ad07f8d2b7db62f \
+    --hash=sha256:8060b32d8586e912a7b7dac2d15b28dbbd63a174ab32f5bc6d107a1c4143f40b \
+    --hash=sha256:80dcd3c938cbcac986c5c92779db8e8ce51a89a849c135172c88ecbdc8c056b7 \
+    --hash=sha256:813720bd57e193391dfe26f4871186cf460848b83df7e23e6bef698a7624b4c9 \
+    --hash=sha256:831d6f35037cf18ca5e80a737a27d822d87cd922521d18ed3dbc8a6967be50ce \
+    --hash=sha256:871b0a8835f9e9d461b7fdaa1b57e3492dd45398e87324c047469ce2fc9f516c \
+    --hash=sha256:952256c2bc5b4ee8df8dfc54fc4de330970bf5d79253c863fb5e6761f00dda35 \
+    --hash=sha256:96d9ea57292f636ec851a9bb961a5cc0f9976900e16e5d5647f19aa36ba6366b \
+    --hash=sha256:9a812224a5fb17a538207e8cf8e86f517df2080c8ee0f8c1ed2bdaccd18f38f4 \
+    --hash=sha256:9adbd8ecf097e34ada8efde9b6fec4dd2a903b1e98037adf72d12993a1c80b51 \
+    --hash=sha256:9de687479faec7db5b198cc365bc34addd256b0028956501f4d4d5e9ca2e240a \
+    --hash=sha256:a048293392d4e058298710a54dfaefcefdf49d287cd33fb1f7d63d55426e4355 \
+    --hash=sha256:aa15a2ec737cb609ed48902b45c5e4ff6044feb5dcdfcf6fa8482379190330d7 \
+    --hash=sha256:abe1ef3d780de56defd0c77c5ba95e152f4e4c4e12d7e11dd8447d338b85a625 \
+    --hash=sha256:ad6fb737e46b8bd63156b8f59ba6cdef46fe2b7db0c5804388a2d0519b8ddb99 \
+    --hash=sha256:b1660a15a446206c8545edc292ab5c48b91ff732f91b3d3b30d9a915d5ec4779 \
+    --hash=sha256:b505fcfc26f4148551826a96f7317e02c400665fa0883fe505d4fcaab1dabfdd \
+    --hash=sha256:b822fab253ac0f330ee807e7485769e3ac85d5eef827ca224feaaefa462dc0d0 \
+    --hash=sha256:bdd696947cd695924aecb3870660b7545a19851f93b9d327ef8236bfc49be705 \
+    --hash=sha256:bdfaeecf8cc705d35d8e6de324bf58427d7eafb55f67050d8f28053a3d57118c \
+    --hash=sha256:be557119bf467d37a8099d91fbf11b2de5eb1fd5fc5b91598407574848dc910f \
+    --hash=sha256:c6b5ce7f40f0e2f8b88c28e6691ca6806814157ff05e794cdd161be928550f4c \
+    --hash=sha256:c94e4e924d09b5a3e37b853fe5924a95eac058cb6f6fb437ebb588b7eda79870 \
+    --hash=sha256:cc3e2679ea13b4de79bdc44b25a0c4fcd5e94e21b8f290791744ac42d34a0353 \
+    --hash=sha256:d1e22c22f7826096ad503e9bb681b05b8c1f5a8138469b255eb91f26a76634f2 \
+    --hash=sha256:d5539f6da3418c3dc002739cb2bb8d169056aa66e0c83f6bacae0cd3ac26b423 \
+    --hash=sha256:d55db1db455c59b46f794346efce896e754b8942817f46a1bada2d29446e305a \
+    --hash=sha256:e09dea87cc91aea5500262993cbd484b41edf8af74f976719dd83fe724644cd6 \
+    --hash=sha256:e52a712c38e5fb4fd68e00dc3caf00b60cb65634d50e32281a9d6431b33b4af1 \
+    --hash=sha256:e693e759e172fa1c2c90d35dea4acbdd1d609b6936115d3739148d5e4cd11947 \
+    --hash=sha256:ecf94aa539e97a8411b5ea52fc6ccd8371be9550c4041011a091eb8b3ca1d810 \
+    --hash=sha256:f351479a6914fd81a55c8e68963609f792d9b067fb8a60a042c585a621e0de4f \
+    --hash=sha256:f47932c434a3c8d3c86d865443fadc1fbf574e9b11d6650b656e602b1797908a
+    # via
+    #   -c requirements/main.txt
+    #   sqlalchemy
 h11==0.14.0 \
     --hash=sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d \
     --hash=sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761
@@ -337,6 +410,14 @@ sqlalchemy[mypy]==2.0.21 \
     # via
     #   -c requirements/main.txt
     #   -r requirements/dev.in
+tomli==2.0.1 \
+    --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \
+    --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f
+    # via
+    #   coverage
+    #   mypy
+    #   pylint
+    #   pytest
 tomlkit==0.12.1 \
     --hash=sha256:38e1ff8edb991273ec9f6181244a6a391ac30e9f5098e7535640ea6be97a7c86 \
     --hash=sha256:712cbd236609acc6a3e2e97253dfc52d4c2082982a88f61b640ecf0817eab899
@@ -356,6 +437,7 @@ typing-extensions==4.8.0 \
     --hash=sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef
     # via
     #   -c requirements/main.txt
+    #   astroid
     #   mypy
     #   sqlalchemy
 virtualenv==20.24.5 \

requirements/main.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
 #    pip-compile --generate-hashes --output-file=requirements/main.txt requirements/main.in
@@ -16,9 +16,9 @@ anyio==3.7.1 \
     #   httpcore
     #   starlette
     #   watchfiles
-anytree==2.9.0 \
-    --hash=sha256:06f7bc294293da2755f4699cc5da5c92d9182a5cfae2842c83fb56f02bd427c8 \
-    --hash=sha256:7f1ad0f9b225705b780ea0593c8ff52af05df9428e7cc34b9379b879fa462663
+anytree==2.10.0 \
+    --hash=sha256:a5e922bef6bb5a154f8d306d37b40ea21885e4143856a9206a14b791cfc26102 \
+    --hash=sha256:f59680d19bd94f76d8984c5df1fb286085bb1964238b714c6229c5c95d8e55cf
     # via idds-workflow
 argcomplete==3.1.2 \
     --hash=sha256:d5d1e5efd41435260b8f85673b74ea2e883affcbec9f4230c582689e8e78251b \
@@ -64,6 +64,10 @@ astropy==5.3.4 \
     #   lsst-daf-butler
     #   lsst-pipe-base
     #   lsst-utils
+async-timeout==4.0.3 \
+    --hash=sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f \
+    --hash=sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028
+    # via redis
 asyncpg==0.28.0 \
     --hash=sha256:0740f836985fd2bd73dca42c50c6074d1d61376e134d7ad3ad7566c4f79f8184 \
     --hash=sha256:0a6d1b954d2b296292ddff4e0060f494bb4270d87fb3655dd23c5c6096d16d83 \
@@ -316,8 +320,6 @@ cryptography==41.0.4 \
     # via
     #   idds-common
     #   moto
-    #   pyjwt
-    #   safir
 decorator==5.1.1 \
     --hash=sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330 \
     --hash=sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186
@@ -333,6 +335,10 @@ dogpile-cache==1.2.2 \
     --hash=sha256:f6c2c6ff3a3dc7dc0d662b3f30983f684502fd7a91a45be680879d7d8cc177d7 \
     --hash=sha256:fd9022c0d9cbadadf20942391a95adaf296be80b42daa8e202f8de1c21f198b2
     # via idds-common
+exceptiongroup==1.1.3 \
+    --hash=sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9 \
+    --hash=sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3
+    # via anyio
 fastapi==0.103.2 \
     --hash=sha256:3270de872f0fe9ec809d4bd3d4d890c6d5cc7b9611d721d6438f9dacc8c4ef2e \
     --hash=sha256:75a11f6bfb8fc4d2bec0bd710c2d5f2829659c0e8c0afd5560fdda6ce25ec653
@@ -382,10 +388,6 @@ futures==3.0.5 \
     --hash=sha256:0542525145d5afc984c88f914a0c85c77527f65946617edb5274f72406f981df \
     --hash=sha256:f7f16b6bf9653a918a03f1f2c2d62aac0cd64b1bc088e93ea279517f6b61120b
     # via idds-doma
-gidgethub==5.3.0 \
-    --hash=sha256:4dd92f2252d12756b13f9dd15cde322bfb0d625b6fb5d680da1567ec74b462c0 \
-    --hash=sha256:9ece7d37fbceb819b80560e7ed58f936e48a65d37ec5f56db79145156b426a25
-    # via safir
 greenlet==3.0.0 \
     --hash=sha256:02a807b2a58d5cdebb07050efe3d7deaf915468d112dfcf5e426d0564aa3aa4a \
     --hash=sha256:0b72b802496cccbd9b31acea72b6f87e7771ccfd7f7927437d592e5c92ed703c \
@@ -1046,12 +1048,10 @@ pyerfa==2.0.0.3 \
     --hash=sha256:f0e9d0b122c454bcad5dbd0c3283b200783031d3f99ca9c550f49a7a7d4c41ea \
     --hash=sha256:f1819e0d95ff8dead80614f8063919d82b2dbb55437b6c0109d3393c1ab55954
     # via astropy
-pyjwt[crypto]==2.8.0 \
+pyjwt==2.8.0 \
     --hash=sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de \
     --hash=sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320
-    # via
-    #   gidgethub
-    #   idds-common
+    # via idds-common
 python-dateutil==2.8.2 \
     --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \
     --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9
@@ -1145,9 +1145,9 @@ s3transfer==0.7.0 \
     --hash=sha256:10d6923c6359175f264811ef4bf6161a3156ce8e350e705396a7557d6293c33a \
     --hash=sha256:fd3889a66f5fe17299fe75b82eae6cf722554edca744ca5d5fe308b104883d2e
     # via boto3
-safir[arq,db]==4.3.0 \
-    --hash=sha256:7a5df1d6a2dd22aa8ba8d19e3390721152fe43540e7dbb754dac321affe8dc43 \
-    --hash=sha256:ce4f3d8ff34a9044650e07bdf2214d1462fc49a07d7ac864e2232433d6630fbe
+safir[arq,db]==3.8.0 \
+    --hash=sha256:25bd4f069d2ae76db0ccf3dcd5e2361a424cc66f138c5c59504abf1908e66161 \
+    --hash=sha256:3bb373ced893155821124cebecb7abaafb15deb24cd717bb9ef00f8b399af228
     # via -r requirements/main.in
 six==1.16.0 \
     --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
@@ -1253,14 +1253,12 @@ typing-extensions==4.8.0 \
     --hash=sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef
     # via
     #   arq
+    #   dogpile-cache
     #   fastapi
     #   pydantic
     #   pydantic-core
     #   sqlalchemy
-uritemplate==4.1.1 \
-    --hash=sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0 \
-    --hash=sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e
-    # via gidgethub
+    #   uvicorn
 urllib3==2.0.6 \
     --hash=sha256:7a7c7003b000adf9e7ca2a377c9688bbc54ed41b985789ed576570342a375cd2 \
     --hash=sha256:b19e1a85d206b56d7df1d5e683df4a7725252a964e3993648dd0fb5a1c157564

src/lsst/cmservice/db/group.py

@@ -77,8 +77,10 @@ async def children(
         self,
         session: async_scoped_session,  # pylint: disable=unused-argument
     ) -> Iterable:
-        """Returns empty list for consistency"""
-        return []
+        """Maps self.g_ to self.children() for consistency"""
+        async with session.begin_nested():
+            await session.refresh(self, attribute_names=["jobs_"])
+            return self.jobs_
 
     @classmethod
     async def get_create_kwargs(

src/lsst/cmservice/db/job.py

@@ -11,13 +11,13 @@
 from .dbid import DbId
 from .element import ElementMixin
 from .group import Group
-from .script import Script
 from .specification import SpecBlock
 from .step import Step
 
 if TYPE_CHECKING:
     from .pipetask_error import PipetaskError
     from .product_set import ProductSet
+    from .script import Script
     from .task_set import TaskSet
     from .wms_task_report import WmsTaskReport
 
@@ -36,6 +36,8 @@ class Job(Base, ElementMixin):
     data: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
     child_config: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
     collections: Mapped[Optional[dict | list]] = mapped_column(type_=JSON)
+    wms_job_id: Mapped[Optional[int]] = mapped_column()
+    stamp_url: Mapped[Optional[str]] = mapped_column()
 
     spec_block_: Mapped["SpecBlock"] = relationship("SpecBlock", viewonly=True)
     s_: Mapped["Step"] = relationship(
@@ -45,6 +47,20 @@ class Job(Base, ElementMixin):
         secondaryjoin="Group.parent_id==Step.id",
         viewonly=True,
     )
+    c_: Mapped["Campaign"] = relationship(
+        "Campaign",
+        primaryjoin="Job.parent_id==Group.id",
+        secondary="join(Group, Step).join(Campaign)",
+        secondaryjoin="and_(Group.parent_id==Step.id, Step.parent_id==Campaign.id) ",
+        viewonly=True,
+    )
+    p_: Mapped["Production"] = relationship(
+        "Production",
+        primaryjoin="Job.parent_id==Group.id",
+        secondary="join(Group, Step).join(Campaign).join(Production)",
+        secondaryjoin="and_(Group.parent_id==Step.id, Step.parent_id==Campaign.id, Campaign.parent_id==Production.id,) ",
+        viewonly=True,
+    )
     parent_: Mapped["Group"] = relationship("Group", viewonly=True)
     scripts_: Mapped[List["Script"]] = relationship("Script", viewonly=True)
     tasks_: Mapped[List["TaskSet"]] = relationship("TaskSet", viewonly=True)

src/lsst/cmservice/db/script.py

@@ -14,6 +14,7 @@
 from .dbid import DbId
 from .element import ElementMixin
 from .group import Group
+from .job import Job
 from .node import NodeMixin
 from .specification import SpecBlock
 from .step import Step
@@ -177,6 +178,9 @@ async def get_create_kwargs(
         elif parent_level == LevelEnum.group:
             element = await Group.get_row_by_fullname(session, parent_name)
             ret_dict["g_id"] = element.id
+        elif parent_level == LevelEnum.job:
+            element = await Job.get_row_by_fullname(session, parent_name)
+            ret_dict["j_id"] = element.id
         else:
             raise ValueError(f"Bad level for script: {parent_level}")
         ret_dict["parent_id"] = element.id

src/lsst/cmservice/db/script_handler.py

@@ -304,6 +304,7 @@ async def prepare(
         if script_method == ScriptMethod.default:
             script_method = self.default_method
 
+        status = script.status
         if script_method == ScriptMethod.no_script:  # pragma: no cover
             raise ValueError("ScriptMethod.no_script can not be set for ScriptHandler")
         if script_method == ScriptMethod.bash:

src/lsst/cmservice/handlers/elements.py

@@ -88,7 +88,7 @@ class RunJobsScriptHandler(RunElementScriptHandler):
     FIXME
     """
 
-    async def _do_run(
+    async def _do_prepare(
         self,
         session: async_scoped_session,
         script: Script,