From c264a0d3c088b4dc76fd20ada48192690259f64d Mon Sep 17 00:00:00 2001 From: stxue1 Date: Tue, 16 Jul 2024 11:50:37 -0700 Subject: [PATCH] Don't prepull containers if extensions are enabled See https://github.com/common-workflow-language/cwl-utils/issues/309 --- src/toil/cwl/cwltoil.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/toil/cwl/cwltoil.py b/src/toil/cwl/cwltoil.py index ad70623975..a37e72d916 100644 --- a/src/toil/cwl/cwltoil.py +++ b/src/toil/cwl/cwltoil.py @@ -102,7 +102,7 @@ from toil.batchSystems.registry import DEFAULT_BATCH_SYSTEM from toil.common import Toil, addOptions from toil.cwl import check_cwltool_version -from toil.lib.misc import call_command +from toil.lib.misc import call_command, CalledProcessErrorStderr from toil.provisioners.clusterScaler import JobTooBigError check_cwltool_version() @@ -247,14 +247,14 @@ def try_prepull(cwl_tool_uri: str, runtime_context: cwltool.context.RuntimeConte """ if runtime_context.singularity: if "CWL_SINGULARITY_CACHE" in os.environ: - logger.info("Prepulling containers with Singularity...") + logger.info("Prepulling the workflow's containers with Singularity...") call_command(["cwl-docker-extract", "--singularity", "--dir", os.environ['CWL_SINGULARITY_CACHE'], cwl_tool_uri]) elif not runtime_context.user_space_docker_cmd and not runtime_context.podman: # For udocker and podman prefetching is unimplemented # This is docker if batchsystem == "single_machine": # Only on single machine will the docker daemon be accessible by all workers and the leader - logger.info("Prepulling containers with Docker...") + logger.info("Prepulling the workflow's containers with Docker...") call_command(["cwl-docker-extract", cwl_tool_uri]) @@ -3768,7 +3768,12 @@ def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int: # Attempt to prepull the containers if not options.no_prepull: - try_prepull(uri, runtime_context, toil.config.batchSystem) + if not options.enable_ext: + # The CWL utils parser does not support cwltool extensions and will crash if encountered, so don't prepull if extensions are enabled + # See https://github.com/common-workflow-language/cwl-utils/issues/309 + try_prepull(uri, runtime_context, toil.config.batchSystem) + else: + logger.debug("Not prepulling containers as cwltool extensions are not supported.") options.tool_help = None options.debug = options.logLevel == "DEBUG"