diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 158dbdae7..85e078a0e 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -136,10 +136,15 @@ jobs: - uses: actions/checkout@v3 - name: Set up Singularity + if: ${{ matrix.container == 'singularity' }} uses: eWaterCycle/setup-singularity@v7 with: singularity-version: ${{ env.singularity_version }} + - name: Set up Podman + if: ${{ matrix.container == 'podman' }} + run: sudo rm -f /usr/bin/docker ; sudo apt-get install -y podman + - name: Set up Python uses: actions/setup-python@v4 with: diff --git a/cwltool/builder.py b/cwltool/builder.py index 57420d67b..3c5d80923 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -17,7 +17,6 @@ from cwl_utils import expression from cwl_utils.file_formats import check_format from rdflib import Graph -from ruamel.yaml.comments import CommentedMap from schema_salad.avro.schema import Names, Schema, make_avsc_object from schema_salad.exceptions import ValidationException from schema_salad.sourceline import SourceLine @@ -25,6 +24,8 @@ from schema_salad.validate import validate from typing_extensions import TYPE_CHECKING, Type # pylint: disable=unused-import +from ruamel.yaml.comments import CommentedMap + from .errors import WorkflowException from .loghandler import _logger from .mutation import MutationManager diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index e48df93ec..bc463413a 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -30,7 +30,6 @@ ) import shellescape -from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.avro.schema import Schema from schema_salad.exceptions import ValidationException from schema_salad.ref_resolver import file_uri, uri_file_path @@ -39,6 +38,8 @@ from schema_salad.validate import validate_ex from typing_extensions import TYPE_CHECKING, Type +from ruamel.yaml.comments import CommentedMap, CommentedSeq + from .builder import ( INPUT_OBJ_VOCAB, Builder, @@ -46,7 +47,7 @@ substitute, ) from .context import LoadingContext, RuntimeContext, getdefault -from .docker import DockerCommandLineJob +from .docker import DockerCommandLineJob, PodmanCommandLineJob from .errors import UnsupportedRequirement, WorkflowException from .flatten import flatten from .job import CommandLineJob, JobBase @@ -460,6 +461,8 @@ def make_job_runner(self, runtimeContext: RuntimeContext) -> Type[JobBase]: raise UnsupportedRequirement( "Both Docker and MPI have been hinted - don't know what to do" ) + if runtimeContext.podman: + return PodmanCommandLineJob return DockerCommandLineJob if dockerRequired: raise UnsupportedRequirement( diff --git a/cwltool/context.py b/cwltool/context.py index 4afb1ea10..b348e4c9d 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -17,13 +17,14 @@ Union, ) -# move to a regular typing import when Python 3.3-3.6 is no longer supported -from ruamel.yaml.comments import CommentedMap from schema_salad.avro.schema import Names from schema_salad.ref_resolver import Loader from schema_salad.utils import FetcherCallableType from typing_extensions import TYPE_CHECKING +# move to a regular typing import when Python 3.3-3.6 is no longer supported +from ruamel.yaml.comments import CommentedMap + from .builder import Builder from .mpi import MpiConfig from .mutation import MutationManager diff --git a/cwltool/cwlrdf.py b/cwltool/cwlrdf.py index 1bfe8adcc..b18c4207e 100644 --- a/cwltool/cwlrdf.py +++ b/cwltool/cwlrdf.py @@ -4,10 +4,11 @@ from rdflib import Graph from rdflib.query import ResultRow -from ruamel.yaml.comments import CommentedMap from schema_salad.jsonld_context import makerdf from schema_salad.utils import ContextType +from ruamel.yaml.comments import CommentedMap + from .cwlviewer import CWLViewer from .process import Process diff --git a/cwltool/docker.py b/cwltool/docker.py index 2fa688b11..af3c49d88 100644 --- a/cwltool/docker.py +++ b/cwltool/docker.py @@ -90,9 +90,10 @@ def __init__( ) -> None: """Initialize a command line builder using the Docker software container engine.""" super().__init__(builder, joborder, make_path_mapper, requirements, hints, name) + self.docker_exec = "docker" - @staticmethod def get_image( + self, docker_requirement: Dict[str, str], pull_image: bool, force_pull: bool, @@ -117,7 +118,7 @@ def get_image( for line in ( subprocess.check_output( # nosec - ["docker", "images", "--no-trunc", "--all"] + [self.docker_exec, "images", "--no-trunc", "--all"] ) .decode("utf-8") .splitlines() @@ -151,7 +152,7 @@ def get_image( if (force_pull or not found) and pull_image: cmd = [] # type: List[str] if "dockerPull" in docker_requirement: - cmd = ["docker", "pull", str(docker_requirement["dockerPull"])] + cmd = [self.docker_exec, "pull", str(docker_requirement["dockerPull"])] _logger.info(str(cmd)) subprocess.check_call(cmd, stdout=sys.stderr) # nosec found = True @@ -160,7 +161,7 @@ def get_image( with open(os.path.join(dockerfile_dir, "Dockerfile"), "w") as dfile: dfile.write(docker_requirement["dockerFile"]) cmd = [ - "docker", + self.docker_exec, "build", "--tag=%s" % str(docker_requirement["dockerImageId"]), dockerfile_dir, @@ -169,7 +170,7 @@ def get_image( subprocess.check_call(cmd, stdout=sys.stderr) # nosec found = True elif "dockerLoad" in docker_requirement: - cmd = ["docker", "load"] + cmd = [self.docker_exec, "load"] _logger.info(str(cmd)) if os.path.exists(docker_requirement["dockerLoad"]): _logger.info( @@ -203,7 +204,7 @@ def get_image( found = True elif "dockerImport" in docker_requirement: cmd = [ - "docker", + self.docker_exec, "import", str(docker_requirement["dockerImport"]), str(docker_requirement["dockerImageId"]), @@ -225,8 +226,8 @@ def get_from_requirements( force_pull: bool, tmp_outdir_prefix: str, ) -> Optional[str]: - if not shutil.which("docker"): - raise WorkflowException("docker executable is not available") + if not shutil.which(self.docker_exec): + raise WorkflowException(f"{self.docker_exec} executable is not available") if self.get_image( cast(Dict[str, str], r), pull_image, force_pull, tmp_outdir_prefix @@ -341,10 +342,10 @@ def create_runtime( runtime = [user_space_docker_cmd, "--quiet", "run", "--nobanner"] else: runtime = [user_space_docker_cmd, "run"] - elif runtimeContext.podman: - runtime = ["podman", "run", "-i", "--userns=keep-id"] else: - runtime = ["docker", "run", "-i"] + runtime = [self.docker_exec, "run", "-i"] + if runtimeContext.podman: + runtime.append("--userns=keep-id") self.append_volume( runtime, os.path.realpath(self.outdir), self.builder.outdir, writable=True ) @@ -460,3 +461,20 @@ def create_runtime( ) return runtime, cidfile_path + + +class PodmanCommandLineJob(DockerCommandLineJob): + """Runs a CommandLineJob in a software container using the podman engine.""" + + def __init__( + self, + builder: Builder, + joborder: CWLObjectType, + make_path_mapper: Callable[..., PathMapper], + requirements: List[CWLObjectType], + hints: List[CWLObjectType], + name: str, + ) -> None: + """Initialize a command line builder using the Podman software container engine.""" + super().__init__(builder, joborder, make_path_mapper, requirements, hints, name) + self.docker_exec = "podman" diff --git a/cwltool/job.py b/cwltool/job.py index a327773c5..8b206e185 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -874,6 +874,7 @@ def run( cidfile, runtimeContext.tmpdir_prefix, not bool(runtimeContext.cidfile_dir), + "podman" if runtimeContext.podman else "docker", ) elif runtimeContext.user_space_docker_cmd: monitor_function = functools.partial(self.process_monitor) @@ -884,6 +885,7 @@ def docker_monitor( cidfile: str, tmpdir_prefix: str, cleanup_cidfile: bool, + docker_exe: str, process, # type: subprocess.Popen[str] ) -> None: """Record memory usage of the running Docker container.""" @@ -901,7 +903,7 @@ def docker_monitor( os.remove(cidfile) except OSError as exc: _logger.warning( - "Ignored error cleaning up Docker cidfile: %s", exc + "Ignored error cleaning up %s cidfile: %s", docker_exe, exc ) return try: @@ -915,15 +917,19 @@ def docker_monitor( stats_file_name = stats_file.name try: with open(stats_file_name, mode="w") as stats_file_handle: + cmds = [docker_exe, "stats"] + if "podman" not in docker_exe: + cmds.append("--no-trunc") + cmds.extend(["--format", "{{.MemPerc}}", cid]) stats_proc = subprocess.Popen( # nosec - ["docker", "stats", "--no-trunc", "--format", "{{.MemPerc}}", cid], + cmds, stdout=stats_file_handle, stderr=subprocess.DEVNULL, ) process.wait() stats_proc.kill() except OSError as exc: - _logger.warning("Ignored error with docker stats: %s", exc) + _logger.warning("Ignored error with %s stats: %s", docker_exe, exc) return max_mem_percent = 0 # type: float mem_percent = 0 # type: float @@ -938,8 +944,10 @@ def docker_monitor( ) if mem_percent > max_mem_percent: max_mem_percent = mem_percent - except ValueError: - break + except ValueError as exc: + _logger.debug( + "%s stats parsing error in line %s: %s", docker_exe, line, exc + ) _logger.info( "[job %s] Max memory used: %iMiB", self.name, diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index f574b67e4..22a5a56c0 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -21,7 +21,6 @@ ) from cwl_utils.parser import cwl_v1_2, cwl_v1_2_utils -from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.exceptions import ValidationException from schema_salad.ref_resolver import Loader, file_uri from schema_salad.schema import validate_doc @@ -34,6 +33,8 @@ json_dumps, ) +from ruamel.yaml.comments import CommentedMap, CommentedSeq + from . import CWL_CONTENT_TYPES, process, update from .context import LoadingContext from .errors import GraphTargetMissingException diff --git a/cwltool/main.py b/cwltool/main.py index 23c12d595..e98ef2d1b 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -36,14 +36,15 @@ import argcomplete import coloredlogs import pkg_resources # part of setuptools -import ruamel.yaml -from ruamel.yaml.comments import CommentedMap, CommentedSeq -from ruamel.yaml.main import YAML from schema_salad.exceptions import ValidationException from schema_salad.ref_resolver import Loader, file_uri, uri_file_path from schema_salad.sourceline import cmap, strip_dup_lineno from schema_salad.utils import ContextType, FetcherCallableType, json_dumps, yaml_no_ts +import ruamel.yaml +from ruamel.yaml.comments import CommentedMap, CommentedSeq +from ruamel.yaml.main import YAML + from . import CWL_CONTENT_TYPES, workflow from .argparser import arg_parser, generate_parser, get_default_args from .context import LoadingContext, RuntimeContext, getdefault @@ -105,6 +106,8 @@ ) from .workflow import Workflow +docker_exe: str + def _terminate_processes() -> None: """Kill all spawned processes. @@ -117,6 +120,7 @@ def _terminate_processes() -> None: continuing to execute while it kills the processes that they've spawned. This may occasionally lead to unexpected behaviour. """ + global docker_exe # It's possible that another thread will spawn a new task while # we're executing, so it's not safe to use a for loop here. while processes_to_kill: @@ -130,7 +134,7 @@ def _terminate_processes() -> None: try: with open(cidfile[0]) as inp_stream: p = subprocess.Popen( # nosec - ["docker", "kill", inp_stream.read()], shell=False # nosec + [docker_exe, "kill", inp_stream.read()], shell=False # nosec ) try: p.wait(timeout=10) @@ -1009,6 +1013,7 @@ def main( stderr_handler = _logger.handlers[-1] workflowobj = None prov_log_handler: Optional[logging.StreamHandler[ProvOut]] = None + global docker_exe try: if args is None: if argsl is None: @@ -1030,6 +1035,10 @@ def main( else: runtimeContext = runtimeContext.copy() + if runtimeContext.podman: + docker_exe = "podman" + else: + docker_exe = "docker" # If caller parsed its own arguments, it may not include every # cwltool option, so fill in defaults to avoid crashing when # dereferencing them in args. diff --git a/cwltool/pack.py b/cwltool/pack.py index efd7aeada..cd26d3483 100644 --- a/cwltool/pack.py +++ b/cwltool/pack.py @@ -14,10 +14,11 @@ cast, ) -from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.ref_resolver import Loader, SubLoader from schema_salad.utils import ResolveType +from ruamel.yaml.comments import CommentedMap, CommentedSeq + from .context import LoadingContext from .load_tool import fetch_document, resolve_and_validate_document from .process import shortname, uniquename diff --git a/cwltool/process.py b/cwltool/process.py index 18017d523..76f18014c 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -35,7 +35,6 @@ from mypy_extensions import mypyc_attr from pkg_resources import resource_stream from rdflib import Graph -from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.avro.schema import ( Names, Schema, @@ -50,6 +49,8 @@ from schema_salad.validate import avro_type_name, validate_ex from typing_extensions import TYPE_CHECKING +from ruamel.yaml.comments import CommentedMap, CommentedSeq + from .builder import INPUT_OBJ_VOCAB, Builder from .context import LoadingContext, RuntimeContext, getdefault from .errors import UnsupportedRequirement, WorkflowException diff --git a/cwltool/procgenerator.py b/cwltool/procgenerator.py index 69780e32c..aabbd354d 100644 --- a/cwltool/procgenerator.py +++ b/cwltool/procgenerator.py @@ -1,10 +1,11 @@ import copy from typing import Dict, Optional, Tuple, cast -from ruamel.yaml.comments import CommentedMap from schema_salad.exceptions import ValidationException from schema_salad.sourceline import indent +from ruamel.yaml.comments import CommentedMap + from .context import LoadingContext, RuntimeContext from .errors import WorkflowException from .load_tool import load_tool diff --git a/cwltool/update.py b/cwltool/update.py index 04a2d95d5..d5c753ef4 100644 --- a/cwltool/update.py +++ b/cwltool/update.py @@ -11,11 +11,12 @@ cast, ) -from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.exceptions import ValidationException from schema_salad.ref_resolver import Loader from schema_salad.sourceline import SourceLine +from ruamel.yaml.comments import CommentedMap, CommentedSeq + from .loghandler import _logger from .utils import CWLObjectType, CWLOutputType, aslist, visit_class, visit_field diff --git a/cwltool/validate_js.py b/cwltool/validate_js.py index 821cadbf2..9c6379f7b 100644 --- a/cwltool/validate_js.py +++ b/cwltool/validate_js.py @@ -19,7 +19,6 @@ from cwl_utils.expression import scanner as scan_expression from cwl_utils.sandboxjs import code_fragment_to_js, exec_js_process from pkg_resources import resource_stream -from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.avro.schema import ( ArraySchema, EnumSchema, @@ -31,6 +30,8 @@ from schema_salad.utils import json_dumps from schema_salad.validate import validate_ex +from ruamel.yaml.comments import CommentedMap, CommentedSeq + from .errors import WorkflowException from .loghandler import _logger diff --git a/cwltool/workflow.py b/cwltool/workflow.py index 0f7983222..0b66fa937 100644 --- a/cwltool/workflow.py +++ b/cwltool/workflow.py @@ -17,10 +17,11 @@ ) from uuid import UUID -from ruamel.yaml.comments import CommentedMap from schema_salad.exceptions import ValidationException from schema_salad.sourceline import SourceLine, indent +from ruamel.yaml.comments import CommentedMap + from . import command_line_tool, context, procgenerator from .checker import circular_dependency_checker, loop_checker, static_checker from .context import LoadingContext, RuntimeContext, getdefault diff --git a/tests/test_anon_types.py b/tests/test_anon_types.py index b4edf0cb0..6909368ab 100644 --- a/tests/test_anon_types.py +++ b/tests/test_anon_types.py @@ -1,11 +1,11 @@ from typing import cast import pytest -from ruamel.yaml.comments import CommentedMap from schema_salad.sourceline import cmap from cwltool.command_line_tool import CommandLineTool from cwltool.context import LoadingContext +from ruamel.yaml.comments import CommentedMap snippet = cast( CommentedMap, diff --git a/tests/test_examples.py b/tests/test_examples.py index c8e4c6fcc..54673a4f9 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -15,7 +15,6 @@ import pytest from cwl_utils.errors import JavascriptException from cwl_utils.sandboxjs import param_re -from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.exceptions import ValidationException import cwltool.checker @@ -29,6 +28,7 @@ from cwltool.main import main from cwltool.process import CWL_IANA from cwltool.utils import CWLObjectType, dedup +from ruamel.yaml.comments import CommentedMap, CommentedSeq from .util import get_data, get_main_output, needs_docker, working_directory diff --git a/tests/test_mpi.py b/tests/test_mpi.py index 8fb4b0e51..0fbedbcdf 100644 --- a/tests/test_mpi.py +++ b/tests/test_mpi.py @@ -8,7 +8,6 @@ import pkg_resources import pytest -from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.avro.schema import Names from schema_salad.utils import yaml_no_ts @@ -19,6 +18,7 @@ from cwltool.context import LoadingContext, RuntimeContext from cwltool.main import main from cwltool.mpi import MpiConfig, MPIRequirementName +from ruamel.yaml.comments import CommentedMap, CommentedSeq from .util import get_data, working_directory diff --git a/tests/test_path_checks.py b/tests/test_path_checks.py index 0e5a56454..0e8cb1214 100644 --- a/tests/test_path_checks.py +++ b/tests/test_path_checks.py @@ -4,7 +4,6 @@ from typing import IO, Any, List, cast import pytest -from ruamel.yaml.comments import CommentedMap from schema_salad.sourceline import cmap from cwltool.command_line_tool import CommandLineTool @@ -13,6 +12,7 @@ from cwltool.stdfsaccess import StdFsAccess from cwltool.update import INTERNAL_VERSION from cwltool.utils import CWLObjectType +from ruamel.yaml.comments import CommentedMap from .util import needs_docker diff --git a/tests/test_streaming.py b/tests/test_streaming.py index 3c5526592..0e23276ac 100644 --- a/tests/test_streaming.py +++ b/tests/test_streaming.py @@ -4,7 +4,6 @@ from typing import cast import pytest -from ruamel.yaml.comments import CommentedMap from schema_salad.sourceline import cmap from cwltool.command_line_tool import CommandLineTool @@ -13,6 +12,7 @@ from cwltool.job import JobBase from cwltool.update import INTERNAL_VERSION, ORIGINAL_CWLVERSION from cwltool.utils import CWLObjectType +from ruamel.yaml.comments import CommentedMap from .util import get_data diff --git a/tests/test_tmpdir.py b/tests/test_tmpdir.py index 19ad253d7..420fefc1a 100644 --- a/tests/test_tmpdir.py +++ b/tests/test_tmpdir.py @@ -5,7 +5,6 @@ from typing import List, cast import pytest -from ruamel.yaml.comments import CommentedMap from schema_salad.avro import schema from schema_salad.sourceline import cmap @@ -19,6 +18,7 @@ from cwltool.stdfsaccess import StdFsAccess from cwltool.update import INTERNAL_VERSION, ORIGINAL_CWLVERSION from cwltool.utils import create_tmp_dir +from ruamel.yaml.comments import CommentedMap from .util import get_data, needs_docker @@ -116,8 +116,39 @@ def test_dockerfile_tmpdir_prefix( monkeypatch.setattr( target=subprocess, name="check_call", value=lambda *args, **kwargs: True ) - tmp_outdir_prefix = tmp_path / "1" - assert DockerCommandLineJob.get_image( + (tmp_path / "out").mkdir() + tmp_outdir_prefix = tmp_path / "out" / "1" + (tmp_path / "3").mkdir() + tmpdir_prefix = str(tmp_path / "3" / "ttmp") + runtime_context = RuntimeContext( + {"tmpdir_prefix": tmpdir_prefix, "user_space_docker_cmd": None} + ) + builder = Builder( + {}, + [], + [], + {}, + schema.Names(), + [], + [], + {}, + None, + None, + StdFsAccess, + StdFsAccess(""), + None, + 0.1, + False, + False, + False, + "", + runtime_context.get_outdir(), + runtime_context.get_tmpdir(), + runtime_context.get_stagedir(), + INTERNAL_VERSION, + "docker", + ) + assert DockerCommandLineJob(builder, {}, PathMapper, [], [], "").get_image( { "class": "DockerRequirement", "dockerFile": "FROM debian:stable-slim",