From 46482012e808a6587ba77607366b4616741767df Mon Sep 17 00:00:00 2001 From: jsitarova-dnanexus <100697844+jsitarova-dnanexus@users.noreply.github.com> Date: Fri, 11 Oct 2024 10:31:23 +0200 Subject: [PATCH 1/6] [COMPUTE-993] nvidiaDriver field support in dx-toolkit (#1399) * add nvidia_driver to dxjob.py * simplify if statement in dxjob.py * add nvidia_driver to dxapplet.py * add nvidia_driver to system_requirements.py * remove duplicated commment * add nvidia_driver to dx.py * add nvidia_driver to dx.py * uncomment tests * remove print * add nvidia check to test * update readme * edit test * add logs * edit job logs * add test logs * remove logs * edit test * remove logging * fix1: edit if condition * fix2: move nvidia_driver at the end * fix3: add test to test_dx_bash_helpers.py * fix4: add job clone TC * add '' * fix command * fix command2 * add 1more TC * fix4: move nvidia driver param * fix5: use if any --- src/python/Readme.md | 8 ++++ src/python/dxpy/bindings/dxapplet.py | 12 ++++-- src/python/dxpy/bindings/dxjob.py | 34 ++++++++------- src/python/dxpy/scripts/dx.py | 14 +++++-- src/python/dxpy/system_requirements.py | 2 +- src/python/test/test_dx_bash_helpers.py | 16 +++++++ src/python/test/test_dxclient.py | 56 +++++++++++++++++++++++-- src/python/test/test_dxpy.py | 10 ++++- 8 files changed, 124 insertions(+), 28 deletions(-) diff --git a/src/python/Readme.md b/src/python/Readme.md index 1a650fec40..3caf2402db 100644 --- a/src/python/Readme.md +++ b/src/python/Readme.md @@ -27,6 +27,14 @@ Example: $ _DX_DEBUG=1 dx ls ``` +### Debugging inside the IDE (PyCharm) +To be able to debug dx-toolkit (dx commands) directly in the IDE, 'Run/Debug Configurations' needs to be changed. +1. Go to Run → Edit Configurations... +2. Add New Configuration (Python) +3. Change script to module (dxpy.scripts.dx) +4. To Script parameters field write dx command you want to run (eg 'ls' runs 'dx ls') +5. Apply and OK (now it is possible to start debugging via main() function in dx.py) + Python coding style ------------------- diff --git a/src/python/dxpy/bindings/dxapplet.py b/src/python/dxpy/bindings/dxapplet.py index 46f9657271..3be1ab9cc9 100644 --- a/src/python/dxpy/bindings/dxapplet.py +++ b/src/python/dxpy/bindings/dxapplet.py @@ -58,11 +58,12 @@ def _get_run_input_common_fields(executable_input, **kwargs): if kwargs.get(arg) is not None: run_input[arg] = kwargs[arg] - if kwargs.get('instance_type') is not None or kwargs.get('cluster_spec') is not None or kwargs.get('fpga_driver') is not None: + if any(kwargs.get(key) is not None for key in ['instance_type', 'cluster_spec', 'fpga_driver', 'nvidia_driver']): instance_type_srd = SystemRequirementsDict.from_instance_type(kwargs.get('instance_type')) cluster_spec_srd = SystemRequirementsDict(kwargs.get('cluster_spec')) fpga_driver_srd = SystemRequirementsDict(kwargs.get('fpga_driver')) - run_input["systemRequirements"] = (instance_type_srd + cluster_spec_srd + fpga_driver_srd).as_dict() + nvidia_driver_srd = SystemRequirementsDict(kwargs.get('nvidia_driver')) + run_input["systemRequirements"] = (instance_type_srd + cluster_spec_srd + fpga_driver_srd + nvidia_driver_srd).as_dict() if kwargs.get('system_requirements') is not None: run_input["systemRequirements"] = kwargs.get('system_requirements') @@ -195,7 +196,7 @@ def run(self, executable_input, project=None, folder=None, name=None, tags=None, depends_on=None, allow_ssh=None, debug=None, delay_workspace_destruction=None, priority=None, head_job_on_demand=None, ignore_reuse=None, ignore_reuse_stages=None, detach=None, cost_limit=None, rank=None, max_tree_spot_wait_time=None, max_job_spot_wait_time=None, preserve_job_outputs=None, detailed_job_metrics=None, extra_args=None, - fpga_driver=None, system_requirements=None, system_requirements_by_executable=None, **kwargs): + fpga_driver=None, system_requirements=None, system_requirements_by_executable=None, nvidia_driver=None, **kwargs): ''' :param executable_input: Hash of the executable's input arguments :type executable_input: dict @@ -252,6 +253,8 @@ def run(self, executable_input, project=None, folder=None, name=None, tags=None, :type system_requirements: dict :param system_requirements_by_executable: System requirement by executable double mapping :type system_requirements_by_executable: dict + :param nvidia_driver: a dict mapping function names to nvidia driver requests + :type nvidia_driver: dict :rtype: :class:`~dxpy.bindings.dxjob.DXJob` Creates a new job that executes the function "main" of this executable with @@ -292,7 +295,8 @@ def run(self, executable_input, project=None, folder=None, name=None, tags=None, extra_args=extra_args, fpga_driver=fpga_driver, system_requirements=system_requirements, - system_requirements_by_executable=system_requirements_by_executable) + system_requirements_by_executable=system_requirements_by_executable, + nvidia_driver=nvidia_driver) return self._run_impl(run_input, **kwargs) diff --git a/src/python/dxpy/bindings/dxjob.py b/src/python/dxpy/bindings/dxjob.py index f13a78dbc4..bcb09b63c7 100644 --- a/src/python/dxpy/bindings/dxjob.py +++ b/src/python/dxpy/bindings/dxjob.py @@ -38,14 +38,15 @@ from ..utils.local_exec_utils import queue_entry_point from ..compat import basestring + ######### # DXJob # ######### -def new_dxjob(fn_input, fn_name, name=None, tags=None, properties=None, details=None, - instance_type=None, depends_on=None, - cluster_spec=None, fpga_driver=None, system_requirements=None, system_requirements_by_executable=None, - **kwargs): + +def new_dxjob(fn_input, fn_name, name=None, tags=None, properties=None, details=None, instance_type=None, + depends_on=None, cluster_spec=None, fpga_driver=None, system_requirements=None, + system_requirements_by_executable=None, nvidia_driver=None, **kwargs): ''' :param fn_input: Function input :type fn_input: dict @@ -71,6 +72,8 @@ def new_dxjob(fn_input, fn_name, name=None, tags=None, properties=None, details= :type system_requirements: dict :param system_requirements_by_executable: System requirement by executable double mapping :type system_requirements_by_executable: dict + :param nvidia_driver: a dict mapping function names to nvidia driver requests + :type nvidia_driver: dict :rtype: :class:`~dxpy.bindings.dxjob.DXJob` Creates and enqueues a new job that will execute a particular @@ -94,12 +97,13 @@ def new_dxjob(fn_input, fn_name, name=None, tags=None, properties=None, details= ''' dxjob = DXJob() - dxjob.new(fn_input, fn_name, name=name, tags=tags, properties=properties, - details=details, instance_type=instance_type, depends_on=depends_on, - cluster_spec=cluster_spec, fpga_driver=fpga_driver, - system_requirements=system_requirements, system_requirements_by_executable=system_requirements_by_executable, **kwargs) + dxjob.new(fn_input, fn_name, name=name, tags=tags, properties=properties, details=details, + instance_type=instance_type, depends_on=depends_on, cluster_spec=cluster_spec, fpga_driver=fpga_driver, + system_requirements=system_requirements, system_requirements_by_executable=system_requirements_by_executable, + nvidia_driver=nvidia_driver, **kwargs) return dxjob + class DXJob(DXObject): ''' Remote job object handler. @@ -112,10 +116,9 @@ def __init__(self, dxid=None): DXObject.__init__(self, dxid=dxid) self.set_id(dxid) - def new(self, fn_input, fn_name, name=None, tags=None, properties=None, details=None, - instance_type=None, depends_on=None, - cluster_spec=None, fpga_driver=None, system_requirements=None, system_requirements_by_executable=None, - **kwargs): + def new(self, fn_input, fn_name, name=None, tags=None, properties=None, details=None, instance_type=None, + depends_on=None, cluster_spec=None, fpga_driver=None, system_requirements=None, + system_requirements_by_executable=None, nvidia_driver=None, **kwargs): ''' :param fn_input: Function input :type fn_input: dict @@ -141,6 +144,8 @@ def new(self, fn_input, fn_name, name=None, tags=None, properties=None, details= :type system_requirements: dict :param system_requirements_by_executable: System requirement by executable double mapping :type system_requirements_by_executable: dict + :param nvidia_driver: a dict mapping function names to nvidia driver requests + :type nvidia_driver: dict Creates and enqueues a new job that will execute a particular function (from the same app or applet as the one the current job @@ -179,11 +184,12 @@ def new(self, fn_input, fn_name, name=None, tags=None, properties=None, details= req_input["tags"] = tags if properties is not None: req_input["properties"] = properties - if instance_type is not None or cluster_spec is not None or fpga_driver is not None: + if any(requirement is not None for requirement in [instance_type, cluster_spec, fpga_driver, nvidia_driver]): instance_type_srd = SystemRequirementsDict.from_instance_type(instance_type, fn_name) cluster_spec_srd = SystemRequirementsDict(cluster_spec) fpga_driver_srd = SystemRequirementsDict(fpga_driver) - req_input["systemRequirements"] = (instance_type_srd + cluster_spec_srd + fpga_driver_srd).as_dict() + nvidia_driver_srd = SystemRequirementsDict(nvidia_driver) + req_input["systemRequirements"] = (instance_type_srd + cluster_spec_srd + fpga_driver_srd + nvidia_driver_srd).as_dict() if system_requirements is not None: req_input["systemRequirements"] = system_requirements if system_requirements_by_executable is not None: diff --git a/src/python/dxpy/scripts/dx.py b/src/python/dxpy/scripts/dx.py index f63d1a0b9c..ce776b20ea 100644 --- a/src/python/dxpy/scripts/dx.py +++ b/src/python/dxpy/scripts/dx.py @@ -3195,10 +3195,12 @@ def run_body(args, executable, dest_proj, dest_path, preset_inputs=None, input_n cloned_instance_type = SystemRequirementsDict.from_sys_requirements(cloned_system_requirements, _type='instanceType') cloned_cluster_spec = SystemRequirementsDict.from_sys_requirements(cloned_system_requirements, _type='clusterSpec') cloned_fpga_driver = SystemRequirementsDict.from_sys_requirements(cloned_system_requirements, _type='fpgaDriver') + cloned_nvidia_driver = SystemRequirementsDict.from_sys_requirements(cloned_system_requirements, _type='nvidiaDriver') cloned_system_requirements_by_executable = args.cloned_job_desc.get("mergedSystemRequirementsByExecutable", {}) or {} else: cloned_system_requirements = {} - cloned_instance_type, cloned_cluster_spec, cloned_fpga_driver = SystemRequirementsDict({}), SystemRequirementsDict({}), SystemRequirementsDict({}) + cloned_instance_type, cloned_cluster_spec, cloned_fpga_driver, cloned_nvidia_driver = ( + SystemRequirementsDict({}), SystemRequirementsDict({}), SystemRequirementsDict({}), SystemRequirementsDict({})) cloned_system_requirements_by_executable = {} # convert runtime --instance-type into mapping {entrypoint:{'instanceType':xxx}} @@ -3227,12 +3229,15 @@ def run_body(args, executable, dest_proj, dest_path, preset_inputs=None, input_n else: requested_cluster_spec = cloned_cluster_spec - # fpga driver now does not have corresponding dx run option, so it can only be requested using the cloned value + # fpga/nvidia driver now does not have corresponding dx run option, + # so it can only be requested using the cloned value requested_fpga_driver = cloned_fpga_driver + requested_nvidia_driver = cloned_nvidia_driver - # combine the requested instance type, full cluster spec, fpga spec + # combine the requested instance type, full cluster spec, fpga spec, nvidia spec # into the runtime systemRequirements - requested_system_requirements = (requested_instance_type + requested_cluster_spec + requested_fpga_driver).as_dict() + requested_system_requirements = (requested_instance_type + requested_cluster_spec + requested_fpga_driver + + requested_nvidia_driver).as_dict() if (args.instance_type and cloned_system_requirements_by_executable): warning = BOLD("WARNING") + ": --instance-type argument: {} may get overridden by".format(args.instance_type) @@ -3283,6 +3288,7 @@ def run_body(args, executable, dest_proj, dest_path, preset_inputs=None, input_n "instance_type": None, "cluster_spec": None, "fpga_driver": None, + "nvidia_driver": None, "stage_instance_types": args.stage_instance_types, "stage_folders": args.stage_folders, "rerun_stages": args.rerun_stages, diff --git a/src/python/dxpy/system_requirements.py b/src/python/dxpy/system_requirements.py index b25504a2db..d0eb768ac2 100644 --- a/src/python/dxpy/system_requirements.py +++ b/src/python/dxpy/system_requirements.py @@ -85,7 +85,7 @@ def from_sys_requirements(cls, system_requirements, _type='all'): It can extract only entrypoints with specific fields ('clusterSpec', 'instanceType', etc), depending on the value of _type. """ - allowed_types = ['all', 'clusterSpec', 'instanceType', 'fpgaDriver'] + allowed_types = ['all', 'clusterSpec', 'instanceType', 'fpgaDriver', 'nvidiaDriver'] if _type not in (allowed_types): raise DXError("Expected '_type' to be one of the following: {}".format(allowed_types)) diff --git a/src/python/test/test_dx_bash_helpers.py b/src/python/test/test_dx_bash_helpers.py index 494286ce49..4348e1597a 100755 --- a/src/python/test/test_dx_bash_helpers.py +++ b/src/python/test/test_dx_bash_helpers.py @@ -935,6 +935,22 @@ def test_job_arguments(self): "other_function": {"fpgaDriver": "edico-1.4.5"}}}})), {"systemRequirementsByExecutable": {"my_applet":{"main": { "instanceType": "mem2_hdd2_x2", "clusterSpec":{"initialInstanceCount": 3}}, "other_function": { "instanceType": "mem3_ssd2_fpga1_x8", "fpgaDriver": "edico-1.4.5"} }}}), + # nvidia driver + ("--instance-type-by-executable " + + pipes.quote(json.dumps({ + "my_applet": { + "main": "mem1_ssd1_v2_x2", + "other_function": "mem2_ssd1_gpu_x16"}})) + + " --extra-args " + + pipes.quote(json.dumps({ + "systemRequirementsByExecutable": { + "my_applet": { + "main": {"instanceType": "mem2_hdd2_x2"}, + "other_function": {"nvidiaDriver": "R535"}}}})), + {"systemRequirementsByExecutable": { + "my_applet": {"main": {"instanceType": "mem2_hdd2_x2"}, + "other_function": {"instanceType": "mem2_ssd1_gpu_x16", + "nvidiaDriver": "R535"}}}}), # properties - mapping ( "--property foo=foo_value --property bar=bar_value", diff --git a/src/python/test/test_dxclient.py b/src/python/test/test_dxclient.py index 3af69f8b2f..471b7473d1 100755 --- a/src/python/test/test_dxclient.py +++ b/src/python/test/test_dxclient.py @@ -3332,6 +3332,53 @@ def test_dx_run_sys_reqs(self): run("dx run " + applet_id + " --instance-type-by-executable not-a-JSON-string") + def test_dx_run_clone_nvidia_driver(self): + """ + Run the applet and clone the origin job. Verify nvidiaDriver value. + """ + build_nvidia_version = "R535" + run_nvidia_version = "R470" + + applet_id = dxpy.api.applet_new({"project": self.project, + "dxapi": "1.0.0", + "runSpec": {"interpreter": "bash", + "distribution": "Ubuntu", + "release": "20.04", + "version": "0", + "code": "echo 'hello'", + "systemRequirements": { + "*": { + "instanceType": "mem2_hdd2_x1", + "nvidiaDriver": build_nvidia_version + } + }} + })['id'] + + # Run with unchanged nvidia version (build value) + origin_job_id = run(f"dx run {applet_id} --brief -y").strip().split('\n')[-1] + origin_job_desc = dxpy.api.job_describe(origin_job_id) + assert origin_job_desc["systemRequirements"]["*"]["nvidiaDriver"] == build_nvidia_version + + cloned_job_id = run(f"dx run --clone {origin_job_id} --brief -y").strip() + cloned_job_desc = dxpy.api.job_describe(cloned_job_id) + assert cloned_job_desc["systemRequirements"]["*"]["nvidiaDriver"] == build_nvidia_version + + # Change nvidia driver version in runtime - origin job (run value) + extra_args = json.dumps({"systemRequirements": {"*": {"nvidiaDriver": run_nvidia_version}}}) + origin_job_id_nvidia_override = run(f"dx run {applet_id} --extra-args '{extra_args}' --brief -y").strip().split('\n')[-1] + origin_job_desc = dxpy.api.job_describe(origin_job_id_nvidia_override) + assert origin_job_desc["systemRequirements"]["*"]["nvidiaDriver"] == run_nvidia_version + + cloned_job_id_nvidia_override = run(f"dx run --clone {origin_job_id_nvidia_override} --brief -y").strip() + cloned_job_desc = dxpy.api.job_describe(cloned_job_id_nvidia_override) + assert cloned_job_desc["systemRequirements"]["*"]["nvidiaDriver"] == run_nvidia_version + + # Change nvidia driver version in runtime - cloned job (build value) + extra_args = json.dumps({"systemRequirements": {"*": {"nvidiaDriver": build_nvidia_version}}}) + cloned_job_id_nvidia_override = run(f"dx run --clone {origin_job_id_nvidia_override} --extra-args '{extra_args}' --brief -y").strip() + cloned_job_desc = dxpy.api.job_describe(cloned_job_id_nvidia_override) + assert cloned_job_desc["systemRequirements"]["*"]["nvidiaDriver"] == build_nvidia_version + def test_dx_run_clone(self): applet_id = dxpy.api.applet_new({"project": self.project, "dxapi": "1.0.0", @@ -3626,12 +3673,14 @@ def check_instance_count(job_desc , entrypoints , expected_counts): check_new_job_metadata(new_job_desc, orig_job_desc, overridden_fields=['systemRequirements']) - # fpgaDriver override: new original job with extra_args + # fpgaDriver/nvidiaDriver override: new original job with extra_args orig_job_id = run("dx run " + other_applet_id + " --instance-count 2 --brief -y " + "--extra-args '" + - json.dumps({"systemRequirements": {"some_ep": {"clusterSpec": {"initialInstanceCount": 12, "bootstrapScript": "z.sh"}, - "fpgaDriver": "edico-1.4.5"}}}) + "'").strip() + json.dumps({"systemRequirements": {"some_ep": + {"clusterSpec": {"initialInstanceCount": 12, "bootstrapScript": "z.sh"}, + "fpgaDriver": "edico-1.4.5", + "nvidiaDriver": "R535"}}}) + "'").strip() orig_job_desc = dxpy.api.job_describe(orig_job_id) check_instance_count(orig_job_desc, ["main", "some_ep","*"], [2, 12, 2]) # --instance-type and --instance-count override: instance type and cluster spec are resolved independently @@ -3650,6 +3699,7 @@ def check_instance_count(job_desc , entrypoints , expected_counts): self.assertEqual(new_job_desc['systemRequirements']['*']['instanceType'], 'mem2_hdd2_v2_x2') self.assertEqual(new_job_desc['systemRequirements']['some_ep']['fpgaDriver'], 'edico-1.4.5') + self.assertEqual(new_job_desc['systemRequirements']['some_ep']['nvidiaDriver'], 'R535') self.assertEqual(new_job_desc['systemRequirements']['some_ep']['clusterSpec']['bootstrapScript'], 'z.sh') # --instance-type and --instance-type-by-executable override diff --git a/src/python/test/test_dxpy.py b/src/python/test/test_dxpy.py index 1a768e7015..8f31ccd302 100755 --- a/src/python/test/test_dxpy.py +++ b/src/python/test/test_dxpy.py @@ -1319,14 +1319,19 @@ def main(): "interpreter": "python3", "distribution": "Ubuntu", "release": "20.04", "version": "0", - "execDepends": [{"name": "python-numpy"}]}) + "execDepends": [{"name": "python-numpy"}], + "systemRequirements": { + "*": { + "nvidiaDriver": "R535" + } + }}) dxrecord = dxpy.new_dxrecord() dxrecord.close() prog_input = {"chromosomes": {"$dnanexus_link": dxrecord.get_id()}, "rowFetchChunk": 100} dxjob = dxapplet.run(applet_input=prog_input, details={"$dnanexus_link": "hello world"}, tags=['foo', '$foo.bar'], properties={'$dnanexus_link.foo': 'barbaz'}, - priority="normal") + priority="normal", nvidia_driver={"*": {"nvidiaDriver": "R470"}}) jobdesc = dxjob.describe() self.assertEqual(jobdesc["class"], "job") self.assertEqual(jobdesc["function"], "main") @@ -1346,6 +1351,7 @@ def main(): self.assertEqual(len(jobdesc["properties"]), 1) self.assertEqual(jobdesc["properties"]["$dnanexus_link.foo"], "barbaz") self.assertEqual(jobdesc["priority"], "normal") + self.assertEqual(jobdesc["systemRequirements"]["*"]["nvidiaDriver"], "R470") # Test setting tags and properties on job dxjob.add_tags(["foo", "bar", "foo"]) From c23d507dd63fa55c96e3b6583690593b69461162 Mon Sep 17 00:00:00 2001 From: Jan Dvorsky Date: Thu, 17 Oct 2024 17:33:21 +0200 Subject: [PATCH 2/6] DEVEX-2411 Throw an error when runSpec.distribution or runSpec.release is not present (#1406) --- src/python/dxpy/scripts/dx_build_app.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/python/dxpy/scripts/dx_build_app.py b/src/python/dxpy/scripts/dx_build_app.py index f37b809856..7a23c93810 100755 --- a/src/python/dxpy/scripts/dx_build_app.py +++ b/src/python/dxpy/scripts/dx_build_app.py @@ -357,11 +357,11 @@ def _verify_app_source_dir_impl(src_dir, temp_dir, mode, enforce=True): if "interpreter" not in manifest['runSpec']: raise dxpy.app_builder.AppBuilderException('runSpec.interpreter field was not present') - if "release" not in manifest['runSpec'] or "distribution" not in manifest['runSpec']: - warn_message = 'runSpec.distribution or runSpec.release was not present. These fields ' - warn_message += 'will be required in a future version of the API. Recommended value ' - warn_message += 'for distribution is \"Ubuntu\" and release - \"14.04\".' - logger.warn(warn_message) + if "distribution" not in manifest['runSpec']: + raise dxpy.app_builder.AppBuilderException('Required field runSpec.distribution is not present') + + if "release" not in manifest['runSpec']: + raise dxpy.app_builder.AppBuilderException('Required field runSpec.release is not present') if manifest['runSpec']['interpreter'] in ["python2.7", "bash", "python3"]: if "file" in manifest['runSpec']: From b92b58c05f420135644203b851d33d4a2535679c Mon Sep 17 00:00:00 2001 From: Austin Locke Date: Fri, 18 Oct 2024 14:08:01 -0500 Subject: [PATCH 3/6] PTFM-38840 Update dx CLI to allow passing drive for symlink 2.0 projects --- src/python/dxpy/scripts/dx.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/python/dxpy/scripts/dx.py b/src/python/dxpy/scripts/dx.py index ce776b20ea..b2a9fd530c 100644 --- a/src/python/dxpy/scripts/dx.py +++ b/src/python/dxpy/scripts/dx.py @@ -1440,6 +1440,8 @@ def new_project(args): inputs["monthlyStorageLimit"] = args.monthly_storage_limit if args.default_symlink is not None: inputs["defaultSymlink"] = json.loads(args.default_symlink) + if args.drive is not None: + inputs["drive"] = args.drive try: resp = dxpy.api.project_new(inputs) if args.brief: @@ -5814,7 +5816,8 @@ def __call__(self, parser, namespace, values, option_string=None): parser_new_project.add_argument('--monthly-compute-limit', type=positive_integer, help='Monthly project spending limit for compute') parser_new_project.add_argument('--monthly-egress-bytes-limit', type=positive_integer, help='Monthly project spending limit for egress (in Bytes)') parser_new_project.add_argument('--monthly-storage-limit', type=positive_number, help='Monthly project spending limit for storage') -parser_new_project.add_argument('--default-symlink', help='Default symlink for external store account') +parser_new_project.add_argument('--default-symlink', help='Default symlink for external storage account') +parser_new_project.add_argument('--drive', help='Drive for external storage account') parser_new_project.set_defaults(func=new_project) register_parser(parser_new_project, subparsers_action=subparsers_new, categories='fs') From 6239fa275d78957314572e5d4f6e2d4c4f04d24d Mon Sep 17 00:00:00 2001 From: Kurt Jensen Date: Mon, 21 Oct 2024 12:13:21 +0000 Subject: [PATCH 4/6] Version updates for v0.385.0 --- src/R/dxR/DESCRIPTION | 2 +- src/R/dxR/R/dxR-package.R | 2 +- src/R/dxR/man/dxR-package.Rd | 2 +- src/python/dxpy/toolkit_version.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/R/dxR/DESCRIPTION b/src/R/dxR/DESCRIPTION index 9ef22d53b2..b21f9d2cae 100644 --- a/src/R/dxR/DESCRIPTION +++ b/src/R/dxR/DESCRIPTION @@ -1,7 +1,7 @@ Package: dxR Type: Package Title: DNAnexus R Client Library -Version: 0.384.0 +Version: 0.385.0 Author: Katherine Lai Maintainer: Katherine Lai Description: dxR is an R extension containing API wrapper functions for diff --git a/src/R/dxR/R/dxR-package.R b/src/R/dxR/R/dxR-package.R index 8387daaf69..3925b6afaa 100644 --- a/src/R/dxR/R/dxR-package.R +++ b/src/R/dxR/R/dxR-package.R @@ -4,7 +4,7 @@ ##' the new DNAnexus platform. ##' ##' \tabular{ll}{ Package: \tab dxR\cr Type: \tab Package\cr Version: \tab -##' 0.384.0\cr License: \tab Apache License (== 2.0)\cr +##' 0.385.0\cr License: \tab Apache License (== 2.0)\cr ##' } ##' ##' @name dxR-package diff --git a/src/R/dxR/man/dxR-package.Rd b/src/R/dxR/man/dxR-package.Rd index 903f5c33fc..fb3d65c038 100644 --- a/src/R/dxR/man/dxR-package.Rd +++ b/src/R/dxR/man/dxR-package.Rd @@ -9,7 +9,7 @@ } \details{ \tabular{ll}{ Package: \tab dxR\cr Type: \tab Package\cr - Version: \tab 0.384.0\cr License: \tab Apache License (== + Version: \tab 0.385.0\cr License: \tab Apache License (== 2.0)\cr } } \author{ diff --git a/src/python/dxpy/toolkit_version.py b/src/python/dxpy/toolkit_version.py index 2561fb547e..55d93a6e5f 100644 --- a/src/python/dxpy/toolkit_version.py +++ b/src/python/dxpy/toolkit_version.py @@ -1 +1 @@ -version = '0.384.0' +version = '0.385.0' From 39b0847e2481453b5d108b016211a5ab66d6fc9d Mon Sep 17 00:00:00 2001 From: Kurt Jensen Date: Mon, 21 Oct 2024 10:54:03 -0400 Subject: [PATCH 5/6] Update CHANGELOG.md (#1411) --- CHANGELOG.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a4be8aa67..11253bf2d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,18 @@ Categories for each release: Added, Changed, Deprecated, Removed, Fixed, Securit ## Unreleased -## [384.0] - beta +## [385.0] - beta + +### Added + +* `nvidiaDriver` field +* `--drive` parameter for `dx new project` + +### Fixed + +* Throw an error when runSpec.distribution or runSpec.release is not present in `dx build` + +## [384.0] - 2024.10.21 ### Fixed From e788e91d8f6f06592765cfc47b3f815e43a5fbc3 Mon Sep 17 00:00:00 2001 From: Branch Vincent Date: Tue, 22 Oct 2024 12:21:15 -0700 Subject: [PATCH 6/6] DEVEX-2424 Replace pipes for python 3.13 (#1410) --- src/python/dxpy/cli/exec_io.py | 4 +-- src/python/dxpy/utils/exec_utils.py | 4 +-- src/python/dxpy/utils/file_load_utils.py | 8 ++--- src/python/dxpy/utils/local_exec_utils.py | 8 ++--- src/python/test/test_dx_app_wizard.py | 1 - src/python/test/test_dx_bash_helpers.py | 20 +++++------ src/python/test/test_dxclient.py | 44 +++++++++++------------ 7 files changed, 42 insertions(+), 47 deletions(-) diff --git a/src/python/dxpy/cli/exec_io.py b/src/python/dxpy/cli/exec_io.py index 982d60450e..e1d927f016 100644 --- a/src/python/dxpy/cli/exec_io.py +++ b/src/python/dxpy/cli/exec_io.py @@ -22,7 +22,7 @@ # TODO: refactor all dx run helper functions here -import os, sys, json, collections, pipes +import os, sys, json, collections, shlex from ..bindings.dxworkflow import DXWorkflow import dxpy @@ -327,7 +327,7 @@ def format_data_object_reference(item): # TODO: in interactive prompts the quotes here may be a bit # misleading. Perhaps it should be a separate mode to print # "interactive-ready" suggestions. - return fill(header + ' ' + ', '.join([pipes.quote(str(item)) for item in items]), + return fill(header + ' ' + ', '.join([shlex.quote(str(item)) for item in items]), initial_indent=initial_indent, subsequent_indent=subsequent_indent) diff --git a/src/python/dxpy/utils/exec_utils.py b/src/python/dxpy/utils/exec_utils.py index ce0b9f5b71..8d02293b5f 100644 --- a/src/python/dxpy/utils/exec_utils.py +++ b/src/python/dxpy/utils/exec_utils.py @@ -23,7 +23,7 @@ import os, sys, json, re, collections, logging, argparse, string, itertools, subprocess, tempfile from functools import wraps from collections import namedtuple -import pipes +import shlex import dxpy from ..compat import USING_PYTHON2, open, Mapping @@ -435,7 +435,7 @@ def _install_dep_bundle(self, bundle): dxpy.download_dxfile(bundle["id"], bundle["name"], project=dxpy.WORKSPACE_ID) except dxpy.exceptions.ResourceNotFound: dxpy.download_dxfile(bundle["id"], bundle["name"]) - self.run("dx-unpack {}".format(pipes.quote(bundle["name"]))) + self.run("dx-unpack {}".format(shlex.quote(bundle["name"]))) else: self.log('Skipping bundled dependency "{name}" because it does not refer to a file'.format(**bundle)) diff --git a/src/python/dxpy/utils/file_load_utils.py b/src/python/dxpy/utils/file_load_utils.py index 89aed97cfb..6f1566401b 100644 --- a/src/python/dxpy/utils/file_load_utils.py +++ b/src/python/dxpy/utils/file_load_utils.py @@ -83,7 +83,7 @@ from __future__ import print_function, unicode_literals, division, absolute_import import json -import pipes +import shlex import os import fnmatch import sys @@ -401,10 +401,6 @@ def factory(): return file_key_descs, rest_hash -# -# Note: pipes.quote() to be replaced with shlex.quote() in Python 3 -# (see http://docs.python.org/2/library/pipes.html#pipes.quote) -# def gen_bash_vars(job_input_file, job_homedir=None, check_name_collision=True): """ :param job_input_file: path to a JSON file describing the job inputs @@ -427,7 +423,7 @@ def string_of_elem(elem): result = json.dumps(dxpy.dxlink(elem)) else: result = json.dumps(elem) - return pipes.quote(result) + return shlex.quote(result) def string_of_value(val): if isinstance(val, list): diff --git a/src/python/dxpy/utils/local_exec_utils.py b/src/python/dxpy/utils/local_exec_utils.py index 72d7981362..6d1e6b0d9d 100755 --- a/src/python/dxpy/utils/local_exec_utils.py +++ b/src/python/dxpy/utils/local_exec_utils.py @@ -16,7 +16,7 @@ from __future__ import print_function, unicode_literals, division, absolute_import -import os, sys, json, subprocess, pipes +import os, sys, json, subprocess, shlex import collections, datetime import dxpy @@ -351,9 +351,9 @@ def run_one_entry_point(job_id, function, input_hash, run_spec, depends_on, name if [[ $(type -t {function}) == "function" ]]; then {function}; else echo "$0: Global scope execution complete. Not invoking entry point function {function} because it was not found" 1>&2; - fi'''.format(homedir=pipes.quote(job_homedir), - env_path=pipes.quote(os.path.join(job_env['HOME'], 'environment')), - code_path=pipes.quote(environ['DX_TEST_CODE_PATH']), + fi'''.format(homedir=shlex.quote(job_homedir), + env_path=shlex.quote(os.path.join(job_env['HOME'], 'environment')), + code_path=shlex.quote(environ['DX_TEST_CODE_PATH']), function=function) invocation_args = ['bash', '-c', '-e'] + (['-x'] if environ.get('DX_TEST_X_FLAG') else []) + [script] elif run_spec['interpreter'] == 'python2.7': diff --git a/src/python/test/test_dx_app_wizard.py b/src/python/test/test_dx_app_wizard.py index b190e12f99..313378b6ed 100755 --- a/src/python/test/test_dx_app_wizard.py +++ b/src/python/test/test_dx_app_wizard.py @@ -19,7 +19,6 @@ import os, sys, unittest, json, tempfile, subprocess import pexpect -import pipes from dxpy_testutil import DXTestCase, check_output import dxpy_testutil as testutil diff --git a/src/python/test/test_dx_bash_helpers.py b/src/python/test/test_dx_bash_helpers.py index 4348e1597a..5b7557982e 100755 --- a/src/python/test/test_dx_bash_helpers.py +++ b/src/python/test/test_dx_bash_helpers.py @@ -23,7 +23,7 @@ import dxpy_testutil as testutil import json import os -import pipes +import shlex import pytest import shutil import tempfile @@ -42,7 +42,7 @@ def run(command, **kwargs): try: if isinstance(command, list) or isinstance(command, tuple): - print("$ %s" % " ".join(pipes.quote(f) for f in command)) + print("$ %s" % " ".join(shlex.quote(f) for f in command)) output = check_output(command, **kwargs) else: print("$ %s" % (command,)) @@ -919,30 +919,30 @@ def test_job_arguments(self): ), # instance type: mapping ("--instance-type " + - pipes.quote(json.dumps({"main": "mem2_hdd2_x2" , "other_function": "mem2_hdd2_x1" })), + shlex.quote(json.dumps({"main": "mem2_hdd2_x2" , "other_function": "mem2_hdd2_x1" })), {"systemRequirements": {"main": { "instanceType": "mem2_hdd2_x2" }, "other_function": { "instanceType": "mem2_hdd2_x1" }}}), ("--instance-type-by-executable " + - pipes.quote(json.dumps({"my_applet": {"main": "mem2_hdd2_x2", + shlex.quote(json.dumps({"my_applet": {"main": "mem2_hdd2_x2", "other_function": "mem3_ssd2_fpga1_x8"}})), {"systemRequirementsByExecutable": {"my_applet": {"main": {"instanceType": "mem2_hdd2_x2"}, "other_function": {"instanceType": "mem3_ssd2_fpga1_x8"}}}}), ("--instance-type-by-executable " + - pipes.quote(json.dumps({"my_applet": {"main": "mem1_ssd1_v2_x2", + shlex.quote(json.dumps({"my_applet": {"main": "mem1_ssd1_v2_x2", "other_function": "mem3_ssd2_fpga1_x8"}})) + " --extra-args " + - pipes.quote(json.dumps({"systemRequirementsByExecutable": {"my_applet": {"main": {"instanceType": "mem2_hdd2_x2", "clusterSpec": {"initialInstanceCount": 3}}, + shlex.quote(json.dumps({"systemRequirementsByExecutable": {"my_applet": {"main": {"instanceType": "mem2_hdd2_x2", "clusterSpec": {"initialInstanceCount": 3}}, "other_function": {"fpgaDriver": "edico-1.4.5"}}}})), {"systemRequirementsByExecutable": {"my_applet":{"main": { "instanceType": "mem2_hdd2_x2", "clusterSpec":{"initialInstanceCount": 3}}, "other_function": { "instanceType": "mem3_ssd2_fpga1_x8", "fpgaDriver": "edico-1.4.5"} }}}), # nvidia driver ("--instance-type-by-executable " + - pipes.quote(json.dumps({ + shlex.quote(json.dumps({ "my_applet": { "main": "mem1_ssd1_v2_x2", "other_function": "mem2_ssd1_gpu_x16"}})) + " --extra-args " + - pipes.quote(json.dumps({ + shlex.quote(json.dumps({ "systemRequirementsByExecutable": { "my_applet": { "main": {"instanceType": "mem2_hdd2_x2"}, @@ -963,14 +963,14 @@ def test_job_arguments(self): self.assertNewJobInputHash(cmd_snippet, arguments_hash) def test_extra_arguments(self): - cmd_snippet = "--extra-args " + pipes.quote( + cmd_snippet = "--extra-args " + shlex.quote( json.dumps({"details": {"d1": "detail1", "d2": 1234}, "foo": "foo_value"}) ) arguments_hash = {"details": {"d1": "detail1", "d2": 1234}, "foo": "foo_value"} self.assertNewJobInputHash(cmd_snippet, arguments_hash) # override previously specified args - cmd_snippet = "--name JobName --extra-args " + pipes.quote( + cmd_snippet = "--name JobName --extra-args " + shlex.quote( json.dumps({"name": "FinalName"}) ) arguments_hash = {"name": "FinalName"} diff --git a/src/python/test/test_dxclient.py b/src/python/test/test_dxclient.py index 471b7473d1..1592147fe6 100755 --- a/src/python/test/test_dxclient.py +++ b/src/python/test/test_dxclient.py @@ -21,7 +21,7 @@ import os, sys, unittest, json, tempfile, subprocess, shutil, re, base64, random, time import filecmp -import pipes +import shlex import stat import hashlib import collections @@ -380,14 +380,14 @@ def test_dx_set_details_with_file(self): # Test -f with valid JSON file. record_id = run("dx new record Ψ2 --brief").strip() - run("dx set_details Ψ2 -f " + pipes.quote(tmp_file.name)) + run("dx set_details Ψ2 -f " + shlex.quote(tmp_file.name)) dxrecord = dxpy.DXRecord(record_id) details = dxrecord.get_details() self.assertEqual({"foo": "bar"}, details, msg="dx set_details -f with valid JSON input file failed.") # Test --details-file with valid JSON file. record_id = run("dx new record Ψ3 --brief").strip() - run("dx set_details Ψ3 --details-file " + pipes.quote(tmp_file.name)) + run("dx set_details Ψ3 --details-file " + shlex.quote(tmp_file.name)) dxrecord = dxpy.DXRecord(record_id) details = dxrecord.get_details() self.assertEqual({"foo": "bar"}, details, @@ -400,16 +400,16 @@ def test_dx_set_details_with_file(self): # Test above with invalid JSON file. record_id = run("dx new record Ψ4 --brief").strip() with self.assertSubprocessFailure(stderr_regexp="JSON", exit_code=3): - run("dx set_details Ψ4 -f " + pipes.quote(tmp_invalid_file.name)) + run("dx set_details Ψ4 -f " + shlex.quote(tmp_invalid_file.name)) # Test command with (-f or --details-file) and CL JSON. with self.assertSubprocessFailure(stderr_regexp="Error: Cannot provide both -f/--details-file and details", exit_code=3): - run("dx set_details Ψ4 '{ \"foo\":\"bar\" }' -f " + pipes.quote(tmp_file.name)) + run("dx set_details Ψ4 '{ \"foo\":\"bar\" }' -f " + shlex.quote(tmp_file.name)) # Test piping JSON from STDIN. record_id = run("dx new record Ψ5 --brief").strip() - run("cat " + pipes.quote(tmp_file.name) + " | dx set_details Ψ5 -f -") + run("cat " + shlex.quote(tmp_file.name) + " | dx set_details Ψ5 -f -") dxrecord = dxpy.DXRecord(record_id) details = dxrecord.get_details() self.assertEqual({"foo": "bar"}, details, msg="dx set_details -f - with valid JSON input failed.") @@ -5504,11 +5504,11 @@ def test_dx_find_data_by_region(self): def test_dx_find_projects(self): unique_project_name = 'dx find projects test ' + str(time.time()) with temporary_project(unique_project_name) as unique_project: - self.assertEqual(run("dx find projects --name " + pipes.quote(unique_project_name)), + self.assertEqual(run("dx find projects --name " + shlex.quote(unique_project_name)), unique_project.get_id() + ' : ' + unique_project_name + ' (ADMINISTER)\n') - self.assertEqual(run("dx find projects --brief --name " + pipes.quote(unique_project_name)), + self.assertEqual(run("dx find projects --brief --name " + shlex.quote(unique_project_name)), unique_project.get_id() + '\n') - json_output = json.loads(run("dx find projects --json --name " + pipes.quote(unique_project_name))) + json_output = json.loads(run("dx find projects --json --name " + shlex.quote(unique_project_name))) self.assertEqual(len(json_output), 1) self.assertEqual(json_output[0]['id'], unique_project.get_id()) @@ -5529,15 +5529,15 @@ def test_dx_find_projects_by_created(self): created_project_name = 'dx find projects test ' + str(time.time()) with temporary_project(created_project_name) as unique_project: self.assertEqual(run("dx find projects --created-after=-1d --brief --name " + - pipes.quote(created_project_name)), unique_project.get_id() + '\n') + shlex.quote(created_project_name)), unique_project.get_id() + '\n') self.assertEqual(run("dx find projects --created-before=" + str(int(time.time() + 1000) * 1000) + - " --brief --name " + pipes.quote(created_project_name)), + " --brief --name " + shlex.quote(created_project_name)), unique_project.get_id() + '\n') self.assertEqual(run("dx find projects --created-after=-1d --created-before=" + str(int(time.time() + 1000) * 1000) + " --brief --name " + - pipes.quote(created_project_name)), unique_project.get_id() + '\n') + shlex.quote(created_project_name)), unique_project.get_id() + '\n') self.assertEqual(run("dx find projects --created-after=" + str(int(time.time() + 1000) * 1000) + " --name " - + pipes.quote(created_project_name)), "") + + shlex.quote(created_project_name)), "") def test_dx_find_projects_by_region(self): awseast = "aws:us-east-1" @@ -5545,7 +5545,7 @@ def test_dx_find_projects_by_region(self): created_project_name = 'dx find projects test ' + str(time.time()) with temporary_project(created_project_name, region=awseast) as unique_project: self.assertEqual(run("dx find projects --region {} --brief --name {}".format( - awseast, pipes.quote(created_project_name))), + awseast, shlex.quote(created_project_name))), unique_project.get_id() + '\n') self.assertIn(unique_project.get_id(), run("dx find projects --region {} --brief".format(awseast))) @@ -5635,10 +5635,10 @@ def test_dx_find_projects_by_property(self): def test_dx_find_projects_phi(self): projectName = "tempProject+{t}".format(t=time.time()) with temporary_project(name=projectName) as project_1: - res = run('dx find projects --phi true --brief --name ' + pipes.quote(projectName)) + res = run('dx find projects --phi true --brief --name ' + shlex.quote(projectName)) self.assertTrue(len(res) == 0, "Expected no PHI projects to be found") - res = run('dx find projects --phi false --brief --name ' + pipes.quote(projectName)).strip().split('\n') + res = run('dx find projects --phi false --brief --name ' + shlex.quote(projectName)).strip().split('\n') self.assertTrue(len(res) == 1, "Expected to find one project") self.assertTrue(res[0] == project_1.get_id()) @@ -6287,10 +6287,10 @@ def test_dx_find_org_projects_phi(self): project1_id = project_1.get_id() dxpy.api.project_update(project1_id, {"billTo": self.org_id}) - res = run('dx find org projects org-piratelabs --phi true --brief --name ' + pipes.quote(projectName)) + res = run('dx find org projects org-piratelabs --phi true --brief --name ' + shlex.quote(projectName)) self.assertTrue(len(res) == 0, "Expected no PHI projects to be found") - res = run('dx find org projects org-piratelabs --phi false --brief --name ' + pipes.quote(projectName)).strip().split("\n") + res = run('dx find org projects org-piratelabs --phi false --brief --name ' + shlex.quote(projectName)).strip().split("\n") self.assertTrue(len(res) == 1, "Expected to find one project") self.assertEqual(res[0], project1_id) @@ -7431,7 +7431,7 @@ def test_update_strings(self): #Update items one by one. for item in update_items: - run(self.cmd.format(pid=self.project, item=item, n=pipes.quote(update_items[item]))) + run(self.cmd.format(pid=self.project, item=item, n=shlex.quote(update_items[item]))) describe_input = {} describe_input[item] = 'true' self.assertEqual(self.project_describe(describe_input)[item], @@ -7447,14 +7447,14 @@ def test_update_multiple_items(self): 'protected': 'false'} update_project_output = check_output(["dx", "update", "project", self.project, "--name", - pipes.quote(update_items['name']), "--summary", update_items['summary'], "--description", + shlex.quote(update_items['name']), "--summary", update_items['summary'], "--description", update_items['description'], "--protected", update_items['protected']]) update_project_json = json.loads(update_project_output); self.assertTrue("id" in update_project_json) self.assertEqual(self.project, update_project_json["id"]) update_project_output = check_output(["dx", "update", "project", self.project, "--name", - pipes.quote(update_items['name']), "--summary", update_items['summary'], "--description", + shlex.quote(update_items['name']), "--summary", update_items['summary'], "--description", update_items['description'], "--protected", update_items['protected'], "--brief"]) self.assertEqual(self.project, update_project_output.rstrip("\n")) @@ -7479,7 +7479,7 @@ def test_update_project_by_name(self): project_name = self.project_describe(describe_input)['name'] new_name = 'Another Project Name' + str(time.time()) - run(self.cmd.format(pid=project_name, item='name', n=pipes.quote(new_name))) + run(self.cmd.format(pid=project_name, item='name', n=shlex.quote(new_name))) result = self.project_describe(describe_input) self.assertEqual(result['name'], new_name)