Merge branch 'master' into TIP-3804-databaseResultsRestricted

CHANGELOG.md

@@ -6,7 +6,18 @@ Categories for each release: Added, Changed, Deprecated, Removed, Fixed, Securit
 
 ## Unreleased
 
-## [384.0] - beta
+## [385.0] - beta
+
+### Added
+
+* `nvidiaDriver` field 
+* `--drive` parameter for `dx new project`
+
+### Fixed
+
+* Throw an error when runSpec.distribution or runSpec.release is not present in `dx build`
+
+## [384.0] - 2024.10.21
 
 ### Fixed
 

src/R/dxR/DESCRIPTION

@@ -1,7 +1,7 @@
 Package: dxR
 Type: Package
 Title: DNAnexus R Client Library
-Version: 0.384.0
+Version: 0.385.0
 Author: Katherine Lai
 Maintainer: Katherine Lai <klai@dnanexus.com>
 Description: dxR is an R extension containing API wrapper functions for

src/R/dxR/R/dxR-package.R

@@ -4,7 +4,7 @@
 ##' the new DNAnexus platform.
 ##' 
 ##' \tabular{ll}{ Package: \tab dxR\cr Type: \tab Package\cr Version: \tab
-##' 0.384.0\cr License: \tab Apache License (== 2.0)\cr
+##' 0.385.0\cr License: \tab Apache License (== 2.0)\cr
 ##' }
 ##' 
 ##' @name dxR-package

src/R/dxR/man/dxR-package.Rd

@@ -9,7 +9,7 @@
 }
 \details{
   \tabular{ll}{ Package: \tab dxR\cr Type: \tab Package\cr
-  Version: \tab 0.384.0\cr License: \tab Apache License (==
+  Version: \tab 0.385.0\cr License: \tab Apache License (==
   2.0)\cr }
 }
 \author{

src/python/Readme.md

@@ -27,6 +27,14 @@ Example:
 $ _DX_DEBUG=1 dx ls
 ```
 
+### Debugging inside the IDE (PyCharm)
+To be able to debug dx-toolkit (dx commands) directly in the IDE, 'Run/Debug Configurations' needs to be changed.
+1. Go to Run → Edit Configurations...
+2. Add New Configuration (Python)
+3. Change script to module (dxpy.scripts.dx)
+4. To Script parameters field write dx command you want to run (eg 'ls' runs 'dx ls')
+5. Apply and OK (now it is possible to start debugging via main() function in dx.py)
+
 Python coding style
 -------------------
 

src/python/dxpy/bindings/dxapplet.py

@@ -58,11 +58,12 @@ def _get_run_input_common_fields(executable_input, **kwargs):
             if kwargs.get(arg) is not None:
                 run_input[arg] = kwargs[arg]
 
-        if kwargs.get('instance_type') is not None or kwargs.get('cluster_spec') is not None or kwargs.get('fpga_driver') is not None:
+        if any(kwargs.get(key) is not None for key in ['instance_type', 'cluster_spec', 'fpga_driver', 'nvidia_driver']):
             instance_type_srd = SystemRequirementsDict.from_instance_type(kwargs.get('instance_type'))
             cluster_spec_srd = SystemRequirementsDict(kwargs.get('cluster_spec'))
             fpga_driver_srd = SystemRequirementsDict(kwargs.get('fpga_driver'))
-            run_input["systemRequirements"] = (instance_type_srd + cluster_spec_srd + fpga_driver_srd).as_dict()
+            nvidia_driver_srd = SystemRequirementsDict(kwargs.get('nvidia_driver'))
+            run_input["systemRequirements"] = (instance_type_srd + cluster_spec_srd + fpga_driver_srd + nvidia_driver_srd).as_dict()
 
         if kwargs.get('system_requirements') is not None:
             run_input["systemRequirements"] = kwargs.get('system_requirements')
@@ -195,7 +196,7 @@ def run(self, executable_input, project=None, folder=None, name=None, tags=None,
             depends_on=None, allow_ssh=None, debug=None, delay_workspace_destruction=None, priority=None, head_job_on_demand=None,
             ignore_reuse=None, ignore_reuse_stages=None, detach=None, cost_limit=None, rank=None, max_tree_spot_wait_time=None,
             max_job_spot_wait_time=None, preserve_job_outputs=None, detailed_job_metrics=None, extra_args=None,
-            fpga_driver=None, system_requirements=None, system_requirements_by_executable=None, **kwargs):
+            fpga_driver=None, system_requirements=None, system_requirements_by_executable=None, nvidia_driver=None, **kwargs):
         '''
         :param executable_input: Hash of the executable's input arguments
         :type executable_input: dict
@@ -252,6 +253,8 @@ def run(self, executable_input, project=None, folder=None, name=None, tags=None,
         :type system_requirements: dict
         :param system_requirements_by_executable: System requirement by executable double mapping
         :type system_requirements_by_executable: dict
+        :param nvidia_driver: a dict mapping function names to nvidia driver requests
+        :type nvidia_driver: dict
         :rtype: :class:`~dxpy.bindings.dxjob.DXJob`
 
         Creates a new job that executes the function "main" of this executable with
@@ -292,7 +295,8 @@ def run(self, executable_input, project=None, folder=None, name=None, tags=None,
                                         extra_args=extra_args,
                                         fpga_driver=fpga_driver,
                                         system_requirements=system_requirements,
-                                        system_requirements_by_executable=system_requirements_by_executable)
+                                        system_requirements_by_executable=system_requirements_by_executable,
+                                        nvidia_driver=nvidia_driver)
         return self._run_impl(run_input, **kwargs)
 
 

src/python/dxpy/bindings/dxjob.py

@@ -38,14 +38,15 @@
 from ..utils.local_exec_utils import queue_entry_point
 from ..compat import basestring
 
+
 #########
 # DXJob #
 #########
 
-def new_dxjob(fn_input, fn_name, name=None, tags=None, properties=None, details=None,
-              instance_type=None, depends_on=None,
-              cluster_spec=None, fpga_driver=None, system_requirements=None, system_requirements_by_executable=None,
-              **kwargs):
+
+def new_dxjob(fn_input, fn_name, name=None, tags=None, properties=None, details=None, instance_type=None,
+              depends_on=None, cluster_spec=None, fpga_driver=None, system_requirements=None,
+              system_requirements_by_executable=None, nvidia_driver=None, **kwargs):
     '''
     :param fn_input: Function input
     :type fn_input: dict
@@ -71,6 +72,8 @@ def new_dxjob(fn_input, fn_name, name=None, tags=None, properties=None, details=
     :type system_requirements: dict
     :param system_requirements_by_executable: System requirement by executable double mapping
     :type system_requirements_by_executable: dict
+    :param nvidia_driver: a dict mapping function names to nvidia driver requests
+    :type nvidia_driver: dict
     :rtype: :class:`~dxpy.bindings.dxjob.DXJob`
 
     Creates and enqueues a new job that will execute a particular
@@ -94,12 +97,13 @@ def new_dxjob(fn_input, fn_name, name=None, tags=None, properties=None, details=
 
     '''
     dxjob = DXJob()
-    dxjob.new(fn_input, fn_name, name=name, tags=tags, properties=properties,
-              details=details, instance_type=instance_type, depends_on=depends_on,
-              cluster_spec=cluster_spec, fpga_driver=fpga_driver, 
-              system_requirements=system_requirements, system_requirements_by_executable=system_requirements_by_executable, **kwargs)
+    dxjob.new(fn_input, fn_name, name=name, tags=tags, properties=properties, details=details,
+              instance_type=instance_type, depends_on=depends_on, cluster_spec=cluster_spec, fpga_driver=fpga_driver,
+              system_requirements=system_requirements, system_requirements_by_executable=system_requirements_by_executable,
+              nvidia_driver=nvidia_driver, **kwargs)
     return dxjob
 
+
 class DXJob(DXObject):
     '''
     Remote job object handler.
@@ -112,10 +116,9 @@ def __init__(self, dxid=None):
         DXObject.__init__(self, dxid=dxid)
         self.set_id(dxid)
 
-    def new(self, fn_input, fn_name, name=None, tags=None, properties=None, details=None,
-            instance_type=None, depends_on=None,
-            cluster_spec=None, fpga_driver=None, system_requirements=None, system_requirements_by_executable=None, 
-            **kwargs):
+    def new(self, fn_input, fn_name, name=None, tags=None, properties=None, details=None, instance_type=None,
+            depends_on=None, cluster_spec=None, fpga_driver=None, system_requirements=None,
+            system_requirements_by_executable=None, nvidia_driver=None, **kwargs):
         '''
         :param fn_input: Function input
         :type fn_input: dict
@@ -141,6 +144,8 @@ def new(self, fn_input, fn_name, name=None, tags=None, properties=None, details=
         :type system_requirements: dict
         :param system_requirements_by_executable: System requirement by executable double mapping
         :type system_requirements_by_executable: dict
+        :param nvidia_driver: a dict mapping function names to nvidia driver requests
+        :type nvidia_driver: dict
 
         Creates and enqueues a new job that will execute a particular
         function (from the same app or applet as the one the current job
@@ -179,11 +184,12 @@ def new(self, fn_input, fn_name, name=None, tags=None, properties=None, details=
                 req_input["tags"] = tags
             if properties is not None:
                 req_input["properties"] = properties
-            if instance_type is not None or cluster_spec is not None or fpga_driver is not None:
+            if any(requirement is not None for requirement in [instance_type, cluster_spec, fpga_driver, nvidia_driver]):
                 instance_type_srd = SystemRequirementsDict.from_instance_type(instance_type, fn_name)
                 cluster_spec_srd = SystemRequirementsDict(cluster_spec)
                 fpga_driver_srd = SystemRequirementsDict(fpga_driver)
-                req_input["systemRequirements"] = (instance_type_srd + cluster_spec_srd + fpga_driver_srd).as_dict()
+                nvidia_driver_srd = SystemRequirementsDict(nvidia_driver)
+                req_input["systemRequirements"] = (instance_type_srd + cluster_spec_srd + fpga_driver_srd + nvidia_driver_srd).as_dict()
             if system_requirements is not None:
                 req_input["systemRequirements"] = system_requirements
             if system_requirements_by_executable is not None:

src/python/dxpy/cli/exec_io.py

@@ -22,7 +22,7 @@
 
 # TODO: refactor all dx run helper functions here
 
-import os, sys, json, collections, pipes
+import os, sys, json, collections, shlex
 from ..bindings.dxworkflow import DXWorkflow
 
 import dxpy
@@ -327,7 +327,7 @@ def format_data_object_reference(item):
         # TODO: in interactive prompts the quotes here may be a bit
         # misleading. Perhaps it should be a separate mode to print
         # "interactive-ready" suggestions.
-        return fill(header + ' ' + ', '.join([pipes.quote(str(item)) for item in items]),
+        return fill(header + ' ' + ', '.join([shlex.quote(str(item)) for item in items]),
                     initial_indent=initial_indent,
                     subsequent_indent=subsequent_indent)
 

src/python/dxpy/scripts/dx.py

@@ -1442,6 +1442,8 @@ def new_project(args):
         inputs["monthlyStorageLimit"] = args.monthly_storage_limit
     if args.default_symlink is not None:
         inputs["defaultSymlink"] = json.loads(args.default_symlink)
+    if args.drive is not None:
+        inputs["drive"] = args.drive
     try:
         resp = dxpy.api.project_new(inputs)
         if args.brief:
@@ -3197,10 +3199,12 @@ def run_body(args, executable, dest_proj, dest_path, preset_inputs=None, input_n
         cloned_instance_type = SystemRequirementsDict.from_sys_requirements(cloned_system_requirements, _type='instanceType')
         cloned_cluster_spec = SystemRequirementsDict.from_sys_requirements(cloned_system_requirements, _type='clusterSpec')
         cloned_fpga_driver = SystemRequirementsDict.from_sys_requirements(cloned_system_requirements, _type='fpgaDriver')
+        cloned_nvidia_driver = SystemRequirementsDict.from_sys_requirements(cloned_system_requirements, _type='nvidiaDriver')
         cloned_system_requirements_by_executable = args.cloned_job_desc.get("mergedSystemRequirementsByExecutable", {}) or {}
     else:
         cloned_system_requirements = {}
-        cloned_instance_type, cloned_cluster_spec, cloned_fpga_driver = SystemRequirementsDict({}), SystemRequirementsDict({}), SystemRequirementsDict({})
+        cloned_instance_type, cloned_cluster_spec, cloned_fpga_driver, cloned_nvidia_driver = (
+            SystemRequirementsDict({}), SystemRequirementsDict({}), SystemRequirementsDict({}), SystemRequirementsDict({}))
         cloned_system_requirements_by_executable = {}
 
     # convert runtime --instance-type into mapping {entrypoint:{'instanceType':xxx}}
@@ -3229,12 +3233,15 @@ def run_body(args, executable, dest_proj, dest_path, preset_inputs=None, input_n
     else:
         requested_cluster_spec = cloned_cluster_spec
 
-    # fpga driver now does not have corresponding dx run option, so it can only be requested using the cloned value
+    # fpga/nvidia driver now does not have corresponding dx run option,
+    # so it can only be requested using the cloned value
     requested_fpga_driver = cloned_fpga_driver
+    requested_nvidia_driver = cloned_nvidia_driver
 
-    # combine the requested instance type, full cluster spec, fpga spec
+    # combine the requested instance type, full cluster spec, fpga spec, nvidia spec
     # into the runtime systemRequirements
-    requested_system_requirements = (requested_instance_type + requested_cluster_spec + requested_fpga_driver).as_dict()
+    requested_system_requirements = (requested_instance_type + requested_cluster_spec + requested_fpga_driver +
+                                     requested_nvidia_driver).as_dict()
 
     if (args.instance_type and cloned_system_requirements_by_executable):
         warning = BOLD("WARNING") + ": --instance-type argument: {} may get overridden by".format(args.instance_type) 
@@ -3285,6 +3292,7 @@ def run_body(args, executable, dest_proj, dest_path, preset_inputs=None, input_n
         "instance_type": None,
         "cluster_spec": None,
         "fpga_driver": None,
+        "nvidia_driver": None,
         "stage_instance_types": args.stage_instance_types,
         "stage_folders": args.stage_folders,
         "rerun_stages": args.rerun_stages,
@@ -5814,7 +5822,8 @@ def __call__(self, parser, namespace, values, option_string=None):
 parser_new_project.add_argument('--monthly-compute-limit', type=positive_integer, help='Monthly project spending limit for compute')
 parser_new_project.add_argument('--monthly-egress-bytes-limit', type=positive_integer, help='Monthly project spending limit for egress (in Bytes)')
 parser_new_project.add_argument('--monthly-storage-limit', type=positive_number, help='Monthly project spending limit for storage')
-parser_new_project.add_argument('--default-symlink', help='Default symlink for external store account')
+parser_new_project.add_argument('--default-symlink', help='Default symlink for external storage account')
+parser_new_project.add_argument('--drive', help='Drive for external storage account')
 parser_new_project.set_defaults(func=new_project)
 register_parser(parser_new_project, subparsers_action=subparsers_new, categories='fs')
 

src/python/dxpy/scripts/dx_build_app.py

@@ -357,11 +357,11 @@ def _verify_app_source_dir_impl(src_dir, temp_dir, mode, enforce=True):
         if "interpreter" not in manifest['runSpec']:
             raise dxpy.app_builder.AppBuilderException('runSpec.interpreter field was not present')
 
-        if "release" not in manifest['runSpec'] or "distribution" not in manifest['runSpec']:
-            warn_message = 'runSpec.distribution or runSpec.release was not present. These fields '
-            warn_message += 'will be required in a future version of the API. Recommended value '
-            warn_message += 'for distribution is \"Ubuntu\" and release - \"14.04\".'
-            logger.warn(warn_message)
+        if "distribution" not in manifest['runSpec']:
+            raise dxpy.app_builder.AppBuilderException('Required field runSpec.distribution is not present')
+
+        if "release" not in manifest['runSpec']:
+            raise dxpy.app_builder.AppBuilderException('Required field runSpec.release is not present')
 
         if manifest['runSpec']['interpreter'] in ["python2.7", "bash", "python3"]:
             if "file" in manifest['runSpec']:

src/python/dxpy/system_requirements.py

@@ -85,7 +85,7 @@ def from_sys_requirements(cls, system_requirements, _type='all'):
         It can extract only entrypoints with specific fields ('clusterSpec',
         'instanceType', etc), depending on the value of _type.
         """
-        allowed_types = ['all', 'clusterSpec', 'instanceType', 'fpgaDriver']
+        allowed_types = ['all', 'clusterSpec', 'instanceType', 'fpgaDriver', 'nvidiaDriver']
         if _type not in (allowed_types):
             raise DXError("Expected '_type' to be one of the following: {}".format(allowed_types))
 

src/python/dxpy/toolkit_version.py

@@ -1 +1 @@
-version = '0.384.0'
+version = '0.385.0'

src/python/dxpy/utils/exec_utils.py

@@ -23,7 +23,7 @@
 import os, sys, json, re, collections, logging, argparse, string, itertools, subprocess, tempfile
 from functools import wraps
 from collections import namedtuple
-import pipes
+import shlex
 
 import dxpy
 from ..compat import USING_PYTHON2, open, Mapping
@@ -435,7 +435,7 @@ def _install_dep_bundle(self, bundle):
                 dxpy.download_dxfile(bundle["id"], bundle["name"], project=dxpy.WORKSPACE_ID)
             except dxpy.exceptions.ResourceNotFound:
                 dxpy.download_dxfile(bundle["id"], bundle["name"])
-            self.run("dx-unpack {}".format(pipes.quote(bundle["name"])))
+            self.run("dx-unpack {}".format(shlex.quote(bundle["name"])))
         else:
             self.log('Skipping bundled dependency "{name}" because it does not refer to a file'.format(**bundle))
 

src/python/dxpy/utils/file_load_utils.py

@@ -83,7 +83,7 @@
 from __future__ import print_function, unicode_literals, division, absolute_import
 
 import json
-import pipes
+import shlex
 import os
 import fnmatch
 import sys
@@ -401,10 +401,6 @@ def factory():
     return file_key_descs, rest_hash
 
 
-#
-# Note: pipes.quote() to be replaced with shlex.quote() in Python 3
-# (see http://docs.python.org/2/library/pipes.html#pipes.quote)
-#
 def gen_bash_vars(job_input_file, job_homedir=None, check_name_collision=True):
     """
     :param job_input_file: path to a JSON file describing the job inputs
@@ -427,7 +423,7 @@ def string_of_elem(elem):
             result = json.dumps(dxpy.dxlink(elem))
         else:
             result = json.dumps(elem)
-        return pipes.quote(result)
+        return shlex.quote(result)
 
     def string_of_value(val):
         if isinstance(val, list):

src/python/dxpy/utils/local_exec_utils.py

@@ -16,7 +16,7 @@
 
 from __future__ import print_function, unicode_literals, division, absolute_import
 
-import os, sys, json, subprocess, pipes
+import os, sys, json, subprocess, shlex
 import collections, datetime
 
 import dxpy
@@ -351,9 +351,9 @@ def run_one_entry_point(job_id, function, input_hash, run_spec, depends_on, name
           if [[ $(type -t {function}) == "function" ]];
           then {function};
           else echo "$0: Global scope execution complete. Not invoking entry point function {function} because it was not found" 1>&2;
-          fi'''.format(homedir=pipes.quote(job_homedir),
-                       env_path=pipes.quote(os.path.join(job_env['HOME'], 'environment')),
-                       code_path=pipes.quote(environ['DX_TEST_CODE_PATH']),
+          fi'''.format(homedir=shlex.quote(job_homedir),
+                       env_path=shlex.quote(os.path.join(job_env['HOME'], 'environment')),
+                       code_path=shlex.quote(environ['DX_TEST_CODE_PATH']),
                        function=function)
         invocation_args = ['bash', '-c', '-e'] + (['-x'] if environ.get('DX_TEST_X_FLAG') else []) + [script]
     elif run_spec['interpreter'] == 'python2.7':

src/python/test/test_dx_app_wizard.py

@@ -19,7 +19,6 @@
 
 import os, sys, unittest, json, tempfile, subprocess
 import pexpect
-import pipes
 
 from dxpy_testutil import DXTestCase, check_output
 import dxpy_testutil as testutil