From eea5fb1a53ea6fb4e0111bdb40bee5379e1340f2 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 24 Aug 2023 13:44:11 -0400 Subject: [PATCH 01/13] Fix mapping parent workflow formal inputs to subworkflow informal replacement parameters. --- lib/galaxy/managers/workflows.py | 2 +- lib/galaxy/workflow/modules.py | 31 +++++++++++++++------ lib/galaxy/workflow/run.py | 12 +++++++++ lib/galaxy_test/api/test_workflows.py | 39 +++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/managers/workflows.py b/lib/galaxy/managers/workflows.py index a268c4ce73c4..4d84e834cd85 100644 --- a/lib/galaxy/managers/workflows.py +++ b/lib/galaxy/managers/workflows.py @@ -948,7 +948,7 @@ def _workflow_to_dict_run(self, trans, stored, workflow, history=None): inputs = step.module.get_runtime_inputs(connections=step.output_connections) step_model = {"inputs": [input.to_dict(trans) for input in inputs.values()]} step_model["when"] = step.when_expression - step_model["replacement_parameters"] = step.module.get_replacement_parameters(step) + step_model["replacement_parameters"] = step.module.get_informal_replacement_parameters(step) step_model["step_type"] = step.type step_model["step_label"] = step.label step_model["step_name"] = step.module.get_name() diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index c373d148da9f..02b6540e5da5 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -497,8 +497,11 @@ def recover_mapping(self, invocation_step, progress): progress.set_step_outputs(invocation_step, outputs, already_persisted=True) - def get_replacement_parameters(self, step): - """Return a list of replacement parameters.""" + def get_informal_replacement_parameters(self, step) -> List[str]: + """Return a list of informal replacement parameters. + + If replacement is handled via formal workflow inputs - do not include it in this list. + """ return [] @@ -832,14 +835,22 @@ def callback(input, prefixed_name, prefixed_label, value=None, **kwds): return inputs - def get_replacement_parameters(self, step): + def get_informal_replacement_parameters(self, step) -> List[str]: """Return a list of replacement parameters.""" replacement_parameters = set() + + formal_parameters = set() + for step in self.subworkflow.input_steps: + if step.label: + formal_parameters.add(step.label) + for subworkflow_step in self.subworkflow.steps: module = subworkflow_step.module assert module - for replacement_parameter in module.get_replacement_parameters(subworkflow_step): - replacement_parameters.add(replacement_parameter) + + for replacement_parameter in module.get_informal_replacement_parameters(subworkflow_step): + if replacement_parameter not in formal_parameters: + replacement_parameters.add(replacement_parameter) return list(replacement_parameters) @@ -2256,7 +2267,9 @@ def callback(input, prefixed_name, **kwargs): invocation_step=invocation_step, max_num_jobs=max_num_jobs, validate_outputs=validate_outputs, - job_callback=lambda job: self._handle_post_job_actions(step, job, progress.replacement_dict), + job_callback=lambda job: self._handle_post_job_actions( + step, job, progress.effective_replacement_dict() + ), completed_jobs=completed_jobs, workflow_resource_parameters=resource_parameters, ) @@ -2280,7 +2293,9 @@ def callback(input, prefixed_name, **kwargs): step_inputs = mapping_params.param_template step_inputs.update(collection_info.collections) - self._handle_mapped_over_post_job_actions(step, step_inputs, step_outputs, progress.replacement_dict) + self._handle_mapped_over_post_job_actions( + step, step_inputs, step_outputs, progress.effective_replacement_dict() + ) if execution_tracker.execution_errors: # TODO: formalize into InvocationFailure ? message = f"Failed to create {len(execution_tracker.execution_errors)} job(s) for workflow step {step.order_index + 1}: {str(execution_tracker.execution_errors[0])}" @@ -2342,7 +2357,7 @@ def __to_pja(self, key, value, step): action_arguments = None return PostJobAction(value["action_type"], step, output_name, action_arguments) - def get_replacement_parameters(self, step): + def get_informal_replacement_parameters(self, step) -> List[str]: """Return a list of replacement parameters.""" replacement_parameters = set() for pja in step.post_job_actions: diff --git a/lib/galaxy/workflow/run.py b/lib/galaxy/workflow/run.py index cf82b2a2c710..6ddc56427672 100644 --- a/lib/galaxy/workflow/run.py +++ b/lib/galaxy/workflow/run.py @@ -353,6 +353,7 @@ def __init__( self.copy_inputs_to_history = copy_inputs_to_history self.use_cached_job = use_cached_job self.replacement_dict = replacement_dict or {} + self.runtime_replacements: Dict[str, str] = {} self.subworkflow_collection_info = subworkflow_collection_info self.subworkflow_structure = subworkflow_collection_info.structure if subworkflow_collection_info else None self.when_values = when_values @@ -547,8 +548,19 @@ def set_outputs_for_input( elif step_id in self.inputs_by_step_id: outputs["output"] = self.inputs_by_step_id[step_id] + if step.label and step.type == "parameter_input" and "output" in outputs: + self.runtime_replacements[step.label] = str(outputs["output"]) self.set_step_outputs(invocation_step, outputs, already_persisted=already_persisted) + def effective_replacement_dict(self): + replacement_dict = {} + for key, value in self.replacement_dict.items(): + replacement_dict[key] = value + for key, value in self.runtime_replacements.items(): + if key not in replacement_dict: + replacement_dict[key] = value + return replacement_dict + def set_step_outputs( self, invocation_step: WorkflowInvocationStep, outputs: Dict[str, Any], already_persisted: bool = False ) -> None: diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py index 7cc9849dc947..a5e42bc562e4 100644 --- a/lib/galaxy_test/api/test_workflows.py +++ b/lib/galaxy_test/api/test_workflows.py @@ -2474,6 +2474,45 @@ def test_placements_from_text_inputs(self): details = self.dataset_populator.get_history_dataset_details(history_id) assert details["name"] == "moocow name 2", details["name"] + def test_placements_from_text_inputs_nested(self): + with self.dataset_populator.test_history() as history_id: + run_def = """ +class: GalaxyWorkflow +inputs: + replacemeouter: text +steps: + nested_workflow: + run: + class: GalaxyWorkflow + inputs: + replacemeinner: text + outputs: + workflow_output_1: + outputSource: create_2/out_file1 + workflow_output_2: + outputSource: create_2/out_file2 + steps: + create_2: + tool_id: create_2 + state: + sleep_time: 0 + outputs: + out_file1: + rename: "${replacemeinner} name" + out_file2: + rename: "${replacemeinner} name 2" + in: + replacemeinner: replacemeouter + +test_data: + replacemeouter: + value: moocow + type: raw +""" + self._run_jobs(run_def, history_id=history_id) + details = self.dataset_populator.get_history_dataset_details(history_id) + assert details["name"] == "moocow name 2", details["name"] + @skip_without_tool("random_lines1") def test_run_runtime_parameters_after_pause(self): with self.dataset_populator.test_history() as history_id: From f39f4b982c21edc26f0aac7dfa7eaabc4b0cc57c Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 25 Aug 2023 11:00:06 +0200 Subject: [PATCH 02/13] make sure that TMP, TEMP, and TMPDIR are set seems needed for docker volume strings (an empty variable would lead to `-v "::rw"` which causes `docker: invalid spec: ::rw: empty section between colons.`) --- .../job_script/DEFAULT_JOB_FILE_TEMPLATE.sh | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh index 2d2c9c71763b..6cb42eba6d92 100644 --- a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh +++ b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh @@ -39,6 +39,29 @@ GALAXY_LIB="$galaxy_lib" _galaxy_setup_environment "$PRESERVE_GALAXY_ENVIRONMENT" export _GALAXY_JOB_HOME_DIR export _GALAXY_JOB_TMP_DIR + +TEMP="${TEMP:-$TMP}" +TMPDIR="${TMPDIR:-$TMP}" + +TMP="${TMP:-$TEMP}" +TMPDIR="${TMPDIR:-$TEMP}" + +TMP="${TMP:-$TMPDIR}" +TEMP="${TEMP:-$TMPDIR}" + +TMP="${TMP:-_GALAXY_JOB_TMP_DIR}" +TEMP="${TEMP:-_GALAXY_JOB_TMP_DIR}" +TMPDIR="${TMPDIR:-_GALAXY_JOB_TMP_DIR}" + +export TMP +export TEMP +export TMPDIR + +echo "_GALAXY_JOB_TMP_DIR $_GALAXY_JOB_TMP_DIR" >> outputs/tmp +echo "TMP $TMP" >> outputs/tmp +echo "TEMP $TEMP" >> outputs/tmp +echo "TMPDIR $TMPDIR" >> outputs/tmp + GALAXY_PYTHON=`command -v python` cd $working_directory $memory_statement From 34c781e604358e26a17a338333f8f2a26cc726ac Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 25 Aug 2023 16:02:22 +0200 Subject: [PATCH 03/13] Fix _GALAXY_JOB_TMP_DIR Co-authored-by: Marius van den Beek --- .../runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh index 6cb42eba6d92..a419dc0f8111 100644 --- a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh +++ b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh @@ -49,9 +49,9 @@ TMPDIR="${TMPDIR:-$TEMP}" TMP="${TMP:-$TMPDIR}" TEMP="${TEMP:-$TMPDIR}" -TMP="${TMP:-_GALAXY_JOB_TMP_DIR}" -TEMP="${TEMP:-_GALAXY_JOB_TMP_DIR}" -TMPDIR="${TMPDIR:-_GALAXY_JOB_TMP_DIR}" +TMP="${TMP:-$_GALAXY_JOB_TMP_DIR}" +TEMP="${TEMP:-$_GALAXY_JOB_TMP_DIR}" +TMPDIR="${TMPDIR:-$_GALAXY_JOB_TMP_DIR}" export TMP export TEMP From b1ffd47ab8f07eafa392a04584ba7cfab84a95c6 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 25 Aug 2023 16:03:11 +0200 Subject: [PATCH 04/13] remove debug code --- .../runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh index a419dc0f8111..b20cb5f4c61a 100644 --- a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh +++ b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh @@ -57,11 +57,6 @@ export TMP export TEMP export TMPDIR -echo "_GALAXY_JOB_TMP_DIR $_GALAXY_JOB_TMP_DIR" >> outputs/tmp -echo "TMP $TMP" >> outputs/tmp -echo "TEMP $TEMP" >> outputs/tmp -echo "TMPDIR $TMPDIR" >> outputs/tmp - GALAXY_PYTHON=`command -v python` cd $working_directory $memory_statement From 1a9a402f2e42d767c00478617f8e44bed18bd927 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 25 Aug 2023 16:52:56 +0200 Subject: [PATCH 05/13] Bump minimum tpv version to 2.3.2 This version fixes the version parsing for tools that don't use PEP 440 versions (https://github.com/galaxyproject/total-perspective-vortex/issues/111) --- lib/galaxy/dependencies/conditional-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index 17b16118f5a8..0806672bfc16 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -13,7 +13,7 @@ ldap3==2.9.1 python-pam galaxycloudrunner pkce -total-perspective-vortex>=2.2.4,<3 +total-perspective-vortex>=2.3.2,<3 # For file sources plugins fs.webdavfs>=0.4.2 # type: webdav From 3c4e1195f7d4572cc12183b86c0fb1bc8740492c Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 25 Aug 2023 15:24:33 +0200 Subject: [PATCH 06/13] Implement downloading directories from S3 --- lib/galaxy/objectstore/s3.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/objectstore/s3.py b/lib/galaxy/objectstore/s3.py index 6fa1b6e7e4ff..a719ba3ec191 100644 --- a/lib/galaxy/objectstore/s3.py +++ b/lib/galaxy/objectstore/s3.py @@ -47,6 +47,21 @@ logging.getLogger("boto").setLevel(logging.INFO) # Otherwise boto is quite noisy +def download_directory(bucket, remote_folder, local_path): + # List objects in the specified S3 folder + objects = bucket.list(prefix=remote_folder) + + for obj in objects: + remote_file_path = obj.key + local_file_path = os.path.join(local_path, os.path.relpath(remote_file_path, remote_folder)) + + # Create directories if they don't exist + os.makedirs(os.path.dirname(local_file_path), exist_ok=True) + + # Download the file + obj.get_contents_to_filename(local_file_path) + + def parse_config_xml(config_xml): try: a_xml = config_xml.findall("auth")[0] @@ -720,7 +735,8 @@ def _get_filename(self, obj, **kwargs): return cache_path # Check if the file exists in persistent storage and, if it does, pull it into cache elif self._exists(obj, **kwargs): - if dir_only: # Directories do not get pulled into cache + if dir_only: + download_directory(self._bucket, rel_path, cache_path) return cache_path else: if self._pull_into_cache(rel_path): From 27991b950d51d9101e0455c3c53ad94977f64762 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 25 Aug 2023 15:25:06 +0200 Subject: [PATCH 07/13] Fix writing data bundle manifest to object store --- lib/galaxy/tool_util/data/__init__.py | 6 +++++- lib/galaxy/tools/__init__.py | 12 +++++++++++- lib/galaxy/tools/data_manager/manager.py | 4 ++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/tool_util/data/__init__.py b/lib/galaxy/tool_util/data/__init__.py index ea2e1a77c474..5e2fdfb7908b 100644 --- a/lib/galaxy/tool_util/data/__init__.py +++ b/lib/galaxy/tool_util/data/__init__.py @@ -1174,8 +1174,10 @@ def write_bundle( out_data: Dict[str, OutputDataset], bundle_description: DataTableBundleProcessorDescription, repo_info: Optional[RepoInfo], - ) -> None: + ) -> Dict[str, OutputDataset]: + """Writes bundle and returns bundle path.""" data_manager_dict = _data_manager_dict(out_data, ensure_single_output=True) + bundle_datasets: Dict[str, OutputDataset] = {} for output_name, dataset in out_data.items(): if dataset.ext != "data_manager_json": continue @@ -1190,6 +1192,8 @@ def write_bundle( bundle_path = os.path.join(extra_files_path, BUNDLE_INDEX_FILE_NAME) with open(bundle_path, "w") as fw: json.dump(bundle.dict(), fw) + bundle_datasets[bundle_path] = dataset + return bundle_datasets SUPPORTED_DATA_TABLE_TYPES = TabularToolDataTable diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index a8ccec0b37ad..58ff950c7c4f 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -3064,7 +3064,17 @@ def exec_after_process(self, app, inp_data, out_data, param_dict, job=None, fina elif data_manager_mode == "dry_run": pass elif data_manager_mode == "bundle": - data_manager.write_bundle(out_data) + for bundle_path, dataset in data_manager.write_bundle(out_data).items(): + dataset = cast(model.HistoryDatasetAssociation, dataset) + dataset.dataset.object_store.update_from_file( + dataset.dataset, + extra_dir=dataset.dataset.extra_files_path_name, + file_name=bundle_path, + alt_name=os.path.basename(bundle_path), + create=True, + preserve_symlinks=True, + ) + else: raise Exception("Unknown data manager mode encountered type...") diff --git a/lib/galaxy/tools/data_manager/manager.py b/lib/galaxy/tools/data_manager/manager.py index 89894db1f303..aaa7e4d23ee3 100644 --- a/lib/galaxy/tools/data_manager/manager.py +++ b/lib/galaxy/tools/data_manager/manager.py @@ -236,9 +236,9 @@ def process_result(self, out_data: Dict[str, OutputDataset]) -> None: def write_bundle( self, out_data: Dict[str, OutputDataset], - ) -> None: + ): tool_data_tables = self.data_managers.app.tool_data_tables - tool_data_tables.write_bundle( + return tool_data_tables.write_bundle( out_data, self.processor_description, self.repo_info, From ff504d9d3d9503a889e411e01804f64ef5ec997f Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 25 Aug 2023 15:25:47 +0200 Subject: [PATCH 08/13] Extend data bundle integration test to use S3 object store, check that cache can be repopulated --- test/integration/test_tool_data_bundles.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/integration/test_tool_data_bundles.py b/test/integration/test_tool_data_bundles.py index 67eca7e3a045..d4ede08b7682 100644 --- a/test/integration/test_tool_data_bundles.py +++ b/test/integration/test_tool_data_bundles.py @@ -1,10 +1,12 @@ import os +import shutil from galaxy.util.compression_utils import decompress_bytes_to_directory +from .objectstore._base import BaseSwiftObjectStoreIntegrationTestCase from .test_tool_data_delete import DataManagerIntegrationTestCase -class TestDataBundlesIntegration(DataManagerIntegrationTestCase): +class TestDataBundlesIntegration(BaseSwiftObjectStoreIntegrationTestCase, DataManagerIntegrationTestCase): def test_admin_build_data_bundle_by_uri(self): original_count = self._testbeta_field_count() @@ -24,6 +26,9 @@ def test_admin_build_data_bundle_by_uri(self): post_job_count = self._testbeta_field_count() assert original_count == post_job_count + shutil.rmtree(self.object_store_cache_path) + os.makedirs(self.object_store_cache_path) + content = self.dataset_populator.get_history_dataset_content( history_id, to_ext="data_manager_json", type="bytes" ) From 0477581217d014cff25edebda4b3c4f1777c3cff Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 25 Aug 2023 14:57:20 +0200 Subject: [PATCH 09/13] Push extra files from object store cache to object store --- lib/galaxy/job_execution/output_collect.py | 7 ++++--- test/integration/test_tool_data_bundles.py | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/job_execution/output_collect.py b/lib/galaxy/job_execution/output_collect.py index a8cd4369c121..9464de383e9a 100644 --- a/lib/galaxy/job_execution/output_collect.py +++ b/lib/galaxy/job_execution/output_collect.py @@ -727,18 +727,19 @@ def collect_extra_files(object_store, dataset, job_working_directory): # Fall back to working dir, remove in 23.2 output_location = "working" temp_file_path = os.path.join(job_working_directory, output_location, file_name) - extra_dir = None + if not os.path.exists(temp_file_path): + # no outputs to working directory, but may still need to push form cache to backend + temp_file_path = dataset.extra_files_path try: # This skips creation of directories - object store # automatically creates them. However, empty directories will # not be created in the object store at all, which might be a # problem. for root, _dirs, files in os.walk(temp_file_path): - extra_dir = root.replace(os.path.join(job_working_directory, output_location), "", 1).lstrip(os.path.sep) for f in files: object_store.update_from_file( dataset.dataset, - extra_dir=extra_dir, + extra_dir=os.path.normpath(os.path.join(file_name, os.path.relpath(root, temp_file_path))), alt_name=f, file_name=os.path.join(root, f), create=True, diff --git a/test/integration/test_tool_data_bundles.py b/test/integration/test_tool_data_bundles.py index d4ede08b7682..54f886334738 100644 --- a/test/integration/test_tool_data_bundles.py +++ b/test/integration/test_tool_data_bundles.py @@ -33,6 +33,7 @@ def test_admin_build_data_bundle_by_uri(self): history_id, to_ext="data_manager_json", type="bytes" ) temp_directory = decompress_bytes_to_directory(content) + assert os.path.exists(os.path.join(temp_directory, "newvalue.txt")) uri = f"file://{os.path.normpath(temp_directory)}" data = { "source": { From e93e986ad6dffea6b0e5f7278c2472783c77acbe Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 25 Aug 2023 15:53:39 +0200 Subject: [PATCH 10/13] Fix new_dataset._extra_files_path just in case something still uses it --- lib/galaxy/metadata/set_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/metadata/set_metadata.py b/lib/galaxy/metadata/set_metadata.py index 3bfd55a36151..47e7eb3dcf49 100644 --- a/lib/galaxy/metadata/set_metadata.py +++ b/lib/galaxy/metadata/set_metadata.py @@ -528,7 +528,7 @@ def write_job_metadata(tool_job_working_directory, job_metadata, set_meta, tool_ new_dataset = Dataset(id=-i, external_filename=new_dataset_filename) extra_files = file_dict.get("extra_files", None) if extra_files is not None: - new_dataset._extra_files_path = os.path.join(tool_job_working_directory, "working", extra_files) + new_dataset._extra_files_path = os.path.join(tool_job_working_directory, "outputs", extra_files) new_dataset.state = new_dataset.states.OK new_dataset_instance = HistoryDatasetAssociation( id=-i, dataset=new_dataset, extension=file_dict.get("ext", "data") From efef30d6ab3bccb4f177792b982b08d055959d3d Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 25 Aug 2023 11:00:06 +0200 Subject: [PATCH 11/13] make sure that TMP, TEMP, and TMPDIR are set seems needed for docker volume strings (an empty variable would lead to `-v "::rw"` which causes `docker: invalid spec: ::rw: empty section between colons.`) --- .../job_script/DEFAULT_JOB_FILE_TEMPLATE.sh | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh index 08d698ab7792..e89cc001e453 100644 --- a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh +++ b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh @@ -35,6 +35,29 @@ GALAXY_LIB="$galaxy_lib" _galaxy_setup_environment "$PRESERVE_GALAXY_ENVIRONMENT" export _GALAXY_JOB_HOME_DIR export _GALAXY_JOB_TMP_DIR + +TEMP="${TEMP:-$TMP}" +TMPDIR="${TMPDIR:-$TMP}" + +TMP="${TMP:-$TEMP}" +TMPDIR="${TMPDIR:-$TEMP}" + +TMP="${TMP:-$TMPDIR}" +TEMP="${TEMP:-$TMPDIR}" + +TMP="${TMP:-_GALAXY_JOB_TMP_DIR}" +TEMP="${TEMP:-_GALAXY_JOB_TMP_DIR}" +TMPDIR="${TMPDIR:-_GALAXY_JOB_TMP_DIR}" + +export TMP +export TEMP +export TMPDIR + +echo "_GALAXY_JOB_TMP_DIR $_GALAXY_JOB_TMP_DIR" >> outputs/tmp +echo "TMP $TMP" >> outputs/tmp +echo "TEMP $TEMP" >> outputs/tmp +echo "TMPDIR $TMPDIR" >> outputs/tmp + GALAXY_PYTHON=`command -v python` cd $working_directory $memory_statement From 913e890c6acab1d25123a2dec4484fababff2569 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 25 Aug 2023 16:02:22 +0200 Subject: [PATCH 12/13] Fix _GALAXY_JOB_TMP_DIR Co-authored-by: Marius van den Beek --- .../runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh index e89cc001e453..0b39cafb159e 100644 --- a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh +++ b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh @@ -45,9 +45,9 @@ TMPDIR="${TMPDIR:-$TEMP}" TMP="${TMP:-$TMPDIR}" TEMP="${TEMP:-$TMPDIR}" -TMP="${TMP:-_GALAXY_JOB_TMP_DIR}" -TEMP="${TEMP:-_GALAXY_JOB_TMP_DIR}" -TMPDIR="${TMPDIR:-_GALAXY_JOB_TMP_DIR}" +TMP="${TMP:-$_GALAXY_JOB_TMP_DIR}" +TEMP="${TEMP:-$_GALAXY_JOB_TMP_DIR}" +TMPDIR="${TMPDIR:-$_GALAXY_JOB_TMP_DIR}" export TMP export TEMP From 99ffa0e5ce7dd346d8a68be2b6a308eb3a93e5db Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 25 Aug 2023 16:03:11 +0200 Subject: [PATCH 13/13] remove debug code --- .../runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh index 0b39cafb159e..514552419dd7 100644 --- a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh +++ b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh @@ -53,11 +53,6 @@ export TMP export TEMP export TMPDIR -echo "_GALAXY_JOB_TMP_DIR $_GALAXY_JOB_TMP_DIR" >> outputs/tmp -echo "TMP $TMP" >> outputs/tmp -echo "TEMP $TEMP" >> outputs/tmp -echo "TMPDIR $TMPDIR" >> outputs/tmp - GALAXY_PYTHON=`command -v python` cd $working_directory $memory_statement