Merge pull request #17016 from bernt-matthias/topic/stdio-regex-extend

[24.0] Extend regex groups in stdio regex matches

client/src/components/JobInformation/JobInformation.test.js

@@ -69,7 +69,7 @@ describe("JobInformation/JobInformation.vue", () => {
     });
 
     it("job messages", async () => {
-        const rendered_link = jobInfoTable.findAll(`#job-messages li`);
+        const rendered_link = jobInfoTable.findAll(`#job-messages .job-message`);
         expect(rendered_link.length).toBe(jobResponse.job_messages.length);
         for (let i = 0; i < rendered_link.length; i++) {
             const msg = rendered_link.at(i).text();

client/src/components/JobInformation/JobInformation.vue

@@ -1,6 +1,75 @@
+<script setup>
+import CopyToClipboard from "components/CopyToClipboard";
+import HelpText from "components/Help/HelpText";
+import { JobDetailsProvider } from "components/providers/JobProvider";
+import UtcDate from "components/UtcDate";
+import { formatDuration, intervalToDuration } from "date-fns";
+import JOB_STATES_MODEL from "utils/job-states-model";
+import { computed, ref } from "vue";
+
+import { invocationForJob } from "@/api/invocations";
+
+import DecodedId from "../DecodedId.vue";
+import CodeRow from "./CodeRow.vue";
+
+const job = ref(null);
+const invocationId = ref(null);
+
+const props = defineProps({
+    job_id: {
+        type: String,
+        required: true,
+    },
+    includeTimes: {
+        type: Boolean,
+        default: false,
+    },
+});
+
+const runTime = computed(() =>
+    formatDuration(intervalToDuration({ start: new Date(job.value.create_time), end: new Date(job.value.update_time) }))
+);
+
+const jobIsTerminal = computed(() => job.value && !JOB_STATES_MODEL.NON_TERMINAL_STATES.includes(job.value.state));
+
+const routeToInvocation = computed(() => `/workflows/invocations/${invocationId.value}`);
+
+const metadataDetail = ref({
+    exit_code: `Tools may use exit codes to indicate specific execution errors. Many programs use 0 to indicate success and non-zero exit codes to indicate errors. Galaxy allows each tool to specify exit codes that indicate errors. https://docs.galaxyproject.org/en/master/dev/schema.html#tool-stdio-exit-code`,
+    error_level: `NO_ERROR = 0</br>LOG = 1</br>QC = 1.1</br>WARNING = 2</br>FATAL = 3</br>FATAL_OOM = 4</br>MAX = 4`,
+});
+
+function updateJob(newJob) {
+    job.value = newJob;
+    if (newJob) {
+        fetchInvocation(newJob.id);
+    }
+}
+
+function filterMetadata(jobMessages) {
+    return jobMessages.map((item) => {
+        return Object.entries(item).reduce((acc, [key, value]) => {
+            if (value) {
+                acc[key] = value;
+            }
+            return acc;
+        }, {});
+    });
+}
+
+async function fetchInvocation(jobId) {
+    if (jobId) {
+        const invocation = await invocationForJob({ jobId: jobId });
+        if (invocation) {
+            invocationId.value = invocation.id;
+        }
+    }
+}
+</script>
+
 <template>
     <div>
-        <JobDetailsProvider auto-refresh :job-id="job_id" @update:result="updateJob" />
+        <JobDetailsProvider auto-refresh :job-id="props.job_id" @update:result="updateJob" />
         <h2 class="h-md">Job Information</h2>
         <table id="job-information" class="tabletip info_data_table">
             <tbody>
@@ -24,19 +93,19 @@
                     <td>Galaxy Tool Version</td>
                     <td id="galaxy-tool-version">{{ job.tool_version }}</td>
                 </tr>
-                <tr v-if="job && includeTimes">
+                <tr v-if="job && props.includeTimes">
                     <td>Created</td>
                     <td v-if="job.create_time" id="created">
                         <UtcDate :date="job.create_time" mode="pretty" />
                     </td>
                 </tr>
-                <tr v-if="job && includeTimes">
+                <tr v-if="job && props.includeTimes">
                     <td>Updated</td>
                     <td v-if="job.update_time" id="updated">
                         <UtcDate :date="job.update_time" mode="pretty" />
                     </td>
                 </tr>
-                <tr v-if="job && includeTimes && jobIsTerminal">
+                <tr v-if="job && props.includeTimes && jobIsTerminal">
                     <td>Time To Finish</td>
                     <td id="runtime">
                         {{ runTime }}
@@ -73,9 +142,28 @@
                 <tr v-if="job && job.job_messages && job.job_messages.length > 0" id="job-messages">
                     <td>Job Messages</td>
                     <td>
-                        <ul style="padding-left: 15px; margin-bottom: 0px">
-                            <li v-for="(message, index) in job.job_messages" :key="index">{{ message }}</li>
+                        <ul v-if="Array.isArray(job.job_messages)" class="pl-2 mb-0">
+                            <div v-for="(message, m) in filterMetadata(job.job_messages)" :key="m" class="job-message">
+                                <div v-if="job.job_messages.length > 1">
+                                    <u>Job Message {{ m + 1 }}:</u>
+                                </div>
+                                <li v-for="(value, name, i) in message" :key="i">
+                                    <span
+                                        v-if="metadataDetail[name]"
+                                        v-b-tooltip.html
+                                        class="tooltipJobInfo"
+                                        :title="metadataDetail[name]"
+                                        ><strong>{{ name }}:</strong></span
+                                    >
+                                    <strong v-else>{{ name }}:</strong>
+                                    {{ value }}
+                                </li>
+                                <hr v-if="m + 1 < job.job_messages.length" />
+                            </div>
                         </ul>
+                        <div v-else>
+                            {{ job.job_messages }}
+                        </div>
                     </td>
                 </tr>
                 <slot></slot>
@@ -100,76 +188,9 @@
     </div>
 </template>
 
-<script>
-import CopyToClipboard from "components/CopyToClipboard";
-import HelpText from "components/Help/HelpText";
-import { JobDetailsProvider } from "components/providers/JobProvider";
-import UtcDate from "components/UtcDate";
-import { formatDuration, intervalToDuration } from "date-fns";
-import { getAppRoot } from "onload/loadConfig";
-import JOB_STATES_MODEL from "utils/job-states-model";
-
-import { invocationForJob } from "@/api/invocations";
-
-import DecodedId from "../DecodedId.vue";
-import CodeRow from "./CodeRow.vue";
-
-export default {
-    components: {
-        CodeRow,
-        DecodedId,
-        JobDetailsProvider,
-        HelpText,
-        UtcDate,
-        CopyToClipboard,
-    },
-    props: {
-        job_id: {
-            type: String,
-            required: true,
-        },
-        includeTimes: {
-            type: Boolean,
-            default: false,
-        },
-    },
-    data() {
-        return {
-            job: null,
-            invocationId: null,
-        };
-    },
-    computed: {
-        runTime: function () {
-            return formatDuration(
-                intervalToDuration({ start: new Date(this.job.create_time), end: new Date(this.job.update_time) })
-            );
-        },
-        jobIsTerminal() {
-            return this.job && !JOB_STATES_MODEL.NON_TERMINAL_STATES.includes(this.job.state);
-        },
-        routeToInvocation() {
-            return `/workflows/invocations/${this.invocationId}`;
-        },
-    },
-    methods: {
-        getAppRoot() {
-            return getAppRoot();
-        },
-        updateJob(job) {
-            this.job = job;
-            if (job) {
-                this.fetchInvocation(job.id);
-            }
-        },
-        async fetchInvocation(jobId) {
-            if (jobId) {
-                const invocation = await invocationForJob({ jobId: jobId });
-                if (invocation) {
-                    this.invocationId = invocation.id;
-                }
-            }
-        },
-    },
-};
-</script>
+<style scoped>
+.tooltipJobInfo {
+    text-decoration-line: underline;
+    text-decoration-style: dashed;
+}
+</style>

lib/galaxy/jobs/__init__.py

@@ -75,6 +75,7 @@
     check_output,
     DETECTED_JOB_STATE,
 )
+from galaxy.tool_util.parser.stdio import StdioErrorLevel
 from galaxy.tools.evaluation import (
     PartialToolEvaluator,
     ToolEvaluator,
@@ -1939,7 +1940,13 @@ def fail(message=job.info, exception=None):
                 self.discover_outputs(job, inp_data, out_data, out_collections, final_job_state=final_job_state)
             except MaxDiscoveredFilesExceededError as e:
                 final_job_state = job.states.ERROR
-                job.job_messages = [str(e)]
+                job.job_messages = [
+                    {
+                        "type": "internal",
+                        "desc": str(e),
+                        "error_level": StdioErrorLevel.FATAL,
+                    }
+                ]
 
             for dataset_assoc in output_dataset_associations:
                 if getattr(dataset_assoc.dataset, "discovered", False):
@@ -2088,12 +2095,8 @@ def discover_outputs(self, job, inp_data, out_data, out_collections, final_job_s
         )
 
     def check_tool_output(self, tool_stdout, tool_stderr, tool_exit_code, job, job_stdout=None, job_stderr=None):
-        job_id_tag = "<unknown job id>"
-        if job is not None:
-            job_id_tag = job.get_id_tag()
-
         state, tool_stdout, tool_stderr, job_messages = check_output(
-            self.tool.stdio_regexes, self.tool.stdio_exit_codes, tool_stdout, tool_stderr, tool_exit_code, job_id_tag
+            self.tool.stdio_regexes, self.tool.stdio_exit_codes, tool_stdout, tool_stderr, tool_exit_code
         )
 
         # Store the modified stdout and stderr in the job:

lib/galaxy/metadata/set_metadata.py

@@ -19,7 +19,12 @@
 import traceback
 from functools import partial
 from pathlib import Path
-from typing import Optional
+from typing import (
+    Any,
+    Dict,
+    List,
+    Optional,
+)
 
 try:
     from pulsar.client.staging import COMMAND_VERSION_FILENAME
@@ -61,13 +66,15 @@
     DETECTED_JOB_STATE,
 )
 from galaxy.tool_util.parser.stdio import (
+    StdioErrorLevel,
     ToolStdioExitCode,
     ToolStdioRegex,
 )
 from galaxy.tool_util.provided_metadata import parse_tool_provided_metadata
 from galaxy.util import (
     safe_contains,
     stringify_dictionary_keys,
+    unicodify,
 )
 from galaxy.util.expressions import ExpressionContext
 
@@ -216,7 +223,7 @@ def set_meta(new_dataset_instance, file_dict):
 
     export_store = None
     final_job_state = Job.states.OK
-    job_messages = []
+    job_messages: List[Dict[str, Any]] = []
     if extended_metadata_collection:
         tool_dict = metadata_params["tool"]
         stdio_exit_code_dicts, stdio_regex_dicts = tool_dict["stdio_exit_codes"], tool_dict["stdio_regexes"]
@@ -237,25 +244,25 @@ def set_meta(new_dataset_instance, file_dict):
         for directory, prefix in locations:
             if directory and os.path.exists(os.path.join(directory, f"{prefix}stdout")):
                 with open(os.path.join(directory, f"{prefix}stdout"), "rb") as f:
-                    tool_stdout = f.read(MAX_STDIO_READ_BYTES)
+                    tool_stdout = unicodify(f.read(MAX_STDIO_READ_BYTES), strip_null=True)
                 with open(os.path.join(directory, f"{prefix}stderr"), "rb") as f:
-                    tool_stderr = f.read(MAX_STDIO_READ_BYTES)
+                    tool_stderr = unicodify(f.read(MAX_STDIO_READ_BYTES), strip_null=True)
                 break
         else:
             if os.path.exists(os.path.join(tool_job_working_directory, "task_0")):
                 # We have a task splitting job
-                tool_stdout = b""
-                tool_stderr = b""
+                tool_stdout = ""
+                tool_stderr = ""
                 paths = tool_job_working_directory.glob("task_*")
                 for path in paths:
                     with open(path / "outputs" / "tool_stdout", "rb") as f:
-                        task_stdout = f.read(MAX_STDIO_READ_BYTES)
+                        task_stdout = unicodify(f.read(MAX_STDIO_READ_BYTES), strip_null=True)
                         if task_stdout:
-                            tool_stdout = b"%s[%s stdout]\n%s\n" % (tool_stdout, path.name.encode(), task_stdout)
+                            tool_stdout = f"{tool_stdout}[{path.name} stdout]\n{task_stdout}\n"
                     with open(path / "outputs" / "tool_stderr", "rb") as f:
-                        task_stderr = f.read(MAX_STDIO_READ_BYTES)
+                        task_stderr = unicodify(f.read(MAX_STDIO_READ_BYTES), strip_null=True)
                         if task_stderr:
-                            tool_stderr = b"%s[%s stdout]\n%s\n" % (tool_stderr, path.name.encode(), task_stderr)
+                            tool_stderr = f"{tool_stderr}[{path.name} stderr]\n{task_stderr}\n"
             else:
                 wdc = os.listdir(tool_job_working_directory)
                 odc = os.listdir(outputs_directory)
@@ -265,15 +272,15 @@ def set_meta(new_dataset_instance, file_dict):
                     log.warning(f"{error_desc}. {error_extra}")
                     raise Exception(error_desc)
                 else:
-                    tool_stdout = tool_stderr = b""
+                    tool_stdout = tool_stderr = ""
 
         job_id_tag = metadata_params["job_id_tag"]
 
         exit_code_file = default_exit_code_file(".", job_id_tag)
         tool_exit_code = read_exit_code_from(exit_code_file, job_id_tag)
 
         check_output_detected_state, tool_stdout, tool_stderr, job_messages = check_output(
-            stdio_regexes, stdio_exit_codes, tool_stdout, tool_stderr, tool_exit_code, job_id_tag
+            stdio_regexes, stdio_exit_codes, tool_stdout, tool_stderr, tool_exit_code
         )
         if check_output_detected_state == DETECTED_JOB_STATE.OK and not tool_provided_metadata.has_failed_outputs():
             final_job_state = Job.states.OK
@@ -340,7 +347,15 @@ def set_meta(new_dataset_instance, file_dict):
             collect_dynamic_outputs(job_context, output_collections)
         except MaxDiscoveredFilesExceededError as e:
             final_job_state = Job.states.ERROR
-            job_messages.append(str(e))
+            job_messages.append(
+                {
+                    "type": "max_discovered_files",
+                    "desc": str(e),
+                    "code_desc": None,
+                    "error_level": StdioErrorLevel.FATAL,
+                }
+            )
+
         if job:
             job.set_streams(tool_stdout=tool_stdout, tool_stderr=tool_stderr, job_messages=job_messages)
             job.state = final_job_state