From 04b1bfeb90f7e428d66ed6c2857b5efac3ff4c43 Mon Sep 17 00:00:00 2001
From: Ralph Castain <rhc@pmix.org>
Date: Sun, 25 Jun 2023 10:48:07 -0600
Subject: [PATCH 1/4] Add psched daemon

Add a daemon that can act as a scheduler in support of
development of PMIx scheduler integration.

Signed-off-by: Ralph Castain <rhc@pmix.org>
---
 .gitignore                                    |   1 +
 config/prte_config_files.m4                   |   1 +
 src/mca/ess/base/ess_base_std_prolog.c        |   7 +-
 src/mca/schizo/psched/Makefile.am             |  44 ++
 src/mca/schizo/psched/configure.m4            |  37 ++
 src/mca/schizo/psched/help-schizo-psched.txt  |  99 ++++
 src/mca/schizo/psched/owner.txt               |   7 +
 src/mca/schizo/psched/schizo_psched.c         | 309 ++++++++++
 src/mca/schizo/psched/schizo_psched.h         |  37 ++
 .../schizo/psched/schizo_psched_component.c   |  66 +++
 src/tools/Makefile.am                         |   6 +-
 src/tools/psched/Makefile.am                  |  54 ++
 src/tools/psched/psched.c                     | 555 ++++++++++++++++++
 13 files changed, 1218 insertions(+), 5 deletions(-)
 create mode 100644 src/mca/schizo/psched/Makefile.am
 create mode 100644 src/mca/schizo/psched/configure.m4
 create mode 100644 src/mca/schizo/psched/help-schizo-psched.txt
 create mode 100644 src/mca/schizo/psched/owner.txt
 create mode 100644 src/mca/schizo/psched/schizo_psched.c
 create mode 100644 src/mca/schizo/psched/schizo_psched.h
 create mode 100644 src/mca/schizo/psched/schizo_psched_component.c
 create mode 100644 src/tools/psched/Makefile.am
 create mode 100644 src/tools/psched/psched.c

diff --git a/.gitignore b/.gitignore
index 9ae0b574d5..d216856cf7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -162,6 +162,7 @@ src/tools/prte/prte
 src/tools/pcc/pcc
 src/tools/pcc/pcc-wrapper-data.txt
 src/tools/pterm/pterm
+src/tools/psched/psched
 
 src/util/hostfile/hostfile_lex.c
 src/util/keyval/keyval_lex.c
diff --git a/config/prte_config_files.m4 b/config/prte_config_files.m4
index 750484975e..ef3f2eb2e1 100644
--- a/config/prte_config_files.m4
+++ b/config/prte_config_files.m4
@@ -26,5 +26,6 @@ AC_DEFUN([PRTE_CONFIG_FILES],[
         src/tools/prte_info/Makefile
         src/tools/prte/Makefile
         src/tools/pterm/Makefile
+        src/tools/psched/Makefile
     ])
 ])
diff --git a/src/mca/ess/base/ess_base_std_prolog.c b/src/mca/ess/base/ess_base_std_prolog.c
index c63d78df2f..c475fd9e58 100644
--- a/src/mca/ess/base/ess_base_std_prolog.c
+++ b/src/mca/ess/base/ess_base_std_prolog.c
@@ -11,7 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2019      Intel, Inc.  All rights reserved.
  * Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved
- * Copyright (c) 2021-2022 Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2021-2023 Nanook Consulting.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -60,8 +60,9 @@ int prte_ess_base_std_prolog(void)
     return PRTE_SUCCESS;
 
 error:
-    pmix_show_help("help-prte-runtime", "prte_init:startup:internal-failure", true, error,
-                   PRTE_ERROR_NAME(ret), ret);
+    pmix_show_help("help-prte-runtime",
+                   "prte_init:startup:internal-failure", true,
+                   error, PRTE_ERROR_NAME(ret), ret);
 
     return ret;
 }
diff --git a/src/mca/schizo/psched/Makefile.am b/src/mca/schizo/psched/Makefile.am
new file mode 100644
index 0000000000..9ee72ed754
--- /dev/null
+++ b/src/mca/schizo/psched/Makefile.am
@@ -0,0 +1,44 @@
+#
+# Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
+# Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved
+# Copyright (c) 2022-2023 Nanook Consulting.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+AM_CFLAGS = \
+    -DPRTE_GREEK_VERSION="\"@PRTE_GREEK_VERSION@\""
+
+dist_prtedata_DATA = \
+    help-schizo-psched.txt
+
+sources = \
+          schizo_psched_component.c \
+          schizo_psched.h \
+          schizo_psched.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_prte_schizo_psched_DSO
+component_noinst =
+component_install = prte_mca_schizo_psched.la
+else
+component_noinst = libprtemca_schizo_psched.la
+component_install =
+endif
+
+mcacomponentdir = $(prtelibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+prte_mca_schizo_psched_la_SOURCES = $(sources)
+prte_mca_schizo_psched_la_LDFLAGS = -module -avoid-version
+prte_mca_schizo_psched_la_LIBADD = $(top_builddir)/src/libprrte.la
+
+noinst_LTLIBRARIES = $(component_noinst)
+libprtemca_schizo_psched_la_SOURCES = $(sources)
+libprtemca_schizo_psched_la_LDFLAGS = -module -avoid-version
diff --git a/src/mca/schizo/psched/configure.m4 b/src/mca/schizo/psched/configure.m4
new file mode 100644
index 0000000000..9efa13e9df
--- /dev/null
+++ b/src/mca/schizo/psched/configure.m4
@@ -0,0 +1,37 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2009-2020 Cisco Systems, Inc.  All rights reserved
+# Copyright (c) 2011-2013 Los Alamos National Security, LLC.
+#                         All rights reserved.
+# Copyright (c) 2019      Intel, Inc.  All rights reserved.
+# Copyright (c) 2022-2023 Nanook Consulting.  All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+#                         All Rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# MCA_schizo_psched_CONFIG([action-if-found], [action-if-not-found])
+# -----------------------------------------------------------
+AC_DEFUN([MCA_prte_schizo_psched_CONFIG],[
+    AC_CONFIG_FILES([src/mca/schizo/psched/Makefile])
+
+    AS_IF([test "yes" = "yes"],
+          [$1], [$2])
+
+    PRTE_SUMMARY_ADD([Personalities], [PRTE Scheduler], [], [yes])
+
+])dnl
diff --git a/src/mca/schizo/psched/help-schizo-psched.txt b/src/mca/schizo/psched/help-schizo-psched.txt
new file mode 100644
index 0000000000..6bf2bc031a
--- /dev/null
+++ b/src/mca/schizo/psched/help-schizo-psched.txt
@@ -0,0 +1,99 @@
+# -*- text -*-
+#
+# Copyright (c) 2021-2023 Nanook Consulting  All rights reserved.
+# Copyright (c) 2022      IBM Corporation.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+#
+[usage]
+%s (%s) %s
+
+Usage: %s [OPTION]...
+
+The following list of command line options are available. Note that
+more detailed help for any option can be obtained by adding that
+option to the help request as "--help <option>".
+
+/*****      General Options      *****/
+
+-h|--help                            This help message
+-h|--help <arg0>                     Help for the specified option
+-v|--verbose                         Enable typical debug options
+-V|--version                         Print version and exit
+
+
+/*****       Debug Options       *****/
+
+   --debug                           Top-level PRTE debug switch (default: false) This option will be
+                                     deprecated, use --debug-devel instead.
+   --debug-daemons-file              Enable debugging of any PRTE daemons used by this application, storing
+                                     output in files
+   --leave-session-attached          Do not daemonize the prted
+   --test-suicide <arg0>             Suicide instead of clean abort after specified delay
+
+
+
+/*****      DVM Options       *****/
+
+   --pmixmca <arg0> <arg1>           Pass context-specific PMIx MCA parameters (arg0 is the
+                                     parameter name; arg1 is the parameter value)
+   --prtemca <arg0> <arg1>           Pass context-specific PRTE MCA parameters; they are considered global if
+                                     --gmca is not used and only one context is specified (arg0 is the
+                                     parameter name; arg1 is the parameter value)
+   --dvm-master-uri <arg0>           Specify the URI of the DVM master, or the name of the file (specified as
+                                     file:filename) that contains that info
+   --parent-uri <arg0>               Specify the URI of the prted acting as the parent of this prted
+                                     in a tree-based spawn operation
+   --tree-spawn                      A tree-based spawn operation is in progress
+   --bootstrap                       Self-construct the DVM based on a configuration file
+
+
+/*****    Specific Options   *****/
+
+   --set-sid                         Direct the DVM daemons to separate from the current session
+   --system-server                   Start the DVM as the system server
+   --pubsub-server <arg0>            Contact information for external PRRTE publish/lookup
+                                     data server
+
+Report bugs to %s
+#
+[version]
+%s (%s) %s
+
+Report bugs to %s
+#
+[dvm-master-uri]
+Specify the URI of the DVM master, or the name of the file (specified as
+file:filename) that contains that info
+#
+[test-suicide]
+Test DVM cleanup upon daemon failure by having one daemon suicide after delay
+#
+[system-server]
+Start the daemon as the system server on its node
+#
+[set-sid]
+Direct the daemon to separate from the current session
+#
+[prtemca]
+#include#help-schizo-cli#prtemca
+#
+[pmixmca]
+#include#help-schizo-cli#pmixmca
+#
+[leave-session-attached]
+#include#help-schizo-cli#leave-session-attached
+#
+[parent-uri]
+Specify the URI of the prted acting as the parent of this prted
+in a tree-based spawn operation
+#
+[tree-spawn]
+A tree-based spawn operation is in progress
+#
+[pubsub-server]
+Contact information for external PRRTE publish/lookup data server
diff --git a/src/mca/schizo/psched/owner.txt b/src/mca/schizo/psched/owner.txt
new file mode 100644
index 0000000000..a6532f448e
--- /dev/null
+++ b/src/mca/schizo/psched/owner.txt
@@ -0,0 +1,7 @@
+#
+# owner/status file
+# owner: institution that is responsible for this package
+# status: e.g. active, maintenance, unmaintained
+#
+owner: Nanook
+status: devel
diff --git a/src/mca/schizo/psched/schizo_psched.c b/src/mca/schizo/psched/schizo_psched.c
new file mode 100644
index 0000000000..2bc726b674
--- /dev/null
+++ b/src/mca/schizo/psched/schizo_psched.c
@@ -0,0 +1,309 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2017 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2009-2020 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2011-2017 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2017      UT-Battelle, LLC. All rights reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2022 IBM Corporation.  All rights reserved.
+ * Copyright (c) 2021-2023 Nanook Consulting  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include "prte_config.h"
+#include "types.h"
+
+#ifdef HAVE_UNISTD_H
+#    include <unistd.h>
+#endif
+#include <ctype.h>
+#include <getopt.h>
+
+
+#include "src/util/name_fns.h"
+#include "src/util/pmix_argv.h"
+#include "src/util/pmix_basename.h"
+#include "src/util/pmix_os_dirpath.h"
+#include "src/util/pmix_os_path.h"
+#include "src/util/pmix_path.h"
+#include "src/util/pmix_environ.h"
+#include "src/util/prte_cmd_line.h"
+#include "src/util/session_dir.h"
+#include "src/util/pmix_show_help.h"
+
+#include "src/mca/errmgr/errmgr.h"
+#include "src/mca/ess/base/base.h"
+#include "src/mca/prteinstalldirs/prteinstalldirs.h"
+#include "src/mca/rmaps/base/base.h"
+#include "src/mca/state/base/base.h"
+#include "src/runtime/prte_globals.h"
+#include "src/runtime/pmix_init_util.h"
+
+#include "schizo_psched.h"
+#include "src/mca/schizo/base/base.h"
+
+static int parse_cli(char **argv, pmix_cli_result_t *results, bool silent);
+static int detect_proxy(char *argv);
+static int parse_env(char **srcenv, char ***dstenv, pmix_cli_result_t *cli);
+static void allow_run_as_root(pmix_cli_result_t *results);
+static void job_info(pmix_cli_result_t *results,
+                     void *jobinfo);
+static int set_default_rto(prte_job_t *jdata,
+                           prte_rmaps_options_t *options);
+
+prte_schizo_base_module_t prte_schizo_prte_module = {
+    .name = "psched",
+    .parse_cli = parse_cli,
+    .parse_env = parse_env,
+    .setup_fork = prte_schizo_base_setup_fork,
+    .detect_proxy = detect_proxy,
+    .allow_run_as_root = allow_run_as_root,
+    .job_info = job_info,
+    .set_default_rto = set_default_rto,
+    .check_sanity = prte_schizo_base_sanity
+};
+
+static struct option pschedoptions[] = {
+    /* basic options */
+    PMIX_OPTION_SHORT_DEFINE(PRTE_CLI_HELP, PMIX_ARG_OPTIONAL, 'h'),
+    PMIX_OPTION_SHORT_DEFINE(PRTE_CLI_VERSION, PMIX_ARG_NONE, 'V'),
+    PMIX_OPTION_SHORT_DEFINE(PRTE_CLI_VERBOSE, PMIX_ARG_NONE, 'v'),
+
+    // MCA parameters
+    PMIX_OPTION_DEFINE(PRTE_CLI_PRTEMCA, PMIX_ARG_REQD),
+    PMIX_OPTION_DEFINE(PRTE_CLI_PMIXMCA, PMIX_ARG_REQD),
+    PMIX_OPTION_DEFINE(PRTE_CLI_TUNE, PMIX_ARG_REQD),
+
+    // DVM options
+    PMIX_OPTION_DEFINE(PRTE_CLI_NO_READY_MSG, PMIX_ARG_NONE),
+    PMIX_OPTION_DEFINE(PRTE_CLI_DAEMONIZE, PMIX_ARG_NONE),
+    PMIX_OPTION_DEFINE(PRTE_CLI_SET_SID, PMIX_ARG_NONE),
+    PMIX_OPTION_DEFINE(PRTE_CLI_REPORT_PID, PMIX_ARG_REQD),
+    PMIX_OPTION_DEFINE(PRTE_CLI_REPORT_URI, PMIX_ARG_REQD),
+    PMIX_OPTION_DEFINE(PRTE_CLI_DEFAULT_HOSTFILE, PMIX_ARG_REQD),
+    PMIX_OPTION_DEFINE(PRTE_CLI_DEBUG, PMIX_ARG_NONE),
+    PMIX_OPTION_DEFINE(PRTE_CLI_LEAVE_SESSION_ATTACHED, PMIX_ARG_NONE),
+    PMIX_OPTION_DEFINE(PRTE_CLI_TMPDIR, PMIX_ARG_REQD),
+    PMIX_OPTION_DEFINE(PRTE_CLI_PREFIX, PMIX_ARG_REQD),
+    PMIX_OPTION_DEFINE(PRTE_CLI_NOPREFIX, PMIX_ARG_NONE),
+    PMIX_OPTION_DEFINE(PRTE_CLI_RUN_AS_ROOT, PMIX_ARG_NONE),
+
+    // Runtime options
+    PMIX_OPTION_DEFINE(PRTE_CLI_RTOS, PMIX_ARG_REQD),
+
+    // output options
+
+    /* developer options */
+    PMIX_OPTION_DEFINE(PRTE_CLI_DISPLAY, PMIX_ARG_REQD),
+
+    // deprecated options
+    PMIX_OPTION_DEFINE("machinefile", PMIX_ARG_REQD),
+
+    PMIX_OPTION_END
+};
+static char *pschedshorts = "h::vVx:H:";
+
+static int parse_cli(char **argv, pmix_cli_result_t *results,
+                     bool silent)
+{
+    int rc, n;
+    pmix_cli_item_t *opt;
+
+    pmix_tool_msg = "Report bugs to: https://github.com/openpmix/prrte";
+    pmix_tool_org = "PRRTE";
+    pmix_tool_version = prte_util_make_version_string("all", PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION,
+                                                      PRTE_RELEASE_VERSION, PRTE_GREEK_VERSION, NULL);
+
+    rc = pmix_cmd_line_parse(argv, pschedshorts, pschedoptions, NULL,
+                             results, "help-schizo-prte.txt");
+    if (PMIX_SUCCESS != rc) {
+        if (PMIX_OPERATION_SUCCEEDED == rc) {
+            /* pmix cmd line interpreter output result
+             * successfully - usually means version or
+             * some other stock output was generated */
+            return PRTE_OPERATION_SUCCEEDED;
+        }
+        rc = prte_pmix_convert_status(rc);
+        return rc;
+    }
+
+    // handle relevant MCA params
+    PMIX_LIST_FOREACH(opt, &results->instances, pmix_cli_item_t) {
+        if (0 == strcmp(opt->key, PRTE_CLI_PRTEMCA)) {
+            for (n=0; NULL != opt->values[n]; n++) {
+                prte_schizo_base_expose(opt->values[n], "PRTE_MCA_");
+            }
+        } else if (0 == strcmp(opt->key, PRTE_CLI_PMIXMCA)) {
+            for (n=0; NULL != opt->values[n]; n++) {
+                prte_schizo_base_expose(opt->values[n], "PMIX_MCA_");
+            }
+        }
+    }
+    return PRTE_SUCCESS;
+}
+
+static int parse_env(char **srcenv, char ***dstenv,
+                     pmix_cli_result_t *cli)
+{
+    int i, j, n;
+    char *p1, *p2;
+    char **env;
+    char **xparams = NULL, **xvals = NULL;
+    char *param, *value;
+    pmix_cli_item_t *opt;
+    PRTE_HIDE_UNUSED_PARAMS(srcenv);
+
+    pmix_output_verbose(1, prte_schizo_base_framework.framework_output,
+                        "%s schizo:prte: parse_env",
+                        PRTE_NAME_PRINT(PRTE_PROC_MY_NAME));
+
+    if (NULL == cli) {
+        return PRTE_SUCCESS;
+    }
+
+    env = *dstenv;
+
+    /* look for -x options - not allowed to conflict with a -mca option */
+    opt = pmix_cmd_line_get_param(cli, PRTE_CLI_FWD_ENVAR);
+    if (NULL != opt) {
+        for (j=0; NULL != opt->values[j]; j++) {
+            /* the value is the envar */
+            p1 = opt->values[j];
+            /* if there is an '=' in it, then they are setting a value */
+            if (NULL != (p2 = strchr(p1, '='))) {
+                *p2 = '\0';
+                ++p2;
+            } else {
+                p2 = getenv(p1);
+                if (NULL == p2) {
+                    pmix_show_help("help-schizo-base.txt", "missing-envar-param", true, p1);
+                    continue;
+                }
+            }
+
+            /* check if it is already present in the environment */
+            for (n = 0; NULL != env && NULL != env[n]; n++) {
+                param = strdup(env[n]);
+                value = strchr(param, '=');
+                *value = '\0';
+                value++;
+                /* check if parameter is already present */
+                if (0 == strcmp(param, p1)) {
+                    /* we do have it - check for same value */
+                    if (0 != strcmp(value, p2)) {
+                        /* this is an error - different values */
+                        pmix_show_help("help-schizo-base.txt", "duplicate-mca-value", true, p1, p2,
+                                       value);
+                        free(param);
+                        PMIX_ARGV_FREE_COMPAT(xparams);
+                        PMIX_ARGV_FREE_COMPAT(xvals);
+                        return PRTE_ERR_BAD_PARAM;
+                    }
+                }
+                free(param);
+            }
+
+            /* check if we already processed a conflicting -x version with MCA prefix */
+            if (NULL != xparams) {
+                for (i = 0; NULL != xparams[i]; i++) {
+                    if (0 == strncmp("PRTE_MCA_", p1, strlen("PRTE_MCA_"))) {
+                        /* this is an error - different values */
+                        pmix_show_help("help-schizo-base.txt", "duplicate-mca-value", true, p1, p2,
+                                       xvals[i]);
+                        PMIX_ARGV_FREE_COMPAT(xparams);
+                        PMIX_ARGV_FREE_COMPAT(xvals);
+                        return PRTE_ERR_BAD_PARAM;
+                    }
+                }
+            }
+
+            /* cache this for later inclusion - do not modify dstenv in this loop */
+            PMIX_ARGV_APPEND_NOSIZE_COMPAT(&xparams, p1);
+            PMIX_ARGV_APPEND_NOSIZE_COMPAT(&xvals, p2);
+        }
+    }
+
+    /* add the -x values */
+    if (NULL != xparams) {
+        for (i = 0; NULL != xparams[i]; i++) {
+            PMIX_SETENV_COMPAT(xparams[i], xvals[i], true, dstenv);
+        }
+        PMIX_ARGV_FREE_COMPAT(xparams);
+        PMIX_ARGV_FREE_COMPAT(xvals);
+    }
+
+    return PRTE_SUCCESS;
+}
+
+static int detect_proxy(char *personalities)
+{
+    pmix_output_verbose(2, prte_schizo_base_framework.framework_output,
+                        "%s[%s]: detect proxy with %s (%s)",
+                        PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), __FILE__,
+                        (NULL == personalities) ? "NULL" : personalities,
+                        prte_tool_basename);
+
+    if (NULL != personalities) {
+        /* this is a list of personalities we need to check -
+         * if it contains "psched", then we are available
+         * to be used */
+        if (NULL != strstr(personalities, "psched")) {
+            return 100;
+        }
+        return 0;
+    }
+
+    /* not us */
+    return -1;
+}
+
+static void allow_run_as_root(pmix_cli_result_t *cli)
+{
+    char *r1, *r2;
+
+    if (pmix_cmd_line_is_taken(cli, "allow-run-as-root")) {
+        prte_allow_run_as_root = true;
+        return;
+    }
+
+    if (NULL != (r1 = getenv("PRTE_ALLOW_RUN_AS_ROOT"))
+        && NULL != (r2 = getenv("PRTE_ALLOW_RUN_AS_ROOT_CONFIRM"))) {
+        if (0 == strcmp(r1, "1") && 0 == strcmp(r2, "1")) {
+            prte_allow_run_as_root = true;
+            return;
+        }
+    }
+
+    prte_schizo_base_root_error_msg();
+}
+
+static void job_info(pmix_cli_result_t *results,
+                     void *jobinfo)
+{
+    PRTE_HIDE_UNUSED_PARAMS(results, jobinfo);
+    return;
+}
+
+static int set_default_rto(prte_job_t *jdata,
+                           prte_rmaps_options_t *options)
+{
+    PRTE_HIDE_UNUSED_PARAMS(options);
+    return prte_state_base_set_runtime_options(jdata, NULL);
+}
diff --git a/src/mca/schizo/psched/schizo_psched.h b/src/mca/schizo/psched/schizo_psched.h
new file mode 100644
index 0000000000..3c00f77c77
--- /dev/null
+++ b/src/mca/schizo/psched/schizo_psched.h
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2019      Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2021-2023 Nanook Consulting.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef _MCA_SCHIZO_PSCHED_H_
+#define _MCA_SCHIZO_PSCHED_H_
+
+#include "prte_config.h"
+
+#include "types.h"
+
+#include "src/mca/base/pmix_base.h"
+#include "src/mca/schizo/schizo.h"
+
+BEGIN_C_DECLS
+
+typedef struct {
+    prte_schizo_base_component_t super;
+    int priority;
+    bool warn_deprecations;
+} prte_schizo_psched_component_t;
+
+PRTE_MODULE_EXPORT extern prte_schizo_psched_component_t prte_mca_schizo_psched_component;
+extern prte_schizo_base_module_t prte_schizo_psched_module;
+
+END_C_DECLS
+
+#endif /* MCA_SCHIZO_PSCHED_H_ */
diff --git a/src/mca/schizo/psched/schizo_psched_component.c b/src/mca/schizo/psched/schizo_psched_component.c
new file mode 100644
index 0000000000..02251cf6f5
--- /dev/null
+++ b/src/mca/schizo/psched/schizo_psched_component.c
@@ -0,0 +1,66 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2016-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2019      Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2021-2023 Nanook Consulting.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "prte_config.h"
+#include "types.h"
+
+#include "src/util/pmix_show_help.h"
+
+#include "src/runtime/prte_globals.h"
+
+#include "schizo_psched.h"
+#include "src/mca/schizo/schizo.h"
+
+static int component_query(pmix_mca_base_module_t **module, int *priority);
+static int component_register(void);
+
+/*
+ * Struct of function pointers and all that to let us be initialized
+ */
+prte_schizo_psched_component_t prte_mca_schizo_sched_component = {
+    .super = {
+        PRTE_MCA_SCHIZO_BASE_VERSION_1_0_0,
+        .pmix_mca_component_name = "psched",
+        PMIX_MCA_BASE_MAKE_VERSION(component,
+                                   PRTE_MAJOR_VERSION,
+                                   PRTE_MINOR_VERSION,
+                                   PMIX_RELEASE_VERSION),
+        .pmix_mca_query_component = component_query,
+        .pmix_mca_register_component_params = component_register,
+    },
+    .priority = 1,
+    .warn_deprecations = true
+};
+
+static int component_register(void)
+{
+    pmix_mca_base_component_t *c = &prte_mca_schizo_psched_component.super;
+
+    prte_mca_schizo_psched_component.warn_deprecations = true;
+    (void) pmix_mca_base_component_var_register(c, "warn_deprecations",
+                                                "Issue warnings about deprecated command line options",
+                                                PMIX_MCA_BASE_VAR_TYPE_BOOL,
+                                                &prte_mca_schizo_psched_component.warn_deprecations);
+
+    return PRTE_SUCCESS;
+}
+
+static int component_query(pmix_mca_base_module_t **module, int *priority)
+{
+    *module = (pmix_mca_base_module_t *) &prte_schizo_psched_module;
+    *priority = prte_mca_schizo_psched_component.priority;
+    return PRTE_SUCCESS;
+}
diff --git a/src/tools/Makefile.am b/src/tools/Makefile.am
index 3a85a9d931..8c22710bc1 100644
--- a/src/tools/Makefile.am
+++ b/src/tools/Makefile.am
@@ -31,7 +31,8 @@ SUBDIRS += \
 	tools/pcc \
     tools/prte_info \
     tools/prte \
-    tools/pterm
+    tools/pterm \
+    tools/psched
 
 DIST_SUBDIRS += \
 	tools/prted \
@@ -39,4 +40,5 @@ DIST_SUBDIRS += \
 	tools/pcc \
     tools/prte_info \
     tools/prte \
-    tools/pterm
+    tools/pterm \
+    tools/psched
diff --git a/src/tools/psched/Makefile.am b/src/tools/psched/Makefile.am
new file mode 100644
index 0000000000..4f4e7ba215
--- /dev/null
+++ b/src/tools/psched/Makefile.am
@@ -0,0 +1,54 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2007-2020 Cisco Systems, Inc.  All rights reserved
+# Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+# Copyright (c) 2016-2020 Intel, Inc.  All rights reserved.
+# Copyright (c) 2021-2023 Nanook Consulting.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+AM_LDFLAGS = $(prte_hwloc_LDFLAGS) $(prte_libevent_LDFLAGS) $(prte_pmix_LDFLAGS)
+AM_CFLAGS = \
+            -DPRTE_CONFIGURE_USER="\"@PRTE_CONFIGURE_USER@\"" \
+            -DPRTE_CONFIGURE_HOST="\"@PRTE_CONFIGURE_HOST@\"" \
+            -DPRTE_CONFIGURE_DATE="\"@PRTE_CONFIGURE_DATE@\"" \
+            -DPRTE_BUILD_USER="\"$$USER\"" \
+            -DPRTE_BUILD_HOST="\"$${HOSTNAME:-`(hostname || uname -n) | sed 1q`}\"" \
+            -DPRTE_BUILD_DATE="\"`$(top_srcdir)/config/getdate.sh`\"" \
+            -DPRTE_BUILD_CFLAGS="\"@CFLAGS@\"" \
+            -DPRTE_BUILD_CPPFLAGS="\"@CPPFLAGS@\"" \
+            -DPRTE_BUILD_LDFLAGS="\"@LDFLAGS@\"" \
+            -DPRTE_BUILD_LIBS="\"@LIBS@\"" \
+            -DPRTE_CC_ABSOLUTE="\"@PRTE_CC_ABSOLUTE@\"" \
+            -DPRTE_GREEK_VERSION="\"@PRTE_GREEK_VERSION@\"" \
+            -DPRTE_REPO_REV="\"@PRTE_REPO_REV@\"" \
+            -DPMIX_RELEASE_DATE="\"@PMIX_RELEASE_DATE@\""
+
+bin_PROGRAMS = psched
+
+psched_SOURCES = psched.c
+# the following empty psched_LDFLAGS is used
+#  so that the psched can be compiled statically
+#  by simply changing the value of this from
+#  nothing to -all-static in the Makefile.in
+#  nice for systems that don't have all the shared
+#  libraries on the computes
+psched_LDFLAGS =
+psched_LDADD = \
+    $(prte_libevent_LIBS) \
+    $(prte_hwloc_LIBS) \
+    $(prte_pmix_LIBS) \
+	$(top_builddir)/src/libprrte.la
diff --git a/src/tools/psched/psched.c b/src/tools/psched/psched.c
new file mode 100644
index 0000000000..6ef4090b23
--- /dev/null
+++ b/src/tools/psched/psched.c
@@ -0,0 +1,555 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2020 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2007-2016 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2009      Institut National de Recherche en Informatique
+ *                         et Automatique. All rights reserved.
+ * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2021-2023 Nanook Consulting  All rights reserved.
+ * Copyright (c) 2022      Triad National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "prte_config.h"
+#include "constants.h"
+
+#include <string.h>
+
+#include <ctype.h>
+#include <stdio.h>
+#ifdef HAVE_UNISTD_H
+#    include <unistd.h>
+#endif
+#ifdef HAVE_NETDB_H
+#    include <netdb.h>
+#endif
+#ifdef HAVE_SYS_PARAM_H
+#    include <sys/param.h>
+#endif
+#include <errno.h>
+#include <fcntl.h>
+#include <signal.h>
+#ifdef HAVE_SYS_TIME_H
+#    include <sys/time.h>
+#endif /* HAVE_SYS_TIME_H */
+#ifdef HAVE_SYS_STAT_H
+#    include <sys/stat.h>
+#endif
+#ifdef HAVE_SYS_WAIT_H
+#    include <sys/wait.h>
+#endif
+
+#include <pmix_server.h>
+
+#include "src/mca/base/pmix_base.h"
+#include "src/mca/base/pmix_mca_base_var.h"
+#include "src/pmix/pmix-internal.h"
+#include "src/runtime/pmix_init_util.h"
+#include "src/util/pmix_argv.h"
+#include "src/util/pmix_basename.h"
+#include "src/util/prte_cmd_line.h"
+#include "src/util/daemon_init.h"
+#include "src/util/pmix_os_path.h"
+#include "src/util/pmix_output.h"
+#include "src/util/pmix_printf.h"
+#include "src/util/pmix_environ.h"
+
+#include "src/util/name_fns.h"
+#include "src/util/pmix_parse_options.h"
+#include "src/util/proc_info.h"
+#include "src/util/pmix_show_help.h"
+#include "src/util/session_dir.h"
+
+#include "src/mca/prtebacktrace/base/base.h"
+#include "src/mca/prteinstalldirs/base/base.h"
+#include "src/mca/schizo/base/base.h"
+#include "src/mca/state/base/base.h"
+#include "src/runtime/runtime.h"
+
+/*
+ * Globals
+ */
+static pmix_cli_result_t results;
+
+typedef struct {
+    prte_pmix_lock_t lock;
+    pmix_info_t *info;
+    size_t ninfo;
+} myxfer_t;
+
+static void infocbfunc(pmix_status_t status, pmix_info_t *info, size_t ninfo, void *cbdata,
+                       pmix_release_cbfunc_t release_fn, void *release_cbdata)
+{
+    myxfer_t *xfer = (myxfer_t *) cbdata;
+    size_t n;
+    PRTE_HIDE_UNUSED_PARAMS(status);
+
+    if (NULL != info) {
+        xfer->ninfo = ninfo;
+        PMIX_INFO_CREATE(xfer->info, xfer->ninfo);
+        for (n = 0; n < ninfo; n++) {
+            PMIX_INFO_XFER(&xfer->info[n], &info[n]);
+        }
+    }
+
+    if (NULL != release_fn) {
+        release_fn(release_cbdata);
+    }
+    PRTE_PMIX_WAKEUP_THREAD(&xfer->lock);
+}
+
+static int wait_pipe[2];
+
+static int wait_dvm(pid_t pid)
+{
+    char reply;
+    int rc;
+    int status;
+
+    close(wait_pipe[1]);
+    do {
+        rc = read(wait_pipe[0], &reply, 1);
+    } while (0 > rc && EINTR == errno);
+
+    if (1 == rc && 'K' == reply) {
+        return 0;
+    } else if (0 == rc) {
+        waitpid(pid, &status, 0);
+        if (WIFEXITED(status)) {
+            return WEXITSTATUS(status);
+        }
+    }
+    return 255;
+}
+
+static bool check_exist(char *path)
+{
+    struct stat buf;
+    /* coverity[TOCTOU] */
+    if (0 == stat(path, &buf)) { /* exists */
+        return true;
+    }
+    return false;
+}
+
+int main(int argc, char *argv[])
+{
+    int ret = 0;
+    int i;
+    pmix_data_buffer_t *buffer;
+    pmix_value_t val;
+    pmix_proc_t proc;
+    pmix_status_t prc;
+    myxfer_t xfer;
+    pmix_data_buffer_t pbuf, *wbuf;
+    pmix_byte_object_t pbo;
+    int8_t flag;
+    uint8_t naliases, ni;
+    char **nonlocal = NULL, *personality;
+    int n;
+    pmix_value_t *vptr;
+    char **pargv;
+    int pargc;
+    prte_schizo_base_module_t *schizo;
+    pmix_cli_item_t *opt;
+    char *path = NULL;
+
+    /* initialize the globals */
+    prte_tool_basename = pmix_basename(argv[0]);
+    prte_tool_actual = "psched";
+    pargc = argc;
+    pargv = pmix_argv_copy_strip(argv);  // strip any quoted arguments
+
+    /* carry across the toolname */
+    pmix_tool_basename = prte_tool_basename;
+
+    /* initialize install dirs code */
+    ret = pmix_mca_base_framework_open(&prte_prteinstalldirs_base_framework,
+                                       PMIX_MCA_BASE_OPEN_DEFAULT);
+    if (PRTE_SUCCESS != ret) {
+        fprintf(stderr,
+                "prte_prteinstalldirs_base_open() failed -- process will likely abort (%s:%d, "
+                "returned %d instead of PRTE_SUCCESS)\n",
+                __FILE__, __LINE__, ret);
+        return ret;
+    }
+
+    /* initialize the MCA infrastructure */
+    if (check_exist(prte_install_dirs.prtelibdir)) {
+        pmix_asprintf(&path, "prte@%s", prte_install_dirs.prtelibdir);
+    }
+    ret = pmix_init_util(NULL, 0, path);
+    if (NULL != path) {
+        free(path);
+    }
+    if (PMIX_SUCCESS != ret) {
+        pmix_show_help("help-prte-runtime",
+                       "prte_init:startup:internal-failure", true,
+                       "pmix_init_util", PRTE_ERROR_NAME(ret), ret);
+        return prte_pmix_convert_status(ret);
+    }
+    ret = pmix_show_help_add_dir(prte_install_dirs.prtedatadir);
+    if (PMIX_SUCCESS != ret) {
+        pmix_show_help("help-prte-runtime",
+                       "prte_init:startup:internal-failure", true,
+                       "show_help_add_dir", PRTE_ERROR_NAME(ret), ret);
+        return prte_pmix_convert_status(ret);
+    }
+
+    /* we always need the prrte and pmix params */
+    ret = prte_schizo_base_parse_prte(pargc, 0, pargv, NULL);
+    if (PRTE_SUCCESS != ret) {
+        pmix_show_help("help-prte-runtime",
+                       "prte_init:startup:internal-failure", true,
+                       "parse_prte params", PRTE_ERROR_NAME(ret), ret);
+        return ret;
+    }
+
+    ret = prte_schizo_base_parse_pmix(pargc, 0, pargv, NULL);
+    if (PRTE_SUCCESS != ret) {
+        pmix_show_help("help-prte-runtime",
+                       "prte_init:startup:internal-failure", true,
+                       "parse_pmix params", PRTE_ERROR_NAME(ret), ret);
+        return ret;
+    }
+
+    /* initialize the memory allocator */
+    prte_malloc_init();
+
+    /* initialize the output system */
+    pmix_output_init();
+
+    /* keyval lex-based parser */
+    /* Setup the parameter system */
+    if (PRTE_SUCCESS != (ret = pmix_mca_base_var_init())) {
+        pmix_show_help("help-prte-runtime",
+                       "prte_init:startup:internal-failure", true,
+                       "var_init", PRTE_ERROR_NAME(ret), ret);
+        return ret;
+    }
+
+    /* set the nodename so anyone who needs it has it - this
+     * must come AFTER we initialize the installdirs */
+    prte_setup_hostname();
+
+    /* pretty-print stack handlers */
+    if (PRTE_SUCCESS != (ret = prte_util_register_stackhandlers())) {
+        pmix_show_help("help-prte-runtime",
+                       "prte_init:startup:internal-failure", true,
+                       "register_stackhandlers", PRTE_ERROR_NAME(ret), ret);
+        return ret;
+    }
+
+    /* pre-load any default mca param files */
+    preload_default_mca_params();
+
+    /* Register all MCA Params */
+    if (PRTE_SUCCESS != (ret = prte_register_params())) {
+        pmix_show_help("help-prte-runtime",
+                       "prte_init:startup:internal-failure", true,
+                       "register params", PRTE_ERROR_NAME(ret), ret);
+        return ret;
+    }
+
+    ret = pmix_mca_base_framework_open(&prte_prtebacktrace_base_framework,
+                                       PMIX_MCA_BASE_OPEN_DEFAULT);
+    if (PRTE_SUCCESS != ret) {
+        pmix_show_help("help-prte-runtime",
+                       "prte_init:startup:internal-failure", true,
+                       "open backtrace", PRTE_ERROR_NAME(ret), ret);
+        return ret;
+    }
+
+    /* open the SCHIZO framework */
+    ret = pmix_mca_base_framework_open(&prte_schizo_base_framework,
+                                       PMIX_MCA_BASE_OPEN_DEFAULT);
+    if (PRTE_SUCCESS != ret) {
+        PRTE_ERROR_LOG(ret);
+        return ret;
+    }
+
+    if (PRTE_SUCCESS != (ret = prte_schizo_base_select())) {
+        PRTE_ERROR_LOG(ret);
+        return ret;
+    }
+
+    /* get our schizo module */
+    personality = "psched";
+    schizo = prte_schizo_base_detect_proxy(personality);
+    if (NULL == schizo) {
+        pmix_show_help("help-schizo-base.txt", "no-proxy", true,
+                       prte_tool_basename, personality);
+        return 1;
+    }
+
+    /* parse the CLI to load the MCA params */
+    PMIX_CONSTRUCT(&results, pmix_cli_result_t);
+    ret = schizo->parse_cli(pargv, &results, PMIX_CLI_SILENT);
+    if (PRTE_SUCCESS != ret) {
+        if (PRTE_OPERATION_SUCCEEDED == ret) {
+            return PRTE_SUCCESS;
+        }
+        if (PRTE_ERR_SILENT != ret) {
+            fprintf(stderr, "%s: command line error (%s)\n", prte_tool_basename,
+                    prte_strerror(ret));
+        }
+        return ret;
+    }
+
+    /* check if we are running as root - if we are, then only allow
+     * us to proceed if the allow-run-as-root flag was given. Otherwise,
+     * exit with a giant warning message
+     */
+    if (0 == geteuid()) {
+        schizo->allow_run_as_root(&results); // will exit us if not allowed
+    }
+
+    /* check for debug options */
+    if (pmix_cmd_line_is_taken(&results, PRTE_CLI_DEBUG)) {
+        prte_debug_flag = true;
+    }
+    // leave_session_attached is used to indicate that we are not to daemonize
+    if (pmix_cmd_line_is_taken(&results, PRTE_CLI_LEAVE_SESSION_ATTACHED)) {
+        prte_leave_session_attached = true;
+    }
+
+    // detach from controlling terminal, if so directed
+    if (!prte_leave_session_attached) {
+        pipe(wait_pipe);
+        prte_state_base.parent_fd = wait_pipe[1];
+        prte_daemon_init_callback(NULL, wait_dvm);
+        close(wait_pipe[0]);
+#if defined(HAVE_SETSID)
+        setsid();
+#endif
+    }
+    /* ensure we silence any compression warnings */
+    PMIX_SETENV_COMPAT("PMIX_MCA_compress_base_silence_warning", "1", true, &environ);
+
+    /* run the prolog */
+    if (PRTE_SUCCESS != (ret = prte_ess_base_std_prolog())) {
+        pmix_show_help("help-prte-runtime",
+                       "prte_init:startup:internal-failure", true,
+                       "prte_ess_base_std_prolog", PRTE_ERROR_NAME(ret), ret);
+        return ret;
+    }
+
+    /* get the local topology */
+    if (NULL == prte_hwloc_topology) {
+        if (PRTE_SUCCESS != (ret = prte_hwloc_base_get_topology())) {
+            pmix_show_help("help-prte-runtime",
+                           "prte_init:startup:internal-failure", true,
+                           "topology discovery", PRTE_ERROR_NAME(ret), ret);
+            return ret;
+        }
+    }
+
+    /* open and setup the state machine */
+    ret = pmix_mca_base_framework_open(&prte_state_base_framework,
+                                       PMIX_MCA_BASE_OPEN_DEFAULT);
+    if (PRTE_SUCCESS != ret) {
+        pmix_show_help("help-prte-runtime",
+                       "prte_init:startup:internal-failure", true,
+                       "state open", PRTE_ERROR_NAME(ret), ret);
+        return ret;
+    }
+    if (PRTE_SUCCESS != (ret = prte_state_base_select())) {
+        pmix_show_help("help-prte-runtime",
+                       "prte_init:startup:internal-failure", true,
+                       "state select", PRTE_ERROR_NAME(ret), ret);
+        return ret;
+    }
+
+    /* open the errmgr */
+    ret = pmix_mca_base_framework_open(&prte_errmgr_base_framework,
+                                       PMIX_MCA_BASE_OPEN_DEFAULT);
+    if (PRTE_SUCCESS != ret) {
+        pmix_show_help("help-prte-runtime",
+                       "prte_init:startup:internal-failure", true,
+                       "prte_errmgr_base_open", PRTE_ERROR_NAME(ret), ret);
+        return ret;
+    }
+    /* set a name */
+    ret = prte_plm_base_set_hnp_name();
+    if (PRTE_SUCCESS != ret) {
+        pmix_show_help("help-prte-runtime",
+                       "prte_init:startup:internal-failure", true,
+                       "set_hnp_name", PRTE_ERROR_NAME(ret), ret);
+        return ret;
+    }
+
+    /* take a pass thru the session directory code to fillin the
+     * tmpdir names - don't create anything yet
+     */
+    ret = prte_session_dir(false, PRTE_PROC_MY_NAME);
+    if (PRTE_SUCCESS != ret) {
+        pmix_show_help("help-prte-runtime",
+                       "prte_init:startup:internal-failure", true,
+                       "session_dir define", PRTE_ERROR_NAME(ret), ret);
+        return ret;
+    }
+    /* clear the session directory just in case there are
+     * stale directories laying around
+     */
+    prte_session_dir_cleanup(PRTE_JOBID_WILDCARD);
+
+    /* now actually create the directory tree */
+    ret = prte_session_dir(true, PRTE_PROC_MY_NAME);
+    if (PRTE_SUCCESS != ret) {
+        pmix_show_help("help-prte-runtime",
+                       "prte_init:startup:internal-failure", true,
+                       "session_dir", PRTE_ERROR_NAME(ret), ret);
+        prte_exit_status = ret;
+        goto DONE;
+    }
+
+    /* setup the PMIx server - we need this here in case the
+     * communications infrastructure wants to register
+     * information */
+    if (PRTE_SUCCESS != (ret = pmix_server_init())) {
+        /* the server code already barked, so let's be quiet */
+        prte_exit_status = PRTE_ERR_SILENT;
+        goto DONE;
+    }
+    pmix_server_start();
+
+    /* setup the error manager */
+    if (PRTE_SUCCESS != (ret = prte_errmgr_base_select())) {
+        pmix_show_help("help-prte-runtime",
+                       "prte_init:startup:internal-failure", true,
+                       "errmgr_base_select", PRTE_ERROR_NAME(ret), ret);
+        prte_exit_status = ret;
+        goto DONE;
+     }
+
+    /* create my job data object */
+    jdata = PMIX_NEW(prte_job_t);
+    PMIX_LOAD_NSPACE(jdata->nspace, PRTE_PROC_MY_NAME->nspace);
+    prte_set_job_data_object(jdata);
+
+    /* set the schizo personality to "psched" by default */
+    jdata->schizo = (struct prte_schizo_base_module_t*)prte_schizo_base_detect_proxy("psched");
+    if (NULL == jdata->schizo) {
+        pmix_show_help("help-schizo-base.txt", "no-proxy", true, prte_tool_basename, "prte");
+        error = "select personality";
+        ret = PRTE_ERR_SILENT;
+        goto error;
+    }
+
+    /* every job requires at least one app */
+    app = PMIX_NEW(prte_app_context_t);
+    app->app = strdup(argv[0]);
+    app->argv = PMIX_ARGV_COPY_COMPAT(argv);
+    pmix_pointer_array_set_item(jdata->apps, 0, app);
+    jdata->num_apps++;
+    /* create and store a node object where we are */
+    node = PMIX_NEW(prte_node_t);
+    node->name = strdup(prte_process_info.nodename);
+    node->index = PRTE_PROC_MY_NAME->rank;
+    PRTE_FLAG_SET(node, PRTE_NODE_FLAG_LOC_VERIFIED);
+    pmix_pointer_array_set_item(prte_node_pool, PRTE_PROC_MY_NAME->rank, node);
+
+    /* create and store a proc object for us */
+    proc = PMIX_NEW(prte_proc_t);
+    PMIX_LOAD_PROCID(&proc->name, PRTE_PROC_MY_NAME->nspace, PRTE_PROC_MY_NAME->rank);
+    proc->job = jdata;
+    proc->rank = proc->name.rank;
+    proc->pid = prte_process_info.pid;
+    prte_oob_base_get_addr(&proc->rml_uri);
+    prte_process_info.my_hnp_uri = strdup(proc->rml_uri);
+    /* store it in the local PMIx repo for later retrieval */
+    PMIX_VALUE_LOAD(&pval, proc->rml_uri, PMIX_STRING);
+    if (PMIX_SUCCESS != (pret = PMIx_Store_internal(PRTE_PROC_MY_NAME, PMIX_PROC_URI, &pval))) {
+        PMIX_ERROR_LOG(pret);
+        ret = PRTE_ERROR;
+        PMIX_VALUE_DESTRUCT(&pval);
+        error = "store uri";
+        goto error;
+    }
+    PMIX_VALUE_DESTRUCT(&pval);
+    proc->state = PRTE_PROC_STATE_RUNNING;
+    PMIX_RETAIN(node); /* keep accounting straight */
+    proc->node = node;
+    pmix_pointer_array_set_item(jdata->procs, PRTE_PROC_MY_NAME->rank, proc);
+
+    /* setup to detect any external allocation */
+    if (PRTE_SUCCESS
+        != (ret = pmix_mca_base_framework_open(&prte_ras_base_framework,
+                                               PMIX_MCA_BASE_OPEN_DEFAULT))) {
+        PRTE_ERROR_LOG(ret);
+        error = "prte_ras_base_open";
+        goto error;
+    }
+    if (PRTE_SUCCESS != (ret = prte_ras_base_select())) {
+        PRTE_ERROR_LOG(ret);
+        error = "prte_ras_base_find_available";
+        goto error;
+    }
+
+    /* add our topology to the array of known topologies */
+    t = PMIX_NEW(prte_topology_t);
+    t->topo = prte_hwloc_topology;
+    /* generate the signature */
+    prte_topo_signature = prte_hwloc_base_get_topo_signature(prte_hwloc_topology);
+    t->sig = strdup(prte_topo_signature);
+    t->index = pmix_pointer_array_add(prte_node_topologies, t);
+    node->topology = t;
+    node->available = prte_hwloc_base_filter_cpus(prte_hwloc_topology);
+    if (15 < pmix_output_get_verbosity(prte_ess_base_framework.framework_output)) {
+        char *output = NULL;
+        pmix_output(0, "%s Topology Info:", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME));
+        prte_hwloc_print(&output, "\t", prte_hwloc_topology);
+        pmix_output(0, "%s", output);
+        free(output);
+    }
+
+    /* output a message indicating we are alive, our name, and our pid
+     * for debugging purposes
+     */
+    if (prte_debug_flag) {
+        fprintf(stderr, "Scheduler %s checking in as pid %ld on host %s\n",
+                PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (long) prte_process_info.pid,
+                prte_process_info.nodename);
+    }
+
+    /* loop the event lib until an exit event is detected */
+    while (prte_event_base_active) {
+        prte_event_loop(prte_event_base, PRTE_EVLOOP_ONCE);
+    }
+    PMIX_ACQUIRE_OBJECT(prte_event_base_active);
+
+DONE:
+    /* update the exit status, in case it wasn't done */
+    PRTE_UPDATE_EXIT_STATUS(ret);
+
+    /* cleanup and leave */
+    prte_finalize();
+
+    prte_session_dir_cleanup(PRTE_PROC_MY_NAME->nspace);
+    /* cleanup the process info */
+    prte_proc_info_finalize();
+
+    if (prte_debug_flag) {
+        fprintf(stderr, "exiting with status %d\n", prte_exit_status);
+    }
+    exit(prte_exit_status);
+}

From 20e76965f49e43da541db283d5ed175536bd8ba9 Mon Sep 17 00:00:00 2001
From: Ralph Castain <rhc@pmix.org>
Date: Wed, 26 Jul 2023 11:08:38 -0600
Subject: [PATCH 2/4] Minor cleanups

Formatting changes and ensuring we return the right error
code in a few places

Signed-off-by: Ralph Castain <rhc@pmix.org>
---
 src/mca/state/dvm/state_dvm.c     | 24 ++++++++++++------------
 src/mca/state/prted/state_prted.c | 24 ++++++++++++------------
 src/prted/pmix/pmix_server.c      |  7 +++++++
 src/runtime/prte_init.c           |  7 +++----
 src/runtime/runtime_internals.h   |  2 ++
 5 files changed, 36 insertions(+), 28 deletions(-)

diff --git a/src/mca/state/dvm/state_dvm.c b/src/mca/state/dvm/state_dvm.c
index da42dc7142..a51479021a 100644
--- a/src/mca/state/dvm/state_dvm.c
+++ b/src/mca/state/dvm/state_dvm.c
@@ -64,18 +64,18 @@ static void ready_for_debug(int fd, short args, void *cbata);
  * DVM module - used when mpirun is persistent
  ******************/
 prte_state_base_module_t prte_state_dvm_module = {
-    init,
-    finalize,
-    prte_state_base_activate_job_state,
-    prte_state_base_add_job_state,
-    prte_state_base_set_job_state_callback,
-    prte_state_base_set_job_state_priority,
-    prte_state_base_remove_job_state,
-    prte_state_base_activate_proc_state,
-    prte_state_base_add_proc_state,
-    prte_state_base_set_proc_state_callback,
-    prte_state_base_set_proc_state_priority,
-    prte_state_base_remove_proc_state
+    .init = init,
+    .finalize = finalize,
+    .activate_job_state = prte_state_base_activate_job_state,
+    .add_job_state = prte_state_base_add_job_state,
+    .set_job_state_callback = prte_state_base_set_job_state_callback,
+    .set_job_state_priority = prte_state_base_set_job_state_priority,
+    .remove_job_state = prte_state_base_remove_job_state,
+    .activate_proc_state = prte_state_base_activate_proc_state,
+    .add_proc_state = prte_state_base_add_proc_state,
+    .set_proc_state_callback = prte_state_base_set_proc_state_callback,
+    .set_proc_state_priority = prte_state_base_set_proc_state_priority,
+    .remove_proc_state = prte_state_base_remove_proc_state
 };
 
 static void dvm_notify(int sd, short args, void *cbdata);
diff --git a/src/mca/state/prted/state_prted.c b/src/mca/state/prted/state_prted.c
index 8ce616d6a1..4c126120b3 100644
--- a/src/mca/state/prted/state_prted.c
+++ b/src/mca/state/prted/state_prted.c
@@ -49,18 +49,18 @@ static int finalize(void);
  * PRTED module
  ******************/
 prte_state_base_module_t prte_state_prted_module = {
-    init,
-    finalize,
-    prte_state_base_activate_job_state,
-    prte_state_base_add_job_state,
-    prte_state_base_set_job_state_callback,
-    prte_state_base_set_job_state_priority,
-    prte_state_base_remove_job_state,
-    prte_state_base_activate_proc_state,
-    prte_state_base_add_proc_state,
-    prte_state_base_set_proc_state_callback,
-    prte_state_base_set_proc_state_priority,
-    prte_state_base_remove_proc_state
+    .init = init,
+    .finalize = finalize,
+    .activate_job_state = prte_state_base_activate_job_state,
+    .add_job_state = prte_state_base_add_job_state,
+    .set_job_state_callback = prte_state_base_set_job_state_callback,
+    .set_job_state_priority = prte_state_base_set_job_state_priority,
+    .remove_job_state = prte_state_base_remove_job_state,
+    .activate_proc_state = prte_state_base_activate_proc_state,
+    .add_proc_state = prte_state_base_add_proc_state,
+    .set_proc_state_callback = prte_state_base_set_proc_state_callback,
+    .set_proc_state_priority = prte_state_base_set_proc_state_priority,
+    .remove_proc_state = prte_state_base_remove_proc_state
 };
 
 /* Local functions */
diff --git a/src/prted/pmix/pmix_server.c b/src/prted/pmix/pmix_server.c
index 1d9965ef0d..3ad4a4ead3 100644
--- a/src/prted/pmix/pmix_server.c
+++ b/src/prted/pmix/pmix_server.c
@@ -596,6 +596,7 @@ int pmix_server_init(void)
     PMIX_INFO_LIST_ADD(prc, ilist, PMIX_HOSTNAME, prte_process_info.nodename, PMIX_STRING);
     if (PMIX_SUCCESS != prc) {
         PMIX_INFO_LIST_RELEASE(ilist);
+        rc = prte_pmix_convert_status(prc);
         return rc;
     }
 
@@ -604,6 +605,7 @@ int pmix_server_init(void)
     PMIX_INFO_LIST_ADD(prc, ilist, PMIX_EXTERNAL_AUX_EVENT_BASE, prte_event_base, PMIX_POINTER);
     if (PMIX_SUCCESS != prc) {
         PMIX_INFO_LIST_RELEASE(ilist);
+        rc = prte_pmix_convert_status(prc);
         return rc;
     }
 #endif
@@ -620,12 +622,14 @@ int pmix_server_init(void)
     PMIX_INFO_LIST_INSERT(prc, ilist, &myinf);
     if (PMIX_SUCCESS != prc) {
         PMIX_INFO_LIST_RELEASE(ilist);
+        rc = prte_pmix_convert_status(prc);
         return rc;
     }
     // tell the server to share this topology for us
     PMIX_INFO_LIST_ADD(prc, ilist, PMIX_SERVER_SHARE_TOPOLOGY, NULL, PMIX_BOOL);
     if (PMIX_SUCCESS != prc) {
         PMIX_INFO_LIST_RELEASE(ilist);
+        rc = prte_pmix_convert_status(prc);
         return rc;
     }
 
@@ -838,6 +842,7 @@ int pmix_server_init(void)
     PMIX_INFO_LIST_ADD(prc, ilist, PMIX_HOSTNAME, prte_process_info.nodename, PMIX_STRING);
     if (PMIX_SUCCESS != rc) {
         PMIX_INFO_LIST_RELEASE(ilist);
+        rc = prte_pmix_convert_status(prc);
         return rc;
     }
 
@@ -848,6 +853,7 @@ int pmix_server_init(void)
         free(tmp);
         if (PMIX_SUCCESS != rc) {
             PMIX_INFO_LIST_RELEASE(ilist);
+            rc = prte_pmix_convert_status(prc);
             return rc;
         }
     }
@@ -872,6 +878,7 @@ int pmix_server_init(void)
     PMIx_Register_event_handler(&prc, 1, NULL, 0, lost_connection_hdlr, regcbfunc, &lock);
     PRTE_PMIX_WAIT_THREAD(&lock);
     PRTE_PMIX_DESTRUCT_LOCK(&lock);
+    rc = prte_pmix_convert_status(prc);
 
     return rc;
 }
diff --git a/src/runtime/prte_init.c b/src/runtime/prte_init.c
index 9c86b51e45..649d8bf701 100644
--- a/src/runtime/prte_init.c
+++ b/src/runtime/prte_init.c
@@ -83,6 +83,7 @@
 #include "src/runtime/prte_globals.h"
 #include "src/runtime/prte_locks.h"
 #include "src/runtime/runtime.h"
+#include "src/runtime/runtime_internals.h"
 
 /*
  * Whether we have completed prte_init or we are in prte_finalize
@@ -116,8 +117,6 @@ static bool min_initialized = false;
 #endif
 const char prte_version_string[] = PRTE_IDENT_STRING;
 
-static void preload_default_mca_params(void);
-
 static bool check_exist(char *path)
 {
     struct stat buf;
@@ -222,7 +221,7 @@ int prte_init_util(prte_proc_type_t flags)
     }
 
     /* pre-load any default mca param files */
-    preload_default_mca_params();
+    prte_preload_default_mca_params();
 
     /* Register all MCA Params */
     if (PRTE_SUCCESS != (ret = prte_register_params())) {
@@ -417,7 +416,7 @@ static bool check_pmix_overlap(char *var, char *value)
     return false;
 }
 
-static void preload_default_mca_params(void)
+void prte_preload_default_mca_params(void)
 {
     char *file, *home, *tmp;
     pmix_list_t params, params2, pfinal;
diff --git a/src/runtime/runtime_internals.h b/src/runtime/runtime_internals.h
index 8433272392..cd48bf4066 100644
--- a/src/runtime/runtime_internals.h
+++ b/src/runtime/runtime_internals.h
@@ -37,6 +37,8 @@ BEGIN_C_DECLS
  */
 PRTE_EXPORT int prte_dt_init(void);
 
+PRTE_EXPORT void prte_preload_default_mca_params(void);
+
 END_C_DECLS
 
 #endif /* PRTE_RUNTIME_INTERNALS_H */

From edad1897783a313d45bb1626b853d36831e009d0 Mon Sep 17 00:00:00 2001
From: Ralph Castain <rhc@pmix.org>
Date: Wed, 26 Jul 2023 11:09:49 -0600
Subject: [PATCH 3/4] Continue work on psched tool

Continue building out new pseudo-scheduler tool for
investigating dynamic environments

Signed-off-by: Ralph Castain <rhc@pmix.org>
---
 src/mca/schizo/psched/Makefile.am             |   44 -
 src/mca/schizo/psched/configure.m4            |   37 -
 src/mca/schizo/psched/owner.txt               |    7 -
 src/mca/schizo/psched/schizo_psched.h         |   37 -
 .../schizo/psched/schizo_psched_component.c   |   66 -
 src/tools/psched/Makefile.am                  |   16 +-
 src/tools/psched/backend.c                    |  734 +++++++++++
 src/tools/psched/errmgr.c                     |  693 +++++++++++
 src/tools/psched/event.c                      |  262 ++++
 .../psched/help-psched.txt}                   |    0
 src/tools/psched/psched.c                     |  313 +++--
 src/tools/psched/psched.h                     |  108 ++
 src/tools/psched/queries.c                    |  832 +++++++++++++
 src/tools/psched/scheduler.c                  |   35 +
 .../schizo_psched.c => tools/psched/schizo.c} |    4 +-
 src/tools/psched/server.c                     |  532 ++++++++
 src/tools/psched/session.c                    |  248 ++++
 src/tools/psched/state.c                      | 1074 +++++++++++++++++
 18 files changed, 4750 insertions(+), 292 deletions(-)
 delete mode 100644 src/mca/schizo/psched/Makefile.am
 delete mode 100644 src/mca/schizo/psched/configure.m4
 delete mode 100644 src/mca/schizo/psched/owner.txt
 delete mode 100644 src/mca/schizo/psched/schizo_psched.h
 delete mode 100644 src/mca/schizo/psched/schizo_psched_component.c
 create mode 100644 src/tools/psched/backend.c
 create mode 100644 src/tools/psched/errmgr.c
 create mode 100644 src/tools/psched/event.c
 rename src/{mca/schizo/psched/help-schizo-psched.txt => tools/psched/help-psched.txt} (100%)
 create mode 100644 src/tools/psched/psched.h
 create mode 100644 src/tools/psched/queries.c
 create mode 100644 src/tools/psched/scheduler.c
 rename src/{mca/schizo/psched/schizo_psched.c => tools/psched/schizo.c} (99%)
 create mode 100644 src/tools/psched/server.c
 create mode 100644 src/tools/psched/session.c
 create mode 100644 src/tools/psched/state.c

diff --git a/src/mca/schizo/psched/Makefile.am b/src/mca/schizo/psched/Makefile.am
deleted file mode 100644
index 9ee72ed754..0000000000
--- a/src/mca/schizo/psched/Makefile.am
+++ /dev/null
@@ -1,44 +0,0 @@
-#
-# Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
-# Copyright (c) 2017      IBM Corporation.  All rights reserved.
-# Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved
-# Copyright (c) 2022-2023 Nanook Consulting.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-AM_CFLAGS = \
-    -DPRTE_GREEK_VERSION="\"@PRTE_GREEK_VERSION@\""
-
-dist_prtedata_DATA = \
-    help-schizo-psched.txt
-
-sources = \
-          schizo_psched_component.c \
-          schizo_psched.h \
-          schizo_psched.c
-
-# Make the output library in this directory, and name it either
-# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
-# (for static builds).
-
-if MCA_BUILD_prte_schizo_psched_DSO
-component_noinst =
-component_install = prte_mca_schizo_psched.la
-else
-component_noinst = libprtemca_schizo_psched.la
-component_install =
-endif
-
-mcacomponentdir = $(prtelibdir)
-mcacomponent_LTLIBRARIES = $(component_install)
-prte_mca_schizo_psched_la_SOURCES = $(sources)
-prte_mca_schizo_psched_la_LDFLAGS = -module -avoid-version
-prte_mca_schizo_psched_la_LIBADD = $(top_builddir)/src/libprrte.la
-
-noinst_LTLIBRARIES = $(component_noinst)
-libprtemca_schizo_psched_la_SOURCES = $(sources)
-libprtemca_schizo_psched_la_LDFLAGS = -module -avoid-version
diff --git a/src/mca/schizo/psched/configure.m4 b/src/mca/schizo/psched/configure.m4
deleted file mode 100644
index 9efa13e9df..0000000000
--- a/src/mca/schizo/psched/configure.m4
+++ /dev/null
@@ -1,37 +0,0 @@
-# -*- shell-script -*-
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2009-2020 Cisco Systems, Inc.  All rights reserved
-# Copyright (c) 2011-2013 Los Alamos National Security, LLC.
-#                         All rights reserved.
-# Copyright (c) 2019      Intel, Inc.  All rights reserved.
-# Copyright (c) 2022-2023 Nanook Consulting.  All rights reserved.
-# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
-#                         All Rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-# MCA_schizo_psched_CONFIG([action-if-found], [action-if-not-found])
-# -----------------------------------------------------------
-AC_DEFUN([MCA_prte_schizo_psched_CONFIG],[
-    AC_CONFIG_FILES([src/mca/schizo/psched/Makefile])
-
-    AS_IF([test "yes" = "yes"],
-          [$1], [$2])
-
-    PRTE_SUMMARY_ADD([Personalities], [PRTE Scheduler], [], [yes])
-
-])dnl
diff --git a/src/mca/schizo/psched/owner.txt b/src/mca/schizo/psched/owner.txt
deleted file mode 100644
index a6532f448e..0000000000
--- a/src/mca/schizo/psched/owner.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-#
-# owner/status file
-# owner: institution that is responsible for this package
-# status: e.g. active, maintenance, unmaintained
-#
-owner: Nanook
-status: devel
diff --git a/src/mca/schizo/psched/schizo_psched.h b/src/mca/schizo/psched/schizo_psched.h
deleted file mode 100644
index 3c00f77c77..0000000000
--- a/src/mca/schizo/psched/schizo_psched.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
- * Copyright (c) 2019      Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved
- * Copyright (c) 2021-2023 Nanook Consulting.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef _MCA_SCHIZO_PSCHED_H_
-#define _MCA_SCHIZO_PSCHED_H_
-
-#include "prte_config.h"
-
-#include "types.h"
-
-#include "src/mca/base/pmix_base.h"
-#include "src/mca/schizo/schizo.h"
-
-BEGIN_C_DECLS
-
-typedef struct {
-    prte_schizo_base_component_t super;
-    int priority;
-    bool warn_deprecations;
-} prte_schizo_psched_component_t;
-
-PRTE_MODULE_EXPORT extern prte_schizo_psched_component_t prte_mca_schizo_psched_component;
-extern prte_schizo_base_module_t prte_schizo_psched_module;
-
-END_C_DECLS
-
-#endif /* MCA_SCHIZO_PSCHED_H_ */
diff --git a/src/mca/schizo/psched/schizo_psched_component.c b/src/mca/schizo/psched/schizo_psched_component.c
deleted file mode 100644
index 02251cf6f5..0000000000
--- a/src/mca/schizo/psched/schizo_psched_component.c
+++ /dev/null
@@ -1,66 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2016-2020 Intel, Inc.  All rights reserved.
- * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2019      Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved
- * Copyright (c) 2021-2023 Nanook Consulting.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "prte_config.h"
-#include "types.h"
-
-#include "src/util/pmix_show_help.h"
-
-#include "src/runtime/prte_globals.h"
-
-#include "schizo_psched.h"
-#include "src/mca/schizo/schizo.h"
-
-static int component_query(pmix_mca_base_module_t **module, int *priority);
-static int component_register(void);
-
-/*
- * Struct of function pointers and all that to let us be initialized
- */
-prte_schizo_psched_component_t prte_mca_schizo_sched_component = {
-    .super = {
-        PRTE_MCA_SCHIZO_BASE_VERSION_1_0_0,
-        .pmix_mca_component_name = "psched",
-        PMIX_MCA_BASE_MAKE_VERSION(component,
-                                   PRTE_MAJOR_VERSION,
-                                   PRTE_MINOR_VERSION,
-                                   PMIX_RELEASE_VERSION),
-        .pmix_mca_query_component = component_query,
-        .pmix_mca_register_component_params = component_register,
-    },
-    .priority = 1,
-    .warn_deprecations = true
-};
-
-static int component_register(void)
-{
-    pmix_mca_base_component_t *c = &prte_mca_schizo_psched_component.super;
-
-    prte_mca_schizo_psched_component.warn_deprecations = true;
-    (void) pmix_mca_base_component_var_register(c, "warn_deprecations",
-                                                "Issue warnings about deprecated command line options",
-                                                PMIX_MCA_BASE_VAR_TYPE_BOOL,
-                                                &prte_mca_schizo_psched_component.warn_deprecations);
-
-    return PRTE_SUCCESS;
-}
-
-static int component_query(pmix_mca_base_module_t **module, int *priority)
-{
-    *module = (pmix_mca_base_module_t *) &prte_schizo_psched_module;
-    *priority = prte_mca_schizo_psched_component.priority;
-    return PRTE_SUCCESS;
-}
diff --git a/src/tools/psched/Makefile.am b/src/tools/psched/Makefile.am
index 4f4e7ba215..0b75942d65 100644
--- a/src/tools/psched/Makefile.am
+++ b/src/tools/psched/Makefile.am
@@ -39,7 +39,21 @@ AM_CFLAGS = \
 
 bin_PROGRAMS = psched
 
-psched_SOURCES = psched.c
+dist_prtedata_DATA = help-psched.txt
+
+psched_SOURCES = \
+	psched.h \
+	psched.c \
+	errmgr.c \
+	state.c \
+	server.c \
+	scheduler.c \
+	backend.c \
+	event.c \
+	queries.c \
+	schizo.c \
+	session.c
+
 # the following empty psched_LDFLAGS is used
 #  so that the psched can be compiled statically
 #  by simply changing the value of this from
diff --git a/src/tools/psched/backend.c b/src/tools/psched/backend.c
new file mode 100644
index 0000000000..32ef637d05
--- /dev/null
+++ b/src/tools/psched/backend.c
@@ -0,0 +1,734 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2009-2020 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2017 Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2014-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2020      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2021-2023 Nanook Consulting  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include "prte_config.h"
+
+#ifdef HAVE_UNISTD_H
+#    include <unistd.h>
+#endif
+
+#include "src/hwloc/hwloc-internal.h"
+#include "src/pmix/pmix-internal.h"
+#include "src/util/pmix_argv.h"
+#include "src/util/pmix_output.h"
+
+#include "src/mca/errmgr/errmgr.h"
+#include "src/mca/grpcomm/base/base.h"
+#include "src/mca/iof/base/base.h"
+#include "src/mca/iof/iof.h"
+#include "src/mca/plm/base/plm_private.h"
+#include "src/mca/plm/plm.h"
+#include "src/mca/plm/base/plm_private.h"
+#include "src/mca/rmaps/rmaps_types.h"
+#include "src/rml/rml.h"
+#include "src/mca/schizo/schizo.h"
+#include "src/mca/state/state.h"
+#include "src/runtime/prte_globals.h"
+#include "src/runtime/prte_locks.h"
+#include "src/threads/pmix_threads.h"
+#include "src/util/name_fns.h"
+#include "src/util/pmix_show_help.h"
+
+#include "src/prted/pmix/pmix_server_internal.h"
+#include "src/tools/psched/psched.h"
+
+static void _toolconn(int sd, short args, void *cbdata)
+{
+    pmix_server_req_t *cd = (pmix_server_req_t *) cbdata;
+    int rc;
+    char *tmp;
+    size_t n;
+    pmix_data_buffer_t *buf;
+    prte_plm_cmd_flag_t command = PRTE_PLM_ALLOC_JOBID_CMD;
+    pmix_status_t xrc;
+    PRTE_HIDE_UNUSED_PARAMS(sd, args);
+
+    PMIX_ACQUIRE_OBJECT(cd);
+
+    pmix_output_verbose(2, prte_pmix_server_globals.output,
+                        "%s PROCESSING TOOL CONNECTION",
+                        PRTE_NAME_PRINT(PRTE_PROC_MY_NAME));
+
+    /* check for directives */
+    if (NULL != cd->info) {
+        for (n = 0; n < cd->ninfo; n++) {
+            if (PMIX_CHECK_KEY(&cd->info[n], PMIX_EVENT_SILENT_TERMINATION)) {
+                cd->flag = PMIX_INFO_TRUE(&cd->info[n]);
+            } else if (PMIX_CHECK_KEY(&cd->info[n], PMIX_VERSION_INFO)) {
+                /* we ignore this for now */
+            } else if (PMIX_CHECK_KEY(&cd->info[n], PMIX_USERID)) {
+                PMIX_VALUE_GET_NUMBER(xrc, &cd->info[n].value, cd->uid, uid_t);
+                if (PMIX_SUCCESS != xrc) {
+                    if (NULL != cd->toolcbfunc) {
+                        cd->toolcbfunc(xrc, NULL, cd->cbdata);
+                    }
+                    PMIX_RELEASE(cd);
+                    return;
+                }
+            } else if (PMIX_CHECK_KEY(&cd->info[n], PMIX_GRPID)) {
+                PMIX_VALUE_GET_NUMBER(xrc, &cd->info[n].value, cd->gid, gid_t);
+                if (PMIX_SUCCESS != xrc) {
+                    if (NULL != cd->toolcbfunc) {
+                        cd->toolcbfunc(xrc, NULL, cd->cbdata);
+                    }
+                    PMIX_RELEASE(cd);
+                    return;
+                }
+            } else if (PMIX_CHECK_KEY(&cd->info[n], PMIX_NSPACE)) {
+                PMIX_LOAD_NSPACE(cd->target.nspace, cd->info[n].value.data.string);
+            } else if (PMIX_CHECK_KEY(&cd->info[n], PMIX_RANK)) {
+                cd->target.rank = cd->info[n].value.data.rank;
+            } else if (PMIX_CHECK_KEY(&cd->info[n], PMIX_HOSTNAME)) {
+                cd->operation = strdup(cd->info[n].value.data.string);
+            } else if (PMIX_CHECK_KEY(&cd->info[n], PMIX_CMD_LINE)) {
+                cd->cmdline = strdup(cd->info[n].value.data.string);
+            } else if (PMIX_CHECK_KEY(&cd->info[n], PMIX_LAUNCHER)) {
+                cd->launcher = PMIX_INFO_TRUE(&cd->info[n]);
+            } else if (PMIX_CHECK_KEY(&cd->info[n], PMIX_SERVER_SCHEDULER)) {
+                cd->scheduler = PMIX_INFO_TRUE(&cd->info[n]);
+            } else if (PMIX_CHECK_KEY(&cd->info[n], PMIX_PROC_PID)) {
+                PMIX_VALUE_GET_NUMBER(xrc, &cd->info[n].value, cd->pid, pid_t);
+                if (PMIX_SUCCESS != xrc) {
+                    if (NULL != cd->toolcbfunc) {
+                        cd->toolcbfunc(xrc, NULL, cd->cbdata);
+                    }
+                    PMIX_RELEASE(cd);
+                    return;
+                }
+            }
+        }
+    }
+
+    pmix_output_verbose(2, prte_pmix_server_globals.output,
+                        "%s %s CONNECTION FROM UID %d GID %d NSPACE %s",
+                        PRTE_NAME_PRINT(PRTE_PROC_MY_NAME),
+                        cd->launcher ? "LAUNCHER" : (cd->scheduler ? "SCHEDULER" : "TOOL"),
+                        cd->uid, cd->gid, cd->target.nspace);
+
+    /* if this is the scheduler and we are not the DVM master, then
+     * this is not allowed */
+    if (cd->scheduler) {
+        if (!PRTE_PROC_IS_MASTER) {
+            cd->toolcbfunc(PMIX_ERR_NOT_SUPPORTED, NULL, cd->cbdata);
+            PMIX_RELEASE(cd);
+            return;
+        } else {
+            /* mark that the scheduler has attached to us */
+            prte_pmix_server_globals.scheduler_connected = true;
+            PMIX_LOAD_PROCID(&prte_pmix_server_globals.scheduler,
+                             cd->target.nspace, cd->target.rank);
+            /* we cannot immediately set the scheduler to be our
+             * PMIx server as the PMIx library hasn't finished
+             * recording it */
+        }
+    }
+
+    /* if we are not the HNP or master, and the tool doesn't
+     * already have a self-assigned name, then
+     * we need to ask the master for one */
+    if (PMIX_NSPACE_INVALID(cd->target.nspace) || PMIX_RANK_INVALID == cd->target.rank) {
+        /* if we are the HNP, we can directly assign the jobid */
+        if (PRTE_PROC_IS_MASTER) {
+            /* the new nspace is our base nspace with an "@N" extension */
+            pmix_asprintf(&tmp, "%s@%u", prte_plm_globals.base_nspace, prte_plm_globals.next_jobid);
+            PMIX_LOAD_PROCID(&cd->target, tmp, 0);
+            free(tmp);
+            prte_plm_globals.next_jobid++;
+        } else {
+            cd->local_index = pmix_pointer_array_add(&prte_pmix_server_globals.local_reqs, cd);
+            /* we need to send this to the HNP for a jobid */
+            PMIX_DATA_BUFFER_CREATE(buf);
+            rc = PMIx_Data_pack(NULL, buf, &command, 1, PMIX_UINT8);
+            if (PMIX_SUCCESS != rc) {
+                PMIX_ERROR_LOG(rc);
+            }
+            rc = PMIx_Data_pack(NULL, buf, &cd->local_index, 1, PMIX_INT);
+            if (PMIX_SUCCESS != rc) {
+                PMIX_ERROR_LOG(rc);
+            }
+            /* send it to the HNP for processing - might be myself! */
+            PRTE_RML_SEND(rc, PRTE_PROC_MY_HNP->rank,
+                          buf, PRTE_RML_TAG_PLM);
+            if (PRTE_SUCCESS != rc) {
+                PRTE_ERROR_LOG(rc);
+                xrc = prte_pmix_convert_rc(rc);
+                pmix_pointer_array_set_item(&prte_pmix_server_globals.local_reqs, cd->local_index, NULL);
+                PMIX_DATA_BUFFER_RELEASE(buf);
+                if (NULL != cd->toolcbfunc) {
+                    cd->toolcbfunc(xrc, NULL, cd->cbdata);
+                }
+                PMIX_RELEASE(cd);
+            }
+            return;
+        }
+    }
+
+    /* the tool is not a client of ours, but we can provide at least some information */
+    rc = prte_pmix_server_register_tool(cd->target.nspace);
+    if (PMIX_SUCCESS != rc) {
+        rc = prte_pmix_convert_rc(rc);
+    }
+    if (NULL != cd->toolcbfunc) {
+        cd->toolcbfunc(rc, &cd->target, cd->cbdata);
+    }
+    PMIX_RELEASE(cd);
+}
+
+void psched_tool_connected_fn(pmix_info_t *info, size_t ninfo,
+                              pmix_tool_connection_cbfunc_t cbfunc,
+                              void *cbdata)
+{
+    pmix_server_req_t *cd;
+
+    pmix_output_verbose(2, prte_pmix_server_globals.output,
+                        "%s TOOL CONNECTION REQUEST RECVD",
+                        PRTE_NAME_PRINT(PRTE_PROC_MY_NAME));
+
+    /* need to threadshift this request */
+    cd = PMIX_NEW(pmix_server_req_t);
+    cd->info = info;
+    cd->ninfo = ninfo;
+    cd->toolcbfunc = cbfunc;
+    cd->cbdata = cbdata;
+    cd->target.rank = 0; // set default for tool
+
+    prte_event_set(prte_event_base, &(cd->ev), -1, PRTE_EV_WRITE, _toolconn, cd);
+    prte_event_set_priority(&(cd->ev), PRTE_MSG_PRI);
+    PMIX_POST_OBJECT(cd);
+    prte_event_active(&(cd->ev), PRTE_EV_WRITE, 1);
+}
+
+static void lgcbfn(int sd, short args, void *cbdata)
+{
+    prte_pmix_server_op_caddy_t *cd = (prte_pmix_server_op_caddy_t *) cbdata;
+    PRTE_HIDE_UNUSED_PARAMS(sd, args);
+
+    if (NULL != cd->cbfunc) {
+        cd->cbfunc(cd->status, cd->cbdata);
+    }
+    PMIX_RELEASE(cd);
+}
+
+void psched_log_fn(const pmix_proc_t *client,
+                   const pmix_info_t data[], size_t ndata,
+                   const pmix_info_t directives[], size_t ndirs,
+                   pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    size_t n, cnt, dcnt;
+    pmix_data_buffer_t *buf;
+    int rc = PRTE_SUCCESS;
+    pmix_data_buffer_t pbuf, dbuf;
+    pmix_byte_object_t pbo, dbo;
+    pmix_status_t ret;
+
+    pmix_output_verbose(2, prte_pmix_server_globals.output,
+                        "%s logging info",
+                        PRTE_NAME_PRINT(PRTE_PROC_MY_NAME));
+
+    PMIX_DATA_BUFFER_CONSTRUCT(&dbuf);
+    /* if we are the one that passed it down, then we don't pass it back */
+    dcnt = 0;
+    for (n = 0; n < ndirs; n++) {
+        if (PMIX_CHECK_KEY(&directives[n], "prte.log.noloop")) {
+            if (PMIX_INFO_TRUE(&directives[n])) {
+                rc = PMIX_SUCCESS;
+                goto done;
+            }
+        }
+        else {
+            ret = PMIx_Data_pack(NULL, &dbuf, (pmix_info_t *) &directives[n], 1, PMIX_INFO);
+            if (PMIX_SUCCESS != ret) {
+                PMIX_ERROR_LOG(ret);
+            }
+            dcnt++;
+        }
+    }
+
+    PMIX_DATA_BUFFER_CONSTRUCT(&pbuf);
+    cnt = 0;
+
+    for (n = 0; n < ndata; n++) {
+        /* ship this to our HNP/MASTER for processing, even if that is us */
+        ret = PMIx_Data_pack(NULL, &pbuf, (pmix_info_t *) &data[n], 1, PMIX_INFO);
+        if (PMIX_SUCCESS != ret) {
+            PMIX_ERROR_LOG(ret);
+        }
+        ++cnt;
+    }
+    if (0 < cnt) {
+        PMIX_DATA_BUFFER_CREATE(buf);
+        /* pack the source of this log request */
+        rc = PMIx_Data_pack(NULL, buf, (void*)client, 1, PMIX_PROC);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+        }
+        /* pack number of info provided */
+        rc = PMIx_Data_pack(NULL, buf, &cnt, 1, PMIX_SIZE);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+        }
+        /* pack number of directives given */
+        rc = PMIx_Data_pack(NULL, buf, &dcnt, 1, PMIX_SIZE);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+        }
+        /* bring over the packed info blob */
+        rc = PMIx_Data_unload(&pbuf, &pbo);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+        }
+        rc = PMIx_Data_pack(NULL, buf, &pbo, 1, PMIX_BYTE_OBJECT);
+        PMIX_BYTE_OBJECT_DESTRUCT(&pbo);
+        /* pack the directives blob */
+        rc = PMIx_Data_unload(&dbuf, &dbo);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+        }
+        rc = PMIx_Data_pack(NULL, buf, &dbo, 1, PMIX_BYTE_OBJECT);
+        PMIX_BYTE_OBJECT_DESTRUCT(&dbo);
+        /* send the result to the HNP */
+        PRTE_RML_SEND(rc, PRTE_PROC_MY_HNP->rank, buf,
+                      PRTE_RML_TAG_LOGGING);
+        if (PRTE_SUCCESS != rc) {
+            PRTE_ERROR_LOG(rc);
+            PMIX_DATA_BUFFER_RELEASE(buf);
+        }
+    }
+
+done:
+    /* we cannot directly execute the callback here
+     * as it would threadlock - so shift to somewhere
+     * safe */
+    PRTE_SERVER_PMIX_THREADSHIFT(PRTE_NAME_WILDCARD, NULL, rc, NULL, NULL, 0, lgcbfn, cbfunc, cbdata);
+}
+
+pmix_status_t psched_job_ctrl_fn(const pmix_proc_t *requestor,
+                                 const pmix_proc_t targets[], size_t ntargets,
+                                 const pmix_info_t directives[], size_t ndirs,
+                                 pmix_info_cbfunc_t cbfunc, void *cbdata)
+{
+    int rc, j;
+    int32_t signum;
+    size_t m, n;
+    prte_proc_t *proc;
+    pmix_nspace_t jobid;
+    pmix_pointer_array_t parray, *ptrarray;
+    pmix_data_buffer_t *cmd;
+    prte_daemon_cmd_flag_t cmmnd;
+    prte_grpcomm_signature_t *sig;
+    pmix_proc_t *proct;
+    PRTE_HIDE_UNUSED_PARAMS(cbfunc, cbdata);
+
+    pmix_output_verbose(2, prte_pmix_server_globals.output,
+                        "%s job control request from %s:%d",
+                        PRTE_NAME_PRINT(PRTE_PROC_MY_NAME),
+                        requestor->nspace, requestor->rank);
+
+    for (m = 0; m < ndirs; m++) {
+        if (0 == strncmp(directives[m].key, PMIX_JOB_CTRL_KILL, PMIX_MAX_KEYLEN)) {
+            /* convert the list of targets to a pointer array */
+            if (NULL == targets) {
+                ptrarray = NULL;
+            } else {
+                PMIX_CONSTRUCT(&parray, pmix_pointer_array_t);
+                for (n = 0; n < ntargets; n++) {
+                    if (PMIX_RANK_WILDCARD == targets[n].rank) {
+                        /* create an object */
+                        proc = PMIX_NEW(prte_proc_t);
+                        PMIX_LOAD_PROCID(&proc->name, targets[n].nspace, PMIX_RANK_WILDCARD);
+                    } else {
+                        /* get the proc object for this proc */
+                        if (NULL == (proc = prte_get_proc_object(&targets[n]))) {
+                            PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND);
+                            continue;
+                        }
+                        PMIX_RETAIN(proc);
+                    }
+                    pmix_pointer_array_add(&parray, proc);
+                }
+                ptrarray = &parray;
+            }
+            if (PRTE_SUCCESS != (rc = prte_plm.terminate_procs(ptrarray))) {
+                PRTE_ERROR_LOG(rc);
+            }
+            if (NULL != ptrarray) {
+                /* cleanup the array */
+                for (j = 0; j < parray.size; j++) {
+                    if (NULL != (proc = (prte_proc_t *) pmix_pointer_array_get_item(&parray, j))) {
+                        PMIX_RELEASE(proc);
+                    }
+                }
+                PMIX_DESTRUCT(&parray);
+            }
+        } else if (0 == strncmp(directives[m].key, PMIX_JOB_CTRL_TERMINATE, PMIX_MAX_KEYLEN)) {
+            if (NULL == targets) {
+                /* terminate the daemons and all running jobs */
+                PMIX_DATA_BUFFER_CREATE(cmd);
+                /* pack the command */
+                cmmnd = PRTE_DAEMON_HALT_VM_CMD;
+                rc = PMIx_Data_pack(NULL, cmd, &cmmnd, 1, PMIX_UINT8);
+                if (PMIX_SUCCESS != rc) {
+                    PMIX_ERROR_LOG(rc);
+                    PMIX_DATA_BUFFER_RELEASE(cmd);
+                    return rc;
+                }
+                /* goes to all daemons */
+                sig = PMIX_NEW(prte_grpcomm_signature_t);
+                sig->signature = (pmix_proc_t *) malloc(sizeof(pmix_proc_t));
+                sig->sz = 1;
+                PMIX_LOAD_PROCID(&sig->signature[0], PRTE_PROC_MY_NAME->nspace, PMIX_RANK_WILDCARD);
+                if (PRTE_SUCCESS != (rc = prte_grpcomm.xcast(sig, PRTE_RML_TAG_DAEMON, cmd))) {
+                    PRTE_ERROR_LOG(rc);
+                }
+                PMIX_DATA_BUFFER_RELEASE(cmd);
+                PMIX_RELEASE(sig);
+            }
+        } else if (0 == strncmp(directives[m].key, PMIX_JOB_CTRL_SIGNAL, PMIX_MAX_KEYLEN)) {
+            PMIX_DATA_BUFFER_CREATE(cmd);
+            cmmnd = PRTE_DAEMON_SIGNAL_LOCAL_PROCS;
+            /* pack the command */
+            rc = PMIx_Data_pack(NULL, cmd, &cmmnd, 1, PMIX_UINT8);
+            if (PMIX_SUCCESS != rc) {
+                PMIX_ERROR_LOG(rc);
+                PMIX_DATA_BUFFER_RELEASE(cmd);
+                return rc;
+            }
+            /* pack the target jobid */
+            if (NULL == targets) {
+                PMIX_LOAD_NSPACE(jobid, NULL);
+            } else {
+                proct = (pmix_proc_t *) &targets[0];
+                PMIX_LOAD_NSPACE(jobid, proct->nspace);
+            }
+            rc = PMIx_Data_pack(NULL, cmd, &jobid, 1, PMIX_PROC_NSPACE);
+            if (PMIX_SUCCESS != rc) {
+                PMIX_ERROR_LOG(rc);
+                PMIX_DATA_BUFFER_RELEASE(cmd);
+                return rc;
+            }
+            /* pack the signal */
+            PMIX_VALUE_GET_NUMBER(rc, &directives[m].value, signum, int32_t);
+            if (PMIX_SUCCESS != rc) {
+                PMIX_DATA_BUFFER_RELEASE(cmd);
+                return rc;
+            }
+            rc = PMIx_Data_pack(NULL, cmd, &signum, 1, PMIX_INT32);
+            if (PMIX_SUCCESS != rc) {
+                PMIX_ERROR_LOG(rc);
+                PMIX_DATA_BUFFER_RELEASE(cmd);
+                return rc;
+            }
+            /* goes to all daemons */
+            sig = PMIX_NEW(prte_grpcomm_signature_t);
+            sig->signature = (pmix_proc_t *) malloc(sizeof(pmix_proc_t));
+            sig->sz = 1;
+            PMIX_LOAD_PROCID(&sig->signature[0], PRTE_PROC_MY_NAME->nspace, PMIX_RANK_WILDCARD);
+            if (PRTE_SUCCESS != (rc = prte_grpcomm.xcast(sig, PRTE_RML_TAG_DAEMON, cmd))) {
+                PRTE_ERROR_LOG(rc);
+            }
+            PMIX_DATA_BUFFER_RELEASE(cmd);
+            PMIX_RELEASE(sig);
+        }
+    }
+
+    return PMIX_OPERATION_SUCCEEDED;
+}
+
+static void relcb(void *cbdata)
+{
+    prte_pmix_mdx_caddy_t *cd = (prte_pmix_mdx_caddy_t *) cbdata;
+
+    if (NULL != cd->info) {
+        PMIX_INFO_FREE(cd->info, cd->ninfo);
+    }
+    PMIX_RELEASE(cd);
+}
+static void group_release(int status, pmix_data_buffer_t *buf, void *cbdata)
+{
+    prte_pmix_mdx_caddy_t *cd = (prte_pmix_mdx_caddy_t *) cbdata;
+    int32_t cnt;
+    int rc = PRTE_SUCCESS;
+    pmix_status_t ret;
+    bool assignedID = false;
+    bool procsadded = false;
+    size_t cid;
+    pmix_proc_t *procs, *members;
+    size_t n, num_members;
+    pmix_data_array_t darray;
+    pmix_info_t info;
+    pmix_data_buffer_t dbuf;
+    pmix_byte_object_t bo;
+    int32_t byused;
+    pmix_server_pset_t *pset;
+
+    PMIX_ACQUIRE_OBJECT(cd);
+
+   pmix_output_verbose(2, prte_pmix_server_globals.output,
+                        "%s group request complete",
+                        PRTE_NAME_PRINT(PRTE_PROC_MY_NAME));
+
+    if (PRTE_SUCCESS != status) {
+        rc = status;
+        goto complete;
+    }
+
+    /* if this was a destruct operation, then there is nothing
+     * further we need do */
+    if (PMIX_GROUP_DESTRUCT == cd->op) {
+        /* find this group ID on our list of groups */
+        PMIX_LIST_FOREACH(pset, &prte_pmix_server_globals.groups, pmix_server_pset_t)
+        {
+            if (0 == strcmp(pset->name, cd->grpid)) {
+                pmix_list_remove_item(&prte_pmix_server_globals.groups, &pset->super);
+                PMIX_RELEASE(pset);
+                break;
+            }
+        }
+        rc = status;
+        goto complete;
+    }
+
+    /* check for any directives */
+    cnt = 1;
+    rc = PMIx_Data_unpack(NULL, buf, &bo, &cnt, PMIX_BYTE_OBJECT);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto complete;
+    }
+    PMIX_DATA_BUFFER_CONSTRUCT(&dbuf);
+    PMIX_DATA_BUFFER_LOAD(&dbuf, bo.bytes, bo.size);
+
+    cd->ninfo = 2;
+    cnt = 1;
+    rc = PMIx_Data_unpack(NULL, &dbuf, &info, &cnt, PMIX_INFO);
+    while (PMIX_SUCCESS == rc) {
+        if (PMIX_CHECK_KEY(&info, PMIX_GROUP_CONTEXT_ID)) {
+            PMIX_VALUE_GET_NUMBER(rc, &info.value, cid, size_t);
+            if (PMIX_SUCCESS != rc) {
+                PMIX_ERROR_LOG(rc);
+                cd->ninfo = 0;
+                PMIX_DATA_BUFFER_DESTRUCT(&dbuf);
+                goto complete;
+            }
+            assignedID = true;
+            cd->ninfo++;
+        } else if (PMIX_CHECK_KEY(&info, PMIX_GROUP_ADD_MEMBERS)) {
+            members = (pmix_proc_t*)info.value.data.darray->array;
+            num_members = info.value.data.darray->size;
+            PMIX_PROC_CREATE(procs, cd->nprocs + num_members);
+            for (n=0; n < cd->nprocs; n++) {
+                PMIX_XFER_PROCID(&procs[n], &cd->procs[n]);
+            }
+            for (n=0; n < num_members; n++) {
+                PMIX_XFER_PROCID(&procs[n+cd->nprocs], &members[n]);
+            }
+            PMIX_PROC_FREE(cd->procs, cd->nprocs);
+            cd->procs = procs;
+            cd->nprocs += num_members;
+            procsadded = true;
+        }
+        /* cleanup */
+        PMIX_INFO_DESTRUCT(&info);
+        /* get the next object */
+        cnt = 1;
+        rc = PMIx_Data_unpack(NULL, &dbuf, &info, &cnt, PMIX_INFO);
+    }
+    PMIX_DATA_BUFFER_DESTRUCT(&dbuf);
+    /* the unpacking loop will have ended when the unpack either
+     * went past the end of the buffer */
+    if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto complete;
+    }
+    rc = PMIX_SUCCESS;
+
+    if (PMIX_GROUP_CONSTRUCT == cd->op) {
+       /* add it to our list of known groups */
+        pset = PMIX_NEW(pmix_server_pset_t);
+        pset->name = strdup(cd->grpid);
+        pset->num_members = cd->nprocs;
+        PMIX_PROC_CREATE(pset->members, pset->num_members);
+        memcpy(pset->members, cd->procs, cd->nprocs * sizeof(pmix_proc_t));
+        pmix_list_append(&prte_pmix_server_globals.groups, &pset->super);
+    }
+
+    /* if anything is left in the buffer, then it is
+     * modex data that needs to be stored */
+    PMIX_BYTE_OBJECT_CONSTRUCT(&bo);
+    byused = buf->bytes_used - (buf->unpack_ptr - buf->base_ptr);
+    if (0 < byused) {
+        bo.bytes = buf->unpack_ptr;
+        bo.size = byused;
+    }
+    if (NULL != bo.bytes && 0 < bo.size) {
+        cd->ninfo++;
+    }
+
+    PMIX_INFO_CREATE(cd->info, cd->ninfo);
+    n = 0;
+    // pass back the final group membership
+    darray.type = PMIX_PROC;
+    darray.array = cd->procs;
+    darray.size = cd->nprocs;
+    PMIX_INFO_LOAD(&cd->info[n], PMIX_GROUP_MEMBERSHIP, &darray, PMIX_DATA_ARRAY);
+    PMIX_PROC_FREE(cd->procs, cd->nprocs);
+    ++n;
+    if (assignedID) {
+        PMIX_INFO_LOAD(&cd->info[n], PMIX_GROUP_CONTEXT_ID, &cid, PMIX_SIZE);
+        ++n;
+    }
+    if (NULL != bo.bytes && 0 < bo.size) {
+        PMIX_INFO_LOAD(&cd->info[n], PMIX_GROUP_ENDPT_DATA, &bo, PMIX_BYTE_OBJECT);
+    }
+
+complete:
+    ret = prte_pmix_convert_rc(rc);
+    /* return to the local procs in the collective */
+    if (NULL != cd->infocbfunc) {
+        cd->infocbfunc(ret, cd->info, cd->ninfo, cd->cbdata, relcb, cd);
+    } else {
+        if (NULL != cd->info) {
+            PMIX_INFO_FREE(cd->info, cd->ninfo);
+        }
+        PMIX_RELEASE(cd);
+    }
+}
+
+pmix_status_t psched_group_fn(pmix_group_operation_t op, char *grpid,
+                              const pmix_proc_t procs[], size_t nprocs,
+                              const pmix_info_t directives[], size_t ndirs,
+                              pmix_info_cbfunc_t cbfunc, void *cbdata)
+{
+    prte_pmix_mdx_caddy_t *cd;
+    int rc;
+    size_t i;
+    bool assignID = false;
+    pmix_server_pset_t *pset;
+    bool fence = false;
+    bool force_local = false;
+    pmix_byte_object_t *bo = NULL;
+    struct timeval tv = {0, 0};
+
+    pmix_output_verbose(2, prte_pmix_server_globals.output,
+                        "%s group request recvd",
+                        PRTE_NAME_PRINT(PRTE_PROC_MY_NAME));
+
+    /* they are required to pass us an id */
+    if (NULL == grpid) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    /* check the directives */
+    for (i = 0; i < ndirs; i++) {
+        /* see if they want a context id assigned */
+        if (PMIX_CHECK_KEY(&directives[i], PMIX_GROUP_ASSIGN_CONTEXT_ID)) {
+            assignID = PMIX_INFO_TRUE(&directives[i]);
+        } else if (PMIX_CHECK_KEY(&directives[i], PMIX_EMBED_BARRIER)) {
+            fence = PMIX_INFO_TRUE(&directives[i]);
+        } else if (PMIX_CHECK_KEY(&directives[i], PMIX_GROUP_ENDPT_DATA)) {
+            bo = (pmix_byte_object_t *) &directives[i].value.data.bo;
+        } else if (PMIX_CHECK_KEY(&directives[i], PMIX_TIMEOUT)) {
+            tv.tv_sec = directives[i].value.data.uint32;
+        } else if (PMIX_CHECK_KEY(&directives[i], PMIX_GROUP_LOCAL_ONLY)) {
+            force_local = PMIX_INFO_TRUE(&directives[i]);
+        }
+    }
+
+    /* if they don't want us to do a fence and they don't want a
+     * context id assigned, or they insist on forcing local
+     * completion of the operation, then we are done */
+    if ((!fence && !assignID) || force_local) {
+        pmix_output_verbose(2, prte_pmix_server_globals.output,
+                            "%s group request - purely local",
+                            PRTE_NAME_PRINT(PRTE_PROC_MY_NAME));
+        if (PMIX_GROUP_CONSTRUCT == op) {
+            /* add it to our list of known groups */
+            pset = PMIX_NEW(pmix_server_pset_t);
+            pset->name = strdup(grpid);
+            pset->num_members = nprocs;
+            PMIX_PROC_CREATE(pset->members, pset->num_members);
+            memcpy(pset->members, procs, nprocs * sizeof(pmix_proc_t));
+            pmix_list_append(&prte_pmix_server_globals.groups, &pset->super);
+        } else if (PMIX_GROUP_DESTRUCT == op) {
+            /* find this group ID on our list of groups */
+            PMIX_LIST_FOREACH(pset, &prte_pmix_server_globals.groups, pmix_server_pset_t)
+            {
+                if (0 == strcmp(pset->name, grpid)) {
+                    pmix_list_remove_item(&prte_pmix_server_globals.groups, &pset->super);
+                    PMIX_RELEASE(pset);
+                    break;
+                }
+            }
+        }
+        return PMIX_OPERATION_SUCCEEDED;
+    }
+
+    cd = PMIX_NEW(prte_pmix_mdx_caddy_t);
+    cd->grpid = grpid;
+    cd->op = op;
+    /* have to copy the procs in case we add members */
+    PMIX_PROC_CREATE(cd->procs, nprocs);
+    memcpy(cd->procs, procs, nprocs * sizeof(pmix_proc_t));
+    cd->nprocs = nprocs;
+    cd->grpcbfunc = group_release;
+    cd->infocbfunc = cbfunc;
+    cd->cbdata = cbdata;
+
+    /* compute the signature of this collective */
+    if (NULL != procs) {
+        cd->sig = PMIX_NEW(prte_grpcomm_signature_t);
+        cd->sig->sz = nprocs;
+        cd->sig->signature = (pmix_proc_t *) malloc(cd->sig->sz * sizeof(pmix_proc_t));
+        memcpy(cd->sig->signature, procs, cd->sig->sz * sizeof(pmix_proc_t));
+    }
+    /* setup the ctrls blob - this will include any "add_members" directive */
+    rc = prte_pack_ctrl_options(&cd->ctrls, directives, ndirs);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_RELEASE(cd);
+        return rc;
+    }
+    PMIX_DATA_BUFFER_CREATE(cd->buf);
+    /* if they provided us with a data blob, send it along */
+    if (NULL != bo) {
+        /* We don't own the byte_object and so we have to
+         * copy it here */
+        rc = PMIx_Data_embed(cd->buf, bo);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+        }
+    }
+    /* pass it to the global collective algorithm */
+    if (PRTE_SUCCESS != (rc = prte_grpcomm.allgather(cd))) {
+        PRTE_ERROR_LOG(rc);
+        PMIX_RELEASE(cd);
+        return PMIX_ERROR;
+    }
+    return PMIX_SUCCESS;
+}
diff --git a/src/tools/psched/errmgr.c b/src/tools/psched/errmgr.c
new file mode 100644
index 0000000000..e298ae54e6
--- /dev/null
+++ b/src/tools/psched/errmgr.c
@@ -0,0 +1,693 @@
+/*
+ * Copyright (c) 2009-2011 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2020 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2010-2017 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2011      Oracle and/or all its affiliates.  All rights reserved.
+ * Copyright (c) 2011-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2021-2023 Nanook Consulting.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "prte_config.h"
+
+#include <sys/types.h>
+#ifdef HAVE_UNISTD_H
+#    include <unistd.h>
+#endif /* HAVE_UNISTD_H */
+#include <string.h>
+#ifdef HAVE_SYS_WAIT_H
+#    include <sys/wait.h>
+#endif
+#include "src/pmix/pmix-internal.h"
+#include <pmix.h>
+#include <pmix_server.h>
+
+#include "src/pmix/pmix-internal.h"
+#include "src/util/pmix_output.h"
+#include "src/util/pmix_printf.h"
+
+#include "src/mca/ess/ess.h"
+#include "src/mca/grpcomm/grpcomm.h"
+#include "src/mca/iof/base/base.h"
+#include "src/mca/odls/base/base.h"
+#include "src/mca/plm/base/base.h"
+#include "src/mca/rmaps/rmaps_types.h"
+#include "src/rml/rml.h"
+#include "src/mca/state/state.h"
+
+#include "src/threads/pmix_threads.h"
+#include "src/util/error_strings.h"
+#include "src/util/name_fns.h"
+#include "src/util/proc_info.h"
+#include "src/util/pmix_show_help.h"
+
+#include "src/runtime/prte_globals.h"
+#include "src/runtime/prte_locks.h"
+#include "src/runtime/prte_quit.h"
+
+#include "src/mca/errmgr/base/base.h"
+#include "src/mca/errmgr/base/errmgr_private.h"
+#include "src/mca/errmgr/errmgr.h"
+
+#include "psched.h"
+
+static int init(void);
+static int finalize(void);
+
+/******************
+ * dvm module
+ ******************/
+prte_errmgr_base_module_t psched_errmgr_module = {
+    .init = init,
+    .finalize = finalize,
+    .logfn = prte_errmgr_base_log,
+    .abort = prte_errmgr_base_abort,
+    .abort_peers = prte_errmgr_base_abort_peers
+};
+
+/*
+ * Local functions
+ */
+static void job_errors(int fd, short args, void *cbdata);
+static void proc_errors(int fd, short args, void *cbdata);
+static void check_send_notification(prte_job_t *jdata,
+                                    prte_proc_t *proc,
+                                    pmix_status_t event);
+
+void psched_errmgr_init(void)
+{
+    prte_errmgr = psched_errmgr_module;
+}
+
+static int init(void)
+{
+    /* setup state machine to trap job errors */
+    prte_state.add_job_state(PRTE_JOB_STATE_ERROR, job_errors, PRTE_ERROR_PRI);
+
+    /* set the lost connection state to run at MSG priority so
+     * we can process any last messages from the proc
+     */
+    prte_state.add_proc_state(PRTE_PROC_STATE_COMM_FAILED, proc_errors, PRTE_MSG_PRI);
+
+    prte_state.add_proc_state(PRTE_PROC_STATE_ERROR, proc_errors, PRTE_ERROR_PRI);
+
+    return PRTE_SUCCESS;
+}
+
+static int finalize(void)
+{
+    return PRTE_SUCCESS;
+}
+
+static void _terminate_job(pmix_nspace_t jobid)
+{
+    pmix_pointer_array_t procs;
+    prte_proc_t pobj;
+
+    PMIX_CONSTRUCT(&procs, pmix_pointer_array_t);
+    pmix_pointer_array_init(&procs, 1, 1, 1);
+    PMIX_CONSTRUCT(&pobj, prte_proc_t);
+    PMIX_LOAD_PROCID(&pobj.name, jobid, PMIX_RANK_WILDCARD);
+    pmix_pointer_array_add(&procs, &pobj);
+    prte_plm.terminate_procs(&procs);
+    PMIX_DESTRUCT(&procs);
+    PMIX_DESTRUCT(&pobj);
+}
+
+static void job_errors(int fd, short args, void *cbdata)
+{
+    prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata;
+    prte_job_t *jdata;
+    prte_job_state_t jobstate;
+    int32_t rc;
+    PRTE_HIDE_UNUSED_PARAMS(fd, args);
+
+    PMIX_ACQUIRE_OBJECT(caddy);
+
+    /*
+     * if prte is trying to shutdown, just let it
+     */
+    if (prte_finalizing) {
+        return;
+    }
+
+    /* if the jdata is NULL, then it is referencing the daemon job */
+    if (NULL == caddy->jdata) {
+        caddy->jdata = prte_get_job_data_object(PRTE_PROC_MY_NAME->nspace);
+        PMIX_RETAIN(caddy->jdata);
+    }
+
+    /* update the state */
+    jdata = caddy->jdata;
+    jobstate = caddy->job_state;
+    jdata->state = jobstate;
+
+    PMIX_OUTPUT_VERBOSE((1, prte_errmgr_base_framework.framework_output,
+                         "%s errmgr:dvm: job %s reported state %s",
+                         PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(jdata->nspace),
+                         prte_job_state_to_str(jobstate)));
+
+    if (PMIX_CHECK_NSPACE(jdata->nspace, PRTE_PROC_MY_NAME->nspace)) {
+        if (PRTE_JOB_STATE_FAILED_TO_START == jdata->state
+            || PRTE_JOB_STATE_NEVER_LAUNCHED == jdata->state
+            || PRTE_JOB_STATE_FAILED_TO_LAUNCH == jdata->state
+            || PRTE_JOB_STATE_CANNOT_LAUNCH == jdata->state) {
+            prte_routing_is_enabled = false;
+            PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_DAEMONS_TERMINATED);
+            PMIX_RELEASE(caddy);
+            return;
+        }
+        /* if the daemon job aborted and we haven't heard from everyone yet,
+         * then this could well have been caused by a daemon not finding
+         * a way back to us. In this case, output a message indicating a daemon
+         * died without reporting. Otherwise, say nothing as we
+         * likely already output an error message */
+        if (PRTE_JOB_STATE_ABORTED == jobstate && jdata->num_procs != jdata->num_reported) {
+            prte_routing_is_enabled = false;
+            pmix_show_help("help-errmgr-base.txt", "failed-daemon", true);
+        }
+        /* there really isn't much else we can do since the problem
+         * is in the DVM itself, so best just to terminate */
+        jdata->num_terminated = jdata->num_procs;
+        /* activate the terminated state so we can exit */
+        PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_TERMINATED);
+        PMIX_RELEASE(caddy);
+        return;
+    }
+
+    /* all other cases involve jobs submitted to the DVM - therefore,
+     * we only inform the submitter of the problem, but do NOT terminate
+     * the DVM itself */
+
+    PMIX_OUTPUT_VERBOSE((5, prte_errmgr_base_framework.framework_output,
+                         "%s errmgr:dvm sending notification of job %s failure to %s",
+                         PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(jdata->nspace),
+                         PRTE_NAME_PRINT(&jdata->originator)));
+
+    /* all jobs were spawned by a requestor, so ensure that requestor
+     * has been notified that the spawn completed - otherwise, a quick-failing
+     * job might not generate a spawn response */
+    rc = prte_pmix_convert_job_state_to_error(jobstate);
+    rc = prte_plm_base_spawn_response(rc, jdata);
+    if (PRTE_SUCCESS != rc) {
+        PRTE_ERROR_LOG(rc);
+    }
+
+    /* ensure we terminate any processes left running in the DVM */
+    _terminate_job(jdata->nspace);
+
+    /* if the job never launched, then we need to let the
+     * state machine know this job failed - it has no
+     * other means of being alerted since no proc states
+     * will be triggered */
+    if (PRTE_JOB_STATE_FAILED_TO_START == jdata->state
+        || PRTE_JOB_STATE_NEVER_LAUNCHED == jdata->state
+        || PRTE_JOB_STATE_FAILED_TO_LAUNCH == jdata->state
+        || PRTE_JOB_STATE_ALLOC_FAILED == jdata->state
+        || PRTE_JOB_STATE_MAP_FAILED == jdata->state
+        || PRTE_JOB_STATE_CANNOT_LAUNCH == jdata->state) {
+        PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_TERMINATED);
+    }
+
+    /* cleanup */
+    PMIX_RELEASE(caddy);
+}
+
+static void proc_errors(int fd, short args, void *cbdata)
+{
+    prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata;
+    prte_job_t *jdata;
+    prte_proc_t *pptr, *proct;
+    pmix_proc_t *proc = &caddy->name;
+    prte_proc_state_t state = caddy->proc_state;
+    int i;
+    int32_t i32, *i32ptr;
+    bool flag;
+    PRTE_HIDE_UNUSED_PARAMS(fd, args);
+
+    PMIX_ACQUIRE_OBJECT(caddy);
+
+    PMIX_OUTPUT_VERBOSE((1, prte_errmgr_base_framework.framework_output,
+                         "%s errmgr:dvm: for proc %s state %s",
+                         PRTE_NAME_PRINT(PRTE_PROC_MY_NAME),
+                         PRTE_NAME_PRINT(proc), prte_proc_state_to_str(state)));
+
+    /* get the job object */
+    if (prte_finalizing || NULL == (jdata = prte_get_job_data_object(proc->nspace))) {
+        /* could be a race condition */
+        PMIX_RELEASE(caddy);
+        return;
+    }
+    pptr = (prte_proc_t *) pmix_pointer_array_get_item(jdata->procs, proc->rank);
+    if (NULL == pptr) {
+        PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND);
+        goto cleanup;
+    }
+
+    if (PMIX_CHECK_NSPACE(jdata->nspace, PRTE_PROC_MY_NAME->nspace)) {
+        /* NOTE: this is a daemon process that had the error */
+        /* we MUST handle a communication failure with special care to
+         * avoid normal termination issues */
+        if (PRTE_PROC_STATE_COMM_FAILED == state ||
+            PRTE_PROC_STATE_HEARTBEAT_FAILED == state ||
+            PRTE_PROC_STATE_UNABLE_TO_SEND_MSG == state ||
+            PRTE_PROC_STATE_FAILED_TO_START == state) {
+            /* if this is my own connection, ignore it */
+            if (PRTE_PROC_MY_NAME->rank == proc->rank) {
+                PMIX_OUTPUT_VERBOSE((5, prte_errmgr_base_framework.framework_output,
+                                     "%s Comm failure on my own connection - ignoring it",
+                                     PRTE_NAME_PRINT(PRTE_PROC_MY_NAME)));
+                goto cleanup;
+            }
+            /* mark the daemon as gone */
+            PRTE_FLAG_UNSET(pptr, PRTE_PROC_FLAG_ALIVE);
+            /* update the state */
+            pptr->state = state;
+            /* adjust our num_procs */
+            --prte_process_info.num_daemons;
+            /* if we have ordered prteds to terminate or abort
+             * is in progress, record it */
+            if (prte_prteds_term_ordered || prte_abnormal_term_ordered) {
+                PMIX_OUTPUT_VERBOSE((5, prte_errmgr_base_framework.framework_output,
+                                     "%s Comm failure: daemons terminating - recording daemon %s as gone",
+                                     PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(proc)));
+                /* remove from dependent routes, if it is one */
+                prte_rml_route_lost(proc->rank);
+                /* if all my routes and local children are gone, then terminate ourselves */
+                if (0 == pmix_list_get_size(&prte_rml_base.children)) {
+                    for (i = 0; i < prte_local_children->size; i++) {
+                        proct = (prte_proc_t *) pmix_pointer_array_get_item(prte_local_children, i);
+                        if (NULL != proct &&
+                            PRTE_FLAG_TEST(pptr, PRTE_PROC_FLAG_ALIVE) &&
+                            proct->state < PRTE_PROC_STATE_UNTERMINATED) {
+                            /* at least one is still alive */
+                            PMIX_OUTPUT_VERBOSE((5, prte_errmgr_base_framework.framework_output,
+                                                 "%s Comm failure: at least one proc (%s) still alive",
+                                                 PRTE_NAME_PRINT(PRTE_PROC_MY_NAME),
+                                                 PRTE_NAME_PRINT(&proct->name)));
+                            goto cleanup;
+                        }
+                    }
+                    /* call our appropriate exit procedure */
+                    PMIX_OUTPUT_VERBOSE((5, prte_errmgr_base_framework.framework_output,
+                                         "%s errmgr_dvm: all routes and children gone - ordering exit",
+                                         PRTE_NAME_PRINT(PRTE_PROC_MY_NAME)));
+                    PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_DAEMONS_TERMINATED);
+                } else {
+                    PMIX_OUTPUT_VERBOSE((5, prte_errmgr_base_framework.framework_output,
+                                         "%s Comm failure: %d routes remain alive",
+                                         PRTE_NAME_PRINT(PRTE_PROC_MY_NAME),
+                                         (int) pmix_list_get_size(&prte_rml_base.children)));
+                }
+                goto cleanup;
+            }
+            PMIX_OUTPUT_VERBOSE((5, prte_errmgr_base_framework.framework_output,
+                                 "%s Comm failure: daemon %s - aborting",
+                                 PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(proc)));
+            /* record the first one to fail */
+            if (!PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_ABORTED)) {
+                /* output an error message so the user knows what happened */
+                pmix_show_help("help-errmgr-base.txt", "node-died", true,
+                               PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), prte_process_info.nodename,
+                               PRTE_NAME_PRINT(proc), pptr->node->name);
+                /* mark the daemon job as failed */
+                jdata->state = PRTE_JOB_STATE_COMM_FAILED;
+                /* point to the lowest rank to cause the problem */
+                prte_set_attribute(&jdata->attributes, PRTE_JOB_ABORTED_PROC, PRTE_ATTR_LOCAL, pptr,
+                                   PMIX_POINTER);
+                /* retain the object so it doesn't get free'd */
+                PMIX_RETAIN(pptr);
+                PRTE_FLAG_SET(jdata, PRTE_JOB_FLAG_ABORTED);
+                /* update our exit code */
+                jdata->exit_code = pptr->exit_code;
+                /* just in case the exit code hadn't been set, do it here - this
+                 * won't override any reported exit code */
+                if (0 == jdata->exit_code) {
+                    jdata->exit_code = PRTE_ERR_COMM_FAILURE;
+                }
+            }
+        } else {
+            pmix_output(0, "UNSUPPORTED DAEMON ERROR STATE: %s", prte_proc_state_to_str(state));
+        }
+        /* since communications have failed, we have to simply force ourselves
+         * to terminate as we cannot rely on the routing tree to get messages
+         * out to other daemons */
+        prte_abnormal_term_ordered = true;
+        PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_DAEMONS_TERMINATED);
+        goto cleanup;
+    }
+
+    /* update the proc state - can get multiple reports on a proc
+     * depending on circumstances, so ensure we only do this once
+     */
+    if (pptr->state < PRTE_PROC_STATE_TERMINATED) {
+        pptr->state = state;
+    }
+
+    /* if we were ordered to terminate, mark this proc as dead and see if
+     * any of our routes or local children remain alive - if not, then
+     * terminate ourselves. */
+    if (prte_prteds_term_ordered) {
+        for (i = 0; i < prte_local_children->size; i++) {
+            proct = (prte_proc_t*)pmix_pointer_array_get_item(prte_local_children, i);
+            if (NULL != proct) {
+                if (PRTE_FLAG_TEST(proct, PRTE_PROC_FLAG_ALIVE)) {
+                    goto keep_going;
+                }
+            }
+        }
+        /* if all my routes and children are gone, then terminate
+           ourselves nicely (i.e., this is a normal termination) */
+        if (0 == pmix_list_get_size(&prte_rml_base.children)) {
+            PMIX_OUTPUT_VERBOSE((2, prte_errmgr_base_framework.framework_output,
+                                 "%s errmgr:default:dvm all routes gone - exiting",
+                                 PRTE_NAME_PRINT(PRTE_PROC_MY_NAME)));
+            PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_DAEMONS_TERMINATED);
+        }
+    }
+
+keep_going:
+    /* always mark the waitpid as having fired */
+    PRTE_ACTIVATE_PROC_STATE(&pptr->name, PRTE_PROC_STATE_WAITPID_FIRED);
+    /* if this is a remote proc, we won't hear anything more about it
+     * as the default behavior would be to terminate the job. So be sure to
+     * mark the IOF as having completed too so we correctly mark this proc
+     * as dead and notify everyone as required */
+    if (!PRTE_FLAG_TEST(pptr, PRTE_PROC_FLAG_LOCAL)) {
+        PRTE_ACTIVATE_PROC_STATE(&pptr->name, PRTE_PROC_STATE_IOF_COMPLETE);
+    }
+
+    /* simplify later checks */
+    flag = (prte_get_attribute(&jdata->attributes, PRTE_JOB_RECOVERABLE, NULL, PMIX_BOOL) ||
+            prte_get_attribute(&jdata->attributes, PRTE_JOB_CONTINUOUS, NULL, PMIX_BOOL));
+
+    /* ensure we record the failed proc properly so we can report
+     * the error once we terminate
+     */
+    switch (state) {
+    case PRTE_PROC_STATE_KILLED_BY_CMD:
+        PMIX_OUTPUT_VERBOSE((5, prte_errmgr_base_framework.framework_output,
+                             "%s errmgr:dvm: proc %s killed by cmd",
+                             PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(proc)));
+        /* we ordered this proc to die, so it isn't an abnormal termination
+         * and we don't flag it as such
+         */
+        if (jdata->num_terminated >= jdata->num_procs) {
+            /* this job has terminated */
+            PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_TERMINATED);
+        } else if (flag) {
+            /* at least one proc survives - send out a notification if one is requested */
+            check_send_notification(jdata, pptr, PMIX_ERR_PROC_KILLED_BY_CMD);
+        }
+        break;
+
+    case PRTE_PROC_STATE_ABORTED_BY_SIG:
+        PMIX_OUTPUT_VERBOSE((5, prte_errmgr_base_framework.framework_output,
+                             "%s errmgr:dvm: proc %s aborted by signal",
+                             PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(proc)));
+        if (flag) {
+            /* send out a notification if one is requested */
+            check_send_notification(jdata, pptr, PMIX_ERR_PROC_ABORTED_BY_SIG);
+        } else {
+            if (!PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_ABORTED)) {
+                jdata->state = PRTE_JOB_STATE_ABORTED_BY_SIG;
+                /* point to the first rank to cause the problem */
+                prte_set_attribute(&jdata->attributes, PRTE_JOB_ABORTED_PROC, PRTE_ATTR_LOCAL, pptr,
+                                   PMIX_POINTER);
+                /* retain the object so it doesn't get free'd */
+                PMIX_RETAIN(pptr);
+                PRTE_FLAG_SET(jdata, PRTE_JOB_FLAG_ABORTED);
+                jdata->exit_code = pptr->exit_code;
+                _terminate_job(jdata->nspace);
+            }
+        }
+        break;
+
+    case PRTE_PROC_STATE_TERM_WO_SYNC:
+        PMIX_OUTPUT_VERBOSE((5, prte_errmgr_base_framework.framework_output,
+                             "%s errmgr:dvm: proc %s terminated without sync",
+                             PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(proc)));
+        if (flag) {
+            /* send out a notification if one is requested */
+            check_send_notification(jdata, pptr, PMIX_ERR_PROC_TERM_WO_SYNC);
+        } else {
+            if (!PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_ABORTED)) {
+                jdata->state = PRTE_JOB_STATE_ABORTED_WO_SYNC;
+                /* point to the first rank to cause the problem */
+                prte_set_attribute(&jdata->attributes, PRTE_JOB_ABORTED_PROC, PRTE_ATTR_LOCAL, pptr,
+                                   PMIX_POINTER);
+                /* retain the object so it doesn't get free'd */
+                PMIX_RETAIN(pptr);
+                PRTE_FLAG_SET(jdata, PRTE_JOB_FLAG_ABORTED);
+                jdata->exit_code = pptr->exit_code;
+                /* send out a notification is one is requested */
+                check_send_notification(jdata, pptr, PMIX_ERR_PROC_TERM_WO_SYNC);
+                /* now treat a special case - if the proc exit'd without a required
+                 * sync, it may have done so with a zero exit code. We want to ensure
+                 * that the user realizes there was an error, so in this -one- case,
+                 * we overwrite the process' exit code with the default error code
+                 */
+                if (0 == jdata->exit_code) {
+                    jdata->exit_code = PRTE_ERROR_DEFAULT_EXIT_CODE;
+                }
+                /* kill the job */
+                _terminate_job(jdata->nspace);
+            }
+        }
+        break;
+
+    case PRTE_PROC_STATE_FAILED_TO_START:
+    case PRTE_PROC_STATE_FAILED_TO_LAUNCH:
+        PMIX_OUTPUT_VERBOSE((5, prte_errmgr_base_framework.framework_output,
+                             "%s errmgr:dvm: proc %s %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME),
+                             PRTE_NAME_PRINT(proc), prte_proc_state_to_str(state)));
+        if (!PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_ABORTED)) {
+            if (PRTE_PROC_STATE_FAILED_TO_START) {
+                jdata->state = PRTE_JOB_STATE_FAILED_TO_START;
+            } else {
+                jdata->state = PRTE_JOB_STATE_FAILED_TO_LAUNCH;
+            }
+            /* point to the first rank to cause the problem */
+            prte_set_attribute(&jdata->attributes, PRTE_JOB_ABORTED_PROC, PRTE_ATTR_LOCAL, pptr,
+                               PMIX_POINTER);
+            /* update our exit code */
+            jdata->exit_code = pptr->exit_code;
+            /* just in case the exit code hadn't been set, do it here - this
+             * won't override any reported exit code */
+            if (0 == jdata->exit_code) {
+                jdata->exit_code = PRTE_ERR_FAILED_TO_START;
+            }
+            /* retain the object so it doesn't get free'd */
+            PMIX_RETAIN(pptr);
+            PRTE_FLAG_SET(jdata, PRTE_JOB_FLAG_ABORTED);
+            /* kill the job */
+            _terminate_job(jdata->nspace);
+        }
+        /* if this was a daemon, report it */
+        if (PMIX_CHECK_NSPACE(jdata->nspace, PRTE_PROC_MY_NAME->nspace)) {
+            /* output a message indicating we failed to launch a daemon */
+            pmix_show_help("help-errmgr-base.txt", "failed-daemon-launch",
+                           true, prte_tool_basename);
+        }
+        PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_FAILED_TO_START);
+        break;
+
+    case PRTE_PROC_STATE_CALLED_ABORT:
+        PMIX_OUTPUT_VERBOSE((5, prte_errmgr_base_framework.framework_output,
+                             "%s errmgr:dvm: proc %s called abort with exit code %d",
+                             PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(proc),
+                             pptr->exit_code));
+        /* this proc ordered the job to abort */
+        if (flag) {
+            /* send out a notification if one is requested */
+            check_send_notification(jdata, pptr, PMIX_ERR_PROC_REQUESTED_ABORT);
+        } else {
+            if (!PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_ABORTED)) {
+                jdata->state = PRTE_JOB_STATE_CALLED_ABORT;
+                /* point to the first proc to cause the problem */
+                prte_set_attribute(&jdata->attributes, PRTE_JOB_ABORTED_PROC, PRTE_ATTR_LOCAL, pptr,
+                                   PMIX_POINTER);
+                /* retain the object so it doesn't get free'd */
+                PMIX_RETAIN(pptr);
+                PRTE_FLAG_SET(jdata, PRTE_JOB_FLAG_ABORTED);
+                jdata->exit_code = pptr->exit_code;
+                /* kill the job */
+                _terminate_job(jdata->nspace);
+            }
+        }
+        break;
+
+    case PRTE_PROC_STATE_TERM_NON_ZERO:
+        PMIX_OUTPUT_VERBOSE((5, prte_errmgr_base_framework.framework_output,
+                             "%s errmgr:dvm: proc %s exited with non-zero status %d",
+                             PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(proc),
+                             pptr->exit_code));
+        jdata->exit_code = pptr->exit_code;
+        PRTE_FLAG_UNSET(pptr, PRTE_PROC_FLAG_ALIVE);
+        jdata->num_terminated++;
+        /* track the number of non-zero exits */
+        i32 = 0;
+        i32ptr = &i32;
+        prte_get_attribute(&jdata->attributes, PRTE_JOB_NUM_NONZERO_EXIT, (void **) &i32ptr,
+                           PMIX_INT32);
+        ++i32;
+        prte_set_attribute(&jdata->attributes, PRTE_JOB_NUM_NONZERO_EXIT, PRTE_ATTR_LOCAL, i32ptr,
+                           PMIX_INT32);
+        if (jdata->num_terminated >= jdata->num_procs) {
+            /* this job has terminated */
+            PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_TERMINATED);
+        } else if (flag && prte_get_attribute(&jdata->attributes, PRTE_JOB_ERROR_NONZERO_EXIT, NULL, PMIX_BOOL)) {
+            check_send_notification(jdata, pptr, PMIX_ERR_EXIT_NONZERO_TERM);
+        } else {
+            if (!PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_ABORTED)) {
+                jdata->state = PRTE_JOB_STATE_NON_ZERO_TERM;
+                /* point to the first rank to cause the problem */
+                prte_set_attribute(&jdata->attributes, PRTE_JOB_ABORTED_PROC,
+                                   PRTE_ATTR_LOCAL, pptr, PMIX_POINTER);
+                /* retain the object so it doesn't get free'd */
+                PMIX_RETAIN(pptr);
+                PRTE_FLAG_SET(jdata, PRTE_JOB_FLAG_ABORTED);
+                /* kill the job */
+                _terminate_job(jdata->nspace);
+            }
+        }
+        break;
+
+    default:
+        /* shouldn't get this, but terminate job if required */
+        PMIX_OUTPUT_VERBOSE((5, prte_errmgr_base_framework.framework_output,
+                             "%s errmgr:dvm: proc %s default error %s",
+                             PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(proc),
+                             prte_proc_state_to_str(state)));
+        if (jdata->num_terminated == jdata->num_procs) {
+            PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_TERMINATED);
+        }
+        break;
+    }
+
+cleanup:
+    PMIX_RELEASE(caddy);
+}
+
+static void check_send_notification(prte_job_t *jdata,
+                                    prte_proc_t *proc,
+                                    pmix_status_t event)
+{
+    prte_grpcomm_signature_t sig;
+    int rc;
+    pmix_info_t *info;
+    size_t ninfo;
+    pmix_proc_t target;
+    pmix_data_buffer_t pbkt;
+    pmix_data_range_t range = PMIX_RANGE_CUSTOM;
+
+    pmix_output_verbose(5, prte_state_base_framework.framework_output,
+                        "%s errmgr:dvm:sending notification %s affected proc %s",
+                        PRTE_NAME_PRINT(PRTE_PROC_MY_NAME),
+                        PMIx_Error_string(event),
+                        PRTE_NAME_PRINT(&proc->name));
+
+    if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_NOTIFY_ERRORS, NULL, PMIX_BOOL) ||
+        prte_dvm_abort_ordered) {
+        return;
+    }
+    /* we checked for termination due to the specific error we encountered, but
+     * it is possible that we received another type of error that wouldn't have
+     * resulted in terminating the job. Thus, we still check to see if the job
+     * is being aborted before sending anything out */
+    if (PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_ABORTED)) {
+        /* this job has already been aborted, so we don't need to notify
+         * about the fate of any proc within it */
+        return;
+    }
+    /* notify the other procs of the termination */
+    PMIX_LOAD_PROCID(&target, jdata->nspace, PMIX_RANK_WILDCARD);
+
+    /* pack the info for sending */
+    PMIX_DATA_BUFFER_CONSTRUCT(&pbkt);
+
+    /* we need to add a flag indicating this came from an invalid proc so that we will
+     * inject it into our own PMIx server library */
+    rc = PMIx_Data_pack(NULL, &pbkt, &PRTE_NAME_INVALID->rank, 1, PMIX_PROC_RANK);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_DATA_BUFFER_DESTRUCT(&pbkt);
+        return;
+    }
+    /* pack the status code */
+    rc = PMIx_Data_pack(NULL, &pbkt, &event, 1, PMIX_STATUS);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_DATA_BUFFER_DESTRUCT(&pbkt);
+        return;
+    }
+    /* pack the source - it cannot be me as that will cause
+     * the pmix server to upcall the event back to me */
+    rc = PMIx_Data_pack(NULL, &pbkt, &proc->name, 1, PMIX_PROC);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_DATA_BUFFER_DESTRUCT(&pbkt);
+        return;
+    }
+    /* pack the range */
+    rc = PMIx_Data_pack(NULL, &pbkt, &range, 1, PMIX_DATA_RANGE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_DATA_BUFFER_DESTRUCT(&pbkt);
+        return;
+    }
+
+    /* setup the info */
+    if (-1 != proc->exit_code) {
+        ninfo = 3;
+    } else {
+        ninfo = 2;
+    }
+    PMIX_INFO_CREATE(info, ninfo);
+    PMIX_INFO_LOAD(&info[0], PMIX_EVENT_AFFECTED_PROC, &proc->name, PMIX_PROC);
+    PMIX_INFO_LOAD(&info[1], PMIX_EVENT_CUSTOM_RANGE, &target, PMIX_PROC);
+    if (-1 != proc->exit_code) {
+        PMIX_INFO_LOAD(&info[2], PMIX_EXIT_CODE, &proc->exit_code, PMIX_INT);
+    }
+
+    /* pack the number of infos */
+    rc = PMIx_Data_pack(NULL, &pbkt, &ninfo, 1, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_INFO_FREE(info, ninfo);
+        PMIX_DATA_BUFFER_DESTRUCT(&pbkt);
+        return;
+    }
+    /* pack the infos themselves */
+    rc = PMIx_Data_pack(NULL, &pbkt, info, ninfo, PMIX_INFO);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_INFO_FREE(info, ninfo);
+        PMIX_DATA_BUFFER_DESTRUCT(&pbkt);
+        return;
+    }
+    PMIX_INFO_FREE(info, ninfo);
+
+    /* xcast it to everyone */
+    PMIX_CONSTRUCT(&sig, prte_grpcomm_signature_t);
+    sig.signature = (pmix_proc_t *) malloc(sizeof(pmix_proc_t));
+    PMIX_LOAD_PROCID(&sig.signature[0], PRTE_PROC_MY_NAME->nspace, PMIX_RANK_WILDCARD);
+    sig.sz = 1;
+
+    if (PRTE_SUCCESS != (rc = prte_grpcomm.xcast(&sig, PRTE_RML_TAG_NOTIFICATION, &pbkt))) {
+        PRTE_ERROR_LOG(rc);
+    }
+    PMIX_DESTRUCT(&sig);
+    PMIX_DATA_BUFFER_DESTRUCT(&pbkt);
+}
diff --git a/src/tools/psched/event.c b/src/tools/psched/event.c
new file mode 100644
index 0000000000..19f4b3c368
--- /dev/null
+++ b/src/tools/psched/event.c
@@ -0,0 +1,262 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2009-2020 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2017 Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2014-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2020      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2021-2023 Nanook Consulting  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include "prte_config.h"
+
+#ifdef HAVE_UNISTD_H
+#    include <unistd.h>
+#endif
+
+#include "src/hwloc/hwloc-internal.h"
+#include "src/pmix/pmix-internal.h"
+#include "src/util/pmix_argv.h"
+#include "src/util/pmix_output.h"
+
+#include "src/mca/errmgr/errmgr.h"
+#include "src/mca/grpcomm/base/base.h"
+#include "src/mca/iof/base/base.h"
+#include "src/mca/iof/iof.h"
+#include "src/mca/plm/base/plm_private.h"
+#include "src/mca/plm/plm.h"
+#include "src/mca/plm/base/plm_private.h"
+#include "src/mca/rmaps/rmaps_types.h"
+#include "src/rml/rml.h"
+#include "src/mca/schizo/schizo.h"
+#include "src/mca/state/state.h"
+#include "src/runtime/prte_globals.h"
+#include "src/runtime/prte_locks.h"
+#include "src/threads/pmix_threads.h"
+#include "src/util/name_fns.h"
+#include "src/util/pmix_show_help.h"
+
+#include "src/prted/pmix/pmix_server_internal.h"
+#include "src/tools/psched/psched.h"
+
+static void _register_events(int sd, short args, void *cbdata)
+{
+    prte_pmix_server_op_caddy_t *cd = (prte_pmix_server_op_caddy_t *) cbdata;
+    PRTE_HIDE_UNUSED_PARAMS(sd, args);
+
+    PMIX_ACQUIRE_OBJECT(cd);
+
+    /* need to implement this */
+
+    if (NULL != cd->cbfunc) {
+        cd->cbfunc(PRTE_SUCCESS, cd->cbdata);
+    }
+    PMIX_RELEASE(cd);
+}
+
+/* hook for the local PMIX server to pass event registrations
+ * up to us - we will assume the responsibility for providing
+ * notifications for registered events */
+pmix_status_t psched_register_events_fn(pmix_status_t *codes, size_t ncodes,
+                                        const pmix_info_t info[], size_t ninfo,
+                                        pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    prte_pmix_server_op_caddy_t *cd;
+
+    /* need to thread-shift this request as we are going
+     * to access our global list of registered events */
+    cd = PMIX_NEW(prte_pmix_server_op_caddy_t);
+    cd->codes = codes;
+    cd->ncodes = ncodes;
+    cd->info = (pmix_info_t *) info;
+    cd->ninfo = ninfo;
+    cd->cbfunc = cbfunc;
+    cd->cbdata = cbdata;
+    prte_event_set(prte_event_base, &(cd->ev), -1, PRTE_EV_WRITE, _register_events, cd);
+    prte_event_set_priority(&(cd->ev), PRTE_MSG_PRI);
+    PMIX_POST_OBJECT(cd);
+    prte_event_active(&(cd->ev), PRTE_EV_WRITE, 1);
+    return PMIX_SUCCESS;
+}
+
+static void _deregister_events(int sd, short args, void *cbdata)
+{
+    prte_pmix_server_op_caddy_t *cd = (prte_pmix_server_op_caddy_t *) cbdata;
+    PRTE_HIDE_UNUSED_PARAMS(sd, args);
+
+    PMIX_ACQUIRE_OBJECT(cd);
+
+    /* need to implement this */
+    if (NULL != cd->cbfunc) {
+        cd->cbfunc(PRTE_SUCCESS, cd->cbdata);
+    }
+    PMIX_RELEASE(cd);
+}
+/* hook for the local PMIX server to pass event deregistrations
+ * up to us */
+pmix_status_t psched_deregister_events_fn(pmix_status_t *codes, size_t ncodes,
+                                          pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    prte_pmix_server_op_caddy_t *cd;
+
+    /* need to thread-shift this request as we are going
+     * to access our global list of registered events */
+    cd = PMIX_NEW(prte_pmix_server_op_caddy_t);
+    cd->codes = codes;
+    cd->ncodes = ncodes;
+    cd->cbfunc = cbfunc;
+    cd->cbdata = cbdata;
+    prte_event_set(prte_event_base, &(cd->ev), -1, PRTE_EV_WRITE, _deregister_events, cd);
+    prte_event_set_priority(&(cd->ev), PRTE_MSG_PRI);
+    PMIX_POST_OBJECT(cd);
+    prte_event_active(&(cd->ev), PRTE_EV_WRITE, 1);
+    return PRTE_SUCCESS;
+}
+
+static void _notify_release(int status, void *cbdata)
+{
+    prte_pmix_server_op_caddy_t *cd = (prte_pmix_server_op_caddy_t *) cbdata;
+    PRTE_HIDE_UNUSED_PARAMS(status);
+
+    PMIX_ACQUIRE_OBJECT(cd);
+
+    if (NULL != cd->info) {
+        PMIX_INFO_FREE(cd->info, cd->ninfo);
+    }
+    PMIX_RELEASE(cd);
+}
+
+pmix_status_t psched_notify_event(pmix_status_t code, const pmix_proc_t *source,
+                                  pmix_data_range_t range, pmix_info_t info[], size_t ninfo,
+                                  pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    int rc;
+    prte_grpcomm_signature_t *sig;
+    pmix_data_buffer_t pbkt;
+    pmix_status_t ret;
+    size_t n;
+    PRTE_HIDE_UNUSED_PARAMS(cbfunc, cbdata);
+
+    pmix_output_verbose(2, prte_pmix_server_globals.output,
+                        "%s local process %s generated event code %s range %s",
+                        PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(source),
+                        PMIx_Error_string(code), PMIx_Data_range_string(range));
+
+    /* we can get events prior to completing prte_init as we have
+     * to init PMIx early so that PRRTE components can use it */
+    PMIX_ACQUIRE_THREAD(&prte_init_lock);
+    if (!prte_initialized) {
+        PMIX_RELEASE_THREAD(&prte_init_lock);
+        goto done;
+    }
+    PMIX_RELEASE_THREAD(&prte_init_lock);
+
+    /* check to see if this is one we sent down */
+    for (n = 0; n < ninfo; n++) {
+        if (0 == strcmp(info[n].key, "prte.notify.donotloop")) {
+            /* yep - do not process */
+            goto done;
+        }
+    }
+
+    /* if this is notification of procs being ready for debug, then
+     * we treat this as a state change */
+    if (PMIX_READY_FOR_DEBUG == code) {
+        PRTE_ACTIVATE_PROC_STATE((pmix_proc_t*)source, PRTE_PROC_STATE_READY_FOR_DEBUG);
+        goto done;
+    }
+
+    /* a local process has generated an event - we need to xcast it
+     * to all the daemons so it can be passed down to their local
+     * procs */
+    PMIX_DATA_BUFFER_CONSTRUCT(&pbkt);
+
+    /* we need to add a flag indicating this came from us as we are going to get it echoed
+     * back to us by the broadcast */
+    if (PMIX_SUCCESS
+        != (rc = PMIx_Data_pack(NULL, &pbkt, &PRTE_PROC_MY_NAME->rank, 1, PMIX_PROC_RANK))) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_DATA_BUFFER_DESTRUCT(&pbkt);
+        return rc;
+    }
+
+    /* pack the status code */
+    if (PMIX_SUCCESS != (ret = PMIx_Data_pack(NULL, &pbkt, &code, 1, PMIX_STATUS))) {
+        PMIX_ERROR_LOG(ret);
+        PMIX_DATA_BUFFER_DESTRUCT(&pbkt);
+        return ret;
+    }
+    /* pack the source */
+    if (PMIX_SUCCESS != (ret = PMIx_Data_pack(NULL, &pbkt, (pmix_proc_t *) source, 1, PMIX_PROC))) {
+        PMIX_ERROR_LOG(ret);
+        PMIX_DATA_BUFFER_DESTRUCT(&pbkt);
+        return ret;
+    }
+    /* pack the range */
+    if (PMIX_SUCCESS != (ret = PMIx_Data_pack(NULL, &pbkt, &range, 1, PMIX_DATA_RANGE))) {
+        PMIX_ERROR_LOG(ret);
+        PMIX_DATA_BUFFER_DESTRUCT(&pbkt);
+        return ret;
+    }
+    /* pack the number of infos */
+    if (PMIX_SUCCESS != (ret = PMIx_Data_pack(NULL, &pbkt, &ninfo, 1, PMIX_SIZE))) {
+        PMIX_ERROR_LOG(ret);
+        PMIX_DATA_BUFFER_DESTRUCT(&pbkt);
+        return ret;
+    }
+    if (0 < ninfo) {
+        if (PMIX_SUCCESS != (ret = PMIx_Data_pack(NULL, &pbkt, info, ninfo, PMIX_INFO))) {
+            PMIX_ERROR_LOG(ret);
+            PMIX_DATA_BUFFER_DESTRUCT(&pbkt);
+            return ret;
+        }
+    }
+
+    /* goes to all daemons */
+    sig = PMIX_NEW(prte_grpcomm_signature_t);
+    if (NULL == sig) {
+        PMIX_DATA_BUFFER_DESTRUCT(&pbkt);
+        return PMIX_ERR_NOMEM;
+    }
+    sig->signature = (pmix_proc_t *) malloc(sizeof(pmix_proc_t));
+    if (NULL == sig->signature) {
+        PMIX_DATA_BUFFER_DESTRUCT(&pbkt);
+        PMIX_RELEASE(sig);
+        return PMIX_ERR_NOMEM;
+    }
+    PMIX_LOAD_PROCID(&sig->signature[0], PRTE_PROC_MY_NAME->nspace, PMIX_RANK_WILDCARD);
+    sig->sz = 1;
+    if (PRTE_SUCCESS != (rc = prte_grpcomm.xcast(sig, PRTE_RML_TAG_NOTIFICATION, &pbkt))) {
+        PRTE_ERROR_LOG(rc);
+        PMIX_DATA_BUFFER_DESTRUCT(&pbkt);
+        PMIX_RELEASE(sig);
+        return PMIX_ERROR;
+    }
+    PMIX_DATA_BUFFER_DESTRUCT(&pbkt);
+    /* maintain accounting */
+    PMIX_RELEASE(sig);
+
+done:
+    /* we do not need to execute a callback as we did this atomically */
+    return PMIX_OPERATION_SUCCEEDED;
+}
diff --git a/src/mca/schizo/psched/help-schizo-psched.txt b/src/tools/psched/help-psched.txt
similarity index 100%
rename from src/mca/schizo/psched/help-schizo-psched.txt
rename to src/tools/psched/help-psched.txt
diff --git a/src/tools/psched/psched.c b/src/tools/psched/psched.c
index 6ef4090b23..319908a531 100644
--- a/src/tools/psched/psched.c
+++ b/src/tools/psched/psched.c
@@ -68,6 +68,9 @@
 #include "src/util/pmix_basename.h"
 #include "src/util/prte_cmd_line.h"
 #include "src/util/daemon_init.h"
+#include "src/util/malloc.h"
+#include "src/util/pmix_if.h"
+#include "src/util/pmix_os_dirpath.h"
 #include "src/util/pmix_os_path.h"
 #include "src/util/pmix_output.h"
 #include "src/util/pmix_printf.h"
@@ -79,11 +82,20 @@
 #include "src/util/pmix_show_help.h"
 #include "src/util/session_dir.h"
 
+#include "src/mca/ess/base/base.h"
+#include "src/mca/plm/base/plm_private.h"
 #include "src/mca/prtebacktrace/base/base.h"
 #include "src/mca/prteinstalldirs/base/base.h"
+#include "src/mca/ras/base/base.h"
 #include "src/mca/schizo/base/base.h"
 #include "src/mca/state/base/base.h"
+
+#include "src/runtime/prte_globals.h"
+#include "src/runtime/prte_locks.h"
 #include "src/runtime/runtime.h"
+#include "src/runtime/runtime_internals.h"
+
+#include "psched.h"
 
 /*
  * Globals
@@ -117,7 +129,14 @@ static void infocbfunc(pmix_status_t status, pmix_info_t *info, size_t ninfo, vo
     PRTE_PMIX_WAKEUP_THREAD(&xfer->lock);
 }
 
+static bool forcibly_die = false;
 static int wait_pipe[2];
+static prte_event_t term_handler;
+static prte_event_t epipe_handler;
+static int term_pipe[2];
+static pmix_mutex_t abort_inprogress_lock = PMIX_MUTEX_STATIC_INIT;
+static void clean_abort(int fd, short flags, void *arg);
+static void abort_signal_callback(int signal);
 
 static int wait_dvm(pid_t pid)
 {
@@ -172,6 +191,11 @@ int main(int argc, char *argv[])
     prte_schizo_base_module_t *schizo;
     pmix_cli_item_t *opt;
     char *path = NULL;
+    prte_job_t *jdata;
+    prte_app_context_t *app;
+    prte_node_t *node;
+    prte_proc_t *pptr;
+    prte_topology_t *t;
 
     /* initialize the globals */
     prte_tool_basename = pmix_basename(argv[0]);
@@ -215,6 +239,9 @@ int main(int argc, char *argv[])
         return prte_pmix_convert_status(ret);
     }
 
+    /* ensure we know the type of proc for when we finalize */
+    prte_process_info.proc_type = PRTE_PROC_MASTER;
+
     /* we always need the prrte and pmix params */
     ret = prte_schizo_base_parse_prte(pargc, 0, pargv, NULL);
     if (PRTE_SUCCESS != ret) {
@@ -238,7 +265,6 @@ int main(int argc, char *argv[])
     /* initialize the output system */
     pmix_output_init();
 
-    /* keyval lex-based parser */
     /* Setup the parameter system */
     if (PRTE_SUCCESS != (ret = pmix_mca_base_var_init())) {
         pmix_show_help("help-prte-runtime",
@@ -250,6 +276,8 @@ int main(int argc, char *argv[])
     /* set the nodename so anyone who needs it has it - this
      * must come AFTER we initialize the installdirs */
     prte_setup_hostname();
+    /* add network aliases to our list of alias hostnames */
+    pmix_ifgetaliases(&prte_process_info.aliases);
 
     /* pretty-print stack handlers */
     if (PRTE_SUCCESS != (ret = prte_util_register_stackhandlers())) {
@@ -260,7 +288,7 @@ int main(int argc, char *argv[])
     }
 
     /* pre-load any default mca param files */
-    preload_default_mca_params();
+    prte_preload_default_mca_params();
 
     /* Register all MCA Params */
     if (PRTE_SUCCESS != (ret = prte_register_params())) {
@@ -279,28 +307,114 @@ int main(int argc, char *argv[])
         return ret;
     }
 
-    /* open the SCHIZO framework */
-    ret = pmix_mca_base_framework_open(&prte_schizo_base_framework,
-                                       PMIX_MCA_BASE_OPEN_DEFAULT);
-    if (PRTE_SUCCESS != ret) {
-        PRTE_ERROR_LOG(ret);
+    /** setup callbacks for abort signals - from this point
+     * forward, we need to abort in a manner that allows us
+     * to cleanup. However, we cannot directly use libevent
+     * to trap these signals as otherwise we cannot respond
+     * to them if we are stuck in an event! So instead use
+     * the basic POSIX trap functions to handle the signal,
+     * and then let that signal handler do some magic to
+     * avoid the hang
+     *
+     * NOTE: posix traps don't allow us to do anything major
+     * in them, so use a pipe tied to a libevent event to
+     * reach a "safe" place where the termination event can
+     * be created
+     */
+    if (0 != pipe(term_pipe)) {
+        exit(1);
+    }
+
+    /*
+     * Initialize the event library
+     */
+    if (PRTE_SUCCESS != (ret = prte_event_base_open())) {
+        pmix_show_help("help-prte-runtime",
+                       "prte_init:startup:internal-failure", true,
+                       "event base open", PRTE_ERROR_NAME(ret), ret);
         return ret;
     }
+    /* setup an event to attempt normal termination on signal */
+    prte_event_set(prte_event_base, &term_handler, term_pipe[0], PRTE_EV_READ, clean_abort, NULL);
+    prte_event_add(&term_handler, NULL);
+    /* point the signal trap to a function that will activate that event */
+    signal(SIGTERM, abort_signal_callback);
+    signal(SIGINT, abort_signal_callback);
+    signal(SIGHUP, abort_signal_callback);
 
-    if (PRTE_SUCCESS != (ret = prte_schizo_base_select())) {
-        PRTE_ERROR_LOG(ret);
+    /* setup the locks */
+    if (PRTE_SUCCESS != (ret = prte_locks_init())) {
+        pmix_show_help("help-prte-runtime",
+                       "prte_init:startup:internal-failure", true,
+                       "locks init", PRTE_ERROR_NAME(ret), ret);
+        return ret;
+    }
+
+    /* Ensure the rest of the process info structure is initialized */
+    if (PRTE_SUCCESS != (ret = prte_proc_info())) {
+        pmix_show_help("help-prte-runtime",
+                       "prte_init:startup:internal-failure", true,
+                       "proc info", PRTE_ERROR_NAME(ret), ret);
         return ret;
     }
 
-    /* get our schizo module */
-    personality = "psched";
-    schizo = prte_schizo_base_detect_proxy(personality);
-    if (NULL == schizo) {
-        pmix_show_help("help-schizo-base.txt", "no-proxy", true,
-                       prte_tool_basename, personality);
-        return 1;
+    if (PRTE_SUCCESS != (ret = prte_hwloc_base_register())) {
+        pmix_show_help("help-prte-runtime",
+                       "prte_init:startup:internal-failure", true,
+                       "register hwloc", PRTE_ERROR_NAME(ret), ret);
+        return ret;
     }
 
+    /* open hwloc */
+    prte_hwloc_base_open();
+    /* get the local topology */
+    ret = prte_hwloc_base_get_topology();
+    if (PRTE_SUCCESS != ret) {
+        pmix_show_help("help-prte-runtime",
+                       "prte_init:startup:internal-failure", true,
+                       "get topology", PRTE_ERROR_NAME(ret), ret);
+        return ret;
+    }
+
+    /* setup the global job and node arrays */
+    prte_job_data = PMIX_NEW(pmix_pointer_array_t);
+    ret = pmix_pointer_array_init(prte_job_data,
+                                  PRTE_GLOBAL_ARRAY_BLOCK_SIZE,
+                                  PRTE_GLOBAL_ARRAY_MAX_SIZE,
+                                  PRTE_GLOBAL_ARRAY_BLOCK_SIZE);
+    if (PMIX_SUCCESS != ret) {
+        pmix_show_help("help-prte-runtime",
+                       "prte_init:startup:internal-failure", true,
+                       "setup job array", PRTE_ERROR_NAME(ret), ret);
+        return ret;
+    }
+    prte_node_pool = PMIX_NEW(pmix_pointer_array_t);
+    ret = pmix_pointer_array_init(prte_node_pool, PRTE_GLOBAL_ARRAY_BLOCK_SIZE,
+                                  PRTE_GLOBAL_ARRAY_MAX_SIZE,
+                                  PRTE_GLOBAL_ARRAY_BLOCK_SIZE);
+    if (PMIX_SUCCESS != ret) {
+        pmix_show_help("help-prte-runtime",
+                       "prte_init:startup:internal-failure", true,
+                       "setup node array", PRTE_ERROR_NAME(ret), ret);
+        return ret;
+    }
+    prte_node_topologies = PMIX_NEW(pmix_pointer_array_t);
+    ret = pmix_pointer_array_init(prte_node_topologies, PRTE_GLOBAL_ARRAY_BLOCK_SIZE,
+                                  PRTE_GLOBAL_ARRAY_MAX_SIZE,
+                                  PRTE_GLOBAL_ARRAY_BLOCK_SIZE);
+    if (PMIX_SUCCESS != ret) {
+        pmix_show_help("help-prte-runtime",
+                       "prte_init:startup:internal-failure", true,
+                       "setup node topologies array", PRTE_ERROR_NAME(ret), ret);
+        return ret;
+    }
+    /* initialize the cache */
+    prte_cache = PMIX_NEW(pmix_pointer_array_t);
+    pmix_pointer_array_init(prte_cache, 1, INT_MAX, 1);
+
+    /* setup the SCHIZO module */
+    schizo = &psched_schizo_module;
+
     /* parse the CLI to load the MCA params */
     PMIX_CONSTRUCT(&results, pmix_cli_result_t);
     ret = schizo->parse_cli(pargv, &results, PMIX_CLI_SILENT);
@@ -342,6 +456,7 @@ int main(int argc, char *argv[])
         setsid();
 #endif
     }
+
     /* ensure we silence any compression warnings */
     PMIX_SETENV_COMPAT("PMIX_MCA_compress_base_silence_warning", "1", true, &environ);
 
@@ -353,41 +468,12 @@ int main(int argc, char *argv[])
         return ret;
     }
 
-    /* get the local topology */
-    if (NULL == prte_hwloc_topology) {
-        if (PRTE_SUCCESS != (ret = prte_hwloc_base_get_topology())) {
-            pmix_show_help("help-prte-runtime",
-                           "prte_init:startup:internal-failure", true,
-                           "topology discovery", PRTE_ERROR_NAME(ret), ret);
-            return ret;
-        }
-    }
+    /* setup the state machine */
+    psched_state_init();
 
-    /* open and setup the state machine */
-    ret = pmix_mca_base_framework_open(&prte_state_base_framework,
-                                       PMIX_MCA_BASE_OPEN_DEFAULT);
-    if (PRTE_SUCCESS != ret) {
-        pmix_show_help("help-prte-runtime",
-                       "prte_init:startup:internal-failure", true,
-                       "state open", PRTE_ERROR_NAME(ret), ret);
-        return ret;
-    }
-    if (PRTE_SUCCESS != (ret = prte_state_base_select())) {
-        pmix_show_help("help-prte-runtime",
-                       "prte_init:startup:internal-failure", true,
-                       "state select", PRTE_ERROR_NAME(ret), ret);
-        return ret;
-    }
+    /* setup the errmgr */
+    psched_errmgr_init();
 
-    /* open the errmgr */
-    ret = pmix_mca_base_framework_open(&prte_errmgr_base_framework,
-                                       PMIX_MCA_BASE_OPEN_DEFAULT);
-    if (PRTE_SUCCESS != ret) {
-        pmix_show_help("help-prte-runtime",
-                       "prte_init:startup:internal-failure", true,
-                       "prte_errmgr_base_open", PRTE_ERROR_NAME(ret), ret);
-        return ret;
-    }
     /* set a name */
     ret = prte_plm_base_set_hnp_name();
     if (PRTE_SUCCESS != ret) {
@@ -422,24 +508,12 @@ int main(int argc, char *argv[])
         goto DONE;
     }
 
-    /* setup the PMIx server - we need this here in case the
-     * communications infrastructure wants to register
-     * information */
-    if (PRTE_SUCCESS != (ret = pmix_server_init())) {
+    /* setup the PMIx scheduler class */
+    if (PRTE_SUCCESS != (ret = psched_server_init())) {
         /* the server code already barked, so let's be quiet */
         prte_exit_status = PRTE_ERR_SILENT;
         goto DONE;
     }
-    pmix_server_start();
-
-    /* setup the error manager */
-    if (PRTE_SUCCESS != (ret = prte_errmgr_base_select())) {
-        pmix_show_help("help-prte-runtime",
-                       "prte_init:startup:internal-failure", true,
-                       "errmgr_base_select", PRTE_ERROR_NAME(ret), ret);
-        prte_exit_status = ret;
-        goto DONE;
-     }
 
     /* create my job data object */
     jdata = PMIX_NEW(prte_job_t);
@@ -447,13 +521,7 @@ int main(int argc, char *argv[])
     prte_set_job_data_object(jdata);
 
     /* set the schizo personality to "psched" by default */
-    jdata->schizo = (struct prte_schizo_base_module_t*)prte_schizo_base_detect_proxy("psched");
-    if (NULL == jdata->schizo) {
-        pmix_show_help("help-schizo-base.txt", "no-proxy", true, prte_tool_basename, "prte");
-        error = "select personality";
-        ret = PRTE_ERR_SILENT;
-        goto error;
-    }
+    jdata->schizo = (struct prte_schizo_base_module_t *)schizo;
 
     /* every job requires at least one app */
     app = PMIX_NEW(prte_app_context_t);
@@ -469,40 +537,28 @@ int main(int argc, char *argv[])
     pmix_pointer_array_set_item(prte_node_pool, PRTE_PROC_MY_NAME->rank, node);
 
     /* create and store a proc object for us */
-    proc = PMIX_NEW(prte_proc_t);
-    PMIX_LOAD_PROCID(&proc->name, PRTE_PROC_MY_NAME->nspace, PRTE_PROC_MY_NAME->rank);
-    proc->job = jdata;
-    proc->rank = proc->name.rank;
-    proc->pid = prte_process_info.pid;
-    prte_oob_base_get_addr(&proc->rml_uri);
-    prte_process_info.my_hnp_uri = strdup(proc->rml_uri);
-    /* store it in the local PMIx repo for later retrieval */
-    PMIX_VALUE_LOAD(&pval, proc->rml_uri, PMIX_STRING);
-    if (PMIX_SUCCESS != (pret = PMIx_Store_internal(PRTE_PROC_MY_NAME, PMIX_PROC_URI, &pval))) {
-        PMIX_ERROR_LOG(pret);
-        ret = PRTE_ERROR;
-        PMIX_VALUE_DESTRUCT(&pval);
-        error = "store uri";
-        goto error;
-    }
-    PMIX_VALUE_DESTRUCT(&pval);
-    proc->state = PRTE_PROC_STATE_RUNNING;
+    pptr = PMIX_NEW(prte_proc_t);
+    PMIX_LOAD_PROCID(&pptr->name, PRTE_PROC_MY_NAME->nspace, PRTE_PROC_MY_NAME->rank);
+    pptr->job = jdata;
+    pptr->rank = pptr->name.rank;
+    pptr->pid = prte_process_info.pid;
+    pptr->state = PRTE_PROC_STATE_RUNNING;
     PMIX_RETAIN(node); /* keep accounting straight */
-    proc->node = node;
-    pmix_pointer_array_set_item(jdata->procs, PRTE_PROC_MY_NAME->rank, proc);
+    pptr->node = node;
+    pmix_pointer_array_set_item(jdata->procs, PRTE_PROC_MY_NAME->rank, pptr);
 
     /* setup to detect any external allocation */
-    if (PRTE_SUCCESS
-        != (ret = pmix_mca_base_framework_open(&prte_ras_base_framework,
-                                               PMIX_MCA_BASE_OPEN_DEFAULT))) {
+    ret = pmix_mca_base_framework_open(&prte_ras_base_framework,
+                                       PMIX_MCA_BASE_OPEN_DEFAULT);
+    if (PRTE_SUCCESS != ret) {
         PRTE_ERROR_LOG(ret);
-        error = "prte_ras_base_open";
-        goto error;
+        prte_exit_status = ret;
+        goto DONE;
     }
     if (PRTE_SUCCESS != (ret = prte_ras_base_select())) {
         PRTE_ERROR_LOG(ret);
-        error = "prte_ras_base_find_available";
-        goto error;
+        prte_exit_status = ret;
+        goto DONE;
     }
 
     /* add our topology to the array of known topologies */
@@ -542,7 +598,7 @@ int main(int argc, char *argv[])
     PRTE_UPDATE_EXIT_STATUS(ret);
 
     /* cleanup and leave */
-    prte_finalize();
+    psched_server_finalize();
 
     prte_session_dir_cleanup(PRTE_PROC_MY_NAME->nspace);
     /* cleanup the process info */
@@ -553,3 +609,64 @@ int main(int argc, char *argv[])
     }
     exit(prte_exit_status);
 }
+
+static void clean_abort(int fd, short flags, void *arg)
+{
+    PRTE_HIDE_UNUSED_PARAMS(fd, flags, arg);
+
+    /* if we have already ordered this once, don't keep
+     * doing it to avoid race conditions
+     */
+    if (pmix_mutex_trylock(&abort_inprogress_lock)) { /* returns 1 if already locked */
+        if (forcibly_die) {
+            /* exit with a non-zero status */
+            exit(1);
+        }
+        fprintf(stderr,
+                "%s: abort is already in progress...hit ctrl-c again to forcibly terminate\n\n",
+                prte_tool_basename);
+        forcibly_die = true;
+        /* reset the event */
+        prte_event_add(&term_handler, NULL);
+        return;
+    }
+
+    fflush(stderr);
+    /* ensure we exit with a non-zero status */
+    PRTE_UPDATE_EXIT_STATUS(PRTE_ERROR_DEFAULT_EXIT_CODE);
+    // stop the event loop
+    prte_event_base_active = false;
+}
+
+static bool first = true;
+static bool second = true;
+
+/*
+ * Attempt to terminate and wait for callback
+ */
+static void abort_signal_callback(int fd)
+{
+    uint8_t foo = 1;
+    char *msg = "Abort is in progress...hit ctrl-c again to forcibly terminate\n\n";
+    PRTE_HIDE_UNUSED_PARAMS(fd);
+
+    /* if this is the first time thru, just get
+     * the current time
+     */
+    if (first) {
+        first = false;
+        /* tell the event lib to attempt to abnormally terminate */
+        if (-1 == write(term_pipe[1], &foo, 1)) {
+            exit(1);
+        }
+    } else if (second) {
+        if (-1 == write(2, (void *) msg, strlen(msg))) {
+            exit(1);
+        }
+        fflush(stderr);
+        second = false;
+    } else {
+        pmix_os_dirpath_destroy(prte_process_info.jobfam_session_dir, true, NULL);
+        exit(1);
+    }
+}
diff --git a/src/tools/psched/psched.h b/src/tools/psched/psched.h
new file mode 100644
index 0000000000..8c066af526
--- /dev/null
+++ b/src/tools/psched/psched.h
@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2020 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved
+ * Copyright (c) 2014-2019 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2021-2023 Nanook Consulting.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef PSCHED_H
+#define PSCHED_H
+
+#include "prte_config.h"
+
+#include "src/pmix/pmix-internal.h"
+#include "src/class/pmix_list.h"
+#include "src/class/pmix_pointer_array.h"
+#include "src/mca/schizo/schizo.h"
+
+BEGIN_C_DECLS
+
+typedef struct {
+    bool initialized;
+    pmix_pointer_array_t requests;
+    pmix_list_t tools;
+    pmix_proc_t syscontroller;
+    bool controller_connected;
+    int verbosity;
+    int output;
+} psched_globals_t;
+
+extern psched_globals_t psched_globals;
+
+void psched_state_init(void);
+void psched_errmgr_init(void);
+int psched_server_init(void);
+void psched_server_finalize(void);
+void psched_scheduler_init(void);
+void psched_scheduler_finalize(void);
+
+extern pmix_status_t psched_register_events_fn(pmix_status_t *codes, size_t ncodes,
+                                               const pmix_info_t info[], size_t ninfo,
+                                               pmix_op_cbfunc_t cbfunc, void *cbdata);
+
+extern pmix_status_t psched_deregister_events_fn(pmix_status_t *codes,
+                                                 size_t ncodes,
+                                                 pmix_op_cbfunc_t cbfunc,
+                                                 void *cbdata);
+
+extern pmix_status_t psched_notify_event(pmix_status_t code,
+                                         const pmix_proc_t *source,
+                                         pmix_data_range_t range,
+                                         pmix_info_t info[], size_t ninfo,
+                                         pmix_op_cbfunc_t cbfunc, void *cbdata);
+
+extern pmix_status_t psched_query_fn(pmix_proc_t *proct,
+                                     pmix_query_t *queries, size_t nqueries,
+                                     pmix_info_cbfunc_t cbfunc, void *cbdata);
+
+extern void psched_tool_connected_fn(pmix_info_t *info, size_t ninfo,
+                                    pmix_tool_connection_cbfunc_t cbfunc, void *cbdata);
+
+extern void psched_log_fn(const pmix_proc_t *client,
+                          const pmix_info_t data[], size_t ndata,
+                          const pmix_info_t directives[], size_t ndirs,
+                          pmix_op_cbfunc_t cbfunc, void *cbdata);
+
+extern pmix_status_t psched_job_ctrl_fn(const pmix_proc_t *requestor,
+                                        const pmix_proc_t targets[], size_t ntargets,
+                                        const pmix_info_t directives[], size_t ndirs,
+                                        pmix_info_cbfunc_t cbfunc, void *cbdata);
+
+extern pmix_status_t psched_group_fn(pmix_group_operation_t op, char *gpid,
+                                     const pmix_proc_t procs[], size_t nprocs,
+                                     const pmix_info_t directives[], size_t ndirs,
+                                     pmix_info_cbfunc_t cbfunc, void *cbdata);
+
+extern pmix_status_t psched_alloc_fn(const pmix_proc_t *client,
+                                     pmix_alloc_directive_t directive,
+                                     const pmix_info_t data[], size_t ndata,
+                                     pmix_info_cbfunc_t cbfunc, void *cbdata);
+
+#if PMIX_NUMERIC_VERSION >= 0x00050000
+extern pmix_status_t psched_session_ctrl_fn(const pmix_proc_t *requestor,
+                                            uint32_t sessionID,
+                                            const pmix_info_t directives[], size_t ndirs,
+                                            pmix_info_cbfunc_t cbfunc, void *cbdata);
+#endif
+
+extern prte_schizo_base_module_t psched_schizo_module;
+
+END_C_DECLS
+
+#endif /* PSCHED_H */
diff --git a/src/tools/psched/queries.c b/src/tools/psched/queries.c
new file mode 100644
index 0000000000..e3c0db5907
--- /dev/null
+++ b/src/tools/psched/queries.c
@@ -0,0 +1,832 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2009-2020 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2017 Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2014-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2020      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2021-2023 Nanook Consulting.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include "prte_config.h"
+
+#ifdef HAVE_UNISTD_H
+#    include <unistd.h>
+#endif
+
+#include "src/hwloc/hwloc-internal.h"
+#include "src/pmix/pmix-internal.h"
+#include "src/util/pmix_argv.h"
+#include "src/util/pmix_os_path.h"
+#include "src/util/pmix_path.h"
+#include "src/util/pmix_output.h"
+
+#include "src/mca/errmgr/errmgr.h"
+#include "src/mca/iof/iof.h"
+#include "src/mca/plm/base/plm_private.h"
+#include "src/mca/plm/plm.h"
+#include "src/mca/rmaps/rmaps_types.h"
+#include "src/rml/rml.h"
+#include "src/mca/schizo/schizo.h"
+#include "src/mca/state/state.h"
+#include "src/runtime/prte_globals.h"
+#include "src/threads/pmix_threads.h"
+#include "src/util/name_fns.h"
+#include "src/util/pmix_show_help.h"
+
+#include "src/prted/pmix/pmix_server_internal.h"
+#include "src/tools/psched/psched.h"
+
+static void qrel(void *cbdata)
+{
+    prte_pmix_server_op_caddy_t *cd = (prte_pmix_server_op_caddy_t *) cbdata;
+    if (NULL != cd->info) {
+        PMIX_INFO_FREE(cd->info, cd->ninfo);
+    }
+    PMIX_RELEASE(cd);
+}
+
+static void _query(int sd, short args, void *cbdata)
+{
+    prte_pmix_server_op_caddy_t *cd = (prte_pmix_server_op_caddy_t *) cbdata;
+    prte_pmix_server_op_caddy_t *rcd;
+    pmix_query_t *q;
+    pmix_status_t ret = PMIX_SUCCESS;
+    void *results, *plist, *stack, *cache;
+    prte_info_item_t *kv;
+    pmix_nspace_t jobid;
+    prte_job_t *jdata;
+    prte_node_t *node, *ndptr;
+    int j, k, rc;
+    size_t m, n, p;
+    uint32_t key, nodeid;
+    char **nspaces, *hostname, *uri;
+    char *cmdline;
+    char **ans, *tmp;
+    char *psetname;
+    prte_app_context_t *app;
+    int matched;
+    pmix_proc_info_t *procinfo;
+    pmix_info_t *info;
+    pmix_data_array_t dry;
+    prte_proc_t *proct;
+    pmix_proc_t *proc;
+    size_t sz;
+    PRTE_HIDE_UNUSED_PARAMS(sd, args);
+
+    PMIX_ACQUIRE_OBJECT(cd);
+
+    pmix_output_verbose(2, prte_pmix_server_globals.output,
+                        "%s processing query",
+                        PRTE_NAME_PRINT(PRTE_PROC_MY_NAME));
+
+    PMIX_INFO_LIST_START(results);
+
+    /* see what they wanted */
+    for (m = 0; m < cd->nqueries; m++) {
+        q = &cd->queries[m];
+        hostname = NULL;
+        nodeid = UINT32_MAX;
+        psetname = NULL;
+        /* default to the requestor's jobid */
+        PMIX_LOAD_NSPACE(jobid, cd->proct.nspace);
+        /* see if they provided any qualifiers */
+        if (NULL != q->qualifiers && 0 < q->nqual) {
+            for (n = 0; n < q->nqual; n++) {
+                pmix_output_verbose(2, prte_pmix_server_globals.output,
+                                    "%s qualifier key \"%s\" : value \"%s\"",
+                                    PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), q->qualifiers[n].key,
+                                    (q->qualifiers[n].value.type == PMIX_STRING
+                                         ? q->qualifiers[n].value.data.string
+                                         : "(not a string)"));
+
+                if (PMIX_CHECK_KEY(&q->qualifiers[n], PMIX_NSPACE)) {
+                    /* Never trust the namespace string that is provided.
+                     * First check to see if we know about this namespace. If
+                     * not then return an error. If so then continue on.
+                     */
+                    /* Make sure the qualifier namespace exists */
+                    matched = 0;
+                    for (k = 0; k < prte_job_data->size; k++) {
+                        jdata = (prte_job_t *) pmix_pointer_array_get_item(prte_job_data, k);
+                        if (NULL != jdata) {
+                            if (PMIX_CHECK_NSPACE(q->qualifiers[n].value.data.string, jdata->nspace)) {
+                                matched = 1;
+                                break;
+                            }
+                        }
+                    }
+                    if (0 == matched) {
+                        pmix_output_verbose(2, prte_pmix_server_globals.output,
+                                            "%s qualifier key \"%s\" : value \"%s\" is an unknown namespace",
+                                            PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), q->qualifiers[n].key,
+                                            q->qualifiers[n].value.data.string);
+                        ret = PMIX_ERR_BAD_PARAM;
+                        goto done;
+                    }
+
+                    PMIX_LOAD_NSPACE(jobid, q->qualifiers[n].value.data.string);
+                    if (PMIX_NSPACE_INVALID(jobid)) {
+                        ret = PMIX_ERR_BAD_PARAM;
+                        goto done;
+                    }
+
+                } else if (PMIX_CHECK_KEY(&q->qualifiers[n], PMIX_GROUP_ID)) {
+                    /* Never trust the group string that is provided.
+                     * First check to see if we know about this group. If
+                     * not then return an error. If so then continue on.
+                     */
+                    /* Make sure the qualifier group exists */
+                    matched = 0;
+                    pmix_server_pset_t *ps;
+                    PMIX_LIST_FOREACH(ps, &prte_pmix_server_globals.groups, pmix_server_pset_t)
+                    {
+                        if (PMIX_CHECK_NSPACE(q->qualifiers[n].value.data.string, ps->name)) {
+                            matched = 1;
+                            break;
+                        }
+                    }
+                    if (0 == matched) {
+                        pmix_output_verbose(2, prte_pmix_server_globals.output,
+                                            "%s qualifier key \"%s\" : value \"%s\" is an unknown group",
+                                            PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), q->qualifiers[n].key,
+                                            q->qualifiers[n].value.data.string);
+                        ret = PMIX_ERR_BAD_PARAM;
+                        goto done;
+                    }
+                    PMIX_LOAD_NSPACE(jobid, q->qualifiers[n].value.data.string);
+                    if (PMIX_NSPACE_INVALID(jobid)) {
+                        ret = PMIX_ERR_BAD_PARAM;
+                        goto done;
+                    }
+
+                } else if (PMIX_CHECK_KEY(&q->qualifiers[n], PMIX_HOSTNAME)) {
+                    hostname = q->qualifiers[n].value.data.string;
+
+                } else if (PMIX_CHECK_KEY(&q->qualifiers[n], PMIX_NODEID)) {
+                    PMIX_VALUE_GET_NUMBER(rc, &q->qualifiers[n].value, nodeid, uint32_t);
+
+                } else if (PMIX_CHECK_KEY(&q->qualifiers[n], PMIX_PSET_NAME)) {
+                    psetname = q->qualifiers[n].value.data.string;
+                }
+
+            }
+        }
+        for (n = 0; NULL != q->keys[n]; n++) {
+            pmix_output_verbose(2, prte_pmix_server_globals.output,
+                                "%s processing key %s",
+                                PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), q->keys[n]);
+
+            if (0 == strcmp(q->keys[n], PMIX_QUERY_NAMESPACES)) {
+                /* get the current jobids */
+                nspaces = NULL;
+                for (k = 0; k < prte_job_data->size; k++) {
+                    jdata = (prte_job_t *) pmix_pointer_array_get_item(prte_job_data, k);
+                    if (NULL == jdata) {
+                        continue;
+                    }
+                    /* don't show the requestor's job */
+                    if (!PMIX_CHECK_NSPACE(PRTE_PROC_MY_NAME->nspace, jdata->nspace)) {
+                        PMIX_ARGV_APPEND_NOSIZE_COMPAT(&nspaces, jdata->nspace);
+                    }
+                }
+                /* join the results into a single comma-delimited string */
+                tmp = PMIX_ARGV_JOIN_COMPAT(nspaces, ',');
+                PMIX_ARGV_FREE_COMPAT(nspaces);
+                PMIX_INFO_LIST_ADD(rc, results, PMIX_QUERY_NAMESPACES, tmp, PMIX_STRING);
+                free(tmp);
+                if (PMIX_SUCCESS != rc) {
+                    PMIX_ERROR_LOG(rc);
+                    goto done;
+                }
+
+            } else if (0 == strcmp(q->keys[n], PMIX_QUERY_NAMESPACE_INFO)) {
+                PMIX_INFO_LIST_START(cache);
+                /* get the current jobids */
+                for (k = 0; k < prte_job_data->size; k++) {
+                    jdata = (prte_job_t *) pmix_pointer_array_get_item(prte_job_data, k);
+                    if (NULL == jdata) {
+                        continue;
+                    }
+                    /* don't show the requestor's job */
+                    if (!PMIX_CHECK_NSPACE(PRTE_PROC_MY_NAME->nspace, jdata->nspace)) {
+                        PMIX_INFO_LIST_START(stack);
+                        /* add the nspace name */
+                        PMIX_INFO_LIST_ADD(rc, stack, PMIX_NSPACE, jdata->nspace, PMIX_STRING);
+                        if (PMIX_SUCCESS != rc) {
+                            PMIX_ERROR_LOG(rc);
+                            PMIX_INFO_LIST_RELEASE(stack);
+                            goto done;
+                        }
+                        /* add the cmd line */
+                        app = (prte_app_context_t *) pmix_pointer_array_get_item(jdata->apps, 0);
+                        if (NULL == app) {
+                            ret = PMIX_ERR_NOT_FOUND;
+                            goto done;
+                        }
+                        cmdline = PMIX_ARGV_JOIN_COMPAT(app->argv, ' ');
+                        PMIX_INFO_LIST_ADD(rc, stack, PMIX_CMD_LINE, cmdline, PMIX_STRING);
+                        free(cmdline);
+                        /* add the job size */
+                        PMIX_INFO_LIST_ADD(rc, stack, PMIX_JOB_SIZE, &jdata->num_procs, PMIX_UINT32);
+                        if (PMIX_SUCCESS != rc) {
+                            PMIX_ERROR_LOG(rc);
+                            PMIX_INFO_LIST_RELEASE(stack);
+                            goto done;
+                        }
+                        /* construct info on each process in the job */
+                        for (j=0; j < jdata->procs->size; j++) {
+                            proct = (prte_proc_t*)pmix_pointer_array_get_item(jdata->procs, j);
+                            if (NULL == proct) {
+                                continue;
+                            }
+                            PMIX_INFO_LIST_START(plist);
+                            /* add the proc's rank */
+                            PMIX_INFO_LIST_ADD(rc, plist, PMIX_RANK, &proct->name.rank, PMIX_PROC_RANK);
+                            if (PMIX_SUCCESS != rc) {
+                                PMIX_ERROR_LOG(rc);
+                                PMIX_INFO_LIST_RELEASE(stack);
+                                PMIX_INFO_LIST_RELEASE(plist);
+                                goto done;
+                            }
+                            /* add the proc's hostname */
+                            PMIX_INFO_LIST_ADD(rc, plist, PMIX_HOSTNAME, proct->node->name, PMIX_STRING);
+                            if (PMIX_SUCCESS != rc) {
+                                PMIX_ERROR_LOG(rc);
+                                PMIX_INFO_LIST_RELEASE(stack);
+                                PMIX_INFO_LIST_RELEASE(plist);
+                                goto done;
+                            }
+                            /* add the proc's local rank */
+                            PMIX_INFO_LIST_ADD(rc, plist, PMIX_LOCAL_RANK, &proct->local_rank, PMIX_UINT16);
+                            if (PMIX_SUCCESS != rc) {
+                                PMIX_ERROR_LOG(rc);
+                                PMIX_INFO_LIST_RELEASE(stack);
+                                PMIX_INFO_LIST_RELEASE(plist);
+                                goto done;
+                            }
+                            /* add to the stack */
+                            PMIX_INFO_LIST_CONVERT(rc, plist, &dry);
+                            if (PMIX_SUCCESS != rc) {
+                                PMIX_ERROR_LOG(rc);
+                                PMIX_INFO_LIST_RELEASE(stack);
+                                PMIX_INFO_LIST_RELEASE(plist);
+                                goto done;
+                            }
+                            PMIX_INFO_LIST_RELEASE(plist);
+                            PMIX_INFO_LIST_ADD(rc, stack, PMIX_PROC_INFO_ARRAY, &dry, PMIX_DATA_ARRAY);
+                            PMIX_DATA_ARRAY_DESTRUCT(&dry);
+                        }
+                        /* add the result to our cache */
+                        PMIX_INFO_LIST_CONVERT(rc, stack, &dry);
+                        if (PMIX_SUCCESS != rc) {
+                            PMIX_ERROR_LOG(rc);
+                            PMIX_INFO_LIST_RELEASE(stack);
+                            goto done;
+                        }
+                        PMIX_INFO_LIST_RELEASE(stack);
+                        PMIX_INFO_LIST_ADD(rc, cache, PMIX_JOB_INFO_ARRAY, &dry, PMIX_DATA_ARRAY);
+                        PMIX_DATA_ARRAY_DESTRUCT(&dry);
+                    }
+                }
+                /* add our findings to the results */
+                PMIX_INFO_LIST_CONVERT(rc, cache, &dry);
+                if (PMIX_SUCCESS != rc) {
+                    PMIX_ERROR_LOG(rc);
+                    PMIX_INFO_LIST_RELEASE(cache);
+                    goto done;
+                }
+                PMIX_INFO_LIST_RELEASE(cache);
+                PMIX_INFO_LIST_ADD(rc, results, PMIX_QUERY_NAMESPACE_INFO, &dry, PMIX_DATA_ARRAY);
+                PMIX_DATA_ARRAY_DESTRUCT(&dry);
+
+            } else if (0 == strcmp(q->keys[n], PMIX_QUERY_SPAWN_SUPPORT)) {
+                ans = NULL;
+                PMIX_ARGV_APPEND_NOSIZE_COMPAT(&ans, PMIX_HOST);
+                PMIX_ARGV_APPEND_NOSIZE_COMPAT(&ans, PMIX_HOSTFILE);
+                PMIX_ARGV_APPEND_NOSIZE_COMPAT(&ans, PMIX_ADD_HOST);
+                PMIX_ARGV_APPEND_NOSIZE_COMPAT(&ans, PMIX_ADD_HOSTFILE);
+                PMIX_ARGV_APPEND_NOSIZE_COMPAT(&ans, PMIX_PREFIX);
+                PMIX_ARGV_APPEND_NOSIZE_COMPAT(&ans, PMIX_WDIR);
+                PMIX_ARGV_APPEND_NOSIZE_COMPAT(&ans, PMIX_MAPPER);
+                PMIX_ARGV_APPEND_NOSIZE_COMPAT(&ans, PMIX_PPR);
+                PMIX_ARGV_APPEND_NOSIZE_COMPAT(&ans, PMIX_MAPBY);
+                PMIX_ARGV_APPEND_NOSIZE_COMPAT(&ans, PMIX_RANKBY);
+                PMIX_ARGV_APPEND_NOSIZE_COMPAT(&ans, PMIX_BINDTO);
+                PMIX_ARGV_APPEND_NOSIZE_COMPAT(&ans, PMIX_COSPAWN_APP);
+                /* create the return kv */
+                tmp = PMIX_ARGV_JOIN_COMPAT(ans, ',');
+                PMIX_ARGV_FREE_COMPAT(ans);
+                PMIX_INFO_LIST_ADD(rc, results, PMIX_QUERY_SPAWN_SUPPORT, tmp, PMIX_STRING);
+                free(tmp);
+                if (PMIX_SUCCESS != rc) {
+                    PMIX_ERROR_LOG(rc);
+                    goto done;
+                }
+
+            } else if (0 == strcmp(q->keys[n], PMIX_QUERY_DEBUG_SUPPORT)) {
+                ans = NULL;
+                PMIX_ARGV_APPEND_NOSIZE_COMPAT(&ans, PMIX_DEBUG_STOP_IN_INIT);
+                PMIX_ARGV_APPEND_NOSIZE_COMPAT(&ans, PMIX_DEBUG_STOP_IN_APP);
+#if PRTE_HAVE_STOP_ON_EXEC
+                PMIX_ARGV_APPEND_NOSIZE_COMPAT(&ans, PMIX_DEBUG_STOP_ON_EXEC);
+#endif
+                PMIX_ARGV_APPEND_NOSIZE_COMPAT(&ans, PMIX_DEBUG_TARGET);
+                /* create the return kv */
+                tmp = PMIX_ARGV_JOIN_COMPAT(ans, ',');
+                PMIX_ARGV_FREE_COMPAT(ans);
+                PMIX_INFO_LIST_ADD(rc, results, PMIX_QUERY_DEBUG_SUPPORT, tmp, PMIX_STRING);
+                free(tmp);
+                if (PMIX_SUCCESS != rc) {
+                    PMIX_ERROR_LOG(rc);
+                    goto done;
+                }
+
+            } else if (0 == strcmp(q->keys[n], PMIX_HWLOC_XML_V1)) {
+                if (NULL != prte_hwloc_topology) {
+                    char *xmlbuffer = NULL;
+                    int len;
+                    kv = PMIX_NEW(prte_info_item_t);
+#if HWLOC_API_VERSION < 0x20000
+                    /* get this from the v1.x API */
+                    if (0 != hwloc_topology_export_xmlbuffer(prte_hwloc_topology, &xmlbuffer, &len)) {
+                        PMIX_RELEASE(kv);
+                        continue;
+                    }
+#else
+                    /* get it from the v2 API */
+                    if (0 != hwloc_topology_export_xmlbuffer(prte_hwloc_topology, &xmlbuffer, &len,
+                                                             HWLOC_TOPOLOGY_EXPORT_XML_FLAG_V1)) {
+                        PMIX_RELEASE(kv);
+                        continue;
+                    }
+#endif
+                    PMIX_INFO_LIST_ADD(rc, results, PMIX_HWLOC_XML_V1, xmlbuffer, PMIX_STRING);
+                    free(xmlbuffer);
+                    if (PMIX_SUCCESS != rc) {
+                        PMIX_ERROR_LOG(rc);
+                        goto done;
+                    }
+                }
+
+            } else if (0 == strcmp(q->keys[n], PMIX_HWLOC_XML_V2)) {
+                /* we cannot provide it if we are using v1.x */
+#if HWLOC_API_VERSION >= 0x20000
+                if (NULL != prte_hwloc_topology) {
+                    char *xmlbuffer = NULL;
+                    int len;
+                    kv = PMIX_NEW(prte_info_item_t);
+                    if (0 != hwloc_topology_export_xmlbuffer(prte_hwloc_topology, &xmlbuffer, &len, 0)) {
+                        PMIX_RELEASE(kv);
+                        continue;
+                    }
+                    PMIX_INFO_LIST_ADD(rc, results, PMIX_HWLOC_XML_V2, xmlbuffer, PMIX_STRING);
+                    free(xmlbuffer);
+                    if (PMIX_SUCCESS != rc) {
+                        PMIX_ERROR_LOG(rc);
+                        goto done;
+                    }
+                }
+#endif
+
+            } else if (0 == strcmp(q->keys[n], PMIX_PROC_URI)) {
+                /* they want our URI */
+                PMIX_INFO_LIST_ADD(rc, results, PMIX_PROC_URI, prte_process_info.my_hnp_uri, PMIX_STRING);
+                if (PMIX_SUCCESS != rc) {
+                    PMIX_ERROR_LOG(rc);
+                    goto done;
+                }
+
+            } else if (0 == strcmp(q->keys[n], PMIX_SERVER_URI)) {
+                /* they want the PMIx URI */
+                if (NULL != hostname) {
+                    /* find the node object */
+                    node = NULL;
+                    for (k = 0; k < prte_node_pool->size; k++) {
+                        ndptr = (prte_node_t *) pmix_pointer_array_get_item(prte_node_pool, k);
+                        if (NULL == ndptr) {
+                            continue;
+                        }
+                        if (0 == strcmp(hostname, ndptr->name)) {
+                            node = ndptr;
+                            break;
+                        }
+                    }
+                    if (NULL == node) {
+                        /* unknown node */
+                        ret = PMIX_ERR_BAD_PARAM;
+                        goto done;
+                    }
+                    /* we want the info for the server on that node */
+                    if (NULL == node->daemon) {
+                        /* not found */
+                        ret = PMIX_ERR_BAD_PARAM;
+                        goto done;
+                    }
+                    proct = node->daemon;
+                } else if (UINT32_MAX != nodeid) {
+                    /* get the node object at that index */
+                    node = (prte_node_t *) pmix_pointer_array_get_item(prte_node_pool, nodeid);
+                    if (NULL == node) {
+                        /* bad index */
+                        ret = PMIX_ERR_BAD_PARAM;
+                        goto done;
+                    }
+                    /* we want the info for the server on that node */
+                    if (NULL == node->daemon) {
+                        /* not found */
+                        ret = PMIX_ERR_BAD_PARAM;
+                        goto done;
+                    }
+                    proct = node->daemon;
+                } else {
+                    /* send them ours */
+                    proct = prte_get_proc_object(PRTE_PROC_MY_NAME);
+                }
+                /* get the server uri value - we can block here as we are in
+                 * an PRTE progress thread */
+                PRTE_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_SERVER_URI, &proct->name,
+                                               (char **) &uri, PMIX_STRING);
+                if (PRTE_SUCCESS != rc) {
+                    ret = prte_pmix_convert_rc(rc);
+                    goto done;
+                }
+                PMIX_INFO_LIST_ADD(rc, results, PMIX_SERVER_URI, uri, PMIX_STRING);
+                free(uri);
+                if (PMIX_SUCCESS != rc) {
+                    PMIX_ERROR_LOG(rc);
+                    goto done;
+                }
+
+            } else if (0 == strcmp(q->keys[n], PMIX_QUERY_PROC_TABLE)) {
+                /* construct a list of values with prte_proc_info_t
+                 * entries for each proc in the indicated job */
+                jdata = prte_get_job_data_object(jobid);
+                if (NULL == jdata) {
+                    ret = PMIX_ERR_NOT_FOUND;
+                    goto done;
+                }
+                /* Check if there are any entries in global proctable */
+                if (0 == jdata->num_procs) {
+                    ret = PMIX_ERR_NOT_FOUND;
+                    goto done;
+                }
+                /* cycle thru the job and create an entry for each proc */
+                PMIX_DATA_ARRAY_CONSTRUCT(&dry, jdata->num_procs, PMIX_PROC_INFO);
+                procinfo = (pmix_proc_info_t*)dry.array;
+                p = 0;
+                for (k = 0; k < jdata->procs->size; k++) {
+                    proct = (prte_proc_t *) pmix_pointer_array_get_item(jdata->procs, k);
+                    if (NULL == proct) {
+                        continue;
+                    }
+                    PMIX_LOAD_PROCID(&procinfo[p].proc, proct->name.nspace, proct->name.rank);
+                    if (NULL != proct->node && NULL != proct->node->name) {
+                        procinfo[p].hostname = strdup(proct->node->name);
+                    }
+                    app = (prte_app_context_t *) pmix_pointer_array_get_item(jdata->apps,
+                                                                             proct->app_idx);
+                    if (NULL != app && NULL != app->app) {
+                        if (pmix_path_is_absolute(app->app)) {
+                            procinfo[p].executable_name = strdup(app->app);
+                        } else {
+                            procinfo[p].executable_name = pmix_os_path(false, app->cwd, app->app, NULL);
+                        }
+                    }
+                    procinfo[p].pid = proct->pid;
+                    procinfo[p].exit_code = proct->exit_code;
+                    procinfo[p].state = prte_pmix_convert_state(proct->state);
+                    ++p;
+                }
+                PMIX_INFO_LIST_ADD(rc, results, PMIX_QUERY_PROC_TABLE, &dry, PMIX_DATA_ARRAY);
+                PMIX_DATA_ARRAY_DESTRUCT(&dry);
+                if (PMIX_SUCCESS != rc) {
+                    PMIX_ERROR_LOG(rc);
+                    goto done;
+                }
+
+            } else if (0 == strcmp(q->keys[n], PMIX_QUERY_LOCAL_PROC_TABLE)) {
+                /* construct a list of values with prte_proc_info_t
+                 * entries for each LOCAL proc in the indicated job */
+                jdata = prte_get_job_data_object(jobid);
+                if (NULL == jdata) {
+                    ret = PMIX_ERR_NOT_FOUND;
+                    goto done;
+                }
+                /* Check if there are any entries in local proctable */
+                if (0 == jdata->num_local_procs) {
+                    ret = PMIX_ERR_NOT_FOUND;
+                    goto done;
+                }
+                /* cycle thru the job and create an entry for each proc */
+                PMIX_DATA_ARRAY_CONSTRUCT(&dry, jdata->num_local_procs, PMIX_PROC_INFO);
+                procinfo = (pmix_proc_info_t *) dry.array;
+                p = 0;
+                for (k = 0; k < jdata->procs->size; k++) {
+                    proct = (prte_proc_t *) pmix_pointer_array_get_item(jdata->procs, k);
+                    if (NULL == proct) {
+                        continue;
+                    }
+                    if (PRTE_FLAG_TEST(proct, PRTE_PROC_FLAG_LOCAL)) {
+                        PMIX_LOAD_PROCID(&procinfo[p].proc, proct->name.nspace, proct->name.rank);
+                        if (NULL != proct->node && NULL != proct->node->name) {
+                            procinfo[p].hostname = strdup(proct->node->name);
+                        }
+                        app = (prte_app_context_t *) pmix_pointer_array_get_item(jdata->apps,
+                                                                                 proct->app_idx);
+                        if (NULL != app && NULL != app->app) {
+                            if (pmix_path_is_absolute(app->app)) {
+                                procinfo[p].executable_name = strdup(app->app);
+                            } else {
+                                procinfo[p].executable_name = pmix_os_path(false, app->cwd, app->app, NULL);
+                            }
+                        }
+                        procinfo[p].pid = proct->pid;
+                        procinfo[p].exit_code = proct->exit_code;
+                        procinfo[p].state = prte_pmix_convert_state(proct->state);
+                        ++p;
+                    }
+                }
+                PMIX_INFO_LIST_ADD(rc, results, PMIX_QUERY_LOCAL_PROC_TABLE, &dry, PMIX_DATA_ARRAY);
+                PMIX_DATA_ARRAY_DESTRUCT(&dry);
+                if (PMIX_SUCCESS != rc) {
+                    PMIX_ERROR_LOG(rc);
+                    goto done;
+                }
+
+            } else if (0 == strcmp(q->keys[n], PMIX_QUERY_NUM_PSETS)) {
+                sz = pmix_list_get_size(&prte_pmix_server_globals.psets);
+                PMIX_INFO_LIST_ADD(rc, results, PMIX_QUERY_NUM_PSETS, &sz, PMIX_SIZE);
+                if (PMIX_SUCCESS != rc) {
+                    PMIX_ERROR_LOG(rc);
+                    goto done;
+                }
+
+            } else if (0 == strcmp(q->keys[n], PMIX_QUERY_PSET_NAMES)) {
+                pmix_server_pset_t *ps;
+                ans = NULL;
+                PMIX_LIST_FOREACH(ps, &prte_pmix_server_globals.psets, pmix_server_pset_t)
+                {
+                    PMIX_ARGV_APPEND_NOSIZE_COMPAT(&ans, ps->name);
+                }
+                if (NULL == ans) {
+                    ret = PMIX_ERR_NOT_FOUND;
+                    goto done;
+                } else {
+                    tmp = PMIX_ARGV_JOIN_COMPAT(ans, ',');
+                    PMIX_ARGV_FREE_COMPAT(ans);
+                    ans = NULL;
+                    PMIX_INFO_LIST_ADD(rc, results, PMIX_QUERY_PSET_NAMES, tmp, PMIX_STRING);
+                    free(tmp);
+                    if (PMIX_SUCCESS != rc) {
+                        PMIX_ERROR_LOG(rc);
+                        goto done;
+                    }
+                }
+
+            } else if (0 == strcmp(q->keys[n], PMIX_QUERY_PSET_MEMBERSHIP)) {
+                pmix_server_pset_t *ps, *psptr;
+                /* must have provided us with a pset name qualifier */
+                if (NULL == psetname) {
+                    ret = PMIX_ERR_BAD_PARAM;
+                    goto done;
+                }
+                ans = NULL;
+                /* find the referenced pset */
+                psptr = NULL;
+                PMIX_LIST_FOREACH(ps, &prte_pmix_server_globals.psets, pmix_server_pset_t) {
+                    if (0 == strcmp(psetname, ps->name)) {
+                        psptr = ps;
+                        break;
+                    }
+                }
+                if (NULL == psptr) {
+                    /* we don't know that pset */
+                    ret = PMIX_ERR_NOT_FOUND;
+                    goto done;
+                }
+                /* define the array that holds the membership - no need to allocate anything */
+                dry.array = psptr->members;
+                dry.type = PMIX_PROC_RANK;
+                dry.size = psptr->num_members;
+                PMIX_INFO_LIST_ADD(rc, results, PMIX_QUERY_PSET_MEMBERSHIP, &dry, PMIX_DATA_ARRAY);
+                PMIX_DATA_ARRAY_DESTRUCT(&dry);
+                if (PMIX_SUCCESS != rc) {
+                    PMIX_ERROR_LOG(rc);
+                    goto done;
+                }
+
+            } else if (0 == strcmp(q->keys[n], PMIX_JOB_SIZE)) {
+                jdata = prte_get_job_data_object(jobid);
+                if (NULL == jdata) {
+                    ret = PMIX_ERR_NOT_FOUND;
+                    goto done;
+                }
+                /* setup the reply */
+                key = jdata->num_procs;
+                PMIX_INFO_LIST_ADD(rc, results, PMIX_JOB_SIZE, &key, PMIX_UINT32);
+                if (PMIX_SUCCESS != rc) {
+                    PMIX_ERROR_LOG(rc);
+                    goto done;
+                }
+
+            } else if (0 == strcmp(q->keys[n], PMIX_QUERY_NUM_GROUPS)) {
+                sz = pmix_list_get_size(&prte_pmix_server_globals.groups);
+                PMIX_INFO_LIST_ADD(rc, results, PMIX_QUERY_NUM_GROUPS, &sz, PMIX_SIZE);
+                if (PMIX_SUCCESS != rc) {
+                    PMIX_ERROR_LOG(rc);
+                    goto done;
+                }
+
+            } else if (0 == strcmp(q->keys[n], PMIX_QUERY_GROUP_NAMES)) {
+                pmix_server_pset_t *ps;
+                ans = NULL;
+                PMIX_LIST_FOREACH(ps, &prte_pmix_server_globals.groups, pmix_server_pset_t)
+                {
+                    PMIX_ARGV_APPEND_NOSIZE_COMPAT(&ans, ps->name);
+                }
+                tmp = PMIX_ARGV_JOIN_COMPAT(ans, ',');
+                PMIX_ARGV_FREE_COMPAT(ans);
+                PMIX_INFO_LIST_ADD(rc, results, PMIX_QUERY_GROUP_NAMES, tmp, PMIX_STRING);
+                free(tmp);
+                if (PMIX_SUCCESS != rc) {
+                    PMIX_ERROR_LOG(rc);
+                    goto done;
+                }
+
+            } else if (0 == strcmp(q->keys[n], PMIX_QUERY_GROUP_MEMBERSHIP)) {
+                /* construct a list of values with pmix_proc_t
+                 * entries for each proc in the indicated group */
+                pmix_server_pset_t *ps, *grp = NULL;
+                PMIX_LIST_FOREACH(ps, &prte_pmix_server_globals.groups, pmix_server_pset_t)
+                {
+                    if (PMIX_CHECK_NSPACE(ps->name, jobid)) {
+                        grp = ps;
+                        break;
+                    }
+                }
+                if (NULL == grp) {
+                    ret = PMIX_ERR_NOT_FOUND;
+                    goto done;
+                }
+                /* Check if there are any entries in the group */
+                if (0 == grp->num_members) {
+                    ret = PMIX_ERR_NOT_FOUND;
+                    goto done;
+                }
+                /* cycle thru the job and create an entry for each proc */
+                PMIX_DATA_ARRAY_CONSTRUCT(&dry, grp->num_members, PMIX_PROC);
+                proc = (pmix_proc_t *) dry.array;
+                for (k = 0; k < grp->num_members; k++) {
+                    PMIX_LOAD_PROCID(&proc[k], grp->members[k].nspace, grp->members[k].rank);
+                }
+                PMIX_INFO_LIST_ADD(rc, results, PMIX_QUERY_GROUP_MEMBERSHIP, &dry, PMIX_DATA_ARRAY);
+                PMIX_DATA_ARRAY_DESTRUCT(&dry);
+                if (PMIX_SUCCESS != rc) {
+                    PMIX_ERROR_LOG(rc);
+                    goto done;
+                }
+
+#if PMIX_NUMERIC_VERSION >= 0x00050000
+            } else if (0 == strcmp(q->keys[n], PMIX_QUERY_ALLOCATION)) {
+                /* collect all the node info */
+                void *nodelist, *nodeinfolist;
+                char *str;
+                prte_topology_t *topo;
+                int len;
+
+                PMIX_INFO_LIST_START(nodelist);
+                p = 0;
+                for (k=0; k < prte_node_pool->size; k++) {
+                    node = (prte_node_t*)pmix_pointer_array_get_item(prte_node_pool, k);
+                    if (NULL == node) {
+                        continue;
+                    }
+                    PMIX_INFO_LIST_START(nodeinfolist);
+                    /* start with the node name */
+                    PMIX_INFO_LIST_ADD(rc, nodeinfolist, PMIX_HOSTNAME, node->name, PMIX_STRING);
+                    /* add any aliases */
+                    if (NULL != node->aliases) {
+                        str = PMIX_ARGV_JOIN_COMPAT(node->aliases, ',');
+                        PMIX_INFO_LIST_ADD(rc, nodeinfolist, PMIX_HOSTNAME_ALIASES, str, PMIX_STRING);
+                        free(str);
+                    }
+                    /* add topology index */
+                    PMIX_INFO_LIST_ADD(rc, nodeinfolist, PMIX_TOPOLOGY_INDEX, &node->topology->index, PMIX_INT);
+                    /* convert to array */
+                    PMIX_INFO_LIST_CONVERT(rc, nodeinfolist, &dry);
+                    PMIX_INFO_LIST_RELEASE(nodeinfolist);
+                    /* now add the entry to the main list */
+                    PMIX_INFO_LIST_ADD(rc, nodelist, PMIX_NODE_INFO, &dry, PMIX_DATA_ARRAY);
+                    ++p;
+                    PMIX_DATA_ARRAY_DESTRUCT(&dry);
+                }
+                /* add topology info */
+                for (k=0; k < prte_node_topologies->size; k++) {
+                    topo = (prte_topology_t*)pmix_pointer_array_get_item(prte_node_topologies, k);
+                    if (NULL == topo) {
+                        continue;
+                    }
+                    /* convert the topology to XML representation */
+#if HWLOC_API_VERSION < 0x20000
+                    /* get this from the v1.x API */
+                    if (0 != hwloc_topology_export_xmlbuffer(topo->topo, &str, &len)) {
+                        continue;
+                    }
+                    PMIX_INFO_LIST_ADD(rc, nodelist, PMIX_HWLOC_XML_V1, str, PMIX_STRING);
+#else
+                    if (0 != hwloc_topology_export_xmlbuffer(topo->topo, &str, &len, 0)) {
+                        continue;
+                    }
+                    PMIX_INFO_LIST_ADD(rc, nodelist, PMIX_HWLOC_XML_V2, str, PMIX_STRING);
+#endif
+                    free(str);
+                }
+                /* convert list to array */
+                PMIX_INFO_LIST_CONVERT(rc, nodelist, &dry);
+                PMIX_INFO_LIST_RELEASE(nodelist);
+                /* add to results */
+                PMIX_INFO_LIST_ADD(rc, nodelist, PMIX_QUERY_ALLOCATION, &dry, PMIX_DATA_ARRAY);
+                PMIX_DATA_ARRAY_DESTRUCT(&dry);
+                if (PMIX_SUCCESS != rc) {
+                    PMIX_ERROR_LOG(rc);
+                    goto done;
+                }
+#endif
+            } else {
+                fprintf(stderr, "Query for unrecognized attribute: %s\n", q->keys[n]);
+            }
+        } // for
+    }     // for
+
+done:
+    rcd = PMIX_NEW(prte_pmix_server_op_caddy_t);
+    PMIX_INFO_LIST_CONVERT(rc, results, &dry);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        ret = rc;
+    }
+    PMIX_INFO_LIST_RELEASE(results);
+    if (PMIX_SUCCESS == ret) {
+        if (0 == dry.size) {
+            ret = PMIX_ERR_NOT_FOUND;
+        } else {
+            if (dry.size < cd->ninfo) {
+                ret = PMIX_QUERY_PARTIAL_SUCCESS;
+            } else {
+                ret = PMIX_SUCCESS;
+            }
+        }
+    }
+    rcd->ninfo = dry.size;
+    rcd->info = (pmix_info_t*)dry.array;
+    // memory allocated in the data array will be free'd when rcd is released
+    cd->infocbfunc(ret, rcd->info, rcd->ninfo, cd->cbdata, qrel, rcd);
+    PMIX_RELEASE(cd);
+}
+
+pmix_status_t psched_query_fn(pmix_proc_t *proct,
+                              pmix_query_t *queries, size_t nqueries,
+                              pmix_info_cbfunc_t cbfunc, void *cbdata)
+{
+    prte_pmix_server_op_caddy_t *cd;
+
+    if (NULL == queries || NULL == cbfunc) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    /* need to threadshift this request */
+    cd = PMIX_NEW(prte_pmix_server_op_caddy_t);
+    memcpy(&cd->proct, proct, sizeof(pmix_proc_t));
+    cd->queries = queries;
+    cd->nqueries = nqueries;
+    cd->infocbfunc = cbfunc;
+    cd->cbdata = cbdata;
+
+    prte_event_set(prte_event_base, &(cd->ev), -1, PRTE_EV_WRITE, _query, cd);
+    prte_event_set_priority(&(cd->ev), PRTE_MSG_PRI);
+    PMIX_POST_OBJECT(cd);
+    prte_event_active(&(cd->ev), PRTE_EV_WRITE, 1);
+
+    return PMIX_SUCCESS;
+}
diff --git a/src/tools/psched/scheduler.c b/src/tools/psched/scheduler.c
new file mode 100644
index 0000000000..84b1869d2c
--- /dev/null
+++ b/src/tools/psched/scheduler.c
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2018-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2020      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2021-2023 Nanook Consulting.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "prte_config.h"
+
+#include <sys/types.h>
+#ifdef HAVE_UNISTD_H
+#    include <unistd.h>
+#endif /* HAVE_UNISTD_H */
+#include <string.h>
+
+#include "src/pmix/pmix-internal.h"
+
+#include "src/tools/psched/psched.h"
+
+void psched_scheduler_init(void)
+{
+    return;
+}
+
+void psched_scheduler_finalize(void)
+{
+    return;
+}
diff --git a/src/mca/schizo/psched/schizo_psched.c b/src/tools/psched/schizo.c
similarity index 99%
rename from src/mca/schizo/psched/schizo_psched.c
rename to src/tools/psched/schizo.c
index 2bc726b674..d4dabbfabf 100644
--- a/src/mca/schizo/psched/schizo_psched.c
+++ b/src/tools/psched/schizo.c
@@ -56,8 +56,8 @@
 #include "src/runtime/prte_globals.h"
 #include "src/runtime/pmix_init_util.h"
 
-#include "schizo_psched.h"
 #include "src/mca/schizo/base/base.h"
+#include "src/tools/psched/psched.h"
 
 static int parse_cli(char **argv, pmix_cli_result_t *results, bool silent);
 static int detect_proxy(char *argv);
@@ -68,7 +68,7 @@ static void job_info(pmix_cli_result_t *results,
 static int set_default_rto(prte_job_t *jdata,
                            prte_rmaps_options_t *options);
 
-prte_schizo_base_module_t prte_schizo_prte_module = {
+prte_schizo_base_module_t psched_schizo_module = {
     .name = "psched",
     .parse_cli = parse_cli,
     .parse_env = parse_env,
diff --git a/src/tools/psched/server.c b/src/tools/psched/server.c
new file mode 100644
index 0000000000..81121c9e17
--- /dev/null
+++ b/src/tools/psched/server.c
@@ -0,0 +1,532 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2009-2020 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2017 Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2014-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2021-2023 Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2023      Triad National Security, LLC. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include "prte_config.h"
+#include "types.h"
+
+#ifdef HAVE_UNISTD_H
+#    include <unistd.h>
+#endif
+#ifdef HAVE_SYS_TYPES_H
+#    include <sys/types.h>
+#endif
+#include <fcntl.h>
+#ifdef HAVE_NETINET_IN_H
+#    include <netinet/in.h>
+#endif
+#ifdef HAVE_ARPA_INET_H
+#    include <arpa/inet.h>
+#endif
+#ifdef HAVE_NETDB_H
+#    include <netdb.h>
+#endif
+#include <ctype.h>
+
+#include "prte_stdint.h"
+#include "src/class/pmix_hotel.h"
+#include "src/class/pmix_list.h"
+#include "src/mca/base/pmix_mca_base_var.h"
+#include "src/pmix/pmix-internal.h"
+#include "src/util/pmix_argv.h"
+#include "src/util/error.h"
+#include "src/util/pmix_os_dirpath.h"
+#include "src/util/pmix_os_path.h"
+#include "src/util/pmix_output.h"
+#include "src/util/pmix_printf.h"
+#include "src/util/pmix_environ.h"
+#include "src/util/pmix_show_help.h"
+
+#include "src/mca/errmgr/errmgr.h"
+#include "src/mca/grpcomm/grpcomm.h"
+#include "src/prted/pmix/pmix_server_internal.h"
+#include "src/rml/rml_contact.h"
+#include "src/rml/rml.h"
+#include "src/runtime/prte_data_server.h"
+#include "src/runtime/prte_globals.h"
+#include "src/threads/pmix_threads.h"
+#include "src/util/name_fns.h"
+#include "src/util/proc_info.h"
+#include "src/util/session_dir.h"
+#include "src/util/pmix_show_help.h"
+
+#include "src/tools/psched/psched.h"
+
+static pmix_topology_t mytopology = {0};
+
+static pmix_server_module_t psched_server = {
+    .register_events = psched_register_events_fn,
+    .deregister_events = psched_deregister_events_fn,
+    .notify_event = psched_notify_event,
+    .query = psched_query_fn,
+    .tool_connected = psched_tool_connected_fn,
+    .log = psched_log_fn,
+    .job_control = psched_job_ctrl_fn,
+    .group = psched_group_fn,
+    .allocate = psched_alloc_fn,
+#if PMIX_NUMERIC_VERSION >= 0x00050000
+    .session_control = psched_session_ctrl_fn
+#endif
+};
+
+psched_globals_t psched_globals = {
+    .initialized = false,
+    .requests = PMIX_POINTER_ARRAY_STATIC_INIT,
+    .tools = PMIX_LIST_STATIC_INIT,
+    .syscontroller = PMIX_PROC_STATIC_INIT,
+    .controller_connected = false,
+    .verbosity = -1,
+    .output = -1
+};
+
+typedef struct {
+    char *function;
+    char **attrs;
+} prte_regattr_input_t;
+
+static prte_regattr_input_t prte_attributes[] = {
+    {.function = "PMIx_Query_info",
+     .attrs = (char *[]){"PMIX_QUERY_NAMESPACES",
+                         "PMIX_QUERY_NAMESPACE_INFO",
+                         "PMIX_QUERY_SPAWN_SUPPORT",
+                         "PMIX_QUERY_DEBUG_SUPPORT",
+                         "PMIX_HWLOC_XML_V1",
+                         "PMIX_HWLOC_XML_V2",
+                         "PMIX_PROC_URI",
+                         "PMIX_QUERY_PROC_TABLE",
+                         "PMIX_QUERY_LOCAL_PROC_TABLE",
+                         "PMIX_QUERY_NUM_PSETS",
+                         "PMIX_QUERY_PSET_NAMES",
+                         "PMIX_JOB_SIZE",
+                         "PMIX_QUERY_NUM_GROUPS",
+                         "PMIX_QUERY_GROUP_NAMES",
+                         "PMIX_QUERY_GROUP_MEMBERSHIP",
+                         NULL}},
+    {.function = "PMIx_Query_info_nb",
+     .attrs = (char *[]){"PMIX_QUERY_NAMESPACES",
+                         "PMIX_QUERY_NAMESPACE_INFO",
+                         "PMIX_QUERY_SPAWN_SUPPORT",
+                         "PMIX_QUERY_DEBUG_SUPPORT",
+                         "PMIX_HWLOC_XML_V1",
+                         "PMIX_HWLOC_XML_V2",
+                         "PMIX_PROC_URI",
+                         "PMIX_QUERY_PROC_TABLE",
+                         "PMIX_QUERY_LOCAL_PROC_TABLE",
+                         "PMIX_QUERY_NUM_PSETS",
+                         "PMIX_QUERY_PSET_NAMES",
+                         "PMIX_JOB_SIZE",
+                         "PMIX_QUERY_NUM_GROUPS",
+                         "PMIX_QUERY_GROUP_NAMES",
+                         "PMIX_QUERY_GROUP_MEMBERSHIP",
+                         "PMIX_QUERY_ALLOCATION",
+                         "PMIX_QUERY_ALLOC_STATUS",
+                         NULL}},
+    {.function = "PMIx_Log", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Log_nb", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Job_control",
+     .attrs = (char *[]){"PMIX_JOB_CTRL_KILL",
+                         "PMIX_JOB_CTRL_TERMINATE",
+                         "PMIX_JOB_CTRL_SIGNAL",
+                         NULL}},
+    {.function = "PMIx_Job_control_nb",
+     .attrs = (char *[]){"PMIX_JOB_CTRL_KILL",
+                         "PMIX_JOB_CTRL_TERMINATE",
+                         "PMIX_JOB_CTRL_SIGNAL",
+                         NULL}},
+    {.function = "PMIx_Group_construct",
+     .attrs = (char *[]){"PMIX_GROUP_ASSIGN_CONTEXT_ID",
+                         "PMIX_EMBED_BARRIER",
+                         "PMIX_GROUP_ENDPT_DATA",
+                         NULL}},
+    {.function = "PMIx_Group_construct_nb",
+     .attrs = (char *[]){"PMIX_GROUP_ASSIGN_CONTEXT_ID",
+                         "PMIX_EMBED_BARRIER",
+                         "PMIX_GROUP_ENDPT_DATA",
+                         NULL}},
+    {.function = "PMIx_Group_invite", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Group_invite_nb", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Group_join", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Group_join_nb", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Group_leave", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Group_leave_nb", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Group_destruct", .attrs = (char *[]){"PMIX_EMBED_BARRIER", NULL}},
+    {.function = "PMIx_Group_destruct_nb", .attrs = (char *[]){"PMIX_EMBED_BARRIER", NULL}},
+    {.function = "PMIx_Register_event_handler", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Deregister_event_handler", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_Notify_event", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Allocate_resources",
+     .attrs = (char *[]){"PMIX_ALLOC_REQ_ID",
+                         "PMIX_ALLOC_NUM_NODES",
+                         "PMIX_ALLOC_NODE_LIST",
+                         "PMIX_ALLOC_NUM_CPUS",
+                         "PMIX_ALLOC_NUM_CPU_LIST",
+                         "PMIX_ALLOC_CPU_LIST",
+                         "PMIX_ALLOC_MEM_SIZE",
+                         "PMIX_ALLOC_TIME",
+                         "PMIX_ALLOC_QUEUE",
+                         "PMIX_ALLOC_PREEMPTIBLE",
+                         NULL}},
+#if PMIX_NUMERIC_VERSION >= 0x00050000
+    {.function = "PMIx_Session_control",
+     .attrs = (char *[]){"PMIX_SESSION_CTRL_ID",
+                         "PMIX_SESSION_APP",
+                         "PMIX_SESSION_PAUSE",
+                         "PMIX_SESSION_RESUME",
+                         "PMIX_SESSION_TERMINATE",
+                         "PMIX_SESSION_PREEMPT",
+                         "PMIX_SESSION_RESTORE",
+                         "PMIX_SESSION_SIGNAL",
+                         "PMIX_SESSION_COMPLETE",
+                         NULL}},
+#endif
+    {.function = ""},
+};
+
+static char *generate_dist = "fabric,gpu,network";
+void psched_register_params(void)
+{
+    /* register a verbosity */
+    (void) pmix_mca_base_var_register("prte", "psched", NULL, "verbose",
+                                      "Debug verbosity for PRRTE Scheduler",
+                                      PMIX_MCA_BASE_VAR_TYPE_INT,
+                                      &psched_globals.verbosity);
+    if (0 <= psched_globals.verbosity) {
+        psched_globals.output = pmix_output_open(NULL);
+        pmix_output_set_verbosity(psched_globals.output,
+                                  psched_globals.verbosity);
+    }
+
+}
+
+/* provide a callback function for lost connections to allow us
+ * to cleanup after any tools once they depart */
+static void lost_connection_hdlr(size_t evhdlr_registration_id, pmix_status_t status,
+                                 const pmix_proc_t *source, pmix_info_t info[], size_t ninfo,
+                                 pmix_info_t *results, size_t nresults,
+                                 pmix_event_notification_cbfunc_fn_t cbfunc, void *cbdata)
+{
+    prte_pmix_tool_t *tl;
+    PRTE_HIDE_UNUSED_PARAMS(evhdlr_registration_id, status,
+                            info, ninfo, results, nresults);
+
+    /* scan the list of attached tools to see if this one is there */
+    PMIX_LIST_FOREACH(tl, &psched_globals.tools, prte_pmix_tool_t)
+    {
+        if (PMIX_CHECK_PROCID(&tl->name, source)) {
+            /* remove the session directory we created for it */
+            if (NULL != tl->nsdir) {
+                pmix_os_dirpath_destroy(tl->nsdir, true, NULL);
+            }
+            /* take this tool off the list */
+            pmix_list_remove_item(&psched_globals.tools, &tl->super);
+            /* release it */
+            PMIX_RELEASE(tl);
+            break;
+        }
+    }
+
+    /* we _always_ have to execute the evhandler callback or
+     * else the event progress engine will hang */
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata);
+    }
+}
+
+static void regcbfunc(pmix_status_t status, size_t ref, void *cbdata)
+{
+    prte_pmix_lock_t *lock = (prte_pmix_lock_t *) cbdata;
+    PRTE_HIDE_UNUSED_PARAMS(ref);
+
+    PMIX_ACQUIRE_OBJECT(lock);
+    lock->status = status;
+    PRTE_PMIX_WAKEUP_THREAD(lock);
+}
+
+/*
+ * Initialize global variables used w/in the server.
+ */
+int psched_server_init(void)
+{
+    int rc;
+    void *ilist;
+    pmix_data_array_t darray;
+    pmix_info_t *info, myinf;
+    size_t n, ninfo;
+    char *tmp;
+    pmix_status_t prc;
+    prte_pmix_lock_t lock;
+    bool flag;
+    pmix_proc_t myproc;
+
+    if (psched_globals.initialized) {
+        return PRTE_SUCCESS;
+    }
+    psched_globals.initialized = true;
+
+    /* setup the server's global variables */
+    PMIX_CONSTRUCT(&psched_globals.requests, pmix_pointer_array_t);
+    pmix_pointer_array_init(&psched_globals.requests, 128, INT_MAX, 2);
+    PMIX_CONSTRUCT(&psched_globals.tools, pmix_list_t);
+    psched_globals.syscontroller = *PRTE_NAME_INVALID;
+
+    PMIX_INFO_LIST_START(ilist);
+
+#ifdef PMIX_SERVER_SCHEDULER
+    /* tell the server that we are the scheduler */
+    PMIX_INFO_LIST_ADD(prc, ilist, PMIX_SERVER_SCHEDULER,
+                       NULL, PMIX_BOOL);
+    if (PMIX_SUCCESS != prc) {
+        PMIX_INFO_LIST_RELEASE(ilist);
+        rc = prte_pmix_convert_status(prc);
+        return rc;
+    }
+#endif
+
+    /* tell the server our name so we agree on our identifier */
+    PMIX_INFO_LIST_ADD(prc, ilist, PMIX_TOOL_NSPACE,
+                       prte_process_info.myproc.nspace, PMIX_STRING);
+    if (PMIX_SUCCESS != prc) {
+        PMIX_INFO_LIST_RELEASE(ilist);
+        rc = prte_pmix_convert_status(prc);
+        return rc;
+    }
+    PMIX_INFO_LIST_ADD(prc, ilist, PMIX_TOOL_RANK,
+                       &prte_process_info.myproc.rank, PMIX_PROC_RANK);
+    if (PMIX_SUCCESS != prc) {
+        PMIX_INFO_LIST_RELEASE(ilist);
+        rc = prte_pmix_convert_status(prc);
+        return rc;
+    }
+
+    /* tell the server our hostname so we agree on it */
+    PMIX_INFO_LIST_ADD(prc, ilist, PMIX_HOSTNAME,
+                       prte_process_info.nodename, PMIX_STRING);
+    if (PMIX_SUCCESS != prc) {
+        PMIX_INFO_LIST_RELEASE(ilist);
+        rc = prte_pmix_convert_status(prc);
+        return rc;
+    }
+
+    // check for aliases
+    if (NULL != prte_process_info.aliases) {
+        tmp = PMIX_ARGV_JOIN_COMPAT(prte_process_info.aliases, ',');
+        PMIX_INFO_LIST_ADD(prc, ilist, PMIX_HOSTNAME_ALIASES,
+                           tmp, PMIX_STRING);
+        free(tmp);
+        if (PMIX_SUCCESS != prc) {
+            PMIX_INFO_LIST_RELEASE(ilist);
+            rc = prte_pmix_convert_status(prc);
+            return rc;
+        }
+    }
+
+    /* tell the server what we are doing with FQDN */
+    PMIX_INFO_LIST_ADD(prc, ilist, PMIX_HOSTNAME_KEEP_FQDN,
+                       &prte_keep_fqdn_hostnames, PMIX_BOOL);
+    if (PMIX_SUCCESS != prc) {
+        PMIX_INFO_LIST_RELEASE(ilist);
+        rc = prte_pmix_convert_status(prc);
+        return rc;
+    }
+
+#ifdef PMIX_EXTERNAL_AUX_EVENT_BASE
+    /* give the server our event base to use for signal trapping */
+    PMIX_INFO_LIST_ADD(prc, ilist, PMIX_EXTERNAL_AUX_EVENT_BASE,
+                       prte_event_base, PMIX_POINTER);
+    if (PMIX_SUCCESS != prc) {
+        PMIX_INFO_LIST_RELEASE(ilist);
+        rc = prte_pmix_convert_status(prc);
+        return rc;
+    }
+#endif
+
+    // pass our topology object down to the server library for its use
+    mytopology.source = "hwloc";
+    mytopology.topology = prte_hwloc_topology;
+    PMIX_INFO_CONSTRUCT(&myinf);
+    PMIX_LOAD_KEY(myinf.key, PMIX_TOPOLOGY2);
+    myinf.value.type = PMIX_TOPO;
+    myinf.value.data.topo = &mytopology;
+    PMIX_INFO_LIST_INSERT(prc, ilist, &myinf);
+    if (PMIX_SUCCESS != prc) {
+        PMIX_INFO_LIST_RELEASE(ilist);
+        rc = prte_pmix_convert_status(prc);
+        return rc;
+    }
+
+    /* tell the server our temp directory */
+    PMIX_INFO_LIST_ADD(prc, ilist, PMIX_SERVER_TMPDIR,
+                       prte_process_info.jobfam_session_dir,
+                       PMIX_STRING);
+    if (PMIX_SUCCESS != prc) {
+        PMIX_INFO_LIST_RELEASE(ilist);
+        rc = prte_pmix_convert_status(prc);
+        return rc;
+    }
+
+    /* if requested, tell the server library to output our PMIx URI */
+    if (NULL != prte_pmix_server_globals.report_uri) {
+        PMIX_INFO_LIST_ADD(prc, ilist, PMIX_TCP_REPORT_URI,
+                           prte_pmix_server_globals.report_uri, PMIX_STRING);
+        if (PMIX_SUCCESS != prc) {
+            PMIX_INFO_LIST_RELEASE(ilist);
+            rc = prte_pmix_convert_status(prc);
+            return rc;
+        }
+    }
+
+    if (NULL != prte_progress_thread_cpus) {
+        PMIX_INFO_LIST_ADD(prc, ilist, PMIX_BIND_PROGRESS_THREAD,
+                           prte_progress_thread_cpus, PMIX_STRING);
+        PMIX_INFO_LIST_ADD(prc, ilist, PMIX_BIND_REQUIRED,
+                           &prte_bind_progress_thread_reqd, PMIX_BOOL);
+    }
+
+    /* schedulers always allow remote tool connections */
+    PMIX_INFO_LIST_ADD(prc, ilist, PMIX_SERVER_REMOTE_CONNECTIONS,
+                       NULL, PMIX_BOOL);
+    if (PMIX_SUCCESS != prc) {
+        PMIX_INFO_LIST_RELEASE(ilist);
+        rc = prte_pmix_convert_status(prc);
+        return rc;
+    }
+
+    /* if the root DVM is present, connect to it */
+    PMIX_INFO_LIST_ADD(prc, ilist, PMIX_CONNECT_TO_SYSTEM,
+                       NULL, PMIX_BOOL);
+    if (PMIX_SUCCESS != prc) {
+        PMIX_INFO_LIST_RELEASE(ilist);
+        rc = prte_pmix_convert_status(prc);
+        return rc;
+    }
+
+    /* don't require the connection else we will abort
+     * if the root DVM isn't found */
+    PMIX_INFO_LIST_ADD(prc, ilist, PMIX_TOOL_CONNECT_OPTIONAL,
+                       NULL, PMIX_BOOL);
+    if (PMIX_SUCCESS != prc) {
+        PMIX_INFO_LIST_RELEASE(ilist);
+        rc = prte_pmix_convert_status(prc);
+        return rc;
+    }
+
+
+    /* convert to an info array */
+    PMIX_INFO_LIST_CONVERT(prc, ilist, &darray);
+    if (PMIX_SUCCESS != prc) {
+        PMIX_INFO_LIST_RELEASE(ilist);
+        rc = prte_pmix_convert_status(prc);
+        return rc;
+    }
+    PMIX_INFO_LIST_RELEASE(ilist);
+    info = (pmix_info_t*)darray.array;
+    ninfo = darray.size;
+
+    /* initialize as a tool */
+    prc = PMIx_tool_init(&myproc, info, ninfo);
+    if (PMIX_SUCCESS != prc) {
+        /* pmix will provide a nice show_help output here */
+        PMIX_INFO_FREE(info, ninfo);
+        return prte_pmix_convert_status(prc);
+    }
+    PMIX_INFO_FREE(info, ninfo);
+    rc = PRTE_SUCCESS;
+
+    /* register our attributes */
+    for (n = 0; 0 != strlen(prte_attributes[n].function); n++) {
+        prc = PMIx_Register_attributes(prte_attributes[n].function, prte_attributes[n].attrs);
+        if (PMIX_SUCCESS != prc) {
+            return prte_pmix_convert_status(prc);
+        }
+    }
+
+    /* register the "lost-connection" event handler */
+    PRTE_PMIX_CONSTRUCT_LOCK(&lock);
+    prc = PMIX_ERR_LOST_CONNECTION;
+    PMIx_Register_event_handler(&prc, 1, NULL, 0, lost_connection_hdlr, regcbfunc, &lock);
+    PRTE_PMIX_WAIT_THREAD(&lock);
+    rc = prte_pmix_convert_status(lock.status);
+    PRTE_PMIX_DESTRUCT_LOCK(&lock);
+    if (PRTE_SUCCESS != rc) {
+        return rc;
+    }
+
+    /* register our server function module */
+    prc = PMIx_tool_set_server_module(&psched_server);
+    if (PMIX_SUCCESS != prc) {
+        rc = prte_pmix_convert_status(prc);
+        return rc;
+    }
+
+    return PRTE_SUCCESS;
+}
+
+void psched_server_finalize(void)
+{
+    pmix_status_t prc;
+
+    if (!psched_globals.initialized) {
+        return;
+    }
+
+    pmix_output_verbose(2, psched_globals.output,
+                        "%s Finalizing PMIX server",
+                        PRTE_NAME_PRINT(PRTE_PROC_MY_NAME));
+
+    prc = PMIx_tool_finalize();
+    if (PMIX_SUCCESS != prc) {
+        PMIX_ERROR_LOG(prc);
+    }
+
+#if 0
+    /* cleanup collectives */
+    pmix_server_req_t *cd;
+    for (int i = 0; i < prte_pmix_server_globals.local_reqs.size; i++) {
+      cd = (pmix_server_req_t*)pmix_pointer_array_get_item(&prte_pmix_server_globals.local_reqs, i);
+      if (NULL != cd) {
+          PMIX_RELEASE(cd);
+      }
+    }
+    for (int i = 0; i < prte_pmix_server_globals.remote_reqs.size; i++) {
+      cd = (pmix_server_req_t*)pmix_pointer_array_get_item(&prte_pmix_server_globals.remote_reqs, i);
+      if (NULL != cd) {
+          PMIX_RELEASE(cd);
+      }
+    }
+
+    PMIX_DESTRUCT(&prte_pmix_server_globals.remote_reqs);
+    PMIX_DESTRUCT(&prte_pmix_server_globals.local_reqs);
+    PMIX_LIST_DESTRUCT(&prte_pmix_server_globals.notifications);
+    PMIX_LIST_DESTRUCT(&prte_pmix_server_globals.psets);
+    PMIX_LIST_DESTRUCT(&prte_pmix_server_globals.groups);
+
+    /* shutdown the local server */
+    psched_globals.initialized = false;
+#endif
+}
diff --git a/src/tools/psched/session.c b/src/tools/psched/session.c
new file mode 100644
index 0000000000..f45946139e
--- /dev/null
+++ b/src/tools/psched/session.c
@@ -0,0 +1,248 @@
+/*
+ * Copyright (c) 2022-2023 Nanook Consulting.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "prte_config.h"
+
+#include "src/pmix/pmix-internal.h"
+#include "src/prted/pmix/pmix_server_internal.h"
+#include "src/rml/rml.h"
+
+#include "src/tools/psched/psched.h"
+
+static void localrelease(void *cbdata)
+{
+    pmix_server_req_t *req = (pmix_server_req_t*)cbdata;
+    pmix_pointer_array_set_item(&prte_pmix_server_globals.local_reqs, req->local_index, NULL);
+    PMIX_RELEASE(req);
+}
+
+static void infocbfunc(pmix_status_t status,
+                       pmix_info_t *info, size_t ninfo,
+                       void *cbdata,
+                       pmix_release_cbfunc_t rel, void *relcbdata)
+{
+    pmix_server_req_t *req = (pmix_server_req_t*)cbdata;
+
+    if (NULL != req->infocbfunc) {
+        req->infocbfunc(status, info, ninfo, req->cbdata, localrelease, req);
+        if (NULL != rel) {
+            rel(relcbdata);
+        }
+        return;
+    }
+    /* need to cleanup ourselves */
+    if (NULL != rel) {
+        rel(relcbdata);
+    }
+    pmix_pointer_array_set_item(&prte_pmix_server_globals.local_reqs, req->local_index, NULL);
+    PMIX_RELEASE(req);
+}
+
+static void pass_request(int sd, short args, void *cbdata)
+{
+    prte_pmix_server_op_caddy_t *cd = (prte_pmix_server_op_caddy_t*)cbdata;
+    pmix_server_req_t *req;
+    pmix_data_buffer_t *buf;
+    uint8_t command;
+    pmix_status_t rc;
+
+    /* create a request tracker for this operation */
+    req = PMIX_NEW(pmix_server_req_t);
+    if (0 < cd->allocdir) {
+        pmix_asprintf(&req->operation, "ALLOCATE: %u", cd->allocdir);
+        command = 0;
+    } else {
+        pmix_asprintf(&req->operation, "SESSIONCTRL: %u", cd->sessionID);
+        command = 1;
+    }
+    req->infocbfunc = cd->infocbfunc;
+    req->cbdata = cd->cbdata;
+    /* add this request to our local request tracker array */
+    req->local_index = pmix_pointer_array_add(&prte_pmix_server_globals.local_reqs, req);
+
+    /* if we are the DVM master, then handle this ourselves */
+    if (PRTE_PROC_IS_MASTER) {
+        if (!prte_pmix_server_globals.scheduler_connected) {
+            /* the scheduler has not attached to us - there is
+             * nothing we can do */
+            rc = PMIX_ERR_NOT_SUPPORTED;
+            goto callback;
+        }
+
+        /* if we have not yet set the scheduler as our server, do so */
+        if (!prte_pmix_server_globals.scheduler_set_as_server) {
+            rc = PMIx_tool_set_server(&prte_pmix_server_globals.scheduler, NULL, 0);
+            if (PMIX_SUCCESS != rc) {
+                goto callback;
+            }
+            prte_pmix_server_globals.scheduler_set_as_server = true;
+        }
+
+        if (0 == command) {
+            rc = PMIx_Allocation_request_nb(cd->allocdir, cd->info, cd->ninfo,
+                                            infocbfunc, req);
+        } else {
+#if PMIX_NUMERIC_VERSION < 0x00050000
+        rc = PMIX_ERR_NOT_SUPPORTED;
+#else
+        rc = PMIx_Session_control(cd->sessionID, cd->info, cd->ninfo,
+                                  infocbfunc, req);
+#endif
+        }
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            goto callback;
+        }
+        return;
+    }
+
+    PMIX_DATA_BUFFER_CREATE(buf);
+
+    /* construct a request message for the command */
+    rc = PMIx_Data_pack(NULL, buf, &command, 1, PMIX_UINT8);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_DATA_BUFFER_RELEASE(buf);
+        pmix_pointer_array_set_item(&prte_pmix_server_globals.local_reqs, req->local_index, NULL);
+        goto callback;
+    }
+
+    /* pack the local requestor ID */
+    rc = PMIx_Data_pack(NULL, buf, &req->local_index, 1, PMIX_INT);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_DATA_BUFFER_RELEASE(buf);
+        pmix_pointer_array_set_item(&prte_pmix_server_globals.local_reqs, req->local_index, NULL);
+        goto callback;
+    }
+
+    /* pack the requestor */
+    rc = PMIx_Data_pack(NULL, buf, &cd->proc, 1, PMIX_PROC);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_DATA_BUFFER_RELEASE(buf);
+        pmix_pointer_array_set_item(&prte_pmix_server_globals.local_reqs, req->local_index, NULL);
+        goto callback;
+    }
+
+    if (0 == command) {
+        /* pack the allocation directive */
+        rc = PMIx_Data_pack(NULL, buf, &cd->allocdir, 1, PMIX_ALLOC_DIRECTIVE);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_DATA_BUFFER_RELEASE(buf);
+            pmix_pointer_array_set_item(&prte_pmix_server_globals.local_reqs, req->local_index, NULL);
+            goto callback;
+        }
+    } else {
+        /* pack the sessionID */
+        rc = PMIx_Data_pack(NULL, buf, &cd->sessionID, 1, PMIX_UINT32);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_DATA_BUFFER_RELEASE(buf);
+            pmix_pointer_array_set_item(&prte_pmix_server_globals.local_reqs, req->local_index, NULL);
+            goto callback;
+        }
+    }
+
+    /* pack the number of info */
+    rc = PMIx_Data_pack(NULL, buf, &cd->ninfo, 1, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_DATA_BUFFER_RELEASE(buf);
+        pmix_pointer_array_set_item(&prte_pmix_server_globals.local_reqs, req->local_index, NULL);
+        goto callback;
+    }
+    if (0 < cd->ninfo) {
+        /* pack the info */
+        rc = PMIx_Data_pack(NULL, buf, cd->info, cd->ninfo, PMIX_INFO);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_DATA_BUFFER_RELEASE(buf);
+            pmix_pointer_array_set_item(&prte_pmix_server_globals.local_reqs, req->local_index, NULL);
+            goto callback;
+        }
+    }
+
+    /* send this request to the DVM controller - might be us */
+    PRTE_RML_SEND(rc, PRTE_PROC_MY_HNP->rank, buf, PRTE_RML_TAG_SCHED);
+    if (PRTE_SUCCESS != rc) {
+        PRTE_ERROR_LOG(rc);
+        pmix_pointer_array_set_item(&prte_pmix_server_globals.local_reqs, req->local_index, NULL);
+        PMIX_DATA_BUFFER_RELEASE(buf);
+        goto callback;
+    }
+    PMIX_RELEASE(cd);
+    return;
+
+callback:
+    PMIX_RELEASE(cd);
+    /* this section gets executed solely upon an error */
+    if (NULL != req->infocbfunc) {
+        req->infocbfunc(rc, req->info, req->ninfo, req->cbdata, localrelease, req);
+        return;
+    }
+    PMIX_RELEASE(req);
+}
+
+pmix_status_t psched_alloc_fn(const pmix_proc_t *client,
+                              pmix_alloc_directive_t directive,
+                              const pmix_info_t data[], size_t ndata,
+                              pmix_info_cbfunc_t cbfunc, void *cbdata)
+{
+    prte_pmix_server_op_caddy_t *cd;
+
+
+    pmix_output_verbose(2, prte_pmix_server_globals.output,
+                        "%s allocate upcalled on behalf of proc %s:%u with %" PRIsize_t " infos",
+                        PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), client->nspace, client->rank, ndata);
+
+    cd = PMIX_NEW(prte_pmix_server_op_caddy_t);
+    PMIX_LOAD_PROCID(&cd->proc, client->nspace, client->rank);
+    cd->allocdir = directive;
+    cd->info = (pmix_info_t *) data;
+    cd->ninfo = ndata;
+    cd->infocbfunc = cbfunc;
+    cd->cbdata = cbdata;
+    prte_event_set(prte_event_base, &cd->ev, -1, PRTE_EV_WRITE, pass_request, cd);
+    prte_event_set_priority(&cd->ev, PRTE_MSG_PRI);
+    PMIX_POST_OBJECT(cd);
+    prte_event_active(&cd->ev, PRTE_EV_WRITE, 1);
+    return PRTE_SUCCESS;
+}
+
+#if PMIX_NUMERIC_VERSION >= 0x00050000
+
+pmix_status_t psched_session_ctrl_fn(const pmix_proc_t *requestor,
+                                     uint32_t sessionID,
+                                     const pmix_info_t directives[], size_t ndirs,
+                                     pmix_info_cbfunc_t cbfunc, void *cbdata)
+{
+    prte_pmix_server_op_caddy_t *cd;
+
+
+    pmix_output_verbose(2, prte_pmix_server_globals.output,
+                        "%s session ctrl upcalled on behalf of proc %s:%u with %" PRIsize_t " directives",
+                        PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), requestor->nspace, requestor->rank, ndirs);
+
+    cd = PMIX_NEW(prte_pmix_server_op_caddy_t);
+    PMIX_LOAD_PROCID(&cd->proc, requestor->nspace, requestor->rank);
+    cd->sessionID = sessionID;
+    cd->info = (pmix_info_t *) directives;
+    cd->ninfo = ndirs;
+    cd->infocbfunc = cbfunc;
+    cd->cbdata = cbdata;
+    prte_event_set(prte_event_base, &cd->ev, -1, PRTE_EV_WRITE, pass_request, cd);
+    prte_event_set_priority(&cd->ev, PRTE_MSG_PRI);
+    PMIX_POST_OBJECT(cd);
+    prte_event_active(&cd->ev, PRTE_EV_WRITE, 1);
+    return PRTE_SUCCESS;
+}
+
+#endif
diff --git a/src/tools/psched/state.c b/src/tools/psched/state.c
new file mode 100644
index 0000000000..fd7f5a5298
--- /dev/null
+++ b/src/tools/psched/state.c
@@ -0,0 +1,1074 @@
+/*
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2018-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2020      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2021-2023 Nanook Consulting.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "prte_config.h"
+
+#include <sys/types.h>
+#ifdef HAVE_UNISTD_H
+#    include <unistd.h>
+#endif /* HAVE_UNISTD_H */
+#include <string.h>
+
+#include "src/pmix/pmix-internal.h"
+#include "src/prted/pmix/pmix_server.h"
+#include "src/prted/pmix/pmix_server_internal.h"
+#include "src/util/pmix_argv.h"
+#include "src/util/nidmap.h"
+#include "src/util/pmix_os_dirpath.h"
+#include "src/util/pmix_output.h"
+#include "src/util/proc_info.h"
+
+#include "src/mca/errmgr/errmgr.h"
+#include "src/mca/filem/filem.h"
+#include "src/mca/grpcomm/grpcomm.h"
+#include "src/mca/iof/base/base.h"
+#include "src/mca/odls/odls_types.h"
+#include "src/mca/plm/base/base.h"
+#include "src/mca/ras/base/base.h"
+#include "src/mca/rmaps/base/base.h"
+#include "src/rml/rml.h"
+#include "src/runtime/prte_data_server.h"
+#include "src/runtime/prte_quit.h"
+#include "src/runtime/prte_wait.h"
+#include "src/threads/pmix_threads.h"
+
+#include "src/mca/state/base/base.h"
+#include "psched.h"
+
+/*
+ * Module functions: Global
+ */
+static int init(void);
+static int finalize(void);
+
+/* local functions */
+static void init_complete(int fd, short args, void *cbdata);
+static void vm_ready(int fd, short args, void *cbata);
+static void check_complete(int fd, short args, void *cbdata);
+static void cleanup_job(int fd, short args, void *cbdata);
+static void job_started(int fd, short args, void *cbata);
+static void ready_for_debug(int fd, short args, void *cbata);
+static void dvm_notify(int sd, short args, void *cbdata);
+
+/* defined default state machine sequence - individual
+ * plm's must add a state for launching daemons
+ */
+static prte_job_state_t launch_states[] = {
+    PRTE_JOB_STATE_INIT,
+    PRTE_JOB_STATE_INIT_COMPLETE,
+    PRTE_JOB_STATE_ALLOCATE,
+    PRTE_JOB_STATE_ALLOCATION_COMPLETE,
+    PRTE_JOB_STATE_DAEMONS_LAUNCHED,
+    PRTE_JOB_STATE_DAEMONS_REPORTED,
+    PRTE_JOB_STATE_VM_READY,
+    PRTE_JOB_STATE_MAP,
+    PRTE_JOB_STATE_MAP_COMPLETE,
+    PRTE_JOB_STATE_SYSTEM_PREP,
+    PRTE_JOB_STATE_LAUNCH_APPS,
+    PRTE_JOB_STATE_SEND_LAUNCH_MSG,
+    PRTE_JOB_STATE_STARTED,
+    PRTE_JOB_STATE_LOCAL_LAUNCH_COMPLETE,
+    PRTE_JOB_STATE_READY_FOR_DEBUG,
+    PRTE_JOB_STATE_RUNNING,
+    PRTE_JOB_STATE_REGISTERED,
+    /* termination states */
+    PRTE_JOB_STATE_TERMINATED,
+    PRTE_JOB_STATE_NOTIFY_COMPLETED,
+    PRTE_JOB_STATE_NOTIFIED,
+    PRTE_JOB_STATE_ALL_JOBS_COMPLETE
+};
+
+static prte_state_cbfunc_t launch_callbacks[] = {
+    prte_plm_base_setup_job,
+    init_complete,
+    prte_ras_base_allocate,
+    prte_plm_base_allocation_complete,
+    prte_plm_base_daemons_launched,
+    prte_plm_base_daemons_reported,
+    vm_ready,
+    prte_rmaps_base_map_job,
+    prte_plm_base_mapping_complete,
+    prte_plm_base_complete_setup,
+    prte_plm_base_launch_apps,
+    prte_plm_base_send_launch_msg,
+    job_started,
+    prte_state_base_local_launch_complete,
+    ready_for_debug,
+    prte_plm_base_post_launch,
+    prte_plm_base_registered,
+    check_complete,
+    dvm_notify,
+    cleanup_job,
+    prte_quit
+};
+
+static prte_proc_state_t proc_states[] = {
+    PRTE_PROC_STATE_RUNNING,
+    PRTE_PROC_STATE_READY_FOR_DEBUG,
+    PRTE_PROC_STATE_REGISTERED,
+    PRTE_PROC_STATE_IOF_COMPLETE,
+    PRTE_PROC_STATE_WAITPID_FIRED,
+    PRTE_PROC_STATE_TERMINATED
+};
+
+static prte_state_cbfunc_t proc_callbacks[] = {
+    prte_state_base_track_procs,
+    prte_state_base_track_procs,
+    prte_state_base_track_procs,
+    prte_state_base_track_procs,
+    prte_state_base_track_procs,
+    prte_state_base_track_procs
+};
+
+static void force_quit(int fd, short args, void *cbdata)
+{
+    PRTE_HIDE_UNUSED_PARAMS(fd, args);
+    prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata;
+
+    /* give us a chance to stop the orteds */
+    prte_plm.terminate_orteds();
+    PMIX_RELEASE(caddy);
+}
+
+/************************
+ * Local variables
+ ************************/
+static bool terminate_dvm = false;
+static bool dvm_terminated = false;
+
+
+prte_state_base_module_t psched_state_module = {
+    .init = init,
+    .finalize = finalize,
+    .activate_job_state = prte_state_base_activate_job_state,
+    .add_job_state = prte_state_base_add_job_state,
+    .set_job_state_callback = prte_state_base_set_job_state_callback,
+    .set_job_state_priority = prte_state_base_set_job_state_priority,
+    .remove_job_state = prte_state_base_remove_job_state,
+    .activate_proc_state = prte_state_base_activate_proc_state,
+    .add_proc_state = prte_state_base_add_proc_state,
+    .set_proc_state_callback = prte_state_base_set_proc_state_callback,
+    .set_proc_state_priority = prte_state_base_set_proc_state_priority,
+    .remove_proc_state = prte_state_base_remove_proc_state
+};
+
+void psched_state_init(void)
+{
+    prte_state = psched_state_module;
+}
+
+/************************
+ * API Definitions
+ ************************/
+static int init(void)
+{
+    int i, rc;
+    int num_states;
+
+    /* setup the state machines */
+    PMIX_CONSTRUCT(&prte_job_states, pmix_list_t);
+    PMIX_CONSTRUCT(&prte_proc_states, pmix_list_t);
+
+    /* setup the job state machine */
+    num_states = sizeof(launch_states) / sizeof(prte_job_state_t);
+    for (i = 0; i < num_states; i++) {
+        if (PRTE_SUCCESS
+            != (rc = prte_state.add_job_state(launch_states[i], launch_callbacks[i],
+                                              PRTE_SYS_PRI))) {
+            PRTE_ERROR_LOG(rc);
+        }
+    }
+    /* add the termination response */
+    rc = prte_state.add_job_state(PRTE_JOB_STATE_DAEMONS_TERMINATED, prte_quit, PRTE_SYS_PRI);
+    if (PRTE_SUCCESS != rc) {
+        PRTE_ERROR_LOG(rc);
+    }
+    /* add a default error response */
+    rc = prte_state.add_job_state(PRTE_JOB_STATE_FORCED_EXIT, force_quit, PRTE_ERROR_PRI);
+    if (PRTE_SUCCESS != rc) {
+        PRTE_ERROR_LOG(rc);
+    }
+    /* add callback to report progress, if requested */
+    rc = prte_state.add_job_state(PRTE_JOB_STATE_REPORT_PROGRESS,
+                                  prte_state_base_report_progress, PRTE_ERROR_PRI);
+    if (PRTE_SUCCESS != rc) {
+        PRTE_ERROR_LOG(rc);
+    }
+    if (5 < pmix_output_get_verbosity(prte_state_base_framework.framework_output)) {
+        prte_state_base_print_job_state_machine();
+    }
+
+    /* populate the proc state machine to allow us to
+     * track proc lifecycle changes
+     */
+    num_states = sizeof(proc_states) / sizeof(prte_proc_state_t);
+    for (i = 0; i < num_states; i++) {
+        rc = prte_state.add_proc_state(proc_states[i], proc_callbacks[i], PRTE_SYS_PRI);
+        if (PRTE_SUCCESS != rc) {
+            PRTE_ERROR_LOG(rc);
+        }
+    }
+    if (5 < pmix_output_get_verbosity(prte_state_base_framework.framework_output)) {
+        prte_state_base_print_proc_state_machine();
+    }
+
+    return PRTE_SUCCESS;
+}
+
+static int finalize(void)
+{
+    /* cleanup the state machines */
+    PMIX_LIST_DESTRUCT(&prte_proc_states);
+    PMIX_LIST_DESTRUCT(&prte_job_states);
+
+    return PRTE_SUCCESS;
+}
+
+static void files_ready(int status, void *cbdata)
+{
+    prte_job_t *jdata = (prte_job_t *) cbdata;
+
+    if (PRTE_SUCCESS != status) {
+        PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_FILES_POSN_FAILED);
+    } else {
+        PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP);
+    }
+}
+
+static void init_complete(int sd, short args, void *cbdata)
+{
+    PRTE_HIDE_UNUSED_PARAMS(sd, args);
+    prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata;
+
+    PMIX_ACQUIRE_OBJECT(caddy);
+
+    /* need to go thru allocate step in case someone wants to
+     * expand the DVM */
+    PRTE_ACTIVATE_JOB_STATE(caddy->jdata, PRTE_JOB_STATE_ALLOCATE);
+    PMIX_RELEASE(caddy);
+}
+
+static void vm_ready(int fd, short args, void *cbdata)
+{
+    prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata;
+    int rc, i;
+    pmix_data_buffer_t buf;
+    prte_grpcomm_signature_t sig;
+    prte_job_t *jptr;
+    prte_proc_t *dmn;
+    int32_t v;
+    pmix_value_t *val;
+    pmix_status_t ret;
+    PRTE_HIDE_UNUSED_PARAMS(fd, args, cbdata);
+
+    PMIX_ACQUIRE_OBJECT(caddy);
+    /* if this is my job, then we are done */
+    if (prte_get_attribute(&caddy->jdata->attributes, PRTE_JOB_LAUNCHED_DAEMONS, NULL, PMIX_BOOL)) {
+        /* if there is more than one daemon in the job, then there
+         * is just a little bit to do */
+        if (!prte_get_attribute(&caddy->jdata->attributes, PRTE_JOB_DO_NOT_LAUNCH, NULL, PMIX_BOOL)
+            && 1 < prte_process_info.num_daemons) {
+            /* send the daemon map to every daemon in this DVM - we
+             * do this here so we don't have to do it for every
+             * job we are going to launch */
+            PMIX_DATA_BUFFER_CONSTRUCT(&buf);
+            rc = prte_util_nidmap_create(prte_node_pool, &buf);
+            if (PRTE_SUCCESS != rc) {
+                PRTE_ERROR_LOG(rc);
+                PMIX_DATA_BUFFER_DESTRUCT(&buf);
+                PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT);
+                return;
+            }
+            /* get wireup info for daemons */
+            jptr = prte_get_job_data_object(PRTE_PROC_MY_NAME->nspace);
+            for (v = 0; v < jptr->procs->size; v++) {
+                if (NULL == (dmn = (prte_proc_t *) pmix_pointer_array_get_item(jptr->procs, v))) {
+                    continue;
+                }
+                val = NULL;
+                if (PMIX_SUCCESS != (ret = PMIx_Get(&dmn->name, PMIX_PROC_URI, NULL, 0, &val)) ||
+                    NULL == val) {
+                    PMIX_ERROR_LOG(ret);
+                    PMIX_DATA_BUFFER_DESTRUCT(&buf);
+                    PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT);
+                    return;
+                }
+                rc = PMIx_Data_pack(NULL, &buf, &dmn->name, 1, PMIX_PROC);
+                if (PMIX_SUCCESS != rc) {
+                    PMIX_ERROR_LOG(ret);
+                    PMIX_DATA_BUFFER_DESTRUCT(&buf);
+                    PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT);
+                    return;
+                }
+                rc = PMIx_Data_pack(NULL, &buf, &val->data.string, 1, PMIX_STRING);
+                if (PMIX_SUCCESS != rc) {
+                    PMIX_ERROR_LOG(ret);
+                    PMIX_DATA_BUFFER_DESTRUCT(&buf);
+                    PMIX_VALUE_RELEASE(val);
+                    PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT);
+                    return;
+                }
+                PMIX_VALUE_RELEASE(val);
+            }
+
+            /* goes to all daemons */
+            PMIX_PROC_CREATE(sig.signature, 1);
+            PMIX_LOAD_PROCID(&sig.signature[0], PRTE_PROC_MY_NAME->nspace, PMIX_RANK_WILDCARD);
+            sig.sz = 1;
+            if (PRTE_SUCCESS != (rc = prte_grpcomm.xcast(&sig, PRTE_RML_TAG_WIREUP, &buf))) {
+                PRTE_ERROR_LOG(rc);
+                PMIX_DATA_BUFFER_DESTRUCT(&buf);
+                PMIX_PROC_FREE(sig.signature, 1);
+                PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT);
+                return;
+            }
+            PMIX_DATA_BUFFER_DESTRUCT(&buf);
+            PMIX_PROC_FREE(sig.signature, 1);
+        }
+    }
+    if (PMIX_CHECK_NSPACE(PRTE_PROC_MY_NAME->nspace, caddy->jdata->nspace)) {
+        prte_dvm_ready = true;
+        /* notify that the vm is ready */
+        if (0 > prte_state_base.parent_fd) {
+            if (prte_state_base.ready_msg && prte_persistent) {
+                fprintf(stdout, "DVM ready\n");
+                fflush(stdout);
+            }
+        } else {
+            char ok = 'K';
+            write(prte_state_base.parent_fd, &ok, 1);
+            close(prte_state_base.parent_fd);
+            prte_state_base.parent_fd = -1;
+        }
+        for (i = 0; i < prte_cache->size; i++) {
+            jptr = (prte_job_t *) pmix_pointer_array_get_item(prte_cache, i);
+            if (NULL != jptr) {
+                pmix_pointer_array_set_item(prte_cache, i, NULL);
+                prte_plm.spawn(jptr);
+            }
+        }
+        /* progress the job */
+        caddy->jdata->state = PRTE_JOB_STATE_VM_READY;
+        PMIX_RELEASE(caddy);
+        return;
+    }
+
+    /* position any required files */
+    if (PRTE_SUCCESS != prte_filem.preposition_files(caddy->jdata, files_ready, caddy->jdata)) {
+        PRTE_ACTIVATE_JOB_STATE(caddy->jdata, PRTE_JOB_STATE_FILES_POSN_FAILED);
+    }
+    PMIX_RELEASE(caddy);
+}
+
+static void job_started(int fd, short args, void *cbdata)
+{
+    prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata;
+    prte_job_t *jdata = caddy->jdata;
+    pmix_info_t *iptr;
+    time_t timestamp;
+    pmix_proc_t *nptr;
+    PRTE_HIDE_UNUSED_PARAMS(fd, args);
+
+    /* if there is an originator for this job, notify them
+     * that the first process of the job has been started */
+    if (prte_get_attribute(&jdata->attributes, PRTE_JOB_DVM_JOB, NULL, PMIX_BOOL)) {
+        /* dvm job => launch was requested by a TOOL, so we notify the launch proxy
+         * and NOT the originator (as that would be us) */
+        nptr = NULL;
+        if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_LAUNCH_PROXY, (void **) &nptr, PMIX_PROC)
+            || NULL == nptr) {
+            PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND);
+            return;
+        }
+        timestamp = time(NULL);
+        PMIX_INFO_CREATE(iptr, 5);
+        /* target this notification solely to that one tool */
+        PMIX_INFO_LOAD(&iptr[0], PMIX_EVENT_CUSTOM_RANGE, nptr, PMIX_PROC);
+        PMIX_PROC_RELEASE(nptr);
+        /* pass the nspace of the spawned job */
+        PMIX_INFO_LOAD(&iptr[1], PMIX_NSPACE, jdata->nspace, PMIX_STRING);
+        /* not to be delivered to a default event handler */
+        PMIX_INFO_LOAD(&iptr[2], PMIX_EVENT_NON_DEFAULT, NULL, PMIX_BOOL);
+        /* provide the timestamp */
+        PMIX_INFO_LOAD(&iptr[3], PMIX_EVENT_TIMESTAMP, &timestamp, PMIX_TIME);
+        PMIX_INFO_LOAD(&iptr[4], "prte.notify.donotloop", NULL, PMIX_BOOL);
+        PMIx_Notify_event(PMIX_EVENT_JOB_START, &prte_process_info.myproc, PMIX_RANGE_CUSTOM, iptr,
+                          5, NULL, NULL);
+        PMIX_INFO_FREE(iptr, 5);
+    }
+
+    PMIX_RELEASE(caddy);
+}
+
+static void ready_for_debug(int fd, short args, void *cbdata)
+{
+    prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata;
+    prte_job_t *jdata = caddy->jdata;
+    pmix_proc_t *nptr;
+    time_t timestamp;
+    pmix_info_t *iptr;
+    size_t ninfo;
+    pmix_data_array_t darray;
+    void *tinfo;
+    pmix_status_t rc;
+    int n;
+    char *name;
+    prte_app_context_t *app;
+    PRTE_HIDE_UNUSED_PARAMS(fd, args);
+
+    /* launch was requested by a TOOL, so we notify the launch proxy
+     * and NOT the originator (as that would be us) */
+    nptr = NULL;
+    if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_LAUNCH_PROXY, (void **) &nptr, PMIX_PROC)
+        || NULL == nptr) {
+        PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND);
+        goto DONE;
+    }
+    timestamp = time(NULL);
+    PMIX_INFO_LIST_START(tinfo);
+    /* target this notification solely to that one tool */
+    PMIX_INFO_LIST_ADD(rc, tinfo, PMIX_EVENT_CUSTOM_RANGE, nptr, PMIX_PROC);
+    PMIX_PROC_RELEASE(nptr);
+    /* pass the nspace of the job */
+    PMIX_INFO_LIST_ADD(rc, tinfo, PMIX_NSPACE, jdata->nspace, PMIX_STRING);
+    for (n=0; n < jdata->apps->size; n++) {
+        app = (prte_app_context_t *) pmix_pointer_array_get_item(jdata->apps, n);
+        if (NULL == app) {
+            continue;
+        }
+        /* if pset name was assigned, pass it */
+       if (prte_get_attribute(&app->attributes, PRTE_APP_PSET_NAME, (void**) &name, PMIX_STRING)) {
+           PMIX_INFO_LIST_ADD(rc, tinfo, PMIX_PSET_NAME, name, PMIX_STRING);
+           free(name);
+        }
+        /* pass the argv from each app */
+        name = PMIX_ARGV_JOIN_COMPAT(app->argv, ' ');
+        PMIX_INFO_LIST_ADD(rc, tinfo, PMIX_APP_ARGV, name, PMIX_STRING);
+        free(name);
+    }
+
+    /* not to be delivered to a default event handler */
+    PMIX_INFO_LIST_ADD(rc, tinfo, PMIX_EVENT_NON_DEFAULT, NULL, PMIX_BOOL);
+    /* provide the timestamp */
+    PMIX_INFO_LIST_ADD(rc, tinfo, PMIX_EVENT_TIMESTAMP, &timestamp, PMIX_TIME);
+    PMIX_INFO_LIST_ADD(rc, tinfo, "prte.notify.donotloop", NULL, PMIX_BOOL);
+    PMIX_INFO_LIST_CONVERT(rc, tinfo, &darray);
+    if (PMIX_ERR_EMPTY == rc) {
+        iptr = NULL;
+        ninfo = 0;
+    } else if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PRTE_UPDATE_EXIT_STATUS(rc);
+        PMIX_INFO_LIST_RELEASE(tinfo);
+        PMIX_PROC_RELEASE(nptr);
+        goto DONE;
+    } else {
+        iptr = (pmix_info_t *) darray.array;
+        ninfo = darray.size;
+    }
+    PMIX_INFO_LIST_RELEASE(tinfo);
+
+    PMIx_Notify_event(PMIX_READY_FOR_DEBUG, PRTE_PROC_MY_NAME, PMIX_RANGE_CUSTOM, iptr,
+                      ninfo, NULL, NULL);
+    PMIX_INFO_FREE(iptr, ninfo);
+
+DONE:
+    PMIX_RELEASE(caddy);
+}
+
+static void opcbfunc(pmix_status_t status, void *cbdata)
+{
+    prte_pmix_lock_t *lk = (prte_pmix_lock_t *) cbdata;
+
+    PMIX_POST_OBJECT(lk);
+    lk->status = prte_pmix_convert_status(status);
+    PRTE_PMIX_WAKEUP_THREAD(lk);
+}
+static void lkcbfunc(pmix_status_t status, void *cbdata)
+{
+    prte_pmix_lock_t *lk = (prte_pmix_lock_t *) cbdata;
+
+    PMIX_POST_OBJECT(lk);
+    lk->status = prte_pmix_convert_status(status);
+    PRTE_PMIX_WAKEUP_THREAD(lk);
+}
+static void check_complete(int fd, short args, void *cbdata)
+{
+    prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata;
+    prte_job_t *jdata, *jptr;
+    prte_proc_t *proc;
+    int i, rc;
+    prte_node_t *node;
+    prte_job_map_t *map;
+    int32_t index;
+    pmix_proc_t pname;
+    prte_pmix_lock_t lock;
+    uint8_t command = PRTE_PMIX_PURGE_PROC_CMD;
+    pmix_data_buffer_t *buf;
+    pmix_pointer_array_t procs;
+    char *tmp;
+    prte_timer_t *timer;
+    prte_app_context_t *app;
+    hwloc_obj_t obj;
+    hwloc_obj_type_t type;
+    hwloc_cpuset_t boundcpus, tgt;
+    bool takeall;
+    PRTE_HIDE_UNUSED_PARAMS(fd, args);
+
+    PMIX_ACQUIRE_OBJECT(caddy);
+    jdata = caddy->jdata;
+
+    pmix_output_verbose(2, prte_state_base_framework.framework_output,
+                        "%s state:dvm:check_job_complete on job %s",
+                        PRTE_NAME_PRINT(PRTE_PROC_MY_NAME),
+                        (NULL == jdata) ? "NULL" : PRTE_JOBID_PRINT(jdata->nspace));
+
+    if (NULL != jdata &&
+        prte_get_attribute(&jdata->attributes, PRTE_JOB_TIMEOUT_EVENT, (void **) &timer, PMIX_POINTER)) {
+        /* timer is an prte_timer_t object */
+        prte_event_evtimer_del(timer->ev);
+        PMIX_RELEASE(timer);
+        prte_remove_attribute(&jdata->attributes, PRTE_JOB_TIMEOUT_EVENT);
+    }
+
+    if (NULL == jdata || PMIX_CHECK_NSPACE(jdata->nspace, PRTE_PROC_MY_NAME->nspace)) {
+        /* just check to see if the daemons are complete */
+        PMIX_OUTPUT_VERBOSE(
+            (2, prte_state_base_framework.framework_output,
+             "%s state:dvm:check_job_complete - received NULL job, checking daemons",
+             PRTE_NAME_PRINT(PRTE_PROC_MY_NAME)));
+        if (0 == pmix_list_get_size(&prte_rml_base.children)) {
+            /* orteds are done! */
+            PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output,
+                                 "%s prteds complete - exiting",
+                                 PRTE_NAME_PRINT(PRTE_PROC_MY_NAME)));
+            if (NULL == jdata) {
+                jdata = prte_get_job_data_object(PRTE_PROC_MY_NAME->nspace);
+            }
+            PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_DAEMONS_TERMINATED);
+            PMIX_RELEASE(caddy);
+            prte_dvm_ready = false;
+            return;
+        }
+        prte_plm.terminate_orteds();
+        PMIX_RELEASE(caddy);
+        return;
+    }
+
+    /* mark the job as terminated, but don't override any
+     * abnormal termination flags
+     */
+    if (jdata->state < PRTE_JOB_STATE_UNTERMINATED) {
+        jdata->state = PRTE_JOB_STATE_TERMINATED;
+    }
+
+    /* see if there was any problem */
+    if (prte_get_attribute(&jdata->attributes, PRTE_JOB_ABORTED_PROC, NULL, PMIX_POINTER)) {
+        rc = prte_pmix_convert_rc(jdata->exit_code);
+        /* or whether we got cancelled by the user */
+    } else if (prte_get_attribute(&jdata->attributes, PRTE_JOB_CANCELLED, NULL, PMIX_BOOL)) {
+        rc = prte_pmix_convert_rc(PRTE_ERR_JOB_CANCELLED);
+    } else {
+        rc = prte_pmix_convert_rc(jdata->exit_code);
+    }
+
+    /* if would be rare, but a very fast terminating job could conceivably
+     * reach here prior to the spawn requestor being notified of spawn */
+    rc = prte_plm_base_spawn_response(rc, jdata);
+    if (PRTE_SUCCESS != rc) {
+        PRTE_ERROR_LOG(rc);
+    }
+
+    /* cleanup any pending server ops */
+    PMIX_LOAD_PROCID(&pname, jdata->nspace, PMIX_RANK_WILDCARD);
+    prte_pmix_server_clear(&pname);
+
+    /* cleanup the procs as these are gone */
+    for (i = 0; i < prte_local_children->size; i++) {
+        if (NULL == (proc = (prte_proc_t *) pmix_pointer_array_get_item(prte_local_children, i))) {
+            continue;
+        }
+        /* if this child is part of the job... */
+        if (PMIX_CHECK_NSPACE(proc->name.nspace, jdata->nspace)) {
+            /* clear the entry in the local children */
+            pmix_pointer_array_set_item(prte_local_children, i, NULL);
+            PMIX_RELEASE(proc); // maintain accounting
+        }
+    }
+
+    /* tell the IOF that the job is complete */
+    if (NULL != prte_iof.complete) {
+        prte_iof.complete(jdata);
+    }
+
+    /* tell the PMIx subsystem the job is complete */
+    PRTE_PMIX_CONSTRUCT_LOCK(&lock);
+    PMIx_server_deregister_nspace(pname.nspace, opcbfunc, &lock);
+    PRTE_PMIX_WAIT_THREAD(&lock);
+    PRTE_PMIX_DESTRUCT_LOCK(&lock);
+
+    if (!prte_persistent) {
+        /* update our exit status */
+        PRTE_UPDATE_EXIT_STATUS(jdata->exit_code);
+        /* if this is an abnormal termination, report it */
+        if (jdata->state > PRTE_JOB_STATE_ERROR) {
+            char *msg;
+            msg = prte_dump_aborted_procs(jdata);
+            if (NULL != msg) {
+                pmix_byte_object_t bo;
+                PMIX_BYTE_OBJECT_CONSTRUCT(&bo);
+                bo.bytes = (char *) msg;
+                bo.size = strlen(msg) + 1;
+                PRTE_PMIX_CONSTRUCT_LOCK(&lock);
+                rc = PMIx_server_IOF_deliver(&prte_process_info.myproc,
+                                             PMIX_FWD_STDDIAG_CHANNEL,
+                                             &bo, NULL, 0, lkcbfunc, (void *) &lock);
+                if (PMIX_SUCCESS != rc) {
+                    PMIX_ERROR_LOG(rc);
+                } else {
+                    /* wait for completion */
+                    PRTE_PMIX_WAIT_THREAD(&lock);
+                    if (PMIX_SUCCESS != lock.status) {
+                        PMIX_ERROR_LOG(lock.status);
+                    }
+                }
+                PRTE_PMIX_DESTRUCT_LOCK(&lock);
+                free(msg);
+            }
+        }
+        /* if all of the jobs we are running are done, then shut us down */
+        for (i = 0; i < prte_job_data->size; i++) {
+            jptr = (prte_job_t *) pmix_pointer_array_get_item(prte_job_data, i);
+            if (NULL == jptr) {
+                continue;
+            }
+            /* skip the daemon job */
+            if (PMIX_CHECK_NSPACE(jptr->nspace, PRTE_PROC_MY_NAME->nspace)) {
+                continue;
+            }
+            if (jptr->state < PRTE_JOB_STATE_TERMINATED) {
+                /* still alive - finish processing this job's termination */
+                goto release;
+            }
+        }
+
+        /* Let the tools know that a job terminated before we shutdown */
+        if (jdata->state != PRTE_JOB_STATE_NOTIFIED) {
+            PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output,
+                                 "%s state:dvm:check_job_completed state is terminated - activating notify",
+                                 PRTE_NAME_PRINT(PRTE_PROC_MY_NAME)));
+            terminate_dvm = true;  // flag that the DVM is to terminate
+            PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_NOTIFY_COMPLETED);
+            return;
+        }
+
+        /* if we fell thru to this point, then nobody is still
+         * alive except the daemons, so just shut us down */
+        prte_plm.terminate_orteds();
+        PMIX_RELEASE(caddy);
+        return;
+    }
+
+    if (NULL != prte_data_server_uri) {
+        /* tell the data server to purge any data from this nspace */
+        PMIX_DATA_BUFFER_CREATE(buf);
+        /* room number is ignored, but has to be included for pack sequencing */
+        i = 0;
+        rc = PMIx_Data_pack(NULL, buf, &i, 1, PMIX_INT);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_DATA_BUFFER_RELEASE(buf);
+            goto release;
+        }
+        rc = PMIx_Data_pack(NULL, buf, &command, 1, PMIX_UINT8);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_DATA_BUFFER_RELEASE(buf);
+            goto release;
+        }
+        /* pack the nspace to be purged */
+        pname.rank = PMIX_RANK_WILDCARD;
+        rc = PMIx_Data_pack(NULL, buf, &pname, 1, PMIX_PROC);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_DATA_BUFFER_RELEASE(buf);
+            goto release;
+        }
+        /* send it to the data server */
+        PRTE_RML_SEND(rc, PRTE_PROC_MY_NAME->rank, buf, PRTE_RML_TAG_DATA_SERVER);
+        if (PRTE_SUCCESS != rc) {
+            PRTE_ERROR_LOG(rc);
+            PMIX_DATA_BUFFER_RELEASE(buf);
+        }
+    }
+
+release:
+    /* Release the resources used by this job. Since some errmgrs may want
+     * to continue using resources allocated to the job as part of their
+     * fault recovery procedure, we only do this once the job is "complete".
+     * Note that an aborted/killed job -is- flagged as complete and will
+     * therefore have its resources released. We need to do this after
+     * we call the errmgr so that any attempt to restart the job will
+     * avoid doing so in the exact same place as the current job
+     */
+    if (NULL != jdata->map) {
+        map = jdata->map;
+        takeall = false;
+        if (prte_get_attribute(&jdata->attributes, PRTE_JOB_HWT_CPUS, NULL, PMIX_BOOL)) {
+            type = HWLOC_OBJ_PU;
+        } else {
+            type = HWLOC_OBJ_CORE;
+        }
+        if (prte_get_attribute(&jdata->attributes, PRTE_JOB_PES_PER_PROC, NULL, PMIX_UINT16) ||
+            PRTE_MAPPING_BYUSER == PRTE_GET_MAPPING_POLICY(map->mapping) ||
+            PRTE_MAPPING_SEQ == PRTE_GET_MAPPING_POLICY(map->mapping)) {
+            takeall = true;
+        }
+        boundcpus = hwloc_bitmap_alloc();
+        for (index = 0; index < map->nodes->size; index++) {
+            node = (prte_node_t *) pmix_pointer_array_get_item(map->nodes, index);
+            if (NULL == node) {
+                continue;
+            }
+            PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output,
+                                 "%s state:dvm releasing procs from node %s",
+                                 PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), node->name));
+            for (i = 0; i < node->procs->size; i++) {
+                proc = (prte_proc_t *) pmix_pointer_array_get_item(node->procs, i);
+                if (NULL == proc) {
+                    continue;
+                }
+                if (!PMIX_CHECK_NSPACE(proc->name.nspace, jdata->nspace)) {
+                    /* skip procs from another job */
+                    continue;
+                }
+                app = (prte_app_context_t*) pmix_pointer_array_get_item(jdata->apps, proc->app_idx);
+                if (!PRTE_FLAG_TEST(app, PRTE_APP_FLAG_TOOL) &&
+                    !PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_TOOL)) {
+                    node->slots_inuse--;
+                    node->num_procs--;
+                    node->next_node_rank--;
+                }
+                /* release the resources held by the proc - only the first
+                 * cpu in the proc's cpuset was used to mark usage */
+                if (NULL != proc->cpuset) {
+                    if (0 != (rc = hwloc_bitmap_list_sscanf(boundcpus, proc->cpuset))) {
+                        pmix_output(0, "hwloc_bitmap_sscanf returned %s for the string %s",
+                                    prte_strerror(rc), proc->cpuset);
+                        continue;
+                    }
+                    if (takeall) {
+                        tgt = boundcpus;
+                    } else {
+                        /* we only want to restore the first CPU of whatever region
+                         * the proc was bound to, so we have to first narrow the
+                         * bitmap down to only that region */
+                        hwloc_bitmap_andnot(prte_rmaps_base.available, boundcpus, node->available);
+                        /* the set bits in the result are the bound cpus that are still
+                         * marked as in-use */
+                        obj = hwloc_get_obj_inside_cpuset_by_type(node->topology->topo,
+                                                                  prte_rmaps_base.available, type, 0);
+                        if (NULL == obj) {
+                            pmix_output(0, "COULD NOT GET BOUND CPU FOR RESOURCE RELEASE");
+                            continue;
+                        }
+#if HWLOC_API_VERSION < 0x20000
+                        tgt = obj->allowed_cpuset;
+#else
+                        tgt = obj->cpuset;
+#endif
+                    }
+                    hwloc_bitmap_or(node->available, node->available, tgt);
+                }
+
+                PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output,
+                                     "%s state:dvm releasing proc %s from node %s",
+                                     PRTE_NAME_PRINT(PRTE_PROC_MY_NAME),
+                                     PRTE_NAME_PRINT(&proc->name), node->name));
+                /* set the entry in the node array to NULL */
+                pmix_pointer_array_set_item(node->procs, i, NULL);
+                /* release the proc once for the map entry */
+                PMIX_RELEASE(proc);
+            }
+            /* set the node location to NULL */
+            pmix_pointer_array_set_item(map->nodes, index, NULL);
+            /* maintain accounting */
+            PMIX_RELEASE(node);
+            /* flag that the node is no longer in a map */
+            PRTE_FLAG_UNSET(node, PRTE_NODE_FLAG_MAPPED);
+        }
+        hwloc_bitmap_free(boundcpus);
+        PMIX_RELEASE(map);
+        jdata->map = NULL;
+    }
+
+    /* if requested, check fd status for leaks */
+    if (prte_state_base.run_fdcheck) {
+        prte_state_base_check_fds(jdata);
+    }
+
+    /* if this job was a launcher, then we need to abort all of its
+     * child jobs that might still be running */
+    if (0 < pmix_list_get_size(&jdata->children)) {
+        PMIX_CONSTRUCT(&procs, pmix_pointer_array_t);
+        pmix_pointer_array_init(&procs, 1, INT_MAX, 1);
+        PMIX_LIST_FOREACH(jptr, &jdata->children, prte_job_t)
+        {
+            proc = PMIX_NEW(prte_proc_t);
+            PMIX_LOAD_PROCID(&proc->name, jptr->nspace, PMIX_RANK_WILDCARD);
+            pmix_pointer_array_add(&procs, proc);
+        }
+        prte_plm.terminate_procs(&procs);
+        for (i = 0; i < procs.size; i++) {
+            if (NULL != (proc = (prte_proc_t *) pmix_pointer_array_get_item(&procs, i))) {
+                PMIX_RELEASE(proc);
+            }
+        }
+        PMIX_DESTRUCT(&procs);
+    }
+
+    /* remove the session directory tree */
+    if (0 > pmix_asprintf(&tmp, "%s/%u", prte_process_info.jobfam_session_dir,
+                          PRTE_LOCAL_JOBID(jdata->nspace))) {
+        PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE);
+    } else {
+        pmix_os_dirpath_destroy(tmp, true, NULL);
+        free(tmp);
+    }
+
+    if (jdata->state != PRTE_JOB_STATE_NOTIFIED) {
+        PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output,
+                             "%s state:dvm:check_job_completed state is terminated - activating notify",
+                             PRTE_NAME_PRINT(PRTE_PROC_MY_NAME)));
+        PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_NOTIFY_COMPLETED);
+        /* mark the job as notified */
+        jdata->state = PRTE_JOB_STATE_NOTIFIED;
+    }
+
+    PMIX_RELEASE(caddy);
+}
+
+static void cleanup_job(int sd, short args, void *cbdata)
+{
+    prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata;
+    PRTE_HIDE_UNUSED_PARAMS(sd, args);
+
+    PMIX_ACQUIRE_OBJECT(caddy);
+
+    if (terminate_dvm && !dvm_terminated) {
+        dvm_terminated = true;
+        prte_plm.terminate_orteds();
+    }
+
+    PMIX_RELEASE(caddy);
+}
+
+static void dvm_notify(int sd, short args, void *cbdata)
+{
+    prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata;
+    prte_job_t *jdata = caddy->jdata;
+    prte_proc_t *pptr = NULL;
+    int rc;
+    pmix_data_buffer_t *reply;
+    prte_daemon_cmd_flag_t command;
+    prte_grpcomm_signature_t sig;
+    bool notify = true, flag;
+    pmix_proc_t *proc, pnotify;
+    pmix_info_t *info;
+    size_t ninfo;
+    pmix_proc_t pname;
+    pmix_data_buffer_t pbkt;
+    pmix_data_range_t range = PMIX_RANGE_SESSION;
+    pmix_status_t code, ret;
+    char *errmsg = NULL;
+    PRTE_HIDE_UNUSED_PARAMS(sd, args);
+
+    PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output,
+                         "%s state:dvm:dvm_notify called",
+                         PRTE_NAME_PRINT(PRTE_PROC_MY_NAME)));
+
+    /* see if there was any problem */
+    if (prte_get_attribute(&jdata->attributes, PRTE_JOB_ABORTED_PROC, (void **) &pptr, PMIX_POINTER)
+        && NULL != pptr) {
+        rc = jdata->exit_code;
+        /* or whether we got cancelled by the user */
+    } else if (prte_get_attribute(&jdata->attributes, PRTE_JOB_CANCELLED, NULL, PMIX_BOOL)) {
+        rc = PRTE_ERR_JOB_CANCELLED;
+    } else {
+        rc = jdata->exit_code;
+    }
+
+    if (0 == rc &&
+        prte_get_attribute(&jdata->attributes, PRTE_JOB_SILENT_TERMINATION, NULL, PMIX_BOOL)) {
+        notify = false;
+    }
+    /* if the jobid matches that of the requestor, then don't notify */
+    if (prte_get_attribute(&jdata->attributes, PRTE_JOB_LAUNCH_PROXY, (void **) &proc, PMIX_PROC)) {
+        if (PMIX_CHECK_NSPACE(proc->nspace, jdata->nspace)) {
+            notify = false;
+        }
+        PMIX_PROC_RELEASE(proc);
+    }
+
+    if (notify) {
+        PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output,
+                             "%s state:dvm:dvm_notify notification requested",
+                             PRTE_NAME_PRINT(PRTE_PROC_MY_NAME)));
+        /* if it was an abnormal termination, then construct an appropriate
+         * error message */
+        if (PRTE_SUCCESS != rc) {
+            errmsg = prte_dump_aborted_procs(jdata);
+        }
+        /* construct the info to be provided */
+        if (NULL == errmsg) {
+            ninfo = 3;
+        } else {
+            ninfo = 4;
+        }
+        PMIX_INFO_CREATE(info, ninfo);
+        /* ensure this only goes to the job terminated event handler */
+        flag = true;
+        PMIX_INFO_LOAD(&info[0], PMIX_EVENT_NON_DEFAULT, &flag, PMIX_BOOL);
+        /* provide the status */
+        PMIX_INFO_LOAD(&info[1], PMIX_JOB_TERM_STATUS, &rc, PMIX_STATUS);
+        /* tell the requestor which job or proc  */
+        PMIX_LOAD_NSPACE(pname.nspace, jdata->nspace);
+        if (NULL != pptr) {
+            pname.rank = pptr->name.rank;
+        } else {
+            pname.rank = PMIX_RANK_WILDCARD;
+        }
+        PMIX_INFO_LOAD(&info[2], PMIX_EVENT_AFFECTED_PROC, &pname, PMIX_PROC);
+        if (NULL != errmsg) {
+            PMIX_INFO_LOAD(&info[3], PMIX_EVENT_TEXT_MESSAGE, errmsg, PMIX_STRING);
+            free(errmsg);
+        }
+
+        /* pack the info for sending */
+        PMIX_DATA_BUFFER_CONSTRUCT(&pbkt);
+
+        /* pack the status code */
+        code = PMIX_EVENT_JOB_END;
+        if (PMIX_SUCCESS != (ret = PMIx_Data_pack(NULL, &pbkt, &code, 1, PMIX_STATUS))) {
+            PMIX_ERROR_LOG(ret);
+            PMIX_INFO_FREE(info, ninfo);
+            PMIX_DATA_BUFFER_DESTRUCT(&pbkt);
+            return;
+        }
+        /* pack the source - it cannot be me as that will cause
+         * the pmix server to upcall the event back to me */
+        PMIX_LOAD_PROCID(&pnotify, jdata->nspace, 0);
+        if (PMIX_SUCCESS != (ret = PMIx_Data_pack(NULL, &pbkt, &pnotify, 1, PMIX_PROC))) {
+            PMIX_ERROR_LOG(ret);
+            PMIX_INFO_FREE(info, ninfo);
+            PMIX_DATA_BUFFER_DESTRUCT(&pbkt);
+            return;
+        }
+        /* pack the range */
+        if (PMIX_SUCCESS != (ret = PMIx_Data_pack(NULL, &pbkt, &range, 1, PMIX_DATA_RANGE))) {
+            PMIX_ERROR_LOG(ret);
+            PMIX_INFO_FREE(info, ninfo);
+            PMIX_DATA_BUFFER_DESTRUCT(&pbkt);
+            return;
+        }
+        /* pack the number of infos */
+        if (PMIX_SUCCESS != (ret = PMIx_Data_pack(NULL, &pbkt, &ninfo, 1, PMIX_SIZE))) {
+            PMIX_ERROR_LOG(ret);
+            PMIX_INFO_FREE(info, ninfo);
+            PMIX_DATA_BUFFER_DESTRUCT(&pbkt);
+            return;
+        }
+        /* pack the infos themselves */
+        if (PMIX_SUCCESS != (ret = PMIx_Data_pack(NULL, &pbkt, info, ninfo, PMIX_INFO))) {
+            PMIX_ERROR_LOG(ret);
+            PMIX_INFO_FREE(info, ninfo);
+            PMIX_DATA_BUFFER_DESTRUCT(&pbkt);
+            return;
+        }
+        PMIX_INFO_FREE(info, ninfo);
+
+        /* insert into pmix_data_buffer_t */
+        PMIX_DATA_BUFFER_CREATE(reply);
+        /* we need to add a flag indicating this came from an invalid proc so that we will
+         * inject it into our own PMIx server library */
+        rc = PMIx_Data_pack(NULL, reply, &PRTE_NAME_INVALID->rank, 1, PMIX_PROC_RANK);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_DATA_BUFFER_DESTRUCT(&pbkt);
+            PMIX_DATA_BUFFER_RELEASE(reply);
+            return;
+        }
+        rc = PMIx_Data_copy_payload(reply, &pbkt);
+        PMIX_DATA_BUFFER_DESTRUCT(&pbkt);
+
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_DATA_BUFFER_RELEASE(reply);
+            return;
+        }
+
+        /* we have to send the notification to all daemons so that
+         * anyone watching for it can receive it */
+        PMIX_PROC_CREATE(sig.signature, 1);
+        PMIX_LOAD_PROCID(&sig.signature[0], PRTE_PROC_MY_NAME->nspace, PMIX_RANK_WILDCARD);
+        sig.sz = 1;
+        if (PRTE_SUCCESS != (rc = prte_grpcomm.xcast(&sig, PRTE_RML_TAG_NOTIFICATION, reply))) {
+            PRTE_ERROR_LOG(rc);
+            PMIX_DATA_BUFFER_RELEASE(reply);
+            PMIX_PROC_FREE(sig.signature, 1);
+            return;
+        }
+        PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output,
+                             "%s state:dvm:dvm_notify notification sent",
+                             PRTE_NAME_PRINT(PRTE_PROC_MY_NAME)));
+        PMIX_DATA_BUFFER_RELEASE(reply);
+        /* maintain accounting */
+        PMIX_PROC_FREE(sig.signature, 1);
+    }
+
+    if (prte_persistent) {
+        /* now ensure that _all_ daemons know that this job has terminated so even
+         * those that did not participate in it will know to cleanup the resources
+         * they assigned to the job. This is necessary now that the mapping function
+         * has been moved to the backend daemons - otherwise, non-participating daemons
+         * retain the slot assignments on the participating daemons, and then incorrectly
+         * map subsequent jobs thinking those nodes are still "busy" */
+        PMIX_DATA_BUFFER_CREATE(reply);
+        command = PRTE_DAEMON_DVM_CLEANUP_JOB_CMD;
+        rc = PMIx_Data_pack(NULL, reply, &command, 1, PMIX_UINT8);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_DATA_BUFFER_RELEASE(reply);
+            return;
+        }
+        rc = PMIx_Data_pack(NULL, reply, &jdata->nspace, 1, PMIX_PROC_NSPACE);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_DATA_BUFFER_RELEASE(reply);
+            return;
+        }
+        PMIX_PROC_CREATE(sig.signature, 1);
+        PMIX_LOAD_PROCID(&sig.signature[0], PRTE_PROC_MY_NAME->nspace, PMIX_RANK_WILDCARD);
+        sig.sz = 1;
+        prte_grpcomm.xcast(&sig, PRTE_RML_TAG_DAEMON, reply);
+        PMIX_DATA_BUFFER_RELEASE(reply);
+        PMIX_PROC_FREE(sig.signature, 1);
+        PMIX_RELEASE(caddy);
+    }
+
+    // We are done with our use of job data and have notified the other daemons
+    if (notify) {
+        PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_NOTIFIED);
+    }
+}

From a99f5f0e474473d6df07593927847632f9f4d9a7 Mon Sep 17 00:00:00 2001
From: Ralph Castain <rhc@pmix.org>
Date: Wed, 26 Jul 2023 20:13:08 -0600
Subject: [PATCH 4/4] Propagate returned status

Signed-off-by: Ralph Castain <rhc@pmix.org>
---
 src/prted/pmix/pmix_server.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/prted/pmix/pmix_server.c b/src/prted/pmix/pmix_server.c
index 3ad4a4ead3..e417aa06d9 100644
--- a/src/prted/pmix/pmix_server.c
+++ b/src/prted/pmix/pmix_server.c
@@ -554,6 +554,7 @@ static void regcbfunc(pmix_status_t status, size_t ref, void *cbdata)
     PRTE_HIDE_UNUSED_PARAMS(status, ref);
 
     PMIX_ACQUIRE_OBJECT(lock);
+    lock->status = status;
     PRTE_PMIX_WAKEUP_THREAD(lock);
 }
 
@@ -877,6 +878,7 @@ int pmix_server_init(void)
     prc = PMIX_ERR_LOST_CONNECTION;
     PMIx_Register_event_handler(&prc, 1, NULL, 0, lost_connection_hdlr, regcbfunc, &lock);
     PRTE_PMIX_WAIT_THREAD(&lock);
+    prc = lock.status;
     PRTE_PMIX_DESTRUCT_LOCK(&lock);
     rc = prte_pmix_convert_status(prc);