diff --git a/nix/workbench/backend/backend.sh b/nix/workbench/backend/backend.sh index 2e75ac1e03f..d0ef16ea64c 100644 --- a/nix/workbench/backend/backend.sh +++ b/nix/workbench/backend/backend.sh @@ -24,6 +24,8 @@ usage_backend() { wait-pools-stopped RUNDIR Wait until all pools are stopped + wait-workloads-stopped RUNDIR + Wait until all workloads are stopped stop-cluster RUNDIR cleanup-cluster RUNDIR Wipe cluster state to pristine @@ -50,8 +52,8 @@ case "${op}" in start-tracers ) backend_$WB_BACKEND "$@";; start-nodes ) backend_$WB_BACKEND "$@";; start-generator ) backend_$WB_BACKEND "$@";; + start-workloads ) backend_$WB_BACKEND "$@";; start-healthchecks ) backend_$WB_BACKEND "$@";; - start-latencies ) backend_$WB_BACKEND "$@";; # Fine grained start-node ) backend_$WB_BACKEND "$@";; stop-node ) backend_$WB_BACKEND "$@";; @@ -59,7 +61,7 @@ case "${op}" in wait-node-stopped ) backend_$WB_BACKEND "$@";; get-node-socket-path ) backend_$WB_BACKEND "$@";; wait-pools-stopped ) backend_$WB_BACKEND "$@";; - wait-latencies-stopped ) backend_$WB_BACKEND "$@";; + wait-workloads-stopped ) backend_$WB_BACKEND "$@";; # Stop functions stop-all ) backend_$WB_BACKEND "$@";; fetch-logs ) backend_$WB_BACKEND "$@";; diff --git a/nix/workbench/backend/nomad-job.nix b/nix/workbench/backend/nomad-job.nix index 57bcdaed097..a7bbe91d710 100644 --- a/nix/workbench/backend/nomad-job.nix +++ b/nix/workbench/backend/nomad-job.nix @@ -902,28 +902,27 @@ let } ]) ++ - # healthcheck - [ - ## healthcheck start.sh script. + # workloads + (builtins.map (workload: + ## workload start.sh script. { env = false; - destination = "local/${stateDir}/healthcheck/start.sh"; - data = escapeTemplate - profileData.healthcheck-service.start.value; + destination = "local/${stateDir}/workloads/${workload.name}/start.sh"; + data = escapeTemplate workload.start.value; change_mode = "noop"; error_on_missing_key = true; perms = "744"; # Only for every "start.sh" script. Default: "644" } - ] + ) profileData.workloads-service) ++ - # latency + # healthcheck [ - ## Latency start.sh script. + ## healthcheck start.sh script. { env = false; - destination = "local/${stateDir}/latency/start.sh"; + destination = "local/${stateDir}/healthcheck/start.sh"; data = escapeTemplate - profileData.latency-service.start.value; + profileData.healthcheck-service.start.value; change_mode = "noop"; error_on_missing_key = true; perms = "744"; # Only for every "start.sh" script. Default: "644" diff --git a/nix/workbench/backend/nomad.sh b/nix/workbench/backend/nomad.sh index 1d9554ee620..75e9c5d0bae 100644 --- a/nix/workbench/backend/nomad.sh +++ b/nix/workbench/backend/nomad.sh @@ -34,8 +34,8 @@ backend_nomad() { # - allocate-run-directory-tracers RUN-DIR (Nomad only) # - allocate-run-directory-nodes RUN-DIR (Nomad only) # - allocate-run-directory-generator RUN-DIR (Nomad only) + # - allocate-run-directory-workloads RUN-DIR (Nomad only) # - allocate-run-directory-healthchecks RUN-DIR (Nomad only) - # - allocate-run-directory-latencies RUN-DIR (Nomad only) # - allocate-run-nomad-job-patch-name RUN-DIR NAME (Nomad only) # - allocate-run-nomad-job-patch-namespace RUN-DIR NAME (Nomad only) # - allocate-run-nomad-job-patch-nix RUN-DIR (Nomad only) @@ -52,10 +52,10 @@ backend_nomad() { # After `allocate-run` the Nomad job is running (supervisord) waiting for # genesis to be deployed and tracer/cardano-nodes/generator to be started. # - # "generator", "tracer", "node", "healthcheck" and "latency" folder contents - # (start.sh, config files, etc) are included in the Nomad Job spec file as - # "template" stanzas and are materialized inside the container when the job - # is started. This is how it works for every environment combination + # "generator", "tracer", "node", "workloads" and "healthcheck" folder + # contents (start.sh, config files, etc) are included in the Nomad Job spec + # file as "template" stanzas and are materialized inside the container when + # the job is started. This is how it works for every environment combination # (exec-(local/cloud)). # # But "genesis" and "CARDANO_MAINNET_MIRROR" are the deployment exceptions: @@ -83,8 +83,8 @@ backend_nomad() { backend_nomad allocate-run-directory-tracers "${dir}" backend_nomad allocate-run-directory-nodes "${dir}" backend_nomad allocate-run-directory-generator "${dir}" + backend_nomad allocate-run-directory-workloads "${dir}" backend_nomad allocate-run-directory-healthchecks "${dir}" - backend_nomad allocate-run-directory-latencies "${dir}" # This one is decided at "setenv-defaults" of each sub-backend. local nomad_environment=$(envjqr 'nomad_environment') @@ -188,26 +188,32 @@ backend_nomad() { mkdir -p "${dir}"/generator ;; - allocate-run-directory-healthchecks ) + allocate-run-directory-workloads ) local usage="USAGE: wb backend $op RUN-DIR" local dir=${1:?$usage}; shift - mkdir "${dir}"/healthcheck - # For every node ... - local nodes=($(jq_tolist keys "${dir}"/node-specs.json)) - for node in ${nodes[*]} + mkdir "${dir}"/workloads + + # For every workload ... + for workload in $(jq_tolist '.workloads | map(.name)' "$dir"/profile.json) do - # File "start.sh" that usually goes in here is copied from the - # Task/container once it's started because the contents are created or - # patched using Nomad's "template" stanza in the job spec and we want to - # hold a copy of what was actually run. - mkdir "${dir}"/healthcheck/"${node}" + mkdir "${dir}"/workloads/"${workload}" + # Workload may or may not run something in all producers. + local nodes=($(jq_tolist 'map(select(.isProducer) | .name)' "$dir"/node-specs.json)) + for node in ${nodes[*]} + do + # File "start.sh" that usually goes in here is copied from the + # Task/container once it's started because the contents are created or + # patched using Nomad's "template" stanza in the job spec and we want + # to hold a copy of what was actually run. + mkdir "${dir}"/workloads/"${workload}"/"${node}" + done done ;; - allocate-run-directory-latencies ) + allocate-run-directory-healthchecks ) local usage="USAGE: wb backend $op RUN-DIR" local dir=${1:?$usage}; shift - mkdir "${dir}"/latency + mkdir "${dir}"/healthcheck # For every node ... local nodes=($(jq_tolist keys "${dir}"/node-specs.json)) for node in ${nodes[*]} @@ -216,7 +222,7 @@ backend_nomad() { # Task/container once it's started because the contents are created or # patched using Nomad's "template" stanza in the job spec and we want to # hold a copy of what was actually run. - mkdir "${dir}"/latency/"${node}" + mkdir "${dir}"/healthcheck/"${node}" done ;; @@ -391,18 +397,7 @@ backend_nomad() { # Last the Tasks' template stanzas. msg "Fetch Nomad generated files ..." local jobs_tasks=() - # The `tx-generator` config files, running in one of the Tasks were - # `cardano-node` is deployed. - backend_nomad download-config-generator "${dir}" & - jobs_tasks+=("$!") - # For every node (not including a possible tracer Task) ... - local nodes=($(jq_tolist keys "$dir"/node-specs.json)) - for node in ${nodes[*]} - do - # `cardano-node` config files. - backend_nomad download-config-node "${dir}" "${node}" & - jobs_tasks+=("$!") - done + # Tracers. if test "${one_tracer_per_node}" = "true" then local nodes=($(jq_tolist keys "$dir"/node-specs.json)) @@ -422,22 +417,35 @@ backend_nomad() { local nodes=($(jq_tolist keys "$dir"/node-specs.json)) for node in ${nodes[*]} do - # Only used for debugging! - backend_nomad download-config-healthcheck "${dir}" "${node}" & + # `cardano-node` config files. + backend_nomad download-config-node "${dir}" "${node}" & jobs_tasks+=("$!") done - # DO NOT DOWNLOAD THE latency SCRIPTS/CONFIG EVERY TIME - if echo "${WB_SHELL_PROFILE}" | grep --quiet "latency" - then + # The `tx-generator` config files, running in one of the Tasks were + # `cardano-node` is deployed. + backend_nomad download-config-generator "${dir}" & + jobs_tasks+=("$!") + # For every workload + for workload in $(jq_tolist '.workloads | map(.name)' "$dir"/profile.json) + do # For every node (not including a possible tracer Task) ... - local nodes=($(jq_tolist keys "$dir"/node-specs.json)) + # Workload may or may not run something in all producers. + local nodes=($(jq_tolist 'map(select(.isProducer) | .name)' "$dir"/node-specs.json)) for node in ${nodes[*]} do # Only used for debugging! - backend_nomad download-config-latency "${dir}" "${node}" & + backend_nomad download-config-workload "${dir}" "${workload}" "${node}" & jobs_tasks+=("$!") done - fi + done + # For every node (not including a possible tracer Task) ... + local nodes=($(jq_tolist keys "$dir"/node-specs.json)) + for node in ${nodes[*]} + do + # Only used for debugging! + backend_nomad download-config-healthcheck "${dir}" "${node}" & + jobs_tasks+=("$!") + done # Wait and check! if test -n "${jobs_tasks}" then @@ -754,7 +762,7 @@ backend_nomad() { # Functions to stop the cluster: # - stop-all RUN-DIR # - stop-all-healthchecks RUN-DIR (Nomad only) - # - stop-all-latencies RUN-DIR (Nomad only) + # - stop-all-workloads RUN-DIR (Nomad only) # - stop-all-generator RUN-DIR (Nomad only) # - stop-all-nodes RUN-DIR (Nomad only) # - stop-all-tracers RUN-DIR (Nomad only) @@ -770,20 +778,8 @@ backend_nomad() { local dir=${1:?$usage}; shift local generator_task=$(envjqr 'generator_task_name') - # Stop latency(s). - ################## - local jobs_latencies_array=() - for node in $(jq_tolist 'keys' "${dir}"/node-specs.json) - do - backend_nomad stop-all-latencies "${dir}" "${node}" & - jobs_latencies_array+=("$!") - done - if ! wait_all "${jobs_latencies_array[@]}" - then - msg "$(red "Failed to stop latency(ies)")" - fi # Stop healthcheck(s). - ##################### + ###################### local jobs_healthchecks_array=() for node in $(jq_tolist 'keys' "${dir}"/node-specs.json) do @@ -794,6 +790,23 @@ backend_nomad() { then msg "$(red "Failed to stop healthcheck(s)")" fi + # Stop workload(s). + ################### + local jobs_workloads_array=() + # For every workload + for workload in $(jq_tolist '.workloads | map(.name)' "$dir"/profile.json) + do + # Workload may or may not run something in all producers. + for node in $(jq_tolist 'map(select(.isProducer) | .name)' "$dir"/node-specs.json) + do + backend_nomad stop-all-workloads "${dir}" "${workload}" "${node}" & + jobs_workloads_array+=("$!") + done + done + if ! wait_all "${jobs_workloads_array[@]}" + then + msg "$(red "Failed to stop workload(s)")" + fi # Stop generator. ################# backend_nomad stop-all-generator "${dir}" "${generator_task}" @@ -833,25 +846,25 @@ backend_nomad() { fi ;; - stop-all-latencies ) + stop-all-healthchecks ) local usage="USAGE: wb backend $op RUN-DIR" local dir=${1:?$usage}; shift local task=${1:?$usage}; shift - local task_dir="${dir}"/latency/"${task}" + local task_dir="${dir}"/healthcheck/"${task}" if test -f "${task_dir}"/started && !(test -f "${task_dir}"/stopped || test -f "${task_dir}"/quit) then - if backend_nomad is-task-program-running "${dir}" "${task}" latency + if backend_nomad is-task-program-running "${dir}" "${task}" healthcheck then - if ! backend_nomad task-program-stop "${dir}" "${task}" latency + if ! backend_nomad task-program-stop "${dir}" "${task}" healthcheck then - msg "$(yellow "WARNING: Program \"latency\" inside Task \"${task}\" failed to stop")" + msg "$(yellow "WARNING: Program \"healthcheck\" inside Task \"${task}\" failed to stop")" else touch "${task_dir}"/stopped - msg "$(green "supervisord program \"latency\" inside Nomad Task \"${task}\" down!")" + msg "$(green "supervisord program \"healthcheck\" inside Nomad Task \"${task}\" down!")" fi else touch "${task_dir}"/quit - if backend_nomad is-task-program-failed "${dir}" "${task}" latency + if backend_nomad is-task-program-failed "${dir}" "${task}" healthcheck then local generator_task=$(envjqr 'generator_task_name') # If the node quits (due to `--shutdown_on_slot_synced X` or @@ -860,36 +873,37 @@ backend_nomad() { if test "${generator_task}" != "${task}" || test "${generator_can_fail}" = "false" || backend_nomad is-task-program-running "${dir}" "${task}" "${task}" then # Do not fail here, because nobody will be able to stop the cluster! - msg "$(red "FATAL: \"latency\" inside Task \"${task}\" quit unexpectedly")" + msg "$(red "FATAL: \"healthcheck\" inside Task \"${task}\" quit unexpectedly")" else - msg "$(yellow "INFO: Program \"latency\" inside Task \"${task}\" failed, but expected when \"${task}\" automatically exits first and makes \"generator\" fail")" + msg "$(yellow "INFO: Program \"healthcheck\" inside Task \"${task}\" failed, but expected when \"${task}\" automatically exits first and makes \"generator\" fail")" fi else - msg "$(yellow "WARNING: Program \"latency\" inside Task \"${task}\" was not running, should it?")" + msg "$(yellow "WARNING: Program \"healthcheck\" inside Task \"${task}\" was not running, should it?")" fi fi fi ;; - stop-all-healthchecks ) + stop-all-workloads ) local usage="USAGE: wb backend $op RUN-DIR" local dir=${1:?$usage}; shift + local workload=${1:?$usage}; shift local task=${1:?$usage}; shift - local task_dir="${dir}"/healthcheck/"${task}" + local task_dir="${dir}"/"${workload}"/"${task}" if test -f "${task_dir}"/started && !(test -f "${task_dir}"/stopped || test -f "${task_dir}"/quit) then - if backend_nomad is-task-program-running "${dir}" "${task}" healthcheck + if backend_nomad is-task-program-running "${dir}" "${task}" "${workload}" then - if ! backend_nomad task-program-stop "${dir}" "${task}" healthcheck + if ! backend_nomad task-program-stop "${dir}" "${task}" "${workload}" then - msg "$(yellow "WARNING: Program \"healthcheck\" inside Task \"${task}\" failed to stop")" + msg "$(yellow "WARNING: Program \"${workload}\" inside Task \"${task}\" failed to stop")" else touch "${task_dir}"/stopped - msg "$(green "supervisord program \"healthcheck\" inside Nomad Task \"${task}\" down!")" + msg "$(green "supervisord program \"${workload}\" inside Nomad Task \"${task}\" down!")" fi else touch "${task_dir}"/quit - if backend_nomad is-task-program-failed "${dir}" "${task}" healthcheck + if backend_nomad is-task-program-failed "${dir}" "${task}" "${workload}" then local generator_task=$(envjqr 'generator_task_name') # If the node quits (due to `--shutdown_on_slot_synced X` or @@ -898,12 +912,12 @@ backend_nomad() { if test "${generator_task}" != "${task}" || test "${generator_can_fail}" = "false" || backend_nomad is-task-program-running "${dir}" "${task}" "${task}" then # Do not fail here, because nobody will be able to stop the cluster! - msg "$(red "FATAL: \"healthcheck\" inside Task \"${task}\" quit unexpectedly")" + msg "$(red "FATAL: \"${workload}\" inside Task \"${task}\" quit unexpectedly")" else - msg "$(yellow "INFO: Program \"healthcheck\" inside Task \"${task}\" failed, but expected when \"${task}\" automatically exits first and makes \"generator\" fail")" + msg "$(yellow "INFO: Program \"${workload}\" inside Task \"${task}\" failed, but expected when \"${task}\" automatically exits first and makes \"generator\" fail")" fi else - msg "$(yellow "WARNING: Program \"healthcheck\" inside Task \"${task}\" was not running, should it?")" + msg "$(yellow "WARNING: Program \"${workload}\" inside Task \"${task}\" was not running, should it?")" fi fi fi @@ -1070,35 +1084,6 @@ backend_nomad() { msg "Fetch logs ..." - # Download latency(ies) logs. ############################################ - ########################################################################## - # Download retry "infinite" loop. - local latencies_array - # Fetch the nodes that don't have all the log files in its directory - latencies_array="$(jq_tolist 'keys' "$dir"/node-specs.json)" - while test -n "${latencies_array:-}" - do - local latencies_jobs_array=() - for node in ${latencies_array[*]} - do - backend_nomad download-logs-latency "${dir}" "${node}" & - latencies_jobs_array+=("$!") - done - if test -n "${latencies_jobs_array:-}" # If = () "unbound variable" error - then - # Wait until all jobs finish, don't use `wait_kill_em_all` that kills - # Returns the exit code of the last failed job, we ignore it! - wait_all "${latencies_jobs_array[@]}" || true - fi - # Fetch the nodes that don't have all the log files in its directory - latencies_array="$(backend_nomad fetch-logs-latencies "${dir}")" - if test -n "${latencies_array:-}" - then - msg "Retrying latency(ies) [${latencies_array[@]}] logs download" - read -p "Hit enter to continue ..." - fi - done - msg "$(green "Finished downloading latency(ies) logs")" # Download healthcheck(s) logs. ########################################## ########################################################################## # Download retry "infinite" loop. @@ -1128,6 +1113,40 @@ backend_nomad() { fi done msg "$(green "Finished downloading Healthcheck(s) logs")" + # Download workload(s) logs. ############################################# + ########################################################################## + # For every workload + for workload in $(jq_tolist '.workloads | map(.name)' "$dir"/profile.json) + do + # Download retry "infinite" loop. + local workloads_array + # Fetch the nodes that don't have all the log files in its directory + # Workload may or may not run something in all producers. + workloads_array=($(jq_tolist 'map(select(.isProducer) | .name)' "$dir"/node-specs.json)) + while test -n "${workloads_array:-}" + do + local workloads_jobs_array=() + for node in ${workloads_array[*]} + do + backend_nomad download-logs-workload "${dir}" "${workload}" "${node}" & + workloads_jobs_array+=("$!") + done + if test -n "${workloads_jobs_array:-}" # If = () "unbound variable" error + then + # Wait until all jobs finish, don't use `wait_kill_em_all` that kills + # Returns the exit code of the last failed job, we ignore it! + wait_all "${workloads_jobs_array[@]}" || true + fi + # Fetch the nodes that don't have all the log files in its directory + workloads_array="$(backend_nomad fetch-logs-workloads "${dir}" "${workload}")" + if test -n "${workloads_array:-}" + then + msg "Retrying workload \"${workload}\" [${workloads_array[@]}] logs download" + read -p "Hit enter to continue ..." + fi + done + done + msg "$(green "Finished downloading workload(s) logs")" # Download generator logs. ############################################### ########################################################################## # Download retry "infinite" loop. @@ -1256,12 +1275,12 @@ backend_nomad() { msg "$(green "Finished downloading entrypoint(s) logs")" # TODO: Check downloads - # ls run/current/nomad/{node-{0..51},explorer}/{stdout,stderr} || msg "" - # ls run/current/tracer/{node-{0..51},explorer}/{exit_code,stdout,stderr} || msg "" - # ls run/current/{node-{0..51},explorer}/{exit_code,stdout,stderr} || msg "" - # ls run/current/generator/{exit_code,stdout,stderr} || msg "" - # ls run/current/healthcheck/{node-{0..51},explorer}/{exit_code,stdout,stderr} || msg "" - # ls run/current/latency/{node-{0..51},explorer}/{exit_code,stdout,stderr} || msg "" + # ls run/current/nomad/{node-{0..51},explorer}/{stdout,stderr} || msg "" + # ls run/current/tracer/{node-{0..51},explorer}/{exit_code,stdout,stderr} || msg "" + # ls run/current/{node-{0..51},explorer}/{exit_code,stdout,stderr} || msg "" + # ls run/current/generator/{exit_code,stdout,stderr} || msg "" + # ls run/current/workloads/WORKLOAD-NAME/{node-{0..51}}/{exit_code,stdout,stderr} || msg "" + # ls run/current/healthcheck/{node-{0..51},explorer}/{exit_code,stdout,stderr} || msg "" msg "$(green "Finished fetching logs")" ;; @@ -1396,47 +1415,49 @@ backend_nomad() { ;; # Array of nodes that don't have all the required log files in its directory - fetch-logs-latencies ) - local usage="USAGE: wb backend $op RUN-DIR" + fetch-logs-workloads ) + local usage="USAGE: wb backend $op RUN-DIR WORKLOAD-NAME" local dir=${1:?$usage}; shift - local latencies_array=() - for node in $(jq_tolist 'keys' "${dir}"/node-specs.json) + local workload=${1:?$usage}; shift + local workloads_array=() + # Workload may or may not run something in all producers. + for node in $(jq_tolist 'map(select(.isProducer) | .name)' "$dir"/node-specs.json) do - # Only if the latency was started. - if test -f "${dir}"/latency/"${node}"/started + # Only if the workload was started. + if test -f "${dir}"/workloads/"${workload}"/"${node}"/started then - local latency_ok="true" + local workload_ok="true" # Check the existance of all the wanted files: - if ! test -f "${dir}"/latency/"${node}"/exit_code + if ! test -f "${dir}"/workloads/"${workload}"/"${node}"/exit_code then - latency_ok="false" + workload_ok="false" fi - if ! test -f "${dir}"/latency/"${node}"/stdout + if ! test -f "${dir}"/workloads/"${workload}"/"${node}"/stdout then - latency_ok="false" + workload_ok="false" fi - if ! test -f "${dir}"/latency/"${node}"/stderr + if ! test -f "${dir}"/workloads/"${workload}"/"${node}"/stderr then - latency_ok="false" + workload_ok="false" fi # Below like errors can end in truncated files, a proper flag is used! # failed to exec into task: read tcp 10.0.0.115:33840->3.72.231.105:443: read: connection reset by peer # tar: Unexpected EOF in archive # tar: Unexpected EOF in archive # tar: Error is not recoverable: exiting now - if test -f "${dir}"/latency/"${node}"/download_failed + if test -f "${dir}"/workloads/"${workload}"/"${node}"/download_failed then - latency_ok="false" + workload_ok="false" fi - # If any error add this latency to the array - if test "${latency_ok}" = "false" + # If any error add this workload to the array + if test "${workload_ok}" = "false" then - latencies_array+=("${node}") + workloads_array+=("${node}") fi fi done # Return array - echo "${latencies_array[@]}" + echo "${workloads_array[@]}" ;; # Array of nodes that don't have all the required log files in its directory @@ -1502,8 +1523,8 @@ backend_nomad() { # Functions to start/stop groups of cluster "programs": # - start-tracers RUN-DIR # - start-nodes RUN-DIR + # - start-workloads RUN-DIR # - start-healthchecks RUN-DIR - # - start-latencies RUN-DIR ############################################################################ # * Functions in the backend "interface" must use `fatal` when errors! @@ -1613,47 +1634,52 @@ backend_nomad() { ;; # Called by `scenario.sh` with the exit trap (`scenario_setup_exit_trap`) set! - start-healthchecks ) + start-workloads ) local usage="USAGE: wb backend $op RUN-DIR" local dir=${1:?$usage}; shift - local jobs_array=() - local nodes=($(jq_tolist keys "$dir"/node-specs.json)) - for node in ${nodes[*]} + # For every workload + for workload in $(jq_tolist '.workloads | map(.name)' "$dir"/profile.json) do - backend_nomad start-healthcheck "${dir}" "${node}" & - jobs_array+=("$!") - done - # Wait and check! - if test -n "${jobs_array}" - then - if ! wait_kill_em_all "${jobs_array[@]}" + local jobs_array=() + # Workload may or may not run something in all producers. + local nodes=($(jq_tolist 'map(select(.isProducer) | .name)' "$dir"/node-specs.json)) + for node in ${nodes[*]} + do + backend_nomad start-workload "${dir}" "${workload}" "${node}" & + jobs_array+=("$!") + done + # Wait and check! + if test -n "${jobs_array}" then - fatal "Failed to start healthcheck(s)" - return 1 - else - for node in ${nodes[*]} - do - if ! test -f "${dir}"/healthcheck/"${node}"/started - then - fatal "Healthcheck for \"${node}\" failed to start!" - fi - done + if ! wait_kill_em_all "${jobs_array[@]}" + then + fatal "Failed to start workload(s)" + return 1 + else + for node in ${nodes[*]} + do + if ! test -f "${dir}"/workloads/"${workload}"/"${node}"/started + then + fatal "Workload \"${workload}\" for \"${node}\" failed to start!" + fi + done + fi fi - fi + done return 0 ;; # Called by `scenario.sh` with the exit trap (`scenario_setup_exit_trap`) set! - start-latencies ) + start-healthchecks ) local usage="USAGE: wb backend $op RUN-DIR" local dir=${1:?$usage}; shift + local jobs_array=() - # explorer node is ignored, it will ping every other node. - local nodes=($(jq_tolist 'map(select(.isProducer) | .name)' "$dir"/node-specs.json)) + local nodes=($(jq_tolist keys "$dir"/node-specs.json)) for node in ${nodes[*]} do - backend_nomad start-latency "${dir}" "${node}" & + backend_nomad start-healthcheck "${dir}" "${node}" & jobs_array+=("$!") done # Wait and check! @@ -1661,14 +1687,14 @@ backend_nomad() { then if ! wait_kill_em_all "${jobs_array[@]}" then - fatal "Failed to start latency(ies)" + fatal "Failed to start healthcheck(s)" return 1 else for node in ${nodes[*]} do - if ! test -f "${dir}"/latency/"${node}"/started + if ! test -f "${dir}"/healthcheck/"${node}"/started then - fatal "Latency for \"${node}\" failed to start!" + fatal "Healthcheck for \"${node}\" failed to start!" fi done fi @@ -1681,17 +1707,17 @@ backend_nomad() { # - start-tracer RUN-DIR (Nomad backend specific subcommand) # - start-node RUN-DIR NODE-NAME # - start-generator RUN-DIR + # - start-workload RUN-DIR WORKLOAD-NAME TASK-NAME (Nomad backend .....) # - start-healthcheck RUN-DIR TASK-NAME (Nomad backend specific subcommand) - # - start-latency RUN-DIR TASK-NAME (Nomad backend specific subcommand) # - wait-tracer RUN-DIR TASK-NAME (Nomad backend specific subcommand) # - wait-node RUN-DIR NODE_NAME (Nomad backend specific subcommand) # - stop-node RUN-DIR NODE-NAME # # TODO: They are up here as "stop-cluster-*" # - stop-generator RUN-DIR TASK-NAME (Nomad backend specific subcommand) + # - stop-workload RUN-DIR WORKLOAD-NAME TASK-NAME (Nomad backend .....) # - stop-tracer RUN-DIR TASK-NAME (Nomad backend specific subcommand) # - stop-healthcheck RUN-DIR TASK-NAME (Nomad backend specific subcommand) - # - stop-latency RUN-DIR TASK-NAME (Nomad backend specific subcommand) ############################################################################ # * Functions in the backend "interface" must use `fatal` when errors! @@ -1950,18 +1976,19 @@ backend_nomad() { fi ;; - # Called by "start-healthchecks" that has no exit trap, don't use fatal here! - start-healthcheck ) # Nomad backend specific subcommands - local usage="USAGE: wb backend $op RUN-DIR TASK" + # Called by "start-workloads" that has no exit trap, don't use fatal here! + start-workload ) # Nomad backend specific subcommands + local usage="USAGE: wb backend $op RUN-DIR WORKLOAD-NAME TASK" local dir=${1:?$usage}; shift + local workload=${1:?$usage}; shift local task=${1:?$usage}; shift - if ! backend_nomad task-program-start "${dir}" "${task}" healthcheck + if ! backend_nomad task-program-start "${dir}" "${task}" "${workload}" then - msg "$(red "FATAL: Program \"healthcheck\" inside Nomad Task \"${task}\" startup failed")" + msg "$(red "FATAL: Program \"${workload}\" workload inside Nomad Task \"${task}\" startup failed")" # TODO: Let the download fail when everything fails? backend_nomad download-logs-entrypoint "${dir}" "${task}" || true - backend_nomad download-logs-healthcheck "${dir}" "${task}" || true + backend_nomad download-logs-workload "${dir}" "${workload}" "${task}" || true # Should show the output/log of `supervisord` (runs as "entrypoint"). msg "$(yellow "${dir}/nomad/${task}/stdout:")" cat \ @@ -1974,57 +2001,57 @@ backend_nomad() { "${dir}"/nomad/"${task}"/stderr \ <(echo "-------------------- log end --------------------") # Depending on when the start command failed, logs may not be available! - if test -f "${dir}"/healthcheck/"${task}"/stdout + if test -f "${dir}"/workloads/"${workload}"/"${task}"/stdout then - msg "$(yellow "${dir}/healthcheck/${task}/stdout:")" + msg "$(yellow "${dir}/workloads/"${workload}"/${task}/stdout:")" cat \ <(echo "-------------------- log start --------------------") \ - "${dir}"/healthcheck/"${task}"/stdout \ + "${dir}"/workloads/"${workload}"/"${task}"/stdout \ <(echo "-------------------- log end --------------------") fi # Depending on when the start command failed, logs may not be available! - if test -f "${dir}"/healthcheck/"${task}"/stderr + if test -f "${dir}"/workloads/"${workload}"/"${task}"/stderr then - msg "$(yellow "${dir}/healthcheck/${task}/stderr:")" + msg "$(yellow "${dir}/workloads/"${workload}"/${task}/stderr:")" cat \ <(echo "-------------------- log start --------------------") \ - "${dir}"/healthcheck/"${task}"/stderr \ + "${dir}"/workloads/"${workload}"/"${task}"/stderr \ <(echo "-------------------- log end --------------------") fi # Let "start" parse the response code and handle the cleanup! - msg "$(red "Failed to start program \"healthcheck\" inside Nomad Task \"${task}\"")" + msg "$(red "Failed to start program \"${workload}\" workload inside Nomad Task \"${task}\"")" return 1 else local nomad_environment=$(envjqr 'nomad_environment') if test "${nomad_environment}" != "cloud" then - ln -s \ - ../../nomad/alloc/"${task}"/local/run/current/healthcheck/stdout \ - "${dir}"/healthcheck/"${task}"/stdout - ln -s \ - ../../nomad/alloc/"${task}"/local/run/current/healthcheck/stderr \ - "${dir}"/healthcheck/"${task}"/stderr - ln -s \ - ../../nomad/alloc/"${task}"/local/run/current/healthcheck/exit_code \ - "${dir}"/healthcheck/"${task}"/exit_code + ln -s \ + ../../../nomad/alloc/"${task}"/local/run/current/workloads/"${workload}"/stdout \ + "${dir}"/workloads/"${workload}"/"${task}"/stdout + ln -s \ + ../../../nomad/alloc/"${task}"/local/run/current/workloads/"${workload}"/stderr \ + "${dir}"/workloads/"${workload}"/"${task}"/stderr + ln -s \ + ../../../nomad/alloc/"${task}"/local/run/current/workloads/"${workload}"/exit_code \ + "${dir}"/workloads/"${workload}"/"${task}"/exit_code fi # It was "intentionally started and should not automagically stop" flag! - touch "${dir}"/healthcheck/"${task}"/started + touch "${dir}"/workloads/"${workload}"/"${task}"/started fi ;; - # Called by "start-latencies" that has no exit trap, don't use fatal here! - start-latency ) # Nomad backend specific subcommands + # Called by "start-healthchecks" that has no exit trap, don't use fatal here! + start-healthcheck ) # Nomad backend specific subcommands local usage="USAGE: wb backend $op RUN-DIR TASK" local dir=${1:?$usage}; shift local task=${1:?$usage}; shift - if ! backend_nomad task-program-start "${dir}" "${task}" latency + if ! backend_nomad task-program-start "${dir}" "${task}" healthcheck then - msg "$(red "FATAL: Program \"latency\" inside Nomad Task \"${task}\" startup failed")" + msg "$(red "FATAL: Program \"healthcheck\" inside Nomad Task \"${task}\" startup failed")" # TODO: Let the download fail when everything fails? backend_nomad download-logs-entrypoint "${dir}" "${task}" || true - backend_nomad download-logs-latency "${dir}" "${task}" || true + backend_nomad download-logs-healthcheck "${dir}" "${task}" || true # Should show the output/log of `supervisord` (runs as "entrypoint"). msg "$(yellow "${dir}/nomad/${task}/stdout:")" cat \ @@ -2037,42 +2064,42 @@ backend_nomad() { "${dir}"/nomad/"${task}"/stderr \ <(echo "-------------------- log end --------------------") # Depending on when the start command failed, logs may not be available! - if test -f "${dir}"/latency/"${task}"/stdout + if test -f "${dir}"/healthcheck/"${task}"/stdout then - msg "$(yellow "${dir}/latency/${task}/stdout:")" + msg "$(yellow "${dir}/healthcheck/${task}/stdout:")" cat \ <(echo "-------------------- log start --------------------") \ - "${dir}"/latency/"${task}"/stdout \ + "${dir}"/healthcheck/"${task}"/stdout \ <(echo "-------------------- log end --------------------") fi # Depending on when the start command failed, logs may not be available! - if test -f "${dir}"/latency/"${task}"/stderr + if test -f "${dir}"/healthcheck/"${task}"/stderr then - msg "$(yellow "${dir}/latency/${task}/stderr:")" + msg "$(yellow "${dir}/healthcheck/${task}/stderr:")" cat \ <(echo "-------------------- log start --------------------") \ - "${dir}"/latency/"${task}"/stderr \ + "${dir}"/healthcheck/"${task}"/stderr \ <(echo "-------------------- log end --------------------") fi # Let "start" parse the response code and handle the cleanup! - msg "$(red "Failed to start program \"latency\" inside Nomad Task \"${task}\"")" + msg "$(red "Failed to start program \"healthcheck\" inside Nomad Task \"${task}\"")" return 1 else local nomad_environment=$(envjqr 'nomad_environment') if test "${nomad_environment}" != "cloud" then ln -s \ - ../../nomad/alloc/"${task}"/local/run/current/latency/stdout \ - "${dir}"/latency/"${task}"/stdout + ../../nomad/alloc/"${task}"/local/run/current/healthcheck/stdout \ + "${dir}"/healthcheck/"${task}"/stdout ln -s \ - ../../nomad/alloc/"${task}"/local/run/current/latency/stderr \ - "${dir}"/latency/"${task}"/stderr + ../../nomad/alloc/"${task}"/local/run/current/healthcheck/stderr \ + "${dir}"/healthcheck/"${task}"/stderr ln -s \ - ../../nomad/alloc/"${task}"/local/run/current/latency/exit_code \ - "${dir}"/latency/"${task}"/exit_code + ../../nomad/alloc/"${task}"/local/run/current/healthcheck/exit_code \ + "${dir}"/healthcheck/"${task}"/exit_code fi # It was "intentionally started and should not automagically stop" flag! - touch "${dir}"/latency/"${task}"/started + touch "${dir}"/healthcheck/"${task}"/started fi ;; @@ -2169,7 +2196,7 @@ backend_nomad() { # - get-node-socket-path RUN-DIR NODE-NAME (Will break when cloud running) # - wait-node-stopped RUN-DIR NODE-NAME # - wait-pools-stopped RUN-DIR - # - wait-latencies-stopped RUN-DIR + # - wait-workloads-stopped RUN-DIR # - cluster-exited-programs RUN-DIR (Nomad backend specific subcommand) ############################################################################ # * Functions in the backend "interface" must use `fatal` when errors! @@ -2320,41 +2347,44 @@ backend_nomad() { fi ;; - wait-latencies-stopped ) + wait-workloads-stopped ) local usage="USAGE: wb backend $op SLEEP-SECONDS RUN-DIR" # This parameters is added by the nomad backend being used. local sleep_seconds=${1:?$usage}; shift local dir=${1:?$usage}; shift local start_time=$(date +%s) - # explorer node is ignored, it will ping every other node. - local nodes=($(jq_tolist 'map(select(.isProducer) | .name)' "${dir}"/node-specs.json)) - msg_ne "nomad: $(blue Waiting) until all latency services are stopped: 000000" - for node in ${nodes[*]} + msg_ne "nomad: $(blue Waiting) until all workloads are stopped: 000000" + for workload in $(jq_tolist '.workloads | map(.name)' "$dir"/profile.json) do - while \ - ! test -f "${dir}"/flag/cluster-stopping \ - && \ - backend_nomad is-task-program-running "${dir}" "${node}" "latency" 5 > /dev/null + # Workload may or may not run something in all producers. + local nodes=($(jq_tolist 'map(select(.isProducer) | .name)' "${dir}"/node-specs.json)) + for node in ${nodes[*]} do + while \ + ! test -f "${dir}"/flag/cluster-stopping \ + && \ + backend_nomad is-task-program-running "${dir}" "${node}" "${workload}" 5 > /dev/null + do + local elapsed="$(($(date +%s) - start_time))" + echo -ne "\b\b\b\b\b\b" + printf "%6d" "${elapsed}" + # This time is different between local and cloud backends to avoid + # unnecesary Nomad specific traffic and at the same time be less + # sensitive to network failures. + sleep "${sleep_seconds}" + done # While + if ! test -f "${dir}"/flag/cluster-stopping + then + echo -ne "\n" + msg "$(yellow "supervisord program \"${workload}\" workload of \"${node}\" stopped")" + msg_ne "nomad: $(blue Waiting) until workload nodes are stopped: 000000" + fi local elapsed="$(($(date +%s) - start_time))" echo -ne "\b\b\b\b\b\b" printf "%6d" "${elapsed}" - # This time is different between local and cloud backends to avoid - # unnecesary Nomad specific traffic and at the same time be less - # sensitive to network failures. - sleep "${sleep_seconds}" - done # While - if ! test -f "${dir}"/flag/cluster-stopping - then - echo -ne "\n" - msg "$(yellow "supervisord program \"latency\" of \"${node}\" stopped")" - msg_ne "nomad: $(blue Waiting) until all pool nodes are stopped: 000000" - fi - local elapsed="$(($(date +%s) - start_time))" - echo -ne "\b\b\b\b\b\b" - printf "%6d" "${elapsed}" - done >&2 # For + done >&2 # For + done echo -ne "\b\b\b\b\b\b" local elapsed=$(($(date +%s) - start_time)) @@ -2365,7 +2395,7 @@ backend_nomad() { else touch "${dir}"/flag/cluster-stopping echo -ne "\n" - msg "All latency services exited -- after $(yellow ${elapsed})s" + msg "All workloads exited -- after $(yellow ${elapsed})s" fi ;; @@ -2412,7 +2442,7 @@ backend_nomad() { ;; # For debugging when something fails, downloads and prints details! - download-logs-latency ) + download-logs-healthcheck ) local usage="USAGE: wb backend pass $op RUN-DIR TASK-NAME" local dir=${1:?$usage}; shift local task=${1:?$usage}; shift @@ -2421,63 +2451,64 @@ backend_nomad() { local nomad_environment=$(envjqr 'nomad_environment') if test "${nomad_environment}" != "cloud" then - rm -f "${dir}"/latency/"${task}"/{stdout,stderr,exit_code} + rm -f "${dir}"/healthcheck/"${task}"/{stdout,stderr,exit_code} fi # Downloads "exit_code", "stdout", "stderr" and GHC files. # Depending on when the start command failed, logs may not be available! - backend_nomad download-zstd-latency "${dir}" "${task}" \ + backend_nomad download-zstd-healthcheck "${dir}" "${task}" \ || download_ok="false" # Return if test "${download_ok}" = "false" then - msg "$(red "Failed to download \"latency\" run files from \"${task}\"")" + msg "$(red "Failed to download \"healthcheck\" run files from \"${task}\"")" # Below like errors can end in truncated files, a proper flag is needed! # failed to exec into task: read tcp 10.0.0.115:33840->3.72.231.105:443: read: connection reset by peer # tar: Unexpected EOF in archive # tar: Unexpected EOF in archive # tar: Error is not recoverable: exiting now - touch "${dir}"/latency/"${task}"/download_failed + touch "${dir}"/healthcheck/"${task}"/download_failed return 1 else - if test -f "${dir}"/latency/"${task}"/download_failed + if test -f "${dir}"/healthcheck/"${task}"/download_failed then - rm "${dir}"/latency/"${task}"/download_failed + rm "${dir}"/healthcheck/"${task}"/download_failed fi return 0 fi ;; # For debugging when something fails, downloads and prints details! - download-logs-healthcheck ) - local usage="USAGE: wb backend pass $op RUN-DIR TASK-NAME" + download-logs-workload ) + local usage="USAGE: wb backend pass $op RUN-DIR WORKLOAD-NAME TASK-NAME" local dir=${1:?$usage}; shift + local workload=${1:?$usage}; shift local task=${1:?$usage}; shift local download_ok="true" # Remove "live" symlinks before downloading the "originals" local nomad_environment=$(envjqr 'nomad_environment') if test "${nomad_environment}" != "cloud" then - rm -f "${dir}"/healthcheck/"${task}"/{stdout,stderr,exit_code} + rm -f "${dir}"/workloads/"${workload}"/"${task}"/{stdout,stderr,exit_code} fi # Downloads "exit_code", "stdout", "stderr" and GHC files. # Depending on when the start command failed, logs may not be available! - backend_nomad download-zstd-healthcheck "${dir}" "${task}" \ + backend_nomad download-zstd-workload "${dir}" "${workload}" "${task}" \ || download_ok="false" # Return if test "${download_ok}" = "false" then - msg "$(red "Failed to download \"healthcheck\" run files from \"${task}\"")" + msg "$(red "Failed to download \"${workload}\" workload run files from \"${task}\"")" # Below like errors can end in truncated files, a proper flag is needed! # failed to exec into task: read tcp 10.0.0.115:33840->3.72.231.105:443: read: connection reset by peer # tar: Unexpected EOF in archive # tar: Unexpected EOF in archive # tar: Error is not recoverable: exiting now - touch "${dir}"/healthcheck/"${task}"/download_failed + touch "${dir}"/workloads/"${workload}"/"${task}"/download_failed return 1 else - if test -f "${dir}"/healthcheck/"${task}"/download_failed + if test -f "${dir}"/workloads/"${workload}"/"${task}"/download_failed then - rm "${dir}"/healthcheck/"${task}"/download_failed + rm "${dir}"/workloads/"${workload}"/"${task}"/download_failed fi return 0 fi @@ -2662,31 +2693,32 @@ backend_nomad() { fi ;; - download-zstd-latency ) + download-zstd-healthcheck ) local usage="USAGE: wb backend pass $op RUN-DIR TASK-NAME" local dir=${1:?$usage}; shift local task=${1:?$usage}; shift - msg "$(blue Fetching) $(yellow "\"latency\"") run files from Nomad $(yellow "Task \"${task}\"") ..." + msg "$(blue Fetching) $(yellow "\"healthcheck\"") run files from Nomad $(yellow "Task \"${task}\"") ..." # TODO: Add compression, either "--zstd" or "--xz" backend_nomad task-exec-program-run-files-tar-zstd \ - "${dir}" "${task}" "latency" \ + "${dir}" "${task}" "healthcheck" \ | tar --extract \ - --directory="${dir}"/latency/"${task}"/ --file=- \ + --directory="${dir}"/healthcheck/"${task}"/ --file=- \ --no-same-owner --no-same-permissions ;; - download-zstd-healthcheck ) - local usage="USAGE: wb backend pass $op RUN-DIR TASK-NAME" + download-zstd-workload ) + local usage="USAGE: wb backend pass $op RUN-DIR WORKLOAD-NAME TASK-NAME" local dir=${1:?$usage}; shift + local workload=${1:?$usage}; shift local task=${1:?$usage}; shift - msg "$(blue Fetching) $(yellow "\"healthcheck\"") run files from Nomad $(yellow "Task \"${task}\"") ..." + msg "$(blue Fetching) $(yellow "\"${workload}\" workload") run files from Nomad $(yellow "Task \"${task}\"") ..." # TODO: Add compression, either "--zstd" or "--xz" - backend_nomad task-exec-program-run-files-tar-zstd \ - "${dir}" "${task}" "healthcheck" \ - | tar --extract \ - --directory="${dir}"/healthcheck/"${task}"/ --file=- \ + backend_nomad task-exec-program-run-files-tar-zstd \ + "${dir}" "${task}" "workloads/${workload}" \ + | tar --extract \ + --directory="${dir}"/workloads/"${workload}"/"${task}"/ --file=- \ --no-same-owner --no-same-permissions ;; @@ -2814,6 +2846,16 @@ backend_nomad() { > "${dir}"/nomad/"${task}"/networking.json ;; + download-config-workload ) + local usage="USAGE: wb backend pass $op RUN-DIR NODE-NAME" + local dir=${1:?$usage}; shift + local workload=${1:?$usage}; shift + local node=${1:?$usage}; shift + backend_nomad task-file-contents "${dir}" "${node}" \ + run/current/workloads/"${workload}"/start.sh \ + > "${dir}"/workloads/"${workload}"/"${node}"/start.sh + ;; + download-config-generator ) local usage="USAGE: wb backend pass $op RUN-DIR" local dir=${1:?$usage}; shift @@ -2881,15 +2923,6 @@ backend_nomad() { > "${dir}"/healthcheck/"${node}"/start.sh ;; - download-config-latency ) - local usage="USAGE: wb backend pass $op RUN-DIR NODE-NAME" - local dir=${1:?$usage}; shift - local node=${1:?$usage}; shift - backend_nomad task-file-contents "${dir}" "${node}" \ - run/current/latency/start.sh \ - > "${dir}"/latency/"${node}"/start.sh - ;; - ## Nomad Job's Tasks supervisord queries ######################################## diff --git a/nix/workbench/backend/nomad/cloud.sh b/nix/workbench/backend/nomad/cloud.sh index 92e60244ea2..21c4efb5bfa 100644 --- a/nix/workbench/backend/nomad/cloud.sh +++ b/nix/workbench/backend/nomad/cloud.sh @@ -81,13 +81,13 @@ backend_nomadcloud() { backend_nomad wait-pools-stopped 60 "$@" ;; - wait-latencies-stopped ) + wait-workloads-stopped ) # It passes the sleep time (in seconds) required argument. # This time is different between local and cloud backends to avoid # unnecesary Nomad specific traffic (~99% happens waiting for node-0, the # first one it waits to stop inside a loop) and at the same time be less # sensitive to network failures. - backend_nomad wait-latencies-stopped 60 "$@" + backend_nomad wait-workloads-stopped 60 "$@" ;; fetch-logs ) @@ -146,12 +146,12 @@ backend_nomadcloud() { backend_nomad start-generator "$@" ;; - start-healthchecks ) - backend_nomad start-healthchecks "$@" + start-workloads ) + backend_nomad start-workloads "$@" ;; - start-latencies ) - backend_nomad start-latencies "$@" + start-healthchecks ) + backend_nomad start-healthchecks "$@" ;; start-node ) @@ -998,18 +998,6 @@ fetch-logs-ssh-node() { local ssh_config_path ssh_command ssh_config_path="$(wb nomad ssh config)" ssh_command="ssh -F ${ssh_config_path} -p 32000 -l nobody" - # Download latency(ies) logs. ################################################ - ############################################################################## - msg "$(blue "Fetching") $(yellow "program \"latency\"") run files from $(yellow "\"${node}\" (\"${public_ipv4}\")") ..." - if ! rsync -e "${ssh_command}" -au \ - -f'- start.sh' \ - "${public_ipv4}":/local/run/current/latency/ \ - "${dir}"/latency/"${node}"/ - then - node_ok="false" - touch "${dir}"/nomad/"${node}"/download_failed - msg "$(red Error fetching) $(yellow "program \"latency\"") $(red "run files from") $(yellow "\"${node}\" (\"${public_ipv4}\")") ..." - fi # Download healthcheck(s) logs. ############################################## ############################################################################## msg "$(blue "Fetching") $(yellow "program \"healthcheck\"") run files from $(yellow "\"${node}\" (\"${public_ipv4}\")") ..." @@ -1022,6 +1010,22 @@ fetch-logs-ssh-node() { touch "${dir}"/nomad/"${node}"/download_failed msg "$(red Error fetching) $(yellow "program \"healthcheck\"") $(red "run files from") $(yellow "\"${node}\" (\"${public_ipv4}\")") ..." fi + # Download workload(s) logs. ################################################# + ############################################################################## + # For every workload + for workload in $(jq_tolist '.workloads | map(.name)' "$dir"/profile.json) + do + msg "$(blue "Fetching") $(yellow "program \"${workload}\" workload") run files from $(yellow "\"${node}\" (\"${public_ipv4}\")") ..." + if ! rsync -e "${ssh_command}" -au \ + -f'- start.sh' \ + "${public_ipv4}":/local/run/current/workloads/"${workload}"/ \ + "${dir}"/workloads/"${workload}"/"${node}"/ + then + node_ok="false" + touch "${dir}"/nomad/"${node}"/download_failed + msg "$(red Error fetching) $(yellow "program \"${workload}\" workload") $(red "run files from") $(yellow "\"${node}\" (\"${public_ipv4}\")") ..." + fi + done # Download generator logs. ################################################### ############################################################################## if test "${node}" = "explorer" diff --git a/nix/workbench/backend/nomad/exec.sh b/nix/workbench/backend/nomad/exec.sh index 38a21875f9e..5c4f61a4364 100644 --- a/nix/workbench/backend/nomad/exec.sh +++ b/nix/workbench/backend/nomad/exec.sh @@ -61,13 +61,13 @@ backend_nomadexec() { backend_nomad wait-pools-stopped 1 "$@" ;; - wait-latencies-stopped ) + wait-workloads-stopped ) # It passes the sleep time (in seconds) required argument. # This time is different between local and cloud backends to avoid # unnecesary Nomad specific traffic (~99% happens waiting for node-0, the # first one it waits to stop inside a loop) and at the same time be less # sensitive to network failures. - backend_nomad wait-latencies-stopped 1 "$@" + backend_nomad wait-workloads-stopped 1 "$@" ;; # All or clean up everything! @@ -107,12 +107,12 @@ backend_nomadexec() { backend_nomad start-generator "$@" ;; - start-healthchecks ) - backend_nomad start-healthchecks "$@" + start-workloads ) + backend_nomad start-workloads "$@" ;; - start-latencies ) - backend_nomad start-latencies "$@" + start-healthchecks ) + backend_nomad start-healthchecks "$@" ;; start-node ) diff --git a/nix/workbench/backend/supervisor-conf.nix b/nix/workbench/backend/supervisor-conf.nix index 052362e35e7..ccf2b3d4ec4 100644 --- a/nix/workbench/backend/supervisor-conf.nix +++ b/nix/workbench/backend/supervisor-conf.nix @@ -184,14 +184,18 @@ let startsecs = 5; }; } + + + // - { - "program:latency" = { + (builtins.listToAttrs (builtins.map (workload: { + name = "program:${workload.name}"; + value = { # "command" below assumes "directory" is set accordingly. - directory = "${stateDir}/latency"; + directory = "${stateDir}/workloads/${workload.name}"; command = "${command}"; - stdout_logfile = "${stateDir}/latency/stdout"; - stderr_logfile = "${stateDir}/latency/stderr"; + stdout_logfile = "${stateDir}/workloads/${workload.name}/stdout"; + stderr_logfile = "${stateDir}/workloads/${workload.name}/stderr"; # Set these values to 0 to indicate an unlimited log size / no rotation. stdout_logfile_maxbytes = 0; stderr_logfile_maxbytes = 0; @@ -204,7 +208,10 @@ let # Seconds it needs to stay running to consider the start successful startsecs = 5; }; - } + }) profileData.workloads)) + + + // lib.attrsets.optionalAttrs withSsh { diff --git a/nix/workbench/backend/supervisor.sh b/nix/workbench/backend/supervisor.sh index c9e08301140..43a4dfd9339 100755 --- a/nix/workbench/backend/supervisor.sh +++ b/nix/workbench/backend/supervisor.sh @@ -58,6 +58,7 @@ case "$op" in local svcs=$dir/profile/node-services.json local gtor=$dir/profile/generator-service.json + local work=$dir/profile/workloads-service.json local trac=$dir/profile/tracer-service.json local hche=$dir/profile/healthcheck-service.json @@ -76,6 +77,15 @@ case "$op" in cp $(jq '."plutus-redeemer"' -r $gtor) "$gen_dir"/plutus-redeemer.json cp $(jq '."plutus-datum"' -r $gtor) "$gen_dir"/plutus-datum.json + local work_dir="$dir"/workloads + mkdir -p "$work_dir" + for workload in $(jq_tolist 'map(.name)' "$work") + do + mkdir -p "$work_dir"/"${workload}" + cp $(jq "map(select(.name == \"${workload}\"))[0] | .start" -r $work) \ + "$work_dir"/"${workload}"/start.sh + done + local trac_dir="$dir"/tracer mkdir -p "$trac_dir" cp $(jq '."start"' -r $trac) "$trac_dir"/start.sh @@ -84,8 +94,6 @@ case "$op" in local hche_dir="$dir"/healthcheck mkdir -p "$hche_dir" cp $(jq '."start"' -r $hche) "$hche_dir"/start.sh - - mkdir -p "$dir"/latency ;; deploy-genesis ) @@ -274,6 +282,30 @@ EOF fi backend_supervisor save-child-pids "$dir";; + start-workloads ) + local usage="USAGE: wb backend $op RUN-DIR" + local dir=${1:?$usage}; shift + + while test $# -gt 0 + do case "$1" in + --* ) msg "FATAL: unknown flag '$1'"; usage_supervisor;; + * ) break;; esac; shift; done + + # For every workload + for workload in $(jq_tolist '.workloads | map(.name)' "$dir"/profile.json) + do + if ! supervisorctl start "${workload}" + then progress "supervisor" "$(red fatal: failed to start) $(white "${workload} workload")" + echo "$(red "${workload}" workload stdout) ----------------------" >&2 + cat "$dir"/workloads/"${workload}"/stdout + echo "$(red "${workload}" workload stderr) ----------------------" >&2 + cat "$dir"/workloads/"${workload}"/stderr + echo "$(white -------------------------------------------------)" >&2 + fatal "could not start $(white "${workload} workload")" + fi + done + backend_supervisor save-child-pids "$dir";; + wait-node-stopped ) local usage="USAGE: wb backend $op RUN-DIR NODE" local dir=${1:?$usage}; shift @@ -322,6 +354,40 @@ EOF fi ;; + wait-workloads-stopped ) + local usage="USAGE: wb backend $op RUN-DIR" + local dir=${1:?$usage}; shift + + local start_time=$(date +%s) + msg_ne "supervisor: waiting until all workloads are stopped: 000000" + for workload in $(jq_tolist '.workloads | map(.name)' "$dir"/profile.json) + do + while \ + ! test -f "${dir}"/flag/cluster-stopping \ + && \ + supervisorctl status "${workload}" > /dev/null + do + echo -ne "\b\b\b\b\b\b" + printf "%6d" "$(($(date +%s) - start_time))" + sleep 1 + done + if ! test -f "${dir}"/flag/cluster-stopping + then + echo -ne "\b\b\b\b\b\b" + echo -n "${workload} 000000" + fi + done >&2 + echo -ne "\b\b\b\b\b\b" + local elapsed=$(($(date +%s) - start_time)) + if test -f "${dir}"/flag/cluster-stopping + then + echo " Termination requested -- after $(yellow ${elapsed})s" >&2 + else + touch "${dir}"/flag/cluster-stopping + echo " All workloads exited -- after $(yellow ${elapsed})s" >&2 + fi + ;; + stop-all ) local usage="USAGE: wb backend $op RUN-DIR" local dir=${1:?$usage}; shift diff --git a/nix/workbench/genesis/genesis.sh b/nix/workbench/genesis/genesis.sh index 2c14b1f97ee..bdcfc30e15b 100644 --- a/nix/workbench/genesis/genesis.sh +++ b/nix/workbench/genesis/genesis.sh @@ -735,7 +735,7 @@ genesis-create-testnet-data() { link_keys utxo-keys utxo-keys local is_voting - is_voting=$(jq --raw-output '.generator.drep_voting' "$profile_json") + is_voting=$(jq --raw-output '.workloads | any( .name == "voting")' "$profile_json") if [[ "$is_voting" == "true" ]]; then info genesis "voting workload specified - keeping one stake key per producer" diff --git a/nix/workbench/profile/prof0-defaults.jq b/nix/workbench/profile/prof0-defaults.jq index 821ec6a57ca..b9e5e6e29fd 100644 --- a/nix/workbench/profile/prof0-defaults.jq +++ b/nix/workbench/profile/prof0-defaults.jq @@ -63,6 +63,8 @@ def era_defaults($era): } } + , workloads: [] + , node: { rts_flags_override: [] , heap_limit: null ## optional: heap limit in MB (translates to RTS flag -M) diff --git a/nix/workbench/profile/prof1-variants.jq b/nix/workbench/profile/prof1-variants.jq index 40940bdaaa8..8f43273eb09 100644 --- a/nix/workbench/profile/prof1-variants.jq +++ b/nix/workbench/profile/prof1-variants.jq @@ -392,7 +392,7 @@ def all_profile_variants: | .node.shutdown_on_slot_synced = 1200 ) as $for_1200slot ## - ### Definition vocabulary: workload + ### Definition vocabulary: generator workload ## | ({}| .generator.tps = 15 @@ -431,7 +431,6 @@ def all_profile_variants: , generator: { inputs_per_tx: 1 , outputs_per_tx: 1 - , drep_voting: true } }) as $voting_base | @@ -545,6 +544,27 @@ def all_profile_variants: | .generator.tx_fee = 940000 ) as $plutus_loop_ripemd ## + ### Definition vocabulary: custom workloads + ## + | + ({ name: "latency" + , parameters: {} + , entrypoints: { + pre_generator: null + , producers: "latency" + } + , wait_pools: false + }) as $latency_workload + | + ({ name: "voting" + , parameters: {} + , entrypoints: { + pre_generator: "workflow_generator" + , producers: "workflow_producer" + } + , wait_pools: true + }) as $voting_workload + ## ### Definition vocabulary: genesis variants ## | @@ -716,9 +736,6 @@ def all_profile_variants: | ({ scenario: "tracer-only" }) as $scenario_tracer_only - | - ({ scenario: "latency" - }) as $scenario_latency | ## ### Definition vocabulary: base variant @@ -830,12 +847,14 @@ def all_profile_variants: } }) as $nomad_perf_plutussecp_base | - ($scenario_latency * $compose_fiftytwo * $dataset_empty * $no_filtering * + ($compose_fiftytwo * $dataset_empty * $no_filtering * { desc: "AWS perf class cluster, stop when all latency services stop" + , workloads: [ $latency_workload ] }) as $nomad_perf_latency_base | - ($scenario_latency * $compose_fiftytwo * $dataset_empty * $no_filtering * + ($compose_fiftytwo * $dataset_empty * $no_filtering * { desc: "AWS perf-ssd class cluster, stop when all latency services stop" + , workloads: [ $latency_workload ] }) as $nomad_perfssd_latency_base | ($scenario_nomad_perfssd_solo * $solo * $dataset_24m * @@ -1325,68 +1344,68 @@ def all_profile_variants: # Split creating 500k UTxO, create the transactions (build-raw) but no submit. , $valuevoting_nomadperf_template * $dreps_large * { name: "value-voting-utxo-volt-nomadperf" - , generator: {drep_voting: true} - , workload: [ - { outs_per_split_transaction: 193 - , submit_vote: false - } - ] + , workloads: + [ $voting_workload * {parameters: + { outs_per_split_transaction: 193 + , submit_vote: false + } + }] } # One vote per voting tx version. , $valuevoting_nomadperf_template * $dreps_large * { name: "value-voting-volt-nomadperf" - , generator: {drep_voting: true} - , workload: [ - { outs_per_split_transaction: 193 - , submit_vote: true - , votes_per_tx: 1 - } - ] + , workloads: + [ $voting_workload * {parameters: + { outs_per_split_transaction: 193 + , submit_vote: true + , votes_per_tx: 1 + } + }] } # Two votes per voting tx version. , $valuevoting_nomadperf_template * $dreps_large * { name: "value-voting-double-volt-nomadperf" - , generator: {drep_voting: true} - , workload: [ - { outs_per_split_transaction: 193 - , submit_vote: true - , votes_per_tx: 2 - } - ] + , workloads: + [ $voting_workload * {parameters: + { outs_per_split_transaction: 193 + , submit_vote: true + , votes_per_tx: 2 + } + }] } ## As "plutus" above with an extra voting workload # Split creating 500k UTxO, create the transactions (build-raw) but no submit. , $plutusvoting_nomadperf_template * $dreps_large * { name: "plutus-voting-utxo-volt-nomadperf" - , generator: {drep_voting: true} - , workload: [ - { outs_per_split_transaction: 193 - , submit_vote: false - } - ] + , workloads: + [ $voting_workload * {parameters: + { outs_per_split_transaction: 193 + , submit_vote: false + } + }] } # One vote per voting tx version. , $plutusvoting_nomadperf_template * $dreps_large * { name: "plutus-voting-volt-nomadperf" - , generator: {drep_voting: true} - , workload: [ - { outs_per_split_transaction: 193 - , submit_vote: true - , votes_per_tx: 1 - } - ] + , workloads: + [ $voting_workload * {parameters: + { outs_per_split_transaction: 193 + , submit_vote: true + , votes_per_tx: 1 + } + }] } # Two votes per voting tx version. , $plutusvoting_nomadperf_template * $dreps_large * { name: "plutus-voting-double-volt-nomadperf" - , generator: {drep_voting: true} - , workload: [ - { outs_per_split_transaction: 193 - , submit_vote: true - , votes_per_tx: 2 - } - ] + , workloads: + [ $voting_workload * {parameters: + { outs_per_split_transaction: 193 + , submit_vote: true + , votes_per_tx: 2 + } + }] } ## P&T Nomad cluster: 52 nodes, PlutusV3 BLST and Plutus SECP workloads @@ -1626,9 +1645,13 @@ def all_profile_variants: ## development profile for voting workload: PV9, Conway costmodel, 1000 DReps injected , $cibench_base * $voting_base * $double_plus_tps_saturation_plutus * $genesis_voting * $dreps_small * { name: "development-voting" - , workload: [ - {votes_per_tx: 2} - ] + , workloads: + [ $voting_workload * {parameters: + { outs_per_split_transaction: 193 + , submit_vote: true + , votes_per_tx: 2 + } + }] } ## Last, but not least, the profile used by "nix-shell -A devops": diff --git a/nix/workbench/profile/profile.nix b/nix/workbench/profile/profile.nix index b23795282ea..839194d5024 100644 --- a/nix/workbench/profile/profile.nix +++ b/nix/workbench/profile/profile.nix @@ -54,6 +54,19 @@ let }) generator-service; + workloads-service = builtins.map (workload: rec { + name = workload.name; + start = rec { + value = '' + ${import ../workload/${name}.nix + {inherit pkgs profile nodeSpecs workload;} + } + ${workload.entrypoints.producers} + ''; + JSON = pkgs.writeScript "startup-${name}.sh" value; + }; + }) profile.workloads; + inherit (pkgs.callPackage ../service/tracer.nix @@ -70,12 +83,6 @@ let inherit backend profile nodeSpecs; }) healthcheck-service; - - inherit - (pkgs.callPackage - ../service/latency.nix - {}) - latency-service; }; materialise-profile = @@ -95,9 +102,9 @@ let }) node-services generator-service + workloads-service tracer-service healthcheck-service - latency-service ; in pkgs.runCommand "workbench-profile-${profileName}" @@ -125,6 +132,11 @@ let plutus-redeemer = plutus-redeemer.JSON; plutus-datum = plutus-datum.JSON; }; + workloadsService = __toJSON (builtins.map (workload: { + name = workload.name; + start = workload.start.JSON; + } + ) workloads-service); tracerService = with tracer-service; __toJSON @@ -138,19 +150,13 @@ let { name = "healthcheck"; start = start.JSON; }; - latencyService = - with healthcheck-service; - __toJSON - { name = "latency"; - start = start.JSON; - }; passAsFile = [ "nodeServices" "generatorService" + "workloadsService" "tracerService" "healthcheckService" - "latencyService" "topologyJson" "topologyDot" ]; @@ -163,9 +169,9 @@ let cp $topologyDotPath $out/topology.dot cp $nodeServicesPath $out/node-services.json cp $generatorServicePath $out/generator-service.json + cp $workloadsServicePath $out/workloads-service.json cp $tracerServicePath $out/tracer-service.json cp $healthcheckServicePath $out/healthcheck-service.json - cp $latencyServicePath $out/latency-service.json '' // ( @@ -180,7 +186,7 @@ let value = (__fromJSON (__readFile "${topologyFiles}/topology.json")); }; node-specs = {JSON = nodeSpecsJson; value = nodeSpecs;}; - inherit node-services generator-service tracer-service healthcheck-service latency-service; + inherit node-services generator-service tracer-service healthcheck-service workloads-service; } ) ; diff --git a/nix/workbench/scenario.sh b/nix/workbench/scenario.sh index 6ea13f2b992..3076da2c30e 100644 --- a/nix/workbench/scenario.sh +++ b/nix/workbench/scenario.sh @@ -71,28 +71,18 @@ case "$op" in ############ backend start-nodes "$dir" backend start-generator "$dir" + backend start-workloads "$dir" backend start-healthchecks "$dir" - scenario_setup_workload_termination "$dir" - # Trap end - ########## - - backend wait-pools-stopped "$dir" - scenario_cleanup_termination - - backend stop-all "$dir" - ;; - - latency ) - - scenario_setup_exit_trap "$dir" - # Trap start - ############ - backend start-nodes "$dir" - backend start-latencies "$dir" + if jqtest '.workloads == []' "$dir"/profile.json \ + || jqtest '.workloads | any(.wait_pools)' "$dir"/profile.json + then + scenario_setup_workload_termination "$dir" + backend wait-pools-stopped "$dir" + else + backend wait-workloads-stopped "$dir" + fi # Trap end ########## - - backend wait-latencies-stopped "$dir" scenario_cleanup_termination backend stop-all "$dir" diff --git a/nix/workbench/service/generator.nix b/nix/workbench/service/generator.nix index 70bd6700b18..bdbf52bbc90 100644 --- a/nix/workbench/service/generator.nix +++ b/nix/workbench/service/generator.nix @@ -129,23 +129,42 @@ let value = '' #!${pkgs.stdenv.shell} - ############################### VOTING ############################### - ############################### VOTING ############################### - ############################### VOTING ############################### - ${if profile.generator.drep_voting or false - then - '' - ${import ./voting.nix {inherit pkgs profile nodeSpecs;}} - workflow_generator \ - ${if profile.composition.with_explorer then "explorer" else "node-0"} - '' - else - '' - '' + ########################################### + # Extra workloads start ################### + ########################################### + ${builtins.concatStringsSep "" (builtins.map (workload: + let workload_name = workload.name; + entrypoint = workload.entrypoints.pre_generator; + node_name = if profile.composition.with_explorer + then "explorer" + else "node-0" + ; + in + '' + ########################################### + ########## workload start: ${workload_name} + ########################################### + ${if entrypoint != null + then + '' + ${import ../workload/${workload_name}.nix + {inherit pkgs profile nodeSpecs workload;} + } + (cd ../workloads/${workload_name} && ${entrypoint} ${node_name}) + '' + else + '' + '' + } + ########################################### + ########## workload end: ${workload_name} + ########################################### + '' + ) (profile.workloads or [])) } - ############################### VOTING ############################### - ############################### VOTING ############################### - ############################### VOTING ############################### + ############################################# + # Extra workloads end ####################### + ############################################# ${service.script} ''; diff --git a/nix/workbench/service/healthcheck.nix b/nix/workbench/service/healthcheck.nix index 6963dd0a933..5cb09c8f9bd 100644 --- a/nix/workbench/service/healthcheck.nix +++ b/nix/workbench/service/healthcheck.nix @@ -124,9 +124,6 @@ let for node in ''${nodes[*]} do - # TODO: A couple of simple pings - # latency_topology_producers "''${node}" - # Cardano cluster connectivity (cardano-ping) connectivity_topology_producers "''${node}" @@ -193,26 +190,9 @@ let done else - # Producers only!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ############################# VOTING ############################# - ############################# VOTING ############################# - ############################# VOTING ############################# - # If running supervisord (local only), only one healthcheck is run - # for all nodes, so sending to background and sleeping forever all - # but the last node on the list fits all backends for all - # producers to vote simultaneously. - for node in ''${nodes[*]} # nodes array is only deployed nodes! - do - if test "''${node}" = "''${nodes[-1]}" - then - workflow_producer "''${node}" - else - workflow_producer "''${node}" & - fi - done - ############################# VOTING ############################# - ############################# VOTING ############################# - ############################# VOTING ############################# + # Seconds supervisor needs to consider the start successful + ${coreutils}/bin/sleep 5 + msg "Done, bye!" fi } @@ -221,8 +201,6 @@ let # Network functions ################################################## ###################################################################### - # TODO: latency_topology_producers "''${node}" - function connectivity_topology_producers() { local node=$1 msg "Connectivity using 'cardano-cli ping' of \"''${node}\"'s Producers" @@ -954,20 +932,6 @@ let exit 22 } - ###################################################################### - # Conway/governance functions! ####################################### - ###################################################################### - - ${if profile.generator.drep_voting or false - then - '' - ${import ./voting.nix {inherit pkgs profile nodeSpecs;}} - '' - else - '' - '' - } - if test -n "''${NOMAD_DEBUG:-}" then DEBUG_FILE="$(${coreutils}/bin/dirname "$(${coreutils}/bin/readlink -f "$0")")"/"$0".debug diff --git a/nix/workbench/service/latency.nix b/nix/workbench/service/latency.nix deleted file mode 100644 index c00d0256300..00000000000 --- a/nix/workbench/service/latency.nix +++ /dev/null @@ -1,144 +0,0 @@ -{ pkgs }: - -with pkgs.lib; - -let - latency-service = - (nodeSpecs: - let - bashInteractive = pkgs.bashInteractive; - coreutils = pkgs.coreutils; - iputils = pkgs.iputils; - jq = pkgs.jq; - in { - start = rec { - # Assumptions: - # - Command `date` and node's log use the same timezone! - value = '' - #!${bashInteractive}/bin/sh - - ###################################################################### - # Set script globals ################################################# - ###################################################################### - - # Strict runtime - ################ - - # e: Immediately exit if any command has a non-zero exit status - # u: Reference to non previously defined variables is an error - # pipefail: Any failed command in a pipeline is used as return code - set -euo pipefail - - # Fetch all defined node names (Including "explorer" nodes) - ########################################################### - - node_specs_nodes=$(${jq}/bin/jq --raw-output \ - "keys | join (\" \")" \ - ../node-specs.json \ - ) - node_specs_pools=$(${jq}/bin/jq \ - 'map(select(.kind == "pool")) | length' \ - ../node-specs.json \ - ) - ${coreutils}/bin/echo "node-specs.json:" - ${coreutils}/bin/echo "- Nodes: [''${node_specs_nodes[*]}]" - ${coreutils}/bin/echo "- Pools: ''${node_specs_pools}" - - # Look for locally deployed nodes and allocate latency - ###################################################### - - nodes=() - started_time=$(${coreutils}/bin/date +%s) - for node in ''${node_specs_nodes[*]} - do - if test -d "../''${node}" - then - nodes+=("''${node}") - # Create latency directory inside available node directories - ${coreutils}/bin/mkdir "../''${node}/latency" - # Save the starting time - ${coreutils}/bin/echo "''${started_time}" > "../''${node}/latency/start_time" - fi - done - ${coreutils}/bin/echo "Found deployed nodes:" - ${coreutils}/bin/echo "- Nodes: [''${nodes[*]}]" - - ###################################################################### - # Main ############################################################### - ###################################################################### - - # The main function, called at the end of the file/script. - function latency() { - - msg "Started!" - - for node in ''${nodes[*]} - do - - msg "Latency of \"''${node}\"'s Producers using 'ping'" - local topology_path="../''${node}/topology.json" - # Merge non-P2P and P2P in the same {addr:"ADDR",port:0} format. - local producers - producers=$(${jq}/bin/jq '.Producers//[] + ((.localRoots[0].accessPoints//[]) | map({addr:.address,port:.port}))' "''${topology_path}") - local keys - keys=$(echo "''${producers}" | ${jq}/bin/jq --raw-output 'keys | join (" ")') - for key in ''${keys[*]} - do - - local host - host=$(echo "''${producers}" | ${jq}/bin/jq --raw-output ".[''${key}].addr") - local port - port=$(echo "''${producers}" | ${jq}/bin/jq --raw-output ".[''${key}].port") - - # If the ping fails the whole script must fail! - - msg "Executing 'ping' to \"''${host}:''${port}\"" - ${coreutils}/bin/sleep 60 - ${iputils}/bin/ping -D -c 60 -i 1 -n -O "''${host}" - ${coreutils}/bin/sleep 60 - ${iputils}/bin/ping -D -c 60 -i 1 -n -O "''${host}" - ${coreutils}/bin/sleep 60 - ${iputils}/bin/ping -D -c 60 -i 1 -n -O "''${host}" - ${coreutils}/bin/sleep 60 - ${iputils}/bin/ping -D -c 60 -i 1 -n -O "''${host}" - - msg "Executing 'ping' to \"''${host}:''${port}\" with size 16" - ${coreutils}/bin/sleep 60 - ${iputils}/bin/ping -D -c 60 -i 1 -n -O -s 16 "''${host}" - ${coreutils}/bin/sleep 60 - ${iputils}/bin/ping -D -c 60 -i 1 -n -O -s 16 "''${host}" - ${coreutils}/bin/sleep 60 - ${iputils}/bin/ping -D -c 60 -i 1 -n -O -s 16 "''${host}" - ${coreutils}/bin/sleep 60 - ${iputils}/bin/ping -D -c 60 -i 1 -n -O -s 16 "''${host}" - - msg "Executing 'ping' to \"''${host}:''${port}\" with size 65507" - ${coreutils}/bin/sleep 60 - ${iputils}/bin/ping -D -c 60 -i 1 -n -O -s 65507 "''${host}" - ${coreutils}/bin/sleep 60 - ${iputils}/bin/ping -D -c 60 -i 1 -n -O -s 65507 "''${host}" - ${coreutils}/bin/sleep 60 - ${iputils}/bin/ping -D -c 60 -i 1 -n -O -s 65507 "''${host}" - ${coreutils}/bin/sleep 60 - ${iputils}/bin/ping -D -c 60 -i 1 -n -O -s 65507 "''${host}" - - done - - done - } - - function msg { - # Outputs to stdout, unbuffered if not the message may be lost! - ${coreutils}/bin/stdbuf -o0 \ - ${bashInteractive}/bin/sh -c \ - "${coreutils}/bin/echo -e \"$(${coreutils}/bin/date --rfc-3339=seconds): $1\"" - } - - latency $@ - ''; - JSON = pkgs.writeScript "startup-latency.sh" value; - }; - }) - nodeSpecs; -in - { inherit latency-service; } diff --git a/nix/workbench/workload/latency.nix b/nix/workbench/workload/latency.nix new file mode 100644 index 00000000000..70be055d360 --- /dev/null +++ b/nix/workbench/workload/latency.nix @@ -0,0 +1,133 @@ +{ pkgs +, profile +, nodeSpecs +, workload +}: + +with pkgs.lib; + +let + bashInteractive = pkgs.bashInteractive; + coreutils = pkgs.coreutils; + iputils = pkgs.iputils; + jq = pkgs.jq; +# Assumptions: +# - Command `date` and node's log use the same timezone! +in '' +#!${bashInteractive}/bin/sh + +###################################################################### +# Set script globals ################################################# +###################################################################### + +# Strict runtime +################ + +# e: Immediately exit if any command has a non-zero exit status +# u: Reference to non previously defined variables is an error +# pipefail: Any failed command in a pipeline is used as return code +set -euo pipefail + +# Fetch all defined node names (Including "explorer" nodes) +########################################################### + +node_specs_nodes=$(${jq}/bin/jq --raw-output \ + "keys | join (\" \")" \ + ../../node-specs.json \ +) +node_specs_pools=$(${jq}/bin/jq \ + 'map(select(.kind == "pool")) | length' \ + ../../node-specs.json \ +) +${coreutils}/bin/echo "node-specs.json:" +${coreutils}/bin/echo "- Nodes: [''${node_specs_nodes[*]}]" +${coreutils}/bin/echo "- Pools: ''${node_specs_pools}" + +# Look for locally deployed nodes and allocate latency +###################################################### + +nodes=() +started_time=$(${coreutils}/bin/date +%s) +for node in ''${node_specs_nodes[*]} +do + if test -d "../../''${node}" + then + nodes+=("''${node}") + # Save the starting time + ${coreutils}/bin/echo "''${started_time}" > "../../workloads/latency/start_time_''${node}" + fi +done +${coreutils}/bin/echo "Found deployed nodes:" +${coreutils}/bin/echo "- Nodes: [''${nodes[*]}]" + +###################################################################### +# Main ############################################################### +###################################################################### + +# The main function, called at the end of the file/script. +function latency() { + + msg "Started!" + + for node in ''${nodes[*]} + do + + msg "Latency of \"''${node}\"'s Producers using 'ping'" + local topology_path="../../''${node}/topology.json" + # Merge non-P2P and P2P in the same {addr:"ADDR",port:0} format. + local producers + producers=$(${jq}/bin/jq '.Producers//[] + ((.localRoots[0].accessPoints//[]) | map({addr:.address,port:.port}))' "''${topology_path}") + local keys + keys=$(echo "''${producers}" | ${jq}/bin/jq --raw-output 'keys | join (" ")') + for key in ''${keys[*]} + do + + local host + host=$(echo "''${producers}" | ${jq}/bin/jq --raw-output ".[''${key}].addr") + local port + port=$(echo "''${producers}" | ${jq}/bin/jq --raw-output ".[''${key}].port") + + # If the ping fails the whole script must fail! + + msg "Executing 'ping' to \"''${host}:''${port}\"" + ${coreutils}/bin/sleep 60 + ${iputils}/bin/ping -D -c 60 -i 1 -n -O "''${host}" + ${coreutils}/bin/sleep 60 + ${iputils}/bin/ping -D -c 60 -i 1 -n -O "''${host}" + ${coreutils}/bin/sleep 60 + ${iputils}/bin/ping -D -c 60 -i 1 -n -O "''${host}" + ${coreutils}/bin/sleep 60 + ${iputils}/bin/ping -D -c 60 -i 1 -n -O "''${host}" + + msg "Executing 'ping' to \"''${host}:''${port}\" with size 16" + ${coreutils}/bin/sleep 60 + ${iputils}/bin/ping -D -c 60 -i 1 -n -O -s 16 "''${host}" + ${coreutils}/bin/sleep 60 + ${iputils}/bin/ping -D -c 60 -i 1 -n -O -s 16 "''${host}" + ${coreutils}/bin/sleep 60 + ${iputils}/bin/ping -D -c 60 -i 1 -n -O -s 16 "''${host}" + ${coreutils}/bin/sleep 60 + ${iputils}/bin/ping -D -c 60 -i 1 -n -O -s 16 "''${host}" + + msg "Executing 'ping' to \"''${host}:''${port}\" with size 65507" + ${coreutils}/bin/sleep 60 + ${iputils}/bin/ping -D -c 60 -i 1 -n -O -s 65507 "''${host}" + ${coreutils}/bin/sleep 60 + ${iputils}/bin/ping -D -c 60 -i 1 -n -O -s 65507 "''${host}" + ${coreutils}/bin/sleep 60 + ${iputils}/bin/ping -D -c 60 -i 1 -n -O -s 65507 "''${host}" + ${coreutils}/bin/sleep 60 + ${iputils}/bin/ping -D -c 60 -i 1 -n -O -s 65507 "''${host}" + + done + + done +} + +function msg { + # Outputs to stdout, unbuffered if not the message may be lost! + ${coreutils}/bin/stdbuf -o0 \ + ${bashInteractive}/bin/sh -c \ + "${coreutils}/bin/echo -e \"$(${coreutils}/bin/date --rfc-3339=seconds): $1\"" +} +'' diff --git a/nix/workbench/service/voting.nix b/nix/workbench/workload/voting.nix similarity index 93% rename from nix/workbench/service/voting.nix rename to nix/workbench/workload/voting.nix index 7e36eaadd6a..686510cd039 100644 --- a/nix/workbench/service/voting.nix +++ b/nix/workbench/workload/voting.nix @@ -1,6 +1,7 @@ { pkgs , profile , nodeSpecs +, workload }: let @@ -17,12 +18,14 @@ let ################ testnet_magic = profile.genesis.network_magic; - gov_action_deposit = profile.genesis.conway.govActionDeposit; + gov_action_deposit = + if __hasAttr "conway" profile.genesis + then profile.genesis.conway.govActionDeposit + else throw "Conway genesis needed!" + ; # Where to obtain the genesis funds from. - genesis_funds_vkey = "../genesis/cache-entry/utxo-keys/utxo2.vkey"; - genesis_funds_skey = "../genesis/cache-entry/utxo-keys/utxo2.skey"; - # Workload - workload = builtins.elemAt profile.workload 0; # Only workload 0 exists now! + genesis_funds_vkey = "../../genesis/cache-entry/utxo-keys/utxo2.vkey"; + genesis_funds_skey = "../../genesis/cache-entry/utxo-keys/utxo2.skey"; # Initial donation from genesis funds to make "valid" withdrawal proposals. treasury_donation = 500000; @@ -143,7 +146,7 @@ function get_socket_path { # Function arguments. local node_str=$1 # node name / folder to find the socket. - local socket_path="../''${node_str}/node.socket" + local socket_path="../../''${node_str}/node.socket" ${coreutils}/bin/echo "''${socket_path}" } @@ -202,14 +205,13 @@ function calculate_next_utxo { function store_address_utxo_expected { # Function arguments. - local node_str=$1 # node name / folder where to store the files. - local tx_signed=$2 - local addr=$3 + local tx_signed=$1 + local addr=$2 - local utxo_file=../"''${node_str}"/addr."''${addr}".json - calculate_next_utxo \ - "''${tx_signed}" \ - "''${addr}" \ + local utxo_file=./addr."''${addr}".json # Store in workload's directory! + calculate_next_utxo \ + "''${tx_signed}" \ + "''${addr}" \ > "''${utxo_file}" } @@ -221,10 +223,9 @@ function store_address_utxo_expected { function get_address_utxo_expected_id { # Function arguments. - local node_str=$1 # node name / folder where to store the files. - local addr=$2 + local addr=$1 - local utxo_file=../"''${node_str}"/addr."''${addr}".json + local utxo_file=./addr."''${addr}".json # Store in workload's directory! if test -f "''${utxo_file}" then ${jq}/bin/jq --raw-output \ @@ -241,10 +242,9 @@ function get_address_utxo_expected_id { function get_address_utxo_expected_value { # Function arguments. - local node_str=$1 # node name / folder where to store the files. - local addr=$2 + local addr=$1 - local utxo_file=../"''${node_str}"/addr."''${addr}".json + local utxo_file=./addr."''${addr}".json # Store in workload's directory! if test -f "''${utxo_file}" then ${jq}/bin/jq --raw-output '.value' "''${utxo_file}" @@ -294,7 +294,7 @@ function funds_submit_retry { socket_path="$(get_socket_path "''${node_str}")" local utxo_id - utxo_id="$(get_address_utxo_expected_id "''${node_str}" "''${addr}")" + utxo_id="$(get_address_utxo_expected_id "''${addr}")" local contains_addr="false" local submit_tries=${toString funds_submit_tries} @@ -458,7 +458,7 @@ function funds_from_to { # Not the first batch. # The input comes from the last transaction submitted. # No need to wait for it because the submission function does this! - tx_in="$(get_address_utxo_expected_id "''${node_str}" "''${funds_addr}")" + tx_in="$(get_address_utxo_expected_id "''${funds_addr}")" # Treasury donation happens only once. treasury_donation_args_array=() fi @@ -468,7 +468,7 @@ function funds_from_to { # Send this batch to each node! # Build transaction. - tx_filename=../"''${node_str}"/funds_from_to."''${funds_addr}"."''${i}" + tx_filename=./funds_from_to."''${funds_addr}"."''${i}" ${cardano-cli}/bin/cardano-cli conway transaction build \ --testnet-magic ${toString testnet_magic} \ --socket-path "''${socket_path}" \ @@ -488,14 +488,12 @@ function funds_from_to { for addr_cache in "''${txOuts_addrs_array[@]}" do store_address_utxo_expected \ - "''${node_str}" \ "''${tx_filename}.signed" \ "''${addr_cache}" done # Without the change address we can't wait for the funds after submission # or calculate the next input to use if an extra batch is needed! store_address_utxo_expected \ - "''${node_str}" \ "''${tx_filename}.signed" \ "''${funds_addr}" @@ -645,7 +643,7 @@ function create_node_prop_drep_key_files { local prop_i=$3 local drep_i=$4 - local filename=../"''${node_str}"-prop-"''${prop_i}"-drep-"''${drep_i}" + local filename=./"''${node_str}"-prop-"''${prop_i}"-drep-"''${drep_i}" # Now with the extensions. local skey="''${filename}".skey local vkey="''${filename}".vkey @@ -764,7 +762,7 @@ function governance_funds_genesis { ${jq}/bin/jq --raw-output \ --arg keyName "''${producer_name}" \ '.[$keyName].i' \ - ../node-specs.json \ + ../../node-specs.json \ )" local producer_addr # Drep 0 is No DRep (funds for the node). @@ -802,7 +800,7 @@ function governance_funds_producer { ${jq}/bin/jq --raw-output \ --arg keyName "''${producer_name}" \ '.[$keyName].i' \ - ../node-specs.json \ + ../../node-specs.json \ )" local producer_addr producer_vkey producer_skey producer_addr="$(build_node_prop_drep_address "''${producer_name}" "''${producer_i}" 0 0)" @@ -914,7 +912,7 @@ function governance_create_constitution { # No waiting! But, if last submitted transaction fails (function # `governance_funds_genesis` in current workflow), everything else fails. local funds_tx - funds_tx="$(get_address_utxo_expected_id "''${node_str}" "''${node_addr}")" + funds_tx="$(get_address_utxo_expected_id "''${node_addr}")" # Show current gov-state. ${cardano-cli}/bin/cardano-cli conway query gov-state \ @@ -925,31 +923,31 @@ function governance_create_constitution { # Create dummy constitution. ${coreutils}/bin/echo "My Constitution: free mate and asado" \ - > ../"''${node_str}"/constitution.txt + > ./constitution.txt # Calculate constitution hash. ${cardano-cli}/bin/cardano-cli hash anchor-data \ - --file-text ../"''${node_str}"/constitution.txt \ - --out-file ../"''${node_str}"/constitution.hash + --file-text ./constitution.txt \ + --out-file ./constitution.hash # Copy guardrails-script. ${coreutils}/bin/cp \ - ../genesis/guardrails-script.plutus \ - ../"''${node_str}"/guardrails-script.plutus + ../../genesis/guardrails-script.plutus \ + ./guardrails-script.plutus # Calculate guardrails-script hash. ${cardano-cli}/bin/cardano-cli hash script \ - --script-file ../"''${node_str}"/guardrails-script.plutus \ - --out-file ../"''${node_str}"/guardrails-script.hash + --script-file ./guardrails-script.plutus \ + --out-file ./guardrails-script.hash # Create action. - local tx_filename=../"''${node_str}"/create-constitution + local tx_filename=./create-constitution ${cardano-cli}/bin/cardano-cli conway governance action create-constitution \ --testnet \ --anchor-url "https://raw.githubusercontent.com/cardano-foundation/CIPs/master/CIP-0100/cip-0100.common.schema.json" \ --anchor-data-hash "9d99fbca260b2d77e6d3012204e1a8658f872637ae94cdb1d8a53f4369400aa9" \ --constitution-url "https://ipfs.io/ipfs/Qmdo2J5vkGKVu2ur43PuTrM7FdaeyfeFav8fhovT6C2tto" \ - --constitution-hash "$(${coreutils}/bin/cat ../"''${node_str}"/constitution.hash)" \ - --constitution-script-hash "$(${coreutils}/bin/cat ../"''${node_str}"/guardrails-script.hash)" \ + --constitution-hash "$(${coreutils}/bin/cat ./constitution.hash)" \ + --constitution-script-hash "$(${coreutils}/bin/cat ./guardrails-script.hash)" \ --governance-action-deposit "''${action_deposit}" \ - --deposit-return-stake-verification-key-file ../genesis/cache-entry/stake-delegators/delegator0/staking.vkey \ + --deposit-return-stake-verification-key-file ../../genesis/cache-entry/stake-delegators/delegator0/staking.vkey \ --out-file "''${tx_filename}".action # Build transaction. ${cardano-cli}/bin/cardano-cli conway transaction build \ @@ -977,7 +975,6 @@ function governance_create_constitution { wait_proposal_id "''${node_str}" "''${tx_filename}".signed >/dev/null store_address_utxo_expected \ - "''${node_str}" \ "''${tx_filename}.signed" \ "''${node_addr}" } @@ -1013,9 +1010,9 @@ function governance_create_withdrawal { # No waiting! But, if last submitted transaction fails (function # `governance_funds_producer` current workflow), everything else fails. local funds_tx - funds_tx="$(get_address_utxo_expected_id "''${node_str}" "''${node_drep_addr}")" + funds_tx="$(get_address_utxo_expected_id "''${node_drep_addr}")" - local tx_filename=../"''${node_str}"/create-withdrawal."''${node_str}"."''${drep_i}" + local tx_filename=./create-withdrawal."''${node_str}"."''${drep_i}" # Create action. ${cardano-cli}/bin/cardano-cli conway governance action create-treasury-withdrawal \ --testnet \ @@ -1023,8 +1020,8 @@ function governance_create_withdrawal { --anchor-data-hash "311b148ca792007a3b1fee75a8698165911e306c3bc2afef6cf0145ecc7d03d4" \ --governance-action-deposit "''${action_deposit}" \ --transfer 50 \ - --deposit-return-stake-verification-key-file ../genesis/cache-entry/stake-delegators/"delegator''${node_i}"/staking.vkey \ - --funds-receiving-stake-verification-key-file ../genesis/cache-entry/stake-delegators/"delegator''${node_i}"/staking.vkey \ + --deposit-return-stake-verification-key-file ../../genesis/cache-entry/stake-delegators/"delegator''${node_i}"/staking.vkey \ + --funds-receiving-stake-verification-key-file ../../genesis/cache-entry/stake-delegators/"delegator''${node_i}"/staking.vkey \ --out-file "''${tx_filename}".action # Build transaction. ${cardano-cli}/bin/cardano-cli conway transaction build \ @@ -1052,7 +1049,6 @@ function governance_create_withdrawal { wait_proposal_id "''${node_str}" "''${tx_filename}".signed >/dev/null store_address_utxo_expected \ - "''${node_str}" \ "''${tx_filename}.signed" \ "''${node_drep_addr}" } @@ -1066,8 +1062,8 @@ function vote_tps_throttle() { local node_str=$1 # node name / folder to find the socket to use. local txs_count=$2 # Actual number of total txs already submitted. - local filepath_first=../"''${node_str}"/first_vote_time - local filepath_last=../"''${node_str}"/last_vote_time + local filepath_first=./first_vote_time + local filepath_last=./last_vote_time if ! test -f "''${filepath_first}" then local start_time @@ -1170,7 +1166,7 @@ function governance_vote_all { dreps_array=() fi done - local proposal_flag="../''${node_str}/proposal.''${proposal_tx_id}.voted" + local proposal_flag="./proposal.''${proposal_tx_id}.voted" ${coreutils}/bin/touch "''${proposal_flag}" done } @@ -1212,26 +1208,26 @@ function governance_vote_proposal { # No waiting! But, if last submitted transaction fails (function # `governance_funds_producer` or `governance_vote_proposal` in current # workflow), everything else fails. - funds_tx="$(get_address_utxo_expected_id "''${node_str}" "''${node_drep_addr}")" + funds_tx="$(get_address_utxo_expected_id "''${node_drep_addr}")" # A next UTxO must be cached by `funds_from_to` when splitting the funds, # we don't check the response to be sure there is an expected UTxO to be # really sure we are not querying the node unnecessarily. - funds_value="$(get_address_utxo_expected_value "''${node_str}" "''${node_drep_addr}")" + funds_value="$(get_address_utxo_expected_value "''${node_drep_addr}")" # Need to be used twice to sign signing_key_file_params_array+=("--signing-key-file ''${node_drep_skey}") fi - local vote_filename=../"''${node_str}"/proposal."''${proposal_tx_id}"."''${drep_i}" + local vote_filename=./proposal."''${proposal_tx_id}"."''${drep_i}" ${cardano-cli}/bin/cardano-cli conway governance vote create \ --yes \ --governance-action-tx-id "''${proposal_tx_id}" \ --governance-action-index "0" \ - --drep-verification-key-file ../genesis/cache-entry/drep-keys/drep"''${drep_i}"/drep.vkey \ + --drep-verification-key-file ../../genesis/cache-entry/drep-keys/drep"''${drep_i}"/drep.vkey \ --out-file "''${vote_filename}".action vote_file_params_array+=("--vote-file ''${vote_filename}.action") - signing_key_file_params_array+=("--signing-key-file ../genesis/cache-entry/drep-keys/drep''${drep_i}/drep.skey") + signing_key_file_params_array+=("--signing-key-file ../../genesis/cache-entry/drep-keys/drep''${drep_i}/drep.skey") done - local tx_filename=../"''${node_str}"/proposal."''${proposal_tx_id}"."''${dreps_array[0]}"-"''${dreps_array[-1]}" + local tx_filename=./proposal."''${proposal_tx_id}"."''${dreps_array[0]}"-"''${dreps_array[-1]}" # Build the transaction. ${if build_vote then ( @@ -1309,7 +1305,7 @@ function workflow_generator_log_proposals { --testnet-magic ${toString testnet_magic} \ --socket-path "''${socket_path}" \ | ${jq}/bin/jq '.proposals' \ - > ../"''${node_str}"/proposals."''$(${coreutils}/bin/date +"%Y-%m-%d-%H-%M-%S-%3N")".json + > ./proposals."''$(${coreutils}/bin/date +"%Y-%m-%d-%H-%M-%S-%3N")".json ${coreutils}/bin/sleep 60 done } @@ -1350,6 +1346,18 @@ function workflow_generator { } function workflow_producer { + # Run the producer workflow for each deployed producer. + local producers=${toString producers_bash_array} + for producer_name in ''${producers[*]} + do + if test -d "../../''${producer_name}" + then + workflow_producer_deployed "''${producer_name}" + fi + done +} + +function workflow_producer_deployed { # Function arguments. local node_str=$1 # node name / folder to find the socket to use. @@ -1358,7 +1366,7 @@ function workflow_producer { ${jq}/bin/jq --raw-output \ --arg keyName "''${node_str}" \ '.[$keyName].i' \ - ../node-specs.json \ + ../../node-specs.json \ )" #- Splitting 1 --------------------------------------------------------------# @@ -1385,7 +1393,7 @@ function workflow_producer { ${cardano-cli}/bin/cardano-cli conway query gov-state \ --testnet-magic ${toString testnet_magic} \ --socket-path "''${socket_path}" \ - > "../''${node_str}/gov-state.start.json" + > "./gov-state.start.json" ${coreutils}/bin/echo "governance_vote_all: Start: $(${coreutils}/bin/date --rfc-3339=seconds)" governance_vote_all "''${node_str}" "''${producer_i}" ${coreutils}/bin/echo "governance_vote_all: End: $(${coreutils}/bin/date --rfc-3339=seconds)" @@ -1394,7 +1402,7 @@ function workflow_producer { ${cardano-cli}/bin/cardano-cli conway query gov-state \ --testnet-magic ${toString testnet_magic} \ --socket-path "''${socket_path}" \ - > "../''${node_str}/gov-state.end.json" + > "./gov-state.end.json" #----------------------------------------------------------------------------# '' else '' diff --git a/wb_profiles.mk b/wb_profiles.mk index d0ce36d8e25..7f7ab2b2dcb 100644 --- a/wb_profiles.mk +++ b/wb_profiles.mk @@ -4,11 +4,11 @@ PROFILES_FORGE_STRESS := forge-stress-solo-xs forge-stress-solo forge-stress-plu PROFILES_PLUTUSCALL := plutuscall-loop-plain plutuscall-loop-half plutuscall-loop-double plutuscall-secp-ecdsa-plain plutuscall-secp-ecdsa-half plutuscall-secp-ecdsa-double plutuscall-secp-schnorr-plain plutuscall-secp-schnorr-half plutuscall-secp-schnorr-double plutuscall-volt-loop plutuscall-volt-blst plutuscall-volt-ripemd PROFILES_MODEL := model-secp-ecdsa-double model-secp-ecdsa-half model-secp-ecdsa-plain model-value model-value-test PROFILES_K3 := k3-3ep-5kTx-10000kU-1300kD-64kbs-fixed-loaded k3-3ep-9kTx-10000kU-1300kD-64kbs-5tps-fixed-loaded k3-3ep-18kTx-10000kU-1300kD-64kbs-10tps-fixed-loaded k3-3ep-22kTx-10000kU-1300kD-64kbs-fixed-loaded -PROFILES_SCENARIOS := chainsync-early-byron chainsync-early-byron-notracer chainsync-early-byron-oldtracing chainsync-early-alonzo chainsync-early-alonzo-notracer chainsync-early-alonzo-p2p chainsync-early-alonzo-oldtracing devops idle latency-nomadperf latency-nomadperfssd tracer-only +PROFILES_SCENARIOS := chainsync-early-byron chainsync-early-byron-notracer chainsync-early-byron-oldtracing chainsync-early-alonzo chainsync-early-alonzo-notracer chainsync-early-alonzo-p2p chainsync-early-alonzo-oldtracing devops idle tracer-only PROFILES_LEGACY := ci-test-dense10 dish dish-10M dish-plutus dish-10M-plutus PROFILES_SCALING := faststartup-24M -PROFILES_NOMAD_PERF := value-nomadperf value-nomadperf-nop2p value-drep1k-nomadperf value-drep2k-nomadperf value-drep10k-nomadperf value-drep100k-nomadperf value-oldtracing-nomadperf value-oldtracing-nomadperf-nop2p value-volt-nomadperf plutus-nomadperf plutus-nomadperf-nop2p plutus-drep1k-nomadperf plutus-drep2k-nomadperf plutus-drep10k-nomadperf plutus-drep100k-nomadperf plutus24-nomadperf plutus-secp-ecdsa-nomadperf plutus-secp-schnorr-nomadperf plutusv3-blst-nomadperf plutusv3-blst-double-nomadperf plutusv3-blst-half-nomadperf plutus-volt-nomadperf fast-nomadperf fast-nomadperf-nop2p ci-test-nomadperf ci-test-nomadperf-nop2p ci-test-oldtracing-nomadperf default-nomadperf-nop2p default-nomadperf oldtracing-nomadperf oldtracing-nomadperf-nop2p ci-bench-nomadperf ci-bench-nomadperf-nop2p ci-bench-oldtracing-nomadperf value-voting-utxo-volt-nomadperf value-voting-volt-nomadperf value-voting-double-volt-nomadperf plutus-voting-utxo-volt-nomadperf plutus-voting-volt-nomadperf plutus-voting-double-volt-nomadperf -PROFILES_NOMAD_PERFSSD := utxoscale-solo-12M16G-nomadperfssd utxoscale-solo-12M64G-nomadperfssd utxoscale-solo-24M64G-nomadperfssd fast-nomadperfssd value-nomadperfssd +PROFILES_NOMAD_PERF := value-nomadperf value-nomadperf-nop2p value-drep1k-nomadperf value-drep2k-nomadperf value-drep10k-nomadperf value-drep100k-nomadperf value-oldtracing-nomadperf value-oldtracing-nomadperf-nop2p value-volt-nomadperf plutus-nomadperf plutus-nomadperf-nop2p plutus-drep1k-nomadperf plutus-drep2k-nomadperf plutus-drep10k-nomadperf plutus-drep100k-nomadperf plutus24-nomadperf plutus-secp-ecdsa-nomadperf plutus-secp-schnorr-nomadperf plutusv3-blst-nomadperf plutusv3-blst-double-nomadperf plutusv3-blst-half-nomadperf plutus-volt-nomadperf fast-nomadperf fast-nomadperf-nop2p ci-test-nomadperf ci-test-nomadperf-nop2p ci-test-oldtracing-nomadperf default-nomadperf-nop2p default-nomadperf oldtracing-nomadperf oldtracing-nomadperf-nop2p ci-bench-nomadperf ci-bench-nomadperf-nop2p ci-bench-oldtracing-nomadperf value-voting-utxo-volt-nomadperf value-voting-volt-nomadperf value-voting-double-volt-nomadperf plutus-voting-utxo-volt-nomadperf plutus-voting-volt-nomadperf plutus-voting-double-volt-nomadperf latency-nomadperf +PROFILES_NOMAD_PERFSSD := utxoscale-solo-12M16G-nomadperfssd utxoscale-solo-12M64G-nomadperfssd utxoscale-solo-24M64G-nomadperfssd fast-nomadperfssd value-nomadperfssd latency-nomadperfssd LOCAL_PROFILES += $(PROFILES_EMPTY) LOCAL_PROFILES += $(PROFILES_MINIATURE)