fix: resolve fork failures during training runs

torchrun jobs create a number of children per GPU which can often exceed the 2k limit. Signed-off-by: Jason T. Greene <jason.greene@redhat.com>
containers · Aug 12, 2024 · ac7a55d · ac7a55d
1 parent 2249889
commit ac7a55d
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 0 deletions.
diff --git a/training/ilab-wrapper/ilab b/training/ilab-wrapper/ilab
@@ -128,6 +128,7 @@ PODMAN_COMMAND=("sudo" "--preserve-env=$PRESERVE_ENV" "podman" "run" "--rm" "-it
     "--device" "${CONTAINER_DEVICE}"
     "--security-opt" "label=disable" "--net" "host"
     "--shm-size" "10G"
+    "--pids-limit" "-1"
     "-v" "$HOME:$HOME"
     "${ADDITIONAL_MOUNT_OPTIONS[@]}"
     # This is intentionally NOT using "--env" "HOME" because we want the HOME

diff --git a/training/nvidia-bootc/duplicated/ilab-wrapper/ilab b/training/nvidia-bootc/duplicated/ilab-wrapper/ilab
@@ -128,6 +128,7 @@ PODMAN_COMMAND=("sudo" "--preserve-env=$PRESERVE_ENV" "podman" "run" "--rm" "-it
     "--device" "${CONTAINER_DEVICE}"
     "--security-opt" "label=disable" "--net" "host"
     "--shm-size" "10G"
+    "--pids-limit" "-1"
     "-v" "$HOME:$HOME"
     "${ADDITIONAL_MOUNT_OPTIONS[@]}"
     # This is intentionally NOT using "--env" "HOME" because we want the HOME