From 4f6c707e499c4b09762b716c07d86089d6c99b72 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Mon, 4 Nov 2024 17:42:23 -0800 Subject: [PATCH] Use spawn instead of forkserver This avoids forkserver errors of the form: OSError: AF_UNIX path too long Bug: https://bugs.gentoo.org/941956 Signed-off-by: Zac Medico --- .github/workflows/ci.yml | 3 +++ lib/portage/locks.py | 3 +++ lib/portage/util/_async/ForkProcess.py | 13 +++++++++---- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 19bdde0510..486e595348 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,4 +96,7 @@ jobs: export PYTEST_ADDOPTS="-vv -ra -l -o console_output_style=count -n $(nproc) --dist=worksteal" # Use pytest-rerunfailures to workaround pytest-xdist worker crashes with spawn start-method (bug 924416). [[ "${{ matrix.start-method }}" == "spawn" ]] && PYTEST_ADDOPTS+=" --reruns 5 --only-rerun 'worker .* crashed while running'" + [[ "${{ matrix.start-method }}" == "fork" ]] && + [[ "${{ matrix.python-version }}" == "3.14-dev" ]] && + PYTEST_ADDOPTS+=" --reruns 5 --only-rerun 'node down: Not properly terminated'" meson test -C /tmp/build --verbose diff --git a/lib/portage/locks.py b/lib/portage/locks.py index ee40451b12..377df22a16 100644 --- a/lib/portage/locks.py +++ b/lib/portage/locks.py @@ -27,6 +27,9 @@ import typing import warnings +if multiprocessing.get_start_method() == "forkserver": + multiprocessing = multiprocessing.get_context("spawn") + import portage from portage import os, _encodings, _unicode_decode from portage.exception import ( diff --git a/lib/portage/util/_async/ForkProcess.py b/lib/portage/util/_async/ForkProcess.py index 946978b301..bca2285337 100644 --- a/lib/portage/util/_async/ForkProcess.py +++ b/lib/portage/util/_async/ForkProcess.py @@ -2,7 +2,7 @@ # Distributed under the terms of the GNU General Public License v2 import fcntl -import multiprocessing +import multiprocessing as _multiprocessing import warnings import signal import sys @@ -17,6 +17,11 @@ _registered_run_exitfuncs = None +if _multiprocessing.get_start_method() == "forkserver": + multiprocessing = _multiprocessing.get_context("spawn") +else: + multiprocessing = _multiprocessing + class ForkProcess(SpawnProcess): # NOTE: This class overrides the meaning of the SpawnProcess 'args' @@ -33,7 +38,7 @@ class ForkProcess(SpawnProcess): _file_names = ("connection", "slave_fd") _files_dict = slot_dict_class(_file_names, prefix="") - _HAVE_SEND_HANDLE = getattr(multiprocessing.reduction, "HAVE_SEND_HANDLE", False) + _HAVE_SEND_HANDLE = getattr(_multiprocessing.reduction, "HAVE_SEND_HANDLE", False) def _start(self): if multiprocessing.get_start_method() == "fork": @@ -157,7 +162,7 @@ def _send_fd_pipes(self): (self._fd_pipes, fd_list), ) for fd in fd_list: - multiprocessing.reduction.send_handle( + _multiprocessing.reduction.send_handle( self._files.connection, fd, self.pid, @@ -297,7 +302,7 @@ def _bootstrap(child_connection, have_send_handle, fd_pipes, target, args, kwarg fd_pipes, fd_list = child_connection.recv() fd_pipes_map = {} for fd in fd_list: - fd_pipes_map[fd] = multiprocessing.reduction.recv_handle( + fd_pipes_map[fd] = _multiprocessing.reduction.recv_handle( child_connection ) child_connection.close()