Merge branch 'master' into docs-master

.ci/azure-pipelines-steps.yml

@@ -2,6 +2,7 @@
 # https://learn.microsoft.com/en-us/azure/devops/pipelines/tasks/reference/?view=azure-pipelines&viewFallbackFrom=azure-devops#tool
 
 # `{script: ...}` is shorthand for `{task: CmdLine@<mumble>, inputs: {script: ...}}`.
+# The shell is bash.
 # https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/steps-script?view=azure-pipelines
 # https://learn.microsoft.com/en-us/azure/devops/pipelines/tasks/reference/cmd-line-v2?view=azure-pipelines
 
@@ -14,20 +15,80 @@ steps:
   condition: ne(variables['python.version'], '')
 
 - script: |
-    type python
-    python --version
-  displayName: Show python version
+    set -o errexit
+    set -o nounset
+    set -o pipefail
 
-- script: |
     sudo apt-get update
     sudo apt-get install -y python2-dev python3-pip virtualenv
   displayName: Install build deps
   condition: and(eq(variables['python.version'], ''), eq(variables['Agent.OS'], 'Linux'))
 
-- script: python -mpip install "tox<4.0"
+- script: |
+    set -o errexit
+    set -o nounset
+    set -o pipefail
+
+    # macOS builders lack a realpath command
+    type python  && python  -c"import os.path;print(os.path.realpath('$(type -p python)'))" &&  python --version
+    type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version
+    type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version
+    echo
+
+    if [ -e /usr/bin/python ]; then
+        echo "/usr/bin/python:    sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')"
+    fi
+
+    if [ -e /usr/bin/python2 ]; then
+        echo "/usr/bin/python2:   sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')"
+    fi
+
+    if [ -e /usr/bin/python2.7 ]; then
+        echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')"
+    fi
+  displayName: Show python versions
+
+- script: |
+    set -o errexit
+    set -o nounset
+    set -o pipefail
+
+    # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12)
+    PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "$(tox.env)"))')
+
+    if [[ -z $PYTHON ]]; then
+        echo 1>&2 "Python interpreter could not be determined"
+        exit 1
+    fi
+
+    if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then
+        "$PYTHON" -m ensurepip --user --altinstall --no-default-pip
+        "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt"
+    elif [[ $PYTHON == "python2.7" ]]; then
+        curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py"
+        "$PYTHON" get-pip.py --user --no-python-version-warning
+        # Avoid Python 2.x pip masking system pip
+        rm -f ~/.local/bin/{easy_install,pip,wheel}
+        "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt"
+    else
+        "$PYTHON" -m pip install -r "tests/requirements-tox.txt"
+    fi
   displayName: Install tooling
 
-- script: python -mtox -e "$(tox.env)"
+- script: |
+    set -o errexit
+    set -o nounset
+    set -o pipefail
+
+    # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12)
+    PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "$(tox.env)"))')
+
+    if [[ -z $PYTHON ]]; then
+        echo 1>&2 "Python interpreter could not be determined"
+        exit 1
+    fi
+
+    "$PYTHON" -m tox -e "$(tox.env)"
   displayName: "Run tests"
   env:
     AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)

.ci/azure-pipelines.yml

@@ -21,25 +21,21 @@ jobs:
     matrix:
       Mito_27:
         tox.env: py27-mode_mitogen
-      Mito_311:
-        python.version: '3.11'
-        tox.env: py311-mode_mitogen
+      Mito_312:
+        python.version: '3.12'
+        tox.env: py312-mode_mitogen
 
-      # TODO: test python3, python3 tests are broken
       Loc_27_210:
         tox.env: py27-mode_localhost-ansible2.10
-      Loc_27_4:
-        tox.env: py27-mode_localhost-ansible4
+      Loc_312_6:
+        python.version: '3.12'
+        tox.env: py312-mode_localhost-ansible6
 
-      # NOTE: this hangs when ran in Ubuntu 18.04
       Van_27_210:
-        tox.env: py27-mode_localhost-ansible2.10
-        STRATEGY: linear
-        ANSIBLE_SKIP_TAGS: resource_intensive
-      Van_27_4:
-        tox.env: py27-mode_localhost-ansible4
-        STRATEGY: linear
-        ANSIBLE_SKIP_TAGS: resource_intensive
+        tox.env: py27-mode_localhost-ansible2.10-strategy_linear
+      Van_312_6:
+        python.version: '3.12'
+        tox.env: py312-mode_localhost-ansible6-strategy_linear
 
 - job: Linux
   pool:
@@ -96,33 +92,33 @@ jobs:
         python.version: '3.6'
         tox.env: py36-mode_mitogen-distro_ubuntu2004
 
-      Mito_311_centos6:
-        python.version: '3.11'
-        tox.env: py311-mode_mitogen-distro_centos6
-      Mito_311_centos7:
-        python.version: '3.11'
-        tox.env: py311-mode_mitogen-distro_centos7
-      Mito_311_centos8:
-        python.version: '3.11'
-        tox.env: py311-mode_mitogen-distro_centos8
-      Mito_311_debian9:
-        python.version: '3.11'
-        tox.env: py311-mode_mitogen-distro_debian9
-      Mito_311_debian10:
-        python.version: '3.11'
-        tox.env: py311-mode_mitogen-distro_debian10
-      Mito_311_debian11:
-        python.version: '3.11'
-        tox.env: py311-mode_mitogen-distro_debian11
-      Mito_311_ubuntu1604:
-        python.version: '3.11'
-        tox.env: py311-mode_mitogen-distro_ubuntu1604
-      Mito_311_ubuntu1804:
-        python.version: '3.11'
-        tox.env: py311-mode_mitogen-distro_ubuntu1804
-      Mito_311_ubuntu2004:
-        python.version: '3.11'
-        tox.env: py311-mode_mitogen-distro_ubuntu2004
+      Mito_312_centos6:
+        python.version: '3.12'
+        tox.env: py312-mode_mitogen-distro_centos6
+      Mito_312_centos7:
+        python.version: '3.12'
+        tox.env: py312-mode_mitogen-distro_centos7
+      Mito_312_centos8:
+        python.version: '3.12'
+        tox.env: py312-mode_mitogen-distro_centos8
+      Mito_312_debian9:
+        python.version: '3.12'
+        tox.env: py312-mode_mitogen-distro_debian9
+      Mito_312_debian10:
+        python.version: '3.12'
+        tox.env: py312-mode_mitogen-distro_debian10
+      Mito_312_debian11:
+        python.version: '3.12'
+        tox.env: py312-mode_mitogen-distro_debian11
+      Mito_312_ubuntu1604:
+        python.version: '3.12'
+        tox.env: py312-mode_mitogen-distro_ubuntu1604
+      Mito_312_ubuntu1804:
+        python.version: '3.12'
+        tox.env: py312-mode_mitogen-distro_ubuntu1804
+      Mito_312_ubuntu2004:
+        python.version: '3.12'
+        tox.env: py312-mode_mitogen-distro_ubuntu2004
 
       Ans_27_210:
         tox.env: py27-mode_ansible-ansible2.10
@@ -148,6 +144,6 @@ jobs:
       Ans_311_5:
         python.version: '3.11'
         tox.env: py311-mode_ansible-ansible5
-      Ans_311_6:
-        python.version: '3.11'
-        tox.env: py311-mode_ansible-ansible6
+      Ans_312_6:
+        python.version: '3.12'
+        tox.env: py312-mode_ansible-ansible6

.ci/localhost_ansible_tests.py

@@ -1,6 +1,10 @@
 #!/usr/bin/env python
 # Run tests/ansible/all.yml under Ansible and Ansible-Mitogen
 
+from __future__ import print_function
+
+import getpass
+import io
 import os
 import subprocess
 import sys
@@ -53,6 +57,38 @@
         os.chdir(IMAGE_PREP_DIR)
         ci_lib.run("ansible-playbook -c local -i localhost, _user_accounts.yml")
 
+    # FIXME Don't hardcode https://github.com/mitogen-hq/mitogen/issues/1022
+    #       and os.environ['USER'] is not populated on Azure macOS runners.
+    os.chdir(HOSTS_DIR)
+    with io.open('default.hosts', 'r+', encoding='utf-8') as f:
+        user = getpass.getuser()
+        content = f.read()
+        content = content.replace("{{ lookup('pipe', 'whoami') }}", user)
+        f.seek(0)
+        f.write(content)
+        f.truncate()
+    ci_lib.dump_file('default.hosts')
+
+    cmd = ';'.join([
+        'from __future__ import print_function',
+        'import os, sys',
+        'print(sys.executable, os.path.realpath(sys.executable))',
+    ])
+    for interpreter in ['/usr/bin/python', '/usr/bin/python2', '/usr/bin/python2.7']:
+        print(interpreter)
+        try:
+            subprocess.call([interpreter, '-c', cmd])
+        except OSError as exc:
+            print(exc)
+
+        print(interpreter, 'with PYTHON_LAUNCHED_FROM_WRAPPER=1')
+        environ = os.environ.copy()
+        environ['PYTHON_LAUNCHED_FROM_WRAPPER'] = '1'
+        try:
+            subprocess.call([interpreter, '-c', cmd], env=environ)
+        except OSError as exc:
+            print(exc)
+
 
 with ci_lib.Fold('ansible'):
     os.chdir(TESTS_DIR)

.gitignore

@@ -6,6 +6,7 @@ venvs/**
 *.pyc
 *.pyd
 *.pyo
+*.retry
 MANIFEST
 build/
 dist/