ci: add pytest-xdist

Add xdist to speed up test runs some.

_appmap/test/conftest.py

@@ -14,7 +14,7 @@
 
 import _appmap
 import appmap
-from _appmap.test.web_framework import TEST_HOST, TEST_PORT
+from _appmap.test.web_framework import TEST_HOST
 from appmap import generation
 
 from .. import utils
@@ -198,14 +198,22 @@ def _starter(controldir, xprocess):
 
     return _starter
 
+@pytest.fixture
+def server_port(worker_id):
+    if worker_id == "master":
+        offset = "0"
+    else:
+        offset = worker_id[2:]
+    return 8000 + int(offset)
+
 
 @pytest.fixture(name="server_base")
-def server_base_fixture(request):
+def server_base_fixture(request, server_port):
     marker = request.node.get_closest_marker("server")
     debug = marker.kwargs.get("debug", False)
     server_env = os.environ.copy()
     server_env.update(marker.kwargs.get("env", {}))
 
-    info = ServerInfo(debug=debug, host=TEST_HOST, port=TEST_PORT, env=server_env)
+    info = ServerInfo(debug=debug, host=TEST_HOST, port=server_port, env=server_env)
     info.factory = partial(server_starter, info)
     return info

_appmap/test/web_framework.py

@@ -5,6 +5,7 @@
 import json
 import multiprocessing
 import os
+import re
 import time
 import traceback
 from os.path import exists
@@ -20,7 +21,6 @@
 from .normalize import normalize_appmap
 
 TEST_HOST = "127.0.0.1"
-TEST_PORT = 8000
 
 _SR = SystemRandom()
 
@@ -323,28 +323,25 @@ def test_can_record(self, data_dir, client):
         res = client.delete("/_appmap/record")
         assert res.status_code == 404
 
+@pytest.mark.xdist_group("group1")
 class _TestRecordRequests:
     """Common tests for per-requests recording (record requests.)"""
 
     @classmethod
-    def server_url(cls):
-        return f"http://{TEST_HOST}:{TEST_PORT}"
-
-    @classmethod
-    def record_request_thread(cls):
+    def record_request_thread(cls, server_url):
         # I've seen occasional test failures, seemingly because the test servers can't handle the
         # barrage of requests. A tiny bit of delay still causes many, many concurrent requests, but
         # eliminates the failures.
         time.sleep(_SR.uniform(0, 0.1))
-        return requests.get(cls.server_url() + "/test", timeout=30)
+        return requests.get(server_url + "/test", timeout=30)
 
-    def record_requests(self, record_remote):
+    def record_requests(self, record_remote, server_url):
         # pylint: disable=too-many-locals
         if record_remote:
             # when remote recording is enabled, this test also
             # verifies the global recorder doesn't save duplicate
             # events when per-request recording is enabled
-            response = requests.post(self.server_url() + "/_appmap/record", timeout=30)
+            response = requests.post(server_url + "/_appmap/record", timeout=30)
             assert response.status_code == 200
 
         with concurrent.futures.ThreadPoolExecutor(
@@ -354,7 +351,7 @@ def record_requests(self, record_remote):
             max_number_of_threads = 400
             future_to_request_number = {}
             for n in range(max_number_of_threads):
-                future = executor.submit(self.record_request_thread)
+                future = executor.submit(self.record_request_thread, server_url)
                 future_to_request_number[future] = n
 
             # wait for all threads to complete
@@ -379,14 +376,21 @@ def record_requests(self, record_remote):
                     # response.headers doesn't exist in Django 2.2
                     assert "AppMap-File-Name" in response
                     appmap_file_name = response["AppMap-File-Name"]
-                assert exists(appmap_file_name)
+                found = False
+                for i in range(1, 10):
+                    found = exists(appmap_file_name)
+                    if found:
+                        break
+                    time.sleep(0.1)
+
+                assert found, f"{appmap_file_name} doesn't exist"
+
                 appmap_file_name_basename = appmap_file_name.split("/")[-1]
                 appmap_file_name_basename_part = "_".join(
                     appmap_file_name_basename.split("_")[2:]
                 )
-                assert (
-                    appmap_file_name_basename_part
-                    == "http_127_0_0_1_8000_test.appmap.json"
+                assert re.match(
+                    r"http_127_0_0_1_8[0-9]*_test.appmap.json", appmap_file_name_basename_part
                 )
 
                 with open(appmap_file_name, encoding="utf-8") as f:
@@ -414,12 +418,12 @@ def record_requests(self, record_remote):
     @pytest.mark.appmap_enabled
     @pytest.mark.server(debug=True)
     def test_record_requests_with_remote(self, server):
-        self.record_requests(server.debug)
+        self.record_requests(server.debug, server.url)
 
     @pytest.mark.appmap_enabled
     @pytest.mark.server(debug=False)
     def test_record_requests_without_remote(self, server):
-        self.record_requests(server.debug)
+        self.record_requests(server.debug, server.url)
 
     @pytest.mark.server(debug=False)
     def test_remote_disabled_in_prod(self, server):

pyproject.toml

@@ -78,6 +78,7 @@ fastapi = "^0.110.0"
 httpx = "^0.27.0"
 pytest-env = "^1.1.3"
 pytest-console-scripts = "^1.4.1"
+pytest-xdist = "^3.6.1"
 
 [build-system]
 requires = ["poetry-core>=1.1.0"]

pytest.ini

@@ -10,9 +10,10 @@ markers =
 testpaths = _appmap/test
 pytester_example_dir = _appmap/test/data
 
-# running in a subprocess ensures that environment variables are set
-# correctly and no classes are loaded.
-addopts = --runpytest subprocess --ignore vendor
+# running in a subprocess ensures that environment variables are set correctly and no classes are
+# loaded. Also, the remote-recording tests can't be run in parallel, so they're marked to run in the
+# same load group and distribution is done by loadgroup.
+addopts = --runpytest subprocess --ignore vendor --tb=short --dist loadgroup
 
 # We're stuck at pytest ~6.1.2. This warning got removed in a later
 # version.

tox.ini

@@ -11,6 +11,8 @@ deps=
     sqlalchemy >=2.0, <3.0
 
 [testenv]
+passenv = 
+    PYTEST_XDIST_AUTO_NUM_WORKERS
 allowlist_externals =
     env
     bash
@@ -26,10 +28,10 @@ deps=
 
 commands =
     poetry install -v
-    web: poetry run appmap-python {posargs:pytest}
-    django3: poetry run appmap-python pytest _appmap/test/test_django.py
-    flask2: poetry run appmap-python pytest _appmap/test/test_flask.py
-    sqlalchemy1: poetry run appmap-python pytest _appmap/test/test_sqlalchemy.py
+    web: poetry run appmap-python {posargs:pytest -n auto}
+    django3: poetry run appmap-python pytest -n auto _appmap/test/test_django.py
+    flask2: poetry run appmap-python pytest -n auto _appmap/test/test_flask.py
+    sqlalchemy1: poetry run appmap-python pytest -n auto _appmap/test/test_sqlalchemy.py
 
 [testenv:lint]
 skip_install = True