test: check flask event ordering

_appmap/test/conftest.py

@@ -217,3 +217,36 @@ def server_base_fixture(request, server_port):
     info = ServerInfo(debug=debug, host=TEST_HOST, port=server_port, env=server_env)
     info.factory = partial(server_starter, info)
     return info
+
+@pytest.fixture(name="testdir")
+def testdir_fixture(request, data_dir, pytester, monkeypatch):
+    # We need to set environment variables to control how tests are run. This will only work
+    # properly if pytester runs pytest in a subprocess.
+    assert (
+        pytester._method == "subprocess"  # pylint:disable=protected-access
+    ), "must run pytest in a subprocess"
+
+    # The init subdirectory contains a sitecustomize.py file that
+    # imports the appmap module. This simulates the way a real
+    # installation works, performing the same function as the the
+    # appmap.pth file that gets put in site-packages.
+    monkeypatch.setenv("PYTHONPATH", "init")
+
+    # Make sure APPMAP isn't the environment, to test that recording-by-default is working as
+    # expected. Individual test cases may set it as necessary.
+    monkeypatch.delenv("_APPMAP", raising=False)
+
+    marker = request.node.get_closest_marker("example_dir")
+    test_type = "unittest" if marker is None else marker.args[0]
+    pytester.copy_example(test_type)
+
+    pytester.expected = data_dir / test_type / "expected"
+    pytester.test_type = test_type
+
+    # this is so test_type can be overriden in test cases
+    def output_dir():
+        return pytester.path / "tmp" / "appmap" / pytester.test_type
+
+    pytester.output = output_dir
+
+    return pytester

_appmap/test/data/flask-instrumented/appmap.yml

@@ -0,0 +1,4 @@
+name: FlaskTest
+packages:
+- path: flaskapp
+- path: flask.app.Flask

_appmap/test/data/flask-instrumented/flaskapp.py

@@ -0,0 +1,20 @@
+"""
+Rudimentary Flask application for testing.
+"""
+# pylint: disable=missing-function-docstring
+
+from appmap.flask import AppmapFlask
+from flask import Flask, make_response
+from markupsafe import escape
+
+app = Flask(__name__)
+AppmapFlask(app).init_app()
+
+
+@app.route("/")
+def hello_world():
+    return "Hello, World!"
+
+@app.route("/exception")
+def raise_exception():
+    raise Exception("An exception")

_appmap/test/data/flask-instrumented/init/sitecustomize.py

@@ -0,0 +1 @@
+import appmap

_appmap/test/data/flask-instrumented/test_app.py

@@ -0,0 +1,19 @@
+import pytest
+from flaskapp import app
+
+
+@pytest.fixture(name="client")
+def test_client():
+    with app.test_client() as c:  # pylint: disable=no-member
+        yield c
+
+
+def test_request(client):
+    response = client.get("/")
+
+    assert response.status_code == 200
+
+def test_exception(client):
+    response = client.get("/exception")
+
+    assert response.status_code == 500

_appmap/test/data/flask/appmap-flask.yml

@@ -0,0 +1,4 @@
+name: FlaskTest
+packages:
+- path: app
+- dist: flask

_appmap/test/test_flask.py

@@ -2,6 +2,7 @@
 # pylint: disable=missing-function-docstring
 
 import importlib
+import json
 import os
 from importlib.metadata import version
 from types import SimpleNamespace as NS
@@ -169,3 +170,58 @@ def test_disabled_for_process(self, pytester, monkeypatch):
         assert (pytester.path / "tmp" / "appmap" / "process").exists()
         assert not (pytester.path / "tmp" / "appmap" / "requests").exists()
         assert not (pytester.path / "tmp" / "appmap" / "pytest").exists()
+
+def call_stack_balanced(events):
+    """Ensure that the call stack in events has balanced call and return events"""
+    stack = []
+    for e in events:
+        if e.get("event") == "call":
+            stack.append(e)
+        elif e.get("event") == "return":
+            if len(stack) > 0:
+                call = stack.pop()
+                if call.get("id") != e.get("parent_id"):
+                    return False
+            else:
+                return False
+    if len(stack) == 0:
+        return True
+
+    return False
+
+
+@pytest.mark.example_dir("flask-instrumented")
+def test_flask_instrumented(testdir, monkeypatch):
+    result = testdir.runpytest("-svv", "-k", "test_request")
+    result.assert_outcomes(passed=1)
+
+    appmap_file = testdir.path / "tmp" / "appmap" / "pytest" / "test_request.appmap.json"
+    appmap = json.load(appmap_file.open())
+    events = appmap["events"]
+    assert len(events) == 32
+    request_event = events[10]
+    assert request_event.get("http_server_request") is not None
+    nested_events = events[11:17]
+    assert call_stack_balanced(nested_events), f"unbalanced call stack\n{events[11:17]}"
+    response_event = events[17]
+    assert response_event.get("http_server_response") is not None and response_event.get(
+        "parent_id"
+    ) == request_event.get("id")
+
+@pytest.mark.example_dir("flask-instrumented")
+def test_flask_instrumented_with_exception(testdir, monkeypatch):
+    result = testdir.runpytest("-svv", "-k", "test_exception")
+    result.assert_outcomes(passed=1)
+
+    appmap_file = testdir.path / "tmp" / "appmap" / "pytest" / "test_exception.appmap.json"
+    appmap = json.load(appmap_file.open())
+    events = appmap["events"]
+    assert len(events) == 41
+    request_event = events[10]
+    assert request_event.get("http_server_request") is not None
+    nested_events = events[11:21]
+    assert call_stack_balanced(nested_events), f"unbalanced call stack\n{events[11:17]}"
+    response_event = events[21]
+    assert len(response_event.get("exceptions")) == 1 and response_event.get(
+        "parent_id"
+    ) == request_event.get("id")

_appmap/test/test_test_frameworks.py

@@ -158,40 +158,6 @@ def test_write_appmap(recorder_outdir):
     assert (recorder_outdir / expected_shortname).read_text().startswith('{"version"')
 
 
-@pytest.fixture(name="testdir")
-def fixture_runner_testdir(request, data_dir, pytester, monkeypatch):
-    # We need to set environment variables to control how tests are run. This will only work
-    # properly if pytester runs pytest in a subprocess.
-    assert (
-        pytester._method == "subprocess"  # pylint:disable=protected-access
-    ), "must run pytest in a subprocess"
-
-    # The init subdirectory contains a sitecustomize.py file that
-    # imports the appmap module. This simulates the way a real
-    # installation works, performing the same function as the the
-    # appmap.pth file that gets put in site-packages.
-    monkeypatch.setenv("PYTHONPATH", "init")
-
-    # Make sure APPMAP isn't the environment, to test that recording-by-default is working as
-    # expected. Individual test cases may set it as necessary.
-    monkeypatch.delenv("_APPMAP", raising=False)
-
-    marker = request.node.get_closest_marker("example_dir")
-    test_type = "unittest" if marker is None else marker.args[0]
-    pytester.copy_example(test_type)
-
-    pytester.expected = data_dir / test_type / "expected"
-    pytester.test_type = test_type
-
-    # this is so test_type can be overriden in test cases
-    def output_dir():
-        return pytester.path / "tmp" / "appmap" / pytester.test_type
-
-    pytester.output = output_dir
-
-    return pytester
-
-
 def verify_expected_appmap(testdir, suffix=""):
     appmap_json = list(testdir.output().glob("*test_hello_world.appmap.json"))
     assert len(appmap_json) == 1  # sanity check