Create a manual GH action we can use to run make-appmaps with.

.github/workflows/make-appmaps.yml

@@ -0,0 +1,86 @@
+name: Make AppMaps
+
+on:
+  workflow_dispatch:
+    inputs:
+      projectFilter:
+        description: 'Project or instance to filter for'
+        required: true
+      appmapMaxEvents:
+        description: 'Max Events per AppMap'
+        required: true
+        default: '25000'
+      appmapMaxTime:
+        description: 'Max time per appmap creation'
+        required: true
+        default: '30'
+
+permissions:
+  contents: write
+  actions: write
+
+jobs:
+  run:
+    runs-on: SWE-Bench_Larger
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Install AppMap tools
+      uses: getappmap/install-action@v1
+      with:
+        install-appmap-library: false
+        tools-url: https://github.com/getappmap/appmap-js/releases/download/%40appland%2Fappmap-v3.147.0/appmap-linux-x64
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        cache: 'pip'
+
+    - name: Set up Conda
+      uses: conda-incubator/setup-miniconda@v3
+      with:
+        python-version: 3.9
+
+    - name: Make dirs
+      run: |
+        mkdir -p appmaps appmap_logs /tmp/swe-appmaps
+
+    - name: Create conda env
+      run: |
+        conda init bash
+        conda env create -f environment.yml
+
+    - name: Make Appmaps
+      run: |
+          source /home/runner/.bashrc
+          conda activate swe-bench
+          conda info
+          APPMAP_MAX_TIME=${{ github.event.inputs.appmapMaxTime }} APPMAP_MAX_EVENTS=${{ github.event.inputs.appmapMaxEvents }} ./appmap/make_appmaps-github-actions.sh --filter ${{ github.event.inputs.projectFilter }}
+
+    - name: Get Artifact Name
+      id: get-artifact-name
+      run: |
+        file=$(ls appmaps/ | head -n 1)
+        echo "file_name=$file" >> $GITHUB_ENV
+        echo "::set-output name=file_name::$file" 
+
+    - name: Upload AppMaps
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ steps.get-artifact-name.outputs.file_name }}
+        path: ./appmaps/${{ steps.get-artifact-name.outputs.file_name }}
+
+    - name: Upload SWE Bench Logs
+      uses: actions/upload-artifact@v4
+      with:
+        name: appmap-logs.zip
+        path: ./appmap_logs/
+
+    # If the build fails leave a terminal open for 5 minutes 
+    # we can investigate if needed
+    - name: Setup upterm session
+      uses: owenthereal/action-upterm@v1
+      if: ${{ failure() }}
+      with:
+        wait-timeout-minutes: 5

appmap/make_appmaps-github-actions.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# find project root directory
+PROJECT_ROOT=$(git rev-parse --show-toplevel)
+# add it to python path
+export PYTHONPATH=$PROJECT_ROOT:$PYTHONPATH
+
+INSTANCES="princeton-nlp/SWE-bench"
+
+python $PROJECT_ROOT/appmap/make_appmaps.py \
+    --instances_path "$INSTANCES" \
+    --log_dir "appmap_logs" \
+    --appmap_archive_dir appmaps \
+    --temp_dir "/tmp/swe-appmaps" \
+    --num_workers 1 \
+    --verbose \
+    --path_conda $(conda info --base) \
+    --appmap_bin /usr/local/bin/appmap \
+    "$@"

appmap/make_appmaps.py

@@ -1,14 +1,23 @@
-import argparse, glob, itertools, os, tarfile, subprocess
-
+import argparse
+import faulthandler
+import glob
+import itertools
+import json
+import os
+import signal
+import subprocess
+import sys
+import tarfile
 from multiprocessing import Pool, cpu_count
 from swebench.harness.constants import MAP_REPO_TO_TEST_FRAMEWORK
 from swebench.harness.context_manager import (
     TaskEnvContextManager,
     TestbedContextManager,
 )
-from swebench.harness.utils import split_instances, DotDict
+from swebench.harness.utils import DotDict, split_instances
 from swebench.metrics.getters import get_eval_refs
 
+faulthandler.register(signal.SIGUSR1)
 
 SKIP_INSTANCES = {"pytest-dev/pytest": ["6387", "7956", "3805"]}
 
@@ -80,12 +89,15 @@ def make_appmaps(data: dict):
         tcm.run_install_task(task_instance)
         tcm.log.write("Installing appmap")
         tcm.exec(["bash", "-c", f"{tcm.cmd_activate} && pip install appmap"])
+        tcm.log.write("Installing pytest-test-groups")
+        tcm.exec(["bash", "-c", f"{tcm.cmd_activate} && pip install pytest-test-groups"])
         task_instance["test_cmd"] = MAP_REPO_TO_TEST_FRAMEWORK[
             task_instance["repo"]
         ]  # run all tests
         tcm.log.write("Running tests with appmap")
-        task_instance["test_cmd"] = f"appmap-python {task_instance['test_cmd']}"
-        tcm.run_tests_task(task_instance)
+        for i in range(1,100):
+            test_cmd = f"APPMAP_DISPLAY_PARAMS=false PYTHONUNBUFFERED=1 appmap-python {task_instance['test_cmd']}  --test-group-count 100 --test-group {i}"
+            tcm.run_tests_task(task_instance, test_cmd)
         tcm.log.write("Uninstalling appmap")
         tcm.exec(["bash", "-c", f"{tcm.cmd_activate} && pip uninstall -y appmap"])
         # count .appmap.json files in testbed/tmp/appmap (recursively)
@@ -166,6 +178,14 @@ def main(args):
             for task_instance in task_instances
             if args.filter in task_instance["instance_id"]
         ]
+        if (args.show_instances):
+            def filter_keys(dicts):
+                keys_to_keep=["instance_id", "version", "environment_setup_commit"]
+                return [{k: v for k, v in d.items() if k in keys_to_keep} for d in dicts]
+
+            json.dump(filter_keys(task_instances), indent=2, fp=sys.stdout)
+            sys.exit(0)
+
 
     # group by repo-version
     rv_groups = itertools.groupby(task_instances, lambda x: (x["repo"], x["version"]))
@@ -267,6 +287,12 @@ def main(args):
         action="store_true",
         help="(Optional) Keep temporary directories after running",
     )
+    parser.add_argument(
+        "--show-instances",
+        action="store_true",
+        help="Only show instance info",
+    )
+
     args = parser.parse_args()
     appmap_bin = os.path.expanduser(args.appmap_bin)
     validate_args(args)

swebench/harness/constants.py

@@ -243,6 +243,7 @@
 MAP_VERSION_TO_INSTALL_PYTEST["8.0"]["pip_packages"] = [
     "iniconfig==2.0.0", "packaging==23.1", "pluggy==1.3.0",
     "exceptiongroup==1.1.3", "tomli==2.0.1"]
+MAP_VERSION_TO_INSTALL_PYTEST["8.0"]["install"] = "pip install -e .[testing]"
 
 MAP_VERSION_TO_INSTALL_MATPLOTLIB = {
     k: {
@@ -571,7 +572,7 @@
 MAP_REPO_TO_INSTALL = {}
 
 # Constants - Task Instance Test Frameworks
-TEST_PYTEST = "pytest --no-header -rA --tb=no -p no:cacheprovider"
+TEST_PYTEST = "pytest --no-header -rA --tb=no -p no:cacheprovider -v"
 MAP_REPO_TO_TEST_FRAMEWORK = {
     "astropy/astropy": TEST_PYTEST,
     "django/django": "./tests/runtests.py --verbosity 2",
@@ -585,7 +586,7 @@
     "pydicom/pydicom": TEST_PYTEST,
     "pylint-dev/astroid": TEST_PYTEST,
     "pylint-dev/pylint": TEST_PYTEST,
-    "pytest-dev/pytest": "pytest -rA",
+    "pytest-dev/pytest": "pytest -vrA",
     "pyvista/pyvista": TEST_PYTEST,
     "scikit-learn/scikit-learn": TEST_PYTEST,
     "sphinx-doc/sphinx": "tox -epy39 -v --",