Parametrize python tests by Python feature flag (#3263)

src/workerd/server/tests/python/BUILD.bazel

@@ -14,38 +14,26 @@ copy_file(
     visibility = ["//visibility:public"],
 )
 
+py_wd_test("hello")
+
+py_wd_test("env-param")
+
 py_wd_test(
-    "hello",
-    args = ["--experimental"],
+    "asgi",
+    # TODO: update bundle to include asgi fix and test on 0.26.0a2
+    python_flags = ["development"],
 )
 
+py_wd_test("random")
+
+py_wd_test("subdirectory")
+
 py_wd_test(
     "sdk",
-    args = ["--experimental"],
     tags = [
         # TODO(someday): Fix asan failure for this, see https://github.com/cloudflare/workerd/pull/3140#discussion_r1858273318
         "no-asan",
     ],
 )
 
-py_wd_test(
-    "env-param",
-    args = ["--experimental"],
-)
-
-py_wd_test(
-    "asgi",
-    args = ["--experimental"],
-)
-
-py_wd_test(
-    "random",
-    args = ["--experimental"],
-)
-
-py_wd_test(
-    "subdirectory",
-    args = ["--experimental"],
-)
-
-# gen_import_tests(PYODIDE_IMPORTS_TO_TEST)
+gen_import_tests(PYODIDE_IMPORTS_TO_TEST)

src/workerd/server/tests/python/asgi/asgi.wd-test

@@ -12,7 +12,7 @@ const unitTests :Workerd.Config = (
           ( name = "SELF", service = "python-asgi" ),
         ],
         compatibilityDate = "2024-10-01",
-        compatibilityFlags = ["python_workers_development"],
+        compatibilityFlags = [%PYTHON_FEATURE_FLAGS],
       )
     )
   ],

src/workerd/server/tests/python/env-param/env.wd-test

@@ -14,7 +14,7 @@ const unitTests :Workerd.Config = (
           ),
         ],
         compatibilityDate = "2024-01-15",
-        compatibilityFlags = ["python_workers_development"],
+        compatibilityFlags = [%PYTHON_FEATURE_FLAGS],
       )
     ),
   ],

src/workerd/server/tests/python/hello/hello.wd-test

@@ -8,7 +8,7 @@ const unitTests :Workerd.Config = (
           (name = "worker.py", pythonModule = embed "worker.py")
         ],
         compatibilityDate = "2024-01-15",
-        compatibilityFlags = ["python_workers_development"],
+        compatibilityFlags = [%PYTHON_FEATURE_FLAGS],
       )
     ),
   ],

src/workerd/server/tests/python/import_tests.bzl

@@ -22,35 +22,37 @@ const unitTests :Workerd.Config = (
           (name = "{}", pythonRequirement = ""),
         ],
         compatibilityDate = "2024-05-02",
-        compatibilityFlags = ["{}"],
+        compatibilityFlags = [%PYTHON_FEATURE_FLAGS],
       )
     ),
   ]
 );"""
 
-def generate_wd_test_file(requirement, compatFlag):
-    return WD_FILE_TEMPLATE.format(requirement, requirement, compatFlag)
+def generate_wd_test_file(requirement):
+    return WD_FILE_TEMPLATE.format(requirement, requirement)
 
 # to_test is a dictionary from library name to list of imports
 def gen_import_tests(to_test):
     for lib in to_test.keys():
-        for compatFlag in ["python_workers", "python_workers_development"]:
-            worker_py_fname = "import/{}@{}/worker.py".format(lib, compatFlag)
-            wd_test_fname = "import/{}@{}/import.wd-test".format(lib, compatFlag)
-            write_file(
-                name = worker_py_fname + "@rule",
-                out = worker_py_fname,
-                content = [generate_import_py_file(to_test[lib])],
-            )
-            write_file(
-                name = wd_test_fname + "@rule",
-                out = wd_test_fname,
-                content = [generate_wd_test_file(lib, compatFlag)],
-            )
+        prefix = "import/" + lib
+        worker_py_fname = prefix + "/worker.py"
+        wd_test_fname = prefix + "/import.wd-test"
+        write_file(
+            name = worker_py_fname + "@rule",
+            out = worker_py_fname,
+            content = [generate_import_py_file(to_test[lib])],
+        )
+        write_file(
+            name = wd_test_fname + "@rule",
+            out = wd_test_fname,
+            content = [generate_wd_test_file(lib)],
+        )
 
-            py_wd_test(
-                src = wd_test_fname,
-                args = ["--experimental", "--pyodide-package-disk-cache-dir", "../all_pyodide_wheels"],
-                data = [worker_py_fname, "@all_pyodide_wheels//:whls"],
-                size = "enormous",
-            )
+        py_wd_test(
+            name = prefix,
+            directory = lib,
+            src = wd_test_fname,
+            args = ["--experimental", "--pyodide-package-disk-cache-dir", "../all_pyodide_wheels"],
+            data = [worker_py_fname, "@all_pyodide_wheels//:whls"],
+            size = "enormous",
+        )

src/workerd/server/tests/python/py_wd_test.bzl

@@ -1,15 +1,24 @@
+load("@bazel_skylib//rules:expand_template.bzl", "expand_template")
 load("//:build/wd_test.bzl", "wd_test")
 
+FEATURE_FLAGS = {
+    "0.26.0a2": ["python_workers"],
+    "development": ["python_workers_development"],
+}
+
 def py_wd_test(
         directory = None,
         src = None,
         data = None,
         name = None,
+        python_flags = "all",
         args = [],
         size = "enormous",
         tags = [],
         **kwargs):
-    if data == None:
+    if python_flags == "all":
+        python_flags = FEATURE_FLAGS.keys()
+    if data == None and directory != None:
         data = native.glob(
             [
                 directory + "/**",
@@ -19,16 +28,29 @@ def py_wd_test(
     if src == None:
         src = native.glob([directory + "/*.wd-test"])[0]
     if name == None and directory != None:
-        name = directory + "@"
+        name = directory
+    elif name == None:
+        name = src.removesuffix(".wd-test")
     data += ["//src/workerd/server/tests/python:pyodide_dev.capnp.bin@rule"]
     args = args + ["--pyodide-bundle-disk-cache-dir", "$(location //src/workerd/server/tests/python:pyodide_dev.capnp.bin@rule)/..", "--experimental"]
+    for python_flag in python_flags:
+        name_flag = name + "_" + python_flag
+        templated_src = name_flag.replace("/", "-") + "@template"
+        templated_src = "/".join(src.split("/")[:-1] + [templated_src])
+        feature_flags_txt = ",".join(['"{}"'.format(flag) for flag in FEATURE_FLAGS[python_flag]])
+        expand_template(
+            name = name_flag + "@rule",
+            out = templated_src,
+            template = src,
+            substitutions = {"%PYTHON_FEATURE_FLAGS": feature_flags_txt},
+        )
 
-    wd_test(
-        src = src,
-        data = data,
-        name = name,
-        args = args,
-        size = size,
-        tags = tags + ["py_wd_test"],
-        **kwargs
-    )
+        wd_test(
+            src = templated_src,
+            data = data,
+            name = name_flag + "@",
+            args = args,
+            size = size,
+            tags = tags + ["py_wd_test"],
+            **kwargs
+        )

src/workerd/server/tests/python/random/random.wd-test

@@ -8,7 +8,7 @@ const unitTests :Workerd.Config = (
           (name = "worker.py", pythonModule = embed "worker.py")
         ],
         compatibilityDate = "2024-01-15",
-        compatibilityFlags = ["python_workers_development"],
+        compatibilityFlags = [%PYTHON_FEATURE_FLAGS],
       )
     ),
   ],

src/workerd/server/tests/python/sdk/sdk.wd-test

@@ -8,15 +8,15 @@ const python :Workerd.Worker = (
     ( name = "SELF", service = "python-sdk" ),
   ],
   compatibilityDate = "2024-10-01",
-  compatibilityFlags = ["python_workers_development"],
+  compatibilityFlags = [%PYTHON_FEATURE_FLAGS],
 );
 
 const server :Workerd.Worker = (
   modules = [
     (name = "server.py", pythonModule = embed "server.py")
   ],
   compatibilityDate = "2024-10-01",
-  compatibilityFlags = ["python_workers_development"],
+  compatibilityFlags = [%PYTHON_FEATURE_FLAGS],
 );
 
 const unitTests :Workerd.Config = (

src/workerd/server/tests/python/subdirectory/subdirectory.wd-test

@@ -10,7 +10,7 @@ const unitTests :Workerd.Config = (
           (name = "subdir/a.py", pythonModule = embed "./subdir/a.py"),
         ],
         compatibilityDate = "2023-12-18",
-        compatibilityFlags = ["python_workers_development"],
+        compatibilityFlags = [%PYTHON_FEATURE_FLAGS],
       )
     ),
   ],