Modernized framework to replace tool shed functional tests.

Goals:

- Replace direct database access with pydantic typed API requests.
- Replace Twill with playwright.
- Eliminate access to the Galaxy UI in the tests - the tests should assume API access from the installing party.

pytest-playwright requires small bits of galaxyproject#13909.

This is a worthy project that could let us remove a bunch of deprecated stuff from the Galaxy admin controllers and a bunch of mako stuff that is unused and could be used to test PRs like galaxyproject#14609 (which prompted me to do this) but I'm anxious about growing emotionally attached to code I want to remove and I'm worried about losing track of which helpers are required for Planemo/Emphemeris/Galaxy and which helpers are just being used to test the tool shed.

.github/workflows/toolshed.yaml

@@ -22,7 +22,12 @@ jobs:
       matrix:
         python-version: ['3.7']
         test-install-client: ['standalone', 'galaxy_api']
-        shed-api: ['v1', 'v2']
+        # v1 is mostly working...
+        shed-api: ['v1']
+        # lets get twill working with twill then try to
+        # make progress on the playwright
+        # shed-browser: ['twill', 'playwright']
+        shed-browser: ['playwright']
     services:
       postgres:
         image: postgres:13
@@ -54,11 +59,21 @@ jobs:
         with:
           path: 'galaxy_root/.venv'
           key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy_root/requirements.txt') }}-toolshed
+      - name: Install dependencies
+        run: ./scripts/common_startup.sh --skip-client-build
+        working-directory: 'galaxy_root'
+      - name: Build Frontend
+        run: '. .venv/bin/activate && cd lib/tool_shed/webapp/frontend && yarn && make client'
+        working-directory: 'galaxy_root'
+      - name: Install playwright
+        run: '. .venv/bin/activate && playwright install'
+        working-directory: 'galaxy_root'
       - name: Run tests
         run: './run_tests.sh -toolshed'
         env:
           TOOL_SHED_TEST_INSTALL_CLIENT: ${{ matrix.test-install-client }}
           TOOL_SHED_API_VERSION: ${{ matrix.shed-api }}
+          TOOL_SHED_TEST_BROWSER: ${{ matrix.shed-browser }}
         working-directory: 'galaxy_root'
       - uses: actions/upload-artifact@v3
         if: failure()

lib/galaxy/dependencies/dev-requirements.txt

@@ -88,6 +88,7 @@ pytest-html==3.2.0 ; python_version >= "3.7" and python_version < "3.12"
 pytest-json-report==1.5.0 ; python_version >= "3.7" and python_version < "3.12"
 pytest-metadata==2.0.4 ; python_version >= "3.7" and python_version < "3.12"
 pytest-mock==3.10.0 ; python_version >= "3.7" and python_version < "3.12"
+pytest-playwright==0.3.0 ; python_version >= "3.7" and python_version < "3.12"
 pytest-postgresql==4.1.1 ; python_version >= "3.7" and python_version < "3.12"
 pytest-shard==0.1.2 ; python_version >= "3.7" and python_version < "3.12"
 pytest==7.2.1 ; python_version >= "3.7" and python_version < "3.12"

lib/tool_shed/test/base/playwrightbrowser.py

@@ -0,0 +1,153 @@
+import time
+from typing import List
+
+from playwright.sync_api import (
+    expect,
+    Locator,
+    Page,
+)
+
+from .browser import (
+    FormValueType,
+    ShedBrowser,
+)
+
+
+class PlaywrightShedBrowser(ShedBrowser):
+    _page: Page
+
+    def __init__(self, page: Page):
+        self._page = page
+
+    def visit_url(self, url: str, allowed_codes: List[int]) -> str:
+        response = self._page.goto(url)
+        assert response is not None
+        return_code = response.status
+        assert return_code in allowed_codes, "Invalid HTTP return code {}, allowed codes: {}".format(
+            return_code,
+            ", ".join(str(code) for code in allowed_codes),
+        )
+        return response.url
+
+    def page_content(self) -> str:
+        self._page.wait_for_load_state("networkidle")
+        return self._page.content()
+
+    def check_page_for_string(self, patt: str) -> None:
+        """Looks for 'patt' in the current browser page"""
+        patt = patt.replace("<b>", "").replace("</b>", "")
+        expect(self._page.locator("body")).to_contain_text(patt)
+
+    def check_string_not_in_page(self, patt: str) -> None:
+        patt = patt.replace("<b>", "").replace("</b>", "")
+        expect(self._page.locator("body")).not_to_contain_text(patt)
+
+    def xcheck_page_for_string(self, patt: str) -> None:
+        page = self.page_content()
+        if page.find(patt) == -1:
+            fname = self.write_temp_file(page)
+            errmsg = f"no match to '{patt}'\npage content written to '{fname}'\npage: [[{page}]]"
+            raise AssertionError(errmsg)
+
+    def xcheck_string_not_in_page(self, patt: str) -> None:
+        page = self.page_content()
+        if page.find(patt) != -1:
+            fname = self.write_temp_file(page)
+            errmsg = f"string ({patt}) incorrectly displayed in page.\npage content written to '{fname}'"
+            raise AssertionError(errmsg)
+
+    def write_temp_file(self, content, suffix=".html"):
+        import tempfile
+
+        from galaxy.util import smart_str
+
+        with tempfile.NamedTemporaryFile(suffix=suffix, prefix="twilltestcase-", delete=False) as fh:
+            fh.write(smart_str(content))
+        return fh.name
+
+    def show_forms(self) -> Locator:
+        """Shows form, helpful for debugging new tests"""
+        return self._page.locator("form")
+
+    def submit_form_with_name(self, form_name: str, button="runtool_btn", **kwd):
+        form = self._form_with_name(form_name)
+        self._submit_form(form, button, **kwd)
+
+    def submit_form(self, form_no=-1, button="runtool_btn", form=None, **kwd):
+        """Populates and submits a form from the keyword arguments."""
+        # An HTMLForm contains a sequence of Controls.  Supported control classes are:
+        # TextControl, FileControl, ListControl, RadioControl, CheckboxControl, SelectControl,
+        # SubmitControl, ImageControl
+        if form is None:
+            try:
+                form = self.show_forms().nth(form_no)
+            except IndexError:
+                raise ValueError("No form to submit found")
+        self._submit_form(form, button, **kwd)
+
+    def _submit_form(self, form: Locator, button="runtool_btn", **kwd):
+        for control_name, control_value in kwd.items():
+            self._fill_form_value(form, control_name, control_value)
+        input = self._page.locator(f"[name='{button}']")
+        if input.count():
+            input.click()
+        else:
+            submit_input = form.locator("input[type=submit]")
+            submit_input.click()
+        time.sleep(0.25)
+        # tc.submit(button)
+
+    def _form_with_name(self, name: str) -> Locator:
+        forms = self.show_forms()
+        count = forms.count()
+        for i in range(count):
+            nth_form = self.show_forms().nth(i)
+            if nth_form.get_attribute("name") == name:
+                return nth_form
+        raise KeyError(f"No form with name [{name}]")
+
+    def fill_form_value(self, form_name: str, control_name: str, value: FormValueType):
+        form: Locator = self._form_with_name(form_name)
+        self._fill_form_value(form, control_name, value)
+
+    def _fill_form_value(self, form: Locator, control_name: str, value: FormValueType):
+        input_i = form.locator(f"input[name='{control_name}']")
+        input_t = form.locator(f"textarea[name='{control_name}']")
+        input_s = form.locator(f"select[name='{control_name}']")
+        if input_i.count():
+            if control_name in ["redirect"]:
+                input_i.input_value = value
+            else:
+                if type(value) is bool:
+                    if value and not input_i.is_checked():
+                        input_i.check()
+                    elif not value and input_i.is_checked():
+                        input_i.uncheck()
+                else:
+                    input_i.fill(value)
+        if input_t.count():
+            input_t.fill(value)
+        if input_s.count():
+            input_s.select_option(value)
+
+    def edit_repository_categories(self, categories_to_add: List[str], categories_to_remove: List[str]) -> None:
+        multi_select = "form[name='categories'] select[name='category_id']"
+        select_locator = self._page.locator(multi_select)
+        select_locator.evaluate("node => node.selectedOptions = []")
+        select_locator.select_option(label=categories_to_add)
+        self.submit_form_with_name("categories", "manage_categories_button")
+
+        select_locator.evaluate("node => node.selectedOptions = []")
+        select_locator.select_option(label=categories_to_remove)
+        self.submit_form_with_name("categories", "manage_categories_button")
+
+    def grant_users_access(self, usernames: List[str]):
+        multi_select = "form[name='user_access'] select[name='allow_push']"
+        select_locator = self._page.locator(multi_select)
+        select_locator.evaluate("node => node.selectedOptions = []")
+        select_locator.select_option(label=usernames)
+        self.submit_form_with_name("user_access", "user_access_button")
+
+    @property
+    def is_twill(self) -> bool:
+        return False

lib/tool_shed/test/base/twilltestcase.py

@@ -239,14 +239,18 @@ def display_installed_jobs_list_page(
         for data_manager_name in data_manager_names:
             params = {"id": data_managers[data_manager_name]["guid"]}
             self._visit_galaxy_url("/data_manager/jobs_list", params=params)
-            self.testcase.check_for_strings(strings_displayed)
+            content = page_content()
+            for expected in strings_displayed:
+                if content.find(expected) == -1:
+                    raise AssertionError(f"Failed to find pattern {expected} in {content}")
 
     def installed_repository_extended_info(
         self, installed_repository: galaxy_model.ToolShedRepository
     ) -> Dict[str, Any]:
         params = {"id": self.testcase.security.encode_id(installed_repository.id)}
         self._visit_galaxy_url("/admin_toolshed/manage_repository_json", params=params)
-        return loads(self.testcase.last_page())
+        json = page_content()
+        return loads(json)
 
     def install_repository(
         self,
@@ -654,6 +658,12 @@ def _browser(self) -> ShedBrowser:
         assert self.__browser
         return self.__browser
 
+    def _escape_page_content_if_needed(self, content: str) -> str:
+        # if twill browser is being used - replace spaces with " "
+        if self._browser.is_twill:
+            content = content.replace(" ", " ")
+        return content
+
     def check_for_strings(self, strings_displayed=None, strings_not_displayed=None):
         strings_displayed = strings_displayed or []
         strings_not_displayed = strings_not_displayed or []

lib/tool_shed/test/functional/conftest.py

@@ -0,0 +1,46 @@
+import os
+from typing import (
+    Any,
+    Dict,
+    Generator,
+)
+
+import pytest
+from playwright.sync_api import (
+    Browser,
+    BrowserContext,
+)
+from typing_extensions import Literal
+
+from ..base.browser import ShedBrowser
+from ..base.playwrightbrowser import PlaywrightShedBrowser
+from ..base.twillbrowser import TwillShedBrowser
+
+DEFAULT_BROWSER: Literal["twill", "playwright"] = "playwright"
+
+
+def twill_browser() -> Generator[ShedBrowser, None, None]:
+    yield TwillShedBrowser()
+
+
+def playwright_browser(class_context: BrowserContext) -> Generator[ShedBrowser, None, None]:
+    page = class_context.new_page()
+    yield PlaywrightShedBrowser(page)
+
+
+if os.environ.get("TOOL_SHED_TEST_BROWSER", DEFAULT_BROWSER) == "twill":
+    shed_browser = pytest.fixture(scope="class")(twill_browser)
+else:
+    shed_browser = pytest.fixture(scope="class")(playwright_browser)
+
+
+@pytest.fixture(scope="class")
+def class_context(
+    browser: Browser,
+    browser_context_args: Dict,
+    pytestconfig: Any,
+    request: pytest.FixtureRequest,
+) -> Generator[BrowserContext, None, None]:
+    from pytest_playwright.pytest_playwright import context
+
+    yield from context.__pytest_wrapped__.obj(browser, browser_context_args, pytestconfig, request)