From 910c5eee56c5ec84bddf0617bf94bc5b751d8828 Mon Sep 17 00:00:00 2001 From: Alex Gibson Date: Mon, 22 Jul 2024 14:37:49 +0100 Subject: [PATCH] Add accessibility test job running against home page (#14773) --- .github/workflows/a11y_tests.yml | 74 +++++++ .github/workflows/integration_tests.yml | 2 +- .gitignore | 1 + docs/testing.rst | 133 +++++++------ tests/playwright/README.md | 96 ---------- tests/playwright/global-setup.js | 31 +++ tests/playwright/package-lock.json | 180 ++++++++++++++++-- tests/playwright/package.json | 8 +- tests/playwright/playwright.config.js | 2 + .../playwright/specs/a11y/components.spec.js | 122 ++++++++++++ .../playwright/specs/a11y/includes/helpers.js | 49 +++++ .../specs/a11y/includes/locators.js | 12 ++ tests/playwright/specs/a11y/includes/urls.js | 12 ++ tests/playwright/specs/a11y/pages.spec.js | 56 ++++++ 14 files changed, 610 insertions(+), 168 deletions(-) create mode 100644 .github/workflows/a11y_tests.yml delete mode 100644 tests/playwright/README.md create mode 100644 tests/playwright/global-setup.js create mode 100644 tests/playwright/specs/a11y/components.spec.js create mode 100644 tests/playwright/specs/a11y/includes/helpers.js create mode 100644 tests/playwright/specs/a11y/includes/locators.js create mode 100644 tests/playwright/specs/a11y/includes/urls.js create mode 100644 tests/playwright/specs/a11y/pages.spec.js diff --git a/.github/workflows/a11y_tests.yml b/.github/workflows/a11y_tests.yml new file mode 100644 index 00000000000..0f95d5eefa3 --- /dev/null +++ b/.github/workflows/a11y_tests.yml @@ -0,0 +1,74 @@ +# Workflow that runs accessibility tests once per day against dev infra. + +name: Accessibility Tests +run-name: Accessibility Tests for ${{ github.sha }} +env: + SLACK_CHANNEL_ID: CBX0KH5GA # #www-notify in MoCo Slack + SLACK_BOT_TOKEN: ${{secrets.SLACK_BOT_TOKEN_FOR_MEAO_NOTIFICATIONS_APP}} +on: + schedule: + - cron: "0 15 * * *" # 3pm daily + workflow_dispatch: + inputs: + mozorg_service_hostname: + description: The root URL of the Mozorg service to run tests against. eg "https://www.mozilla.org/" + required: true + +jobs: + notify-of-test-run-start: + if: github.repository == 'mozilla/bedrock' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Notify via Slack that tests are starting + uses: ./.github/actions/slack + with: + env_name: test + label: "Accessibility Tests [${{ github.sha }}]" + status: info + channel_id: ${{ env.SLACK_CHANNEL_ID }} + slack_bot_token: ${{ env.SLACK_BOT_TOKEN }} + ref: ${{ github.sha }} + message: "Accessibility Tests started" + + accessibility-tests: + runs-on: ubuntu-latest + needs: notify-of-test-run-start + env: + PLAYWRIGHT_BASE_URL: ${{ github.event.inputs.mozorg_service_hostname || 'https://dev.bedrock.nonprod.webservices.mozgcp.net/' }} + CI: true + CI_JOB_ID: ${{ github.run_id }} + DRIVER: "" + LABEL: accessibility-tests + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + - name: Install dependencies + run: cd tests/playwright && npm ci && npm run install-deps + - name: Run Playwright tests + run: npm run a11y-tests + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: a11y-report-${{ github.sha }} + path: a11y-report/ + retention-days: 30 + + notify-of-test-run-completion: + if: github.repository == 'mozilla/bedrock' && always() + runs-on: ubuntu-latest + needs: [notify-of-test-run-start, accessibility-tests] + steps: + - uses: actions/checkout@v4 + - name: Notify via Slack of test-run outcome + uses: ./.github/actions/slack + with: + env_name: test + label: "Playwright tests [${{ github.sha }}]" + status: ${{ needs.accessibility-tests.result }} + channel_id: ${{ env.SLACK_CHANNEL_ID }} + slack_bot_token: ${{ env.SLACK_BOT_TOKEN }} + ref: ${{ github.sha }} + message: "Playwright tests completed. Status: ${{ needs.accessibility-tests.result }}" diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index d0335ec7de4..92a53813e31 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -124,7 +124,7 @@ jobs: - name: Install dependencies run: cd tests/playwright && npm ci && npm run install-deps - name: Run Playwright tests - run: cd tests/playwright && npx playwright test + run: cd tests/playwright && npm run functional-tests notify-of-test-run-completion: if: always() diff --git a/.gitignore b/.gitignore index 19690f0ea2e..ca30ca3f23d 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,7 @@ tests/functional/results.html tests/unit/coverage tests/unit/dist/ tests/playwright/test-results/ +tests/playwright/a11y-report/ Thumbs.db tmp/* venv diff --git a/docs/testing.rst b/docs/testing.rst index efb2f281556..47f85f13186 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -8,103 +8,127 @@ Front-end testing ================= -Bedrock runs a suite of front-end `Jasmine`_ behavioral/unit tests, which use -`Jasmine Browser Runner`_ as a test runner. We also have a suite of functional tests using -`Selenium`_ and `pytest`_. This allows us to emulate users interacting with a -real browser. All these test suites live in the ``tests`` directory. To run the tests locally, -you must also first download `geckodriver`_ and `chromedriver`_ and make it available -in your system path. You can alternatively specify the path to geckodriver and chromedriver -using the command line (see the `pytest-selenium documentation`_ for more information). +Bedrock runs several different types of front-end tests to ensure that the site is working +correctly and that new changes don't break existing functionality. -The ``tests`` directory comprises of: +- `Jasmine`_ unit/behavioral tests are used to test JavaScript code that runs in the browser. + These tests are run against both Firefox and Chrome browsers via a GitHub action, which is + triggered against all pull requests and commits to the main branch. +- `Playwright`_ integration tests are used to run end-to-end tests in a real browser + environment. These tests are run automatically post-merge as part of our CI deployment + process against dev, stage, and prod. Playwright tests are run against Firefox, Chromium, + and Webkit headless browsers for cross-engine coverage. +- `Axe Core`_ accessibility (a11y) tests are run as sub-suite of the Playwright tests, and + are used to check for a11y issues on key pages of the site, as well as common global + elements such as the site header and footer. These tests are not part of our CI deployment + pipeline, but are instead run via a GitHub action as a daily job against www-dev. +- `Selenium`_ integration tests are bedrock legacy test suite, which will eventually be + replaced by Playwright. These tests are run against Firefox, Chrome, and Internet Explorer + (via a mix of both a local Selenium Grid and Sauce Labs), as part of our CI pipeline, and + run alongside the Playwright tests. New tests should be written using Playwright, but we + will continue to run the Selenium tests until they are all ported over. -* ``/functional`` contains pytest tests. -* ``/pages`` contains Python page objects. -* ``/unit`` contains the Jasmine tests and Jasmine Browser Runner config file. +The test specs for all of the above suites can be found in the root ``./tests`` directory: -.. note:: +* ``./tests/functional/`` for Selenium tests. +* ``./tests/pages/`` for Selenium page objects. +* ``./tests/playwright/`` Playwright and Axe tests. +* ``./tests/unit/`` for Jasmine tests. - As of July 2024, bedrock has started to use `Playwright`_ for some integration tests, with - the aim of replacing Selenium in the near future. Playwright is a Node.js library for - automating modern browsers, and aims to make it easier to write, run, and debug tests. - This documentation will be updated as we transition to Playwright, but for now you can - find more information in the `README`_ file in the ``tests/playwright`` directory. +Automating the browser +====================== -Installation ------------- +Jasmine, Playwright and Selenium all require a browser to run. In order to automate browsers +such as Firefox and Chrome, you also need to have the appropriate drivers installed. To +download ``geckodriver`` and ``chromedriver`` and have them ready to run in your system, +there are a couple of ways: -First follow the :ref:`installation instructions for bedrock`, which -will install the dependencies required to run the various front-end test suites. +Download `geckodriver`_ and add it to your system path: -To download geckodriver and chromedriver and have it ready to run in your system, there are a couple of ways: + .. code-block:: bash -- `Download geckdriver `_ and add it to your system path: + cd /path/to/your/downloaded/files/ + mv geckodriver /usr/local/bin/ - .. code-block:: bash +If you're on MacOS, download ``geckodriver`` directly using Homebrew, which automatically +places it in your system path: - cd /path/to/your/downloaded/files/ - mv geckodriver /usr/local/bin/ + .. code-block:: bash -- If you're on MacOS, download geckodriver directly using Homebrew, which automatically places it in your system path: + brew install geckodriver - .. code-block:: bash - brew install geckodriver +Download `chromedriver`_ and add it to your system path: + .. code-block:: bash -- `Download chromedriver `_ and add it to your system path: + cd /path/to/your/downloaded/files/ + mv chromedriver /usr/local/bin/ - .. code-block:: bash +If you're on MacOS, download ``chromedriver`` directly using Homebrew/Cask, which +automatically places it in your system path: - cd /path/to/your/downloaded/files/ - mv chromedriver /usr/local/bin/ + .. code-block:: bash -- If you're on MacOS, download chromedriver directly using Homebrew/Cask, which automatically places it in your system path: + brew tap homebrew/cask + brew cask install chromedriver - .. code-block:: bash +Running Jasmine tests +===================== - brew tap homebrew/cask - - brew cask install chromedriver +Jasmine and its dependencies are installed via npm and are included when you run +``make preflight`` to install bedrock's main dependencies. +.. code-block:: bash -Running Jasmine tests using Jasmine Browser Runner --------------------------------------------------- + $ make preflight -To perform a single run of the Jasmine test suite using Firefox and Chrome, -first make sure you have both browsers installed locally, and then activate -your bedrock virtual env. +Next, make sure you activate your bedrock virtual env. .. code-block:: bash $ pyenv activate bedrock -You can then run the tests with the following command: +You can then run the full suite of Jasmine tests with the following command: .. code-block:: bash $ npm run test -This will run all our front-end linters and formatting checks before running +This will also run all our front-end linters and formatting checks before running the Jasmine test suite. If you only want to run the tests themselves, you can run: .. code-block:: bash - $ npm run test + $ npm run jasmine + +Writing Jasmine tests +--------------------- See the `Jasmine`_ documentation for tips on how to write JS behavioral or unit tests. We also use `Sinon`_ for creating test spies, stubs and mocks. -Running functional tests ------------------------- +Debugging Jasmine tests +----------------------- -.. Note:: +If you need to debug Jasmine tests, you can also run them interactively in the +browser using the following command: + +.. code-block:: bash + + $ npm run jasmine-serve + +Running Playwright tests +======================== + +Running Selenium tests +====================== - Before running the functional tests, please make sure to follow the bedrock - :ref:`installation docs`, including the database sync that is needed - to pull in external data such as event/blog feeds etc. These are required for - some of the tests to pass. +Before running the Selenium tests, please make sure to follow the bedrock +:ref:`installation docs`, including the database sync that is needed +to pull in external data such as event/blog feeds etc. These are required for +some of the tests to pass. To run the full functional test suite against your local bedrock instance in Mozorg mode: @@ -322,7 +346,7 @@ Guidelines for writing functional tests See also the `Web QA style guide`_ for Python based testing. Testing Basket email forms --------------------------- +========================== When writing functional tests for front-end email newsletter forms that submit to `Basket`_, we have some special case email addresses that can be used just for testing: @@ -346,6 +370,7 @@ via product details are well formed and return valid 200 responses. .. _Jasmine: https://jasmine.github.io/index.html .. _Jasmine Browser Runner: https://jasmine.github.io/setup/browser.html .. _Sinon: http://sinonjs.org/ +.. _Axe Core: https://github.com/dequelabs/axe-core .. _Selenium: http://docs.seleniumhq.org/ .. _pytest: http://pytest.org/latest/ .. _pytest documentation: http://pytest.org/latest/ diff --git a/tests/playwright/README.md b/tests/playwright/README.md deleted file mode 100644 index 9837012999b..00000000000 --- a/tests/playwright/README.md +++ /dev/null @@ -1,96 +0,0 @@ -# Playwright integration tests - -See the [Playwright documentation](https://playwright.dev/docs/intro) -for detailed information on how to write, run and debug tests. - -## Installation - -Install Playwright and headless browser engines: - -``` -npm install && npm run install-deps -``` - -## Specifying an environment - -By default tests will run against `http://localhost:8000/`. -You can set `PLAYWRIGHT_BASE_URL` in your `.env` to point to -a different environment. Remember to have your development -server running before starting the test suite locally. - -## Running the test suite - -To run the full suite of tests: - -``` -npx playwright test -``` - -This will run all tests against three different headless browser -engines (Firefox, Chromium, WebKit). - -### Running specific tests - -Tests are grouped using tags, such as `@mozorg`, `@firefox`, `@vpn`. -To run only one group of tests, instead of the whole suite, you -can use `--grep`: - -``` -npx playwright test --grep @firefox -``` - -To run only a specific test file, such as `firefox-new.spec.js`, -you can pass the filename: - -``` -npx playwright test firefox-new -``` - -See the [Playwright CLI docs](https://playwright.dev/docs/test-cli) -for more options when running and debugging tests. - -### Writing tests - -Test spec files are found in the `./tests/playwright/specs/` -directory. See the Playwright docs for more examples on [how -to write tests](https://playwright.dev/docs/writing-tests). - -#### Specifying browsers and operating systems. - -Tests use User Agent string overrides to mock different browser -and operating systems combinations. By default tests run with -User Agent strings configured for Firefox and Chrome running -on Windows 10, and Safari running on macOS 10.15.7. This is -accomplished using an `OpenPage()` helper that can be imported -in each test file: - -```javascript -const openPage = require('../scripts/open-page'); -const url = '/en-US/firefox/new/'; - -test.beforeEach(async ({ page, browserName }) => { - await openPage(url, page, browserName); -}); -``` - -To mock a different browser or operating system combination, -tests can manually load a different override instead of using -`openPage`: - -```javascript -test.beforeEach(async ({ page, browserName }) => { - if (browserName === 'webkit') { - // Set macOS 10.14 UA strings. - await page.addInitScript({ - path: `./scripts/useragent/mac-old/${browserName}.js` - }); - } else { - // Set Windows 8.1 UA strings (64-bit). - await page.addInitScript({ - path: `./scripts/useragent/win-old/${browserName}.js` - }); - } - - await page.goto(url + '?automation=true'); -}); -``` diff --git a/tests/playwright/global-setup.js b/tests/playwright/global-setup.js new file mode 100644 index 00000000000..15d9f34b272 --- /dev/null +++ b/tests/playwright/global-setup.js @@ -0,0 +1,31 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +'use strict'; + +const fs = require('fs'); + +/** + * Clean up /a11y-report/ directory before running tests. + */ +function cleanA11yReportDir() { + const dir = 'a11y-report/'; + if (fs.existsSync(dir)) { + fs.rmdirSync(dir, { + recursive: true + }); + } + + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir); + } +} + +function globalSetup() { + cleanA11yReportDir(); +} + +module.exports = globalSetup; diff --git a/tests/playwright/package-lock.json b/tests/playwright/package-lock.json index f18eb1a2200..bdb09a6ad62 100644 --- a/tests/playwright/package-lock.json +++ b/tests/playwright/package-lock.json @@ -9,24 +9,79 @@ "version": "0.1.0", "license": "MPL", "dependencies": { - "@playwright/test": "^1.44.1", + "@axe-core/playwright": "^4.9.1", + "@playwright/test": "^1.45.2", + "axe-html-reporter": "^2.2.5", "dotenv": "^16.4.1" } }, + "node_modules/@axe-core/playwright": { + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/@axe-core/playwright/-/playwright-4.9.1.tgz", + "integrity": "sha512-8m4WZbZq7/aq7ZY5IG8GqV+ZdvtGn/iJdom+wBg+iv/3BAOBIfNQtIu697a41438DzEEyptXWmC3Xl5Kx/o9/g==", + "dependencies": { + "axe-core": "~4.9.1" + }, + "peerDependencies": { + "playwright-core": ">= 1.0.0" + } + }, "node_modules/@playwright/test": { - "version": "1.44.1", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.44.1.tgz", - "integrity": "sha512-1hZ4TNvD5z9VuhNJ/walIjvMVvYkZKf71axoF/uiAqpntQJXpG64dlXhoDXE3OczPuTuvjf/M5KWFg5VAVUS3Q==", + "version": "1.45.2", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.45.2.tgz", + "integrity": "sha512-JxG9eq92ET75EbVi3s+4sYbcG7q72ECeZNbdBlaMkGcNbiDQ4cAi8U2QP5oKkOx+1gpaiL1LDStmzCaEM1Z6fQ==", "dependencies": { - "playwright": "1.44.1" + "playwright": "1.45.2" }, "bin": { "playwright": "cli.js" }, "engines": { - "node": ">=16" + "node": ">=18" + } + }, + "node_modules/axe-core": { + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.9.1.tgz", + "integrity": "sha512-QbUdXJVTpvUTHU7871ppZkdOLBeGUKBQWHkHrvN2V9IQWGMt61zf3B45BtzjxEJzYuj0JBjBZP/hmYS/R9pmAw==", + "engines": { + "node": ">=4" + } + }, + "node_modules/axe-html-reporter": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/axe-html-reporter/-/axe-html-reporter-2.2.5.tgz", + "integrity": "sha512-3+w+R7gjTLLtBn8wFPIkSb4VBV/CuMkk4F9+0/Iza7KBK6nxWFQ05LYY39pUf43OxORVhlf6wWT3YXdkLBrFxQ==", + "dependencies": { + "mustache": "^4.0.1", + "rimraf": "^3.0.2" + }, + "engines": { + "node": ">=8.9.0" + }, + "peerDependencies": { + "axe-core": ">=3" } }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" + }, "node_modules/dotenv": { "version": "16.4.5", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", @@ -38,6 +93,11 @@ "url": "https://dotenvx.com" } }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" + }, "node_modules/fsevents": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", @@ -51,33 +111,123 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "bin": { + "mustache": "bin/mustache" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/playwright": { - "version": "1.44.1", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.44.1.tgz", - "integrity": "sha512-qr/0UJ5CFAtloI3avF95Y0L1xQo6r3LQArLIg/z/PoGJ6xa+EwzrwO5lpNr/09STxdHuUoP2mvuELJS+hLdtgg==", + "version": "1.45.2", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.45.2.tgz", + "integrity": "sha512-ReywF2t/0teRvNBpfIgh5e4wnrI/8Su8ssdo5XsQKpjxJj+jspm00jSoz9BTg91TT0c9HRjXO7LBNVrgYj9X0g==", "dependencies": { - "playwright-core": "1.44.1" + "playwright-core": "1.45.2" }, "bin": { "playwright": "cli.js" }, "engines": { - "node": ">=16" + "node": ">=18" }, "optionalDependencies": { "fsevents": "2.3.2" } }, "node_modules/playwright-core": { - "version": "1.44.1", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.44.1.tgz", - "integrity": "sha512-wh0JWtYTrhv1+OSsLPgFzGzt67Y7BE/ZS3jEqgGBlp2ppp1ZDj8c+9IARNW4dwf1poq5MgHreEM2KV/GuR4cFA==", + "version": "1.45.2", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.45.2.tgz", + "integrity": "sha512-ha175tAWb0dTK0X4orvBIqi3jGEt701SMxMhyujxNrgd8K0Uy5wMSwwcQHtyB4om7INUkfndx02XnQ2p6dvLDw==", "bin": { "playwright-core": "cli.js" }, "engines": { - "node": ">=16" + "node": ">=18" } + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" } } } diff --git a/tests/playwright/package.json b/tests/playwright/package.json index bd00263e48d..bf8423beac5 100644 --- a/tests/playwright/package.json +++ b/tests/playwright/package.json @@ -4,7 +4,9 @@ "description": "Making mozilla.org awesome, one pebble at a time", "private": true, "dependencies": { - "@playwright/test": "^1.44.1", + "@axe-core/playwright": "^4.9.1", + "@playwright/test": "^1.45.2", + "axe-html-reporter": "^2.2.5", "dotenv": "^16.4.1" }, "repository": { @@ -17,6 +19,8 @@ "url": "https://bugzilla.mozilla.org/" }, "scripts": { - "install-deps": "npx playwright install --with-deps" + "install-deps": "npx playwright install --with-deps", + "functional-tests": "npx playwright test --grep-invert @a11y", + "a11y-tests": "npx playwright test --grep @a11y --project=chromium" } } diff --git a/tests/playwright/playwright.config.js b/tests/playwright/playwright.config.js index ae65069b0c1..ef052a2a49d 100644 --- a/tests/playwright/playwright.config.js +++ b/tests/playwright/playwright.config.js @@ -23,6 +23,8 @@ const desktopViewportSize = { * @see https://playwright.dev/docs/test-configuration */ module.exports = defineConfig({ + /* Global setup file to prepare the environment for the test run. */ + globalSetup: require.resolve('./global-setup'), testDir: './specs', /* Run tests in files in parallel */ fullyParallel: true, diff --git a/tests/playwright/specs/a11y/components.spec.js b/tests/playwright/specs/a11y/components.spec.js new file mode 100644 index 00000000000..28c90991350 --- /dev/null +++ b/tests/playwright/specs/a11y/components.spec.js @@ -0,0 +1,122 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +'use strict'; + +const openPage = require('../../scripts/open-page'); +const testURL = '/en-US/'; +const { createReport, scanPageElement } = require('./includes/helpers'); +const { navigationLocator, footerLocator } = require('./includes/locators'); +const { test, expect } = require('@playwright/test'); + +test.describe( + `Navigation (desktop)`, + { + tag: '@a11y' + }, + () => { + test.beforeEach(async ({ page, browserName }) => { + await openPage(testURL, page, browserName); + }); + + test('should not have any detectable a11y issues', async ({ page }) => { + const firefoxLink = page.getByTestId('navigation-link-firefox'); + const firefoxMenu = page.getByTestId('navigation-menu-firefox'); + + // Hover over Firefox link to open menu + await firefoxLink.hover(); + await expect(firefoxMenu).toBeVisible(); + + const results = await scanPageElement(page, navigationLocator); + await createReport('navigation', 'desktop', results); + expect(results.violations.length).toEqual(0); + }); + } +); + +test.describe( + `Navigation (mobile)`, + { + tag: '@a11y' + }, + () => { + test.use({ viewport: { width: 360, height: 780 } }); + + test.beforeEach(async ({ page, browserName }) => { + await openPage(testURL, page, browserName); + }); + + test('should not have any detectable a11y issues', async ({ page }) => { + const navigationMenuButton = page.getByTestId( + 'navigation-menu-button' + ); + const navigationMenuItems = page.getByTestId( + 'navigation-menu-items' + ); + const firefoxLink = page.getByTestId('navigation-link-firefox'); + const firefoxMenu = page.getByTestId('navigation-menu-firefox'); + + // Open navigation menu + await navigationMenuButton.click(); + await expect(navigationMenuItems).toBeVisible(); + + // Open and close Firefox menu + await firefoxLink.click(); + await expect(firefoxMenu).toBeVisible(); + + const results = await scanPageElement(page, navigationLocator); + await createReport('navigation', 'mobile', results); + expect(results.violations.length).toEqual(0); + }); + } +); + +test.describe( + `Footer (desktop)`, + { + tag: '@a11y' + }, + () => { + test.beforeEach(async ({ page, browserName }) => { + await openPage(testURL, page, browserName); + }); + + test('should not have any detectable a11y issues', async ({ page }) => { + const results = await scanPageElement(page, footerLocator); + await createReport('footer', 'desktop', results); + expect(results.violations.length).toEqual(0); + }); + } +); + +test.describe( + `Footer (mobile)`, + { + tag: '@a11y' + }, + () => { + test.use({ viewport: { width: 360, height: 780 } }); + + test.beforeEach(async ({ page, browserName }) => { + await openPage(testURL, page, browserName); + }); + + test('should not have any detectable a11y issues', async ({ page }) => { + const footerHeadingCompany = page.getByTestId( + 'footer-heading-company' + ); + const footerListCompany = page.getByTestId('footer-list-company'); + + // Open and Company section + await expect(footerListCompany).not.toBeVisible(); + await footerHeadingCompany.click(); + + const results = await scanPageElement(page, footerLocator); + await createReport('footer', 'mobile', results); + expect(results.violations.length).toEqual(0); + }); + } +); diff --git a/tests/playwright/specs/a11y/includes/helpers.js b/tests/playwright/specs/a11y/includes/helpers.js new file mode 100644 index 00000000000..963c7eec284 --- /dev/null +++ b/tests/playwright/specs/a11y/includes/helpers.js @@ -0,0 +1,49 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +'use strict'; + +const fs = require('fs'); +const { AxeBuilder } = require('@axe-core/playwright'); +const { createHtmlReport } = require('axe-html-reporter'); +const { navigationLocator, footerLocator } = require('./locators'); + +async function createReport(slug, type, results) { + const fileName = `results-${slug.replaceAll('/', '-').toLowerCase()}-${type}.html`; + const reportHTML = createHtmlReport({ + results: results, + options: { + projectKey: `${slug} (${type})`, + doNotCreateReportFile: true + } + }); + + fs.writeFileSync(`a11y-report/${fileName}`, reportHTML); +} + +/** + * Scan a whole page for a11y issues. + * Exclude global elements such as navigation and footer. + * @param {Object} page + * @returns {Promise} results + */ +async function scanPage(page) { + return await new AxeBuilder({ page }) + .exclude(navigationLocator) + .exclude(footerLocator) + .analyze(); +} + +/** + * Scan a specific element on the page for a11y issues. + * @param {Object} page + * @returns {Promise} results + */ +async function scanPageElement(page, locator) { + return new AxeBuilder({ page }).include(locator).analyze(); +} + +module.exports = { createReport, scanPage, scanPageElement }; diff --git a/tests/playwright/specs/a11y/includes/locators.js b/tests/playwright/specs/a11y/includes/locators.js new file mode 100644 index 00000000000..f6d493713ac --- /dev/null +++ b/tests/playwright/specs/a11y/includes/locators.js @@ -0,0 +1,12 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +'use strict'; + +const navigationLocator = '.c-navigation'; +const footerLocator = '.mzp-c-footer'; + +module.exports = { navigationLocator, footerLocator }; diff --git a/tests/playwright/specs/a11y/includes/urls.js b/tests/playwright/specs/a11y/includes/urls.js new file mode 100644 index 00000000000..15a4571e100 --- /dev/null +++ b/tests/playwright/specs/a11y/includes/urls.js @@ -0,0 +1,12 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +'use strict'; + +const desktopTestURLs = ['/en-US/', '/en-US/about/']; +const mobileTestURLs = []; + +module.exports = { desktopTestURLs, mobileTestURLs }; diff --git a/tests/playwright/specs/a11y/pages.spec.js b/tests/playwright/specs/a11y/pages.spec.js new file mode 100644 index 00000000000..ad54a0e1b47 --- /dev/null +++ b/tests/playwright/specs/a11y/pages.spec.js @@ -0,0 +1,56 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +'use strict'; + +const { test, expect } = require('@playwright/test'); +const openPage = require('../../scripts/open-page'); +const { createReport, scanPage } = require('./includes/helpers'); +const { desktopTestURLs, mobileTestURLs } = require('./includes/urls'); + +for (const url of desktopTestURLs) { + test.describe( + `${url} page (desktop)`, + { + tag: '@a11y' + }, + () => { + test.beforeEach(async ({ page, browserName }) => { + await openPage(url, page, browserName); + }); + + test('should not have any detectable a11y issues', async ({ + page + }) => { + const results = await scanPage(page); + await createReport(url, 'desktop', results); + expect(results.violations.length).toEqual(0); + }); + } + ); +} + +for (const url of mobileTestURLs) { + test.describe( + `${url} page (mobile)`, + { + tag: '@a11y' + }, + () => { + test.beforeEach(async ({ page, browserName }) => { + await openPage(url, page, browserName); + }); + + test('should not have any detectable a11y issues', async ({ + page + }) => { + const results = await scanPage(page); + await createReport(url, 'mobile', results); + expect(results.violations.length).toEqual(0); + }); + } + ); +}