Merge pull request #372 from GoogleChromeLabs/playwright-mvp

Playwright Test Suite: realtime-sine (MVP)
GoogleChromeLabs · Jul 17, 2024 · 2d19312 · 2d19312
2 parents ec8dc80 + 87a4138
commit 2d19312
Show file tree

Hide file tree

Showing 18 changed files with 2,206 additions and 4,677 deletions.
diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml
@@ -0,0 +1,30 @@
+name: Playwright Tests
+on:
+  push:
+    branches: [ main, master ]
+  pull_request:
+    branches: [ main, master ]
+jobs:
+  test:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-node@v4
+      with:
+        node-version: lts/*
+    - name: Install dependencies
+      run: npm ci
+    - name: Install Playwright Browsers
+      run: npx playwright install --with-deps chromium
+    - name: Run Playwright tests
+      # Only test on Chromium
+      # Each version of playwright is tied to the latest version of Chromium (at the time). The version is listed here:
+      # https://github.com/microsoft/playwright?tab=readme-ov-file#documentation--api-reference
+      run: npx playwright test --project=chromium
+    - uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: playwright-report
+        path: playwright-report/
+        retention-days: 30
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,9 @@
 .DS_Store
 node_modules/
-_site/
+_site/
+
+## Playwright-specific ignores
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -11,7 +11,10 @@
     "start": "run-p start:*",
     "start:eleventy": "eleventy --serve",
     "start:postcss": "postcss src/styles/*.css --dir _site --watch",
-    "format": "npx eslint --fix _site/audio-worklet/**/*.js && npx prettier --write --loglevel silent _site/audio-worklet/**/*.html"
+    "format": "npx eslint --fix _site/audio-worklet/**/*.js && npx prettier --write --loglevel silent _site/audio-worklet/**/*.html",
+    "test": "npx playwright test",
+    "test-server": "npx http-server",
+    "test-live": "npx http-server ./src/tests/playwright/pages/"
   },
   "license": "MIT",
   "dependencies": {
@@ -29,5 +32,9 @@
     "prettier": "^3.2.5",
     "rimraf": "^5.0.7",
     "tailwindcss": "^3.4.3"
+  },
+  "devDependencies": {
+    "@playwright/test": "latest",
+    "@types/node": "^20.12.13"
   }
 }
diff --git a/playwright.config.ts b/playwright.config.ts
@@ -0,0 +1,46 @@
+import { defineConfig, devices } from '@playwright/test';
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: './src/tests/playwright',
+  // Run tests in files in parallel
+  fullyParallel: true,
+  // Fail the build on CI if you accidentally left test.only in the source code.
+  forbidOnly: !!process.env.CI,
+  // Retry on CI only
+  retries: process.env.CI ? 2 : 0,
+  // Opt out of parallel tests on CI.
+  workers: process.env.CI ? 1 : undefined,
+  // Reporter to use. See https://playwright.dev/docs/test-reporters
+  reporter: 'html',
+  // Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions.
+  use: {
+    // Base URL to use in actions like `await page.goto('/')`.
+    baseURL: 'http://127.0.0.1:8080/src/tests/playwright/',
+    // Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer
+    trace: 'on-first-retry',
+  },
+
+  // Configure projects for major browsers
+  projects: [
+    {
+      name: 'chromium',
+      use: {
+        ...devices['Desktop Chrome'],
+        launchOptions: {
+          ignoreDefaultArgs: ['--mute-audio'],
+          args: ['--autoplay-policy=no-user-gesture-required']
+        },
+      }
+    }
+  ],
+
+  // Run your local dev server before starting the tests
+  webServer: {
+    command: 'npm run test-server',
+    url: 'http://127.0.0.1:8080',
+    reuseExistingServer: !process.env.CI,
+  },
+});
diff --git a/src/_data/build_info.json b/src/_data/build_info.json
@@ -1 +1 @@
-{"version":"3.2.0","revision":"c722df0","lastUpdated":"2024-05-14","copyrightYear":2024}
+{"version":"3.2.0","revision":"cb35e25","lastUpdated":"2024-05-17","copyrightYear":2024}
diff --git a/src/tests/playwright/pages/index.html b/src/tests/playwright/pages/index.html
@@ -0,0 +1,34 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8">
+    <link rel="stylesheet" href="https://unpkg.com/terminal.css@0.7.4/dist/terminal.min.css" />
+    <link rel="stylesheet" href="live-suite/style.css" />
+    <script defer type="module" src="live-suite/scripts/main.js"></script>
+    <title>Web Audio Test Suite</title>
+  </head>
+  <body class="terminal">
+    <h1 class="terminal-prompt">Web Audio Test Suite</h1>
+    <table>
+      <thead>
+        <tr>
+          <th>Test Name</th>
+          <th>Result</th>
+          <th>Time</th>
+          <th>Output</th>
+          <th>Run</th>
+        </tr>
+      </thead>
+      <tbody></tbody>
+    </table>
+    <template id="row">
+      <tr>
+        <td><slot name="name">Unnamed Test</slot></td>
+        <td><slot name="result">---</slot></td>
+        <td><slot name="time">---</slot></td>
+        <td><pre><slot name="output">---</slot></pre></td>
+        <td><button class="btn btn-block btn-primary">start</button></td>
+      </tr>
+    </template>
+  </body>
+</html>
diff --git a/src/tests/playwright/pages/live-suite/scripts/console-override.js b/src/tests/playwright/pages/live-suite/scripts/console-override.js
@@ -0,0 +1,19 @@
+/**
+ * @fileoverview Overrides console methods to capture outputs in `output` array
+ * for debugging/tests. Handles common methods like log, error, info, and more,
+ * storing { method, arguments }.
+ */
+export const output = [];
+
+[
+  'log', 'debug', 'info', 'error', 'warning', 'dir', 'dirxml',
+  'table', 'trace', 'clear', 'group', 'groupCollapsed', 'groupEnd',
+  'assert', 'profile', 'profileEnd', 'count', 'timeEnd',
+].forEach((method) => {
+  const original = console[method];
+  console[method] = (...args) => {
+    (method !== 'assert' || (method === 'assert' && !args[0])) &&
+        output.push({method, args});
+    return original.apply(console, args);
+  };
+});
diff --git a/src/tests/playwright/pages/live-suite/scripts/main.js b/src/tests/playwright/pages/live-suite/scripts/main.js
@@ -0,0 +1,14 @@
+/**
+ * @fileoverview Initializes the Web Audio Test Suite by converting specified
+ * HTML test files to interactive DOM elements.
+ */
+import {convertTestFiles} from './test-file-converter.js';
+
+// Flag for live test suite environment
+window._isTestSuiteMode = true;
+
+const files = [
+  'realtime-sine.html',
+];
+
+convertTestFiles(files);
diff --git a/src/tests/playwright/pages/live-suite/scripts/test-file-converter.js b/src/tests/playwright/pages/live-suite/scripts/test-file-converter.js
@@ -0,0 +1,60 @@
+/**
+ * @fileoverview Provides utilities for converting HTML test files to
+ * interactive DOM elements in the Web Audio Test Suite, including dynamic test
+ * execution, results display, and console output management.
+ */
+import {output} from './console-override.js';
+
+export const convertTestFiles = async (tests) => {
+  const htmls = await Promise.all(
+      tests.map(async (t) => (await fetch(t)).text()));
+
+  const template = document.querySelector('#row');
+  htmls.forEach((html) => {
+    const dom = new DOMParser()
+        .parseFromString(html, 'text/html');
+    const scriptContent = dom.querySelector('script').innerText;
+
+    const tr = template.content.cloneNode(true);
+    const id = dom.title
+        .replace(/[^a-z0-9]/gi, '-')
+        .toLowerCase();
+    tr.childNodes[1].id = id;
+    tr.querySelector('slot[name=name]').textContent = dom.title;
+    tr.querySelector('button').addEventListener('click', async () => {
+      document.querySelectorAll('button').forEach((b) => b.disabled = true);
+
+      const script = document.createElement('script');
+      script.defer = true;
+      script.async = true;
+      script.type = 'module';
+      script.textContent = scriptContent;
+      document.head.appendChild(script);
+
+      // await until script loads in the live suite
+      await new Promise((resolve) => window._webAudioTestIsRunning = resolve);
+      const start = performance.now();
+      await window._webAudioTest;
+      const diff = performance.now() - start;
+
+      document.querySelector(`#${id} slot[name=result]`).textContent =
+          await window.webAudioEvaluate() ? '✅': '❌';
+      document.querySelector(`#${id} slot[name=time]`).textContent =
+          `${(diff).toFixed(2)}ms`;
+      document.querySelector(`#${id} pre slot[name=output]`).textContent =
+          output.map(({method, args}) =>
+            `${method}: ${args.join(' ')}`,
+          ).join('\n') || '---';
+
+      output.length = 0;
+
+      delete window._webAudioTest;
+      delete window.webAudioEvaluate;
+      document.head.removeChild(script);
+
+      document.querySelectorAll('button').forEach((b) => b.disabled = false);
+    });
+
+    document.querySelector('tbody').appendChild(tr);
+  });
+};
diff --git a/src/tests/playwright/pages/live-suite/style.css b/src/tests/playwright/pages/live-suite/style.css
@@ -0,0 +1,36 @@
+:root {
+  --global-font-size: 15px; /* The Base font size */
+  --global-line-height: 1.4em; /* The baseline height. Modify this to achieve the best readability. */
+  --global-space: 10px; /* A global spacer used to consistently space things */
+  --font-stack: Menlo, Monaco, Courier New, monospace, serif; /* The fonts for the website. Use @font-face or any other font provider to include your custom fonts. */
+  --mono-font-stack: Menlo, Monaco, Courier New, monospace, serif; /* Same as above but for code. */
+  --background-color: #222225; /* The page background color */
+  --font-color: #e8e9ed; /* The base font color for text, headlines, blockquotes, lists, etc. */
+  --invert-font-color: #222225; /* Sometimes text appears on a colored background. Adjust this color to improve readability. */
+  --primary-color: #62c4ff; /* The primary color is used for links and buttons. */
+  --secondary-color: #a3abba; /* The secondary color is more subtle than the primary color. It's used for code highlighting and image captions. */
+  --error-color: #ff3c74; /* Used for error alerts and form validation. */
+  --progress-bar-background: #3f3f44; /* The background color of progress bars. */
+  --progress-bar-fill: #62c4ff; /* The fill color, indicating the progress in progress bars. */
+  --code-bg-color: #3f3f44; /* The background color of <code> elements. */
+  --block-background-color: #3f3f44; /* The background color of <pre> elements. Also applies to <code> elements inside a <pre>. */
+}
+
+body {
+  margin: 1vh 10vw;
+}
+
+button[disabled], button[disabled]:hover {
+  background: var(--secondary-color);
+  cursor: not-allowed;
+}
+
+pre {
+  font-size: 8pt;
+  max-height: 40vh;
+  overflow: auto;
+}
+
+td {
+  vertical-align: middle !important;
+}
diff --git a/src/tests/playwright/pages/realtime-sine.html b/src/tests/playwright/pages/realtime-sine.html
@@ -0,0 +1,47 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <script defer type="module">
+      import {record} from './util/recorder/recorder-main.js';
+      import {beCloseTo, test, evaluateTest} from './util/audit.js';
+
+      const expectedAccuracy = 0.998;
+
+      test((async () => {
+        const context = new AudioContext({sampleRate: 48000});
+        const oscillator = new OscillatorNode(context);
+        const duration = 1; // second
+        const {recorder, recordingCompletePromise} = await record(context, duration);
+        oscillator.connect(recorder).connect(context.destination);
+
+        oscillator.start();
+
+        const recordedChannelBuffers = await recordingCompletePromise;
+        const float32Buffer = recordedChannelBuffers[0];
+
+        await context.close();
+        return {buffer: float32Buffer};
+      })());
+
+      evaluateTest(async testResult => {
+        const bufferData = testResult.buffer;
+        const myRefData = await (await fetch('./reference/440@48k-sine-octx.json')).json();
+
+        // compare bufferData samples to reference
+        let numberOfAcceptableSamples = 0;
+        for (let i = 0; i < bufferData.length; i++) {
+          numberOfAcceptableSamples += beCloseTo(bufferData[i], myRefData[i], 0.01) ? 1 : 0;
+        }
+        console.info('% match:', numberOfAcceptableSamples / bufferData.length);
+
+        // return true if 99.8% pass
+        return numberOfAcceptableSamples / bufferData.length > expectedAccuracy;
+      });
+    </script>
+    <title>Realtime Sine Oscillator Test</title>
+  </head>
+  <body>
+    <h1>Realtime Sine Oscillator Test</h1>
+    <p>Play a sine wave at 440Hz for 1 second at 48kHz sample rate.</p>
+  </body>
+</html>
diff --git a/src/tests/playwright/pages/reference/440@48k-sine-octx.json b/src/tests/playwright/pages/reference/440@48k-sine-octx.json
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		{"version":"3.2.0","revision":"c722df0","lastUpdated":"2024-05-14","copyrightYear":2024}
		{"version":"3.2.0","revision":"cb35e25","lastUpdated":"2024-05-17","copyrightYear":2024}