workflow

.github/workflows/wpt-test.yml

@@ -0,0 +1,96 @@
+name: Tests
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "src/technologies/*.json"
+      - "src/categories.json"
+      - "src/groups.json"
+      - "tests/*.js"
+  workflow_dispatch:
+
+jobs:
+  test:
+    name: WebPageTest Test Cases
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Lint
+        run: npm run lint
+
+      - name: Run WebPageTest with unit tests
+        env:
+          WPT_SERVER: "webpagetest.httparchive.org"
+          WPT_API_KEY: ${{ secrets.HA_API_KEY }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        if: ${{ env.WPT_API_KEY != '' }}
+        run: |
+          echo "::group::Unit tests"
+            npm test
+          echo "::endgroup::"
+
+      - name: Run WebPageTest for more websites
+        env:
+          WPT_SERVER: "webpagetest.httparchive.org"
+          WPT_API_KEY: ${{ secrets.HA_API_KEY }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          # Get the PR body
+          PR_BODY="$(cat <<'EOF'
+          ${{ github.event.pull_request.body }}
+          EOF
+          )"
+
+          # Read PR body into an array, removing line breaks and carriage returns
+          declare -a lines
+          while IFS= read -r line; do
+            lines+=("${line//[$'\r\n']}")
+          done <<< "$PR_BODY"
+
+          # Find the index of the line after "**Test websites**:"
+          start_index=-1
+          for ((i=0; i<${#lines[@]}; i++)); do
+            if [[ "${lines[$i]}" == *"**Test websites**:"* ]]; then
+              start_index=$((i + 1))
+              break
+            fi
+          done
+
+          # If the index is valid, then parse the URLs
+          if [ $start_index -gt -1 ]; then
+            # Initialize an array for URLs
+            declare -a URLS
+            url_pattern="((http|https|ftp):\/\/[a-zA-Z0-9.-]+(\.[a-zA-Z]{2,4})(\/[a-zA-Z0-9_.-]+)*(\/?)(\?[a-zA-Z0-9_.-]+=[a-zA-Z0-9%_.-]+)*(\#?)([a-zA-Z0-9%_.-=]+)*)"
+
+            for ((i=start_index; i<${#lines[@]}; i++)); do
+              if [[ ${lines[$i]} =~ $url_pattern ]]; then
+                URLS+=("${BASH_REMATCH[1]}")
+              fi
+            done
+
+            # Run WebPageTest for each URL
+            for TEST_WEBSITE in "${URLS[@]}"; do
+              echo "::group::Detecting technologies for $TEST_WEBSITE"
+              node tests/wpt.js "$TEST_WEBSITE"
+              echo "::endgroup::"
+            done
+          else
+            echo "No websites found."
+          fi
+
+      - name: Add comment with results
+        uses: mshick/add-pr-comment@v2
+        if: always()
+        with:
+          refresh-message-position: true
+          message-path: test-results.md

tests/unit-tests.test.js

@@ -0,0 +1,12 @@
+const assert = require('assert')
+const { runWPTTest } = require('./wpt.js')
+const testWebsite = 'https://almanac.httparchive.org/en/2022/'
+
+let responseData
+beforeAll(async () => {
+  responseData = await runWPTTest(testWebsite)
+}, 400000)
+
+test('wappalyzer successful', () => {
+  assert.ok(responseData.runs['1'].firstView.wappalyzer_failed === 0)
+})

tests/wpt.js

@@ -0,0 +1,58 @@
+const fs = require('fs')
+const { argv } = require('node:process')
+const WebPageTest = require('webpagetest')
+
+const isDirectRun = require.main === module
+
+const wptServer = process.env.WPT_SERVER
+const wptApiKey = process.env.WPT_API_KEY
+const PRnumber = parseInt(process.env.PR_NUMBER)
+const wpt = new WebPageTest(wptServer, wptApiKey)
+
+/**
+ * Runs a WebPageTest (WPT) test for a given URL.
+ *
+ * @param {string} url - The URL to run the test on.
+ * @returns {Promise<object>} A promise that resolves with an object containing the custom metrics.
+ * @throws {Error} If the test run fails or the response status code is not 200.
+ */
+function runWPTTest(url) {
+  const options = { key: wptApiKey, wappalyzerpr: PRnumber }
+
+  return new Promise((resolve, reject) => {
+    wpt.runTestAndWait(url, options, (error, response) => {
+      if (error || response.statusCode !== 200) {
+        reject(error || response)
+      } else {
+        const technologies = {
+          detected: response.data.runs['1'].firstView.detected,
+          detected_apps: response.data.runs['1'].firstView.detected_apps,
+          detected_technologies:
+            response.data.runs['1'].firstView.detected_technologies,
+          detected_raw: response.data.runs['1'].firstView.detected_raw,
+        }
+
+        fs.appendFileSync(
+          'test-results.md',
+          '<details>\n' +
+            `<summary><strong>Custom metrics for ${url}</strong></summary>\n\n` +
+            `WPT test run results: ${response.data.summary}\n` +
+            (isDirectRun
+              ? 'Changed custom metrics values:\n' +
+                `\`\`\`json\n${JSON.stringify(technologies, null, 4)}\n\`\`\`\n`
+              : '') +
+            '</details>\n'
+        )
+
+        resolve(response.data)
+      }
+    })
+  })
+}
+
+if (isDirectRun) {
+  const url = argv[2]
+  runWPTTest(url)
+}
+
+module.exports = { runWPTTest }