Fix mobile listeners and other improvements (#177)

* Trim whitespace from labels and discard long ones

Signed-off-by: Emanuele Feliziani <feliziani.emanuele@gmail.com>

* Improve matchers and selectors

Signed-off-by: Emanuele Feliziani <feliziani.emanuele@gmail.com>

* Improve statistics with real false positives

Signed-off-by: Emanuele Feliziani <feliziani.emanuele@gmail.com>

* Improve tests to support false positive stats

Signed-off-by: Emanuele Feliziani <feliziani.emanuele@gmail.com>

* Use proper userPreferences in Android test mocks

Signed-off-by: Emanuele Feliziani <feliziani.emanuele@gmail.com>

* Use platform.name for device detection

Signed-off-by: Emanuele Feliziani <feliziani.emanuele@gmail.com>

* Move onPointerDown from Apple to InterfacePrototyp

Signed-off-by: Emanuele Feliziani <feliziani.emanuele@gmail.com>

* Move the event handler and options from Overlay to UIController

Signed-off-by: Emanuele Feliziani <feliziani.emanuele@gmail.com>

* Add capture to the keypress event listener

Signed-off-by: Emanuele Feliziani <feliziani.emanuele@gmail.com>

* Improve selector

Signed-off-by: Emanuele Feliziani <feliziani.emanuele@gmail.com>

* Improve matching for German

Signed-off-by: Emanuele Feliziani <feliziani.emanuele@gmail.com>

* Extract buttonMatchesFormType to utils

Signed-off-by: Emanuele Feliziani <feliziani.emanuele@gmail.com>

* Improve form submission and focus detection

Signed-off-by: Emanuele Feliziani <feliziani.emanuele@gmail.com>

* Submission detection now tries to read buttons outside the form

Signed-off-by: Emanuele Feliziani <feliziani.emanuele@gmail.com>

* Delay autofill startup until document is visible

Signed-off-by: Emanuele Feliziani <feliziani.emanuele@gmail.com>

* Pass onPointerDown to the NativeUIController

Signed-off-by: Emanuele Feliziani <feliziani.emanuele@gmail.com>

* Commit compiled files

Signed-off-by: Emanuele Feliziani <feliziani.emanuele@gmail.com>

* Add Android listeners and tests

Signed-off-by: Emanuele Feliziani <feliziani.emanuele@gmail.com>

* remove androidContentScopeReplacements

* android (current) doesn't support `platform.name` yet, so fallback to UA check

Co-authored-by: Shane Osbourne <shane.osbourne8@gmail.com>

integration-test/helpers/mocks.js

@@ -6,7 +6,8 @@ export const constants = {
         'overlay': 'overlay.html',
         'email-autofill': 'email-autofill.html',
         'signup': 'signup.html',
-        'login': 'login.html'
+        'login': 'login.html',
+        'loginWithPoorForm': 'login-poor-form.html'
     },
     fields: {
         email: {

integration-test/helpers/pages.js

@@ -233,10 +233,16 @@ export function loginPage (page, server, opts = {}) {
             await page.type('#email', data.username)
             await page.click('#login button[type="submit"]')
         },
-        async shouldNotPromptToSave () {
-            const calls = await page.evaluate('window.__playwright.mocks.calls')
-            // todo(Shane): is it too apple specific?
-            const mockCalls = calls.filter(([name]) => name === 'pmHandlerStoreData')
+        /** @param {Platform} platform */
+        async shouldNotPromptToSave (platform = 'ios') {
+            let mockCalls = []
+            if (['ios', 'macos'].includes(platform)) {
+                mockCalls = await mockedCalls(page, ['pmHandlerStoreData'])
+            }
+            if (platform === 'android') {
+                mockCalls = await mockedCalls(page, ['storeFormData'])
+            }
+
             expect(mockCalls.length).toBe(0)
         },
         /** @param {string} mockCallName */
@@ -301,6 +307,23 @@ export function loginPage (page, server, opts = {}) {
     }
 }
 
+/**
+ * A wrapper around interactions for `integration-test/pages/login-poor-form.html`
+ *
+ * @param {import("playwright").Page} page
+ * @param {ServerWrapper} server
+ * @param {{overlay?: boolean, clickLabel?: boolean}} [opts]
+ */
+export function loginPageWithPoorForm (page, server, opts) {
+    const originalLoginPage = loginPage(page, server, opts)
+    return {
+        ...originalLoginPage,
+        async navigate () {
+            await page.goto(server.urlForPath(constants.pages['loginWithPoorForm']))
+        }
+    }
+}
+
 /**
  * A wrapper around interactions for `integration-test/pages/email-autofill.html`
  *

integration-test/pages/login-poor-form.html

@@ -0,0 +1,30 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width">
+  <title>Login form</title>
+  <link rel="stylesheet" href="./style.css" />
+</head>
+
+<body>
+<p><a href="../index.html">[Home]</a></p>
+
+<p id="demo"></p>
+
+<div class="dialog">
+  <div id="login-form">
+    <label for="email">Email</label>
+    <input id="email" type="email">
+    <label for="password">Password</label>
+    <input id="password" type="password">
+  </div>
+</div>
+<div class="fixed">
+  <div role="button" class="button">Log in</div>
+  <a href="#" role="button">Sign up</a>
+</div>
+</body>
+
+</html>

integration-test/tests/email-autofill.android.spec.js

@@ -38,6 +38,7 @@ test.describe('android', () => {
 
             // create + inject the script
             await createAutofillScript()
+                .replaceAll(androidStringReplacements())
                 .platform('android')
                 .applyTo(page)
 

integration-test/tests/save-prompts.android.spec.js

@@ -4,7 +4,7 @@ import {
     setupServer, withAndroidContext
 } from '../helpers/harness.js'
 import {test as base} from '@playwright/test'
-import {loginPage, signupPage} from '../helpers/pages.js'
+import {loginPage, loginPageWithPoorForm, signupPage} from '../helpers/pages.js'
 import {androidStringReplacements, createAndroidMocks} from '../helpers/mocks.android.js'
 import {constants} from '../helpers/mocks.js'
 
@@ -124,4 +124,60 @@ test.describe('Android Save prompts', () => {
             await login.promptWasNotShown()
         })
     })
+
+    test.describe('Prompting to save from a poor login form (using Enter and click on a button outside the form)', () => {
+        const credentials = {
+            username: 'dax@wearejh.com',
+            password: '123456'
+        }
+        /**
+         * @param {import("playwright").Page} page
+         */
+        async function setup (page) {
+            await forwardConsoleMessages(page)
+            const login = loginPageWithPoorForm(page, server)
+            await login.navigate()
+
+            await createAndroidMocks()
+                .applyTo(page)
+            await createAutofillScript()
+                .replaceAll(androidStringReplacements({
+                    featureToggles: {
+                        credentials_saving: true
+                    }
+                }))
+                .platform('android')
+                .applyTo(page)
+
+            await page.type('#password', credentials.password)
+            await page.type('#email', credentials.username)
+
+            // Check that we haven't detected any submission at this point
+            await login.shouldNotPromptToSave()
+
+            return login
+        }
+
+        test('submit by clicking on the out-of-form button', async ({page}) => {
+            const login = await setup(page)
+
+            await page.click('"Log in"')
+            await login.assertWasPromptedToSave(credentials, 'android')
+        })
+        test('should not prompt if the out-of-form button does not match the form type', async ({page}) => {
+            const login = await setup(page)
+
+            await page.click('"Sign up"')
+            await login.shouldNotPromptToSave()
+        })
+        test('should prompt when hitting enter while an input is focused', async ({page}) => {
+            const login = await setup(page)
+
+            await page.press('#email', 'Tab')
+            await login.shouldNotPromptToSave()
+
+            await page.press('#password', 'Enter')
+            await login.assertWasPromptedToSave(credentials, 'android')
+        })
+    })
 })

integration-test/tests/save-prompts.ios.spec.js

@@ -4,7 +4,7 @@ import {
     withIOSContext, withIOSFeatureToggles
 } from '../helpers/harness.js'
 import {test as base} from '@playwright/test'
-import {loginPage, signupPage} from '../helpers/pages.js'
+import {loginPage, loginPageWithPoorForm, signupPage} from '../helpers/pages.js'
 import {createWebkitMocks} from '../helpers/mocks.webkit.js'
 import {constants} from '../helpers/mocks.js'
 
@@ -123,5 +123,54 @@ test.describe('iOS Save prompts', () => {
                 await login.shouldNotPromptToSave()
             })
         })
+
+        test.describe('Prompting to save from a poor login form (using Enter and click on a button outside the form)', () => {
+            const credentials = {
+                username: 'dax@wearejh.com',
+                password: '123456'
+            }
+            /**
+             * @param {import("playwright").Page} page
+             */
+            async function setup (page) {
+                await forwardConsoleMessages(page)
+                await createWebkitMocks().applyTo(page)
+                await withIOSFeatureToggles(page, {
+                    credentials_saving: true
+                })
+                const login = loginPageWithPoorForm(page, server)
+                await login.navigate()
+
+                await page.type('#password', credentials.password)
+                await page.type('#email', credentials.username)
+
+                // Check that we haven't detected any submission at this point
+                await login.shouldNotPromptToSave()
+
+                return login
+            }
+
+            test('submit by clicking on the out-of-form button', async ({page}) => {
+                const login = await setup(page)
+
+                await page.click('"Log in"')
+                await login.assertWasPromptedToSave(credentials)
+            })
+            test('should not prompt if the out-of-form button does not match the form type', async ({page}) => {
+                const login = await setup(page)
+
+                await page.click('"Sign up"')
+                await login.shouldNotPromptToSave()
+            })
+            test('should prompt when hitting enter while an input is focused', async ({page}) => {
+                const login = await setup(page)
+
+                await page.press('#email', 'Tab')
+                await login.shouldNotPromptToSave()
+
+                await page.press('#password', 'Enter')
+                await login.assertWasPromptedToSave(credentials)
+            })
+        })
     })
 })

src/DeviceInterface/AndroidInterface.js

@@ -19,7 +19,9 @@ class AndroidInterface extends InterfacePrototype {
      * @override
      */
     createUIController () {
-        return new NativeUIController()
+        return new NativeUIController({
+            onPointerDown: (event) => this._onPointerDown(event)
+        })
     }
 
     /**

src/DeviceInterface/AppleDeviceInterface.js

@@ -29,7 +29,9 @@ class AppleDeviceInterface extends InterfacePrototype {
      */
     createUIController () {
         if (this.globalConfig.userPreferences?.platform?.name === 'ios') {
-            return new NativeUIController()
+            return new NativeUIController({
+                onPointerDown: (event) => this._onPointerDown(event)
+            })
         }
 
         if (!this.globalConfig.supportsTopFrame) {
@@ -314,32 +316,6 @@ class AppleDeviceInterface extends InterfacePrototype {
             poll()
         })
     }
-    /**
-     * on macOS we try to detect if a click occurred within a form
-     * @param {PointerEvent} event
-     */
-    _onPointerDown (event) {
-        if (this.settings.featureToggles.credentials_saving) {
-            this._detectFormSubmission(event)
-        }
-    }
-    /**
-     * @param {PointerEvent} event
-     */
-    _detectFormSubmission (event) {
-        const matchingForm = [...this.scanner.forms.values()].find(
-            (form) => {
-                const btns = [...form.submitButtons]
-                // @ts-ignore
-                if (btns.includes(event.target)) return true
-
-                // @ts-ignore
-                if (btns.find((btn) => btn.contains(event.target))) return true
-            }
-        )
-
-        matchingForm?.submitHandler()
-    }
 }
 
 export {AppleDeviceInterface}

src/DeviceInterface/InterfacePrototype.js

@@ -4,10 +4,10 @@ import {
     sendAndWaitForAnswer,
     formatDuckAddress,
     autofillEnabled,
-    notifyWebApp, getDaxBoundingBox
+    notifyWebApp, getDaxBoundingBox, buttonMatchesFormType
 } from '../autofill-utils'
 
-import { getInputType, getSubtypeFromType } from '../Form/matching'
+import {getInputType, getSubtypeFromType, removeExcessWhitespace} from '../Form/matching'
 import { formatFullName } from '../Form/formatters'
 import listenForGlobalFormSubmission from '../Form/listenForFormSubmission'
 import { fromPassword, appendGeneratedId, AUTOGENERATED_KEY } from '../InputTypes/Credentials'
@@ -19,6 +19,7 @@ import {createTransport} from '../deviceApiCalls/transports/transports'
 import {Settings} from '../Settings'
 import {DeviceApi} from '../../packages/device-api'
 import {StoreFormDataCall} from '../deviceApiCalls/__generated__/deviceApiCalls'
+import {SUBMIT_BUTTON_SELECTOR} from '../Form/selectors-css'
 
 /**
  * @typedef {import('../deviceApiCalls/__generated__/validators-ts').StoreFormData} StoreFormData
@@ -610,6 +611,48 @@ class InterfacePrototype {
             this.storeFormData(withAutoGeneratedFlag)
         }
     }
+    /**
+     * on macOS we try to detect if a click occurred within a form
+     * @param {PointerEvent} event
+     */
+    _onPointerDown (event) {
+        if (this.settings.featureToggles.credentials_saving) {
+            this._detectFormSubmission(event)
+        }
+    }
+    /**
+     * @param {PointerEvent} event
+     */
+    _detectFormSubmission (event) {
+        const matchingForm = [...this.scanner.forms.values()].find(
+            (form) => {
+                const btns = [...form.submitButtons]
+                // @ts-ignore
+                if (btns.includes(event.target)) return true
+
+                // @ts-ignore
+                if (btns.find((btn) => btn.contains(event.target))) return true
+            }
+        )
+
+        matchingForm?.submitHandler()
+
+        if (!matchingForm) {
+            // check if the click happened on a button
+            const button = /** @type HTMLElement */(event.target)?.closest(SUBMIT_BUTTON_SELECTOR)
+            if (!button) return
+
+            const text = removeExcessWhitespace(button?.textContent)
+            const hasRelevantText = /(log|sign).?(in|up)|continue|next|submit/i.test(text)
+            if (hasRelevantText && text.length < 25) {
+                // check if there's a form with values
+                const filledForm = [...this.scanner.forms.values()].find(form => form.hasValues())
+                if (filledForm && buttonMatchesFormType(/** @type HTMLElement */(button), filledForm)) {
+                    filledForm?.submitHandler()
+                }
+            }
+        }
+    }
 
     /**
      * This serves as a single place to create a default instance

src/Form/Form.js

@@ -6,7 +6,7 @@ import {
     setValue,
     isEventWithinDax,
     isLikelyASubmitButton,
-    isVisible
+    isVisible, buttonMatchesFormType
 } from '../autofill-utils'
 
 import { getInputSubtype, getInputMainType, createMatching } from './matching'
@@ -85,11 +85,12 @@ class Form {
     }
 
     /**
-     * Checks if the form element contains the activeElement
+     * Checks if the form element contains the activeElement or the event target
      * @return {boolean}
+     * @param {KeyboardEvent | null} [e]
      */
-    hasFocus () {
-        return this.form.contains(document.activeElement)
+    hasFocus (e) {
+        return this.form.contains(document.activeElement) || this.form.contains(/** @type HTMLElement */(e?.target))
     }
 
     /**
@@ -225,17 +226,9 @@ class Form {
         const allButtons = /** @type {HTMLElement[]} */([...this.form.querySelectorAll(selector)])
 
         return allButtons
-            .filter(isLikelyASubmitButton)
-            // filter out buttons of the wrong type - login buttons on a signup form, signup buttons on a login form
-            .filter((button) => {
-                if (this.isLogin) {
-                    return !/sign.?up/i.test(button.textContent || '')
-                } else if (this.isSignup) {
-                    return !/(log|sign).?([io])n/i.test(button.textContent || '')
-                } else {
-                    return true
-                }
-            })
+            .filter((btn) =>
+                isLikelyASubmitButton(btn) && buttonMatchesFormType(btn, this)
+            )
     }
 
     /**