From 89dc52c456d3b5f2ff5d9095a4bc208e652d4c36 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Tue, 3 Oct 2023 12:16:05 +0700 Subject: [PATCH 01/12] feat(web): start of gestureSetForKeyboard --- .../src/input/gestures/specsForKeyboard.ts | 77 ++++++++++++++++++- web/src/engine/osk/src/visualKeyboard.ts | 4 +- 2 files changed, 76 insertions(+), 5 deletions(-) diff --git a/web/src/engine/osk/src/input/gestures/specsForKeyboard.ts b/web/src/engine/osk/src/input/gestures/specsForKeyboard.ts index 8e5b49e56a6..59468c573fb 100644 --- a/web/src/engine/osk/src/input/gestures/specsForKeyboard.ts +++ b/web/src/engine/osk/src/input/gestures/specsForKeyboard.ts @@ -17,7 +17,26 @@ import specs = gestures.specs; * @param keyboard * @returns */ -export function modelSetForKeyboard(keyboard: Keyboard): GestureModelDefs { +export function gestureSetForKeyboard(keyboard: Keyboard): GestureModelDefs { + // To be used among the `allowsInitialState` contact-model specifications as needed. + const gestureKeyFilter = (key: KeyElement, gestureId: string) => { + const keySpec = key.key.spec; + switch(gestureId) { + case 'special-key-start': + return ['K_LOPT', 'K_ROPT', 'K_BKSP'].indexOf(keySpec.baseKeyID) != -1; + case 'longpress': + return !!keySpec.sk; + case 'multitap': + //return !!keySpec. // no field specified for this within KMW yet! + return keySpec.baseKeyID == 'K_SHIFT'; + case 'flick': + //return !!keySpec. // no field specified for this within KMW yet! + return false; + default: + return true; + } + }; + // TODO: keyboard-specific config stuff // if `null`, assume a no-flick keyboard (assuming our default layout has no flicks) @@ -31,13 +50,15 @@ export function modelSetForKeyboard(keyboard: Keyboard): GestureModelDefs; // - modipress: keyboard-specific modifier keys - which may require inspection of a // key's properties. +export const SpecialKeyStartModel: GestureModel = { + id: 'special-key-start', + resolutionPriority: 0, + contacts : [ + { + model: { + ...InstantContactResolutionModel, + allowsInitialState: (incoming, dummy, baseItem) => { + // TODO: needs better abstraction, probably. + + // But, to get started... we can just use a simple hardcoded approach. + const modifierKeyIds = ['K_LOPT', 'K_ROPT', 'K_BKSP']; + for(const modKeyId of modifierKeyIds) { + if(baseItem.key.spec.id == modKeyId) { + return true; + } + } + + return false; + } + }, + endOnResolve: false // keyboard-selection longpress - would be nice to not need to lift the finger + // in app/browser form. + } + ], + resolutionAction: { + type: 'chain', + next: 'special-key-end', + item: 'current' + } +} + +export const SpecialKeyEndModel: GestureModel = { + id: 'special-key-end', + resolutionPriority: 0, + contacts : [ + { + model: { + ...SimpleTapContactModel, + itemChangeAction: 'resolve' + }, + endOnResolve: true, + } + ], + resolutionAction: { + type: 'complete', + item: 'none' + } +} + // Is kind of a mix of the two longpress styles. export const LongpressModel: GestureModel = { id: 'longpress', diff --git a/web/src/engine/osk/src/visualKeyboard.ts b/web/src/engine/osk/src/visualKeyboard.ts index 2e821ac41e6..1b19ea88a85 100644 --- a/web/src/engine/osk/src/visualKeyboard.ts +++ b/web/src/engine/osk/src/visualKeyboard.ts @@ -42,7 +42,7 @@ import InternalPendingLongpress from './input/gestures/browser/pendingLongpress. import InternalKeyTip from './input/gestures/browser/keytip.js'; import CommonConfiguration from './config/commonConfiguration.js'; -import { modelSetForKeyboard } from './input/gestures/specsForKeyboard.js'; +import { gestureSetForKeyboard } from './input/gestures/specsForKeyboard.js'; import { getViewportScale } from './screenUtils.js'; @@ -353,7 +353,7 @@ export default class VisualKeyboard extends EventEmitter implements Ke } }; - const recognizer = new GestureRecognizer(modelSetForKeyboard(this.layoutKeyboard), config); + const recognizer = new GestureRecognizer(gestureSetForKeyboard(this.layoutKeyboard), config); recognizer.stateToken = this.layerId; const sourceTrackingMap: Record Date: Wed, 4 Oct 2023 10:21:56 +0700 Subject: [PATCH 02/12] feat(web): layout-based gesture-set selection, key-based filtering --- .../src/keyboards/activeLayout.ts | 70 ++++++++- .../tests/node/keyboard-loading.js | 20 +++ common/web/utils/src/test/deepCopy.js | 74 +++++++++ common/web/utils/src/test/timeoutPromise.js | 13 +- ...{specsForKeyboard.ts => specsForLayout.ts} | 143 ++++++++++++++---- web/src/engine/osk/src/visualKeyboard.ts | 4 +- 6 files changed, 281 insertions(+), 43 deletions(-) create mode 100644 common/web/utils/src/test/deepCopy.js rename web/src/engine/osk/src/input/gestures/{specsForKeyboard.ts => specsForLayout.ts} (73%) diff --git a/common/web/keyboard-processor/src/keyboards/activeLayout.ts b/common/web/keyboard-processor/src/keyboards/activeLayout.ts index 21caac2ee52..230366e4f03 100644 --- a/common/web/keyboard-processor/src/keyboards/activeLayout.ts +++ b/common/web/keyboard-processor/src/keyboards/activeLayout.ts @@ -7,6 +7,7 @@ import type Keyboard from "./keyboard.js"; import { TouchLayout } from "@keymanapp/common-types"; import TouchLayoutDefaultHint = TouchLayout.TouchLayoutDefaultHint; +import TouchLayoutFlick = TouchLayout.TouchLayoutFlick; import { type DeviceSpec } from "@keymanapp/web-utils"; // TS 3.9 changed behavior of getters to make them @@ -22,6 +23,16 @@ function Enumerable( descriptor.enumerable = true; }; +/** + * Designed for use by call-by-reference objects during keyboard-load preprocessing + * to note properties of a keyboard that are only specified at lower levels of the + * layout object. + */ +interface AnalysisMetadata { + hasFlicks: boolean; + hasMultitaps: boolean; + hasLongpresses: boolean; +} class ActiveKeyBase { static readonly DEFAULT_PAD=15; // Padding to left of key, in virtual units @@ -207,7 +218,13 @@ class ActiveKeyBase { rawKey.sp ||= 0; // The default button class. } - static polyfill(key: LayoutKey, keyboard: Keyboard, layout: ActiveLayout, displayLayer: string) { + static polyfill(key: LayoutKey, keyboard: Keyboard, layout: ActiveLayout, displayLayer: string, analysisFlagObj?: AnalysisMetadata) { + analysisFlagObj ||= { + hasFlicks: false, + hasLongpresses: false, + hasMultitaps: false + } + // Add class functions to the existing layout object, allowing it to act as an ActiveLayout. let dummy = new ActiveKeyBase(); let proto = Object.getPrototypeOf(dummy); @@ -226,8 +243,25 @@ class ActiveKeyBase { // Ensure subkeys are also properly extended. if(key.sk) { + analysisFlagObj.hasLongpresses = true; for(let subkey of key.sk) { - ActiveSubkey.polyfill(subkey, keyboard, layout, displayLayer); + ActiveSubkey.polyfill(subkey, keyboard, layout, displayLayer, analysisFlagObj); + } + } + + // Also multitap keys. + if(key.multitap) { + analysisFlagObj.hasMultitaps = true; + for(let mtKey of key.multitap) { + ActiveSubkey.polyfill(mtKey, keyboard, layout, displayLayer, analysisFlagObj); + } + } + + + if(key.flick) { + analysisFlagObj.hasFlicks = true; + for(let flickKey in key.flick) { + ActiveSubkey.polyfill(key.flick[flickKey as keyof TouchLayoutFlick], keyboard, layout, displayLayer, analysisFlagObj); } } @@ -353,7 +387,15 @@ export class ActiveRow implements LayoutRow { } } - static polyfill(row: LayoutRow, keyboard: Keyboard, layout: ActiveLayout, displayLayer: string, totalWidth: number, proportionalY: number) { + static polyfill( + row: LayoutRow, + keyboard: Keyboard, + layout: ActiveLayout, + displayLayer: string, + totalWidth: number, + proportionalY: number, + analysisFlagObj: AnalysisMetadata + ) { // Apply defaults, setting the width and other undefined properties for each key let keys=row['key']; for(let j=0; j { + // -- START: Standard Recorder-based unit test loading boilerplate -- + let harness = new KeyboardHarness({}, MinimalKeymanGlobal); + let keyboardLoader = new NodeKeyboardLoader(harness); + let km_keyboard = await keyboardLoader.loadKeyboardFromPath(khmerPath); + // -- END: Standard Recorder-based unit test loading boilerplate -- + + // `khmer_angkor` - supports longpresses, but not flicks or multitaps. + + const desktopLayout = km_keyboard.layout('desktop'); + assert.isFalse(desktopLayout.hasFlicks); + assert.isFalse(desktopLayout.hasLongpresses); + assert.isFalse(desktopLayout.hasMultitaps); + + const mobileLayout = km_keyboard.layout('phone'); + assert.isFalse(mobileLayout.hasFlicks); + assert.isTrue(mobileLayout.hasLongpresses); + assert.isFalse(mobileLayout.hasMultitaps); + }); }); describe('Full harness loading', () => { diff --git a/common/web/utils/src/test/deepCopy.js b/common/web/utils/src/test/deepCopy.js new file mode 100644 index 00000000000..90e809dc116 --- /dev/null +++ b/common/web/utils/src/test/deepCopy.js @@ -0,0 +1,74 @@ +import { assert } from 'chai'; + +import { deepCopy } from '@keymanapp/web-utils'; + + +describe('deepCopy', function() { + it('simple object', () => { + const original = { + a: 1, + b: '2', + c: () => 3 + }; + const clone = deepCopy(original); + + assert.deepEqual(clone, original); + assert.notEqual(clone, original); + + original.b = 'two'; + assert.equal(clone.b, '2'); + assert.equal(clone.c(), 3); + }); + + it('object with simple array', () => { + const original = { arr: [1, 2, 3, 4, 5] }; + const clone = deepCopy(original); + + assert.deepEqual(clone, original) + assert.sameDeepOrderedMembers(clone.arr, original.arr); + assert.notEqual(clone.arr, original.arr); + assert.notEqual(clone, original); + + original.arr[2] = 13; + assert.equal(clone.arr[2], 3); + }); + + it('complex object', () => { + const original = { + arr: [1, 2, {entries: [3, [4, 5]]}], + nested: { + character: { + first: 'inigo', + last: 'montoya' + }, + actor: { + first: 'mandy', + last: 'patinkin' + }, + getQuote: () => "My name is Inigo Montoya. You killed my father; prepare to die!" + } + } + + const clone = deepCopy(original); + + assert.deepEqual(clone, original); + assert.notEqual(clone, original); + + assert.sameDeepOrderedMembers(clone.arr, original.arr); + assert.notEqual(clone.arr, original.arr); + assert.notEqual(clone.arr[2], original.arr[2]); + assert.sameDeepOrderedMembers(clone.arr[2].entries, original.arr[2].entries); + assert.notEqual(clone.arr[2].entries, original.arr[2].entries); + assert.notEqual(clone.arr[2].entries[1], original.arr[2].entries[1]); + + assert.deepEqual(clone.nested, original.nested); + assert.notEqual(clone.nested, original.nested); + + assert.notEqual(clone.nested.character, original.nested.character); + assert.notEqual(clone.nested.actor, original.nested.actor); + + assert.isFunction(clone.nested.getQuote); + assert.equal(clone.nested.getQuote(), original.nested.getQuote()); + // the function will actually be the same instance. + }); +}); diff --git a/common/web/utils/src/test/timeoutPromise.js b/common/web/utils/src/test/timeoutPromise.js index 24f37a243d6..337cf286e2d 100644 --- a/common/web/utils/src/test/timeoutPromise.js +++ b/common/web/utils/src/test/timeoutPromise.js @@ -1,6 +1,6 @@ import { assert } from 'chai'; -import { TimeoutPromise } from '@keymanapp/web-utils'; +import { TimeoutPromise, timedPromise } from '@keymanapp/web-utils'; // Set this long enough to allow a bit of delay from OS context-switching (often on the // order of ~16ms for some OSes) to occur multiple times without breaking these tests. @@ -18,6 +18,17 @@ describe("TimeoutPromise", () => { assert.isAtLeast(end-start, INTERVAL-1); }); + it('standard use (simpler format)', async () => { + const start = Date.now(); + const promise = timedPromise(INTERVAL); + + assert.isTrue(await promise); + + const end = Date.now(); + // https://github.com/nodejs/node/issues/26578 - setTimeout() may resolve 1ms earlier than requested. + assert.isAtLeast(end-start, INTERVAL-1); + }); + it('simple early fulfillment', async () => { const start = Date.now(); const promise = new TimeoutPromise(INTERVAL); diff --git a/web/src/engine/osk/src/input/gestures/specsForKeyboard.ts b/web/src/engine/osk/src/input/gestures/specsForLayout.ts similarity index 73% rename from web/src/engine/osk/src/input/gestures/specsForKeyboard.ts rename to web/src/engine/osk/src/input/gestures/specsForLayout.ts index 59468c573fb..159a214df18 100644 --- a/web/src/engine/osk/src/input/gestures/specsForKeyboard.ts +++ b/web/src/engine/osk/src/input/gestures/specsForLayout.ts @@ -4,8 +4,8 @@ import { } from '@keymanapp/gesture-recognizer'; import { - Codes, - Keyboard + type ActiveLayout, + deepCopy } from '@keymanapp/keyboard-processor'; import { type KeyElement } from '../../keyElement.js'; @@ -17,7 +17,7 @@ import specs = gestures.specs; * @param keyboard * @returns */ -export function gestureSetForKeyboard(keyboard: Keyboard): GestureModelDefs { +export function gestureSetForLayout(layout: ActiveLayout): GestureModelDefs { // To be used among the `allowsInitialState` contact-model specifications as needed. const gestureKeyFilter = (key: KeyElement, gestureId: string) => { const keySpec = key.key.spec; @@ -27,38 +27,82 @@ export function gestureSetForKeyboard(keyboard: Keyboard): GestureModelDefs { + if((contactIndices as number[]).indexOf(index) != -1) { + const baseInitialStateCheck = contact.model.allowsInitialState ?? (() => true); + + contact.model = { + ...contact.model, + allowsInitialState: (sample, ancestorSample, key) => { + return baseInitialStateCheck(sample, ancestorSample, key) && gestureKeyFilter(key, modelId); + } + }; + } + }); + + return model; + } + + const gestureModels = [ + withKeySpecFiltering(longpressModel, 0), + // FIXME: needs a special version - we need access to the corresponding GestureSequence's + // stage 0 to properly resolve this! + withKeySpecFiltering(MultitapModel, 0), + SimpleTapModel, + withKeySpecFiltering(SpecialKeyStartModel, 0), + SpecialKeyEndModel, + SubkeySelectModel, + withKeySpecFiltering(ModipressStartModel, 0), + ModipressEndModel + ]; + + const defaultSet = [ + BasicLongpressModel.id, SimpleTapModel.id, ModipressStartModel.id, SpecialKeyStartModel.id + ]; + + if(layout.hasFlicks) { + // TODO: + // gestureModels.push // flick-start + // gestureModels.push // flick-end + + // defaultSet.push('flick-start'); + } - // Idea: if we want to get fancy, we could detect if the keyboard even _supports_ some of - // the less common gestures and just... not include the model if it doesn't. Should only - // do that if it's computationally "cheap", though. return { - gestures: [ - // TODO: some, if not all, will probably utilize methods, rather than constant definitions. - // But, this should be fine for a first-pass integration attempt. - LongpressModel, - MultitapModel, - SimpleTapModel, - SpecialKeyStartModel, - SpecialKeyEndModel, - SubkeySelectModel, - ModipressStartModel, - ModipressEndModel - ], + gestures: gestureModels, sets: { - default: [LongpressModel.id, SimpleTapModel.id, ModipressStartModel.id, SpecialKeyStartModel.id], - modipress: [LongpressModel.id, SimpleTapModel.id, SpecialKeyStartModel.id], // no nested modipressing + default: defaultSet, + modipress: defaultSet.filter((entry) => entry != ModipressStartModel.id), // no nested modipressing none: [] } } @@ -85,7 +129,7 @@ export const InstantContactResolutionModel: ContactModel = { } export const LongpressDistanceThreshold = 10; -export const MainContactLongpressSourceModel: ContactModel = { +export const BasicLongpressContactModel: ContactModel = { itemChangeAction: 'reject', itemPriority: 0, pathResolutionAction: 'resolve', @@ -108,8 +152,8 @@ export const MainContactLongpressSourceModel: ContactModel = { }; export const LongpressFlickDistanceThreshold = 6; -export const MainContactLongpressSourceModelWithShortcut: ContactModel = { - ...MainContactLongpressSourceModel, +export const LongpressContactModelWithShortcut: ContactModel = { + ...BasicLongpressContactModel, pathModel: { evaluate: (path) => { const stats = path.stats; @@ -119,7 +163,7 @@ export const MainContactLongpressSourceModelWithShortcut: ContactModel = { return 'resolve'; } - return MainContactLongpressSourceModel.pathModel.evaluate(path); + return BasicLongpressContactModel.pathModel.evaluate(path); } } } @@ -231,15 +275,47 @@ export const SpecialKeyEndModel: GestureModel = { } } -// Is kind of a mix of the two longpress styles. -export const LongpressModel: GestureModel = { +/** + * The flickless, roaming-touch-less version. + */ +export const BasicLongpressModel: GestureModel = { + id: 'longpress', + resolutionPriority: 0, + contacts: [ + { + model: { + // Is the version without the up-flick shortcut. + ...BasicLongpressContactModel, + itemPriority: 1, + pathInheritance: 'chop' + }, + endOnResolve: false + }, { + model: InstantContactRejectionModel + } + ], + resolutionAction: { + type: 'chain', + next: 'subkey-select', + selectionMode: 'none', + item: 'none' + } +} + +/** + * For use when a layout doesn't have flicks; has the up-flick shortcut + * and facilitates roaming-touch. + */ +export const LongpressModelWithShortcut: GestureModel = { + ...BasicLongpressModel, + id: 'longpress', resolutionPriority: 0, contacts: [ { model: { // Is the version without the up-flick shortcut. - ...MainContactLongpressSourceModel, + ...LongpressContactModelWithShortcut, itemPriority: 1, pathInheritance: 'chop' }, @@ -254,6 +330,7 @@ export const LongpressModel: GestureModel = { selectionMode: 'none', item: 'none' }, + /* * Note: these actions make sense in a 'roaming-touch' context, but not when * flicks are also enabled. diff --git a/web/src/engine/osk/src/visualKeyboard.ts b/web/src/engine/osk/src/visualKeyboard.ts index 1b19ea88a85..59335ea3468 100644 --- a/web/src/engine/osk/src/visualKeyboard.ts +++ b/web/src/engine/osk/src/visualKeyboard.ts @@ -42,7 +42,7 @@ import InternalPendingLongpress from './input/gestures/browser/pendingLongpress. import InternalKeyTip from './input/gestures/browser/keytip.js'; import CommonConfiguration from './config/commonConfiguration.js'; -import { gestureSetForKeyboard } from './input/gestures/specsForKeyboard.js'; +import { gestureSetForLayout } from './input/gestures/specsForLayout.js'; import { getViewportScale } from './screenUtils.js'; @@ -353,7 +353,7 @@ export default class VisualKeyboard extends EventEmitter implements Ke } }; - const recognizer = new GestureRecognizer(gestureSetForKeyboard(this.layoutKeyboard), config); + const recognizer = new GestureRecognizer(gestureSetForLayout(this.kbdLayout), config); recognizer.stateToken = this.layerId; const sourceTrackingMap: Record Date: Wed, 4 Oct 2023 13:02:31 +0700 Subject: [PATCH 03/12] feat(web): gesture configuration for layout, gesture key filtering --- .../src/engine/headless/gestureSource.ts | 16 +++- .../gestures/matchers/gestureMatcher.ts | 7 +- .../headless/gestures/specs/contactModel.ts | 21 ++++- .../osk/src/input/gestures/specsForLayout.ts | 79 ++++++++++++++----- .../osk/src/keyboard-layout/oskLayerGroup.ts | 4 +- web/src/engine/osk/src/visualKeyboard.ts | 30 +++---- 6 files changed, 116 insertions(+), 41 deletions(-) diff --git a/common/web/gesture-recognizer/src/engine/headless/gestureSource.ts b/common/web/gesture-recognizer/src/engine/headless/gestureSource.ts index 96087cb4172..adfe560784b 100644 --- a/common/web/gesture-recognizer/src/engine/headless/gestureSource.ts +++ b/common/web/gesture-recognizer/src/engine/headless/gestureSource.ts @@ -133,12 +133,20 @@ export class GestureSource { } /** - * The first path sample (coordinate) under consideration for this `GestureSource`. + * The 'base item' for the path of this `GestureSource`. + * + * May be set independently after construction for cases where one GestureSource conceptually + * "succeeds" another one, as with multitap gestures. (Though, those generally constrain + * new paths to have the same base item.) */ public get baseItem(): HoveredItemType { return this._baseItem; } + public set baseItem(value: HoveredItemType) { + this._baseItem = value; + } + /** * The most recent path sample (coordinate) under consideration for this `GestureSource`. */ @@ -299,12 +307,18 @@ export class GestureSourceSubview extends Ges subpath = new GesturePath(); for(let i=0; i < length; i++) { + // IMPORTANT: also acts as a deep-copy of the sample; edits to it do not propagate to other + // subviews or the original `baseSource`. Needed for multitaps that trigger system + // `stateToken` changes. subpath.extend(translateSample(baseSource.path.coords[start + i])); } this._path = subpath; if(preserveBaseItem) { + // IMPORTANT: inherits the _subview's_ base item, not the baseSource's version thereof. + // This allows gesture models based upon 'sustain timers' to have a different base item + // than concurrent models that aren't sustain-followups. this._baseItem = source.baseItem; } else { this._baseItem = lastSample?.item; diff --git a/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureMatcher.ts b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureMatcher.ts index 3b1d0457107..ae6ad8f827b 100644 --- a/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureMatcher.ts +++ b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureMatcher.ts @@ -266,7 +266,7 @@ export class GestureMatcher implements PredecessorMatch { */ public get primaryPath(): GestureSource { let bestMatcher: PathMatcher; - let highestPriority = Number.MIN_VALUE; + let highestPriority = Number.NEGATIVE_INFINITY; for(let matcher of this.pathMatchers) { if(matcher.model.itemPriority > highestPriority) { highestPriority = matcher.model.itemPriority; @@ -359,6 +359,11 @@ export class GestureMatcher implements PredecessorMatch { baseItem = this.predecessor.result.action.item; break; } + + // Under 'sustain timer' mode, the concept is that the first new source is the + // continuation and successor to `predecessor.primaryPath`. Its base `item` + // should reflect this. + simpleSource.baseItem = baseItem ?? simpleSource.baseItem; } else { // just use the highest-priority item source's base item and call it a day. // There's no need to refer to some previously-existing source for comparison. diff --git a/common/web/gesture-recognizer/src/engine/headless/gestures/specs/contactModel.ts b/common/web/gesture-recognizer/src/engine/headless/gestures/specs/contactModel.ts index 14cca09fac6..2fe1b589c8d 100644 --- a/common/web/gesture-recognizer/src/engine/headless/gestures/specs/contactModel.ts +++ b/common/web/gesture-recognizer/src/engine/headless/gestures/specs/contactModel.ts @@ -56,9 +56,22 @@ export interface ContactModel { /** * Is needed to define whether or not the contact-point should be ignored by this gesture type. * If undefined, defaults to () => true. - * Param 2 only matters for slots of multitouch gesture spec added after initialization of - * the gesture's corresponding matcher; it will receive the latest sample from the highest-priority - * active path (or predecessor gesture, if no path is active due to a 'sustain' state) + * + * Param 2 allows use of the ongoing sequence's properties to provide further filtering; + * this is needed by KMW's OSK to determine the base key for a multitouch. + * + * Param 4 should prove useful in the future for multitouch gestures dependent upon relative + * location of the touches involved, such as caret-panning. + * + * @param incomingSample The first input sample of the path to be modeled. + * @param comparisonSample The most recent sample related to the same gesture component, if one exists. + * May be `null`. + * @param baseItem The 'base item' for the path corresponding to `comparisonSample` + * @returns */ - readonly allowsInitialState?: (incomingSample: InputSample, comparisonSample?: InputSample, baseItem?: Type) => boolean; + readonly allowsInitialState?: ( + incomingSample: InputSample, + comparisonSample?: InputSample, + baseItem?: Type + ) => boolean; } \ No newline at end of file diff --git a/web/src/engine/osk/src/input/gestures/specsForLayout.ts b/web/src/engine/osk/src/input/gestures/specsForLayout.ts index 159a214df18..bfd85f7e80a 100644 --- a/web/src/engine/osk/src/input/gestures/specsForLayout.ts +++ b/web/src/engine/osk/src/input/gestures/specsForLayout.ts @@ -1,12 +1,13 @@ import { gestures, - GestureModelDefs + GestureModelDefs, + InputSample } from '@keymanapp/gesture-recognizer'; import { - type ActiveLayout, deepCopy } from '@keymanapp/keyboard-processor'; +import OSKLayerGroup from '../../keyboard-layout/oskLayerGroup.js'; import { type KeyElement } from '../../keyElement.js'; @@ -17,7 +18,9 @@ import specs = gestures.specs; * @param keyboard * @returns */ -export function gestureSetForLayout(layout: ActiveLayout): GestureModelDefs { +export function gestureSetForLayout(layerGroup: OSKLayerGroup): GestureModelDefs { + const layout = layerGroup.spec; + // To be used among the `allowsInitialState` contact-model specifications as needed. const gestureKeyFilter = (key: KeyElement, gestureId: string) => { const keySpec = key.key.spec; @@ -45,8 +48,10 @@ export function gestureSetForLayout(layout: ActiveLayout): GestureModelDefs { + if((contactIndices as number[]).indexOf(index) != -1) { + const baseInitialStateCheck = contact.model.allowsInitialState ?? (() => true); + + contact.model = { + ...contact.model, + // And now for the true purpose of the method. + allowsInitialState: (sample, ancestorSample, baseKey) => { + // By default, the state token is set to whatever the current layer is for a source. + // + // So, if the first tap of a key swaps layers, the second tap will be on the wrong layer and + // thus have a different state token. This is the perfect place to detect and correct that. + if(ancestorSample.stateToken != sample.stateToken) { + sample.stateToken = ancestorSample.stateToken; + + // Specialized item lookup is required here for proper 'correction' - we want the key + // corresponding to our original layer, not the new layer here. Now that we've identified + // the original OSK layer (state) for the gesture, we can find the best matching key + // from said layer instead of the current layer. + // + // Matters significantly for multitaps if and when they include layer-switching specs. + sample.item = layerGroup.findNearestKey(sample); + } + + return baseInitialStateCheck(sample, ancestorSample, baseKey); + } + }; + } + }); + + return model; + } + // #endregion + const gestureModels = [ withKeySpecFiltering(longpressModel, 0), - // FIXME: needs a special version - we need access to the corresponding GestureSequence's - // stage 0 to properly resolve this! - withKeySpecFiltering(MultitapModel, 0), - SimpleTapModel, + withLayerChangeItemFix(withKeySpecFiltering(MultitapModel, 0), 0), + simpleTapModel, withKeySpecFiltering(SpecialKeyStartModel, 0), SpecialKeyEndModel, SubkeySelectModel, @@ -357,17 +403,6 @@ export const MultitapModel: GestureModel = { itemPriority: 1, pathInheritance: 'reject', allowsInitialState(incomingSample, comparisonSample, baseItem) { - // By default, the state token is set to whatever the current layer is for a source. - // - // So, if the first tap of a key swaps layers, the second tap will be on the wrong layer and - // thus have a different state token. This is the perfect place to detect and correct that. - if(comparisonSample.stateToken != incomingSample.stateToken) { - incomingSample.stateToken = comparisonSample.stateToken; - - // TODO: specialized item lookup required here for proper 'correction', corresponding to - // the owning VisualKeyboard. That rigging doesn't exist quite yet, at the time of writing this. - incomingSample.item = undefined; - } return incomingSample.item == baseItem; }, }, @@ -408,7 +443,11 @@ export const SimpleTapModel: GestureModel = { type: 'chain', next: 'multitap', item: 'current' - }, + } +} + +export const SimpleTapModelWithReset: GestureModel = { + ...SimpleTapModel, rejectionActions: { item: { type: 'replace', diff --git a/web/src/engine/osk/src/keyboard-layout/oskLayerGroup.ts b/web/src/engine/osk/src/keyboard-layout/oskLayerGroup.ts index 541425eeae4..763e1b5ac6e 100644 --- a/web/src/engine/osk/src/keyboard-layout/oskLayerGroup.ts +++ b/web/src/engine/osk/src/keyboard-layout/oskLayerGroup.ts @@ -1,4 +1,4 @@ -import { ActiveLayer, type DeviceSpec, Keyboard, LayoutLayer } from '@keymanapp/keyboard-processor'; +import { ActiveLayer, type DeviceSpec, Keyboard, LayoutLayer, ActiveLayout } from '@keymanapp/keyboard-processor'; import { InputSample } from '@keymanapp/gesture-recognizer'; @@ -10,9 +10,11 @@ import OSKRow from './oskRow.js'; export default class OSKLayerGroup { public readonly element: HTMLDivElement; public readonly layers: {[layerID: string]: OSKLayer} = {}; + public readonly spec: ActiveLayout; public constructor(vkbd: VisualKeyboard, keyboard: Keyboard, formFactor: DeviceSpec.FormFactor) { let layout = keyboard.layout(formFactor); + this.spec = layout; const lDiv = this.element = document.createElement('div'); const ls=lDiv.style; diff --git a/web/src/engine/osk/src/visualKeyboard.ts b/web/src/engine/osk/src/visualKeyboard.ts index 59335ea3468..675f1812235 100644 --- a/web/src/engine/osk/src/visualKeyboard.ts +++ b/web/src/engine/osk/src/visualKeyboard.ts @@ -342,29 +342,27 @@ export default class VisualKeyboard extends EventEmitter implements Ke mouseEventRoot: document.body, // touchEventRoot: this.element, // is the default itemIdentifier: (sample, target) => { - if(sample.stateToken == this.layerId) { - let resolvedTarget = this.keyTarget(target); - if(resolvedTarget) { - return resolvedTarget; - } - } + /* ALWAYS use the findNearestKey function. + * MDN spec for `target`, which comes from Touch.target for touch-based interactions: + * + * > The read-only target property of the Touch interface returns the (EventTarget) on which the touch contact + * started when it was first placed on the surface, even if the touch point has since moved outside the + * interactive area of that element[...] + */ return this.layerGroup.findNearestKey(sample); } }; - const recognizer = new GestureRecognizer(gestureSetForLayout(this.kbdLayout), config); + const recognizer = new GestureRecognizer(gestureSetForLayout(this.layerGroup), config); recognizer.stateToken = this.layerId; const sourceTrackingMap: Record, - roamingHandler: typeof roamingTouchHighlighting, + roamingHighlightHandler: (sample: InputSample) => void, key: KeyElement }> = {}; - // TODO: Obviously, this is extremely rough at present. - const roamingTouchHighlighting: (sample: InputSample) => void = null; - // Now to set up event-handling links. // This handler should probably vary based on the keyboard: do we allow roaming touches or not? recognizer.on('inputstart', (source) => { @@ -373,7 +371,7 @@ export default class VisualKeyboard extends EventEmitter implements Ke // highlighting it) const trackingEntry = sourceTrackingMap[source.identifier] = { source: source, - roamingHandler: (sample) => { + roamingHighlightHandler: (sample) => { // Maintain highlighting const key = sample.item; const oldKey = sourceTrackingMap[source.identifier].key; @@ -404,7 +402,7 @@ export default class VisualKeyboard extends EventEmitter implements Ke // If so, separate handler - it likely needs to be disabled once the first gesture-component // match happens, unlike the highlighting part. - source.path.on('step', trackingEntry.roamingHandler); + source.path.on('step', trackingEntry.roamingHighlightHandler); source.path.on('step', (sample) => { console.log(`New sample for source ${source.identifier}:`); @@ -436,7 +434,7 @@ export default class VisualKeyboard extends EventEmitter implements Ke // if( /* not in subkey-select mode */) { for(let id of gestureStage.allSourceIds) { const trackingEntry = sourceTrackingMap[id]; - trackingEntry.source.path.off('step', trackingEntry.roamingHandler); + trackingEntry.source.path.off('step', trackingEntry.roamingHighlightHandler); } // } @@ -454,6 +452,10 @@ export default class VisualKeyboard extends EventEmitter implements Ke coord = coordSource.currentSample; } + if(gestureStage.matchedId == 'multitap') { + // TODO: examine sequence, determine rota-style index to apply; select THAT item instead. + } + // Once the best coord to use for fat-finger calculations has been determined: this.modelKeyClick(gestureStage.item, coord); From 69757852fb48436e77611c6883f98fde59e25276 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 4 Oct 2023 15:27:00 +0700 Subject: [PATCH 04/12] fix(web): roaming simple-touch slightly more stable --- .../src/engine/headless/gestureSource.ts | 4 +++- .../gestures/matchers/gestureMatcher.ts | 2 +- .../engine/headless/touchpointCoordinator.ts | 17 +++++++++++++++-- web/src/engine/osk/src/visualKeyboard.ts | 6 ------ 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/common/web/gesture-recognizer/src/engine/headless/gestureSource.ts b/common/web/gesture-recognizer/src/engine/headless/gestureSource.ts index adfe560784b..ce46b956192 100644 --- a/common/web/gesture-recognizer/src/engine/headless/gestureSource.ts +++ b/common/web/gesture-recognizer/src/engine/headless/gestureSource.ts @@ -269,7 +269,7 @@ export class GestureSourceSubview extends Ges // Check against the full remaining length of the original source; does // the subview provided to us include its source's most recent point? const sampleCountSinceStart = source.baseSource.path.coords.length; - if(expectedLength != start + sampleCountSinceStart) { + if(expectedLength != sampleCountSinceStart) { mayUpdate = false; } } @@ -380,6 +380,8 @@ export class GestureSourceSubview extends Ges */ public disconnect() { if(this.subviewDisconnector) { + console.log("disconnecting - most recent sample follows"); + console.log(this.currentSample); this.subviewDisconnector(); this.subviewDisconnector = null; } diff --git a/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureMatcher.ts b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureMatcher.ts index ae6ad8f827b..f2c33ef5123 100644 --- a/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureMatcher.ts +++ b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureMatcher.ts @@ -99,7 +99,7 @@ export class GestureMatcher implements PredecessorMatch { return entry.isPathComplete ? null : entry; }).reduce((cleansed, entry) => { return entry ? cleansed.concat(entry) : cleansed; - }, []); + }, [] as GestureSource[]); if(model.sustainTimer && sourceTouchpoints.length > 0) { // If a sustain timer is set, it's because we expect to have NO gesture-source _initially_. diff --git a/common/web/gesture-recognizer/src/engine/headless/touchpointCoordinator.ts b/common/web/gesture-recognizer/src/engine/headless/touchpointCoordinator.ts index 7112227e2c1..4061fa3729b 100644 --- a/common/web/gesture-recognizer/src/engine/headless/touchpointCoordinator.ts +++ b/common/web/gesture-recognizer/src/engine/headless/touchpointCoordinator.ts @@ -1,9 +1,10 @@ import EventEmitter from "eventemitter3"; import { InputEngineBase } from "./inputEngineBase.js"; import { buildGestureMatchInspector, GestureSource } from "./gestureSource.js"; -import { MatcherSelector } from "./gestures/matchers/matcherSelector.js"; +import { MatcherSelection, MatcherSelector } from "./gestures/matchers/matcherSelector.js"; import { GestureSequence } from "./gestures/matchers/gestureSequence.js"; -import { GestureModelDefs, getGestureModelSet } from "./gestures/specs/gestureModelDefs.js"; +import { GestureModelDefs, getGestureModel, getGestureModelSet } from "./gestures/specs/gestureModelDefs.js"; +import { GestureModel } from "./gestures/specs/gestureModel.js"; interface EventMap { /** @@ -45,10 +46,21 @@ export class TouchpointCoordinator extends Even this.addEngine(engine); } } + + this.selectorStack[0].on('rejectionwithaction', this.modelResetHandler) } + private readonly modelResetHandler = (selection: MatcherSelection, replaceModelWith: (model: GestureModel) => void) => { + if(selection.result.action.type == 'replace') { + replaceModelWith(getGestureModel(this.gestureModelDefinitions, selection.result.action.replace)); + } else { + throw new Error("Missed a case in implementation!"); + } + }; + public pushSelector(selector: MatcherSelector) { this.selectorStack.push(selector); + selector.on('rejectionwithaction', this.modelResetHandler); } public popSelector(selector: MatcherSelector) { @@ -63,6 +75,7 @@ export class TouchpointCoordinator extends Even } /* c8 ignore end */ + selector.off('rejectionwithaction', this.modelResetHandler); selector.cascadeTermination(); this.selectorStack.splice(index, 1); diff --git a/web/src/engine/osk/src/visualKeyboard.ts b/web/src/engine/osk/src/visualKeyboard.ts index 675f1812235..1a0a4e3898d 100644 --- a/web/src/engine/osk/src/visualKeyboard.ts +++ b/web/src/engine/osk/src/visualKeyboard.ts @@ -366,7 +366,6 @@ export default class VisualKeyboard extends EventEmitter implements Ke // Now to set up event-handling links. // This handler should probably vary based on the keyboard: do we allow roaming touches or not? recognizer.on('inputstart', (source) => { - console.log(source); // Make sure we're tracking the source and its currently-selected item (the latter, as we're // highlighting it) const trackingEntry = sourceTrackingMap[source.identifier] = { @@ -405,8 +404,6 @@ export default class VisualKeyboard extends EventEmitter implements Ke source.path.on('step', trackingEntry.roamingHighlightHandler); source.path.on('step', (sample) => { - console.log(`New sample for source ${source.identifier}:`); - console.log(sample); // // Do... something based on the potential gesture types that could arise, as appropriate. // // Should be useful for selecting a hint type, etc. // source.potentialModelMatchIds @@ -422,8 +419,6 @@ export default class VisualKeyboard extends EventEmitter implements Ke // This should probably be universal in some manner. gestureSequence.on('complete', () => { - console.log("Complete:"); - console.log(gestureSequence); }); // This should probably vary based on the type of gesture. @@ -438,7 +433,6 @@ export default class VisualKeyboard extends EventEmitter implements Ke } // } - console.log(gestureStage); // First, if we've configured the gesture to generate a keystroke, let's handle that. const gestureKey = gestureStage.item; From 8f5e7d2980136c152a4171dc097f1aa0691a38d8 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 5 Oct 2023 08:42:51 +0700 Subject: [PATCH 05/12] fix(web): 'rejectionwithaction' handler filtering, cleanup --- .../src/engine/headless/gestureSource.ts | 2 -- .../gestures/matchers/gestureMatcher.ts | 6 +++++ .../gestures/matchers/gestureSequence.ts | 24 +++++++++++++++---- .../engine/headless/touchpointCoordinator.ts | 10 ++++++++ 4 files changed, 35 insertions(+), 7 deletions(-) diff --git a/common/web/gesture-recognizer/src/engine/headless/gestureSource.ts b/common/web/gesture-recognizer/src/engine/headless/gestureSource.ts index ce46b956192..b1dc3fb7a2e 100644 --- a/common/web/gesture-recognizer/src/engine/headless/gestureSource.ts +++ b/common/web/gesture-recognizer/src/engine/headless/gestureSource.ts @@ -380,8 +380,6 @@ export class GestureSourceSubview extends Ges */ public disconnect() { if(this.subviewDisconnector) { - console.log("disconnecting - most recent sample follows"); - console.log(this.currentSample); this.subviewDisconnector(); this.subviewDisconnector = null; } diff --git a/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureMatcher.ts b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureMatcher.ts index f2c33ef5123..795147068d4 100644 --- a/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureMatcher.ts +++ b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureMatcher.ts @@ -115,9 +115,15 @@ export class GestureMatcher implements PredecessorMatch { for(let touchpointIndex = 0; touchpointIndex < sourceTouchpoints.length; touchpointIndex++) { const srcContact = sourceTouchpoints[touchpointIndex]; + let baseContact = srcContact; if(srcContact instanceof GestureSourceSubview) { srcContact.disconnect(); // prevent further updates from mangling tracked path info. + baseContact = srcContact.baseSource; + } + + if(baseContact.isPathComplete) { + throw new Error("GestureMatcher may not be built against already-completed contact points"); } const contactSpec = model.contacts[touchpointIndex]; diff --git a/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureSequence.ts b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureSequence.ts index 9df07018872..0654a600961 100644 --- a/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureSequence.ts +++ b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureSequence.ts @@ -107,6 +107,14 @@ export class GestureSequence extends EventEmitter> { this.stageReports = []; this.selector = selector; this.selector.on('rejectionwithaction', this.modelResetHandler); + this.once('complete', () => { + this.selector.off('rejectionwithaction', this.modelResetHandler) + + // Dropping the reference here gives us two benefits: + // 1. Allows garbage collection to do its thing; this might be the last reference left to the selector instance. + // 2. Acts as an obvious flag / indicator of sequence completion. + this.selector = null; + }); this.gestureConfig = gestureModelDefinitions; // So that we can... @@ -260,17 +268,23 @@ export class GestureSequence extends EventEmitter> { this.pushedSelector = null; } - // Dropping the reference here gives us two benefits: - // 1. Allows garbage collection to do its thing; this might be the last reference left to the selector instance. - // 2. Acts as an obvious flag / indicator of sequence completion. - this.selector = null; - // Any extra finalization stuff should go here, before the event, if needed. this.emit('complete'); } } private readonly modelResetHandler = (selection: MatcherSelection, replaceModelWith: (model: GestureModel) => void) => { + const sourceIds = selection.matcher.allSourceIds; + + // If none of the sources involved match a source already included in the sequence, bypass + // this handler; it belongs to a different sequence or one that's beginning. + // + // This works even for multitaps because we include the most recent ancestor sources in + // `allSourceIds` - that one will match here. + if(this.allSourceIds.find((a) => sourceIds.indexOf(a) == -1)) { + return; + } + if(selection.result.action.type == 'replace') { replaceModelWith(getGestureModel(this.gestureConfig, selection.result.action.replace)); } else { diff --git a/common/web/gesture-recognizer/src/engine/headless/touchpointCoordinator.ts b/common/web/gesture-recognizer/src/engine/headless/touchpointCoordinator.ts index 4061fa3729b..312378f8fb6 100644 --- a/common/web/gesture-recognizer/src/engine/headless/touchpointCoordinator.ts +++ b/common/web/gesture-recognizer/src/engine/headless/touchpointCoordinator.ts @@ -51,6 +51,16 @@ export class TouchpointCoordinator extends Even } private readonly modelResetHandler = (selection: MatcherSelection, replaceModelWith: (model: GestureModel) => void) => { + const sourceIds = selection.matcher.allSourceIds; + + // If there's an active gesture that uses a source noted in the selection, it's the responsibility + // of an existing GestureSequence to handle this one. The handler should bypass it for this round. + if(this.activeGestures.find((sequence) => { + return sequence.allSourceIds.find((a) => sourceIds.indexOf(a) != -1); + })) { + return; + } + if(selection.result.action.type == 'replace') { replaceModelWith(getGestureModel(this.gestureModelDefinitions, selection.result.action.replace)); } else { From e4dd8fa176d4580847ed2856f83bca1cbb3c3e0f Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 5 Oct 2023 09:00:44 +0700 Subject: [PATCH 06/12] fix(web): out-of-bounds nearestKey nit --- web/src/engine/osk/src/keyboard-layout/oskLayerGroup.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/web/src/engine/osk/src/keyboard-layout/oskLayerGroup.ts b/web/src/engine/osk/src/keyboard-layout/oskLayerGroup.ts index 763e1b5ac6e..110b0da5b8b 100644 --- a/web/src/engine/osk/src/keyboard-layout/oskLayerGroup.ts +++ b/web/src/engine/osk/src/keyboard-layout/oskLayerGroup.ts @@ -124,6 +124,11 @@ export default class OSKLayerGroup { } } + // If the coordinate isn't even on the keyboard... abort. + if(row == null) { + return null; + } + // Assertion: row no longer `null`. // Warning: am not 100% sure that what follows is actually fully correct. From 4934f8b61552ebd5b54432de1c98c9689379cce7 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 5 Oct 2023 09:47:29 +0700 Subject: [PATCH 07/12] chore(web): restoration of held-backspace --- .../gesture-recognizer/src/engine/index.ts | 1 + .../osk/src/input/gestures/heldRepeater.ts | 31 +++++++++++++++++++ web/src/engine/osk/src/visualKeyboard.ts | 16 +++++++++- 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 web/src/engine/osk/src/input/gestures/heldRepeater.ts diff --git a/common/web/gesture-recognizer/src/engine/index.ts b/common/web/gesture-recognizer/src/engine/index.ts index f95dd69282b..2ced8a26ef6 100644 --- a/common/web/gesture-recognizer/src/engine/index.ts +++ b/common/web/gesture-recognizer/src/engine/index.ts @@ -6,6 +6,7 @@ export { GestureRecognizerConfiguration } from "./configuration/gestureRecognize export { InputEngineBase } from "./headless/inputEngineBase.js"; export { InputSample } from "./headless/inputSample.js"; export { SerializedGesturePath, GesturePath } from "./headless/gesturePath.js"; +export { GestureStageReport, GestureSequence } from "./headless/gestures/matchers/gestureSequence.js"; export { SerializedGestureSource, GestureSource, buildGestureMatchInspector } from "./headless/gestureSource.js"; export { MouseEventEngine } from "./mouseEventEngine.js"; export { PathSegmenter, Subsegmentation } from "./headless/subsegmentation/pathSegmenter.js"; diff --git a/web/src/engine/osk/src/input/gestures/heldRepeater.ts b/web/src/engine/osk/src/input/gestures/heldRepeater.ts new file mode 100644 index 00000000000..93b80be99f2 --- /dev/null +++ b/web/src/engine/osk/src/input/gestures/heldRepeater.ts @@ -0,0 +1,31 @@ +import { GestureSequence } from "@keymanapp/gesture-recognizer"; + +import { KeyElement } from "../../keyElement.js"; + +export class HeldRepeater { + static readonly INITIAL_DELAY = 500; + static readonly REPEAT_DELAY = 100; + + readonly source: GestureSequence; + readonly repeatClosure: () => void; + + timerHandle: number; + + constructor(source: GestureSequence, closureToRepeat: () => void) { + this.source = source; + this.repeatClosure = closureToRepeat; + + this.timerHandle = window.setTimeout(this.deleteRepeater, HeldRepeater.INITIAL_DELAY); + + this.source.on('complete', () => { + window.clearTimeout(this.timerHandle); + this.timerHandle = undefined; + }); + } + + readonly deleteRepeater = () => { + this.repeatClosure(); + + this.timerHandle = window.setTimeout(this.deleteRepeater, HeldRepeater.REPEAT_DELAY); + } +} \ No newline at end of file diff --git a/web/src/engine/osk/src/visualKeyboard.ts b/web/src/engine/osk/src/visualKeyboard.ts index 1a0a4e3898d..2924f511a28 100644 --- a/web/src/engine/osk/src/visualKeyboard.ts +++ b/web/src/engine/osk/src/visualKeyboard.ts @@ -45,6 +45,7 @@ import CommonConfiguration from './config/commonConfiguration.js'; import { gestureSetForLayout } from './input/gestures/specsForLayout.js'; import { getViewportScale } from './screenUtils.js'; +import { HeldRepeater } from './input/gestures/heldRepeater.js'; export interface VisualKeyboardConfiguration extends CommonConfiguration { /** @@ -436,13 +437,15 @@ export default class VisualKeyboard extends EventEmitter implements Ke // First, if we've configured the gesture to generate a keystroke, let's handle that. const gestureKey = gestureStage.item; - if(gestureKey /* && gestureKey is appropriate for the gesture */) { + if(gestureKey) { let coordSource = gestureStage.sources[0]; let coord: InputSample = null; if(coordSource) { // TODO: should probably vary depending upon `gestureStage.matchedId` // (certain types should probably use the base coord... or even from // a prior stage of the sequence as appropriate.) + // + // This is the coordinate used as the basis for fat-finger calculations. coord = coordSource.currentSample; } @@ -453,6 +456,17 @@ export default class VisualKeyboard extends EventEmitter implements Ke // Once the best coord to use for fat-finger calculations has been determined: this.modelKeyClick(gestureStage.item, coord); + // -- Scratch-space as gestures start becoming integrated -- + // Reordering may follow at some point. + if(gestureStage.matchedId == 'special-key-start' && gestureKey.key.spec.baseKeyID == 'K_BKSP') { + // Possible enhancement: maybe update the held location for the backspace if there's movement? + // But... that seems pretty low-priority. + // + // Merely constructing the instance is enough; it'll link into the sequence's events and + // handle everything that remains for the backspace from here. + new HeldRepeater(gestureSequence, () => this.modelKeyClick(gestureKey, coord)); + } + // TODO: depending upon the gesture type, what sort of UI shifts should happen to // facilitate follow-up stages? } From ac2403522980c4f920e69fa0691cd1b3e1e52f58 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 5 Oct 2023 10:27:57 +0700 Subject: [PATCH 08/12] chore(web): re-establishes base-keyboard out-of-bounds roaming --- .../engine/configuration/paddedZoneSource.ts | 34 ++++++++++++++++--- .../osk/src/keyboard-layout/oskLayerGroup.ts | 15 +++++--- web/src/engine/osk/src/visualKeyboard.ts | 21 ++++++++---- 3 files changed, 54 insertions(+), 16 deletions(-) diff --git a/common/web/gesture-recognizer/src/engine/configuration/paddedZoneSource.ts b/common/web/gesture-recognizer/src/engine/configuration/paddedZoneSource.ts index 05c8dc8b79d..98a9f21d496 100644 --- a/common/web/gesture-recognizer/src/engine/configuration/paddedZoneSource.ts +++ b/common/web/gesture-recognizer/src/engine/configuration/paddedZoneSource.ts @@ -4,13 +4,17 @@ import { ViewportZoneSource } from "./viewportZoneSource.js"; export class PaddedZoneSource implements RecognitionZoneSource { private readonly root: RecognitionZoneSource; - private edgePadding: { + private _edgePadding: { x: number, y: number, w: number, h: number }; + public get edgePadding() { + return this._edgePadding; + } + /** * Provides a dynamic 'padded' recognition zone based upon offsetting from the borders * of the active page's viewport. @@ -54,12 +58,32 @@ export class PaddedZoneSource implements RecognitionZoneSource { // In case it isn't yet defined. edgePadding = edgePadding || [0, 0, 0, 0]; + this.updatePadding(edgePadding); + } + + /** + * Provides a dynamic 'padded' recognition zone based upon offsetting from the borders + * of another defined zone. + * + * Padding is defined using the standard CSS border & padding spec style: + * - [a]: equal and even padding on all sides + * - [a, b]: top & bottom use `a`, left & right use `b` + * - [a, b, c]: top uses `a`, left & right use `b`, bottom uses `c` + * - [a, b, c, d]: top, right, bottom, then left. + * + * Positive padding reduces the size of the resulting zone; negative padding expands it. + * + * @param rootZoneSource The root zone source object/element to be 'padded' + * @param edgePadding A set of 1 to 4 numbers defining padding per the standard CSS border & padding spec style. + */ + updatePadding(edgePadding: number[]) { + // Modeled after CSS styling definitions... just with preprocessed numbers, not strings. switch(edgePadding.length) { case 1: // all sides equal const val = edgePadding[0]; - this.edgePadding = { + this._edgePadding = { x: val, y: val, w: 2 * val, @@ -68,7 +92,7 @@ export class PaddedZoneSource implements RecognitionZoneSource { break; case 2: // top & bottom, left & right - this.edgePadding = { + this._edgePadding = { x: edgePadding[1], y: edgePadding[0], w: 2 * edgePadding[1], @@ -77,7 +101,7 @@ export class PaddedZoneSource implements RecognitionZoneSource { break; case 3: // top, left & right, bottom - this.edgePadding = { + this._edgePadding = { x: edgePadding[1], y: edgePadding[0], w: 2 * edgePadding[1], @@ -85,7 +109,7 @@ export class PaddedZoneSource implements RecognitionZoneSource { }; case 4: // top, right, bottom, left - this.edgePadding = { + this._edgePadding = { x: edgePadding[3], y: edgePadding[0], w: edgePadding[1] + edgePadding[3], diff --git a/web/src/engine/osk/src/keyboard-layout/oskLayerGroup.ts b/web/src/engine/osk/src/keyboard-layout/oskLayerGroup.ts index 110b0da5b8b..4237c5fe2da 100644 --- a/web/src/engine/osk/src/keyboard-layout/oskLayerGroup.ts +++ b/web/src/engine/osk/src/keyboard-layout/oskLayerGroup.ts @@ -116,17 +116,22 @@ export default class OSKLayerGroup { } let row: OSKRow = null; + let bestMatchDistance = Number.MAX_VALUE; + + // Find the row that the touch-coordinate lies within. for(const r of layer.rows) { const rowRect = translation(r.element.getBoundingClientRect()); if(rowRect.top <= coord.targetY && coord.targetY < rowRect.bottom) { row = r; break; - } - } + } else { + const distance = rowRect.top > coord.targetY ? rowRect.top - coord.targetY : coord.targetY - rowRect.bottom; - // If the coordinate isn't even on the keyboard... abort. - if(row == null) { - return null; + if(distance < bestMatchDistance) { + bestMatchDistance = distance; + row = r; + } + } } // Assertion: row no longer `null`. diff --git a/web/src/engine/osk/src/visualKeyboard.ts b/web/src/engine/osk/src/visualKeyboard.ts index 2924f511a28..762bccc759d 100644 --- a/web/src/engine/osk/src/visualKeyboard.ts +++ b/web/src/engine/osk/src/visualKeyboard.ts @@ -19,7 +19,8 @@ import { GestureRecognizer, GestureRecognizerConfiguration, GestureSource, - InputSample + InputSample, + PaddedZoneSource } from '@keymanapp/gesture-recognizer'; import { createStyleSheet, getAbsoluteX, getAbsoluteY, StylesheetManager } from 'keyman/engine/dom-utils'; @@ -336,11 +337,16 @@ export default class VisualKeyboard extends EventEmitter implements Ke } private constructGestureEngine(): GestureRecognizer { + const rowCount = this.kbdLayout.layerMap['default'].row.length; + const config: GestureRecognizerConfiguration = { targetRoot: this.element, // document.body is the event root for mouse interactions b/c we need to track // when the mouse leaves the VisualKeyboard's hierarchy. mouseEventRoot: document.body, + // Note: at this point in execution, the value will evaluate to NaN! Height hasn't been set yet. + // BUT: we need to establish the instance now; we can update it later when height _is_ set. + maxRoamingBounds: new PaddedZoneSource(this.element, [-0.333 * this.height / rowCount]), // touchEventRoot: this.element, // is the default itemIdentifier: (sample, target) => { /* ALWAYS use the findNearestKey function. @@ -458,6 +464,10 @@ export default class VisualKeyboard extends EventEmitter implements Ke // -- Scratch-space as gestures start becoming integrated -- // Reordering may follow at some point. + // + // Potential long-term idea: only handle the first stage; delegate future stages to + // specialized handlers for the remainder of the sequence. + // Should work for modipresses, too... I think. if(gestureStage.matchedId == 'special-key-start' && gestureKey.key.spec.baseKeyID == 'K_BKSP') { // Possible enhancement: maybe update the held location for the backspace if there's movement? // But... that seems pretty low-priority. @@ -1465,8 +1475,6 @@ export default class VisualKeyboard extends EventEmitter implements Ke gs.fontSize = this.fontSize.styleString; bs.fontSize = ParsedLengthStyle.forScalar(fs).styleString; - // NEW CODE ------ - // Step 1: have the necessary conditions been met? const fixedSize = this.width && this.height; const computedStyle = getComputedStyle(this.kbdDiv); @@ -1492,9 +1500,10 @@ export default class VisualKeyboard extends EventEmitter implements Ke return; } - // Step 3: perform layout operations. (Handled by 'old code' section below.) - - // END NEW CODE ----------- + // Step 3: perform layout operations. + const paddingZone = this.gestureEngine.config.maxRoamingBounds as PaddedZoneSource; + const rowCount = this.currentLayer.rows.length; + paddingZone.updatePadding([-0.333 * this._computedHeight / rowCount]); // Needs the refreshed layout info to work correctly. if(this.currentLayer) { From 7c81fc6bac322bbb695884debc0bc67772391d3a Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 6 Oct 2023 11:41:11 +0700 Subject: [PATCH 09/12] fix(web): null-guard for a new property --- .../engine/headless/gestures/matchers/gestureSequence.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureSequence.ts b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureSequence.ts index 0654a600961..6838647cb96 100644 --- a/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureSequence.ts +++ b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureSequence.ts @@ -142,6 +142,12 @@ export class GestureSequence extends EventEmitter> { * current state. They will be specified in descending `resolutionPriority` order. */ public get potentialModelMatchIds(): string[] { + // If `this.selector` is null, it's because no further matches are possible. + // We've already emitted the 'complete' event as well. + if(!this.selector) { + return []; + } + // The new round of model-matching is based on the sources used by the previous round. // This is important; 'sustainTimer' gesture models may rely on a now-terminated source // from that previous round (like with multitaps). From e199cd05fdac8dfd77b75425744b849d621a6b5f Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 12 Oct 2023 09:00:40 +0700 Subject: [PATCH 10/12] docs(web): cleans up an errant doc-comment --- .../src/engine/headless/gestures/specs/contactModel.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/common/web/gesture-recognizer/src/engine/headless/gestures/specs/contactModel.ts b/common/web/gesture-recognizer/src/engine/headless/gestures/specs/contactModel.ts index 2fe1b589c8d..494e896b4bf 100644 --- a/common/web/gesture-recognizer/src/engine/headless/gestures/specs/contactModel.ts +++ b/common/web/gesture-recognizer/src/engine/headless/gestures/specs/contactModel.ts @@ -57,12 +57,6 @@ export interface ContactModel { * Is needed to define whether or not the contact-point should be ignored by this gesture type. * If undefined, defaults to () => true. * - * Param 2 allows use of the ongoing sequence's properties to provide further filtering; - * this is needed by KMW's OSK to determine the base key for a multitouch. - * - * Param 4 should prove useful in the future for multitouch gestures dependent upon relative - * location of the touches involved, such as caret-panning. - * * @param incomingSample The first input sample of the path to be modeled. * @param comparisonSample The most recent sample related to the same gesture component, if one exists. * May be `null`. From 3e609f55ff8121c866fc34dc8ae057626d16d677 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Tue, 17 Oct 2023 09:25:15 +0700 Subject: [PATCH 11/12] fix(web): fixes input location mapping for corrections --- web/src/engine/osk/src/visualKeyboard.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/web/src/engine/osk/src/visualKeyboard.ts b/web/src/engine/osk/src/visualKeyboard.ts index 92ac956981f..757a782dc7a 100644 --- a/web/src/engine/osk/src/visualKeyboard.ts +++ b/web/src/engine/osk/src/visualKeyboard.ts @@ -661,12 +661,8 @@ export default class VisualKeyboard extends EventEmitter implements Ke //#region OSK touch handlers getTouchCoordinatesOnKeyboard(input: InputSample) { - // We need to compute the 'local', keyboard-based coordinates for the touch. - let kbdCoords = { - x: getAbsoluteX(this.kbdDiv), - y: getAbsoluteY(this.kbdDiv) - } - let offsetCoords = { x: input.targetX - kbdCoords.x, y: input.targetY - kbdCoords.y }; + // `input` is already in keyboard-local coordinates. It's not scaled, though. + let offsetCoords = { x: input.targetX, y: input.targetY }; // The layer group's element always has the proper width setting, unlike kbdDiv itself. offsetCoords.x /= this.layerGroup.element.offsetWidth; From 9fee40e74035a0eaee74b03d8f9bc0de80349f8c Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Fri, 20 Oct 2023 16:14:05 +0700 Subject: [PATCH 12/12] chore(web): Apply suggestions from code review Co-authored-by: Marc Durdin --- web/src/engine/osk/src/visualKeyboard.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/engine/osk/src/visualKeyboard.ts b/web/src/engine/osk/src/visualKeyboard.ts index 757a782dc7a..e7b1fbbb07b 100644 --- a/web/src/engine/osk/src/visualKeyboard.ts +++ b/web/src/engine/osk/src/visualKeyboard.ts @@ -346,7 +346,7 @@ export default class VisualKeyboard extends EventEmitter implements Ke mouseEventRoot: document.body, // Note: at this point in execution, the value will evaluate to NaN! Height hasn't been set yet. // BUT: we need to establish the instance now; we can update it later when height _is_ set. - maxRoamingBounds: new PaddedZoneSource(this.element, [-0.333 * this.height / rowCount]), + maxRoamingBounds: new PaddedZoneSource(this.element, [NaN]), // touchEventRoot: this.element, // is the default itemIdentifier: (sample, target) => { /* ALWAYS use the findNearestKey function.