feat(developer) marker steps 🙀

common/web/types/src/ldml-keyboard/pattern-parser.ts

@@ -51,6 +51,9 @@ export class MarkerParser {
    * @returns `[]` or an array of all markers referenced
    */
   public static allReferences(str: string): string[] {
+    if (!str) {
+      return [];
+    }
     return matchArray(str, this.REFERENCE);
   }
 }

core/src/ldml/C9134_ldml_markers.md

@@ -27,7 +27,6 @@ Markers can appear in both 'emitting' and 'matching-only' areas:
 #### Match only
 
 - `transform from=` to match markers
-- `transform after=` to match markers
 - `display to=` for matching keys which contain markers
 
 ## Theory / Encoding

developer/src/kmc-ldml/src/compiler/disp.ts

@@ -1,5 +1,5 @@
 import { constants } from "@keymanapp/ldml-keyboard-constants";
-import { KMXPlus } from '@keymanapp/common-types';
+import { KMXPlus, LDMLKeyboard, MarkerParser } from '@keymanapp/common-types';
 
 import { CompilerMessages } from "./messages.js";
 import { SectionCompiler } from "./section-compiler.js";
@@ -9,6 +9,12 @@ import Disp = KMXPlus.Disp;
 import DispItem = KMXPlus.DispItem;
 
 export class DispCompiler extends SectionCompiler {
+  static validateMarkers(keyboard: LDMLKeyboard.LKKeyboard, emitMarkers: Set<string>, matchMarkers: Set<string>): boolean {
+    keyboard.displays?.display?.forEach(({ to }) => {
+      MarkerParser.allReferences(to).forEach(marker => matchMarkers.add(marker));
+    });
+    return true;
+  }
 
   public get id() {
     return constants.section.disp;

developer/src/kmc-ldml/src/compiler/keys.ts

@@ -1,5 +1,5 @@
 import { constants } from '@keymanapp/ldml-keyboard-constants';
-import { LDMLKeyboard, KMXPlus, Constants } from '@keymanapp/common-types';
+import { LDMLKeyboard, KMXPlus, Constants, MarkerParser } from '@keymanapp/common-types';
 import { CompilerMessages } from './messages.js';
 import { SectionCompiler } from "./section-compiler.js";
 
@@ -10,6 +10,12 @@ import KeysFlicks = KMXPlus.KeysFlicks;
 import { allUsedKeyIdsInLayers, calculateUniqueKeys, translateLayerAttrToModifier, validModifier } from '../util/util.js';
 
 export class KeysCompiler extends SectionCompiler {
+  static validateMarkers(keyboard: LDMLKeyboard.LKKeyboard, emitMarkers: Set<string>, matchMarkers: Set<string>): boolean {
+    keyboard.keys?.key?.forEach(({ to }) => {
+      MarkerParser.allReferences(to).forEach(marker => emitMarkers.add(marker));
+    });
+    return true;
+  }
 
   public get id() {
     return constants.section.keys;

developer/src/kmc-ldml/src/compiler/messages.ts

@@ -130,5 +130,9 @@ export class CompilerMessages {
   static Error_CantReferenceSetFromUnicodeSet = (o:{id: string}) =>
   m(this.ERROR_CantReferenceSetFromUnicodeSet, `Illegal use of set variable from within UnicodeSet: \$[${o.id}]`);
   static ERROR_CantReferenceSetFromUnicodeSet = SevError | 0x0020;
+
+  static Error_MissingMarkers = (o: { ids: string[] }) =>
+  m(this.ERROR_MissingMarkers, `Markers used for matching but not defined: ${o.ids?.join(',')}`);
+  static ERROR_MissingMarkers = SevError | 0x0021;
 }
 

developer/src/kmc-ldml/src/compiler/tran.ts

@@ -1,5 +1,5 @@
 import { constants, SectionIdent } from "@keymanapp/ldml-keyboard-constants";
-import { KMXPlus, LDMLKeyboard, CompilerCallbacks, VariableParser } from '@keymanapp/common-types';
+import { KMXPlus, LDMLKeyboard, CompilerCallbacks, VariableParser, MarkerParser } from '@keymanapp/common-types';
 import { SectionCompiler } from "./section-compiler.js";
 
 import Bksp = KMXPlus.Bksp;
@@ -18,7 +18,18 @@ import { CompilerMessages } from "./messages.js";
 
 type TransformCompilerType = 'simple' | 'backspace';
 
-class TransformCompiler<T extends TransformCompilerType, TranBase extends Tran> extends SectionCompiler {
+export class TransformCompiler<T extends TransformCompilerType, TranBase extends Tran> extends SectionCompiler {
+
+  static validateMarkers(keyboard: LDMLKeyboard.LKKeyboard, emitMarkers: Set<string>, matchMarkers: Set<string>): boolean {
+    keyboard?.transforms?.forEach(transforms =>
+      transforms.transformGroup.forEach(transformGroup => {
+        transformGroup.transform?.forEach(({ to, from }) => {
+          MarkerParser.allReferences(from).forEach(marker => matchMarkers.add(marker));
+          MarkerParser.allReferences(to).forEach(marker => emitMarkers.add(marker));
+        });
+      }));
+    return true;
+  }
 
   protected type: T;
 

developer/src/kmc-ldml/src/compiler/vars.ts

@@ -1,5 +1,5 @@
 import { SectionIdent, constants } from "@keymanapp/ldml-keyboard-constants";
-import { KMXPlus, LDMLKeyboard, CompilerCallbacks } from '@keymanapp/common-types';
+import { KMXPlus, LDMLKeyboard, CompilerCallbacks, MarkerParser } from '@keymanapp/common-types';
 import { VariableParser } from '@keymanapp/common-types';
 import { SectionCompiler } from "./section-compiler.js";
 import Vars = KMXPlus.Vars;
@@ -9,6 +9,9 @@ import UnicodeSetItem = KMXPlus.UnicodeSetItem;
 import DependencySections = KMXPlus.DependencySections;
 import LDMLKeyboardXMLSourceFile = LDMLKeyboard.LDMLKeyboardXMLSourceFile;
 import { CompilerMessages } from "./messages.js";
+import { KeysCompiler } from "./keys.js";
+import { TransformCompiler } from "./tran.js";
+import { DispCompiler } from "./disp.js";
 export class VarsCompiler extends SectionCompiler {
   public get id() {
     return constants.section.vars;
@@ -130,9 +133,60 @@ export class VarsCompiler extends SectionCompiler {
       }));
       valid = false;
     }
+
+    valid = this.validateMarkers() && valid; // accumulate validity
+
     return valid;
   }
 
+  private collectMarkers(emitMarkers : Set<string>, matchMarkers : Set<string>) : boolean {
+    let valid = true;
+
+    // call our friends to validate
+    valid = this.validateVarsMarkers(this.keyboard, emitMarkers, matchMarkers) && valid; // accumulate validity
+    valid = KeysCompiler.validateMarkers(this.keyboard, emitMarkers, matchMarkers) && valid; // accumulate validity
+    valid = TransformCompiler.validateMarkers(this.keyboard, emitMarkers, matchMarkers) && valid; // accumulate validity
+    valid = DispCompiler.validateMarkers(this.keyboard, emitMarkers, matchMarkers) && valid; // accumulate validity
+
+    return valid;
+  }
+
+  private validateMarkers(): boolean {
+    /** only the markers used in emitters */
+    const emitMarkers : Set<string> = new Set<string>();
+    /** only the markers used in matchers */
+    const matchMarkers : Set<string> = new Set<string>();
+
+
+    let valid = this.collectMarkers(emitMarkers, matchMarkers);
+
+    // see if there are any matched-but-not-emitted
+    const matchedNotEmitted : string[] = [];
+    for (const m of matchMarkers.values()) {
+      if (m === '.') continue; // match-all marker
+      if (!emitMarkers.has(m)) {
+        matchedNotEmitted.push(m);
+      }
+    }
+
+    // report once
+    if (matchedNotEmitted.length) {
+      matchedNotEmitted.sort();
+      this.callbacks.reportMessage(CompilerMessages.Error_MissingMarkers({ ids: matchedNotEmitted }));
+      valid = false;
+    }
+    return valid;
+  }
+
+  validateVarsMarkers(keyboard: LDMLKeyboard.LKKeyboard, emitMarkers: Set<string>, matchMarkers: Set<string>) : boolean {
+    keyboard?.variables?.string?.forEach(({value}) =>
+          MarkerParser.allReferences(value).forEach(marker => {
+            emitMarkers.add(marker);
+            matchMarkers.add(marker);
+          }));
+    return true;
+  }
+
   public compile(sections: DependencySections): Vars {
     const result =  new Vars();
 

developer/src/kmc-ldml/test/fixtures/sections/vars/fail-markers-badref-0.xml

@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+This will fail because the two markers given don't exist anywhere.
+
+-->
+<!DOCTYPE keyboard SYSTEM "../../../../../../../resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboard.dtd">
+<keyboard locale="mt" conformsTo="techpreview">
+  <names>
+    <name value="vars-markers-badref-0" />
+  </names>
+
+  <displays>
+    <!-- Not ok, nobody emits this, so this display doesn't make sense -->
+    <display to="\m{doesnt_exist_1}" display="¬" />
+    <!-- OK - it si in the strings list.  Could warn that it matches no keys -->
+    <display to="\m{maybe_emitted}" display="‽" />
+  </displays>
+
+  <variables>
+    <!-- This may be emitted, so at present it does not cause an error. -->
+    <!-- A linter could, however, hint on an unused variable. -->
+    <!-- Removing this variable would cause \m{maybe-emitted}  to error in the transform group -->
+    <string id="unused_variable" value="\m{maybe_emitted}" />
+  </variables>
+
+  <transforms type="simple">
+    <transformGroup>
+      <!-- not OK, nobody emits this. -->
+      <transform from="\m{doesnt_exist_3}" to="3" />
+    </transformGroup>
+  </transforms>
+  <transforms type="backspace">
+    <transformGroup>
+      <!-- not OK, nobody emits this. -->
+      <transform from="\m{doesnt_exist_2}" />
+      <!-- OK - it's in the variable list -->
+      <transform from="\m{maybe_emitted}" />
+      <!-- OK - matches all markers -->
+      <transform from="\m{.}" />
+    </transformGroup>
+  </transforms>
+</keyboard>

developer/src/kmc-ldml/test/fixtures/sections/vars/markers-maximal.xml

@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+  This keyboard won't make a lot of sense, it's mostly
+  a test of validating markers.
+
+  When we get further on in implementation, we'll shape this up
+  into something testable for implementation.
+
+-->
+<!DOCTYPE keyboard SYSTEM "../../../../../../../resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboard.dtd">
+<keyboard locale="mt" conformsTo="techpreview">
+  <names>
+    <name value="vars-markers-maximal" />
+  </names>
+
+  <displays>
+    <display to="\{m}" display="Ⓜ️" />
+  </displays>
+
+  <keys>
+    <key id="m" to="\{m}" />
+  </keys>
+
+  <!-- from spec -->
+  <variables>
+    <string id="x" value="\{x}" />
+  </variables>
+
+  <transforms type="simple">
+    <transformGroup>
+      <transform from="m" to="\{m}"/>
+      <transform from="x" to="\{x}"/>
+    </transformGroup>
+    <transformGroup>
+      <transform from="e\{x}" to="é" />
+    </transformGroup>
+  </transforms>
+  <transforms type="backspace">
+    <transformGroup>
+      <transform from="A\{.}B" />
+    </transformGroup>
+  </transforms>
+</keyboard>

developer/src/kmc-ldml/test/test-vars.ts

@@ -183,4 +183,25 @@ describe('vars', function () {
       ],
     },
   ]);
+  describe('markers', function () {
+    this.slow(500); // 0.5 sec -- json schema validation takes a while
+
+    testCompilationCases(VarsCompiler, [
+      {
+        subpath: 'sections/vars/markers-maximal.xml',
+      },
+      {
+        subpath: 'sections/vars/fail-markers-badref-0.xml',
+        errors: [
+          CompilerMessages.Error_MissingMarkers({
+            ids: [
+              'doesnt_exist_1',
+              'doesnt_exist_2',
+              'doesnt_exist_3',
+            ]
+          }),
+        ],
+      },
+    ]);
+  });
 });