Merge pull request #10230 from keymanapp/fix/core/clear-core-context-…

…invalidate

fix(core): clear core context on invalidate cache

common/test/keyboards/baseline/baseline.kpj

@@ -323,5 +323,12 @@
       <FileVersion>1.0</FileVersion>
       <FileType>.kmn</FileType>
     </File>
+    <File>
+      <ID>id_k_049___enter_invalidates_context</ID>
+      <Filename>k_049___enter_invalidates_context.kmn</Filename>
+      <Filepath>k_049___enter_invalidates_context.kmn</Filepath>
+      <FileVersion>1.0</FileVersion>
+      <FileType>.kmn</FileType>
+    </File>
   </Files>
 </KeymanDeveloperProject>

common/test/keyboards/baseline/k_000___null_keyboard.kmn

@@ -1,8 +1,9 @@
 store(&NAME) '000 - null keyboard'
-c Description: Tests null keyboard 
+c Description: Tests null keyboard
 c keys: [K_A][RALT K_B][SHIFT K_C]
 c expected: aC
-c context: 
+c expected context: C
+c context:
 
 store(&version) '6.0'
 

common/test/keyboards/baseline/k_008___vkey_input__ctrl_alt_2_.kmn

@@ -2,12 +2,13 @@ store(&NAME) '008 - vkey input (ctrl alt 2)'
 c Description: Tests basic vkey input with control and alt (Unicode)
 c keys: [LCTRL LALT K_A][LCTRL LALT K_B][LCTRL LALT K_C]
 c expected: \u0E01\u0E03
-c context: 
+c expected context: \u0E03
+c context:
 
 store(&version) '6.0'
 
 begin Unicode > use(Main)
-      
+
 group(Main) using keys
 
 + [CTRL ALT K_A] > U+0E01

common/test/keyboards/baseline/k_049___enter_invalidates_context.kmn

@@ -0,0 +1,18 @@
+store(&NAME) '049 - enter_invalidates_context'
+c Description: Tests that core context is cleared when kb processor
+c              determines its invalid. eg Enter key See #10182.
+c keys: [K_A][K_B][K_ENTER][K_C]
+c expected: abc
+c expected context: c
+c context:
+
+store(&version) '10.0'
+
+begin Unicode > use(Main)
+
+group(Main) using keys
+
+c Test if context was not invalidated by [K_ENTER] context will be 'abd'
+
+'a' 'b' + [K_C] > 'abd'
+

core/src/kmx/kmx_processor.cpp

@@ -247,6 +247,7 @@ kmx_processor::internal_process_queued_actions(km_core_state *state) {
       }
       break;
     case QIT_INVALIDATECONTEXT:
+      state->context().clear();
       state->actions().push_invalidate_context();
       break;
     default:

core/tests/kmx_test_source/kmx_test_source.cpp

@@ -110,11 +110,13 @@ KmxTestSource::load_source(
     const km::core::path &path,
     std::string &keys,
     std::u16string &expected,
+    std::u16string &expected_context,
     std::u16string &context,
     kmx_options &options,
     bool &expected_beep) {
   const std::string s_keys = "c keys: ";
   const std::string s_expected = "c expected: ";
+  const std::string s_expected_context = "c expected context: ";
   const std::string s_context = "c context: ";
   const std::string s_option = "c option: ";
   const std::string s_option_expected = "c expected option: ";
@@ -143,6 +145,8 @@ KmxTestSource::load_source(
       } else {
         expected = parse_source_string(line);
       }
+    } else if (is_token(s_expected_context, line)) {
+      expected_context = parse_source_string(line);
     } else if (is_token(s_context, line)) {
       context = parse_source_string(line);
     } else if (is_token(s_option, line)) {

core/tests/kmx_test_source/kmx_test_source.hpp

@@ -28,6 +28,7 @@ class KmxTestSource {
       const km::core::path &path,
       std::string &keys,
       std::u16string &expected,
+      std::u16string &expected_context,
       std::u16string &context,
       kmx_options &options,
       bool &expected_beep);

core/tests/unit/kmx/kmx.cpp

@@ -63,7 +63,9 @@ apply_action(
     std::u16string &text_store,
     std::vector<km_core_context_item> &context,
     km::tests::kmx_options &options,
-    km::tests::KmxTestSource &test_source) {
+    km::tests::KmxTestSource &test_source,
+    bool &context_invalidated
+    ) {
   switch (act.type) {
   case KM_CORE_IT_END:
     assert(false);
@@ -147,6 +149,8 @@ apply_action(
     }
   } break;
   case KM_CORE_IT_INVALIDATE_CONTEXT:
+    context.clear();
+    context_invalidated = true;
     std::cout << "action: context invalidated (markers cleared)" << std::endl;
     break;
   case KM_CORE_IT_EMIT_KEYSTROKE:
@@ -165,14 +169,20 @@ apply_action(
 int
 run_test(const km::core::path &source, const km::core::path &compiled) {
   std::string keys = "";
-  std::u16string expected = u"", context = u"";
+  std::u16string expected = u"", expected_context = u"NOT SUPPLIED", context = u"";
   km::tests::kmx_options options;
   bool expected_beep = false;
   km::tests::KmxTestSource test_source;
 
-  int result = test_source.load_source(source, keys, expected, context, options, expected_beep);
+  int result = test_source.load_source(source, keys, expected, expected_context, context, options, expected_beep);
   if (result != 0) return result;
 
+  // If an expect_context is not provided, default to expected == expect_context
+  std::cout << "expected_context  = " << string_to_hex(expected_context) <<  " [" << expected_context << "]" << std::endl;
+  if (expected_context == u"NOT SUPPLIED") {
+    expected_context = expected;
+  }
+
   std::cout << "source file   = " << source << std::endl
             << "compiled file = " << compiled << std::endl;
 
@@ -208,6 +218,8 @@ run_test(const km::core::path &source, const km::core::path &compiled) {
 
   // Setup baseline text store
   std::u16string text_store = context;
+  // used to stop comparing text_store to context if this flag is set
+  bool context_invalidated = false;
 
   // Run through key events, applying output for each event
   for (auto p = test_source.next_key(keys); p.vk != 0; p = test_source.next_key(keys)) {
@@ -223,16 +235,17 @@ run_test(const km::core::path &source, const km::core::path &compiled) {
       try_status(km_core_process_event(test_state, p.vk, p.modifier_state | test_source.caps_lock_state(), key_down, KM_CORE_EVENT_FLAG_DEFAULT));
 
       for (auto act = km_core_state_action_items(test_state, nullptr); act->type != KM_CORE_IT_END; act++) {
-        apply_action(test_state, *act, text_store, test_context, options, test_source);
+        apply_action(test_state, *act, text_store, test_context, options, test_source, context_invalidated);
       }
     }
 
-    // Compare context and text store at each step - should be identical
+    // Compare context and text store at each step
+    // should be identical unless an action has caused the context to be invalidated
     size_t n = 0;
     try_status(km_core_context_get(km_core_state_context(test_state), &citems));
     try_status(km_core_context_items_to_utf16(citems, nullptr, &n));
-    km_core_cp *buf = new km_core_cp[n];
-    try_status(km_core_context_items_to_utf16(citems, buf, &n));
+    km_core_cp *core_context_str = new km_core_cp[n];
+    try_status(km_core_context_items_to_utf16(citems, core_context_str, &n));
 
     // Verify that both our local test_context and the core's test_state.context have
     // not diverged
@@ -241,13 +254,12 @@ run_test(const km::core::path &source, const km::core::path &compiled) {
       assert(ci->type != KM_CORE_CT_END && test_ci != test_context.end()); // Verify that both lists are same length
       assert(test_ci->type == ci->type && test_ci->marker == ci->marker);
     }
-
     km_core_context_items_dispose(citems);
-    if (text_store != buf) {
-      std::cerr << "text store has diverged from buf" << std::endl;
-      std::cerr << "text store: " << string_to_hex(text_store) << " [" << text_store << "]" << std::endl;
-      std::cerr << "context   : " << string_to_hex(buf) << " [" << buf << "]" << std::endl;
-      assert(false);
+    if ((!context_invalidated) && (text_store != core_context_str)) {
+      std::cerr << "text store has unexpectedly diverged from core_context" << std::endl;
+      std::cerr << "text store  : " << string_to_hex(text_store) << " [" << text_store << "]" << std::endl;
+      std::cerr << "core context: " << string_to_hex(core_context_str) << " [" << core_context_str << "]" << std::endl;
+     assert(false);
     }
   }
 
@@ -259,8 +271,8 @@ run_test(const km::core::path &source, const km::core::path &compiled) {
   size_t n = 0;
   try_status(km_core_context_get(km_core_state_context(test_state), &citems));
   try_status(km_core_context_items_to_utf16(citems, nullptr, &n));
-  km_core_cp *buf = new km_core_cp[n];
-  try_status(km_core_context_items_to_utf16(citems, buf, &n));
+  km_core_cp *core_context_str = new km_core_cp[n];
+  try_status(km_core_context_items_to_utf16(citems, core_context_str, &n));
 
   // Verify that both our local test_context and the core's test_state.context have
   // not diverged
@@ -272,12 +284,13 @@ run_test(const km::core::path &source, const km::core::path &compiled) {
 
   km_core_context_items_dispose(citems);
 
-  std::cout << "expected  : " << string_to_hex(expected) << " [" << expected << "]" << std::endl;
-  std::cout << "text store: " << string_to_hex(text_store) << " [" << text_store << "]" << std::endl;
-  std::cout << "context   : " << string_to_hex(buf) << " [" << buf << "]" << std::endl;
+  std::cout << "expected        : " << string_to_hex(expected) << " [" << expected << "]" << std::endl;
+  std::cout << "text store      : " << string_to_hex(text_store) << " [" << text_store << "]" << std::endl;
+  std::cout << "expected context: " << string_to_hex(expected_context) << " [" << expected_context << "]" << std::endl;
+  std::cout << "context         : " << string_to_hex(core_context_str) << " [" << core_context_str << "]" << std::endl;
 
-  // Compare internal context with expected result
-  if (buf != expected) return __LINE__;
+  // Compare internal context with expected context result
+  if (core_context_str != expected_context) return __LINE__;
 
   // Compare text store with expected result
   if (text_store != expected) return __LINE__;

core/tests/unit/kmx/meson.build

@@ -79,7 +79,8 @@ tests = [
   'k_045___deadkey_and_context',
   'k_046___deadkey_and_contextex',
   'k_047___caps_always_off_initially_on',
-  'k_048___modifier_keys_keep_context'
+  'k_048___modifier_keys_keep_context',
+  'k_049___enter_invalidates_context'
 ]
 
 node = find_program('node', required: true)

linux/ibus-keyman/tests/testfixture.cpp

@@ -309,12 +309,12 @@ static void test_source(IBusKeymanTestsFixture *fixture, gconstpointer user_data
 
   km::tests::KmxTestSource test_source;
   std::string keys        = "";
-  std::u16string expected = u"", context = u"";
+  std::u16string expected = u"", expected_context = u"", context = u"";
   km::tests::kmx_options options;
   bool expected_beep = false;
   // NOTE: we don't verify expected beeps since engine.c directly calls a gdk method so we
   // don't know when it gets called.
-  g_assert_cmpint(test_source.load_source(sourcefile.c_str(), keys, expected, context, options, expected_beep), ==, 0);
+  g_assert_cmpint(test_source.load_source(sourcefile.c_str(), keys, expected, expected_context, context, options, expected_beep), ==, 0);
 
   for (auto & option : options) {
     if (option.type == km::tests::KOT_INPUT) {

mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMEngineTests.m

@@ -506,9 +506,6 @@ - (void)testContextMatch_InvertedPlatformLogic_NotTouch {
 
 
 - (void)testEngine_ipaKeyboardAction_DoesNotCrash_Issue1892 {
-//  KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileForSilIpaTests];
-//  KMEngine *engine = [[KMEngine alloc] initWithKMX:kmxFile contextBuffer:@"a" verboseLogging:YES];
-//  NSEvent *event = [NSEvent keyEventWithType:NSEventTypeKeyDown location:NSMakePoint(0, 0) modifierFlags:0 timestamp:0 windowNumber:0 context:nil characters:@"=" charactersIgnoringModifiers:@"=" isARepeat:NO keyCode:kVK_ANSI_Equal];
     KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileForIndexOffsetTests];
     KMEngine *engine = [[KMEngine alloc] initWithKMX:kmxFile context:@"z" verboseLogging:YES];
     NSEvent *event = [NSEvent keyEventWithType:NSEventTypeKeyDown location:NSMakePoint(0, 0) modifierFlags:0 timestamp:0 windowNumber:0 context:nil characters:@"a" charactersIgnoringModifiers:@"a" isARepeat:NO keyCode:kVK_ANSI_A];
@@ -517,21 +514,6 @@ - (void)testEngine_ipaKeyboardAction_DoesNotCrash_Issue1892 {
     XCTAssert([output.textToInsert isEqualToString:@"Z"], @"Expected output to be 'Z'.");
 }
 
-/*
-- (void)testLegacyProcessEvent_eventForFWithElNuerKmx_ReturnsCorrectCharacter {
-    KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileForElNuerTests];
-    KMEngine *engine = [[KMEngine alloc] initWithKMX:kmxFile contextBuffer:@"" verboseLogging:YES];
-    NSEvent *event = [NSEvent keyEventWithType:NSEventTypeKeyDown location:NSMakePoint(0, 0) modifierFlags:0 timestamp:0 windowNumber:0 context:nil characters:@"f" charactersIgnoringModifiers:@"f" isARepeat:NO keyCode:kVK_ANSI_F];
-    NSArray *actions = [engine experimentallyProcessEventForUnitTestingOnly:event usingCore:NO];
-    XCTAssert(actions.count == 1, @"Expected one action");
-    NSDictionary *action = actions[0];
-    NSString *actionType = [[action allKeys] objectAtIndex:0];
-    XCTAssert([actionType isEqualToString:Q_STR], @"Expected Q_STR action");
-    NSString *output = [action objectForKey:actionType];
-    XCTAssert([output isEqualToString:@"ɣ"], @"Expected output to be 'ɣ'.");
-}
-*/
-
 - (void)testCoreProcessEvent_eventForFWithElNuerKmx_ReturnsCorrectCharacter {
     KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileForElNuerTests];
     KMEngine *engine = [[KMEngine alloc] initWithKMX:kmxFile context:@"" verboseLogging:YES];
@@ -574,62 +556,25 @@ - (void)testCoreProcessEvent_backspaceElNuerWithContext_EmptiesContextReturnsDel
     XCTAssert([context isEqualToString:@""], @"Context should be empty.");
 }
 
-/*
-- (void)testLegacyProcessEvent_eventDeleteWithElNuerKmx_ReturnsEmptyActionListContextUnchanged {
-    KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileForElNuerTests];
-    KMEngine *engine = [[KMEngine alloc] initWithKMX:kmxFile contextBuffer:@"ɣ" verboseLogging:YES];
-    NSEvent *event = [NSEvent keyEventWithType:NSEventTypeKeyDown location:NSMakePoint(0, 0) modifierFlags:0 timestamp:0 windowNumber:0 context:nil characters:@"\b" charactersIgnoringModifiers:@"\b" isARepeat:NO keyCode:kVK_Delete];
-    NSArray *actions = [engine experimentallyProcessEventForUnitTestingOnly:event usingCore:NO];
-    XCTAssert(actions.count == 0, @"Expected no actions");
-    NSString *context = engine.contextBuffer;
-    XCTAssert([context isEqualToString:@"ɣ"], @"Context should be unchanged.");
-}
-*/
-
-
-- (void)testCoreProcessEvent_eventReturnWithElNuerKmx_ContextUnchangedReturnsReturn {
+- (void)testCoreProcessEvent_eventReturnWithElNuerKmx_EmitWithContextEmpty {
     KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileForElNuerTests];
     KMEngine *engine = [[KMEngine alloc] initWithKMX:kmxFile context:@"ɣ" verboseLogging:YES];
     NSEvent *event = [NSEvent keyEventWithType:NSEventTypeKeyDown location:NSMakePoint(0, 0) modifierFlags:0 timestamp:0 windowNumber:0 context:nil characters:@"\n" charactersIgnoringModifiers:@"\n" isARepeat:NO keyCode:kVK_Return];
     CoreKeyOutput *output = [engine processEvent:event];
     XCTAssert(output.emitKeystroke, @"Expected emitKeystroke==YES");
     NSString *context = engine.getCoreContext;
-    XCTAssert([context isEqualToString:@"ɣ"], @"Context should be unchanged.");
+    XCTAssert([context isEqualToString:@""], @"Context should be cleared.");
 }
 
-/*
-- (void)testLegacyProcessEvent_eventReturnWithElNuerKmx_ReturnsEmptyActionListContextUnchanged {
-    KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileForElNuerTests];
-    KMEngine *engine = [[KMEngine alloc] initWithKMX:kmxFile contextBuffer:@"ɣ" verboseLogging:YES];
-    NSEvent *event = [NSEvent keyEventWithType:NSEventTypeKeyDown location:NSMakePoint(0, 0) modifierFlags:0 timestamp:0 windowNumber:0 context:nil characters:@"\n" charactersIgnoringModifiers:@"\n" isARepeat:NO keyCode:kVK_Return];
-    NSArray *actions = [engine experimentallyProcessEventForUnitTestingOnly:event usingCore:NO];
-    XCTAssert(actions.count == 0, @"Expected no actions");
-    NSString *context = engine.contextBuffer;
-    XCTAssert([context isEqualToString:@"ɣ"], @"Context should be unchanged.");
-}
-*/
-
-- (void)testCoreProcessEvent_eventTabWithElNuerKmx_ContextUnchangedReturnsTab {
+- (void)testCoreProcessEvent_eventTabWithElNuerKmx_EmitWithContextEmpty {
     KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileForElNuerTests];
     KMEngine *engine = [[KMEngine alloc] initWithKMX:kmxFile context:@"ɣ" verboseLogging:YES];
     NSEvent *event = [NSEvent keyEventWithType:NSEventTypeKeyDown location:NSMakePoint(0, 0) modifierFlags:0 timestamp:0 windowNumber:0 context:nil characters:@"\t" charactersIgnoringModifiers:@"\t" isARepeat:NO keyCode:kVK_Tab];
     CoreKeyOutput *output = [engine processEvent:event];
     XCTAssert(output.emitKeystroke, @"Expected emitKeystroke==YES");
     NSString *context = engine.getCoreContext;
-    XCTAssert([context isEqualToString:@"ɣ"], @"Context should be unchanged.");
-}
-
-/*
-- (void)testLegacyProcessEvent_eventTabWithElNuerKmx_ReturnsEmptyActionListContextUnchanged {
-    KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileForElNuerTests];
-    KMEngine *engine = [[KMEngine alloc] initWithKMX:kmxFile contextBuffer:@"ɣ" verboseLogging:YES];
-    NSEvent *event = [NSEvent keyEventWithType:NSEventTypeKeyDown location:NSMakePoint(0, 0) modifierFlags:0 timestamp:0 windowNumber:0 context:nil characters:@"\t" charactersIgnoringModifiers:@"\n" isARepeat:NO keyCode:kVK_Tab];
-    NSArray *actions = [engine experimentallyProcessEventForUnitTestingOnly:event usingCore:NO];
-    XCTAssert(actions.count == 0, @"Expected no actions");
-    NSString *context = engine.contextBuffer;
-    XCTAssert([context isEqualToString:@"ɣ"], @"Context should be unchanged.");
+    XCTAssert([context isEqualToString:@""], @"Context should be cleared.");
 }
-*/
 
 - (void)testCoreProcessEvent_eventSingleQuoteWithElNuerKmx_ReturnsDiacritic {
     KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileForElNuerTests];
@@ -642,24 +587,4 @@ - (void)testCoreProcessEvent_eventSingleQuoteWithElNuerKmx_ReturnsDiacritic {
     XCTAssert([context isEqualToString:@"\u025B\u0308"], @"Context updated with diacritic.");
 }
 
-/*
-- (void)testLegacyProcessEvent_eventSingleQuoteWithElNuerKmx_ReturnsTwoActions {
-    KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileForElNuerTests];
-    NSString *context = @"ɛ";
-    KMEngine *engine = [[KMEngine alloc] initWithKMX:kmxFile contextBuffer:context verboseLogging:YES];
-    NSEvent *event = [NSEvent keyEventWithType:NSEventTypeKeyDown location:NSMakePoint(0, 0) modifierFlags:0 timestamp:0 windowNumber:0 context:nil characters:@"'" charactersIgnoringModifiers:@"'" isARepeat:NO keyCode:kVK_ANSI_Quote];
-    NSArray *actions = [engine experimentallyProcessEventForUnitTestingOnly:event usingCore:NO];
-    XCTAssert(actions.count == 2, @"Expected two actions");
-    NSDictionary *action = actions[0];
-    NSString *actionType = [[action allKeys] objectAtIndex:0];
-    XCTAssert([actionType isEqualToString:Q_BACK], @"Expected Q_BACK action");
-    action = actions[1];
-    actionType = [[action allKeys] objectAtIndex:0];
-    NSString *characters = [action objectForKey:actionType];
-    XCTAssert([characters isEqualToString:@"\u025B\u0308"], @"Expected \u025B\u0308");
-    context = engine.contextBuffer;
-    XCTAssert([context isEqualToString:@"\u025B\u0308"], @"Context updated with diacritic.");
-}
-*/
-
 @end