Merge branch 'master' into feat/core/9468-normalization-epic-ldml

.github/workflows/deb-packaging.yml

@@ -105,10 +105,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        # Currently not building mantic until ibus version on mantic stabilizied
-        # and we can provide a patched version
-        # dist: [focal, jammy, lunar, mantic]
-        dist: [focal, jammy, lunar]
+        dist: [focal, jammy, lunar, mantic]
         arch: [amd64]
 
     runs-on: ubuntu-latest

HISTORY.md

@@ -1,5 +1,23 @@
 # Keyman Version History
 
+## 17.0.194 alpha 2023-10-18
+
+* chore(linux): Re-enable building for Ubuntu 23.10 Mantic (#9780)
+* chore(linux): Add missing tests (#9783)
+* fix(web): fixes touch form-factor default kbd on cookieless keymanweb.com page load (#9786)
+* fix(developer): three kmc .keyboard_info generation bugs (#9784)
+* fix(developer): handle invalid project folders cleanly (#9785)
+* chore(linux): Fix build scripts (#9781)
+
+## 17.0.193 alpha 2023-10-17
+
+* fix(developer): kmc crash on start (#9771)
+* fix(core): don't use double newlines in debuglog (#9258)
+* chore(linux): Add code coverage index page (#9758)
+* feat(linux): Allow installing keyboards with arbitrary language  ️ (#9756)
+* fix(linux): Fix crash initializing Sentry with Python < 3.10 (#9774)
+* chore(linux): Rename `kbp_state_get_intermediate_context` to `km_core…`  ️ (#9775)
+
 ## 17.0.192 alpha 2023-10-16
 
 * fix(ios): missing backslash in build script (#9765)

VERSION.md

@@ -1 +1 @@
-17.0.193
+17.0.195

common/web/types/src/kpj/keyman-developer-project.ts

@@ -30,6 +30,9 @@ export class KeymanDeveloperProject {
       throw new Error('populateFiles can only be called on a v2.0 project');
     }
     let sourcePath = this.resolveProjectPath(this.options.sourcePath);
+    if(!this.callbacks.fs.existsSync(sourcePath)) {
+      return false;
+    }
     let files = this.callbacks.fs.readdirSync(sourcePath);
     for(let filename of files) {
       let fullPath = this.callbacks.path.join(sourcePath, filename);
@@ -44,6 +47,7 @@ export class KeymanDeveloperProject {
         this.files.push(file);
       }
     }
+    return true;
   }
 
   public isKeyboardProject() {

common/web/types/src/main.ts

@@ -36,7 +36,7 @@ export { TouchLayoutFileWriter, TouchLayoutFileWriterOptions } from './keyman-to
 
 export * as KPJ from './kpj/kpj-file.js';
 export { KPJFileReader } from './kpj/kpj-file-reader.js';
-export { KeymanDeveloperProject, KeymanDeveloperProjectFile, } from './kpj/keyman-developer-project.js';
+export { KeymanDeveloperProject, KeymanDeveloperProjectFile, KeymanDeveloperProjectType, } from './kpj/keyman-developer-project.js';
 
 export * as KpsFile from './package/kps-file.js';
 export * as KmpJsonFile from './package/kmp-json-file.js';

core/include/keyman/keyman_core_api.h

@@ -936,7 +936,7 @@ km_core_state_context(km_core_state *state);
 
 /*
 ```
-### `kbp_state_get_intermediate_context`
+### `km_core_state_get_intermediate_context`
 ##### Description:
 Get access to the state object's keyboard processor's intermediate context. This context
 is used during an IMX callback, part way through processing a keystroke.
@@ -950,7 +950,7 @@ to `km_core_context_items_dispose`.
 */
 KMN_API
 km_core_status
-kbp_state_get_intermediate_context(km_core_state *state, km_core_context_item ** context_items);
+km_core_state_get_intermediate_context(km_core_state *state, km_core_context_item ** context_items);
 
 /*
 ```

core/src/debuglog.cpp

@@ -358,7 +358,7 @@ int DebugLog_1(const char *file, int line, const char *function, const char *fmt
           "%ld" TAB   //"TickCount" TAB
           "%s:%d" TAB //"SourceFile" TAB
           "%s" TAB    //"Function"
-          "%s" NL,    //"Message"
+          "%s",    //"Message"
 
           GetTickCount(), //"TickCount" TAB
           file, line,     //"SourceFile" TAB

core/src/km_core_state_api.cpp

@@ -69,7 +69,7 @@ km_core_context *km_core_state_context(km_core_state *state)
   return static_cast<km_core_context *>(&state->context());
 }
 
-km_core_status kbp_state_get_intermediate_context(
+km_core_status km_core_state_get_intermediate_context(
   km_core_state *state,
   km_core_context_item ** context_items
 ) {

core/tests/unit/kmx/kmx_imx.cpp

@@ -74,7 +74,7 @@ uint8_t test_imx_callback(km_core_state *state, uint32_t imx_id, void *callback_
 
   // Test get intermediate context
   km_core_context_item *entry_context = nullptr;
-  kbp_state_get_intermediate_context(state, &entry_context);
+  km_core_state_get_intermediate_context(state, &entry_context);
 
   size_t n = 0;
   try_status(km_core_context_items_to_utf16(entry_context, nullptr, &n))
@@ -142,7 +142,7 @@ uint8_t test_imx_callback(km_core_state *state, uint32_t imx_id, void *callback_
   }
   // Test Exit Context
   km_core_context_item *exit_context = nullptr;
-  kbp_state_get_intermediate_context(state, &exit_context);
+  km_core_state_get_intermediate_context(state, &exit_context);
 
   n = 0;
   try_status(km_core_context_items_to_utf16(exit_context, nullptr, &n))

developer/src/common/web/utils/src/validate-mit-license.ts

@@ -9,7 +9,7 @@ export function validateMITLicense(license: string) {
   // lexical-models repositories, suggest we don't worry about this
 
   // split input text into paragraphs, trimming whitespace
-  const text = license.split('\n').map(line => line.trim()).join('\n').split(/\n\s*\n/);
+  const text = license.trim().split('\n').map(line => line.trim()).join('\n').split(/\n\s*\n/);
 
   // MIT license, cleanup whitespace
   const clauses = [

developer/src/kmc-keyboard-info/src/index.ts

@@ -389,8 +389,8 @@ export class KeyboardInfoCompiler {
       keyboard_info.languages[language] = {};
     }
 
-    const fontSource = kmpJsonData.keyboards.map(e => e.displayFont).concat(...kmpJsonData.keyboards.map(e => e.webDisplayFonts ?? []));
-    const oskFontSource = kmpJsonData.keyboards.map(e => e.oskFont).concat(...kmpJsonData.keyboards.map(e => e.webOskFonts ?? []));
+    const fontSource = [].concat(...kmpJsonData.keyboards.map(e => e.displayFont ? [e.displayFont] : []), ...kmpJsonData.keyboards.map(e => e.webDisplayFonts ?? []));
+    const oskFontSource = [].concat(...kmpJsonData.keyboards.map(e => e.oskFont ? [e.oskFont] : []), ...kmpJsonData.keyboards.map(e => e.webOskFonts ?? []));
 
     for(const bcp47 of Object.keys(keyboard_info.languages)) {
       const language = keyboard_info.languages[bcp47];

developer/src/kmc/src/commands/analyze.ts

@@ -21,13 +21,13 @@ interface AnalysisActivityOptions /* not inheriting from CompilerBaseOptions */
 };
 
 export function declareAnalyze(program: Command) {
-  let command = program.command('analyze [infile...]');
+  let command = program.command('analyze');
   declareOskCharUse(command);
   declareOskRewrite(command);
 }
 
 function declareOskCharUse(command: Command) {
-  let subCommand = command.command('osk-char-use');
+  let subCommand = command.command('osk-char-use [infile...]');
   BaseOptions.addLogLevel(subCommand);
   subCommand
     .description('Analyze On Screen Keyboards for character usage')
@@ -52,7 +52,7 @@ function declareOskCharUse(command: Command) {
 }
 
 function declareOskRewrite(command: Command) {
-  let subCommand = command.command('osk-rewrite-from-char-use');
+  let subCommand = command.command('osk-rewrite-from-char-use [infile...]');
   subCommand
     .description('Rewrite On Screen Keyboard files from source mapping; given file.ext, output is written to file-pua.ext')
     .addOption(new Option('-m, --mapping-file <filename>', 'JSON mapping file to read from').makeOptionMandatory())

developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts

@@ -1,6 +1,6 @@
 import * as fs from 'fs';
 import { BuildActivity } from './BuildActivity.js';
-import { CompilerCallbacks, KeymanDeveloperProject, KeymanFileTypes } from '@keymanapp/common-types';
+import { CompilerCallbacks, KeymanFileTypes } from '@keymanapp/common-types';
 import { KeyboardInfoCompiler } from '@keymanapp/kmc-keyboard-info';
 import { loadProject } from '../../util/projectLoader.js';
 import { InfrastructureMessages } from '../../messages/infrastructureMessages.js';
@@ -26,21 +26,21 @@ export class BuildKeyboardInfo extends BuildActivity {
       return false;
     }
 
-    const keyboard = findProjectFile(callbacks, project, KeymanFileTypes.Source.KeymanKeyboard);
-    const kps = findProjectFile(callbacks, project, KeymanFileTypes.Source.Package);
-    if(!keyboard || !kps)  {
-      // Error messages written by findProjectFile
+    const kps = project.files.find(file => file.fileType == KeymanFileTypes.Source.Package);
+    if(!kps)  {
+      callbacks.reportMessage(InfrastructureMessages.Error_FileTypeNotFound({ext: KeymanFileTypes.Source.Package}));
       return false;
     }
 
-    const jsFilename = project.resolveOutputFilePath(keyboard, KeymanFileTypes.Source.KeymanKeyboard, KeymanFileTypes.Binary.WebKeyboard);
+    const keyboard = project.files.find(file => file.fileType == KeymanFileTypes.Source.KeymanKeyboard);
+    const jsFilename = keyboard ? project.resolveOutputFilePath(keyboard, KeymanFileTypes.Source.KeymanKeyboard, KeymanFileTypes.Binary.WebKeyboard) : null;
     const lastCommitDate = getLastGitCommitDate(project.projectPath);
 
     const compiler = new KeyboardInfoCompiler(callbacks);
     const data = compiler.writeKeyboardInfoFile({
       kmpFilename:  project.resolveOutputFilePath(kps, KeymanFileTypes.Source.Package, KeymanFileTypes.Binary.Package),
       kpsFilename: project.resolveInputFilePath(kps),
-      jsFilename: fs.existsSync(jsFilename) ? jsFilename : undefined,
+      jsFilename: jsFilename && fs.existsSync(jsFilename) ? jsFilename : undefined,
       sourcePath: calculateSourcePath(infile),
       lastCommitDate,
       forPublishing: !!options.forPublishing,
@@ -61,11 +61,3 @@ export class BuildKeyboardInfo extends BuildActivity {
     return true;
   }
 }
-
-function findProjectFile(callbacks: CompilerCallbacks, project: KeymanDeveloperProject, ext: KeymanFileTypes.Source) {
-  const file = project.files.find(file => file.fileType == ext);
-  if(!file) {
-    callbacks.reportMessage(InfrastructureMessages.Error_FileTypeNotFound({ext}));
-  }
-  return file;
-}

developer/src/kmc/src/commands/buildClasses/BuildProject.ts

@@ -1,6 +1,6 @@
 import * as path from 'path';
 import * as fs from 'fs';
-import { CompilerCallbacks, CompilerFileCallbacks, KeymanDeveloperProject, KeymanDeveloperProjectFile, KeymanFileTypes } from '@keymanapp/common-types';
+import { CompilerCallbacks, CompilerFileCallbacks, KeymanDeveloperProject, KeymanDeveloperProjectFile, KeymanDeveloperProjectType, KeymanFileTypes } from '@keymanapp/common-types';
 import { BuildActivity } from './BuildActivity.js';
 import { buildActivities, buildKeyboardInfoActivity, buildModelInfoActivity } from './buildActivities.js';
 import { InfrastructureMessages } from '../../messages/infrastructureMessages.js';
@@ -55,10 +55,25 @@ class ProjectBuilder {
 
     // Build project metadata
     if(this.options.forPublishing || !this.project.options.skipMetadataFiles) {
-      if(!await (this.buildProjectTargets(
-          this.project.isKeyboardProject()
-          ? buildKeyboardInfoActivity
-          : buildModelInfoActivity))) {
+      let activity: BuildActivity = null;
+
+      // Determine activity type first of all by examining files in the project
+      // Then if that is inconclusive, examine the declared project type
+      if(this.project.isKeyboardProject()) {
+        activity = buildKeyboardInfoActivity;
+      } else if(this.project.isLexicalModelProject()) {
+        activity = buildModelInfoActivity;
+      } else if(this.project.options.projectType == KeymanDeveloperProjectType.Keyboard) {
+        activity = buildKeyboardInfoActivity;
+      } else if(this.project.options.projectType == KeymanDeveloperProjectType.LexicalModel) {
+        activity = buildModelInfoActivity;
+      } else {
+        // If we get here, then we are not sure what type of project this is, so
+        // we'll assume keyboard
+        activity = buildKeyboardInfoActivity;
+      }
+
+      if(!await (this.buildProjectTargets(activity))) {
         return false;
       }
     }

developer/src/kmc/src/messages/infrastructureMessages.ts

@@ -79,5 +79,9 @@ export class InfrastructureMessages {
   static Error_CannotCreateFolder = (o:{folderName:string, e: any}) => m(this.ERROR_CannotCreateFolder, null,
     `Unable to create folder ${o.folderName}: ${o.e ?? 'unknown error'}`);
   static ERROR_CannotCreateFolder = SevError | 0x0011;
+
+  static Error_InvalidProjectFolder = (o:{folderName:string}) => m(this.ERROR_InvalidProjectFolder,
+    `The folder ${o.folderName} does not appear to be a Keyman Developer project`);
+  static ERROR_InvalidProjectFolder = SevError | 0x0012;
 }
 

developer/src/kmc/src/util/projectLoader.ts

@@ -11,18 +11,15 @@ export const isProject = (filename: string): boolean =>
   );
 
 export function loadProject(infile: string, callbacks: CompilerCallbacks) {
-  // TODO: move path requirement out of here?
   infile = path.resolve(infile.replace(/\\/g, '/'));
 
-  // TODO: move fs requirement out of here?
-  if(fs.statSync(infile).isDirectory()) {
+  if(fs.existsSync(infile) && fs.statSync(infile).isDirectory()) {
     // This is a project folder, look for folder-name.kpj
     infile = path.join(infile, path.basename(infile) + KeymanFileTypes.Source.Project);
   }
 
   // infile should be the .kpj
   if(!infile.endsWith(KeymanFileTypes.Source.Project)) {
-    // TODO separate error code
     callbacks.reportMessage(InfrastructureMessages.Error_NotAProjectFile({filename:infile}));
     return null;
   }
@@ -37,7 +34,10 @@ export function loadProject(infile: string, callbacks: CompilerCallbacks) {
 function loadDefaultProjectFromFolder(infile: string, callbacks: CompilerCallbacks): KeymanDeveloperProject {
   // The folder does not contain a .kpj, so construct a default 2.0 .kpj
   const project = new KeymanDeveloperProject(infile, '2.0', callbacks);
-  project.populateFiles();
+  if(!project.populateFiles()) {
+    callbacks.reportMessage(InfrastructureMessages.Error_InvalidProjectFolder({folderName:path.dirname(infile)}));
+    return null;
+  }
   return project;
 }
 

docs/linux/README.md

@@ -87,6 +87,36 @@ scripts/run-tests
 The `run-tests` script accepts different arguments which can be seen with
 `scripts/run-tests --help`.
 
+### Code Coverage Report
+
+#### Prerequisites
+
+Code coverage reports require some additional tools: lcov, gcovr,
+libdatetime-perl, and coverage.
+
+You can install these with:
+
+```bash
+sudo apt update
+sudo apt install -y lcov libdatetime-perl gcovr
+pip3 install coverage
+```
+
+#### Creating and displaying code coverage reports
+
+All three projects (ibus-keyman, keyman-config, and keyman-system-service)
+can produce code coverage reports.
+
+Run `./build.sh --debug --coverage --report test` to create the report.
+
+**Note:** You might have to run `./build.sh clean` first.
+
+There's also an index page (`linux/CodecoverageReports.html`) that links to all
+three reports. You can run
+`linux/build.sh --debug --coverage --report --open --no-integration test`
+to create the coverage reports for all three Linux projects and open the
+index page in the browser.
+
 ## Running Keyman for Linux
 
 ### Setting up Ibus

linux/CodeCoverageReports.html

@@ -0,0 +1,37 @@
+<html>
+<head>
+  <title>Keyman for Linux Code Coverage</title>
+  <style type="text/css">
+    body, html {width: 100%; height: 100%; margin-left: 0; padding-left: 0;}
+    .row-container {display: flex; width: 100%; height: 100%; flex-direction: column; overflow: hidden;}
+    .second-row { flex-grow: 1; border: none; margin: 0; padding: 0; border: none; }
+    .tablink { background-color: #555; color: white; float: left; border: none; border-right: 1px solid black; outline: none; cursor: pointer; padding: 14px 16px; font-size: 17px; width: 33.3%; }
+    .tablink:hover { background-color: #777; }
+  </style>
+</head>
+  <body>
+  <div class="row-container">
+    <div class="first-row">
+      <h1>Keyman for Linux Code Coverage Reports</h1>
+      <button class="tablink" onclick="openPage(this, 'build/x86_64/debug/meson-logs/coveragereport/index.html')" id="defaultOpen">ibus-keyman</button>
+      <button class="tablink" onclick="openPage(this, 'keyman-config/build/coveragereport/index.html')">keyman-config</button>
+      <button class="tablink" onclick="openPage(this, 'keyman-system-service/build/x86_64/debug/meson-logs/coveragereport/index.html')">keyman-system-service</button>
+    </div>
+    <iframe id="iframe_a" class="second-row"></iframe>
+  </div>
+
+  <script>
+    function openPage(elmnt, filename) {
+      document.getElementById('iframe_a').src = filename;
+      tablinks = document.getElementsByClassName("tablink");
+      for (i = 0; i < tablinks.length; i++) {
+        tablinks[i].style.backgroundColor = "";
+      }
+      elmnt.style.backgroundColor = "#777";
+    }
+
+    document.getElementById("defaultOpen").click();
+  </script>
+
+  </body>
+</html>

linux/README.md

@@ -1,5 +1,3 @@
 # Keyman for Linux
 
 See [/docs/linux/README.md](../docs/linux/README.md) for documentation.
-
-.

linux/build.sh

@@ -24,9 +24,19 @@ builder_describe \
   "install                   install artifacts" \
   "uninstall                 uninstall artifacts" \
   "--no-integration+         don't run integration tests" \
+  "--coverage+               capture test coverage" \
   "--report+                 create coverage report" \
-  "--coverage+               capture test coverage"
+  "--open                    open the coverage reports in the browser"
 
 builder_parse "$@"
 
 builder_run_child_actions clean configure build test install uninstall
+
+test_action() {
+  if builder_has_option --open; then
+    builder_echo "Opening coverage reports in browser..."
+    xdg-open "file://${THIS_SCRIPT_PATH}/CodeCoverageReports.html"
+  fi
+}
+
+builder_run_action test test_action