feat(core): implement loading KMX from blob

- split keyboard loading into loading KMX file into blob and then
  loading the keyboard processor from the blob.
- deprecate `km_core_keyboard_load`
- move file access next to deprecated method. This is now the only place
  that loads a file in Core; unit tests have some more places that
  load files.
- introduce GTest and add unit tests for loading from blob

Part-of: #11293

core/include/keyman/keyman_core_api.h

@@ -1007,7 +1007,10 @@ Provides read-only information about a keyboard.
 typedef struct {
   km_core_cu const * version_string;
   km_core_cu const * id;
+
+  KMN_DEPRECATED
   km_core_path_name  folder_path;
+
   km_core_option_item const * default_options;
 } km_core_keyboard_attrs;
 
@@ -1022,7 +1025,7 @@ typedef struct {
 : Keyman keyboard ID string.
 
 `folder_path`
-: Path to the unpacked folder containing the keyboard and associated resources.
+: Path to the unpacked folder containing the keyboard and associated resources (deprecated).
 
 `default_options`
 : Set of default values for any options included in the keyboard.
@@ -1096,13 +1099,15 @@ typedef struct {
 
 ## Description
 
+DEPRECATED: use [km_core_keyboard_load_from_blob] instead.
+
 Parse and load keyboard from the supplied path and a pointer to the loaded keyboard
-into the out paramter.
+into the out parameter.
 
 ## Specification
 
 ```c */
-KMN_API
+KMN_DEPRECATED_API
 km_core_status
 km_core_keyboard_load(km_core_path_name kb_path,
                      km_core_keyboard **keyboard);
@@ -1140,6 +1145,60 @@ km_core_keyboard_load(km_core_path_name kb_path,
 
 -------------------------------------------------------------------------------
 
+# km_core_keyboard_load_from_blob()
+
+## Description
+
+Parse and load keyboard from the supplied blob and a pointer to the loaded keyboard
+into the out paramter.
+
+## Specification
+
+```c */
+KMN_API
+km_core_status km_core_keyboard_load_from_blob(char const* kb_name,
+                                               void* blob,
+                                               size_t blob_size,
+                                               km_core_keyboard** keyboard);
+
+/*
+```
+
+## Parameters
+
+`kb_name`
+: a string with the name of the keyboard.
+
+`blob`
+: a byte array containing the content of a KMX/KMX+ file.
+
+`blob_size`
+: A pointer to a size_t variable with the size of the blob in bytes.
+
+`keyboard`
+: A pointer to result variable: A pointer to the opaque keyboard
+  object returned by the Processor. This memory must be freed with a
+  call to [km_core_keyboard_dispose].
+
+## Returns
+
+`KM_CORE_STATUS_OK`
+: On success.
+
+`KM_CORE_STATUS_NO_MEM`
+: In the event an internal memory allocation fails.
+
+`KM_CORE_STATUS_IO_ERROR`
+: In the event the keyboard file is unparseable for any reason
+
+`KM_CORE_STATUS_INVALID_ARGUMENT`
+: In the event `keyboard` is null.
+
+`KM_CORE_STATUS_OS_ERROR`
+: Bit 31 (high bit) set, bits 0-30 are an OS-specific error code.
+
+-------------------------------------------------------------------------------
+
 # km_core_keyboard_dispose()
 
 ## Description

core/include/keyman/keyman_core_api_bits.h

@@ -23,7 +23,6 @@
   #define _kmn_unused(x)        UNUSED_ ## x __attribute__((__unused__))
 #else
   #define _kmn_unused(x)        UNUSED_ ## x
-
 #endif
 
 #if defined _WIN32 || defined __CYGWIN__
@@ -36,7 +35,7 @@
     #undef _kmn_static_flag
   #else  // How MSVC sepcifies function level attributes adn deprecation
     #define _kmn_and
-    #define _kmn_tag_fn(a)       __declspec(a)
+    #define _kmn_tag_fn(a)     __declspec(a)
     #define _kmn_deprecated_flag deprecated
   #endif
   #define _kmn_export_flag     dllexport
@@ -48,6 +47,8 @@
   #define _KM_CORE_EXT_SEPARATOR ('.')
 #endif
 
+#define KMN_DEPRECATED        _kmn_tag_fn(_kmn_deprecated_flag)
+
 #if defined KM_CORE_LIBRARY_STATIC
   #define KMN_API             _kmn_tag_fn(_kmn_static_flag)
   #define KMN_DEPRECATED_API  _kmn_tag_fn(_kmn_deprecated_flag _kmn_and _kmn_static_flag)

core/src/keyboard.cpp

@@ -17,18 +17,16 @@ void keyboard_attributes::render()
   // Make attributes point to the stored values above.
   id              = _keyboard_id.c_str();
   version_string  = _version_string.c_str();
-  folder_path     = _folder_path.c_str();
   default_options = _default_opts.data();
 }
 
 
 keyboard_attributes::keyboard_attributes(std::u16string const & kbid,
     std::u16string const & version,
-    path_type const & path,
     options_store const &opts)
 : _keyboard_id(kbid),
   _version_string(version),
-  _folder_path(path),
+  _folder_path(""),
   _default_opts(opts)
 {
   // Ensure that the default_options array will be properly terminated.
@@ -40,7 +38,7 @@ keyboard_attributes::keyboard_attributes(std::u16string const & kbid,
 keyboard_attributes::keyboard_attributes(keyboard_attributes &&rhs)
 : _keyboard_id(std::move(rhs._keyboard_id)),
   _version_string(std::move(rhs._version_string)),
-  _folder_path(std::move(rhs._folder_path)),
+  _folder_path(""),
   _default_opts(std::move(rhs._default_opts))
 {
     rhs.id = rhs.version_string = nullptr;
@@ -58,7 +56,6 @@ json & km::core::operator << (json & j, km::core::keyboard_attributes const & kb
 {
   j << json::object
       << "id" << kb.id
-      << "folder" << kb._folder_path
       << "version" << kb.version_string
       << "rules" << json::array << json::close;
 

core/src/keyboard.hpp

@@ -26,14 +26,14 @@ namespace core
   {
     std::u16string      _keyboard_id;
     std::u16string      _version_string;
+    // unused and deprecated
     core::path           _folder_path;
     std::vector<option> _default_opts;
 
     void render();
 
   public:
     using options_store = decltype(_default_opts);
-    using path_type = decltype(_folder_path);
 
     keyboard_attributes()
     : km_core_keyboard_attrs {nullptr, nullptr, nullptr, nullptr} {}
@@ -42,7 +42,6 @@ namespace core
 
     keyboard_attributes(std::u16string const & id,
              std::u16string const & version,
-             path_type const & path,
              options_store const &opts);
 
     keyboard_attributes & operator = (keyboard_attributes const &) = delete;
@@ -52,8 +51,6 @@ namespace core
 
     options_store const &   default_opts_store() const noexcept { return _default_opts; }
     options_store &         default_opts_store() noexcept { return _default_opts; }
-
-    path_type const     &   path() const noexcept { return _folder_path; }
   };
 
   json & operator << (json &, km::core::keyboard_attributes const &);

core/src/km_core_keyboard_api.cpp

@@ -14,47 +14,122 @@
 #include "keyman_core.h"
 
 #include "keyboard.hpp"
-#include "processor.hpp"
 #include "kmx/kmx_processor.hpp"
 #include "ldml/ldml_processor.hpp"
 #include "mock/mock_processor.hpp"
+#include "processor.hpp"
+#include "utfcodec.hpp"
 
 using namespace km::core;
 
 namespace
 {
-  abstract_processor * processor_factory(path const & kb_path) {
-    // Some legacy packages may include upper-case file extensions
-    // TODO-LDML: move file io out of core and into engine
-    if (kb_path.suffix() == ".kmx" || kb_path.suffix() == ".KMX") {
-      std::vector<uint8_t> buf;
-      if(ldml_processor::is_kmxplus_file(kb_path, buf)) {
-        abstract_processor * result = new ldml_processor(kb_path, buf);
-        return result;
-      }
-      return new kmx_processor(kb_path);
-    }
-    else if (kb_path.suffix() == ".mock") {
-      return new mock_processor(kb_path);
+  abstract_processor* processor_factory(std::u16string const& kb_name, std::vector<uint8_t> buf) {
+    if (ldml_processor::is_kmxplus_file(buf)) {
+      abstract_processor* result = new ldml_processor(kb_name, buf);
+      return result;
     }
-    else {
-      return new null_processor();
+    return new kmx_processor(kb_name, buf);
+  }
+
+}  // namespace
+
+km_core_status
+km_core_keyboard_load_from_blob_internal(
+  char const * kb_name,
+  std::vector<uint8_t> buf,
+  km_core_keyboard** keyboard
+) {
+  assert(keyboard);
+  if (!keyboard)
+    return KM_CORE_STATUS_INVALID_ARGUMENT;
+
+  if (buf.size() < 64) { // a KMX file is at least 64 bytes (KMX header)
+    return KM_CORE_STATUS_IO_ERROR;
+  }
+
+  if (*PKMX_DWORD((KMX_BYTE*)buf.data()) != KMX_DWORD(FILEID_COMPILED)) {
+    return KM_CORE_STATUS_IO_ERROR;
+  }
+  try {
+    abstract_processor* kp = processor_factory(convert<char, char16_t>(kb_name), buf);
+    km_core_status status  = kp->validate();
+    if (status != KM_CORE_STATUS_OK) {
+      delete kp;
+      return status;
     }
+    *keyboard = static_cast<km_core_keyboard*>(kp);
+  } catch (std::bad_alloc&) {
+    return KM_CORE_STATUS_NO_MEM;
   }
+  return KM_CORE_STATUS_OK;
+}
 
+km_core_status
+km_core_keyboard_load_from_blob(
+  char const * kb_name,
+  void* blob,
+  size_t blob_size,
+  km_core_keyboard** keyboard
+) {
+  assert(keyboard);
+  if (!keyboard || !blob)
+    return KM_CORE_STATUS_INVALID_ARGUMENT;
 
+  std::vector<uint8_t> buf((uint8_t*)blob, (uint8_t*)blob + blob_size);
+  return km_core_keyboard_load_from_blob_internal(kb_name, buf, keyboard);
 }
+
+// BEGIN DEPRECATED: Remove this code when we remove the deprecated km_core_keyboard_load method
+#include <fstream>
+std::vector<uint8_t> load_kmx_file(path const& kb_path) {
+  std::vector<uint8_t> data;
+  std::ifstream file(static_cast<std::string>(kb_path), std::ios::binary | std::ios::ate);
+  if (!file.good()) {
+    return std::vector<uint8_t>();
+  }
+  const std::streamsize size = file.tellg();
+  if (size >= KMX_MAX_ALLOWED_FILE_SIZE) {
+    return std::vector<uint8_t>();
+  }
+
+  file.seekg(0, std::ios::beg);
+
+  data.resize((size_t)size);
+  if (!file.read((char*)data.data(), size)) {
+    return std::vector<uint8_t>();
+  }
+
+  file.close();
+  return data;
+}
+
+KMN_DEPRECATED_API
 km_core_status
-km_core_keyboard_load(km_core_path_name kb_path, km_core_keyboard **keyboard)
+km_core_keyboard_load(km_core_path_name kb, km_core_keyboard **keyboard)
 {
   assert(keyboard);
-  if (!keyboard)
+  if (!keyboard || !kb)
     return KM_CORE_STATUS_INVALID_ARGUMENT;
 
+  path const kb_path(kb);
   try
   {
-    abstract_processor *kp = processor_factory(kb_path);
-    km_core_status status = kp->validate();
+    abstract_processor* kp = nullptr;
+    km_core_status status = KM_CORE_STATUS_OK;
+    // Some legacy packages may include upper-case file extensions
+    if (kb_path.suffix() == ".kmx" || kb_path.suffix() == ".KMX") {
+      std::vector<uint8_t> buf = load_kmx_file(kb_path);
+      status = km_core_keyboard_load_from_blob_internal(kb_path.stem().c_str(), buf, (km_core_keyboard**)&kp);
+      if (status != KM_CORE_STATUS_OK) {
+        return status;
+      }
+    } else if (kb_path.suffix() == ".mock") {
+      kp = new mock_processor(kb_path);
+    } else {
+      kp = new null_processor();
+    }
+    status = kp->validate();
     if (status != KM_CORE_STATUS_OK) {
       delete kp;
       return status;
@@ -67,6 +142,7 @@ km_core_keyboard_load(km_core_path_name kb_path, km_core_keyboard **keyboard)
   }
   return KM_CORE_STATUS_OK;
 }
+// END DEPRECATED
 
 void
 km_core_keyboard_dispose(km_core_keyboard *keyboard)