From 094373f7d1f711793c13b30daa6d9d208f8e6b03 Mon Sep 17 00:00:00 2001 From: benStre Date: Wed, 7 Feb 2024 16:33:53 +0100 Subject: [PATCH 1/3] move storage modules to storage directory --- datex_all.ts | 3 ++- init.ts | 12 ++++++------ network/supranet.ts | 2 +- runtime/crypto.ts | 2 +- runtime/pointers.ts | 2 +- runtime/runtime.ts | 2 +- {runtime => storage}/storage-locations/deno-kv.ts | 4 ++-- {runtime => storage}/storage-locations/indexed-db.ts | 2 +- .../storage-locations/local-storage-compat.ts | 2 +- .../storage-locations/local-storage.ts | 4 ++-- {runtime => storage}/storage-locations/sql-db.ts | 2 +- .../storage-locations/sql-definitions.ts | 0 .../storage-locations/sql-queries.ts | 0 .../storage-locations/sql-type-map.ts | 0 {runtime => storage}/storage.ts | 8 ++++---- types/storage-map.ts | 2 +- types/storage-set.ts | 2 +- utils/eternals.ts | 2 +- utils/utils.ts | 2 +- 19 files changed, 27 insertions(+), 26 deletions(-) rename {runtime => storage}/storage-locations/deno-kv.ts (97%) rename {runtime => storage}/storage-locations/indexed-db.ts (98%) rename {runtime => storage}/storage-locations/local-storage-compat.ts (98%) rename {runtime => storage}/storage-locations/local-storage.ts (96%) rename {runtime => storage}/storage-locations/sql-db.ts (99%) rename {runtime => storage}/storage-locations/sql-definitions.ts (100%) rename {runtime => storage}/storage-locations/sql-queries.ts (100%) rename {runtime => storage}/storage-locations/sql-type-map.ts (100%) rename {runtime => storage}/storage.ts (99%) diff --git a/datex_all.ts b/datex_all.ts index 3e42188f..fc4428d0 100644 --- a/datex_all.ts +++ b/datex_all.ts @@ -35,10 +35,11 @@ export * from "./runtime/io_handler.ts"; export * from "./runtime/js_interface.ts"; export * from "./runtime/performance_measure.ts"; export * from "./runtime/pointers.ts"; -export * from "./runtime/storage.ts"; export * from "./runtime/cli.ts"; export * from "./runtime/cache_path.ts"; +// storage +export * from "./storage/storage.ts"; // types export * from "./types/abstract_types.ts"; diff --git a/init.ts b/init.ts index 40918bd2..113bbbce 100644 --- a/init.ts +++ b/init.ts @@ -2,11 +2,11 @@ import { Runtime } from "./runtime/runtime.ts"; import { Pointer } from "./runtime/pointers.ts"; import { LOCAL_ENDPOINT } from "./types/addressing.ts"; import { client_type } from "./utils/constants.ts"; -import { Storage, registerStorageAsPointerSource } from "./runtime/storage.ts"; +import { Storage, registerStorageAsPointerSource } from "./storage/storage.ts"; import { cwdURL, logger } from "./utils/global_values.ts"; -import { IndexedDBStorageLocation } from "./runtime/storage-locations/indexed-db.ts"; -import { LocalStorageLocation } from "./runtime/storage-locations/local-storage.ts"; -import { DenoKVStorageLocation } from "./runtime/storage-locations/deno-kv.ts"; +import { IndexedDBStorageLocation } from "./storage/storage-locations/indexed-db.ts"; +import { LocalStorageLocation } from "./storage/storage-locations/local-storage.ts"; +import { DenoKVStorageLocation } from "./storage/storage-locations/deno-kv.ts"; import { loadEternalValues } from "./utils/eternals.ts"; import { DX_BOUND_LOCAL_SLOT } from "./runtime/constants.ts"; import { verboseArg } from "./utils/logger.ts"; @@ -45,9 +45,9 @@ export async function init() { } if (await storageInitModule?.fsExists()) { - logger.info("Initializing custom storage configuration (" + storageInitModule!.normal_pathname + ")") + logger.info("Initializing custom storage configuration (" + storageInitModule!.toString() + ")") try { - await import(storageInitModule!.normal_pathname); + await import(storageInitModule!.toString()); } catch (e) { console.error(e) diff --git a/network/supranet.ts b/network/supranet.ts index 01eae861..5bfc9f36 100644 --- a/network/supranet.ts +++ b/network/supranet.ts @@ -18,7 +18,7 @@ import { Logger } from "../utils/logger.ts"; import { endpoint_config } from "../runtime/endpoint_config.ts"; import { endpoint_name, UnresolvedEndpointProperty } from "../datex_all.ts"; import { Datex } from "../mod.ts"; -import { Storage } from "../runtime/storage.ts"; +import { Storage } from "../storage/storage.ts"; import { WebSocketClientInterface } from "./communication-interfaces/websocket-client-interface.ts"; import { communicationHub } from "./communication-hub.ts"; diff --git a/runtime/crypto.ts b/runtime/crypto.ts index 31415756..6dc4e333 100644 --- a/runtime/crypto.ts +++ b/runtime/crypto.ts @@ -3,7 +3,7 @@ import { logger } from "../utils/global_values.ts"; import { Endpoint, Target } from "../types/addressing.ts"; import { SecurityError, ValueError } from "../types/errors.ts"; import { NetworkUtils } from "../network/network_utils.ts"; -import { Storage } from "../runtime/storage.ts"; +import { Storage } from "../storage/storage.ts"; import { Runtime } from "./runtime.ts"; import { displayFatalError } from "./display.ts"; import { Supranet } from "../network/supranet.ts"; diff --git a/runtime/pointers.ts b/runtime/pointers.ts index 18ea266a..619e0d33 100644 --- a/runtime/pointers.ts +++ b/runtime/pointers.ts @@ -30,7 +30,7 @@ import { LazyPointer } from "./lazy-pointer.ts"; import { ReactiveArrayMethods } from "../types/reactive-methods/array.ts"; import { Assertion } from "../types/assertion.ts"; import { StorageSet } from "../types/storage-set.ts"; -import { Storage } from "./storage.ts"; +import { Storage } from "../storage/storage.ts"; export type observe_handler = (value:V extends RefLike ? T : V, key?:K, type?:Ref.UPDATE_TYPE, transform?:boolean, is_child_update?:boolean, previous?: any, atomic_id?:symbol)=>void|boolean export type observe_options = {types?:Ref.UPDATE_TYPE[], ignore_transforms?:boolean, recursive?:boolean} diff --git a/runtime/runtime.ts b/runtime/runtime.ts index f6d7adef..8b13e410 100644 --- a/runtime/runtime.ts +++ b/runtime/runtime.ts @@ -32,7 +32,7 @@ import { BROADCAST, Endpoint, endpoints, IdEndpoint, LOCAL_ENDPOINT, Target, tar import { RuntimePerformance } from "./performance_measure.ts"; import { NetworkError, PermissionError, PointerError, RuntimeError, SecurityError, ValueError, Error as DatexError, CompilerError, TypeError, SyntaxError, AssertionError } from "../types/errors.ts"; import { Function as DatexFunction } from "../types/function.ts"; -import { Storage } from "../runtime/storage.ts"; +import { Storage } from "../storage/storage.ts"; import { Observers } from "../utils/observers.ts"; import { BinaryCode } from "../compiler/binary_codes.ts"; import type { ExecConditions, trace, compile_info, datex_meta, datex_scope, dxb_header, routing_info } from "../utils/global_types.ts"; diff --git a/runtime/storage-locations/deno-kv.ts b/storage/storage-locations/deno-kv.ts similarity index 97% rename from runtime/storage-locations/deno-kv.ts rename to storage/storage-locations/deno-kv.ts index 521f9db9..615d0eee 100644 --- a/runtime/storage-locations/deno-kv.ts +++ b/storage/storage-locations/deno-kv.ts @@ -2,9 +2,9 @@ import { Runtime } from "../../runtime/runtime.ts"; import { Compiler } from "../../compiler/compiler.ts"; import { Pointer } from "../../runtime/pointers.ts"; -import { NOT_EXISTING } from "../constants.ts"; +import { NOT_EXISTING } from "../../runtime/constants.ts"; import { AsyncStorageLocation } from "../storage.ts"; -import { ptr_cache_path } from "../cache_path.ts"; +import { ptr_cache_path } from "../../runtime/cache_path.ts"; import { client_type } from "../../utils/constants.ts"; import { normalizePath } from "../../utils/normalize-path.ts"; import { ExecConditions } from "../../utils/global_types.ts"; diff --git a/runtime/storage-locations/indexed-db.ts b/storage/storage-locations/indexed-db.ts similarity index 98% rename from runtime/storage-locations/indexed-db.ts rename to storage/storage-locations/indexed-db.ts index deb33d6d..e4cbe1c3 100644 --- a/runtime/storage-locations/indexed-db.ts +++ b/storage/storage-locations/indexed-db.ts @@ -2,7 +2,7 @@ import { Runtime } from "../../runtime/runtime.ts"; import { Compiler } from "../../compiler/compiler.ts"; import { Pointer } from "../../runtime/pointers.ts"; -import { NOT_EXISTING } from "../constants.ts"; +import { NOT_EXISTING } from "../../runtime/constants.ts"; import { AsyncStorageLocation, site_suffix } from "../storage.ts"; import localforage from "../../lib/localforage/localforage.js"; diff --git a/runtime/storage-locations/local-storage-compat.ts b/storage/storage-locations/local-storage-compat.ts similarity index 98% rename from runtime/storage-locations/local-storage-compat.ts rename to storage/storage-locations/local-storage-compat.ts index cfe10122..2da7b05f 100644 --- a/runtime/storage-locations/local-storage-compat.ts +++ b/storage/storage-locations/local-storage-compat.ts @@ -2,7 +2,7 @@ import { logger } from "../../utils/global_values.ts"; import { client_type } from "../../utils/constants.ts"; -import { ptr_cache_path } from "../cache_path.ts"; +import { ptr_cache_path } from "../../runtime/cache_path.ts"; import { normalizePath } from "../../utils/normalize-path.ts"; class LocalStorage implements Storage { diff --git a/runtime/storage-locations/local-storage.ts b/storage/storage-locations/local-storage.ts similarity index 96% rename from runtime/storage-locations/local-storage.ts rename to storage/storage-locations/local-storage.ts index e16b6738..551104ca 100644 --- a/runtime/storage-locations/local-storage.ts +++ b/storage/storage-locations/local-storage.ts @@ -1,8 +1,8 @@ import { Compiler } from "../../compiler/compiler.ts"; import { Storage, SyncStorageLocation } from "../storage.ts"; -import { Pointer } from "../pointers.ts"; +import { Pointer } from "../../runtime/pointers.ts"; import { Runtime } from "../../runtime/runtime.ts"; -import { NOT_EXISTING } from "../constants.ts"; +import { NOT_EXISTING } from "../../runtime/constants.ts"; import { base64ToArrayBuffer } from "../../utils/utils.ts"; import { arrayBufferToBase64 } from "../../datex_all.ts"; import { localStorage } from "./local-storage-compat.ts"; diff --git a/runtime/storage-locations/sql-db.ts b/storage/storage-locations/sql-db.ts similarity index 99% rename from runtime/storage-locations/sql-db.ts rename to storage/storage-locations/sql-db.ts index 6222b9f4..9be442fc 100644 --- a/runtime/storage-locations/sql-db.ts +++ b/storage/storage-locations/sql-db.ts @@ -11,7 +11,7 @@ import { NOT_EXISTING } from "../../runtime/constants.ts"; import { client_type } from "../../utils/constants.ts"; import { Compiler } from "../../compiler/compiler.ts"; import { ExecConditions } from "../../utils/global_types.ts"; -import { Runtime } from "../runtime.ts"; +import { Runtime } from "../../runtime/runtime.ts"; const logger = new Logger("SQL Storage"); diff --git a/runtime/storage-locations/sql-definitions.ts b/storage/storage-locations/sql-definitions.ts similarity index 100% rename from runtime/storage-locations/sql-definitions.ts rename to storage/storage-locations/sql-definitions.ts diff --git a/runtime/storage-locations/sql-queries.ts b/storage/storage-locations/sql-queries.ts similarity index 100% rename from runtime/storage-locations/sql-queries.ts rename to storage/storage-locations/sql-queries.ts diff --git a/runtime/storage-locations/sql-type-map.ts b/storage/storage-locations/sql-type-map.ts similarity index 100% rename from runtime/storage-locations/sql-type-map.ts rename to storage/storage-locations/sql-type-map.ts diff --git a/runtime/storage.ts b/storage/storage.ts similarity index 99% rename from runtime/storage.ts rename to storage/storage.ts index b5315ccb..f7b85207 100644 --- a/runtime/storage.ts +++ b/storage/storage.ts @@ -4,11 +4,11 @@ import { Runtime } from "../runtime/runtime.ts"; import type { ExecConditions, PointerSource } from "../utils/global_types.ts"; import { logger } from "../utils/global_values.ts"; import { client_type } from "../utils/constants.ts"; -import { NOT_EXISTING } from "./constants.ts"; -import { Pointer, type MinimalJSRef, Ref } from "./pointers.ts"; +import { NOT_EXISTING } from "../runtime/constants.ts"; +import { Pointer, type MinimalJSRef, Ref } from "../runtime/pointers.ts"; import { localStorage } from "./storage-locations/local-storage-compat.ts"; import { MessageLogger } from "../utils/message_logger.ts"; -import { displayFatalError } from "./display.ts" +import { displayFatalError } from "../runtime/display.ts" import { Type } from "../types/type.ts"; import { addPersistentListener } from "../utils/persistent-listeners.ts"; import { Endpoint, LOCAL_ENDPOINT } from "../types/addressing.ts"; @@ -16,7 +16,7 @@ import { ESCAPE_SEQUENCES } from "../utils/logger.ts"; import { StorageMap } from "../types/storage-map.ts"; import { StorageSet } from "../types/storage-set.ts"; import { IterableWeakSet } from "../utils/iterable-weak-set.ts"; -import { LazyPointer } from "./lazy-pointer.ts"; +import { LazyPointer } from "../runtime/lazy-pointer.ts"; // displayInit(); diff --git a/types/storage-map.ts b/types/storage-map.ts index 65bd9b22..dfd3ebfb 100644 --- a/types/storage-map.ts +++ b/types/storage-map.ts @@ -3,7 +3,7 @@ import { Compiler } from "../compiler/compiler.ts"; import { DX_PTR } from "../runtime/constants.ts"; import { Pointer } from "../runtime/pointers.ts"; -import { Storage } from "../runtime/storage.ts"; +import { Storage } from "../storage/storage.ts"; import { Logger } from "../utils/logger.ts"; const logger = new Logger("StorageMap"); diff --git a/types/storage-set.ts b/types/storage-set.ts index 95635256..1e4b0163 100644 --- a/types/storage-set.ts +++ b/types/storage-set.ts @@ -3,7 +3,7 @@ import { Compiler } from "../compiler/compiler.ts"; import { DX_PTR } from "../runtime/constants.ts"; import { Pointer } from "../runtime/pointers.ts"; -import { Storage } from "../runtime/storage.ts"; +import { Storage } from "../storage/storage.ts"; import { Logger } from "../utils/logger.ts"; const logger = new Logger("StorageSet"); diff --git a/utils/eternals.ts b/utils/eternals.ts index 60c7c849..ad9ebd79 100644 --- a/utils/eternals.ts +++ b/utils/eternals.ts @@ -1,5 +1,5 @@ import { NOT_EXISTING } from "../runtime/constants.ts"; -import { Storage } from "../runtime/storage.ts"; +import { Storage } from "../storage/storage.ts"; import { getCallerInfo } from "./caller_metadata.ts"; import { logger } from "./global_values.ts"; diff --git a/utils/utils.ts b/utils/utils.ts index d568edab..abd7ba78 100644 --- a/utils/utils.ts +++ b/utils/utils.ts @@ -1,6 +1,6 @@ -//import { Storage } from "../runtime/storage.ts"; TODO Storage cannot be importet here, handle file caching somehow (somewhere else) +//import { Storage } from "../storage/storage.ts"; TODO Storage cannot be importet here, handle file caching somehow (somewhere else) import { ValueError } from "../types/errors.ts"; import { baseURL, Deno } from "./global_values.ts"; import { client_type } from "./constants.ts"; From 2eaaba4ee2c1d8987b2405336831380271ac7974 Mon Sep 17 00:00:00 2001 From: benStre Date: Thu, 8 Feb 2024 11:25:43 +0100 Subject: [PATCH 2/3] improve sql db --- storage/storage-locations/sql-db.ts | 97 +++++++++++++++++++++-------- 1 file changed, 72 insertions(+), 25 deletions(-) diff --git a/storage/storage-locations/sql-db.ts b/storage/storage-locations/sql-db.ts index 9be442fc..33b04242 100644 --- a/storage/storage-locations/sql-db.ts +++ b/storage/storage-locations/sql-db.ts @@ -34,16 +34,23 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { ] }, pointerMapping: { - name: "datex_pointers", + name: "datex_pointer_mapping", columns: [ [this.#pointerMysqlColumnName, this.#pointerMysqlType, "PRIMARY KEY"], ["table_name", "varchar(50)"] ] }, + rawPointers: { + name: "datex_pointers_raw", + columns: [ + ["id", "varchar(50)", "PRIMARY KEY"], + ["value", "blob"] + ] + }, items: { name: "datex_items", columns: [ - ["key", "varchar(50)", "PRIMARY KEY"], + ["key", "varchar(200)", "PRIMARY KEY"], ["value", "blob"] ] } @@ -85,7 +92,23 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { } async #query(query_string:string, query_params?:any[]): Promise { - if (typeof query_string != "string") {console.error("invalid query:", query_string); throw new Error("invalid query")} + + // handle arraybuffers + if (query_params) { + for (let i = 0; i < query_params.length; i++) { + const param = query_params[i]; + if (param instanceof ArrayBuffer) { + query_params[i] = new TextDecoder().decode(param) + } + if (param instanceof Array) { + query_params[i] = param.map(p => p instanceof ArrayBuffer ? new TextDecoder().decode(p) : p) + } + } + } + + // console.log("QUERY: " + query_string, query_params) + + if (typeof query_string != "string") {console.error("invalid query:", query_string); throw new Error("invalid query")} if (!query_string) throw new Error("empty query"); try { const result = await this.#sqlClient!.execute(query_string, query_params); @@ -143,10 +166,13 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { async #createTableForType(type: Datex.Type) { const columns:ColumnDefinition[] = [ - [this.#pointerMysqlColumnName, this.#pointerMysqlType, "PRIMARY KEY INVISIBLE"] + [this.#pointerMysqlColumnName, this.#pointerMysqlType, 'PRIMARY KEY INVISIBLE DEFAULT "0"'] ] const constraints: ConstraintsDefinition[] = [] + this.log?.("Creating table for type " + type) + console.log(type) + for (const [propName, propType] of Object.entries(type.template as {[key:string]:Datex.Type})) { let mysqlType: mysql_data_type|undefined @@ -161,14 +187,16 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { columns.push([propName, mysqlType!]) } // no matching primitive type found - else if (!mysqlType && propType.template) { - const foreignTable = await this.#getTableForType(propType); + else if (!mysqlType) { + let foreignTable = propType.template ? await this.#getTableForType(propType) : null; - if (!foreignTable) throw new Error("Cannot map type " + propType + " to a SQL table") - else { - columns.push([propName, this.#pointerMysqlType]) - constraints.push(`FOREIGN KEY (\`${propName}\`) REFERENCES \`${foreignTable}\`(\`${this.#pointerMysqlColumnName}\`)`) + if (!foreignTable) { + logger.warn("Cannot map type " + propType + " to a SQL table, falling back to raw DXB storage") + foreignTable = this.#metaTables.rawPointers.name; } + + columns.push([propName, this.#pointerMysqlType]) + constraints.push(`FOREIGN KEY (\`${propName}\`) REFERENCES \`${foreignTable}\`(\`${this.#pointerMysqlColumnName}\`)`) } else { @@ -252,8 +280,8 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { } // this.log("cols", insertData) - await this.#query('INSERT INTO ?? ?? VALUES ?;', [table, Object.keys(insertData), Object.values(insertData)]) + await this.#query('INSERT INTO ?? ?? VALUES ?;', [table, Object.keys(insertData), Object.values(insertData)]) } async #updatePointer(pointer: Datex.Pointer, keys:string[]) { @@ -288,7 +316,8 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { await this.#init(); const dependencies = new Set() const encoded = Compiler.encodeValue(value, dependencies); - await this.#query('INSERT INTO ?? ?? VALUES ?;', [this.#metaTables.items.name, ["key", "value"], [key, encoded]]) + console.log("db set item", key) + await this.#query('INSERT INTO ?? ?? VALUES ? ON DUPLICATE KEY UPDATE value=?;', [this.#metaTables.items.name, ["key", "value"], [key, encoded], encoded]) // await datex_item_storage.setItem(key, Compiler.encodeValue(value, dependencies)); return dependencies; } @@ -331,24 +360,42 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { const dependencies = new Set() await this.#init(); - this.log?.("update " + pointer.id + " - " + pointer.type, partialUpdateKey, await this.#pointerEntryExists(pointer)) - // new full insert - if (!await this.#pointerEntryExists(pointer)) - await this.#insertPointer(pointer) - else { - // partial update - if (partialUpdateKey !== NOT_EXISTING) { - if (typeof partialUpdateKey !== "string") throw new Error("invalid key type for SQL table: " + Datex.Type.ofValue(partialUpdateKey)) - await this.#updatePointer(pointer, [partialUpdateKey]) - } - // full udpdate + // is templatable pointer type + if (pointer.type.template) { + this.log?.("update " + pointer.id + " - " + pointer.type, partialUpdateKey, await this.#pointerEntryExists(pointer)) + + // new full insert + if (!await this.#pointerEntryExists(pointer)) + await this.#insertPointer(pointer) else { - // TODO + // partial update + if (partialUpdateKey !== NOT_EXISTING) { + if (typeof partialUpdateKey !== "string") throw new Error("invalid key type for SQL table: " + Datex.Type.ofValue(partialUpdateKey)) + await this.#updatePointer(pointer, [partialUpdateKey]) + } + // full udpdate + else { + // TODO + } } } + + // no template, just add a raw DXB entry + else { + await this.setPointerRaw(pointer) + } - + return dependencies; + } + + async setPointerRaw(pointer: Pointer) { + console.log("storing raw pointer: " + Runtime.valueToDatexStringExperimental(pointer, true, true)) + await this.#init(); + const dependencies = new Set() + const encoded = Compiler.encodeValue(pointer, dependencies, true, false, true); + await this.#query('INSERT INTO ?? ?? VALUES ? ON DUPLICATE KEY UPDATE value=?;', [this.#metaTables.rawPointers.name, ["id", "value"], [pointer.id, encoded], encoded]) + // await datex_item_storage.setItem(key, Compiler.encodeValue(value, dependencies)); return dependencies; } From a77e2f7310a79e611f17aa8af4b7b8a015a609cf Mon Sep 17 00:00:00 2001 From: benStre Date: Fri, 9 Feb 2024 18:40:51 +0100 Subject: [PATCH 3/3] remove uix path dependency --- threads/threads.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/threads/threads.ts b/threads/threads.ts index e2d5b9ba..e512b4e7 100644 --- a/threads/threads.ts +++ b/threads/threads.ts @@ -6,7 +6,7 @@ const logger = new Logger("thread-runner"); import { Datex, f } from "../mod.ts"; import { blobifyFile, blobifyScript } from "../utils/blobify.ts"; import { RuntimeError } from "../types/errors.ts"; -import { Path } from "https://dev.cdn.unyt.org/uix/utils/path.ts"; +import { Path } from "../utils/path.ts"; import { getCallerDir } from "../utils/caller_metadata.ts"; import { PromiseMapReturnType, PromiseMappingFn } from "./promise-fn-types.ts"; import { JSTransferableFunction } from "../types/js-function.ts";