Merge pull request #78 from unyt-org/storage-improvements

Storage improvements

datex_all.ts

@@ -35,10 +35,11 @@ export * from "./runtime/io_handler.ts";
 export * from "./runtime/js_interface.ts";
 export * from "./runtime/performance_measure.ts";
 export * from "./runtime/pointers.ts";
-export * from "./runtime/storage.ts";
 export * from "./runtime/cli.ts";
 export * from "./runtime/cache_path.ts";
 
+// storage
+export * from "./storage/storage.ts";
 
 // types
 export * from "./types/abstract_types.ts";

init.ts

@@ -2,11 +2,11 @@ import { Runtime } from "./runtime/runtime.ts";
 import { Pointer } from "./runtime/pointers.ts";
 import { LOCAL_ENDPOINT } from "./types/addressing.ts";
 import { client_type } from "./utils/constants.ts";
-import { Storage, registerStorageAsPointerSource } from "./runtime/storage.ts";
+import { Storage, registerStorageAsPointerSource } from "./storage/storage.ts";
 import { cwdURL, logger } from "./utils/global_values.ts";
-import { IndexedDBStorageLocation } from "./runtime/storage-locations/indexed-db.ts";
-import { LocalStorageLocation } from "./runtime/storage-locations/local-storage.ts";
-import { DenoKVStorageLocation } from "./runtime/storage-locations/deno-kv.ts";
+import { IndexedDBStorageLocation } from "./storage/storage-locations/indexed-db.ts";
+import { LocalStorageLocation } from "./storage/storage-locations/local-storage.ts";
+import { DenoKVStorageLocation } from "./storage/storage-locations/deno-kv.ts";
 import { loadEternalValues } from "./utils/eternals.ts";
 import { DX_BOUND_LOCAL_SLOT } from "./runtime/constants.ts";
 import { verboseArg } from "./utils/logger.ts";
@@ -46,9 +46,9 @@ export async function init() {
 		}
 
 		if (await storageInitModule?.fsExists()) {
-			logger.info("Initializing custom storage configuration (" + storageInitModule!.normal_pathname + ")")
+			logger.info("Initializing custom storage configuration (" + storageInitModule!.toString() + ")")
 			try {
-				await import(storageInitModule!.normal_pathname);
+				await import(storageInitModule!.toString());
 			}
 			catch (e) {
 				console.error(e)

network/supranet.ts

@@ -18,7 +18,7 @@ import { Logger } from "../utils/logger.ts";
 import { endpoint_config } from "../runtime/endpoint_config.ts";
 import { endpoint_name, UnresolvedEndpointProperty } from "../datex_all.ts";
 import { Datex } from "../mod.ts";
-import { Storage } from "../runtime/storage.ts";
+import { Storage } from "../storage/storage.ts";
 import { WebSocketClientInterface } from "./communication-interfaces/websocket-client-interface.ts";
 import { communicationHub } from "./communication-hub.ts";
 

runtime/crypto.ts

@@ -3,7 +3,7 @@ import { logger } from "../utils/global_values.ts";
 import { Endpoint, Target, WildcardTarget } from "../types/addressing.ts";
 import { SecurityError, ValueError } from "../types/errors.ts";
 import { NetworkUtils } from "../network/network_utils.ts";
-import { Storage } from "../runtime/storage.ts";
+import { Storage } from "../storage/storage.ts";
 import { Runtime } from "./runtime.ts";
 import { displayFatalError } from "./display.ts";
 import { Supranet } from "../network/supranet.ts";

runtime/pointers.ts

@@ -30,7 +30,7 @@ import { LazyPointer } from "./lazy-pointer.ts";
 import { ReactiveArrayMethods } from "../types/reactive-methods/array.ts";
 import { Assertion } from "../types/assertion.ts";
 import { StorageSet } from "../types/storage-set.ts";
-import { Storage } from "./storage.ts";
+import { Storage } from "../storage/storage.ts";
 
 export type observe_handler<K=any, V extends RefLike = any> = (value:V extends RefLike<infer T> ? T : V, key?:K, type?:Ref.UPDATE_TYPE, transform?:boolean, is_child_update?:boolean, previous?: any, atomic_id?:symbol)=>void|boolean
 export type observe_options = {types?:Ref.UPDATE_TYPE[], ignore_transforms?:boolean, recursive?:boolean}

runtime/runtime.ts

@@ -32,7 +32,7 @@ import { BROADCAST, Endpoint, endpoints, IdEndpoint, LOCAL_ENDPOINT, Target, tar
 import { RuntimePerformance } from "./performance_measure.ts";
 import { NetworkError, PermissionError, PointerError, RuntimeError, SecurityError, ValueError, Error as DatexError, CompilerError, TypeError, SyntaxError, AssertionError } from "../types/errors.ts";
 import { Function as DatexFunction } from "../types/function.ts";
-import { Storage } from "../runtime/storage.ts";
+import { Storage } from "../storage/storage.ts";
 import { Observers } from "../utils/observers.ts";
 import { BinaryCode } from "../compiler/binary_codes.ts";
 import type { ExecConditions, trace, compile_info, datex_meta, datex_scope, dxb_header, routing_info } from "../utils/global_types.ts";

runtime/storage-locations/deno-kv.ts → storage/storage-locations/deno-kv.ts

@@ -2,9 +2,9 @@ import { Runtime } from "../../runtime/runtime.ts";
 import { Compiler } from "../../compiler/compiler.ts";
 import { Pointer } from "../../runtime/pointers.ts";
 
-import { NOT_EXISTING } from "../constants.ts";
+import { NOT_EXISTING } from "../../runtime/constants.ts";
 import { AsyncStorageLocation } from "../storage.ts";
-import { ptr_cache_path } from "../cache_path.ts";
+import { ptr_cache_path } from "../../runtime/cache_path.ts";
 import { client_type } from "../../utils/constants.ts";
 import { normalizePath } from "../../utils/normalize-path.ts";
 import { ExecConditions } from "../../utils/global_types.ts";

runtime/storage-locations/indexed-db.ts → storage/storage-locations/indexed-db.ts

@@ -2,7 +2,7 @@ import { Runtime } from "../../runtime/runtime.ts";
 import { Compiler } from "../../compiler/compiler.ts";
 import { Pointer } from "../../runtime/pointers.ts";
 
-import { NOT_EXISTING } from "../constants.ts";
+import { NOT_EXISTING } from "../../runtime/constants.ts";
 import { AsyncStorageLocation, site_suffix } from "../storage.ts";
 
 import localforage from "../../lib/localforage/localforage.js";

runtime/storage-locations/local-storage-compat.ts → storage/storage-locations/local-storage-compat.ts

@@ -2,7 +2,7 @@
 
 import { logger } from "../../utils/global_values.ts";
 import { client_type } from "../../utils/constants.ts";
-import { ptr_cache_path } from "../cache_path.ts";
+import { ptr_cache_path } from "../../runtime/cache_path.ts";
 import { normalizePath } from "../../utils/normalize-path.ts";
 
 class LocalStorage implements Storage {

runtime/storage-locations/local-storage.ts → storage/storage-locations/local-storage.ts

@@ -1,8 +1,8 @@
 import { Compiler } from "../../compiler/compiler.ts";
 import { Storage, SyncStorageLocation } from "../storage.ts";
-import { Pointer } from "../pointers.ts";
+import { Pointer } from "../../runtime/pointers.ts";
 import { Runtime } from "../../runtime/runtime.ts";
-import { NOT_EXISTING } from "../constants.ts";
+import { NOT_EXISTING } from "../../runtime/constants.ts";
 import { base64ToArrayBuffer } from "../../utils/utils.ts";
 import { arrayBufferToBase64 } from "../../datex_all.ts";
 import { localStorage } from "./local-storage-compat.ts";

runtime/storage-locations/sql-db.ts → storage/storage-locations/sql-db.ts

@@ -11,7 +11,7 @@ import { NOT_EXISTING } from "../../runtime/constants.ts";
 import { client_type } from "../../utils/constants.ts";
 import { Compiler } from "../../compiler/compiler.ts";
 import { ExecConditions } from "../../utils/global_types.ts";
-import { Runtime } from "../runtime.ts";
+import { Runtime } from "../../runtime/runtime.ts";
 
 const logger = new Logger("SQL Storage");
 
@@ -34,16 +34,23 @@ export class SQLDBStorageLocation extends AsyncStorageLocation {
 			]
 		},
 		pointerMapping: {
-			name: "datex_pointers",
+			name: "datex_pointer_mapping",
 			columns: [
 				[this.#pointerMysqlColumnName, this.#pointerMysqlType, "PRIMARY KEY"],
 				["table_name", "varchar(50)"]
 			]
 		},
+		rawPointers: {
+			name: "datex_pointers_raw",
+			columns: [
+				["id", "varchar(50)", "PRIMARY KEY"],
+				["value", "blob"]
+			]
+		},
 		items: {
 			name: "datex_items",
 			columns: [
-				["key", "varchar(50)", "PRIMARY KEY"],
+				["key", "varchar(200)", "PRIMARY KEY"],
 				["value", "blob"]
 			]
 		}
@@ -85,7 +92,23 @@ export class SQLDBStorageLocation extends AsyncStorageLocation {
 	}
 
 	async #query<row=object>(query_string:string, query_params?:any[]): Promise<row[]> {
-        if (typeof query_string != "string") {console.error("invalid query:", query_string); throw new Error("invalid query")}
+
+		// handle arraybuffers
+		if (query_params) {
+			for (let i = 0; i < query_params.length; i++) {
+				const param = query_params[i];
+				if (param instanceof ArrayBuffer) {
+					query_params[i] = new TextDecoder().decode(param)
+				}
+				if (param instanceof Array) {
+					query_params[i] = param.map(p => p instanceof ArrayBuffer ? new TextDecoder().decode(p) : p)
+				}
+			}
+		}
+
+        // console.log("QUERY: " + query_string, query_params)
+
+		if (typeof query_string != "string") {console.error("invalid query:", query_string); throw new Error("invalid query")}
         if (!query_string) throw new Error("empty query");
         try {
             const result = await this.#sqlClient!.execute(query_string, query_params);
@@ -143,10 +166,13 @@ export class SQLDBStorageLocation extends AsyncStorageLocation {
 
 	async #createTableForType(type: Datex.Type) {
 		const columns:ColumnDefinition[] = [
-			[this.#pointerMysqlColumnName, this.#pointerMysqlType, "PRIMARY KEY INVISIBLE"]
+			[this.#pointerMysqlColumnName, this.#pointerMysqlType, 'PRIMARY KEY INVISIBLE DEFAULT "0"']
 		]
 		const constraints: ConstraintsDefinition[] = []
 
+		this.log?.("Creating table for type " + type)
+		console.log(type)
+
 		for (const [propName, propType] of Object.entries(type.template as {[key:string]:Datex.Type})) {
 			let mysqlType: mysql_data_type|undefined
 
@@ -161,14 +187,16 @@ export class SQLDBStorageLocation extends AsyncStorageLocation {
 				columns.push([propName, mysqlType!])
 			}
 			// no matching primitive type found
-			else if (!mysqlType && propType.template) {
-				const foreignTable = await this.#getTableForType(propType);
+			else if (!mysqlType) {
+				let foreignTable = propType.template ? await this.#getTableForType(propType) : null;
 
-				if (!foreignTable) throw new Error("Cannot map type " + propType + " to a SQL table")
-				else {
-					columns.push([propName, this.#pointerMysqlType])
-					constraints.push(`FOREIGN KEY (\`${propName}\`) REFERENCES \`${foreignTable}\`(\`${this.#pointerMysqlColumnName}\`)`)
+				if (!foreignTable) {
+					logger.warn("Cannot map type " + propType + " to a SQL table, falling back to raw DXB storage")
+					foreignTable = this.#metaTables.rawPointers.name;
 				}
+
+				columns.push([propName, this.#pointerMysqlType])
+				constraints.push(`FOREIGN KEY (\`${propName}\`) REFERENCES \`${foreignTable}\`(\`${this.#pointerMysqlColumnName}\`)`)
 			}
 
 			else {
@@ -252,8 +280,8 @@ export class SQLDBStorageLocation extends AsyncStorageLocation {
 		}
 		// this.log("cols", insertData)
 
-		await this.#query('INSERT INTO ?? ?? VALUES ?;', [table, Object.keys(insertData), Object.values(insertData)])
 
+		await this.#query('INSERT INTO ?? ?? VALUES ?;', [table, Object.keys(insertData), Object.values(insertData)])
 	}
 
 	async #updatePointer(pointer: Datex.Pointer, keys:string[]) {
@@ -288,7 +316,8 @@ export class SQLDBStorageLocation extends AsyncStorageLocation {
 		await this.#init();
 		const dependencies = new Set<Pointer>()
 		const encoded = Compiler.encodeValue(value, dependencies);
-		await this.#query('INSERT INTO ?? ?? VALUES ?;', [this.#metaTables.items.name, ["key", "value"], [key, encoded]])
+		console.log("db set item", key)
+		await this.#query('INSERT INTO ?? ?? VALUES ? ON DUPLICATE KEY UPDATE value=?;', [this.#metaTables.items.name, ["key", "value"], [key, encoded], encoded])
         // await datex_item_storage.setItem(key, Compiler.encodeValue(value, dependencies));
 		return dependencies;
 	}
@@ -331,24 +360,42 @@ export class SQLDBStorageLocation extends AsyncStorageLocation {
 		const dependencies = new Set<Pointer>()
 
 		await this.#init();
-		this.log?.("update " + pointer.id + " - " + pointer.type, partialUpdateKey, await this.#pointerEntryExists(pointer))
 
-		// new full insert
-		if (!await this.#pointerEntryExists(pointer))
-			await this.#insertPointer(pointer)
-		else {
-			// partial update
-			if (partialUpdateKey !== NOT_EXISTING) {
-				if (typeof partialUpdateKey !== "string") throw new Error("invalid key type for SQL table: " + Datex.Type.ofValue(partialUpdateKey))
-				await this.#updatePointer(pointer, [partialUpdateKey])
-			}
-			// full udpdate
+		// is templatable pointer type
+		if (pointer.type.template) {
+			this.log?.("update " + pointer.id + " - " + pointer.type, partialUpdateKey, await this.#pointerEntryExists(pointer))
+
+			// new full insert
+			if (!await this.#pointerEntryExists(pointer))
+				await this.#insertPointer(pointer)
 			else {
-				// TODO
+				// partial update
+				if (partialUpdateKey !== NOT_EXISTING) {
+					if (typeof partialUpdateKey !== "string") throw new Error("invalid key type for SQL table: " + Datex.Type.ofValue(partialUpdateKey))
+					await this.#updatePointer(pointer, [partialUpdateKey])
+				}
+				// full udpdate
+				else {
+					// TODO
+				}
 			}
 		}
+
+		// no template, just add a raw DXB entry
+		else {
+			await this.setPointerRaw(pointer)
+		}
 
-
+		return dependencies;
+	}
+
+	async setPointerRaw(pointer: Pointer) {
+		console.log("storing raw pointer: " + Runtime.valueToDatexStringExperimental(pointer, true, true))
+		await this.#init();
+		const dependencies = new Set<Pointer>()
+		const encoded = Compiler.encodeValue(pointer, dependencies, true, false, true);
+		await this.#query('INSERT INTO ?? ?? VALUES ? ON DUPLICATE KEY UPDATE value=?;', [this.#metaTables.rawPointers.name, ["id", "value"], [pointer.id, encoded], encoded])
+        // await datex_item_storage.setItem(key, Compiler.encodeValue(value, dependencies));
 		return dependencies;
 	}
 

runtime/storage.ts → storage/storage.ts

@@ -4,19 +4,19 @@ import { Runtime } from "../runtime/runtime.ts";
 import type { ExecConditions, PointerSource } from "../utils/global_types.ts";
 import { logger } from "../utils/global_values.ts";
 import { client_type } from "../utils/constants.ts";
-import { NOT_EXISTING } from "./constants.ts";
-import { Pointer, type MinimalJSRef, Ref } from "./pointers.ts";
+import { NOT_EXISTING } from "../runtime/constants.ts";
+import { Pointer, type MinimalJSRef, Ref } from "../runtime/pointers.ts";
 import { localStorage } from "./storage-locations/local-storage-compat.ts";
 import { MessageLogger } from "../utils/message_logger.ts";
-import { displayFatalError } from "./display.ts"
+import { displayFatalError } from "../runtime/display.ts"
 import { Type } from "../types/type.ts";
 import { addPersistentListener } from "../utils/persistent-listeners.ts";
 import { Endpoint, LOCAL_ENDPOINT } from "../types/addressing.ts";
 import { ESCAPE_SEQUENCES } from "../utils/logger.ts";
 import { StorageMap } from "../types/storage-map.ts";
 import { StorageSet } from "../types/storage-set.ts";
 import { IterableWeakSet } from "../utils/iterable-weak-set.ts";
-import { LazyPointer } from "./lazy-pointer.ts";
+import { LazyPointer } from "../runtime/lazy-pointer.ts";
 
 
 // displayInit();

threads/threads.ts

@@ -6,7 +6,7 @@ const logger = new Logger("thread-runner");
 import { Datex, f } from "../mod.ts";
 import { blobifyFile, blobifyScript } from "../utils/blobify.ts";
 import { RuntimeError } from "../types/errors.ts";
-import { Path } from "https://dev.cdn.unyt.org/uix/utils/path.ts";
+import { Path } from "../utils/path.ts";
 import { getCallerDir } from "../utils/caller_metadata.ts";
 import { PromiseMapReturnType, PromiseMappingFn } from "./promise-fn-types.ts";
 import { JSTransferableFunction } from "../types/js-function.ts";

types/storage-map.ts

@@ -3,7 +3,7 @@
 import { Compiler } from "../compiler/compiler.ts";
 import { DX_PTR } from "../runtime/constants.ts";
 import { Pointer } from "../runtime/pointers.ts";
-import { Storage } from "../runtime/storage.ts";
+import { Storage } from "../storage/storage.ts";
 import { Logger } from "../utils/logger.ts";
 
 const logger = new Logger("StorageMap");

types/storage-set.ts

@@ -3,7 +3,7 @@
 import { Compiler } from "../compiler/compiler.ts";
 import { DX_PTR } from "../runtime/constants.ts";
 import { Pointer } from "../runtime/pointers.ts";
-import { Storage } from "../runtime/storage.ts";
+import { Storage } from "../storage/storage.ts";
 import { Logger } from "../utils/logger.ts";
 
 const logger = new Logger("StorageSet");

utils/eternals.ts

@@ -1,5 +1,5 @@
 import { NOT_EXISTING } from "../runtime/constants.ts";
-import { Storage } from "../runtime/storage.ts";
+import { Storage } from "../storage/storage.ts";
 import { getCallerInfo } from "./caller_metadata.ts";
 import { logger } from "./global_values.ts";
 

utils/utils.ts

@@ -1,6 +1,6 @@
 
 
-//import { Storage } from "../runtime/storage.ts"; TODO Storage cannot be importet here, handle file caching somehow (somewhere else)
+//import { Storage } from "../storage/storage.ts"; TODO Storage cannot be importet here, handle file caching somehow (somewhere else)
 import { ValueError } from "../types/errors.ts";
 import { baseURL, Deno } from "./global_values.ts";
 import { client_type } from "./constants.ts";