From 19a0a2d46a3eb78b11c4d1afa60142f6be993c61 Mon Sep 17 00:00:00 2001 From: benStre Date: Sat, 20 Jan 2024 22:58:50 +0100 Subject: [PATCH 01/16] implement storage memory management --- runtime/storage.ts | 242 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 238 insertions(+), 4 deletions(-) diff --git a/runtime/storage.ts b/runtime/storage.ts index 200d81ab..6ac29ce4 100644 --- a/runtime/storage.ts +++ b/runtime/storage.ts @@ -12,6 +12,9 @@ import { displayFatalError } from "./display.ts" import { Type } from "../types/type.ts"; import { addPersistentListener } from "../utils/persistent-listeners.ts"; import { LOCAL_ENDPOINT } from "../types/addressing.ts"; +import { ESCAPE_SEQUENCES } from "../utils/logger.ts"; +import { StorageMap } from "../types/storage_map.ts"; +import { StorageSet } from "../types/storage_set.ts"; // displayInit(); @@ -123,6 +126,10 @@ type storage_options = { type storage_location_options = L extends StorageLocation ? storage_options : never +type StorageSnapshotOptions = { + internalItems: boolean, + expandStorageMapsAndSets: boolean +} export class Storage { @@ -135,7 +142,9 @@ export class Storage { static item_prefix = "dxitem::"+site_suffix+"::" static meta_prefix = "dxmeta::"+site_suffix+"::" - + static rc_prefix = "rc::" + static pointer_deps_prefix = "deps::dxptr::" + static item_deps_prefix = "deps::dxitem::" static #storage_active_pointers = new Set(); static #storage_active_pointer_ids = new Set(); @@ -385,6 +394,7 @@ export class Storage { this.setDirty(location, true) // store value (might be pointer reference) const dependencies = await location.setItem(key, value); + this.updateItemDependencies(key, [...dependencies].map(p=>p.id)); await this.saveDependencyPointersAsync(dependencies, listen_for_pointer_changes, location); this.setDirty(location, false) return true; @@ -392,6 +402,7 @@ export class Storage { static setItemSync(location:SyncStorageLocation, key: string,value: unknown,listen_for_pointer_changes: boolean) { const dependencies = location.setItem(key, value); + this.updateItemDependencies(key, [...dependencies].map(p=>p.id)); this.saveDependencyPointersSync(dependencies, listen_for_pointer_changes, location); return true; } @@ -416,6 +427,7 @@ export class Storage { const dependencies = this.updatePointerSync(location, pointer, partialUpdateKey); dependencies.delete(pointer); + this.updatePointerDependencies(pointer.id, [...dependencies].map(p=>p.id)); this.saveDependencyPointersSync(dependencies, listen_for_changes, location); // listen for changes @@ -656,12 +668,21 @@ export class Storage { } } - private static async removePointer(pointer_id:string, location?:StorageLocation) { - // remove from specific location + private static async removePointer(pointer_id:string, location?:StorageLocation, force_remove = false) { + if (!force_remove && this.getReferenceCount(pointer_id) > 0) { + logger.warn("Cannot remove pointer $" + pointer_id + ", still referenced"); + return; + } + logger.debug("Removing pointer $" + pointer_id + " from storage" + (location ? " (" + location.name + ")" : "")); + + // remove from specific location if (location) return location.removePointer(pointer_id); // remove from all else { + // clear dependencies + this.updatePointerDependencies(pointer_id, []) + for (const location of this.#locations.keys()) { await location.removePointer(pointer_id); } @@ -838,18 +859,107 @@ export class Storage { } public static async removeItem(key:string, location?:StorageLocation):Promise { - if (Storage.cache.has(key)) Storage.cache.delete(key); // delete from cache + + logger.debug("Removing item '" + key + "' from storage" + (location ? " (" + location.name + ")" : "")) // remove from specific location if (location) return location.removeItem(key); // remove from all else { + if (Storage.cache.has(key)) Storage.cache.delete(key); // delete from cache + + // clear dependencies + this.updateItemDependencies(key, []) + for (const location of this.#locations.keys()) { await location.removeItem(key); } } } + /** + * Increase the reference count of a pointer in storage + */ + private static increaseReferenceCount(ptrId:string) { + localStorage.setItem(this.rc_prefix+ptrId, (this.getReferenceCount(ptrId) + 1).toString()); + } + /** + * Decrease the reference count of a pointer in storage + */ + private static decreaseReferenceCount(ptrId:string) { + const newCount = this.getReferenceCount(ptrId) - 1; + // RC is 0, delet pointer from storage + if (newCount <= 0) { + localStorage.removeItem(this.rc_prefix+ptrId); + this.removePointer(ptrId, undefined, true); + } + // decrease RC + else localStorage.setItem(this.rc_prefix+ptrId, newCount.toString()); + } + /** + * Get the current reference count of a pointer (number of entries that have a reference to this pointer) + * @returns + */ + private static getReferenceCount(ptrId:string) { + const entry = localStorage.getItem(this.rc_prefix+ptrId); + return entry ? Number(entry) : 0; + } + + private static addDependency(key:string, depPtrId:string, prefix:string) { + const uniqueKey = prefix+key; + if (localStorage.getItem(uniqueKey)?.includes(depPtrId)) return; + else localStorage.setItem(uniqueKey, (localStorage.getItem(uniqueKey) ?? "") + depPtrId); + } + private static removeDependency(key:string, depPtrId:string, prefix:string) { + const uniqueKey = prefix+key; + if (!localStorage.getItem(uniqueKey)?.includes(depPtrId)) return; + else { + const newDeps = localStorage.getItem(uniqueKey)!.replace(depPtrId, ""); + // remove key if no more dependencies + if (newDeps.length == 0) localStorage.removeItem(uniqueKey); + // remove dependency + else localStorage.setItem(uniqueKey, newDeps); + } + } + private static setDependencies(key:string, depPtrIds:string[], prefix:string) { + const uniqueKey = prefix+key; + if (!depPtrIds.length) localStorage.removeItem(uniqueKey); + else localStorage.setItem(uniqueKey, depPtrIds.join(",")); + } + + private static setItemDependencies(key:string, depPtrIds:string[]) { + this.setDependencies(key, depPtrIds, this.item_deps_prefix); + } + private static setPointerDependencies(key:string, depPtrIds:string[]) { + this.setDependencies(key, depPtrIds, this.pointer_deps_prefix); + } + private static getItemDependencies(key:string) { + const uniqueKey = this.item_deps_prefix+key; + return localStorage.getItem(uniqueKey)?.split(",") ?? []; + } + private static getPointerDependencies(key:string) { + const uniqueKey = this.pointer_deps_prefix+key; + return localStorage.getItem(uniqueKey)?.split(",") ?? []; + } + + private static updateItemDependencies(key:string, newDeps:string[]) { + const oldDeps = this.getItemDependencies(key); + const added = newDeps.filter(p=>!oldDeps.includes(p)); + const removed = oldDeps.filter(p=>!newDeps.includes(p)); + for (const ptrId of added) this.increaseReferenceCount(ptrId); + for (const ptrId of removed) this.decreaseReferenceCount(ptrId); + this.setItemDependencies(key, newDeps); + } + private static updatePointerDependencies(key:string, newDeps:string[]) { + const oldDeps = this.getPointerDependencies(key); + const added = newDeps.filter(p=>!oldDeps.includes(p)); + const removed = oldDeps.filter(p=>!newDeps.includes(p)); + for (const ptrId of added) this.increaseReferenceCount(ptrId); + for (const ptrId of removed) this.decreaseReferenceCount(ptrId); + this.setPointerDependencies(key, newDeps); + } + + public static clearAll(){ return this.clear() } @@ -903,6 +1013,130 @@ export class Storage { else throw new Error("Cannot find or create the state '" + id + "'") } + static #replaceLastOccurenceOf(search: string, replace: string, str: string) { + const n = str.lastIndexOf(search); + return str.substring(0, n) + replace + str.substring(n + search.length); + } + + public static async printSnapshot(options: StorageSnapshotOptions = {internalItems: false, expandStorageMapsAndSets: true}) { + const {items, pointers} = await this.getSnapshot(); + + const COLOR_PTR = `\x1b[38;2;${[65,102,238].join(';')}m` + const COLOR_NUMBER = `\x1b[38;2;${[253,139,25].join(';')}m` + + let string = "Storage Locations:" + for (const [location, options] of this.#locations) { + string += `\n ${location.name} ${ESCAPE_SEQUENCES.GREY}(${options.modes.map(m=>Storage.Mode[m]).join(', ')})${ESCAPE_SEQUENCES.RESET}` + } + console.log(string); + + // pointers + string = "Pointers:\n\n" + for (const [key, [value, location]] of pointers.snapshot) { + string += ` ${COLOR_PTR}$${key}${ESCAPE_SEQUENCES.GREY} (${location.name}) = ${this.#replaceLastOccurenceOf("\n ", "", value.replaceAll("\n", "\n "))}\n` + } + console.log(string); + + // items + string = "Items:\n\n" + for (const [key, [value, location]] of items.snapshot) { + string += ` ${key}${ESCAPE_SEQUENCES.GREY} (${location.name}) = ${value}` + } + console.log(string); + + // memory management + if (options?.internalItems) { + string = "Memory Management:\n\n" + let rc_string = "" + let item_deps_string = "" + let pointer_deps_string = "" + for (let i = 0, len = localStorage.length; i < len; ++i ) { + const key = localStorage.key(i)!; + if (key.startsWith(this.rc_prefix)) { + const ptrId = key.substring(this.rc_prefix.length); + const count = this.getReferenceCount(ptrId); + rc_string += `\x1b[0m ${key} = ${COLOR_NUMBER}${count}\n` + } + else if (key.startsWith(this.item_deps_prefix)) { + let deps = localStorage.getItem(key)!.split(",").join(`\x1b[0m,\n ${COLOR_PTR}$`) + if (deps) deps = ` ${COLOR_PTR}$`+deps + item_deps_string += `\x1b[0m ${key} = (\n${COLOR_PTR}${deps}\x1b[0m\n )\n` + } + else if (key.startsWith(this.pointer_deps_prefix)) { + let deps = localStorage.getItem(key)!.split(",").join(`\x1b[0m,\n ${COLOR_PTR}$`) + if (deps) deps = ` ${COLOR_PTR}$`+deps + pointer_deps_string += `\x1b[0m ${key} = (\n${COLOR_PTR}${deps}\x1b[0m\n )\n` + } + } + + string += rc_string + "\n" + item_deps_string + "\n" + pointer_deps_string; + console.log(string); + + } + + } + + public static async getSnapshot(options: StorageSnapshotOptions = {internalItems: false, expandStorageMapsAndSets: true}) { + const items = await this.createSnapshot(this.getItemKeys.bind(this), this.getItemDecompiled.bind(this)); + const pointers = await this.createSnapshot(this.getPointerKeys.bind(this), this.getPointerDecompiledFromLocation.bind(this)); + + if (!options.internalItems) { + for (const [key] of items.snapshot) { + if (key.startsWith("keys_")) items.snapshot.delete(key); + } + } + + if (options.expandStorageMapsAndSets) { + for (const [ptrId, [value, location]] of pointers.snapshot) { + if (value.startsWith("\x1b[38;2;50;153;220m") || value.startsWith("\x1b[38;2;50;153;220m")) { + const ptr = await Pointer.load(ptrId, undefined, true); + if (ptr.val instanceof StorageMap) { + const map = ptr.val; + let inner = ""; + for await (const [key, val] of map) { + inner += ` ${Runtime.valueToDatexStringExperimental(key, true, true)}\x1b[0m => ${Runtime.valueToDatexStringExperimental(val, true, true)}\n` + } + if (inner) pointers.snapshot.set(ptrId, ["\x1b[38;2;50;153;220m \x1b[0m{\n"+inner+"\x1b[0m\n}", location]) + } + else if (ptr.val instanceof StorageSet) { + const set = ptr.val; + let inner = ""; + for await (const val of set) { + inner += ` ${Runtime.valueToDatexStringExperimental(val, true, true)},\n` + } + if (inner) pointers.snapshot.set(ptrId, ["\x1b[38;2;50;153;220m \x1b[0m{\n"+inner+"\x1b[0m\n}", location]) + } + } + } + } + + return {items, pointers} + } + + private static async createSnapshot( + keyGenerator: (location?: StorageLocation | undefined) => Promise>, + itemGetter: (key: string, colorized: boolean, location: StorageLocation) => Promise, + ) { + const snapshot = new Map(); + const inconsistentItems = new Map>().setAutoDefault(Map); + for (const location of new Set([this.#primary_location!, ...this.#locations.keys()].filter(l=>!!l))) { + for (const key of await keyGenerator(location)) { + const decompiled = await itemGetter(key, true, location); + if (typeof decompiled !== "string") { + console.error("Invalid entry in storage (" + location.name + "): " + key); + } + else if (snapshot.has(key) && snapshot.get(key)![0] != decompiled) { + inconsistentItems.getAuto(key).set(snapshot.get(key)![1], snapshot.get(key)![0]); + inconsistentItems.getAuto(key).set(location, decompiled); + } + else { + snapshot.set(key, [decompiled, location]); + } + } + } + return {snapshot, inconsistentItems}; + } + } export namespace Storage { From e811fbff099251868332fcfa41787d4235b94b7b Mon Sep 17 00:00:00 2001 From: benStre Date: Sat, 20 Jan 2024 22:59:02 +0100 Subject: [PATCH 02/16] fix pointer loading --- runtime/pointers.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/runtime/pointers.ts b/runtime/pointers.ts index 56b024c4..4d3ddc7e 100644 --- a/runtime/pointers.ts +++ b/runtime/pointers.ts @@ -1394,13 +1394,16 @@ export class Pointer extends Ref { } if (stored!=NOT_EXISTING) { - // if the value is a pointer with a tranform scope, copy the transform, not the value (TODO still just a workaround to preserve transforms in storage, maybe better solution?) - if (stored instanceof Pointer && stored.transform_scope) { - await pointer.handleTransformAsync(stored.transform_scope.internal_vars, stored.transform_scope); + // set value if pointer still not loaded during source.getPointer + if (!pointer.#loaded) { + // if the value is a pointer with a tranform scope, copy the transform, not the value (TODO still just a workaround to preserve transforms in storage, maybe better solution?) + if (stored instanceof Pointer && stored.transform_scope) { + await pointer.handleTransformAsync(stored.transform_scope.internal_vars, stored.transform_scope); + } + // set normal value + else pointer = pointer.setValue(stored); } - // set normal value - else pointer = pointer.setValue(stored); - + // now sync if source (pointer storage) can sync pointer if (source?.syncPointer) source.syncPointer(pointer); From 317dee2f6914da3800ad0ba7fbeedb5ac99cbabb Mon Sep 17 00:00:00 2001 From: benStre Date: Sat, 20 Jan 2024 22:59:40 +0100 Subject: [PATCH 03/16] fix set key generation --- types/storage_set.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/types/storage_set.ts b/types/storage_set.ts index 5c4116fd..d73c3f00 100644 --- a/types/storage_set.ts +++ b/types/storage_set.ts @@ -18,7 +18,7 @@ export class StorageSet { constructor(){ // TODO: does not work with eternal pointers! - // Pointer.proxifyValue(this) + Pointer.proxifyValue(this) } static async from(values: readonly V[]) { @@ -66,8 +66,8 @@ export class StorageSet { }, 60_000); } - protected getStorageKey(value: V) { - const keyHash = Compiler.getUniqueValueIdentifier(value); + protected async getStorageKey(value: V) { + const keyHash = await Compiler.getUniqueValueIdentifier(value); // @ts-ignore DX_PTR return this.prefix + keyHash; } From fa5ae779198bc9f52850c8db0a36b52b339a85d4 Mon Sep 17 00:00:00 2001 From: benStre Date: Sat, 20 Jan 2024 23:24:10 +0100 Subject: [PATCH 04/16] fix pointer already created during async pointer loading --- runtime/storage.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/runtime/storage.ts b/runtime/storage.ts index 6ac29ce4..41b0f716 100644 --- a/runtime/storage.ts +++ b/runtime/storage.ts @@ -646,15 +646,22 @@ export class Storage { // create pointer with saved id and value + start syncing, if pointer not already created in DATEX if (pointerify) { - let pointer: Pointer; + let pointer = Pointer.get(pointer_id) // if the value is a pointer with a tranform scope, copy the transform, not the value (TODO still just a workaround to preserve transforms in storage, maybe better solution?) if (val instanceof Pointer && val.transform_scope) { console.log("init value",val); pointer = await Pointer.createTransformAsync(val.transform_scope.internal_vars, val.transform_scope); } - // normal pointer from value - else pointer = Pointer.create(pointer_id, val, false, Runtime.endpoint); + // set value of existing pointer + else if (pointer) { + if (pointer.value_initialized) logger.warn("pointer value " + pointer.idString() + " already initialized, setting new value from storage"); + pointer = pointer.setValue(val); + } + // create new pointer from value + else { + pointer = Pointer.create(pointer_id, val, false, Runtime.endpoint); + } this.syncPointer(pointer); this.#storage_active_pointers.add(pointer); From 1cdfcd0899b0ce6659602b46781135dc0c148818 Mon Sep 17 00:00:00 2001 From: benStre Date: Sat, 20 Jan 2024 23:28:56 +0100 Subject: [PATCH 05/16] fix recursive set initialization --- types/native_types.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/types/native_types.ts b/types/native_types.ts index 717c1966..ff9b1ab5 100644 --- a/types/native_types.ts +++ b/types/native_types.ts @@ -278,6 +278,11 @@ Type.std.Set.setJSInterface({ get_property: (parent:Set, key) => NOT_EXISTING, has_property: (parent:Set, key) => parent.has(key), + // implemented to support self-referencing serialization, not actual properties + // small issue with this approach: the Set always contains 'undefined' + set_property_silently: (parent:Set, key, value, pointer) => Set.prototype.add.call(parent, value), + set_property: (parent:Set, key, value) => parent.add(value), + count: (parent:Set) => parent.size, keys: (parent:Set) => [...parent], From 3ac89d8fe0287ab5190ef08568e9c119fe26b1d8 Mon Sep 17 00:00:00 2001 From: benStre Date: Sun, 21 Jan 2024 00:42:37 +0100 Subject: [PATCH 06/16] improve snapshot print --- runtime/storage.ts | 135 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 103 insertions(+), 32 deletions(-) diff --git a/runtime/storage.ts b/runtime/storage.ts index 41b0f716..37b8b6af 100644 --- a/runtime/storage.ts +++ b/runtime/storage.ts @@ -1031,29 +1031,50 @@ export class Storage { const COLOR_PTR = `\x1b[38;2;${[65,102,238].join(';')}m` const COLOR_NUMBER = `\x1b[38;2;${[253,139,25].join(';')}m` - let string = "Storage Locations:" + let string = ESCAPE_SEQUENCES.BOLD+"Storage Locations\n\n"+ESCAPE_SEQUENCES.RESET + string += `${ESCAPE_SEQUENCES.ITALIC}A list of all currently used storage locations and their corresponding store strategies.\n${ESCAPE_SEQUENCES.RESET}` + for (const [location, options] of this.#locations) { - string += `\n ${location.name} ${ESCAPE_SEQUENCES.GREY}(${options.modes.map(m=>Storage.Mode[m]).join(', ')})${ESCAPE_SEQUENCES.RESET}` + string += `\n • ${location.name} ${ESCAPE_SEQUENCES.GREY}(${options.modes.map(m=>Storage.Mode[m]).join(', ')})${ESCAPE_SEQUENCES.RESET}` } - console.log(string); + + string += `\n\n${ESCAPE_SEQUENCES.BOLD}Trusted Location:${ESCAPE_SEQUENCES.RESET} ${this.#trusted_location?.name ?? "none"}` + string += `\n${ESCAPE_SEQUENCES.BOLD}Primary Location:${ESCAPE_SEQUENCES.RESET} ${this.#primary_location?.name ?? "none"}` + + console.log(string+"\n\n"); // pointers - string = "Pointers:\n\n" - for (const [key, [value, location]] of pointers.snapshot) { - string += ` ${COLOR_PTR}$${key}${ESCAPE_SEQUENCES.GREY} (${location.name}) = ${this.#replaceLastOccurenceOf("\n ", "", value.replaceAll("\n", "\n "))}\n` + string = ESCAPE_SEQUENCES.BOLD+"Pointers\n\n"+ESCAPE_SEQUENCES.RESET + string += `${ESCAPE_SEQUENCES.ITALIC}A list of all pointers stored in any storage location. Pointers are only stored as long as they are referenced somwhere else in the storage.\n\n${ESCAPE_SEQUENCES.RESET}` + + for (const [key, storageMap] of pointers.snapshot) { + // check if stored in all locations, otherwise print in which location it is stored (functional programming) + const locations = [...storageMap.keys()] + const storedInAll = [...this.#locations.keys()].every(l => locations.includes(l)); + + const value = [...storageMap.values()][0]; + string += ` • ${COLOR_PTR}$${key}${ESCAPE_SEQUENCES.GREY}${storedInAll ? "" : (` (only in ${locations.map(l=>l.name).join(",")})`)} = ${this.#replaceLastOccurenceOf("\n ", "", value.replaceAll("\n", "\n "))}\n` } - console.log(string); + console.log(string+"\n"); // items - string = "Items:\n\n" - for (const [key, [value, location]] of items.snapshot) { - string += ` ${key}${ESCAPE_SEQUENCES.GREY} (${location.name}) = ${value}` + string = ESCAPE_SEQUENCES.BOLD+"Items\n\n"+ESCAPE_SEQUENCES.RESET + string += `${ESCAPE_SEQUENCES.ITALIC}A list of all named items stored in any storage location.\n\n${ESCAPE_SEQUENCES.RESET}` + + for (const [key, storageMap] of items.snapshot) { + // check if stored in all locations, otherwise print in which location it is stored (functional programming) + const locations = [...storageMap.keys()] + const storedInAll = [...this.#locations.keys()].every(l => locations.includes(l)); + + const value = [...storageMap.values()][0]; + string += ` • ${key}${ESCAPE_SEQUENCES.GREY}${storedInAll ? "" : (` (only in ${locations.map(l=>l.name).join(",")})`)} = ${value}` } - console.log(string); + console.log(string+"\n"); // memory management if (options?.internalItems) { - string = "Memory Management:\n\n" + string = ESCAPE_SEQUENCES.BOLD+"Memory Management\n\n"+ESCAPE_SEQUENCES.RESET + string += `${ESCAPE_SEQUENCES.ITALIC}This section shows the reference count (rc::) of pointers and the dependencies (deps::) of items and pointers. The reference count of a pointer tracks the number of items and pointers that reference this pointer.\n\n${ESCAPE_SEQUENCES.RESET}` let rc_string = "" let item_deps_string = "" let pointer_deps_string = "" @@ -1062,24 +1083,60 @@ export class Storage { if (key.startsWith(this.rc_prefix)) { const ptrId = key.substring(this.rc_prefix.length); const count = this.getReferenceCount(ptrId); - rc_string += `\x1b[0m ${key} = ${COLOR_NUMBER}${count}\n` + rc_string += `\x1b[0m • ${key} = ${COLOR_NUMBER}${count}\n` } else if (key.startsWith(this.item_deps_prefix)) { - let deps = localStorage.getItem(key)!.split(",").join(`\x1b[0m,\n ${COLOR_PTR}$`) - if (deps) deps = ` ${COLOR_PTR}$`+deps - item_deps_string += `\x1b[0m ${key} = (\n${COLOR_PTR}${deps}\x1b[0m\n )\n` + const depsRaw = localStorage.getItem(key); + // single entry + if (!depsRaw?.includes(",")) { + item_deps_string += `\x1b[0m • ${key} = (${COLOR_PTR}${depsRaw}\x1b[0m)\n` + } + // multiple entries + else { + let deps = localStorage.getItem(key)!.split(",").join(`\x1b[0m,\n ${COLOR_PTR}$`) + if (deps) deps = ` ${COLOR_PTR}$`+deps + item_deps_string += `\x1b[0m • ${key} = (\n${COLOR_PTR}${deps}\x1b[0m\n )\n` + } } else if (key.startsWith(this.pointer_deps_prefix)) { - let deps = localStorage.getItem(key)!.split(",").join(`\x1b[0m,\n ${COLOR_PTR}$`) - if (deps) deps = ` ${COLOR_PTR}$`+deps - pointer_deps_string += `\x1b[0m ${key} = (\n${COLOR_PTR}${deps}\x1b[0m\n )\n` + const depsRaw = localStorage.getItem(key); + // single entry + if (!depsRaw?.includes(",")) { + pointer_deps_string += `\x1b[0m • ${key} = (${COLOR_PTR}${depsRaw}\x1b[0m)\n` + } + // multiple entries + else { + let deps = localStorage.getItem(key)!.split(",").join(`\x1b[0m,\n ${COLOR_PTR}$`) + if (deps) deps = ` ${COLOR_PTR}$`+deps + pointer_deps_string += `\x1b[0m • ${key} = (\n${COLOR_PTR}${deps}\x1b[0m\n )\n` + } } } string += rc_string + "\n" + item_deps_string + "\n" + pointer_deps_string; - console.log(string); + console.log(string+"\n"); + } + + // inconsistencies + if (pointers.inconsistencies.size > 0 || items.inconsistencies.size > 0) { + string = ESCAPE_SEQUENCES.BOLD+"Inconsistencies\n\n"+ESCAPE_SEQUENCES.RESET + string += `${ESCAPE_SEQUENCES.ITALIC}Inconsistencies between storage locations don't necessarily indicate that something is wrong. They can occur when a storage location is not updated immediately (e.g. when only using SAVE_ON_EXIT).\n\n${ESCAPE_SEQUENCES.RESET}` + for (const [key, storageMap] of pointers.inconsistencies) { + for (const [location, value] of storageMap) { + string += ` • ${COLOR_PTR}$${key}${ESCAPE_SEQUENCES.GREY} (${(location.name+")").padEnd(15, " ")} = ${this.#replaceLastOccurenceOf("\n ", "", value.replaceAll("\n", "\n "))}\n` + } + string += `\n` + } + for (const [key, storageMap] of items.inconsistencies) { + for (const [location, value] of storageMap) { + string += ` • ${key}${ESCAPE_SEQUENCES.GREY} (${(location.name+")").padEnd(15, " ")} = ${value}` + } + string += `\n` + } + console.info(string+"\n"); } + } @@ -1087,14 +1144,19 @@ export class Storage { const items = await this.createSnapshot(this.getItemKeys.bind(this), this.getItemDecompiled.bind(this)); const pointers = await this.createSnapshot(this.getPointerKeys.bind(this), this.getPointerDecompiledFromLocation.bind(this)); + // remove internal items if (!options.internalItems) { for (const [key] of items.snapshot) { if (key.startsWith("keys_")) items.snapshot.delete(key); } } + // iterate over storage maps and sets and render all entries if (options.expandStorageMapsAndSets) { - for (const [ptrId, [value, location]] of pointers.snapshot) { + for (const [ptrId, storageMap] of pointers.snapshot) { + // display entry from first storage + const [location, value] = [...storageMap.entries()][0]; + if (value.startsWith("\x1b[38;2;50;153;220m") || value.startsWith("\x1b[38;2;50;153;220m")) { const ptr = await Pointer.load(ptrId, undefined, true); if (ptr.val instanceof StorageMap) { @@ -1103,7 +1165,7 @@ export class Storage { for await (const [key, val] of map) { inner += ` ${Runtime.valueToDatexStringExperimental(key, true, true)}\x1b[0m => ${Runtime.valueToDatexStringExperimental(val, true, true)}\n` } - if (inner) pointers.snapshot.set(ptrId, ["\x1b[38;2;50;153;220m \x1b[0m{\n"+inner+"\x1b[0m\n}", location]) + if (inner) storageMap.set(location, "\x1b[38;2;50;153;220m \x1b[0m{\n"+inner+"\x1b[0m\n}") } else if (ptr.val instanceof StorageSet) { const set = ptr.val; @@ -1111,7 +1173,7 @@ export class Storage { for await (const val of set) { inner += ` ${Runtime.valueToDatexStringExperimental(val, true, true)},\n` } - if (inner) pointers.snapshot.set(ptrId, ["\x1b[38;2;50;153;220m \x1b[0m{\n"+inner+"\x1b[0m\n}", location]) + if (inner) storageMap.set(location, "\x1b[38;2;50;153;220m \x1b[0m{\n"+inner+"\x1b[0m\n}") } } } @@ -1124,24 +1186,33 @@ export class Storage { keyGenerator: (location?: StorageLocation | undefined) => Promise>, itemGetter: (key: string, colorized: boolean, location: StorageLocation) => Promise, ) { - const snapshot = new Map(); - const inconsistentItems = new Map>().setAutoDefault(Map); + const snapshot = new Map>().setAutoDefault(Map); + const inconsistencies = new Map>().setAutoDefault(Map); for (const location of new Set([this.#primary_location!, ...this.#locations.keys()].filter(l=>!!l))) { for (const key of await keyGenerator(location)) { const decompiled = await itemGetter(key, true, location); if (typeof decompiled !== "string") { console.error("Invalid entry in storage (" + location.name + "): " + key); + continue; } - else if (snapshot.has(key) && snapshot.get(key)![0] != decompiled) { - inconsistentItems.getAuto(key).set(snapshot.get(key)![1], snapshot.get(key)![0]); - inconsistentItems.getAuto(key).set(location, decompiled); - } - else { - snapshot.set(key, [decompiled, location]); + snapshot.getAuto(key).set(location, decompiled); + } + } + + + // find inconsistencies + for (const [key, storageMap] of snapshot) { + const [location, value] = [...storageMap.entries()][0]; + // compare with first entry + for (const [location2, value2] of storageMap) { + if (value !== value2) { + inconsistencies.getAuto(key).set(location, value); + inconsistencies.getAuto(key).set(location2, value2); } } } - return {snapshot, inconsistentItems}; + + return {snapshot, inconsistencies}; } } From a3ec668e5eba8f1e2e0fee0c8657ab7d82120145 Mon Sep 17 00:00:00 2001 From: benStre Date: Sun, 21 Jan 2024 00:53:33 +0100 Subject: [PATCH 07/16] fix subscriber cache init --- init.ts | 50 ++++++++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/init.ts b/init.ts index 0d3fc566..5dd25a28 100644 --- a/init.ts +++ b/init.ts @@ -13,12 +13,34 @@ import { verboseArg } from "./utils/logger.ts"; import { MessageLogger } from "./utils/message_logger.ts"; +/** + * Load the subscriber cache from storage and + * reset if a remote pointer is required + */ +async function initSubscriberCache() { + try { + Runtime.subscriber_cache = (await Storage.loadOrCreate( + "Datex.Runtime.SUBSCRIBER_CACHE", + () => new Map(), + {onlyLocalPointers: true} + )).setAutoDefault(Set); + } + catch (e) { + logger.debug("resetting subscriber cache (" + e?.message + ")") + Runtime.subscriber_cache = (await Storage.loadOrCreate( + "Datex.Runtime.SUBSCRIBER_CACHE", + () => new Map(), + undefined, + true + )).setAutoDefault(Set); + } +} + /** * Runtime init (sets ENV, storage, endpoint, ...) */ export async function init() { - // register DatexStorage as pointer source registerStorageAsPointerSource(); // default storage config: @@ -56,7 +78,11 @@ export async function init() { Runtime.onEndpointChanged((endpoint) => { Pointer.pointer_prefix = endpoint.getPointerPrefix(); // has only local endpoint id (%0000) or global id? - if (endpoint != LOCAL_ENDPOINT) Pointer.is_local = false; + if (endpoint != LOCAL_ENDPOINT) { + Pointer.is_local = false; + // init subscriber cache as soon as endpoint is available + initSubscriberCache(); + } else Pointer.is_local = true; }) @@ -105,26 +131,6 @@ export async function init() { // init persistent memory Runtime.persistent_memory = (await Storage.loadOrCreate("Datex.Runtime.MEMORY", ()=>new Map())).setAutoDefault(Object); - // init persistent subscriber cache - (async () => { - try { - Runtime.subscriber_cache = (await Storage.loadOrCreate( - "Datex.Runtime.SUBSCRIBER_CACHE", - () => new Map(), - {onlyLocalPointers: true} - )).setAutoDefault(Set); - } - catch (e) { - logger.debug("resetting subscriber cache (" + e?.message + ")") - Runtime.subscriber_cache = (await Storage.loadOrCreate( - "Datex.Runtime.SUBSCRIBER_CACHE", - () => new Map(), - undefined, - true - )).setAutoDefault(Set); - } - })() - if (!globalThis.NO_INIT) { Runtime.init(); From 94252c80e12ea64460389687f67eaeaa8faa9cec Mon Sep 17 00:00:00 2001 From: benStre Date: Sun, 21 Jan 2024 01:06:17 +0100 Subject: [PATCH 08/16] fix pointer remove and init for storage --- runtime/storage.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/runtime/storage.ts b/runtime/storage.ts index 37b8b6af..281515ee 100644 --- a/runtime/storage.ts +++ b/runtime/storage.ts @@ -447,6 +447,7 @@ export class Storage { const dependencies = await this.updatePointerAsync(location, pointer, partialUpdateKey); dependencies.delete(pointer); + this.updatePointerDependencies(pointer.id, [...dependencies].map(p=>p.id)); await this.saveDependencyPointersAsync(dependencies, listen_for_changes, location); // listen for changes @@ -690,6 +691,10 @@ export class Storage { // clear dependencies this.updatePointerDependencies(pointer_id, []) + const ptr = Pointer.get(pointer_id) + if (ptr) this.#storage_active_pointers.delete(ptr); + this.#storage_active_pointer_ids.delete(pointer_id); + for (const location of this.#locations.keys()) { await location.removePointer(pointer_id); } @@ -873,7 +878,7 @@ export class Storage { if (location) return location.removeItem(key); // remove from all else { - if (Storage.cache.has(key)) Storage.cache.delete(key); // delete from cache + Storage.cache.delete(key); // delete from cache // clear dependencies this.updateItemDependencies(key, []) From b6e21f1852345f25e62b87b43b723a70274691f5 Mon Sep 17 00:00:00 2001 From: benStre Date: Sun, 21 Jan 2024 02:16:19 +0100 Subject: [PATCH 09/16] fix unintended pointer id reassignment --- runtime/pointers.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runtime/pointers.ts b/runtime/pointers.ts index 4d3ddc7e..b5b51a32 100644 --- a/runtime/pointers.ts +++ b/runtime/pointers.ts @@ -1198,7 +1198,8 @@ export class Pointer extends Ref { // update pointer ids if no longer local if (!this.#is_local) { for (const pointer of this.#local_pointers) { - pointer.id = Pointer.getUniquePointerID(pointer); + // still local? + if (pointer.origin == LOCAL_ENDPOINT) pointer.id = Pointer.getUniquePointerID(pointer); } this.#local_pointers.clear(); } From 5b23faa403323089c392b9ba6e42f125d4baf6e2 Mon Sep 17 00:00:00 2001 From: benStre Date: Sun, 21 Jan 2024 02:16:52 +0100 Subject: [PATCH 10/16] storage fixes --- runtime/storage.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/runtime/storage.ts b/runtime/storage.ts index 281515ee..0c465bd8 100644 --- a/runtime/storage.ts +++ b/runtime/storage.ts @@ -380,6 +380,7 @@ export class Storage { static setItem(key:string, value:any, listen_for_pointer_changes = true, location:StorageLocation|null|undefined = this.#primary_location):Promise|boolean { Storage.cache.set(key, value); // save in cache + // cache deletion does not work, problems with storage item backup // setTimeout(()=>Storage.cache.delete(key), 10000); @@ -845,6 +846,7 @@ export class Storage { Storage.cache.set(key, val); await this.initItemFromTrustedLocation(key, val, location) + return val; } @@ -1031,7 +1033,7 @@ export class Storage { } public static async printSnapshot(options: StorageSnapshotOptions = {internalItems: false, expandStorageMapsAndSets: true}) { - const {items, pointers} = await this.getSnapshot(); + const {items, pointers} = await this.getSnapshot(options); const COLOR_PTR = `\x1b[38;2;${[65,102,238].join(';')}m` const COLOR_NUMBER = `\x1b[38;2;${[253,139,25].join(';')}m` @@ -1149,11 +1151,9 @@ export class Storage { const items = await this.createSnapshot(this.getItemKeys.bind(this), this.getItemDecompiled.bind(this)); const pointers = await this.createSnapshot(this.getPointerKeys.bind(this), this.getPointerDecompiledFromLocation.bind(this)); - // remove internal items - if (!options.internalItems) { - for (const [key] of items.snapshot) { - if (key.startsWith("keys_")) items.snapshot.delete(key); - } + // remove keys items that are unrelated to normal storage + for (const [key] of items.snapshot) { + if (key.startsWith("keys_")) items.snapshot.delete(key); } // iterate over storage maps and sets and render all entries From a2626bf16b8a6f6409571ca33ec3ef0743769f1a Mon Sep 17 00:00:00 2001 From: benStre Date: Sun, 21 Jan 2024 02:17:35 +0100 Subject: [PATCH 11/16] StorageMap refactoring, add comments --- types/storage_map.ts | 44 +++++++++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/types/storage_map.ts b/types/storage_map.ts index 252e6ae4..11314f41 100644 --- a/types/storage_map.ts +++ b/types/storage_map.ts @@ -21,8 +21,7 @@ export class StorageWeakMap { #prefix?: string; constructor(){ - // TODO: does not work with eternal pointers! - // Pointer.proxifyValue(this) + Pointer.proxifyValue(this) } @@ -32,9 +31,8 @@ export class StorageWeakMap { return map; } - get prefix() { - // @ts-ignore - if (!this.#prefix) this.#prefix = 'dxmap::'+this[DX_PTR].idString()+'.'; + protected get _prefix() { + if (!this.#prefix) this.#prefix = 'dxmap::'+(this as any)[DX_PTR].idString()+'.'; return this.#prefix; } @@ -83,12 +81,12 @@ export class StorageWeakMap { protected async getStorageKey(key: K) { const keyHash = await Compiler.getUniqueValueIdentifier(key); // @ts-ignore DX_PTR - return this.prefix + keyHash; + return this._prefix + keyHash; } async clear() { const promises = []; - for (const key of await Storage.getItemKeysStartingWith(this.prefix)) { + for (const key of await Storage.getItemKeysStartingWith(this._prefix)) { promises.push(await Storage.removeItem(key)); } await Promise.all(promises); @@ -128,11 +126,14 @@ export class StorageMap extends StorageWeakMap { return Storage.removeItem(storage_item_key) } + /** + * Async iterator that returns all keys. + */ keys() { const self = this; const key_prefix = this.#key_prefix; return (async function*(){ - const keyGenerator = await Storage.getItemKeysStartingWith(self.prefix); + const keyGenerator = await Storage.getItemKeysStartingWith(self._prefix); for (const key of keyGenerator) { const keyValue = await Storage.getItem(key_prefix+key); @@ -140,16 +141,24 @@ export class StorageMap extends StorageWeakMap { } })() } + + /** + * Returns an array containing all keys. + * This can be used to iterate over the keys without using a (for await of) loop. + */ async keysArray() { const keys = []; for await (const key of this.keys()) keys.push(key); return keys; } + /** + * Async iterator that returns all values. + */ values() { const self = this; return (async function*(){ - const keyGenerator = await Storage.getItemKeysStartingWith(self.prefix); + const keyGenerator = await Storage.getItemKeysStartingWith(self._prefix); for (const key of keyGenerator) { const value = await Storage.getItem(key); @@ -157,15 +166,28 @@ export class StorageMap extends StorageWeakMap { } })() } + + /** + * Returns an array containing all values. + * This can be used to iterate over the values without using a (for await of) loop. + */ async valuesArray() { const values = []; for await (const value of this.values()) values.push(value); return values; } + /** + * Async iterator that returns all entries. + */ entries() { return this[Symbol.asyncIterator]() } + + /** + * Returns an array containing all entries. + * This can be used to iterate over the entries without using a (for await of) loop. + */ async entriesArray() { const entries = []; for await (const entry of this.entries()) entries.push(entry); @@ -173,7 +195,7 @@ export class StorageMap extends StorageWeakMap { } async *[Symbol.asyncIterator]() { - const keyGenerator = await Storage.getItemKeysStartingWith(this.prefix); + const keyGenerator = await Storage.getItemKeysStartingWith(this._prefix); for (const key of keyGenerator) { const keyValue = await Storage.getItem(this.#key_prefix+key); @@ -184,7 +206,7 @@ export class StorageMap extends StorageWeakMap { override async clear() { const promises = []; - for (const key of await Storage.getItemKeysStartingWith(this.prefix)) { + for (const key of await Storage.getItemKeysStartingWith(this._prefix)) { promises.push(await Storage.removeItem(key)); promises.push(await Storage.removeItem(this.#key_prefix+key)); } From 3bd4cdc57d85deebeb1e1c1af56753e850da7804 Mon Sep 17 00:00:00 2001 From: benStre Date: Sun, 21 Jan 2024 02:38:45 +0100 Subject: [PATCH 12/16] complete StorageWeakSet, update jsdocs --- types/storage_map.ts | 11 +++---- types/storage_set.ts | 68 +++++++++++++++++++++++++++++++++++--------- types/type.ts | 10 ++++++- 3 files changed, 70 insertions(+), 19 deletions(-) diff --git a/types/storage_map.ts b/types/storage_map.ts index 11314f41..464b05fd 100644 --- a/types/storage_map.ts +++ b/types/storage_map.ts @@ -11,10 +11,10 @@ const logger = new Logger("StorageMap"); /** * WeakMap that outsources values to storage. - * In contrast to JS WeakMaps, primitive keys are also allowed - * Entries are not automatically garbage collected but must be - * explicitly deleted - * all methods are async + * The API is similar to the JS WeakMap API, but all methods are async. + * In contrast to JS WeakMaps, primitive keys are also allowed. + * The StorageWeakMap holds no strong references to its keys in storage. + * This means that the pointer of a key can be garbage collected. */ export class StorageWeakMap { @@ -95,7 +95,8 @@ export class StorageWeakMap { } /** - * Map that outsources values to storage. + * Set that outsources values to storage. + * The API is similar to the JS Map API, but all methods are async. */ export class StorageMap extends StorageWeakMap { diff --git a/types/storage_set.ts b/types/storage_set.ts index d73c3f00..2d2197b5 100644 --- a/types/storage_set.ts +++ b/types/storage_set.ts @@ -9,15 +9,17 @@ import { Logger } from "../utils/logger.ts"; const logger = new Logger("StorageSet"); /** - * Set that outsources values to storage. - * all methods are async + * WeakSet that outsources values to storage. + * The API is similar to the JS WeakSet API, but all methods are async. + * In contrast to JS WeakSets, primitive values are also allowed. + * The StorageWeakSet holds no strong references to its values in storage. + * This means that the pointer of a value can be garbage collected. */ -export class StorageSet { +export class StorageWeakSet { #prefix?: string; constructor(){ - // TODO: does not work with eternal pointers! Pointer.proxifyValue(this) } @@ -27,18 +29,17 @@ export class StorageSet { return set; } - get prefix() { - // @ts-ignore - if (!this.#prefix) this.#prefix = 'dxset::'+this[DX_PTR].idString()+'.'; + protected get _prefix() { + if (!this.#prefix) this.#prefix = 'dxset::'+(this as any)[DX_PTR].idString()+'.'; return this.#prefix; } async add(value: V) { const storage_key = await this.getStorageKey(value); if (await this._has(storage_key)) return; // already exists - return this._add(storage_key, value); + return this._add(storage_key, null); } - protected _add(storage_key:string, value:V) { + protected _add(storage_key:string, value:V|null) { this.activateCacheTimeout(storage_key); return Storage.setItem(storage_key, value); } @@ -69,40 +70,76 @@ export class StorageSet { protected async getStorageKey(value: V) { const keyHash = await Compiler.getUniqueValueIdentifier(value); // @ts-ignore DX_PTR - return this.prefix + keyHash; + return this._prefix + keyHash; } async clear() { const promises = []; - for (const key of await Storage.getItemKeysStartingWith(this.prefix)) { + for (const key of await Storage.getItemKeysStartingWith(this._prefix)) { promises.push(await Storage.removeItem(key)); } await Promise.all(promises); } +} + +/** + * Set that outsources values to storage. + * The API is similar to the JS Set API, but all methods are async. + */ +export class StorageSet extends StorageWeakSet { + + /** + * Appends a new value to the StorageWeakSet. + */ + async add(value: V) { + const storage_key = await this.getStorageKey(value); + if (await this._has(storage_key)) return; // already exists + return this._add(storage_key, value); + } + + /** + * Async iterator that returns all keys. + */ keys() { return this[Symbol.asyncIterator]() } + + /** + * Returns an array containing all keys. + * This can be used to iterate over the keys without using a (for await of) loop. + */ async keysArray() { const keys = []; for await (const key of this.keys()) keys.push(key); return keys; } + /** + * Async iterator that returns all values. + */ values() { return this[Symbol.asyncIterator]() } + + /** + * Returns an array containing all values. + * This can be used to iterate over the values without using a (for await of) loop. + */ async valuesArray() { const values = []; for await (const value of this.values()) values.push(value); return values; } + /** + * Async iterator that returns all entries. + */ entries() { const self = this; return (async function*(){ - const keyGenerator = await Storage.getItemKeysStartingWith(self.prefix); + const keyGenerator = await Storage.getItemKeysStartingWith(self._prefix); for (const key of keyGenerator) { const value = await Storage.getItem(key); @@ -110,6 +147,11 @@ export class StorageSet { } })() } + + /** + * Returns an array containing all entries. + * This can be used to iterate over the entries without using a (for await of) loop. + */ async entriesArray() { const entries = []; for await (const entry of this.entries()) entries.push(entry); @@ -117,7 +159,7 @@ export class StorageSet { } async *[Symbol.asyncIterator]() { - const keyGenerator = await Storage.getItemKeysStartingWith(this.prefix); + const keyGenerator = await Storage.getItemKeysStartingWith(this._prefix); for (const key of keyGenerator) { const value = await Storage.getItem(key); diff --git a/types/type.ts b/types/type.ts index 72ea2c0f..fd1834a4 100644 --- a/types/type.ts +++ b/types/type.ts @@ -20,7 +20,7 @@ import type { Task } from "./task.ts"; import { Assertion } from "./assertion.ts"; import type { Iterator } from "./iterator.ts"; import {StorageMap, StorageWeakMap} from "./storage_map.ts" -import {StorageSet} from "./storage_set.ts" +import {StorageSet, StorageWeakSet} from "./storage_set.ts" import { ExtensibleFunction } from "./function-utils.ts"; import type { JSTransferableFunction } from "./js-function.ts"; @@ -1012,6 +1012,7 @@ export class Type extends ExtensibleFunction { StorageMap: Type.get>("std:StorageMap"), StorageWeakMap: Type.get>("std:StorageWeakMap"), StorageSet: Type.get>("std:StorageSet"), + StorageWeakSet: Type.get>("std:StorageWeakSet"), Error: Type.get("std:Error"), SyntaxError: Type.get("std:SyntaxError"), @@ -1095,6 +1096,13 @@ Type.std.StorageMap.setJSInterface({ visible_children: new Set(), }) +Type.std.StorageWeakSet.setJSInterface({ + class: StorageWeakSet, + is_normal_object: true, + proxify_children: true, + visible_children: new Set(), +}) + Type.std.StorageSet.setJSInterface({ class: StorageSet, is_normal_object: true, From 51db55159fb9c22ec72345a05feacf72ca24ca6d Mon Sep 17 00:00:00 2001 From: benStre Date: Sun, 21 Jan 2024 02:47:16 +0100 Subject: [PATCH 13/16] fix: only allow http(s) type modules per default (unrelated) --- compiler/compiler.ts | 3 +-- types/type.ts | 6 +++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/compiler/compiler.ts b/compiler/compiler.ts index 3d0ed886..72262c0c 100644 --- a/compiler/compiler.ts +++ b/compiler/compiler.ts @@ -2017,8 +2017,7 @@ export class Compiler { SCOPE.addJSTypeDefs = receiver != Runtime.endpoint && receiver != LOCAL_ENDPOINT; } - // add js type module only if http(s) url - const addTypeDefs = SCOPE.addJSTypeDefs && jsTypeDefModule && (jsTypeDefModule.toString().startsWith("http://") || jsTypeDefModule.toString().startsWith("https://")); + const addTypeDefs = SCOPE.addJSTypeDefs && jsTypeDefModule; if (addTypeDefs) { Compiler.builder.handleRequiredBufferSize(SCOPE.b_index+4, SCOPE); diff --git a/types/type.ts b/types/type.ts index fd1834a4..2afe772a 100644 --- a/types/type.ts +++ b/types/type.ts @@ -59,8 +59,12 @@ export class Type extends ExtensibleFunction { get jsTypeDefModule():string|URL|undefined {return this.#jsTypeDefModule} set jsTypeDefModule(url: string|URL) { + // custom module mapper if (Type.#jsTypeDefModuleMapper) this.#jsTypeDefModule = Type.#jsTypeDefModuleMapper(url, this); - else this.#jsTypeDefModule = url; + // default: only allow http/https modules + else if (url.toString().startsWith("http://") || url.toString().startsWith("https://")) { + this.#jsTypeDefModule = url; + } } root_type: Type; // DatexType without parameters and variation From daf6d25d5335c1e4a0d88cc67e34068f0cdbb210 Mon Sep 17 00:00:00 2001 From: benStre Date: Sun, 21 Jan 2024 02:55:00 +0100 Subject: [PATCH 14/16] fix: only allow http(s) type modules per default (unrelated) --- types/type.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/types/type.ts b/types/type.ts index 2afe772a..a6abb3a1 100644 --- a/types/type.ts +++ b/types/type.ts @@ -56,6 +56,7 @@ export class Type extends ExtensibleFunction { parameters:any[] // special type parameters #jsTypeDefModule?: string|URL // URL for the JS module that creates the corresponding type definition + #potentialJsTypeDefModule?: string|URL // remember jsTypeDefModule if jsTypeDefModuleMapper is added later get jsTypeDefModule():string|URL|undefined {return this.#jsTypeDefModule} set jsTypeDefModule(url: string|URL) { @@ -65,6 +66,7 @@ export class Type extends ExtensibleFunction { else if (url.toString().startsWith("http://") || url.toString().startsWith("https://")) { this.#jsTypeDefModule = url; } + this.#potentialJsTypeDefModule = url; } root_type: Type; // DatexType without parameters and variation @@ -88,7 +90,7 @@ export class Type extends ExtensibleFunction { this.#jsTypeDefModuleMapper = fn; // update existing typedef modules for (const type of this.types.values()) { - if (type.#jsTypeDefModule) type.jsTypeDefModule = type.#jsTypeDefModule; + if (type.#potentialJsTypeDefModule) type.jsTypeDefModule = type.#potentialJsTypeDefModule; } } From 7cae613e3f2d6e50bf9a6abb5351c598807d8b3d Mon Sep 17 00:00:00 2001 From: benStre Date: Sun, 21 Jan 2024 11:58:48 +0100 Subject: [PATCH 15/16] add docs unique pointers section --- docs/manual/04 Pointer Synchronisation.md | 36 +++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/docs/manual/04 Pointer Synchronisation.md b/docs/manual/04 Pointer Synchronisation.md index 90d34cdb..a0dbe07f 100644 --- a/docs/manual/04 Pointer Synchronisation.md +++ b/docs/manual/04 Pointer Synchronisation.md @@ -167,3 +167,39 @@ const x2 = await $.ABCDEF assert (x1 === x2) ``` +The identity of a pointer is always preserved. Even if you receive the pointer +from a remote endpoint call, it is still identical to the local instance: + +```ts +// remote endpoint + +type User = {name: string, age: number} + +const happyBirthday = $$( + function (user: User) { + user.age++; + return user; + } +) +``` + +```ts +// local endpoint + +const user = $$({ + name: "Luke", + age: 20 +}) + +const happyBirthday = await $.DEFABCDEF // <- pointer id for the remote "happyBirthday" function +const olderUser = await happyBirthday(user); + +user === olderUser // true +olderUser.age === 21 // true +user.age === 21 // true +``` + +Like we would expect if this was a normal, local JavaScript function call, the returned +`user` object is identical to the object we passed to the function. +This is not only true for this simple example, but also for more complex scenarios. +For example, reference identities are also preserved within [eternal values](./05%20Eternal%20Pointers.md). \ No newline at end of file From 2cf7e12bf1c79f2f589fa23821343b39450122f2 Mon Sep 17 00:00:00 2001 From: benStre Date: Tue, 23 Jan 2024 19:34:17 +0100 Subject: [PATCH 16/16] remove unnecesary await --- types/storage_map.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/storage_map.ts b/types/storage_map.ts index 464b05fd..65bd9b22 100644 --- a/types/storage_map.ts +++ b/types/storage_map.ts @@ -87,7 +87,7 @@ export class StorageWeakMap { async clear() { const promises = []; for (const key of await Storage.getItemKeysStartingWith(this._prefix)) { - promises.push(await Storage.removeItem(key)); + promises.push(Storage.removeItem(key)); } await Promise.all(promises); }