diff --git a/compiler/compiler.ts b/compiler/compiler.ts index 4ff3aca7..7f5a27f8 100644 --- a/compiler/compiler.ts +++ b/compiler/compiler.ts @@ -27,7 +27,7 @@ import { BinaryCode } from "./binary_codes.ts"; import { Scope } from "../types/scope.ts"; import { ProtocolDataType } from "./protocol_types.ts"; import { Quantity } from "../types/quantity.ts"; -import { EXTENDED_OBJECTS, INHERITED_PROPERTIES, VOID, SLOT_WRITE, SLOT_READ, SLOT_EXEC, NOT_EXISTING, SLOT_GET, SLOT_SET, DX_IGNORE, DX_BOUND_LOCAL_SLOT } from "../runtime/constants.ts"; +import { EXTENDED_OBJECTS, INHERITED_PROPERTIES, VOID, SLOT_WRITE, SLOT_READ, SLOT_EXEC, NOT_EXISTING, SLOT_GET, SLOT_SET, DX_IGNORE, DX_BOUND_LOCAL_SLOT, DX_REPLACE } from "../runtime/constants.ts"; import { arrayBufferToBase64, base64ToArrayBuffer, buffer2hex, hex2buffer } from "../utils/utils.ts"; import { RuntimePerformance } from "../runtime/performance_measure.ts"; import { Conjunction, Disjunction, Logical, Negation } from "../types/logic.ts"; @@ -2278,7 +2278,7 @@ export class Compiler { } // subscribing to own pointers is not allowed // should not happen because pointers are not sent with preemptive loading to own endpoint anyway - if (Runtime.endpoint.equals(endpoint)) continue; + if (Runtime.endpoint.equals(endpoint) || endpoint.equals(LOCAL_ENDPOINT)) continue; logger.debug("auto subscribing " + endpoint + " to " + ptr.idString()); ptr.addSubscriber(endpoint); } @@ -2744,6 +2744,8 @@ export class Compiler { insert: (value:any, SCOPE:compiler_scope, is_root=true, parents?:Set, unassigned_children?:[number, any, number][], add_insert_index = true) => { + if (value?.[DX_REPLACE]) value = value[DX_REPLACE]; + // make sure normal pointers are collapsed (ignore error if uninitialized pointer is passed in) try { value = Ref.collapseValue(value); diff --git a/js_adapter/js_class_adapter.ts b/js_adapter/js_class_adapter.ts index 8a0e8a30..9121b186 100644 --- a/js_adapter/js_class_adapter.ts +++ b/js_adapter/js_class_adapter.ts @@ -23,7 +23,7 @@ import { Error as DatexError, ValueError } from "../types/errors.ts"; import { Function as DatexFunction } from "../types/function.ts"; import { DatexObject } from "../types/object.ts"; import { Tuple } from "../types/tuple.ts"; -import { DX_PERMISSIONS, DX_TYPE, INIT_PROPS } from "../runtime/constants.ts"; +import { DX_PERMISSIONS, DX_TYPE, DX_ROOT, INIT_PROPS } from "../runtime/constants.ts"; import { type Class } from "../utils/global_types.ts"; import { Conjunction, Disjunction, Logical } from "../types/logic.ts"; import { client_type } from "../utils/constants.ts"; @@ -1087,6 +1087,8 @@ export function createTemplateClass(original_class:{ new(...args: any[]): any; } DatexObject.extend(template, prototype[DX_TYPE].template); break; } + // is root of dx prototype chain, stop + if (prototype[DX_ROOT]) break; } diff --git a/runtime/constants.ts b/runtime/constants.ts index ff49c3be..14849425 100644 --- a/runtime/constants.ts +++ b/runtime/constants.ts @@ -11,8 +11,10 @@ export const UNKNOWN_TYPE: unique symbol = Symbol("Unknown type") // return for export const DX_PTR: unique symbol = Symbol("DX_PTR"); // key for pointer objects to access the respective DatexPointer export const DX_TYPE: unique symbol = Symbol("DX_TYPE"); +export const DX_ROOT: unique symbol = Symbol("DX_ROOT"); export const DX_SERIALIZED: unique symbol = Symbol("DX_SERIALIZED"); export const DX_VALUE: unique symbol = Symbol("DX_VALUE"); +export const DX_REPLACE: unique symbol = Symbol("DX_REPLACE"); // value that is used as a replacement when serializing export const DX_SOURCE: unique symbol = Symbol("DX_SOURCE"); // used to override the default loading behaviour for a pointer (fetching by id). Can be an arbitrary DATEX Script that can be resolved with datex.get. Currently only used by the interface generator for JS modules. // TODO: remove? replaced with DX_SLOTS export const DX_TEMPLATE: unique symbol = Symbol("DX_TEMPLATE"); diff --git a/runtime/lazy-pointer.ts b/runtime/lazy-pointer.ts index e0881673..6bc4a55e 100644 --- a/runtime/lazy-pointer.ts +++ b/runtime/lazy-pointer.ts @@ -7,12 +7,12 @@ export class LazyPointer { return "Unresolved Pointer ($" + this.id + ")" } - onLoad(callback:(val:MinimalJSRef)=>void) { - Pointer.onPointerForIdAdded(this.id, p => callback(Pointer.collapseValue(p) as MinimalJSRef)) + onLoad(callback:(val:MinimalJSRef, ref: Pointer)=>void) { + Pointer.onPointerForIdAdded(this.id, p => callback(Pointer.collapseValue(p) as MinimalJSRef, p)) } - static withVal(val:any, callback:(val:MinimalJSRef)=>void) { + static withVal(val:any, callback:(val:MinimalJSRef)=>void) { if (val instanceof LazyPointer) val.onLoad(callback); - else callback(val); + else callback(val, val); } } \ No newline at end of file diff --git a/runtime/pointers.ts b/runtime/pointers.ts index 86c9a8b7..869c53c2 100644 --- a/runtime/pointers.ts +++ b/runtime/pointers.ts @@ -118,6 +118,7 @@ export abstract class Ref extends EventTarget { // deno-lint-ignore no-this-alias let pointer:Pointer = this; + // double pointer property..TODO: improve, currently tries to collapse current value if (pointer instanceof PointerProperty) { pointer = Pointer.getByValue(pointer.val); @@ -554,11 +555,40 @@ export class PointerProperty extends Ref { #leak_js_properties: boolean - private constructor(public pointer: Pointer, public key: any, leak_js_properties = false) { + public pointer?: Pointer; + private lazy_pointer?: LazyPointer; + + private constructor(pointer?: Pointer|LazyPointer, public key: any, leak_js_properties = false) { super(); + + if (pointer instanceof Pointer) this.setPointer(pointer); + else if (pointer instanceof LazyPointer) { + this.lazy_pointer = pointer; + this.lazy_pointer.onLoad((_, ptr) => { + this.lazy_pointer = undefined; + this.setPointer(ptr); + }) + } + else throw new Error("Pointer or lazy pointer required") + this.#leak_js_properties = leak_js_properties; - pointer.is_persistent = true; // TODO: make unpersistent when pointer property deleted - PointerProperty.synced_pairs.get(pointer)!.set(key, this); // save in map + } + + private setPointer(ptr: Pointer) { + this.pointer = ptr; + this.pointer.is_persistent = true; // TODO: make unpersistent when pointer property deleted + if (!PointerProperty.synced_pairs.has(ptr)) PointerProperty.synced_pairs.set(ptr, new Map()); + PointerProperty.synced_pairs.get(ptr)!.set(this.key, this); // save in map + } + + /** + * Called when the bound lazy pointer is loaded. + * If there is no lazy pointer, the callback is called immediately + * @param callback + */ + public onLoad(callback: (val:PointerProperty, ref: PointerProperty)=>void) { + if (this.lazy_pointer) this.lazy_pointer.onLoad(() => callback(this, this)); + else callback(this, this); } private static synced_pairs = new WeakMap>() @@ -571,26 +601,27 @@ export class PointerProperty extends Ref { * @param leak_js_properties * @returns */ - public static get>(parent: Parent|Pointer, key: Key, leak_js_properties = false): PointerProperty ? MV : Parent[Key&keyof Parent]> { - console.warn("getpp",parent,key) + public static get>(parent: Parent|Pointer|LazyPointer, key: Key, leak_js_properties = false): PointerProperty ? MV : Parent[Key&keyof Parent]> { if (Pointer.isRef(key)) throw new Error("Cannot use a reference as a pointer property key"); - let pointer:Pointer; - if (parent instanceof Pointer) pointer = parent; - else pointer = Pointer.createOrGet(parent); + const pointer = Pointer.createOrGetLazy(parent as any); - if (!this.synced_pairs.has(pointer)) this.synced_pairs.set(pointer, new Map()); + if (pointer instanceof Pointer) { + if (!this.synced_pairs.has(pointer)) this.synced_pairs.set(pointer, new Map()); + if (this.synced_pairs.get(pointer)!.has(key)) return this.synced_pairs.get(pointer)!.get(key)!; + } - if (this.synced_pairs.get(pointer)!.has(key)) return this.synced_pairs.get(pointer)!.get(key)!; - else return new PointerProperty(pointer, key, leak_js_properties); + return new PointerProperty(pointer, key, leak_js_properties); } // get current pointer property public override get val():T { // this.handleBeforePrimitiveValueGet(); - const val = this.pointer.getProperty(this.key, this.#leak_js_properties); - if (this.pointer instanceof LazyPointer) return "lazy..." - else if (val === NOT_EXISTING) { + if (this.lazy_pointer) return undefined + + const val = this.pointer!.getProperty(this.key, this.#leak_js_properties); + + if (val === NOT_EXISTING) { console.log(this) throw new Error(`Property ${this.key} does not exist in ${this.pointer}`); } @@ -598,16 +629,25 @@ export class PointerProperty extends Ref { } public override get current_val():T { - return this.pointer.getProperty(this.key, this.#leak_js_properties); + if (this.lazy_pointer) return undefined + return this.pointer!.getProperty(this.key, this.#leak_js_properties); } // update pointer property public override set val(value: T) { - this.pointer.handleSet(this.key, Ref.collapseValue(value, true, true)); + if (this.lazy_pointer) { + console.warn("Cannot set value of lazy pointer property"); + return; + } + this.pointer!.handleSet(this.key, Ref.collapseValue(value, true, true)); } // same as val setter, but can be awaited public override setVal(value: T) { - return this.pointer.handleSet(this.key, Ref.collapseValue(value, true, true)); + if (this.lazy_pointer) { + console.warn("Cannot set value of lazy pointer property"); + return; + } + return this.pointer!.handleSet(this.key, Ref.collapseValue(value, true, true)); } #observer_internal_handlers = new WeakMap() @@ -615,6 +655,10 @@ export class PointerProperty extends Ref { // callback on property value change and when the property value changes internally public override observe(handler: observe_handler, bound_object?:Record, options?:observe_options) { + if (this.lazy_pointer) { + console.warn("Cannot observe lazy pointer"); + return; + } const value_pointer = Pointer.pointerifyValue(this.current_val); if (value_pointer instanceof Ref) value_pointer.observe(handler, bound_object, options); // also observe internal value changes @@ -625,7 +669,7 @@ export class PointerProperty extends Ref { // if arrow function else handler(v,undefined,Ref.UPDATE_TYPE.INIT) }; - this.pointer.observe(internal_handler, bound_object, this.key, options) + this.pointer!.observe(internal_handler, bound_object, this.key, options) if (bound_object) { if (!this.#observer_internal_bound_handlers.has(bound_object)) this.#observer_internal_bound_handlers.set(bound_object, new WeakMap); @@ -635,6 +679,10 @@ export class PointerProperty extends Ref { } public override unobserve(handler: observe_handler, bound_object?:object) { + if (this.lazy_pointer) { + console.warn("Cannot unobserve lazy pointer"); + return; + } const value_pointer = Pointer.pointerifyValue(this.current_val); if (value_pointer instanceof Ref) value_pointer.unobserve(handler, bound_object); // also unobserve internal value changes @@ -649,7 +697,7 @@ export class PointerProperty extends Ref { this.#observer_internal_handlers.delete(handler); } - if (internal_handler) this.pointer.unobserve(internal_handler, bound_object, this.key); // get associated internal handler and unobserve + if (internal_handler) this.pointer!.unobserve(internal_handler, bound_object, this.key); // get associated internal handler and unobserve } } @@ -959,7 +1007,7 @@ export class UpdateScheduler { // Success }) .catch((e) => { - logger.error("forwarding failed", e); + console.error("forwarding failed", e); }); } } @@ -1199,16 +1247,28 @@ export class Pointer extends Ref { } static #is_local = true; + // all pointers with a @@local id, must be mapped to new endpoint ids when endpoint id is available static #local_pointers = new IterableWeakSet(); + // all pointers for which the is_origin property must be updated once the endpoint id is available + static #undetermined_pointers = new IterableWeakSet(); + + public static set is_local(local: boolean) { this.#is_local = local; // update pointer ids if no longer local if (!this.#is_local) { + // update local pointers for (const pointer of this.#local_pointers) { // still local? if (pointer.origin == LOCAL_ENDPOINT) pointer.id = Pointer.getUniquePointerID(pointer); } this.#local_pointers.clear(); + + // update undetermined pointers + for (const pointer of this.#undetermined_pointers) { + pointer.#updateIsOrigin() + } + this.#undetermined_pointers.clear(); } } public static get is_local() {return this.#is_local} @@ -1551,7 +1611,8 @@ export class Pointer extends Ref { } // create a new pointer or return the existing pointer/pointer property for this value - static createOrGet(value:RefOrValue, sealed = false, allowed_access?:target_clause, anonymous = false, persistant= false):Pointer{ + static createOrGet(value:RefOrValue, sealed = false, allowed_access?:target_clause, anonymous = false, persistant = false):Pointer{ + if (value instanceof LazyPointer) throw new PointerError("Lazy Pointer not supported in this context"); if (value instanceof Pointer) return >value; // return pointer by reference //if (value instanceof PointerProperty) return value; // return pointerproperty TODO: handle pointer properties? value = Ref.collapseValue(value, true, true); @@ -1567,6 +1628,12 @@ export class Pointer extends Ref { else return >Pointer.create(undefined, value, sealed, undefined, persistant, anonymous, false, allowed_access); } + // same as createOrGet, but also return lazy pointer if it exists + static createOrGetLazy(value:RefOrValue, sealed = false, allowed_access?:target_clause, anonymous = false, persistant = false):Pointer|LazyPointer{ + if (value instanceof LazyPointer) return value; + return this.createOrGet(value, sealed, allowed_access, anonymous, persistant); + } + // create a new pointer or return the existing pointer + add a label static createLabel(value:RefOrValue, label:string|number):Pointer{ let ptr = Pointer.getByValue(value); // try proxify @@ -1934,6 +2001,11 @@ export class Pointer extends Ref { get is_js_primitive(){return this.#is_js_primitive} // true if js primitive (number, boolean, ...) or 'single instance' class (Type, Endpoint) that cannot be directly addressed by reference get is_anonymous(){return this.#is_anonymous} get origin(){return this.#origin} + private set origin(origin:Endpoint){ + this.#origin = origin + this.#updateIsOrigin() + if (Runtime.endpoint == LOCAL_ENDPOINT) Pointer.#undetermined_pointers.add(this); + } get is_persistent() { return this.#is_persistent || this.subscribers?.size != 0} // change the persistant state of this pointer @@ -1977,9 +2049,9 @@ export class Pointer extends Ref { } } - private set origin(origin:Endpoint){ - this.#origin = origin - this.#is_origin = !!Runtime.endpoint?.equals(this.#origin); + + #updateIsOrigin() { + this.#is_origin = !!Runtime.endpoint.equals(this.#origin) || !!Runtime.endpoint.main.equals(this.#origin) || this.#origin.equals(LOCAL_ENDPOINT); } @@ -2120,7 +2192,7 @@ export class Pointer extends Ref { const endpoint = override_endpoint ?? this.origin; // early return, trying to subscribe to the own main endpoint, guaranteed to be routed back to self, which is not allowed - if (endpoint.equals(Runtime.endpoint.main)) { + if (endpoint.equals(Runtime.endpoint.main) || endpoint.equals(Runtime.endpoint) || endpoint.equals(LOCAL_ENDPOINT)) { logger.warn("tried to subscribe to own pointer: " + this.idString() + "(pointer origin: " + this.origin + ", own endpoint instance: " + Runtime.endpoint + ")"); return this; } @@ -3147,7 +3219,7 @@ export class Pointer extends Ref { // TODO also check pointer permission for 'to' // request sync endpoint is self, cannot subscribe to own pointers! - if (Runtime.endpoint.equals(subscriber)) { + if (Runtime.endpoint.equals(subscriber) || subscriber.equals(LOCAL_ENDPOINT)) { throw new PointerError("Cannot sync pointer with own origin"); } @@ -4059,7 +4131,7 @@ export class Pointer extends Ref { await Runtime.datexOut([datex, data, {collapse_first_inserted, type:ProtocolDataType.UPDATE, preemptive_pointer_init: true}], receiver, undefined, false, undefined, undefined, false, undefined, this.datex_timeout); } catch(e) { //throw e; - logger.error("forwarding failed", e, datex, data) + console.error("forwarding failed", e, datex, data) } } diff --git a/runtime/runtime.ts b/runtime/runtime.ts index 7cf2bc1b..906220ce 100644 --- a/runtime/runtime.ts +++ b/runtime/runtime.ts @@ -3472,10 +3472,9 @@ export class Runtime { // get parent[key] as DatexPointerProperty if possible getReferencedProperty(SCOPE: datex_scope, parent:any, key:any){ - const pointer = Pointer.createOrGet(parent); + const pointer = Pointer.createOrGetLazy(parent); if (pointer) { - pointer.assertEndpointCanRead(SCOPE?.sender) - + if (pointer instanceof Pointer) pointer.assertEndpointCanRead(SCOPE?.sender) return PointerProperty.get(pointer, key); } else throw new RuntimeError("Could not get a child reference"); diff --git a/runtime/storage.ts b/runtime/storage.ts index 0c465bd8..6e3a297e 100644 --- a/runtime/storage.ts +++ b/runtime/storage.ts @@ -1052,7 +1052,7 @@ export class Storage { // pointers string = ESCAPE_SEQUENCES.BOLD+"Pointers\n\n"+ESCAPE_SEQUENCES.RESET - string += `${ESCAPE_SEQUENCES.ITALIC}A list of all pointers stored in any storage location. Pointers are only stored as long as they are referenced somwhere else in the storage.\n\n${ESCAPE_SEQUENCES.RESET}` + string += `${ESCAPE_SEQUENCES.ITALIC}A list of all pointers stored in any storage location. Pointers are only stored as long as they are referenced somewhere else in the storage.\n\n${ESCAPE_SEQUENCES.RESET}` for (const [key, storageMap] of pointers.snapshot) { // check if stored in all locations, otherwise print in which location it is stored (functional programming) diff --git a/types/function-utils.ts b/types/function-utils.ts index 6b717aeb..3bad6f8b 100644 --- a/types/function-utils.ts +++ b/types/function-utils.ts @@ -135,10 +135,15 @@ const isArrowFunction = (fnSrc:string) => { return !!fnSrc.match(/^(async\s+)?\([^)]*\)\s*=>/) } -function resolveLazyDependencies(deps:Record) { +function resolveLazyDependencies(deps:Record, resolve?: ()=>void) { + let resolved = false; for (const [key, value] of Object.entries(deps)) { if (value instanceof LazyPointer) value.onLoad((v) => { - deps[key] = v + deps[key] = v; + if (!resolved && resolve && !hasUnresolvedLazyDependencies(deps)) { + resolved = true; + resolve(); + } }); } } @@ -150,6 +155,13 @@ function assertLazyDependenciesResolved(deps:Record) { } } +export function hasUnresolvedLazyDependencies(deps:Record) { + for (const value of Object.values(deps)) { + if (value instanceof LazyPointer) return true; + } + return false; +} + /** * Create a new function from JS source code with injected dependency variables * Also resolves LazyPointer dependencies @@ -157,9 +169,11 @@ function assertLazyDependenciesResolved(deps:Record) { * @param dependencies * @returns */ -export function createFunctionWithDependencyInjectionsResolveLazyPointers(source: string, dependencies: Record, allowValueMutations = true): ((...args:unknown[]) => unknown) { +export function createFunctionWithDependencyInjectionsResolveLazyPointers(source: string, dependencies: Record, allowValueMutations = true): {intermediateFn: ((...args:unknown[]) => unknown), lazyResolved: Promise} { let fn: Function|undefined; + const {promise: lazyResolved, resolve} = Promise.withResolvers() + const intermediateFn = (...args:any[]) => { if (!fn) { assertLazyDependenciesResolved(dependencies); @@ -167,8 +181,10 @@ export function createFunctionWithDependencyInjectionsResolveLazyPointers(source } return fn(...args) } - resolveLazyDependencies(dependencies) - return intermediateFn; + + if (!hasUnresolvedLazyDependencies(dependencies)) resolve() + else resolveLazyDependencies(dependencies, resolve); + return {intermediateFn, lazyResolved}; } /** diff --git a/types/js-function.ts b/types/js-function.ts index 8b66dea3..f9ced748 100644 --- a/types/js-function.ts +++ b/types/js-function.ts @@ -4,7 +4,7 @@ import { Pointer } from "../runtime/pointers.ts"; import { Runtime } from "../runtime/runtime.ts"; -import { ExtensibleFunction, getDeclaredExternalVariables, getDeclaredExternalVariablesAsync, getSourceWithoutUsingDeclaration, Callable, createFunctionWithDependencyInjectionsResolveLazyPointers } from "./function-utils.ts"; +import { ExtensibleFunction, getDeclaredExternalVariables, getDeclaredExternalVariablesAsync, getSourceWithoutUsingDeclaration, Callable, createFunctionWithDependencyInjectionsResolveLazyPointers, hasUnresolvedLazyDependencies } from "./function-utils.ts"; export type JSTransferableFunctionOptions = { @@ -16,8 +16,11 @@ export class JSTransferableFunction extends ExtensibleFunction { #fn: (...args:unknown[])=>unknown + // promise that resolves when all lazy dependencies are resolved + public lazyResolved: Promise + // deno-lint-ignore constructor-super - private constructor(intermediateFn: (...args:unknown[])=>unknown, public deps: Record, public source: string, public flags?: string[], options?: JSTransferableFunctionOptions) { + private constructor(intermediateFn: (...args:unknown[])=>unknown, lazyResolved: Promise, public deps: Record, public source: string, public flags?: string[], options?: JSTransferableFunctionOptions) { if (options?.errorOnOriginContext) { const invalidIntermediateFunction = () => {throw options.errorOnOriginContext}; super(invalidIntermediateFunction); @@ -39,7 +42,9 @@ export class JSTransferableFunction extends ExtensibleFunction { super(fn); this.#fn = fn; } - + + this.lazyResolved = lazyResolved; + this.source = source; } @@ -48,6 +53,16 @@ export class JSTransferableFunction extends ExtensibleFunction { return this.#fn(...args) } + // waits until all lazy dependencies are resolved and then calls the function + async callLazy() { + await this.lazyResolved; + return this.call() + } + + public get hasUnresolvedLazyDependencies() { + return hasUnresolvedLazyDependencies(this.deps) + } + /** * Returns JS source */ @@ -97,8 +112,8 @@ export class JSTransferableFunction extends ExtensibleFunction { } static #createTransferableFunction(source: string, dependencies: Record, flags?: string[], options?:JSTransferableFunctionOptions) { - const intermediateFn = createFunctionWithDependencyInjectionsResolveLazyPointers(source, dependencies, !options?.isLocal); - return new JSTransferableFunction(intermediateFn, dependencies, source, flags, options); + const {intermediateFn, lazyResolved} = createFunctionWithDependencyInjectionsResolveLazyPointers(source, dependencies, !options?.isLocal); + return new JSTransferableFunction(intermediateFn, lazyResolved, dependencies, source, flags, options); } } diff --git a/utils/interface-generator.ts b/utils/interface-generator.ts index f0d374d3..fee70b73 100644 --- a/utils/interface-generator.ts +++ b/utils/interface-generator.ts @@ -99,7 +99,7 @@ async function getModuleExports(path_or_specifier:URL|string, caller:string|unde } } catch (e) { - throw "error loading module:" + e?.message??e; + throw new Error("Could not load module '"+path_or_specifier+"' " + e?.message??e); } // network error, etc.., TODO: show warning somewhere