From 13a69b04d003f6b977b181b0f7d159681ab99427 Mon Sep 17 00:00:00 2001 From: benStre Date: Sat, 2 Dec 2023 16:26:41 +0100 Subject: [PATCH] fix js:Function, add support for js motype module loading, LazyPointer --- compiler/compiler.ts | 58 +++++++++++++++++++++++++--------- js_adapter/js_class_adapter.ts | 13 +++++++- runtime/lazy-pointer.ts | 18 +++++++++++ runtime/pointers.ts | 36 +++++++++++++++++++-- runtime/runtime.ts | 7 ++-- types/function-utils.ts | 2 +- types/js-function.ts | 23 ++++++++++++-- types/type.ts | 18 +++++++++++ utils/persistent-listeners.ts | 2 +- 9 files changed, 152 insertions(+), 25 deletions(-) create mode 100644 runtime/lazy-pointer.ts diff --git a/compiler/compiler.ts b/compiler/compiler.ts index 91cc43c5..0f0381a8 100644 --- a/compiler/compiler.ts +++ b/compiler/compiler.ts @@ -14,7 +14,7 @@ import { Logger } from "../utils/logger.ts"; const logger = new Logger("datex compiler"); import { ReadableStream, Runtime, StaticScope} from "../runtime/runtime.ts"; -import { Endpoint, IdEndpoint, Target, WildcardTarget, Institution, Person, BROADCAST, target_clause, endpoints } from "../types/addressing.ts"; +import { Endpoint, IdEndpoint, Target, WildcardTarget, Institution, Person, BROADCAST, target_clause, endpoints, LOCAL_ENDPOINT } from "../types/addressing.ts"; import { Pointer, PointerProperty, Ref } from "../runtime/pointers.ts"; import { CompilerError, RuntimeError, Error as DatexError, ValueError } from "../types/errors.ts"; import { Function as DatexFunction } from "../types/function.ts"; @@ -163,6 +163,8 @@ export type compiler_scope = { used_lbls: string[], // already used lbls + addJSTypeDefs?: boolean, // should add url() imports for types to load via JS modules + last_cache_point?: number, // index of last cache point (at LBL) add_header: boolean, @@ -850,7 +852,7 @@ export class Compiler { const block_size = pre_header.byteLength + header_and_body.byteLength; if (block_size > this.MAX_DXB_BLOCK_SIZE) { pre_header_data_view.setUint16(3, 0, true); - logger.warn("DXB block size exceeds maximum size of " + this.MAX_DXB_BLOCK_SIZE + " bytes") + logger.debug("DXB block size exceeds maximum size of " + this.MAX_DXB_BLOCK_SIZE + " bytes") } else pre_header_data_view.setUint16(3, block_size, true); @@ -1968,9 +1970,27 @@ export class Compiler { }, - addTypeByNamespaceAndName: (SCOPE:compiler_scope, namespace:string, name:string, variation?:string, parameters?:any[]|true) => { + addTypeByNamespaceAndName: (SCOPE:compiler_scope, namespace:string, name:string, variation?:string, parameters?:any[]|true, jsTypeDefModule?:string|URL) => { Compiler.builder.handleRequiredBufferSize(SCOPE.b_index, SCOPE); + // remember if js type def modules should be added to this scope + if (SCOPE.addJSTypeDefs == undefined) { + let receiver = Compiler.builder.getScopeReceiver(SCOPE); + SCOPE.addJSTypeDefs = !!jsTypeDefModule && receiver != Runtime.endpoint && receiver != LOCAL_ENDPOINT; + } + + if (SCOPE.addJSTypeDefs) { + Compiler.builder.handleRequiredBufferSize(SCOPE.b_index+4, SCOPE); + SCOPE.uint8[SCOPE.b_index++] = BinaryCode.SUBSCOPE_START; + SCOPE.uint8[SCOPE.b_index++] = BinaryCode.GET; + if (jsTypeDefModule instanceof URL) Compiler.builder.addUrl(jsTypeDefModule.toString(), SCOPE); + else { + SCOPE.uint8[SCOPE.b_index++] = BinaryCode.STD_TYPE_URL; + Compiler.builder.addText(jsTypeDefModule.toString(), SCOPE); + } + SCOPE.uint8[SCOPE.b_index++] = BinaryCode.CLOSE_AND_STORE; + } + Compiler.builder.valueIndex(SCOPE); const is_extended_type = !!(variation || parameters); @@ -2006,7 +2026,7 @@ export class Compiler { const ns_bin = Compiler.utf8_encoder.encode(namespace); // convert type namespace to binary const variation_bin = variation ? Compiler.utf8_encoder.encode(variation) : undefined; // convert type namespace to binary - Compiler.builder.handleRequiredBufferSize(SCOPE.b_index+name_bin.byteLength+ns_bin.byteLength+3+(variation_bin ? variation_bin.byteLength : 0)+(is_extended_type?2:0), SCOPE); + Compiler.builder.handleRequiredBufferSize(SCOPE.b_index+name_bin.byteLength+ns_bin.byteLength+4+(variation_bin ? variation_bin.byteLength : 0)+(is_extended_type?2:0), SCOPE); SCOPE.uint8[SCOPE.b_index++] = is_extended_type ? BinaryCode.EXTENDED_TYPE : BinaryCode.TYPE; SCOPE.uint8[SCOPE.b_index++] = ns_bin.byteLength; SCOPE.uint8[SCOPE.b_index++] = name_bin.byteLength; @@ -2028,6 +2048,10 @@ export class Compiler { if (parameters instanceof Array) { Compiler.builder.addTuple(new Tuple(parameters), SCOPE); } + + if (SCOPE.addJSTypeDefs) { + SCOPE.uint8[SCOPE.b_index++] = BinaryCode.SUBSCOPE_END; + } }, @@ -2573,21 +2597,25 @@ export class Compiler { serializeValue: (v:any, SCOPE:compiler_scope):any => { if (SCOPE.serialized_values.has(v)) return SCOPE.serialized_values.get(v); else { - let receiver:target_clause = Runtime.endpoint; - let options:compiler_options | undefined = SCOPE.options; - while(options) { - if (options.to) { - receiver = options.to as target_clause; - break; - } - options = options.parent_scope?.options; - } + let receiver = Compiler.builder.getScopeReceiver(SCOPE); const s = Runtime.serializeValue(v, receiver); SCOPE.serialized_values.set(v,s); return s; } }, + getScopeReceiver: (SCOPE: compiler_scope) => { + let receiver:target_clause = Runtime.endpoint; + let options:compiler_options | undefined = SCOPE.options; + while(options) { + if (options.to) { + receiver = options.to as target_clause; + break; + } + options = options.parent_scope?.options; + } + return receiver; + }, // // insert Maybe // insertMaybe: async (maybe:Maybe, SCOPE:compiler_scope) => { @@ -2715,7 +2743,7 @@ export class Compiler { // convert to + serialized object ; also always for type variations // exception for explicit type quantity, type variation is always included in primitive representation without explicit cast if (type?.is_complex || type.root_type !== type && !Type.std.quantity.matchesType(type)) { - Compiler.builder.addTypeByNamespaceAndName(SCOPE, type.namespace, type.name, type.variation, type.parameters); + Compiler.builder.addTypeByNamespaceAndName(SCOPE, type.namespace, type.name, type.variation, type.parameters, type.jsTypeDefModule); value = Compiler.builder.serializeValue(value, SCOPE); } else if (type?.serializable_not_complex) { // for UintArray Buffers @@ -2832,7 +2860,7 @@ export class Compiler { else if (value instanceof WildcardTarget) Compiler.builder.addTarget(value.target, SCOPE); // Filter Target: ORG, APP, LABEL, ALIAS else if (value instanceof Endpoint) Compiler.builder.addTarget(value, SCOPE); // Filter Target: ORG, APP, LABEL, ALIAS else if (value instanceof Type) { - Compiler.builder.addTypeByNamespaceAndName(SCOPE, value.namespace, value.name, value.variation, value.parameters); // Type + Compiler.builder.addTypeByNamespaceAndName(SCOPE, value.namespace, value.name, value.variation, value.parameters, value.jsTypeDefModule); // Type } else if (value instanceof Uint8Array) Compiler.builder.addBuffer(value, SCOPE); // Uint8Array else if (value instanceof ArrayBuffer) Compiler.builder.addBuffer(new Uint8Array(value), SCOPE); // Buffer diff --git a/js_adapter/js_class_adapter.ts b/js_adapter/js_class_adapter.ts index 931ce4aa..9d59931d 100644 --- a/js_adapter/js_class_adapter.ts +++ b/js_adapter/js_class_adapter.ts @@ -28,6 +28,7 @@ import { type Class } from "../utils/global_types.ts"; import { Conjunction, Disjunction, Logical } from "../types/logic.ts"; import { client_type } from "../utils/constants.ts"; import { Assertion } from "../types/assertion.ts"; +import { getCallerInfo } from "../utils/caller_metadata.ts"; const { Reflect: MetadataReflect } = client_type == 'deno' ? await import("https://deno.land/x/reflect_metadata@v0.1.12/mod.ts") : {Reflect}; @@ -417,7 +418,17 @@ export class Decorators { else if (params[0] instanceof Type) type = params[0]; else if (original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor) type = original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor else type = Type.get("ext", original_class.name); - + + if (client_type == "deno" && type.namespace !== "std") { + const callerFile = getCallerInfo()?.[2]?.file; + if (!callerFile) { + logger.error("Could not determine JS module URL for type '" + type + "'") + } + else { + type.jsTypeDefModule = callerFile; + } + } + // return new templated class return createTemplateClass(original_class, type); } diff --git a/runtime/lazy-pointer.ts b/runtime/lazy-pointer.ts new file mode 100644 index 00000000..e0881673 --- /dev/null +++ b/runtime/lazy-pointer.ts @@ -0,0 +1,18 @@ +import { Pointer, MinimalJSRef } from "./pointers.ts"; + +export class LazyPointer { + constructor(public id: string) {} + + toString() { + return "Unresolved Pointer ($" + this.id + ")" + } + + onLoad(callback:(val:MinimalJSRef)=>void) { + Pointer.onPointerForIdAdded(this.id, p => callback(Pointer.collapseValue(p) as MinimalJSRef)) + } + + static withVal(val:any, callback:(val:MinimalJSRef)=>void) { + if (val instanceof LazyPointer) val.onLoad(callback); + else callback(val); + } +} \ No newline at end of file diff --git a/runtime/pointers.ts b/runtime/pointers.ts index 9c7f349a..19dd862f 100644 --- a/runtime/pointers.ts +++ b/runtime/pointers.ts @@ -27,6 +27,7 @@ import { sha256 } from "../utils/sha256.ts"; import { AutoMap } from "../utils/auto_map.ts"; import { IterableWeakSet } from "../utils/iterable-weak-set.ts"; import { IterableWeakMap } from "../utils/iterable-weak-map.ts"; +import { LazyPointer } from "./lazy-pointer.ts"; export type observe_handler = (value:V extends RefLike ? T : V, key?:K, type?:Ref.UPDATE_TYPE, transform?:boolean, is_child_update?:boolean)=>void|boolean export type observe_options = {types?:Ref.UPDATE_TYPE[], ignore_transforms?:boolean, recursive?:boolean} @@ -983,6 +984,7 @@ export class Pointer extends Ref { private static pointer_property_delete_listeners = new Set<(p:Pointer, key:any)=>void>(); private static pointer_value_change_listeners =new Set<(p:Pointer)=>void>(); private static pointer_for_value_created_listeners = new WeakMapvoid)>)>(); + private static pointer_for_id_created_listeners = new Mapvoid)>)>(); public static onPointerAdded(listener: (p:Pointer)=>void) { this.pointer_add_listeners.add(listener); @@ -1003,6 +1005,23 @@ export class Pointer extends Ref { this.pointer_value_change_listeners.add(listener); } + /** + * Callback when pointer for a given id was added + * @param id + * @param listener + * @returns + */ + public static onPointerForIdAdded(id:string, listener: (p:Pointer)=>void) { + const ptr = Pointer.get(id); + if (ptr && ptr.value_initialized) { + listener(ptr); + return; + } + // set listener + if (!this.pointer_for_id_created_listeners.has(id)) this.pointer_for_id_created_listeners.set(id, new Set()); + this.pointer_for_id_created_listeners.get(id)?.add(listener); + } + public static onPointerForValueCreated(value:any, listener: (p:Pointer)=>void, trigger_if_exists = true){ // value already has a pointer? if (trigger_if_exists) { @@ -1223,9 +1242,8 @@ export class Pointer extends Ref { if (SCOPE) { // recursive pointer loading! TODO if (this.loading_pointers.get(id_string)?.scopeList.has(SCOPE)) { - logger.error("recursive pointer loading: $"+ id_string); - return "placeholder" - // throw new PointerError("recursive pointer loading: $"+ id_string); + // logger.debug("recursive pointer loading: $"+ id_string); + return new LazyPointer(id_string) } } @@ -2259,6 +2277,12 @@ export class Pointer extends Ref { // pointer for value listeners? if (Pointer.pointer_for_value_created_listeners.has(val)) { for (const l of Pointer.pointer_for_value_created_listeners.get(val)!) l(this); + Pointer.pointer_for_value_created_listeners.delete(val) + } + // pointer for id listeners + if (Pointer.pointer_for_id_created_listeners.has(this.id)) { + for (const l of Pointer.pointer_for_id_created_listeners.get(this.id)!) l(this); + Pointer.pointer_for_id_created_listeners.delete(this.id) } // seal original value @@ -2282,6 +2306,12 @@ export class Pointer extends Ref { Pointer.primitive_pointers.set(this.#id, new WeakRef(this)); Pointer.pointers.delete(this.#id); + // pointer for id listeners + if (Pointer.pointer_for_id_created_listeners.has(this.id)) { + for (const l of Pointer.pointer_for_id_created_listeners.get(this.id)!) l(this); + Pointer.pointer_for_id_created_listeners.delete(this.id) + } + // adds garbage collection listener this.updateGarbageCollection(); } diff --git a/runtime/runtime.ts b/runtime/runtime.ts index ca88fbd1..a2a97d3b 100644 --- a/runtime/runtime.ts +++ b/runtime/runtime.ts @@ -569,6 +569,9 @@ export class Runtime { public static async getURLContent(url_string:string, raw?:RAW, cached?:boolean):Promise public static async getURLContent(url:URL, raw?:RAW, cached?:boolean):Promise public static async getURLContent(url_string:string|URL, raw:RAW=false, cached = false):Promise { + + if (url_string.toString().startsWith("route:") && window.location?.origin) url_string = new URL(url_string.toString().replace("route:", ""), window.location.origin) + const url = url_string instanceof URL ? url_string : new URL(url_string, baseURL); url_string = url.toString(); @@ -2434,7 +2437,7 @@ export class Runtime { else { // cannot fetch type, just cast default - logger.warn("Cannot further resolve unknown type '"+type.toString()+"'"); + logger.warn("Cannot find a type definition for "+type.toString()+". Make sure the module for this type is imported. If this type is no longer used, try to clear your eternal caches."); new_value = type.cast(old_value, context, origin, false, false, assigningPtrId); } @@ -4459,7 +4462,7 @@ export class Runtime { if (!SCOPE.sender.equals(to)) throw new PointerError("Sender has no permission to stop sync pointer to another origin", SCOPE); } - logger.success(SCOPE.sender + " unsubscribed from " + pointer.idString()); + logger.debug(SCOPE.sender + " unsubscribed from " + pointer.idString()); // not existing pointer or no access to this pointer if (!pointer.value_initialized) throw new PointerError("Pointer does not exist", SCOPE) diff --git a/types/function-utils.ts b/types/function-utils.ts index 5e94fb70..dfaedfd5 100644 --- a/types/function-utils.ts +++ b/types/function-utils.ts @@ -157,7 +157,7 @@ export function createFunctionWithDependencyInjections(source: string, dependenc try { let creatorFn = new Function(...renamedVars, `"use strict";${varMapping?createStaticFn:''}${varMapping}; return (${source})`) if (hasThis) creatorFn = creatorFn.bind(dependencies['this']) - return creatorFn(...Object.values(dependencies)); + return creatorFn(...Object.entries(dependencies).filter(([d]) => d!=='this').map(([_,v]) => v)); } catch (e) { console.error(source) diff --git a/types/js-function.ts b/types/js-function.ts index 19a022e3..ba02cf56 100644 --- a/types/js-function.ts +++ b/types/js-function.ts @@ -2,6 +2,7 @@ * Represents a JS function with source code that can be transferred between endpoints */ +import { LazyPointer } from "../runtime/lazy-pointer.ts"; import { Pointer } from "../runtime/pointers.ts"; import { Runtime } from "../runtime/runtime.ts"; import { ExtensibleFunction, getDeclaredExternalVariables, getDeclaredExternalVariablesAsync, createFunctionWithDependencyInjections, getSourceWithoutUsingDeclaration, Callable } from "./function-utils.ts"; @@ -24,22 +25,40 @@ export class JSTransferableFunction extends ExtensibleFunction { } else { let ptr: Pointer|undefined; - const fn = (...args:unknown[]) => { + const fn = (...args:any[]) => { if (!ptr) ptr = Pointer.getByValue(this); if (!ptr) throw new Error("Cannot execute js:Function, must be bound to a pointer"); const origin = ptr.origin.main; if (origin !== Runtime.endpoint && !Runtime.trustedEndpoints.get(origin)?.includes("remote-js-execution")) { throw new Error("Cannot execute js:Function, origin "+origin+" has no permission to execute js source code on this endpoint"); } - return intermediateFn(...args) + this.assertLazyDependenciesResolved(); + if (this.deps.this) return intermediateFn.apply(this.deps.this, args) + else return intermediateFn(...args) } super(fn); + this.resolveLazyDependencies(); // make sure LazyPointer deps are resolved this.#fn = fn; } this.source = source; } + private resolveLazyDependencies() { + for (const [key, value] of Object.entries(this.deps)) { + if (value instanceof LazyPointer) value.onLoad((v) => this.deps[key] = v); + } + } + + #resolved = false; + private assertLazyDependenciesResolved() { + if (this.#resolved) return true; + for (const [key, value] of Object.entries(this.deps)) { + if (value instanceof LazyPointer) throw new Error("Cannot call , dependency variable '"+key+"' is not yet initialized") + } + this.#resolved = true; + } + call(...args:any[]) { return this.#fn(...args) } diff --git a/types/type.ts b/types/type.ts index 4e077aeb..4a304fed 100644 --- a/types/type.ts +++ b/types/type.ts @@ -55,6 +55,14 @@ export class Type extends ExtensibleFunction { variation:string = '' parameters:any[] // special type parameters + #jsTypeDefModule?: string|URL // URL for the JS module that creates the corresponding type definition + + get jsTypeDefModule():string|URL|undefined {return this.#jsTypeDefModule} + set jsTypeDefModule(url: string|URL) { + if (Type.#jsTypeDefModuleMapper) this.#jsTypeDefModule = Type.#jsTypeDefModuleMapper(url); + else this.#jsTypeDefModule = url; + } + root_type: Type; // DatexType without parameters and variation base_type: Type; // DatexType without parameters @@ -69,6 +77,16 @@ export class Type extends ExtensibleFunction { // TODO: make true per default? currently results in stack overflows for some std types #proxify_children = false // proxify all (new) children of this type children_timeouts?: Map // individual timeouts for children + + static #jsTypeDefModuleMapper?: (url:string|URL) => string|URL + + static setJSTypeDefModuleMapper(fn: (url:string|URL) => string|URL) { + this.#jsTypeDefModuleMapper = fn; + // update existing typedef modules + for (const type of this.types.values()) { + if (type.#jsTypeDefModule) type.jsTypeDefModule = type.#jsTypeDefModule; + } + } /** * Should proxify all children with proxify_as_child=true diff --git a/utils/persistent-listeners.ts b/utils/persistent-listeners.ts index c946c416..a1d63e08 100644 --- a/utils/persistent-listeners.ts +++ b/utils/persistent-listeners.ts @@ -31,7 +31,7 @@ export function removePersistentListener(target: EventTarget, event: string, han export function recreatePersistentListeners() { for (const [target, {event, handler, options}] of listeners) { - console.debug("recreated a persistent event listener for '" + event + "'") + // console.debug("recreated a persistent event listener for '" + event + "'") originalAddEventListener.call(target, event, handler, options) } }