fix js:Function, add support for js motype module loading, LazyPointer

compiler/compiler.ts

@@ -14,7 +14,7 @@ import { Logger } from "../utils/logger.ts";
 const logger = new Logger("datex compiler");
 
 import { ReadableStream, Runtime, StaticScope} from "../runtime/runtime.ts";
-import { Endpoint, IdEndpoint, Target, WildcardTarget, Institution, Person, BROADCAST, target_clause, endpoints } from "../types/addressing.ts";
+import { Endpoint, IdEndpoint, Target, WildcardTarget, Institution, Person, BROADCAST, target_clause, endpoints, LOCAL_ENDPOINT } from "../types/addressing.ts";
 import { Pointer, PointerProperty, Ref } from "../runtime/pointers.ts";
 import { CompilerError, RuntimeError, Error as DatexError, ValueError } from "../types/errors.ts";
 import { Function as DatexFunction } from "../types/function.ts";
@@ -163,6 +163,8 @@ export type compiler_scope = {
 
     used_lbls: string[], // already used lbls
 
+    addJSTypeDefs?: boolean, // should add url() imports for types to load via JS modules
+
     last_cache_point?: number, // index of last cache point (at LBL)
 
     add_header: boolean,
@@ -850,7 +852,7 @@ export class Compiler {
         const block_size = pre_header.byteLength + header_and_body.byteLength;
         if (block_size > this.MAX_DXB_BLOCK_SIZE) {
             pre_header_data_view.setUint16(3, 0, true);
-            logger.warn("DXB block size exceeds maximum size of " + this.MAX_DXB_BLOCK_SIZE + " bytes")
+            logger.debug("DXB block size exceeds maximum size of " + this.MAX_DXB_BLOCK_SIZE + " bytes")
         }
         else pre_header_data_view.setUint16(3, block_size, true);
 
@@ -1968,9 +1970,27 @@ export class Compiler {
         },
 
 
-        addTypeByNamespaceAndName: (SCOPE:compiler_scope, namespace:string, name:string, variation?:string, parameters?:any[]|true) => {
+        addTypeByNamespaceAndName: (SCOPE:compiler_scope, namespace:string, name:string, variation?:string, parameters?:any[]|true, jsTypeDefModule?:string|URL) => {
             Compiler.builder.handleRequiredBufferSize(SCOPE.b_index, SCOPE);
 
+            // remember if js type def modules should be added to this scope
+            if (SCOPE.addJSTypeDefs == undefined) {
+                let receiver = Compiler.builder.getScopeReceiver(SCOPE);
+                SCOPE.addJSTypeDefs = !!jsTypeDefModule && receiver != Runtime.endpoint && receiver != LOCAL_ENDPOINT;
+            }
+
+            if (SCOPE.addJSTypeDefs) {
+                Compiler.builder.handleRequiredBufferSize(SCOPE.b_index+4, SCOPE);
+                SCOPE.uint8[SCOPE.b_index++] = BinaryCode.SUBSCOPE_START;
+                SCOPE.uint8[SCOPE.b_index++] = BinaryCode.GET;
+                if (jsTypeDefModule instanceof URL) Compiler.builder.addUrl(jsTypeDefModule.toString(), SCOPE);
+                else {
+                    SCOPE.uint8[SCOPE.b_index++] = BinaryCode.STD_TYPE_URL;
+                    Compiler.builder.addText(jsTypeDefModule.toString(), SCOPE);
+                }
+                SCOPE.uint8[SCOPE.b_index++] = BinaryCode.CLOSE_AND_STORE;
+            }
+
             Compiler.builder.valueIndex(SCOPE);
 
             const is_extended_type = !!(variation || parameters);
@@ -2006,7 +2026,7 @@ export class Compiler {
             const ns_bin = Compiler.utf8_encoder.encode(namespace);  // convert type namespace to binary
             const variation_bin = variation ? Compiler.utf8_encoder.encode(variation) : undefined;  // convert type namespace to binary
 
-            Compiler.builder.handleRequiredBufferSize(SCOPE.b_index+name_bin.byteLength+ns_bin.byteLength+3+(variation_bin ? variation_bin.byteLength : 0)+(is_extended_type?2:0), SCOPE);
+            Compiler.builder.handleRequiredBufferSize(SCOPE.b_index+name_bin.byteLength+ns_bin.byteLength+4+(variation_bin ? variation_bin.byteLength : 0)+(is_extended_type?2:0), SCOPE);
             SCOPE.uint8[SCOPE.b_index++] = is_extended_type ? BinaryCode.EXTENDED_TYPE : BinaryCode.TYPE;  
             SCOPE.uint8[SCOPE.b_index++] = ns_bin.byteLength;
             SCOPE.uint8[SCOPE.b_index++] = name_bin.byteLength;
@@ -2028,6 +2048,10 @@ export class Compiler {
             if (parameters instanceof Array) {
                 Compiler.builder.addTuple(new Tuple(parameters), SCOPE);
             }
+
+            if (SCOPE.addJSTypeDefs) {
+                SCOPE.uint8[SCOPE.b_index++] = BinaryCode.SUBSCOPE_END;
+            }
         },
 
 
@@ -2573,21 +2597,25 @@ export class Compiler {
         serializeValue: (v:any, SCOPE:compiler_scope):any => {
             if (SCOPE.serialized_values.has(v)) return SCOPE.serialized_values.get(v);
             else {
-                let receiver:target_clause = Runtime.endpoint;
-                let options:compiler_options | undefined = SCOPE.options;
-                while(options) {
-                    if (options.to) {
-                        receiver = options.to as target_clause;
-                        break;
-                    }
-                    options = options.parent_scope?.options;
-                }
+                let receiver = Compiler.builder.getScopeReceiver(SCOPE);
                 const s = Runtime.serializeValue(v, receiver);
                 SCOPE.serialized_values.set(v,s);
                 return s;
             }
         },
 
+        getScopeReceiver: (SCOPE: compiler_scope) => {
+            let receiver:target_clause = Runtime.endpoint;
+            let options:compiler_options | undefined = SCOPE.options;
+            while(options) {
+                if (options.to) {
+                    receiver = options.to as target_clause;
+                    break;
+                }
+                options = options.parent_scope?.options;
+            }
+            return receiver;
+        },
 
         // // insert Maybe
         // insertMaybe: async (maybe:Maybe, SCOPE:compiler_scope) => {
@@ -2715,7 +2743,7 @@ export class Compiler {
                 // convert to <type> + serialized object ; also always for type variations
                 // exception for explicit type quantity, type variation is always included in primitive representation without explicit cast
                 if (type?.is_complex || type.root_type !== type && !Type.std.quantity.matchesType(type)) {
-                    Compiler.builder.addTypeByNamespaceAndName(SCOPE, type.namespace, type.name, type.variation, type.parameters);
+                    Compiler.builder.addTypeByNamespaceAndName(SCOPE, type.namespace, type.name, type.variation, type.parameters, type.jsTypeDefModule);
                     value = Compiler.builder.serializeValue(value, SCOPE);
                 }
                 else if (type?.serializable_not_complex) { // for UintArray Buffers
@@ -2832,7 +2860,7 @@ export class Compiler {
             else if (value instanceof WildcardTarget)   Compiler.builder.addTarget(value.target, SCOPE); // Filter Target: ORG, APP, LABEL, ALIAS
             else if (value instanceof Endpoint)         Compiler.builder.addTarget(value, SCOPE); // Filter Target: ORG, APP, LABEL, ALIAS
             else if (value instanceof Type) {
-                Compiler.builder.addTypeByNamespaceAndName(SCOPE, value.namespace, value.name, value.variation, value.parameters); // Type
+                Compiler.builder.addTypeByNamespaceAndName(SCOPE, value.namespace, value.name, value.variation, value.parameters, value.jsTypeDefModule); // Type
             }
             else if (value instanceof Uint8Array)        Compiler.builder.addBuffer(value, SCOPE); // Uint8Array
             else if (value instanceof ArrayBuffer)       Compiler.builder.addBuffer(new Uint8Array(value), SCOPE); // Buffer

js_adapter/js_class_adapter.ts

@@ -28,6 +28,7 @@ import { type Class } from "../utils/global_types.ts";
 import { Conjunction, Disjunction, Logical } from "../types/logic.ts";
 import { client_type } from "../utils/constants.ts";
 import { Assertion } from "../types/assertion.ts";
+import { getCallerInfo } from "../utils/caller_metadata.ts";
 
 const { Reflect: MetadataReflect } = client_type == 'deno' ? await import("https://deno.land/x/reflect_metadata@v0.1.12/mod.ts") : {Reflect};
 
@@ -417,7 +418,17 @@ export class Decorators {
                 else if (params[0] instanceof Type) type = params[0];
                 else if (original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor) type = original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor
                 else type = Type.get("ext", original_class.name);
-
+
+                if (client_type == "deno" && type.namespace !== "std") {
+                    const callerFile = getCallerInfo()?.[2]?.file;
+                    if (!callerFile) {
+                        logger.error("Could not determine JS module URL for type '" + type + "'")
+                    }
+                    else {
+                        type.jsTypeDefModule = callerFile;
+                    }
+                }
+
                 // return new templated class
                 return createTemplateClass(original_class, type);
             }

runtime/lazy-pointer.ts

@@ -0,0 +1,18 @@
+import { Pointer, MinimalJSRef } from "./pointers.ts";
+
+export class LazyPointer<T> {
+	constructor(public id: string) {}
+
+	toString() {
+		return "Unresolved Pointer ($" + this.id + ")"
+	}
+
+	onLoad(callback:(val:MinimalJSRef<T>)=>void) {
+		Pointer.onPointerForIdAdded(this.id, p => callback(Pointer.collapseValue(p) as MinimalJSRef<T>))
+	}
+
+	static withVal(val:any, callback:(val:MinimalJSRef<any>)=>void) {
+		if (val instanceof LazyPointer) val.onLoad(callback);
+		else callback(val);
+	}
+}

runtime/pointers.ts

@@ -27,6 +27,7 @@ import { sha256 } from "../utils/sha256.ts";
 import { AutoMap } from "../utils/auto_map.ts";
 import { IterableWeakSet } from "../utils/iterable-weak-set.ts";
 import { IterableWeakMap } from "../utils/iterable-weak-map.ts";
+import { LazyPointer } from "./lazy-pointer.ts";
 
 export type observe_handler<K=any, V extends RefLike = any> = (value:V extends RefLike<infer T> ? T : V, key?:K, type?:Ref.UPDATE_TYPE, transform?:boolean, is_child_update?:boolean)=>void|boolean
 export type observe_options = {types?:Ref.UPDATE_TYPE[], ignore_transforms?:boolean, recursive?:boolean}
@@ -983,6 +984,7 @@ export class Pointer<T = any> extends Ref<T> {
     private static pointer_property_delete_listeners = new Set<(p:Pointer, key:any)=>void>();
     private static pointer_value_change_listeners =new Set<(p:Pointer)=>void>();
     private static pointer_for_value_created_listeners = new WeakMap<any, (Set<((p:Pointer)=>void)>)>();
+    private static pointer_for_id_created_listeners = new Map<string, (Set<((p:Pointer)=>void)>)>();
 
     public static onPointerAdded(listener: (p:Pointer)=>void) {
         this.pointer_add_listeners.add(listener);
@@ -1003,6 +1005,23 @@ export class Pointer<T = any> extends Ref<T> {
         this.pointer_value_change_listeners.add(listener);
     }
 
+    /**
+     * Callback when pointer for a given id was added
+     * @param id
+     * @param listener 
+     * @returns 
+     */
+    public static onPointerForIdAdded(id:string, listener: (p:Pointer)=>void) {
+        const ptr = Pointer.get(id);
+        if (ptr && ptr.value_initialized) {
+            listener(ptr);
+            return;
+        }
+        // set listener
+        if (!this.pointer_for_id_created_listeners.has(id)) this.pointer_for_id_created_listeners.set(id, new Set());
+        this.pointer_for_id_created_listeners.get(id)?.add(listener);
+    }
+
     public static onPointerForValueCreated(value:any, listener: (p:Pointer)=>void, trigger_if_exists = true){
         // value already has a pointer?
         if (trigger_if_exists) {
@@ -1223,9 +1242,8 @@ export class Pointer<T = any> extends Ref<T> {
         if (SCOPE) {
             // recursive pointer loading! TODO
             if (this.loading_pointers.get(id_string)?.scopeList.has(SCOPE)) {
-                logger.error("recursive pointer loading: $"+ id_string);
-                return "placeholder"
-                // throw new PointerError("recursive pointer loading: $"+ id_string);
+                // logger.debug("recursive pointer loading: $"+ id_string);
+                return new LazyPointer(id_string)
             }
         }
 
@@ -2259,6 +2277,12 @@ export class Pointer<T = any> extends Ref<T> {
                 // pointer for value listeners?
                 if (Pointer.pointer_for_value_created_listeners.has(val)) {
                     for (const l of Pointer.pointer_for_value_created_listeners.get(val)!) l(this);
+                    Pointer.pointer_for_value_created_listeners.delete(val)
+                }
+                // pointer for id listeners
+                if (Pointer.pointer_for_id_created_listeners.has(this.id)) {
+                    for (const l of Pointer.pointer_for_id_created_listeners.get(this.id)!) l(this);
+                    Pointer.pointer_for_id_created_listeners.delete(this.id)
                 }
 
                 // seal original value
@@ -2282,6 +2306,12 @@ export class Pointer<T = any> extends Ref<T> {
             Pointer.primitive_pointers.set(this.#id, new WeakRef(this)); 
             Pointer.pointers.delete(this.#id); 
 
+            // pointer for id listeners
+            if (Pointer.pointer_for_id_created_listeners.has(this.id)) {
+                for (const l of Pointer.pointer_for_id_created_listeners.get(this.id)!) l(this);
+                Pointer.pointer_for_id_created_listeners.delete(this.id)
+            }
+
             // adds garbage collection listener
             this.updateGarbageCollection(); 
         }

runtime/runtime.ts

@@ -569,6 +569,9 @@ export class Runtime {
     public static async getURLContent<T=unknown, RAW extends boolean = false>(url_string:string, raw?:RAW, cached?:boolean):Promise<RAW extends false ? T : [data:unknown, type?:string]>
     public static async getURLContent<T=unknown, RAW extends boolean = false>(url:URL, raw?:RAW, cached?:boolean):Promise<RAW extends false ? T : [data:unknown, type?:string]>
     public static async getURLContent<T=unknown, RAW extends boolean = false>(url_string:string|URL, raw:RAW=false, cached = false):Promise<RAW extends false ? T : [data:unknown, type?:string]> {
+
+        if (url_string.toString().startsWith("route:") && window.location?.origin) url_string = new URL(url_string.toString().replace("route:", ""), window.location.origin)
+
         const url = url_string instanceof URL ? url_string : new URL(url_string, baseURL);
         url_string = url.toString();
 
@@ -2434,7 +2437,7 @@ export class Runtime {
 
             else {
                 // cannot fetch type, just cast default
-                logger.warn("Cannot further resolve unknown type '"+type.toString()+"'");
+                logger.warn("Cannot find a type definition for "+type.toString()+". Make sure the module for this type is imported. If this type is no longer used, try to clear your eternal caches.");
                 new_value = type.cast(old_value, context, origin, false, false, assigningPtrId);
             }
 
@@ -4459,7 +4462,7 @@ export class Runtime {
                         if (!SCOPE.sender.equals(to)) throw new PointerError("Sender has no permission to stop sync pointer to another origin", SCOPE);
                     }
 
-                    logger.success(SCOPE.sender + " unsubscribed from " + pointer.idString());
+                    logger.debug(SCOPE.sender + " unsubscribed from " + pointer.idString());
 
                     // not existing pointer or no access to this pointer
                     if (!pointer.value_initialized) throw new PointerError("Pointer does not exist", SCOPE)

types/function-utils.ts

@@ -157,7 +157,7 @@ export function createFunctionWithDependencyInjections(source: string, dependenc
     try {
         let creatorFn = new Function(...renamedVars, `"use strict";${varMapping?createStaticFn:''}${varMapping}; return (${source})`)
         if (hasThis) creatorFn = creatorFn.bind(dependencies['this'])
-        return creatorFn(...Object.values(dependencies));
+        return creatorFn(...Object.entries(dependencies).filter(([d]) => d!=='this').map(([_,v]) => v));
     }
     catch (e) {
         console.error(source)

types/js-function.ts

@@ -2,6 +2,7 @@
  * Represents a JS function with source code that can be transferred between endpoints
  */
 
+import { LazyPointer } from "../runtime/lazy-pointer.ts";
 import { Pointer } from "../runtime/pointers.ts";
 import { Runtime } from "../runtime/runtime.ts";
 import { ExtensibleFunction, getDeclaredExternalVariables, getDeclaredExternalVariablesAsync, createFunctionWithDependencyInjections, getSourceWithoutUsingDeclaration, Callable } from "./function-utils.ts";
@@ -24,22 +25,40 @@ export class JSTransferableFunction extends ExtensibleFunction {
 		}
 		else {
 			let ptr: Pointer|undefined;
-			const fn = (...args:unknown[]) => {
+			const fn = (...args:any[]) => {
 				if (!ptr) ptr = Pointer.getByValue(this);
 				if (!ptr) throw new Error("Cannot execute js:Function, must be bound to a pointer");
 				const origin = ptr.origin.main;
 				if (origin !== Runtime.endpoint && !Runtime.trustedEndpoints.get(origin)?.includes("remote-js-execution")) {
 					throw new Error("Cannot execute js:Function, origin "+origin+" has no permission to execute js source code on this endpoint");
 				}
-				return intermediateFn(...args)
+				this.assertLazyDependenciesResolved();
+				if (this.deps.this) return intermediateFn.apply(this.deps.this, args)
+				else return intermediateFn(...args)
 			}
 			super(fn);
+			this.resolveLazyDependencies(); // make sure LazyPointer deps are resolved
 			this.#fn = fn;
 		}
 
 		this.source = source;
 	}
 
+	private resolveLazyDependencies() {
+		for (const [key, value] of Object.entries(this.deps)) {
+			if (value instanceof LazyPointer) value.onLoad((v) => this.deps[key] = v);
+		}
+	}
+
+	#resolved = false;
+	private assertLazyDependenciesResolved() {
+		if (this.#resolved) return true;
+		for (const [key, value] of Object.entries(this.deps)) {
+			if (value instanceof LazyPointer) throw new Error("Cannot call <js:Function>, dependency variable '"+key+"' is not yet initialized")
+		}
+		this.#resolved = true;
+	}
+
 	call(...args:any[]) {
 		return this.#fn(...args)
 	}

types/type.ts

@@ -55,6 +55,14 @@ export class Type<T = any> extends ExtensibleFunction {
     variation:string = ''
     parameters:any[] // special type parameters
 
+    #jsTypeDefModule?: string|URL // URL for the JS module that creates the corresponding type definition
+
+    get jsTypeDefModule():string|URL|undefined {return this.#jsTypeDefModule}
+    set jsTypeDefModule(url: string|URL) {
+        if (Type.#jsTypeDefModuleMapper) this.#jsTypeDefModule = Type.#jsTypeDefModuleMapper(url);
+        else this.#jsTypeDefModule = url;
+    }
+
     root_type: Type; // DatexType without parameters and variation
     base_type: Type; // DatexType without parameters
 
@@ -69,6 +77,16 @@ export class Type<T = any> extends ExtensibleFunction {
     // TODO: make true per default? currently results in stack overflows for some std types
     #proxify_children = false // proxify all (new) children of this type
     children_timeouts?: Map<string, number> // individual timeouts for children
+
+    static #jsTypeDefModuleMapper?: (url:string|URL) => string|URL
+
+    static setJSTypeDefModuleMapper(fn:  (url:string|URL) => string|URL) {
+        this.#jsTypeDefModuleMapper = fn;
+        // update existing typedef modules
+        for (const type of this.types.values()) {
+            if (type.#jsTypeDefModule) type.jsTypeDefModule = type.#jsTypeDefModule;
+        }
+    }
 
     /**
      * Should proxify all children with proxify_as_child=true

utils/persistent-listeners.ts

@@ -31,7 +31,7 @@ export function removePersistentListener(target: EventTarget, event: string, han
 
 export function recreatePersistentListeners() {
 	for (const [target, {event, handler, options}] of listeners) {
-		console.debug("recreated a persistent event listener for '" + event + "'")
+		// console.debug("recreated a persistent event listener for '" + event + "'")
 		originalAddEventListener.call(target, event, handler, options)
 	}
 }