diff --git a/functions.ts b/functions.ts index 7da72b75..ad3e7ed5 100644 --- a/functions.ts +++ b/functions.ts @@ -133,7 +133,7 @@ export function map(iterable: Iterable< if (options?.outType == "map") { mapped = $$(new Map()) - new IterableHandler(iterable, { + const iterableHandler = new IterableHandler(iterable, { map: (v,k)=>{ return mapFn(v,k,iterable) }, @@ -143,6 +143,8 @@ export function map(iterable: Iterable< onNewEntry: (v,k) => (mapped as Map).set(k,v), onEmpty: () => (mapped as Map).clear() }) + // reverse transform binding + Datex.Pointer.bindDisposable(mapped, iterableHandler) } // return array @@ -152,7 +154,7 @@ export function map(iterable: Iterable< // no gaps in a set -> array splice required const spliceArray = iterable instanceof Set; - new IterableHandler(iterable, { + const iterableHandler = new IterableHandler(iterable, { map: (v,k)=>{ return mapFn(v,k,iterable) }, @@ -167,6 +169,8 @@ export function map(iterable: Iterable< (mapped as U[]).length = 0 } }) + // reverse transform binding + Datex.Pointer.bindDisposable(mapped, iterableHandler) } } diff --git a/network/unyt.ts b/network/unyt.ts index fbbf0501..050c5de4 100644 --- a/network/unyt.ts +++ b/network/unyt.ts @@ -79,6 +79,16 @@ export class Unyt { return info.app?.dynamicData?.domains ? info.app.dynamicData.domains.map(d=>'https://'+d) : []; } + /** + * Don't delete: required in docker-host + */ + public formatEndpointURL(endpoint: Endpoint) { + const endpointName = endpoint.toString(); + if (endpointName.startsWith("@+")) return `${endpointName.replace("@+","")}.unyt.app` + else if (endpointName.startsWith("@@")) return `${endpointName.replace("@@","")}.unyt.app` + else if (endpointName.startsWith("@")) return `${endpointName.replace("@","")}.unyt.me` + } + // TODO add colored logo dark - light mode public static async logEndpointInfo(){ const info = this.endpoint_info; diff --git a/runtime/pointers.ts b/runtime/pointers.ts index 8c6145af..71077e33 100644 --- a/runtime/pointers.ts +++ b/runtime/pointers.ts @@ -1770,10 +1770,38 @@ export class Pointer extends Ref { Pointer.pointers.delete(this.#id); Pointer.primitive_pointers.delete(this.#id) + // remove disposables + for (const disposable of this.#boundDisposables) { + console.log("disposing", disposable) + disposable[Symbol.dispose]?.() + } + // call remove listeners (todo: also for primitive pointers) if (!this.is_anonymous) for (const l of Pointer.pointer_remove_listeners) l(this); } + #boundDisposables = new Set<{[Symbol.dispose]: ()=>any}>() + + // binds a disposable object to this pointer that gets disposed as soon as the pointer is garbage collected + public bindDisposable(disposable: {[Symbol.dispose]: ()=>any}) { + this.#boundDisposables.add(disposable); + if (!this.is_js_primitive) { + if (!this.val[Pointer.DISPOSABLES]) this.val[Pointer.DISPOSABLES] = [] + this.val[Pointer.DISPOSABLES].push(disposable); + } + } + + static DISPOSABLES = Symbol("DISPOSABLES") + + public static bindDisposable(value: any, disposable: {[Symbol.dispose]: ()=>any}) { + const ptr = value instanceof Pointer ? value : this.getByValue(value); + if (ptr) { + ptr.bindDisposable(disposable); + } + else throw new Error("Cannot bind a disposable value to a non-pointer value") + } + + [Symbol.dispose]() { this.delete() } diff --git a/runtime/runtime.ts b/runtime/runtime.ts index 67f1652a..79ffd117 100644 --- a/runtime/runtime.ts +++ b/runtime/runtime.ts @@ -595,7 +595,7 @@ export class Runtime { // possible js module import: fetch headers first and check content type: if (!raw && (url_string.endsWith("js") || url_string.endsWith("ts") || url_string.endsWith("tsx") || url_string.endsWith("jsx") || url_string.endsWith("dx") || url_string.endsWith("dxb"))) { try { - response = await fetch(url, {method: 'HEAD'}); + response = await fetch(url, {method: 'HEAD', cache: 'no-store'}); const type = response.headers.get('content-type'); if (type?.startsWith("text/javascript") || type?.startsWith("application/javascript")) { doFetch = false; // no body fetch required, can directly import() module diff --git a/types/function-utils.ts b/types/function-utils.ts index 07685ab5..6e61a8f7 100644 --- a/types/function-utils.ts +++ b/types/function-utils.ts @@ -80,7 +80,7 @@ export function getDeclaredExternalVariables(fn: (...args:unknown[])=>unknown) { // call the function with EXTRACT_USED_VARS metadata try { - callWithMetadata({[EXTRACT_USED_VARS]: true}, fn as any) + callWithMetadata({[EXTRACT_USED_VARS]: true}, fn as any, [{}]) // TODO: provide call arguments that don't lead to a {}/[] destructuring error } catch (e) { // capture returned variables from use() @@ -157,13 +157,13 @@ function assertLazyDependenciesResolved(deps:Record) { * @param dependencies * @returns */ -export function createFunctionWithDependencyInjectionsResolveLazyPointers(source: string, dependencies: Record): ((...args:unknown[]) => unknown) { +export function createFunctionWithDependencyInjectionsResolveLazyPointers(source: string, dependencies: Record, allowValueMutations = true): ((...args:unknown[]) => unknown) { let fn: Function|undefined; const intermediateFn = (...args:any[]) => { if (!fn) { assertLazyDependenciesResolved(dependencies); - fn = createFunctionWithDependencyInjections(source, dependencies) + fn = createFunctionWithDependencyInjections(source, dependencies, allowValueMutations) } return fn(...args) } @@ -178,10 +178,10 @@ export function createFunctionWithDependencyInjectionsResolveLazyPointers(source * @returns * @deprecated use createFunctionWithDependencyInjectionsResolveLazyPointers */ -export function createFunctionWithDependencyInjections(source: string, dependencies: Record): ((...args:unknown[]) => unknown) { +export function createFunctionWithDependencyInjections(source: string, dependencies: Record, allowValueMutations = true): ((...args:unknown[]) => unknown) { const hasThis = Object.keys(dependencies).includes('this'); const renamedVars = Object.keys(dependencies).filter(d => d!=='this').map(k=>'_'+k); - const varMapping = renamedVars.map(k=>`const ${k.slice(1)} = createStaticObject(${k});`).join("\n"); + const varMapping = renamedVars.map(k=>`const ${k.slice(1)} = ${allowValueMutations ? 'createStaticObject' : ''}(${k});`).join("\n"); const createStaticFn = `function createStaticObject(val) { if (val && typeof val == "object" && !globalThis.Datex?.Ref.isRef(val)) { @@ -192,7 +192,7 @@ export function createFunctionWithDependencyInjections(source: string, dependenc };` try { - let creatorFn = new Function(...renamedVars, `"use strict";${varMapping?createStaticFn:''}${varMapping}; return (${source})`) + let creatorFn = new Function(...renamedVars, `"use strict";${(varMapping&&allowValueMutations)?createStaticFn:''}${varMapping}; return (${source})`) if (hasThis) creatorFn = creatorFn.bind(dependencies['this']) return creatorFn(...Object.entries(dependencies).filter(([d]) => d!=='this').map(([_,v]) => v)); } diff --git a/types/js-function.ts b/types/js-function.ts index bf3c25c8..8bf6bc32 100644 --- a/types/js-function.ts +++ b/types/js-function.ts @@ -2,14 +2,14 @@ * Represents a JS function with source code that can be transferred between endpoints */ -import { LazyPointer } from "../runtime/lazy-pointer.ts"; import { Pointer } from "../runtime/pointers.ts"; import { Runtime } from "../runtime/runtime.ts"; import { ExtensibleFunction, getDeclaredExternalVariables, getDeclaredExternalVariablesAsync, getSourceWithoutUsingDeclaration, Callable, createFunctionWithDependencyInjectionsResolveLazyPointers } from "./function-utils.ts"; export type JSTransferableFunctionOptions = { - errorOnOriginContext?: Error + errorOnOriginContext?: Error, + isLocal?: boolean } export class JSTransferableFunction extends ExtensibleFunction { @@ -26,11 +26,13 @@ export class JSTransferableFunction extends ExtensibleFunction { else { let ptr: Pointer|undefined; const fn = (...args:any[]) => { - if (!ptr) ptr = Pointer.getByValue(this); - if (!ptr) throw new Error("Cannot execute js:Function, must be bound to a pointer"); - const origin = ptr.origin.main; - if (origin !== Runtime.endpoint && !Runtime.trustedEndpoints.get(origin)?.includes("remote-js-execution")) { - throw new Error("Cannot execute js:Function, origin "+origin+" has no permission to execute js source code on this endpoint"); + if (!options?.isLocal) { + if (!ptr) ptr = Pointer.getByValue(this); + if (!ptr) throw new Error("Cannot execute js:Function, must be bound to a pointer"); + const origin = ptr.origin.main; + if (origin !== Runtime.endpoint.main && !Runtime.trustedEndpoints.get(origin)?.includes("remote-js-execution")) { + throw new Error("Cannot execute js:Function, origin "+origin+" has no permission to execute js source code on this endpoint"); + } } return intermediateFn(...args) } @@ -68,8 +70,9 @@ export class JSTransferableFunction extends ExtensibleFunction { * Important: use createAsync for async functions instead * @param fn */ - static createunknown>(fn: T, options?:JSTransferableFunctionOptions): JSTransferableFunction & Callable, ReturnType> { + static createunknown>(fn: T, options:JSTransferableFunctionOptions = {}): JSTransferableFunction & Callable, ReturnType> { const {vars, flags} = getDeclaredExternalVariables(fn); + options.isLocal ??= true; return this.#createTransferableFunction(getSourceWithoutUsingDeclaration(fn), vars, flags, options) as any; } @@ -78,8 +81,9 @@ export class JSTransferableFunction extends ExtensibleFunction { * Automatically determines dependency variables declared with use() * @param fn */ - static async createAsyncPromise>(fn: T, options?:JSTransferableFunctionOptions): Promise, ReturnType>> { + static async createAsyncPromise>(fn: T, options:JSTransferableFunctionOptions = {}): Promise, ReturnType>> { const {vars, flags} = await getDeclaredExternalVariablesAsync(fn) + options.isLocal ??= true; return this.#createTransferableFunction(getSourceWithoutUsingDeclaration(fn), vars, flags, options) as any; } @@ -93,7 +97,7 @@ export class JSTransferableFunction extends ExtensibleFunction { } static #createTransferableFunction(source: string, dependencies: Record, flags?: string[], options?:JSTransferableFunctionOptions) { - const intermediateFn = createFunctionWithDependencyInjectionsResolveLazyPointers(source, dependencies); + const intermediateFn = createFunctionWithDependencyInjectionsResolveLazyPointers(source, dependencies, !options?.isLocal); return new JSTransferableFunction(intermediateFn, dependencies, source, flags, options); } diff --git a/utils/isolated-scope.ts b/utils/isolated-scope.ts new file mode 100644 index 00000000..a7570b30 --- /dev/null +++ b/utils/isolated-scope.ts @@ -0,0 +1,5 @@ +import { JSTransferableFunction } from "../types/js-function.ts"; + +export function isolatedScope(handler:(...args: any[]) => any) { + return JSTransferableFunction.create(handler); +} \ No newline at end of file diff --git a/utils/iterable-handler.ts b/utils/iterable-handler.ts index d2326f02..fb31b334 100644 --- a/utils/iterable-handler.ts +++ b/utils/iterable-handler.ts @@ -2,6 +2,18 @@ import { Datex } from "../mod.ts"; import { ValueError } from "../datex_all.ts"; import { weakAction } from "./weak-action.ts"; + +function workaroundGetHandler(iterableHandler: WeakRef>) { + return (v:any, k:any, t:any) => { + const deref = iterableHandler.deref(); + if (!deref) { + console.warn("Undetected garbage collection (datex-w0001)"); + return; + } + deref.onValueChanged(v, k, t) + } +} + export class IterableHandler { private map: ((value: T, index: number, array: Iterable) => U) | undefined @@ -33,27 +45,42 @@ export class IterableHandler { observe() { // deno-lint-ignore no-this-alias const self = this; - const iterable = this.iterable; + const iterableRef = new WeakRef(this.iterable); + + // const handler = this.workaroundGetHandler(self) + // Datex.Ref.observeAndInit(iterable, handler); + weakAction( {self}, ({self}) => { - const handler = this.workaroundGetHandler(self) - Datex.Ref.observeAndInit(iterable, handler); + use (iterableRef, Datex); + + const iterable = iterableRef.deref()! // only here to fix closure scope bug, should always exist at this point + const callback = (v:any, k:any, t:any) => { + const deref = self.deref(); + if (!deref) { + console.warn("Undetected garbage collection (datex-w0001)"); + return; + } + deref.onValueChanged(v, k, t) + } + Datex.Ref.observeAndInit(iterable, callback); + return callback; + }, + (callback) => { + use (iterableRef, Datex); + + const deref = iterableRef.deref() + if (deref) Datex.Ref.unobserve(deref, callback); } ); } - workaroundGetHandler(handler: WeakRef>) { - return (v:any, k:any, t:any) => { - const deref = handler.deref(); - if (!deref) { - console.warn("Undetected garbage collection (datex-w0001)"); - return; - } - deref.onValueChanged(v, k, t) - } + [Symbol.dispose]() { + // TODO: unobserve } + #entries?: Map; public get entries() { if (!this.#entries) this.#entries = new Map(); diff --git a/utils/volatile-map.ts b/utils/volatile-map.ts index cdfa7fef..864d75b8 100644 --- a/utils/volatile-map.ts +++ b/utils/volatile-map.ts @@ -6,15 +6,19 @@ export class VolatileMap extends Map { static readonly MAX_MAP_ENTRIES = 2**24 static readonly MAP_ENTRIES_CLEANUP_CHUNK = 1000 + static readonly DEFAULT_ENTRYPOINT_LIFETIME = 30*60 // 30min #options: VolatileMapOptions - #timeouts = new Map() + #liftimeStartTimes = new Map() + #customLifetimes = new Map() constructor(iterable?: Iterable | null, options: Partial = {}) { super(iterable) this.#options = options as VolatileMapOptions; - this.#options.entryLifetime ??= 30*60 // 30min + this.#options.entryLifetime ??= VolatileMap.DEFAULT_ENTRYPOINT_LIFETIME this.#options.preventMapOverflow ??= true + + this.#startInterval() } /** @@ -24,6 +28,10 @@ export class VolatileMap extends Map { * @returns the current value for the key */ keepalive(key: K, overrideLifetime?: number) { + if (!this.has(key)) { + console.warn("key does not exist in VolatileMap") + return; + } this.#setTimeout(key, overrideLifetime); return this.get(key) } @@ -54,22 +62,33 @@ export class VolatileMap extends Map { return super.clear(); } + #startInterval() { + setInterval( + () => { + const currentTime = new Date().getTime(); + for (const [key, time] of this.#liftimeStartTimes) { + const lifetime = 1000 * (this.#customLifetimes.get(key) ?? this.#options.entryLifetime); + if (currentTime-time > lifetime) { + this.delete(key); + } + } + }, + Math.min(this.#options.entryLifetime, (VolatileMap.DEFAULT_ENTRYPOINT_LIFETIME)) * 1000 / 5 + ) + } #clearTimeout(key: K) { - if (this.#timeouts.has(key)) { - clearTimeout(this.#timeouts.get(key)); - this.#timeouts.delete(key); - } + this.#liftimeStartTimes.delete(key); } #setTimeout(key: K, overrideLifetime?: number) { // reset previous timeout this.#clearTimeout(key); + // store custom lifetime + if (overrideLifetime != undefined) this.#customLifetimes.set(key, overrideLifetime) const lifetime = overrideLifetime ?? this.#options.entryLifetime; if (Number.isFinite(lifetime)) { - this.#timeouts.set(key, setTimeout(() => { - this.delete(key) - }, lifetime * 1000)) + this.#liftimeStartTimes.set(key, new Date().getTime()) } } } diff --git a/utils/weak-action.ts b/utils/weak-action.ts index 96ee9ad6..e7de9fd7 100644 --- a/utils/weak-action.ts +++ b/utils/weak-action.ts @@ -1,30 +1,52 @@ +import { isolatedScope } from "./isolated-scope.ts"; + /** * Run a weak action (function that is called once with weak dependencies). * If one of the weak dependencies is garbage collected, an optional deinit function is called. * @param weakRefs - * @param action - * @param deinit + * @param action an isolated callback function that provides weak references. External dependency variable must be explicitly added with use() + * @param deinit an isolated callback function that is callled on garbage collection. External dependency variable must be explicitly added with use() */ export function weakAction, R>(weakDependencies: T, action: (values: {[K in keyof T]: WeakRef}) => R, deinit?: (actionResult: R, collectedVariable: keyof T) => unknown) { - const weakRefs = Object.fromEntries( - Object.entries(weakDependencies) - .map(([k, v]) => [k, new WeakRef(v)]) - ) as {[K in keyof T]: WeakRef}; + const weakRefs = _getWeakRefs(weakDependencies); let result:R; + action = isolatedScope(action); + // optional deinit if (deinit) { - const registry = new FinalizationRegistry((k: string) => { - console.log("deinitalized weak action (variable '" + k + "' was garbage collected)") - deinit(result, k); - }); + deinit = isolatedScope(deinit); + + const deinitFn = deinit; + + const deinitHandler = (k: string) => { + registries.delete(registry) + deinitFn(result, k); + } + const registry = new FinalizationRegistry(deinitHandler); + registries.add(registry) for (const [k, v] of Object.entries(weakDependencies)) { + // if (v.constructor.name.startsWith("Iterable")) { + // const t = "x".repeat(30) + // .replace(/./g, c => "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"[Math.floor(Math.random() * 62) ] ); + // console.log(t); + // v[Symbol()] = t; + // } registry.register(v, k); } } // call action once result = action(weakRefs); -} \ No newline at end of file +} + +function _getWeakRefs>(weakDependencies: T) { + return Object.fromEntries( + Object.entries(weakDependencies) + .map(([k, v]) => [k, new WeakRef(v)]) + ) as {[K in keyof T]: WeakRef}; +} + +const registries = new Set>(); \ No newline at end of file