diff --git a/compiler/compiler.ts b/compiler/compiler.ts index 4ff3aca7..ab801a09 100644 --- a/compiler/compiler.ts +++ b/compiler/compiler.ts @@ -2021,7 +2021,8 @@ export class Compiler { SCOPE.addJSTypeDefs = receiver != Runtime.endpoint && receiver != LOCAL_ENDPOINT; } - const addTypeDefs = SCOPE.addJSTypeDefs && jsTypeDefModule; + // add js type module only if http(s) url + const addTypeDefs = SCOPE.addJSTypeDefs && jsTypeDefModule && (jsTypeDefModule.toString().startsWith("http://") || jsTypeDefModule.toString().startsWith("https://")); if (addTypeDefs) { Compiler.builder.handleRequiredBufferSize(SCOPE.b_index+4, SCOPE); diff --git a/datex_short.ts b/datex_short.ts index 565c42a3..15357797 100644 --- a/datex_short.ts +++ b/datex_short.ts @@ -5,7 +5,7 @@ import { baseURL, Runtime, PrecompiledDXB, Type, Pointer, Ref, PointerProperty, /** make decorators global */ import { assert as _assert, property as _property, sync as _sync, endpoint as _endpoint, template as _template, jsdoc as _jsdoc} from "./datex_all.ts"; -import { effect as _effect, always as _always, toggle as _toggle, map as _map, equals as _equals, selectProperty as _selectProperty, not as _not } from "./functions.ts"; +import { effect as _effect, always as _always, asyncAlways as _asyncAlways, toggle as _toggle, map as _map, equals as _equals, selectProperty as _selectProperty, not as _not } from "./functions.ts"; export * from "./functions.ts"; import { NOT_EXISTING, DX_SLOTS, SLOT_GET, SLOT_SET } from "./runtime/constants.ts"; import { AssertionError } from "./types/errors.ts"; @@ -24,6 +24,7 @@ declare global { const sync: typeof _sync; const endpoint: typeof _endpoint; const always: typeof _always; + const asyncAlways: typeof _asyncAlways; const toggle: typeof _toggle; const map: typeof _map; const equals: typeof _equals; @@ -611,6 +612,7 @@ export function translocate|Set|Array {searchUser(searchName.val, searchAge.val)}) +``` + +> [!NOTE] +> With sequential async execution, it is not guaranteed that the effect is triggered for each state change - some states might be skipped. +> However, it is always guaranteed that the effect is triggered for the latest state at some point in time. + ## Observing pointer changes diff --git a/docs/manual/09 Functional Programming.md b/docs/manual/09 Functional Programming.md index 984e4214..ae9f63d5 100644 --- a/docs/manual/09 Functional Programming.md +++ b/docs/manual/09 Functional Programming.md @@ -84,7 +84,7 @@ c.val = 20; > The `always` transform function must always be synchronous and must not return a Promise -### Caching `always` output values +## Caching `always` output values Since `always` functions are always required to be pure functions, it is possible to cache the result of a calculation with given input values and return it at a later point in time. @@ -170,6 +170,35 @@ const urlContent = transformAsync([url], async url => (await fetch(url)).json()) The same restrictions as for `transform` functions apply + +### The `asyncAlways` transform function + +The `asyncAlways` function is similar to the `always` function, but can be used for async transforms. +The `asyncAlways` function does not accept `async` functions as transform functions, but allows promises as return values: + +```ts +const input = $$(10); + +const output = await asyncAlways(() => input.val * 10) // 🔶 Runtime warning: use 'always' instead +const output = await asyncAlways(async () => input.val * 10) // ❌ Runtime error: asyncAlways cannot be used with async functions +const output = await asyncAlways(() => (async () => input.val * 10)()) // ❌ No runtime error, but not recommended + +const fn = async () => { + const res = await asyncOperation(); + return res + input.val // input is not captured here! +} +const output = await asyncAlways(() => fn()) // ❌ No runtime error, but 'output' is not recalculated when 'input' changes! + +const output = await asyncAlways(() => asyncFunction(input.val)) // ✅ Correct usage +const output = await asyncAlways(() => (async (val) => val * 10)(input.val) ) // ✅ Correct usage +``` + +> [!NOTE] +> In some cases, async transform functions would work correctly with `asyncAlways`, but +> any dependency value after the first `await` is not captured. +> To avoid confusion, async transform functions are always disallowed for `asyncAlways`. + + ## Dedicated transform functions The DATEX JavaSccript Library provides some standard transform functions for common operations. diff --git a/functions.ts b/functions.ts index ad3e7ed5..74d8b259 100644 --- a/functions.ts +++ b/functions.ts @@ -4,8 +4,9 @@ */ -import { AsyncTransformFunction, BooleanRef, CollapsedValue, CollapsedValueAdvanced, Decorators, INSERT_MARK, METADATA, MaybeObjectRef, MinimalJSRef, Pointer, Ref, RefLike, RefOrValue, Runtime, SmartTransformFunction, SmartTransformOptions, TransformFunction, TransformFunctionInputs, handleDecoratorArgs, primitive } from "./datex_all.ts"; +import { AsyncTransformFunction, BooleanRef, CollapsedValue, CollapsedValueAdvanced, Decorators, INSERT_MARK, METADATA, MaybeObjectRef, MinimalJSRef, Pointer, Ref, RefLike, RefOrValue, Runtime, SmartTransformFunction, SmartTransformOptions, TransformFunction, TransformFunctionInputs, handleDecoratorArgs, logger, primitive } from "./datex_all.ts"; import { Datex } from "./mod.ts"; +import { PointerError } from "./types/errors.ts"; import { IterableHandler } from "./utils/iterable-handler.ts"; @@ -31,12 +32,55 @@ export function always(transform:SmartTransformFunction, options?: SmartTr export function always(script:TemplateStringsArray, ...vars:any[]): Promise> export function always(scriptOrJSTransform:TemplateStringsArray|SmartTransformFunction, ...vars:any[]) { // js function - if (typeof scriptOrJSTransform == "function") return Ref.collapseValue(Pointer.createSmartTransform(scriptOrJSTransform, undefined, undefined, undefined, vars[0])); + if (typeof scriptOrJSTransform == "function") { + // make sure handler is not an async function + if (scriptOrJSTransform.constructor.name == "AsyncFunction") { + throw new Error("Async functions are not allowed as always transforms") + } + const ptr = Pointer.createSmartTransform(scriptOrJSTransform, undefined, undefined, undefined, vars[0]); + if (!ptr.value_initialized && ptr.waiting_for_always_promise) { + throw new PointerError(`Promises cannot be returned from always transforms - use 'asyncAlways' instead`); + } + else { + return Ref.collapseValue(ptr); + } + } // datex script else return (async ()=>Ref.collapseValue(await datex(`always (${scriptOrJSTransform.raw.join(INSERT_MARK)})`, vars)))() } +/** + * A generic transform function, creates a new pointer containing the result of the callback function. + * At any point in time, the pointer is the result of the callback function. + * In contrast to the always function, this function can return a Promise, but the callback function cannot be an async function. + * ```ts + * const x = $$(42); + * const y = await asyncAlways (() => complexCalculation(x.val * 10)); + * + * async function complexCalculation(input: number) { + * const res = await ...// some async operation + * return res + * } + * ``` + */ +export async function asyncAlways(transform:SmartTransformFunction, options?: SmartTransformOptions): Promise> { // return signature from Value.collapseValue(Pointer.smartTransform()) + // make sure handler is not an async function + if (transform.constructor.name == "AsyncFunction") { + throw new Error("asyncAlways cannot be used with async functions, but with functions returning a Promise") + } + const ptr = Pointer.createSmartTransform(transform, undefined, undefined, undefined, options); + if (!ptr.value_initialized && ptr.waiting_for_always_promise) { + await ptr.waiting_for_always_promise; + } + else { + logger.warn("asyncAlways: transform function did not return a Promise, you should use 'always' instead") + } + return Ref.collapseValue(ptr) as MinimalJSRef +} + + + /** * Runs each time a dependency reference value changes. * Dependency references are automatically detected. @@ -55,10 +99,15 @@ export function always(scriptOrJSTransform:TemplateStringsArray|SmartTransformFu * x.val = 6; // no log * ``` */ -export function effect|undefined>(handler:W extends undefined ? () => void :(weakVariables: W) => void, weakVariables?: W): {dispose: () => void, [Symbol.dispose]: () => void} { +export function effect|undefined>(handler:W extends undefined ? () => void|Promise :(weakVariables: W) => void|Promise, weakVariables?: W): {dispose: () => void, [Symbol.dispose]: () => void} { let ptr: Pointer; + // make sure handler is not an async function + if (handler.constructor.name == "AsyncFunction") { + throw new Error("Async functions are not allowed as effect handlers") + } + // weak variable binding if (weakVariables) { const weakVariablesProxy = {}; diff --git a/js_adapter/js_class_adapter.ts b/js_adapter/js_class_adapter.ts index c1a520d2..8a0e8a30 100644 --- a/js_adapter/js_class_adapter.ts +++ b/js_adapter/js_class_adapter.ts @@ -574,9 +574,11 @@ export class Decorators { function normalizeType(type:Type|string, allowTypeParams = true, defaultNamespace = "std") { if (typeof type == "string") { // extract type name and parameters - const [typeName, paramsString] = type.replace(/^\$/,'').match(/^((?:\w+\:)?\w*)(?:\((.*)\))?$/)?.slice(1) ?? []; + const [typeName, paramsString] = type.replace(/^\$/,'').match(/^((?:[\w-]+\:)?[\w-]*)(?:\((.*)\))?$/)?.slice(1) ?? []; if (paramsString && !allowTypeParams) throw new Error(`Type parameters not allowed (${type})`); + if (!typeName) throw new Error("Invalid type: " + type); + // TODO: only json-compatible params are allowed for now to avoid async const parsedParams = paramsString ? JSON.parse(`[${paramsString}]`) : undefined; return Type.get(typeName.includes(":") ? typeName : defaultNamespace+":"+typeName, parsedParams) diff --git a/runtime/pointers.ts b/runtime/pointers.ts index a90d6151..4dcc9312 100644 --- a/runtime/pointers.ts +++ b/runtime/pointers.ts @@ -1005,10 +1005,22 @@ export type pointer_type = number; // mock pointer used for garbage collection type MockPointer = {id: string, origin: Endpoint, subscribed?: Endpoint|false, is_origin: boolean} -export type SmartTransformOptions = { +export type SmartTransformOptions = { + initial?: T, cache?: boolean } +type TransformState = { + isLive: boolean; + isFirst: boolean; + deps: IterableWeakSet>; + keyedDeps: AutoMap>; + returnCache: Map; + getDepsHash: () => string; + update: () => void; +} + + const observableArrayMethods = new Set([ "entries", "filter", @@ -1036,6 +1048,7 @@ const observableArrayMethods = new Set([ "with" ]) + /** Wrapper class for all pointer values ($xxxxxxxx) */ export class Pointer extends Ref { @@ -1571,7 +1584,7 @@ export class Pointer extends Ref { * @returns */ static createSmartTransform(transform:SmartTransformFunction, persistent_datex_transform?:string, forceLive = false, ignoreReturnValue = false, options?:SmartTransformOptions):Pointer { - return Pointer.create(undefined, NOT_EXISTING).smartTransform(transform, persistent_datex_transform, forceLive, ignoreReturnValue, options); + return Pointer.create(undefined, options?.initial??NOT_EXISTING).smartTransform(transform, persistent_datex_transform, forceLive, ignoreReturnValue, options); } static createTransformAsync(observe_values:V, transform:AsyncTransformFunction, persistent_datex_transform?:string):Promise> @@ -1928,6 +1941,9 @@ export class Pointer extends Ref { get labels(){return this.#labels} get pointer_type(){return this.#pointer_type} + #waiting_for_always_promise?: Promise; + get waiting_for_always_promise() {return this.#waiting_for_always_promise} + #updateIsJSPrimitive(val:any = this.val) { const type = this.#type ?? Type.ofValue(val); this.#is_js_primitive = (typeof val !== "symbol") && !(Object(val) === val && !type.is_js_pseudo_primitive && !(type == Type.js.NativeObject && globalThis.Element && val instanceof globalThis.Element)) @@ -2253,7 +2269,10 @@ export class Pointer extends Ref { } //placeholder replacement if (Pointer.pointer_value_map.has(v)) { - if (this.#loaded) {throw new PointerError("Cannot assign a new value to an already initialized pointer")} + if (this.#loaded) { + console.log("value",v) + throw new PointerError("Cannot assign a new value to an already initialized pointer") + } const existing_pointer = Pointer.pointer_value_map.get(v)!; existing_pointer.unPlaceholder(this.id) // no longer placeholder, this pointer gets 'overriden' by existing_pointer return existing_pointer; @@ -2655,168 +2674,189 @@ export class Pointer extends Ref { protected smartTransform(transform:SmartTransformFunction, persistent_datex_transform?:string, forceLive = false, ignoreReturnValue = false, options?:SmartTransformOptions): Pointer { if (persistent_datex_transform) this.setDatexTransform(persistent_datex_transform) // TODO: only workaround - const deps = new IterableWeakSet(); - const keyedDeps = new IterableWeakMap>().setAutoDefault(Set); - - let isLive = false; - let isFirst = true; - - const returnCache = new Map() - - const getDepsHash = () => { - const norm = - [...deps].map(v=>Runtime.valueToDatexStringExperimental(v, false, false, false, true)).join("\n") + "\n" + - [...keyedDeps].map(([ptr, keys]) => - [...keys].map(key => Runtime.valueToDatexStringExperimental(ptr.getProperty(key), false, false, false, false)).join("\n") - ).join("\n") - const hash = sha256(norm) - return hash; - } + const state: TransformState = { + isLive: false, + isFirst: true, + deps: new IterableWeakSet(), + keyedDeps: new IterableWeakMap>().setAutoDefault(Set), + returnCache: new Map(), + + getDepsHash: () => { + const norm = + [...state.deps].map(v=>Runtime.valueToDatexStringExperimental(v, false, false, false, true)).join("\n") + "\n" + + [...state.keyedDeps].map(([ptr, keys]) => + [...keys].map(key => Runtime.valueToDatexStringExperimental(ptr.getProperty(key), false, false, false, false)).join("\n") + ).join("\n") + const hash = sha256(norm) as string + return hash; + }, - const update = () => { - // no live transforms needed, just get current value - // capture getters in first update() call to check if there - // is a static transform and show a warning - if (!isLive && !isFirst) { - this.setVal(transform() as T, true, true); - } - // get transform value and update dependency observers - else { - isFirst = false; - - let val!: T - let capturedGetters: Set> | undefined; - let capturedGettersWithKeys: AutoMap, Set> | undefined; - - if (options?.cache) { - const hash = getDepsHash() - if (returnCache.has(hash)) { - logger.debug("using cached transform result with hash " + hash) - val = returnCache.get(hash) - } + update: () => { + // no live transforms needed, just get current value + // capture getters in first update() call to check if there + // is a static transform and show a warning + if (!state.isLive && !state.isFirst) { + this.setVal(transform() as T, true, true); } - - // no cached value found, run transform function - if (val === undefined) { - Ref.captureGetters(); + // get transform value and update dependency observers + else { + state.isFirst = false; - try { - val = transform() as T; - // also trigger getter if pointer is returned - Ref.collapseValue(val, true, true); + let val!: T + let capturedGetters: Set> | undefined; + let capturedGettersWithKeys: AutoMap, Set> | undefined; + + if (options?.cache) { + const hash = state.getDepsHash() + if (state.returnCache.has(hash)) { + logger.debug("using cached transform result with hash " + hash) + val = state.returnCache.get(hash) + } } - catch (e) { - if (e !== Pointer.WEAK_EFFECT_DISPOSED) console.error(e); - // invalid result, no update - return; + + // no cached value found, run transform function + if (val === undefined) { + Ref.captureGetters(); + + try { + val = transform() as T; + // also trigger getter if pointer is returned + Ref.collapseValue(val, true, true); + } + catch (e) { + if (e !== Pointer.WEAK_EFFECT_DISPOSED) console.error(e); + // invalid result, no update + return; + } + // always cleanup capturing + finally { + ({capturedGetters, capturedGettersWithKeys} = Ref.getCapturedGetters()); + } } - // always cleanup capturing - finally { - ({capturedGetters, capturedGettersWithKeys} = Ref.getCapturedGetters()); + + + // promise returned, wait for promise to resolve + if (val instanceof Promise) { + // force live required for async transforms (cannot synchronously calculate the value in a getter) + this.enableLiveTransforms(false) + + // remember latest transform promise + this.#waiting_for_always_promise = val; + + // wait until val promise resolves + val.then((resolvedVal)=>{ + // got a more recent promise result in the meantime, ignore this one + if (val !== this.#waiting_for_always_promise) { + return; + } + this.#waiting_for_always_promise = undefined; + this.handleTransformValue( + resolvedVal, + capturedGetters, + capturedGettersWithKeys, + state, + ignoreReturnValue, + options + ) + }) + } + // normal sync transform + else { + this.handleTransformValue( + val, + capturedGetters, + capturedGettersWithKeys, + state, + ignoreReturnValue, + options + ) } - } - - if (!ignoreReturnValue && !this.value_initialized) { - if (val == undefined) this.#is_js_primitive = true; - else this.#updateIsJSPrimitive(Ref.collapseValue(val,true,true)); } - // set isLive to true, if not primitive - if (!this.is_js_primitive) { - isLive = true; - this._liveTransform = true - } - - // update value - if (!ignoreReturnValue) this.setVal(val, true, true); + } + } - // remove return value if captured by getters - // TODO: this this work as intended? - capturedGetters?.delete(val instanceof Pointer ? val : Pointer.getByValue(val)); - capturedGettersWithKeys?.delete(val instanceof Pointer ? val : Pointer.getByValue(val)); + // only for effects (indicated by ignoreReturnValue=true): + // execute an *async* transform call after the previous one has finished, not in parallel + if (ignoreReturnValue) { + let blocked = false; // if true, the update() method is blocked until the previous transform resolves + let requestingUpdate = false; // if true, an update is requested after the previous transform promise resolves - const hasGetters = capturedGetters||capturedGettersWithKeys; - const gettersCount = (capturedGetters?.size??0) + (capturedGettersWithKeys?.size??0); + const originalUpdate = state.update; - // no dependencies, will never change, this is not the intention of the transform - if (!ignoreReturnValue && hasGetters && !gettersCount) { - logger.warn("The transform value for " + this.idString() + " is a static value:", val); + /** + * execute update() and block if the transform method returns a promise + */ + const safeUpdate = () => { + try { + originalUpdate() } + finally { + blockLoop() + } + } + - if (isLive) { - - if (capturedGetters) { - // unobserve no longer relevant dependencies - for (const dep of deps) { - if (!capturedGetters?.has(dep)) { - dep.unobserve(update, this.#unique); - deps.delete(dep) - } - } - // observe newly discovered dependencies - for (const getter of capturedGetters) { - if (deps.has(getter)) continue; - deps.add(getter) - getter.observe(update, this.#unique); + /** + * if the transform triggeed by update() method returns a promise, block further updates until the promise resolves + */ + const blockLoop = () => { + if (this.#waiting_for_always_promise) { + blocked = true; + this.#waiting_for_always_promise.then(()=>{ + // now trigger requested update + if (requestingUpdate) { + requestingUpdate = false; + safeUpdate() } - } - - if (capturedGettersWithKeys) { - // unobserve no longer relevant dependencies - for (const [ptr, keys] of keyedDeps) { - const capturedKeys = capturedGettersWithKeys.get(ptr); - for (const key of keys) { - if (!capturedKeys?.has(key)) { - ptr.unobserve(update, this.#unique, key); - keys.delete(key) - } - } - if (keys.size == 0) keyedDeps.delete(ptr); + // unblock + else { + blocked = false; } - - // observe newly discovered dependencies - for (const [ptr, keys] of capturedGettersWithKeys) { - const storedKeys = keyedDeps.getAuto(ptr); - - for (const key of keys) { - if (storedKeys.has(key)) continue; - ptr.observe(update, this.#unique, key); - storedKeys.add(key); - } - - } - } - - - if (options?.cache) { - const hash = getDepsHash() - returnCache.set(hash, val); - } + }) + } + else { + blocked = false; } } + /** + * overridden update method that makes sure transforms returning a Promise are executed in order + */ + state.update = () => { + // already awaiting a promise, update is requested and handled by blockLoop() + if (blocked) { + requestingUpdate = true; + } + // not awaiting a promise, execute update immediately + else { + safeUpdate() + } + + } } + // set transform source with TransformSource interface this.setTransformSource({ enableLive: (doUpdate = true) => { - isLive = true; + state.isLive = true; // get current value and automatically reenable observers - if (doUpdate) update(); + if (doUpdate) state.update(); }, disableLive: () => { - isLive = false; + state.isLive = false; // disable all observers - for (const dep of deps) dep.unobserve(update, this.#unique); - for (const [ptr, keys] of keyedDeps) { - for (const key of keys) ptr.unobserve(update, this.#unique, key); + for (const dep of state.deps) dep.unobserve(state.update, this.#unique); + for (const [ptr, keys] of state.keyedDeps) { + for (const key of keys) ptr.unobserve(state.update, this.#unique, key); } - deps.clear(); + state.deps.clear(); }, - deps, - keyedDeps, - update + deps: state.deps, + keyedDeps: state.keyedDeps, + update: state.update }) if (forceLive) this.enableLiveTransforms(false); @@ -2825,6 +2865,94 @@ export class Pointer extends Ref { } + private handleTransformValue( + val: T, + capturedGetters: Set>|undefined, + capturedGettersWithKeys: AutoMap, Set>|undefined, + state: TransformState, + ignoreReturnValue: boolean, + options?: SmartTransformOptions + ) { + if (!ignoreReturnValue && !this.value_initialized) { + if (val == undefined) this.#is_js_primitive = true; + else this.#updateIsJSPrimitive(Ref.collapseValue(val,true,true)); + } + + // set isLive to true, if not primitive + if (!this.is_js_primitive) { + state.isLive = true; + this._liveTransform = true + } + + // update value + if (!ignoreReturnValue) this.setVal(val, true, true); + + // remove return value if captured by getters + // TODO: this this work as intended? + capturedGetters?.delete(val instanceof Pointer ? val : Pointer.getByValue(val)); + capturedGettersWithKeys?.delete(val instanceof Pointer ? val : Pointer.getByValue(val)); + + const hasGetters = capturedGetters||capturedGettersWithKeys; + const gettersCount = (capturedGetters?.size??0) + (capturedGettersWithKeys?.size??0); + + // no dependencies, will never change, this is not the intention of the transform + if (!ignoreReturnValue && hasGetters && !gettersCount) { + logger.warn("The transform value for " + this.idString() + " is a static value:", val); + } + + if (state.isLive) { + + if (capturedGetters) { + // unobserve no longer relevant dependencies + for (const dep of state.deps) { + if (!capturedGetters?.has(dep)) { + dep.unobserve(state.update, this.#unique); + state.deps.delete(dep) + } + } + // observe newly discovered dependencies + for (const getter of capturedGetters) { + if (state.deps.has(getter)) continue; + state.deps.add(getter) + getter.observe(state.update, this.#unique); + } + } + + if (capturedGettersWithKeys) { + // unobserve no longer relevant dependencies + for (const [ptr, keys] of state.keyedDeps) { + const capturedKeys = capturedGettersWithKeys.get(ptr); + for (const key of keys) { + if (!capturedKeys?.has(key)) { + ptr.unobserve(state.update, this.#unique, key); + keys.delete(key) + } + } + if (keys.size == 0) state.keyedDeps.delete(ptr); + } + + // observe newly discovered dependencies + for (const [ptr, keys] of capturedGettersWithKeys) { + const storedKeys = state.keyedDeps.getAuto(ptr); + + for (const key of keys) { + if (storedKeys.has(key)) continue; + ptr.observe(state.update, this.#unique, key); + storedKeys.add(key); + } + + } + } + + + if (options?.cache) { + const hash = state.getDepsHash() + state.returnCache.set(hash, val); + } + } + } + + // TODO: JUST A WORKAROUND - if transform is a JS function, a DATEX Script can be provided to be stored as a transform method async setDatexTransform(datex_transform:string) { // TODO: fix and reenable