Merge pull request #52 from unyt-org/feat-async-effects

Add async effects and async always

compiler/compiler.ts

@@ -2021,7 +2021,8 @@ export class Compiler {
                 SCOPE.addJSTypeDefs = receiver != Runtime.endpoint && receiver != LOCAL_ENDPOINT;
             }
 
-            const addTypeDefs = SCOPE.addJSTypeDefs && jsTypeDefModule;
+            // add js type module only if http(s) url
+            const addTypeDefs = SCOPE.addJSTypeDefs && jsTypeDefModule && (jsTypeDefModule.toString().startsWith("http://") || jsTypeDefModule.toString().startsWith("https://"));
 
             if (addTypeDefs) {
                 Compiler.builder.handleRequiredBufferSize(SCOPE.b_index+4, SCOPE);

datex_short.ts

@@ -5,7 +5,7 @@ import { baseURL, Runtime, PrecompiledDXB, Type, Pointer, Ref, PointerProperty,
 
 /** make decorators global */
 import { assert as _assert, property as _property, sync as _sync, endpoint as _endpoint, template as _template, jsdoc as _jsdoc} from "./datex_all.ts";
-import { effect as _effect, always as _always, toggle as _toggle, map as _map, equals as _equals, selectProperty as _selectProperty, not as _not } from "./functions.ts";
+import { effect as _effect, always as _always, asyncAlways as _asyncAlways, toggle as _toggle, map as _map, equals as _equals, selectProperty as _selectProperty, not as _not } from "./functions.ts";
 export * from "./functions.ts";
 import { NOT_EXISTING, DX_SLOTS, SLOT_GET, SLOT_SET } from "./runtime/constants.ts";
 import { AssertionError } from "./types/errors.ts";
@@ -24,6 +24,7 @@ declare global {
 	const sync: typeof _sync;
 	const endpoint: typeof _endpoint;
     const always: typeof _always;
+    const asyncAlways: typeof _asyncAlways;
     const toggle: typeof _toggle;
     const map: typeof _map;
     const equals: typeof _equals;
@@ -611,6 +612,7 @@ export function translocate<T extends Map<unknown,unknown>|Set<unknown>|Array<un
 
 Object.defineProperty(globalThis, 'once', {value:once, configurable:false})
 Object.defineProperty(globalThis, 'always', {value:_always, configurable:false})
+Object.defineProperty(globalThis, 'asyncAlways', {value:_asyncAlways, configurable:false})
 Object.defineProperty(globalThis, 'toggle', {value:_toggle, configurable:false})
 Object.defineProperty(globalThis, 'map', {value:_map, configurable:false})
 Object.defineProperty(globalThis, 'equals', {value:_equals, configurable:false})

docs/manual/03 Pointers.md

@@ -248,7 +248,7 @@ Weak value bindings can be used with all *object* values, not just with pointers
 ### Async effects
 
 Effect callbacks cannot be `async` functions.
-To handle async operations, you can always call an async function from inside the
+If you need to handle async operations, you can instead call an async function from inside the
 effect callback:
 
 ```ts
@@ -257,7 +257,7 @@ const searchAge = $$(18);
 
 // async function that searches for a user and shows the result somewhere
 async function searchUser(name: string, age: number) {
-    const user = await query({type: "user", name, age});
+    const user = await fetchUserFromServer({name, age});
     showUser(user);
 }
 
@@ -270,6 +270,28 @@ This means that the variables inside the async function don't trigger the effect
 into the `searchUser` call.
 
 
+#### Sequential async effect execution
+
+Due to the nature of JavaScript, synchronous effects are always executed sequentially.
+But often, effects need to be asynchronous, e.g. to fetch some data from the network.
+
+Per default, if an effect is triggered multiple times in quick succession (e.g. due to a user input in a search field), the fetch requests are executed in parallel, but it cannot be guaranteed that the fetch for the last triggered effect is also resolved last. This leads to indeterministic behaviour.
+
+To prevent this, you can force sequential execution of effects by returning a `Promise` from the 
+effect handler function. 
+With this, it is guaranteed that the effect will not be triggered again before the last `Promise` has resolved.
+
+In the [example above](#async-effects), sequential execution is enabled because the `Promise` returned by the `searchValue()` call is returned from the effect handler.
+
+You can achieve parallel effect execution by not returning the `Promise`, e.g.:
+```ts
+effect(() => {searchUser(searchName.val, searchAge.val)})
+```
+
+> [!NOTE]
+> With sequential async execution, it is not guaranteed that the effect is triggered for each state change - some states might be skipped.
+> However, it is always guaranteed that the effect is triggered for the latest state at some point in time.
+
 
 ## Observing pointer changes
 

docs/manual/09 Functional Programming.md

@@ -84,7 +84,7 @@ c.val = 20;
 >  The `always` transform function must always be synchronous and must not return a Promise
 
 
-### Caching `always` output values
+## Caching `always` output values
 
 Since `always` functions are always required to be pure functions, it is possible to
 cache the result of a calculation with given input values and return it at a later point in time.
@@ -170,6 +170,35 @@ const urlContent = transformAsync([url], async url => (await fetch(url)).json())
 
 The same restrictions as for `transform` functions apply
 
+
+### The `asyncAlways` transform function
+
+The `asyncAlways` function is similar to the `always` function, but can be used for async transforms.
+The `asyncAlways` function does not accept `async` functions as transform functions, but allows promises as return values:
+
+```ts
+const input = $$(10);
+
+const output = await asyncAlways(() => input.val * 10)       // 🔶 Runtime warning: use 'always' instead
+const output = await asyncAlways(async () => input.val * 10) // ❌ Runtime error: asyncAlways cannot be used with async functions
+const output = await asyncAlways(() => (async () => input.val * 10)()) // ❌ No runtime error, but not recommended
+
+const fn = async () => {
+    const res = await asyncOperation();
+    return res + input.val // input is not captured here!
+}
+const output = await asyncAlways(() => fn()) // ❌ No runtime error, but 'output' is not recalculated when 'input' changes!
+
+const output = await asyncAlways(() => asyncFunction(input.val)) // ✅ Correct usage
+const output = await asyncAlways(() => (async (val) => val * 10)(input.val) ) // ✅ Correct usage
+``` 
+
+> [!NOTE]
+> In some cases, async transform functions would work correctly with `asyncAlways`, but
+> any dependency value after the first `await` is not captured. 
+> To avoid confusion, async transform functions are always disallowed for `asyncAlways`.
+
+
 ## Dedicated transform functions
 
 The DATEX JavaSccript Library provides some standard transform functions for common operations.

functions.ts

@@ -4,8 +4,9 @@
  */
 
 
-import { AsyncTransformFunction, BooleanRef, CollapsedValue, CollapsedValueAdvanced, Decorators, INSERT_MARK, METADATA, MaybeObjectRef, MinimalJSRef, Pointer, Ref, RefLike, RefOrValue, Runtime, SmartTransformFunction, SmartTransformOptions, TransformFunction, TransformFunctionInputs, handleDecoratorArgs, primitive } from "./datex_all.ts";
+import { AsyncTransformFunction, BooleanRef, CollapsedValue, CollapsedValueAdvanced, Decorators, INSERT_MARK, METADATA, MaybeObjectRef, MinimalJSRef, Pointer, Ref, RefLike, RefOrValue, Runtime, SmartTransformFunction, SmartTransformOptions, TransformFunction, TransformFunctionInputs, handleDecoratorArgs, logger, primitive } from "./datex_all.ts";
 import { Datex } from "./mod.ts";
+import { PointerError } from "./types/errors.ts";
 import { IterableHandler } from "./utils/iterable-handler.ts";
 
 
@@ -31,12 +32,55 @@ export function always<T>(transform:SmartTransformFunction<T>, options?: SmartTr
 export function always<T=unknown>(script:TemplateStringsArray, ...vars:any[]): Promise<MinimalJSRef<T>>
 export function always(scriptOrJSTransform:TemplateStringsArray|SmartTransformFunction<any>, ...vars:any[]) {
     // js function
-    if (typeof scriptOrJSTransform == "function") return Ref.collapseValue(Pointer.createSmartTransform(scriptOrJSTransform, undefined, undefined, undefined, vars[0]));
+    if (typeof scriptOrJSTransform == "function") {
+		// make sure handler is not an async function
+		if (scriptOrJSTransform.constructor.name == "AsyncFunction") {
+			throw new Error("Async functions are not allowed as always transforms")
+		}
+		const ptr = Pointer.createSmartTransform(scriptOrJSTransform, undefined, undefined, undefined, vars[0]);
+		if (!ptr.value_initialized && ptr.waiting_for_always_promise) {
+			throw new PointerError(`Promises cannot be returned from always transforms - use 'asyncAlways' instead`);
+		}
+		else {
+			return Ref.collapseValue(ptr);
+		}
+	}
     // datex script
     else return (async ()=>Ref.collapseValue(await datex(`always (${scriptOrJSTransform.raw.join(INSERT_MARK)})`, vars)))()
 }
 
 
+/**
+ * A generic transform function, creates a new pointer containing the result of the callback function.
+ * At any point in time, the pointer is the result of the callback function.
+ * In contrast to the always function, this function can return a Promise, but the callback function cannot be an async function.
+ * ```ts
+ * const x = $$(42);
+ * const y = await asyncAlways (() => complexCalculation(x.val * 10));
+ * 
+ * async function complexCalculation(input: number) {
+ *    const res = await ...// some async operation
+ *    return res
+ * }
+ * ```
+ */
+export async function asyncAlways<T>(transform:SmartTransformFunction<T>, options?: SmartTransformOptions): Promise<MinimalJSRef<T>> { // return signature from Value.collapseValue(Pointer.smartTransform())
+    // make sure handler is not an async function
+	if (transform.constructor.name == "AsyncFunction") {
+		throw new Error("asyncAlways cannot be used with async functions, but with functions returning a Promise")
+	}
+	const ptr = Pointer.createSmartTransform(transform, undefined, undefined, undefined, options);
+	if (!ptr.value_initialized && ptr.waiting_for_always_promise) {
+		await ptr.waiting_for_always_promise;
+	}
+	else {
+		logger.warn("asyncAlways: transform function did not return a Promise, you should use 'always' instead")
+	}
+	return Ref.collapseValue(ptr) as MinimalJSRef<T>
+}
+
+
+
 /**
  * Runs each time a dependency reference value changes.
  * Dependency references are automatically detected.
@@ -55,10 +99,15 @@ export function always(scriptOrJSTransform:TemplateStringsArray|SmartTransformFu
  * x.val = 6; // no log
  * ```
  */
-export function effect<W extends Record<string, WeakKey>|undefined>(handler:W extends undefined ? () => void :(weakVariables: W) => void, weakVariables?: W): {dispose: () => void, [Symbol.dispose]: () => void} {
+export function effect<W extends Record<string, WeakKey>|undefined>(handler:W extends undefined ? () => void|Promise<void> :(weakVariables: W) => void|Promise<void>, weakVariables?: W): {dispose: () => void, [Symbol.dispose]: () => void} {
 
 	let ptr: Pointer;
 
+	// make sure handler is not an async function
+	if (handler.constructor.name == "AsyncFunction") {
+		throw new Error("Async functions are not allowed as effect handlers")
+	}
+
 	// weak variable binding
 	if (weakVariables) {
 		const weakVariablesProxy = {};

js_adapter/js_class_adapter.ts

@@ -574,9 +574,11 @@ export class Decorators {
 function normalizeType(type:Type|string, allowTypeParams = true, defaultNamespace = "std") {
     if (typeof type == "string") {
         // extract type name and parameters
-        const [typeName, paramsString] = type.replace(/^\</,'').replace(/\>$/,'').match(/^((?:\w+\:)?\w*)(?:\((.*)\))?$/)?.slice(1) ?? [];
+        const [typeName, paramsString] = type.replace(/^\</,'').replace(/\>$/,'').match(/^((?:[\w-]+\:)?[\w-]*)(?:\((.*)\))?$/)?.slice(1) ?? [];
         if (paramsString && !allowTypeParams) throw new Error(`Type parameters not allowed (${type})`);
 
+        if (!typeName) throw new Error("Invalid type: " + type);
+
         // TODO: only json-compatible params are allowed for now to avoid async
         const parsedParams = paramsString ? JSON.parse(`[${paramsString}]`) : undefined;
         return Type.get(typeName.includes(":") ? typeName : defaultNamespace+":"+typeName, parsedParams)