diff --git a/compiler/compiler.ts b/compiler/compiler.ts index 33770abd..76c1429c 100644 --- a/compiler/compiler.ts +++ b/compiler/compiler.ts @@ -27,7 +27,7 @@ import { BinaryCode } from "./binary_codes.ts"; import { Scope } from "../types/scope.ts"; import { ProtocolDataType } from "./protocol_types.ts"; import { Quantity } from "../types/quantity.ts"; -import { EXTENDED_OBJECTS, INHERITED_PROPERTIES, VOID, SLOT_WRITE, SLOT_READ, SLOT_EXEC, NOT_EXISTING, SLOT_GET, SLOT_SET, DX_IGNORE, DX_BOUND_LOCAL_SLOT, DX_REPLACE } from "../runtime/constants.ts"; +import { EXTENDED_OBJECTS, INHERITED_PROPERTIES, VOID, SLOT_WRITE, SLOT_READ, SLOT_EXEC, NOT_EXISTING, SLOT_GET, SLOT_SET, DX_IGNORE, DX_BOUND_LOCAL_SLOT, DX_REPLACE, DX_PTR } from "../runtime/constants.ts"; import { arrayBufferToBase64, base64ToArrayBuffer, buffer2hex, hex2buffer } from "../utils/utils.ts"; import { RuntimePerformance } from "../runtime/performance_measure.ts"; import { Conjunction, Disjunction, Logical, Negation } from "../types/logic.ts"; @@ -48,6 +48,8 @@ import { VolatileMap } from "../utils/volatile-map.ts"; // await wasm_init(); // wasm_init_runtime(); +let WebRTCInterface: undefined|typeof import("../network/communication-interfaces/webrtc-interface.ts").WebRTCInterface; + export const activePlugins:string[] = []; // for actions on variables, pointers, ... @@ -2513,7 +2515,7 @@ export class Compiler { Compiler.builder.insertVariable(SCOPE, assignment[0], ACTION_TYPE.GET, undefined, BinaryCode.INTERNAL_VAR); // parent Compiler.builder.handleRequiredBufferSize(SCOPE.b_index, SCOPE); SCOPE.uint8[SCOPE.b_index++] = BinaryCode.CHILD_SET; - Compiler.builder.insert(assignment[1], SCOPE, true, undefined, undefined, false); // insert key (don't save insert index for key value) + Compiler.builder.insert(assignment[1], SCOPE, true, undefined, undefined, false, false); // insert key (don't save insert index for key value) Compiler.builder.insertVariable(SCOPE, assignment[2], ACTION_TYPE.GET, undefined, BinaryCode.INTERNAL_VAR); // value } Compiler.builder.handleRequiredBufferSize(SCOPE.b_index+1, SCOPE); @@ -2832,6 +2834,15 @@ export class Compiler { // exception for functions: convert to Datex.Function & create Pointer reference (proxifyValue required!) if (value instanceof Function && !(value instanceof DatexFunction) && !(value instanceof JSTransferableFunction)) value = Pointer.proxifyValue(DatexFunction.createFromJSFunction(value)); + // excpetion for MediaStream: always register as WebRTC mediastream when transmitting + if (globalThis.MediaStream && value instanceof MediaStream) { + if (!WebRTCInterface) throw new Error("Cannot bind MediaStream to WebRTCInterface (not yet initialized)") + WebRTCInterface.registerMediaStream(value); + } + + // streams: always create pointer binding + if (value instanceof Stream) value = Pointer.proxifyValue(value); + // exception for Date: convert to Time (TODO: different approach?) if (value instanceof Date && !(value instanceof Time)) { try { @@ -2902,6 +2913,10 @@ export class Compiler { if (option_collapse && !SCOPE.options.no_create_pointers && !skip_first_collapse) SCOPE.uint8[SCOPE.b_index++] = BinaryCode.CREATE_POINTER; } + // temporary to find errors: throw if a cloned html element without a unique ptr id + if (globalThis.HTMLElement && value instanceof HTMLElement && value.hasAttribute("uix-ptr") && !(value as any)[DX_PTR]) console.error("Invalid cloned HTMLElement " + value.tagName + (value.hasAttribute("id")?"#"+value.getAttribute("id"):"")); + + // first value was collapsed (if no_proxify == false, it was still collapsed because it's not a pointer reference) // if (SCOPE.options.collapse_first_inserted) SCOPE.options.collapse_first_inserted = false; // reset SCOPE.options._first_insert_done = true; @@ -3172,7 +3187,7 @@ export class Compiler { // special exception: insert raw datex script (dxb Scope can be inserted normally (synchronous)) if (d instanceof DatexResponse && !(d.datex instanceof Scope)) await Compiler.builder.compilerInsert(SCOPE, d); - else Compiler.builder.insert(d, SCOPE); + else Compiler.builder.insert(d, SCOPE, undefined, undefined, undefined, undefined, false); // disable replace optimization for now to prevent invalid replacement, e.g. for path properties } isEffectiveValue = true; } @@ -5156,10 +5171,28 @@ export class Compiler { ) : SCOPE.buffer; } + /** + * Workaround: recursively cache all blob values in the iterable + */ + static cacheValues(iterable: Iterable, promises:Promise[] = []): Promise[] { + for (const val of iterable) { + if (val instanceof Blob) promises.push(Runtime.cacheValue(val)); + else if (typeof val == "object" && (val as any)?.[Symbol.iterator]) this.cacheValues((val as any), promises); + } + return promises; + } + // compile loop static async compileLoop(SCOPE:compiler_scope):Promise> { + // make sure WebRTC interface is loaded + ({ WebRTCInterface } = await import("../network/communication-interfaces/webrtc-interface.ts")); + + // cache inserted blob values + const promises = SCOPE.data ? this.cacheValues(SCOPE.data) : []; + if (promises.length) await Promise.all(promises); + const body_compile_measure = RuntimePerformance.enabled ? RuntimePerformance.startMeasure("compile time", "body") : undefined; if (!SCOPE.datex) SCOPE.datex = ";";//throw new CompilerError("DATEX Script is empty"); @@ -5541,12 +5574,12 @@ export class Compiler { /** create a dxb file created from a DATEX Script string and convert to data url */ static async datexScriptToDataURL(dx:string, type = ProtocolDataType.DATA):Promise { - let dxb = await Compiler.compile(dx, [], {sign:false, encrypt: false, type}) + const dxb = await Compiler.compile(dx, [], {sign:false, encrypt: false, type}) - let blob = new Blob([dxb], {type: "text/dxb"}); // workaround to prevent download + const blob = new Blob([dxb], {type: "text/dxb"}); // workaround to prevent download return new Promise(resolve=>{ - var a = new FileReader(); + const a = new FileReader(); a.onload = function(e) {resolve(e.target.result);} a.readAsDataURL(blob); }); @@ -5554,9 +5587,9 @@ export class Compiler { /** create a dxb file created from a DATEX Script string and convert to object url */ static async datexScriptToObjectURL(dx:string, type = ProtocolDataType.DATA):Promise { - let dxb = await Compiler.compile(dx, [], {sign:false, encrypt: false, type}) + const dxb = await Compiler.compile(dx, [], {sign:false, encrypt: false, type}) - let blob = new Blob([dxb], {type: "text/dxb"}); // workaround to prevent download + const blob = new Blob([dxb], {type: "text/dxb"}); // workaround to prevent download return URL.createObjectURL(blob); } diff --git a/datex_short.ts b/datex_short.ts index 9380445e..956dfe9a 100644 --- a/datex_short.ts +++ b/datex_short.ts @@ -52,6 +52,11 @@ declare global { */ const isolate: typeof Ref.disableCapturing + /** + * The local endpoint of the current runtime (alias for Datex.Runtime.endpoint) + */ + const localEndpoint: Endpoint + // conflict with UIX.template (confusing) // const template: typeof _template; } @@ -78,6 +83,7 @@ globalThis.timeout = _timeout; // @ts-ignore global globalThis.sync = _sync; + // can be used instead of import(), calls a DATEX get instruction, works for urls, endpoint, ... export async function get(dx:string|URL|Endpoint, assert_type?:Type | Class | string, context_location?:URL|string, plugins?:string[]):Promise { // auto retrieve location from stack @@ -170,7 +176,16 @@ Object.defineProperty(_datex, 'get', {value:(res:string, type?:Class|Type, locat // add globalThis.meta // Object.defineProperty(globalThis, 'meta', {get:()=>getMeta(), set:()=>{}, configurable:false}) -export const datex = _datex; +export const datex = _datex; // @ts-ignore global datex globalThis.datex = datex; // global access to datex and meta @@ -668,6 +683,8 @@ Object.defineProperty(globalThis, 'grantAccess', {value:grantAccess, configurabl Object.defineProperty(globalThis, 'grantPublicAccess', {value:grantPublicAccess, configurable:false}) Object.defineProperty(globalThis, 'revokeAccess', {value:revokeAccess, configurable:false}) +Object.defineProperty(globalThis, 'localEndpoint', {get: ()=>Runtime.endpoint, configurable:false}) + // @ts-ignore globalThis.get = get // @ts-ignore diff --git a/docs/api/runtime/js_interface.md b/docs/api/runtime/js_interface.md index aee02647..e877d67f 100644 --- a/docs/api/runtime/js_interface.md +++ b/docs/api/runtime/js_interface.md @@ -2,7 +2,7 @@ create a custom DATEX JS Interface for a type with handlers - serialize efficiently with the serialize function and de-serialize in the cast function -- do not use @sync classes in combination with an additional js_interface_configuration!; +- do not use struct classes in combination with an additional js_interface_configuration!; ## class **JSInterface** ### Properties diff --git a/docs/manual/01 Introduction.md b/docs/manual/01 Introduction.md index 58d517fc..7efbb31b 100644 --- a/docs/manual/01 Introduction.md +++ b/docs/manual/01 Introduction.md @@ -51,24 +51,26 @@ how pointers are synchronized between endpoints. ### Creating DATEX-compatible classes -With the `@sync` decorator, a class can be bound to a new DATEX type. +With the `struct` wrapper, a class can be bound to a new DATEX type. All instance properties decorated with `@property` are bound to the DATEX value and also visible when the value is shared between endpoints. Per default, the properties are local and only available in the current JavaScript context. ```ts -@sync class MyObject { - @property a = 10 - @property b = 20 - localProp = 4 -} +const MyObject = struct( + class { + @property a = 10 + @property b = 20 + localProp = 4 + } +) const obj = new MyObject(); ``` -Instances of a class marked with `@sync` are also automatically bound to a pointer when created (The value does not have to be explicitly wrapped in `$$()`). +Instances of a class wrapped with `struct` are also automatically bound to a pointer when created (The value does not have to be explicitly wrapped in `$$()`). -Read more about `@sync` classes [here](./11%20Classes.md). +Read more about `struct` classes [here](./12%20Classes.md). ### Persistent data diff --git a/docs/manual/03 Pointers.md b/docs/manual/03 Pointers.md index bcdf8f94..f20df457 100644 --- a/docs/manual/03 Pointers.md +++ b/docs/manual/03 Pointers.md @@ -88,9 +88,9 @@ Datex.Ref.isRef(map.get('y')) // true, was implicitly bound to a pointer There are some exceptions to this behaviour: 1. Primitive property values are not converted to pointers per default 2. Normal [class instances](./10%20Types.md#jsobject) (`js:Object`) are not converted to pointers per default. - Instances of [`@sync`](11%20Classes.md) classes are still converted to pointers + Instances of [`struct`](12%20Classes.md) classes are still converted to pointers 3. When a [class instances](./10%20Types.md#jsobject) is directly bound to a pointer with `$$()`, its - properties are not converted to pointers per default (like 2., this does not affect `@sync` class instances + properties are not converted to pointers per default (like 2., this does not affect `struct` class instances diff --git a/docs/manual/05 Eternal Pointers.md b/docs/manual/05 Eternal Pointers.md index adc1422d..a6864e84 100644 --- a/docs/manual/05 Eternal Pointers.md +++ b/docs/manual/05 Eternal Pointers.md @@ -71,7 +71,7 @@ This guarantees that `eternal` can be used synchronously (without `await`). For the following usecases, the asynchronous `lazyEternal`/`lazyEternalVar` label should be used instead of `eternal`/`eternalVar`: * A value that consumes lots of memory and is only actually needed when certain conditions are met - * A value that requires custom JavaScript bindings (e.g. a `@sync` class instance). JavaScript bindings cannnot be properly initialized at endpoint startup if the corresponding JavaScript class definition is not yet loaded. + * A value that requires custom JavaScript bindings (e.g. a `struct` class instance). JavaScript bindings cannnot be properly initialized at endpoint startup if the corresponding JavaScript class definition is not yet loaded. The `lazyEternal`/`lazyEternalVar` label can be used the same was as the `eternal` label, only requiring an additional `await`: diff --git a/docs/manual/11 Types.md b/docs/manual/11 Types.md index de90ec53..34ccb704 100644 --- a/docs/manual/11 Types.md +++ b/docs/manual/11 Types.md @@ -1,7 +1,61 @@ # Types -The DATEX Runtime comes with its own type system which can be mapped to JavaScript types. -DATEX types can be access via `Datex.Type`. +The DATEX Runtime comes with its own type system which is mapped to JavaScript types. +DATEX types can be accessed via `Datex.Type`. + +## Supported built-in JavaScript and Web types +| **JavaScript Type** | **Support** | **DATEX Type** | **Synchronizable** | **Limitations** | +|:-------------------------------|:----------------------|:----------------|:-------------------|:------------------------------------------------------------------------------------------| +| **string** | Full | `std:text` | Yes 1) | 3) | +| **number** | Full | `std:decimal` | Yes 1) | 3) | +| **bigint** | Full | `std:integer` | Yes 1) | 3) | +| **boolean** | Full | `std:boolean` | Yes 1) | 3) | +| **null** | Full | `std:null` | Yes 1) | - | +| **undefined** | Full | `std:void` | Yes 1) | - | +| **symbol** | Partial | `js:Symbol` | Yes 1) | Registered and well-known symbols are not yet supported | +| **Object** | Full | `std:Object` | Yes | Objects with prototypes other than `Object.prototype` or `null` are mapped to `js:Object` | +| **Object (with prototype)** | Sufficient | `js:Object` | Yes | No synchronisation for nested objects per default | +| **Array** | Full | `std:Array` | Yes | - | +| **Set** | Full | `std:Set` | Yes | - | +| **Map** | Full | `std:Map` | Yes | - | +| **WeakSet** | None | - | - | Cannot be implemented because `WeakSet` internals are not accessible. Alternative: `StorageWeakSet` | +| **WeakMap** | None | - | - | Cannot be implemented because `WeakMap` internals are not accessible. Alternative: `StorageWeakMap` | +| **Function** | Sufficient | `std:Function` | No (Immutable) | Functions always return a Promise, even if they are synchronous | +| **AsyncFunction** | Sufficient | `std:Function` | No (Immutable) | - | +| **Generator** | Sufficient | `js:AsyncGenerator` | No (Immutable) | Generators are always mapped to AsyncGenerators. Generators cannot be stored persistently | +| **AsyncGenerator** | Sufficient | `js:AsyncGenerator` | No (Immutable) | Generators cannot be stored persistently | +| **ArrayBuffer** | Partial | `std:buffer` | No | ArrayBuffer mutations are currently not synchronized | +| **Uint8Array** | Partial | `js:TypedArray/u8` | No | ArrayBuffer mutations are currently not synchronized | +| **Uint16Array** | Partial | `js:TypedArray/u16` | No | ArrayBuffer mutations are currently not synchronized | +| **Uint32Array** | Partial | `js:TypedArray/u32` | No | ArrayBuffer mutations are currently not synchronized | +| **BigUint64Array** | Partial | `js:TypedArray/u64` | No | ArrayBuffer mutations are currently not synchronized | +| **Int8Array** | Partial | `js:TypedArray/i8` | No | ArrayBuffer mutations are currently not synchronized | +| **Int16Array** | Partial | `js:TypedArray/i16` | No | ArrayBuffer mutations are currently not synchronized | +| **Int32Array** | Partial | `js:TypedArray/i32` | No | ArrayBuffer mutations are currently not synchronized | +| **BigInt64Array** | Partial | `js:TypedArray/i64` | No | ArrayBuffer mutations are currently not synchronized | +| **Float32Array** | Partial | `js:TypedArray/f32` | No | ArrayBuffer mutations are currently not synchronized | +| **Float64Array** | Partial | `js:TypedArray/f64` | No | ArrayBuffer mutations are currently not synchronized | +| **Promise** | Full | `js:Promise` | No (Immutable) | Promises cannot be stored persistently | +| **URL** | Partial | `std:url` | No | URL mutations are currently not synchronized | +| **Date** | Partial | `std:time` | No | `Date` objects are currently asymetrically mapped to DATEX `Time` objects | +| **Blob** | Full | `std:*/*,` | No (Immutable) | - | +| **File** | Full | `js:File` | No (Immutable) | - | +| **ReadableStream** | Full | `js:ReadableStream` | Yes | - | +| **WritableStream** | Full | `js:WritableStream` | Yes | - | +| **RegExp** | Partial | `js:RegExp` | No (Immutable) | RegExp values wrapped in a Ref are currently not synchronized | +| **WeakRef** | Full | `std:WeakRef` | No (Immutable) | - | +| **MediaStream** | Partial | `js:MediaStream` | Yes | MediaStreams are only supported in browsers that provide a [WebRTC API](https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API) | +| **Error** | Partial | `std:Error` | No | Error subclasses are not correctly mapped | +| **HTMLElement** | Partial 2) | `std:html` | No | HTML element mutations are currently not synchronized | +| **SVGElement** | Partial 2) | `std:svg` | No | SVG element mutations are currently not synchronized | +| **MathMLElement** | Partial 2) | `std:mathml` | No | MathML element mutations are currently not synchronized | +| **Document** | Partial 2) | `std:htmldocument` | No | Document mutations are currently not synchronized | +| **DocumentFragment** | Partial 2) | `std:htmlfragment` | No | DocumentFragment mutations are currently not synchronized | + + +1) Primitive JS values are immutable and cannot be synchronized on their own, but when wrapped in a Ref.
+2) [UIX-DOM](https://github.com/unyt-org/uix-dom) required
+3) The corresponding object values of primitive values (e.g. `new Number()` for `number`) are not supported
## Std types The `Datex.Type.std` namespace contains all the builtin (*std*) DATEX types that can be accessed as runtime values, e.g.: @@ -27,43 +81,6 @@ Datex.Type.std.boolean === boolean Datex.Type.std.Any === any ``` -## Supported built-in JS and Web types -| **JS Type** | **Support** | **DATEX Type** | **Synchronizable** | **Limitations** | -|:-------------------------------|:----------------------|:---------------|:-------------------|:------------------------------------------------------------------------------------------| -| **string** | Full | std:text | Yes 1) | 3) | -| **number** | Full | std:decimal | Yes 1) | 3) | -| **bigint** | Full | std:integer | Yes 1) | 3) | -| **boolean** | Full | std:boolean | Yes 1) | 3) | -| **null** | Full | std:null | Yes 1) | - | -| **undefined** | Full | std:void | Yes 1) | - | -| **symbol** | Partial | js:Symbol | Yes 1) | Registered and well-known symbols are not yet supported | -| **Object (without prototype)** | Full | std:Object | Yes | Objects with prototypes other than `Object.prototype` or `null` are mapped to `js:Object` | -| **Object** | Sufficient | js:Object | Yes | No synchronisation for nested objects per default | -| **Array** | Full | std:Array | Yes | - | -| **Set** | Full | std:Set | Yes | - | -| **Map** | Full | std:Map | Yes | - | -| **WeakSet** | None | - | - | Cannot be implemented because `WeakSet` internals are not accessible. Alternative: `StorageWeakSet` | -| **WeakMap** | None | - | - | Cannot be implemented because `WeakMap` internals are not accessible. Alternative: `StorageWeakMap` | -| **Function** | Sufficient | std:Function | No (Immutable) | Functions always return a Promise, even if they are synchronous | -| **AsyncFunction** | Sufficient | std:Function | No (Immutable) | - | -| **GeneratorFunction** | None | - | - | - | -| **ArrayBuffer** | Partial | std:buffer | No | ArrayBuffer mutations are currently not synchronized | -| **URL** | Partial | std:url | No | URL mutations are currently not synchronized | -| **Date** | Partial | std:time | No | `Date` objects are currently asymetrically mapped to DATEX `Time` objects | -| **RegExp** | Partial | js:RegExp | No (Immutable) | RegExp values wrapped in a Ref are currently not synchronized | -| **WeakRef** | Full | std:WeakRef | No (Immutable) | - | -| **Error** | Partial | std:Error | No | Error subclasses are not correctly mapped | -| **HTMLElement** | Partial 2) | std:html | No | HTML element mutations are currently not synchronized | -| **SVGElement** | Partial 2) | std:svg | No | SVG element mutations are currently not synchronized | -| **MathMLElement** | Partial 2) | std:mathml | No | MathML element mutations are currently not synchronized | -| **Document** | Partial 2) | std:htmldocument | No | Document mutations are currently not synchronized | -| **DocumentFragment** | Partial 2) | std:htmlfragment | No | DocumentFragment mutations are currently not synchronized | - - -1) Primitive JS values are immutable and cannot be synchronized on their own, but when wrapped in a Ref.
-2) [UIX-DOM](https://github.com/unyt-org/uix-dom) required
-3) The corresponding object values of primitive values (e.g. `new Number()` for `number`) are not supported
- ## Special JS types Most builtin JavaScript types, like Map, Set or Array have equivalent types in the DATEX std library. @@ -102,7 +119,7 @@ In contrast to `std:Object`, `js:Object` is used for JavaScript object with a pr Examples for `std:Object`s: * A plain object like `{x: 1, y: new Set()}` - * Object with `null` prototype `{__proto__: null, x: 1}` + * Object with `null` prototype (e.g. `{__proto__: null, x: 1}`) Examples for `js:Object`s: * A builtin object like `console` @@ -115,9 +132,15 @@ The property values of a `js:Object` are never automatically bound to pointers w DATEX has no native symbol type. JavaScript symbols are mapped to `js:Symbol` values. +### js:MediaStream + +This type mapping allows sharing `MediaStream` objects with audio/video tracks between browser endpoints. +Backend (Deno) endpoints are not yet supported due to missing support for WebRTC. + + ## Structs -The `struct` helper function allows you to define DATEX types with a +The `struct` helper function allows you to define custom DATEX types with a fixed structure. All `struct` values are represented as plain objects in JavaScripts. They can take any DATEX compatible value as properties. @@ -129,7 +152,7 @@ structs are more efficient than plain objects. **Usage**: ```ts -import { struct } from "datex-core-legacy/types/struct.ts"; +import { struct, inferType } from "datex-core-legacy/types/struct.ts"; // struct definition const MyStruct = struct({ diff --git a/docs/manual/12 Classes.md b/docs/manual/12 Classes.md index 5c296666..fe8ad762 100644 --- a/docs/manual/12 Classes.md +++ b/docs/manual/12 Classes.md @@ -4,17 +4,19 @@ Per default, most native JavaScript types (Arrays, Maps, Sets, primitive values, Instances of custom classes are mapped to a DATEX representation of the generic type `js:Object` per default and thus lose their class type and prototypes. -With the `@sync` decorator, a class can be bound to a new DATEX type. +With the struct wrapper, a class can be bound to a new DATEX type. All instance properties decorated with `@property` are bound to the DATEX value and also visible when the value is shared between endpoints. Per default, the properties are local and only available in the current JavaScript context. ```ts -@sync class MyObject { - @property a = 10 - @property b = 20 - localProp = 4 -} +const MyObject = struct( + class { + @property a = 10 + @property b = 20 + localProp = 4 + } +) const obj = new MyObject(); obj.a // 10 @@ -22,11 +24,11 @@ obj.$.a // Datex.Ref<10> ``` > [!NOTE] -> A `@sync` class instance can only be reconstructed correctly on another endpoint or in a later session if the JavaScript class definition is already loaded. Otherwise, the DATEX Runtime can only map the value to a generic object. +> A class instances bound to a DATEX type can only be reconstructed correctly on another endpoint or in a later session if the JavaScript class definition is already loaded. Otherwise, the DATEX Runtime can only map the value to a generic object. ## Automatic Pointer Binding -Instances of a class marked with `@sync` are also automatically bound to a pointer when created (The value does not have to be explicitly wrapped in `$$()`). +Instances of a struct class are also automatically bound to a pointer when created (The value does not have to be explicitly wrapped in `$$()`). All non-primitive properties of an instance (that are decorated with `@property`) are automatically bound to a new pointer if they don't have a pointer yet. @@ -38,13 +40,15 @@ But there is one significant difference: The calculated value returned by the ge This has essentially the same effect as [using the `always()` function](./03%20Pointers.md#creating-pointers). Whenever a pointer value that is referenced in the getter function is updated, the pointer value of the property is also updated. ```ts -@sync class MyObject { - @property a = 10 - @property b = 20 - @property get sum() { - return this.a + this.b - } +const MyObject = struct( + class MyObject { + @property a = 10 + @property b = 20 + @property get sum() { + return this.a + this.b + } } +) const obj = new MyObject(); obj.a // 10 @@ -61,22 +65,24 @@ obj.sum // 26 ## Constructors The normal JavaScript class constructor gets called every time an instance of a sync class is created. -When an existing instance of a sync class is shared with another endpoint, the constructor is +When an existing instance of a struct class is shared with another endpoint, the constructor is called again locally on the endpoint, which is not intended but can't be prevented. -We recommend using DATEX-compatible constructors instead, which are only ever called once at the initial creation of a sync class instance. -The DATEX constructor method is named `construct` and must be decorated with `@constructor`: +For this reason, struct classes cannot use normal constructors. +Instead, you can define a DATEX-compatible constructors named `construct` instead, which is only ever called once at the initial creation of a sync class instance: ```ts -@sync class MyObject { - @property a = 0 - @property b = 0 - - // DATEX-compatible constructor - @constructor construct() { - console.log("constructed a new MyObject") +const MyObject = struct( + class MyObject { + @property a = 0 + @property b = 0 + + // DATEX-compatible constructor + construct() { + console.log("constructed a new MyObject") + } } -} +) const obj = new MyObject() // "constructed a new MyObject" is logged ``` @@ -86,53 +92,38 @@ is not called again on the remote endpoint. You can also access constructor arguments like in a normal constructor: ```ts -@sync class MyObject { - @constructor construct(a: number, b: string) { - console.log("a", a) - console.log("b", a) +const MyObject = struct( + class { + construct(a: number, b: string) { + console.log("a", a) + console.log("b", a) + } } -} +) const obj = new MyObject(42, 'text') ``` -For correct typing, take a look at [this workaround](#workaround-to-get-correct-types). - ## Creating instances without `new` Class instances can also be created by calling the class as a function, passing in an object with the initial property values: ```ts -@sync class MyObject { - @property a = 0 - @property b = 0 -} +const MyObject = struct( + class { + @property a = 0 + @property b = 0 + } +) const obj = MyObject({a: 1, b: 2}) ``` -Currently, this results in a TypeScript error, but it runs without problems. -You can use [this workaround](#workaround-to-get-correct-types) to get rid of the TypeScript errors. - +> [!NOTE] +> Currently, when creating a struct instance this way, the TypeScript tranpiler wants you to set initial values for all properties, even if they +> are getters. This will be fixed in future versions as soon as TS decorator types are improved. -## Workaround to get correct types - -Currently, it is not possible to get the correct types for a sync class without some additional work. -You can add the following lines to a sync class to make the TypeScript compiler happy (this has no effect on the runtime behavior): -```ts -// sync class definition (private) -@sync class _MyObject { - @property a = 0 - @property b = 0 -} -// use these as public proxies for the actual class -export const MyObject = datexClass(_MyObject) -export type MyObject = datexClassType - -const obj1: MyObject = new MyObject() -const obj2: MyObject = MyObject({a: 1, b: 2}) -``` ## Using the raw API For more customization, you can directly use the [JavaScript interface API] which allows you to define custom DATEX mapping behaviours for specific JavaScript types. diff --git a/docs/manual/15 Storage Collections.md b/docs/manual/15 Storage Collections.md index 7e44488d..a14cf51a 100644 --- a/docs/manual/15 Storage Collections.md +++ b/docs/manual/15 Storage Collections.md @@ -49,7 +49,7 @@ Entries of a `StorageSet` can be efficiently queried by using the builtin patter For supported storage locations (e.g. sql storage), the pattern matching is directly performed in storage and non-matching entries are never loaded into RAM. > [!NOTE] -> Pattern matching currently only works with @sync class objects. +> Pattern matching currently only works with struct objects. ### Selecting by property @@ -59,11 +59,12 @@ The easiest way to match entries in a storage set is to provide one or multiple import { StorageSet } from "datex-core-legacy/types/storage-set.ts"; import { Time } from "unyt_core/datex_all.ts"; -@sync class User { - @property(string) name!: string - @property(number) age!: number - @property(Time) created!: Time -} +const User = struct({ + name: string, + age: nubmer, + created: Time +}) +type User = inferType const users = new StorageSet(); @@ -180,15 +181,18 @@ Example: ```ts import { ComputedProperty } from "datex-core-legacy/storage/storage.ts"; -@sync class Location { - @property(number) lat!: number - @property(number) lon!: number -} +const Location = struct({ + lat: number, + lon: number +}); +type Location = inferType -@sync class User { - @property(string) name!: string - @property(Location) location!: Location -} + +const User = struct({ + name: string, + location: Location +}) +type User = inferType const myPosition = {lat: 70.48, lon: -21.96} @@ -225,12 +229,13 @@ Calculates the sum of multiple properties or literal values Example: ```ts -@sync class TodoItem { - @property(number) completedTaskCount!: number - @property(number) openTaskCount!: number -} -const todoItems = new StorageSet() +const TodoItem = struct({ + completedTaskCount: number, + openTaskCount: number +}) +type TodoItem = inferType +const todoItems = new StorageSet() // sum of completedTaskCount and openTaskCount for a given TodoItem const totalTaskCount = ComputedProperty.sum( diff --git a/docs/manual/16 Communication Interfaces.md b/docs/manual/16 Communication Interfaces.md new file mode 100644 index 00000000..765ede02 --- /dev/null +++ b/docs/manual/16 Communication Interfaces.md @@ -0,0 +1,77 @@ +# Communication Interfaces + +DATEX communication is not restricted to a specific underlying communication channel. +DATEX blocks can be transmitted via WebSockets, TCP, WebRTC, HTTP or with Channel Messaging between browser workers or windows. +The DATEX JS Library provides a `CommunicationInterface` API which allows the implementation of other custom communication channels. + +## Connecting communication interfaces + +When calling `Supranet.connect()`, the DATEX Runtime automatically creates a connection with a communication interface. +Per default, WebSocket communication interfaces for public unyt.org relay endpoints (https://unyt.cc/nodes.dx) are used. + +To establish a connection over another communiction channel, you need to create a new instance of the corresponding `CommunicationInterface` and add it to the `communicationHub`: + +```ts +import { WebSocketClientInterface } from "datex-core-legacy/network/communication-interfaces/websocket-client-interface.ts"; +import { communicationHub } from "datex-core-legacy/network/communication-hub.ts"; + +// create a new WebSocketClientInterface connected to wss://example.com +const webSocketInterface = new WebSocketClientInterface("wss://example.com"); + +// add the interface to the communication hub +const connected = await communicationHub.addInterface(webSocketInterface) + +// true if connection could be establised: +console.log("connected:", connected); +``` + +The interface constructor signature can be different for other communication interfaces, but everything else works the same way. + +## Builtin commmunication interfaces + +### WebSocket +- +### HTTP +- +### Worker +- +### Window +- +### WebRTC + +#### Legend + + +#### Establishing a WebRTC connection + +To establish a [WebRTC](https://webrtc.org/) connection between two endpoints, each endpoint first gets its own public IP address from a STUN Server (Step 1 and 3). + +The two endpoints then negotiate a session by exchanging SDP offers and answers (Step 2 and 4). The DATEX WebRTC communication interface does not require an explicit [signaling server](https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Signaling_and_video_calling#the_signaling_server). Instead, SDP negotation happens directly between the two endpoints via DATEX (this might require a relay endpoint, which could be regarded as a kind of signaling server). + +![](./assets/webrtc/webrtc-connecting.png) + +The negotiated session can either use a direct or an indirect connection via a [TURN server](https://webrtc.org/getting-started/turn-server). + +#### Direct WebRTC connection + +If the two clients are in the same network or if [NAT traversal](https://en.wikipedia.org/wiki/NAT_traversal) is possible using public ip addresses of the two clients, +a direct connection can be established: + +![](./assets/webrtc/webrtc-direct-connection.png) + + +#### Indirect WebRTC connection + +As a fallback, WebRTC connections are relayed using a public TURN server. + +![](./assets/webrtc/webrtc-turn-connection.png) + + +#### WebRTC data exchange + +Once a connection is established, it can be used to transmit video or audio streams, as well as DATEX blocks using a [data channel](https://developer.mozilla.org/en-US/docs/Web/API/RTCDataChannel). +`MediaStream` tracks are transmitted directly via WebRTC, any other data is transmitted via the DATEX data channel. + +## Implementing custom communication interfaces + +- \ No newline at end of file diff --git a/docs/manual/assets/webrtc/legend.png b/docs/manual/assets/webrtc/legend.png new file mode 100644 index 00000000..ec6d2d02 Binary files /dev/null and b/docs/manual/assets/webrtc/legend.png differ diff --git a/docs/manual/assets/webrtc/webrtc-connecting.png b/docs/manual/assets/webrtc/webrtc-connecting.png new file mode 100644 index 00000000..545650cb Binary files /dev/null and b/docs/manual/assets/webrtc/webrtc-connecting.png differ diff --git a/docs/manual/assets/webrtc/webrtc-direct-connection.png b/docs/manual/assets/webrtc/webrtc-direct-connection.png new file mode 100644 index 00000000..a65823ef Binary files /dev/null and b/docs/manual/assets/webrtc/webrtc-direct-connection.png differ diff --git a/docs/manual/assets/webrtc/webrtc-turn-connection.png b/docs/manual/assets/webrtc/webrtc-turn-connection.png new file mode 100644 index 00000000..e65bf61e Binary files /dev/null and b/docs/manual/assets/webrtc/webrtc-turn-connection.png differ diff --git a/js_adapter/js_class_adapter.ts b/js_adapter/js_class_adapter.ts index 9ed2a12f..77ea763a 100644 --- a/js_adapter/js_class_adapter.ts +++ b/js_adapter/js_class_adapter.ts @@ -288,7 +288,10 @@ export class Decorators { originalClass[METADATA]?.[Decorators.FORCE_TYPE] && Object.hasOwn(originalClass[METADATA]?.[Decorators.FORCE_TYPE], 'constructor') ) normalizedType = originalClass[METADATA]?.[Decorators.FORCE_TYPE]?.constructor - else normalizedType = Type.get("ext", originalClass.name.replace(/^_/, '')); // remove leading _ from type name + else { + if (!originalClass.name) throw new Error("Cannot create DATEX type mapping for an anonymous class") + normalizedType = Type.get("ext", originalClass.name.replace(/^_/, '')); // remove leading _ from type name + } if (!callerFile && client_type == "deno" && normalizedType.namespace !== "std") { callerFile = getCallerInfo()?.[3]?.file ?? undefined; @@ -726,10 +729,8 @@ export type MethodKeys = { }[keyof T]; export type dc&{new (...args:unknown[]):unknown}, OT extends {new (...args:unknown[]):unknown} = ObjectRef> = - DatexClass & - OT & - // TODO: required instead of OT to disable default constructor, but leads to problems with typing - // Pick & + DatexClass & + Pick & ((struct:Omit, MethodKeys>>) => datexClassType); /** diff --git a/network/communication-hub.ts b/network/communication-hub.ts index 5b743094..a6691083 100644 --- a/network/communication-hub.ts +++ b/network/communication-hub.ts @@ -214,7 +214,7 @@ export class CommunicationHubHandler { ) const identifier = comInterface.toString() if (!mapping.has(identifier)) mapping.set(identifier, new Map()) - sockets.forEach(([endpoint, socket]) => mapping.get(identifier)!.set(socket, {directEndpoint: endpoint, directEndpointDynamicProperties: this.#endpointSockets.get(endpoint)!.get(socket)!, indirectEndpoints: new Map()})) + sockets.forEach(([endpoint, socket]) => mapping.get(identifier)!.set(socket, {directEndpoint: endpoint, directEndpointDynamicProperties: this.#endpointSockets.get(endpoint)?.get(socket), indirectEndpoints: new Map()})) } // indirect connections diff --git a/network/communication-interfaces/webrtc-interface.ts b/network/communication-interfaces/webrtc-interface.ts index 2f957919..6df18340 100644 --- a/network/communication-interfaces/webrtc-interface.ts +++ b/network/communication-interfaces/webrtc-interface.ts @@ -1,23 +1,45 @@ +import { logger } from "../../datex_all.ts"; +import { Pointer } from "../../runtime/pointers.ts"; import { Endpoint } from "../../types/addressing.ts"; +import { PermissionError } from "../../types/errors.ts"; +import { communicationHub } from "../communication-hub.ts"; import { CommunicationInterface, CommunicationInterfaceSocket, InterfaceDirection, InterfaceProperties } from "../communication-interface.ts"; @endpoint class WebRTCSignaling { - @property static offer(data:any) { - InterfaceManager.connect("webrtc", datex.meta!.sender, [data]); + /** + * signaling: offer|accept + */ + @property static negotiation(data: RTCSessionDescriptionInit) { + if (!datex.meta.signed) throw new PermissionError("unauthorized"); + if (data.type == "offer") WebRTCInterface.handleOffer(datex.meta.caller, data); + else if (data.type == "answer") WebRTCInterface.handleAnswer(datex.meta.caller, data); + else throw new Error("Unsupported session description type: " + data.type); } - @property static accept(data:any) { - WebRTCClientInterface.waiting_interfaces_by_endpoint.get(datex.meta!.sender)?.setRemoteDescription(data); - } + /** + * signaling: candidate + */ + @property static candidate(data: RTCIceCandidateInit) { + if (!datex.meta.signed) throw new PermissionError("unauthorized"); + WebRTCInterface.handleCandidate(datex.meta.caller, data); + } - @property static candidate(data:any) { - WebRTCClientInterface.waiting_interfaces_by_endpoint.get(datex.meta!.sender)?.addICECandidate(data); - } + /** + * request a specfic media stream - only works once a webrtc connection is established + */ + @property static requestMediaStream(ptrId: string) { + if (!datex.meta.signed) throw new PermissionError("unauthorized"); + return WebRTCInterface.requestMediaStream(datex.meta.caller, ptrId); + } } export class WebRTCInterfaceSocket extends CommunicationInterfaceSocket { + constructor(public inChannel: RTCDataChannel, public outChannel: RTCDataChannel) { + super() + } + handleReceive = (event: MessageEvent) => { if (event.data instanceof ArrayBuffer) { this.receive(event.data) @@ -25,16 +47,16 @@ export class WebRTCInterfaceSocket extends CommunicationInterfaceSocket { } open() { - this.worker.addEventListener("message", this.handleReceive); + this.inChannel.addEventListener("message", this.handleReceive); } close() { - this.worker.removeEventListener('message', this.handleReceive); + this.inChannel.removeEventListener('message', this.handleReceive); } send(dxb: ArrayBuffer) { try { - this.worker.postMessage(dxb) + this.outChannel.send(dxb) return true; } catch { @@ -55,21 +77,254 @@ export class WebRTCInterface extends CommunicationInterface { bandwidth: 50_000 } - constructor(endpoint: Endpoint) { + #socket?: WebRTCInterfaceSocket; + #sessionInit?: RTCSessionDescriptionInit + #endpoint: Endpoint; + #connection?: RTCPeerConnection; + + #stunServers: RTCIceServer[] + #turnServers: RTCIceServer[] + + #resolveTrackReceivedPromise!: (track: MediaStreamTrack) => void; + #trackReceivedPromise!: Promise + + + constructor(endpoint: Endpoint, sesionInit?: RTCSessionDescriptionInit, stunServers?: RTCIceServer[], turnServers?: RTCIceServer[]) { + if (WebRTCInterface.connectedInterfaces.has(endpoint)) throw new Error("A WebRTCInterface for " + endpoint + " already exists"); super() - const socket = new WebRTCInterfaceSocket(); - socket.endpoint = endpoint; - this.addSocket(socket); + + this.generateTrackReceivedPromise(); + this.#endpoint = endpoint; + this.#sessionInit = sesionInit; + this.properties.name = this.#endpoint.toString(); + this.#stunServers = stunServers ?? [{urls: 'stun:195.201.173.190:3478'}]; + this.#turnServers = turnServers ?? [{urls: 'turn:195.201.173.190:3478', username: '1525325424', credential: 'YuzkH/Th9BBaRj4ivR03PiCfr+E='}]; // TODO: get turn server credentials from server } connect() { - return true; + + WebRTCInterface.connectingInterfaces.set(this.#endpoint, this); + + const {promise, resolve} = Promise.withResolvers() + + // try to establish a WebRTC connection, exchange keys first + this.#connection = new RTCPeerConnection({ + iceServers: [...this.#stunServers, ...this.#turnServers] + }); + + const dataChannelOut = this.#connection.createDataChannel("datex", {protocol: "datex"}); + + // listeners + + this.#connection.onicecandidate = (e) => { + if (e.candidate) WebRTCSignaling.candidate.to(this.#endpoint)(e.candidate.toJSON()) + }; + + // connected + this.#connection.onconnectionstatechange = _ => { + switch(this.#connection?.connectionState) { + case "disconnected": + case "closed": + case "failed": { + if (this.#socket) this.removeSocket(this.#socket); + resolve(false); + } + + } + }; + + // received data channel + this.#connection.ondatachannel = (event) => { + this.logger.debug("received WebRTC data channel"); + const dataChannelIn = event.channel + + this.#socket = new WebRTCInterfaceSocket(dataChannelIn, dataChannelOut); + this.#socket.endpoint = this.#endpoint; + this.addSocket(this.#socket); + + if (WebRTCInterface.connectingInterfaces.has(this.#endpoint)) { + WebRTCInterface.connectingInterfaces.delete(this.#endpoint); + WebRTCInterface.connectedInterfaces.set(this.#endpoint, this); + } + resolve(true); + }; + + // received track + this.#connection.ontrack = (event) => { + this.#resolveTrackReceivedPromise(event.track); + this.generateTrackReceivedPromise() + } + + this.#connection.onnegotiationneeded = async () => { + try { + await this.#connection!.setLocalDescription(); + WebRTCSignaling.negotiation.to(this.#endpoint)(this.#connection!.localDescription!.toJSON()) + } + catch (e) { + console.error(e) + } + } + + // handle initial offer + if (this.#sessionInit) { + this.handleOffer(this.#sessionInit); + } + + return promise; + } + + generateTrackReceivedPromise() { + const {promise, resolve} = Promise.withResolvers(); + this.#trackReceivedPromise = promise; + this.#resolveTrackReceivedPromise = resolve; } - disconnect() {} + async collectMediaStreamTracks(count: number) { + if (!this.#connection) throw new Error("No WebRTC connection found to collect media stream tracks"); + + let tracks: MediaStreamTrack[]; + let previousCount = -1; + while ((tracks = this.#connection.getReceivers().map(receiver => receiver.track)).length < count) { + // throw if no new track was added (would lead to infinite loop) + if (previousCount == tracks.length) throw new Error("Track promise was resolved, but no new track added to connection"); + previousCount = tracks.length; + await this.#trackReceivedPromise + } + const mediaStream = new MediaStream(); + for (const track of tracks) { + mediaStream.addTrack(track); + } + return mediaStream; + } + + + async handleOffer(data: RTCSessionDescriptionInit) { + await this.#connection!.setRemoteDescription(data); + const answer = await this.#connection!.createAnswer(); + await this.#connection!.setLocalDescription(answer); + WebRTCSignaling.negotiation.to(this.#endpoint)(this.#connection!.localDescription!.toJSON()) + } + + + disconnect() { + this.#connection?.close(); + WebRTCInterface.connectedInterfaces.delete(this.#endpoint); + } + + override removeSocket(socket: WebRTCInterfaceSocket) { + super.removeSocket(socket); + WebRTCInterface.connectedInterfaces.delete(this.#endpoint); + WebRTCInterface.connectingInterfaces.delete(this.#endpoint); + } - cloneSocket(_socket: WebRTCInterfaceSocket) { - return new WebRTCInterfaceSocket(); + cloneSocket(socket: WebRTCInterfaceSocket) { + return new WebRTCInterfaceSocket(socket.inChannel, socket.outChannel); + } + + /** + * MediaStream handling + */ + + attachMediaStream(mediaStream: MediaStream) { + if (!this.#connection) throw new Error("No WebRTC connection found to attach media stream"); + for (const track of mediaStream.getTracks()) { + this.#connection.addTrack(track, mediaStream); + } + } + + static async getMediaStream(ptrId: string) { + const pointerOrigin = Pointer.getOriginFromPointerId(ptrId); + console.debug("requesting mediastream $" + ptrId + ", origin " + pointerOrigin) + if (!this.connectedInterfaces.has(pointerOrigin)) await communicationHub.addInterface(new WebRTCInterface(pointerOrigin)); + const interf = this.connectedInterfaces.get(pointerOrigin)!; + if (!interf.#connection) throw new Error("No WebRTC connection could be established to get media stream"); + + const tracksCount = await WebRTCSignaling.requestMediaStream.to(pointerOrigin)(ptrId); + const mediaStream = await interf.collectMediaStreamTracks(tracksCount); + console.debug("received mediastream",mediaStream) + return mediaStream; + } + + /** + * Register a media stream to be used by the WebRTC interface + * @param mediaStream + */ + static registerMediaStream(mediaStream: MediaStream) { + mediaStream = Pointer.proxifyValue(mediaStream) // make sure the media stream is bound to a pointer + this.registeredMediaStreams.set(Pointer.getId(mediaStream)!, new WeakRef(mediaStream)); + } + + + static registeredMediaStreams = new Map>(); // TODO: garbage collect weakrefs + static connectingInterfaces = new Map(); + static connectedInterfaces = new Map(); + + static getInterfaceForEndpoint(endpoint: Endpoint, findConnecting = true, findConnected = true) { + return ( + (findConnecting ? this.connectingInterfaces.get(endpoint) : undefined) ?? + (findConnected ? this.connectedInterfaces.get(endpoint) : undefined) + ) + } + + static getInterfaceConnection(endpoint: Endpoint, findConnecting = true, findConnected = true) { + if (!findConnecting && !findConnected) throw new Error("Cannot find WebRTC connection: both findConnecting and findConnected are false"); + + let interf = this.getInterfaceForEndpoint(endpoint, findConnecting, findConnected) + + // try with main instance + if (!interf) { + interf = this.getInterfaceForEndpoint(endpoint.main, findConnecting, findConnected); + if (interf) interf.#endpoint = endpoint; // specify received endpoint instance + } + + if (!interf) { + console.warn("No WebRTC interface found for endpoint " + endpoint); + return null; + } + else if (!interf.#connection) { + console.warn("No WebRTC connection found for endpoint " + endpoint); + return null; + } + else return interf.#connection + } + + static handleAnswer(endpoint: Endpoint, data: RTCSessionDescriptionInit) { + const connection = this.getInterfaceConnection(endpoint, true, true); + if (connection) connection.setRemoteDescription(data); + } + + static async handleOffer(endpoint: Endpoint, data: RTCSessionDescriptionInit) { + const interf = this.getInterfaceForEndpoint(endpoint, true, true); + // offer for existing interface + if (interf) { + await interf.handleOffer(data); + } + // create new interface + else { + logger.info("Received WebRTC offer from " + endpoint + ", creating new interface...") + await communicationHub.addInterface(new WebRTCInterface(endpoint, data)); + } + } + + + static handleCandidate(endpoint: Endpoint, data: RTCIceCandidateInit) { + const connection = this.getInterfaceConnection(endpoint, true, true); + if (connection) connection.addIceCandidate(data); + } + + static requestMediaStream(endpoint: Endpoint, ptrId: string) { + const connection = this.getInterfaceConnection(endpoint, false, true); + if (!connection) throw new Error("Cannot request MediaStream $" + ptrId + ": no WebRTC connection available"); + + const mediaStream = this.registeredMediaStreams.get(ptrId)?.deref(); + if (!mediaStream) throw new Error("MediaStream $" + ptrId + " not found"); + + console.debug(endpoint + " requested mediastream $" + ptrId) + const tracks = mediaStream.getTracks(); + for (const track of tracks) { + connection.addTrack(track, mediaStream); + } + return tracks.length } } \ No newline at end of file diff --git a/network/unyt.ts b/network/unyt.ts index 7336c270..c8f22cc6 100644 --- a/network/unyt.ts +++ b/network/unyt.ts @@ -16,6 +16,7 @@ import { Supranet } from "./supranet.ts"; import { Endpoint } from "../types/addressing.ts"; import { CommunicationInterfaceSocket } from "./communication-interface.ts"; import { communicationHub } from "./communication-hub.ts"; +import { endpoint_config } from "../runtime/endpoint_config.ts"; const logger = new Logger("unyt"); @@ -24,6 +25,7 @@ Supranet.onConnect = ()=>{ Unyt.endpoint_info.node = communicationHub.defaultSocket?.endpoint, Unyt.endpoint_info.interface = communicationHub.defaultSocket; Unyt.endpoint_info.datex_version = Runtime.VERSION; + Unyt.using_http_over_datex = endpoint_config.usingHTTPoverDATEX Unyt.logEndpointInfo(); } @@ -53,6 +55,8 @@ export class Unyt { static endpoint_info:EndpointInfo = {} + static using_http_over_datex = false; + static setAppInfo(app_info:AppInfo) { this.endpoint_info.app = app_info; } @@ -77,7 +81,18 @@ export class Unyt { // const urlEndpoint = (client_type == "browser" ? info.app?.backend : info.endpoint); // const endpointURLs = urlEndpoint ? [this.formatEndpointURL(urlEndpoint)] : []; // if (info.app?.domains) endpointURLs.unshift(...info.app.domains.map(d=>'https://'+d)) - return info.app?.dynamicData?.domains ? info.app.dynamicData.domains.map(d=>'https://'+d) : []; + const domains = info.app?.dynamicData?.domains ? info.app.dynamicData.domains.map(d=>'https://'+d.toLowerCase()) : []; + + // remove own domain + const ownDomain = globalThis.location?.origin.toLowerCase(); + if (ownDomain && domains.includes(ownDomain)) { + domains.splice(domains.indexOf(ownDomain),1); + } + + // add own domain to the front + if (ownDomain) domains.unshift(ownDomain + (this.using_http_over_datex?ESCAPE_SEQUENCES.UNYT_CYAN+' (HTTP-over-DATEX)'+ESCAPE_SEQUENCES.RESET:'')); + + return domains; } /** @@ -122,7 +137,7 @@ export class Unyt { else if (info.datex_version) content += `${ESCAPE_SEQUENCES.UNYT_GREY}DATEX VERSION${ESCAPE_SEQUENCES.COLOR_DEFAULT} ${info.datex_version.replaceAll('\n','')}\n` content += `\n` - if (info.app?.stage == "Development" && info.app.backend) content += `Worbench Access for this App: https://workbench.unyt.org/\?e=${info.app.backend.toString()}\n` + // if (info.app?.stage == "dev" && info.app.backend) content += `Worbench Access for this App: https://workbench.unyt.org/\?e=${info.app.backend.toString()}\n` content += `${ESCAPE_SEQUENCES.UNYT_GREY}© ${new Date().getFullYear().toString()} unyt.org` diff --git a/network/web_rtc_interface.ts b/network/web_rtc_interface.ts deleted file mode 100644 index b83b8df6..00000000 --- a/network/web_rtc_interface.ts +++ /dev/null @@ -1,137 +0,0 @@ -import { remote, expose } from "../mod.ts"; -import { Endpoint, Target, datex_advanced, scope } from "../datex_all.ts"; -import { client_type } from "../utils/constants.ts"; - -import InterfaceManager, { CommonInterface } from "./client.ts"; - -// signaling for WebRTC connections (used by WebRTCClientInterface) -@scope("webrtc") class _WebRTCSignaling { - - @expose @remote static offer(data:any) { - InterfaceManager.connect("webrtc", datex.meta!.sender, [data]); - } - - @expose @remote static accept(data:any) { - WebRTCClientInterface.waiting_interfaces_by_endpoint.get(datex.meta!.sender)?.setRemoteDescription(data); - } - - @expose @remote static candidate(data:any) { - WebRTCClientInterface.waiting_interfaces_by_endpoint.get(datex.meta!.sender)?.addICECandidate(data); - } -} -const WebRTCSignaling = datex_advanced(_WebRTCSignaling); - - -/** 'Relayed' interface */ -export class WebRTCClientInterface extends CommonInterface { - - override type = "webrtc" - - connection?: RTCPeerConnection - data_channel_out?: RTCDataChannel - data_channel_in?: RTCDataChannel - - override in = true - override out = true - override global = false - - static waiting_interfaces_by_endpoint:Map = new Map() - - constructor(endpoint: Endpoint){ - super(endpoint); - if (client_type != "browser") return; - WebRTCClientInterface.waiting_interfaces_by_endpoint.set(endpoint, this); - } - - public override disconnect(){ - super.disconnect(); - this.connection?.close() - } - - connect() { - const description:RTCSessionDescription = this.initial_arguments[0]; - - // deno-lint-ignore no-async-promise-executor - return new Promise(async resolve=>{ - - // try to establish a WebRTC connection, exchange keys first - this.connection = new RTCPeerConnection({ - iceServers: [{ urls: 'stun:stun.l.google.com:19302' }], - }); - - // listeners - this.connection.onicecandidate = (e) => { - if (e.candidate) WebRTCSignaling.to(this.endpoint).candidate(e.candidate.toJSON()) - }; - - // connected - this.connection.addEventListener('connectionstatechange', event => { - switch(this.connection?.connectionState) { - case "connected": - WebRTCClientInterface.waiting_interfaces_by_endpoint.delete(this.endpoint); - //resolve(true); - break; - case "disconnected": this.connected = false;resolve(false);break; - case "closed": this.connected = false;resolve(false);break; - case "failed": resolve(false); - } - }); - - // received data channel - this.connection.ondatachannel = (event) => { - this.data_channel_in = event.channel; - this.logger.success("received data channel"); - console.log(this.data_channel_in) - this.data_channel_in.onmessage = (event)=>{ - console.log("in>") - InterfaceManager.handleReceiveBlock(event.data, this.endpoint, this); - } - this.connected = true; - resolve(true); - }; - - // create an offer - if (!description) { - this.logger.success("initializing a WebRTC connection ...", this.connection); - - this.data_channel_out = this.connection.createDataChannel("datex", {protocol: "datex"}); - - // this.data_channel_out.addEventListener('open', e => console.warn('local data channel opened', e)); - // this.data_channel_out.addEventListener('close', e => console.warn('local data channel closed')); - - const offer = await this.connection.createOffer(); - await this.connection.setLocalDescription(offer); - WebRTCSignaling.to(this.endpoint).offer(this.connection.localDescription!.toJSON()) - } - - // accept offer - else { - this.logger.success("accepting a WebRTC connection request ..."); - - this.data_channel_out = this.connection.createDataChannel("datex", {protocol: "datex"}); - - await this.connection.setRemoteDescription(description) - const answer = await this.connection.createAnswer(); - await this.connection.setLocalDescription(answer); - - WebRTCSignaling.to(this.endpoint).accept(this.connection.localDescription!.toJSON()) - } - }) - - } - - async setRemoteDescription(description:any) { - await this.connection?.setRemoteDescription(description) - } - - async addICECandidate(candidate:object) { - await this.connection?.addIceCandidate(new RTCIceCandidate(candidate)); - } - - sendBlock(datex:ArrayBuffer){ - console.log("send",this,this.data_channel_out) - this.data_channel_out?.send(datex) - } -} - -InterfaceManager.registerInterface("webrtc", WebRTCClientInterface); \ No newline at end of file diff --git a/runtime/crypto.ts b/runtime/crypto.ts index 3a3393e4..d3982173 100644 --- a/runtime/crypto.ts +++ b/runtime/crypto.ts @@ -75,7 +75,10 @@ export class Crypto { /** Sign + Verify */ static async sign(buffer:ArrayBuffer): Promise { - if (!this.available) throw new SecurityError("Cannot sign DATEX requests, missing private keys"); + if (!this.available) { + displayFatalError('missing-private-keys'); + throw new SecurityError("Cannot sign DATEX requests, missing private keys"); + } return await crypto.subtle.sign(this.sign_key_options, this.rsa_sign_key, buffer); } static async verify(data:ArrayBuffer, signature:ArrayBuffer, endpoint:Endpoint): Promise { @@ -86,7 +89,10 @@ export class Crypto { /** Encypt + Decrypt (RSA) */ static async encrypt(buffer:ArrayBuffer, endpoint:Endpoint): Promise { - if (!this.available) throw new SecurityError("Cannot encrypt DATEX requests, missing private keys"); + if (!this.available) { + displayFatalError('missing-private-keys'); + throw new SecurityError("Cannot encrypt DATEX requests, missing private keys"); + } const keys = await this.getKeysForEndpoint(endpoint); if (!keys || keys[1]==null) throw new SecurityError("Cannot encrypt DATEX requests, could not get keys for endpoint"); return await crypto.subtle.encrypt("RSA-OAEP", keys[1], buffer); diff --git a/runtime/display.ts b/runtime/display.ts index d57f6b94..f6f88892 100644 --- a/runtime/display.ts +++ b/runtime/display.ts @@ -35,6 +35,11 @@ function setup() { export function displayFatalError(code:string, reset_btn = true) { + + // disable error screen (at least for now, immediately reset page) + errorReset(); + return; + // @ts-ignore if (client_type !== "deno" && globalThis.window && globalThis.document) { // @ts-ignore diff --git a/runtime/endpoint_config.ts b/runtime/endpoint_config.ts index e5f5e38f..4c50df50 100644 --- a/runtime/endpoint_config.ts +++ b/runtime/endpoint_config.ts @@ -10,6 +10,7 @@ import { cache_path } from "./cache_path.ts"; import { DatexObject } from "../types/object.ts"; import { Ref } from "./pointers.ts"; import { normalizePath } from "../utils/normalize-path.ts"; +import { ESCAPE_SEQUENCES } from "../datex_all.ts"; type channel_type = 'websocket'|'http' type node_config = { @@ -42,6 +43,8 @@ class EndpointConfig implements EndpointConfigData { public blockchain_relay?: Endpoint /*****************/ + public usingHTTPoverDATEX = false; + // not saved in endpoint config, loaded from https://unyt.cc/nodes.dx public publicNodes?: Map @@ -97,25 +100,32 @@ class EndpointConfig implements EndpointConfigData { // try to get from .dx url if (!path) path = new URL('/'+this.DX_FILE_NAME, globalThis.location.href) try { - const configUpdate = await datex.get(path); - if (!config) { - config = configUpdate; - logger.info("loaded endpoint config from " + path); - } - else { - for (const [key, value] of DatexObject.entries(configUpdate as Record)) { - DatexObject.set(config as Record, key as string, value); + + const dxResponse = await fetch(path); + + // check headers for http-over-datex + if (dxResponse.headers.get("x-http-over-datex") == "yes") this.usingHTTPoverDATEX = true; + + if (dxResponse.ok) { + const content = await dxResponse.text(); + const configUpdate = await Runtime.executeDatexLocally(content, undefined, undefined, path) as EndpointConfigData; + if (!config) { + config = configUpdate; + logger.info("loaded endpoint config from " + path); + } + else { + for (const [key, value] of DatexObject.entries(configUpdate as Record)) { + DatexObject.set(config as Record, key as string, value); + } + logger.debug("updated endpoint config from " + path); } - logger.debug("updated endpoint config from " + path); } + // ignore if no .dx file found + } catch (e) { - // ignore if no .dx file found - if (!(await fetch(path)).ok) {} - else { - logger.error `Could not read config file ${path}: ${e.toString()}`; - throw "invalid config file" - } + logger.error `Could not read config file ${path}: ${e.toString()}`; + throw "invalid config file" } } diff --git a/runtime/io_handler.ts b/runtime/io_handler.ts index d8f98c22..48327b81 100644 --- a/runtime/io_handler.ts +++ b/runtime/io_handler.ts @@ -18,7 +18,7 @@ export class IOHandler { // redirect std/print private static std_out:(data:any[])=>void|Promise = async (data:any[])=>{ for (let d=0; d = { handle_transform?: (value: T, pointer: Pointer) => void, // gets called when a transform function produces a new value, default override behaviour is ignored allow_transform_value?: (type: Type, pointer: Pointer) => string|true, // returns true if the value can be wrapped with wrap transform, allows pointer union types class?: Class, // the corresponding JS class or a prototype + no_instanceof?: boolean, // if true, don't use instanceof checks for the class, instead use isPrototypeOf prototype?: object, // the inherited JS prototype detect_class?: (value:any)=>boolean, // a function that returns whether the value has the type of the pseudo class @@ -158,7 +159,7 @@ export class JSInterface { public static handleConfigUpdate(type:Type, config:js_interface_configuration){ if (!type) throw new Error ("A type is required for a type configuration") - if (!config.class && !config.prototype) throw new Error ("The 'class' or 'prototype' property is required for a type configuration") + if (!config.class && !config.prototype && !config.detect_class) throw new Error ("The 'class', or 'prototype' property is required for a type configuration") config.__type = type; // save type to config for faster type reference @@ -181,14 +182,14 @@ export class JSInterface { // return if a value has a matching pseudo class configuration static hasPseudoClass(value:any):boolean { - for (let [_class, config] of this.configurations_by_class) { + for (const [_class, config] of this.configurations_by_class) { if (value instanceof _class) { // is class instance if (config.detect_class instanceof globalThis.Function && !(config.detect_class)(value)) return false; // detect class invalid return true; } } - for (let [proto, config] of this.configurations_by_prototype) { + for (const [proto, config] of this.configurations_by_prototype) { if (proto.isPrototypeOf(value)) { // has prototype if (config.detect_class instanceof globalThis.Function && !(config.detect_class)(value)) return false; // detect class invalid return true; @@ -271,20 +272,20 @@ export class JSInterface { // value -> static getValueDatexType(value:any):Type { - for (let [_class, config] of this.configurations_by_class) { - if (value instanceof _class) { + for (const [_class, config] of this.configurations_by_class) { + if (!config.detect_class && value instanceof _class) { return config.get_type ? config.get_type(value) : config.__type; } } - for (let [proto, config] of this.configurations_by_prototype) { + for (const [proto, config] of this.configurations_by_prototype) { if (proto.isPrototypeOf(value)) { return config.get_type ? config.get_type(value) : config.__type; } } // try afterwards (less likely to happen) - for (let [_class, config] of this.configurations_by_class) { + for (const [_class, config] of this.configurations_by_class) { if (config.detect_class instanceof globalThis.Function && (config.detect_class)(value) ) { return config.get_type ? config.get_type(value) : config.__type; } diff --git a/runtime/pointers.ts b/runtime/pointers.ts index 8ce5cda0..aede9948 100644 --- a/runtime/pointers.ts +++ b/runtime/pointers.ts @@ -828,11 +828,15 @@ export type ObjectRef = T // TODO: // {[K in keyof T]: MaybeObjectRef} - & - { - $:Proxy$ // reference to value (might generate pointer property, if no underlying pointer reference) - $$:PropertyProxy$ // always returns a pointer property reference - }; + & ( + // add $ and $$ properties if not already present + T extends {$: any, $$: any} ? + unknown: + { + $:Proxy$ // reference to value (might generate pointer property, if no underlying pointer reference) + $$:PropertyProxy$ // always returns a pointer property reference + } + ); export type MaybeObjectRef = T extends primitive|Function ? T : ObjectRef @@ -1389,11 +1393,11 @@ export class Pointer extends Ref { // returns the existing pointer for a value, or the value, if no pointer exists public static pointerifyValue(value:any):Pointer|any { - return value instanceof Pointer ? value : this.pointer_value_map.get(value) ?? value; + return value?.[DX_PTR] ?? (value instanceof Pointer ? value : this.pointer_value_map.get(value)) ?? value; } // returns pointer only if pointer exists - public static getByValue(value:RefOrValue):Pointer|undefined{ - return >this.pointer_value_map.get(Pointer.collapseValue(value)); + public static getByValue(value:RefOrValue): Pointer|undefined{ + return (value?.[DX_PTR] ?? this.pointer_value_map.get(Pointer.collapseValue(value))) as Pointer; } // returns pointer only if pointer exists @@ -1406,6 +1410,15 @@ export class Pointer extends Ref { return this.pointer_label_map.has(label) } + /** + * returns the pointer of a value if bound to a pointer, otherwise null + */ + public static getId(value:unknown) { + const pointer = this.pointerifyValue(value); + if (pointer instanceof Pointer) return pointer.id; + else return null + } + // get pointer by id, only returns pointer if pointer already exists static get(id:Uint8Array|string):Pointer|undefined { id = Pointer.normalizePointerId(id); @@ -1423,7 +1436,7 @@ export class Pointer extends Ref { private static loading_pointers:Map, scopeList: WeakSet}> = new Map(); // load from storage or request from remote endpoint if pointer not yet loaded - static load(id:string|Uint8Array, SCOPE?:datex_scope, only_load_local = false, sender_knows_pointer = true, allow_failure = true): Promise|Pointer|LazyPointer { + static load(id:string|Uint8Array, SCOPE?:datex_scope, only_load_local = false, sender_knows_pointer = true, allow_failure = false): Promise|Pointer|LazyPointer { // pointer already exists const existing_pointer = Pointer.get(id); @@ -2260,7 +2273,14 @@ export class Pointer extends Ref { this.finalizeSubscribe(override_endpoint, keep_pointer_origin) - if (!this.#loaded) return this.setValue(pointer_value); // set value + if (!this.#loaded) { + // special case: intercept MediaStream + if (globalThis.MediaStream && pointer_value instanceof MediaStream) { + const {WebRTCInterface} = await import("../network/communication-interfaces/webrtc-interface.ts") + return this.setValue(await WebRTCInterface.getMediaStream(this.id) as any); + } + else return this.setValue(pointer_value); // set value + } else return this; } @@ -3619,7 +3639,7 @@ export class Pointer extends Ref { } // fake primitives TODO: dynamic mime types - if (obj instanceof Quantity || obj instanceof Time || obj instanceof Type || obj instanceof URL || obj instanceof Target || obj instanceof Blob || (globalThis.HTMLImageElement && obj instanceof HTMLImageElement)) { + if (obj instanceof Quantity || obj instanceof Time || obj instanceof Type || obj instanceof URL || obj instanceof Target || obj instanceof Blob || (globalThis.MediaStream && obj instanceof MediaStream) || (globalThis.HTMLImageElement && obj instanceof HTMLImageElement)) { return obj; } diff --git a/runtime/runtime.ts b/runtime/runtime.ts index 3fa0fc0d..3dad8f69 100644 --- a/runtime/runtime.ts +++ b/runtime/runtime.ts @@ -215,6 +215,8 @@ type Source = '_source'|{}; */ export type trustedEndpointPermission = "remote-js-execution" | "protected-pointer-access" | "fallback-pointer-source"; +export const WRAPPED_PROMISE = Symbol("WRAPPED_PROMISE"); + export class Runtime { @@ -230,7 +232,8 @@ export class Runtime { ERROR_STACK_TRACES: true, // create detailed stack traces with all DATEX Errors NATIVE_ERROR_STACK_TRACES: true, // create detailed stack traces of JS Errors (NATIVE_ERROR_MESSAGES must be true) NATIVE_ERROR_DEBUG_STACK_TRACES: false, // also display internal DATEX library stack traces (hidden per default) - NATIVE_ERROR_MESSAGES: true // expose native error messages + NATIVE_ERROR_MESSAGES: true, // expose native error messages + DATEX_HTTP_ORIGIN: globalThis.location?.origin // http origin to use to send datex-over-http messages (e.g. GOODBYE messages), default is current origin for browsers } public static MIME_TYPE_MAPPING: Record> = { @@ -689,7 +692,7 @@ export class Runtime { else { if (!type) throw Error("Cannot infer type from URL content"); const mime_type = type.split("/"); - result = Runtime.castValue(Type.get("std",mime_type[0], mime_type[1].split(/;| /)[0]), content); + result = Runtime.collapseValueCast(await Runtime.castValue(Type.get("std",mime_type[0], mime_type[1].split(/;| /)[0]), content)); } } } @@ -738,7 +741,7 @@ export class Runtime { } else { if (!mime) throw Error("Cannot infer type from URL content - missing mime module"); - const content = (await getFileContent(url, true, true)); + const content = ((await getFileContent(url, true, true)) as Uint8Array).buffer; const ext = url.toString().match(/\.[^./]*$/)?.[0].replace(".",""); if (!ext) throw Error("Cannot infer type from URL content (no extension)"); const mime_type = mime.getType(ext); @@ -747,7 +750,7 @@ export class Runtime { if (raw) result = [content, mime_type]; else { const type = mime_type.split("/"); - result = Runtime.castValue(Type.get("std",type[0], type[1].split(/;| /)[0]), content); + result = Runtime.collapseValueCast(await Runtime.castValue(Type.get("std",type[0], type[1].split(/;| /)[0]), content)); } } @@ -1215,7 +1218,7 @@ export class Runtime { this.ownLastEndpoint = endpoint; this.lastEndpointUnloadHandler = () => { // send goodbye - if (this.goodbyeMessage) sendDatexViaHTTPChannel(this.goodbyeMessage); + if (this.goodbyeMessage) sendDatexViaHTTPChannel(this.goodbyeMessage, this.OPTIONS.DATEX_HTTP_ORIGIN); if (client_type == "browser") { try { // remove from localstorage list @@ -2261,7 +2264,18 @@ export class Runtime { return this.persistent_memory?.get(scope_identifier) } - /** casts an object, handles all types */ + /** + * Collapses WRAPPED_PROMISE values returned from Runtime.castValue + */ + public static collapseValueCast(value: unknown) { + if (value && value[WRAPPED_PROMISE]) return value[WRAPPED_PROMISE]; + else return value; + } + + /** + * Cast a value to a specific type + * Result must be awaited and collapsed with Runtime.collapseValueCast + */ public static async castValue(type:Type, value:any, context?:any, context_location?:URL, origin:Endpoint = Runtime.endpoint, no_fetch?:boolean, assigningPtrId?: string): Promise { let old_type = Type.ofValue(value); @@ -2272,12 +2286,12 @@ export class Runtime { let new_value:any = UNKNOWN_TYPE; // only handle std namespace / js:Object / js:Symbol - if (type.namespace == "std" || type == Type.js.NativeObject || type == Type.js.Symbol || type == Type.js.RegExp) { + if (type.namespace == "std" || type == Type.js.NativeObject || type == Type.js.Symbol || type == Type.js.RegExp || type == Type.js.MediaStream || type == Type.js.File || type.root_type == Type.js.TypedArray) { const uncollapsed_old_value = old_value if (old_value instanceof Pointer) old_value = old_value.val; // handle default casts - switch (type) { + switch (type.root_type) { // get case Type.std.Type:{ @@ -2299,6 +2313,7 @@ export class Runtime { if (old_value === VOID) new_value = globalThis.String() else if (old_value instanceof Markdown) new_value = old_value.toString(); else if (old_value instanceof ArrayBuffer) new_value = Runtime.utf8_decoder.decode(old_value); // cast to + else if (old_value instanceof TypedArray) new_value = Runtime.utf8_decoder.decode(old_value.buffer); // cast to else if (old_value instanceof Blob) new_value = await old_value.text() else new_value = this.valueToDatexString(value, false, true); break; @@ -2371,6 +2386,49 @@ export class Runtime { else new_value = INVALID; break; } + case Type.js.File: { + if (old_value && typeof old_value == "object") { + new_value = new File([old_value.content], old_value.name, { + type: old_value.type, + lastModified: old_value.lastModified + }); + } + else new_value = INVALID; + break; + } + case Type.js.TypedArray: { + if (old_value instanceof ArrayBuffer) { + switch (type.variation) { + case "u8": new_value = new Uint8Array(old_value); break; + case "u16": new_value = new Uint16Array(old_value); break; + case "u32": new_value = new Uint32Array(old_value); break; + case "u64": new_value = new BigUint64Array(old_value); break; + case "i8": new_value = new Int8Array(old_value); break; + case "i16": new_value = new Int16Array(old_value); break; + case "i32": new_value = new Int32Array(old_value); break; + case "i64": new_value = new BigInt64Array(old_value); break; + case "f32": new_value = new Float32Array(old_value); break; + case "f64": new_value = new Float64Array(old_value); break; + default: new_value = INVALID; + } + } + else new_value = INVALID; + + break; + } + + case Type.js.MediaStream: { + if (!globalThis.MediaStream) throw new Error("MediaStreams are not supported on this endpoint") + if (old_value === VOID || typeof old_value == "object") { + if (assigningPtrId) { + const {WebRTCInterface} = await import("../network/communication-interfaces/webrtc-interface.ts") + new_value = await WebRTCInterface.getMediaStream(assigningPtrId) + } + else new_value = new MediaStream(); + } + else new_value = INVALID; + break; + } case Type.std.Tuple: { if (old_value === VOID) new_value = new Tuple().seal(); else if (old_value instanceof Array){ @@ -2670,13 +2728,19 @@ export class Runtime { } // return new value - return new_value; + // dont' collapes Promise on return + if (new_value instanceof Promise) return {[WRAPPED_PROMISE]: new_value} + else return new_value; } static async cacheValue(value:unknown){ if (value instanceof Blob) { - (>value)[DX_SERIALIZED] = await value.arrayBuffer(); + if (!(value as any)[DX_SERIALIZED]) { + (value as any)[DX_SERIALIZED] = await value.arrayBuffer(); + // remove serialized value cache after 1 minute + setTimeout(()=>{delete (value as any)[DX_SERIALIZED]}, 60_000); + } } else { throw new ValueError("Cannot cache value of type " + Type.ofValue(value)); @@ -2692,6 +2756,12 @@ export class Runtime { let type:Type; + // file (special handling of DX_SERIALIZED inherited from Blob) + if (value instanceof File) { + if (!(value as any)[DX_SERIALIZED]) throw new RuntimeError("Cannot serialize file without cached content"); + return {name: value.name, type: value.type, size: value.size, lastModified: value.lastModified, content: (value as any)[DX_SERIALIZED]}; + } + // cached serialized (e.g. for mime types) if ((>value)?.[DX_SERIALIZED]) return (>value)[DX_SERIALIZED]; @@ -2710,6 +2780,8 @@ export class Runtime { // regex if (value instanceof RegExp) return value.flags ? new Tuple([value.source, value.flags]) : value.source; + if (globalThis.MediaStream && value instanceof MediaStream) return {}; + // weakref if (value instanceof WeakRef) { const deref = value.deref(); @@ -2870,7 +2942,7 @@ export class Runtime { const compiled = new Uint8Array(Compiler.encodeValue(value, undefined, false, deep_clone, collapse_value, false, true, false, true)); return wasm_decompile(compiled, formatted, colorized, resolve_slots).replace(/\r\n$/, ''); } catch (e) { - console.debug(e); + // console.debug(e); return this.valueToDatexString(value, formatted) } // return Decompiler.decompile(Compiler.encodeValue(value, undefined, false, deep_clone, collapse_value), true, formatted, formatted, false); @@ -2938,7 +3010,19 @@ export class Runtime { string = value.toString(); } else if (value instanceof ArrayBuffer || value instanceof TypedArray) { - string = "`"+buffer2hex(value instanceof Uint8Array ? value : new Uint8Array(value instanceof TypedArray ? value.buffer : value), null, null)+"`" + let type = "" + if (value instanceof Uint8Array) type = ""; + else if (value instanceof Uint16Array) type = ""; + else if (value instanceof Uint32Array) type = ""; + else if (value instanceof BigInt64Array) type = ""; + else if (value instanceof Int8Array) type = ""; + else if (value instanceof Int16Array) type = ""; + else if (value instanceof Int32Array) type = ""; + else if (value instanceof BigUint64Array) type = ""; + else if (value instanceof Float32Array) type = ""; + else if (value instanceof Float64Array) type = ""; + + string = type+"`"+buffer2hex(value instanceof Uint8Array ? value : new Uint8Array(value instanceof TypedArray ? value.buffer : value), undefined, null)+"`" } else if (value instanceof Scope) { const spaces = Array(this.FORMAT_INDENT*(depth+1)).join(' '); @@ -3228,14 +3312,14 @@ export class Runtime { let el = INNER_SCOPE.type_casts.pop(); let type:Type | undefined; // iterate over now remaining type casts - while (type = INNER_SCOPE.type_casts.pop()) el = await Runtime.castValue(type, el, INNER_SCOPE.ctx_intern, SCOPE.context_location, SCOPE.origin) + while ((type = INNER_SCOPE.type_casts.pop())) el = await Runtime.castValue(type, el, INNER_SCOPE.ctx_intern, SCOPE.context_location, SCOPE.origin) INNER_SCOPE.active_value = el; } // assignments: // get current active value - let el = INNER_SCOPE.active_value; + const el = INNER_SCOPE.active_value; let did_assignment = false; // make sure endpoint has access (TODO: INNER_SCOPE.active_value should never be set if no access) @@ -3266,7 +3350,7 @@ export class Runtime { } // subscribe for updates at pointer origin else { - ptr.subscribeForPointerUpdates(); + await ptr.subscribeForPointerUpdates(); } } @@ -4267,6 +4351,8 @@ export class Runtime { const pointer = el instanceof Pointer ? el : Pointer.getByValue(el); if (pointer instanceof Pointer) pointer.assertEndpointCanRead(SCOPE?.sender) + // make sure promise wrappers are collapsed + el = Runtime.collapseValueCast(el) // first make sure pointers are collapsed el = Ref.collapseValue(el) @@ -4405,8 +4491,8 @@ export class Runtime { // apply all casts if (INNER_SCOPE.type_casts) { - let type:Type - while (type = INNER_SCOPE.type_casts.pop()) { + let type:Type|undefined + while ((type = INNER_SCOPE.type_casts.pop())) { // workaround to get pointer that the new cast value will be assigned to const waitingPtr = [...INNER_SCOPE.waiting_ptrs??[]][0]; let ptrId: string|undefined; @@ -5055,7 +5141,7 @@ export class Runtime { // create conjunctive (&) value by extending const base_type = Type.ofValue(val); - const base = await base_type.createDefaultValue(); + const base = Runtime.collapseValueCast(await base_type.createDefaultValue()); DatexObject.extend(base, val); DatexObject.extend(base, el); INNER_SCOPE.active_value = base; @@ -5383,6 +5469,7 @@ export class Runtime { Object.defineProperty(scope.meta, 'encrypted', {value: header?.encrypted, writable: false, enumerable:true}); Object.defineProperty(scope.meta, 'signed', {value: header?.signed, writable: false, enumerable:true}); Object.defineProperty(scope.meta, 'sender', {value: header?.sender, writable: false, enumerable:true}); + Object.defineProperty(scope.meta, 'caller', {value: header?.sender, writable: false, enumerable:true}); Object.defineProperty(scope.meta, 'timestamp', {value: header?.timestamp, writable: false, enumerable:true}); Object.defineProperty(scope.meta, 'type', {value: header?.type, writable: false, enumerable:true}); diff --git a/types/function.ts b/types/function.ts index 8fca6c28..80857386 100644 --- a/types/function.ts +++ b/types/function.ts @@ -6,17 +6,17 @@ import { BROADCAST, Endpoint, endpoint_name, LOCAL_ENDPOINT, target_clause } fro import { Markdown } from "./markdown.ts"; import { Scope } from "./scope.ts"; import { Tuple } from "./tuple.ts"; -import type { datex_scope, compile_info, datex_meta } from "../utils/global_types.ts"; +import type { datex_scope, compile_info } from "../utils/global_types.ts"; import { Compiler } from "../compiler/compiler.ts"; import { Stream } from "./stream.ts" import { PermissionError, RuntimeError, TypeError, ValueError } from "./errors.ts"; import { ProtocolDataType } from "../compiler/protocol_types.ts"; import { DX_EXTERNAL_FUNCTION_NAME, DX_EXTERNAL_SCOPE_NAME, DX_TIMEOUT, VOID } from "../runtime/constants.ts"; import { Type, type_clause } from "./type.ts"; -import { callWithMetadata, callWithMetadataAsync, getMeta } from "../utils/caller_metadata.ts"; +import { callWithMetadata, callWithMetadataAsync } from "../utils/caller_metadata.ts"; import { Datex } from "../mod.ts"; import { Callable, ExtensibleFunction, getDeclaredExternalVariables, getDeclaredExternalVariablesAsync, getSourceWithoutUsingDeclaration } from "./function-utils.ts"; -import { Conjunction, Disjunction, Target } from "../datex_all.ts"; +import { Conjunction, Disjunction, Target, datex_meta } from "../datex_all.ts"; @@ -24,9 +24,10 @@ import { Conjunction, Disjunction, Target } from "../datex_all.ts"; * inject meta info to stack trace */ -export function getDefaultLocalMeta(){ +export function getDefaultLocalMeta(): datex_meta { return Object.seal({ sender: Runtime.endpoint, + caller: Runtime.endpoint, timestamp: new Date(), signed: true, encrypted: false, @@ -35,9 +36,10 @@ export function getDefaultLocalMeta(){ }) } -export function getUnknownMeta(){ +export function getUnknownMeta(): datex_meta { return Object.seal({ sender: BROADCAST, + caller: BROADCAST, timestamp: new Date(), signed: false, encrypted: false, @@ -289,7 +291,7 @@ export class Function any = (...args: any) => any> e // run in scope, get result try { - const res = await Runtime.datexOut(compile_info, endpoint, undefined, true, undefined, undefined, false, this.datex_timeout); + const res = await Runtime.datexOut(compile_info, endpoint, undefined, true, undefined, undefined, false, this.datex_timeout??(this as any)[DX_TIMEOUT]); return res; } catch (e) { // error occured during scope execution => scope is broken, can no longer be used => create new scope diff --git a/types/native_types.ts b/types/native_types.ts index ff9b1ab5..5a1c0db9 100644 --- a/types/native_types.ts +++ b/types/native_types.ts @@ -3,11 +3,12 @@ import { ValueError } from "./errors.ts"; import { Type } from "./type.ts"; import { Pointer } from "../runtime/pointers.ts"; import type { any_class } from "../utils/global_types.ts"; -import { INVALID, NOT_EXISTING } from "../runtime/constants.ts"; +import { DX_TIMEOUT, INVALID, NOT_EXISTING } from "../runtime/constants.ts"; import { Tuple } from "./tuple.ts"; import "../utils/auto_map.ts" import { ReactiveMapMethods } from "./reactive-methods/map.ts"; +import { Stream } from "./stream.ts"; // @ts-ignore accecssible to dev console globalThis.serializeImg = (img:HTMLImageElement)=> { @@ -289,6 +290,116 @@ Type.std.Set.setJSInterface({ values: (parent:Set) => [...parent], }) + +const AsyncGenerator = Object.getPrototypeOf(Object.getPrototypeOf((async function*(){})())); +const Generator = Object.getPrototypeOf(Object.getPrototypeOf((function*(){})())) + +Type.js.AsyncGenerator.setJSInterface({ + no_instanceof: true, + class: AsyncGenerator, + detect_class: (val) => AsyncGenerator.isPrototypeOf(val) || Generator.isPrototypeOf(val), + + is_normal_object: true, + proxify_children: true, + + serialize: value => { + return { + next: (...args: []|[unknown]) => value.next(...args), + return: (v: any) => value.return(v), + throw: (v: any) => value.throw(v), + } + }, + + + cast: value => { + if (value && value.next && value.return && value.throw) { + return async function*() { + let res = await value.next(); + while (!res.done) { + yield res.value; + res = await value.next(); + } + return value.return(); + }() + } + else return INVALID; + } +}) + +Type.js.Promise.setJSInterface({ + class: Promise, + + is_normal_object: true, + proxify_children: true, + + serialize: value => { + return { + then: (onFulfilled:any, onRejected:any) => { + // fails if promise was serialized and js function was reconstructed locally + try {value} + catch { + throw new Error("Promise cannot be restored - storing Promises persistently is not supported") + } + return value.then(onFulfilled, onRejected) + }, + catch: (onRejected:any) => { + // fails if promise was serialized and js function was reconstructed locally + try {value} + catch { + throw new Error("Promise cannot be restored - storing Promises persistently is not supported") + } + return value.catch(onRejected) + } + } + }, + + + cast: value => { + if (value && value.then && value.catch) { + (value.then as any)[DX_TIMEOUT] = Infinity; + (value.catch as any)[DX_TIMEOUT] = Infinity; + return new Promise((resolve, reject) => { + value + .then(resolve) + .catch(reject); + }) + } + else return INVALID; + } +}) + +Type.js.ReadableStream.setJSInterface({ + class: ReadableStream, + + serialize: value => { + return {stream:new Stream(value)} + }, + + cast: value => { + if (value?.stream instanceof Stream) { + return (value.stream as Stream).readable_stream + } + else return INVALID; + } +}) + +Type.js.WritableStream.setJSInterface({ + class: WritableStream, + + serialize: value => { + const stream = new Stream(); + stream.pipeTo(value); + return {stream}; + }, + + cast: value => { + if (value?.stream instanceof Stream) { + return (value.stream as Stream).writable_stream + } + else return INVALID; + } +}) + // override set prototype to make sure all sets are sorted at runtime when calling [...set] (TODO is that good?) // const set_iterator = Set.prototype[Symbol.iterator]; // Set.prototype[Symbol.iterator] = function() { diff --git a/types/stream.ts b/types/stream.ts index 44743a5d..89547daf 100644 --- a/types/stream.ts +++ b/types/stream.ts @@ -13,27 +13,50 @@ export class Stream implements StreamConsumer { controller?: ReadableStreamDefaultController readable_stream: ReadableStream + #writable_stream?: WritableStream + + get writable_stream() { + if (!this.#writable_stream) { + this.#writable_stream = new WritableStream({ + write: (chunk) => { + this.write(chunk); + } + }); + } + return this.#writable_stream; + } constructor(readable_stream?:ReadableStream) { this.readable_stream = readable_stream ?? new ReadableStream({ start: controller => {this.controller = controller} }); + // immediately start stream out if readable_stream is given + if (readable_stream) this.#startStreamOut() } started_ptr_stream = false + #startStreamOut() { + const ptr = Pointer.createOrGet(this); + if (ptr instanceof Pointer) { + logger.info("Start stream out for " + ptr.idString()); + setTimeout(() => { + ptr.startStreamOut(); // stream to all subscribers or origin, workaround: timeout to prevent stream init too early (TODO: fix) + }, 100) + } + else { + throw new Error("Could not bind stream to pointer.") + } + this.started_ptr_stream = true; + } + write(chunk: T, scope?: datex_scope) { // convert buffers // if (chunk instanceof TypedArray) chunk = (chunk).buffer; if (!this.started_ptr_stream && !scope) { // no scope -> assume called from JS, not DATEX - this.started_ptr_stream = true; - const ptr = Pointer.getByValue(this); - if (ptr instanceof Pointer) { - logger.info("Start stream out for " + ptr.idString()); - ptr.startStreamOut(); // stream to all subscribers or origin - } + this.#startStreamOut() } try { @@ -54,6 +77,17 @@ export class Stream implements StreamConsumer { } } + async pipeTo(out_stream:WritableStream) { + const reader = this.getReader(); + const writer = out_stream.getWriter(); + let next:ReadableStreamReadResult; + while (true) { + next = await reader.read() + if (next.done) break; + writer.write(next.value); + } + } + close() { this.controller?.close() this.controller = undefined; @@ -66,4 +100,11 @@ export class Stream implements StreamConsumer { return streams[0].getReader() } + values() { + return this.readable_stream.values() + } + + get [Symbol.asyncIterator]() { + return this.readable_stream.values.bind(this.readable_stream) + } } diff --git a/types/struct.ts b/types/struct.ts index 5448c2a4..f1b211fd 100644 --- a/types/struct.ts +++ b/types/struct.ts @@ -112,6 +112,12 @@ export function struct(defOrTypeName: StructuralTypeDefIn|Class|string, def?: St const hash = typeName ?? sha256(Runtime.valueToDatexStringExperimental(template)) const type = new Type("struct", hash).setTemplate(template); + // custom instanceof handling for structs + // TODO: does not work in Deno (throws runtime error when checking instanceof) + (type as any)[Symbol.hasInstance] = (val: unknown) => { + return Type.ofValue(val).matchesType(type); + } + if (callerFile) type.jsTypeDefModule = callerFile; type.proxify_children = true; return type as any diff --git a/types/type.ts b/types/type.ts index 1441a271..040ef00a 100644 --- a/types/type.ts +++ b/types/type.ts @@ -256,6 +256,10 @@ export class Type extends ExtensibleFunction { return (assign_to_object instanceof DatexObject ? DatexObject.seal(assign_to_object) : assign_to_object); } + + /** + * Result must be awaited and collapsed with Runtime.collapseValueCast + */ public createDefaultValue(context?:any, origin:Endpoint = Runtime.endpoint, context_location?: URL): Promise{ return Runtime.castValue(this, VOID, context, context_location, origin); } @@ -814,11 +818,11 @@ export class Type extends ExtensibleFunction { if (value?.[DX_TYPE]) return value[DX_TYPE]; // get type from pointer - let type:Type - if (type = Pointer.getByValue(value)?.type) return type; + let type:Type|undefined + if ((type = Pointer.getByValue(value)?.type)) return type; // get custom type - let custom_type = JSInterface.getValueDatexType(value); + const custom_type = JSInterface.getValueDatexType(value); if (!custom_type) { if (value === VOID) return Type.std.void; @@ -833,13 +837,34 @@ export class Type extends ExtensibleFunction { if (typeof value == "boolean") return >Type.std.boolean; if (typeof value == "symbol") return Type.js.Symbol as unknown as Type; if (value instanceof RegExp) return Type.js.RegExp as unknown as Type; - - if (value instanceof ArrayBuffer || value instanceof TypedArray) return >Type.std.buffer; + if (globalThis.MediaStream && value instanceof MediaStream) return Type.js.MediaStream as unknown as Type; + + if (value instanceof TypedArray) { + switch (value.constructor) { + case Uint8Array: return Type.js.TypedArray.getVariation('u8') as unknown as Type; + case Uint16Array: return Type.js.TypedArray.getVariation('u16') as unknown as Type; + case Uint32Array: return Type.js.TypedArray.getVariation('u32') as unknown as Type; + case BigUint64Array: return Type.js.TypedArray.getVariation('u64') as unknown as Type; + case Int8Array: return Type.js.TypedArray.getVariation('i8') as unknown as Type; + case Int16Array: return Type.js.TypedArray.getVariation('i16') as unknown as Type; + case Int32Array: return Type.js.TypedArray.getVariation('i32') as unknown as Type; + case BigInt64Array: return Type.js.TypedArray.getVariation('i64') as unknown as Type; + case Float32Array: return Type.js.TypedArray.getVariation('f32') as unknown as Type; + case Float64Array: return Type.js.TypedArray.getVariation('f64') as unknown as Type; + default: throw new ValueError("Invalid TypedArray"); + } + } + + if (value instanceof ArrayBuffer) return >Type.std.buffer; if (value instanceof Tuple) return >Type.std.Tuple; if (value instanceof Array) return >Type.std.Array; + if (value instanceof File) return Type.js.File as unknown as Type; + // mime types - if (value instanceof Blob) return Type.get("std", ...<[string, string]>value.type.split("/")) + if (value instanceof Blob) { + return Type.get("std", ...(value.type ? value.type.split("/") : ["application","octet-stream"]) as [string, string]) + } if (Runtime.mime_type_classes.has(value.constructor)) return Type.get("std", ...<[string, string]>Runtime.mime_type_classes.get(value.constructor).split("/")) if (value instanceof SyntaxError) return Type.std.SyntaxError; @@ -907,9 +932,27 @@ export class Type extends ExtensibleFunction { if (_forClass == globalThis.Boolean || globalThis.Boolean.isPrototypeOf(_forClass)) return >Type.std.boolean; if (_forClass == Symbol || Symbol.isPrototypeOf(_forClass)) return >Type.js.Symbol; if (_forClass == RegExp || RegExp.isPrototypeOf(_forClass)) return Type.js.RegExp as unknown as Type; + if (_forClass == File || File.isPrototypeOf(_forClass)) return Type.js.File as unknown as Type; + if (globalThis.MediaStream && _forClass == MediaStream) return Type.js.MediaStream as unknown as Type; if (_forClass == WeakRef || WeakRef.isPrototypeOf(_forClass)) return >Type.std.WeakRef; - if (_forClass == ArrayBuffer || TypedArray.isPrototypeOf(_forClass)) return >Type.std.buffer; + if (TypedArray.isPrototypeOf(_forClass)) { + switch (_forClass) { + case Uint8Array: return Type.js.TypedArray.getVariation('u8') as unknown as Type; + case Uint16Array: return Type.js.TypedArray.getVariation('u16') as unknown as Type; + case Uint32Array: return Type.js.TypedArray.getVariation('u32') as unknown as Type; + case BigUint64Array: return Type.js.TypedArray.getVariation('u64') as unknown as Type; + case Int8Array: return Type.js.TypedArray.getVariation('i8') as unknown as Type; + case Int16Array: return Type.js.TypedArray.getVariation('i16') as unknown as Type; + case Int32Array: return Type.js.TypedArray.getVariation('i32') as unknown as Type; + case BigInt64Array: return Type.js.TypedArray.getVariation('i64') as unknown as Type; + case Float32Array: return Type.js.TypedArray.getVariation('f32') as unknown as Type; + case Float64Array: return Type.js.TypedArray.getVariation('f64') as unknown as Type; + default: throw new ValueError("Invalid TypedArray"); + } + } + + if (_forClass == ArrayBuffer) return >Type.std.buffer; if (_forClass == Tuple || Tuple.isPrototypeOf(_forClass)) return >Type.std.Tuple; if (_forClass == Array || Array.isPrototypeOf(_forClass)) return >Type.std.Array; @@ -968,7 +1011,14 @@ export class Type extends ExtensibleFunction { NativeObject: Type.get("js:Object"), // special object type for non-plain objects (objects with prototype) - no automatic children pointer initialization TransferableFunction: Type.get("js:Function"), Symbol: Type.get("js:Symbol"), - RegExp: Type.get("js:RegExp") + RegExp: Type.get("js:RegExp"), + MediaStream: Type.get("js:MediaStream"), + File: Type.get("js:File"), + TypedArray: Type.get("js:TypedArray"), + AsyncGenerator: Type.get("js:AsyncGenerator"), + Promise: Type.get>("js:Promise"), + ReadableStream: Type.get("js:ReadableStream"), + WritableStream: Type.get("js:WritableStream"), } /** diff --git a/utils/cookies.ts b/utils/cookies.ts index e763db4d..22d0c239 100644 --- a/utils/cookies.ts +++ b/utils/cookies.ts @@ -4,8 +4,7 @@ import { client_type } from "./constants.ts"; const port = globalThis.location?.port; -const isSafari = (/^((?!chrome|android).)*safari/i.test(navigator.userAgent)); - +const browserIsSafariLocalhost = window.location?.hostname == "localhost" && (/^((?!chrome|android).)*safari/i.test(navigator.userAgent)); export function deleteCookie(name: string) { if (client_type !== "browser") { @@ -29,7 +28,7 @@ export function setCookie(name: string, value: string, expDays?: number) { expiryDate.setTime(expiryDate.getTime() + (expDays * 24 * 60 * 60 * 1000)); } const expires = expDays == 0 ? "" : "expires=" + expiryDate.toUTCString() + ";"; - document.cookie = name + "=" + value + "; " + expires + " path=/; SameSite=None;" + (isSafari ? "" :" Secure;") + document.cookie = name + "=" + value + "; " + expires + " path=/; SameSite=None;" + (browserIsSafariLocalhost ? "" :" Secure;") } export function getCookie(name: string) { diff --git a/utils/global_types.ts b/utils/global_types.ts index 045649b7..87a42501 100644 --- a/utils/global_types.ts +++ b/utils/global_types.ts @@ -143,7 +143,36 @@ export type datex_variables_scope = { [key: string]: any } & { // all available __encrypted: boolean, } -export type datex_meta = {encrypted?:boolean, signed?:boolean, sender:Endpoint, timestamp:Date, type:ProtocolDataType, local?:boolean}; +export type datex_meta = { + /** + * indicates if the datex block initiating the function call was encrypted + */ + encrypted?:boolean, + /** + * indicates if the datex block initiating the function call was signed + */ + signed?:boolean, + /** + * @deprecated use caller instead + */ + sender:Endpoint, + /** + * the endpoint that initiated the function call + */ + caller: Endpoint, + /** + * the time when the function call was initiated on the caller + */ + timestamp:Date, + /** + * the type of the datex block initiating the function call + */ + type:ProtocolDataType, + /** + * indicates if the function was called from the local endpoint + */ + local?:boolean +}; export type trace = { endpoint: Endpoint, diff --git a/utils/global_values.ts b/utils/global_values.ts index bbb1624d..8c9898a9 100644 --- a/utils/global_values.ts +++ b/utils/global_values.ts @@ -5,6 +5,7 @@ import {Logger} from "./logger.ts"; if (globalThis.process) throw new Error("node.js is currently not supported - use deno instead") export const TypedArray:typeof Uint8Array|typeof Uint16Array|typeof Uint32Array|typeof Int8Array|typeof Int16Array|typeof Int32Array|typeof BigInt64Array|typeof BigUint64Array|typeof Float32Array|typeof Float64Array = Object.getPrototypeOf(Uint8Array); +export type TypedArray = Uint8Array|Uint16Array|Uint32Array|Int8Array|Int16Array|Int32Array|BigInt64Array|BigUint64Array|Float32Array|Float64Array; // @ts-ignore const is_worker = (typeof WorkerGlobalScope !== 'undefined' && self instanceof WorkerGlobalScope); diff --git a/utils/iterable-weak-map.ts b/utils/iterable-weak-map.ts index 0dd02e55..42f6c92e 100644 --- a/utils/iterable-weak-map.ts +++ b/utils/iterable-weak-map.ts @@ -3,11 +3,20 @@ */ export class IterableWeakMap extends Map { + // additional internal WeakSet for faster lookups + #weakMap = new WeakMap() + set(key: K, value: V): this { + // already added + if (this.#weakMap.has(key)) return this; + this.#weakMap.set(key, value); return super.set(new WeakRef(key), value); } delete(key: K): boolean { + if (!this.#weakMap.has(key)) return false; + this.#weakMap.delete(key); + const deleting = new Set>() try { for (const keyRef of super.keys() as Iterable>) { @@ -30,10 +39,7 @@ export class IterableWeakMap extends Map { } has(key: K): boolean { - for (const unwrappedKey of this.keys()) { - if (unwrappedKey === key) return true; - } - return false; + return this.#weakMap.has(key); } *keys(): IterableIterator { @@ -54,9 +60,12 @@ export class IterableWeakMap extends Map { } get(key: K): V|undefined { - for (const [unwrappedKey, val] of this.entries()) { - if (unwrappedKey === key) return val; - } + return this.#weakMap.get(key); + } + + clear() { + this.#weakMap = new WeakMap() + super.clear() } *values(): IterableIterator { diff --git a/utils/iterable-weak-set.ts b/utils/iterable-weak-set.ts index 66d34dc3..7920934c 100644 --- a/utils/iterable-weak-set.ts +++ b/utils/iterable-weak-set.ts @@ -3,11 +3,20 @@ */ export class IterableWeakSet extends Set { + // additional internal WeakSet for faster lookups + #weakSet = new WeakSet() + add(value: T): this { + // already added + if (this.#weakSet.has(value)) return this; + this.#weakSet.add(value); return super.add(new WeakRef(value)); } delete(value: T): boolean { + if (!this.#weakSet.has(value)) return false; + this.#weakSet.delete(value); + const deleting = new Set>() try { for (const valRef of super.values() as Iterable>) { @@ -29,10 +38,7 @@ export class IterableWeakSet extends Set { } has(value: T): boolean { - for (const val of this.values()) { - if (val === value) return true; - } - return false; + return this.#weakSet.has(value); } *values(): IterableIterator { @@ -52,6 +58,11 @@ export class IterableWeakSet extends Set { } } + clear() { + this.#weakSet = new WeakSet() + super.clear() + } + keys(): IterableIterator { return this.values() } diff --git a/utils/polyfills.ts b/utils/polyfills.ts index 3e43910b..1e9c0efd 100644 --- a/utils/polyfills.ts +++ b/utils/polyfills.ts @@ -48,3 +48,52 @@ Map.groupBy ??= function groupBy (iterable, callbackfn) { } return map } + + +/** + * A polyfill for `ReadableStream.protototype[Symbol.asyncIterator]`, + * aligning as closely as possible to the specification. + * + * @see https://streams.spec.whatwg.org/#rs-asynciterator + * @see https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream#async_iteration + */ +ReadableStream.prototype.values ??= function({ preventCancel = false } = {}) { + const reader = this.getReader(); + return { + async next() { + try { + const result = await reader.read(); + if (result.done) { + reader.releaseLock(); + } + return result; + } catch (e) { + reader.releaseLock(); + throw e; + } + }, + async return(value) { + if (!preventCancel) { + const cancelPromise = reader.cancel(value); + reader.releaseLock(); + await cancelPromise; + } else { + reader.releaseLock(); + } + return { done: true, value }; + }, + [Symbol.asyncIterator]() { + return this; + } + }; + }; + + +ReadableStream.prototype[Symbol.asyncIterator] ??= ReadableStream.prototype.values; + +declare global { + interface ReadableStream { + values(options?: { preventCancel?: boolean }): AsyncIterable; + [Symbol.asyncIterator](): AsyncIterableIterator; + } +} \ No newline at end of file