diff --git a/compiler/compiler.ts b/compiler/compiler.ts index c233b072..3f6fae9d 100644 --- a/compiler/compiler.ts +++ b/compiler/compiler.ts @@ -13,7 +13,7 @@ import { Logger } from "../utils/logger.ts"; const logger = new Logger("datex compiler"); -import { ReadableStream, Runtime, StaticScope} from "../runtime/runtime.ts"; +import { ReadableStream, Runtime } from "../runtime/runtime.ts"; import { Endpoint, IdEndpoint, Target, WildcardTarget, Institution, Person, BROADCAST, target_clause, endpoints, LOCAL_ENDPOINT } from "../types/addressing.ts"; import { Pointer, PointerProperty, Ref } from "../runtime/pointers.ts"; import { CompilerError, RuntimeError, Error as DatexError, ValueError } from "../types/errors.ts"; @@ -45,8 +45,8 @@ import { client_type } from "../utils/constants.ts"; import { normalizePath } from "../utils/normalize-path.ts"; import { VolatileMap } from "../utils/volatile-map.ts"; -await wasm_init(); -wasm_init_runtime(); +// await wasm_init(); +// wasm_init_runtime(); export const activePlugins:string[] = []; @@ -2860,12 +2860,32 @@ export class Compiler { // handle pointers with transform (always ...) // only if not ignore_first_collapse or, if ignore_first_collapse and keep_first_transform is enabled - if (!SCOPE.options.no_create_pointers && value instanceof Pointer && value.transform_scope && (value.force_local_transform || !skip_first_collapse || SCOPE.options.keep_first_transform)) { - SCOPE.options._first_insert_done = true; // set to true before next insert + if (!SCOPE.options.no_create_pointers && value instanceof Pointer && (value.force_local_transform || !skip_first_collapse || SCOPE.options.keep_first_transform)) { + + if (value.transform_scope) { + SCOPE.options._first_insert_done = true; // set to true before next insert + Compiler.builder.insert_transform_scope(SCOPE, value.transform_scope); + return; + } - Compiler.builder.insert_transform_scope(SCOPE, value.transform_scope); - - return; + else if (value.smart_transform_method) { + console.warn("DATEX serialization of JS transforms is not yet supported: ", value.smart_transform_method.toString()) + // TODO: + // const isTransferableFn = value.smart_transform_method instanceof JSTransferableFunction; + // const transferableFn = isTransferableFn ? value.smart_transform_method as unknown as JSTransferableFunction : JSTransferableFunction.create(value.smart_transform_method); + // if (!isTransferableFn) transferableFn.source = `() => always(${transferableFn.source})`; + + // value.smart_transform_method = transferableFn; + // Compiler.builder.handleRequiredBufferSize(SCOPE.b_index+2, SCOPE); + // SCOPE.uint8[SCOPE.b_index++] = BinaryCode.SUBSCOPE_START; + // SCOPE.uint8[SCOPE.b_index++] = BinaryCode.CREATE_POINTER; + // Compiler.builder.insert(transferableFn, SCOPE, is_root, parents, unassigned_children); + // Compiler.builder.handleRequiredBufferSize(SCOPE.b_index+2, SCOPE); + // SCOPE.uint8[SCOPE.b_index++] = BinaryCode.SUBSCOPE_END; + // SCOPE.uint8[SCOPE.b_index++] = BinaryCode.VOID; + // return; + } + } // indirect reference pointer diff --git a/compiler/unit_codes.ts b/compiler/unit_codes.ts index 5060f55d..e8ee576c 100644 --- a/compiler/unit_codes.ts +++ b/compiler/unit_codes.ts @@ -174,6 +174,17 @@ export const UnitAliases = { } as const; +export type UnitAliasUnits = { + [k in keyof typeof UnitAliases]: typeof UnitAliases[k][2] +} + +// reverse mapping for all aliases of a unit e.g. Unit.SECOND -> "min" | "h" | "d" | "a" | "yr" +export type UnitAliasesMap = { + [key in UnitAliasUnits[keyof UnitAliasUnits]]: { + [k in keyof UnitAliasUnits]: UnitAliasUnits[k] extends key ? k : never + }[keyof UnitAliasUnits] +} + // prefixes ----------------------------------------------------------------------------------------- @@ -282,8 +293,10 @@ export type unit_symbol = unit_base_symbol | keyof typeof UnitCodeBySymbolShortF export type unit_prefix = keyof typeof UnitPrefixCodeBySymbol; -export type code_to_symbol = Combine; -export type symbol_to_code = Combine; +export type code_to_symbol = typeof UnitSymbol[C] | typeof UnitSymbolShortFormsByCode[C & keyof typeof UnitSymbolShortFormsByCode]; +export type symbol_to_code = typeof UnitCodeBySymbol & UnitAliasUnits & typeof UnitCodeBySymbolShortForms; + +export type symbol_prefix_combinations = S|`${unit_prefix}${S}` -export type code_to_extended_symbol = C extends null ? string : code_to_symbol[C]|`${unit_prefix}${code_to_symbol[C]}` +export type code_to_extended_symbol = C extends null ? string : symbol_prefix_combinations|UnitAliasesMap[C & keyof UnitAliasesMap]> export type symbol_with_prefix = unit_symbol|`${unit_prefix}${unit_symbol}` \ No newline at end of file diff --git a/datex_all.ts b/datex_all.ts index fc4428d0..bd30ba9c 100644 --- a/datex_all.ts +++ b/datex_all.ts @@ -4,10 +4,10 @@ export * from "./runtime/runtime.ts"; // js_adapter export * from "./js_adapter/js_class_adapter.ts"; -export * from "./js_adapter/legacy_decorators.ts"; +export * from "./js_adapter/decorators.ts"; // utils -export * from "./utils/global_types.ts"; +export type * from "./utils/global_types.ts"; export * from "./utils/global_values.ts"; export * from "./utils/logger.ts"; export * from "./utils/observers.ts"; @@ -42,7 +42,7 @@ export * from "./runtime/cache_path.ts"; export * from "./storage/storage.ts"; // types -export * from "./types/abstract_types.ts"; +export type * from "./types/abstract_types.ts"; export * from "./types/addressing.ts"; export * from "./types/assertion.ts"; export * from "./types/logic.ts"; diff --git a/datex_short.ts b/datex_short.ts index 0b782493..e5fea4d9 100644 --- a/datex_short.ts +++ b/datex_short.ts @@ -1,10 +1,10 @@ // shortcut functions // import { Datex } from "./datex.ts"; -import { baseURL, Runtime, PrecompiledDXB, Type, Pointer, Ref, PointerProperty, primitive, any_class, Target, IdEndpoint, TransformFunctionInputs, AsyncTransformFunction, TransformFunction, Markdown, MinimalJSRef, RefOrValue, PartialRefOrValueObject, datex_meta, ObjectWithDatexValues, Compiler, endpoint_by_endpoint_name, endpoint_name, Storage, compiler_scope, datex_scope, DatexResponse, target_clause, ValueError, logger, Class, getUnknownMeta, Endpoint, INSERT_MARK, CollapsedValueAdvanced, CollapsedValue, SmartTransformFunction, compiler_options, activePlugins, METADATA, handleDecoratorArgs, RefOrValueObject, PointerPropertyParent, InferredPointerProperty, RefLike } from "./datex_all.ts"; +import { baseURL, Runtime, PrecompiledDXB, Type, Pointer, Ref, PointerProperty, primitive, Target, IdEndpoint, Markdown, MinimalJSRef, RefOrValue, PartialRefOrValueObject, datex_meta, ObjectWithDatexValues, Compiler, endpoint_by_endpoint_name, endpoint_name, Storage, compiler_scope, datex_scope, DatexResponse, target_clause, ValueError, logger, Class, getUnknownMeta, Endpoint, INSERT_MARK, CollapsedValueAdvanced, CollapsedValue, SmartTransformFunction, compiler_options, activePlugins, METADATA, handleDecoratorArgs, RefOrValueObject, PointerPropertyParent, InferredPointerProperty, RefLike, dc } from "./datex_all.ts"; /** make decorators global */ -import { assert as _assert, property as _property, sync as _sync, endpoint as _endpoint, template as _template, jsdoc as _jsdoc} from "./datex_all.ts"; +import { assert as _assert, timeout as _timeout, entrypoint as _entrypoint, entrypointProperty as _entrypointProperty, property as _property, struct as _struct, endpoint as _endpoint, sync as _sync} from "./datex_all.ts"; import { effect as _effect, always as _always, reactiveFn as _reactiveFn, asyncAlways as _asyncAlways, toggle as _toggle, map as _map, equals as _equals, selectProperty as _selectProperty, not as _not } from "./functions.ts"; export * from "./functions.ts"; import { NOT_EXISTING, DX_SLOTS, SLOT_GET, SLOT_SET } from "./runtime/constants.ts"; @@ -22,10 +22,16 @@ declare global { const property: typeof _property; const assert: typeof _assert; - const jsdoc: typeof _jsdoc; - const sync: typeof _sync; + const struct: typeof _struct; const endpoint: typeof _endpoint; + const entrypoint: typeof _entrypoint; + const entrypointProperty: typeof _entrypointProperty; + const timeout: typeof _timeout; const always: typeof _always; + /** + * @deprecated Use struct(class {...}) instead; + */ + const sync: typeof _sync; const asyncAlways: typeof _asyncAlways; const reactiveFn: typeof _reactiveFn; const toggle: typeof _toggle; @@ -53,13 +59,17 @@ globalThis.property = _property; globalThis.assert = _assert; // @ts-ignore global -globalThis.sync = _sync; +globalThis.struct = _struct; // @ts-ignore global globalThis.endpoint = _endpoint; -// // @ts-ignore global -// globalThis.template = _template; // @ts-ignore global -globalThis.jsdoc = _jsdoc; +globalThis.entrypoint = _entrypoint; +// @ts-ignore global +globalThis.entrypointProperty = _entrypointProperty; +// @ts-ignore global +globalThis.timeout = _timeout; +// @ts-ignore global +globalThis.sync = _sync; // can be used instead of import(), calls a DATEX get instruction, works for urls, endpoint, ... export async function get(dx:string|URL|Endpoint, assert_type?:Type | Class | string, context_location?:URL|string, plugins?:string[]):Promise { diff --git a/docs/manual/11 Types.md b/docs/manual/11 Types.md index 3f0c9753..de90ec53 100644 --- a/docs/manual/11 Types.md +++ b/docs/manual/11 Types.md @@ -4,7 +4,7 @@ The DATEX Runtime comes with its own type system which can be mapped to JavaScri DATEX types can be access via `Datex.Type`. ## Std types -The `Datex.Type.std` namespace contains all the builtin (*std*) DATEX types, e.g.: +The `Datex.Type.std` namespace contains all the builtin (*std*) DATEX types that can be accessed as runtime values, e.g.: ```ts // primitive types Datex.Type.std.text @@ -27,6 +27,43 @@ Datex.Type.std.boolean === boolean Datex.Type.std.Any === any ``` +## Supported built-in JS and Web types +| **JS Type** | **Support** | **DATEX Type** | **Synchronizable** | **Limitations** | +|:-------------------------------|:----------------------|:---------------|:-------------------|:------------------------------------------------------------------------------------------| +| **string** | Full | std:text | Yes 1) | 3) | +| **number** | Full | std:decimal | Yes 1) | 3) | +| **bigint** | Full | std:integer | Yes 1) | 3) | +| **boolean** | Full | std:boolean | Yes 1) | 3) | +| **null** | Full | std:null | Yes 1) | - | +| **undefined** | Full | std:void | Yes 1) | - | +| **symbol** | Partial | js:Symbol | Yes 1) | Registered and well-known symbols are not yet supported | +| **Object (without prototype)** | Full | std:Object | Yes | Objects with prototypes other than `Object.prototype` or `null` are mapped to `js:Object` | +| **Object** | Sufficient | js:Object | Yes | No synchronisation for nested objects per default | +| **Array** | Full | std:Array | Yes | - | +| **Set** | Full | std:Set | Yes | - | +| **Map** | Full | std:Map | Yes | - | +| **WeakSet** | None | - | - | Cannot be implemented because `WeakSet` internals are not accessible. Alternative: `StorageWeakSet` | +| **WeakMap** | None | - | - | Cannot be implemented because `WeakMap` internals are not accessible. Alternative: `StorageWeakMap` | +| **Function** | Sufficient | std:Function | No (Immutable) | Functions always return a Promise, even if they are synchronous | +| **AsyncFunction** | Sufficient | std:Function | No (Immutable) | - | +| **GeneratorFunction** | None | - | - | - | +| **ArrayBuffer** | Partial | std:buffer | No | ArrayBuffer mutations are currently not synchronized | +| **URL** | Partial | std:url | No | URL mutations are currently not synchronized | +| **Date** | Partial | std:time | No | `Date` objects are currently asymetrically mapped to DATEX `Time` objects | +| **RegExp** | Partial | js:RegExp | No (Immutable) | RegExp values wrapped in a Ref are currently not synchronized | +| **WeakRef** | Full | std:WeakRef | No (Immutable) | - | +| **Error** | Partial | std:Error | No | Error subclasses are not correctly mapped | +| **HTMLElement** | Partial 2) | std:html | No | HTML element mutations are currently not synchronized | +| **SVGElement** | Partial 2) | std:svg | No | SVG element mutations are currently not synchronized | +| **MathMLElement** | Partial 2) | std:mathml | No | MathML element mutations are currently not synchronized | +| **Document** | Partial 2) | std:htmldocument | No | Document mutations are currently not synchronized | +| **DocumentFragment** | Partial 2) | std:htmlfragment | No | DocumentFragment mutations are currently not synchronized | + + +1) Primitive JS values are immutable and cannot be synchronized on their own, but when wrapped in a Ref.
+2) [UIX-DOM](https://github.com/unyt-org/uix-dom) required
+3) The corresponding object values of primitive values (e.g. `new Number()` for `number`) are not supported
+ ## Special JS types Most builtin JavaScript types, like Map, Set or Array have equivalent types in the DATEX std library. @@ -141,6 +178,6 @@ A struct definition accepts strings a keys and `Datex.Type`s, JavaScript classes or other struct definitions as values. -## Mapping JS classes to DATEX types +## Mapping your own JS classes to DATEX types Check out the chapter [11 Classes](./11%20Classes.md) for more information. \ No newline at end of file diff --git a/docs/manual/15 Storage Collections.md b/docs/manual/15 Storage Collections.md index d23a6305..7e44488d 100644 --- a/docs/manual/15 Storage Collections.md +++ b/docs/manual/15 Storage Collections.md @@ -1,16 +1,20 @@ # Storage Collections -DATEX allows the usage of native Set and Map objects. Those types are stored in RAM and may impact performance when it's data is getting to large. -For this reason DATEX provides special storage collections that allow the handling of massive amounts of data more efficiently. -The API is similar to the native JavaScript collections with the major difference that their instance methods are asynchronous. -To get the size of the collection it is recommended to use the asynchronous `getSize` method. +The native `Set` and `Map` objects can be used with DATEX cross-network and as persistent values, +but because their entries are completely stored in RAM, they are not ideal for large amounts of data. + +For this reason, DATEX provides special collection types (`StorageMap`/`StorageSet`) that handle large amounts of data more efficiently by outsourcing entries to a pointer storage location instead of keeping everything in RAM. + +The API is similar to `Set`/`Map` with the major difference that the instance methods are asynchronous. > [!NOTE] -> The storage collections are not stored persistently by default. To store persistent data refer to [Eternal Pointers](./05%20Eternal%20Pointers.md). +> Storage collections are not stored persistently by default as their name might imply. To store storage collections persistently, use [Eternal Pointers](./05%20Eternal%20Pointers.md). + +## StorageSets -## StorageSet ```ts -import "datex-core-legacy/types/storage-set.ts"; +import { StorageSet } from "datex-core-legacy/types/storage-set.ts"; + const mySet = new StorageSet(); await mySet.add(123); // Add 123 to the StorageSet @@ -22,9 +26,11 @@ await mySet.getSize(); // Returns the size of the StorageSet (1) await mySet.clear(); // Clear StorageSet ``` -## StorageMap +## StorageMaps + ```ts -import "datex-core-legacy/types/storage-map.ts"; +import { StorageMap } from "datex-core-legacy/types/storage-map.ts"; + const myMap = new StorageMap(); await myMap.set("myKey", 123); // Add key 'myKey' with value 123 to the StorageMap @@ -35,3 +41,211 @@ for await (const [key, value] of myMap) { // Iterate over entries await mySet.getSize(); // Returns the size of the StorageMap (1) await myMap.clear(); // Clear StorageMap ``` + + +## Pattern Matching + +Entries of a `StorageSet` can be efficiently queried by using the builtin pattern matcher. +For supported storage locations (e.g. sql storage), the pattern matching is directly performed in storage and non-matching entries are never loaded into RAM. + +> [!NOTE] +> Pattern matching currently only works with @sync class objects. + +### Selecting by property + +The easiest way to match entries in a storage set is to provide one or multiple required property values: + +```ts +import { StorageSet } from "datex-core-legacy/types/storage-set.ts"; +import { Time } from "unyt_core/datex_all.ts"; + +@sync class User { + @property(string) name!: string + @property(number) age!: number + @property(Time) created!: Time +} + +const users = new StorageSet(); + +// get all users with age == 18 +const usersAge18 = await users.match( + User, + { + age: 18 + } +); +``` + +### Match Conditions + +Besides exact matches, you can also match properties with certain constraints using match conditions: + +Match between to numbers/dates: +```ts +import { MatchCondition } from "unyt_core/storage/storage.ts"; + +// all users where the "created" timestamp is between now and 7 days ago: +const newUsersLastWeek = users.match( + User, + { + created: MatchCondition.between( + new Time().minus(7, "d"), + new Time() + ) + } +) +``` + +Match not equal: +```ts +// all users which do not have the name "John": +const notJohn = users.match( + User, + { + name: MatchCondition.notEqual("John") + } +) +``` + + +### Return value customization + +#### Limiting + +You can limit the maximum number of returned entries by setting the `limit` option to a number: + +```ts +// get all users with name "Josh", limit to 10 +const joshes = await users.match( + User, + { + name: "Josh" + }, + {limit: 10} +); +``` + +#### Sorting + +You can sort the returned entries by setting the `sortBy` option to a property path: + +```ts +// get all users with age == 18, sorted by their creation timestamp +const usersAge18 = await users.match( + User, + { + age: 18 + }, + {sortBy: 'created'} +); +``` + +Directly sorting values this way in the match query has two significant advantages over sorting +the returned values afterwards, e.g. using `Array.sort`: + * The sorting is normally faster + * When using the `limit` option, sorting is done before applying the `limit`, otherwise only the values remaining within the limit would be sorted + + +#### Returning additional metadata + +When the `returnAdvanced` option is set to `true`, the `match` function returns an object with additional metadata: + +```ts +const {matches, total} = await users.match( + User, + { + name: "Josh" + }, + { + limit: 10, + returnAdvanced: true + } +); + +matches // matching entries: Set +total // total number of matches that would be returned without the limit +``` + + +### Computed properties + +Computed properties provide a way to efficiently match entries in the StorageSet with more complex conditions. +One or multiple computed properties can be specified in the `computedProperties` option. + +#### Geographic Distance + +Calculates the geographic distance of two points provided from literal values or properties: + +Example: +```ts +import { ComputedProperty } from "datex-core-legacy/storage/storage.ts"; + +@sync class Location { + @property(number) lat!: number + @property(number) lon!: number +} + +@sync class User { + @property(string) name!: string + @property(Location) location!: Location +} + + +const myPosition = {lat: 70.48, lon: -21.96} + +// computed geographic distance between myPosition and a user position +const distance = ComputedProperty.geographicDistance( + // point A (user position) + { + lat: 'location.lat', + lon: 'location.lon' + }, + // point B (my position) + { + lat: myPosition.lat, + lon: myPosition.lon + } +) + +const nearbyJoshes = await users.match( + User, + { + name: "Josh", // name = "Josh" + distance: MatchCondition.lessThan(1000) // distance < 1000m + }, + { + computedProperties: { distance } + } +); +``` + +#### Sum + +Calculates the sum of multiple properties or literal values + +Example: +```ts +@sync class TodoItem { + @property(number) completedTaskCount!: number + @property(number) openTaskCount!: number +} +const todoItems = new StorageSet() + + +// sum of completedTaskCount and openTaskCount for a given TodoItem +const totalTaskCount = ComputedProperty.sum( + 'completedTaskCount', + 'openTaskCount' +) + +// match all todo items where the total task count is > 100 +const bigTodoItems = await todoItems.match( + User, + { + totalTaskCount: MatchCondition.greaterThan(100) // totalTaskCount > 100 + }, + { + computedProperties: { totalTaskCount } + } +); +``` diff --git a/functions.ts b/functions.ts index 5d1e050c..4e7b01a0 100644 --- a/functions.ts +++ b/functions.ts @@ -325,7 +325,7 @@ export function map(iterable: Iterable< * @param if_true value selected if true * @param if_false value selected if false */ -export function toggle(value:RefLike, if_true:T, if_false:T): MinimalJSRef { +export function toggle(value:RefLike, if_true:T, if_false:T = null as T): MinimalJSRef { return transform([value], v=>v?if_true:if_false, // dx transforms not working correctly (with uix) /*` diff --git a/js_adapter/decorators.ts b/js_adapter/decorators.ts new file mode 100644 index 00000000..14842015 --- /dev/null +++ b/js_adapter/decorators.ts @@ -0,0 +1,107 @@ +import { endpoint_name, target_clause } from "../datex_all.ts"; +import type { Type } from "../types/type.ts"; +import type { Class } from "../utils/global_types.ts"; +import { Decorators } from "./js_class_adapter.ts"; + + +export function handleClassFieldDecoratorWithOptionalArgs(args:T, context: C|undefined, callback: (arg: T, context: C)=>R): ((value: undefined, context: C) => R)|R { + if (!isDecoratorContext(context)) return (_value: undefined, context: C) => callback(args, context) + else return callback([] as unknown as T, context!) +} +export function handleClassFieldDecoratorWithArgs(args:T, callback: (arg: T, context: C)=>R): ((value: undefined, context: C) => R) { + return (_value: undefined, context: C) => callback(args, context) +} +export function handleClassDecoratorWithOptionalArgs<_Class extends Class, C extends ClassDecoratorContext<_Class>, const T extends unknown[], R>(args:T, value: _Class, context: C|undefined, callback: (arg: T, value: _Class, context: C)=>R): ((value: _Class, context: C) => R)|R { + if (!isDecoratorContext(context)) return (value: _Class, context: C) => callback(args, value, context) + else return callback([] as unknown as T, value, context!) +} +export function handleClassDecoratorWithArgs<_Class extends Class, C extends ClassDecoratorContext<_Class>, const T extends unknown[], R>(args:T, callback: (arg: T, value: _Class, context: C)=>R): ((value: _Class, context: C) => R) { + return (value: _Class, context: C) => callback(args, value, context) +} +export function handleClassMethodDecoratorWithArgs(args:T, callback: (arg: T, value:(...args:any[])=>any, context: C)=>R): ((value: (...args:any[])=>any, context: C) => R) { + return (value: (...args:any[])=>any, context: C) => callback(args, value, context) +} + +function isDecoratorContext(context: unknown) { + return context && typeof context === "object" && "kind" in context +} + + + +type PropertyDecoratorContext = ClassFieldDecoratorContext|ClassGetterDecoratorContext|ClassMethodDecoratorContextany)> + +/** + * Marks a (static) class field as a property accessible by DATEX. + * @param type optional type for the property, must match the declared TypeScript type + */ +export function property(type: string|Type|Class): (value: ((...args: any[])=>any)|undefined, context: PropertyDecoratorContext)=>void +export function property(value: ((...args: any[])=>any)|undefined, context: PropertyDecoratorContext): void +export function property(type: ((...args: any[])=>any)|undefined|string|Type|Class, context?: PropertyDecoratorContext) { + return handleClassFieldDecoratorWithOptionalArgs([type], context as ClassFieldDecoratorContext, ([type], context:PropertyDecoratorContext) => { + return Decorators.property(type as Type, context) + }) +} + + +/** + * Adds an assertion to a class field that is checked before the field is set + * @returns + */ +export function assert(assertion:(val:T)=>boolean|string|undefined): (value: undefined, context: ClassFieldDecoratorContext)=>void { + return handleClassFieldDecoratorWithArgs([assertion], ([assertion], context) => { + return Decorators.assert(assertion, context) + }) +} + +/** + * Make a class publicly accessible for an endpoint (only static methods and properties marked with @property are exposed) + * Also enables calling static class methods on other endpoints + * @param endpoint + * @param scope_name + */ +export function endpoint(endpoint:target_clause|endpoint_name, scope_name?:string): (value: Class, context: ClassDecoratorContext)=>void +export function endpoint(value: Class, context: ClassDecoratorContext): void +export function endpoint(value: Class|target_clause|endpoint_name, context?: ClassDecoratorContext|string) { + return handleClassDecoratorWithOptionalArgs([value as target_clause|endpoint_name, context as string], value as Class, context as ClassDecoratorContext, ([endpoint, scope_name], value, context) => { + return Decorators.endpoint(endpoint, scope_name, value, context) + }) +} + + +/** + * Sets a class as the entrypoint for the current endpoint + */ +export function entrypoint(value: Class, context: ClassDecoratorContext) { + return Decorators.entrypoint(value, context) +} + +/** + * Adds a class as a property of entrypoint for the current endpoint + */ +export function entrypointProperty(value: Class, context: ClassDecoratorContext) { + return Decorators.entrypointProperty(value, context) +} + +/** + * Sets the maximum allowed time (in ms) for a remote function execution + * before a timeout error is thrown (default: 5s) + * @param timeMs timeout in ms + * @returns + */ +export function timeout(timeMs:number): (value: (...args:any[])=>any, context: ClassMethodDecoratorContext)=>void { + return handleClassMethodDecoratorWithArgs([timeMs], ([timeMs], _value, context) => { + return Decorators.timeout(timeMs, context) + }); +} + +/** + * Maps a class to a corresponding DATEX type + * @deprecated Use struct(class {...}) instead; + */ +export function sync(type: string): (value: Class, context: ClassDecoratorContext)=>void +export function sync(value: Class, context: ClassDecoratorContext): void +export function sync(value: Class|string, context?: ClassDecoratorContext) { + return handleClassDecoratorWithOptionalArgs([value as string], value as Class, context as ClassDecoratorContext, ([type], value, context) => { + return Decorators.sync(type, value, context) + }) +} \ No newline at end of file diff --git a/js_adapter/js_class_adapter.ts b/js_adapter/js_class_adapter.ts index f56f445b..a11a4a0d 100644 --- a/js_adapter/js_class_adapter.ts +++ b/js_adapter/js_class_adapter.ts @@ -15,16 +15,14 @@ import { Runtime, StaticScope } from "../runtime/runtime.ts"; import { Logger } from "../utils/logger.ts"; -import { Endpoint, endpoint_name, IdEndpoint, LOCAL_ENDPOINT, Target, target_clause } from "../types/addressing.ts"; -import { context_kind, context_meta_getter, context_meta_setter, context_name } from "./legacy_decorators.ts"; +import { endpoint_name, LOCAL_ENDPOINT, Target, target_clause } from "../types/addressing.ts"; import { Type } from "../types/type.ts"; import { getProxyFunction, getProxyStaticValue, ObjectRef, Pointer, UpdateScheduler } from "../runtime/pointers.ts"; -import { Error as DatexError, ValueError } from "../types/errors.ts"; import { Function as DatexFunction } from "../types/function.ts"; import { DatexObject } from "../types/object.ts"; import { Tuple } from "../types/tuple.ts"; -import { DX_PERMISSIONS, DX_TYPE, DX_ROOT, INIT_PROPS, DX_EXTERNAL_SCOPE_NAME, DX_EXTERNAL_FUNCTION_NAME } from "../runtime/constants.ts"; -import { type Class } from "../utils/global_types.ts"; +import { DX_PERMISSIONS, DX_TYPE, DX_ROOT, INIT_PROPS, DX_EXTERNAL_SCOPE_NAME, DX_EXTERNAL_FUNCTION_NAME, DX_TIMEOUT } from "../runtime/constants.ts"; +import type { Class } from "../utils/global_types.ts"; import { Conjunction, Disjunction, Logical } from "../types/logic.ts"; import { client_type } from "../utils/constants.ts"; import { Assertion } from "../types/assertion.ts"; @@ -51,38 +49,6 @@ export function instance(fromClassOrType:{new(...params:any[]):T}|Type, pr } -/** - * List of decorators - * - * @meta: mark method parameter that should contain meta data about the datex request / declare index of 'meta' parameter in method - * @docs: add docs to a static scope class / pseudo class - * - * @allow: define which endpoints have access to a class / method / property - * @to: define which on endpoints a method should be called / from which endpoint a property should be fetched - * - * @no_result: don't wait for result - * - * Static: - * @scope(name?:string): declare a class as a static scope, or add a static scope to a static property/method - * @root_extension: root extends this static scope in every executed DATEX scope (all static scope members become variables) - * @root_variable: static scope becomes a root variable in every executed DATEX scope (scope name is variable name) - * - * @remote: get a variable from a remote static scope or call a method in a remote static scope - * @expose: make a method/variable in a static scope available to others - * - * Sync: - * @sync: make a class syncable, or sync a property/method - * @sealed: make a sync class sealed, or seal individual properties/methods - * @anonymous: force prevent creating a pointer reference for an object, always transmit serialized - * - * @constructor: called after constructor, if instance is newly generated - * @generator: called after @constructor, if instance is newly generated - * @replicator: called after @constructor, if instance is a clone - * @destructor: called when pointer is garbage collected, or triggers garbage collection - */ - - - // handles all decorators export class Decorators { @@ -123,121 +89,50 @@ export class Decorators { static FROM_TYPE = Symbol("FROM_TYPE"); - static CONSTRUCTOR = Symbol("CONSTRUCTOR"); - static REPLICATOR = Symbol("REPLICATOR"); - static DESTRUCTOR = Symbol("DESTRUCTOR"); - - /** @expose(allow?:filter): make a method in a static scope available to be called by others */ - static public(value:any, name:context_name, kind:context_kind, is_static:boolean, is_private:boolean, setMetadata:context_meta_setter, getMetadata:context_meta_getter, params:[target_clause?] = []) { - - // invalid decorator call - if (kind != "method" && kind != "field") logger.error("Cannot use @expose for value '" + name.toString() +"'"); - else if (!is_static) logger.error("Cannot use @expose for non-static field '" + name.toString() +"'"); - - // handle decorator - else { - setMetadata(Decorators.IS_EXPOSED, true) - if (params.length) Decorators.addMetaFilter( - params[0], - setMetadata, getMetadata, Decorators.ALLOW_FILTER - ) + public static setMetadata(context:DecoratorContext, key:string|symbol, value:unknown) { + if (!context.metadata[key]) context.metadata[key] = {} + const data = context.metadata[key] as {public?:Record, constructor?:any} + if (context.kind == "class") { + data.constructor = value; } - } - - /** @namespace(name?:string): declare a class as a #public property */ - static namespace(value:any, name:context_name, kind:context_kind, is_static:boolean, is_private:boolean, setMetadata:context_meta_setter, getMetadata:context_meta_getter, params:[string?] = []) { - - // invalid decorator call - if (!is_static && kind != "class") logger.error("Cannot use @scope for non-static field '" + name!.toString() +"'"); - - // handle decorator else { - setMetadata(Decorators.NAMESPACE, params[0] ?? value?.name) - - // class @namespace - if (kind == "class") _old_publicStaticClass(value); - - // @namespace for static field -> @remote + @expose - else { - setMetadata(Decorators.IS_REMOTE, true) - setMetadata(Decorators.IS_EXPOSED, true) - } - } + if (!data.public) data.public = {}; + data.public[context.name] = value; + } } - /** @endpoint(endpoint?:string|Datex.Endpoint, namespace?:string): declare a class as a #public property */ - static endpoint(value:any, name:context_name, kind:context_kind, is_static:boolean, is_private:boolean, setMetadata:context_meta_setter, getMetadata:context_meta_getter, params:[(target_clause|endpoint_name)?, string?] = []) { - - // invalid decorator call - if (!is_static && kind != "class") logger.error("Cannot use @endpoint for non-static field '" + name!.toString() +"'"); - - // handle decorator - else { - // target endpoint - if (params[0]) { - Decorators.addMetaFilter( - params[0], - setMetadata, getMetadata, Decorators.SEND_FILTER - ) - } - else { - setMetadata(Decorators.SEND_FILTER, true); // indicate to always use local endpoint (expose) - } - - // custom namespace name - setMetadata(Decorators.NAMESPACE, params[1] ?? value?.name) - - // class @endpoint - if (kind == "class") registerPublicStaticClass(value); - - else logger.error("@endpoint can only be used for classes"); - } + /** @endpoint(endpoint?:string|Datex.Endpoint, namespace?:string): declare a class as a #public property */ + static endpoint(endpoint:target_clause|endpoint_name, scope_name:string|undefined, value: Class, context: ClassDecoratorContext) { + // target endpoint + if (endpoint) { + Decorators.addMetaFilter( + endpoint, + context, + Decorators.SEND_FILTER + ) } - - /** @root_extension: root extends this static scope in every executed DATEX scope (all static scope members become variables) */ - static default(value:any, name:context_name, kind:context_kind, is_static:boolean, is_private:boolean, setMetadata:context_meta_setter, getMetadata:context_meta_getter, params:undefined) { - - // invalid decorator call - if (!is_static && kind != "class") logger.error("Cannot use @root_extension for non-static field '" + name.toString() +"'"); - - // handle decorator else { - setMetadata(Decorators.DEFAULT, true) - - if (kind == "class") _old_publicStaticClass(value); + this.setMetadata(context, Decorators.SEND_FILTER, true) // indicate to always use local endpoint (expose) } - } - /** @root_variable: static scope becomes a root variable in every executed DATEX scope (scope name is variable name) */ - static default_property(value:any, name:context_name, kind:context_kind, is_static:boolean, is_private:boolean, setMetadata:context_meta_setter, getMetadata:context_meta_getter, params:undefined) { - - // invalid decorator call - if (!is_static && kind != "class") logger.error("Cannot use @root_variable for non-static field '" + name.toString() +"'"); - - // handle decorator - else { - setMetadata(Decorators.DEFAULT_PROPERTY, true) - - if (kind == "class") _old_publicStaticClass(value); - } + // custom namespace name + this.setMetadata(context, Decorators.NAMESPACE, scope_name ?? value.name); + registerPublicStaticClass(value, 'public', context.metadata); } - /** @remote(from?:filter): get a variable from a static scope or call a function */ - static remote(value:any, name:context_name, kind:context_kind, is_static:boolean, is_private:boolean, setMetadata:context_meta_setter, getMetadata:context_meta_getter, params:[target_clause?] = []) { - - // invalid decorator call - if (kind == "class") logger.error("Cannot use @remote for a class"); - else if (!is_static) logger.error("Cannot use @remote for non-static field '" + name.toString() +"'"); + /** @entrypoint is set as endpoint entrypoint */ + static entrypoint(value:Class, context: ClassDecoratorContext) { + this.setMetadata(context, Decorators.SEND_FILTER, true) // indicate to always use local endpoint (expose) + this.setMetadata(context, Decorators.NAMESPACE, value.name); + registerPublicStaticClass(value, 'entrypoint', context.metadata); + } - // handle decorator - else { - setMetadata(Decorators.IS_REMOTE, true) - if (params.length) Decorators.addMetaFilter( - params[0], - setMetadata, getMetadata, Decorators.SEND_FILTER - ) - } + /** @entrypointProperty is set as a property of the endpoint entrypoint */ + static entrypointProperty(value:Class, context: ClassDecoratorContext) { + this.setMetadata(context, Decorators.SEND_FILTER, true) // indicate to always use local endpoint (expose) + this.setMetadata(context, Decorators.NAMESPACE, value.name); + registerPublicStaticClass(value, 'entrypointProperty', context.metadata); } @@ -253,18 +148,6 @@ export class Decorators { } } - /** @meta(index?:number): declare index of meta parameter (before method), or inline parameter decorator */ - static meta(value:any, name:context_name, kind:context_kind, is_static:boolean, is_private:boolean, setMetadata:context_meta_setter, getMetadata:context_meta_getter, params:[string?] = []) { - - if (kind == "method") { - setMetadata(Decorators.META_INDEX, params[0] ?? -1); - } - - // invalid decorator call - else logger.error("@meta can only be used for methods"); - } - - /** @sign(sign?:boolean): sign outgoing DATEX requests (default:true) */ static sign(value:any, name:context_name, kind:context_kind, is_static:boolean, is_private:boolean, setMetadata:context_meta_setter, getMetadata:context_meta_getter, params:[boolean?] = []) { setMetadata(Decorators.SIGN, params[0]) @@ -281,9 +164,9 @@ export class Decorators { } /** @timeout(msecs?:number): DATEX request timeout */ - static timeout(value:any, name:context_name, kind:context_kind, is_static:boolean, is_private:boolean, setMetadata:context_meta_setter, getMetadata:context_meta_getter, params:[number?] = []) { - if (params[0] && params[0] > 2**31) throw new Error("@timeout: timeout too big (max value is 2^31), use Infinity if you want to disable the timeout") - setMetadata(Decorators.TIMEOUT, params[0]) + static timeout(timeMs:number, context:ClassMethodDecoratorContext) { + if (isFinite(timeMs) && timeMs > 2**31) throw new Error("@timeout: timeout too big (max value is 2^31), use Infinity if you want to disable the timeout") + this.setMetadata(context, Decorators.TIMEOUT, timeMs) } /** @allow(allow:filter): Allowed endpoints for class/method/field */ @@ -316,25 +199,29 @@ export class Decorators { /** @property: add a field as a template property */ - static property(value:any, name:context_name, kind:context_kind, is_static:boolean, is_private:boolean, setMetadata:context_meta_setter, getMetadata:context_meta_getter, params:[string|number]) { - if (kind != "field" && kind != "getter" && kind != "setter" && kind != "method") logger.error("Invalid use of @property decorator"); + static property(type:string|Type|Class, context: ClassFieldDecoratorContext|ClassGetterDecoratorContext|ClassMethodDecoratorContext) { + if (context.static) { + this.setMetadata(context, Decorators.STATIC_PROPERTY, context.name) + } else { - if (is_static) setMetadata(Decorators.STATIC_PROPERTY, params?.[0] ?? name) - else setMetadata(Decorators.PROPERTY, params?.[0] ?? name) + this.setMetadata(context, Decorators.PROPERTY, context.name) } + // type + if (type) { + const normalizedType = normalizeType(type); + this.setMetadata(context, Decorators.FORCE_TYPE, normalizedType) + } } + /** @assert: add type assertion function */ - static assert(value:any, name:context_name, kind:context_kind, is_static:boolean, is_private:boolean, setMetadata:context_meta_setter, getMetadata:context_meta_getter, params:[((value:any)=>boolean)?] = []) { - if (kind != "field" && kind != "getter" && kind != "setter" && kind != "method") logger.error("Invalid use of @assert decorator"); + static assert(assertion: (val:T) => boolean|string|undefined, context: ClassFieldDecoratorContext) { + if (context.static) logger.error("Cannot use @assert with static fields"); else { - if (typeof params[0] !== "function") logger.error("Invalid @assert decorator value, must be a function"); - else { - const assertionType = new Conjunction(Assertion.get(undefined, params[0], false)); - setMetadata(Decorators.FORCE_TYPE, assertionType) - } + const assertionType = new Conjunction(Assertion.get(undefined, assertion, false)); + this.setMetadata(context, Decorators.FORCE_TYPE, assertionType) } } @@ -371,136 +258,39 @@ export class Decorators { } - /** - * @deprecated use \@sync - */ - static template(value:any, name:context_name, kind:context_kind, is_static:boolean, is_private:boolean, setMetadata:context_meta_setter, getMetadata:context_meta_getter, params:[(string|Type)?] = []) { - if (kind != "class") logger.error("@template can only be used as a class decorator"); - - else { - //initPropertyTypeAssigner(); - - const original_class = value; - let type: Type; - - // get template type - if (typeof params[0] == "string" || params[0] instanceof Type) { - type = normalizeType(params[0], false, "ext"); - } - else if (original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor) type = original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor - else type = Type.get("ext", original_class.name.replace(/^_/, '')); // remove leading _ from type name - - - // return new templated class - return createTemplateClass(original_class, type); - } - - } - /** @sync: sync class/property */ - static sync(value:any, name:context_name, kind:context_kind, is_static:boolean, is_private:boolean, setMetadata:context_meta_setter, getMetadata:context_meta_getter, params:[(string|Type)?] = []) { + static sync(type: string|Type|undefined, value: Class, context?: ClassDecoratorContext, callerFile?:string) { - // invalid decorator call - if (is_static) logger.error("Cannot use @sync for static field '" + name.toString() +"'"); - if (is_static) logger.error("Cannot use @sync for static field '" + name.toString() +"'"); - - // handle decorator - else { - setMetadata(Decorators.IS_SYNC, true) - - // is auto sync class -> create class proxy (like in template) - if (kind == "class") { - //initPropertyTypeAssigner(); - - const original_class = value; - let type: Type; - - // get template type - if (typeof params[0] == "string" || params[0] instanceof Type) { - type = normalizeType(params[0], false, "ext"); - } - else if (original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor) type = original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor - else type = Type.get("ext", original_class.name.replace(/^_/, '')); // remove leading _ from type name - - let callerFile:string|undefined; - - if (client_type == "deno" && type.namespace !== "std") { - callerFile = getCallerInfo()?.[2]?.file ?? undefined; - if (!callerFile) { - logger.error("Could not determine JS module URL for type '" + type + "'") - } - } - - // return new templated class - return createTemplateClass(original_class, type, true, true, callerFile); - } - + if (context) { + this.setMetadata(context ?? {kind: "class", metadata:(value as any)[METADATA]}, Decorators.IS_SYNC, true) } - } - /** @sealed: sealed class/property */ - static sealed(value:any, name:context_name, kind:context_kind, is_static:boolean, is_private:boolean, setMetadata:context_meta_setter, getMetadata:context_meta_getter, params:undefined) { - - // invalid decorator call - if (is_static) logger.error("Cannot use @sealed for static field '" + name.toString() +"'"); + const originalClass = value; - // handle decorator - else { - setMetadata(Decorators.IS_SEALED, true) - } - } - - /** @anonymous: anonymous class/property */ - static anonymous(value:any, name:context_name, kind:context_kind, is_static:boolean, is_private:boolean, setMetadata:context_meta_setter, getMetadata:context_meta_getter, params:undefined) { - - // invalid decorator call - if (is_static) logger.error("Cannot use @anonymous for static field '" + name.toString() +"'"); + let normalizedType: Type; - // handle decorator - else { - setMetadata(Decorators.IS_ANONYMOUS, true) + // get template type + if (typeof type == "string" || type instanceof Type) { + normalizedType = normalizeType(type, false, "ext"); } - } - + else if ( + originalClass[METADATA]?.[Decorators.FORCE_TYPE] && + Object.hasOwn(originalClass[METADATA]?.[Decorators.FORCE_TYPE], 'constructor') + ) normalizedType = originalClass[METADATA]?.[Decorators.FORCE_TYPE]?.constructor + else normalizedType = Type.get("ext", originalClass.name.replace(/^_/, '')); // remove leading _ from type name - /** @observe(handler:Function): listen to value changes */ - static observe(value:any, name:context_name, kind:context_kind, is_static:boolean, is_private:boolean, setMetadata:context_meta_setter, getMetadata:context_meta_getter, params:[Function?] = []) { - setMetadata(Decorators.OBSERVER, params[0]) - } - - - - /** @anonymize: serialize return values (no pointers), only the first layer */ - static anonymize(value:any, name:context_name, kind:context_kind, is_static:boolean, is_private:boolean, setMetadata:context_meta_setter, getMetadata:context_meta_getter, params:undefined) { - - // invalid decorator call - if (kind == "class") logger.error("Cannot use @anonymize for classes"); - - // handle decorator - else { - setMetadata(Decorators.ANONYMIZE, true) + if (!callerFile && client_type == "deno" && normalizedType.namespace !== "std") { + callerFile = getCallerInfo()?.[3]?.file ?? undefined; + if (!callerFile) { + logger.error("Could not determine JS module URL for type '" + normalizedType + "'") + } } + + // return new templated class + return createTemplateClass(originalClass, normalizedType, true, true, callerFile, context?.metadata); } - /** @type(type:string|DatexType)/ (namespace:name) - * sync class with type */ - static type(value:any, name:context_name, kind:context_kind, is_static:boolean, is_private:boolean, setMetadata:context_meta_setter, getMetadata:context_meta_getter, params:[(string|Type)] = []) { - const type = normalizeType(params[0]); - setMetadata(Decorators.FORCE_TYPE, type) - } - - /** @from(type:string|DatexType): sync class from type */ - static from(value:any, name:context_name, kind:context_kind, is_static:boolean, is_private:boolean, setMetadata:context_meta_setter, getMetadata:context_meta_getter, params:[(string|Type)?] = []) { - // invalid decorator call - if (kind !== "class") logger.error("Can use @from only for classes"); - - // handle decorator - else { - setMetadata(Decorators.FROM_TYPE, params[0]) - } - } - /** @update(interval:number|scheduler:DatexUpdateScheduler): set update interval / scheduler */ static update(value:any, name:context_name, kind:context_kind, is_static:boolean, is_private:boolean, setMetadata:context_meta_setter, getMetadata:context_meta_getter, params:[(number|UpdateScheduler)?] = []) { @@ -508,60 +298,11 @@ export class Decorators { else setMetadata(Decorators.SCHEDULER, new UpdateScheduler(params[0])); } - - - /** @constructor: called after constructor if newly generateds */ - static ["constructor"](value:any, name:context_name, kind:context_kind, is_static:boolean, is_private:boolean, setMetadata:context_meta_setter, getMetadata:context_meta_getter, params:undefined) { - - // invalid decorator call - if (is_static) logger.error("Cannot use @constructor for static field '" + name.toString() +"'"); - else if (kind != "method") logger.error("Cannot only use @constructor for methods"); - - // handle decorator - else { - setMetadata(Decorators.CONSTRUCTOR, true) - } - } - - - - /** @replicator: called after constructor if cloned */ - static replicator(value:any, name:context_name, kind:context_kind, is_static:boolean, is_private:boolean, setMetadata:context_meta_setter, getMetadata:context_meta_getter, params:undefined) { - - // invalid decorator call - if (is_static) logger.error("Cannot use @replicator for static field '" + name.toString() +"'"); - else if (kind != "method") logger.error("Cannot only use @replicator for methods"); - - // handle decorator - else { - setMetadata(Decorators.REPLICATOR, true) - } - } - - /** @destructor: called after constructor if cloned */ - static destructor(value:any, name:context_name, kind:context_kind, is_static:boolean, is_private:boolean, setMetadata:context_meta_setter, getMetadata:context_meta_getter, params:undefined) { - - // invalid decorator call - if (is_static) logger.error("Cannot use @destructor for static field '" + name.toString() +"'"); - else if (kind != "method") logger.error("Cannot only use @destructor for methods"); - - // handle decorator - else { - setMetadata(Decorators.DESTRUCTOR, true) - } - } - - // handle ALLOW_FILTER for classes, methods and fields // adds filter - private static addMetaFilter(new_filter:target_clause|endpoint_name, setMetadata:context_meta_setter, getMetadata:context_meta_getter, filter_symbol:symbol){ - // // create filter if not existing - // let filter:Filter = getMetadata(filter_symbol) - // if (!filter) {filter = new Filter(); setMetadata(filter_symbol, filter)} - // filter.appendFilter(new_filter); - - if (typeof new_filter == "string") setMetadata(filter_symbol, Target.get(new_filter)) - else setMetadata(filter_symbol, new_filter) + private static addMetaFilter(new_filter:target_clause|endpoint_name, context: DecoratorContext, filter_symbol:symbol){ + if (typeof new_filter == "string") this.setMetadata(context, filter_symbol, Target.get(new_filter)) + else this.setMetadata(context, filter_symbol, new_filter) } } @@ -571,7 +312,7 @@ export class Decorators { * @param allowTypeParams * @returns */ -function normalizeType(type:Type|string, allowTypeParams = true, defaultNamespace = "std") { +function normalizeType(type:Type|string|Class, allowTypeParams = true, defaultNamespace = "std") { if (typeof type == "string") { // extract type name and parameters const [typeName, paramsString] = type.replace(/^\$/,'').match(/^((?:[\w-]+\:)?[\w-]*)(?:\((.*)\))?$/)?.slice(1) ?? []; @@ -587,65 +328,88 @@ function normalizeType(type:Type|string, allowTypeParams = true, defaultNamespac if (!allowTypeParams && type.parameters?.length) throw new Error(`Type parameters not allowed (${type})`); return type } + else if (typeof type == "function") { + const classType = Type.getClassDatexType(type) + if (!classType) throw new Error("Could not get a DATEX type for class " + type.name + ". Only @sync classes can be used as types"); + return classType + } else { console.error("invalid type",type) throw new Error("Invalid type") } } +type class_data = {name:string, static_scope:StaticScope, properties: string[], metadata:any} -const initialized_static_scope_classes = new Map(); - - -const registered_static_classess = new Set(); -function registerPublicStaticClass(original_class:Class){ - registered_static_classess.add(original_class); +const PROPERTY_COLLECTION = Symbol("PROPERTY_COLLECTION"); +const registeredClasses = new Map(); +const pendingClassRegistrations = new Map>().setAutoDefault(Set); - // if endpoint already loaded, init class - initPublicStaticClasses() +function registerPublicStaticClass(publicClass:Class, type:'public'|'entrypoint'|'entrypointProperty', metadata?:Record){ + pendingClassRegistrations.getAuto(publicClass).add(type); + initPublicStaticClass(publicClass, type, metadata) } -type class_data = {name:string, static_scope:StaticScope, properties: string[], metadata:any} +export function initPublicStaticClasses(){ + for (const [reg_class, types] of [...pendingClassRegistrations]) { + for (const type of [...types]) { + initPublicStaticClass(reg_class, type) + } + } + pendingClassRegistrations.clear(); +} -export function initPublicStaticClasses(){ +function initPublicStaticClass(publicClass: Class, type: 'public'|'entrypoint'|'entrypointProperty', metadata?:Record) { if (!Runtime.endpoint || Runtime.endpoint === LOCAL_ENDPOINT) return; - - for (const reg_class of registered_static_classess) { - - if (initialized_static_scope_classes.has(reg_class)) continue; // already initialized - const metadata = (reg_class)[METADATA]; - let targets = metadata[Decorators.SEND_FILTER]?.constructor; - if (targets == true) targets = Runtime.endpoint; // use own endpoint per default + metadata ??= (publicClass)[METADATA]; + if (!metadata) throw new Error(`Missing metadata for class ${publicClass.name}`) + let targets = metadata[Decorators.SEND_FILTER]?.constructor; + if (targets == true) targets = Runtime.endpoint; // use own endpoint per default - let data:any; - - // expose if current endpoint matches class endpoint - if (Logical.matches(Runtime.endpoint, targets, Target)) { - data ??= getStaticClassData(reg_class); - if (!data) throw new Error("Could not get data for static class") - exposeStaticClass(reg_class, data); - } + let data = registeredClasses.get(publicClass); + + // expose if current endpoint matches class endpoint + if (Logical.matches(Runtime.endpoint, targets, Target)) { + data ??= getStaticClassData(publicClass, true, type == 'public', metadata); + if (!data) throw new Error("Could not get data for static class") + exposeStaticClass(publicClass, data); + } - // also enable remote access if not exactly and only the current endpoint - if (Runtime.endpoint !== targets) { - data ??= getStaticClassData(reg_class, false); - if (!data) throw new Error("Could not get data for static class") - remoteStaticClass(reg_class, data, targets) - } + // also enable remote access if not exactly and only the current endpoint + if (Runtime.endpoint !== targets) { + data ??= getStaticClassData(publicClass, false, false, metadata); + if (!data) throw new Error("Could not get data for static class") + remoteStaticClass(publicClass, data, targets) + } - - DatexObject.seal(data.static_scope); - initialized_static_scope_classes.set(reg_class, data.static_scope); + // set method timeouts + for (const [method_name, timeout] of Object.entries(metadata[Decorators.TIMEOUT]?.public??{})) { + const method = (publicClass as any)[method_name]; + if (method) method[DX_TIMEOUT] = timeout } + // set entrypoint + if (type == 'entrypoint') { + data ??= getStaticClassData(publicClass, true, false, metadata); + if (Runtime.endpoint_entrypoint) logger.error("Existing entrypoint was overridden with @entrypoint class " + publicClass.name); + Runtime.endpoint_entrypoint = data.static_scope; + } + else if (type == 'entrypointProperty') { + data ??= getStaticClassData(publicClass, true, false, metadata); + if (Runtime.endpoint_entrypoint == undefined) Runtime.endpoint_entrypoint = {[PROPERTY_COLLECTION]:true} + if (typeof Runtime.endpoint_entrypoint !== "object" || !Runtime.endpoint_entrypoint[PROPERTY_COLLECTION]) logger.error("Cannot set endpoint property " + publicClass.name + ". The entrypoint is already set to another value."); + Runtime.endpoint_entrypoint[publicClass.name] = data.static_scope; + } + + // DatexObject.seal(data.static_scope); + registeredClasses.set(publicClass, data); + pendingClassRegistrations.get(publicClass)!.delete(type); } function exposeStaticClass(original_class:Class, data:class_data) { - // console.log("expose class", data, data.metadata[Decorators.STATIC_PROPERTY]); - const exposed_public = data.metadata[Decorators.STATIC_PROPERTY]?.public; const exposed_private = data.metadata[Decorators.STATIC_PROPERTY]?.private; @@ -721,7 +485,6 @@ function remoteStaticClass(original_class:Class, data:class_data, targets:target const timeout_public = data.metadata[Decorators.TIMEOUT]?.public; const timeout_private = data.metadata[Decorators.TIMEOUT]?.private; - // prototype for all options objects of static proxy methods (Contains the dynamic_filter) let options_prototype: {[key:string]:any} = {}; @@ -730,7 +493,7 @@ function remoteStaticClass(original_class:Class, data:class_data, targets:target Object.defineProperty(original_class, 'to', { value: function(...targets:(Target|endpoint_name)[]){ options_prototype.dynamic_filter = new Disjunction(); - for (let target of targets) { + for (const target of targets) { if (typeof target == "string") options_prototype.dynamic_filter.add(Target.get(target)) else options_prototype.dynamic_filter.add(target) } @@ -778,15 +541,15 @@ function remoteStaticClass(original_class:Class, data:class_data, targets:target } -function getStaticClassData(original_class:Class, staticScope = true) { - const metadata = (original_class)[METADATA]; +function getStaticClassData(original_class:Class, staticScope = true, expose = true, metadata?:Record) { + metadata ??= (original_class)[METADATA]; if (!metadata) return; const static_scope_name = typeof metadata[Decorators.NAMESPACE]?.constructor == 'string' ? metadata[Decorators.NAMESPACE]?.constructor : original_class.name; const static_properties = Object.getOwnPropertyNames(original_class) return { metadata, - static_scope: staticScope ? new StaticScope(static_scope_name) : null, + static_scope: staticScope ? StaticScope.get(static_scope_name, expose) : null, name: static_scope_name, properties: static_properties } @@ -794,262 +557,11 @@ function getStaticClassData(original_class:Class, staticScope = true) { -function _old_publicStaticClass(original_class:Class) { - - // already initialized - if (initialized_static_scope_classes.has(original_class)) { - - // is default property - if (original_class[METADATA]?.[Decorators.DEFAULT_PROPERTY]?.constructor) { - const static_scope = initialized_static_scope_classes.get(original_class); - if (!Runtime.endpoint_entrypoint || typeof Runtime.endpoint_entrypoint != "object") Runtime.endpoint_entrypoint = {}; - Runtime.endpoint_entrypoint[static_scope.name] = static_scope - } - // is default value - if (original_class[METADATA]?.[Decorators.DEFAULT]?.constructor) { - const static_scope = initialized_static_scope_classes.get(original_class); - Runtime.endpoint_entrypoint = static_scope; - } - - - return; - } - - - let static_properties = Object.getOwnPropertyNames(original_class) - - const metadata = original_class[METADATA]; - if (!metadata) return; - - // prototype for all options objects of static proxy methods (Contains the dynamic_filter) - let options_prototype: {[key:string]:any} = {}; - - const static_scope_name = typeof metadata[Decorators.NAMESPACE]?.constructor == 'string' ? metadata[Decorators.NAMESPACE]?.constructor : original_class.name; - let static_scope:StaticScope; - - // add builtin methods - - Object.defineProperty(original_class, 'to', { - value: function(...targets:(Target|endpoint_name)[]){ - options_prototype.dynamic_filter = new Disjunction(); - for (let target of targets) { - if (typeof target == "string") options_prototype.dynamic_filter.add(Target.get(target)) - else options_prototype.dynamic_filter.add(target) - } - return this; - }, - configurable: false, - enumerable: false, - writable: false - }); - /* - target.list = async function (...filters:ft[]|[Datex.filter]) { - if (!DATEX_CLASS_ENDPOINTS.has(this)) return false; - DATEX_CLASS_ENDPOINTS.get(this).dynamic_filter.appenddatex_filter(...filters) - return (await DATEX_CLASS_ENDPOINTS.get(this).__sendHandler("::list"))?.data || new Set(); - } - target.on_result = function(call: (data:datex_res, meta:{station_id:number, station_bundle:number[]})=>any){ - if (!DATEX_CLASS_ENDPOINTS.has(this)) return this; - DATEX_CLASS_ENDPOINTS.get(this).current_dynamic_callback = call; - return this; - } - target.no_result = function() { - if (!DATEX_CLASS_ENDPOINTS.has(this)) return this; - DATEX_CLASS_ENDPOINTS.get(this).current_no_result = true; - return this; - } - - target.ping = async function(...filters:ft[]|[Datex.filter]){ - if (!DATEX_CLASS_ENDPOINTS.has(this)) return false; - return new Promise(resolve=>{ - DATEX_CLASS_ENDPOINTS.get(this).dynamic_filter.appenddatex_filter(...filters) - let pings = {} - let start_time = new Date().getTime(); - DATEX_CLASS_ENDPOINTS.get(this).current_dynamic_callback = (data, meta) => { - pings[meta.station_id] = (new Date().getTime() - start_time) + "ms"; - if (Object.keys(pings).length == meta.station_bundle.length) resolve(pings); - } - setTimeout(()=>resolve(pings), 10000); - DATEX_CLASS_ENDPOINTS.get(this).__sendHandler("::ping") - }) - } - - target.self = function(){ - if (!DATEX_CLASS_ENDPOINTS.has(this)) return false; - DATEX_CLASS_ENDPOINTS.get(this).dynamic_self = true; - } - target.encrypt = function(encrypt=true){ - if(DATEX_CLASS_ENDPOINTS.has(this)) DATEX_CLASS_ENDPOINTS.get(this).current_encrypt = encrypt; - return this; - } - */ - - let class_send_filter:target_clause = metadata[Decorators.SEND_FILTER]?.constructor - // @ts-ignore - if (class_send_filter == Object) class_send_filter = undefined; - let class_allow_filter:target_clause = metadata[Decorators.ALLOW_FILTER]?.constructor - // @ts-ignore - if (class_allow_filter == Object) class_allow_filter = undefined; - - // per-property metadata - const exposed_public = metadata[Decorators.IS_EXPOSED]?.public; - const exposed_private = metadata[Decorators.IS_EXPOSED]?.private; - - const remote_public = metadata[Decorators.IS_REMOTE]?.public; - const remote_private = metadata[Decorators.IS_REMOTE]?.private; - const timeout_public = metadata[Decorators.TIMEOUT]?.public; - const timeout_private = metadata[Decorators.TIMEOUT]?.private; - const send_filter = metadata[Decorators.SEND_FILTER]?.public; - - - for (const name of static_properties) { - - const current_value = original_class[name]; - - // expose - if ((exposed_public?.hasOwnProperty(name) && exposed_public[name]) || (exposed_private?.hasOwnProperty(name) && exposed_private[name])) { - - if (!static_scope) static_scope = StaticScope.get(static_scope_name) - - // function - if (typeof current_value == "function") { - // set allowed endpoints for this method - //static_scope.setAllowedEndpointsForProperty(name, this.method_a_filters.get(name)) - - const dx_function = Pointer.proxifyValue(DatexFunction.createFromJSFunction(current_value, original_class, name), true, undefined, false, true) ; // generate - - // public function - const ptr = Pointer.pointerifyValue(dx_function); - if (ptr instanceof Pointer) ptr.grantPublicAccess(true); - - static_scope.setVariable(name, dx_function); // add to static scope - } - - // field - else { - // set static value (datexified) - const setProxifiedValue = (val:any) => { - static_scope.setVariable(name, Pointer.proxifyValue(val, true, undefined, false, true)) - // public function - const ptr = Pointer.proxifyValue(val); - if (ptr instanceof Pointer) ptr.grantPublicAccess(true); - }; - setProxifiedValue(current_value); - - /*** handle new value assignments to this property: **/ - - // similar to addObjProxy in DatexRuntime / DatexPointer - const property_descriptor = Object.getOwnPropertyDescriptor(original_class, name); - - // add original getters/setters to static_scope if they exist - if (property_descriptor?.set || property_descriptor?.get) { - Object.defineProperty(static_scope, name, { - set: val => { - property_descriptor.set?.call(original_class,val); - }, - get: () => { - return property_descriptor.get?.call(original_class); - } - }); - } - - // new getter + setter - Object.defineProperty(original_class, name, { - get:()=>static_scope.getVariable(name), - set:(val)=>setProxifiedValue(val) - }); - } - } - - // remote - - if ((remote_public?.hasOwnProperty(name) && remote_public[name]) || (remote_private?.hasOwnProperty(name) && remote_private[name])) { - - const timeout = timeout_public?.[name]??timeout_private?.[name]; - const filter = new Conjunction(class_send_filter, send_filter?.[name]); - - // function - if (typeof current_value == "function") { - const options = Object.create(options_prototype); - Object.assign(options, {filter, sign:true, scope_name:static_scope_name, timeout}); - const proxy_fn = getProxyFunction(name, options); - Object.defineProperty(original_class, name, {value:proxy_fn}) - } - - // field - else { - const options = Object.create(options_prototype); - Object.assign(options, {filter, sign:true, scope_name:static_scope_name, timeout}); - const proxy_fn = getProxyStaticValue(name, options); - Object.defineProperty(original_class, name, { - get: proxy_fn // set proxy function for getting static value - }); - } - } - - } - - - // each methods - - //const each_private = original_class.prototype[METADATA]?.[Decorators.IS_EACH]?.private; - const each_public = original_class.prototype[METADATA]?.[Decorators.IS_EACH]?.public; - - let each_scope: any; - - for (let [name, is_each] of Object.entries(each_public??{})) { - if (!is_each) continue; - - if (!static_scope) static_scope = StaticScope.get(static_scope_name) - - // add _e to current static scope - if (!each_scope) { - each_scope = {}; - static_scope.setVariable("_e", each_scope); // add to static scope - } - - let method:Function = original_class.prototype[name]; - let type = Type.getClassDatexType(original_class); - - if (typeof method != "function") throw new DatexError("@each can only be used with functions") - - - - /****** expose _e */ - // let meta_index = getMetaParamIndex(original_class.prototype, name); - // if (typeof meta_index == "number") meta_index ++; // shift meta_index (insert 'this' before) - - let proxy_method = function(_this:any, ...args:any[]) { - if (!(_this instanceof original_class)) { - console.warn(_this, args); - throw new ValueError("Invalid argument 'this': type should be " + type) - } - return method.call(_this, ...args) - }; - // add ' this' as first argument - //params?.unshift([type, "this"]) - - let dx_function = Pointer.proxifyValue(DatexFunction.createFromJSFunction(proxy_method, original_class, name), true, undefined, false, true) ; // generate - - each_scope[name] = dx_function // add to static scope - - } - - - // finally seal the static scope - if (static_scope) { - DatexObject.seal(static_scope); - initialized_static_scope_classes.set(original_class, static_scope); - } -} - - - const templated_classes = new Map() // original class, templated class -export function createTemplateClass(original_class:{ new(...args: any[]): any; }, type:Type, sync = true, add_js_interface = true, callerFile?:string){ +export function createTemplateClass(original_class: Class, type:Type, sync = true, add_js_interface = true, callerFile?:string, metadata?:Record){ - if (templated_classes.has(original_class)) return templated_classes.get(original_class); + if (templated_classes.has(original_class)) return templated_classes.get(original_class)!; original_class[DX_TYPE] = type; @@ -1066,19 +578,20 @@ export function createTemplateClass(original_class:{ new(...args: any[]): any; } type.jsTypeDefModule = callerFile; } + metadata ??= original_class[METADATA]; // set constructor, replicator, destructor - const constructor_name = Object.keys(original_class.prototype[METADATA]?.[Decorators.CONSTRUCTOR]?.public??{})[0] - const replicator_name = Object.keys(original_class.prototype[METADATA]?.[Decorators.REPLICATOR]?.public??{})[0] - const destructor_name = Object.keys(original_class.prototype[METADATA]?.[Decorators.DESTRUCTOR]?.public??{})[0] + const constructor_name = original_class.prototype['construct'] ? 'construct' : null; // Object.keys(metadata?.[Decorators.CONSTRUCTOR]?.public??{})[0] + const replicator_name = original_class.prototype['replicate'] ? 'replicate' : null; // Object.keys(metadata?.[Decorators.REPLICATOR]?.public??{})[0] + const destructor_name = original_class.prototype['destruct'] ? 'destruct' : null; // Object.keys(metadata?.[Decorators.DESTRUCTOR]?.public??{})[0] if (constructor_name) type.setConstructor(original_class.prototype[constructor_name]); if (replicator_name) type.setReplicator(original_class.prototype[replicator_name]); if (destructor_name) type.setDestructor(original_class.prototype[destructor_name]); // set template - const property_types = original_class.prototype[METADATA]?.[Decorators.FORCE_TYPE]?.public; - const allow_filters = original_class.prototype[METADATA]?.[Decorators.ALLOW_FILTER]?.public; + const property_types = metadata?.[Decorators.FORCE_TYPE]?.public; + const allow_filters = metadata?.[Decorators.ALLOW_FILTER]?.public; const template = {}; template[DX_PERMISSIONS] = {} @@ -1098,7 +611,7 @@ export function createTemplateClass(original_class:{ new(...args: any[]): any; } // iterate over all properties TODO different dx_name? - for (const [name, dx_name] of Object.entries(original_class.prototype[METADATA]?.[Decorators.PROPERTY]?.public??{})) { + for (const [name, dx_name] of Object.entries(metadata?.[Decorators.PROPERTY]?.public??{})) { let metadataConstructor = MetadataReflect.getMetadata && MetadataReflect.getMetadata("design:type", original_class.prototype, name); // if type is Object -> std:Any if (metadataConstructor == Object) metadataConstructor = null; @@ -1109,12 +622,8 @@ export function createTemplateClass(original_class:{ new(...args: any[]): any; } type.setTemplate(template) - - // has static scope methods? - _old_publicStaticClass(original_class); - // create shadow class extending the actual class - const sync_auto_cast_class = proxyClass(original_class, type, original_class[METADATA]?.[Decorators.IS_SYNC]?.constructor ?? sync) + const sync_auto_cast_class = proxyClass(original_class, type, metadata?.[Decorators.IS_SYNC]?.constructor ?? sync) // only for debugging / dev console TODO remove globalThis[sync_auto_cast_class.name] = sync_auto_cast_class; @@ -1124,27 +633,6 @@ export function createTemplateClass(original_class:{ new(...args: any[]): any; } return sync_auto_cast_class; } -// TODO each -// if (is_each) { - -// // call _e - -// const static_scope_name = original_class[METADATA]?.[Decorators.SCOPE_NAME]?.constructor ?? original_class.name -// let filter:DatexFilter; // contains all endpoints that have the pointer - -// Object.defineProperty(instance, p, {value: async function(...args:any[]) { -// if (!filter) { -// let ptr = DatexPointer.getByValue(this); -// if (!(ptr instanceof DatexPointer)) throw new DatexError("called @each method on non-pointer"); -// filter = DatexFilter.OR(await ptr.getSubscribersFilter(), ptr.origin); -// } -// console.log("all endpoints filter: " + filter); - -// return DatexRuntime.datexOut([`--static.${static_scope_name}._e.${p} ?`, [new DatexTuple(this, ...args)], {to:filter, sign:true}], filter); -// }}) -// } - - // Reflect metadata / decorator metadata, get parameters & types if available function getMethodParams(target:Function, method_name:string, meta_param_index?:number):Tuple{ @@ -1210,50 +698,28 @@ function normalizeFunctionParams(params: string) { } -// let _assigner_init = false; -// function initPropertyTypeAssigner(){ -// if (_assigner_init) return; -// _assigner_init = true; -// // TODO just a workaround, handle PropertyTypeAssigner different (currently nodejs error!! DatexPointer not yet defined) -// Pointer.setPropertyTypeAssigner({getMethodMetaParamIndex:getMetaParamIndex, getMethodParams:getMethodParams}) -// } -//Pointer.setPropertyTypeAssigner({getMethodMetaParamIndex:getMetaParamIndex, getMethodParams:getMethodParams}) - DatexFunction.setMethodParamsSource(getMethodParams) DatexFunction.setMethodMetaIndexSource(getMetaParamIndex) -/** @meta: mark meta parameter in a datex method with @meta */ -// export function meta(target: Object, propertyKey: string | symbol, parameterIndex: number) { -// Reflect.defineMetadata( -// "unyt:meta", -// parameterIndex, -// target, -// propertyKey -// ); -// } - // new version for implemented feature functions / attributes: call datex_advanced() on the class (ideally usa as a decorator, currently not supported by ts) -interface DatexClass unknown) = (new (...args: unknown[]) => unknown), Construct = InstanceType["construct"]> { +export interface DatexClass unknown) = (new (...args: unknown[]) => unknown), Construct = InstanceType["construct" & keyof InstanceType]> { new(...args: Construct extends (...args: any) => any ? Parameters : ConstructorParameters): datexClassType; - // special functions - on_result: (call: (data:any, meta:{station_id:number, station_bundle:number[]})=>any) => dc; - options: (options:any)=>T; - - // decorator equivalents - to: (target:target_clause) => dc; - no_result: () => dc; - - // @sync objects - is_origin?: boolean; - origin_id?: number; - room_id?: number; } -type dc&{new (...args:unknown[]):unknown}> = DatexClass & T & ((struct:InstanceType) => datexClassType); +export type MethodKeys = { + [K in keyof T]: T[K] extends (...args: any) => any ? K : never; +}[keyof T]; + +export type dc&{new (...args:unknown[]):unknown}, OT extends {new (...args:unknown[]):unknown} = ObjectRef> = + DatexClass & + OT & + // TODO: required instead of OT to disable default constructor, but leads to problems with typing + // Pick & + ((struct:Omit, MethodKeys>>) => datexClassType); /** * Workaround to enable correct @sync class typing, until new decorators support it. diff --git a/js_adapter/legacy_decorators.ts b/js_adapter/legacy_decorators.ts deleted file mode 100644 index 6df45eb2..00000000 --- a/js_adapter/legacy_decorators.ts +++ /dev/null @@ -1,326 +0,0 @@ -/** - ╔══════════════════════════════════════════════════════════════════════════════════════╗ - ║ Typescript Legacy Decorators for the DATEX JS Interface ║ - ║ - Use until the JS decorator proposal (TC39 Stage 2) is fully implemented ║ - ╠══════════════════════════════════════════════════════════════════════════════════════╣ - ║ Unyt core library ║ - ║ Visit docs.unyt.org/unyt_js for more information ║ - ╠═════════════════════════════════════════╦════════════════════════════════════════════╣ - ║ © 2021 unyt.org ║ ║ - ╚═════════════════════════════════════════╩════════════════════════════════════════════╝ - */ - -import { Decorators, METADATA } from "./js_class_adapter.ts"; -import { } from "../runtime/runtime.ts"; -import { endpoint_name, Target, target_clause } from "../types/addressing.ts"; -import { Type } from "../types/type.ts"; -import { UpdateScheduler, Pointer } from "../runtime/pointers.ts"; - -// decorator types -export type context_kind = 'class'|'method'|'getter'|'setter'|'field'|'auto-accessor'; -export type context_name = string|symbol|undefined; -export type context_meta_setter = (key:symbol, value:any) => void -export type context_meta_getter = (key:symbol ) => any - -type decorator_target = {[key: string]: any} & Partial, never>>; -type decorator_target_optional_params = decorator_target | Function; // only working for static methods! - -const __metadataPrivate = new WeakMap(); -const createObjectWithPrototype = (obj:object, key:any) => Object.hasOwnProperty.call(obj, key) ? obj[key] : Object.create(obj[key] || Object.prototype); - - -// get context kind (currently only supports class, method, field) -function getContextKind(args:any[]):context_kind { - if (typeof args[0] == "function" && args[1] == null && args[2] == null) return 'class'; - if ((typeof args[0] == "function" || typeof args[0] == "object") && (typeof args[2] == "function" || typeof args[2]?.value == "function")) return 'method'; - if ((typeof args[0] == "function" || typeof args[0] == "object") && typeof args[1] == "string") return 'field'; -} -// is context static field/method? -function isContextStatic(args:any[]):boolean { - return typeof args[0] == "function" && args[1] != null; -} - - -// add optional arguments, then call JS Interface decorator handler -export function handleDecoratorArgs(args:any[], method:(value:any, name:context_name, kind:context_kind, is_static:boolean, is_private:boolean, setMetadata:context_meta_setter, getMetadata:context_meta_getter, params?:any[]) => any, first_argument_is_function = false):(_invalid_param_0_: decorator_target_optional_params, _invalid_param_1_?: string, _invalid_param_2_?: PropertyDescriptor) => any { - let kind = getContextKind(args); - // is @decorator(x,y,z) - if (!kind || first_argument_is_function) { - // inject args as decorator params - const params = args; // x,y,z - return (...args:any[]) => { - let kind = getContextKind(args); - // same as below (in else), + params - let is_static = isContextStatic(args); - let target = args[0]; - let name = kind == 'class' ? args[0].name : args[1]; - let value = kind == 'class' ? args[0] : args[2]?.value; - let meta_setter = createMetadataSetter(target, name, kind == 'class'); - let meta_getter = createMetadataGetter(target, name, kind == 'class'); - //console.log("@"+method.name + " name: " + name + ", kind: " + kind + ", is_static:" + is_static + ", params:", params, value) - return method(value, name, kind, is_static, false, meta_setter, meta_getter, params); - } - } - // is direct @decorator - else { - let is_static = isContextStatic(args); - let target = args[0]; - let name = kind == 'class' ? args[0].name : args[1]; - let value = kind == 'class' ? args[0] : args[2]?.value; - let meta_setter = createMetadataSetter(target, name, kind == 'class'); - let meta_getter = createMetadataGetter(target, name, kind == 'class'); - //console.log("@"+method.name + " name: " + name + ", kind: " + kind + ", is_static:" + is_static, value) - return method(value, name, kind, is_static, false, meta_setter, meta_getter); - } -} - -function createMetadataSetter(target:Function, name:string, is_constructor = false, is_private=false) { - return (key:symbol, value:unknown)=>{ - if (typeof key !== "symbol") { - throw new TypeError("the key must be a Symbol"); - } - - target[METADATA] = createObjectWithPrototype(target, METADATA); - target[METADATA][key] = createObjectWithPrototype(target[METADATA], key); - target[METADATA][key].public = createObjectWithPrototype(target[METADATA][key], "public"); - - if (!Object.hasOwnProperty.call(target[METADATA][key], "private")) { - Object.defineProperty(target[METADATA][key], "private", { - get() { - return Object.values(__metadataPrivate.get(target[METADATA][key]) || {}).concat(Object.getPrototypeOf(target[METADATA][key])?.private || []); - } - }); - } - // constructor - if (is_constructor) { - target[METADATA][key].constructor = value; - } - // private - else if (is_private) { - if (!__metadataPrivate.has(target[METADATA][key])) { - __metadataPrivate.set(target[METADATA][key], {}); - } - __metadataPrivate.get(target[METADATA][key])[name] = value; - } - // public - else { - target[METADATA][key].public[name] = value; - } - } -} -function createMetadataGetter(target:Function, name:string, is_constructor = false, is_private=false) { - return (key:symbol) => { - if (target[METADATA] && target[METADATA][key]) { - if (is_constructor) return target[METADATA][key]["constructor"]?.[name]; - else if (is_private) return (__metadataPrivate.has(target[METADATA][key]) ? __metadataPrivate.get(target[METADATA][key])?.[name] : undefined) - else return target[METADATA][key].public?.[name] - } - } -} - -// legacy decorator functions - -// @deprecated -// TODO: remove, use @property (also for static methods) -export function expose(allow?: target_clause):any -export function expose(_invalid_param_0_: decorator_target_optional_params, _invalid_param_1_?: string, _invalid_param_2_?: PropertyDescriptor):any -export function expose(...args:any[]) { - return handleDecoratorArgs(args, Decorators.public); -} - - - -// @deprecated -// TODO: remove, use endpoint -export function scope(name:string):any -export function scope(_invalid_param_0_: decorator_target_optional_params, _invalid_param_1_?: string, _invalid_param_2_?: PropertyDescriptor):any -export function scope(...args:any[]) { - return handleDecoratorArgs(args, Decorators.namespace); -} - -// @deprecated -// TODO: remove, use endpoint -export function namespace(name:string):any -export function namespace(_invalid_param_0_: decorator_target_optional_params, _invalid_param_1_?: string, _invalid_param_2_?: PropertyDescriptor):any -export function namespace(...args:any[]) { - return handleDecoratorArgs(args, Decorators.namespace); -} - -// use instead of @namespace @to -export function endpoint(endpoint:target_clause|endpoint_name, scope_name?:string):any -export function endpoint(_invalid_param_0_: decorator_target_optional_params, _invalid_param_1_?: string, _invalid_param_2_?: PropertyDescriptor):any -export function endpoint(...args:any[]) { - return handleDecoratorArgs(args, Decorators.endpoint); -} - - -export function endpoint_default(target: any, name?: string, method?:any):any -export function endpoint_default(...args:any[]): any { - return handleDecoratorArgs(args, Decorators.default); -} - -export function default_property(target: any, name?: string, method?:any):any -export function default_property(...args:any[]): any { - return handleDecoratorArgs(args, Decorators.default_property); -} - -// @deprecated -// TODO: remove, use @property (also for static methods) -export function remote(from?: target_clause):any -export function remote(_invalid_param_0_: decorator_target_optional_params, _invalid_param_1_: string) -export function remote(...args:any[]) { - return handleDecoratorArgs(args, Decorators.remote); -} - - -export function docs(content: string):any -export function docs(...args:any[]) { - return handleDecoratorArgs(args, Decorators.docs); -} - -export function meta(index: number):any -export function meta(target: any, name?: string, method?:any) -export function meta(...args:any[]) { - return handleDecoratorArgs(args, Decorators.meta); -} - -export function sign(sign: boolean):any -export function sign(_invalid_param_0_: decorator_target_optional_params, _invalid_param_1_?: string, _invalid_param_2_?: PropertyDescriptor) -export function sign(...args:any[]) { - return handleDecoratorArgs(args, Decorators.sign); -} - -export function encrypt(encrypt: boolean):any -export function encrypt(_invalid_param_0_: decorator_target_optional_params, _invalid_param_1_?: string, _invalid_param_2_?: PropertyDescriptor) -export function encrypt(...args:any[]) { - return handleDecoratorArgs(args, Decorators.encrypt); -} - -export function no_result(_invalid_param_0_: decorator_target_optional_params, _invalid_param_1_?: string, _invalid_param_2_?: PropertyDescriptor) -export function no_result(...args:any[]) { - return handleDecoratorArgs(args, Decorators.no_result); -} - -export function timeout(msecs: number):any -export function timeout(...args:any[]) { - return handleDecoratorArgs(args, Decorators.timeout); -} - -export function allow(allow?: target_clause):any -export function allow(...args:any[]) { - return handleDecoratorArgs(args, Decorators.allow); -} - -export function to(to?: target_clause|endpoint_name):any -export function to(...args:any[]) { - return handleDecoratorArgs(args, Decorators.to); -} - - -export function sealed(target: any, name?: string, method?:any) -export function sealed(...args:any[]): any { - return handleDecoratorArgs(args, Decorators.sealed); -} - - -export function each(target: any, name?: string, method?:any) -export function each(...args:any[]): any { - return handleDecoratorArgs(args, Decorators.each); -} - -export function sync(type:string):any -export function sync(target: any, name?: string, method?:any):any -export function sync(...args:any[]): any { - return handleDecoratorArgs(args, Decorators.sync); -} - -/** - * @deprecated use \@sync - */ -export function template(type:string):any -/** - * @deprecated use \@sync - */ -export function template(target: any, name?: string, method?:any):any -export function template(...args:any[]): any { - return handleDecoratorArgs(args, Decorators.template); -} - -export function property(name:string|number):any -export function property(target: any, name?: string, method?:any):any -export function property(...args:any[]): any { - return handleDecoratorArgs(args, Decorators.property); -} - -export function jsdoc(target: any, name?: string, method?:any):any -export function jsdoc(...args:any[]): any { - return handleDecoratorArgs(args, Decorators.jsdoc); -} - - -export function serialize(serializer:(parent:any, value:any)=>any):any -export function serialize(...args:any[]): any { - return handleDecoratorArgs(args, Decorators.serialize, true); -} - - -export function observe(handler:Function):any -export function observe(...args:any[]): any { - return handleDecoratorArgs(args, Decorators.observe); -} - -export function anonymize(_invalid_param_0_: decorator_target_optional_params, _invalid_param_1_?: string, _invalid_param_2_?: PropertyDescriptor) -export function anonymize(...args:any[]): any { - return handleDecoratorArgs(args, Decorators.anonymize); -} - -export function anonymous(_target: T):T -export function anonymous(target: any, name?: string, method?: PropertyDescriptor) -export function anonymous(...args:any[]) { - // no decorator, but function encapsulating object to make it syncable (proxy) - if (args[0]==undefined || args[0] == null || (args[1]===undefined && args[0] && typeof args[0] == "object")) { - return Pointer.create(null, args[0], /*TODO*/false, undefined, false, true).val; - } - // decorator - return handleDecoratorArgs(args, Decorators.anonymous); -} - - -export function type(type:string|Type):any -export function type(...args:any[]): any { - return handleDecoratorArgs(args, Decorators.type, true); -} - -export function assert(assertion:(val:any)=>boolean|string|undefined|null):any -export function assert(...args:any[]): any { - return handleDecoratorArgs(args, Decorators.assert, true); -} - - -export function from(type:string|Type):any -export function from(...args:any[]): any { - return handleDecoratorArgs(args, Decorators.from); -} - -export function update(interval:number):any -export function update(scheduler:UpdateScheduler):any -export function update(...args:any[]): any { - return handleDecoratorArgs(args, Decorators.update); -} - -// special sync class methods -export function constructor(target: any, propertyKey: string, descriptor: PropertyDescriptor) -export function constructor(...args:any[]): any { - return handleDecoratorArgs(args, Decorators.constructor); -} - -export function replicator(target: any, propertyKey: string, descriptor: PropertyDescriptor) -export function replicator(...args:any[]): any { - return handleDecoratorArgs(args, Decorators.replicator); -} - -export function destructor(target: any, propertyKey: string, descriptor: PropertyDescriptor) -export function destructor(...args:any[]): any { - return handleDecoratorArgs(args, Decorators.destructor); -} - diff --git a/mod.ts b/mod.ts index cbc7e9fc..661ca2c0 100644 --- a/mod.ts +++ b/mod.ts @@ -18,7 +18,7 @@ import * as Datex from "./datex_all.ts"; export {Datex}; -export * from "./js_adapter/legacy_decorators.ts"; +export * from "./js_adapter/decorators.ts"; export * from "./datex_short.ts"; export {init} from "./init.ts"; diff --git a/network/blockchain_adapter.ts b/network/blockchain_adapter.ts index 5a194ab7..8c76e313 100644 --- a/network/blockchain_adapter.ts +++ b/network/blockchain_adapter.ts @@ -66,11 +66,11 @@ export type BCData = @sync export class BCEntry { - @property declare index: entry_index - @property declare type:T - @property declare data:BCData - @property declare creator?:Endpoint - @property declare signature?:ArrayBuffer + @property index!: entry_index + @property type!:T + @property data!:BCData + @property creator?:Endpoint + @property signature?:ArrayBuffer constructor(data?: { index: entry_index, diff --git a/network/communication-hub.ts b/network/communication-hub.ts index e9b0b703..5b743094 100644 --- a/network/communication-hub.ts +++ b/network/communication-hub.ts @@ -1,4 +1,4 @@ -import { dxb_header } from "../utils/global_types.ts"; +import type { dxb_header } from "../utils/global_types.ts"; import { Endpoint, BROADCAST, LOCAL_ENDPOINT } from "../types/addressing.ts"; import { CommunicationInterface, CommunicationInterfaceSocket, ConnectedCommunicationInterfaceSocket } from "./communication-interface.ts"; import { Disjunction } from "../types/logic.ts"; diff --git a/network/communication-interface.ts b/network/communication-interface.ts index 266175b9..c16adb3c 100644 --- a/network/communication-interface.ts +++ b/network/communication-interface.ts @@ -6,7 +6,7 @@ import { COM_HUB_SECRET, communicationHub } from "./communication-hub.ts"; import { IOHandler } from "../runtime/io_handler.ts"; import { LOCAL_ENDPOINT } from "../types/addressing.ts"; import { Runtime } from "../runtime/runtime.ts"; -import { dxb_header } from "../utils/global_types.ts"; +import type { dxb_header } from "../utils/global_types.ts"; export enum InterfaceDirection { /** diff --git a/network/communication-interfaces/webrtc-interface.ts b/network/communication-interfaces/webrtc-interface.ts new file mode 100644 index 00000000..2f957919 --- /dev/null +++ b/network/communication-interfaces/webrtc-interface.ts @@ -0,0 +1,75 @@ +import { Endpoint } from "../../types/addressing.ts"; +import { CommunicationInterface, CommunicationInterfaceSocket, InterfaceDirection, InterfaceProperties } from "../communication-interface.ts"; + +@endpoint class WebRTCSignaling { + + @property static offer(data:any) { + InterfaceManager.connect("webrtc", datex.meta!.sender, [data]); + } + + @property static accept(data:any) { + WebRTCClientInterface.waiting_interfaces_by_endpoint.get(datex.meta!.sender)?.setRemoteDescription(data); + } + + @property static candidate(data:any) { + WebRTCClientInterface.waiting_interfaces_by_endpoint.get(datex.meta!.sender)?.addICECandidate(data); + } +} + +export class WebRTCInterfaceSocket extends CommunicationInterfaceSocket { + + handleReceive = (event: MessageEvent) => { + if (event.data instanceof ArrayBuffer) { + this.receive(event.data) + } + } + + open() { + this.worker.addEventListener("message", this.handleReceive); + } + + close() { + this.worker.removeEventListener('message', this.handleReceive); + } + + send(dxb: ArrayBuffer) { + try { + this.worker.postMessage(dxb) + return true; + } + catch { + return false; + } + } +} + +/** + * Creates a direct DATEX communication channel between two WebRTC clients + */ +export class WebRTCInterface extends CommunicationInterface { + + public properties: InterfaceProperties = { + type: "webrtc", + direction: InterfaceDirection.IN_OUT, + latency: 20, + bandwidth: 50_000 + } + + constructor(endpoint: Endpoint) { + super() + const socket = new WebRTCInterfaceSocket(); + socket.endpoint = endpoint; + this.addSocket(socket); + } + + connect() { + return true; + } + + disconnect() {} + + cloneSocket(_socket: WebRTCInterfaceSocket) { + return new WebRTCInterfaceSocket(); + } + +} \ No newline at end of file diff --git a/network/communication-interfaces/websocket-interface.ts b/network/communication-interfaces/websocket-interface.ts index 7edec920..a7bbc1d8 100644 --- a/network/communication-interfaces/websocket-interface.ts +++ b/network/communication-interfaces/websocket-interface.ts @@ -51,7 +51,7 @@ export abstract class WebSocketInterface extends CommunicationInterface { // don't trigger any further errorHandlers - // webSocket.removeEventListener('close', errorHandler); + webSocket.removeEventListener('close', errorHandler); webSocket.removeEventListener('error', errorHandler); this.#webSockets.delete(webSocket); @@ -59,6 +59,7 @@ export abstract class WebSocketInterface extends CommunicationInterface - static getClassDatexType(class_constructor:Class):Type { + static getClassDatexType(class_constructor:Class):Type|undefined { let config:js_interface_configuration; // get directly from class - if (config = this.configurations_by_class.get(class_constructor)) return config.__type; + if ((config = this.configurations_by_class.get(class_constructor))) return config.__type; // get from prototype of class - if (config = this.configurations_by_class.get(Object.getPrototypeOf(class_constructor))) return config.__type; + if ((config = this.configurations_by_class.get(Object.getPrototypeOf(class_constructor)))) return config.__type; // check full prototype chain (should not happen normally, unnessary to loop through every time) // for (let [_class, config] of this.configurations_by_class) { diff --git a/runtime/pointers.ts b/runtime/pointers.ts index a35bf19f..ea7ae1c6 100644 --- a/runtime/pointers.ts +++ b/runtime/pointers.ts @@ -11,7 +11,7 @@ import { BinaryCode } from "../compiler/binary_codes.ts"; import { JSInterface } from "./js_interface.ts"; import { Stream } from "../types/stream.ts"; import { Tuple } from "../types/tuple.ts"; -import { primitive } from "../types/abstract_types.ts"; +import type { primitive } from "../types/abstract_types.ts"; import { Function as DatexFunction } from "../types/function.ts"; import { Quantity } from "../types/quantity.ts"; import { buffer2hex, hex2buffer } from "../utils/utils.ts"; @@ -29,8 +29,8 @@ import { IterableWeakMap } from "../utils/iterable-weak-map.ts"; import { LazyPointer } from "./lazy-pointer.ts"; import { ReactiveArrayMethods } from "../types/reactive-methods/array.ts"; import { Assertion } from "../types/assertion.ts"; -import { StorageSet } from "../types/storage-set.ts"; import { Storage } from "../storage/storage.ts"; +import { client_type } from "../utils/constants.ts"; export type observe_handler = (value:V extends RefLike ? T : V, key?:K, type?:Ref.UPDATE_TYPE, transform?:boolean, is_child_update?:boolean, previous?: any, atomic_id?:symbol)=>void|boolean export type observe_options = {types?:Ref.UPDATE_TYPE[], ignore_transforms?:boolean, recursive?:boolean} @@ -574,7 +574,9 @@ export class PointerProperty extends Ref { #leak_js_properties: boolean - public pointer?: Pointer; + private _strongRef?: any // strong reference to own pointer to prevent garbage collection + + public readonly pointer?: Pointer; private lazy_pointer?: LazyPointer; private constructor(pointer: Pointer|LazyPointer|undefined, public key: any, leak_js_properties = false) { @@ -594,12 +596,16 @@ export class PointerProperty extends Ref { } private setPointer(ptr: Pointer) { + // @ts-ignore private this.pointer = ptr; - this.pointer.is_persistent = true; // TODO: make unpersistent when pointer property deleted + + this._strongRef = ptr.val; + if (!PointerProperty.synced_pairs.has(ptr)) PointerProperty.synced_pairs.set(ptr, new Map()); - PointerProperty.synced_pairs.get(ptr)!.set(this.key, this); // save in map + PointerProperty.synced_pairs.get(ptr)!.set(this.key, new WeakRef(this)); // save in map } + /** * Called when the bound lazy pointer is loaded. * If there is no lazy pointer, the callback is called immediately @@ -610,7 +616,7 @@ export class PointerProperty extends Ref { else callback(this, this); } - private static synced_pairs = new WeakMap>() + private static synced_pairs = new WeakMap>>() // TODO: use InferredPointerProperty (does not collapse) /** @@ -627,7 +633,14 @@ export class PointerProperty extends Ref { if (pointer instanceof Pointer) { if (!this.synced_pairs.has(pointer)) this.synced_pairs.set(pointer, new Map()); - if (this.synced_pairs.get(pointer)!.has(key)) return this.synced_pairs.get(pointer)!.get(key)!; + if (this.synced_pairs.get(pointer)!.has(key)) { + const weakRef = this.synced_pairs.get(pointer)!.get(key); + const pointerProperty = weakRef.deref(); + if (pointerProperty) return pointerProperty; + else { + this.synced_pairs.get(pointer)!.delete(key); + } + } } return new PointerProperty(pointer, key, leak_js_properties); @@ -778,18 +791,18 @@ type _Proxy$ = _Proxy$Function & T extends Array ? // array { - [key: number]: RefLikeOut, + [key: number]: RefLike, map(callbackfn: (value: MaybeObjectRef, index: number, array: V[]) => U, thisArg?: any): Pointer } : ( T extends Map ? { - get(key: K): RefLikeOut + get(key: K): RefLike } // normal object - : {readonly [K in keyof T]: RefLikeOut} // always map properties to pointer property references + : {[K in keyof T]: RefLike} // always map properties to pointer property references ) type _PropertyProxy$ = _Proxy$Function & @@ -1285,6 +1298,18 @@ export class Pointer extends Ref { } public static get is_local() {return this.#is_local} + #createdInContext = true; + /** + * Indicates if the pointer was created in the current context + * or fetched (from storage or network) + */ + public get createdInContext() { + return this.#createdInContext; + } + public set createdInContext(fetched: boolean) { + this.#createdInContext = fetched; + } + /** 21 bytes address: 1 byte address type () 18/16 byte origin id - 2/4 byte origin instance - 4 bytes timestamp - 1 byte counter*/ /** * Endpoint id types: @@ -1462,6 +1487,9 @@ export class Pointer extends Ref { // get value if pointer value not yet loaded if (!pointer.#loaded) { + + // was not created new in current context + pointer.createdInContext = false; // first try loading from storage let stored:any = NOT_EXISTING; @@ -2710,7 +2738,8 @@ export class Pointer extends Ref { // potential storage pointer initialized Storage.providePointer(this); - if (this.isStored) { + // only in frontend, disabled for backend (TODO) + if (this.isStored && client_type == "browser") { // get subsriber caches Storage.getPointerSubscriberCache(this.id).then(cache => { if (cache) { @@ -2737,6 +2766,10 @@ export class Pointer extends Ref { #transform_scope?:Scope; get transform_scope() {return this.#transform_scope} + #smart_transform_method?: (...args:any[])=>any + get smart_transform_method() {return this.#smart_transform_method} + set smart_transform_method(method: (...args:any[])=>any) {this.#smart_transform_method = method} + #force_transform = false; // if true, the pointer transform function is always sent via DATEX set force_local_transform(force_transform: boolean) {this.#force_transform = force_transform} get force_local_transform() {return this.#force_transform} @@ -2817,6 +2850,7 @@ export class Pointer extends Ref { protected smartTransform(transform:SmartTransformFunction, persistent_datex_transform?:string, forceLive = false, ignoreReturnValue = false, options?:SmartTransformOptions): Pointer { if (persistent_datex_transform) this.setDatexTransform(persistent_datex_transform) // TODO: only workaround + this.#smart_transform_method = transform; const state: TransformState = { isLive: false, @@ -3276,8 +3310,9 @@ export class Pointer extends Ref { // already subscribed if (this.subscribers.has(subscriber)) return; - // also store in subscriber cache - if (this.isStored) { + // also store in subscriber cache - only in frontend + // (TODO: required for backend? currently disabled because backend is not stopped frequently, only leads to overhead) + if (this.isStored && client_type == "browser") { if (this.#subscriberCache) this.#subscriberCache.add(subscriber) else { Storage.requestSubscriberCache(this.id).then(cache => { @@ -3381,6 +3416,8 @@ export class Pointer extends Ref { if (!(await endpoint.isOnline())) { this.clearEndpointSubscriptions(endpoint); this.clearEndpointPermissions(endpoint) + // TODO: this should ideally directly be handleded by the runtime + Runtime.clearEndpointScopes(endpoint); } } } @@ -3460,8 +3497,8 @@ export class Pointer extends Ref { } // proxify a (child) value, use the pointer context - private proxifyChild(name:string, value:unknown) { - if (NOT_EXISTING && !this.shadow_object) throw new Error("Cannot proxify child of non-object value"); + proxifyChild(name:string, value:unknown) { + if (value === NOT_EXISTING && !this.shadow_object) throw new Error("Cannot proxify child of non-object value"); let child = value === NOT_EXISTING ? this.shadow_object![name] : value; // special native function -> conversion; @@ -4381,7 +4418,9 @@ export class Pointer extends Ref { // key specific observers if (key!=undefined) { for (const [o, options] of this.change_observers.get(key)||[]) { - if ((!options?.types || options.types.includes(type)) && !(is_transform && options?.ignore_transforms) && (!is_child_update || !options || options.recursive)) promises.push(o(value, key, type, is_transform, is_child_update, previous, atomic_id)); + if ((!options?.types || options.types.includes(type)) && !(is_transform && options?.ignore_transforms) && (!is_child_update || !options || options.recursive)) { + promises.push(o(value, key, type, is_transform, is_child_update, previous, atomic_id)); + } } // bound observers for (const [object, entries] of this.bound_change_observers.entries()) { diff --git a/runtime/runtime.ts b/runtime/runtime.ts index 4a68e775..e39868c4 100644 --- a/runtime/runtime.ts +++ b/runtime/runtime.ts @@ -32,7 +32,7 @@ import { BROADCAST, Endpoint, endpoints, IdEndpoint, LOCAL_ENDPOINT, Target, tar import { RuntimePerformance } from "./performance_measure.ts"; import { NetworkError, PermissionError, PointerError, RuntimeError, SecurityError, ValueError, Error as DatexError, CompilerError, TypeError, SyntaxError, AssertionError } from "../types/errors.ts"; import { Function as DatexFunction } from "../types/function.ts"; -import { Storage } from "../storage/storage.ts"; +import { MatchCondition, Storage } from "../storage/storage.ts"; import { Observers } from "../utils/observers.ts"; import { BinaryCode } from "../compiler/binary_codes.ts"; import type { ExecConditions, trace, compile_info, datex_meta, datex_scope, dxb_header, routing_info } from "../utils/global_types.ts"; @@ -52,7 +52,7 @@ import { JSInterface } from "./js_interface.ts"; import { Stream } from "../types/stream.ts"; import { Quantity } from "../types/quantity.ts"; import { Scope } from "../types/scope.ts"; -import { fundamental } from "../types/abstract_types.ts"; +import type { fundamental } from "../types/abstract_types.ts"; import { IterationFunction as IteratorFunction, Iterator, RangeIterator } from "../types/iterator.ts"; import { Assertion } from "../types/assertion.ts"; import { Deferred } from "../types/deferred.ts"; @@ -105,6 +105,7 @@ RuntimePerformance.marker("module loading time", "modules_loaded", "runtime_star // TODO reader for node.js const ReadableStreamDefaultReader = globalThis.ReadableStreamDefaultReader ?? class {}; +const EXPOSE = Symbol("EXPOSE"); export class StaticScope { @@ -115,11 +116,15 @@ export class StaticScope { public static readonly DOCS: unique symbol = Symbol("docs"); // return a scope with a given name, if it already exists - public static get(name?:string):StaticScope { - return this.scopes.get(name) || new StaticScope(name); + public static get(name?:string, expose = true): StaticScope { + if (!expose) return new StaticScope(name, false); + else return this.scopes.get(name) || new StaticScope(name); } - private constructor(name?:string){ + [EXPOSE]: boolean + + private constructor(name?:string, expose = true){ + this[EXPOSE] = expose; const proxy = Pointer.proxifyValue(this, false, undefined, false); DatexObject.setWritePermission(>proxy, undefined); // make readonly @@ -143,9 +148,9 @@ export class StaticScope { // update/set the name of this static scope set name(name:string){ - if (this[StaticScope.NAME]) StaticScope.scopes.delete(this[StaticScope.NAME]); + if (this[StaticScope.NAME] && this[EXPOSE]) StaticScope.scopes.delete(this[StaticScope.NAME]); this[StaticScope.NAME] = name; - StaticScope.scopes.set(this[StaticScope.NAME], this); + if (this[EXPOSE]) StaticScope.scopes.set(this[StaticScope.NAME], this); if (this[StaticScope.NAME] == "std") StaticScope.STD = this; } @@ -585,9 +590,9 @@ export class Runtime { } // get content of https://, file://, ... - public static async getURLContent(url_string:string, raw?:RAW, cached?:boolean):Promise - public static async getURLContent(url:URL, raw?:RAW, cached?:boolean):Promise - public static async getURLContent(url_string:string|URL, raw:RAW=false, cached = false):Promise { + public static async getURLContent(url_string:string, raw?:RAW, cached?:boolean, potentialDatexAsJsModule?: boolean):Promise + public static async getURLContent(url:URL, raw?:RAW, cached?:boolean, potentialDatexAsJsModule?:boolean):Promise + public static async getURLContent(url_string:string|URL, raw:RAW=false, cached = false, potentialDatexAsJsModule = true):Promise { if (url_string.toString().startsWith("route:") && window.location?.origin) url_string = new URL(url_string.toString().replace("route:", ""), window.location.origin) @@ -609,11 +614,23 @@ export class Runtime { if (url.protocol == "https:" || url.protocol == "http:" || url.protocol == "blob:") { let response:Response|undefined = undefined; + let overrideContentType: string|undefined; let doFetch = true; - // possible js module import: fetch headers first and check content type: - if (!raw && (url_string.endsWith("js") || url_string.endsWith("ts") || url_string.endsWith("tsx") || url_string.endsWith("jsx") || url_string.endsWith("dx") || url_string.endsWith("dxb"))) { + + // exceptions to force potentialDatexAsJsModule (definitely dx files) + if (url_string.endsWith("/.dxb") || url_string.endsWith("/.dx") || url_string == "https://unyt.cc/nodes.dx") { + potentialDatexAsJsModule = false; + } + + // js module import + if (!raw && (url_string.endsWith(".js") || url_string.endsWith(".ts") || url_string.endsWith(".tsx") || url_string.endsWith(".jsx"))) { + doFetch = false; // no body fetch required, can directly import() module + overrideContentType = "application/javascript" + } + // potential js module as dxb/dx: fetch headers first and check content type + else if (!raw && potentialDatexAsJsModule && (url_string.endsWith(".dx") || url_string.endsWith(".dxb"))) { try { response = await fetch(url, {method: 'HEAD', cache: 'no-store'}); const type = response.headers.get('content-type'); @@ -641,33 +658,33 @@ export class Runtime { } } - const type = response.headers.get('content-type'); + const type = overrideContentType ?? response?.headers.get('content-type'); if (type == "application/datex" || type == "text/dxb" || url_string.endsWith(".dxb")) { - const content = await response.arrayBuffer(); + const content = await response!.arrayBuffer(); if (raw) result = [content, type]; else result = await this.executeDXBLocally(content, url); } else if (type?.startsWith("text/datex") || url_string.endsWith(".dx")) { - const content = await response.text() + const content = await response!.text() if (raw) result = [content, type]; else result = await this.executeDatexLocally(content, undefined, undefined, url); } else if (type?.startsWith("application/json5") || url_string.endsWith(".json5")) { - const content = await response.text(); + const content = await response!.text(); if (raw) result = [content, type]; else result = await Runtime.datexOut([content, [], {sign:false, encrypt:false, type:ProtocolDataType.DATA}]); } else if (type?.startsWith("application/json") || type?.endsWith("+json")) { - if (raw) result = [await response.text(), type]; - else result = await response.json() + if (raw) result = [await response!.text(), type]; + else result = await response!.json() } else if (type?.startsWith("text/javascript") || type?.startsWith("application/javascript")) { - if (raw) result = [await response.text(), type]; + if (raw) result = [await response!.text(), type]; else result = await import(url_string) } else { - const content = await response.arrayBuffer() + const content = await response!.arrayBuffer() if (raw) result = [content, type]; else { if (!type) throw Error("Cannot infer type from URL content"); @@ -1262,6 +1279,15 @@ export class Runtime { else return [] } + /** + * Removes all active datex scopes for an endpoint + */ + public static clearEndpointScopes(endpoint: Endpoint) { + const removeCount = this.active_datex_scopes.get(endpoint)?.size; + this.active_datex_scopes.delete(endpoint); + if (removeCount) logger.debug("removed " + removeCount + " datex scopes for " + endpoint); + } + /** * Creates default static scopes * + other async initializations @@ -1816,6 +1842,7 @@ export class Runtime { header.sender.setOnline(false) Pointer.clearEndpointSubscriptions(header.sender) Pointer.clearEndpointPermissions(header.sender) + this.clearEndpointScopes(header.sender); } else { logger.error("ignoring unsigned GOODBYE message") @@ -2015,7 +2042,7 @@ export class Runtime { // save persistent memory if (scope.persistent_vars) { const identifier = scope.context_location.toString() - for (let name of scope.persistent_vars) Runtime.saveScopeMemoryValue(identifier, name, scope.internal_vars[name]); + for (const name of scope.persistent_vars) Runtime.saveScopeMemoryValue(identifier, name, scope.internal_vars[name]); } // cleanup @@ -2245,7 +2272,7 @@ export class Runtime { let new_value:any = UNKNOWN_TYPE; // only handle std namespace / js:Object / js:Symbol - if (type.namespace == "std" || type == Type.js.NativeObject || type == Type.js.Symbol) { + if (type.namespace == "std" || type == Type.js.NativeObject || type == Type.js.Symbol || type == Type.js.RegExp) { const uncollapsed_old_value = old_value if (old_value instanceof Pointer) old_value = old_value.val; @@ -2332,6 +2359,18 @@ export class Runtime { else new_value = INVALID; break; } + case Type.js.RegExp: { + if (typeof old_value == "string") new_value = new RegExp(old_value); + else if (old_value instanceof Tuple) { + const array = old_value.toArray() as [string, string?]; + new_value = new RegExp(...array); + } + else if (old_value instanceof Array) { + new_value = new RegExp(...old_value as [string, string?]); + } + else new_value = INVALID; + break; + } case Type.std.Tuple: { if (old_value === VOID) new_value = new Tuple().seal(); else if (old_value instanceof Array){ @@ -2668,6 +2707,9 @@ export class Runtime { // symbol if (typeof value == "symbol") return value.toString().slice(7,-1) || undefined + // regex + if (value instanceof RegExp) return value.flags ? new Tuple([value.source, value.flags]) : value.source; + // weakref if (value instanceof WeakRef) { const deref = value.deref(); @@ -2828,8 +2870,8 @@ export class Runtime { const compiled = new Uint8Array(Compiler.encodeValue(value, undefined, false, deep_clone, collapse_value, false, true, false, true)); return wasm_decompile(compiled, formatted, colorized, resolve_slots).replace(/\r\n$/, ''); } catch (e) { - console.log(e); - return "/* ERROR: Decompiler Error */"; + console.debug(e); + return this.valueToDatexString(value, formatted) } // return Decompiler.decompile(Compiler.encodeValue(value, undefined, false, deep_clone, collapse_value), true, formatted, formatted, false); } @@ -5318,7 +5360,7 @@ export class Runtime { persistent_vars: persistent_memory ? Object.keys(persistent_memory): [], execution_permission: header?.executable, // allow execution? - impersonation_permission: Runtime.endpoint.equals(header?.sender), // at the moment: only allow endpoint to impersonate itself + impersonation_permission: Runtime.endpoint?.equals(header?.sender), // at the moment: only allow endpoint to impersonate itself inner_scope: null, // has to be copied from sub_scopes[0] @@ -5372,7 +5414,7 @@ export class Runtime { scope.header = header; scope.execution_permission = header?.executable // allow execution? - scope.impersonation_permission = Runtime.endpoint.equals(header?.sender) // at the moment: only allow endpoint to impersonate itself + scope.impersonation_permission = Runtime.endpoint?.equals(header?.sender) // at the moment: only allow endpoint to impersonate itself // enter outer scope ? if (scope.sub_scopes.length == 0) { @@ -7208,7 +7250,8 @@ RuntimePerformance.createMeasureGroup("compile time", [ // automatically sync newly added pointers if they are in the storage Pointer.onPointerAdded(async (pointer)=>{ - if (await Storage.hasPointer(pointer)) { + // assume that already synced if createdInContext and stored in storage + if (!pointer.createdInContext && await Storage.hasPointer(pointer)) { Storage.syncPointer(pointer); } }) @@ -7350,6 +7393,14 @@ Type.std.time.setJSInterface({ }) +Type.std.MatchCondition.setJSInterface({ + class: MatchCondition, + visible_children: new Set([ + "type", + "data" + ]) +}) + Type.get("js:Function").setJSInterface({ @@ -7363,8 +7414,8 @@ Type.get("js:Function").setJSInterface({ }, apply_value(parent, args = []) { - if (args instanceof Tuple) return parent.call(...args.toArray()) - else return parent.call(args) + if (args instanceof Tuple) return parent.handleCall(...args.toArray()) + else return parent.handleCall(args) }, }); diff --git a/storage/storage-locations/deno-kv.ts b/storage/storage-locations/deno-kv.ts index 615d0eee..a75ad9c6 100644 --- a/storage/storage-locations/deno-kv.ts +++ b/storage/storage-locations/deno-kv.ts @@ -7,7 +7,7 @@ import { AsyncStorageLocation } from "../storage.ts"; import { ptr_cache_path } from "../../runtime/cache_path.ts"; import { client_type } from "../../utils/constants.ts"; import { normalizePath } from "../../utils/normalize-path.ts"; -import { ExecConditions } from "../../utils/global_types.ts"; +import type { ExecConditions } from "../../utils/global_types.ts"; const denoKvDir = new URL("./deno-kv/", ptr_cache_path); // @ts-ignore global Deno diff --git a/storage/storage-locations/indexed-db.ts b/storage/storage-locations/indexed-db.ts index e4cbe1c3..aaddaf81 100644 --- a/storage/storage-locations/indexed-db.ts +++ b/storage/storage-locations/indexed-db.ts @@ -6,7 +6,7 @@ import { NOT_EXISTING } from "../../runtime/constants.ts"; import { AsyncStorageLocation, site_suffix } from "../storage.ts"; import localforage from "../../lib/localforage/localforage.js"; -import { ExecConditions } from "../../utils/global_types.ts"; +import type { ExecConditions } from "../../utils/global_types.ts"; // db based storage for DATEX value caching (IndexDB in the browser) const datex_item_storage = localforage.createInstance({name: "dxitem::"+site_suffix}); diff --git a/storage/storage-locations/sql-db.ts b/storage/storage-locations/sql-db.ts index 33b04242..1fd233d8 100644 --- a/storage/storage-locations/sql-db.ts +++ b/storage/storage-locations/sql-db.ts @@ -1,5 +1,5 @@ -import { Client } from "https://deno.land/x/mysql@v2.12.1/mod.ts"; -import { Query, replaceParams } from "https://deno.land/x/sql_builder@v1.9.2/mod.ts"; +import { Client, ExecuteResult } from "https://deno.land/x/mysql@v2.12.1/mod.ts"; +import { Query } from "https://deno.land/x/sql_builder@v1.9.2/mod.ts"; import { Where } from "https://deno.land/x/sql_builder@v1.9.2/where.ts"; import { Pointer } from "../../runtime/pointers.ts"; import { AsyncStorageLocation } from "../storage.ts"; @@ -10,15 +10,32 @@ import { datex_type_mysql_map } from "./sql-type-map.ts"; import { NOT_EXISTING } from "../../runtime/constants.ts"; import { client_type } from "../../utils/constants.ts"; import { Compiler } from "../../compiler/compiler.ts"; -import { ExecConditions } from "../../utils/global_types.ts"; +import type { ExecConditions } from "../../utils/global_types.ts"; import { Runtime } from "../../runtime/runtime.ts"; +import { Storage } from "../storage.ts"; +import { Type } from "../../types/type.ts"; +import { TypedArray } from "../../utils/global_values.ts"; +import { MessageLogger } from "../../utils/message_logger.ts"; +import { Join } from "https://deno.land/x/sql_builder@v1.9.2/join.ts"; +import { LazyPointer } from "../../runtime/lazy-pointer.ts"; +import { MatchOptions, MatchCondition, MatchConditionType, ComputedProperty, ComputedPropertyType} from "../storage.ts"; +import { MatchResult } from "../storage.ts"; +import { Time } from "../../types/time.ts"; +import { Order } from "https://deno.land/x/sql_builder@v1.9.2/order.ts"; +import { configLogger } from "https://deno.land/x/mysql@v2.12.1/src/logger.ts"; + +configLogger({level: "WARNING"}) const logger = new Logger("SQL Storage"); export class SQLDBStorageLocation extends AsyncStorageLocation { name = "SQL_DB" + supportsPrefixSelection = true; + supportsMatchSelection = true; #connected = false; + #initializing = false + #initialized = false #options: dbOptions #sqlClient: Client|undefined @@ -27,36 +44,64 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { readonly #metaTables = { typeMapping: { - name: "datex_types", + name: "__datex_types", columns: [ ["type", "varchar(50)", "PRIMARY KEY"], ["table_name", "varchar(50)"] ] }, pointerMapping: { - name: "datex_pointer_mapping", + name: "__datex_pointer_mapping", columns: [ [this.#pointerMysqlColumnName, this.#pointerMysqlType, "PRIMARY KEY"], ["table_name", "varchar(50)"] ] }, rawPointers: { - name: "datex_pointers_raw", + name: "__datex_pointers_raw", columns: [ - ["id", "varchar(50)", "PRIMARY KEY"], + [this.#pointerMysqlColumnName, this.#pointerMysqlType, "PRIMARY KEY"], ["value", "blob"] ] }, + sets: { + name: "__datex_sets", + columns: [ + [this.#pointerMysqlColumnName, this.#pointerMysqlType, "PRIMARY KEY"], + ["hash", "varchar(50)", "PRIMARY KEY"], + ["value_dxb", "blob"], + ["value_text", "text"], + ["value_integer", "int"], + ["value_decimal", "double"], + ["value_boolean", "boolean"], + ["value_time", "datetime"], + ["value_pointer", this.#pointerMysqlType] + ] + }, items: { - name: "datex_items", + name: "__datex_items", columns: [ ["key", "varchar(200)", "PRIMARY KEY"], - ["value", "blob"] + ["value", "blob"], + [this.#pointerMysqlColumnName, this.#pointerMysqlType], ] } } satisfies Record; - #tableColumns = new Map>() + // cached table columns + #tableColumns = new Map>() + // cached table -> type mapping + #tableTypes = new Map() + + #existingItemsCache = new Set() + #existingPointersCache = new Set() + #tableCreationTasks = new Map>() + #tableColumnTasks = new Map>>() + #tableLoadingTasks = new Map>() + + // remember tables for pointers that still need to be loaded + #pointerTables = new Map() + #templateMultiQueries = new Map, result: Promise[]>}>() constructor(options:dbOptions, private log?:(...args:unknown[])=>void) { super() @@ -64,44 +109,64 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { } async #connect(){ if (this.#connected) return; - this.#sqlClient = await new Client().connect(this.#options); - this.log?.("Connected to SQL database " + this.#options.db + " on " + this.#options.hostname + ":" + this.#options.port) + this.#sqlClient = await new Client().connect({poolSize: 30, ...this.#options}); + logger.info("Using SQL database " + this.#options.db + " on " + this.#options.hostname + ":" + this.#options.port + " as storage location") this.#connected = true; } async #init() { + if (this.#initialized) return; + this.#initializing = true; await this.#connect(); await this.#setupMetaTables(); + await MessageLogger.init(); // required for decompiling + this.#initializing = false; + this.#initialized = true; } - async resetAll() { - await this.#init(); - + async #resetAll() { + + // drop all custom type tables const tables = await this.#query<{table_name:string}>( new Query() .table(this.#metaTables.typeMapping.name) .select("table_name") .build() ) + const tableNames = tables.map(({table_name})=>'`'+table_name+'`'); + + // TODO: better solution to handle drop with foreign constraints + // currently just runs multiple drop table queries on failure, which is not ideal + const iterations = 10; + for (let i = 0; i < iterations; i++) { + try { + await this.#query<{table_name:string}>(`DROP TABLE IF EXISTS ${tableNames.join(',')};`) + break; + } + catch (e) { + console.error("Failed to drop some tables due to foreign constraints, repeating", e) + } + } - const tableNames = tables.map(({table_name})=>'`'+table_name+'`') - - await this.#query<{table_name:string}>(`DROP TABLE IF EXISTS ${tableNames.join(',')};`) - await this.#query<{table_name:string}>(`TRUNCATE TABLE ${this.#metaTables.typeMapping.name};`) - await this.#query<{table_name:string}>(`TRUNCATE TABLE ${this.#metaTables.pointerMapping.name};`) + // truncate meta tables + await Promise.all(Object.values(this.#metaTables).map(table => this.#query(`TRUNCATE TABLE ${table.name};`))) } - async #query(query_string:string, query_params?:any[]): Promise { + async #query(query_string:string, query_params:any[]|undefined, returnRawResult: true): Promise<{rows:row[], result:ExecuteResult}> + async #query(query_string:string, query_params?:any[]): Promise + async #query(query_string:string, query_params?:any[], returnRawResult?: boolean): Promise { + // prevent infinite recursion if calling query from within init() + if (!this.#initializing) await this.#init(); // handle arraybuffers if (query_params) { for (let i = 0; i < query_params.length; i++) { const param = query_params[i]; if (param instanceof ArrayBuffer) { - query_params[i] = new TextDecoder().decode(param) + query_params[i] = this.#binaryToString(param) } if (param instanceof Array) { - query_params[i] = param.map(p => p instanceof ArrayBuffer ? new TextDecoder().decode(p) : p) + query_params[i] = param.map(p => p instanceof ArrayBuffer ? this.#binaryToString(p) : p) } } } @@ -112,7 +177,8 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { if (!query_string) throw new Error("empty query"); try { const result = await this.#sqlClient!.execute(query_string, query_params); - return result.rows ?? []; + if (returnRawResult) return {rows: result.rows ?? [], result}; + else return result.rows ?? []; } catch (e){ if (this.log) this.log("SQL error:", e) else console.error("SQL error:", e); @@ -120,12 +186,19 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { } } - async #queryFirst(query_string:string, query_params?:any[]): Promise { + #stringToBinary(value: string){ + return Uint8Array.from(value, x => x.charCodeAt(0)).buffer + } + #binaryToString(value: ArrayBuffer){ + return String.fromCharCode.apply(null, new Uint8Array(value) as unknown as number[]) + } + + async #queryFirst(query_string:string, query_params?:any[]): Promise { return (await this.#query(query_string, query_params))?.[0] } async #createTableIfNotExists(definition: TableDefinition) { - const exists = await this.#queryFirst( + const exists = this.#tableColumns.has(definition.name) || await this.#queryFirst( new Query() .table("information_schema.tables") .select("*") @@ -134,46 +207,135 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { .build() ) if (!exists) { - await this.#createTable(definition); + await this.#createTableFromDefinition(definition); return true; } return false; } - async #createTable(definition: TableDefinition) { - await this.#queryFirst(`CREATE TABLE ?? (${definition.columns.map(col => + + /** + * Creates a new table + */ + async #createTableFromDefinition(definition: TableDefinition) { + const compositePrimaryKeyColumns = definition.columns.filter(col => col[2]?.includes("PRIMARY KEY")); + if (compositePrimaryKeyColumns.length > 1) { + for (const col of compositePrimaryKeyColumns) { + col[2] = col[2]?.replace("PRIMARY KEY", "") + } + } + const primaryKeyDefinition = compositePrimaryKeyColumns.length > 1 ? `, PRIMARY KEY (${compositePrimaryKeyColumns.map(col => `\`${col[0]}\``).join(', ')})` : ''; + + // create + await this.#queryFirst(`CREATE TABLE IF NOT EXISTS ?? (${definition.columns.map(col => `\`${col[0]}\` ${col[1]} ${col[2]??''}` - ).join(', ')}${definition.constraints?.length ? ',' + definition.constraints.join(',') : ''});`, [definition.name]) + ).join(', ')}${definition.constraints?.length ? ',' + definition.constraints.join(',') : ''}${primaryKeyDefinition});`, [definition.name]) + // load column definitions + await this.#getTableColumns(definition.name); } + /** + * Returns the table name for a given type, creates a new table if it does not exist + * @param type + * @returns + */ async #getTableForType(type: Datex.Type) { + // type does not have a template, use raw pointer table + if (!type.template) return null + + // already has a table + const tableName = this.#typeToTableName(type); + if (this.#tableTypes.has(tableName)) return tableName; + + + // already creating table + if (this.#tableLoadingTasks.has(type)) { + return this.#tableLoadingTasks.get(type); + } + + const {promise, resolve} = Promise.withResolvers(); + this.#tableLoadingTasks.set(type, promise); + const existingTable = (await this.#queryFirst<{table_name: string}|undefined>( new Query() .table(this.#metaTables.typeMapping.name) .select("table_name") - .where(Where.eq("type", type.toString())) + .where(Where.eq("type", this.#typeToString(type))) .build() ))?.table_name; - if (!existingTable) { - return this.#createTableForType(type) + + const table = existingTable ?? await this.#createTableForType(type); + resolve(table); + this.#tableLoadingTasks.delete(type); + return table; + } + + async #getTypeForTable(table: string) { + if (!this.#tableTypes.has(table)) { + const type = await this.#queryFirst<{type:string}>( + new Query() + .table(this.#metaTables.typeMapping.name) + .select("type") + .where(Where.eq("table_name", table)) + .build() + ) + if (!type) { + logger.error("No type found for table " + table); + } + else this.#tableTypes.set(table, Datex.Type.get(type.type)); } - else return existingTable + + return this.#tableTypes.get(table) } + /** + * Returns the table name for a given type. + * Converts UpperCamelCase to snake_case and pluralizes the name. + * Does not validate if the table exists + * Throws if the type is not templated + */ #typeToTableName(type: Datex.Type) { - return type.namespace=="ext" ? type.name : `${type.namespace}_${type.name}`; + if (!type.template) throw new Error("Cannot create table for non-templated type " + type) + const snakeCaseName = type.name.replace(/([A-Z])/g, "_$1").toLowerCase().slice(1).replace(/__+/g, '_'); + const snakeCasePlural = snakeCaseName + (snakeCaseName.endsWith("s") ? "es" : "s"); + const name = type.namespace=="ext"||type.namespace=="struct" ? snakeCasePlural : `${type.namespace}_${snakeCasePlural}`; + if (name.length > 64) throw new Error("Type name too long: " + type); + else return name; + } + + #typeToString(type: Datex.Type) { + return type.namespace + ":" + type.name; + } + + *#iterateTableColumns(type: Datex.Type) { + const table = this.#typeToTableName(type); + const columns = this.#tableColumns.get(table); + if (!columns) throw new Error("Table columns for type " + type + " are not loaded"); + for (const data of columns) { + yield data + } } async #createTableForType(type: Datex.Type) { + + // already creating table + if (this.#tableCreationTasks.has(type)) { + return this.#tableCreationTasks.get(type); + } + + const {promise, resolve} = Promise.withResolvers(); + this.#tableCreationTasks.set(type, promise); + const columns:ColumnDefinition[] = [ [this.#pointerMysqlColumnName, this.#pointerMysqlType, 'PRIMARY KEY INVISIBLE DEFAULT "0"'] ] const constraints: ConstraintsDefinition[] = [] - this.log?.("Creating table for type " + type) - console.log(type) - for (const [propName, propType] of Object.entries(type.template as {[key:string]:Datex.Type})) { + + // invalid prop name for now: starting/ending with _ + if (propName.startsWith("_") || propName.endsWith("_")) throw new Error("Invalid property name: " + propName + " (Property names cannot start or end with an underscore)"); + let mysqlType: mysql_data_type|undefined if (propType.base_type == Datex.Type.std.text && typeof propType.parameters?.[0] == "number") { @@ -188,15 +350,32 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { } // no matching primitive type found else if (!mysqlType) { - let foreignTable = propType.template ? await this.#getTableForType(propType) : null; - if (!foreignTable) { - logger.warn("Cannot map type " + propType + " to a SQL table, falling back to raw DXB storage") - foreignTable = this.#metaTables.rawPointers.name; + // is a primitive type -> assume no pointer, just store as dxb inline + if (propType == Type.std.Any || propType.is_primitive || propType.is_js_pseudo_primitive ) { + logger.warn("Cannot map type " + propType + " to a SQL table, falling back to raw DXB") + columns.push([propName, "blob"]) + } + else { + let foreignTable = await this.#getTableForType(propType); + + if (!foreignTable) { + + // "set" table + if (propType.base_type == Type.std.Set) { + foreignTable = this.#metaTables.sets.name; + } + else { + logger.warn("Cannot map type " + propType + " to a SQL table, falling back to raw pointer storage") + foreignTable = this.#metaTables.rawPointers.name; + } + + } + + columns.push([propName, this.#pointerMysqlType]) + constraints.push(`FOREIGN KEY (\`${propName}\`) REFERENCES \`${foreignTable}\`(\`${this.#pointerMysqlColumnName}\`)`) } - columns.push([propName, this.#pointerMysqlType]) - constraints.push(`FOREIGN KEY (\`${propName}\`) REFERENCES \`${foreignTable}\`(\`${this.#pointerMysqlColumnName}\`)`) } else { @@ -208,7 +387,7 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { const name = this.#typeToTableName(type); // create table - await this.#createTable({ + await this.#createTableFromDefinition({ name, columns, constraints @@ -219,12 +398,18 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { new Query() .table(this.#metaTables.typeMapping.name) .insert({ - type: type.toString(), + type: this.#typeToString(type), table_name: name }) .build() ) + // remember table type mapping + this.#tableTypes.set(name, type); + + // resolve promise + resolve(name); + this.#tableCreationTasks.delete(type); return name; } @@ -233,191 +418,1070 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { * makes sure all DATEX meta tables exist in the database */ async #setupMetaTables() { - for (const definition of Object.values(this.#metaTables)) { - const createdNew = await this.#createTableIfNotExists(definition); - if (createdNew) this.log?.("Created meta table '" + definition.name + "'") - } + await Promise.all( + Object + .values(this.#metaTables) + .map(definition => this.#createTableIfNotExists(definition)) + ) } async #getTableColumns(tableName: string) { + if (!this.#tableColumns.has(tableName)) { - const columnData = new Map() - const columns = await this.#query<{COLUMN_NAME:string, COLUMN_KEY:string}>( + if (this.#tableColumnTasks.has(tableName)) return this.#tableColumnTasks.get(tableName)!; + const {promise, resolve} = Promise.withResolvers>(); + this.#tableColumnTasks.set(tableName, promise); + + const columnData = new Map() + const columns = await this.#query<{COLUMN_NAME:string, COLUMN_KEY:string, DATA_TYPE:string}>( new Query() .table("information_schema.columns") - .select("COLUMN_NAME", "COLUMN_KEY") + .select("COLUMN_NAME", "COLUMN_KEY", "DATA_TYPE") .where(Where.eq("table_schema", this.#options.db)) .where(Where.eq("table_name", tableName)) .build() ) + const constraints = (await this.#query<{COLUMN_NAME:string, REFERENCED_TABLE_NAME:string}>( + new Query() + .table("information_schema.key_column_usage") + .select("COLUMN_NAME", "REFERENCED_TABLE_NAME") + .where(Where.eq("table_schema", this.#options.db)) + .where(Where.eq("table_name", tableName)) + .build() + )); + const columnTables = new Map() + for (const {COLUMN_NAME, REFERENCED_TABLE_NAME} of constraints) { + columnTables.set(COLUMN_NAME, REFERENCED_TABLE_NAME) + } + for (const col of columns) { if (col.COLUMN_NAME == this.#pointerMysqlColumnName) continue; - columnData.set(col.COLUMN_NAME, {foreignPtr: col.COLUMN_KEY == "MUL"}) + columnData.set(col.COLUMN_NAME, {foreignPtr: columnTables.has(col.COLUMN_NAME), foreignTable: columnTables.get(col.COLUMN_NAME), type: col.DATA_TYPE}) } this.#tableColumns.set(tableName, columnData) + + resolve(this.#tableColumns.get(tableName)!); + this.#tableColumnTasks.delete(tableName); } + return this.#tableColumns.get(tableName)!; } - async #insertPointer(pointer: Datex.Pointer) { + /** + * Insert a pointer into the database, pointer type must be templated + */ + async #insertTemplatedPointer(pointer: Datex.Pointer) { const table = await this.#getTableForType(pointer.type) + if (!table) throw new Error("Cannot store pointer of type " + pointer.type + " in a custom table") const columns = await this.#getTableColumns(table); + const dependencies = new Set() + const insertData:Record = { [this.#pointerMysqlColumnName]: pointer.id } - for (const [name, {foreignPtr}] of columns) { + for (const [name, {foreignPtr, type}] of columns) { + const value = pointer.val[name]; if (foreignPtr) { - const propPointer = Datex.Pointer.getByValue(pointer.val[name]); - if (!propPointer) throw new Error("Cannot reference non-pointer value in SQL table") - insertData[name] = propPointer.id - await this.#insertPointer(propPointer) + const propPointer = Datex.Pointer.getByValue(value); + // no pointer value + if (!propPointer) { + // null values are okay, otherwise error + if (value !== undefined) { + logger.error("Cannot reference non-pointer value in SQL table") + } + } + else { + insertData[name] = propPointer.id + // must immediately add entry for foreign constraint to work + await Storage.setPointer(propPointer, true) + dependencies.add(propPointer) + } } - else insertData[name] = pointer.val[name]; + // is raw dxb value (exception for blob <->A rrayBuffer, TODO: better solution, can lead to issues) + else if (type == "blob" && !(value instanceof ArrayBuffer || value instanceof TypedArray)) { + insertData[name] = Compiler.encodeValue(value, dependencies, true, false, false); + } + else insertData[name] = value; } - // this.log("cols", insertData) + // replace if entry already exists - await this.#query('INSERT INTO ?? ?? VALUES ?;', [table, Object.keys(insertData), Object.values(insertData)]) + await this.#query('INSERT INTO ?? ?? VALUES ? ON DUPLICATE KEY UPDATE '+Object.keys(insertData).map((key) => `\`${key}\` = ?`).join(', '), [table, Object.keys(insertData), Object.values(insertData), ...Object.values(insertData)]) + // await this.#query('INSERT INTO ?? ?? VALUES ?;', [table, Object.keys(insertData), Object.values(insertData)]) + + // add to pointer mapping + await this.#updatePointerMapping(pointer.id, table) + return dependencies; } - async #updatePointer(pointer: Datex.Pointer, keys:string[]) { - const table = await this.#getTableForType(pointer.type) + /** + * Update a pointer in the database, pointer type must be templated + */ + async #updatePointer(pointer: Datex.Pointer, keys:string[], dependencies?: Set) { + const table = await this.#getTableForType(pointer.type); + if (!table) throw new Error("Cannot store pointer of type " + pointer.type + " in a custom table") const columns = await this.#getTableColumns(table); for (const key of keys) { - const val = columns.get(key)?.foreignPtr ? Datex.Pointer.getByValue(pointer.val[key])!.id : pointer.val[key]; + const column = columns.get(key); + const val = + column?.foreignPtr ? + // foreign pointer id + Datex.Pointer.getByValue(pointer.val[key])!.id : + ( + (column?.type == "blob" && !(pointer.val[key] instanceof ArrayBuffer || pointer.val[key] instanceof TypedArray)) ? + // raw dxb value + Compiler.encodeValue(pointer.val[key], dependencies, true, false, false) : + // normal value + pointer.val[key] + ) await this.#query('UPDATE ?? SET ?? = ? WHERE ?? = ?;', [table, key, val, this.#pointerMysqlColumnName, pointer.id]) } } - async #pointerEntryExists(pointer: Datex.Pointer) { - const table = await this.#getTableForType(pointer.type) + async #getTemplatedPointerValueString(pointerId: string, table?: string) { + table = table ?? await this.#getPointerTable(pointerId); + if (!table) { + logger.error("No table found for pointer " + pointerId); + return null; + } - const exists = await this.#queryFirst<{COUNT:number}>( - `SELECT COUNT(*) as COUNT FROM ?? WHERE ??=?`, [ - table, - this.#pointerMysqlColumnName, - pointer.id - ] - ); - return exists.COUNT > 0; + const type = await this.#getTypeForTable(table); + if (!type) { + logger.error("No type found for table " + table); + return null; + } + + const object = await this.#getTemplatedPointerObject(pointerId, table); + if (!object) return null; + + // resolve foreign pointers + const foreignPointerPlaceholders: string[] = [] + // const foreignPointerPlaceholderPromises: Promise[] = [] + const columns = await this.#getTableColumns(table); + if (!columns) throw new Error("No columns found for table " + table) + for (const [colName, {foreignPtr, type}] of columns.entries()) { + + // convert blob strings to ArrayBuffer + if (type == "blob" && typeof object[colName] == "string") { + object[colName] = this.#stringToBinary(object[colName] as string) + } + // convert Date to Time + else if (object[colName] instanceof Date) { + object[colName] = new Time(object[colName] as Date) + } + + // convert to boolean + else if (typeof object[colName] == "number" && (type == "tinyint" || type == "boolean")) { + object[colName] = Boolean(object[colName]) + } + + // is an object type with a template + if (foreignPtr) { + if (typeof object[colName] == "string") { + const ptrId = object[colName] as string + object[colName] = `\u0001${foreignPointerPlaceholders.length}` + foreignPointerPlaceholders.push("$"+ptrId) + } + // otherwise, property is null/undefined + } + // is blob, assume it is a DXB value + else if (type == "blob") { + try { + // TODO: fix decompiling + foreignPointerPlaceholders.push(Storage.removeTrailingSemicolon(MessageLogger.decompile(object[colName] as ArrayBuffer, false, false, false)||"'error'")) + } + catch (e) { + console.error("error decompiling", object[colName], e) + foreignPointerPlaceholders.push("'error'") + } + object[colName] = `\u0001${foreignPointerPlaceholders.length-1}` + } + } + + // const foreignPointerPlaceholders = await Promise.all(foreignPointerPlaceholderPromises) + + const objectString = Datex.Runtime.valueToDatexStringExperimental(object, false, false) + .replace(/"\u0001(\d+)"/g, (_, index) => foreignPointerPlaceholders[parseInt(index)]||"'error: no placeholder'") + + return `${type.toString()} ${objectString}` + } + + + async #getTemplatedPointerObject(pointerId: string, table?: string) { + table = table ?? await this.#getPointerTable(pointerId); + if (!table) { + logger.error("No table found for pointer " + pointerId); + return null; + } + + let result: Promise[]>; + + if (this.#templateMultiQueries.has(table)) { + const multiQuery = this.#templateMultiQueries.get(table)! + multiQuery.pointers.add(pointerId) + result = multiQuery.result; + } + else { + const pointers = new Set([pointerId]) + result = (async () => { + await sleep(30); + this.#templateMultiQueries.delete(table) + return this.#query>( + new Query() + .table(table) + .select("*", this.#pointerMysqlColumnName) + .where(Where.in(this.#pointerMysqlColumnName, Array.from(pointers))) + .build() + ) + })() + this.#templateMultiQueries.set(table, {pointers, result}) + } + + const res = (await result) + .find(obj => obj[this.#pointerMysqlColumnName] == pointerId) + if (res) delete res[this.#pointerMysqlColumnName]; + return res; + } + + async #getTemplatedPointerValueDXB(pointerId: string, table?: string) { + const string = await this.#getTemplatedPointerValueString(pointerId, table); + if (!string) return null; + const compiled = await Compiler.compile(string, [], {sign: false, encrypt: false, to: Datex.Runtime.endpoint, preemptive_pointer_init: false}, false) as ArrayBuffer; + return compiled + } + + async #getPointerTable(pointerId: string) { + if (this.#pointerTables.has(pointerId)) { + const table = this.#pointerTables.get(pointerId); + this.#pointerTables.delete(pointerId); + return table; + } + return (await this.#queryFirst<{table_name:string}>( + new Query() + .table(this.#metaTables.pointerMapping.name) + .select("table_name") + .where(Where.eq(this.#pointerMysqlColumnName, pointerId)) + .build() + ))?.table_name; + } + + async #setPointerRaw(pointer: Pointer) { + const dependencies = new Set() + const encoded = Compiler.encodeValue(pointer, dependencies, true, false, true); + await this.#setPointerInRawTable(pointer.id, encoded); + return dependencies; } + async #setPointerSet(pointer: Pointer) { + if (!(pointer.val instanceof Set)) throw new Error("Pointer value must be a Set"); + + const dependencies = new Set() + + const builder = new Query().table(this.#metaTables.sets.name) + const entries = [] + // add default entry (also for empty set) + entries.push({ + [this.#pointerMysqlColumnName]: pointer.id, + hash: "", + value_dxb: null, + value_text: null, + value_integer: null, + value_decimal: null, + value_boolean: null, + value_time: null, + value_pointer: null + }) + + for (const val of pointer.val) { + const hash = await Compiler.getValueHashString(val) + const data = {[this.#pointerMysqlColumnName]: pointer.id, hash} as Record; + const valPtr = Datex.Pointer.pointerifyValue(val); + + if (typeof val == "string") data.value_text = val + else if (typeof val == "number") data.value_decimal = val + else if (typeof val == "bigint") data.value_boolean = val + else if (typeof val == "boolean") data.value_boolean = val + else if (val instanceof Date) data.value_time = val + else if (valPtr instanceof Pointer) { + data.value_pointer = valPtr.id + dependencies.add(valPtr); + } + else data.value_dxb = this.#binaryToString(Compiler.encodeValue(val, dependencies, true, false, false)) + entries.push(data) + } + builder.insert(entries); + + // replace INSERT with INSERT IGNORE to prevent duplicate key errors + const {result} = await this.#query(builder.build().replace("INSERT", "INSERT IGNORE"), undefined, true) + // add to pointer mapping TODO: better decision if to add to pointer mapping + if (result.affectedRows) await this.#updatePointerMapping(pointer.id, this.#metaTables.sets.name) + return dependencies; + } + + async #updatePointerMapping(pointerId: string, tableName: string) { + await this.#query('INSERT INTO ?? ?? VALUES ? ON DUPLICATE KEY UPDATE table_name=?;', [this.#metaTables.pointerMapping.name, [this.#pointerMysqlColumnName, "table_name"], [pointerId, tableName], tableName]) + } + + async #setItemPointer(key: string, pointer: Pointer) { + await this.#query('INSERT INTO ?? ?? VALUES ? ON DUPLICATE KEY UPDATE '+this.#pointerMysqlColumnName+'=?;', [this.#metaTables.items.name, ["key", this.#pointerMysqlColumnName], [key, pointer.id], pointer.id]) + } isSupported() { return client_type === "deno"; } + supportsMatchForType(type: Datex.Type) { + // only templated types are supported because they are stored in custom tables + return !!type.template + } + + async matchQuery(itemPrefix: string, valueType: Datex.Type, match: Datex.MatchInput, options: Options): Promise> { + + // measure total query time + const start = Date.now(); + + const joins = new Map() + const collectedTableTypes = new Set([valueType]) + const collectedIdentifiers = new Set() + const builder = new Query() + .table(this.#metaTables.items.name) + .where(Where.like(this.#metaTables.items.name + ".key", itemPrefix + "%")) + .join( + Join.left(this.#typeToTableName(valueType)).on(`${this.#metaTables.items.name}.${this.#pointerMysqlColumnName}`, `${this.#typeToTableName(valueType)}.${this.#pointerMysqlColumnName}`) + ) + const where = this.buildQueryConditions(builder, match, joins, collectedTableTypes, collectedIdentifiers, valueType, undefined, undefined, options.computedProperties) + let query = "error"; + + const rootTableName = this.#typeToTableName(valueType); + + // computed properties - nested select + if (options.computedProperties || options.returnRaw) { + + // add property joins for returnRaw + if (options.returnRaw) { + this.addPropertyJoins( + options.returnRaw, + builder, joins, valueType, collectedTableTypes + ) + for (const property of options.returnRaw) { + collectedIdentifiers.add(property.replaceAll(".", "__")) + } + } + + const select = [...collectedIdentifiers, this.#pointerMysqlColumnName].map(identifier => { + if (identifier.includes("__")) { + return `${this.getTableProperty(identifier)} as ${identifier}` + } + else return rootTableName + '.' + identifier; + }); + + for (const [name, value] of Object.entries(options.computedProperties??{})) { + if (value.type == ComputedPropertyType.GEOGRAPHIC_DISTANCE) { + const computedProperty = value as ComputedProperty + const {pointA, pointB} = computedProperty.data; + + this.addPropertyJoins( + [pointA.lat, pointA.lon, pointB.lat, pointB.lon].filter(v => typeof v == "string") as string[], + builder, joins, valueType, collectedTableTypes + ) + + select.push( + `ST_Distance_Sphere(point(${ + typeof pointA.lon == "string" ? this.formatProperty(pointA.lon) : pointA.lon + },${ + typeof pointA.lat == "string" ? this.formatProperty(pointA.lat) : pointA.lat + }), point(${ + typeof pointB.lon == "string" ? this.formatProperty(pointB.lon) : pointB.lon + },${ + typeof pointB.lat == "string" ? this.formatProperty(pointB.lat) : pointB.lat + })) as ${name}` + ) + } + else if (value.type == ComputedPropertyType.SUM) { + const computedProperty = value as ComputedProperty + this.addPropertyJoins( + computedProperty.data.filter(v => typeof v == "string") as string[], + builder, joins, valueType, collectedTableTypes + ) + select.push(`SUM(${computedProperty.data.map(p => { + if (typeof p == "string") return this.formatProperty(p) + else return p + })}) as ${name}`) + } + else { + throw new Error("Unsupported computed property type " + value.type) + } + } + builder.select(...select); + joins.forEach(join => builder.join(join)); + + const outerBuilder = new Query() + .select(options.returnRaw ? `*` :`DISTINCT SQL_CALC_FOUND_ROWS ${this.#pointerMysqlColumnName} as ptrId`) + .table('__placeholder__'); + + this.appendBuilderConditions(outerBuilder, options, where) + // nested select + query = outerBuilder.build().replace('`__placeholder__`', `(${builder.build()}) as _inner_res`) + } + + // no computed properties + else { + builder.select(`DISTINCT SQL_CALC_FOUND_ROWS ${this.#typeToTableName(valueType)}.${this.#pointerMysqlColumnName} as ptrId`); + this.appendBuilderConditions(builder, options, where) + joins.forEach(join => builder.join(join)); + query = builder.build(); + } + + // make sure all tables are created + for (const type of collectedTableTypes) { + await this.#getTableForType(type) + } + + const queryResult = await this.#query<{ptrId:string}>(query); + const ptrIds = queryResult.map(({ptrId}) => ptrId) + const limitedPtrIds = options.returnPointerIds ? + // offset and limit manually after query + ptrIds.slice(options.offset ?? 0, options.limit ? (options.offset ?? 0) + options.limit : undefined) : + // use ptrIds returned from query (already limited) + ptrIds + + // TODO: atomic operations for multiple queries + const {foundRows} = (options?.returnAdvanced ? await this.#queryFirst<{foundRows: number}>("SELECT FOUND_ROWS() as foundRows") : null) ?? {foundRows: -1} + + // remember pointer table + for (const ptrId of ptrIds) { + this.#pointerTables.set(ptrId, rootTableName) + } + + const loadStart = Date.now(); + + const result = options.returnRaw ? null : new Set((await Promise.all(limitedPtrIds.map(ptrId => Pointer.load(ptrId)))).filter(ptr => { + if (ptr instanceof LazyPointer) { + logger.warn("Cannot return lazy pointer from match query (" + ptr.id + ")"); + return false; + } + return true; + }).map(ptr => (ptr as Pointer).val as T)) + + console.log("load time", (Date.now() - loadStart) + "ms") + console.log("total query time", (Date.now() - start) + "ms") + + const matches = options.returnRaw ? + await Promise.all(queryResult.map(async entry => (this.mergeNestedObjects(await Promise.all(Object.entries(entry).map( + ([key, value]) => this.collapseNestedObjectEntry(key, value, rootTableName) + )))))) : + result; + + if (options?.returnAdvanced) { + return { + matches: matches, + total: foundRows, + ...options?.returnPointerIds ? {pointerIds: new Set(ptrIds)} : {} + } as MatchResult; + } + else { + return matches as MatchResult; + } + } + + private mergeNestedObjects(insertObjects: Record[], existingObject:Record = {}): Record { + for (const insertObject of insertObjects) { + for (const [key, value] of Object.entries(insertObject)) { + if (key in existingObject && typeof value == "object" && value !== null) { + this.mergeNestedObjects([value], existingObject[key]) + } + else existingObject[key] = value; + } + } + return existingObject; + } + + private async collapseNestedObjectEntry(key: string, value: unknown, tableName: string): Promise<{[key: string]: unknown}> { + const tableDefinition = await this.#getTableColumns(tableName); + if (key.includes("__")) { + const [firstKey, ...rest] = key.split("__"); + const subTable = tableDefinition?.get(firstKey)?.foreignTable; + if (!subTable) throw new Error("No foreign table found for key " + firstKey); + return {[firstKey]: await this.collapseNestedObjectEntry(rest.join("__"), value, subTable)} + } + else { + // buffer + if (tableDefinition?.get(key)?.type == "blob" && typeof value == "string") { + value = await Runtime.decodeValue(this.#stringToBinary(value)) + } + + return {[key]: value} + } + } + + private addPropertyJoins(properties: string[], builder: Query, joins: Map, valueType: Type, collectedTableTypes: Set) { + const mockObject = {} + for (const property of properties) { + let object:Record = mockObject; + let lastParent:Record = mockObject; + let lastProperty: string|undefined + for (const part of property.split(".")) { + if (!object[part]) object[part] = {}; + lastParent = object; + lastProperty = part; + object = object[part]; + } + if (lastParent && lastProperty!=undefined) lastParent[lastProperty] = null; + } + // get correct joins + this.buildQueryConditions(builder, mockObject, joins, collectedTableTypes, new Set(), valueType) + } + + private appendBuilderConditions(builder: Query, options: MatchOptions, where?: Where) { + // limit, do limit later if options.returnPointerIds + if (options && (options.limit !== undefined && isFinite(options.limit) && !options.returnPointerIds)) { + builder.limit(options.offset ?? 0, options.limit) + } + // sort + if (options.sortBy) { + builder.order(Order.by(this.formatProperty(options.sortBy))[options.sortDesc ? "desc" : "asc"]) + } + if (where) builder.where(where); + } + + /** + * replace all .s with __s, except the last one + */ + private formatProperty(prop: string) { + // + return prop.replace(/\.(?=.*[.].*)/g, '__') + } + + /** + * replace last __ with . + */ + private getTableProperty(prop: string) { + return prop.replace(/__(?!.*__.*)/, '.') + } + + private buildQueryConditions(builder: Query, match: unknown, joins: Map, collectedTableTypes:Set, collectedIdentifiers:Set, valueType:Type, namespacedKey?: string, previousKey?: string, computedProperties?: Record>): Where|undefined { + + const matchOrs = match instanceof Array ? match : [match] + let entryIdentifier = previousKey ? previousKey + '.' + namespacedKey : namespacedKey // address.street + const underscoreIdentifier = entryIdentifier?.replaceAll(".", "__") + + let where: Where|undefined; + let insertedConditionForIdentifier = true; // only set to false if recursing + let isPrimitiveArray = true; + + for (const or of matchOrs) { + if (typeof or == "object" || or === null || or instanceof Date) { + isPrimitiveArray = false; + break; + } + } + + const rememberEntryIdentifier = computedProperties && entryIdentifier && !(entryIdentifier in computedProperties); + + // rename entry identifier + if (rememberEntryIdentifier && entryIdentifier) { + entryIdentifier = underscoreIdentifier + } + + // only primitive array, use IN selector + if (isPrimitiveArray) { + if (!namespacedKey) throw new Error("missing namespacedKey"); + if (matchOrs.length == 1) where = Where.eq(entryIdentifier!, matchOrs[0]) + else where = Where.in(entryIdentifier!, matchOrs) + } + + else { + const wheresOr = [] + for (const or of matchOrs) { + + // regex + if (or instanceof RegExp) { + if (!namespacedKey) throw new Error("missing namespacedKey"); + wheresOr.push(Where.expr(`${entryIdentifier!} REGEXP ?`, or.source)) + } + + // match condition + else if (or instanceof MatchCondition) { + if (!namespacedKey) throw new Error("missing namespacedKey"); + + if (or.type == MatchConditionType.BETWEEN) { + const condition = or as MatchCondition + wheresOr.push(Where.between(entryIdentifier!, condition.data[0], condition.data[1])) + } + else if (or.type == MatchConditionType.GREATER_THAN) { + const condition = or as MatchCondition + wheresOr.push(Where.gt(entryIdentifier!, condition.data)) + } + else if (or.type == MatchConditionType.LESS_THAN) { + const condition = or as MatchCondition + wheresOr.push(Where.lt(entryIdentifier!, condition.data)) + } + else if (or.type == MatchConditionType.GREATER_OR_EQUAL) { + const condition = or as MatchCondition + wheresOr.push(Where.gte(entryIdentifier!, condition.data)) + } + else if (or.type == MatchConditionType.LESS_OR_EQUAL) { + const condition = or as MatchCondition + wheresOr.push(Where.lte(entryIdentifier!, condition.data)) + } + else if (or.type == MatchConditionType.NOT_EQUAL) { + const condition = or as MatchCondition + wheresOr.push(Where.ne(entryIdentifier!, condition.data)) + } + else if (or.type == MatchConditionType.CONTAINS) { + insertedConditionForIdentifier = false; + const condition = or as MatchCondition + const propertyType = valueType.template[namespacedKey]; + const tableAName = this.#typeToTableName(valueType) + '.' + namespacedKey // User.address + + if (propertyType.base_type == Type.std.Set) { + joins.set( + namespacedKey, + Join + .left(`${this.#metaTables.sets.name}`, namespacedKey) + .on(`${namespacedKey}.${this.#pointerMysqlColumnName}`, tableAName)) + ; + const values = [...condition.data]; + // group values by type + const valuesByType = Map.groupBy(values, v => v instanceof Date ? "time" : typeof v); + for (const [type, vals] of valuesByType) { + const columnName = { + string: "value_text", + number: "value_decimal", + bigint: "value_integer", + boolean: "value_boolean", + function: "value_dxb", + time: "value_time", + object: "value_dxb", + symbol: "value_dxb", + undefined: "value_dxb", + }[type]; + + if (columnName) { + const identifier = rememberEntryIdentifier ? `${namespacedKey}__${columnName}` : `${namespacedKey}.${columnName}` + if (rememberEntryIdentifier) collectedIdentifiers.add(identifier) + + if (vals.length == 1) wheresOr.push(Where.eq(identifier, vals[0])) + else wheresOr.push(Where.in(identifier, vals)) + } + else { + throw new Error("Unsupported type for MatchConditionType.CONTAINS: " + type); + } + } + + } + else throw new Error("Unsupported type for MatchConditionType.CONTAINS: " + or.type); + } + else { + throw new Error("Unsupported match condition type " + or.type) + } + } + + else if (typeof or == "object" && !(or == null || or instanceof Date)) { + + // is pointer + const ptr = Pointer.pointerifyValue(or); + if (ptr instanceof Pointer) { + if (!namespacedKey) throw new Error("missing namespacedKey"); + wheresOr.push(Where.eq(entryIdentifier!, ptr.id)) + } + + else { + insertedConditionForIdentifier = false; + + // only enter after first recursion + if (namespacedKey) { + + const propertyType = valueType.template[namespacedKey]; + if (!propertyType) throw new Error("Property '" + namespacedKey + "' does not exist in type " + valueType); + if (propertyType.is_primitive) throw new Error("Tried to match primitive type " + propertyType + " against an object (" + entryIdentifier?.replaceAll("__",".")??namespacedKey + ")") + + collectedTableTypes.add(valueType); + collectedTableTypes.add(propertyType); + + const tableAName = rememberEntryIdentifier ? this.getTableProperty(entryIdentifier!) : entryIdentifier!// this.#typeToTableName(valueType) + '.' + namespacedKey // User.address + const tableBName = this.#typeToTableName(propertyType); // Address + const tableBIdentifier = underscoreIdentifier + '.' + this.#pointerMysqlColumnName + // Join Adddreess on address._ptr_id = User.address + joins.set( + underscoreIdentifier!, + Join + .left(`${tableBName}`, underscoreIdentifier) + .on(tableBIdentifier, tableAName) + ); + valueType = valueType.template[namespacedKey]; + } + + const whereAnds:Where[] = [] + for (const [key, value] of Object.entries(or)) { + + // make sure the key exists in the type + if (!valueType.template[key] && !(computedProperties && key in computedProperties)) throw new Error("Property '" + key + "' does not exist in type " + valueType); + + const condition = this.buildQueryConditions(builder, value, joins, collectedTableTypes, collectedIdentifiers, valueType, key, underscoreIdentifier, computedProperties); + if (condition) whereAnds.push(condition) + } + if (whereAnds.length > 1) wheresOr.push(Where.and(...whereAnds)) + else if (whereAnds.length) wheresOr.push(whereAnds[0]) + } + } + else { + if (!namespacedKey) throw new Error("missing namespacedKey"); + wheresOr.push(Where.eq(entryIdentifier!, or)) + } + } + if (wheresOr.length) where = Where.or(...wheresOr) + } + + + if (rememberEntryIdentifier && insertedConditionForIdentifier && entryIdentifier) { + collectedIdentifiers.add(entryIdentifier); + } + + return where; + + } + + async setItem(key: string,value: unknown) { - await this.#init(); const dependencies = new Set() - const encoded = Compiler.encodeValue(value, dependencies); - console.log("db set item", key) - await this.#query('INSERT INTO ?? ?? VALUES ? ON DUPLICATE KEY UPDATE value=?;', [this.#metaTables.items.name, ["key", "value"], [key, encoded], encoded]) - // await datex_item_storage.setItem(key, Compiler.encodeValue(value, dependencies)); + + // value is pointer + const ptr = Pointer.pointerifyValue(value); + if (ptr instanceof Pointer) { + dependencies.add(ptr); + this.#setItemPointer(key, ptr) + } + else { + const encoded = Compiler.encodeValue(value, dependencies); + await this.setItemValueDXB(key, encoded) + } return dependencies; } async getItem(key: string, conditions: ExecConditions): Promise { - const encoded = (await this.#queryFirst<{value: ArrayBuffer}>( + const encoded = await this.getItemValueDXB(key); + if (encoded === null) return NOT_EXISTING; + return Runtime.decodeValue(encoded, false, conditions); + } + + async hasItem(key:string) { + if (this.#existingItemsCache.has(key)) return true; + const count = (await this.#queryFirst<{COUNT: number}>( new Query() .table(this.#metaTables.items.name) - .select("value") + .select("COUNT(*) as COUNT") .where(Where.eq("key", key)) .build() )); - console.log("encoded",encoded) - if (!encoded) return null; - else return Runtime.decodeValue(encoded, false, conditions); + const exists = !!count && count.COUNT > 0; + if (exists) { + this.#existingItemsCache.add(key); + // delete from cache after 2 minutes + setTimeout(()=>this.#existingItemsCache.delete(key), 1000*60*2) + } + return exists; } - async hasItem(key:string) { - return false - } + async getItemKeys(prefix: string) { + const builder = new Query() + .table(this.#metaTables.items.name) + .select("key") - async getItemKeys() { - return function*(){}() + if (prefix != undefined) builder.where(Where.like("key", prefix + "%")) + + const keys = await this.#query<{key:string}>(builder.build()) + return function*(){ + for (const {key} of keys) { + yield key; + } + }() } async getPointerIds() { - return function*(){}() + const pointerIds = await this.#query<{ptrId:string}>( + new Query() + .table(this.#metaTables.pointerMapping.name) + .select(`${this.#pointerMysqlColumnName} as ptrId`) + .build() + ) + return function*(){ + for (const {ptrId} of pointerIds) { + yield ptrId; + } + }() } async removeItem(key: string): Promise { - + this.#existingItemsCache.delete(key) + await this.#query('DELETE FROM ?? WHERE ??=?;', [this.#metaTables.items.name, "key", key]) } async getItemValueDXB(key: string): Promise { - + const encoded = (await this.#queryFirst<{value: string, ptrId: string}>( + new Query() + .table(this.#metaTables.items.name) + .select("value", `${this.#pointerMysqlColumnName} as ptrId`) + .where(Where.eq("key", key)) + .build() + )); + if (encoded?.ptrId) return Compiler.compile(`$${encoded.ptrId}`, undefined, {sign: false, encrypt: false, to: Datex.Runtime.endpoint, preemptive_pointer_init: false}, false) as Promise; + else if (encoded?.value) return this.#stringToBinary(encoded.value); + else return null; } async setItemValueDXB(key: string, value: ArrayBuffer) { - + const stringBinary = this.#binaryToString(value) + await this.#query('INSERT INTO ?? ?? VALUES ? ON DUPLICATE KEY UPDATE value=?;', [this.#metaTables.items.name, ["key", "value"], [key, stringBinary], stringBinary]) } async setPointer(pointer: Pointer, partialUpdateKey: unknown|typeof NOT_EXISTING): Promise>> { - const dependencies = new Set() - - await this.#init(); // is templatable pointer type if (pointer.type.template) { - this.log?.("update " + pointer.id + " - " + pointer.type, partialUpdateKey, await this.#pointerEntryExists(pointer)) + // this.log?.("update " + pointer.id + " - " + pointer.type, partialUpdateKey, await this.#pointerEntryExists(pointer)) // new full insert - if (!await this.#pointerEntryExists(pointer)) - await this.#insertPointer(pointer) + if (partialUpdateKey === NOT_EXISTING || !await this.hasPointer(pointer.id)) { + return this.#insertTemplatedPointer(pointer) + } + // partial update else { - // partial update - if (partialUpdateKey !== NOT_EXISTING) { - if (typeof partialUpdateKey !== "string") throw new Error("invalid key type for SQL table: " + Datex.Type.ofValue(partialUpdateKey)) - await this.#updatePointer(pointer, [partialUpdateKey]) - } - // full udpdate - else { - // TODO + if (typeof partialUpdateKey !== "string") throw new Error("invalid key type for SQL table: " + Datex.Type.ofValue(partialUpdateKey)) + const dependencies = new Set() + // add all pointer properties to dependencies + // dependencies must be added to database before the update to prevent foreign key constraint errors + const promises = [] + for (const [name, {foreignPtr}] of this.#iterateTableColumns(pointer.type)) { + if (foreignPtr) { + const ptr = Pointer.pointerifyValue(pointer.getProperty(name)); + if (ptr instanceof Pointer) { + dependencies.add(ptr) + promises.push(Storage.setPointer(ptr)) + } + } } + await Promise.all(promises) + await this.#updatePointer(pointer, [partialUpdateKey], dependencies) + return dependencies; } } - // no template, just add a raw DXB entry + // is set, store in set table + else if (pointer.type == Type.std.Set) { + return this.#setPointerSet(pointer) + } + + // no template, just add a raw DXB entry, partial updates are not supported else { - await this.setPointerRaw(pointer) + return this.#setPointerRaw(pointer) } - return dependencies; } - async setPointerRaw(pointer: Pointer) { - console.log("storing raw pointer: " + Runtime.valueToDatexStringExperimental(pointer, true, true)) - await this.#init(); - const dependencies = new Set() - const encoded = Compiler.encodeValue(pointer, dependencies, true, false, true); - await this.#query('INSERT INTO ?? ?? VALUES ? ON DUPLICATE KEY UPDATE value=?;', [this.#metaTables.rawPointers.name, ["id", "value"], [pointer.id, encoded], encoded]) - // await datex_item_storage.setItem(key, Compiler.encodeValue(value, dependencies)); - return dependencies; + async getPointerValue(pointerId: string, outer_serialized: boolean): Promise { + // get table where pointer is stored + const table = await this.#getPointerTable(pointerId); + if (!table) { + console.warn("No table found for pointer " + pointerId); + return NOT_EXISTING; + } + + // is raw pointer + if (table == this.#metaTables.rawPointers.name) { + const value = (await this.#queryFirst<{value: string}>( + new Query() + .table(this.#metaTables.rawPointers.name) + .select("value") + .where(Where.eq(this.#pointerMysqlColumnName, pointerId)) + .build() + ))?.value; + if (value) this.#existingPointersCache.add(pointerId); + return value ? Runtime.decodeValue(this.#stringToBinary(value), outer_serialized) : NOT_EXISTING; + } + + // is set pointer + else if (table == this.#metaTables.sets.name) { + const values = await this.#query<{value_text:string, value_integer:number, value_decimal:number, value_boolean:boolean, value_time:Date, value_pointer:string, value_dxb:string}>( + new Query() + .table(this.#metaTables.sets.name) + .select("value_text", "value_integer", "value_decimal", "value_boolean", "value_time", "value_pointer", "value_dxb") + .where(Where.eq(this.#pointerMysqlColumnName, pointerId)) + .where(Where.ne("hash", "")) + .build() + ) + + const result = new Set() + for (const {value_text, value_integer, value_decimal, value_boolean, value_time, value_pointer, value_dxb} of values) { + if (value_text != undefined) result.add(value_text) + else if (value_integer != undefined) result.add(BigInt(value_integer)) + else if (value_decimal != undefined) result.add(value_decimal) + else if (value_boolean != undefined) result.add(Boolean(value_boolean)) + else if (value_time != undefined) result.add(value_time) + else if (value_pointer != undefined) result.add(await Pointer.load(value_pointer)) + else if (value_dxb != undefined) result.add(await Runtime.decodeValue(this.#stringToBinary(value_dxb))) + } + this.#existingPointersCache.add(pointerId); + return result; + } + + // is templated pointer + else { + const type = await this.#getTypeForTable(table); + if (!type) { + logger.error("No type found for table " + table); + return NOT_EXISTING; + } + const object = await this.#getTemplatedPointerObject(pointerId, table); + if (!object) return NOT_EXISTING; + + // resolve foreign pointers + const columns = await this.#getTableColumns(table); + if (!columns) throw new Error("No columns found for table " + table) + + await Promise.all( + [...columns.entries()] + .map(([colName, {foreignPtr, foreignTable, type}]) => this.assignPointerProperty(object, colName, type, foreignPtr, foreignTable)) + ) + + this.#existingPointersCache.add(pointerId); + return type.cast(object, undefined, undefined, false); + } } - async getPointerValue(pointerId: string, outer_serialized: boolean): Promise { + private async assignPointerProperty(object:Record, colName:string, type:string, foreignPtr:boolean, foreignTable?:string) { + // custom conversions: + // convert blob strings to ArrayBuffer + if (type == "blob" && typeof object[colName] == "string") { + object[colName] = this.#stringToBinary(object[colName] as string) + } + // convert Date ot Time + else if (object[colName] instanceof Date) { + object[colName] = new Time(object[colName] as Date) + } + + // convert to boolean + else if (typeof object[colName] == "number" && (type == "tinyint" || type == "boolean")) { + object[colName] = Boolean(object[colName]) + } + // is an object type with a template + if (foreignPtr) { + if (typeof object[colName] == "string") { + const ptrId = object[colName] as string; + if (foreignTable) this.#pointerTables.set(ptrId, foreignTable) + object[colName] = await Pointer.load(ptrId); + } + // else property is null/undefined + } + // is blob, assume it is a DXB value + else if (type == "blob") { + object[colName] = await Runtime.decodeValue(object[colName] as ArrayBuffer, true); + } } - async removePointer(pointerId: string): Promise { + + + async removePointer(pointerId: string): Promise { + this.#existingPointersCache.delete(pointerId) + // get table where pointer is stored + const table = await this.#getPointerTable(pointerId); + if (table) { + await this.#query('DELETE FROM ?? WHERE ??=?;', [table, this.#pointerMysqlColumnName, pointerId]) + } + // delete from pointer mapping + await this.#query('DELETE FROM ?? WHERE ??=?;', [this.#metaTables.pointerMapping.name, this.#pointerMysqlColumnName, pointerId]) } + async getPointerValueDXB(pointerId: string): Promise { - + // get table where pointer is stored + const table = await this.#getPointerTable(pointerId); + + // is raw pointer + if (table == this.#metaTables.rawPointers.name) { + const value = (await this.#queryFirst<{value: string}>( + new Query() + .table(this.#metaTables.rawPointers.name) + .select("value") + .where(Where.eq(this.#pointerMysqlColumnName, pointerId)) + .build() + ))?.value; + return value ? this.#stringToBinary(value) : null; + } + + // is set pointer + else if (table == this.#metaTables.sets.name) { + const values = await this.#query<{value_text:string, value_integer:number, value_decimal:number, value_boolean:boolean, value_time:Date, value_pointer:string, value_dxb:string}>( + new Query() + .table(this.#metaTables.sets.name) + .select("value_text", "value_integer", "value_decimal", "value_boolean", "value_time", "value_pointer", "value_dxb") + .where(Where.eq(this.#pointerMysqlColumnName, pointerId)) + .where(Where.ne("hash", "")) + .build() + ) + let setString = ` [` + const setEntries:string[] = [] + + for (const {value_text, value_integer, value_decimal, value_boolean, value_time, value_pointer, value_dxb} of values) { + if (value_text != undefined) setEntries.push(Runtime.valueToDatexStringExperimental(value_text)) + else if (value_integer != undefined) setEntries.push(Runtime.valueToDatexStringExperimental(value_integer)) + else if (value_decimal != undefined) setEntries.push(Runtime.valueToDatexStringExperimental(value_decimal)) + else if (value_boolean != undefined) setEntries.push(Runtime.valueToDatexStringExperimental(value_boolean)) + else if (value_time != undefined) setEntries.push(Runtime.valueToDatexStringExperimental(value_time)) + else if (value_pointer != undefined) setEntries.push(Runtime.valueToDatexStringExperimental(await Pointer.load(value_pointer))) + else if (value_dxb != undefined) setEntries.push(MessageLogger.decompile(this.#stringToBinary(value_dxb), false, false, false)) + } + setString += setEntries.join(",") + "]" + return Compiler.compile(setString, [], {sign: false, encrypt: false, to: Datex.Runtime.endpoint, preemptive_pointer_init: false}, false) as ArrayBuffer; + } + + // is templated pointer + else { + return this.#getTemplatedPointerValueDXB(pointerId, table); + } + } + async setPointerValueDXB(pointerId: string, value: ArrayBuffer) { + // check if raw pointer, otherwise not yet supported + const table = await this.#getPointerTable(pointerId); + if (table == this.#metaTables.rawPointers.name) { + await this.#setPointerInRawTable(pointerId, value); + } + else { + logger.error("Setting raw dxb value for templated pointer is not yet supported in SQL storage (pointer: " + pointerId + ", table: " + table + ")"); + } + } + async #setPointerInRawTable(pointerId: string, encoded: ArrayBuffer) { + const table = this.#metaTables.rawPointers.name; + const {result} = await this.#query('INSERT INTO ?? ?? VALUES ? ON DUPLICATE KEY UPDATE value=?;', [table, [this.#pointerMysqlColumnName, "value"], [pointerId, encoded], encoded], true) + // is newly inserted, add to pointer mapping + if (result.affectedRows == 1) await this.#updatePointerMapping(pointerId, table) } async hasPointer(pointerId: string): Promise { - return false; + if (this.#existingPointersCache.has(pointerId)) return true; + const count = (await this.#queryFirst<{COUNT: number}>( + new Query() + .table(this.#metaTables.pointerMapping.name) + .select("COUNT(*) as COUNT") + .where(Where.eq(this.#pointerMysqlColumnName, pointerId)) + .build() + )); + const exists = !!count && count.COUNT > 0; + if (exists) { + this.#existingPointersCache.add(pointerId); + // delete from cache after 2 minutes + setTimeout(()=>this.#existingPointersCache.delete(pointerId), 1000*60*2) + } + return exists; } async clear() { - // TODO! + await this.#resetAll(); } } \ No newline at end of file diff --git a/storage/storage-locations/sql-type-map.ts b/storage/storage-locations/sql-type-map.ts index 390c6607..48253f88 100644 --- a/storage/storage-locations/sql-type-map.ts +++ b/storage/storage-locations/sql-type-map.ts @@ -56,6 +56,8 @@ export const datex_type_mysql_map = new Map([ [Datex.Type.std.text, 'text'], + [Datex.Type.std.boolean, 'boolean'], + // ['smallint', Datex.Type.std.integer], // ['mediumint', Datex.Type.std.integer], // ['tinyint', Datex.Type.std.integer], diff --git a/storage/storage.ts b/storage/storage.ts index f94b0814..ab7b416c 100644 --- a/storage/storage.ts +++ b/storage/storage.ts @@ -17,6 +17,8 @@ import { StorageMap } from "../types/storage-map.ts"; import { StorageSet } from "../types/storage-set.ts"; import { IterableWeakSet } from "../utils/iterable-weak-set.ts"; import { LazyPointer } from "../runtime/lazy-pointer.ts"; +import { AutoMap } from "../utils/auto_map.ts"; +import { JSInterface } from "../runtime/js_interface.ts"; // displayInit(); @@ -34,11 +36,188 @@ export const site_suffix = (()=>{ })(); +type AtomicMatchInput = T | + ( + T extends string ? + RegExp : + never + ) + +type _MatchInput = + MatchCondition | + ( + T extends object ? + { + [K in Exclude]?: MatchInputValue + } : + AtomicMatchInput|AtomicMatchInput[] + ) +type MatchInputValue = + _MatchInput| // exact match + _MatchInput[] // or match + +export type MatchInput = MatchInputValue + +type ObjectKeyPaths = + T extends object ? + ( + ObjectKeyPaths extends never ? + `${string & Exclude}` : + `${string & Exclude}`|`${string & Exclude}.${ObjectKeyPaths]>}` + ): + never + +export type MatchOptions = { + /** + * Maximum number of matches to return + */ + limit?: number, + /** + * Sort by key (e.g. address.street) + */ + sortBy?: string // TODO: T extends object ? ObjectKeyPaths : string, + /** + * Sort in descending order (only if sortBy is set) + */ + sortDesc?: boolean, + /** + * Offset for match results + */ + offset?: number, + /** + * Return advanced match results (e.g. total count of matches) + */ + returnAdvanced?: boolean, + /** + * Provide a list of properties that should be returned as raw values. If provided, only the raw properties are returned + * and pointers are not loaded + */ + returnRaw?: string[] + /** + * Return pointer ids of matched items + */ + returnPointerIds?: boolean, + /** + * Custom computed properties for match query + */ + computedProperties?: Record> +} + +export type MatchResult = Options["returnAdvanced"] extends true ? + AdvancedMatchResult & ( + Options["returnPointerIds"] extends true ? + { + pointerIds: Set + } : + unknown + ) : + Set + +export type AdvancedMatchResult = { + total: number, + pointerIds?: Set, + matches: Set +} + +export enum MatchConditionType { + BETWEEN = "BETWEEN", + LESS_THAN = "LESS_THAN", + GREATER_THAN = "GREATER_THAN", + LESS_OR_EQUAL = "LESS_OR_EQUAL", + GREATER_OR_EQUAL = "GREATER_OR_EQUAL", + NOT_EQUAL = "NOT_EQUAL", + CONTAINS = "CONTAINS" +} +export type MatchConditionData = + T extends MatchConditionType.BETWEEN ? + [V, V] : + T extends MatchConditionType.LESS_THAN|MatchConditionType.GREATER_THAN|MatchConditionType.LESS_OR_EQUAL|MatchConditionType.GREATER_OR_EQUAL|MatchConditionType.NOT_EQUAL ? + V : + T extends MatchConditionType.CONTAINS ? + V : + never + +export class MatchCondition { + + private constructor( + public type: Type, + public data: MatchConditionData + ) {} + + static between(lower: V, upper: V) { + return new MatchCondition(MatchConditionType.BETWEEN, [lower, upper]) + } + + static lessThan(value: V) { + return new MatchCondition(MatchConditionType.LESS_THAN, value) + } + + static greaterThan(value: V) { + return new MatchCondition(MatchConditionType.GREATER_THAN, value) + } + + static lessOrEqual(value: V) { + return new MatchCondition(MatchConditionType.LESS_OR_EQUAL, value) + } + + static greaterOrEqual(value: V) { + return new MatchCondition(MatchConditionType.GREATER_OR_EQUAL, value) + } + + static notEqual(value: V) { + return new MatchCondition(MatchConditionType.NOT_EQUAL, value) + } + + static contains(...values: V[]) { + return new MatchCondition(MatchConditionType.CONTAINS, new Set(values)) + } +} + +export enum ComputedPropertyType { + GEOGRAPHIC_DISTANCE = "GEOGRAPHIC_DISTANCE", + SUM = "SUM", +} + +export type ComputedPropertyData = + Type extends ComputedPropertyType.GEOGRAPHIC_DISTANCE ? + {pointA: {lat: number|string, lon: number|string}, pointB: {lat: number|string, lon: number|string}} : + Type extends ComputedPropertyType.SUM ? + (number|string)[] : + never + +export class ComputedProperty { + + private constructor( + public type: Type, + public data: ComputedPropertyData + ) {} + + static geographicDistance(pointA: {lat: number|string, lon: number|string}, pointB: {lat: number|string, lon: number|string}) { + return new ComputedProperty(ComputedPropertyType.GEOGRAPHIC_DISTANCE, {pointA, pointB}) + } + + static sum(...values: (number|string)[]) { + return new ComputedProperty(ComputedPropertyType.SUM, values) + } +} + export interface StorageLocation { name: string isAsync: boolean + /** + * This storage location supports exec conditions for get operations + */ supportsExecConditions?: boolean + /** + * This storage location supports prefix selection for get operations + */ + supportsPrefixSelection?: boolean + /** + * This storage location supports match selection for get operations + * Must implement supportsMatchForType if true + */ + supportsMatchSelection?: boolean isSupported(): boolean onAfterExit?(): void // called when deno process exits @@ -50,7 +229,7 @@ export interface StorageLocation|void getItemValueDXB(key:string): Promise|ArrayBuffer|null setItemValueDXB(key:string, value: ArrayBuffer):Promise|void - getItemKeys(): Promise> | Generator + getItemKeys(prefix?:string): Promise> | Generator setPointer(pointer:Pointer, partialUpdateKey: unknown|typeof NOT_EXISTING): Promise>|Set getPointerValue(pointerId:string, outer_serialized:boolean, conditions?:ExecConditions):Promise|unknown @@ -59,6 +238,9 @@ export interface StorageLocation> | Generator getPointerValueDXB(pointerId:string): Promise|ArrayBuffer|null setPointerValueDXB(pointerId:string, value: ArrayBuffer):Promise|void + + supportsMatchForType?(type: Type): Promise|boolean + matchQuery?>(itemPrefix: string, valueType: Type, match: MatchInput, options:Options): Promise>|MatchResult clear(): Promise|void } @@ -73,7 +255,7 @@ export abstract class SyncStorageLocation implements StorageLocation abstract getItem(key:string, conditions?:ExecConditions): Promise|unknown abstract hasItem(key:string): boolean - abstract getItemKeys(): Generator + abstract getItemKeys(prefix?:string): Generator abstract removeItem(key: string): void abstract getItemValueDXB(key: string): ArrayBuffer|null @@ -88,6 +270,9 @@ export abstract class SyncStorageLocation implements StorageLocation>(itemPrefix: string, valueType: Type, match: MatchInput, options:Options): MatchResult + abstract clear(): void } @@ -101,7 +286,7 @@ export abstract class AsyncStorageLocation implements StorageLocation> abstract getItem(key:string, conditions?:ExecConditions): Promise abstract hasItem(key:string): Promise - abstract getItemKeys(): Promise> + abstract getItemKeys(prefix?:string): Promise> abstract removeItem(key: string): Promise abstract getItemValueDXB(key: string): Promise @@ -116,6 +301,9 @@ export abstract class AsyncStorageLocation implements StorageLocation abstract hasPointer(pointerId: string): Promise + supportsMatchForType?(type: Type): Promise|boolean + matchQuery?>(itemPrefix: string, valueType: Type, match: MatchInput, options:Options): Promise> + abstract clear(): Promise } @@ -129,8 +317,22 @@ type storage_location_options = L extends StorageLocation ? storage_options : never type StorageSnapshotOptions = { + /** + * Display all internally used items (e.g. for garbage collection) + */ internalItems: boolean, - expandStorageMapsAndSets: boolean + /** + * List all items and pointers of storage maps and sets + */ + expandStorageMapsAndSets: boolean, + /** + * Only display items (and related pointers) that contain the given string in their key + */ + itemFilter?: string, + /** + * Only display general information about storage data, no items or pointers + */ + onlyHeaders?: boolean } export class Storage { @@ -149,7 +351,7 @@ export class Storage { static item_deps_prefix = "deps::dxitem::" static subscriber_cache_prefix = "subscribers::" - static #storage_active_pointers = new Set(); + static #storage_active_pointers = new IterableWeakSet(); static #storage_active_pointer_ids = new Set(); /** @@ -299,14 +501,14 @@ export class Storage { } // update pointers - for (const ptr of this.#storage_active_pointers) { + for (const ptr of [...this.#storage_active_pointers]) { try { c++; const res = this.setPointer(ptr, true, location); if (res instanceof Promise) res.catch(()=>{}) } catch (e) {} } - for (const id of this.#storage_active_pointer_ids) { + for (const id of [...this.#storage_active_pointer_ids]) { try { c++; const ptr = Pointer.get(id); @@ -338,14 +540,26 @@ export class Storage { return Number(localStorage.getItem(this.meta_prefix+'__saved__' + location.name) ?? 0); } - static #dirty_locations = new Set() - // handle dirty states for async storage operations: + static #dirty_locations = new Map() // called when a full backup to this storage location was made public static setDirty(location:StorageLocation, dirty = true) { - if (dirty) this.#dirty_locations.add(location); - else this.#dirty_locations.delete(location); + // update counter + if (dirty) { + const currentCount = this.#dirty_locations.get(location)??0; + this.#dirty_locations.set(location, currentCount + 1); + } + else { + if (!this.#dirty_locations.has(location)) logger.warn("Invalid dirty state reset for location '"+location.name + "', dirty state was not set"); + else { + const newCount = this.#dirty_locations.get(location)! - 1; + if (newCount <= 0) { + this.#dirty_locations.delete(location); + } + else this.#dirty_locations.set(location, newCount); + } + } } static #dirty = false; @@ -359,7 +573,7 @@ export class Storage { */ private static saveDirtyState(){ if (this.#exit_without_save) return; // currently exiting - for (const location of this.#dirty_locations) { + for (const [location] of this.#dirty_locations) { localStorage.setItem(this.meta_prefix+'__dirty__' + location.name, new Date().getTime().toString()); } } @@ -418,21 +632,23 @@ export class Storage { static async setItemAsync(location:AsyncStorageLocation, key: string, value: unknown,listen_for_pointer_changes: boolean) { this.setDirty(location, true) + const itemExisted = await location.hasItem(key); // store value (might be pointer reference) const dependencies = await location.setItem(key, value); if (Pointer.is_local) this.checkUnresolvedLocalDependenciesForItem(key, value, dependencies); this.updateItemDependencies(key, [...dependencies].map(p=>p.id)); await this.saveDependencyPointersAsync(dependencies, listen_for_pointer_changes, location); this.setDirty(location, false) - return true; + return itemExisted; } static setItemSync(location:SyncStorageLocation, key: string, value: unknown,listen_for_pointer_changes: boolean) { + const itemExisted = location.hasItem(key); const dependencies = location.setItem(key, value); if (Pointer.is_local) this.checkUnresolvedLocalDependenciesForItem(key, value, dependencies); this.updateItemDependencies(key, [...dependencies].map(p=>p.id)); this.saveDependencyPointersSync(dependencies, listen_for_pointer_changes, location); - return true; + return itemExisted; } /** @@ -557,14 +773,14 @@ export class Storage { * @param location storage location */ private static async saveDependencyPointersAsync(dependencies: Set, listen_for_changes = true, location: AsyncStorageLocation) { - for (const ptr of dependencies) { + await Promise.all([...dependencies].map(async ptr=>{ // add if not yet in storage if (!await location.hasPointer(ptr.id)) await this.setPointer(ptr, listen_for_changes, location) - } + })); } - private static synced_pointers = new Set(); + private static synced_pointers = new WeakSet(); static syncPointer(pointer: Pointer, location: StorageLocation|undefined = this.#primary_location) { if (!this.#auto_sync_enabled) return; @@ -655,7 +871,7 @@ export class Storage { if (this.#primary_location != undefined && this.isInDirtyState(this.#primary_location) && this.#trusted_location != undefined && this.#trusted_location!=this.#primary_location) { await this.copyStorage(this.#trusted_location, this.#primary_location) logger.warn `restored dirty state of ${this.#primary_location.name} from trusted location ${this.#trusted_location.name}` - this.setDirty(this.#primary_location, false) // remove from dirty set + if (this.#dirty_locations.has(this.#primary_location)) this.setDirty(this.#primary_location, false) // remove from dirty set this.clearDirtyState(this.#primary_location) // remove from localstorage this.#dirty = false; // primary location is now trusted, update @@ -862,16 +1078,16 @@ export class Storage { return NOT_EXISTING; } - public static async getItemKeys(location?:StorageLocation){ + public static async getItemKeys(location?:StorageLocation, prefix?: string){ // for specific location - if (location) return location.getItemKeys(); + if (location) return location.getItemKeys(prefix); // ... iterate over keys from all locations const generators = []; for (const location of this.#locations.keys()) { - generators.push(await location.getItemKeys()) + generators.push(await location.getItemKeys(prefix)) } return (function*(){ @@ -889,7 +1105,7 @@ export class Storage { public static async getItemKeysStartingWith(prefix:string, location?:StorageLocation) { - const keyIterator = await Storage.getItemKeys(location); + const keyIterator = await Storage.getItemKeys(location, prefix); return (function*(){ for (const key of keyIterator) { if (key.startsWith(prefix)) yield key; @@ -898,7 +1114,7 @@ export class Storage { } public static async getItemCountStartingWith(prefix:string, location?:StorageLocation) { - const keyIterator = await Storage.getItemKeys(location); + const keyIterator = await Storage.getItemKeys(location, prefix); let count = 0; for (const key of keyIterator) { if (key.startsWith(prefix)) count++; @@ -906,8 +1122,18 @@ export class Storage { return count } + public static async supportsMatchQueries(type: Type) { + return (this.#primary_location?.supportsMatchSelection && await this.#primary_location?.supportsMatchForType!(type)) ?? false; + } + + public static itemMatchQuery>(itemPrefix: string, valueType:Type, match: MatchInput, options?:Options) { + options ??= {} as Options; + if (!this.#primary_location?.supportsMatchSelection) throw new Error("Primary storage location does not support match queries"); + return this.#primary_location!.matchQuery!(itemPrefix, valueType, match, options); + } + - public static async getPointerKeys(location?:StorageLocation){ + public static async getPointerIds(location?:StorageLocation){ // for specific location if (location) return location.getPointerIds(); @@ -938,7 +1164,7 @@ export class Storage { const promises = []; - for (const pointer_id of await this.getPointerKeys(from)) { + for (const pointer_id of await this.getPointerIds(from)) { const buffer = await from.getPointerValueDXB(pointer_id); if (!buffer) logger.error("could not copy empty pointer value: " + pointer_id) else promises.push(to.setPointerValueDXB(pointer_id, buffer)) @@ -989,7 +1215,7 @@ export class Storage { public static async hasItem(key:string, location?:StorageLocation):Promise { - + if (Storage.cache.has(key)) return true; // get from cache // try to find item at a storage location @@ -1009,6 +1235,9 @@ export class Storage { else return false; } + /** + * Remove an item from storage, returns true if the item existed + */ public static async removeItem(key:string, location?:StorageLocation):Promise { logger.debug("Removing item '" + key + "' from storage" + (location ? " (" + location.name + ")" : "")) @@ -1135,6 +1364,13 @@ export class Storage { } } + // remove internal localstorage entries + for (const key of Object.keys(localStorage)) { + if (key.startsWith(this.rc_prefix) || key.startsWith(this.item_deps_prefix) || key.startsWith(this.pointer_deps_prefix) || key.startsWith(this.meta_prefix)) { + localStorage.removeItem(key); + } + } + } /** @@ -1174,7 +1410,7 @@ export class Storage { } - public static async printSnapshot(options: StorageSnapshotOptions = {internalItems: false, expandStorageMapsAndSets: true}) { + public static async printSnapshot(options: StorageSnapshotOptions = {internalItems: false, expandStorageMapsAndSets: true, onlyHeaders: false}) { const {items, pointers} = await this.getSnapshot(options); const COLOR_PTR = `\x1b[38;2;${[65,102,238].join(';')}m` @@ -1196,13 +1432,19 @@ export class Storage { string = ESCAPE_SEQUENCES.BOLD+"Pointers\n\n"+ESCAPE_SEQUENCES.RESET string += `${ESCAPE_SEQUENCES.ITALIC}A list of all pointers stored in any storage location. Pointers are only stored as long as they are referenced somewhere else in the storage.\n\n${ESCAPE_SEQUENCES.RESET}` - for (const [key, storageMap] of pointers.snapshot) { - // check if stored in all locations, otherwise print in which location it is stored (functional programming) - const locations = [...storageMap.keys()] - const storedInAll = [...this.#locations.keys()].every(l => locations.includes(l)); - - const value = [...storageMap.values()][0]; - string += ` • ${COLOR_PTR}$${key}${ESCAPE_SEQUENCES.GREY}${storedInAll ? "" : (` (only in ${locations.map(l=>l.name).join(",")})`)} = ${value.replaceAll("\n", "\n ")}\n` + const pointersInMemory = [...pointers.snapshot.keys()].filter(id => Pointer.get(id)).length; + string += `\nTotal: ${ESCAPE_SEQUENCES.BOLD}${pointers.snapshot.size}${ESCAPE_SEQUENCES.RESET} pointers` + string += `\nIn memory: ${ESCAPE_SEQUENCES.BOLD}${pointersInMemory}${ESCAPE_SEQUENCES.RESET} pointers\n\n` + + if (!options.onlyHeaders) { + for (const [key, storageMap] of pointers.snapshot) { + // check if stored in all locations, otherwise print in which location it is stored (functional programming) + const locations = [...storageMap.keys()] + const storedInAll = [...this.#locations.keys()].every(l => locations.includes(l)); + + const value = [...storageMap.values()][0]; + string += ` • ${COLOR_PTR}$${key}${ESCAPE_SEQUENCES.GREY}${storedInAll ? "" : (` (only in ${locations.map(l=>l.name).join(",")})`)} = ${value.replaceAll("\n", "\n ")}\n` + } } console.log(string+"\n"); @@ -1210,13 +1452,15 @@ export class Storage { string = ESCAPE_SEQUENCES.BOLD+"Items\n\n"+ESCAPE_SEQUENCES.RESET string += `${ESCAPE_SEQUENCES.ITALIC}A list of all named items stored in any storage location.\n\n${ESCAPE_SEQUENCES.RESET}` - for (const [key, storageMap] of items.snapshot) { - // check if stored in all locations, otherwise print in which location it is stored (functional programming) - const locations = [...storageMap.keys()] - const storedInAll = [...this.#locations.keys()].every(l => locations.includes(l)); - - const value = [...storageMap.values()][0]; - string += ` • ${key}${ESCAPE_SEQUENCES.GREY}${storedInAll ? "" : (` (only in ${locations.map(l=>l.name).join(",")})`)} = ${value}\n` + if (!options.onlyHeaders) { + for (const [key, storageMap] of items.snapshot) { + // check if stored in all locations, otherwise print in which location it is stored (functional programming) + const locations = [...storageMap.keys()] + const storedInAll = [...this.#locations.keys()].every(l => locations.includes(l)); + + const value = [...storageMap.values()][0]; + string += ` • ${key}${ESCAPE_SEQUENCES.GREY}${storedInAll ? "" : (` (only in ${locations.map(l=>l.name).join(",")})`)} = ${value}\n` + } } console.log(string+"\n"); @@ -1227,37 +1471,39 @@ export class Storage { let rc_string = "" let item_deps_string = "" let pointer_deps_string = "" - for (let i = 0, len = localStorage.length; i < len; ++i ) { - const key = localStorage.key(i)!; - if (key.startsWith(this.rc_prefix)) { - const ptrId = key.substring(this.rc_prefix.length); - const count = this.getReferenceCount(ptrId); - rc_string += `\x1b[0m • ${key} = ${COLOR_NUMBER}${count}\n` - } - else if (key.startsWith(this.item_deps_prefix)) { - const depsRaw = localStorage.getItem(key); - // single entry - if (!depsRaw?.includes(",")) { - item_deps_string += `\x1b[0m • ${key} = (${COLOR_PTR}${depsRaw}\x1b[0m)\n` - } - // multiple entries - else { - let deps = localStorage.getItem(key)!.split(",").join(`\x1b[0m,\n ${COLOR_PTR}$`) - if (deps) deps = ` ${COLOR_PTR}$`+deps - item_deps_string += `\x1b[0m • ${key} = (\n${COLOR_PTR}${deps}\x1b[0m\n )\n` + if (!options.onlyHeaders) { + for (let i = 0, len = localStorage.length; i < len; ++i ) { + const key = localStorage.key(i)!; + if (key.startsWith(this.rc_prefix)) { + const ptrId = key.substring(this.rc_prefix.length); + const count = this.getReferenceCount(ptrId); + rc_string += `\x1b[0m • ${key} = ${COLOR_NUMBER}${count}\n` } - } - else if (key.startsWith(this.pointer_deps_prefix)) { - const depsRaw = localStorage.getItem(key); - // single entry - if (!depsRaw?.includes(",")) { - pointer_deps_string += `\x1b[0m • ${key} = (${COLOR_PTR}${depsRaw}\x1b[0m)\n` + else if (key.startsWith(this.item_deps_prefix)) { + const depsRaw = localStorage.getItem(key); + // single entry + if (!depsRaw?.includes(",")) { + item_deps_string += `\x1b[0m • ${key} = (${COLOR_PTR}${depsRaw}\x1b[0m)\n` + } + // multiple entries + else { + let deps = localStorage.getItem(key)!.split(",").join(`\x1b[0m,\n ${COLOR_PTR}$`) + if (deps) deps = ` ${COLOR_PTR}$`+deps + item_deps_string += `\x1b[0m • ${key} = (\n${COLOR_PTR}${deps}\x1b[0m\n )\n` + } } - // multiple entries - else { - let deps = localStorage.getItem(key)!.split(",").join(`\x1b[0m,\n ${COLOR_PTR}$`) - if (deps) deps = ` ${COLOR_PTR}$`+deps - pointer_deps_string += `\x1b[0m • ${key} = (\n${COLOR_PTR}${deps}\x1b[0m\n )\n` + else if (key.startsWith(this.pointer_deps_prefix)) { + const depsRaw = localStorage.getItem(key); + // single entry + if (!depsRaw?.includes(",")) { + pointer_deps_string += `\x1b[0m • ${key} = (${COLOR_PTR}${depsRaw}\x1b[0m)\n` + } + // multiple entries + else { + let deps = localStorage.getItem(key)!.split(",").join(`\x1b[0m,\n ${COLOR_PTR}$`) + if (deps) deps = ` ${COLOR_PTR}$`+deps + pointer_deps_string += `\x1b[0m • ${key} = (\n${COLOR_PTR}${deps}\x1b[0m\n )\n` + } } } } @@ -1270,17 +1516,21 @@ export class Storage { if (pointers.inconsistencies.size > 0 || items.inconsistencies.size > 0) { string = ESCAPE_SEQUENCES.BOLD+"Inconsistencies\n\n"+ESCAPE_SEQUENCES.RESET string += `${ESCAPE_SEQUENCES.ITALIC}Inconsistencies between storage locations don't necessarily indicate that something is wrong. They can occur when a storage location is not updated immediately (e.g. when only using SAVE_ON_EXIT).\n\n${ESCAPE_SEQUENCES.RESET}` - for (const [key, storageMap] of pointers.inconsistencies) { - for (const [location, value] of storageMap) { - string += ` • ${COLOR_PTR}$${key}${ESCAPE_SEQUENCES.GREY} (${(location.name+")").padEnd(15, " ")} = ${value.replaceAll("\n", "\n ")}\n` + + + if (!options.onlyHeaders) { + for (const [key, storageMap] of pointers.inconsistencies) { + for (const [location, value] of storageMap) { + string += ` • ${COLOR_PTR}$${key}${ESCAPE_SEQUENCES.GREY} (${(location.name+")").padEnd(15, " ")} = ${value.replaceAll("\n", "\n ")}\n` + } + string += `\n` } - string += `\n` - } - for (const [key, storageMap] of items.inconsistencies) { - for (const [location, value] of storageMap) { - string += ` • ${key}${ESCAPE_SEQUENCES.GREY} (${(location.name+")").padEnd(15, " ")} = ${value}` + for (const [key, storageMap] of items.inconsistencies) { + for (const [location, value] of storageMap) { + string += ` • ${key}${ESCAPE_SEQUENCES.GREY} (${(location.name+")").padEnd(15, " ")} = ${value}` + } + string += `\n` } - string += `\n` } console.info(string+"\n"); @@ -1289,9 +1539,15 @@ export class Storage { } + public static removeTrailingSemicolon(str:string) { + // replace ; and reset sequences with nothing + return str.replace(/;(\x1b\[0m)?$/g, "") + } + public static async getSnapshot(options: StorageSnapshotOptions = {internalItems: false, expandStorageMapsAndSets: true}) { - const items = await this.createSnapshot(this.getItemKeys.bind(this), this.getItemDecompiled.bind(this)); - const pointers = await this.createSnapshot(this.getPointerKeys.bind(this), this.getPointerDecompiledFromLocation.bind(this)); + const allowedPointerIds = options.itemFilter ? new Set() : undefined; + const items = await this.createSnapshot(this.getItemKeys.bind(this), this.getItemDecompiled.bind(this), options.itemFilter, allowedPointerIds); + const pointers = await this.createSnapshot(this.getPointerIds.bind(this), this.getPointerDecompiledFromLocation.bind(this), options.itemFilter, allowedPointerIds); // remove keys items that are unrelated to normal storage for (const [key] of items.snapshot) { @@ -1305,6 +1561,9 @@ export class Storage { } } + // additional pointer entries from storage maps/sets + const additionalEntries = new Set(); + // iterate over storage maps and sets and render all entries if (options.expandStorageMapsAndSets) { for (const [ptrId, storageMap] of pointers.snapshot) { @@ -1319,46 +1578,148 @@ export class Storage { } if (ptr.val instanceof StorageMap) { const map = ptr.val; + const keyIterator = await this.getItemKeysStartingWith((map as any)._prefix) + const pointerIds = new Set(); let inner = ""; - for await (const [key, val] of map) { - inner += ` ${Runtime.valueToDatexStringExperimental(key, true, true)}\x1b[0m => ${Runtime.valueToDatexStringExperimental(val, true, true)}\n` + for await (const key of keyIterator) { + const valString = await this.getItemDecompiled(key, true, location); + if (valString === NOT_EXISTING) { + logger.error("Invalid entry in storage (" + location.name + "): " + key); + continue; + } + const keyString = await this.getItemDecompiled('key.' + key, true, location); + if (keyString === NOT_EXISTING) { + logger.error("Invalid key in storage (" + location.name + "): " + key); + continue; + } + inner += ` ${this.removeTrailingSemicolon(keyString)}\x1b[0m => ${this.removeTrailingSemicolon(valString)}\n` + + // additional pointer ids included in value or key + if (allowedPointerIds) { + const valMatches = valString.match(/\$[a-zA-Z0-9]+/g)??[] + const keyMatches = keyString.match(/\$[a-zA-Z0-9]+/g)??[]; + + for (const match of valMatches) { + const id = match.substring(1); + pointerIds.add(id) + if (!allowedPointerIds.has(id)) additionalEntries.add(id); + } + for (const match of keyMatches) { + const id = match.substring(1); + if (!allowedPointerIds.has(id)) additionalEntries.add(id); + + } + } } + + // size in memory / total size + const totalSize = await (ptr.val as StorageMap).getSize(); + const totalDirectPointerSize = pointerIds.size; + const inMemoryPointersSize= [...pointerIds].filter(id => Pointer.get(id)).length; + const sizeInfo = ` ${ESCAPE_SEQUENCES.GREY}total size: ${totalSize}, in memory: ${inMemoryPointersSize}/${totalDirectPointerSize} pointers${ESCAPE_SEQUENCES.RESET}\n` + // substring: remove last \n - if (inner) storageMap.set(location, "\x1b[38;2;50;153;220m \x1b[0m{\n"+inner.substring(0, inner.length-1)+"\x1b[0m\n}") + if (inner) storageMap.set(location, "\x1b[38;2;50;153;220m \x1b[0m{\n"+sizeInfo+inner.substring(0, inner.length-1)+"\x1b[0m\n}") } else if (ptr.val instanceof StorageSet) { const set = ptr.val; + const keyIterator = await this.getItemKeysStartingWith((set as any)._prefix) + const pointerIds = new Set(); + let inner = ""; - for await (const val of set) { - inner += ` ${Runtime.valueToDatexStringExperimental(val, true, true)},\n` + for await (const key of keyIterator) { + const valString = await this.getItemDecompiled(key, true, location); + if (valString === NOT_EXISTING) { + logger.error("Invalid entry in storage (" + location.name + "): " + key); + continue; + } + inner += ` ${this.removeTrailingSemicolon(valString)},\n` + + // additional pointer ids included in value + if (allowedPointerIds) { + const matches = valString.match(/\$[a-zA-Z0-9]+/g)??[]; + for (const match of matches) { + const id = match.substring(1); + pointerIds.add(id) + if (!allowedPointerIds.has(id)) additionalEntries.add(id); + } + } } + + // size in memory / total size + const totalSize = await (ptr.val as StorageSet).getSize(); + const totalDirectPointerSize = pointerIds.size; + const inMemoryPointersSize= [...pointerIds].filter(id => Pointer.get(id)).length; + const sizeInfo = ` ${ESCAPE_SEQUENCES.GREY}total size: ${totalSize}, in memory: ${inMemoryPointersSize}/${totalDirectPointerSize} pointers${ESCAPE_SEQUENCES.RESET}\n` + // substring: remove last \n - if (inner) storageMap.set(location, "\x1b[38;2;50;153;220m \x1b[0m{\n"+inner.substring(0, inner.length-1)+"\x1b[0m\n}") + if (inner) storageMap.set(location, "\x1b[38;2;50;153;220m \x1b[0m{\n"+sizeInfo+inner.substring(0, inner.length-1)+"\x1b[0m\n}") } } } } + if (additionalEntries.size > 0) { + await this.createSnapshot(this.getPointerIds.bind(this), this.getPointerDecompiledFromLocation.bind(this), options.itemFilter, additionalEntries, { + snapshot: pointers.snapshot, + inconsistencies: pointers.inconsistencies + }); + } + return {items, pointers} } private static async createSnapshot( keyGenerator: (location?: StorageLocation | undefined) => Promise>, itemGetter: (key: string, colorized: boolean, location: StorageLocation) => Promise, + filter?: string, + allowedPointerIds?: Set, + baseSnapshot?: { + snapshot: AutoMap, string>>; + inconsistencies: AutoMap, string>>; + } ) { - const snapshot = new Map>().setAutoDefault(Map); - const inconsistencies = new Map>().setAutoDefault(Map); + const snapshot = baseSnapshot?.snapshot ?? new Map>().setAutoDefault(Map); + const inconsistencies = baseSnapshot?.inconsistencies ?? new Map>().setAutoDefault(Map); + + const skippedEntries = new Set(); + const additionalEntries = new Set(); + for (const location of new Set([this.#primary_location!, ...this.#locations.keys()].filter(l=>!!l))) { for (const key of await keyGenerator(location)) { + if (filter && !key.includes(filter) && !allowedPointerIds?.has(key)) { + if (allowedPointerIds) skippedEntries.add(key); // remember skipped entries that might be added later + continue; + } const decompiled = await itemGetter(key, true, location); + if (typeof decompiled !== "string") { console.error("Invalid entry in storage (" + location.name + "): " + key); continue; } - snapshot.getAuto(key).set(location, decompiled); + + // collect referenced pointer ids + if (allowedPointerIds) { + const matches = decompiled.match(/\$[a-zA-Z0-9]+/g); + if (matches) { + for (const match of matches) { + const id = match.substring(1); + if (skippedEntries.has(id)) additionalEntries.add(id); + allowedPointerIds.add(id); + } + } + } + snapshot.getAuto(key).set(location, this.removeTrailingSemicolon(decompiled)); } } - + + // run again with additional entries + if (additionalEntries.size > 0) { + await this.createSnapshot(keyGenerator, itemGetter, filter, additionalEntries, { + snapshot, + inconsistencies + }); + } // find inconsistencies for (const [key, storageMap] of snapshot) { diff --git a/tests/sql-storage.test.ts b/tests/sql-storage.test.ts index 9a42853a..165d7f1f 100644 --- a/tests/sql-storage.test.ts +++ b/tests/sql-storage.test.ts @@ -27,23 +27,23 @@ Datex.Storage.addLocation(sqlStorage, { }) @sync class Position { - @property declare x: number - @property declare y: number + @property x!: number + @property y!: number } @sync class Player { - @property declare name: string + @property name!: string @property @type('text(20)') declare username: string - @property declare color: bigint - @property declare pos: Position + @property color!: bigint + @property pos!: Position } @sync class ScoreboardEntry { - @property declare player: Player - @property declare score: number + @property player!: Player + @property score!: number } diff --git a/threads/promise-fn-types.ts b/threads/promise-fn-types.ts index eeadb5ff..3be11d94 100644 --- a/threads/promise-fn-types.ts +++ b/threads/promise-fn-types.ts @@ -3,7 +3,7 @@ * Promise methods return type inference magic */ -import { Equals } from "../utils/global_types.ts"; +import type { Equals } from "../utils/global_types.ts"; class _PromiseWrapper { all(e: T[]) { diff --git a/threads/threads.ts b/threads/threads.ts index e512b4e7..fd6eb2b4 100644 --- a/threads/threads.ts +++ b/threads/threads.ts @@ -1,5 +1,5 @@ import { Logger, console_theme } from "../utils/logger.ts"; -import { Equals } from "../utils/global_types.ts"; +import type { Equals } from "../utils/global_types.ts"; const logger = new Logger("thread-runner"); diff --git a/types/addressing.ts b/types/addressing.ts index dafcd767..d5ac1aaa 100644 --- a/types/addressing.ts +++ b/types/addressing.ts @@ -1,6 +1,6 @@ import { BinaryCode } from "../compiler/binary_codes.ts"; import { Pointer } from "../runtime/pointers.ts"; -import { ValueConsumer } from "./abstract_types.ts"; +import type { ValueConsumer } from "./abstract_types.ts"; import { ValueError } from "./errors.ts"; import { Compiler, ProtocolDataTypesMap } from "../compiler/compiler.ts"; import type { datex_scope, dxb_header, trace } from "../utils/global_types.ts"; @@ -43,6 +43,7 @@ export type endpoints = Endpoint|Disjunction export class Target implements ValueConsumer { + // TODO: remove entry when Endpoint WeakRef was garbage collected protected static targets = new Map>(); // target string -> target element static readonly prefix:target_prefix = "@" static readonly type:BinaryCode diff --git a/types/assertion.ts b/types/assertion.ts index db01e5cf..716f4c7b 100644 --- a/types/assertion.ts +++ b/types/assertion.ts @@ -1,6 +1,6 @@ import { VOID } from "../runtime/constants.ts"; import type { datex_scope } from "../utils/global_types.ts"; -import { ValueConsumer } from "./abstract_types.ts"; +import type { ValueConsumer } from "./abstract_types.ts"; import { AssertionError, RuntimeError, ValueError } from "./errors.ts"; import { ExtensibleFunction } from "./function-utils.ts"; import { Callable } from "./function.ts"; diff --git a/types/function-utils.ts b/types/function-utils.ts index 1c5d14ab..08491168 100644 --- a/types/function-utils.ts +++ b/types/function-utils.ts @@ -60,7 +60,7 @@ declare global { function getUsedVars(fn: (...args:unknown[])=>unknown) { const source = fn.toString(); - const usedVarsSource = source.match(/^(?:(?:[\w\s*])+\(.*\)\s*{|\(.*\)\s*=>\s*{?|.*\s*=>\s*{?)\s*use\s*\(([\s\S]*?)\)/)?.[1] + const usedVarsSource = source.match(/^(?:(?:[\w\s*])+\(.*?\)\s*{|\(.*?\)\s*=>\s*{?|.*?\s*=>\s*{?)\s*use\s*\(([\s\S]*?)\)/)?.[1] if (!usedVarsSource) return {}; const usedVars = usedVarsSource.split(",").map(v=>v.trim()).filter(v=>!!v) @@ -121,6 +121,7 @@ export function getSourceWithoutUsingDeclaration(fn: (...args:unknown[])=>unknow if (fnSource.startsWith("async")) fnSource = fnSource.replace("async", "async function") else fnSource = "function " + fnSource } + return fnSource .replace(/(?<=(?:(?:[\w\s*])+\(.*\)\s*{|\(.*\)\s*=>\s*{?|.*\s*=>\s*{?)\s*)(use\s*\((?:[\s\S]*?)\))/, 'true /*$1*/') } @@ -132,7 +133,7 @@ const isNormalFunction = (fnSrc:string) => { return !!fnSrc.match(/^(async\s+)?function(\(| |\*)/) } const isArrowFunction = (fnSrc:string) => { - return !!fnSrc.match(/^(async\s+)?\([^)]*\)\s*=>/) + return !!fnSrc.match(/^(async\s+)?(\([^)]*\)|\w+)\s*=>/) } const isNativeFunction = (fnSrc:string) => { diff --git a/types/function.ts b/types/function.ts index aba7ceef..6a3d24b5 100644 --- a/types/function.ts +++ b/types/function.ts @@ -1,7 +1,7 @@ import { Pointer, Ref } from "../runtime/pointers.ts"; import { Runtime } from "../runtime/runtime.ts"; import { logger } from "../utils/global_values.ts"; -import { StreamConsumer, ValueConsumer } from "./abstract_types.ts"; +import type { StreamConsumer, ValueConsumer } from "./abstract_types.ts"; import { BROADCAST, Endpoint, endpoint_name, LOCAL_ENDPOINT, target_clause } from "./addressing.ts"; import { Markdown } from "./markdown.ts"; import { Scope } from "./scope.ts"; @@ -11,7 +11,7 @@ import { Compiler } from "../compiler/compiler.ts"; import { Stream } from "./stream.ts" import { PermissionError, RuntimeError, TypeError, ValueError } from "./errors.ts"; import { ProtocolDataType } from "../compiler/protocol_types.ts"; -import { DX_EXTERNAL_FUNCTION_NAME, DX_EXTERNAL_SCOPE_NAME, VOID } from "../runtime/constants.ts"; +import { DX_EXTERNAL_FUNCTION_NAME, DX_EXTERNAL_SCOPE_NAME, DX_TIMEOUT, VOID } from "../runtime/constants.ts"; import { Type, type_clause } from "./type.ts"; import { callWithMetadata, callWithMetadataAsync, getMeta } from "../utils/caller_metadata.ts"; import { Datex } from "../mod.ts"; @@ -511,7 +511,14 @@ function to (this:Function, receivers:Receiver) { apply: (target, _thisArg, argArray:unknown[]) => { const externalScopeName = target[DX_EXTERNAL_SCOPE_NAME]; const externalFunctionName = target[DX_EXTERNAL_FUNCTION_NAME]; - return datex(`#public.?.? ?`, [externalScopeName, externalFunctionName, new Tuple(argArray)], receivers as target_clause) + const timeout = target[DX_TIMEOUT]; + return datex( + `#public.?.? ?`, + [externalScopeName, externalFunctionName, new Tuple(argArray)], + receivers as target_clause, + undefined, undefined, undefined, undefined, + timeout + ) } }) } diff --git a/types/iterator.ts b/types/iterator.ts index d26402bf..d44bc2ae 100644 --- a/types/iterator.ts +++ b/types/iterator.ts @@ -1,6 +1,6 @@ import { Tuple } from "./tuple.ts"; import type { datex_scope } from "../utils/global_types.ts"; -import { ValueConsumer } from "./abstract_types.ts"; +import type { ValueConsumer } from "./abstract_types.ts"; export class Iterator { diff --git a/types/js-function.ts b/types/js-function.ts index f9ced748..ee7a1cfc 100644 --- a/types/js-function.ts +++ b/types/js-function.ts @@ -49,14 +49,14 @@ export class JSTransferableFunction extends ExtensibleFunction { } - call(...args:any[]) { + handleCall(...args:any[]) { return this.#fn(...args) } // waits until all lazy dependencies are resolved and then calls the function async callLazy() { await this.lazyResolved; - return this.call() + return this.handleCall() } public get hasUnresolvedLazyDependencies() { diff --git a/types/logic.ts b/types/logic.ts index 13dd178b..b6879b6f 100644 --- a/types/logic.ts +++ b/types/logic.ts @@ -155,7 +155,9 @@ export class Logical extends Set { // wrong atomic type at runtime // guard for: against is T if (!(against instanceof atomic_class)) { - throw new RuntimeError(`Invalid match check: atomic value has wrong type (expected ${Type.getClassDatexType(atomic_class)}, found ${Type.ofValue(against)})`); + console.error(`Invalid match check: atomic value has wrong type (expected ${Type.getClassDatexType(atomic_class)}, found ${Type.ofValue(against)})`) + // throw new RuntimeError(`Invalid match check: atomic value has wrong type (expected ${Type.getClassDatexType(atomic_class)}, found ${Type.ofValue(against)})`); + return true; } // match diff --git a/types/quantity.ts b/types/quantity.ts index a0c359b2..006c5ee6 100644 --- a/types/quantity.ts +++ b/types/quantity.ts @@ -38,7 +38,7 @@ type expanded_symbol = [factor_num:number|bigint, factor_den:number|bigint, unit // Quantity with unit -export class Quantity { +export class Quantity { static cached_binaries = new Map(); @@ -102,7 +102,7 @@ export class Quantity { * @param value can be a number, bigint, or string: '1.25', '1', '0.5e12', '1/10', or [numerator, denominator] * @param unit */ - constructor(value?:number|bigint|string|[num:number|bigint, den:number|bigint], unit?:U extends Unit ? code_to_extended_symbol : unknown) + constructor(value?:number|bigint|string|[num:number|bigint, den:number|bigint], unit?:code_to_extended_symbol) constructor(value?:number|bigint|string|[num:number|bigint, den:number|bigint], encoded_unit?:unit) constructor(value:number|bigint|string|[num:number|bigint, den:number|bigint] = 1, symbol_or_encoded_unit:string|unit = 'x') { diff --git a/types/storage-map.ts b/types/storage-map.ts index dfd3ebfb..4e71abe4 100644 --- a/types/storage-map.ts +++ b/types/storage-map.ts @@ -4,9 +4,6 @@ import { Compiler } from "../compiler/compiler.ts"; import { DX_PTR } from "../runtime/constants.ts"; import { Pointer } from "../runtime/pointers.ts"; import { Storage } from "../storage/storage.ts"; -import { Logger } from "../utils/logger.ts"; - -const logger = new Logger("StorageMap"); /** @@ -20,10 +17,31 @@ export class StorageWeakMap { #prefix?: string; + /** + * Time in milliseconds after which a value is removed from the in-memory cache + * Default: 5min + */ + cacheTimeout = 5 * 60 * 1000; + + /** + * If true, non-pointer objects are allowed as + * values in the map (default) + * Otherwise, object values are automatically proxified + * when added to the map. + */ + allowNonPointerObjectValues = false; + + constructor(){ Pointer.proxifyValue(this) } + #_pointer?: Pointer; + get #pointer() { + if (!this.#_pointer) this.#_pointer = Pointer.getByValue(this); + if (!this.#_pointer) throw new Error(this.constructor.name + " not bound to a pointer") + return this.#_pointer; + } static async from(entries: readonly (readonly [K, V])[]){ const map = $$(new StorageWeakMap()); @@ -66,16 +84,20 @@ export class StorageWeakMap { const storage_key = await this.getStorageKey(key); return this._set(storage_key, value); } - protected _set(storage_key:string, value:V) { + protected async _set(storage_key:string, value:V) { + // proxify value + if (!this.allowNonPointerObjectValues) { + value = this.#pointer.proxifyChild("", value); + } this.activateCacheTimeout(storage_key); - return Storage.setItem(storage_key, value) + await Storage.setItem(storage_key, value) + return this; } protected activateCacheTimeout(storage_key:string){ setTimeout(()=>{ - logger.debug("removing item from cache: " + storage_key); Storage.cache.delete(storage_key) - }, 60_000); + }, this.cacheTimeout); } protected async getStorageKey(key: K) { @@ -108,14 +130,51 @@ export class StorageMap extends StorageWeakMap { #key_prefix = 'key.' - override async set(key: K, value: V): Promise { + #size?: number; + + get size() { + if (this.#size == undefined) throw new Error("size not yet available. use getSize() instead"); + return this.#size; + } + + async getSize() { + if (this.#size != undefined) return this.#size; + else { + await this.#determineSizeFromStorage(); + return this.#size! + } + } + + /** + * Sets this.#size to the correct value determined from storage. + */ + async #determineSizeFromStorage() { + const calculatedSize = await Storage.getItemCountStartingWith(this._prefix); + this.#updateSize(calculatedSize); + } + + #updateSize(newSize: number) { + this.#size = newSize; + } + + async #incrementSize() { + this.#updateSize(await this.getSize() + 1); + } + + async #decrementSize() { + this.#updateSize(await this.getSize() - 1); + } + + override async set(key: K, value: V): Promise { const storage_key = await this.getStorageKey(key); const storage_item_key = this.#key_prefix + storage_key; // store value await this._set(storage_key, value); // store key this.activateCacheTimeout(storage_item_key); - return Storage.setItem(storage_item_key, key) + const alreadyExisted = await Storage.setItem(storage_item_key, key); + if (!alreadyExisted) await this.#incrementSize(); + return this; } override async delete(key: K) { @@ -124,7 +183,9 @@ export class StorageMap extends StorageWeakMap { // delete value await this._delete(storage_key); // delete key - return Storage.removeItem(storage_item_key) + const existed = await Storage.removeItem(storage_item_key) + if (existed) await this.#decrementSize(); + return existed; } /** diff --git a/types/storage-set.ts b/types/storage-set.ts index 1e4b0163..4774d679 100644 --- a/types/storage-set.ts +++ b/types/storage-set.ts @@ -3,8 +3,12 @@ import { Compiler } from "../compiler/compiler.ts"; import { DX_PTR } from "../runtime/constants.ts"; import { Pointer } from "../runtime/pointers.ts"; -import { Storage } from "../storage/storage.ts"; +import { MatchResult, Storage } from "../storage/storage.ts"; import { Logger } from "../utils/logger.ts"; +import { MatchInput, match } from "../utils/match.ts"; +import type { Class } from "../utils/global_types.ts"; +import { MatchOptions } from "../utils/match.ts"; +import { Type } from "./type.ts"; const logger = new Logger("StorageSet"); @@ -17,8 +21,29 @@ const logger = new Logger("StorageSet"); */ export class StorageWeakSet { + /** + * Time in milliseconds after which a value is removed from the in-memory cache + * Default: 5min + */ + cacheTimeout = 5 * 60 * 1000; + + /** + * If true, non-pointer objects are allowed as + * values in the map (default) + * Otherwise, object values are automatically proxified + * when added to the map. + */ + allowNonPointerObjectValues = false; + #prefix?: string; + #_pointer?: Pointer; + get #pointer() { + if (!this.#_pointer) this.#_pointer = Pointer.getByValue(this); + if (!this.#_pointer) throw new Error(this.constructor.name + " not bound to a pointer") + return this.#_pointer; + } + constructor(){ Pointer.proxifyValue(this) } @@ -29,7 +54,7 @@ export class StorageWeakSet { return set; } - protected get _prefix() { + get _prefix() { if (!this.#prefix) this.#prefix = 'dxset::'+(this as any)[DX_PTR].idString()+'.'; return this.#prefix; } @@ -41,6 +66,10 @@ export class StorageWeakSet { return this; } protected _add(storage_key:string, value:V|null) { + // proxify value + if (!this.allowNonPointerObjectValues) { + value = this.#pointer.proxifyChild("", value); + } this.activateCacheTimeout(storage_key); return Storage.setItem(storage_key, value); } @@ -63,9 +92,8 @@ export class StorageWeakSet { protected activateCacheTimeout(storage_key:string){ setTimeout(()=>{ - logger.debug("removing item from cache: " + storage_key); Storage.cache.delete(storage_key) - }, 60_000); + }, this.cacheTimeout); } protected async getStorageKey(value: V) { @@ -91,11 +119,6 @@ export class StorageWeakSet { */ export class StorageSet extends StorageWeakSet { - #size_key?: string - protected get _size_key() { - if (!this.#size_key) this.#size_key = 'dxset.size::'+(this as any)[DX_PTR].idString()+'.'; - return this.#size_key; - } #size?: number; @@ -116,27 +139,20 @@ export class StorageSet extends StorageWeakSet { * Sets this.#size to the correct value determined from storage. */ async #determineSizeFromStorage() { - const cachedSize = await Storage.getItem(this._size_key); const calculatedSize = await Storage.getItemCountStartingWith(this._prefix); - if (cachedSize !== calculatedSize) { - if (cachedSize != undefined) - logger.warn(`Size mismatch for StorageSet (${(this as any)[DX_PTR].idString()}) detected. Setting size to ${calculatedSize}`) - await this.#updateSize(calculatedSize); - } - else this.#size = calculatedSize; + this.#updateSize(calculatedSize); } - async #updateSize(newSize: number) { + #updateSize(newSize: number) { this.#size = newSize; - await Storage.setItem(this._size_key, newSize); } async #incrementSize() { - await this.#updateSize(await this.getSize() + 1); + this.#updateSize(await this.getSize() + 1); } async #decrementSize() { - await this.#updateSize(await this.getSize() - 1); + this.#updateSize(await this.getSize() - 1); } @@ -230,4 +246,7 @@ export class StorageSet extends StorageWeakSet { } } + match(valueType:Class|Type, matchInput: MatchInput, options?: Options): Promise> { + return match(this as unknown as StorageSet, valueType, matchInput, options) + } } \ No newline at end of file diff --git a/types/stream.ts b/types/stream.ts index 942f40e6..44743a5d 100644 --- a/types/stream.ts +++ b/types/stream.ts @@ -1,7 +1,7 @@ import { ReadableStream } from "../runtime/runtime.ts"; import { Pointer } from "../runtime/pointers.ts"; import type { datex_scope } from "../utils/global_types.ts"; -import { StreamConsumer } from "./abstract_types.ts"; +import type { StreamConsumer } from "./abstract_types.ts"; import { Logger } from "../utils/logger.ts"; const logger = new Logger("Stream") diff --git a/types/struct.ts b/types/struct.ts index a4d68077..5448c2a4 100644 --- a/types/struct.ts +++ b/types/struct.ts @@ -1,7 +1,12 @@ +import { dc } from "../js_adapter/js_class_adapter.ts"; import type { ObjectRef } from "../runtime/pointers.ts"; import { Runtime } from "../runtime/runtime.ts"; +import { Class } from "../utils/global_types.ts"; import { sha256 } from "../utils/sha256.ts"; import { Type } from "./type.ts"; +import { Decorators } from "../js_adapter/js_class_adapter.ts"; +import { getCallerFile } from "../utils/caller_metadata.ts"; +import { client_type } from "../utils/constants.ts"; type StructuralTypeDefIn = { [key: string]: Type|(new () => unknown)|StructuralTypeDefIn @@ -19,7 +24,12 @@ type collapseType = { ) } -export type inferType = DXType extends Type ? Def : never; +export type inferType = + DXTypeOrClass extends Type ? + ObjectRef : + DXTypeOrClass extends Class ? + InstanceType : + never; /** * Define a structural type without a class or prototype. @@ -57,10 +67,25 @@ export type inferType = DXType extends Type ? De * ``` */ - -export function struct(def: Def): Type> & ((val: collapseType)=>ObjectRef>) { +export function struct & Class>(classDefinition: T): dc +export function struct & Class>(type: string, classDefinition: T): dc +export function struct(typeName: string, def: Def): Type> & ((val: collapseType)=>ObjectRef>) +export function struct(def: Def): Type> & ((val: collapseType)=>ObjectRef>) +export function struct(defOrTypeName: StructuralTypeDefIn|Class|string, def?: StructuralTypeDefIn|Class): any { // create unique type name from template hash + const callerFile = client_type == "deno" ? getCallerFile() : undefined; + + const hasType = typeof defOrTypeName == "string"; + const typeName = hasType ? defOrTypeName : undefined; + def = hasType ? def : defOrTypeName; + + // is class definition + if (typeof def == "function") { + return Decorators.sync(typeName, def, undefined, callerFile); + } + + // is struct definition if (!def || typeof def !== "object") throw new Error("Struct definition must of type object"); const template:StructuralTypeDef = {}; @@ -84,8 +109,10 @@ export function struct(def: Def): Type): Time + plus(amount: number, unit: code_to_extended_symbol): Time + plus(time:Quantity|number, unit?: code_to_extended_symbol) { + if (typeof time == "number") { + if (unit == undefined) throw new Error("unit is required") + else time = new Quantity(time, unit) + } - plus(time:Quantity) { if (time.hasBaseUnit('s')) { return new Time(this.getTime()+(time.value*1000)) } @@ -24,9 +30,19 @@ export class Time extends Date { new_time.add(time); return new_time } + else { + throw new Error("Invalid time unit") + } } - minus(time:Quantity) { + minus(time:Quantity): Time + minus(amount: number, unit: code_to_extended_symbol): Time + minus(time:Quantity|number, unit?: code_to_extended_symbol) { + if (typeof time == "number") { + if (unit == undefined) throw new Error("unit is required") + else time = new Quantity(time, unit) + } + if (time.hasBaseUnit('s')) { return new Time(this.getTime()-(time.value*1000)) } @@ -35,28 +51,49 @@ export class Time extends Date { new_time.subtract(time); return new_time } + else { + throw new Error("Invalid time unit") + } } - add(time:Quantity) { + add(time:Quantity): void + add(amount: number, unit: code_to_extended_symbol): void + add(time:Quantity|number, unit?: code_to_extended_symbol) { + if (typeof time == "number") { + if (unit == undefined) throw new Error("unit is required") + else time = new Quantity(time, unit) + } + if (time.hasBaseUnit('s')) { this.setTime(this.getTime()+(time.value*1000)) - console.log(this.getTime(), time.value*1000, this.getTime()+(time.value*1000)) - } else if (time.hasBaseUnit('Cmo')) { this.setMonth(this.getMonth()+time.value); } + else { + throw new Error("Invalid time unit") + } } - subtract(time:Quantity) { + subtract(time:Quantity): void + subtract(amount: number, unit: code_to_extended_symbol): void + subtract(time:Quantity|number, unit?: code_to_extended_symbol) { + if (typeof time == "number") { + if (unit == undefined) throw new Error("unit is required") + else time = new Quantity(time, unit) + } + if (time.hasBaseUnit('s')) { this.setTime(this.getTime()-(time.value*1000)) } else if (time.hasBaseUnit('Cmo')) { this.setMonth(this.getMonth()-time.value); } + else { + throw new Error("Invalid time unit") + } } } \ No newline at end of file diff --git a/types/type.ts b/types/type.ts index 91dccbed..74d51703 100644 --- a/types/type.ts +++ b/types/type.ts @@ -23,6 +23,7 @@ import {StorageMap, StorageWeakMap} from "./storage-map.ts" import {StorageSet, StorageWeakSet} from "./storage-set.ts" import { ExtensibleFunction } from "./function-utils.ts"; import type { JSTransferableFunction } from "./js-function.ts"; +import type { MatchCondition } from "../storage/storage.ts"; export type inferDatexType = T extends Type ? JST : any; @@ -41,7 +42,7 @@ export class Type extends ExtensibleFunction { // should be serialized, but is not a complex type (per default, only complex types are serialized) static serializable_not_complex_types = ["buffer"] // values that are represented js objects but have a single instance per value, handle like normal js primitives - static pseudo_js_primitives = ["Type", "endpoint", "target", "url"] + static pseudo_js_primitives = ["Type", "endpoint", "target", "url", "RegExp"] public static types = new Map(); // type name -> type @@ -258,7 +259,7 @@ export class Type extends ExtensibleFunction { return Runtime.castValue(this, VOID, context, context_location, origin); } - static #current_constructor:globalThis.Function; + static #current_constructor:globalThis.Function|null; public static isConstructing(value:object) { return value.constructor == this.#current_constructor; @@ -326,10 +327,10 @@ export class Type extends ExtensibleFunction { }) } - public newJSInstance(is_constructor = true, args:any[]|undefined, propertyInitializer:{[INIT_PROPS]:(instance:any)=>void}) { + public newJSInstance(is_constructor = true, args?:any[], propertyInitializer?:{[INIT_PROPS]:(instance:any)=>void}) { // create new instance - TODO 'this' as last constructor argument still required? - Type.#current_constructor = this.interface_config?.class; - const instance = (this.interface_config?.class ? Reflect.construct(Type.#current_constructor, is_constructor?[...args]:[propertyInitializer]) : {[DX_TYPE]: this}); + Type.#current_constructor = this.interface_config?.class??null; + const instance = (this.interface_config?.class ? Reflect.construct(Type.#current_constructor, is_constructor?[...args]:(propertyInitializer ? [propertyInitializer] : [])) : {[DX_TYPE]: this}); Type.#current_constructor = null; return instance; } @@ -380,7 +381,7 @@ export class Type extends ExtensibleFunction { // never call the constructor directly!! should be private constructor(namespace?:string, name?:string, variation?:string, parameters?:any[]) { - super(namespace && namespace != "std" ? (val:any) => this.cast(val) : undefined) + super(namespace && namespace != "std" ? (val:any) => this.cast(val, undefined, undefined, true) : undefined) if (name) this.name = name; if (namespace) this.namespace = namespace; if (variation) this.variation = variation; @@ -391,7 +392,7 @@ export class Type extends ExtensibleFunction { this.is_primitive = namespace=="std" && Type.primitive_types.includes(this.name); this.is_complex = namespace!="std" || !Type.fundamental_types.includes(this.name); - this.is_js_pseudo_primitive = namespace=="std" && Type.pseudo_js_primitives.includes(this.name); + this.is_js_pseudo_primitive = (namespace=="std"||namespace=="js") && Type.pseudo_js_primitives.includes(this.name); this.has_compact_rep = namespace=="std" && (this.is_primitive || Type.compact_rep_types.includes(this.name)); this.serializable_not_complex = Type.serializable_not_complex_types.includes(this.name); @@ -814,7 +815,8 @@ export class Type extends ExtensibleFunction { if (typeof value == "bigint") return >Type.std.integer; if (typeof value == "number") return >Type.std.decimal; if (typeof value == "boolean") return >Type.std.boolean; - if (typeof value == "symbol") return Type.js.Symbol; + if (typeof value == "symbol") return Type.js.Symbol as unknown as Type; + if (value instanceof RegExp) return Type.js.RegExp as unknown as Type; if (value instanceof ArrayBuffer || value instanceof TypedArray) return >Type.std.buffer; if (value instanceof Tuple) return >Type.std.Tuple; @@ -878,7 +880,7 @@ export class Type extends ExtensibleFunction { if (_forClass == Negation) return >Type.std.Negation; - let custom_type = JSInterface.getClassDatexType(_forClass); + const custom_type = JSInterface.getClassDatexType(_forClass); if (!custom_type) { @@ -888,6 +890,7 @@ export class Type extends ExtensibleFunction { if (_forClass == Number || Number.isPrototypeOf(_forClass)) return >Type.std.decimal; if (_forClass == globalThis.Boolean || globalThis.Boolean.isPrototypeOf(_forClass)) return >Type.std.boolean; if (_forClass == Symbol || Symbol.isPrototypeOf(_forClass)) return >Type.js.Symbol; + if (_forClass == RegExp || RegExp.isPrototypeOf(_forClass)) return Type.js.RegExp as unknown as Type; if (_forClass == WeakRef || WeakRef.isPrototypeOf(_forClass)) return >Type.std.WeakRef; if (_forClass == ArrayBuffer || TypedArray.isPrototypeOf(_forClass)) return >Type.std.buffer; @@ -948,7 +951,8 @@ export class Type extends ExtensibleFunction { static js = { NativeObject: Type.get("js:Object"), // special object type for non-plain objects (objects with prototype) - no automatic children pointer initialization TransferableFunction: Type.get("js:Function"), - Symbol: Type.get("js:Symbol") + Symbol: Type.get("js:Symbol"), + RegExp: Type.get("js:RegExp") } /** @@ -1025,6 +1029,8 @@ export class Type extends ExtensibleFunction { Assertion: Type.get("std:Assertion"), Iterator: Type.get>("std:Iterator"), + MatchCondition: Type.get>("std:MatchCondition"), + StorageMap: Type.get>("std:StorageMap"), StorageWeakMap: Type.get>("std:StorageWeakMap"), StorageSet: Type.get>("std:StorageSet"), diff --git a/utils/auto_map.ts b/utils/auto_map.ts index 127abc40..c4c2014f 100644 --- a/utils/auto_map.ts +++ b/utils/auto_map.ts @@ -9,8 +9,6 @@ const DEFAULT_CLASS_PRIMITIVE = Symbol('DEFAULT_CLASS_PRIMITIVE') const DEFAULT_CREATOR_FUNCTION = Symbol('DEFAULT_CREATOR_FUNCTION') const DEFAULT_VALUE = Symbol('DEFAULT_VALUE') -export const _ = "_"; - export type AutoMap = Map & { getAuto(key: K): V; enableAutoRemove(): AutoRemoveMap & AutoMap diff --git a/utils/interface-generator.ts b/utils/interface-generator.ts index fee70b73..4c1148c1 100644 --- a/utils/interface-generator.ts +++ b/utils/interface-generator.ts @@ -230,7 +230,6 @@ function getClassTSCode(name:string, interf: interf, no_pointer = false) { const meta_is_sync = metadata[Datex.Decorators.IS_SYNC]?.constructor; const meta_is_sealed = metadata[Datex.Decorators.IS_SEALED]?.constructor; const meta_timeout = metadata[Datex.Decorators.TIMEOUT]?.public; - const meta_meta_index = metadata[Datex.Decorators.META_INDEX]?.public; let fields = ""; diff --git a/utils/local_files.ts b/utils/local_files.ts index 32dbf32e..f3523110 100644 --- a/utils/local_files.ts +++ b/utils/local_files.ts @@ -1,6 +1,6 @@ import { DATEX_FILE_TYPE, FILE_TYPE } from "../compiler/compiler.ts"; import { Runtime } from "../runtime/runtime.ts"; -import { decompile } from "../wasm/adapter/pkg/datex_wasm.js"; +import wasm_init, { decompile } from "../wasm/adapter/pkg/datex_wasm.js"; export async function uploadDatexFile(){ const pickerOpts = { @@ -30,6 +30,8 @@ export async function uploadDatexFile(){ export type datex_file_data = {type: DATEX_FILE_TYPE, text:string, binary?:ArrayBuffer, fileHandle:any}; export async function getDatexContentFromFileHandle(fileHandle:any) { + // init wasm + await wasm_init(); const fileData = await fileHandle.getFile(); const data:datex_file_data = { diff --git a/utils/match.ts b/utils/match.ts new file mode 100644 index 00000000..fffbea65 --- /dev/null +++ b/utils/match.ts @@ -0,0 +1,78 @@ +import { StorageSet } from "../types/storage_set.ts"; +import { Type } from "../types/type.ts"; +import type { Class } from "./global_types.ts"; +import { MatchInput, MatchResult, MatchOptions, Storage } from "../storage/storage.ts"; + +export type { MatchInput, MatchOptions, MatchResult } from "../storage/storage.ts"; + +/** + * Returns all entries of a StorageSet that match the given match descriptor. + * @param inputSet + * @param match + * @param limit + * @returns + */ +export async function match(inputSet: StorageSet, valueType:Class|Type, match: MatchInput, options?: Options): Promise> { + options ??= {} as Options; + const found = new Set(); + const matchOrEntries = (match instanceof Array ? match : [match]).map(m => Object.entries(m)) as [keyof T, T[keyof T]][][]; + + if (!(valueType instanceof Type)) valueType = Type.getClassDatexType(valueType); + + // match queries supported + if (await Storage.supportsMatchQueries(valueType)) { + return Storage.itemMatchQuery(inputSet._prefix, valueType, match, options); + } + + // fallback: match by iterating over all entries + // TODO: implement full match query support + + for await (const input of inputSet) { + // ors + for (const matchOrs of matchOrEntries) { + let isMatch = true; + for (const [key, value] of matchOrs) { + if (!_match(input[key], value)) { + isMatch = false; + break; + } + } + if (isMatch) found.add(input); + if (found.size >= (options.limit??Infinity)) break; + } + + } + return found as MatchResult; +} + +function _match(value: unknown, match: unknown) { + const matchOrs = (match instanceof Array ? match : [match]); + + for (const matchEntry of matchOrs) { + let isMatch = true; + // nested object + if (value && typeof value == "object") { + // identical object + if (value === matchEntry) isMatch = true; + // nested match + else if (matchEntry && typeof matchEntry === "object") { + for (const [key, val] of Object.entries(matchEntry)) { + // an object entry does not match + if (!_match((value as any)[key], val)) { + isMatch = false; + break; + } + } + } + else isMatch = false; + } + // primitive, other value + else isMatch = value === matchEntry; + + // match? + if (isMatch) return true; + } + + // no match found + return false; +} \ No newline at end of file diff --git a/utils/message_logger.ts b/utils/message_logger.ts index dbb0650e..486a9873 100644 --- a/utils/message_logger.ts +++ b/utils/message_logger.ts @@ -5,7 +5,7 @@ import { Logical } from "../types/logic.ts"; import { Logger } from "./logger.ts"; // WASM -import {decompile as wasm_decompile} from "../wasm/adapter/pkg/datex_wasm.js"; +import wasm_init, {decompile as wasm_decompile} from "../wasm/adapter/pkg/datex_wasm.js"; import { console } from "./ansi_compat.ts"; import { ESCAPE_SEQUENCES } from "./logger.ts"; @@ -14,6 +14,7 @@ export class MessageLogger { static logger:Logger static decompile(dxb:ArrayBuffer, has_header = true, colorized = true, resolve_slots = true){ + if (!this.#initialized) return "[DATEX Decompiler not enabled]" try { // extract body (TODO: just temporary, rust impl does not yet support header decompilation) if (has_header) { @@ -28,7 +29,20 @@ export class MessageLogger { } } - static enable(showRedirectedMessages = true){ + static #initialized = false; + static async init() { + if (this.#initialized) return; + try { + await wasm_init() + } + catch (e) { + console.error(e) + } + this.#initialized = true; + } + + static async enable(showRedirectedMessages = true){ + await this.init(); IOHandler.resetDatexHandlers(); if (!this.logger) this.logger = new Logger("DATEX Message"); diff --git a/utils/weak-action.ts b/utils/weak-action.ts index e7de9fd7..395ce3d0 100644 --- a/utils/weak-action.ts +++ b/utils/weak-action.ts @@ -5,12 +5,13 @@ import { isolatedScope } from "./isolated-scope.ts"; * If one of the weak dependencies is garbage collected, an optional deinit function is called. * @param weakRefs * @param action an isolated callback function that provides weak references. External dependency variable must be explicitly added with use() - * @param deinit an isolated callback function that is callled on garbage collection. External dependency variable must be explicitly added with use() + * @param deinit an isolated callback function that is called on garbage collection. External dependency variable must be explicitly added with use() */ -export function weakAction, R>(weakDependencies: T, action: (values: {[K in keyof T]: WeakRef}) => R, deinit?: (actionResult: R, collectedVariable: keyof T) => unknown) { +export function weakAction, R, D extends Record|undefined>(weakDependencies: T, action: (values: {[K in keyof T]: WeakRef}) => R, deinit?: (actionResult: R, collectedVariable: keyof T, weakDeinitDependencies: D) => unknown, weakDeinitDependencies?: D) { const weakRefs = _getWeakRefs(weakDependencies); + const weakDeinitRefs = weakDeinitDependencies ? _getWeakRefs(weakDeinitDependencies) : undefined; - let result:R; + let result:R|WeakRef; action = isolatedScope(action); @@ -22,7 +23,26 @@ export function weakAction, R>(weakDependencie const deinitHandler = (k: string) => { registries.delete(registry) - deinitFn(result, k); + + // unwrap all deinit weak refs + const weakDeinitDeps = weakDeinitRefs && Object.fromEntries( + Object.entries(weakDeinitRefs).map(([k, v]) => [k, v.deref()]) + ) + // check if all deinit weak refs are still alive, otherwise return + if (weakDeinitDeps) { + for (const v of Object.values(weakDeinitDeps)) { + if (v === undefined) { + return; + } + } + } + + const unwrappedResult = result instanceof WeakRef ? result.deref() : result; + if (result instanceof WeakRef && unwrappedResult === undefined) { + return; + } + + deinitFn(unwrappedResult!, k, weakDeinitDeps as D); } const registry = new FinalizationRegistry(deinitHandler); registries.add(registry) @@ -39,7 +59,8 @@ export function weakAction, R>(weakDependencie } // call action once - result = action(weakRefs); + const actionResult = action(weakRefs); + result = (actionResult && (typeof actionResult === "object" || typeof actionResult == "function")) ? new WeakRef(actionResult) : actionResult; } function _getWeakRefs>(weakDependencies: T) { diff --git a/wasm/adapter/Cargo.lock b/wasm/adapter/Cargo.lock index c9927f04..3dc44a63 100644 --- a/wasm/adapter/Cargo.lock +++ b/wasm/adapter/Cargo.lock @@ -187,6 +187,7 @@ dependencies = [ "num-bigint", "num-integer", "num_enum", + "pad", "pest", "pest_derive", "regex", @@ -660,6 +661,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "pad" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ad9b889f1b12e0b9ee24db044b5129150d5eada288edc800f789928dc8c0e3" +dependencies = [ + "unicode-width", +] + [[package]] name = "parking_lot" version = "0.9.0" @@ -1314,6 +1324,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + [[package]] name = "url" version = "1.7.2" diff --git a/wasm/adapter/pkg/datex_wasm_bg.wasm b/wasm/adapter/pkg/datex_wasm_bg.wasm index 78c62bee..ea316a35 100644 Binary files a/wasm/adapter/pkg/datex_wasm_bg.wasm and b/wasm/adapter/pkg/datex_wasm_bg.wasm differ