From 41da7dc82221c2b1b25c21ae3ef7570cd3b8c9e9 Mon Sep 17 00:00:00 2001 From: benStre Date: Wed, 17 Jan 2024 16:37:27 +0100 Subject: [PATCH 01/38] fix URL spelling error --- types/type.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/type.ts b/types/type.ts index 44f92b45..ad1b9ab2 100644 --- a/types/type.ts +++ b/types/type.ts @@ -78,7 +78,7 @@ export class Type extends ExtensibleFunction { #proxify_children = false // proxify all (new) children of this type children_timeouts?: Map // individual timeouts for children - static #jsTypeDefModuleMapper?: (url:string|URLL, type: Type) => string|URL|undefined + static #jsTypeDefModuleMapper?: (url:string|URL, type: Type) => string|URL|undefined static setJSTypeDefModuleMapper(fn: (url:string|URL, type: Type) => string|URL|undefined) { this.#jsTypeDefModuleMapper = fn; From 8a6779aa267c7a7a6d6a9a2c292468d46b6ad4f0 Mon Sep 17 00:00:00 2001 From: benStre Date: Wed, 17 Jan 2024 16:38:47 +0100 Subject: [PATCH 02/38] cleanup boolean expression --- compiler/compiler.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/compiler/compiler.ts b/compiler/compiler.ts index a03b3612..948467bb 100644 --- a/compiler/compiler.ts +++ b/compiler/compiler.ts @@ -2178,8 +2178,13 @@ export class Compiler { const pointer_origin = (id_buffer[0]==BinaryCode.ENDPOINT || id_buffer[0]==BinaryCode.PERSON_ALIAS || id_buffer[0]==BinaryCode.INSTITUTION_ALIAS) ? Target.get(id_buffer.slice(1,19), id_buffer.slice(19,21), id_buffer[0]) : null; const singleReceiver = - (SCOPE.options.to instanceof Endpoint && SCOPE.options.to) || - (SCOPE.options.to instanceof Disjunction && SCOPE.options.to.size == 1 && [...SCOPE.options.to][0] instanceof Endpoint && [...SCOPE.options.to][0]) + SCOPE.options.to instanceof Endpoint || + ( + SCOPE.options.to instanceof Disjunction && + SCOPE.options.to.size == 1 && + [...SCOPE.options.to][0] instanceof Endpoint && + [...SCOPE.options.to][0] + ) if ( pointer_origin && From eab61bf1493747c76108974760e7dd426a994eb1 Mon Sep 17 00:00:00 2001 From: benStre Date: Wed, 17 Jan 2024 16:45:24 +0100 Subject: [PATCH 03/38] refactor cleanupSubscribers --- runtime/pointers.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/runtime/pointers.ts b/runtime/pointers.ts index 17d486bd..5b7eb8e5 100644 --- a/runtime/pointers.ts +++ b/runtime/pointers.ts @@ -3055,17 +3055,12 @@ export class Pointer extends Ref { */ public static async cleanupSubscribers() { logger.debug("cleaning up subscribers"); - let removeCount = 0; - for (const [endpoint, pointers] of Pointer.#endpoint_subscriptions) { - if (await endpoint.isOnline()) continue; - for (const pointer of pointers) { - pointer.removeSubscriber(endpoint); - removeCount++; + for (const endpoint of Pointer.#endpoint_subscriptions.keys()) { + if (!(await endpoint.isOnline())) { + this.clearEndpointSubscriptions(endpoint); } } - - logger.debug("removed " + removeCount + " subscriptions"); } /** From 02197e4eb76b0c61922e40f8292034125cb953cc Mon Sep 17 00:00:00 2001 From: benStre Date: Wed, 17 Jan 2024 17:31:07 +0100 Subject: [PATCH 04/38] refactor type normalization, @sync and @type --- js_adapter/js_class_adapter.ts | 52 +++++++++++++++++++++------------ js_adapter/legacy_decorators.ts | 6 ++-- 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/js_adapter/js_class_adapter.ts b/js_adapter/js_class_adapter.ts index 150d06d7..f279ab8b 100644 --- a/js_adapter/js_class_adapter.ts +++ b/js_adapter/js_class_adapter.ts @@ -384,14 +384,13 @@ export class Decorators { let type: Type; // get template type - if (typeof params[0] == "string") { - const typeString = params[0].replace(/^\$/,'') - type = Type.get(typeString.includes(":") ? typeString : "ext:"+typeString) + if (typeof params[0] == "string" || params[0] instanceof Type) { + type = normalizeType(params[0], false, "ext"); } - else if (params[0] instanceof Type) type = params[0]; else if (original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor) type = original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor else type = Type.get("ext", original_class.name); + // return new templated class return createTemplateClass(original_class, type); } @@ -417,11 +416,9 @@ export class Decorators { let type: Type; // get template type - if (typeof params[0] == "string") { - const typeString = params[0].replace(/^\$/,'') - type = Type.get(typeString.includes(":") ? typeString : "ext:"+typeString) + if (typeof params[0] == "string" || params[0] instanceof Type) { + type = normalizeType(params[0], false, "ext"); } - else if (params[0] instanceof Type) type = params[0]; else if (original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor) type = original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor else type = Type.get("ext", original_class.name); @@ -488,15 +485,9 @@ export class Decorators { /** @type(type:string|DatexType)/ (namespace:name) * sync class with type */ - static type(value:any, name:context_name, kind:context_kind, is_static:boolean, is_private:boolean, setMetadata:context_meta_setter, getMetadata:context_meta_getter, params:[(string|Type)?] = []) { - - // handle decorator - if (typeof params[0] == "string") { - const [typeName, paramsString] = params[0].replace(/^\$/,'').match(/^(\w*)(?:\((.*)\))?$/)?.slice(1) ?? []; - const parsedParams = paramsString ? JSON.parse(`[${paramsString}]`) : undefined; - setMetadata(Decorators.FORCE_TYPE, Type.get(typeName, parsedParams)) - } - else if (params[0] instanceof Type) setMetadata(Decorators.FORCE_TYPE, params[0]) + static type(value:any, name:context_name, kind:context_kind, is_static:boolean, is_private:boolean, setMetadata:context_meta_setter, getMetadata:context_meta_getter, params:[(string|Type)] = []) { + const type = normalizeType(params[0]); + setMetadata(Decorators.FORCE_TYPE, type) } /** @from(type:string|DatexType): sync class from type */ @@ -574,7 +565,32 @@ export class Decorators { } } -globalThis.Decorators = Decorators; +/** + * Converts strings into Datex.Type and checks if type parameters are allowed + * @param type + * @param allowTypeParams + * @returns + */ +function normalizeType(type:Type|string, allowTypeParams = true, defaultNamespace = "std") { + if (typeof type == "string") { + // extract type name and parameters + const [typeName, paramsString] = type.replace(/^\$/,'').match(/^((?:\w+\:)?\w*)(?:\((.*)\))?$/)?.slice(1) ?? []; + if (paramsString && !allowTypeParams) throw new Error(`Type parameters not allowed (${type})`); + + // TODO: only json-compatible params are allowed for now to avoid async + const parsedParams = paramsString ? JSON.parse(`[${paramsString}]`) : undefined; + return Type.get(typeName.includes(":") ? typeName : defaultNamespace+":"+typeName, parsedParams) + } + else if (type instanceof Type) { + if (!allowTypeParams && type.parameters?.length) throw new Error(`Type parameters not allowed (${type})`); + return type + } + else { + console.log(type) + throw new Error("Invalid type") + } +} + const initialized_static_scope_classes = new Map(); diff --git a/js_adapter/legacy_decorators.ts b/js_adapter/legacy_decorators.ts index 4e0d2755..6df45eb2 100644 --- a/js_adapter/legacy_decorators.ts +++ b/js_adapter/legacy_decorators.ts @@ -228,7 +228,7 @@ export function each(...args:any[]): any { return handleDecoratorArgs(args, Decorators.each); } -export function sync(type:string|Type):any +export function sync(type:string):any export function sync(target: any, name?: string, method?:any):any export function sync(...args:any[]): any { return handleDecoratorArgs(args, Decorators.sync); @@ -237,7 +237,7 @@ export function sync(...args:any[]): any { /** * @deprecated use \@sync */ -export function template(type:string|Type):any +export function template(type:string):any /** * @deprecated use \@sync */ @@ -288,7 +288,7 @@ export function anonymous(...args:any[]) { export function type(type:string|Type):any export function type(...args:any[]): any { - return handleDecoratorArgs(args, Decorators.type); + return handleDecoratorArgs(args, Decorators.type, true); } export function assert(assertion:(val:any)=>boolean|string|undefined|null):any From 6b0fe0a27935e6df207674b80a1d09c9cb3c0efa Mon Sep 17 00:00:00 2001 From: benStre Date: Wed, 17 Jan 2024 17:35:14 +0100 Subject: [PATCH 05/38] fix assertMatches to use Type.matches, enables correct handling of special types like --- types/type.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/types/type.ts b/types/type.ts index c4bd5c36..da123753 100644 --- a/types/type.ts +++ b/types/type.ts @@ -695,12 +695,12 @@ export class Type extends ExtensibleFunction { } public static assertMatches(value:RefOrValue, type:type_clause): asserts value is (T extends Type ? TT : any) { - const res = Type.matchesType(Type.ofValue(value), type, value, true); + const res = Type.matches(value, type, true); if (!res) throw new ValueError("Value must be of type " + type) } // check if root type of value matches exactly - public static matches(value:RefOrValue, type:type_clause): value is (T extends Type ? TT : any) { + public static matches(value:RefOrValue, type:type_clause, throwInvalidAssertion = false): value is (T extends Type ? TT : any) { value = Ref.collapseValue(value, true, true); // value has a matching DX_TEMPLATE if (type instanceof Type && type.template && value[DX_TEMPLATE] && this.matchesTemplate(value[DX_TEMPLATE], type.template)) return true; @@ -712,7 +712,7 @@ export class Type extends ExtensibleFunction { return value.length <= type.parameters[0]; } - return Type.matchesType(Type.ofValue(value), type, value); + return Type.matchesType(Type.ofValue(value), type, value, throwInvalidAssertion); } public static extends(type:Type, extends_type:type_clause){ From cac72c2965e5f532c61d109af763127e10033b58 Mon Sep 17 00:00:00 2001 From: benStre Date: Wed, 17 Jan 2024 19:39:59 +0100 Subject: [PATCH 06/38] add workflow --- .github/workflows/update-prs.js | 22 ++++++++++++++++++++++ .github/workflows/update-prs.yml | 14 ++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 .github/workflows/update-prs.js create mode 100644 .github/workflows/update-prs.yml diff --git a/.github/workflows/update-prs.js b/.github/workflows/update-prs.js new file mode 100644 index 00000000..7e8225aa --- /dev/null +++ b/.github/workflows/update-prs.js @@ -0,0 +1,22 @@ +const owner = 'your_owner'; +const repo = 'your_repo'; +const accessToken = 'your_access_token'; + +const apiUrl = `https://api.github.com/repos/${owner}/${repo}/pulls`; + +try { + const response = await fetch(apiUrl, { + headers: { + Authorization: `Bearer ${accessToken}`, + }, + }); + + if (!response.ok) { + throw new Error(`Failed to fetch pull requests: ${response.statusText}`); + } + + const data = await response.json(); + console.log(data); +} catch (error) { + console.error(error); +} \ No newline at end of file diff --git a/.github/workflows/update-prs.yml b/.github/workflows/update-prs.yml new file mode 100644 index 00000000..da5062b1 --- /dev/null +++ b/.github/workflows/update-prs.yml @@ -0,0 +1,14 @@ +on: + push: + + +jobs: + trigger-pr-update: + runs-on: ubuntu-latest + + steps: + - name: Setup Node.js + uses: actions/setup-node@v3 + + - name: Run JavaScript file + run: node ./update-prs.js From 1419d5419c23d79cd76b6ccbdc43826dcfcd7d19 Mon Sep 17 00:00:00 2001 From: benStre Date: Wed, 17 Jan 2024 19:46:54 +0100 Subject: [PATCH 07/38] add github-script --- .github/workflows/update-prs.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/update-prs.yml b/.github/workflows/update-prs.yml index da5062b1..7df927f5 100644 --- a/.github/workflows/update-prs.yml +++ b/.github/workflows/update-prs.yml @@ -7,8 +7,15 @@ jobs: runs-on: ubuntu-latest steps: - - name: Setup Node.js - uses: actions/setup-node@v3 - - - name: Run JavaScript file - run: node ./update-prs.js + - uses: actions/github-script@v7 + with: + script: | + // Get a list of all issues created by the PR opener + // See: https://octokit.github.io/rest.js/#pagination + const creator = context.payload.sender.login + const prs = github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo + }); + + console.log(prs) \ No newline at end of file From fa32b4696be5492f3cd945368b4847479e3d96dc Mon Sep 17 00:00:00 2001 From: benStre Date: Wed, 17 Jan 2024 19:50:03 +0100 Subject: [PATCH 08/38] add github-script --- .github/workflows/update-prs.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/update-prs.yml b/.github/workflows/update-prs.yml index 7df927f5..7a5fbdca 100644 --- a/.github/workflows/update-prs.yml +++ b/.github/workflows/update-prs.yml @@ -13,9 +13,12 @@ jobs: // Get a list of all issues created by the PR opener // See: https://octokit.github.io/rest.js/#pagination const creator = context.payload.sender.login - const prs = github.rest.pulls.get({ + const prs = gawait github.rest.request('GET /repos/{owner}/{repo}/pulls', { owner: context.repo.owner, repo: context.repo.repo - }); + headers: { + 'X-GitHub-Api-Version': '2022-11-28' + } + }) console.log(prs) \ No newline at end of file From 15c989810dce4f467bc22c6fd1171dd83ae3eb81 Mon Sep 17 00:00:00 2001 From: benStre Date: Wed, 17 Jan 2024 19:50:35 +0100 Subject: [PATCH 09/38] add github-script --- .github/workflows/update-prs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-prs.yml b/.github/workflows/update-prs.yml index 7a5fbdca..63ddbbc8 100644 --- a/.github/workflows/update-prs.yml +++ b/.github/workflows/update-prs.yml @@ -13,7 +13,7 @@ jobs: // Get a list of all issues created by the PR opener // See: https://octokit.github.io/rest.js/#pagination const creator = context.payload.sender.login - const prs = gawait github.rest.request('GET /repos/{owner}/{repo}/pulls', { + const prs = await github.rest.request('GET /repos/{owner}/{repo}/pulls', { owner: context.repo.owner, repo: context.repo.repo headers: { From db7396ba95c98a9a1f617a1e1fd55acbbf70b415 Mon Sep 17 00:00:00 2001 From: benStre Date: Wed, 17 Jan 2024 19:51:03 +0100 Subject: [PATCH 10/38] add github-script --- .github/workflows/update-prs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-prs.yml b/.github/workflows/update-prs.yml index 63ddbbc8..ec3c981f 100644 --- a/.github/workflows/update-prs.yml +++ b/.github/workflows/update-prs.yml @@ -15,7 +15,7 @@ jobs: const creator = context.payload.sender.login const prs = await github.rest.request('GET /repos/{owner}/{repo}/pulls', { owner: context.repo.owner, - repo: context.repo.repo + repo: context.repo.repo, headers: { 'X-GitHub-Api-Version': '2022-11-28' } From e95477fe2c9a8094b533b21ade3d03d0c5b80425 Mon Sep 17 00:00:00 2001 From: benStre Date: Wed, 17 Jan 2024 19:52:52 +0100 Subject: [PATCH 11/38] add github-script --- .github/workflows/update-prs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-prs.yml b/.github/workflows/update-prs.yml index ec3c981f..1c91ac4c 100644 --- a/.github/workflows/update-prs.yml +++ b/.github/workflows/update-prs.yml @@ -13,7 +13,7 @@ jobs: // Get a list of all issues created by the PR opener // See: https://octokit.github.io/rest.js/#pagination const creator = context.payload.sender.login - const prs = await github.rest.request('GET /repos/{owner}/{repo}/pulls', { + const prs = await github.request('GET /repos/{owner}/{repo}/pulls', { owner: context.repo.owner, repo: context.repo.repo, headers: { From 96621d9e86ed0dea264b16e91ade0a98cf3bf4df Mon Sep 17 00:00:00 2001 From: benStre Date: Wed, 17 Jan 2024 19:56:19 +0100 Subject: [PATCH 12/38] add github-script --- .github/workflows/update-prs.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/update-prs.yml b/.github/workflows/update-prs.yml index 1c91ac4c..7e2d1877 100644 --- a/.github/workflows/update-prs.yml +++ b/.github/workflows/update-prs.yml @@ -20,5 +20,17 @@ jobs: 'X-GitHub-Api-Version': '2022-11-28' } }) - - console.log(prs) \ No newline at end of file + + console.log(prs); + for (const pr of prs.data) { + await github.request('PATCH /repos/{owner}/{repo}/pulls/{pull_number}', { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pr.number, + base: 'develop', + headers: { + 'X-GitHub-Api-Version': '2022-11-28' + } + }) + break; + } From 86983595b06ceb436b097021f760437d207a6b2a Mon Sep 17 00:00:00 2001 From: benStre Date: Wed, 17 Jan 2024 19:57:24 +0100 Subject: [PATCH 13/38] update permissions --- .github/workflows/update-prs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/update-prs.yml b/.github/workflows/update-prs.yml index 7e2d1877..94ec6e3d 100644 --- a/.github/workflows/update-prs.yml +++ b/.github/workflows/update-prs.yml @@ -1,6 +1,8 @@ on: push: +permissions: + pull-requests: write jobs: trigger-pr-update: From d0b18060fc52ab8023e5df09703e8d4b2577a5cb Mon Sep 17 00:00:00 2001 From: benStre Date: Wed, 17 Jan 2024 19:59:57 +0100 Subject: [PATCH 14/38] update github-script --- .github/workflows/update-prs.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/update-prs.yml b/.github/workflows/update-prs.yml index 94ec6e3d..52fad32b 100644 --- a/.github/workflows/update-prs.yml +++ b/.github/workflows/update-prs.yml @@ -23,8 +23,9 @@ jobs: } }) - console.log(prs); for (const pr of prs.data) { + if (pr.state !== 'open') continue; + console.log("Refreshing PR diff for #" + pr.number + " (" + pr.title + ")"); await github.request('PATCH /repos/{owner}/{repo}/pulls/{pull_number}', { owner: context.repo.owner, repo: context.repo.repo, @@ -34,5 +35,13 @@ jobs: 'X-GitHub-Api-Version': '2022-11-28' } }) - break; + await github.request('PATCH /repos/{owner}/{repo}/pulls/{pull_number}', { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pr.number, + base: 'main', + headers: { + 'X-GitHub-Api-Version': '2022-11-28' + } + }) } From 9ea160ec93c6c54b91c2d496a5f51f73c7bd1b9a Mon Sep 17 00:00:00 2001 From: benStre Date: Wed, 17 Jan 2024 20:01:49 +0100 Subject: [PATCH 15/38] trigger on push to main --- .github/workflows/update-prs.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/update-prs.yml b/.github/workflows/update-prs.yml index 52fad32b..deaa4199 100644 --- a/.github/workflows/update-prs.yml +++ b/.github/workflows/update-prs.yml @@ -1,5 +1,8 @@ +name: Refresh PRs on: push: + branches: + - main permissions: pull-requests: write From 8dec545b5ee8b4ea23e963b2b49e142622b92dd8 Mon Sep 17 00:00:00 2001 From: benStre Date: Wed, 17 Jan 2024 20:04:58 +0100 Subject: [PATCH 16/38] remove js file --- .github/workflows/update-prs.js | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 .github/workflows/update-prs.js diff --git a/.github/workflows/update-prs.js b/.github/workflows/update-prs.js deleted file mode 100644 index 7e8225aa..00000000 --- a/.github/workflows/update-prs.js +++ /dev/null @@ -1,22 +0,0 @@ -const owner = 'your_owner'; -const repo = 'your_repo'; -const accessToken = 'your_access_token'; - -const apiUrl = `https://api.github.com/repos/${owner}/${repo}/pulls`; - -try { - const response = await fetch(apiUrl, { - headers: { - Authorization: `Bearer ${accessToken}`, - }, - }); - - if (!response.ok) { - throw new Error(`Failed to fetch pull requests: ${response.statusText}`); - } - - const data = await response.json(); - console.log(data); -} catch (error) { - console.error(error); -} \ No newline at end of file From ab54a05753c49dfac33b09e02420ea71c6a449e4 Mon Sep 17 00:00:00 2001 From: benStre Date: Wed, 17 Jan 2024 23:37:53 +0100 Subject: [PATCH 17/38] debug --- runtime/pointers.ts | 1 + runtime/storage.ts | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/pointers.ts b/runtime/pointers.ts index 17d486bd..35017b0c 100644 --- a/runtime/pointers.ts +++ b/runtime/pointers.ts @@ -1506,6 +1506,7 @@ export class Pointer extends Ref { // check read permissions pointer.assertEndpointCanRead(SCOPE?.sender) + return pointer; } diff --git a/runtime/storage.ts b/runtime/storage.ts index 43f71230..ce4c8f72 100644 --- a/runtime/storage.ts +++ b/runtime/storage.ts @@ -407,7 +407,6 @@ export class Storage { } public static setPointer(pointer:Pointer, listen_for_changes = true, location:StorageLocation|undefined = this.#primary_location, partialUpdateKey: unknown = NOT_EXISTING): Promise|boolean { - if (!pointer.value_initialized) { // logger.warn("pointer value " + pointer.idString() + " not available, cannot save in storage"); this.#storage_active_pointers.delete(pointer); From ca5d69d2486e38ec9300a8589cca522af62d3be9 Mon Sep 17 00:00:00 2001 From: benStre Date: Wed, 17 Jan 2024 23:57:41 +0100 Subject: [PATCH 18/38] disable function constructor if namespace is std --- types/function-utils.ts | 4 ++-- types/type.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/types/function-utils.ts b/types/function-utils.ts index 6e61a8f7..ab888f49 100644 --- a/types/function-utils.ts +++ b/types/function-utils.ts @@ -204,8 +204,8 @@ export function createFunctionWithDependencyInjections(source: string, dependenc } export class ExtensibleFunction { - constructor(f:globalThis.Function) { - return Object.setPrototypeOf(f, new.target.prototype); + constructor(f?:globalThis.Function) { + if (f) return Object.setPrototypeOf(f, new.target.prototype); } } diff --git a/types/type.ts b/types/type.ts index da123753..0824b195 100644 --- a/types/type.ts +++ b/types/type.ts @@ -367,7 +367,7 @@ export class Type extends ExtensibleFunction { // never call the constructor directly!! should be private constructor(namespace?:string, name?:string, variation?:string, parameters?:any[]) { - super((val:any) => this.cast(val)) + super(namespace && namespace != "std" ? (val:any) => this.cast(val) : undefined) if (name) this.name = name; if (namespace) this.namespace = namespace; if (variation) this.variation = variation; From 9fae0d6a457bf958d7d021589fc88b16cbb4eaa5 Mon Sep 17 00:00:00 2001 From: benStre Date: Thu, 18 Jan 2024 00:40:08 +0100 Subject: [PATCH 19/38] add debug message for potentially undefined header --- runtime/runtime.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/runtime/runtime.ts b/runtime/runtime.ts index 9fc00b39..fda166ec 100644 --- a/runtime/runtime.ts +++ b/runtime/runtime.ts @@ -1806,6 +1806,10 @@ export class Runtime { * @param header DXB header of incoming message */ private static updateEndpointOnlineState(header: dxb_header) { + if (!header) { + logger.error("updateEndpointOnlineState: no header provided"); + return; + } if (header.sender) { // received signed GOODBYE message -> endpoint is offline if (header.type == ProtocolDataType.GOODBYE) { From 33afb677e793427cb103a50a23ec71543a97d154 Mon Sep 17 00:00:00 2001 From: benStre Date: Thu, 18 Jan 2024 14:06:44 +0100 Subject: [PATCH 20/38] fix storagemap order --- types/type.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/types/type.ts b/types/type.ts index 0824b195..72ea2c0f 100644 --- a/types/type.ts +++ b/types/type.ts @@ -1081,15 +1081,15 @@ Type.std.Assertion.setJSInterface({ }) -Type.std.StorageMap.setJSInterface({ - class: StorageMap, +Type.std.StorageWeakMap.setJSInterface({ + class: StorageWeakMap, is_normal_object: true, proxify_children: true, visible_children: new Set(), }) -Type.std.StorageWeakMap.setJSInterface({ - class: StorageWeakMap, +Type.std.StorageMap.setJSInterface({ + class: StorageMap, is_normal_object: true, proxify_children: true, visible_children: new Set(), From 3787f4e937183e74591efd407a6b7176d66141a9 Mon Sep 17 00:00:00 2001 From: benStre Date: Thu, 18 Jan 2024 14:07:07 +0100 Subject: [PATCH 21/38] rename pointer error message --- runtime/pointers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/pointers.ts b/runtime/pointers.ts index 35017b0c..f8ab7b3d 100644 --- a/runtime/pointers.ts +++ b/runtime/pointers.ts @@ -1497,7 +1497,7 @@ export class Pointer extends Ref { // else if (!allow_failure) displayFatalError('pointer-not-found'); pointer.delete(); - throw new PointerError("Pointer $"+id_string+" has no assigned value", SCOPE); + throw new PointerError("Pointer $"+id_string+" does not exist", SCOPE); } } From f3de7e0919f40b1f433a27930d2a06959c4488ed Mon Sep 17 00:00:00 2001 From: benStre Date: Thu, 18 Jan 2024 14:35:16 +0100 Subject: [PATCH 22/38] record dependency pointers for setItem --- runtime/storage-locations/deno-kv.ts | 7 +- runtime/storage-locations/indexed-db.ts | 7 +- runtime/storage-locations/local-storage.ts | 7 +- runtime/storage-locations/sql-db.ts | 2 +- runtime/storage.ts | 84 ++++++++++++---------- 5 files changed, 60 insertions(+), 47 deletions(-) diff --git a/runtime/storage-locations/deno-kv.ts b/runtime/storage-locations/deno-kv.ts index ad9e9eab..9071dda3 100644 --- a/runtime/storage-locations/deno-kv.ts +++ b/runtime/storage-locations/deno-kv.ts @@ -34,9 +34,10 @@ export class DenoKVStorageLocation extends AsyncStorageLocation { return client_type == "deno" && !!globalThis.Deno?.openKv; } - async setItem(key: string, value: unknown): Promise { - await this.set(itemDB!, key, Compiler.encodeValue(value)); - return true; + async setItem(key: string, value: unknown) { + const inserted_ptrs = new Set(); + await this.set(itemDB!, key, Compiler.encodeValue(value, inserted_ptrs)); + return inserted_ptrs; } async getItem(key: string, conditions?: ExecConditions): Promise { const result = await this.get(itemDB!, key); diff --git a/runtime/storage-locations/indexed-db.ts b/runtime/storage-locations/indexed-db.ts index aea89a1a..4cdb4265 100644 --- a/runtime/storage-locations/indexed-db.ts +++ b/runtime/storage-locations/indexed-db.ts @@ -23,9 +23,10 @@ export class IndexedDBStorageLocation extends AsyncStorageLocation { return !!globalThis.indexedDB; } - async setItem(key: string,value: unknown): Promise { - await datex_item_storage.setItem(key, Compiler.encodeValue(value)); // value to buffer (no header) - return true; + async setItem(key: string,value: unknown) { + const inserted_ptrs = new Set(); + await datex_item_storage.setItem(key, Compiler.encodeValue(value, inserted_ptrs)); + return inserted_ptrs; } async getItem(key: string, conditions: ExecConditions): Promise { const buffer = await datex_item_storage.getItem(key); diff --git a/runtime/storage-locations/local-storage.ts b/runtime/storage-locations/local-storage.ts index 48c911bb..e16b6738 100644 --- a/runtime/storage-locations/local-storage.ts +++ b/runtime/storage-locations/local-storage.ts @@ -22,9 +22,10 @@ export class LocalStorageLocation extends SyncStorageLocation { if (!isExit && localStorage.saveFile) localStorage.saveFile(); // deno local storage, save file afer save on exit or interval } - setItem(key: string, value: unknown): boolean { - localStorage.setItem(Storage.item_prefix+key, Compiler.encodeValueBase64(value)) - return true; + setItem(key: string, value: unknown) { + const inserted_ptrs = new Set(); + localStorage.setItem(Storage.item_prefix+key, Compiler.encodeValueBase64(value, inserted_ptrs)); // serialized pointer + return inserted_ptrs; } getItem(key: string, conditions?: ExecConditions) { diff --git a/runtime/storage-locations/sql-db.ts b/runtime/storage-locations/sql-db.ts index 7efba8f1..d39125c5 100644 --- a/runtime/storage-locations/sql-db.ts +++ b/runtime/storage-locations/sql-db.ts @@ -273,7 +273,7 @@ export class SQLDBStorageLocation extends AsyncStorageLocation { return client_type === "deno"; } - async setItem(key: string,value: unknown): Promise { + async setItem(key: string,value: unknown) { } async getItem(key: string): Promise { diff --git a/runtime/storage.ts b/runtime/storage.ts index ce4c8f72..200d81ab 100644 --- a/runtime/storage.ts +++ b/runtime/storage.ts @@ -39,7 +39,7 @@ export interface StorageLocation|boolean + setItem(key:string, value:unknown): Promise>|Set getItem(key:string, conditions?:ExecConditions): Promise|unknown hasItem(key:string):Promise|boolean removeItem(key:string): Promise|void @@ -65,7 +65,7 @@ export abstract class SyncStorageLocation implements StorageLocation abstract getItem(key:string, conditions?:ExecConditions): Promise|unknown abstract hasItem(key:string): boolean abstract getItemKeys(): Generator @@ -74,7 +74,7 @@ export abstract class SyncStorageLocation implements StorageLocation, partialUpdateKey: unknown|typeof NOT_EXISTING): Set> + abstract setPointer(pointer: Pointer, partialUpdateKey: unknown|typeof NOT_EXISTING): Set abstract getPointerValue(pointerId: string, outer_serialized:boolean, conditions?:ExecConditions): unknown abstract getPointerIds(): Generator @@ -93,7 +93,7 @@ export abstract class AsyncStorageLocation implements StorageLocation + abstract setItem(key: string,value: unknown): Promise> abstract getItem(key:string, conditions?:ExecConditions): Promise abstract hasItem(key:string): Promise abstract getItemKeys(): Promise> @@ -102,7 +102,7 @@ export abstract class AsyncStorageLocation implements StorageLocation abstract setItemValueDXB(key:string, value: ArrayBuffer):Promise - abstract setPointer(pointer: Pointer, partialUpdateKey: unknown|typeof NOT_EXISTING): Promise>> + abstract setPointer(pointer: Pointer, partialUpdateKey: unknown|typeof NOT_EXISTING): Promise> abstract getPointerValue(pointerId: string, outer_serialized:boolean, conditions?:ExecConditions): Promise abstract getPointerIds(): Promise> @@ -373,37 +373,27 @@ export class Storage { Storage.cache.set(key, value); // save in cache // cache deletion does not work, problems with storage item backup // setTimeout(()=>Storage.cache.delete(key), 10000); - const pointer = value instanceof Pointer ? value : Pointer.getByValue(value); if (location) { - if (location.isAsync) return this.setItemAsync(location as AsyncStorageLocation, key, value, pointer, listen_for_pointer_changes); - else return this.setItemSync(location as SyncStorageLocation, key, value, pointer, listen_for_pointer_changes); + if (location.isAsync) return this.setItemAsync(location as AsyncStorageLocation, key, value, listen_for_pointer_changes); + else return this.setItemSync(location as SyncStorageLocation, key, value, listen_for_pointer_changes); } else return false; } - static async setItemAsync(location:AsyncStorageLocation, key: string,value: unknown,pointer: Pointer|undefined,listen_for_pointer_changes: boolean): Promise { - this.setDirty(location, true) - // also store pointer - if (pointer) { - const res = await this.setPointer(pointer, listen_for_pointer_changes, location); - if (!res) return false; - } + static async setItemAsync(location:AsyncStorageLocation, key: string,value: unknown,listen_for_pointer_changes: boolean) { this.setDirty(location, true) // store value (might be pointer reference) - const res = await location.setItem(key, value); + const dependencies = await location.setItem(key, value); + await this.saveDependencyPointersAsync(dependencies, listen_for_pointer_changes, location); this.setDirty(location, false) - return res; + return true; } - static setItemSync(location:SyncStorageLocation, key: string,value: unknown,pointer: Pointer|undefined,listen_for_pointer_changes: boolean): boolean { - // also store pointer - if (pointer) { - const res = this.setPointer(pointer, listen_for_pointer_changes, location); - if (!res) return false; - } - - return location.setItem(key, value); + static setItemSync(location:SyncStorageLocation, key: string,value: unknown,listen_for_pointer_changes: boolean) { + const dependencies = location.setItem(key, value); + this.saveDependencyPointersSync(dependencies, listen_for_pointer_changes, location); + return true; } public static setPointer(pointer:Pointer, listen_for_changes = true, location:StorageLocation|undefined = this.#primary_location, partialUpdateKey: unknown = NOT_EXISTING): Promise|boolean { @@ -425,12 +415,8 @@ export class Storage { // if (pointer.transform_scope && this.hasPointer(pointer)) return true; // ignore transform pointer, initial transform scope already stored, does not change const dependencies = this.updatePointerSync(location, pointer, partialUpdateKey); - - // add required pointers for this pointer (only same-origin pointers) - for (const ptr of dependencies) { - // add if not yet in storage - if (ptr != pointer && /*ptr.is_origin &&*/ !localStorage.getItem(this.pointer_prefix+ptr.id)) this.setPointer(ptr, listen_for_changes, location) - } + dependencies.delete(pointer); + this.saveDependencyPointersSync(dependencies, listen_for_changes, location); // listen for changes if (listen_for_changes) this.syncPointer(pointer, location); @@ -448,12 +434,8 @@ export class Storage { // if (pointer.transform_scope && await this.hasPointer(pointer)) return true; // ignore transform pointer, initial transform scope already stored, does not change const dependencies = await this.updatePointerAsync(location, pointer, partialUpdateKey); - - // add required pointers for this pointer (only same-origin pointers) - for (const ptr of dependencies) { - // add if not yet in storage - if (ptr != pointer && /*ptr.is_origin &&*/ !await this.hasPointer(ptr)) await this.setPointer(ptr, listen_for_changes, location) - } + dependencies.delete(pointer); + await this.saveDependencyPointersAsync(dependencies, listen_for_changes, location); // listen for changes if (listen_for_changes) this.syncPointer(pointer, location); @@ -470,6 +452,34 @@ export class Storage { return res; } + /** + * Save dependency pointers to storage (SyncStorageLocation) + * Not all pointers in the set are saved, only those which are not yet in storage or not accessible in other ways + * @param dependencies List of dependency pointers + * @param listen_for_changes should update pointers in storage when value changes + * @param location storage location + */ + private static saveDependencyPointersSync(dependencies: Set, listen_for_changes = true, location: SyncStorageLocation) { + for (const ptr of dependencies) { + // add if not yet in storage + if (!location.hasPointer(ptr.id)) this.setPointer(ptr, listen_for_changes, location) + } + } + + /** + * Save dependency pointers to storage (AsyncStorageLocation) + * Not all pointers in the set are saved, only those which are not yet in storage or not accessible in other ways + * @param dependencies List of dependency pointers + * @param listen_for_changes should update pointers in storage when value changes + * @param location storage location + */ + private static async saveDependencyPointersAsync(dependencies: Set, listen_for_changes = true, location: AsyncStorageLocation) { + for (const ptr of dependencies) { + // add if not yet in storage + if (!await location.hasPointer(ptr.id)) await this.setPointer(ptr, listen_for_changes, location) + } + } + private static synced_pointers = new Set(); From 7dab1660e0d57d10546bfcd20e003829b8155f4d Mon Sep 17 00:00:00 2001 From: benStre Date: Thu, 18 Jan 2024 14:50:47 +0100 Subject: [PATCH 23/38] remove unnecessary check --- compiler/compiler.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/compiler/compiler.ts b/compiler/compiler.ts index 948467bb..f5ec7f52 100644 --- a/compiler/compiler.ts +++ b/compiler/compiler.ts @@ -2182,8 +2182,7 @@ export class Compiler { ( SCOPE.options.to instanceof Disjunction && SCOPE.options.to.size == 1 && - [...SCOPE.options.to][0] instanceof Endpoint && - [...SCOPE.options.to][0] + [...SCOPE.options.to][0] instanceof Endpoint ) if ( From a5dfaf8c5eb2d344a5e8320d5c394686f4a2e584 Mon Sep 17 00:00:00 2001 From: benStre Date: Thu, 18 Jan 2024 15:59:24 +0100 Subject: [PATCH 24/38] add distinction between arrow and normal functions for 'this' injection --- types/function-utils.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/types/function-utils.ts b/types/function-utils.ts index ab888f49..6b717aeb 100644 --- a/types/function-utils.ts +++ b/types/function-utils.ts @@ -182,6 +182,7 @@ export function createFunctionWithDependencyInjections(source: string, dependenc const hasThis = Object.keys(dependencies).includes('this'); const renamedVars = Object.keys(dependencies).filter(d => d!=='this').map(k=>'_'+k); const varMapping = renamedVars.map(k=>`const ${k.slice(1)} = ${allowValueMutations ? 'createStaticObject' : ''}(${k});`).join("\n"); + const isArrow = isArrowFunction(source); const createStaticFn = `function createStaticObject(val) { if (val && typeof val == "object" && !globalThis.Datex?.Ref.isRef(val)) { @@ -193,8 +194,12 @@ export function createFunctionWithDependencyInjections(source: string, dependenc try { let creatorFn = new Function(...renamedVars, `"use strict";${(varMapping&&allowValueMutations)?createStaticFn:''}${varMapping}; return (${source})`) - if (hasThis) creatorFn = creatorFn.bind(dependencies['this']) - return creatorFn(...Object.entries(dependencies).filter(([d]) => d!=='this').map(([_,v]) => v)); + // arrow function without own this context - bind creatorFn to this + if (hasThis && isArrow) creatorFn = creatorFn.bind(dependencies['this']) + const fn = creatorFn(...Object.entries(dependencies).filter(([d]) => d!=='this').map(([_,v]) => v)); + // normal function - bind directly to this + if (hasThis && !isArrow) return fn.bind(dependencies['this']) + else return fn; } catch (e) { console.error(source) From d513fcc89e8247732ca553187875a0aaeda17919 Mon Sep 17 00:00:00 2001 From: benStre Date: Thu, 18 Jan 2024 18:17:22 +0100 Subject: [PATCH 25/38] add ignoreHEllo --- network/client.ts | 14 +++++++++++++- runtime/pointers.ts | 1 + runtime/runtime.ts | 4 ++-- threads/worker-com-interface.ts | 1 + types/addressing.ts | 4 ++++ 5 files changed, 21 insertions(+), 3 deletions(-) diff --git a/network/client.ts b/network/client.ts index 0b04e23a..b7a80334 100644 --- a/network/client.ts +++ b/network/client.ts @@ -32,6 +32,7 @@ export interface ComInterface { endpoint?: Endpoint // connected directly to a single endpoint endpoints?: Set // multiple endpoints is_bidirectional_hub?: boolean, // allow the same block to go in and out eg a -> this interface -> this runtime -> this interface again -> b + immediate?: boolean // can send immediately (eg. for local interfaces, workers) isEqualSource?:(source: Partial, to: Endpoint) => boolean in: boolean // can receive data out: boolean // can send data @@ -162,6 +163,7 @@ export abstract class CommonInterface implements Co public in = true public out = true public global = true + public immediate = false public get endpoint() { return this._endpoint @@ -197,7 +199,17 @@ export abstract class CommonInterface implements Co this.initial_arguments = args; this.connected = await this.connect(); - if (this.connected) this.updateEndpoint(); + console.log("connect",this) + if (this.connected) { + this.updateEndpoint(); + // immediately consider endpoint as online + if (this.endpoint && this.immediate) { + console.warn("immediate online " + endpoint,this) + this.endpoint.setOnline(true) + // don't trigger online state change (to offline) once first HELLO message is received + this.endpoint.ignoreHello = true; + } + } return this.connected; } diff --git a/runtime/pointers.ts b/runtime/pointers.ts index f8ab7b3d..8ac24c98 100644 --- a/runtime/pointers.ts +++ b/runtime/pointers.ts @@ -3073,6 +3073,7 @@ export class Pointer extends Ref { * Removes all subscriptions for an endpoint */ public static clearEndpointSubscriptions(endpoint: Endpoint) { + console.warn("cler frpr " + endpoint) let removeCount = 0; for (const pointer of Pointer.#endpoint_subscriptions.get(endpoint) ?? []) { diff --git a/runtime/runtime.ts b/runtime/runtime.ts index fda166ec..b7d8d4c1 100644 --- a/runtime/runtime.ts +++ b/runtime/runtime.ts @@ -1824,9 +1824,9 @@ export class Runtime { } // other message, assume sender endpoint is online now else { + // HELLO message received, regard as new login to network, reset previous subscriptions + if (header.type == ProtocolDataType.HELLO && !header.sender.ignoreHello) Pointer.clearEndpointSubscriptions(header.sender) header.sender.setOnline(true) - // new login to network, reset previous subscriptions - if (header.type == ProtocolDataType.HELLO) Pointer.clearEndpointSubscriptions(header.sender) } } } diff --git a/threads/worker-com-interface.ts b/threads/worker-com-interface.ts index 83fcd9ed..74cf804b 100644 --- a/threads/worker-com-interface.ts +++ b/threads/worker-com-interface.ts @@ -12,6 +12,7 @@ export class WorkerCommunicationInterface extends CommonInterface<[Worker]> { override global = false; override authorization_required = false; // don't connect with public keys override type = "worker"; + override immediate = true; protected connect() { diff --git a/types/addressing.ts b/types/addressing.ts index 5d2396b1..5223ced7 100644 --- a/types/addressing.ts +++ b/types/addressing.ts @@ -473,6 +473,10 @@ export class Endpoint extends Target { setTimeout(() => this.#online=undefined, this.#current_online ? Endpoint.cache_life_online : Endpoint.cache_life_offline); } + /** + * Ignore HELLO messages from this endpoint in regards to online state + */ + public ignoreHello = false; // get endpoint from string public static fromString(string:string) { From 2a0fa18cbbe5916811fb528e3a3f5f7e665e7c80 Mon Sep 17 00:00:00 2001 From: benStre Date: Thu, 18 Jan 2024 18:20:23 +0100 Subject: [PATCH 26/38] update comments --- network/client.ts | 2 +- types/addressing.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/network/client.ts b/network/client.ts index b7a80334..348b503b 100644 --- a/network/client.ts +++ b/network/client.ts @@ -206,7 +206,7 @@ export abstract class CommonInterface implements Co if (this.endpoint && this.immediate) { console.warn("immediate online " + endpoint,this) this.endpoint.setOnline(true) - // don't trigger online state change (to offline) once first HELLO message is received + // don't trigger subscription cleanup once first HELLO message is received this.endpoint.ignoreHello = true; } } diff --git a/types/addressing.ts b/types/addressing.ts index 5223ced7..e31efd42 100644 --- a/types/addressing.ts +++ b/types/addressing.ts @@ -474,7 +474,7 @@ export class Endpoint extends Target { } /** - * Ignore HELLO messages from this endpoint in regards to online state + * Ignore HELLO messages from this endpoint (don't clean up subscriptions) */ public ignoreHello = false; From 71250f9e82b8583cc54e8bbdf77021974a71f89e Mon Sep 17 00:00:00 2001 From: benStre Date: Thu, 18 Jan 2024 18:24:37 +0100 Subject: [PATCH 27/38] remove debug logs --- network/client.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/network/client.ts b/network/client.ts index 348b503b..d5da3576 100644 --- a/network/client.ts +++ b/network/client.ts @@ -199,12 +199,10 @@ export abstract class CommonInterface implements Co this.initial_arguments = args; this.connected = await this.connect(); - console.log("connect",this) if (this.connected) { this.updateEndpoint(); // immediately consider endpoint as online if (this.endpoint && this.immediate) { - console.warn("immediate online " + endpoint,this) this.endpoint.setOnline(true) // don't trigger subscription cleanup once first HELLO message is received this.endpoint.ignoreHello = true; From 521ef9bc34e36783a4471bedc64f243691692710 Mon Sep 17 00:00:00 2001 From: benStre Date: Thu, 18 Jan 2024 18:25:49 +0100 Subject: [PATCH 28/38] remove debug logs --- runtime/pointers.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/runtime/pointers.ts b/runtime/pointers.ts index 8ac24c98..f8ab7b3d 100644 --- a/runtime/pointers.ts +++ b/runtime/pointers.ts @@ -3073,7 +3073,6 @@ export class Pointer extends Ref { * Removes all subscriptions for an endpoint */ public static clearEndpointSubscriptions(endpoint: Endpoint) { - console.warn("cler frpr " + endpoint) let removeCount = 0; for (const pointer of Pointer.#endpoint_subscriptions.get(endpoint) ?? []) { From 3b9d341f849bcb001dde6314496ff8e063f4ae86 Mon Sep 17 00:00:00 2001 From: benStre Date: Thu, 18 Jan 2024 18:28:37 +0100 Subject: [PATCH 29/38] updated thread worker log --- threads/thread-worker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/threads/thread-worker.ts b/threads/thread-worker.ts index a4840aa7..a6615985 100644 --- a/threads/thread-worker.ts +++ b/threads/thread-worker.ts @@ -3,7 +3,7 @@ import type { Datex as DatexType } from "../mod.ts"; const isServiceWorker = 'registration' in globalThis && (globalThis as any).registration instanceof ServiceWorkerRegistration; -console.log("initialized thread worker", {isServiceWorker}) +console.log("spawned new thread worker") if (isServiceWorker) { // https://developer.mozilla.org/en-US/docs/Web/API/Clients/claim From f8c50281d22503899f40d3589e855d34d94107b0 Mon Sep 17 00:00:00 2001 From: benStre Date: Thu, 18 Jan 2024 18:37:49 +0100 Subject: [PATCH 30/38] add dark/light theme inheritance for worker loggers --- threads/thread-worker.ts | 3 +++ threads/threads.ts | 7 ++++--- utils/logger.ts | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/threads/thread-worker.ts b/threads/thread-worker.ts index a6615985..9f3245b4 100644 --- a/threads/thread-worker.ts +++ b/threads/thread-worker.ts @@ -58,6 +58,9 @@ addEventListener("message", async function (event) { // await import("https://ga.jspm.io/npm:es-module-shims@1.8.0/dist/es-module-shims.wasm.js"); // if (data.importMap) importShim.addImportMap(data.importMap); + // inherit theme from parent + (globalThis as any)._override_console_theme = data.theme; + await initDatex(data.datexURL); await initWorkerComInterface(data.comInterfaceURL); await initTsInterfaceGenerator(data.tsInterfaceGeneratorURL); diff --git a/threads/threads.ts b/threads/threads.ts index 6e39ddb0..1ca2705e 100644 --- a/threads/threads.ts +++ b/threads/threads.ts @@ -1,4 +1,4 @@ -import { Logger } from "../utils/logger.ts"; +import { Logger, console_theme } from "../utils/logger.ts"; import "./worker-com-interface.ts"; import { Equals } from "../utils/global_types.ts"; @@ -23,7 +23,7 @@ export type ThreadPool = Recordvoid} export type MessageToWorker = - {type: "INIT", datexURL: string, comInterfaceURL: string, moduleURL: string, tsInterfaceGeneratorURL:string, endpoint: string, importMap:Record} | + {type: "INIT", datexURL: string, comInterfaceURL: string, moduleURL: string, tsInterfaceGeneratorURL:string, endpoint: string, importMap:Record, theme:"dark"|"light"} | {type: "INIT_PORT"} export type MessageFromWorker = @@ -503,7 +503,8 @@ export async function _initWorker(worker: Worker|ServiceWorkerRegistration, modu comInterfaceURL: import.meta.resolve("./worker-com-interface.ts"), tsInterfaceGeneratorURL: import.meta.resolve("../utils/interface-generator.ts"), moduleURL: modulePath ? import.meta.resolve(modulePath.toString()): null, - endpoint: Datex.Runtime.endpoint.toString() + endpoint: Datex.Runtime.endpoint.toString(), + theme: console_theme }); let resolve: Function; diff --git a/utils/logger.ts b/utils/logger.ts index ce4cfd63..ebb2aec7 100644 --- a/utils/logger.ts +++ b/utils/logger.ts @@ -127,7 +127,7 @@ const COLOR = { POINTER: [ESCAPE_SEQUENCES.BLUE, ESCAPE_SEQUENCES.UNYT_POINTER] as COLOR, } as const; -export let console_theme:"dark"|"light" = (client_type=="deno" || (globalThis).matchMedia && (globalThis).matchMedia('(prefers-color-scheme: dark)')?.matches) ? "dark" : "light"; +export let console_theme:"dark"|"light" = (globalThis as any)._override_console_theme ?? ((client_type=="deno" || (globalThis).matchMedia && (globalThis).matchMedia('(prefers-color-scheme: dark)')?.matches) ? "dark" : "light"); try { (globalThis).matchMedia && (globalThis).matchMedia('(prefers-color-scheme: dark)')?.addEventListener("change", (e:any)=>{ From 10b22169275680c51f7c8f1b428e263c13bb67e2 Mon Sep 17 00:00:00 2001 From: benStre Date: Thu, 18 Jan 2024 20:50:39 +0100 Subject: [PATCH 31/38] update workaround helper type --- js_adapter/js_class_adapter.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/js_adapter/js_class_adapter.ts b/js_adapter/js_class_adapter.ts index f279ab8b..07626fef 100644 --- a/js_adapter/js_class_adapter.ts +++ b/js_adapter/js_class_adapter.ts @@ -1198,7 +1198,9 @@ DatexFunction.setMethodMetaIndexSource(getMetaParamIndex) // new version for implemented feature functions / attributes: call datex_advanced() on the class (ideally usa as a decorator, currently not supported by ts) -interface DatexClass { +interface DatexClass unknown) = (new (...args: unknown[]) => unknown), Construct = InstanceType["construct"]> { + + new(...args: Construct extends (...args: any) => any ? Parameters : ConstructorParameters): datexClassType; // special functions on_result: (call: (data:any, meta:{station_id:number, station_bundle:number[]})=>any) => dc; @@ -1225,7 +1227,7 @@ type dc&{new (...args:unknown[]):unknown}> = DatexC * export type MyClass = datexClassType * ``` */ -export function datexClass&{new (...args:unknown[]):unknown}>(_class:T) { +export function datexClass&{new (...args:any[]):any}>(_class:T) { return >> _class; } From cac1ad49725373b934aac3b8ce42da5aa8cfe685 Mon Sep 17 00:00:00 2001 From: benStre Date: Thu, 18 Jan 2024 20:51:05 +0100 Subject: [PATCH 32/38] update docs --- docs/manual/12 Classes.md | 76 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/docs/manual/12 Classes.md b/docs/manual/12 Classes.md index 0370115a..5c296666 100644 --- a/docs/manual/12 Classes.md +++ b/docs/manual/12 Classes.md @@ -58,5 +58,81 @@ obj.b = 15 // triggers observer obj.sum // 26 ``` +## Constructors + +The normal JavaScript class constructor gets called every time an instance of a sync class is created. +When an existing instance of a sync class is shared with another endpoint, the constructor is +called again locally on the endpoint, which is not intended but can't be prevented. + +We recommend using DATEX-compatible constructors instead, which are only ever called once at the initial creation of a sync class instance. +The DATEX constructor method is named `construct` and must be decorated with `@constructor`: + +```ts +@sync class MyObject { + @property a = 0 + @property b = 0 + + // DATEX-compatible constructor + @constructor construct() { + console.log("constructed a new MyObject") + } +} + +const obj = new MyObject() // "constructed a new MyObject" is logged +``` + +When the `obj` pointer is now accessed on a remote endpoint, the `construct` method +is not called again on the remote endpoint. + +You can also access constructor arguments like in a normal constructor: +```ts +@sync class MyObject { + @constructor construct(a: number, b: string) { + console.log("a", a) + console.log("b", a) + } +} + +const obj = new MyObject(42, 'text') +``` + +For correct typing, take a look at [this workaround](#workaround-to-get-correct-types). + +## Creating instances without `new` + +Class instances can also be created by calling the class as a function, passing +in an object with the initial property values: + +```ts +@sync class MyObject { + @property a = 0 + @property b = 0 +} + +const obj = MyObject({a: 1, b: 2}) +``` + +Currently, this results in a TypeScript error, but it runs without problems. +You can use [this workaround](#workaround-to-get-correct-types) to get rid of the TypeScript errors. + + +## Workaround to get correct types + +Currently, it is not possible to get the correct types for a sync class without some additional work. +You can add the following lines to a sync class to make the TypeScript compiler happy (this has no effect on the runtime behavior): +```ts +// sync class definition (private) +@sync class _MyObject { + @property a = 0 + @property b = 0 +} +// use these as public proxies for the actual class +export const MyObject = datexClass(_MyObject) +export type MyObject = datexClassType + +const obj1: MyObject = new MyObject() +const obj2: MyObject = MyObject({a: 1, b: 2}) +``` + ## Using the raw API For more customization, you can directly use the [JavaScript interface API] which allows you to define custom DATEX mapping behaviours for specific JavaScript types. From be94a231e6ac77e96425aaa07f88ccd09c3838f9 Mon Sep 17 00:00:00 2001 From: benStre Date: Thu, 18 Jan 2024 20:51:23 +0100 Subject: [PATCH 33/38] change pointer error message (unrelated) --- runtime/pointers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/pointers.ts b/runtime/pointers.ts index f8ab7b3d..aefba010 100644 --- a/runtime/pointers.ts +++ b/runtime/pointers.ts @@ -1946,7 +1946,7 @@ export class Pointer extends Ref { && (!endpoint || !Logical.matches(endpoint, this.allowed_access, Target)) && (endpoint && !Runtime.trustedEndpoints.get(endpoint.main)?.includes("protected-pointer-access")) ) { - throw new PermissionError("Endpoint has no read permissions for this pointer") + throw new PermissionError("Endpoint has no read permissions for this pointer ("+this.idString()+")"); } } From fadd762a2daa82ea863738e753265a2e2f7b8d4e Mon Sep 17 00:00:00 2001 From: benStre Date: Thu, 18 Jan 2024 21:15:28 +0100 Subject: [PATCH 34/38] remove leading _ from class names --- js_adapter/js_class_adapter.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/js_adapter/js_class_adapter.ts b/js_adapter/js_class_adapter.ts index 07626fef..ea509f34 100644 --- a/js_adapter/js_class_adapter.ts +++ b/js_adapter/js_class_adapter.ts @@ -387,7 +387,7 @@ export class Decorators { if (typeof params[0] == "string" || params[0] instanceof Type) { type = normalizeType(params[0], false, "ext"); } - else if (original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor) type = original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor + else if (original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor) type = original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor.replace(/^_/, '') // remove leading _ from type name else type = Type.get("ext", original_class.name); @@ -419,7 +419,7 @@ export class Decorators { if (typeof params[0] == "string" || params[0] instanceof Type) { type = normalizeType(params[0], false, "ext"); } - else if (original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor) type = original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor + else if (original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor) type = original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor.replace(/^_/, '') // remove leading _ from type name else type = Type.get("ext", original_class.name); let callerFile:string|undefined; From c2356bdd768df8e2efa602621058b4d6ce4eff8e Mon Sep 17 00:00:00 2001 From: benStre Date: Thu, 18 Jan 2024 21:17:36 +0100 Subject: [PATCH 35/38] fix class name --- js_adapter/js_class_adapter.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/js_adapter/js_class_adapter.ts b/js_adapter/js_class_adapter.ts index ea509f34..c1a520d2 100644 --- a/js_adapter/js_class_adapter.ts +++ b/js_adapter/js_class_adapter.ts @@ -387,8 +387,8 @@ export class Decorators { if (typeof params[0] == "string" || params[0] instanceof Type) { type = normalizeType(params[0], false, "ext"); } - else if (original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor) type = original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor.replace(/^_/, '') // remove leading _ from type name - else type = Type.get("ext", original_class.name); + else if (original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor) type = original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor + else type = Type.get("ext", original_class.name.replace(/^_/, '')); // remove leading _ from type name // return new templated class @@ -419,8 +419,8 @@ export class Decorators { if (typeof params[0] == "string" || params[0] instanceof Type) { type = normalizeType(params[0], false, "ext"); } - else if (original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor) type = original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor.replace(/^_/, '') // remove leading _ from type name - else type = Type.get("ext", original_class.name); + else if (original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor) type = original_class[METADATA]?.[Decorators.FORCE_TYPE]?.constructor + else type = Type.get("ext", original_class.name.replace(/^_/, '')); // remove leading _ from type name let callerFile:string|undefined; From 768bc6d0e2fcc0034d58da98531527cd68a452d2 Mon Sep 17 00:00:00 2001 From: benStre Date: Fri, 19 Jan 2024 12:35:50 +0100 Subject: [PATCH 36/38] update dx in cache with dx from server --- runtime/endpoint_config.ts | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/runtime/endpoint_config.ts b/runtime/endpoint_config.ts index 03d066ad..cf520c23 100644 --- a/runtime/endpoint_config.ts +++ b/runtime/endpoint_config.ts @@ -94,21 +94,29 @@ class EndpointConfig implements EndpointConfigData { config = await Runtime.executeDatexLocally(serialized, undefined, undefined, globalThis.location?.href ? new URL(globalThis.location.href) : undefined) } // try to get from .dx url - else { - if (!path) path = new URL('/'+this.DX_FILE_NAME, globalThis.location.href) - try { - config = await datex.get(path); + if (!path) path = new URL('/'+this.DX_FILE_NAME, globalThis.location.href) + try { + const configUpdate = await datex.get(path); + if (!config) { + config = configUpdate; logger.info("loaded endpoint config from " + path); } - catch (e) { - // ignore if no .dx file found - if (!(await fetch(path)).ok) {} - else { - logger.error `Could not read config file ${path}: ${e.toString()}`; - throw "invalid config file" + else { + for (const [key, value] of DatexObject.entries(configUpdate as Record)) { + DatexObject.set(config as Record, key as string, value); } + logger.debug("updated endpoint config from " + path); + } + } + catch (e) { + // ignore if no .dx file found + if (!(await fetch(path)).ok) {} + else { + logger.error `Could not read config file ${path}: ${e.toString()}`; + throw "invalid config file" } } + } else { logger.debug("Cannot load endpoint config file for client type '" + client_type + "'") From 8d50e14a4ad21f0bda568c447e76071bc478cf3e Mon Sep 17 00:00:00 2001 From: benStre Date: Fri, 19 Jan 2024 15:00:47 +0100 Subject: [PATCH 37/38] fix typo in docs --- docs/manual/03 Pointers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/manual/03 Pointers.md b/docs/manual/03 Pointers.md index 2e30b51d..692e8f05 100644 --- a/docs/manual/03 Pointers.md +++ b/docs/manual/03 Pointers.md @@ -154,7 +154,7 @@ Read more about transform functions in the chapter [Functional Programming](./09 ## Using effects -With transform functions, value can be defined declaratively. +With transform functions, values can be defined declaratively. Still, there are some scenarios where the actual pointer value change event must be handled with custom logic. For this scenario, the `effect()` function can be used. From a3d478f6a0c44dbc72299d1c1207f901188950d2 Mon Sep 17 00:00:00 2001 From: benStre Date: Fri, 19 Jan 2024 15:12:55 +0100 Subject: [PATCH 38/38] add section 'Async effects' --- docs/manual/03 Pointers.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/manual/03 Pointers.md b/docs/manual/03 Pointers.md index 692e8f05..569a8b82 100644 --- a/docs/manual/03 Pointers.md +++ b/docs/manual/03 Pointers.md @@ -245,6 +245,32 @@ it is garbage colleted and the effect is removed. Weak value bindings can be used with all *object* values, not just with pointers. +### Async effects + +Effect callbacks cannot be `async` functions. +To handle async operations, you can always call an async function from inside the +effect callback: + +```ts +const searchName = $$(""); +const searchAge = $$(18); + +// async function that searches for a user and shows the result somewhere +async function searchUser(name: string, age: number) { + const user = await query({type: "user", name, age}); + showUser(user); +} + +// effect that triggers the user search every time searchName or searchAge is changed +effect(() => searchUser(searchName.val, searchAge.val)) +``` + +All dependency values of the effect must be accessed synchronously. +This means that the variables inside the async function don't trigger the effect, only the ones passed +into the `searchUser` call. + + + ## Observing pointer changes For more fine grained control, the `observe()` function can be used to handle pointer value updates.