Skip to content

Commit

Permalink
Merge pull request #61 from unyt-org/feat-lazy-pointer-properties
Browse files Browse the repository at this point in the history 
Add support for lazy pointer properties
  • Loading branch information
@benStre
benStre authored Jan 24, 2024
2 parents a135ede + f32651a commit 33cf4a9
Show file tree
Hide file tree
Showing 10 changed files with 157 additions and 49 deletions.
6 changes: 4 additions & 2 deletions compiler/compiler.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import { BinaryCode } from "./binary_codes.ts";
import { Scope } from "../types/scope.ts";
import { ProtocolDataType } from "./protocol_types.ts";
import { Quantity } from "../types/quantity.ts";
import { EXTENDED_OBJECTS, INHERITED_PROPERTIES, VOID, SLOT_WRITE, SLOT_READ, SLOT_EXEC, NOT_EXISTING, SLOT_GET, SLOT_SET, DX_IGNORE, DX_BOUND_LOCAL_SLOT } from "../runtime/constants.ts";
import { EXTENDED_OBJECTS, INHERITED_PROPERTIES, VOID, SLOT_WRITE, SLOT_READ, SLOT_EXEC, NOT_EXISTING, SLOT_GET, SLOT_SET, DX_IGNORE, DX_BOUND_LOCAL_SLOT, DX_REPLACE } from "../runtime/constants.ts";
import { arrayBufferToBase64, base64ToArrayBuffer, buffer2hex, hex2buffer } from "../utils/utils.ts";
import { RuntimePerformance } from "../runtime/performance_measure.ts";
import { Conjunction, Disjunction, Logical, Negation } from "../types/logic.ts";
Expand Down Expand Up @@ -2278,7 +2278,7 @@ export class Compiler {
}
// subscribing to own pointers is not allowed
// should not happen because pointers are not sent with preemptive loading to own endpoint anyway
if (Runtime.endpoint.equals(endpoint)) continue;
if (Runtime.endpoint.equals(endpoint) || endpoint.equals(LOCAL_ENDPOINT)) continue;
logger.debug("auto subscribing " + endpoint + " to " + ptr.idString());
ptr.addSubscriber(endpoint);
}
Expand Down Expand Up @@ -2744,6 +2744,8 @@ export class Compiler {

insert: (value:any, SCOPE:compiler_scope, is_root=true, parents?:Set<any>, unassigned_children?:[number, any, number][], add_insert_index = true) => {

if (value?.[DX_REPLACE]) value = value[DX_REPLACE];

// make sure normal pointers are collapsed (ignore error if uninitialized pointer is passed in)
try {
value = Ref.collapseValue(value);
Expand Down
4 changes: 3 additions & 1 deletion js_adapter/js_class_adapter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import { Error as DatexError, ValueError } from "../types/errors.ts";
import { Function as DatexFunction } from "../types/function.ts";
import { DatexObject } from "../types/object.ts";
import { Tuple } from "../types/tuple.ts";
import { DX_PERMISSIONS, DX_TYPE, INIT_PROPS } from "../runtime/constants.ts";
import { DX_PERMISSIONS, DX_TYPE, DX_ROOT, INIT_PROPS } from "../runtime/constants.ts";
import { type Class } from "../utils/global_types.ts";
import { Conjunction, Disjunction, Logical } from "../types/logic.ts";
import { client_type } from "../utils/constants.ts";
Expand Down Expand Up @@ -1087,6 +1087,8 @@ export function createTemplateClass(original_class:{ new(...args: any[]): any; }
DatexObject.extend(template, prototype[DX_TYPE].template);
break;
}
// is root of dx prototype chain, stop
if (prototype[DX_ROOT]) break;
}


Expand Down
2 changes: 2 additions & 0 deletions runtime/constants.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,10 @@ export const UNKNOWN_TYPE: unique symbol = Symbol("Unknown type") // return for

export const DX_PTR: unique symbol = Symbol("DX_PTR"); // key for pointer objects to access the respective DatexPointer
export const DX_TYPE: unique symbol = Symbol("DX_TYPE");
export const DX_ROOT: unique symbol = Symbol("DX_ROOT");
export const DX_SERIALIZED: unique symbol = Symbol("DX_SERIALIZED");
export const DX_VALUE: unique symbol = Symbol("DX_VALUE");
export const DX_REPLACE: unique symbol = Symbol("DX_REPLACE"); // value that is used as a replacement when serializing
export const DX_SOURCE: unique symbol = Symbol("DX_SOURCE"); // used to override the default loading behaviour for a pointer (fetching by id). Can be an arbitrary DATEX Script that can be resolved with datex.get. Currently only used by the interface generator for JS modules.
// TODO: remove? replaced with DX_SLOTS
export const DX_TEMPLATE: unique symbol = Symbol("DX_TEMPLATE");
Expand Down
8 changes: 4 additions & 4 deletions runtime/lazy-pointer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,12 @@ export class LazyPointer<T> {
return "Unresolved Pointer ($" + this.id + ")"
}

onLoad(callback:(val:MinimalJSRef<T>)=>void) {
Pointer.onPointerForIdAdded(this.id, p => callback(Pointer.collapseValue(p) as MinimalJSRef<T>))
onLoad(callback:(val:MinimalJSRef<T>, ref: Pointer<T>)=>void) {
Pointer.onPointerForIdAdded(this.id, p => callback(Pointer.collapseValue(p) as MinimalJSRef<T>, p))
}

static withVal(val:any, callback:(val:MinimalJSRef<any>)=>void) {
static withVal(val:any, callback:(val:MinimalJSRef<unknown>)=>void) {
if (val instanceof LazyPointer) val.onLoad(callback);
else callback(val);
else callback(val, val);
}
}
126 changes: 99 additions & 27 deletions runtime/pointers.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ export abstract class Ref<T = any> extends EventTarget {
// deno-lint-ignore no-this-alias
let pointer:Pointer = this;


// double pointer property..TODO: improve, currently tries to collapse current value
if (pointer instanceof PointerProperty) {
pointer = <Pointer>Pointer.getByValue(pointer.val);
Expand Down Expand Up @@ -554,11 +555,40 @@ export class PointerProperty<T=any> extends Ref<T> {

#leak_js_properties: boolean

private constructor(public pointer: Pointer, public key: any, leak_js_properties = false) {
public pointer?: Pointer;
private lazy_pointer?: LazyPointer<unknown>;

private constructor(pointer?: Pointer|LazyPointer<unknown>, public key: any, leak_js_properties = false) {
super();

if (pointer instanceof Pointer) this.setPointer(pointer);
else if (pointer instanceof LazyPointer) {
this.lazy_pointer = pointer;
this.lazy_pointer.onLoad((_, ptr) => {
this.lazy_pointer = undefined;
this.setPointer(ptr);
})
}
else throw new Error("Pointer or lazy pointer required")

this.#leak_js_properties = leak_js_properties;
pointer.is_persistent = true; // TODO: make unpersistent when pointer property deleted
PointerProperty.synced_pairs.get(pointer)!.set(key, this); // save in map
}

private setPointer(ptr: Pointer) {
this.pointer = ptr;
this.pointer.is_persistent = true; // TODO: make unpersistent when pointer property deleted
if (!PointerProperty.synced_pairs.has(ptr)) PointerProperty.synced_pairs.set(ptr, new Map());
PointerProperty.synced_pairs.get(ptr)!.set(this.key, this); // save in map
}

/**
* Called when the bound lazy pointer is loaded.
* If there is no lazy pointer, the callback is called immediately
* @param callback
*/
public onLoad(callback: (val:PointerProperty<T>, ref: PointerProperty<T>)=>void) {
if (this.lazy_pointer) this.lazy_pointer.onLoad(() => callback(this, this));
else callback(this, this);
}

private static synced_pairs = new WeakMap<Pointer, Map<any, PointerProperty>>()
Expand All @@ -571,50 +601,64 @@ export class PointerProperty<T=any> extends Ref<T> {
* @param leak_js_properties
* @returns
*/
public static get<const Key, Parent extends PointerPropertyParent<Key,unknown>>(parent: Parent|Pointer<Parent>, key: Key, leak_js_properties = false): PointerProperty<Parent extends Map<unknown, infer MV> ? MV : Parent[Key&keyof Parent]> {
console.warn("getpp",parent,key)
public static get<const Key, Parent extends PointerPropertyParent<Key,unknown>>(parent: Parent|Pointer<Parent>|LazyPointer<Parent>, key: Key, leak_js_properties = false): PointerProperty<Parent extends Map<unknown, infer MV> ? MV : Parent[Key&keyof Parent]> {
if (Pointer.isRef(key)) throw new Error("Cannot use a reference as a pointer property key");

let pointer:Pointer;
if (parent instanceof Pointer) pointer = parent;
else pointer = Pointer.createOrGet(parent);
const pointer = Pointer.createOrGetLazy(parent as any);

if (!this.synced_pairs.has(pointer)) this.synced_pairs.set(pointer, new Map());
if (pointer instanceof Pointer) {
if (!this.synced_pairs.has(pointer)) this.synced_pairs.set(pointer, new Map());
if (this.synced_pairs.get(pointer)!.has(key)) return this.synced_pairs.get(pointer)!.get(key)!;
}

if (this.synced_pairs.get(pointer)!.has(key)) return this.synced_pairs.get(pointer)!.get(key)!;
else return new PointerProperty(pointer, key, leak_js_properties);
return new PointerProperty(pointer, key, leak_js_properties);
}

// get current pointer property
public override get val():T {
// this.handleBeforePrimitiveValueGet();
const val = this.pointer.getProperty(this.key, this.#leak_js_properties);
if (this.pointer instanceof LazyPointer) return "lazy..."
else if (val === NOT_EXISTING) {
if (this.lazy_pointer) return undefined

const val = this.pointer!.getProperty(this.key, this.#leak_js_properties);

if (val === NOT_EXISTING) {
console.log(this)
throw new Error(`Property ${this.key} does not exist in ${this.pointer}`);
}
else return val;
}

public override get current_val():T {
return this.pointer.getProperty(this.key, this.#leak_js_properties);
if (this.lazy_pointer) return undefined
return this.pointer!.getProperty(this.key, this.#leak_js_properties);
}

// update pointer property
public override set val(value: T) {
this.pointer.handleSet(this.key, Ref.collapseValue(value, true, true));
if (this.lazy_pointer) {
console.warn("Cannot set value of lazy pointer property");
return;
}
this.pointer!.handleSet(this.key, Ref.collapseValue(value, true, true));
}
// same as val setter, but can be awaited
public override setVal(value: T) {
return this.pointer.handleSet(this.key, Ref.collapseValue(value, true, true));
if (this.lazy_pointer) {
console.warn("Cannot set value of lazy pointer property");
return;
}
return this.pointer!.handleSet(this.key, Ref.collapseValue(value, true, true));
}

#observer_internal_handlers = new WeakMap<observe_handler, observe_handler>()
#observer_internal_bound_handlers = new WeakMap<object, WeakMap<observe_handler, observe_handler>>()

// callback on property value change and when the property value changes internally
public override observe(handler: observe_handler, bound_object?:Record<string, unknown>, options?:observe_options) {
if (this.lazy_pointer) {
console.warn("Cannot observe lazy pointer");
return;
}
const value_pointer = Pointer.pointerifyValue(this.current_val);
if (value_pointer instanceof Ref) value_pointer.observe(handler, bound_object, options); // also observe internal value changes

Expand All @@ -625,7 +669,7 @@ export class PointerProperty<T=any> extends Ref<T> {
// if arrow function
else handler(v,undefined,Ref.UPDATE_TYPE.INIT)
};
this.pointer.observe(internal_handler, bound_object, this.key, options)
this.pointer!.observe(internal_handler, bound_object, this.key, options)

if (bound_object) {
if (!this.#observer_internal_bound_handlers.has(bound_object)) this.#observer_internal_bound_handlers.set(bound_object, new WeakMap);
Expand All @@ -635,6 +679,10 @@ export class PointerProperty<T=any> extends Ref<T> {
}

public override unobserve(handler: observe_handler, bound_object?:object) {
if (this.lazy_pointer) {
console.warn("Cannot unobserve lazy pointer");
return;
}
const value_pointer = Pointer.pointerifyValue(this.current_val);
if (value_pointer instanceof Ref) value_pointer.unobserve(handler, bound_object); // also unobserve internal value changes

Expand All @@ -649,7 +697,7 @@ export class PointerProperty<T=any> extends Ref<T> {
this.#observer_internal_handlers.delete(handler);
}

if (internal_handler) this.pointer.unobserve(internal_handler, bound_object, this.key); // get associated internal handler and unobserve
if (internal_handler) this.pointer!.unobserve(internal_handler, bound_object, this.key); // get associated internal handler and unobserve
}

}
Expand Down Expand Up @@ -959,7 +1007,7 @@ export class UpdateScheduler {
// Success
})
.catch((e) => {
logger.error("forwarding failed", e);
console.error("forwarding failed", e);
});
}
}
Expand Down Expand Up @@ -1199,16 +1247,28 @@ export class Pointer<T = any> extends Ref<T> {
}

static #is_local = true;
// all pointers with a @@local id, must be mapped to new endpoint ids when endpoint id is available
static #local_pointers = new IterableWeakSet<Pointer>();
// all pointers for which the is_origin property must be updated once the endpoint id is available
static #undetermined_pointers = new IterableWeakSet<Pointer>();


public static set is_local(local: boolean) {
this.#is_local = local;
// update pointer ids if no longer local
if (!this.#is_local) {
// update local pointers
for (const pointer of this.#local_pointers) {
// still local?
if (pointer.origin == LOCAL_ENDPOINT) pointer.id = Pointer.getUniquePointerID(pointer);
}
this.#local_pointers.clear();

// update undetermined pointers
for (const pointer of this.#undetermined_pointers) {
pointer.#updateIsOrigin()
}
this.#undetermined_pointers.clear();
}
}
public static get is_local() {return this.#is_local}
Expand Down Expand Up @@ -1551,7 +1611,8 @@ export class Pointer<T = any> extends Ref<T> {
}

// create a new pointer or return the existing pointer/pointer property for this value
static createOrGet<T>(value:RefOrValue<T>, sealed = false, allowed_access?:target_clause, anonymous = false, persistant= false):Pointer<T>{
static createOrGet<T>(value:RefOrValue<T>, sealed = false, allowed_access?:target_clause, anonymous = false, persistant = false):Pointer<T>{
if (value instanceof LazyPointer) throw new PointerError("Lazy Pointer not supported in this context");
if (value instanceof Pointer) return <Pointer<T>>value; // return pointer by reference
//if (value instanceof PointerProperty) return value; // return pointerproperty TODO: handle pointer properties?
value = Ref.collapseValue(value, true, true);
Expand All @@ -1567,6 +1628,12 @@ export class Pointer<T = any> extends Ref<T> {
else return <Pointer<T>>Pointer.create(undefined, value, sealed, undefined, persistant, anonymous, false, allowed_access);
}

// same as createOrGet, but also return lazy pointer if it exists
static createOrGetLazy<T>(value:RefOrValue<T>, sealed = false, allowed_access?:target_clause, anonymous = false, persistant = false):Pointer<T>|LazyPointer<T>{
if (value instanceof LazyPointer) return value;
return this.createOrGet(value, sealed, allowed_access, anonymous, persistant);
}

// create a new pointer or return the existing pointer + add a label
static createLabel<T>(value:RefOrValue<T>, label:string|number):Pointer<T>{
let ptr = Pointer.getByValue(value); // try proxify
Expand Down Expand Up @@ -1934,6 +2001,11 @@ export class Pointer<T = any> extends Ref<T> {
get is_js_primitive(){return this.#is_js_primitive} // true if js primitive (number, boolean, ...) or 'single instance' class (Type, Endpoint) that cannot be directly addressed by reference
get is_anonymous(){return this.#is_anonymous}
get origin(){return this.#origin}
private set origin(origin:Endpoint){
this.#origin = origin
this.#updateIsOrigin()
if (Runtime.endpoint == LOCAL_ENDPOINT) Pointer.#undetermined_pointers.add(this);
}

get is_persistent() { return this.#is_persistent || this.subscribers?.size != 0}
// change the persistant state of this pointer
Expand Down Expand Up @@ -1977,9 +2049,9 @@ export class Pointer<T = any> extends Ref<T> {
}
}

private set origin(origin:Endpoint){
this.#origin = origin
this.#is_origin = !!Runtime.endpoint?.equals(this.#origin);

#updateIsOrigin() {
this.#is_origin = !!Runtime.endpoint.equals(this.#origin) || !!Runtime.endpoint.main.equals(this.#origin) || this.#origin.equals(LOCAL_ENDPOINT);
}


Expand Down Expand Up @@ -2120,7 +2192,7 @@ export class Pointer<T = any> extends Ref<T> {
const endpoint = override_endpoint ?? this.origin;

// early return, trying to subscribe to the own main endpoint, guaranteed to be routed back to self, which is not allowed
if (endpoint.equals(Runtime.endpoint.main)) {
if (endpoint.equals(Runtime.endpoint.main) || endpoint.equals(Runtime.endpoint) || endpoint.equals(LOCAL_ENDPOINT)) {
logger.warn("tried to subscribe to own pointer: " + this.idString() + "(pointer origin: " + this.origin + ", own endpoint instance: " + Runtime.endpoint + ")");
return this;
}
Expand Down Expand Up @@ -3147,7 +3219,7 @@ export class Pointer<T = any> extends Ref<T> {
// TODO also check pointer permission for 'to'

// request sync endpoint is self, cannot subscribe to own pointers!
if (Runtime.endpoint.equals(subscriber)) {
if (Runtime.endpoint.equals(subscriber) || subscriber.equals(LOCAL_ENDPOINT)) {
throw new PointerError("Cannot sync pointer with own origin");
}

Expand Down Expand Up @@ -4059,7 +4131,7 @@ export class Pointer<T = any> extends Ref<T> {
await Runtime.datexOut([datex, data, {collapse_first_inserted, type:ProtocolDataType.UPDATE, preemptive_pointer_init: true}], receiver, undefined, false, undefined, undefined, false, undefined, this.datex_timeout);
} catch(e) {
//throw e;
logger.error("forwarding failed", e, datex, data)
console.error("forwarding failed", e, datex, data)
}
}

Expand Down
5 changes: 2 additions & 3 deletions runtime/runtime.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3472,10 +3472,9 @@ export class Runtime {

// get parent[key] as DatexPointerProperty if possible
getReferencedProperty(SCOPE: datex_scope, parent:any, key:any){
const pointer = Pointer.createOrGet(parent);
const pointer = Pointer.createOrGetLazy(parent);
if (pointer) {
pointer.assertEndpointCanRead(SCOPE?.sender)

if (pointer instanceof Pointer) pointer.assertEndpointCanRead(SCOPE?.sender)
return PointerProperty.get(pointer, key);
}
else throw new RuntimeError("Could not get a child reference");
Expand Down
2 changes: 1 addition & 1 deletion runtime/storage.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1052,7 +1052,7 @@ export class Storage {

// pointers
string = ESCAPE_SEQUENCES.BOLD+"Pointers\n\n"+ESCAPE_SEQUENCES.RESET
string += `${ESCAPE_SEQUENCES.ITALIC}A list of all pointers stored in any storage location. Pointers are only stored as long as they are referenced somwhere else in the storage.\n\n${ESCAPE_SEQUENCES.RESET}`
string += `${ESCAPE_SEQUENCES.ITALIC}A list of all pointers stored in any storage location. Pointers are only stored as long as they are referenced somewhere else in the storage.\n\n${ESCAPE_SEQUENCES.RESET}`

for (const [key, storageMap] of pointers.snapshot) {
// check if stored in all locations, otherwise print in which location it is stored (functional programming)
Expand Down
Loading

0 comments on commit 33cf4a9

Please sign in to comment.