Merge pull request #58 from unyt-org/feat-storage-memory-management

Storage memory management

compiler/compiler.ts

@@ -2021,8 +2021,7 @@ export class Compiler {
                 SCOPE.addJSTypeDefs = receiver != Runtime.endpoint && receiver != LOCAL_ENDPOINT;
             }
 
-            // add js type module only if http(s) url
-            const addTypeDefs = SCOPE.addJSTypeDefs && jsTypeDefModule && (jsTypeDefModule.toString().startsWith("http://") || jsTypeDefModule.toString().startsWith("https://"));
+            const addTypeDefs = SCOPE.addJSTypeDefs && jsTypeDefModule;
 
             if (addTypeDefs) {
                 Compiler.builder.handleRequiredBufferSize(SCOPE.b_index+4, SCOPE);

docs/manual/04 Pointer Synchronisation.md

@@ -167,3 +167,39 @@ const x2 = await $.ABCDEF
 assert (x1 === x2)
 ```
 
+The identity of a pointer is always preserved. Even if you receive the pointer
+from a remote endpoint call, it is still identical to the local instance:
+
+```ts
+// remote endpoint
+
+type User = {name: string, age: number}
+
+const happyBirthday = $$(
+   function (user: User) {
+      user.age++;
+      return user;
+   }
+)
+```
+
+```ts
+// local endpoint
+
+const user = $$({
+   name: "Luke",
+   age: 20
+})
+
+const happyBirthday = await $.DEFABCDEF // <- pointer id for the remote "happyBirthday" function
+const olderUser = await happyBirthday(user);
+
+user === olderUser // true
+olderUser.age === 21 // true
+user.age === 21 // true
+```
+
+Like we would expect if this was a normal, local JavaScript function call, the returned
+`user` object is identical to the object we passed to the function.
+This is not only true for this simple example, but also for more complex scenarios.
+For example, reference identities are also preserved within [eternal values](./05%20Eternal%20Pointers.md).

init.ts

@@ -13,12 +13,34 @@ import { verboseArg } from "./utils/logger.ts";
 import { MessageLogger } from "./utils/message_logger.ts";
 
 
+/**
+ * Load the subscriber cache from storage and
+ * reset if a remote pointer is required
+ */
+async function initSubscriberCache() {
+	try {
+		Runtime.subscriber_cache = (await Storage.loadOrCreate(
+			"Datex.Runtime.SUBSCRIBER_CACHE", 
+			() => new Map(), 
+			{onlyLocalPointers: true}
+		)).setAutoDefault(Set);
+	}
+	catch (e) {
+		logger.debug("resetting subscriber cache (" + e?.message + ")")
+		Runtime.subscriber_cache = (await Storage.loadOrCreate(
+			"Datex.Runtime.SUBSCRIBER_CACHE", 
+			() => new Map(), 
+			undefined, 
+			true
+		)).setAutoDefault(Set);
+	}
+}
+
 /**
  * Runtime init (sets ENV, storage, endpoint, ...)
  */
 export async function init() {
 
-
 	// register DatexStorage as pointer source
 	registerStorageAsPointerSource();
 	// default storage config:
@@ -56,7 +78,11 @@ export async function init() {
 	Runtime.onEndpointChanged((endpoint) => {
 		Pointer.pointer_prefix = endpoint.getPointerPrefix();
 		// has only local endpoint id (%0000) or global id?
-		if (endpoint != LOCAL_ENDPOINT) Pointer.is_local = false;
+		if (endpoint != LOCAL_ENDPOINT) {
+			Pointer.is_local = false;
+			// init subscriber cache as soon as endpoint is available
+			initSubscriberCache();
+		}
 		else Pointer.is_local = true;
 	})
 
@@ -105,26 +131,6 @@ export async function init() {
 	// init persistent memory
 	Runtime.persistent_memory = (await Storage.loadOrCreate("Datex.Runtime.MEMORY", ()=>new Map())).setAutoDefault(Object);
 
-	// init persistent subscriber cache
-	(async () => {
-		try {
-			Runtime.subscriber_cache = (await Storage.loadOrCreate(
-				"Datex.Runtime.SUBSCRIBER_CACHE", 
-				() => new Map(), 
-				{onlyLocalPointers: true}
-			)).setAutoDefault(Set);
-		}
-		catch (e) {
-			logger.debug("resetting subscriber cache (" + e?.message + ")")
-			Runtime.subscriber_cache = (await Storage.loadOrCreate(
-				"Datex.Runtime.SUBSCRIBER_CACHE", 
-				() => new Map(), 
-				undefined, 
-				true
-			)).setAutoDefault(Set);
-		}
-	})()
-
 
 	if (!globalThis.NO_INIT) {
 		Runtime.init();

runtime/pointers.ts

@@ -1199,7 +1199,8 @@ export class Pointer<T = any> extends Ref<T> {
         // update pointer ids if no longer local
         if (!this.#is_local) {
             for (const pointer of this.#local_pointers) {
-                pointer.id = Pointer.getUniquePointerID(pointer);
+                // still local?
+                if (pointer.origin == LOCAL_ENDPOINT) pointer.id = Pointer.getUniquePointerID(pointer);
             }
             this.#local_pointers.clear();
         }
@@ -1395,13 +1396,16 @@ export class Pointer<T = any> extends Ref<T> {
             }
 
             if (stored!=NOT_EXISTING) {
-                // if the value is a pointer with a tranform scope, copy the transform, not the value (TODO still just a workaround to preserve transforms in storage, maybe better solution?)
-                if (stored instanceof Pointer && stored.transform_scope) {
-                    await pointer.handleTransformAsync(stored.transform_scope.internal_vars, stored.transform_scope);
+                // set value if pointer still not loaded during source.getPointer
+                if (!pointer.#loaded) {
+                    // if the value is a pointer with a tranform scope, copy the transform, not the value (TODO still just a workaround to preserve transforms in storage, maybe better solution?)
+                    if (stored instanceof Pointer && stored.transform_scope) {
+                        await pointer.handleTransformAsync(stored.transform_scope.internal_vars, stored.transform_scope);
+                    }
+                    // set normal value
+                    else pointer = pointer.setValue(stored);
                 }
-                // set normal value
-                else pointer = pointer.setValue(stored);
-
+
                 // now sync if source (pointer storage) can sync pointer
                 if (source?.syncPointer) source.syncPointer(pointer);