Skip to content

Commit

Permalink
Merge pull request #40 from unyt-org/fix-storage-items
Browse files Browse the repository at this point in the history 
Fix nested pointer dependencies in storage items
  • Loading branch information
@jonasstrehle
jonasstrehle authored Jan 21, 2024
2 parents 20eaee1 + ba32e39 commit a8b8949
Show file tree
Hide file tree
Showing 5 changed files with 60 additions and 47 deletions.
7 changes: 4 additions & 3 deletions runtime/storage-locations/deno-kv.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,10 @@ export class DenoKVStorageLocation extends AsyncStorageLocation {
return client_type == "deno" && !!globalThis.Deno?.openKv;
}

async setItem(key: string, value: unknown): Promise<boolean> {
await this.set(itemDB!, key, Compiler.encodeValue(value));
return true;
async setItem(key: string, value: unknown) {
const inserted_ptrs = new Set<Pointer>();
await this.set(itemDB!, key, Compiler.encodeValue(value, inserted_ptrs));
return inserted_ptrs;
}
async getItem(key: string, conditions?: ExecConditions): Promise<unknown> {
const result = await this.get(itemDB!, key);
Expand Down
7 changes: 4 additions & 3 deletions runtime/storage-locations/indexed-db.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,10 @@ export class IndexedDBStorageLocation extends AsyncStorageLocation {
return !!globalThis.indexedDB;
}

async setItem(key: string,value: unknown): Promise<boolean> {
await datex_item_storage.setItem(key, <any>Compiler.encodeValue(value)); // value to buffer (no header)
return true;
async setItem(key: string,value: unknown) {
const inserted_ptrs = new Set<Pointer>();
await datex_item_storage.setItem(key, <any>Compiler.encodeValue(value, inserted_ptrs));
return inserted_ptrs;
}
async getItem(key: string, conditions: ExecConditions): Promise<unknown> {
const buffer = <ArrayBuffer><any>await datex_item_storage.getItem(key);
Expand Down
7 changes: 4 additions & 3 deletions runtime/storage-locations/local-storage.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,10 @@ export class LocalStorageLocation extends SyncStorageLocation {
if (!isExit && localStorage.saveFile) localStorage.saveFile(); // deno local storage, save file afer save on exit or interval
}

setItem(key: string, value: unknown): boolean {
localStorage.setItem(Storage.item_prefix+key, Compiler.encodeValueBase64(value))
return true;
setItem(key: string, value: unknown) {
const inserted_ptrs = new Set<Pointer>();
localStorage.setItem(Storage.item_prefix+key, Compiler.encodeValueBase64(value, inserted_ptrs)); // serialized pointer
return inserted_ptrs;
}

getItem(key: string, conditions?: ExecConditions) {
Expand Down
2 changes: 1 addition & 1 deletion runtime/storage-locations/sql-db.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,7 @@ export class SQLDBStorageLocation extends AsyncStorageLocation {
return client_type === "deno";
}

async setItem(key: string,value: unknown): Promise<boolean> {
async setItem(key: string,value: unknown) {

}
async getItem(key: string): Promise<unknown> {
Expand Down
84 changes: 47 additions & 37 deletions runtime/storage.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ export interface StorageLocation<SupportedModes extends Storage.Mode = Storage.M
onAfterExit?(): void // called when deno process exits
onAfterSnapshot?(isExit: boolean): void // called after a snapshot was saved to the storage (e.g. triggered by interval or exit event)

setItem(key:string, value:unknown): Promise<boolean>|boolean
setItem(key:string, value:unknown): Promise<Set<Pointer>>|Set<Pointer>
getItem(key:string, conditions?:ExecConditions): Promise<unknown>|unknown
hasItem(key:string):Promise<boolean>|boolean
removeItem(key:string): Promise<void>|void
Expand All @@ -65,7 +65,7 @@ export abstract class SyncStorageLocation implements StorageLocation<Storage.Mod
abstract isSupported(): boolean
onAfterExit() {}

abstract setItem(key: string,value: unknown): boolean
abstract setItem(key: string,value: unknown): Set<Pointer>
abstract getItem(key:string, conditions?:ExecConditions): Promise<unknown>|unknown
abstract hasItem(key:string): boolean
abstract getItemKeys(): Generator<string, void, unknown>
Expand All @@ -74,7 +74,7 @@ export abstract class SyncStorageLocation implements StorageLocation<Storage.Mod
abstract getItemValueDXB(key: string): ArrayBuffer|null
abstract setItemValueDXB(key:string, value: ArrayBuffer):void

abstract setPointer(pointer: Pointer<any>, partialUpdateKey: unknown|typeof NOT_EXISTING): Set<Pointer<any>>
abstract setPointer(pointer: Pointer, partialUpdateKey: unknown|typeof NOT_EXISTING): Set<Pointer>
abstract getPointerValue(pointerId: string, outer_serialized:boolean, conditions?:ExecConditions): unknown
abstract getPointerIds(): Generator<string, void, unknown>

Expand All @@ -93,7 +93,7 @@ export abstract class AsyncStorageLocation implements StorageLocation<Storage.Mo
abstract isSupported(): boolean
onAfterExit() {}

abstract setItem(key: string,value: unknown): Promise<boolean>
abstract setItem(key: string,value: unknown): Promise<Set<Pointer>>
abstract getItem(key:string, conditions?:ExecConditions): Promise<unknown>
abstract hasItem(key:string): Promise<boolean>
abstract getItemKeys(): Promise<Generator<string, void, unknown>>
Expand All @@ -102,7 +102,7 @@ export abstract class AsyncStorageLocation implements StorageLocation<Storage.Mo
abstract getItemValueDXB(key: string): Promise<ArrayBuffer|null>
abstract setItemValueDXB(key:string, value: ArrayBuffer):Promise<void>

abstract setPointer(pointer: Pointer<any>, partialUpdateKey: unknown|typeof NOT_EXISTING): Promise<Set<Pointer<any>>>
abstract setPointer(pointer: Pointer, partialUpdateKey: unknown|typeof NOT_EXISTING): Promise<Set<Pointer>>
abstract getPointerValue(pointerId: string, outer_serialized:boolean, conditions?:ExecConditions): Promise<unknown>
abstract getPointerIds(): Promise<Generator<string, void, unknown>>

Expand Down Expand Up @@ -373,37 +373,27 @@ export class Storage {
Storage.cache.set(key, value); // save in cache
// cache deletion does not work, problems with storage item backup
// setTimeout(()=>Storage.cache.delete(key), 10000);
const pointer = value instanceof Pointer ? value : Pointer.getByValue(value);

if (location) {
if (location.isAsync) return this.setItemAsync(location as AsyncStorageLocation, key, value, pointer, listen_for_pointer_changes);
else return this.setItemSync(location as SyncStorageLocation, key, value, pointer, listen_for_pointer_changes);
if (location.isAsync) return this.setItemAsync(location as AsyncStorageLocation, key, value, listen_for_pointer_changes);
else return this.setItemSync(location as SyncStorageLocation, key, value, listen_for_pointer_changes);
}
else return false;
}

static async setItemAsync(location:AsyncStorageLocation, key: string,value: unknown,pointer: Pointer<any>|undefined,listen_for_pointer_changes: boolean): Promise<boolean> {
this.setDirty(location, true)
// also store pointer
if (pointer) {
const res = await this.setPointer(pointer, listen_for_pointer_changes, location);
if (!res) return false;
}
static async setItemAsync(location:AsyncStorageLocation, key: string,value: unknown,listen_for_pointer_changes: boolean) {
this.setDirty(location, true)
// store value (might be pointer reference)
const res = await location.setItem(key, value);
const dependencies = await location.setItem(key, value);
await this.saveDependencyPointersAsync(dependencies, listen_for_pointer_changes, location);
this.setDirty(location, false)
return res;
return true;
}

static setItemSync(location:SyncStorageLocation, key: string,value: unknown,pointer: Pointer<any>|undefined,listen_for_pointer_changes: boolean): boolean {
// also store pointer
if (pointer) {
const res = this.setPointer(pointer, listen_for_pointer_changes, location);
if (!res) return false;
}

return location.setItem(key, value);
static setItemSync(location:SyncStorageLocation, key: string,value: unknown,listen_for_pointer_changes: boolean) {
const dependencies = location.setItem(key, value);
this.saveDependencyPointersSync(dependencies, listen_for_pointer_changes, location);
return true;
}

public static setPointer(pointer:Pointer, listen_for_changes = true, location:StorageLocation|undefined = this.#primary_location, partialUpdateKey: unknown = NOT_EXISTING): Promise<boolean>|boolean {
Expand All @@ -425,12 +415,8 @@ export class Storage {
// if (pointer.transform_scope && this.hasPointer(pointer)) return true; // ignore transform pointer, initial transform scope already stored, does not change

const dependencies = this.updatePointerSync(location, pointer, partialUpdateKey);

// add required pointers for this pointer (only same-origin pointers)
for (const ptr of dependencies) {
// add if not yet in storage
if (ptr != pointer && /*ptr.is_origin &&*/ !localStorage.getItem(this.pointer_prefix+ptr.id)) this.setPointer(ptr, listen_for_changes, location)
}
dependencies.delete(pointer);
this.saveDependencyPointersSync(dependencies, listen_for_changes, location);

// listen for changes
if (listen_for_changes) this.syncPointer(pointer, location);
Expand All @@ -448,12 +434,8 @@ export class Storage {
// if (pointer.transform_scope && await this.hasPointer(pointer)) return true; // ignore transform pointer, initial transform scope already stored, does not change

const dependencies = await this.updatePointerAsync(location, pointer, partialUpdateKey);

// add required pointers for this pointer (only same-origin pointers)
for (const ptr of dependencies) {
// add if not yet in storage
if (ptr != pointer && /*ptr.is_origin &&*/ !await this.hasPointer(ptr)) await this.setPointer(ptr, listen_for_changes, location)
}
dependencies.delete(pointer);
await this.saveDependencyPointersAsync(dependencies, listen_for_changes, location);

// listen for changes
if (listen_for_changes) this.syncPointer(pointer, location);
Expand All @@ -470,6 +452,34 @@ export class Storage {
return res;
}

/**
* Save dependency pointers to storage (SyncStorageLocation)
* Not all pointers in the set are saved, only those which are not yet in storage or not accessible in other ways
* @param dependencies List of dependency pointers
* @param listen_for_changes should update pointers in storage when value changes
* @param location storage location
*/
private static saveDependencyPointersSync(dependencies: Set<Pointer>, listen_for_changes = true, location: SyncStorageLocation) {
for (const ptr of dependencies) {
// add if not yet in storage
if (!location.hasPointer(ptr.id)) this.setPointer(ptr, listen_for_changes, location)
}
}

/**
* Save dependency pointers to storage (AsyncStorageLocation)
* Not all pointers in the set are saved, only those which are not yet in storage or not accessible in other ways
* @param dependencies List of dependency pointers
* @param listen_for_changes should update pointers in storage when value changes
* @param location storage location
*/
private static async saveDependencyPointersAsync(dependencies: Set<Pointer>, listen_for_changes = true, location: AsyncStorageLocation) {
for (const ptr of dependencies) {
// add if not yet in storage
if (!await location.hasPointer(ptr.id)) await this.setPointer(ptr, listen_for_changes, location)
}
}


private static synced_pointers = new Set<Pointer>();

Expand Down

0 comments on commit a8b8949

Please sign in to comment.