From 8259b95a47f13bf2c815eb6845af354a168c0665 Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Sat, 28 Dec 2024 18:53:17 +0100 Subject: [PATCH] chore: cleanup comments --- .../programs/data/shared-log/src/debounce.ts | 43 ----- packages/programs/data/shared-log/src/pid.ts | 1 - .../data/shared-log/src/sync/rateless-iblt.ts | 160 ------------------ .../data/shared-log/src/sync/simple.ts | 9 - .../data/shared-log/test/sharding.spec.ts | 36 ++-- packages/utils/indexer/sqlite3/src/engine.ts | 21 --- .../indexer/sqlite3/src/query-planner.ts | 20 --- 7 files changed, 14 insertions(+), 276 deletions(-) diff --git a/packages/programs/data/shared-log/src/debounce.ts b/packages/programs/data/shared-log/src/debounce.ts index ca6c318f7..f163c16cb 100644 --- a/packages/programs/data/shared-log/src/debounce.ts +++ b/packages/programs/data/shared-log/src/debounce.ts @@ -1,46 +1,3 @@ -/* export const debounceFixedInterval = < - T extends (...args: any[]) => any | Promise, ->( - fn: T, - delay: number, - options?: { debug?: boolean, onError?: (error: Error) => void }, -) => { - // a debounce function that will always wait for the delay to pass before invoking the function - // though if delay time has passed it will call the function immediately instead of resetting the timer - - let onError = options?.onError || ((error: Error) => console.error(error)); - let timeout: NodeJS.Timeout | null = null; - let lastArgs: any[] = []; - let lastThis: any; - let invokePromise = Promise.resolve(); - const invoke = async () => { - const fnCall = fn.apply(lastThis, lastArgs); - invokePromise = Promise.resolve(fnCall ?? {}); - await invokePromise.then((res) => { - - timeout = null; - return res; - }).catch(onError); - }; - - const debounced = (...args: Parameters) => { - lastArgs = args; - lastThis = this; - if (timeout) { - return; - } - invokePromise.then(() => { - timeout && clearTimeout(timeout); - timeout = setTimeout(invoke, delay); - if (options?.debug) { - console.log("debounceFixedInterval: timeout set", timeout); - } - }); - }; - - return debounced as T; -}; - */ export const debounceFixedInterval = < T extends (...args: any[]) => any | Promise, >( diff --git a/packages/programs/data/shared-log/src/pid.ts b/packages/programs/data/shared-log/src/pid.ts index 20c3ce554..4f49c3094 100644 --- a/packages/programs/data/shared-log/src/pid.ts +++ b/packages/programs/data/shared-log/src/pid.ts @@ -93,7 +93,6 @@ export class PIDReplicationController { totalError = errorMemory * errorMemoryFactor + totalError * (1 - errorMemoryFactor); } - // (this.id === "rRcHKy8yCun+32/dvRAkNMqvmXVb/N/X3Sis/wkDxKQ=") && console.log("MEMORY ERROR ? ", { errorMemory, errorMemoryFactor, memoryLimit: this.maxMemoryLimit, estimatedTotalSize, currentFactor, memoryUsage }); // Computer is getting too hot? if (this.maxCPUUsage != null && (cpuUsage || 0) > this.maxCPUUsage) { diff --git a/packages/programs/data/shared-log/src/sync/rateless-iblt.ts b/packages/programs/data/shared-log/src/sync/rateless-iblt.ts index 8dbc5f7c0..104a5ce97 100644 --- a/packages/programs/data/shared-log/src/sync/rateless-iblt.ts +++ b/packages/programs/data/shared-log/src/sync/rateless-iblt.ts @@ -178,148 +178,8 @@ const buildEncoderOrDecoderFromRange = async < } } - /* console.log( - "symbol count", - symbolCount + - " from " + - (Number(ranges.start1 - ranges.end1) / Number(MAX_U64) + - Number(ranges.start2 - ranges.end2) / Number(MAX_U64)), - ranges, - ); */ - - /* console.log( - (type === "encoder" ? "Encoder" : "Decoder") + " build time (s): ", - (+new Date() - buildDecoderStart) / 1000, - "Symbols: ", - symbolCount, - ", Hashes size: ", - +hashes.size, - ", Range: ", - ranges, - ); */ return encoder as E; }; -/* -class RangeToEncoders { - encoders: Map; - - constructor( - readonly me: PublicSignKey, - readonly rangeIndex: Index>, - readonly entryIndex: Index>, - ) { - this.encoders = new Map(); - } - - async build() { - // for all ranges in rangeIndex that belong to me - // fetch all cursors from entryIndex and build encoder with key from rangeId - for (const range of await this.rangeIndex - .iterate({ query: { hash: this.me.hashcode() } }) - .all()) { - const encoder = await buildEncoderOrDecoderFromRange( - range.value, - this.entryIndex, - "encoder", - ); - this.encoders.set(range.value.toUniqueSegmentId(), encoder); - } - } - - createSymbolGenerator(range: ReplicationRangeIndexable<"u64">): { - next: () => Symbol; - free: () => void; - } { - let encoder = this.encoders.get(range.toUniqueSegmentId()); - if (!encoder) { - throw new Error("No encoder found for range"); - } - const cloned = encoder.clone(); - return { - next: (): Symbol => { - return cloned.produce_next_coded_symbol(); - }, - free: () => { - // TODO? - }, - }; - } -} - - - -const getAllOverlappingRanges = async (properties: { - range: { - // To match - start1: bigint | number; - end1: bigint | number; - start2: bigint | number; - end2: bigint | number; - }; - publicKey: PublicSignKey; - rangeIndex: Index, any>; -}): Promise>> => { - const ranges = await properties.rangeIndex - .iterate({ - query: [ - ...getCoveringRangeQuery(properties.range), - new StringMatch({ - key: "hash", - value: properties.publicKey.hashcode(), - }), - ], - }) - .all(); - return ranges; -}; */ - -/* const getMissingValuesInRemote = async (properties: { - myEncoder: RangeToEncoders; - remoteRange: { - start1: bigint; - end1: bigint; - start2: bigint; - end2: bigint; - }; -}) => { - const findOverlappingRangesIOwn = await getAllOverlappingRanges({ - range: properties.remoteRange, - publicKey: properties.myEncoder.me, - rangeIndex: properties.myEncoder.rangeIndex, - }); - - const decoders: Map = new Map(); - for (const range of findOverlappingRangesIOwn) { - const segmentId = range.value.toUniqueSegmentId(); - const encoder: EncoderWrapper | undefined = - properties.myEncoder.encoders.get(segmentId); - if (encoder) { - decoders.set(segmentId, encoder.to_decoder()); - } - } - - return { - process: (encodedSymbol: any) => { - let allMissingSymbols: any[] = []; - for (const [k, decoder] of decoders) { - decoder.add_coded_symbol(encodedSymbol); - decoder.try_decode(); - if (decoder.decoded()) { - for (const missingSymbol of decoder.get_local_symbols()) { - allMissingSymbols.push(missingSymbol); - } - decoders.delete(k); - } - } - return { - missing: allMissingSymbols, - done: decoders.size === 0, - }; - }, - }; -}; - -export { RangeToEncoders, getMissingValuesInRemote }; */ export class RatelessIBLTSynchronizer< D extends "u32" | "u64", @@ -564,13 +424,6 @@ export class RatelessIBLTSynchronizer< return true; } - /* console.log( - "ALREADY HAVE ENTRIES", - await this.properties.entryIndex.count(), - "but log ?", - this.properties.log.length, - ); */ - const createTimeout = () => { return setTimeout(() => { // decoder.free(); TODO? @@ -629,19 +482,6 @@ export class RatelessIBLTSynchronizer< allMissingSymbolsInRemote.push(missingSymbol); } - /* let t1 = +new Date(); - console.log("Done decoding after", count, "symbols", "allMissingSymbolsInRemote: ", allMissingSymbolsInRemote.length, "time: ", (t1 - t0) / 1000, "s"); */ - - // now we want to resolve the hashes from the symbols - /* console.log( - "NODE: " + - this.properties.rpc.node.identity.publicKey.hashcode() + - " --> " + - context.from?.hashcode() + - " is missing in remote: ", - allMissingSymbolsInRemote.length + " remote is missing from me ", - decoder.get_local_symbols().length + " consumed symbols " + count, - ); */ this.simple.queueSync(allMissingSymbolsInRemote, context.from!, { skipCheck: true, }); diff --git a/packages/programs/data/shared-log/src/sync/simple.ts b/packages/programs/data/shared-log/src/sync/simple.ts index 089521b31..e90b826aa 100644 --- a/packages/programs/data/shared-log/src/sync/simple.ts +++ b/packages/programs/data/shared-log/src/sync/simple.ts @@ -226,17 +226,8 @@ export class SimpleSyncronizer options?.skipCheck || !(await this.checkHasCoordinateOrHash(coordinateOrHash)) ) { - /* if ((this.dbg)) { - console.log("I NEED TO SYNC!", coordinateOrHash) - } */ - this.syncInFlightQueue.set(coordinateOrHash, []); requestHashes.push(coordinateOrHash); // request immediately (first time we have seen this hash) - } else { - /* if ((this.dbg)) { - console.log("ALREAD HAVE SYNC?", coordinateOrHash) - - } */ } } diff --git a/packages/programs/data/shared-log/test/sharding.spec.ts b/packages/programs/data/shared-log/test/sharding.spec.ts index ed4f18deb..72b98d428 100644 --- a/packages/programs/data/shared-log/test/sharding.spec.ts +++ b/packages/programs/data/shared-log/test/sharding.spec.ts @@ -1030,30 +1030,22 @@ testSetups.forEach((setup) => { // insert 1mb await db2.add(data, { meta: { next: [] } }); } - try { - await waitForConverged(async () => { - const diff = Math.abs( - (await db2.log.calculateMyTotalParticipation()) - - (await db1.log.calculateMyTotalParticipation()), - ); - - return Math.round(diff * 100); - }); + await waitForConverged(async () => { + const diff = Math.abs( + (await db2.log.calculateMyTotalParticipation()) - + (await db1.log.calculateMyTotalParticipation()), + ); - await waitForResolved( - async () => - expect( - Math.abs(memoryLimit - (await db2.log.getMemoryUsage())), - ).lessThan((memoryLimit / 100) * 10), // 10% error at most - { timeout: 20 * 1000, delayInterval: 1000 }, - ); // 10% error at most - } catch (error) { - const weight1 = await db2.log.getMemoryUsage(); + return Math.round(diff * 100); + }); - const weight2 = await db2.log.getMemoryUsage(); - console.log("weight", weight1, weight2); - throw error; - } + await waitForResolved( + async () => + expect( + Math.abs(memoryLimit - (await db2.log.getMemoryUsage())), + ).lessThan((memoryLimit / 100) * 10), // 10% error at most + { timeout: 20 * 1000, delayInterval: 1000 }, + ); // 10% error at most }); it("underflow limited", async () => { diff --git a/packages/utils/indexer/sqlite3/src/engine.ts b/packages/utils/indexer/sqlite3/src/engine.ts index f1af9336a..9544ca1f8 100644 --- a/packages/utils/indexer/sqlite3/src/engine.ts +++ b/packages/utils/indexer/sqlite3/src/engine.ts @@ -432,28 +432,7 @@ export class SQLLiteIndex> }, ); - /* if (indexesToCreate) { - for (const index of indexesToCreate) { - console.log(index); - await this.properties.db.exec(index); - } - } */ - sqlFetch = sql; - // sqlFetch = ` select NULL as 'v_0#id', NULL as 'v_0#value', v_1."id" as 'v_1#id', json_group_array(distinct json_object('__id', v_1__nested__class_DocumentWithProperties."__id", '__index', v_1__nested__class_DocumentWithProperties."__index", 'a', v_1__nested__class_DocumentWithProperties."a", 'b', v_1__nested__class_DocumentWithProperties."b", 'bool', v_1__nested__class_DocumentWithProperties."bool", 'c', v_1__nested__class_DocumentWithProperties."c", 'd', v_1__nested__class_DocumentWithProperties."d")) as v_1__nested__class_DocumentWithProperties, v_1."id" as 'v_1.id' FROM v_1 INDEXED BY v_1_index_id LEFT JOIN v_1__nested__class_DocumentWithProperties AS v_1__nested__class_DocumentWithProperties INDEXED BY v_1__nested__class_DocumentWithProperties_index___parent_id ON v_1.id = v_1__nested__class_DocumentWithProperties.__parent_id LEFT JOIN v_1__nested__class_DocumentWithProperties AS _query_v_1__nested__class_DocumentWithProperties INDEXED BY v_1__nested__class_DocumentWithProperties_index___parent_id_bool ON v_1.id = _query_v_1__nested__class_DocumentWithProperties.__parent_id where _query_v_1__nested__class_DocumentWithProperties."bool" = ? GROUP BY v_1."id" ORDER BY "v_0#id" ASC limit ? offset ?` - /* this.x++; - if (this.x % 1000 === 0) { - console.log("SQL FETCH", sqlFetch); - } */ - /* console.log("SQL FETCH", sqlFetch); */ - - /* if (sqlFetch.trim() === `select class_NumberQueryDocument."id" as 'class_NumberQueryDocument#id', class_NumberQueryDocument."number" as 'class_NumberQueryDocument#number' FROM class_NumberQueryDocument where class_NumberQueryDocument."number" < ? ORDER BY class_NumberQueryDocument.id ASC limit ? offset ?`) { - // sqlFetch = `select class_NumberQueryDocument."id" as 'class_NumberQueryDocument#id', class_NumberQueryDocument."number" as 'class_NumberQueryDocument#number' FROM class_NumberQueryDocument INDEXED BY class_NumberQueryDocument_index where class_NumberQueryDocument.number < ? ORDER BY class_NumberQueryDocument.id ASC limit ? offset ?` - sqlFetch = `select class_NumberQueryDocument.id as 'class_NumberQueryDocument#id', class_NumberQueryDocument.number as 'class_NumberQueryDocument#number' FROM class_NumberQueryDocument where class_NumberQueryDocument.number < ? ORDER BY class_NumberQueryDocument.id ASC limit ? offset ?` - - } */ - - /* sqlFetch = `explain query plan ${sqlFetch}`; */ bindable = toBind; await planningScope.beforePrepare(); diff --git a/packages/utils/indexer/sqlite3/src/query-planner.ts b/packages/utils/indexer/sqlite3/src/query-planner.ts index cf7997213..51b396daa 100644 --- a/packages/utils/indexer/sqlite3/src/query-planner.ts +++ b/packages/utils/indexer/sqlite3/src/query-planner.ts @@ -273,24 +273,6 @@ export class QueryPlanner { fastestIndex.used++; pickedIndexKeys.set(fastestIndex.indexKey, sortedNameKey); - /* if (fastestIndex.used % 300 === 0) { - console.log("INDEX STATS", indexStats.results.map(x => { - return { - key: x.indexKey, - used: x.used, - avg: x.avg, - } - })); - } */ - /* console.log("INDEX STATS", indexStats.results.map(x => { - return { - key: x.indexKey, - used: x.used, - avg: x.avg, - } - }), columns); */ - - // console.log("FASTEST", fastestIndex.indexKey) return fastestIndex.indexKey!; }, perform: async (fn: () => Promise): Promise => { @@ -299,7 +281,6 @@ export class QueryPlanner { const out = await fn(); let t1 = hrtime.bigint(); const time = Number(t1 - t0); - // console.log("MEASURE TIME", time, "FOR", [...pickedIndexKeys.keys()]); for (const [indexKey, columnsKey] of pickedIndexKeys) { const indexStats = obj.columnsToIndexes.get(columnsKey); @@ -320,7 +301,6 @@ export class QueryPlanner { index.times.reduce((a, b) => a + b, 0) / index.times.length; indexStats.results.sort((a, b) => a.avg - b.avg); // make sure fastest is first - // console.log("INDEX STATS", indexStats.results.map(x => x.lastTime)); } return out;