Skip to content

Commit

Permalink
chore: cleanup comments
Browse files Browse the repository at this point in the history
  • Loading branch information
@marcus-pousette
marcus-pousette committed Dec 28, 2024
1 parent e067a97 commit 8259b95
Show file tree
Hide file tree
Showing 7 changed files with 14 additions and 276 deletions.
43 changes: 0 additions & 43 deletions packages/programs/data/shared-log/src/debounce.ts
View file
Original file line number Diff line number Diff line change
@@ -1,46 +1,3 @@
/* export const debounceFixedInterval = <
T extends (...args: any[]) => any | Promise<any>,
>(
fn: T,
delay: number,
options?: { debug?: boolean, onError?: (error: Error) => void },
) => {
// a debounce function that will always wait for the delay to pass before invoking the function
// though if delay time has passed it will call the function immediately instead of resetting the timer
let onError = options?.onError || ((error: Error) => console.error(error));
let timeout: NodeJS.Timeout | null = null;
let lastArgs: any[] = [];
let lastThis: any;
let invokePromise = Promise.resolve();
const invoke = async () => {
const fnCall = fn.apply(lastThis, lastArgs);
invokePromise = Promise.resolve(fnCall ?? {});
await invokePromise.then((res) => {
timeout = null;
return res;
}).catch(onError);
};
const debounced = (...args: Parameters<T>) => {
lastArgs = args;
lastThis = this;
if (timeout) {
return;
}
invokePromise.then(() => {
timeout && clearTimeout(timeout);
timeout = setTimeout(invoke, delay);
if (options?.debug) {
console.log("debounceFixedInterval: timeout set", timeout);
}
});
};
return debounced as T;
};
*/
export const debounceFixedInterval = <
T extends (...args: any[]) => any | Promise<any>,
>(
Expand Down
1 change: 0 additions & 1 deletion packages/programs/data/shared-log/src/pid.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,6 @@ export class PIDReplicationController {
totalError =
errorMemory * errorMemoryFactor + totalError * (1 - errorMemoryFactor);
}
// (this.id === "rRcHKy8yCun+32/dvRAkNMqvmXVb/N/X3Sis/wkDxKQ=") && console.log("MEMORY ERROR ? ", { errorMemory, errorMemoryFactor, memoryLimit: this.maxMemoryLimit, estimatedTotalSize, currentFactor, memoryUsage });

// Computer is getting too hot?
if (this.maxCPUUsage != null && (cpuUsage || 0) > this.maxCPUUsage) {
Expand Down
160 changes: 0 additions & 160 deletions packages/programs/data/shared-log/src/sync/rateless-iblt.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,148 +178,8 @@ const buildEncoderOrDecoderFromRange = async <
}
}

/* console.log(
"symbol count",
symbolCount +
" from " +
(Number(ranges.start1 - ranges.end1) / Number(MAX_U64) +
Number(ranges.start2 - ranges.end2) / Number(MAX_U64)),
ranges,
); */

/* console.log(
(type === "encoder" ? "Encoder" : "Decoder") + " build time (s): ",
(+new Date() - buildDecoderStart) / 1000,
"Symbols: ",
symbolCount,
", Hashes size: ",
+hashes.size,
", Range: ",
ranges,
); */
return encoder as E;
};
/*
class RangeToEncoders {
encoders: Map<string, EncoderWrapper>;
constructor(
readonly me: PublicSignKey,
readonly rangeIndex: Index<ReplicationRangeIndexable<"u64">>,
readonly entryIndex: Index<EntryReplicated<"u64">>,
) {
this.encoders = new Map();
}
async build() {
// for all ranges in rangeIndex that belong to me
// fetch all cursors from entryIndex and build encoder with key from rangeId
for (const range of await this.rangeIndex
.iterate({ query: { hash: this.me.hashcode() } })
.all()) {
const encoder = await buildEncoderOrDecoderFromRange(
range.value,
this.entryIndex,
"encoder",
);
this.encoders.set(range.value.toUniqueSegmentId(), encoder);
}
}
createSymbolGenerator(range: ReplicationRangeIndexable<"u64">): {
next: () => Symbol;
free: () => void;
} {
let encoder = this.encoders.get(range.toUniqueSegmentId());
if (!encoder) {
throw new Error("No encoder found for range");
}
const cloned = encoder.clone();
return {
next: (): Symbol => {
return cloned.produce_next_coded_symbol();
},
free: () => {
// TODO?
},
};
}
}
const getAllOverlappingRanges = async (properties: {
range: {
// To match
start1: bigint | number;
end1: bigint | number;
start2: bigint | number;
end2: bigint | number;
};
publicKey: PublicSignKey;
rangeIndex: Index<ReplicationRangeIndexable<"u64">, any>;
}): Promise<IndexedResults<ReplicationRangeIndexable<"u64">>> => {
const ranges = await properties.rangeIndex
.iterate({
query: [
...getCoveringRangeQuery(properties.range),
new StringMatch({
key: "hash",
value: properties.publicKey.hashcode(),
}),
],
})
.all();
return ranges;
}; */

/* const getMissingValuesInRemote = async (properties: {
myEncoder: RangeToEncoders;
remoteRange: {
start1: bigint;
end1: bigint;
start2: bigint;
end2: bigint;
};
}) => {
const findOverlappingRangesIOwn = await getAllOverlappingRanges({
range: properties.remoteRange,
publicKey: properties.myEncoder.me,
rangeIndex: properties.myEncoder.rangeIndex,
});
const decoders: Map<string, DecoderWrapper> = new Map();
for (const range of findOverlappingRangesIOwn) {
const segmentId = range.value.toUniqueSegmentId();
const encoder: EncoderWrapper | undefined =
properties.myEncoder.encoders.get(segmentId);
if (encoder) {
decoders.set(segmentId, encoder.to_decoder());
}
}
return {
process: (encodedSymbol: any) => {
let allMissingSymbols: any[] = [];
for (const [k, decoder] of decoders) {
decoder.add_coded_symbol(encodedSymbol);
decoder.try_decode();
if (decoder.decoded()) {
for (const missingSymbol of decoder.get_local_symbols()) {
allMissingSymbols.push(missingSymbol);
}
decoders.delete(k);
}
}
return {
missing: allMissingSymbols,
done: decoders.size === 0,
};
},
};
};
export { RangeToEncoders, getMissingValuesInRemote }; */

export class RatelessIBLTSynchronizer<
D extends "u32" | "u64",
Expand Down Expand Up @@ -564,13 +424,6 @@ export class RatelessIBLTSynchronizer<
return true;
}

/* console.log(
"ALREADY HAVE ENTRIES",
await this.properties.entryIndex.count(),
"but log ?",
this.properties.log.length,
); */

const createTimeout = () => {
return setTimeout(() => {
// decoder.free(); TODO?
Expand Down Expand Up @@ -629,19 +482,6 @@ export class RatelessIBLTSynchronizer<
allMissingSymbolsInRemote.push(missingSymbol);
}

/* let t1 = +new Date();
console.log("Done decoding after", count, "symbols", "allMissingSymbolsInRemote: ", allMissingSymbolsInRemote.length, "time: ", (t1 - t0) / 1000, "s"); */

// now we want to resolve the hashes from the symbols
/* console.log(
"NODE: " +
this.properties.rpc.node.identity.publicKey.hashcode() +
" --> " +
context.from?.hashcode() +
" is missing in remote: ",
allMissingSymbolsInRemote.length + " remote is missing from me ",
decoder.get_local_symbols().length + " consumed symbols " + count,
); */
this.simple.queueSync(allMissingSymbolsInRemote, context.from!, {
skipCheck: true,
});
Expand Down
9 changes: 0 additions & 9 deletions packages/programs/data/shared-log/src/sync/simple.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,17 +226,8 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
options?.skipCheck ||
!(await this.checkHasCoordinateOrHash(coordinateOrHash))
) {
/* if ((this.dbg)) {
console.log("I NEED TO SYNC!", coordinateOrHash)
} */

this.syncInFlightQueue.set(coordinateOrHash, []);
requestHashes.push(coordinateOrHash); // request immediately (first time we have seen this hash)
} else {
/* if ((this.dbg)) {
console.log("ALREAD HAVE SYNC?", coordinateOrHash)
} */
}
}

Expand Down
36 changes: 14 additions & 22 deletions packages/programs/data/shared-log/test/sharding.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1030,30 +1030,22 @@ testSetups.forEach((setup) => {
// insert 1mb
await db2.add(data, { meta: { next: [] } });
}
try {
await waitForConverged(async () => {
const diff = Math.abs(
(await db2.log.calculateMyTotalParticipation()) -
(await db1.log.calculateMyTotalParticipation()),
);

return Math.round(diff * 100);
});
await waitForConverged(async () => {
const diff = Math.abs(
(await db2.log.calculateMyTotalParticipation()) -
(await db1.log.calculateMyTotalParticipation()),
);

await waitForResolved(
async () =>
expect(
Math.abs(memoryLimit - (await db2.log.getMemoryUsage())),
).lessThan((memoryLimit / 100) * 10), // 10% error at most
{ timeout: 20 * 1000, delayInterval: 1000 },
); // 10% error at most
} catch (error) {
const weight1 = await db2.log.getMemoryUsage();
return Math.round(diff * 100);
});

const weight2 = await db2.log.getMemoryUsage();
console.log("weight", weight1, weight2);
throw error;
}
await waitForResolved(
async () =>
expect(
Math.abs(memoryLimit - (await db2.log.getMemoryUsage())),
).lessThan((memoryLimit / 100) * 10), // 10% error at most
{ timeout: 20 * 1000, delayInterval: 1000 },
); // 10% error at most
});

it("underflow limited", async () => {
Expand Down
21 changes: 0 additions & 21 deletions packages/utils/indexer/sqlite3/src/engine.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -432,28 +432,7 @@ export class SQLLiteIndex<T extends Record<string, any>>
},
);

/* if (indexesToCreate) {
for (const index of indexesToCreate) {
console.log(index);
await this.properties.db.exec(index);
}
} */

sqlFetch = sql;
// sqlFetch = ` select NULL as 'v_0#id', NULL as 'v_0#value', v_1."id" as 'v_1#id', json_group_array(distinct json_object('__id', v_1__nested__class_DocumentWithProperties."__id", '__index', v_1__nested__class_DocumentWithProperties."__index", 'a', v_1__nested__class_DocumentWithProperties."a", 'b', v_1__nested__class_DocumentWithProperties."b", 'bool', v_1__nested__class_DocumentWithProperties."bool", 'c', v_1__nested__class_DocumentWithProperties."c", 'd', v_1__nested__class_DocumentWithProperties."d")) as v_1__nested__class_DocumentWithProperties, v_1."id" as 'v_1.id' FROM v_1 INDEXED BY v_1_index_id LEFT JOIN v_1__nested__class_DocumentWithProperties AS v_1__nested__class_DocumentWithProperties INDEXED BY v_1__nested__class_DocumentWithProperties_index___parent_id ON v_1.id = v_1__nested__class_DocumentWithProperties.__parent_id LEFT JOIN v_1__nested__class_DocumentWithProperties AS _query_v_1__nested__class_DocumentWithProperties INDEXED BY v_1__nested__class_DocumentWithProperties_index___parent_id_bool ON v_1.id = _query_v_1__nested__class_DocumentWithProperties.__parent_id where _query_v_1__nested__class_DocumentWithProperties."bool" = ? GROUP BY v_1."id" ORDER BY "v_0#id" ASC limit ? offset ?`
/* this.x++;
if (this.x % 1000 === 0) {
console.log("SQL FETCH", sqlFetch);
} */
/* console.log("SQL FETCH", sqlFetch); */

/* if (sqlFetch.trim() === `select class_NumberQueryDocument."id" as 'class_NumberQueryDocument#id', class_NumberQueryDocument."number" as 'class_NumberQueryDocument#number' FROM class_NumberQueryDocument where class_NumberQueryDocument."number" < ? ORDER BY class_NumberQueryDocument.id ASC limit ? offset ?`) {
// sqlFetch = `select class_NumberQueryDocument."id" as 'class_NumberQueryDocument#id', class_NumberQueryDocument."number" as 'class_NumberQueryDocument#number' FROM class_NumberQueryDocument INDEXED BY class_NumberQueryDocument_index where class_NumberQueryDocument.number < ? ORDER BY class_NumberQueryDocument.id ASC limit ? offset ?`
sqlFetch = `select class_NumberQueryDocument.id as 'class_NumberQueryDocument#id', class_NumberQueryDocument.number as 'class_NumberQueryDocument#number' FROM class_NumberQueryDocument where class_NumberQueryDocument.number < ? ORDER BY class_NumberQueryDocument.id ASC limit ? offset ?`
} */

/* sqlFetch = `explain query plan ${sqlFetch}`; */
bindable = toBind;

await planningScope.beforePrepare();
Expand Down
20 changes: 0 additions & 20 deletions packages/utils/indexer/sqlite3/src/query-planner.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,24 +273,6 @@ export class QueryPlanner {
fastestIndex.used++;
pickedIndexKeys.set(fastestIndex.indexKey, sortedNameKey);

/* if (fastestIndex.used % 300 === 0) {
console.log("INDEX STATS", indexStats.results.map(x => {
return {
key: x.indexKey,
used: x.used,
avg: x.avg,
}
}));
} */
/* console.log("INDEX STATS", indexStats.results.map(x => {
return {
key: x.indexKey,
used: x.used,
avg: x.avg,
}
}), columns); */

// console.log("FASTEST", fastestIndex.indexKey)
return fastestIndex.indexKey!;
},
perform: async <T>(fn: () => Promise<T>): Promise<T> => {
Expand All @@ -299,7 +281,6 @@ export class QueryPlanner {
const out = await fn();
let t1 = hrtime.bigint();
const time = Number(t1 - t0);
// console.log("MEASURE TIME", time, "FOR", [...pickedIndexKeys.keys()]);

for (const [indexKey, columnsKey] of pickedIndexKeys) {
const indexStats = obj.columnsToIndexes.get(columnsKey);
Expand All @@ -320,7 +301,6 @@ export class QueryPlanner {
index.times.reduce((a, b) => a + b, 0) / index.times.length;

indexStats.results.sort((a, b) => a.avg - b.avg); // make sure fastest is first
// console.log("INDEX STATS", indexStats.results.map(x => x.lastTime));
}

return out;
Expand Down

0 comments on commit 8259b95

Please sign in to comment.