diff --git a/packages/transport/libp2p-test-utils/src/session.ts b/packages/transport/libp2p-test-utils/src/session.ts index 63e3edd34..7cde71ade 100644 --- a/packages/transport/libp2p-test-utils/src/session.ts +++ b/packages/transport/libp2p-test-utils/src/session.ts @@ -88,11 +88,22 @@ export class TestSession { const result = async () => { const definedOptions: Libp2pOptions | undefined = (options as any)?.[i] || options; + + const services: any = { + identify: identify(), + ...definedOptions?.services, + }; + if (definedOptions?.services?.relay !== null) { + services.relay = relay(); + } else { + delete services.relay; + } + const node = await createLibp2p({ addresses: { listen: listen(), }, - connectionManager: definedOptions?.connectionManager ?? {}, + connectionManager: definedOptions?.connectionManager, privateKey: definedOptions?.privateKey, datastore: definedOptions?.datastore, transports: definedOptions?.transports ?? transports(), @@ -100,11 +111,7 @@ export class TestSession { enabled: false, }, - services: { - relay: relay(), - identify: identify(), - ...definedOptions?.services, - } as any, + services, connectionEncrypters: [noise()], streamMuxers: definedOptions?.streamMuxers || [yamux()], start: definedOptions?.start, diff --git a/packages/transport/stream/test/stream.spec.ts b/packages/transport/stream/test/stream.spec.ts index 46cb6632d..ae40b525a 100644 --- a/packages/transport/stream/test/stream.spec.ts +++ b/packages/transport/stream/test/stream.spec.ts @@ -3239,6 +3239,25 @@ describe("start/stop", () => { await session.peers[0].stop(); await session.peers[0].start(); }); + + it("streams are pruned on disconnect", async () => { + // https://github.com/libp2p/js-libp2p/issues/2794 + session = await disconnected(2, { + services: { + relay: null, + directstream: (c: any) => new TestDirectStream(c), + }, + } as any); + await session.connect([[session.peers[0], session.peers[1]]]); + await waitForResolved(() => + expect(session.peers[0].services.directstream.peers.size).to.equal(1), + ); + + await session.peers[0].hangUp(session.peers[1].peerId); + await waitForResolved(() => + expect(session.peers[0].services.directstream.peers.size).to.equal(0), + ); + }); }); describe("multistream", () => { diff --git a/packages/utils/indexer/interface/src/id.ts b/packages/utils/indexer/interface/src/id.ts index cb1cf0d7a..4ef87595d 100644 --- a/packages/utils/indexer/interface/src/id.ts +++ b/packages/utils/indexer/interface/src/id.ts @@ -52,8 +52,8 @@ export class BigUnsignedIntegerValue extends IntegerValue { constructor(number: bigint) { super(); - if (number > 18446744073709551615n || number < 0) { - throw new Error("Number is not u32"); + if (number > 18446744073709551615n || number < 0n) { + throw new Error("Number is not u64"); } this.number = number; } @@ -129,10 +129,26 @@ export class IntegerKey extends IdKey { } } +@variant(3) +export class LargeIntegerKey extends IdKey { + @field({ type: "u64" }) // max value is 2^63 - 1 (9007199254740991) + key: bigint; + + constructor(key: bigint) { + super(); + this.key = key; + } + + get primitive() { + return this.key; + } +} + export type Ideable = string | number | bigint | Uint8Array; const idKeyTypes = new Set(["string", "number", "bigint"]); +const u64Max = 18446744073709551615n; export const toId = (obj: Ideable): IdKey => { if (typeof obj === "string") { return new StringKey(obj); @@ -141,11 +157,14 @@ export const toId = (obj: Ideable): IdKey => { return new IntegerKey(obj); } if (typeof obj === "bigint") { - if (obj <= Number.MAX_SAFE_INTEGER && obj >= 0) { - return new IntegerKey(Number(obj)); + if (obj <= u64Max && obj >= 0n) { + return new LargeIntegerKey(obj); } throw new Error( - "BigInt is not less than 2^53. Max value is 9007199254740991", + "BigInt is not less than 2^64 - 1. Max value is " + + (2 ** 64 - 1) + + ". Provided value: " + + obj, ); } if (obj instanceof Uint8Array) { diff --git a/packages/utils/indexer/simple/src/index.ts b/packages/utils/indexer/simple/src/index.ts index 4a8c05eae..911143496 100644 --- a/packages/utils/indexer/simple/src/index.ts +++ b/packages/utils/indexer/simple/src/index.ts @@ -188,8 +188,8 @@ export class HashmapIndex, NestedType = any> } iterate( - query: types.IterateOptions, - properties: { shape?: S; reference?: boolean }, + query?: types.IterateOptions, + properties?: { shape?: S; reference?: boolean }, ): types.IndexIterator { let done: boolean | undefined = undefined; let queue: @@ -205,7 +205,7 @@ export class HashmapIndex, NestedType = any> const indexedDocuments = await this.queryAll(query); if (indexedDocuments.length > 1) { // Sort - if (query.sort) { + if (query?.sort) { const sortArr = Array.isArray(query.sort) ? query.sort : [query.sort]; diff --git a/packages/utils/indexer/simple/test/index.spec.ts b/packages/utils/indexer/simple/test/index.spec.ts index 71727d13b..77e18ce9f 100644 --- a/packages/utils/indexer/simple/test/index.spec.ts +++ b/packages/utils/indexer/simple/test/index.spec.ts @@ -2,5 +2,8 @@ import { tests } from "@peerbit/indexer-tests"; import { create } from "../src"; describe("all", () => { - tests(create, "transient", false); + tests(create, "transient", { + shapingSupported: false, + u64SumSupported: true, + }); }); diff --git a/packages/utils/indexer/sqlite3/src/engine.ts b/packages/utils/indexer/sqlite3/src/engine.ts index ea4a76939..75514786c 100644 --- a/packages/utils/indexer/sqlite3/src/engine.ts +++ b/packages/utils/indexer/sqlite3/src/engine.ts @@ -13,9 +13,11 @@ import { buildJoin, convertCountRequestToQuery, convertDeleteRequestToQuery, + convertFromSQLType, convertSearchRequestToQuery, /* getTableName, */ convertSumRequestToQuery, + convertToSQLType, escapeColumnName, generateSelectQuery, getInlineTableFieldName, @@ -251,9 +253,13 @@ export class SQLLiteIndex> table, options?.shape, ); - const sql = `${generateSelectQuery(table, selects)} ${buildJoin(joinMap, true)} where ${this.primaryKeyString} = ? `; + const sql = `${generateSelectQuery(table, selects)} ${buildJoin(joinMap, true)} where ${this.primaryKeyString} = ? limit 1`; const stmt = await this.properties.db.prepare(sql, sql); - const rows = await stmt.get([id.key]); + const rows = await stmt.get([ + table.primaryField?.from?.type + ? convertToSQLType(id.key, table.primaryField.from.type) + : id.key, + ]); if (!rows) { continue; } @@ -324,12 +330,6 @@ export class SQLLiteIndex> ): types.IndexIterator { // create a sql statement where the offset and the limit id dynamic and can be updated // TODO don't use offset but sort and limit 'next' calls by the last value of the sort - let { sql: sqlFetch, bindable } = convertSearchRequestToQuery( - request, - this.tables, - this._rootTables, - options?.shape, - ); /* const totalCountKey = "count"; */ /* const sqlTotalCount = convertCountRequestToQuery(new types.CountRequest({ query: request.query }), this.tables, this.tables.get(this.rootTableName)!) @@ -342,11 +342,25 @@ export class SQLLiteIndex> let stmt: Statement; let kept: number | undefined = undefined; + let bindable: any[] = []; + let sqlFetch: string | undefined = undefined; /* let totalCount: undefined | number = undefined; */ - const fetch = async (amount: number) => { + const fetch = async (amount: number | "all") => { kept = undefined; if (!once) { + let { sql, bindable: toBind } = convertSearchRequestToQuery( + request, + this.tables, + this._rootTables, + { + shape: options?.shape, + stable: typeof amount === "number", // if we are to fetch all, we dont need stable sorting + }, + ); + sqlFetch = sql; + bindable = toBind; + stmt = await this.properties.db.prepare(sqlFetch, sqlFetch); // stmt.reset?.(); // TODO dont invoke reset if not needed /* countStmt.reset?.(); */ @@ -360,12 +374,11 @@ export class SQLLiteIndex> } once = true; - const offsetStart = offset; const allResults: Record[] = await stmt.all([ ...bindable, - amount, - offsetStart, + amount === "all" ? Number.MAX_SAFE_INTEGER : amount, + offset, ]); let results: IndexedResult>[] = @@ -389,9 +402,12 @@ export class SQLLiteIndex> return { value, id: types.toId( - row[ - getTablePrefixedField(selectedTable, this.primaryKeyString) - ], + convertFromSQLType( + row[ + getTablePrefixedField(selectedTable, this.primaryKeyString) + ], + selectedTable.primaryField!.from!.type, + ), ), }; }), @@ -410,7 +426,7 @@ export class SQLLiteIndex> iterator.kept = 0; } */ - if (results.length < amount) { + if (amount === "all" || results.length < amount) { hasMore = false; await this.clearupIterator(requestId); clearTimeout(iterator.timeout); @@ -548,10 +564,12 @@ export class SQLLiteIndex> const stmt = await this.properties.db.prepare(sql, sql); const result = await stmt.get(bindable); if (result != null) { + const value = result.sum as number; + if (ret == null) { - (ret as any) = result.sum as number; + ret = value; } else { - (ret as any) += result.sum as number; + ret += value; } once = true; } diff --git a/packages/utils/indexer/sqlite3/src/schema.ts b/packages/utils/indexer/sqlite3/src/schema.ts index c72ede9a4..6613f6edb 100644 --- a/packages/utils/indexer/sqlite3/src/schema.ts +++ b/packages/utils/indexer/sqlite3/src/schema.ts @@ -54,17 +54,27 @@ export type BindableValue = | ArrayBuffer | null; +export const u64ToI64 = (u64: bigint | number) => { + return (typeof u64 === "number" ? BigInt(u64) : u64) - 9223372036854775808n; +}; +export const i64ToU64 = (i64: number | bigint) => + (typeof i64 === "number" ? BigInt(i64) : i64) + 9223372036854775808n; + export const convertToSQLType = ( value: boolean | bigint | string | number | Uint8Array, type?: FieldType, ): BindableValue => { // add bigint when https://github.com/TryGhost/node-sqlite3/pull/1501 fixed - if (type === "bool") { - if (value != null) { + if (value != null) { + if (type === "bool") { return value ? 1 : 0; } - return null; + if (type === "u64") { + // shift to fit in i64 + + return u64ToI64(value as number | bigint); + } } return value as BindableValue; }; @@ -101,9 +111,15 @@ export const convertFromSQLType = ( : nullAsUndefined(value); } if (type === "u64") { - return typeof value === "number" || typeof value === "string" - ? BigInt(value) - : nullAsUndefined(value); + if (typeof value === "number" || typeof value === "bigint") { + return i64ToU64(value as number | bigint); // TODO is not always value type bigint? + } + if (value == null) { + return nullAsUndefined(value); + } + throw new Error( + `Unexpected value type for value ${value} expected number or bigint for u64 field`, + ); } return nullAsUndefined(value); }; @@ -145,7 +161,8 @@ export interface Table { name: string; ctor: Constructor; primary: string | false; - primaryIndex: number; + primaryIndex: number; // can be -1 for nested tables TODO make it more clear + primaryField?: SQLField; // can be undefined for nested tables TODO make it required path: string[]; parentPath: string[] | undefined; // field path of the parent where this table originates from fields: SQLField[]; @@ -195,6 +212,7 @@ export const getSQLTable = ( ctor, parentPath: path, path: newPath, + primaryField: fields.find((x) => x.isPrimary)!, primary, primaryIndex: fields.findIndex((x) => x.isPrimary), children: dependencies, @@ -1254,8 +1272,16 @@ export const convertSumRequestToQuery = ( tables, table, ); + + const inlineName = getInlineTableFieldName(request.key); + const field = table.fields.find((x) => x.name === inlineName); + if (unwrapNestedType(field!.from!.type) === "u64") { + throw new Error("Summing is not supported for u64 fields"); + } + const column = `${table.name}.${getInlineTableFieldName(request.key)}`; + return { - sql: `SELECT SUM(${table.name}.${getInlineTableFieldName(request.key)}) as sum FROM ${table.name} ${query}`, + sql: `SELECT SUM(${column}) as sum FROM ${table.name} ${query}`, bindable, }; }; @@ -1281,7 +1307,10 @@ export const convertSearchRequestToQuery = ( request: types.IterateOptions | undefined, tables: Map, rootTables: Table[], - shape: types.Shape | undefined, + options?: { + shape?: types.Shape | undefined; + stable?: boolean; + }, ): { sql: string; bindable: any[] } => { let unionBuilder = ""; let orderByClause: string = ""; @@ -1289,7 +1318,7 @@ export const convertSearchRequestToQuery = ( let matchedOnce = false; let lastError: Error | undefined = undefined; - const selectsPerTable = selectAllFieldsFromTables(rootTables, shape); + const selectsPerTable = selectAllFieldsFromTables(rootTables, options?.shape); let bindableBuilder: any[] = []; for (const [i, table] of rootTables.entries()) { const { selects, joins: joinFromSelect } = selectsPerTable[i]; @@ -1301,6 +1330,10 @@ export const convertSearchRequestToQuery = ( tables, table, joinFromSelect, + [], + { + stable: options?.stable, + }, ); unionBuilder += `${unionBuilder.length > 0 ? " UNION ALL " : ""} ${selectQuery} ${query}`; orderByClause = @@ -1358,6 +1391,9 @@ const convertRequestToQuery = < table: Table, extraJoin?: Map, path: string[] = [], + options?: { + stable?: boolean; + }, ): R => { let whereBuilder = ""; let bindableBuilder: any[] = []; @@ -1388,8 +1424,16 @@ const convertRequestToQuery = < } if (isIterateRequest(request, type)) { - if (request?.sort) { - let sortArr = Array.isArray(request.sort) ? request.sort : [request.sort]; + let sort = request?.sort; + if (!sort && options?.stable) { + sort = + table.primary && path.length === 0 + ? [{ key: [table.primary], direction: types.SortDirection.ASC }] + : undefined; + } + + if (sort) { + let sortArr = Array.isArray(sort) ? sort : [sort]; if (sortArr.length > 0) { orderByBuilder = ""; let once = false; @@ -1793,7 +1837,13 @@ const convertStateFieldQuery = ( } else { throw new Error(`Unsupported compare type: ${query.compare}`); } - bindable.push(query.value.value); + + if (unwrapNestedType(tableField.from!.type) === "u64") { + // shift left because that is how we insert the value + bindable.push(u64ToI64(query.value.value)); + } else { + bindable.push(query.value.value); + } } } else if (query instanceof types.IsNull) { where = `${keyWithTable} IS NULL`; diff --git a/packages/utils/indexer/sqlite3/src/sqlite3.ts b/packages/utils/indexer/sqlite3/src/sqlite3.ts index 48e2cbb89..71aefe58a 100644 --- a/packages/utils/indexer/sqlite3/src/sqlite3.ts +++ b/packages/utils/indexer/sqlite3/src/sqlite3.ts @@ -40,7 +40,8 @@ let create = async (directory?: string) => { fileMustExist: false, readonly: false /* , verbose: (message) => console.log(message) */, }); - /* db.pragma('journal_mode = WAL'); */ + // TODO this test makes things faster, but for benchmarking it might yield wierd results where some runs are faster than others + db.pragma("journal_mode = WAL"); db.pragma("foreign_keys = on"); db.defaultSafeIntegers(true); }; diff --git a/packages/utils/indexer/sqlite3/test/index.spec.ts b/packages/utils/indexer/sqlite3/test/index.spec.ts index b28cbbc20..e35fbce75 100644 --- a/packages/utils/indexer/sqlite3/test/index.spec.ts +++ b/packages/utils/indexer/sqlite3/test/index.spec.ts @@ -2,6 +2,9 @@ import { tests } from "@peerbit/indexer-tests"; import { create } from "../src/index.js"; describe("all", () => { - tests(create, "persist", true); - tests(create, "transient", true); + tests(create, "persist", { shapingSupported: true, u64SumSupported: false }); + tests(create, "transient", { + shapingSupported: true, + u64SumSupported: false, + }); }); diff --git a/packages/utils/indexer/sqlite3/test/sort.spec.ts b/packages/utils/indexer/sqlite3/test/sort.spec.ts new file mode 100644 index 000000000..d3c24fe88 --- /dev/null +++ b/packages/utils/indexer/sqlite3/test/sort.spec.ts @@ -0,0 +1,47 @@ +import { id } from "@peerbit/indexer-interface"; +import { expect, use } from "chai"; +import chaiAsPromised from "chai-as-promised"; +import { SQLLiteIndex } from "../src/engine.js"; +import { create } from "../src/index.js"; +import { setup } from "./utils.js"; + +use(chaiAsPromised); + +describe("sort", () => { + // u64 is a special case since we need to shift values to fit into signed 64 bit integers + + let index: Awaited>>; + + afterEach(async () => { + await index.store.stop(); + }); + + class Document { + @id({ type: "string" }) + id: string; + + constructor(id: string) { + this.id = id; + } + } + + it("sorts by default by id ", async () => { + // this test is to insure that the iterator is stable. I.e. default sorting is applied + index = await setup({ schema: Document }, create); + const store = index.store as SQLLiteIndex; + expect(store.tables.size).to.equal(1); + await index.store.put(new Document("3")); + await index.store.put(new Document("2")); + await index.store.put(new Document("1")); + + const iterator = await index.store.iterate(); + const [first, second, third] = [ + ...(await iterator.next(1)), + ...(await iterator.next(1)), + ...(await iterator.next(1)), + ]; + expect(first.value.id).to.equal("1"); + expect(second.value.id).to.equal("2"); + expect(third.value.id).to.equal("3"); + }); +}); diff --git a/packages/utils/indexer/sqlite3/test/statement.spec.ts b/packages/utils/indexer/sqlite3/test/statement.spec.ts index e0c7c8a27..168c8d685 100644 --- a/packages/utils/indexer/sqlite3/test/statement.spec.ts +++ b/packages/utils/indexer/sqlite3/test/statement.spec.ts @@ -1,34 +1,14 @@ import { field } from "@dao-xyz/borsh"; import { - type Index, - type IndexEngineInitProperties, - type Indices, StringMatch, StringMatchMethod, - getIdProperty, id, toId, } from "@peerbit/indexer-interface"; import { expect } from "chai"; import { SQLLiteIndex } from "../src/engine.js"; import { create } from "../src/index.js"; - -const setup = async >( - properties: Partial> & { schema: any }, - createIndicies: (directory?: string) => Indices | Promise, -): Promise<{ indices: Indices; store: Index; directory?: string }> => { - const indices = await createIndicies(); - await indices.start(); - const indexProps: IndexEngineInitProperties = { - ...{ - indexBy: getIdProperty(properties.schema) || ["id"], - iterator: { batch: { maxSize: 5e6, sizeProperty: ["__size"] } }, - }, - ...properties, - }; - const store = await indices.init(indexProps); - return { indices, store }; -}; +import { setup } from "./utils.js"; describe("statement", () => { let index: Awaited>>; diff --git a/packages/utils/indexer/sqlite3/test/table.spec.ts b/packages/utils/indexer/sqlite3/test/table.spec.ts index 3512eb6ae..a799609c7 100644 --- a/packages/utils/indexer/sqlite3/test/table.spec.ts +++ b/packages/utils/indexer/sqlite3/test/table.spec.ts @@ -1,31 +1,9 @@ import { field } from "@dao-xyz/borsh"; -import { - type Index, - type IndexEngineInitProperties, - type Indices, - getIdProperty, - id, -} from "@peerbit/indexer-interface"; +import { id } from "@peerbit/indexer-interface"; import { expect } from "chai"; import { SQLLiteIndex } from "../src/engine.js"; import { create } from "../src/index.js"; - -const setup = async >( - properties: Partial> & { schema: any }, - createIndicies: (directory?: string) => Indices | Promise, -): Promise<{ indices: Indices; store: Index; directory?: string }> => { - const indices = await createIndicies(); - await indices.start(); - const indexProps: IndexEngineInitProperties = { - ...{ - indexBy: getIdProperty(properties.schema) || ["id"], - iterator: { batch: { maxSize: 5e6, sizeProperty: ["__size"] } }, - }, - ...properties, - }; - const store = await indices.init(indexProps); - return { indices, store }; -}; +import { setup } from "./utils.js"; describe("table", () => { let index: Awaited>>; diff --git a/packages/utils/indexer/sqlite3/test/u64.spec.ts b/packages/utils/indexer/sqlite3/test/u64.spec.ts new file mode 100644 index 000000000..8a329c3c2 --- /dev/null +++ b/packages/utils/indexer/sqlite3/test/u64.spec.ts @@ -0,0 +1,65 @@ +import { field } from "@dao-xyz/borsh"; +import { type IndexedResults, id } from "@peerbit/indexer-interface"; +import { expect, use } from "chai"; +import chaiAsPromised from "chai-as-promised"; +import { SQLLiteIndex } from "../src/engine.js"; +import { create } from "../src/index.js"; +import { setup } from "./utils.js"; + +use(chaiAsPromised); + +describe("u64", () => { + // u64 is a special case since we need to shift values to fit into signed 64 bit integers + + let index: Awaited>>; + + afterEach(async () => { + await index.store.stop(); + }); + + class DocumentWithBigint { + @id({ type: "u64" }) + id: bigint; + + @field({ type: "u64" }) + value: bigint; + + constructor(id: bigint, value: bigint) { + this.id = id; + this.value = value; + } + } + + it("fetch bounds ", async () => { + index = await setup({ schema: DocumentWithBigint }, create); + const store = index.store as SQLLiteIndex; + expect(store.tables.size).to.equal(1); + await index.store.put(new DocumentWithBigint(0n, 0n)); + await index.store.put( + new DocumentWithBigint(18446744073709551615n, 18446744073709551615n), + ); + await index.store.put(new DocumentWithBigint(123n, 123n)); + + const checkValue = async (value: bigint) => { + const max: IndexedResults = await index.store + .iterate({ query: { value: value } }) + .all(); + expect(max.length).to.equal(1); + expect(max[0].id.primitive).to.equal(value); + expect(max[0].value.id).to.equal(value); + expect(max[0].value.value).to.equal(value); + }; + + await checkValue(0n); + await checkValue(18446744073709551615n); + await checkValue(123n); + }); + + it("summing not supported", async () => { + index = await setup({ schema: DocumentWithBigint }, create); + const store = index.store as SQLLiteIndex; + await expect(store.sum({ key: "value" })).eventually.rejectedWith( + "Summing is not supported for u64 fields", + ); + }); +}); diff --git a/packages/utils/indexer/sqlite3/test/utils.ts b/packages/utils/indexer/sqlite3/test/utils.ts new file mode 100644 index 000000000..0008d02da --- /dev/null +++ b/packages/utils/indexer/sqlite3/test/utils.ts @@ -0,0 +1,22 @@ +import { + type Index, + type IndexEngineInitProperties, + type Indices, + getIdProperty, +} from "@peerbit/indexer-interface"; + +export const setup = async >( + properties: Partial> & { schema: any }, + createIndicies: (directory?: string) => Indices | Promise, +): Promise<{ indices: Indices; store: Index; directory?: string }> => { + const indices = await createIndicies(); + await indices.start(); + const indexProps: IndexEngineInitProperties = { + ...{ + indexBy: getIdProperty(properties.schema) || ["id"], + }, + ...properties, + }; + const store = await indices.init(indexProps); + return { indices, store }; +}; diff --git a/packages/utils/indexer/tests/src/tests.ts b/packages/utils/indexer/tests/src/tests.ts index 3b00dc8cb..93188d989 100644 --- a/packages/utils/indexer/tests/src/tests.ts +++ b/packages/utils/indexer/tests/src/tests.ts @@ -142,7 +142,10 @@ const assertIteratorIsDone = async (iterator: IndexIterator) => { export const tests = ( createIndicies: (directory?: string) => Indices | Promise, type: "transient" | "persist" = "transient", - shapingSupported: boolean, + properties: { + shapingSupported: boolean; + u64SumSupported: boolean; + }, ) => { return describe("index", () => { let store: Index; @@ -494,10 +497,10 @@ export const tests = ( @field({ type: "u64" }) id: bigint; - @field({ type: "string" }) - value: string; + @field({ type: "u64" }) + value: bigint; - constructor(properties: { id: bigint; value: string }) { + constructor(properties: { id: bigint; value: bigint }) { this.id = properties.id; this.value = properties.value; } @@ -507,10 +510,10 @@ export const tests = ( const { store } = await setup({ schema: DocumentBigintId }); // make the id less than 2^53, but greater than u32 max - const id = BigInt(2 ** 53 - 1); + const id = BigInt(2 ** 63 - 1); const doc = new DocumentBigintId({ id, - value: "Hello world", + value: id, }); await testIndex(store, doc); }); @@ -2212,7 +2215,7 @@ export const tests = ( ); expect(results).to.have.length(4); for (const result of results) { - if (shapingSupported) { + if (properties.shapingSupported) { expect(Object.keys(result.value)).to.have.length(1); expect(result.value["id"]).to.exist; } else { @@ -2238,7 +2241,7 @@ export const tests = ( if (arr.length > 0) { for (const element of arr) { expect(element.number).to.exist; - if (shapingSupported) { + if (properties.shapingSupported) { expect(Object.keys(element)).to.have.length(1); } } @@ -2314,7 +2317,7 @@ export const tests = ( expect(shapedResults).to.have.length(1); expect(shapedResults[0].value.id).to.equal("2"); - if (shapingSupported) { + if (properties.shapingSupported) { expect(shapedResults[0].value["nested"]).to.be.undefined; } else { expect(shapedResults[0].value["nested"]).to.exist; @@ -2366,7 +2369,7 @@ export const tests = ( expect(shapedResults).to.have.length(1); expect(shapedResults[0].value.id).to.equal(d2.id); - if (shapingSupported) { + if (properties.shapingSupported) { expect({ ...shapedResults[0].value.nested }).to.deep.equal({ bool: false, }); @@ -2466,7 +2469,7 @@ export const tests = ( expect(shapedResults).to.have.length(1); expect(shapedResults[0].value.id).to.equal("2"); - if (shapingSupported) { + if (properties.shapingSupported) { expect(shapedResults[0].value["nested"]).to.be.undefined; } else { expect(shapedResults[0].value["nested"]).to.exist; @@ -2519,7 +2522,7 @@ export const tests = ( expect(shapedResults).to.have.length(1); expect(shapedResults[0].value.id).to.equal(d2.id); - if (shapingSupported) { + if (properties.shapingSupported) { expect({ ...shapedResults[0].value.nested }).to.deep.equal({ bool: false, }); @@ -2596,7 +2599,7 @@ export const tests = ( expect(shapedResults).to.have.length(1); expect(shapedResults[0].value.id).to.equal("2"); - if (shapingSupported) { + if (properties.shapingSupported) { expect(shapedResults[0].value["nested"]).to.be.undefined; } else { expect(shapedResults[0].value["nested"]).to.exist; @@ -2638,7 +2641,7 @@ export const tests = ( expect(shapedResults).to.have.length(1); expect(shapedResults[0].value.id).to.equal(d2.id); - if (shapingSupported) { + if (properties.shapingSupported) { expect({ ...shapedResults[0].value.nested[0] }).to.deep.equal({ bool: false, }); @@ -2690,9 +2693,9 @@ export const tests = ( }); describe("sort", () => { - const put = async (id: number) => { + const put = async (id: number, stringId?: string) => { const doc = new Document({ - id: String(id), + id: stringId ?? String(id), name: String(id), number: BigInt(id), tags: [], @@ -2768,7 +2771,7 @@ export const tests = ( await put(0); await put(1); await put(2); - { + const f1 = async () => { const iterator = store.iterate({ query: [], sort: [new Sort({ direction: SortDirection.ASC, key: "name" })], @@ -2777,8 +2780,8 @@ export const tests = ( const next = await iterator.next(3); expect(next.map((x) => x.value.name)).to.deep.equal(["0", "1", "2"]); await assertIteratorIsDone(iterator); - } - { + }; + const f2 = async () => { const iterator = store.iterate({ query: [], sort: [new Sort({ direction: SortDirection.DESC, key: "name" })], @@ -2787,15 +2790,104 @@ export const tests = ( const next = await iterator.next(3); expect(next.map((x) => x.value.name)).to.deep.equal(["2", "1", "0"]); await assertIteratorIsDone(iterator); - } + }; + await f1(); + await f2(); }); + it("sorts by order", async () => { + await put(0); + await put(1); + await put(2); + const f1 = async () => { + const iterator = store.iterate({ + query: [], + sort: [new Sort({ direction: SortDirection.ASC, key: "name" })], + }); + expect(iterator.done()).to.be.undefined; + const next = await iterator.next(3); + expect(next.map((x) => x.value.name)).to.deep.equal(["0", "1", "2"]); + await assertIteratorIsDone(iterator); + }; + const f2 = async () => { + const iterator = store.iterate({ + query: [], + sort: [new Sort({ direction: SortDirection.DESC, key: "name" })], + }); + expect(iterator.done()).to.be.undefined; + const next = await iterator.next(3); + expect(next.map((x) => x.value.name)).to.deep.equal(["2", "1", "0"]); + await assertIteratorIsDone(iterator); + }; + const f3 = async () => { + const iterator = store.iterate({ + query: [], + sort: [new Sort({ direction: SortDirection.ASC, key: "name" })], + }); + expect(iterator.done()).to.be.undefined; + let next = await iterator.next(2); + expect(next.map((x) => x.value.name)).to.deep.equal(["0", "1"]); + next = await iterator.next(1); + expect(next.map((x) => x.value.name)).to.deep.equal(["2"]); + await assertIteratorIsDone(iterator); + }; + const f4 = async () => { + const iterator = store.iterate({ + query: [], + sort: [new Sort({ direction: SortDirection.DESC, key: "name" })], + }); + expect(iterator.done()).to.be.undefined; + let next = await iterator.next(2); + expect(next.map((x) => x.value.name)).to.deep.equal(["2", "1"]); + next = await iterator.next(1); + expect(next.map((x) => x.value.name)).to.deep.equal(["0"]); + await assertIteratorIsDone(iterator); + }; + const f5 = async () => { + const iterator = store.iterate({ + query: [], + sort: [new Sort({ direction: SortDirection.ASC, key: "name" })], + }); + expect(iterator.done()).to.be.undefined; + let next = await iterator.next(1); + expect(next.map((x) => x.value.name)).to.deep.equal(["0"]); + next = await iterator.next(1); + expect(next.map((x) => x.value.name)).to.deep.equal(["1"]); + next = await iterator.next(1); + expect(next.map((x) => x.value.name)).to.deep.equal(["2"]); + await assertIteratorIsDone(iterator); + }; + await f1(); + await f2(); + await f3(); + await f4(); + await f5(); + }); + + /* it("no sort is stable", async () => { + // TODO this test is actually not a good predictor of stability + + const insertCount = 500; + for (let i = 0; i < insertCount; i++) { + await put(i, uuid()); + } + + const resolvedValues: Set = new Set() + const batchSize = 123; + const iterator = store.iterate(); + while (!iterator.done()) { + const next = await iterator.next(batchSize); + next.map((x) => resolvedValues.add(Number(x.value.number))); + } + expect(resolvedValues.size).to.equal(insertCount); + }); */ + it("strings", async () => { await put(0); await put(1); await put(2); - const iterator = await store.iterate({ + const iterator = store.iterate({ query: [], sort: [new Sort({ direction: SortDirection.ASC, key: "name" })], }); @@ -3107,24 +3199,70 @@ export const tests = ( }); describe("sum", () => { + class SummableDocument { + @field({ type: "string" }) + id: string; + + @field({ type: option("u32") }) + value?: number; + + constructor(opts: SummableDocument) { + this.id = opts.id; + this.value = opts.value; + } + } it("it returns sum", async () => { - await setupDefault(); - const sum = await store.sum({ key: "number" }); + await setup({ schema: SummableDocument }); + await store.put( + new SummableDocument({ + id: "1", + value: 1, + }), + ); + await store.put( + new SummableDocument({ + id: "2", + value: 2, + }), + ); + const sum = await store.sum({ key: "value" }); typeof sum === "bigint" - ? expect(sum).to.equal(6n) - : expect(sum).to.equal(6); + ? expect(sum).to.equal(3n) + : expect(sum).to.equal(3); }); + if (properties.u64SumSupported) { + it("u64", async () => { + await setupDefault(); + const sum = await store.sum({ key: "number" }); + typeof sum === "bigint" + ? expect(sum).to.equal(6n) + : expect(sum).to.equal(6); + }); + } + it("it returns sum with query", async () => { - await setupDefault(); + await setup({ schema: SummableDocument }); + await store.put( + new SummableDocument({ + id: "1", + value: 1, + }), + ); + await store.put( + new SummableDocument({ + id: "2", + value: 2, + }), + ); + const sum = await store.sum({ - key: "number", + key: "value", query: [ - new StringMatch({ - key: "tags", - value: "world", - method: StringMatchMethod.contains, - caseInsensitive: true, + new IntegerCompare({ + key: "value", + compare: Compare.Greater, + value: 1, }), ], });