Merge pull request #291 from dao-xyz/persistant-indexer

feat!: Persistant indexer

.aegir.js

@@ -1,17 +1,19 @@
-
 // get monorepo root location using esm and .git folder
-import path from 'path'
-import findUp from 'find-up'
-const root = path.dirname(findUp.sync('.git', { type: 'directory' }))
+import findUp from "find-up";
+import path from "path";
+
+const root = path.dirname(findUp.sync(".git", { type: "directory" }));
 
 export default {
-    // global options
-    debug: false,
-    test: {
-        concurrency: 2,
-        files: [],
-        before: () => {
-            return { env: { TS_NODE_PROJECT: path.join(root, 'tsconfig.test.json') } }
-        }
-    }
-}
+	// global options
+	debug: false,
+	test: {
+		/* concurrency: 2, */
+		files: [],
+		before: () => {
+			return {
+				env: { TS_NODE_PROJECT: path.join(root, "tsconfig.test.json") },
+			};
+		},
+	},
+};

.mocharc.json

@@ -1,23 +1,14 @@
 {
-  "require": [
-    "ts-node/register",
-    "chai-extend/chai-global.js"
-  ],
-  "node-option": [
-    "experimental-specifier-resolution=node",
-    "loader=ts-node/esm",
-    "no-warnings"
-  ],
-  "extension": [
-    "ts"
-  ],
-  "parallel": true,
-  "spec": [
-    "**/test/**/*.spec.ts"
-  ],
-  "ignore": [
-    "**/node_modules/**",
-    "**/dist/**"
-  ],
-  "timeout": "20000"
-}
+	"require": ["ts-node/register", "chai-extend/chai-global.js"],
+	"node-option": [
+		"experimental-specifier-resolution=node",
+		"loader=ts-node/esm",
+		"no-warnings"
+	],
+	"extension": ["ts"],
+	"parallel": false,
+	"spec": ["**/*.spec.ts"],
+	"ignore": ["**/node_modules/**", "**/dist/**"],
+	"timeout": "40000",
+	"exit": true
+}

.prettierignore

@@ -0,0 +1 @@
+**/public/peerbit/**

.prettierrc

@@ -0,0 +1,18 @@
+{
+    "useTabs": true,
+    "tabWidth": 2,
+    "plugins": [
+        "@trivago/prettier-plugin-sort-imports"
+    ],
+    "importOrder": [
+        "<THIRD_PARTY_MODULES>",
+        "^[./]"
+    ],
+    "importOrderParserPlugins": [
+        "typescript",
+        "tsx",
+        "jsx",
+        "decorators"
+    ],
+    "importOrderSortSpecifiers": true
+}

.release-please-manifest.json

@@ -1,29 +1,29 @@
 {
-    "packages/transport/blocks": "2.0.32",
-    "packages/transport/blocks-interface": "1.2.19",
-    "packages/utils/cache": "2.0.6",
-    "packages/programs/clock-service": "2.0.40",
-    "packages/utils/crypto": "2.2.0",
-    "packages/programs/data/document/document": "6.0.7",
-    "packages/programs/acl/identity-access-controller": "3.0.40",
-    "packages/transport/libp2p-test-utils": "2.1.3",
-    "packages/log": "3.0.34",
-    "packages/utils/logger": "1.0.2",
-    "packages/programs/program": "4.0.7",
-    "packages/clients/peerbit-proxy/proxy": "3.0.7",
-    "packages/transport/pubsub": "3.0.5",
-    "packages/transport/pubsub-interface": "3.0.3",
-    "packages/programs/rpc": "4.0.7",
-    "packages/clients/peerbit-server/node": "5.0.33",
-    "packages/programs/data/shared-log": "8.0.7",
-    "packages/transport/stream": "4.0.5",
-    "packages/transport/stream-interface": "5.0.2",
-    "packages/programs/data/string": "4.1.26",
-    "packages/clients/test-utils": "2.0.33",
-    "packages/utils/time": "2.0.6",
-    "packages/programs/acl/trusted-network": "3.0.40",
-    "packages/clients/peerbit": "4.0.7",
-    "packages/clients/peerbit-proxy/window": "3.0.33",
-    "packages/utils/any-store/any-store": "2.0.2",
-    "packages/utils/keychain": "1.0.17"
-}
+	"packages/transport/blocks": "2.0.32",
+	"packages/transport/blocks-interface": "1.2.19",
+	"packages/utils/cache": "2.0.6",
+	"packages/programs/clock-service": "2.0.40",
+	"packages/utils/crypto": "2.2.0",
+	"packages/programs/data/document/document": "6.0.7",
+	"packages/programs/acl/identity-access-controller": "3.0.40",
+	"packages/transport/libp2p-test-utils": "2.1.3",
+	"packages/log": "3.0.34",
+	"packages/utils/logger": "1.0.2",
+	"packages/programs/program": "4.0.7",
+	"packages/clients/peerbit-proxy/proxy": "3.0.7",
+	"packages/transport/pubsub": "3.0.5",
+	"packages/transport/pubsub-interface": "3.0.3",
+	"packages/programs/rpc": "4.0.7",
+	"packages/clients/peerbit-server/node": "5.0.33",
+	"packages/programs/data/shared-log": "8.0.7",
+	"packages/transport/stream": "4.0.5",
+	"packages/transport/stream-interface": "5.0.2",
+	"packages/programs/data/string": "4.1.26",
+	"packages/clients/test-utils": "2.0.33",
+	"packages/utils/time": "2.0.6",
+	"packages/programs/acl/trusted-network": "3.0.40",
+	"packages/clients/peerbit": "4.0.7",
+	"packages/clients/peerbit-proxy/window": "3.0.33",
+	"packages/utils/any-store/any-store": "2.0.2",
+	"packages/utils/keychain": "1.0.17"
+}

.release-please.json

@@ -1,9 +1,7 @@
 {
 	"bootstrap-sha": "148d23a1bc0e9fad48ea6589cd861d47bfd2b75c",
 	"last-release-sha": "c3343e8e19a44343036ed20ed6f15bd13beddf6f",
-	"plugins": [
-		"node-workspace"
-	],
+	"plugins": ["node-workspace"],
 	"changelog-sections": [
 		{
 			"type": "feat",
@@ -45,25 +43,27 @@
 		"packages/clients/peerbit-proxy/proxy": {},
 		"packages/clients/peerbit-server/node": {},
 		"packages/clients/test-utils": {},
+		"packages/clients/vite": {},
 		"packages/programs/rpc": {},
 		"packages/programs/acl/identity-access-controller": {},
 		"packages/programs/acl/trusted-network": {},
 		"packages/programs/program": {},
 		"packages/programs/clock-service": {},
-		"packages/programs/data/document/document": {},
-		"packages/programs/data/document/index/simple": {},
-		"packages/programs/data/document/index/sqlite3": {},
-		"packages/programs/data/document/index/tests": {},
-		"packages/programs/data/document/interface": {},
+		"packages/programs/data/document/indexer": {},
 		"packages/programs/data/string": {},
 		"packages/programs/data/shared-log": {},
+		"packages/programs/data/document/document/interface": {},
 		"packages/utils/crypto": {},
 		"packages/utils/any-store/any-store": {},
 		"packages/utils/any-store/interface": {},
 		"packages/utils/any-store/opfs": {},
 		"packages/utils/cache": {},
 		"packages/utils/time": {},
 		"packages/utils/logger": {},
-		"packages/utils/keychain": {}
+		"packages/utils/keychain": {},
+		"packages/utils/indexer/simple": {},
+		"packages/utils/indexer/sqlite3": {},
+		"packages/utils/indexer/tests": {},
+		"packages/utils/indexer/interface": {}
 	}
-}
+}

chai-global.js

@@ -1,7 +1,7 @@
 import { use } from "chai";
-import chaiAsPromised from 'chai-as-promised';
+import chaiAsPromised from "chai-as-promised";
+
 use(chaiAsPromised);
 /* 
 process.env.TS_NODE_PROJECT = 'tsconfig.test.json';
 console.log(process.env.TS_NODE_PROJECT) */
-

docs/.mocharc.json

@@ -1,22 +1,13 @@
 {
-  "require": [
-    "ts-node/register",
-    "chai-extend/chai-global.js"
-  ],
-  "node-option": [
-    "experimental-specifier-resolution=node",
-    "loader=ts-node/esm",
-    "no-warnings"
-  ],
-  "extension": [
-    "ts"
-  ],
-  "spec": [
-    "**/*.spec.ts"
-  ],
-  "ignore": [
-    "**/node_modules/**",
-    "dist/**"
-  ],
-  "timeout": "20000"
-}
+	"require": ["ts-node/register", "chai-extend/chai-global.js"],
+	"node-option": [
+		"experimental-specifier-resolution=node",
+		"loader=ts-node/esm",
+		"no-warnings"
+	],
+	"extension": ["ts"],
+	"spec": ["**/*.spec.ts"],
+	"ignore": ["**/node_modules/**", "dist/**"],
+	"timeout": "20000",
+	"exit": true
+}

docs/examples/document-store.ts

@@ -9,14 +9,15 @@
  * website to be able to render parts of the code.
  * If you are to copy code from this example, you can safely remove these
  */
-
 /// [imports]
 import { field, variant } from "@dao-xyz/borsh";
+import { Documents } from "@peerbit/document";
+import { SearchRequest } from "@peerbit/indexer-interface";
 import { Program } from "@peerbit/program";
+import assert from "node:assert";
 import { Peerbit } from "peerbit";
-import { Documents, SearchRequest } from "@peerbit/document";
 import { v4 as uuid } from "uuid";
-import assert from 'node:assert'
+
 /// [imports]
 
 /// [client]
@@ -57,7 +58,7 @@ class PostsDB extends Program {
 		// We need to setup the store in the setup hook
 		// we can also modify properties of our store here, for example set access control
 		await this.posts.open({
-			type: Post
+			type: Post,
 			// You can add more properties here, like
 			/* canPerform: (entry) => true */
 		});
@@ -84,19 +85,22 @@ await store.posts.log.waitForReplicator(peer2.identity.publicKey);
 
 const responses: Post[] = await store2.posts.index.search(
 	new SearchRequest({
-		query: [] // query all
+		query: [], // query all
 	}),
 	{
 		local: true,
-		remote: true
-	}
+		remote: true,
+	},
 );
 
 assert.equal(responses.length, 1);
-assert.deepEqual(responses.map((x) => x.message), ["hello world"]);
+assert.deepEqual(
+	responses.map((x) => x.message),
+	["hello world"],
+);
 /// [another-client]
 
 /// [disconnecting]
 await peer.stop();
 await peer2.stop();
-/// [disconnecting]
+/// [disconnecting]

docs/examples/text-store.ts

@@ -1,14 +1,14 @@
-import { Peerbit } from "peerbit";
-import { Program } from "@peerbit/program";
+import { field, variant } from "@dao-xyz/borsh";
 import { PublicSignKey } from "@peerbit/crypto";
+import { Program } from "@peerbit/program";
 import {
-	Range,
 	DString,
+	Range,
 	StringOperation,
-	type TransactionContext
+	type TransactionContext,
 } from "@peerbit/string";
-import { field, variant } from "@dao-xyz/borsh";
 import assert from "node:assert";
+import { Peerbit } from "peerbit";
 
 @variant("collaborative_text") // You have to give the program a unique name
 class CollaborativeText extends Program {
@@ -23,13 +23,13 @@ class CollaborativeText extends Program {
 	async open() {
 		await this.string.open({
 			canPerform: this.canPerform,
-			canRead: this.canRead
+			canRead: this.canRead,
 		});
 	}
 
 	async canPerform(
 		operation: StringOperation,
-		context: TransactionContext
+		context: TransactionContext,
 	): Promise<boolean> {
 		// .. acl logic writers
 		return true;

docs/modules/client/connectivity-direct.ts

@@ -1,6 +1,6 @@
-import { Peerbit } from "peerbit";
 import type { Multiaddr } from "@multiformats/multiaddr";
-import { expect } from 'chai';
+import { expect } from "chai";
+import { Peerbit } from "peerbit";
 
 const peerA = await Peerbit.create();
 const peerB = await Peerbit.create();
@@ -23,9 +23,7 @@ await peerA.dial(multaddrs);
 // Connected!
 
 // We can dial an address directly from a string. This one below is malformed and will fail
-await expect(peerA.dial("/ip4/123.4.5...")).rejectedWith(
-	"invalid ip address"
-);
+await expect(peerA.dial("/ip4/123.4.5...")).rejectedWith("invalid ip address");
 
 await peerA.stop();
 await peerB.stop();

docs/modules/client/example.ts

@@ -6,7 +6,7 @@ const clientInMemory = await Peerbit.create();
 
 // Store content on disc when possible
 const clientWithStorage = await Peerbit.create({
-	directory: "/path/somewhere"
+	directory: "/path/somewhere",
 });
 /// [memory]
 

docs/modules/encoding/borsh.ts

@@ -1,11 +1,12 @@
 import {
-	variant,
+	deserialize,
 	field,
 	fixedArray,
 	serialize,
-	deserialize
+	variant,
 } from "@dao-xyz/borsh";
 import { randomBytes } from "@peerbit/crypto";
+
 abstract class AbstractPost {}
 
 @variant(0) // V0