diff --git a/kylin/.gitignore b/kylin/.gitignore new file mode 100644 index 0000000..4c37193 --- /dev/null +++ b/kylin/.gitignore @@ -0,0 +1,52 @@ +# These are some examples of commonly ignored file patterns. +# You should customize this list as applicable to your project. +# Learn more about .gitignore: +# https://www.atlassian.com/git/tutorials/saving-changes/gitignore + +# Node artifact files +node_modules/ +dist/ + +# Compiled Java class files +*.class + +# Compiled Python bytecode +*.py[cod] + +# Log files +*.log + +# Package files +*.jar + +# Maven +target/ +dist/ +src/types + +# JetBrains IDE +.idea/ + +# Unit test reports +TEST*.xml + +# Generated by MacOS +.DS_Store + +# Generated by Windows +Thumbs.db + +# Applications +*.app +*.exe +*.war + +# Large media files +*.mp4 +*.tiff +*.avi +*.flv +*.mov +*.wmv + +.data \ No newline at end of file diff --git a/kylin/README.md b/kylin/README.md new file mode 100644 index 0000000..1237e25 --- /dev/null +++ b/kylin/README.md @@ -0,0 +1,28 @@ +# SubQuery - Dictionary + +This special SubQuery Project provides a dictionary of data that pre-indexes events on chain to dramatically improve indexing the performance of your own SubQuery Project, sometimes up to 10x faster. + +It scans over the network, and simply records the module and method for every event/extrinsic on each block - please see the standard entities in `schema.graphql`. + +**If you want to create your SubQuery Dictionary to speed up indexing of your own Substrate chain, fork this project and let us know** + +# Geting Started +### 1. Install dependencies +```shell +yarn +``` + +### 2. Generate types +```shell +yarn codegen +``` + +### 3. Build +```shell +yarn build +``` + +### 4. Run locally +```shell +yarn start:docker +``` diff --git a/kylin/docker-compose.yml b/kylin/docker-compose.yml new file mode 100644 index 0000000..9d56aeb --- /dev/null +++ b/kylin/docker-compose.yml @@ -0,0 +1,44 @@ +version: '3' + +services: + postgres: + image: postgres:12-alpine + ports: + - 5432:5432 + volumes: + - .data/postgres:/var/lib/postgresql/data + environment: + POSTGRES_PASSWORD: postgres + + subquery-node: + image: onfinality/subql-node:latest + depends_on: + - "postgres" + restart: always + environment: + DB_USER: postgres + DB_PASS: postgres + DB_DATABASE: postgres + DB_HOST: postgres + DB_PORT: 5432 + volumes: + - ./:/app + command: + - -f=/app + + graphql-engine: + image: onfinality/subql-query:latest + ports: + - 3000:3000 + depends_on: + - "postgres" + restart: always + environment: + DB_USER: postgres + DB_PASS: postgres + DB_DATABASE: postgres + DB_HOST: postgres + DB_PORT: 5432 + command: + - --name=app + - --playground diff --git a/kylin/package.json b/kylin/package.json new file mode 100644 index 0000000..a8d2bbe --- /dev/null +++ b/kylin/package.json @@ -0,0 +1,31 @@ +{ + "name": "subql-dictionary", + "version": "1.0.0", + "description": "A SubQuery Dictionary Project that provides increased indexing speed to all projects", + "main": "dist/index.js", + "scripts": { + "build": "tsc -b", + "prepack": "rm -rf dist && npm build", + "test": "jest", + "codegen": "./node_modules/.bin/subql codegen", + "start:docker": "docker-compose pull && docker-compose up --remove-orphans" + }, + "homepage": "https://github.com/subquery/subql-dictionary", + "repository": "github:subquery/subql-dictionary", + "files": [ + "dist", + "schema.graphql", + "project.yaml" + ], + "author": "SubQuery Network", + "license": "Apache-2.0", + "devDependencies": { + "@polkadot/api": "^7", + "@subql/types": "latest", + "typescript": "^4.1.3", + "@subql/cli": "latest" + }, + "exports": { + "chaintypes": "src/chaintypes.ts" + } +} diff --git a/kylin/project.yaml b/kylin/project.yaml new file mode 100644 index 0000000..1ceeb2c --- /dev/null +++ b/kylin/project.yaml @@ -0,0 +1,20 @@ +specVersion: 0.2.0 +name: SubQuery Dictionary - Kylin +version: 1.0.0 +description: "A SubQuery Dictionary Project that provides increased indexing speed to all projects" +repository: https://github.com/subquery/subql-dictionary +schema: + file: ./schema.graphql +network: + genesisHash: "0xdb034d86d4228b2c5b007a7b363446cf918d473962325dd08d80995827fcc9f7" + endpoint: wss://pichiu-rococo-01.onebitdev.com/public-ws + chaintypes: + file: ./dist/chaintypes.js +dataSources: + - kind: substrate/Runtime + startBlock: 1 + mapping: + file: ./dist/index.js + handlers: + - handler: handleBlock + kind: substrate/BlockHandler diff --git a/kylin/schema.graphql b/kylin/schema.graphql new file mode 100644 index 0000000..4306706 --- /dev/null +++ b/kylin/schema.graphql @@ -0,0 +1,21 @@ +type SpecVersion @entity { + id: ID! #specVersion + blockHeight: BigInt! +} + +type Event @entity { + id: ID! + module: String! @index + event: String! @index + blockHeight: BigInt! @index +} + +type Extrinsic @entity { + id: ID! + txHash: String! @index + module: String! @index + call: String! @index + blockHeight: BigInt! @index + success: Boolean! + isSigned: Boolean! +} diff --git a/kylin/src/chaintypes.ts b/kylin/src/chaintypes.ts new file mode 100644 index 0000000..cd01ee4 --- /dev/null +++ b/kylin/src/chaintypes.ts @@ -0,0 +1,30 @@ +import type { OverrideBundleDefinition } from "@polkadot/types/types"; + +// structs need to be in order +/* eslint-disable sort-keys */ + +const definitions: OverrideBundleDefinition = { + types: [ + { + // on all versions + minmax: [0, undefined], + types: { + "Address": "MultiAddress", + "LookupSource": "MultiAddress", + "DataRequest": { + "para_id": "Option", + "account_id": "Option", + "requested_block_number": "BlockNumber", + "processed_block_number": "Option", + "requested_timestamp": "u128", + "processed_timestamp": "Option", + "payload": "Text", + "feed_name": "Text", + "url": "Text" + } + }, + }, + ], +}; + +export default { typesBundle: definitions }; diff --git a/kylin/src/index.ts b/kylin/src/index.ts new file mode 100644 index 0000000..5091615 --- /dev/null +++ b/kylin/src/index.ts @@ -0,0 +1,3 @@ +//Exports all handler functions +export * from "./mappings/mappingHandlers"; +import "@polkadot/api-augment"; diff --git a/kylin/src/mappings/mappingHandlers.ts b/kylin/src/mappings/mappingHandlers.ts new file mode 100644 index 0000000..22cccff --- /dev/null +++ b/kylin/src/mappings/mappingHandlers.ts @@ -0,0 +1,79 @@ +import { EventRecord } from "@polkadot/types/interfaces"; +import { SubstrateExtrinsic, SubstrateBlock } from "@subql/types"; +import { SpecVersion, Event, Extrinsic } from "../types"; + +let specVersion: SpecVersion; +export async function handleBlock(block: SubstrateBlock): Promise { + // Initialise Spec Version + if (!specVersion) { + specVersion = await SpecVersion.get(block.specVersion.toString()); + } + + // Check for updates to Spec Version + if (!specVersion || specVersion.id !== block.specVersion.toString()) { + specVersion = new SpecVersion(block.specVersion.toString()); + specVersion.blockHeight = block.block.header.number.toBigInt(); + await specVersion.save(); + } + + // Process all events in block + const events = block.events + .filter( + (evt) => + !(evt.event.section === "system" && + evt.event.method === "ExtrinsicSuccess") + ) + .map((evt, idx) => + handleEvent(block.block.header.number.toString(), idx, evt) + ); + + // Process all calls in block + const calls = wrapExtrinsics(block).map((ext, idx) => + handleCall(`${block.block.header.number.toString()}-${idx}`, ext) + ); + + // Save all data + await Promise.all([ + store.bulkCreate("Event", events), + store.bulkCreate("Extrinsic", calls), + ]); +} + +function handleEvent( + blockNumber: string, + eventIdx: number, + event: EventRecord +): Event { + const newEvent = new Event(`${blockNumber}-${eventIdx}`); + newEvent.blockHeight = BigInt(blockNumber); + newEvent.module = event.event.section; + newEvent.event = event.event.method; + return newEvent; +} + +function handleCall(idx: string, extrinsic: SubstrateExtrinsic): Extrinsic { + const newExtrinsic = new Extrinsic(idx); + newExtrinsic.txHash = extrinsic.extrinsic.hash.toString(); + newExtrinsic.module = extrinsic.extrinsic.method.section; + newExtrinsic.call = extrinsic.extrinsic.method.method; + newExtrinsic.blockHeight = extrinsic.block.block.header.number.toBigInt(); + newExtrinsic.success = extrinsic.success; + newExtrinsic.isSigned = extrinsic.extrinsic.isSigned; + return newExtrinsic; +} + +function wrapExtrinsics(wrappedBlock: SubstrateBlock): SubstrateExtrinsic[] { + return wrappedBlock.block.extrinsics.map((extrinsic, idx) => { + const events = wrappedBlock.events.filter( + ({ phase }) => phase.isApplyExtrinsic && phase.asApplyExtrinsic.eqn(idx) + ); + return { + idx, + extrinsic, + block: wrappedBlock, + events, + success: + events.findIndex((evt) => evt.event.method === "ExtrinsicSuccess") > -1, + }; + }); +} diff --git a/kylin/tsconfig.json b/kylin/tsconfig.json new file mode 100644 index 0000000..9b95212 --- /dev/null +++ b/kylin/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "emitDecoratorMetadata": true, + "experimentalDecorators": true, + "esModuleInterop": true, + "declaration": true, + "importHelpers": true, + "resolveJsonModule": true, + "module": "commonjs", + "outDir": "dist", + "rootDir": "src", + "target": "es2017" + }, + "include": [ + "src/**/*", + "node_modules/@subql/types/dist/global.d.ts" + ] +} diff --git a/kylin/types.json b/kylin/types.json new file mode 100644 index 0000000..ca20bc7 --- /dev/null +++ b/kylin/types.json @@ -0,0 +1,15 @@ +{ + "Address": "MultiAddress", + "LookupSource": "MultiAddress", + "DataRequest": { + "para_id": "Option", + "account_id": "Option", + "requested_block_number": "BlockNumber", + "processed_block_number": "Option", + "requested_timestamp": "u128", + "processed_timestamp": "Option", + "payload": "Text", + "feed_name": "Text", + "url": "Text" + } +} \ No newline at end of file