Merge pull request #854 from autonomys/feat/fix-accounts-sub-query

Fix Accounts Sub Query Node
autonomys · Sep 24, 2024 · de09818 · de09818
2 parents 766bea2 + ea8e09a
commit de09818
Show file tree

Hide file tree

Showing 30 changed files with 2,616 additions and 158 deletions.
diff --git a/.env b/.env
@@ -24,3 +24,9 @@ HASURA_GRAPHQL_JWT_SECRET='{"type":"HS256","key":"this_is_a_mock_secret_with_32_
 HASURA_GRAPHQL_CORS_DOMAIN="*"
 
 GEMINI_3H_DICTIONARY_SUBQUERY_NODE_PORT=3010
+
+REDIS_PORT=6379
+
+BULL_USERNAME=bull
+BULL_PASSWORD=board
+BULL_PORT=3020
diff --git a/.vscode/astral.code-workspace b/.vscode/astral.code-workspace
@@ -52,6 +52,7 @@
       "subscan",
       "subspace",
       "subsquid",
+      "taskboard",
       "typegen"
     ]
   }

diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -25,6 +25,7 @@
     "subscan",
     "subspace",
     "subsquid",
+    "taskboard",
     "typegen"
   ]
 }
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
@@ -60,3 +60,15 @@ services:
       driver: loki
       options:
         loki-url: "https://logging.subspace.network/loki/api/v1/push"
+
+  redis:
+    logging:
+      driver: loki
+      options:
+        loki-url: "https://logging.subspace.network/loki/api/v1/push"
+
+  taskboard:
+    logging:
+      driver: loki
+      options:
+        loki-url: "https://logging.subspace.network/loki/api/v1/push"
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -2,6 +2,7 @@ version: "3.9"
 
 volumes:
   postgres_db: {}
+  redis_db: {}
 
 services:
   # Postgres Database
@@ -11,10 +12,12 @@ services:
       - postgres_db:/var/lib/postgresql/data
       - ./indexers/db/docker-entrypoint-initdb.d/init-db.sql:/docker-entrypoint-initdb.d/init-db.sql
     restart: unless-stopped
+    hostname: postgres
     environment:
       POSTGRES_DB: ${DB_DATABASE}
       POSTGRES_USER: ${DB_USER}
       POSTGRES_PASSWORD: ${DB_PASSWORD}
+      POSTGRES_MAX_CONNECTIONS: 500
     ports:
       - "${DB_PORT}:5432"
     healthcheck:
@@ -324,3 +327,54 @@ services:
       interval: 3s
       timeout: 5s
       retries: 10
+
+  # Redis
+  redis:
+    profiles: [task]
+    image: redis:latest
+    restart: unless-stopped
+    ports:
+      - "${REDIS_PORT}:6379"
+    volumes:
+      - redis_db:/data
+    hostname: redis
+    command: ["redis-server", "--bind", "0.0.0.0", "--port", "6379"]
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "redis-cli -h localhost -p 6379 ping | grep PONG || exit 1",
+        ]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+
+  # Express + BullMQ
+  taskboard:
+    profiles: [task]
+    build:
+      context: ./indexers/taskboard
+    depends_on:
+      redis:
+        condition: service_healthy
+    volumes:
+      - ./indexers/taskboard:/app
+    hostname: taskboard
+    restart: unless-stopped
+    ports:
+      - "${BULL_PORT}:3000"
+    environment:
+      BULL_USERNAME: ${BULL_USERNAME}
+      BULL_PASSWORD: ${BULL_PASSWORD}
+      BULL_PORT: ${BULL_PORT}
+
+      REDIS_HOST: redis
+      REDIS_PORT: ${REDIS_PORT}
+
+      DB_USER: ${DB_USER}
+      DB_PASSWORD: ${DB_PASSWORD}
+      DB_DATABASE: ${DB_DATABASE}
+      DB_HOST: ${DB_HOST}
+      DB_PORT: ${DB_PORT}
+
+      DB_GEMINI_3H: ${DB_GEMINI_3H}
diff --git a/indexers/db/metadata/databases/gemini_3h/tables/accounts_accounts.yaml b/indexers/db/metadata/databases/gemini_3h/tables/accounts_accounts.yaml
@@ -0,0 +1,20 @@
+table:
+  name: accounts
+  schema: accounts
+select_permissions:
+  - role: user
+    permission:
+      columns:
+        - _block_range
+        - created_at
+        - free
+        - nonce
+        - reserved
+        - total
+        - updated_at
+        - account_id
+        - id
+        - _id
+      filter: {}
+      allow_aggregations: true
+    comment: ""
diff --git a/indexers/db/metadata/databases/gemini_3h/tables/accounts_balance_histories.yaml b/indexers/db/metadata/databases/gemini_3h/tables/accounts_balance_histories.yaml
@@ -0,0 +1,18 @@
+table:
+  name: balance_histories
+  schema: accounts
+select_permissions:
+  - role: user
+    permission:
+      columns:
+        - _block_range
+        - created_at
+        - free
+        - reserved
+        - total
+        - account_id
+        - id
+        - _id
+      filter: {}
+      allow_aggregations: true
+    comment: ""
diff --git a/indexers/db/metadata/databases/gemini_3h/tables/accounts_transfers.yaml b/indexers/db/metadata/databases/gemini_3h/tables/accounts_transfers.yaml
@@ -0,0 +1,23 @@
+table:
+  name: transfers
+  schema: accounts
+select_permissions:
+  - role: user
+    permission:
+      columns:
+        - success
+        - _block_range
+        - created_at
+        - fee
+        - timestamp
+        - value
+        - event_id
+        - extrinsic_id
+        - from
+        - id
+        - to
+        - date
+        - _id
+      filter: {}
+      allow_aggregations: true
+    comment: ""
diff --git a/indexers/db/metadata/databases/gemini_3h/tables/tables.yaml b/indexers/db/metadata/databases/gemini_3h/tables/tables.yaml
@@ -1,3 +1,6 @@
+- "!include accounts_accounts.yaml"
+- "!include accounts_balance_histories.yaml"
+- "!include accounts_transfers.yaml"
 - "!include consensus_blocks.yaml"
 - "!include consensus_event_modules.yaml"
 - "!include consensus_events.yaml"

diff --git a/indexers/gemini-3h/accounts/package.json b/indexers/gemini-3h/accounts/package.json
@@ -31,13 +31,17 @@
   },
   "license": "MIT",
   "devDependencies": {
+    "@autonomys/auto-consensus": "^0.5.0",
+    "@autonomys/auto-utils": "^0.5.0",
     "@polkadot/api": "^12",
     "@subql/cli": "latest",
     "@subql/common-substrate": "latest",
     "@subql/testing": "latest",
     "@subql/types": "latest",
     "@types/node": "latest",
+    "axios": "^1.7.7",
     "dotenv": "latest",
+    "http": "^0.0.1-security",
     "typescript": "^5.2.2"
   },
   "resolutions": {

diff --git a/indexers/gemini-3h/accounts/project.ts b/indexers/gemini-3h/accounts/project.ts
@@ -59,67 +59,23 @@ const project: SubstrateProject = {
             },
           },
           {
-            kind: SubstrateHandlerKind.Event,
-            handler: "handleTransferEvent",
-            filter: {
-              module: "system",
-              method: "ExtrinsicSuccess",
-            },
-          },
-          {
-            kind: SubstrateHandlerKind.Event,
-            handler: "handleTransferEvent",
-            filter: {
-              module: "transactionPayment",
-              method: "TransactionFeePaid",
-            },
-          },
-          {
-            kind: SubstrateHandlerKind.Event,
-            handler: "handleTransferEvent",
-            filter: {
-              module: "domains",
-              method: "OperatorRewarded",
-            },
-          },
-          {
-            kind: SubstrateHandlerKind.Event,
-            handler: "handleTransferEvent",
-            filter: {
-              module: "domains",
-              method: "OperatorTaxCollected",
-            },
-          },
-          {
-            kind: SubstrateHandlerKind.Event,
-            handler: "handleTransferEvent",
-            filter: {
-              module: "domains",
-              method: "OperatorRegistered",
-            },
+            kind: SubstrateHandlerKind.Call,
+            handler: "handleExtrinsic",
           },
           {
             kind: SubstrateHandlerKind.Event,
-            handler: "handleTransferEvent",
+            handler: "handleFarmerVoteRewardEvent",
             filter: {
-              module: "domains",
-              method: "OperatorNominated",
+              module: "rewards",
+              method: "VoteReward",
             },
           },
           {
             kind: SubstrateHandlerKind.Event,
-            handler: "handleTransferEvent",
-            filter: {
-              module: "domains",
-              method: "WithdrewStake",
-            },
-          },
-          {
-            kind: SubstrateHandlerKind.Event,
-            handler: "handleTransferEvent",
+            handler: "handleFarmerBlockRewardEvent",
             filter: {
-              module: "domains",
-              method: "StorageFeeDeposited",
+              module: "rewards",
+              method: "BlockReward",
             },
           },
         ],

diff --git a/indexers/gemini-3h/accounts/schema.graphql b/indexers/gemini-3h/accounts/schema.graphql
@@ -4,12 +4,22 @@
 
 type Account @entity {
   id: ID!
+  accountId: String! @index
   nonce: BigInt!
   free: BigInt!
   reserved: BigInt!
   total: BigInt @index
-  createdAt: Int! @index
-  updatedAt: Int! @index
+  createdAt: BigInt! @index
+  updatedAt: BigInt! @index
+}
+
+type BalanceHistory @entity {
+  id: ID!
+  accountId: String! @index
+  free: BigInt!
+  reserved: BigInt!
+  total: BigInt @index
+  createdAt: BigInt! @index
 }
 
 type Transfer @entity {
@@ -23,5 +33,5 @@ type Transfer @entity {
   success: Boolean!
   timestamp: BigInt! @index
   date: Date! @index
-  createdAt: Int! @index
+  createdAt: BigInt! @index
 }
diff --git a/indexers/gemini-3h/accounts/src/mappings/db.ts b/indexers/gemini-3h/accounts/src/mappings/db.ts
@@ -0,0 +1,82 @@
+import { Account, BalanceHistory, Transfer } from "../types";
+import { dateEntry } from "./utils";
+
+export async function createAndSaveAccountIfNotExists(
+  accountId: string,
+  blockNumber: bigint,
+  nonce: bigint,
+  free: bigint,
+  reserved: bigint,
+  total: bigint
+): Promise<Account> {
+  const id = accountId.toLowerCase();
+  const accounts = await Account.getByAccountId(id);
+  let account = accounts ? accounts[0] : undefined;
+  if (!account) {
+    account = Account.create({
+      id,
+      accountId: id,
+      nonce,
+      free,
+      reserved,
+      total,
+      ...dateEntry(blockNumber),
+    });
+    await account.save();
+  }
+  return account;
+}
+
+export async function createAndSaveBalanceHistory(
+  accountId: string,
+  blockNumber: bigint,
+  free: bigint,
+  reserved: bigint,
+  total: bigint
+): Promise<BalanceHistory> {
+  const id = accountId.toLowerCase() + "-" + blockNumber.toString();
+  const balanceHistory = BalanceHistory.create({
+    id,
+    accountId: accountId.toLowerCase(),
+    free,
+    reserved,
+    total,
+    createdAt: blockNumber,
+  });
+  await balanceHistory.save();
+  return balanceHistory;
+}
+
+export async function createAndSaveTransfer(
+  blockNumber: bigint,
+  extrinsicId: string,
+  eventId: string,
+  from: string,
+  to: string,
+  value: bigint,
+  fee: bigint,
+  success: boolean,
+  timestamp: bigint,
+  date: Date
+): Promise<Transfer> {
+  const id = blockNumber + "-" + extrinsicId.toLowerCase();
+  const transfers = await Transfer.getByExtrinsicId(id);
+  let transfer = transfers ? transfers[0] : undefined;
+  if (!transfer) {
+    transfer = Transfer.create({
+      id,
+      extrinsicId,
+      eventId,
+      from,
+      to,
+      value,
+      fee,
+      success,
+      timestamp,
+      date,
+      createdAt: blockNumber,
+    });
+    await transfer.save();
+  }
+  return transfer;
+}