Merge pull request #852 from autonomys/feat/add-sub-query-dictionary

Add dictionary repo as submodule in indexers/dictionary
autonomys · Sep 22, 2024 · db7d193 · db7d193
2 parents e9791d5 + a673abd
commit db7d193
Show file tree

Hide file tree

Showing 60 changed files with 224 additions and 612 deletions.
diff --git a/.env b/.env
@@ -12,17 +12,15 @@ GEMINI_3G_RPC="wss://rpc-0.gemini-3g.subspace.network/ws"
 GEMINI_3H_CHAIN_ID="0x0c121c75f4ef450f40619e1fca9d1e8e7fbabc42c895bc4790801e85d5a91c34"
 GEMINI_3G_CHAIN_ID="0x418040fc282f5e5ddd432c46d05297636f6f75ce68d66499ff4cbda69ccd180b"
 
-DB_GEMINI_3H_CONSENSUS=gemini_3h_consensus
-DB_GEMINI_3H_ACCOUNTS=gemini_3h_accounts
-DB_GEMINI_3H_LEADERBOARD=gemini_3h_leaderboard
-DB_GEMINI_3H_STAKING=gemini_3h_staking
-DB_GEMINI_3H_TESTNET_REWARDS=gemini_3h_testnet_rewards
+DB_GEMINI_3H=gemini_3h
 
 DB_GEMINI_3G_TESTNET_REWARDS=gemini_3g_testnet_rewards
 
 HASURA_GRAPHQL_PORT=8080
 HASURA_GRAPHQL_ADMIN_SECRET=helloworld
 
 HASURA_GRAPHQL_ENABLE_CONSOLE=false
-HASURA_GRAPHQL_JWT_SECRET='{ "type": "HS256", "key": "this_is_a_mock_secret_with_32_chars" }'
-HASURA_GRAPHQL_CORS_DOMAIN="*"
+HASURA_GRAPHQL_JWT_SECRET='{"type":"HS256","key":"this_is_a_mock_secret_with_32_chars"}'
+HASURA_GRAPHQL_CORS_DOMAIN="*"
+
+GEMINI_3H_DICTIONARY_SUBQUERY_NODE_PORT=3010
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "indexers/dictionary"]
+	path = indexers/dictionary
+	url = https://github.com/autonomys/subql-dictionary
diff --git a/.vscode/astral.code-workspace b/.vscode/astral.code-workspace
@@ -41,6 +41,7 @@
     "cSpell.words": [
       "autonomys",
       "autoid",
+      "codegen",
       "hasura",
       "deregister",
       "extrinsics",

diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -14,6 +14,7 @@
   "cSpell.words": [
     "autonomys",
     "autoid",
+    "codegen",
     "hasura",
     "deregister",
     "extrinsics",

diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
@@ -12,37 +12,50 @@ services:
       options:
         loki-url: "https://logging.subspace.network/loki/api/v1/push"
 
-  hasura_engine:
+  hasura:
+    image: hasura/graphql-engine:v2.40.0.cli-migrations-v3
     logging:
       driver: loki
       options:
         loki-url: "https://logging.subspace.network/loki/api/v1/push"
 
-  gemini_3h_accounts-subquery-node:
+  gemini_3h_dictionary_subquery_node:
     logging:
       driver: loki
       options:
         loki-url: "https://logging.subspace.network/loki/api/v1/push"
 
-  gemini_3h_consensus-subquery-node:
+  gemini_3h_dictionary_graphql_engine:
     logging:
       driver: loki
       options:
         loki-url: "https://logging.subspace.network/loki/api/v1/push"
 
-  gemini_3h_leaderboard-subquery-node:
+  gemini_3h_accounts_subquery_node:
     logging:
       driver: loki
       options:
         loki-url: "https://logging.subspace.network/loki/api/v1/push"
 
-  gemini_3h_staking-subquery-node:
+  gemini_3h_consensus_subquery_node:
     logging:
       driver: loki
       options:
         loki-url: "https://logging.subspace.network/loki/api/v1/push"
 
-  gemini_3g_testnet_rewards-subquery-node:
+  gemini_3h_leaderboard_subquery_node:
+    logging:
+      driver: loki
+      options:
+        loki-url: "https://logging.subspace.network/loki/api/v1/push"
+
+  gemini_3h_staking_subquery_node:
+    logging:
+      driver: loki
+      options:
+        loki-url: "https://logging.subspace.network/loki/api/v1/push"
+
+  gemini_3g_testnet_rewards_subquery_node:
     logging:
       driver: loki
       options:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -25,7 +25,7 @@ services:
 
   # Hasura GraphQL Engine
   hasura:
-    image: hasura/graphql-engine:v2.40.0.cli-migrations-v3
+    image: hasura/graphql-engine:v2.40.0
     depends_on:
       - "postgres"
     volumes:
@@ -34,8 +34,8 @@ services:
     restart: unless-stopped
     environment:
       # Essential Environment Variables
-      HASURA_GRAPHQL_METADATA_DATABASE_URL: postgres://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:5432/${DB_GEMINI_3H_CONSENSUS}
-      HASURA_GRAPHQL_DATABASE_URL: postgres://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:5432/${DB_GEMINI_3H_CONSENSUS} # Main database connection
+      HASURA_GRAPHQL_METADATA_DATABASE_URL: postgres://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:5432/${DB_DATABASE}
+      HASURA_GRAPHQL_DATABASE_URL: postgres://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:5432/${DB_DATABASE} # Main database connection
       HASURA_GRAPHQL_ADMIN_SECRET: ${HASURA_GRAPHQL_ADMIN_SECRET} # Admin access secret
       HASURA_GRAPHQL_JWT_SECRET: ${HASURA_GRAPHQL_JWT_SECRET} # JWT authentication secret
 
@@ -63,12 +63,7 @@ services:
       HASURA_GRAPHQL_ENABLE_ALLOWLIST: "true" # Enable query allowlisting
 
       # Gemini 3H Databases
-      HASURA_GRAPHQL_GEMINI_3H_CONSENSUS_DATABASE_URL: postgres://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:5432/${DB_GEMINI_3H_CONSENSUS}
-      HASURA_GRAPHQL_GEMINI_3H_ACCOUNTS_DATABASE_URL: postgres://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:5432/${DB_GEMINI_3H_ACCOUNTS}
-      HASURA_GRAPHQL_GEMINI_3H_LEADERBOARD_DATABASE_URL: postgres://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:5432/${DB_GEMINI_3H_LEADERBOARD}
-      HASURA_GRAPHQL_GEMINI_3H_STAKING_DATABASE_URL: postgres://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:5432/${DB_GEMINI_3H_STAKING}
-      HASURA_GRAPHQL_GEMINI_3H_TESTNET_REWARDS_DATABASE_URL: postgres://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:5432/${DB_GEMINI_3H_TESTNET_REWARDS}
-
+      HASURA_GRAPHQL_GEMINI_3H_DATABASE_URL: postgres://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:5432/${DB_GEMINI_3H}
       # Gemini 3G Database
       HASURA_GRAPHQL_GEMINI_3G_TESTNET_REWARDS_DATABASE_URL: postgres://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:5432/${DB_GEMINI_3G_TESTNET_REWARDS}
 
@@ -81,132 +76,219 @@ services:
       - graphql-engine
       - serve
 
+  # Gemini 3H Dictionary
+  gemini_3h_dictionary_subquery_node:
+    profiles: [dictionary, gemini_3h, gemini_3h_dictionary]
+    build:
+      context: ./indexers/dictionary/autonomys-gemini-3h
+    depends_on:
+      "postgres":
+        condition: service_healthy
+    restart: unless-stopped
+    environment:
+      DB_USER: ${DB_USER}
+      DB_PASS: ${DB_PASSWORD}
+      DB_DATABASE: ${DB_GEMINI_3H}
+      DB_HOST: ${DB_HOST}
+      DB_PORT: 5432
+    volumes:
+      - ./indexers/dictionary/autonomys-gemini-3h:/dictionary
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "curl",
+          "-f",
+          "http://gemini_3h_dictionary_subquery_node:3000/ready",
+        ]
+      interval: 3s
+      timeout: 5s
+      retries: 10
+    command:
+      - -f=/dictionary
+      - --disable-historical=true
+
+  gemini_3h_dictionary_graphql_engine:
+    profiles: [dictionary, gemini_3h, gemini_3h_dictionary]
+    image: onfinality/subql-query:latest
+    ports:
+      - "${GEMINI_3H_DICTIONARY_SUBQUERY_NODE_PORT}:3000"
+    depends_on:
+      - "postgres"
+      - "gemini_3h_dictionary_subquery_node"
+    restart: unless-stopped
+    environment:
+      DB_USER: ${DB_USER}
+      DB_PASS: ${DB_PASSWORD}
+      DB_DATABASE: ${DB_GEMINI_3H}
+      DB_HOST: ${DB_HOST}
+      DB_PORT: 5432
+    command:
+      - --name=dictionary
+      - --playground
+
   # Gemini 3H Subquery Nodes
-  gemini_3h_accounts-subquery-node:
+  gemini_3h_accounts_subquery_node:
     profiles: [gemini_3h, gemini_3h_accounts]
     image: subquerynetwork/subql-node-substrate:latest
     depends_on:
       "postgres":
         condition: service_healthy
+      "gemini_3h_dictionary_subquery_node":
+        condition: service_healthy
     restart: unless-stopped
     environment:
       ENDPOINT: ${GEMINI_3H_RPC}
       CHAIN_ID: ${GEMINI_3H_CHAIN_ID}
+      DICTIONARY: http://gemini_3h_dictionary_subquery_node:3000
       DB_USER: ${DB_USER}
       DB_PASS: ${DB_PASSWORD}
-      DB_DATABASE: ${DB_GEMINI_3H_ACCOUNTS}
+      DB_DATABASE: ${DB_GEMINI_3H}
       DB_HOST: ${DB_HOST}
       DB_PORT: 5432
     volumes:
       - ./indexers/gemini-3h/accounts:/gemini-3h/accounts
     command:
       - ${SUB_COMMAND:-} # set SUB_COMMAND env variable to "test" to run tests
       - -f=/gemini-3h/accounts
-      - --db-schema=${DB_GEMINI_3H_ACCOUNTS}
+      - --db-schema=accounts
       - --workers=4
       - --unsafe
       - --batch-size=30
       - --unfinalized-blocks=true
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://accounts-subquery-node:3000/ready"]
+      test:
+        [
+          "CMD",
+          "curl",
+          "-f",
+          "http://gemini_3h_accounts_subquery_node:3000/ready",
+        ]
       interval: 3s
       timeout: 5s
       retries: 10
 
-  gemini_3h_consensus-subquery-node:
+  gemini_3h_consensus_subquery_node:
     profiles: [gemini_3h, gemini_3h_consensus]
     image: subquerynetwork/subql-node-substrate:latest
     depends_on:
       "postgres":
         condition: service_healthy
+      "gemini_3h_dictionary_subquery_node":
+        condition: service_healthy
     restart: unless-stopped
     environment:
       ENDPOINT: ${GEMINI_3H_RPC}
       CHAIN_ID: ${GEMINI_3H_CHAIN_ID}
+      DICTIONARY: http://gemini_3h_dictionary_subquery_node:3000
       DB_USER: ${DB_USER}
       DB_PASS: ${DB_PASSWORD}
-      DB_DATABASE: ${DB_GEMINI_3H_CONSENSUS}
+      DB_DATABASE: ${DB_GEMINI_3H}
       DB_HOST: ${DB_HOST}
       DB_PORT: 5432
     volumes:
       - ./indexers/gemini-3h/consensus:/gemini-3h/consensus
     command:
       - ${SUB_COMMAND:-} # set SUB_COMMAND env variable to "test" to run tests
       - -f=/gemini-3h/consensus
-      - --db-schema=${DB_GEMINI_3H_CONSENSUS}
+      - --db-schema=consensus
       - --workers=4
       - --unsafe
       - --batch-size=30
       - --unfinalized-blocks=true
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://consensus-subquery-node:3000/ready"]
+      test:
+        [
+          "CMD",
+          "curl",
+          "-f",
+          "http://gemini_3h_consensus_subquery_node:3000/ready",
+        ]
       interval: 3s
       timeout: 5s
       retries: 10
 
-  gemini_3h_leaderboard-subquery-node:
+  gemini_3h_leaderboard_subquery_node:
     profiles: [gemini_3h, gemini_3h_leaderboard]
     image: subquerynetwork/subql-node-substrate:latest
     depends_on:
       "postgres":
         condition: service_healthy
+      "gemini_3h_dictionary_subquery_node":
+        condition: service_healthy
     restart: unless-stopped
     environment:
       ENDPOINT: ${GEMINI_3H_RPC}
       CHAIN_ID: ${GEMINI_3H_CHAIN_ID}
+      DICTIONARY: http://gemini_3h_dictionary_subquery_node:3000
       DB_USER: ${DB_USER}
       DB_PASS: ${DB_PASSWORD}
-      DB_DATABASE: ${DB_GEMINI_3H_LEADERBOARD}
+      DB_DATABASE: ${DB_GEMINI_3H}
       DB_HOST: ${DB_HOST}
       DB_PORT: 5432
     volumes:
       - ./indexers/gemini-3h/leaderboard:/gemini-3h/leaderboard
     command:
       - ${SUB_COMMAND:-} # set SUB_COMMAND env variable to "test" to run tests
       - -f=/gemini-3h/leaderboard
-      - --db-schema=${DB_GEMINI_3H_LEADERBOARD}
+      - --db-schema=leaderboard
       - --workers=4
       - --unsafe
       - --batch-size=30
       - --unfinalized-blocks=true
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://leaderboard-subquery-node:3000/ready"]
+      test:
+        [
+          "CMD",
+          "curl",
+          "-f",
+          "http://gemini_3h_leaderboard_subquery_node:3000/ready",
+        ]
       interval: 3s
       timeout: 5s
       retries: 10
 
-  gemini_3h_staking-subquery-node:
+  gemini_3h_staking_subquery_node:
     profiles: [gemini_3h, gemini_3h_staking]
     image: subquerynetwork/subql-node-substrate:latest
     depends_on:
       "postgres":
         condition: service_healthy
+      "gemini_3h_dictionary_subquery_node":
+        condition: service_healthy
     restart: unless-stopped
     environment:
       ENDPOINT: ${GEMINI_3H_RPC}
       CHAIN_ID: ${GEMINI_3H_CHAIN_ID}
+      DICTIONARY: http://gemini_3h_dictionary_subquery_node:3000
       DB_USER: ${DB_USER}
       DB_PASS: ${DB_PASSWORD}
-      DB_DATABASE: ${DB_GEMINI_3H_STAKING}
+      DB_DATABASE: ${DB_GEMINI_3H}
       DB_HOST: ${DB_HOST}
       DB_PORT: 5432
     volumes:
       - ./indexers/gemini-3h/staking:/gemini-3h/staking
     command:
       - ${SUB_COMMAND:-} # set SUB_COMMAND env variable to "test" to run tests
       - -f=/gemini-3h/staking
-      - --db-schema=${DB_GEMINI_3H_STAKING}
+      - --db-schema=staking
       - --workers=4
       - --unsafe
       - --batch-size=30
       - --unfinalized-blocks=true
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://staking-subquery-node:3000/ready"]
+      test:
+        [
+          "CMD",
+          "curl",
+          "-f",
+          "http://gemini_3h_staking_subquery_node:3000/ready",
+        ]
       interval: 3s
       timeout: 5s
       retries: 10
 
-  gemini_3g_testnet_rewards-subquery-node:
+  gemini_3g_testnet_rewards_subquery_node:
     profiles: [gemini_3g, gemini_3g_testnet_rewards]
     image: subquerynetwork/subql-node-substrate:latest
     depends_on:
@@ -233,7 +315,12 @@ services:
       - --unfinalized-blocks=true
     healthcheck:
       test:
-        ["CMD", "curl", "-f", "http://testnet-rewards-subquery-node:3000/ready"]
+        [
+          "CMD",
+          "curl",
+          "-f",
+          "http://gemini_3g_testnet_rewards_subquery_node:3000/ready",
+        ]
       interval: 3s
       timeout: 5s
       retries: 10
diff --git a/indexers/.gitignore b/indexers/.gitignore
@@ -1 +1,3 @@
-node_modules/
+node_modules/
+
+.nx/*