Add subsquid parquet indexing

Synthetixio · Sep 27, 2024 · 49da37b · 49da37b
1 parent 38fc5be
commit 49da37b
Show file tree

Hide file tree

Showing 29 changed files with 25,205 additions and 289 deletions.
diff --git a/.gitignore b/.gitignore
@@ -175,6 +175,7 @@ logs
 # local data
 postgres-data
 parquet-data
+parquet-data-indexers
 data
 
 # streamlit

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -19,6 +19,7 @@ services:
       - ./postgres/initdb:/docker-entrypoint-initdb.d
       - ./postgres-data:/var/lib/postgresql/data
       - ./parquet-data:/parquet-data
+      - ./parquet-data-indexers:/parquet-data-indexers
     ports:
       - "${DB_PORT}:5432"
     deploy:
@@ -80,16 +81,12 @@ services:
       context: ./indexers/arbitrum-sepolia
     networks:
       - data
-    depends_on:
-      - db
     restart: always
     environment:
-      DB_HOST: db
-      DB_PORT: 5432
-      DB_NAME: arbitrum_sepolia
-      DB_PASS: $PG_PASSWORD
       GQL_PORT: 4350
       RPC_ENDPOINT: https://sepolia-rollup.arbitrum.io/rpc
+    volumes:
+      - ./parquet-data-indexers:/parquet-data-indexers
 
   arbitrum-mainnet-processor:
     build:
@@ -167,6 +164,7 @@ services:
     volumes:
       - ./parquet-data:/parquet-data
       - ./transformers/synthetix:/app/synthetix
+      - ./parquet-data-indexers:/parquet-data-indexers
     networks:
       - data
 

diff --git a/indexers/arbitrum-sepolia/Dockerfile b/indexers/arbitrum-sepolia/Dockerfile
@@ -4,11 +4,11 @@ WORKDIR /app
 
 COPY package*.json ./
 
+RUN npm install
 RUN npm ci
 
 COPY . .
 
-RUN npm run generate:processor
 RUN npm run build
 
-CMD npm run generate:migration ; npm run start
+CMD npm run start
diff --git a/indexers/arbitrum-sepolia/commands.json b/indexers/arbitrum-sepolia/commands.json
@@ -5,77 +5,25 @@
         "description": "delete all build artifacts",
         "cmd": ["npx", "--yes", "rimraf", "lib"]
       },
-      "generate": {
-        "description": "Generate a squid from an ABI file",
-        "cmd": ["squid-gen-abi"]
-      },
-      "squid-gen-abi": {
-        "description": "Generate a squid from an ABI file",
-        "cmd": ["squid-gen-abi"],
-        "hidden": true
-      },
       "build": {
         "description": "Build the squid project",
         "deps": ["clean"],
         "cmd": ["tsc"]
       },
-      "up": {
-        "description": "Start a PG database",
-        "cmd": ["docker-compose", "up", "-d"]
-      },
-      "down": {
-        "description": "Drop a PG database",
-        "cmd": ["docker-compose", "down"]
-      },
-      "migration:apply": {
-        "description": "Apply the DB migrations",
-        "cmd": ["squid-typeorm-migration", "apply"]
-      },
-      "migration:generate": {
-        "description": "Generate a DB migration matching the TypeORM entities",
-        "deps": ["build", "migration:clean"],
-        "cmd": ["squid-typeorm-migration", "generate"],
-      },
-      "migration:clean": {
-        "description": "Clean the migrations folder",
-        "cmd": ["npx", "--yes", "rimraf", "./db/migrations"],
-      },
-      "migration": {
-        "deps": ["build"],
-        "cmd": ["squid-typeorm-migration", "generate"],
-        "hidden": true
-      },
-      "codegen": {
-        "description": "Generate TypeORM entities from the schema file",
-        "cmd": ["squid-typeorm-codegen"]
-      },
       "typegen": {
         "description": "Generate data access classes for an ABI file(s) in the ./abi folder",
         "cmd": ["squid-evm-typegen", "./src/abi", {"glob": "./abi/*.json"}, "--multicall"]
       },
       "process": {
         "description": "Load .env and start the squid processor",
-        "deps": ["build", "migration:apply"],
+        "deps": ["build"],
         "cmd": ["node", "--require=dotenv/config", "lib/main.js"]
       },
       "process:prod": {
         "description": "Start the squid processor",
-        "deps": ["migration:apply"],
         "cmd": ["node", "lib/main.js"],
         "hidden": true
       },
-      "serve": {
-        "description": "Start the GraphQL API server",
-        "cmd": ["squid-graphql-server"]
-      },
-      "serve:prod": {
-        "description": "Start the GraphQL API server with caching and limits",
-        "cmd": ["squid-graphql-server",
-          "--dumb-cache", "in-memory",
-          "--dumb-cache-ttl", "1000",
-          "--dumb-cache-size", "100",
-          "--dumb-cache-max-age", "1000" ]
-      },
       "check-updates": {
         "cmd": ["npx", "--yes", "npm-check-updates", "--filter=/subsquid/", "--upgrade"],
         "hidden": true
@@ -84,10 +32,6 @@
         "description": "Bump @subsquid packages to the latest versions",
         "deps": ["check-updates"],
         "cmd": ["npm", "i", "-f"]
-      },
-      "open": {
-        "description": "Open a local browser window",
-        "cmd": ["npx", "--yes", "opener"]
       }
     }
   }