From 0d20d7265cc8f73622849a0153914903afa0b3de Mon Sep 17 00:00:00 2001
From: Altug Keyder <7550161+akeyder@users.noreply.github.com>
Date: Fri, 21 Jul 2023 15:17:52 +0300
Subject: [PATCH] (#18) silo support

---
 contrib/bin/common.sh                     |   8 +
 contrib/config/build/refiner.sh           |  74 ++++---
 contrib/config/docker/silo_datalake.yaml  |  93 ++++++++
 contrib/config/docker/silo_nearcore.yaml  |  94 +++++++++
 contrib/config/nginx/silo.conf            |  24 +++
 contrib/config/refiner/silo_datalake.json |  17 ++
 contrib/config/refiner/silo_nearcore.json |  17 ++
 contrib/config/relayer/silo.yaml          |  80 +++++++
 contrib/config/silo/template.conf         |  22 ++
 install.sh                                | 245 +++++++++++++++-------
 10 files changed, 562 insertions(+), 112 deletions(-)
 create mode 100644 contrib/config/docker/silo_datalake.yaml
 create mode 100644 contrib/config/docker/silo_nearcore.yaml
 create mode 100644 contrib/config/nginx/silo.conf
 create mode 100644 contrib/config/refiner/silo_datalake.json
 create mode 100644 contrib/config/refiner/silo_nearcore.json
 create mode 100644 contrib/config/relayer/silo.yaml
 create mode 100644 contrib/config/silo/template.conf

diff --git a/contrib/bin/common.sh b/contrib/bin/common.sh
index f7d76e4..1b8fd17 100755
--- a/contrib/bin/common.sh
+++ b/contrib/bin/common.sh
@@ -28,4 +28,12 @@ confirmed() {
             return 1
             ;;
     esac
+}
+
+beginswith() { case "$2" in "$1"*) true;; *) false;; esac; }
+
+to_upper_first() {
+  upper_first=$(echo "$1" | cut -c1 | tr [a-z] [A-Z])
+  rest=$(echo "$1" | cut -c2-)
+  echo "$upper_first$rest"
 }
\ No newline at end of file
diff --git a/contrib/config/build/refiner.sh b/contrib/config/build/refiner.sh
index 250fb67..381b385 100755
--- a/contrib/config/build/refiner.sh
+++ b/contrib/config/build/refiner.sh
@@ -8,44 +8,54 @@ cmd="/usr/local/bin/aurora-refiner -c /config/refiner.json run"
 
 if [ "x$curr_version" != "x" ] && [ "$long_version" != "$curr_version" ]; then
   echo "successfully updated to $long_version"
-  echo "$long_version $(date)" >> $version_log_file
+  echo "$long_version $(date)" >> "$version_log_file"
 fi
 
-while [ "x$height" = "x" ]; do
-  # get latest block from relayer to start refiner from specific height
-  resp=$(curl http://srpc2-relayer:8545 -X POST -H "Content-Type: application/json" \
-             -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params": [],"id":1}' 2>/dev/null)
-  if test $? -eq 0; then
-    echo "received response from relayer: $resp"
-    result=$(echo "$resp" | jq -r '.result' 2>/dev/null)
-    if test $? -eq 0 && { [ "0x" = "$(echo "$result" | cut -c-2)" ] || [ "0X" = "$(echo "$result" | cut -c-2)" ]; }; then
-      height=$(printf %d "$result" 2>/dev/null)
-      if test $? -eq 0; then
-        if [ $height -gt 0 ]; then
-          # start refiner with received height
-          cmd="$cmd --height $height"
-          break
+if [ -f "/data/.REFINER_LAST_BLOCK" ]; then
+  echo "starting with last block recorded"
+  height=$(cat "/data/.REFINER_LAST_BLOCK" | xargs printf "%'d")
+elif [ "x$1" != "x" ]; then
+  echo "starting with last block argument"
+  height=$(printf '%d' "$1")
+else
+  while [ "x$height" = "x" ]; do
+    # get latest block from relayer to start refiner from specific height
+    resp=$(curl http://srpc2-relayer:8545 -X POST -H "Content-Type: application/json" \
+               -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params": [],"id":1}' 2>/dev/null)
+    if test $? -eq 0; then
+      echo "received response from relayer: $resp"
+      result=$(echo "$resp" | jq -r '.result' 2>/dev/null)
+      if test $? -eq 0 && { [ "0x" = "$(echo "$result" | cut -c-2)" ] || [ "0X" = "$(echo "$result" | cut -c-2)" ]; }; then
+        height=$(printf %d "$result" 2>/dev/null)
+        if test $? -eq 0; then
+          if [ "$height" -gt 0 ]; then
+            # start refiner with received height
+            echo "starting with last block received from relayer"
+            break
+          fi
         fi
       fi
-    fi
-    error=$(echo "$resp" | jq -r '.error.message' 2>/dev/null)
-    if test $? -eq 0 && [ "$error" = "record not found in DB" ]; then
-      near_config=$(grep DataLake /config/refiner.json)
-      if [ "x$near_config" != "x" ]; then
-        # if datalake config and there is no record in DB, start refiner without height. i.e: starts from earliest block
-        # in AWS S3 where near datalake resides
-        break
+      error=$(echo "$resp" | jq -r '.error.message' 2>/dev/null)
+      if test $? -eq 0 && [ "$error" = "record not found in DB" ]; then
+        near_config=$(grep DataLake /config/refiner.json)
+        if [ "x$near_config" != "x" ]; then
+          # if datalake config and there is no record in DB, start refiner without height. i.e: starts from earliest block
+          # in AWS S3 where near datalake resides
+          break
+        fi
+        # If refiner is configured to index from nearcore, then there is no way refiner can index anything earlier than
+        # 2 epochs of the current block height from the near core, so keep waiting if the received height is 0.
+        # This can happen if relayer is started without any snapshots, possible solution;
+        #   - manually wiring relayer data or let installation downloads relayer snapshots
+        #   - and restart relayer
       fi
-      # If refiner is configured to index from nearcore, then there is no way refiner can index anything earlier than
-      # 2 epochs of the current block height from the near core, so keep waiting if the received height is 0.
-      # This can happen if relayer is started without any snapshots, possible solution;
-      #   - manually wiring relayer data or let installation downloads relayer snapshots
-      #   - and restart relayer
     fi
-  fi
-  echo "waiting for relayer response for block height..."
-  sleep 5
-done
+    echo "waiting for relayer response for block height..."
+    sleep 5
+  done
+fi
+
+cmd="$cmd --height $height"
 
 echo "starting refiner: [$cmd]"
 
diff --git a/contrib/config/docker/silo_datalake.yaml b/contrib/config/docker/silo_datalake.yaml
new file mode 100644
index 0000000..3385d13
--- /dev/null
+++ b/contrib/config/docker/silo_datalake.yaml
@@ -0,0 +1,93 @@
+---
+version: '3.8'
+services:
+  relayer:
+    image: nearaurora/srpc2-relayer
+    container_name: srpc2-relayer
+#    pull_policy: missing
+#    build:
+#      dockerfile: ../contrib/config/build/Dockerfile.relayer
+    restart: unless-stopped
+    init: true
+    environment:
+      - NEAR_ENV=%%SILO_NETWORK%%
+      - NODE_ENV=%%SILO_NETWORK%%
+    expose:
+      - '8545'
+    volumes:
+      - ./config/relayer:/config
+      - ./data/relayer:/data/relayer
+      - ./data/refiner:/data/refiner
+      - ./near:/near
+    logging:
+      driver: "json-file"
+      options:
+        max-size: 5m
+        max-file: "5"
+    networks:
+      static-network:
+  refiner:
+    image: nearaurora/srpc2-refiner
+    container_name: srpc2-refiner
+    entrypoint: ["/docker-entrypoint.sh", "%%SILO_FROM_BLOCK%%"]
+#    pull_policy: missing
+#    build:
+#      dockerfile: ../contrib/config/build/Dockerfile.refiner
+    restart: unless-stopped
+    environment:
+      - RUST_BACKTRACE=${RUST_BACKTRACE:-1}
+      - TZ=UTC0
+    volumes:
+      - ./config/refiner:/config # config files
+      - ./data/refiner:/data # Output directory
+      - "%%AWS%%:/root/.aws/credentials:ro"
+    logging:
+      driver: "json-file"
+      options:
+        max-size: 5m
+        max-file: "5"
+    ports:
+      - '127.0.0.1:23030:3030'
+    networks:
+      static-network:
+        ipv4_address: 10.123.49.254
+  proxy:
+    image: nearaurora/reverseproxy:latest
+    container_name: srpc2-proxy
+    restart: unless-stopped
+    init: true
+    expose:
+      - '80'
+    ports:
+      - '20080:80'  # Remove this line to prevent listening on public IP address.
+      # - '127.0.0.1:20080:80' # Remove the hashtag in the beginning of this line to enable listening on 127.0.0.1.
+    volumes:
+      - ./config/nginx:/config
+    logging:
+      driver: "json-file"
+      options:
+        max-size: 5m
+        max-file: "5"
+    networks:
+      static-network:
+        ipv4_address: 10.123.49.253
+  watchtower:
+    image: containrrr/watchtower
+    container_name: srpc2-watchtower
+    restart: unless-stopped
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+#      - ./config/docker/config.json:/config.json
+    logging:
+      driver: "json-file"
+      options:
+        max-size: 5m
+        max-file: "5"
+    command: --interval 30 srpc2-relayer srpc2-refiner srpc2-proxy
+networks:
+  static-network:
+    ipam:
+      config:
+        - subnet: 10.123.49.0/24
+volumes:
+  database:
\ No newline at end of file
diff --git a/contrib/config/docker/silo_nearcore.yaml b/contrib/config/docker/silo_nearcore.yaml
new file mode 100644
index 0000000..e210778
--- /dev/null
+++ b/contrib/config/docker/silo_nearcore.yaml
@@ -0,0 +1,94 @@
+---
+version: '3.8'
+services:
+  relayer:
+    image: nearaurora/srpc2-relayer
+    container_name: srpc2-relayer
+#    pull_policy: missing
+#    build:
+#      dockerfile: ../contrib/config/build/Dockerfile.relayer
+    restart: unless-stopped
+    init: true
+    environment:
+      - NEAR_ENV=%%SILO_NETWORK%%
+      - NODE_ENV=%%SILO_NETWORK%%
+    expose:
+      - '8545'
+    volumes:
+      - ./config/relayer:/config
+      - ./data/relayer:/data/relayer
+      - ./data/refiner:/data/refiner
+      - ./near:/near
+    logging:
+      driver: "json-file"
+      options:
+        max-size: 5m
+        max-file: "5"
+    networks:
+      static-network:
+  refiner:
+    image: nearaurora/srpc2-refiner
+    container_name: srpc2-refiner
+    entrypoint: ["/docker-entrypoint.sh", "%%SILO_FROM_BLOCK%%"]
+#    pull_policy: missing
+#    build:
+#      dockerfile: ../contrib/config/build/Dockerfile.refiner
+    restart: unless-stopped
+    environment:
+      - RUST_BACKTRACE=${RUST_BACKTRACE:-1}
+      - TZ=UTC0
+    volumes:
+      - ./config/refiner:/config # config files
+      - ./data/refiner:/data # Output directory
+      - ./near:/near # Path to Nearcore data
+      - ./engine:/engine
+    logging:
+      driver: "json-file"
+      options:
+        max-size: 5m
+        max-file: "5"
+    ports:
+      - '127.0.0.1:23030:3030'
+    networks:
+      static-network:
+        ipv4_address: 10.123.49.254
+  proxy:
+    image: nearaurora/reverseproxy:latest
+    container_name: srpc2-proxy
+    restart: unless-stopped
+    init: true
+    expose:
+      - '80'
+    ports:
+      - '20080:80'  # Remove this line to prevent listening on public IP address.
+      # - '127.0.0.1:20080:80' # Remove the hashtag in the beginning of this line to enable listening on 127.0.0.1.
+    volumes:
+      - ./config/nginx:/config
+    logging:
+      driver: "json-file"
+      options:
+        max-size: 5m
+        max-file: "5"
+    networks:
+      static-network:
+        ipv4_address: 10.123.49.253
+  watchtower:
+    image: containrrr/watchtower
+    container_name: srpc2-watchtower
+    restart: unless-stopped
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+#      - ./config/docker/config.json:/config.json
+    logging:
+      driver: "json-file"
+      options:
+        max-size: 5m
+        max-file: "5"
+    command: --interval 30 srpc2-relayer srpc2-refiner srpc2-proxy
+networks:
+  static-network:
+    ipam:
+      config:
+        - subnet: 10.123.49.0/24
+volumes:
+  database:
\ No newline at end of file
diff --git a/contrib/config/nginx/silo.conf b/contrib/config/nginx/silo.conf
new file mode 100644
index 0000000..4f8d7e1
--- /dev/null
+++ b/contrib/config/nginx/silo.conf
@@ -0,0 +1,24 @@
+upstream endpoint {
+	server 10.123.49.2:8545;
+}
+
+server {
+    listen *:80;
+
+    location / {
+	client_body_buffer_size 64k;
+        proxy_pass http://endpoint;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "Upgrade";
+        proxy_socket_keepalive on;
+        nchan_websocket_ping_interval 5;
+        nchan_websocket_client_heartbeat PING PONG;
+        proxy_read_timeout 360000s;
+        proxy_send_timeout 360000s;
+    }
+}
diff --git a/contrib/config/refiner/silo_datalake.json b/contrib/config/refiner/silo_datalake.json
new file mode 100644
index 0000000..ad3af1f
--- /dev/null
+++ b/contrib/config/refiner/silo_datalake.json
@@ -0,0 +1,17 @@
+{
+    "refiner": {
+        "chain_id": %%SILO_CHAIN_ID%%,
+        "engine_path": "/engine",
+        "tx_tracker_path": "/engine/tx_tracker",
+        "engine_account_id": "aurora"
+    },
+    "input_mode": {
+        "DataLake": {
+            "network": "%%SILO_DATALAKE_NETWORK%%"
+        }
+    },
+    "output_storage": {
+        "path": "/data",
+        "batch_size": 10000
+    }
+}
diff --git a/contrib/config/refiner/silo_nearcore.json b/contrib/config/refiner/silo_nearcore.json
new file mode 100644
index 0000000..e5e2e91
--- /dev/null
+++ b/contrib/config/refiner/silo_nearcore.json
@@ -0,0 +1,17 @@
+{
+    "refiner": {
+        "chain_id": %%SILO_CHAIN_ID%%,
+        "engine_path": "/engine",
+        "tx_tracker_path": "/engine/tx_tracker",
+        "engine_account_id": "aurora"
+    },
+    "input_mode": {
+        "Nearcore": {
+            "path": "/near"
+        }
+    },
+    "output_storage": {
+        "path": "/data",
+        "batch_size": 10000
+    }
+}
diff --git a/contrib/config/relayer/silo.yaml b/contrib/config/relayer/silo.yaml
new file mode 100644
index 0000000..f7c4414
--- /dev/null
+++ b/contrib/config/relayer/silo.yaml
@@ -0,0 +1,80 @@
+logger:
+  level: info
+  filePath: /log/relayer.log
+  logToConsole: true
+  logToFile: false
+db:
+  badger:
+    core:
+      gcIntervalSeconds: 10
+      scanRangeThreshold: 2000
+      maxScanIterators: 10000
+      filterTtlMinutes: 15
+      options:
+        Dir: /data/relayer
+        ValueDir: /data/relayer
+        InMemory: false
+        DetectConflicts: false
+endpoint:
+  chainId: %%SILO_CHAIN_ID%%
+  filterFilePath: /config/filter.yaml
+  engine:
+    nearNetworkID: %%SILO_NETWORK%%
+    nearNodeURL: https://archival-rpc.%%SILO_NETWORK%%.near.org
+    signer: %%SIGNER%%
+    signerKey: /config/relayer.json
+    asyncSendRawTxs: true
+    minGasPrice: 0
+    minGasLimit: 21000
+    gasForNearTxsCall: 300000000000000
+    depositForNearTxsCall: 0
+    retryWaitTimeMsForNearTxsCall: 3000
+    retryNumberForNearTxsCall: 3
+  eth:
+    protocolVersion: 0x41
+    hashrate: 0
+    gasEstimate: 0x6691b7
+    gasPrice: 0x0
+  proxyEndpoints:
+    url: "https://%%SILO_NETWORK%%.aurora.dev:443"
+    endpoints:
+      - eth_estimateGas
+      - debug_traceTransaction
+  disabledEndpoints:
+    -
+rpcNode:
+  httpHost: 0.0.0.0
+  httpPort: 8545
+  httpCors:
+    - "*"
+  httpCompress: true
+  httpTimeout: 300 # in seconds
+  httpPathPrefix: ""
+  wsHost: 0.0.0.0 # Both wsHost and wsPort are mandatory to start the websocket
+  wsPort: 8545 # Both wsHost and wsPort are mandatory to start the websocket
+  wsHandshakeTimeout: 10 # in seconds
+  wsPathPrefix: ""
+  maxBatchRequests: 1000
+indexer:
+  sourceFolder: /data/refiner/
+  subFolderBatchSize: 10000   # should match with Aurora refiner batch size, see https://github.com/aurora-is-near/borealis-engine-lib
+  keepFiles: true
+  genesisBlock: %%SILO_GENESIS%% # 37157758 = Aurora Genesis
+  fromBlock: %%SILO_FROM_BLOCK%% # cannot be lower than Genesis
+  toBlock: 0                  # 0 = indefinite
+  retryCountOnFailure: 10
+  waitForBlockMs: 500
+backupIndexer: # N/A for silo
+  indexFromBackup: false
+  backupDir:
+  backupNamePrefix:
+  from:
+prehistoryIndexer: # N/A for silo
+  indexFromPrehistory: false
+  prehistoryHeight:
+  from:
+  to:
+  batchSize:
+  prehistoryChainId:
+  archiveURL:
+
diff --git a/contrib/config/silo/template.conf b/contrib/config/silo/template.conf
new file mode 100644
index 0000000..149aabb
--- /dev/null
+++ b/contrib/config/silo/template.conf
@@ -0,0 +1,22 @@
+# THIS A TEMPLATE FILE DEFINING A SILO. PLEASE DO NOT EDIT THIS FILE DIRECTLY,
+# INSTEAD MAKE A COPY AND USE IT AS THE SILO CONFIGURATION DURING INSTALLATION.
+
+# Defines on which network of Aurora and Near, the Silo resides. Accepted values are mainnet or testnet only
+# [required config]
+SILO_NETWORK:
+
+# Chain ID of the Silo
+# [required config]
+SILO_CHAIN_ID:
+
+# A decimal number which defines the genesis block number of the Silo
+# [required config]
+SILO_GENESIS:
+
+# A decimal number, defining the block number, from which the indexing of blockchain starts
+# [required config]
+SILO_FROM_BLOCK:
+
+# A path to JSON file which contains the Engine Account ID and keys of the SILO
+# [required config]
+SILO_ENGINE_ACCOUNT_JSON:
\ No newline at end of file
diff --git a/install.sh b/install.sh
index a98639f..c68148d 100755
--- a/install.sh
+++ b/install.sh
@@ -4,9 +4,12 @@ src_dir="contrib"
 near_postfix="near"
 
 network="mainnet"
+near_network="mainnet"
+silo_config_file=""
 near_source="nearcore" # nearcore or datalake
 migrate_from=""
-use_snapshots=1
+use_aurora_snapshot=1
+use_near_snapshot=1
 download_workers=256
 
 trap "echo Exited!; exit 2;" INT TERM
@@ -18,11 +21,113 @@ fi
 
 . ./${src_dir}/bin/common.sh
 
-install() {
+valid_silo_config() {
+  if [ ! -f "$1" ]; then
+      return 1
+  fi
+  while read -r config || [ -n "$config" ]; do
+      if [ -n "$config" ] && [ "$config" != " " ] && ! beginswith "#" "$config"; then
+          value=$(echo "$config" | cut -d ':' -f2- | awk '{$1=$1};1')
+          if [ -n "$value" ] && [ "$value" != " " ] && [ -z "${value##*:*}" ]; then
+              return 1
+          fi
+      fi
+  done < "$1"
+}
+
+apply_silo_config() {
+    silo_network=$(grep "SILO_NETWORK" "$1" | cut -d ':' -f2- | awk '{$1=$1};1')
+    sed "s/%%SILO_NETWORK%%/${silo_network}/" "${INSTALL_DIR}/config/relayer/relayer.yaml" > "${INSTALL_DIR}/config/relayer/relayer.yaml2" && \
+    mv "${INSTALL_DIR}/config/relayer/relayer.yaml2" "${INSTALL_DIR}/config/relayer/relayer.yaml"
+    sed "s/%%SILO_NETWORK%%/${silo_network}/" "${INSTALL_DIR}/docker-compose.yaml" > "${INSTALL_DIR}/docker-compose.yaml2" && \
+    mv "${INSTALL_DIR}/docker-compose.yaml2" "${INSTALL_DIR}/docker-compose.yaml"
+
+    silo_chain_id=$(grep "SILO_CHAIN_ID" "$1" | cut -d ':' -f2- | awk '{$1=$1};1')
+    sed "s/%%SILO_CHAIN_ID%%/${silo_chain_id}/" "${INSTALL_DIR}/config/refiner/refiner.json" > "${INSTALL_DIR}/config/refiner/refiner.json2" && \
+    mv "${INSTALL_DIR}/config/refiner/refiner.json2" "${INSTALL_DIR}/config/refiner/refiner.json"
+    sed "s/%%SILO_CHAIN_ID%%/${silo_chain_id}/" "${INSTALL_DIR}/config/relayer/relayer.yaml" > "${INSTALL_DIR}/config/relayer/relayer.yaml2" && \
+    mv "${INSTALL_DIR}/config/relayer/relayer.yaml2" "${INSTALL_DIR}/config/relayer/relayer.yaml"
+
+    silo_genesis=$(grep "SILO_GENESIS" "$1" | cut -d ':' -f2- | awk '{$1=$1};1')
+    sed "s/%%SILO_GENESIS%%/${silo_genesis}/" "${INSTALL_DIR}/config/relayer/relayer.yaml" > "${INSTALL_DIR}/config/relayer/relayer.yaml2" && \
+    mv "${INSTALL_DIR}/config/relayer/relayer.yaml2" "${INSTALL_DIR}/config/relayer/relayer.yaml"
+
+    silo_from_block=$(grep "SILO_FROM_BLOCK" "$1" | cut -d ':' -f2- | awk '{$1=$1};1')
+    sed "s/%%SILO_FROM_BLOCK%%/${silo_from_block}/" "${INSTALL_DIR}/config/relayer/relayer.yaml" > "${INSTALL_DIR}/config/relayer/relayer.yaml2" && \
+    mv "${INSTALL_DIR}/config/relayer/relayer.yaml2" "${INSTALL_DIR}/config/relayer/relayer.yaml"
+    sed "s/%%SILO_FROM_BLOCK%%/${silo_from_block}/" "${INSTALL_DIR}/docker-compose.yaml" > "${INSTALL_DIR}/docker-compose.yaml2" && \
+    mv "${INSTALL_DIR}/docker-compose.yaml2" "${INSTALL_DIR}/docker-compose.yaml"
+
+    engine_account_json=$(grep "SILO_ENGINE_ACCOUNT_JSON" "$silo_config_file" | cut -d ':' -f2- | awk '{$1=$1};1')
+    if [ ! -f "$engine_account_json" ]; then
+      echo "Engine account JSON file could not be found at [$engine_account_json], please check your config file [$silo_config_file]"
+      exit 1
+    fi
+    cp "$engine_account_json" "${INSTALL_DIR}/config/relayer/relayer.json"
 
-  if [ "${network}" = "testnet" ]; then
-	  near_postfix="testnet"
+    if [ ${near_source} = "datalake" ]; then
+      silo_datalake_network=$(to_upper_first "$silo_network")
+      sed "s/%%SILO_DATALAKE_NETWORK%%/${silo_datalake_network}/" "${INSTALL_DIR}/config/refiner/refiner.json" > "${INSTALL_DIR}/config/refiner/refiner.json2" && \
+      mv "${INSTALL_DIR}/config/refiner/refiner.json2" "${INSTALL_DIR}/config/refiner/refiner.json"
+    fi
+}
+
+apply_nearcore_config() {
+  if [ "x$migrate_from" != "x" ]; then
+    ln -s "$migrate_from/near" "${INSTALL_DIR}/near"
+    ln -s "$migrate_from/engine" "${INSTALL_DIR}/engine"
+  else
+    mkdir -p "${INSTALL_DIR}/near" "${INSTALL_DIR}/engine" 2> /dev/null
+    if [ ! -f "${INSTALL_DIR}/near/config.json" ]; then
+      echo "Downloading default configuration..."
+      curl -sSf -o "${INSTALL_DIR}/near/config.json" https://files.deploy.aurora.dev/"${near_network}"-new-rpc/config.json
+    fi
+    if [ ! -f "${INSTALL_DIR}/near/genesis.json" ]; then
+      echo "Downloading genesis file..."
+      curl -sSf -o "${INSTALL_DIR}/near/genesis.json.gz" https://files.deploy.aurora.dev/"${near_network}"-new-rpc/genesis.json.gz
+      echo "Extracting genesis file..."
+      gzip -d "${INSTALL_DIR}/near/genesis.json.gz"
+    fi
+    if [ ! -f "${INSTALL_DIR}/near/node_key.json" ]; then
+      echo "Generating node_key..."
+      docker run --rm --name near_keygen -v "$(pwd)/${INSTALL_DIR}"/near:/near:rw --entrypoint /bin/sh nearaurora/srpc2-relayer -c "/usr/local/bin/nearkey node%.${near_postfix} > /near/node_key.json"
+    fi
+    if [ ! -f "${INSTALL_DIR}/near/validator_key.json" ]; then
+      echo "Generating validator_key..."
+      docker run --rm --name near_keygen -v "$(pwd)/${INSTALL_DIR}"/near:/near:rw --entrypoint /bin/sh nearaurora/srpc2-relayer -c "/usr/local/bin/nearkey node%.${near_postfix} > /near/validator_key.json"
+    fi
+    if [ $use_near_snapshot -eq 1 ] && [ ! -f "${INSTALL_DIR}/near/data/CURRENT" ]; then
+      echo "Downloading near chain snapshot..."
+      latest=$(docker run --rm --entrypoint /bin/sh nearaurora/srpc2-relayer -c "/usr/local/bin/s5cmd --no-sign-request --numworkers $download_workers cat s3://near-protocol-public/backups/${near_network}/rpc/latest")
+      finish=0
+      while [ ${finish} -eq 0 ]; do
+        echo "Fetching, this can take some time..."
+        docker run --rm --name near_downloader -v "$(pwd)/${INSTALL_DIR}"/near:/near:rw --entrypoint /bin/sh nearaurora/srpc2-relayer -c "/usr/local/bin/s5cmd --stat --no-sign-request cp s3://near-protocol-public/backups/${near_network}/rpc/"${latest}"/* /near/data/"
+        if [ -f "${INSTALL_DIR}/near/data/CURRENT" ]; then
+          finish=1
+        fi
+      done
+    fi
   fi
+}
+
+apply_datalake_config() {
+  if [ "x$AWS_SHARED_CREDENTIALS_FILE" = "x" ]; then
+    echo "Installation failed, environment variable AWS_SHARED_CREDENTIALS_FILE is needed for datalake config." \
+         "Please set environment variable AWS_SHARED_CREDENTIALS_FILE for [$USER] user or run installer as 'AWS_SHARED_CREDENTIALS_FILE={path to AWS credentials file} ./install.sh'"
+    echo "For more details, also see https://docs.aws.amazon.com/sdkref/latest/guide/file-location.html"
+    exit 1
+  fi
+  if [ ! -f "$AWS_SHARED_CREDENTIALS_FILE" ]; then
+    echo "Installation failed, datalake config requires AWS account." \
+         "Create [$AWS_SHARED_CREDENTIALS_FILE] file and run install script again!"
+    exit 1
+  fi
+  sed "s|%%AWS%%|${AWS_SHARED_CREDENTIALS_FILE}|" "${INSTALL_DIR}/docker-compose.yaml" > "${INSTALL_DIR}/docker-compose.yaml2"
+  mv "${INSTALL_DIR}/docker-compose.yaml2" "${INSTALL_DIR}/docker-compose.yaml"
+}
+
+install() {
 
   if [ -f "${VERSION_FILE}" ]; then
     echo "There is already an Aurora Standalone RPC installation running or an unfinished installation exists"
@@ -36,6 +141,7 @@ install() {
   echo "Installing" && version | tee "${VERSION_FILE}"
 
   set -e
+
   mkdir -p  "${INSTALL_DIR}/data/relayer" \
             "${INSTALL_DIR}/data/refiner" \
             "${INSTALL_DIR}/config/relayer" \
@@ -46,14 +152,6 @@ install() {
     cp "./${src_dir}/config/relayer/${network}.yaml" "${INSTALL_DIR}/config/relayer/relayer.yaml"
   fi
 
-  if [ ! -f "${INSTALL_DIR}/config/relayer/relayer.json" ]; then
-    echo "Generating relayer key..."
-    docker run --rm --name near_keygen -v "$(pwd)/${INSTALL_DIR}"/config/relayer:/config:rw --entrypoint /bin/sh nearaurora/srpc2-relayer -c "/usr/local/bin/nearkey > /config/relayer.json"
-    relayerName=$(cat "${INSTALL_DIR}/config/relayer/relayer.json" | grep account_id | cut -d\" -f4)
-    sed "s/%%SIGNER%%/${relayerName}/" "${INSTALL_DIR}/config/relayer/relayer.yaml" > "${INSTALL_DIR}/config/relayer/relayer.yaml2" && \
-    mv "${INSTALL_DIR}/config/relayer/relayer.yaml2" "${INSTALL_DIR}/config/relayer/relayer.yaml"
-  fi
-
   if [ ! -f "${INSTALL_DIR}/config/relayer/filter.yaml" ]; then
     cp "./${src_dir}/config/relayer/filter.yaml" "${INSTALL_DIR}/config/relayer/filter.yaml"
   fi
@@ -66,7 +164,11 @@ install() {
     cp "./${src_dir}/config/nginx/${network}.conf" "${INSTALL_DIR}/config/nginx/endpoint.conf"
   fi
 
-  if [ $use_snapshots -eq 1 ] || [ "x$migrate_from" != "x" ]; then
+  if [ ! -f "${INSTALL_DIR}/docker-compose.yaml" ]; then
+    cp "./${src_dir}/config/docker/${network}_${near_source}.yaml" "${INSTALL_DIR}/docker-compose.yaml"
+  fi
+
+  if [ $use_aurora_snapshot -eq 1 ] || [ "x$migrate_from" != "x" ]; then
     latest=""
     if [ ! -f "${INSTALL_DIR}/.latest" ]; then
       echo Initial
@@ -87,64 +189,30 @@ install() {
     fi
   fi
 
-  if [ ${near_source} = "nearcore" ]; then
-    if [ "x$migrate_from" != "x" ]; then
-      ln -s "$migrate_from/near" "${INSTALL_DIR}/near"
-      ln -s "$migrate_from/engine" "${INSTALL_DIR}/engine"
-    else
-      mkdir -p "${INSTALL_DIR}/near" "${INSTALL_DIR}/engine" 2> /dev/null
-      if [ ! -f "${INSTALL_DIR}/near/config.json" ]; then
-        echo "Downloading default configuration..."
-        curl -sSf -o "${INSTALL_DIR}/near/config.json" https://files.deploy.aurora.dev/"${network}"-new-rpc/config.json
-      fi
-      if [ ! -f "${INSTALL_DIR}/near/genesis.json" ]; then
-        echo "Downloading genesis file..."
-        curl -sSf -o "${INSTALL_DIR}/near/genesis.json.gz" https://files.deploy.aurora.dev/"${network}"-new-rpc/genesis.json.gz
-        echo "Extracting genesis file..."
-        gzip -d "${INSTALL_DIR}/near/genesis.json.gz"
-      fi
-      if [ ! -f "${INSTALL_DIR}/near/node_key.json" ]; then
-        echo "Generating node_key..."
-        docker run --rm --name near_keygen -v "$(pwd)/${INSTALL_DIR}"/near:/near:rw --entrypoint /bin/sh nearaurora/srpc2-relayer -c "/usr/local/bin/nearkey node%.${near_postfix} > /near/node_key.json"
-      fi
-      if [ ! -f "${INSTALL_DIR}/near/validator_key.json" ]; then
-        echo "Generating validator_key..."
-        docker run --rm --name near_keygen -v "$(pwd)/${INSTALL_DIR}"/near:/near:rw --entrypoint /bin/sh nearaurora/srpc2-relayer -c "/usr/local/bin/nearkey node%.${near_postfix} > /near/validator_key.json"
-      fi
-      if [ $use_snapshots -eq 1 ] && [ ! -f "${INSTALL_DIR}/near/data/CURRENT" ]; then
-        echo "Downloading near chain snapshot..."
-        latest=$(docker run --rm --entrypoint /bin/sh nearaurora/srpc2-relayer -c "/usr/local/bin/s5cmd --no-sign-request --numworkers $download_workers cat s3://near-protocol-public/backups/${network}/rpc/latest")
-        finish=0
-        while [ ${finish} -eq 0 ]; do
-          echo "Fetching, this can take some time..."
-          docker run --rm --name near_downloader -v "$(pwd)/${INSTALL_DIR}"/near:/near:rw --entrypoint /bin/sh nearaurora/srpc2-relayer -c "/usr/local/bin/s5cmd --stat --no-sign-request cp s3://near-protocol-public/backups/${network}/rpc/"${latest}"/* /near/data/"
-          if [ -f "${INSTALL_DIR}/near/data/CURRENT" ]; then
-            finish=1
-          fi
-        done
-      fi
-    fi
-    cp "./${src_dir}/config/docker/${network}_${near_source}.yaml" "${INSTALL_DIR}/docker-compose.yaml"
-  elif [ ${near_source} = "datalake" ]; then
-    if [ "x$AWS_SHARED_CREDENTIALS_FILE" = "x" ]; then
-      echo "Installation failed, environment variable AWS_SHARED_CREDENTIALS_FILE is needed for datalake config." \
-           "Please set environment variable AWS_SHARED_CREDENTIALS_FILE for [$USER] user or run installer as 'AWS_SHARED_CREDENTIALS_FILE={path to AWS credentials file} ./install.sh'"
-      echo "For more details, also see https://docs.aws.amazon.com/sdkref/latest/guide/file-location.html"
-      exit 1
-    fi
-    if [ ! -f "$AWS_SHARED_CREDENTIALS_FILE" ]; then
-      echo "Installation failed, datalake config requires AWS account." \
-           "Create [$AWS_SHARED_CREDENTIALS_FILE] file and run install script again!"
-      exit 1
-    fi
-    sed "s|%%AWS%%|${AWS_SHARED_CREDENTIALS_FILE}|" "./${src_dir}/config/docker/${network}_${near_source}.yaml" > "${INSTALL_DIR}/docker-compose.yaml"
+  if [ "${network}" = "silo" ]; then
+    apply_silo_config "$silo_config_file"
+    near_network="$silo_network"
   else
-    echo "Installation failed, invalid near data source. It should either be 'datalake' or 'nearcore' !"
-    exit 1
+    near_network="$network"
+  fi
+
+  if [ "${near_source}" = "nearcore" ]; then
+    apply_nearcore_config
+  else
+    apply_datalake_config
   fi
 
-  if [ $use_snapshots -eq 0 ] \
-    || [ ${near_source} = "nearcore" -a -f "${INSTALL_DIR}/near/data/CURRENT" -a -f "${INSTALL_DIR}/data/relayer/.version" ] \
+  if [ ! -f "${INSTALL_DIR}/config/relayer/relayer.json" ]; then
+    echo "Generating relayer key..."
+    docker run --rm --name near_keygen -v "$(pwd)/${INSTALL_DIR}"/config/relayer:/config:rw --entrypoint /bin/sh nearaurora/srpc2-relayer -c "/usr/local/bin/nearkey > /config/relayer.json"
+  fi
+
+  account_id=$(grep "account_id" "${INSTALL_DIR}/config/relayer/relayer.json" | cut -d\" -f4)
+  sed "s/%%SIGNER%%/${account_id}/" "${INSTALL_DIR}/config/relayer/relayer.yaml" > "${INSTALL_DIR}/config/relayer/relayer.yaml2" && \
+  mv "${INSTALL_DIR}/config/relayer/relayer.yaml2" "${INSTALL_DIR}/config/relayer/relayer.yaml"
+
+  if [ $use_aurora_snapshot -eq 0 -a $use_near_snapshot -eq 0 ] \
+    || [ $use_near_snapshot -eq 1 -a ${near_source} = "nearcore" -a -f "${INSTALL_DIR}/near/data/CURRENT" -a -f "${INSTALL_DIR}/data/relayer/.version" ] \
     || [ ${near_source} = "datalake" -a -f "${INSTALL_DIR}/data/relayer/.version" ]; then
     echo "Setup complete [${network}, ${near_source}]"
   fi
@@ -177,26 +245,34 @@ version() {
 
 usage() {
   printf '\nUsage: %s [options]' "$(basename "$0")"
-  printf '\nOptions\n'
-  printf ' %s\t%s\n' "-n {mainnet|testnet}" "network to use, default is mainnet."
-  printf ' %s\t%s\n' "-r {nearcore|datalake}" "near source for indexing, default is nearcore."
-  printf ' %s\t\t%s\n\t\t\t%s\n' "-m {path}" "use the existing nearcore data at 'path' instead of downloading snapshots from scratch." \
+  printf '\n\nOptions\n'
+  printf ' %s\t%s\n\n' "-n {mainnet|testnet|silo}" "network to use, default is mainnet."
+  printf ' %s\t\t\t%s\n\t\t\t\t%s\n\n' "-f {path}" "for silo networks, this is the path to your silo configuration file." \
+  "This option is valid only if silo network is used, and '-s' option is ignored if this option is given."
+  printf ' %s\t\t%s\n\n' "-r {nearcore|datalake}" "near source for indexing, default is nearcore."
+  printf ' %s\t\t\t%s\n\t\t\t\t%s\n\n' "-m {path}" "use the existing nearcore data at 'path' instead of downloading snapshots from scratch." \
   "This option is valid only if nearcore config is used, and '-s' option is ignored if this option is given."
-  printf ' %s\t%s\n\t\t\t%s\n' "-w {number [1-256]}" "number of workers used for downloading near snapshots, default is 256." \
+  printf ' %s\t\t%s\n\t\t\t\t%s\n\n' "-w {number [1-256]}" "number of workers used for downloading near snapshots, default is 256." \
   "NOTE: On some OS and HW configurations, default number of workers may cause high CPU consumption during download."
-  printf ' %s\t\t\t%s\n\t\t\t%s\n\t\t\t%s\n' "-s" "if specified then snapshots are ignored during installation, default downloads and uses snapshots." \
+  printf ' %s\t\t\t\t%s\n\t\t\t\t%s\n\t\t\t\t%s\n\n' "-s" "if specified then snapshots are ignored during installation, default downloads and uses snapshots." \
   "NOTE: Ignoring snapshots may cause refiner not to index near chain. This can only be a valid option" \
   "if near source is selected as datalake otherwise refiner will not be sync with near core from scratch."
-  printf ' %s\t\t\t%s\n' "-v" "prints version"
-  printf ' %s\t\t\t%s\n\t\t\t%s\n' "-h" "prints usage"
-  printf 'Example\n%s\n\n' "./install.sh -n mainnet -r datalake -s"
+  printf ' %s\t\t\t\t%s\n\n' "-v" "prints version"
+  printf ' %s\t\t\t\t%s\n\n' "-h" "prints usage"
+  printf 'Examples\n'
+  printf ' %s\t\t-> %s\n\n' "./install.sh -n mainnet -r datalake -s" "use mainnet with near data lake but do not download snapshots"
+  printf ' %s\t\t-> %s\n\n' "./install.sh -n silo -f ./silo.conf" "use sile network whose config is defined in silo.conf, near source for indexing is nearcore"
 }
 
-while getopts ":n:r:m:w:svh" opt; do
+while getopts ":n:r:m:f:w:svh" opt; do
   case "${opt}" in
     n)
       network="${OPTARG}"
-      if [ "$network" != "mainnet" ] && [ "$network" != "testnet" ]; then
+      if [ "${network}" = "silo" ]; then
+        use_aurora_snapshot=0
+      elif [ "${network}" = "testnet" ]; then
+        near_postfix="testnet"
+      elif [ "${network}" != "mainnet" ]; then
         echo "Invalid Value: -${opt} cannot be '${OPTARG}'"
         usage
         exit 1
@@ -227,7 +303,11 @@ while getopts ":n:r:m:w:svh" opt; do
       fi
       ;;
     s)
-      use_snapshots=0
+      use_aurora_snapshot=0
+      use_near_snapshot=0
+      ;;
+    f)
+      silo_config_file="${OPTARG}"
       ;;
     v)
       version
@@ -251,4 +331,9 @@ while getopts ":n:r:m:w:svh" opt; do
 done
 shift $((OPTIND-1))
 
+if [ "${network}" = "silo" ] && ! valid_silo_config "$silo_config_file"; then
+  echo "Invalid silo config, $silo_config_file. For more information, see template config ./${src_dir}/silo/template.conf"
+  exit 1
+fi
+
 install
\ No newline at end of file