(#18) silo support

contrib/bin/common.sh

@@ -28,4 +28,12 @@ confirmed() {
             return 1
             ;;
     esac
+}
+
+beginswith() { case "$2" in "$1"*) true;; *) false;; esac; }
+
+to_upper_first() {
+  upper_first=$(echo "$1" | cut -c1 | tr [a-z] [A-Z])
+  rest=$(echo "$1" | cut -c2-)
+  echo "$upper_first$rest"
 }

contrib/config/build/refiner.sh

@@ -8,44 +8,54 @@ cmd="/usr/local/bin/aurora-refiner -c /config/refiner.json run"
 
 if [ "x$curr_version" != "x" ] && [ "$long_version" != "$curr_version" ]; then
   echo "successfully updated to $long_version"
-  echo "$long_version $(date)" >> $version_log_file
+  echo "$long_version $(date)" >> "$version_log_file"
 fi
 
-while [ "x$height" = "x" ]; do
-  # get latest block from relayer to start refiner from specific height
-  resp=$(curl http://srpc2-relayer:8545 -X POST -H "Content-Type: application/json" \
-             -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params": [],"id":1}' 2>/dev/null)
-  if test $? -eq 0; then
-    echo "received response from relayer: $resp"
-    result=$(echo "$resp" | jq -r '.result' 2>/dev/null)
-    if test $? -eq 0 && { [ "0x" = "$(echo "$result" | cut -c-2)" ] || [ "0X" = "$(echo "$result" | cut -c-2)" ]; }; then
-      height=$(printf %d "$result" 2>/dev/null)
-      if test $? -eq 0; then
-        if [ $height -gt 0 ]; then
-          # start refiner with received height
-          cmd="$cmd --height $height"
-          break
+if [ -f "/data/.REFINER_LAST_BLOCK" ]; then
+  echo "starting with last block recorded"
+  height=$(cat "/data/.REFINER_LAST_BLOCK" | xargs printf "%'d")
+elif [ "x$1" != "x" ]; then
+  echo "starting with last block argument"
+  height=$(printf '%d' "$1")
+else
+  while [ "x$height" = "x" ]; do
+    # get latest block from relayer to start refiner from specific height
+    resp=$(curl http://srpc2-relayer:8545 -X POST -H "Content-Type: application/json" \
+               -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params": [],"id":1}' 2>/dev/null)
+    if test $? -eq 0; then
+      echo "received response from relayer: $resp"
+      result=$(echo "$resp" | jq -r '.result' 2>/dev/null)
+      if test $? -eq 0 && { [ "0x" = "$(echo "$result" | cut -c-2)" ] || [ "0X" = "$(echo "$result" | cut -c-2)" ]; }; then
+        height=$(printf %d "$result" 2>/dev/null)
+        if test $? -eq 0; then
+          if [ "$height" -gt 0 ]; then
+            # start refiner with received height
+            echo "starting with last block received from relayer"
+            break
+          fi
         fi
       fi
-    fi
-    error=$(echo "$resp" | jq -r '.error.message' 2>/dev/null)
-    if test $? -eq 0 && [ "$error" = "record not found in DB" ]; then
-      near_config=$(grep DataLake /config/refiner.json)
-      if [ "x$near_config" != "x" ]; then
-        # if datalake config and there is no record in DB, start refiner without height. i.e: starts from earliest block
-        # in AWS S3 where near datalake resides
-        break
+      error=$(echo "$resp" | jq -r '.error.message' 2>/dev/null)
+      if test $? -eq 0 && [ "$error" = "record not found in DB" ]; then
+        near_config=$(grep DataLake /config/refiner.json)
+        if [ "x$near_config" != "x" ]; then
+          # if datalake config and there is no record in DB, start refiner without height. i.e: starts from earliest block
+          # in AWS S3 where near datalake resides
+          break
+        fi
+        # If refiner is configured to index from nearcore, then there is no way refiner can index anything earlier than
+        # 2 epochs of the current block height from the near core, so keep waiting if the received height is 0.
+        # This can happen if relayer is started without any snapshots, possible solution;
+        #   - manually wiring relayer data or let installation downloads relayer snapshots
+        #   - and restart relayer
       fi
-      # If refiner is configured to index from nearcore, then there is no way refiner can index anything earlier than
-      # 2 epochs of the current block height from the near core, so keep waiting if the received height is 0.
-      # This can happen if relayer is started without any snapshots, possible solution;
-      #   - manually wiring relayer data or let installation downloads relayer snapshots
-      #   - and restart relayer
     fi
-  fi
-  echo "waiting for relayer response for block height..."
-  sleep 5
-done
+    echo "waiting for relayer response for block height..."
+    sleep 5
+  done
+fi
+
+cmd="$cmd --height $height"
 
 echo "starting refiner: [$cmd]"
 

contrib/config/docker/silo_datalake.yaml

@@ -0,0 +1,93 @@
+---
+version: '3.8'
+services:
+  relayer:
+    image: nearaurora/srpc2-relayer
+    container_name: srpc2-relayer
+#    pull_policy: missing
+#    build:
+#      dockerfile: ../contrib/config/build/Dockerfile.relayer
+    restart: unless-stopped
+    init: true
+    environment:
+      - NEAR_ENV=%%SILO_NETWORK%%
+      - NODE_ENV=%%SILO_NETWORK%%
+    expose:
+      - '8545'
+    volumes:
+      - ./config/relayer:/config
+      - ./data/relayer:/data/relayer
+      - ./data/refiner:/data/refiner
+      - ./near:/near
+    logging:
+      driver: "json-file"
+      options:
+        max-size: 5m
+        max-file: "5"
+    networks:
+      static-network:
+  refiner:
+    image: nearaurora/srpc2-refiner
+    container_name: srpc2-refiner
+    entrypoint: ["/docker-entrypoint.sh", "%%SILO_FROM_BLOCK%%"]
+#    pull_policy: missing
+#    build:
+#      dockerfile: ../contrib/config/build/Dockerfile.refiner
+    restart: unless-stopped
+    environment:
+      - RUST_BACKTRACE=${RUST_BACKTRACE:-1}
+      - TZ=UTC0
+    volumes:
+      - ./config/refiner:/config # config files
+      - ./data/refiner:/data # Output directory
+      - "%%AWS%%:/root/.aws/credentials:ro"
+    logging:
+      driver: "json-file"
+      options:
+        max-size: 5m
+        max-file: "5"
+    ports:
+      - '127.0.0.1:23030:3030'
+    networks:
+      static-network:
+        ipv4_address: 10.123.49.254
+  proxy:
+    image: nearaurora/reverseproxy:latest
+    container_name: srpc2-proxy
+    restart: unless-stopped
+    init: true
+    expose:
+      - '80'
+    ports:
+      - '20080:80'  # Remove this line to prevent listening on public IP address.
+      # - '127.0.0.1:20080:80' # Remove the hashtag in the beginning of this line to enable listening on 127.0.0.1.
+    volumes:
+      - ./config/nginx:/config
+    logging:
+      driver: "json-file"
+      options:
+        max-size: 5m
+        max-file: "5"
+    networks:
+      static-network:
+        ipv4_address: 10.123.49.253
+  watchtower:
+    image: containrrr/watchtower
+    container_name: srpc2-watchtower
+    restart: unless-stopped
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+#      - ./config/docker/config.json:/config.json
+    logging:
+      driver: "json-file"
+      options:
+        max-size: 5m
+        max-file: "5"
+    command: --interval 30 srpc2-relayer srpc2-refiner srpc2-proxy
+networks:
+  static-network:
+    ipam:
+      config:
+        - subnet: 10.123.49.0/24
+volumes:
+  database:

contrib/config/docker/silo_nearcore.yaml

@@ -0,0 +1,94 @@
+---
+version: '3.8'
+services:
+  relayer:
+    image: nearaurora/srpc2-relayer
+    container_name: srpc2-relayer
+#    pull_policy: missing
+#    build:
+#      dockerfile: ../contrib/config/build/Dockerfile.relayer
+    restart: unless-stopped
+    init: true
+    environment:
+      - NEAR_ENV=%%SILO_NETWORK%%
+      - NODE_ENV=%%SILO_NETWORK%%
+    expose:
+      - '8545'
+    volumes:
+      - ./config/relayer:/config
+      - ./data/relayer:/data/relayer
+      - ./data/refiner:/data/refiner
+      - ./near:/near
+    logging:
+      driver: "json-file"
+      options:
+        max-size: 5m
+        max-file: "5"
+    networks:
+      static-network:
+  refiner:
+    image: nearaurora/srpc2-refiner
+    container_name: srpc2-refiner
+    entrypoint: ["/docker-entrypoint.sh", "%%SILO_FROM_BLOCK%%"]
+#    pull_policy: missing
+#    build:
+#      dockerfile: ../contrib/config/build/Dockerfile.refiner
+    restart: unless-stopped
+    environment:
+      - RUST_BACKTRACE=${RUST_BACKTRACE:-1}
+      - TZ=UTC0
+    volumes:
+      - ./config/refiner:/config # config files
+      - ./data/refiner:/data # Output directory
+      - ./near:/near # Path to Nearcore data
+      - ./engine:/engine
+    logging:
+      driver: "json-file"
+      options:
+        max-size: 5m
+        max-file: "5"
+    ports:
+      - '127.0.0.1:23030:3030'
+    networks:
+      static-network:
+        ipv4_address: 10.123.49.254
+  proxy:
+    image: nearaurora/reverseproxy:latest
+    container_name: srpc2-proxy
+    restart: unless-stopped
+    init: true
+    expose:
+      - '80'
+    ports:
+      - '20080:80'  # Remove this line to prevent listening on public IP address.
+      # - '127.0.0.1:20080:80' # Remove the hashtag in the beginning of this line to enable listening on 127.0.0.1.
+    volumes:
+      - ./config/nginx:/config
+    logging:
+      driver: "json-file"
+      options:
+        max-size: 5m
+        max-file: "5"
+    networks:
+      static-network:
+        ipv4_address: 10.123.49.253
+  watchtower:
+    image: containrrr/watchtower
+    container_name: srpc2-watchtower
+    restart: unless-stopped
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+#      - ./config/docker/config.json:/config.json
+    logging:
+      driver: "json-file"
+      options:
+        max-size: 5m
+        max-file: "5"
+    command: --interval 30 srpc2-relayer srpc2-refiner srpc2-proxy
+networks:
+  static-network:
+    ipam:
+      config:
+        - subnet: 10.123.49.0/24
+volumes:
+  database:

contrib/config/nginx/silo.conf

@@ -0,0 +1,24 @@
+upstream endpoint {
+	server 10.123.49.2:8545;
+}
+
+server {
+    listen *:80;
+
+    location / {
+	client_body_buffer_size 64k;
+        proxy_pass http://endpoint;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "Upgrade";
+        proxy_socket_keepalive on;
+        nchan_websocket_ping_interval 5;
+        nchan_websocket_client_heartbeat PING PONG;
+        proxy_read_timeout 360000s;
+        proxy_send_timeout 360000s;
+    }
+}

contrib/config/refiner/silo_datalake.json

@@ -0,0 +1,17 @@
+{
+    "refiner": {
+        "chain_id": %%SILO_CHAIN_ID%%,
+        "engine_path": "/engine",
+        "tx_tracker_path": "/engine/tx_tracker",
+        "engine_account_id": "%%SILO_ENGINE_ACCOUNT%%"
+    },
+    "input_mode": {
+        "DataLake": {
+            "network": "%%SILO_DATALAKE_NETWORK%%"
+        }
+    },
+    "output_storage": {
+        "path": "/data",
+        "batch_size": 10000
+    }
+}

contrib/config/refiner/silo_nearcore.json

@@ -0,0 +1,17 @@
+{
+    "refiner": {
+        "chain_id": %%SILO_CHAIN_ID%%,
+        "engine_path": "/engine",
+        "tx_tracker_path": "/engine/tx_tracker",
+        "engine_account_id": "%%SILO_ENGINE_ACCOUNT%%"
+    },
+    "input_mode": {
+        "Nearcore": {
+            "path": "/near"
+        }
+    },
+    "output_storage": {
+        "path": "/data",
+        "batch_size": 10000
+    }
+}