Merge pull request #59 from warp-contracts/rc/sync_timestamp

Rc/sync timestamp

.env.defaults

@@ -1,18 +1,36 @@
 # test, prod
-ENV=test
+ENV=prod
 
+WARP_GW_URL='https://gw.warp.cc'
+
+# PG config
+PG_HOST='localhost'
+PG_DATABASE='postgres'
+PG_USER_WARP='warp'
+PG_USER_WARP_PASSWORD='warp_password'
+PG_USER_DRE='dre'
+PG_USER_DRE_PASSWORD='dre_password'
+PG_PORT=21726
+PG_SSL='false'
+PG_MIN_CONTRACT_ENTRIES=100
+PG_MAX_CONTRACT_ENTRIES=1000
+
+FIRST_INTERACTION_TIMESTAMP=1592991945000
+REDIS_PUBLISH_STATE=false
 APPSYNC_PUBLISH_STATE=false
 # required only if APPSYNC_PUBLISH_STATE=true
 APPSYNC_KEY=''
+APPSYNC_STREAM=''
 
 NODE_JWK_KEY=''
+UPDATE_MODE='poll'
+# 3h (3 * 60 * 60s), 1h (60 * 60s), 1m (60s), 2s
+SYNC_WINDOW_SECONDS='[10800,3600,60,2]'
+POLL_RESPONSE_LENGTH_LIMIT=30000
+POLL_INTERACTIONS_URL='http://api-dre-sync.warp.cc/v1/ro/interactions'
+# Spawns a separate node process to evaluate a partition of interactions
+POLL_FORK_PROCESS=false
 
-# redis, streamr
-PUBSUB_TYPE=redis
-# required only if PUBSUB_TYPE=streamr
-STREAMR_STREAM_ID=''
-STREAMR_STREAM_HOST=''
-STREAMR_STREAM_PORT=''
 
 ## Warp GW Redis connection, required only if PUBSUB_TYPE=redis
 # required only if the PUBSUB_TYPE=redis #
@@ -50,6 +68,8 @@ EVALUATION_MAXINTERACTIONEVALUATIONTIMESECONDS=10
 EVALUATION_ALLOWBIGINT=true
 EVALUATION_UNSAFECLIENT=skip
 EVALUATION_INTERNALWRITES=true
+EVALUATION_WHITELIST_SOURCES='[]'
+EVALUATION_BLACKLISTED_CONTRACTS='[]'
 
 ### Bull MQ redis connection ###
 BULLMQ_PORT=6379
@@ -60,9 +80,18 @@ BULLMQ_TLS=
 BULLMQ_ENABLE_OFFLINE_QUEUE=true
 BULLMQ_LAZY_CONNECT=true
 
+### Available functionality
+FUNC_VIEW_STATE=true
+PROCESS_CONTRACT_EVENTS=false
+
 ### Workers ###
-WORKERS_REGISTER=2
-WORKERS_UPDATE=2
-WORKERS_JOB_ID_REFRESH_SECONDS=1
+WORKERS_REGISTER=1
+WORKERS_UPDATE=1
+# By default maintenance is turned off
+WORKERS_MAINTENANCE=0
+# Every 5 minutes (value is in ms)
+WORKERS_MAINTENANCE_WINDOW=300000
+WORKERS_POST_EVAL=1
+WORKERS_JOB_ID_REFRESH_SECONDS=2
 WORKERS_MAX_FAILURES=3
 WORKERS_MAX_STATESIZE=30000000

.env.defaults.u

@@ -0,0 +1,145 @@
+# test, prod
+ENV=prod
+
+MY_NAME_IS='DRE-U'
+WARP_GW_URL='https://gw.warp.cc'
+
+# PG config
+PG_HOST='localhost'
+PG_DATABASE='postgres'
+PG_USER_WARP='warp'
+PG_USER_WARP_PASSWORD='warp_password'
+PG_USER_DRE='dre'
+PG_USER_DRE_PASSWORD='dre_password'
+PG_PORT=21726
+PG_SSL='false'
+PG_MIN_CONTRACT_ENTRIES=100000
+PG_MAX_CONTRACT_ENTRIES=1000000
+
+FIRST_INTERACTION_TIMESTAMP=1666220657408
+REDIS_PUBLISH_STATE=false
+APPSYNC_PUBLISH_STATE=false
+# required only if APPSYNC_PUBLISH_STATE=true
+APPSYNC_KEY=''
+APPSYNC_STREAM=''
+
+NODE_JWK_KEY=''
+UPDATE_MODE='poll'
+# 6h (6 * 60 * 60s), 1h (60 * 60s), 1m (60s), 2s
+SYNC_WINDOW_SECONDS='[21600,3600,60,2]'
+POLL_RESPONSE_LENGTH_LIMIT=30000
+POLL_INTERACTIONS_URL='http://api-dre-sync.warp.cc/v1/interactions'
+# Spawns a separate node process to evaluate a partition of interactions
+POLL_FORK_PROCESS=false
+
+
+## Warp GW Redis connection, required only if PUBSUB_TYPE=redis
+# required only if the PUBSUB_TYPE=redis #
+GW_PORT=6379
+GW_HOST=dre-redis-read.warp.cc
+GW_USERNAME=contracts
+GW_PASSWORD=''
+GW_TLS=true
+GW_ENABLE_OFFLINE_QUEUE=true
+GW_LAZY_CONNECT=true
+GW_TLS_CA_CERT="-----BEGIN CERTIFICATE-----
+MIIDETCCAfmgAwIBAgIQHXVHBz5eF6OFL5LG9vGB1zANBgkqhkiG9w0BAQsFADAT
+MREwDwYDVQQDEwhyZWRpcy1jYTAeFw0yMzAyMjMxNDU1MTNaFw0yNDAyMjMxNDU1
+MTNaMBMxETAPBgNVBAMTCHJlZGlzLWNhMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A
+MIIBCgKCAQEAqqOn6V/e69b+A5mp+u1b79duxX59UoKk21cjywol2tg5GfTxq16x
+hYne+g1jy91wRz36K1El9Qa8OPyJCGbe+Ab2iMQ4361X4CTMSMd18dLjjy+urm2x
+oyCM82MZO14oLr2J2yJk1DFERwW5GFVFluJto/LmwY5eA/7GK3nm5bqZQaYgqgpH
+GuypcjM1AMubw7m9n55Nol93jytr3eFUQcZKKFqJlP6xJJgFltsGwDwSu3sjolwu
+y7JHvNTgyC/nkKwQ899nF4UN3QaYtH9WMShTHzIrIFQLjxk/qq3UKgIqah/Wv/nV
+G9JWRGaodu/suSVM7w4RrR1KTTmWzdkpiQIDAQABo2EwXzAOBgNVHQ8BAf8EBAMC
+AqQwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMA8GA1UdEwEB/wQFMAMB
+Af8wHQYDVR0OBBYEFPFhc8RVSce6AH5fG2Wq++gw/EbuMA0GCSqGSIb3DQEBCwUA
+A4IBAQCcAIJiEP3SITKiMVPPvbHTSfkwSIi33LC5JwozXqDH8I9J2PCgbTGrnvlf
+au3VzhRuve7kobWsWtZoiLuo08P1dvEfV2mCgyknIvP0vrY6qqO9YnOObEGQASIk
+Tb5RoAjK/ccNXUP7n6Ck21xAbOXd2JITADZtLlsDPYvmR7IWdVgDFUAlhUf8IrHM
+tz/XOyBHYX38rEvY7+5UMNUvRwqZ4xrDE/bwIfmBjLMZCuNkQhCrd0SseHECjWuH
+NIcUeuv6s0p1SpLBlbDtBVoaOQUqKURS2ynqYnLvqNwQuoNG69n4U2IbLNkoV7Sz
+rirJbWgiegob4xr6fkr+n7z41EwO
+-----END CERTIFICATE-----"
+
+### Evaluation options
+EVALUATION_USEVM2=true
+EVALUATION_MAXCALLDEPTH=5
+EVALUATION_MAXINTERACTIONEVALUATIONTIMESECONDS=10
+EVALUATION_ALLOWBIGINT=true
+EVALUATION_UNSAFECLIENT=skip
+EVALUATION_INTERNALWRITES=true
+# The following whitelist is intended for the U contract evaluation only.
+EVALUATION_WHITELIST_SOURCES='
+[
+    "Of9pi--Gj7hCTawhgxOwbuWnFI1h24TTgO5pw8ENJNQ",
+    "W78KEzU8vIODFHzwa9ab7SS7Pjchc7KenivCiwSHxBY",
+    "kP1Ed8AMvaaBrEFjatP4pSmiE_fsRrGS0EcBMQYYiyc",
+    "mGxosQexdvrvzYCshzBvj18Xh1QmZX16qFJBuh4qobo",
+    "7qv5x9A0NgAlTdMnBc1H2HFvN-te0kzzuT9RNt_66g8",
+    "eIAyBgHH-H7Qzw9fj7Austj30QKPQn27eaakvpOUSR8",
+    "Of9pi--Gj7hCTawhgxOwbuWnFI1h24TTgO5pw8ENJNQ",
+    "ovWCp0xKuHtq-bADXbtiNr6umwb_AE73kVZWtfHlX3w",
+    "1hDZBRSptTNgnACuO9qGHLbaOfnAcMBKCHcHPRhMWUY",
+    "LBcYEl2zwKDApj1Cow1_BYyiicxVV7OCZTexsjk6mB4",
+    "dRTFmLwJ3cNqdNvFK4yUvwc13CrJtFOmLymLxL4HWOE",
+    "yXPm9-9VyxH9otGf7xim0EJsnt21IJN8qJjanFTC_kc",
+    "qOd7mNAJdju9PxtsRJbel4Zu3xYgEwUbxW8U14czjD8",
+    "0GOnb0o9c232d6SXF_HXHbGzfIdiYeos7U5jobOSZ_c",
+    "8kPgNMm7dZUVk93T7wq05otEy1oDNqZhyD3L7WrcMTY",
+    "yDAppVePqGU1qcRnxdk-AShpIJ0RHCZixOMXtJTgm4Y",
+    "W7V0n7g2UKhCee1QDTpvAq6eI6pP9jCS860uF70TbYY",
+    "h9v17KHV4SXwdW2-JHU6a23f6R0YtbXZJJht8LfP8QM",
+    "DXc7dRQe8AZRPNqHzjewWWeT7dUeEQxBEEYOHlomp2g",
+    "j9Lk3cTmukZS2-hae9GYxK1CuHtWtHcA1V5-tkIfu5k",
+    "JcELSCid4ZWZxW8Mj0nza12mm4bFjyAOfpW9anDgaMY"
+]'
+
+# "Of9pi--Gj7hCTawhgxOwbuWnFI1h24TTgO5pw8ENJNQ"  // Atomic Asset
+# "W78KEzU8vIODFHzwa9ab7SS7Pjchc7KenivCiwSHxBY"  // STAMP
+# "kP1Ed8AMvaaBrEFjatP4pSmiE_fsRrGS0EcBMQYYiyc"  // STAMP-evolve
+# "mGxosQexdvrvzYCshzBvj18Xh1QmZX16qFJBuh4qobo"  // U
+# "7qv5x9A0NgAlTdMnBc1H2HFvN-te0kzzuT9RNt_66g8"  // UCM contract old
+# "eIAyBgHH-H7Qzw9fj7Austj30QKPQn27eaakvpOUSR8"  // Facts
+# "Of9pi--Gj7hCTawhgxOwbuWnFI1h24TTgO5pw8ENJNQ"  // Pages
+# "ovWCp0xKuHtq-bADXbtiNr6umwb_AE73kVZWtfHlX3w"  // VouchDAO
+# "1hDZBRSptTNgnACuO9qGHLbaOfnAcMBKCHcHPRhMWUY"  // VouchDAO-evolve
+# "LBcYEl2zwKDApj1Cow1_BYyiicxVV7OCZTexsjk6mB4"  // UCM contract new
+# "dRTFmLwJ3cNqdNvFK4yUvwc13CrJtFOmLymLxL4HWOE"  // UCM contract evolve
+# "yXPm9-9VyxH9otGf7xim0EJsnt21IJN8qJjanFTC_kc"  // UCM contract evolve
+# "qOd7mNAJdju9PxtsRJbel4Zu3xYgEwUbxW8U14czjD8"  // UCM contract evolve
+# "0GOnb0o9c232d6SXF_HXHbGzfIdiYeos7U5jobOSZ_c"  // UCM contract evolve
+# "8kPgNMm7dZUVk93T7wq05otEy1oDNqZhyD3L7WrcMTY"  // UCM contract evolve
+# "yDAppVePqGU1qcRnxdk-AShpIJ0RHCZixOMXtJTgm4Y"  // UCM contract evolve
+# "W7V0n7g2UKhCee1QDTpvAq6eI6pP9jCS860uF70TbYY"  // UCM contract evolve
+# "h9v17KHV4SXwdW2-JHU6a23f6R0YtbXZJJht8LfP8QM"  // FAIR
+# "DXc7dRQe8AZRPNqHzjewWWeT7dUeEQxBEEYOHlomp2g"  // UCM contract evolve
+# "j9Lk3cTmukZS2-hae9GYxK1CuHtWtHcA1V5-tkIfu5k"  // PST ASSET
+# "JcELSCid4ZWZxW8Mj0nza12mm4bFjyAOfpW9anDgaMY"  // AR.IO ANT
+
+EVALUATION_BLACKLISTED_CONTRACTS='[]'
+
+### Bull MQ redis connection ###
+BULLMQ_PORT=6379
+BULLMQ_HOST=
+BULLMQ_USERNAME=
+BULLMQ_PASSWORD=
+BULLMQ_TLS=
+BULLMQ_ENABLE_OFFLINE_QUEUE=true
+BULLMQ_LAZY_CONNECT=true
+
+### Available functionality
+FUNC_VIEW_STATE=false
+PROCESS_CONTRACT_EVENTS=false
+
+### Workers ###
+WORKERS_REGISTER=1
+WORKERS_UPDATE=1
+# By default maintenance is turned off
+WORKERS_MAINTENANCE=0
+# Every 5 minutes (value is in ms)
+WORKERS_MAINTENANCE_WINDOW=300000
+WORKERS_POST_EVAL=1
+WORKERS_JOB_ID_REFRESH_SECONDS=2
+WORKERS_MAX_FAILURES=3
+WORKERS_MAX_STATESIZE=30000000

.gitignore

@@ -5,6 +5,7 @@ db*.sqlite
 .secrets
 .secrets/.env
 .secrets/local.env
+certs
 gateway.lock
 network-cache.json
 .DS_Store

Dockerfile

@@ -11,7 +11,10 @@ RUN mv .env.defaults .env
 RUN echo $(git rev-parse HEAD) > GIT_HASH
 RUN rm -rf .git/
 
+# DRE cache
 VOLUME /app/sqlite
+
+# Warp SDK cache
 VOLUME /app/cache
 
 CMD ["node", "src/listener.js"]

README.md

@@ -1,4 +1,58 @@
 # Warp D.R.E.
 
 A Delegated Resolution Environment for Warp Contracts.
-Docs are availalbe [here](https://academy.warp.cc/docs/dre/overview).
+Docs are available [here](https://academy.warp.cc/docs/dre/overview).
+
+
+## Update mode
+DRE can run in two different modes.
+Each of them listens on gateways notification and registers new contracts, but upcoming new interactions are handled differently. 
+
+#### Poll
+Node fetches interactions directly from gateway in time intervals.
+Interaction are grouped and executed sequentially.
+This is the default mode.
+
+
+#### Subscription
+Node listens on new interactions events and evaluates contracts immediately after every interaction.
+This offers a faster contract evaluation and serves the latest contract state 
+
+Update mode can be set up using env config
+``UPDATE_MODE='poll'`` or
+``UPDATE_MODE='subscription'``
+
+## Deployment
+
+### 1.  Database
+##### 1. Local env  
+1. `npm run run-docker:local`
+2. Confirm that database is running locally
+```bash
+docker ps -a
+
+# should display something similar to:
+CONTAINER ID   IMAGE              COMMAND                  CREATED          STATUS          PORTS                      NAMES
+f240114e69a0   postgres:14        "docker-entrypoint.s…"   11 minutes ago   Up 11 minutes   0.0.0.0:21726->5432/tcp    dre-postgres
+c3ee9a099cda   redis:7.0-alpine   "docker-entrypoint.s…"   11 minutes ago   Up 11 minutes   127.0.0.1:6379->6379/tcp   warp-dre-node-bullmq-1
+```
+
+3. Run database initialization script:
+ `npm run db-setup:local` (the default password for user `postgres` is `postgres`)
+
+
+
+##### 2. Prod env
+TBA
+
+### Starting DRE node
+1. `npm install --force`
+2. `npm install pm2 -g`
+3. `cp .env.defaults .env`
+4. Update `.env` config file.
+   1. Set `NODE_JWK_KEY` with your Arweave wallet JWK
+   2. Update `.env` passwords for your local postgres (only if you changed them!).
+   3. Optionally you can change the 'UPDATE_MODE'. By default, it is set to poll mode.
+5. `pm2 install pm2-logrotate` 
+6. `pm2 start src/syncer.js`
+7. `pm2 start src/listener.js`

docker-compose-local.yml

@@ -1,29 +1,19 @@
 version: '3.8'
 services:
-  dre:
-    build: .
-    env_file:
-      - .env
-    environment:
-      BULLMQ_HOST: "${BULLMQ_HOST:-bullmq}"
-    ports:
-      - '80:8080'
-
-    volumes:
-      - dre-sqlite:/app/sqlite
-      - dre-cache:/app/cache
   bullmq:
     image: redis:7.0-alpine
     restart: always
     ports:
       - '127.0.0.1:6379:6379'
     command: redis-server --save 20 1 --loglevel warning --maxmemory-policy noeviction
-    volumes:
-      - cache:/data
-volumes:
-  cache:
-    driver: local
-  dre-sqlite:
-    driver: local
-  dre-cache:
-    driver: local
+
+  postgres:
+    image: postgres:14
+    container_name: dre-postgres
+    restart: always
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DB=postgres
+    ports:
+      - '21726:5432'

docker-compose.yml

@@ -11,20 +11,20 @@ services:
       - '80:8080'
 
     volumes:
-      - dre-sqlite:/app/sqlite
-      - dre-cache:/app/cache
+      - dre-cache:/app/sqlite
+      - warp-cache:/app/cache
   bullmq:
     image: redis:7.0-alpine
     restart: always
     ports:
       - '127.0.0.1:6379:6379'
     command: redis-server --save 20 1 --loglevel warning --maxmemory-policy noeviction
     volumes:
-      - cache:/data
+      - bullmq-cache:/data
 volumes:
-  cache:
-    driver: local
-  dre-sqlite:
+  bullmq-cache:
     driver: local
   dre-cache:
     driver: local
+  warp-cache:
+    driver: local

knexConfigEventsDb.js

@@ -0,0 +1,19 @@
+const path = require('path');
+
+module.exports = {
+  client: 'better-sqlite3',
+  connection: {
+    filename: path.resolve(__dirname, `sqlite/node-events.sqlite`)
+  },
+  useNullAsDefault: true,
+  pool: {
+    afterCreate: (conn, cb) => {
+      // https://github.com/knex/knex/issues/4971#issuecomment-1030701574
+      conn.pragma('journal_mode = WAL');
+      cb();
+    }
+  },
+  migrations: {
+    directory: './src/db/migrations/eventsDb'
+  }
+};