feat: delayed fulfillments (#91)

* feat: order delay

* chore: version bump

* feat: dst constraints, new dst constraint: delayed fulfillment

* chore: docs

README.md

@@ -12,6 +12,7 @@
 - [Managing cross-chain risk/reward ratio](#managing-cross-chain-riskreward-ratio)
   - [Reducing transaction finality constraint](#reducing-transaction-finality-constraint)
   - [Setting a budget for non-finalized orders](#setting-a-budget-for-non-finalized-orders)
+  - [Delayed fulfillments](#delayed-fulfillments)
 - [Testing the order execution flow in the wild](#testing-the-order-execution-flow-in-the-wild)
   - [Restricting orders from fulfillment](#restricting-orders-from-fulfillment)
   - [Placing new orders](#placing-new-orders)
@@ -222,6 +223,72 @@ and there is an accidental flood of 100,000 orders worth $1 occurs, you probably
 
 This budged is a hard cap for orders that were not yet finalized after your `dln-taker`'s instance have successfully fulfilled them. As soon as such orders got a finalization status, they got removed effectively releasing the room for other non-finalized orders that can be attempted to be fulfulled.
 
+### Delayed fulfillments
+
+Sometimes you may be willing to run several instances of dln-taker to reduce risks of managing a full bag of assets under one roof, or create a virtual fault tolerant cluster of `dln-taker`'s in which second instance catches up with the new orders when the first instance runs out money.
+
+To avoid a race when all of the instances are attempting to fulfill the same order simultaneously and thus burning each other's gas, there is an opt-in feature that allows configuring delayed fulfillments. This gives the ability to make specific instance wait the given amount of time before starting its attempt to fulfill newly arrived order.
+
+For example, when you want your instance#1 to delay the fulfillment of orders under $1000 by 30s, and all other orders by 60s, you may use `constraints.requiredConfirmationsThresholds[].fulfillmentDelay` and `constraints.fulfillmentDelay` accordingly for each source chain:
+
+```ts
+{
+  chain: ChainId.Avalanche,
+  chainRpc: `${process.env.AVALANCHE_RPC}`,
+
+  constraints: {
+    requiredConfirmationsThresholds: [
+      // expects to receive orders under $1,000 coming from Avalanche as soon as 1 block confirmations,
+      // and starts processing it after a 30s delay
+      {
+        thresholdAmountInUSD: 1000, // USD
+        minBlockConfirmations: 1,   // see transaction finality
+        fulfillmentDelay: 30        // seconds
+      },
+
+      // expects to receive orders under $10,000 coming from Avalanche as soon as 6 block confirmations,
+      // and starts processing it after a 60s delay (see higher level default value)
+      {
+        thresholdAmountInUSD: 10_000, // USD
+        minBlockConfirmations: 6,     // see transaction finality
+        // default value of fulfillmentDelay is implicitly inherited and is set to 60s
+      },
+    ],
+
+    // optional: start processing orders over >$1,000 coming from Avalanche after a 60s delay
+    fulfillmentDelay: 60 // seconds
+  },
+},
+```
+
+At the same time, instance#2 may be configured without such timeouts, so if it goes offline or runs out of money, your instance#1 will catch up with the orders missed by instance#2 after the given time frame.
+
+Additionally, you may be willing to delay orders coming to the specific chain (regardless its source chain). In this case, `dstConstraints` must be used:
+
+```ts
+{
+  chain: ChainId.Ethereum,
+  chainRpc: `${process.env.ETHEREUM_RPC}`,
+
+  dstConstraints: {
+    perOrderValueUpperThreshold: [
+      // start processing all orders under $1,000 coming to Ethereum (from any supported chain) after a 30s delay,
+      // regardless of constraints specified for the supported chains
+      {
+        upperThreshold: 1000, // USD
+        fulfillmentDelay: 30  // seconds
+      },
+    ],
+
+    // optional: start processing orders over >$1,000 coming to ethereum (from any supported chain) after a 40s delay,
+    // regardless of constraints specified for the supported chains
+    fulfillmentDelay: 40 // seconds
+  },
+},
+```
+
+Mind that `dstConstraints` property has precedence over `constraints`.
+
 ## Testing the order execution flow in the wild
 
 After you have set up and launched `dln-taker`, you may wish to give it a try in a limited conditions.

package.json

@@ -1,7 +1,7 @@
 {
   "name": "@debridge-finance/dln-executor",
   "description": "DLN executor is the rule-based daemon service developed to automatically execute orders placed on the deSwap Liquidity Network (DLN) across supported blockchains",
-  "version": "2.4.0",
+  "version": "2.5.0",
   "author": "deBridge",
   "license": "GPL-3.0-only",
   "homepage": "https://debridge.finance",

src/config.ts

@@ -81,6 +81,22 @@ export type ChainEnvironment = {
   };
 };
 
+export type DstOrderConstraints = {
+    /**
+     * Defines a delay (in seconds) the dln-taker should wait before starting to process each new (non-archival) order
+     * coming to this chain after it first saw it.
+     */
+    fulfillmentDelay?: number;
+}
+
+export type SrcOrderConstraints = {
+    /**
+     * Defines a delay (in seconds) the dln-taker should wait before starting to process each new (non-archival) order
+     * coming from this chain after it first saw it.
+     */
+    fulfillmentDelay?: number;
+}
+
 /**
  * Represents a chain configuration where orders can be fulfilled.
  */
@@ -113,7 +129,7 @@ export interface ChainDefinition {
   /**
    * Defines constraints imposed on all orders coming from this chain
    */
-  constraints?: {
+  constraints?: SrcOrderConstraints & {
     /**
      * Defines necessary and sufficient block confirmation thresholds per worth of order expressed in dollars.
      * For example, you may want to fulfill orders coming from Ethereum:
@@ -129,7 +145,10 @@ export interface ChainDefinition {
      * ]
      * ```
      */
-    requiredConfirmationsThresholds?: Array<{thresholdAmountInUSD: number, minBlockConfirmations: number}>;
+    requiredConfirmationsThresholds?: Array<SrcOrderConstraints & {
+      thresholdAmountInUSD: number,
+      minBlockConfirmations?: number,
+    }>;
 
     /**
      * Defines a budget (a hard cap) of all successfully fulfilled orders' value (expressed in USD) that
@@ -143,7 +162,21 @@ export interface ChainDefinition {
      * one by one as soon as fulfilled orders are being finalized.
      */
     nonFinalizedTVLBudget?: number;
-  }
+  },
+
+  /**
+   * Defines constraints imposed on all orders coming to this chain. These properties have precedence over `constraints` property
+   */
+  dstConstraints?: DstOrderConstraints & {
+    /**
+     * Defines custom constraints for orders falling into the given upper thresholds expressed in US dollars.
+     *
+     * Mind that these constraints have precedence over higher order constraints
+     */
+    perOrderValueUpperThreshold?: Array<DstOrderConstraints & {
+      upperThreshold: number
+    }>
+  },
 
   //
   // taker related

src/executors/executor.ts

@@ -29,7 +29,7 @@ import { ProviderAdapter } from "../providers/provider.adapter";
 import { SolanaProviderAdapter } from "../providers/solana.provider.adapter";
 import { HooksEngine } from "../hooks/HooksEngine";
 import { NonFinalizedOrdersBudgetController } from "../processors/NonFinalizedOrdersBudgetController";
-
+import { DstOrderConstraints as RawDstOrderConstraints, SrcOrderConstraints as RawSrcOrderConstraints } from "../config";
 
 const BLOCK_CONFIRMATIONS_HARD_CAPS: { [key in SupportedChain]: number } = {
   [SupportedChain.Avalanche]: 15,
@@ -49,25 +49,45 @@ export type ExecutorInitializingChain = Readonly<{
   client: Solana.PmmClient | Evm.PmmEvmClient;
 }>;
 
-type UsdWorthBlockConfirmationConstraints = Array<{
-  usdWorthFrom: number,
-  usdWorthTo: number,
-  minBlockConfirmations: number,
-}>;
+type DstOrderConstraints = Readonly<{
+  fulfillmentDelay: number;
+}>
+
+type DstConstraintsPerOrderValue = Array<
+    DstOrderConstraints & Readonly<{
+      upperThreshold: number;
+  }>
+>;
+
+type SrcOrderConstraints = Readonly<{
+  fulfillmentDelay: number;
+}>
 
-export type ExecutorSupportedChain = {
+type SrcConstraintsPerOrderValue = Array<
+  SrcOrderConstraints & Readonly<{
+      upperThreshold: number;
+      minBlockConfirmations: number;
+  }>
+>;
+
+export type ExecutorSupportedChain = Readonly<{
   chain: ChainId;
   chainRpc: string;
   srcFilters: OrderFilter[];
   dstFilters: OrderFilter[];
-  usdAmountConfirmations: UsdWorthBlockConfirmationConstraints;
   nonFinalizedOrdersBudgetController: NonFinalizedOrdersBudgetController;
+  srcConstraints: Readonly<SrcOrderConstraints & {
+    perOrderValue: SrcConstraintsPerOrderValue
+  }>,
+  dstConstraints: Readonly<DstOrderConstraints & {
+    perOrderValue: DstConstraintsPerOrderValue
+  }>,
   orderProcessor: processors.IOrderProcessor;
   unlockProvider: ProviderAdapter;
   fulfillProvider: ProviderAdapter;
   beneficiary: string;
   client: Solana.PmmClient | Evm.PmmEvmClient;
-};
+}>;
 
 export interface IExecutor {
   readonly tokenPriceService: PriceTokenService;
@@ -259,8 +279,15 @@ export class Executor implements IExecutor {
           this.logger
         ),
         client,
-        usdAmountConfirmations: this.getConfirmationRanges(chain.chain as unknown as SupportedChain, chain),
         beneficiary: chain.beneficiary,
+        srcConstraints: {
+          ...this.getSrcConstraints(chain.constraints || {}),
+          perOrderValue: this.getSrcConstraintsPerOrderValue(chain.chain as unknown as SupportedChain, chain.constraints || {})
+        },
+        dstConstraints: {
+          ...this.getDstConstraints(chain.dstConstraints || {}),
+          perOrderValue: this.getDstConstraintsPerOrderValue(chain.dstConstraints || {}),
+        }
       };
 
       clients[chain.chain] = client;
@@ -284,10 +311,14 @@ export class Executor implements IExecutor {
         address: chain.unlockProvider.address as string,
       };
     });
-    const minConfirmationThresholds = Object.values(this.chains).map(chain => ({
-      chainId: chain.chain,
-      points: chain.usdAmountConfirmations.map(t => t.minBlockConfirmations)
-    }))
+    const minConfirmationThresholds = Object.values(this.chains)
+      .map(chain => ({
+        chainId: chain.chain,
+        points: chain.srcConstraints.perOrderValue
+          .map(t => t.minBlockConfirmations)
+          .filter(t => t > 0) // skip empty block confirmations
+      }))
+      .filter(range => range.points.length > 0); // skip chains without necessary confirmation points
     orderFeed.init(this.execute.bind(this), unlockAuthorities, minConfirmationThresholds, hooksEngine);
 
     // Override internal slippage calculation: do not reserve slippage buffer for pre-fulfill swap
@@ -296,29 +327,43 @@ export class Executor implements IExecutor {
     this.isInitialized = true;
   }
 
-  private getConfirmationRanges(chain: SupportedChain, definition: ChainDefinition): UsdWorthBlockConfirmationConstraints {
-    const ranges: UsdWorthBlockConfirmationConstraints = [];
-    const requiredConfirmationsThresholds = definition.constraints?.requiredConfirmationsThresholds || [];
-    requiredConfirmationsThresholds
-      .sort((a, b) => a.thresholdAmountInUSD < b.thresholdAmountInUSD ? -1 : 1) // sort by usdWorth ASC
-      .forEach((threshold, index, thresholdsSortedByUsdWorth) => {
-        const prev = index === 0 ? {minBlockConfirmations: 0, thresholdAmountInUSD: 0} : thresholdsSortedByUsdWorth[index - 1];
+  private getDstConstraintsPerOrderValue(configDstConstraints: ChainDefinition['dstConstraints']): DstConstraintsPerOrderValue {
+    return (configDstConstraints?.perOrderValueUpperThreshold || [])
+      .map(constraint => ({
+        upperThreshold: constraint.upperThreshold,
+        ...this.getDstConstraints(constraint, configDstConstraints)
+      }))
+      // important to sort by upper bound ASC for easier finding of the corresponding range
+      .sort((constraintA, constraintB) => constraintA.upperThreshold - constraintB.upperThreshold);
+  }
 
-        if (threshold.minBlockConfirmations <= prev.minBlockConfirmations) {
-          throw new Error(`Unable to set required confirmation threshold for $${threshold.thresholdAmountInUSD} on ${SupportedChain[chain]}: minBlockConfirmations (${threshold.minBlockConfirmations}) must be greater than ${prev.minBlockConfirmations}`)
-        }
-        if (BLOCK_CONFIRMATIONS_HARD_CAPS[chain] <= threshold.minBlockConfirmations) {
-          throw new Error(`Unable to set required confirmation threshold for $${threshold.thresholdAmountInUSD} on ${SupportedChain[chain]}: minBlockConfirmations (${threshold.minBlockConfirmations}) must be less than max block confirmations (${BLOCK_CONFIRMATIONS_HARD_CAPS[chain]})`)
+  private getDstConstraints(primaryConstraints: RawDstOrderConstraints, defaultConstraints?: RawDstOrderConstraints): DstOrderConstraints {
+    return {
+      fulfillmentDelay: primaryConstraints?.fulfillmentDelay || defaultConstraints?.fulfillmentDelay || 0
+    }
+  }
+
+  private getSrcConstraintsPerOrderValue(chain: SupportedChain, configDstConstraints: ChainDefinition['constraints']): SrcConstraintsPerOrderValue {
+    return (configDstConstraints?.requiredConfirmationsThresholds || [])
+      .map(constraint => {
+        if (BLOCK_CONFIRMATIONS_HARD_CAPS[chain] <= (constraint.minBlockConfirmations || 0)) {
+          throw new Error(`Unable to set required confirmation threshold for $${constraint.thresholdAmountInUSD} on ${SupportedChain[chain]}: minBlockConfirmations (${constraint.minBlockConfirmations}) must be less than max block confirmations (${BLOCK_CONFIRMATIONS_HARD_CAPS[chain]})`);
         }
 
-        ranges.push({
-          usdWorthFrom: prev.thresholdAmountInUSD,
-          usdWorthTo: threshold.thresholdAmountInUSD,
-          minBlockConfirmations: threshold.minBlockConfirmations
-        })
-      });
+        return {
+          upperThreshold: constraint.thresholdAmountInUSD,
+          minBlockConfirmations: constraint.minBlockConfirmations || 0,
+          ...this.getSrcConstraints(constraint, configDstConstraints)
+          }
+      })
+      // important to sort by upper bound ASC for easier finding of the corresponding range
+      .sort((constraintA, constraintB) => constraintA.upperThreshold - constraintB.upperThreshold);
+  }
 
-    return ranges;
+  private getSrcConstraints(primaryConstraints: RawSrcOrderConstraints, defaultConstraints?: RawSrcOrderConstraints): SrcOrderConstraints {
+    return {
+      fulfillmentDelay: primaryConstraints?.fulfillmentDelay || defaultConstraints?.fulfillmentDelay || 0
+    }
   }
 
   async execute(nextOrderInfo: IncomingOrder<any>) {
@@ -410,8 +455,8 @@ export class Executor implements IExecutor {
         logger,
         config: this,
         giveChain,
+        takeChain
       },
-      attempts: 0
     });
 
     return true;

src/hooks/HookEnums.ts

@@ -49,6 +49,11 @@ export enum PostponingReason {
    * Unexpected error
    */
   UNHANDLED_ERROR,
+
+  /**
+   * indicates that this order is forcibly delayed according to this dln-takers instance configuration
+   */
+  FORCED_DELAY,
 }
 
 export enum RejectionReason {

src/interfaces.ts

@@ -1,7 +1,7 @@
 import { ChainId, OrderData } from "@debridge-finance/dln-client";
 import { Logger } from "pino";
 
-import { OrderProcessorContext } from "./processors/base";
+import { OrderId, OrderProcessorContext } from "./processors/base";
 import { HooksEngine } from "./hooks/HooksEngine";
 
 export enum OrderInfoStatus {
@@ -35,12 +35,11 @@ export type IncomingOrder<T extends OrderInfoStatus> = {
 ) & (T extends OrderInfoStatus.Fulfilled ? { unlockAuthority: string } : {}
 ) & (T extends OrderInfoStatus.Created ? { finalization_info: FinalizationInfo } : {})
 
-export type ProcessOrder = (params: IncomingOrderContext) => Promise<void>;
+export type ProcessOrder = (orderId: OrderId) => Promise<void>;
 
 export type IncomingOrderContext = {
   orderInfo: IncomingOrder<OrderInfoStatus>;
   context: OrderProcessorContext;
-  attempts: number;
 };
 
 export type OrderProcessorFunc = (order: IncomingOrder<any>) => Promise<void>;

src/processors/base.ts

@@ -16,10 +16,13 @@ import {
 import { IncomingOrderContext } from "../interfaces";
 import { HooksEngine } from "../hooks/HooksEngine";
 
+export type OrderId = string;
+
 export class OrderProcessorContext {
   logger: Logger;
   config: IExecutor;
   giveChain: ExecutorSupportedChain;
+  takeChain: ExecutorSupportedChain;
 }
 
 export class OrderProcessorInitContext {