Merge remote-tracking branch 'sweeper-source/master' into sweeper

fee-sweeper/Cargo.toml

@@ -0,0 +1,42 @@
+[package]
+name = "fee-sweeper"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+# === CLI + Runtime === #
+clap = { version = "4.5.3", features = ["derive", "env"] }
+tokio = { version = "1.10", features = ["full"] }
+
+# === Infra === #
+aws-sdk-secretsmanager = "1.37"
+aws-config = "1.5"
+diesel = { version = "2.1", features = ["postgres", "numeric", "uuid"] }
+
+# === Blockchain Interaction === #
+alloy-sol-types = "0.3.1"
+ethers = "2"
+
+# === Renegade Dependencies === #
+arbitrum-client = { git = "https://github.com/renegade-fi/renegade.git", features = [
+    "rand",
+] }
+renegade-api = { package = "external-api", git = "https://github.com/renegade-fi/renegade.git" }
+renegade-common = { package = "common", git = "https://github.com/renegade-fi/renegade.git" }
+renegade-constants = { package = "constants", git = "https://github.com/renegade-fi/renegade.git" }
+renegade-circuits = { package = "circuits", git = "https://github.com/renegade-fi/renegade.git" }
+renegade-circuit-types = { package = "circuit-types", git = "https://github.com/renegade-fi/renegade.git" }
+renegade-crypto = { git = "https://github.com/renegade-fi/renegade.git" }
+renegade-util = { package = "util", git = "https://github.com/renegade-fi/renegade.git" }
+
+# === Misc Dependencies === #
+base64 = "0.22"
+bigdecimal = { version = "0.3", features = ["serde"] }
+futures = "0.3"
+http = "1.1"
+num-bigint = "0.4"
+reqwest = { version = "0.12", features = ["json"] }
+serde = "1.0"
+serde_json = "1.0"
+tracing = "0.1"
+uuid = "1.8"

fee-sweeper/diesel.toml

@@ -0,0 +1,9 @@
+# For documentation on how to configure this file,
+# see https://diesel.rs/guides/configuring-diesel-cli
+
+[print_schema]
+file = "src/db/schema.rs"
+custom_type_derives = ["diesel::query_builder::QueryId", "Clone"]
+
+[migrations_directory]
+dir = "./migrations"

fee-sweeper/migrations/00000000000000_diesel_initial_setup/down.sql

@@ -0,0 +1,6 @@
+-- This file was automatically created by Diesel to setup helper functions
+-- and other internal bookkeeping. This file is safe to edit, any future
+-- changes will be added to existing projects as new migrations.
+
+DROP FUNCTION IF EXISTS diesel_manage_updated_at(_tbl regclass);
+DROP FUNCTION IF EXISTS diesel_set_updated_at();

fee-sweeper/migrations/00000000000000_diesel_initial_setup/up.sql

@@ -0,0 +1,36 @@
+-- This file was automatically created by Diesel to setup helper functions
+-- and other internal bookkeeping. This file is safe to edit, any future
+-- changes will be added to existing projects as new migrations.
+
+
+
+
+-- Sets up a trigger for the given table to automatically set a column called
+-- `updated_at` whenever the row is modified (unless `updated_at` was included
+-- in the modified columns)
+--
+-- # Example
+--
+-- ```sql
+-- CREATE TABLE users (id SERIAL PRIMARY KEY, updated_at TIMESTAMP NOT NULL DEFAULT NOW());
+--
+-- SELECT diesel_manage_updated_at('users');
+-- ```
+CREATE OR REPLACE FUNCTION diesel_manage_updated_at(_tbl regclass) RETURNS VOID AS $$
+BEGIN
+    EXECUTE format('CREATE TRIGGER set_updated_at BEFORE UPDATE ON %s
+                    FOR EACH ROW EXECUTE PROCEDURE diesel_set_updated_at()', _tbl);
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION diesel_set_updated_at() RETURNS trigger AS $$
+BEGIN
+    IF (
+        NEW IS DISTINCT FROM OLD AND
+        NEW.updated_at IS NOT DISTINCT FROM OLD.updated_at
+    ) THEN
+        NEW.updated_at := current_timestamp;
+    END IF;
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;

fee-sweeper/migrations/2024-06-15-202249_create_last_indexed_table/down.sql

@@ -0,0 +1,2 @@
+-- Drop the indexing metadata table
+DROP TABLE IF EXISTS indexing_metadata;

fee-sweeper/migrations/2024-06-15-202249_create_last_indexed_table/up.sql

@@ -0,0 +1,8 @@
+-- Create the table that stores indexing metadata
+CREATE TABLE indexing_metadata (
+    key TEXT PRIMARY KEY,
+    value TEXT NOT NULL
+);
+
+-- Insert a row with the latest block number set to zero
+INSERT INTO indexing_metadata (key, value) VALUES ('latest_block', '0');

fee-sweeper/migrations/2024-06-15-203503_create_fees_table/down.sql

@@ -0,0 +1,4 @@
+-- Drop the fees table and indexes
+DROP TABLE IF EXISTS fees;
+DROP INDEX IF EXISTS idx_fees_mint;
+DROP INDEX IF EXISTS idx_fees_amount;

fee-sweeper/migrations/2024-06-15-203503_create_fees_table/up.sql

@@ -0,0 +1,13 @@
+-- Stores fees and index by mint, amount
+CREATE TABLE fees(
+    id SERIAL PRIMARY KEY,
+    tx_hash TEXT NOT NULL UNIQUE,
+    mint TEXT NOT NULL,
+    amount NUMERIC NOT NULL,
+    blinder NUMERIC NOT NULL,
+    receiver TEXT NOT NULL,
+    redeemed BOOLEAN NOT NULL DEFAULT FALSE
+);
+
+CREATE INDEX idx_fees_mint ON fees(mint);
+CREATE INDEX idx_fees_amount ON fees(amount);

fee-sweeper/migrations/2024-06-16-003335_create_wallets_table/down.sql

@@ -0,0 +1,2 @@
+-- Drop the wallets table
+DROP TABLE IF EXISTS wallets;

fee-sweeper/migrations/2024-06-16-003335_create_wallets_table/up.sql

@@ -0,0 +1,7 @@
+-- Create a table for storing wallets and the mints they hold
+-- The `secret_id` is the id of the AWS Secrets Manager secret that holds recovery information for the wallet
+CREATE TABLE wallets (
+    id UUID PRIMARY KEY,
+    mints TEXT[] not null,
+    secret_id TEXT not null
+);

fee-sweeper/src/db/mod.rs

@@ -0,0 +1,5 @@
+//! Database code
+
+pub mod models;
+#[allow(missing_docs)]
+pub mod schema;

fee-sweeper/src/db/models.rs

@@ -0,0 +1,88 @@
+#![allow(missing_docs)]
+#![allow(trivial_bounds)]
+
+use bigdecimal::BigDecimal;
+use diesel::prelude::*;
+use num_bigint::BigInt;
+use renegade_circuit_types::note::Note;
+use renegade_crypto::fields::scalar_to_bigint;
+use renegade_util::hex::{biguint_to_hex_addr, jubjub_to_hex_string};
+use uuid::Uuid;
+
+use crate::db::schema::fees;
+
+/// A fee that has been indexed by the indexer
+#[derive(Queryable, Selectable)]
+#[diesel(table_name = crate::db::schema::fees)]
+#[diesel(check_for_backend(diesel::pg::Pg))]
+#[allow(missing_docs, clippy::missing_docs_in_private_items)]
+pub struct Fee {
+    pub id: i32,
+    pub tx_hash: String,
+    pub mint: String,
+    pub amount: BigDecimal,
+    pub blinder: BigDecimal,
+    pub receiver: String,
+    pub redeemed: bool,
+}
+
+/// A new fee inserted into the database
+#[derive(Insertable)]
+#[diesel(table_name = fees)]
+pub struct NewFee {
+    pub tx_hash: String,
+    pub mint: String,
+    pub amount: BigDecimal,
+    pub blinder: BigDecimal,
+    pub receiver: String,
+}
+
+impl NewFee {
+    /// Construct a fee from a note
+    pub fn new_from_note(note: &Note, tx_hash: String) -> Self {
+        let mint = biguint_to_hex_addr(&note.mint);
+        let amount = BigInt::from(note.amount).into();
+        let blinder = scalar_to_bigint(&note.blinder).into();
+        let receiver = jubjub_to_hex_string(&note.receiver);
+
+        NewFee {
+            tx_hash,
+            mint,
+            amount,
+            blinder,
+            receiver,
+        }
+    }
+}
+
+/// Metadata information maintained by the indexer
+#[derive(Clone, Queryable, Selectable)]
+#[diesel(table_name = crate::db::schema::indexing_metadata)]
+#[diesel(check_for_backend(diesel::pg::Pg))]
+#[allow(missing_docs, clippy::missing_docs_in_private_items)]
+pub struct Metadata {
+    pub key: String,
+    pub value: String,
+}
+
+/// A metadata entry for a wallet managed by the indexer
+#[derive(Clone, Queryable, Selectable, Insertable)]
+#[diesel(table_name = crate::db::schema::wallets)]
+#[diesel(check_for_backend(diesel::pg::Pg))]
+#[allow(missing_docs, clippy::missing_docs_in_private_items)]
+pub struct WalletMetadata {
+    pub id: Uuid,
+    pub mints: Vec<Option<String>>,
+    pub secret_id: String,
+}
+
+impl WalletMetadata {
+    /// Construct a new wallet metadata entry
+    pub fn empty(id: Uuid, secret_id: String) -> Self {
+        WalletMetadata {
+            id,
+            mints: vec![],
+            secret_id,
+        }
+    }
+}

fee-sweeper/src/db/schema.rs

@@ -0,0 +1,34 @@
+// @generated automatically by Diesel CLI.
+
+diesel::table! {
+    fees (id) {
+        id -> Int4,
+        tx_hash -> Text,
+        mint -> Text,
+        amount -> Numeric,
+        blinder -> Numeric,
+        receiver -> Text,
+        redeemed -> Bool,
+    }
+}
+
+diesel::table! {
+    indexing_metadata (key) {
+        key -> Text,
+        value -> Text,
+    }
+}
+
+diesel::table! {
+    wallets (id) {
+        id -> Uuid,
+        mints -> Array<Nullable<Text>>,
+        secret_id -> Text,
+    }
+}
+
+diesel::allow_tables_to_appear_in_same_query!(
+    fees,
+    indexing_metadata,
+    wallets,
+);

fee-sweeper/src/indexer/index_fees.rs

@@ -0,0 +1,105 @@
+//! Phase one of the sweeper's execution; index all fees since the last consistent block
+
+use alloy_sol_types::SolCall;
+use arbitrum_client::abi::settleOfflineFeeCall;
+use arbitrum_client::{
+    abi::NotePostedFilter, constants::SELECTOR_LEN,
+    helpers::parse_note_ciphertext_from_settle_offline_fee,
+};
+use ethers::contract::LogMeta;
+use ethers::middleware::Middleware;
+use renegade_circuit_types::elgamal::ElGamalCiphertext;
+use renegade_circuit_types::native_helpers::elgamal_decrypt;
+use renegade_circuit_types::note::{Note, NOTE_CIPHERTEXT_SIZE};
+use renegade_circuit_types::wallet::NoteCommitment;
+use renegade_constants::Scalar;
+use renegade_crypto::fields::{scalar_to_biguint, scalar_to_u128, u256_to_scalar};
+use renegade_util::raw_err_str;
+use tracing::info;
+
+use crate::db::models::NewFee;
+use crate::Indexer;
+
+impl Indexer {
+    /// Index all fees since the given block
+    pub async fn index_fees(&mut self) -> Result<(), String> {
+        let block_number = self.get_latest_block()?;
+        info!("indexing fees from block {block_number}");
+
+        let filter = self
+            .arbitrum_client
+            .get_darkpool_client()
+            .event::<NotePostedFilter>()
+            .from_block(block_number);
+
+        let events = filter
+            .query_with_meta()
+            .await
+            .map_err(raw_err_str!("failed to create note posted stream: {}"))?;
+
+        let mut most_recent_block = block_number;
+        for (event, meta) in events {
+            let block = meta.block_number.as_u64();
+            let note_comm = u256_to_scalar(&event.note_commitment);
+            self.index_note(note_comm, meta).await?;
+
+            if block > most_recent_block {
+                most_recent_block = block;
+                self.update_latest_block(most_recent_block)?;
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Index a note
+    async fn index_note(&mut self, note_comm: NoteCommitment, meta: LogMeta) -> Result<(), String> {
+        // Parse the note from the tx
+        let tx = self
+            .arbitrum_client
+            .get_darkpool_client()
+            .client()
+            .get_transaction(meta.transaction_hash)
+            .await
+            .map_err(raw_err_str!("failed to query tx: {}"))?
+            .ok_or_else(|| format!("tx not found: {}", meta.transaction_hash))?;
+
+        let calldata: Vec<u8> = tx.input.to_vec();
+        let selector: [u8; 4] = calldata[..SELECTOR_LEN].try_into().unwrap();
+        let encryption = match selector {
+            <settleOfflineFeeCall as SolCall>::SELECTOR => {
+                parse_note_ciphertext_from_settle_offline_fee(&calldata)
+                    .map_err(raw_err_str!("failed to parse ciphertext: {}"))?
+            }
+            sel => return Err(format!("invalid selector when parsing note: {sel:?}")),
+        };
+
+        // Decrypt the note and check that the commitment matches the expected value; if not we are not the receiver
+        let note = self.decrypt_note(&encryption);
+        let tx = format!("{:#x}", meta.transaction_hash);
+        if note.commitment() != note_comm {
+            info!("not receiver, skipping");
+            return Ok(());
+        } else {
+            info!("indexing note from tx: {tx}");
+        }
+
+        // Otherwise, index the note
+        let fee = NewFee::new_from_note(&note, tx);
+        self.insert_fee(fee)
+    }
+
+    /// Decrypt a note using the decryption key
+    fn decrypt_note(&self, note: &ElGamalCiphertext<NOTE_CIPHERTEXT_SIZE>) -> Note {
+        // The ciphertext stores all note values except the encryption key
+        let cleartext_values: [Scalar; NOTE_CIPHERTEXT_SIZE] =
+            elgamal_decrypt(note, &self.decryption_key);
+
+        Note {
+            mint: scalar_to_biguint(&cleartext_values[0]),
+            amount: scalar_to_u128(&cleartext_values[1]),
+            receiver: self.decryption_key.public_key(),
+            blinder: cleartext_values[2],
+        }
+    }
+}