feat(torii): fetch and process erc721 metadata and image

commit-id:274aaa3a

Cargo.toml

@@ -157,13 +157,16 @@ clap_complete = "4.3"
 console = "0.15.7"
 convert_case = "0.6.0"
 crypto-bigint = { version = "0.5.3", features = [ "serde" ] }
+data-url = "0.3"
 derive_more = "0.99.17"
 flate2 = "1.0.24"
+fluent-uri = "0.3"
 futures = "0.3.30"
 futures-util = "0.3.30"
 hashlink = "0.9.1"
 hex = "0.4.3"
 http = "0.2.9"
+image = "0.25.2"
 indexmap = "2.2.5"
 indoc = "1.0.7"
 itertools = "0.12.1"
@@ -209,6 +212,9 @@ tracing-log = "0.1.3"
 tracing-subscriber = { version = "0.3.16", features = [ "env-filter", "json" ] }
 url = { version = "2.4.0", features = [ "serde" ] }
 walkdir = "2.5.0"
+# TODO: see if we still need the git version
+ipfs-api-backend-hyper = { git = "https://github.com/ferristseng/rust-ipfs-api", rev = "af2c17f7b19ef5b9898f458d97a90055c3605633", features = [ "with-hyper-rustls", "with-send-sync" ] }
+mime_guess = "2.0"
 
 # server
 hyper = "0.14.27"

bin/torii/Cargo.toml

@@ -41,12 +41,12 @@ torii-relay.workspace = true
 torii-server.workspace = true
 tower.workspace = true
 
+tempfile.workspace = true
 tower-http.workspace = true
 tracing-subscriber.workspace = true
 tracing.workspace = true
 url.workspace = true
 webbrowser = "0.8"
-tempfile.workspace = true
 
 [dev-dependencies]
 camino.workspace = true

bin/torii/src/main.rs

@@ -19,6 +19,7 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use anyhow::Context;
+use camino::Utf8PathBuf;
 use clap::{ArgAction, Parser};
 use dojo_metrics::exporters::prometheus::PrometheusRecorder;
 use dojo_utils::parse::{parse_socket_address, parse_url};
@@ -30,7 +31,7 @@ use sqlx::SqlitePool;
 use starknet::core::types::Felt;
 use starknet::providers::jsonrpc::HttpTransport;
 use starknet::providers::JsonRpcClient;
-use tempfile::NamedTempFile;
+use tempfile::{NamedTempFile, TempDir};
 use tokio::sync::broadcast;
 use tokio::sync::broadcast::Sender;
 use tokio_stream::StreamExt;
@@ -146,6 +147,10 @@ struct Args {
     /// Configuration file
     #[arg(long)]
     config: Option<PathBuf>,
+
+    /// Path to a directory to store ERC artifacts
+    #[arg(long)]
+    artifacts_path: Option<Utf8PathBuf>,
 }
 
 #[tokio::main]
@@ -213,7 +218,13 @@ async fn main() -> anyhow::Result<()> {
     let contracts =
         config.contracts.iter().map(|contract| (contract.address, contract.r#type)).collect();
 
-    let (mut executor, sender) = Executor::new(pool.clone(), shutdown_tx.clone()).await?;
+    let (mut executor, sender) = Executor::new(
+        pool.clone(),
+        shutdown_tx.clone(),
+        provider.clone(),
+        args.max_concurrent_tasks,
+    )
+    .await?;
     tokio::spawn(async move {
         executor.run().await.unwrap();
     });
@@ -254,10 +265,24 @@ async fn main() -> anyhow::Result<()> {
         Arc::new(contracts),
     );
 
-    let shutdown_rx = shutdown_tx.subscribe();
-    let (grpc_addr, grpc_server) =
-        torii_grpc::server::new(shutdown_rx, &pool, block_rx, world_address, Arc::clone(&provider))
-            .await?;
+    let (grpc_addr, grpc_server) = torii_grpc::server::new(
+        shutdown_tx.subscribe(),
+        &pool,
+        block_rx,
+        world_address,
+        Arc::clone(&provider),
+    )
+    .await?;
+
+    let temp_dir = TempDir::new()?;
+    let artifacts_path =
+        args.artifacts_path.unwrap_or_else(|| Utf8PathBuf::from(temp_dir.path().to_str().unwrap()));
+
+    tokio::fs::create_dir_all(&artifacts_path).await?;
+    let absolute_path = artifacts_path.canonicalize_utf8()?;
+
+    let (artifacts_addr, artifacts_server) =
+        torii_server::artifacts::new(shutdown_tx.subscribe(), &absolute_path, pool.clone()).await?;
 
     let mut libp2p_relay_server = torii_relay::server::Relay::new(
         db,
@@ -270,7 +295,13 @@ async fn main() -> anyhow::Result<()> {
     )
     .expect("Failed to start libp2p relay server");
 
-    let proxy_server = Arc::new(Proxy::new(args.addr, args.allowed_origins, Some(grpc_addr), None));
+    let proxy_server = Arc::new(Proxy::new(
+        args.addr,
+        args.allowed_origins,
+        Some(grpc_addr),
+        None,
+        Some(artifacts_addr),
+    ));
 
     let graphql_server = spawn_rebuilding_graphql_server(
         shutdown_tx.clone(),
@@ -288,6 +319,7 @@ async fn main() -> anyhow::Result<()> {
     info!(target: LOG_TARGET, endpoint = %endpoint, "Starting torii endpoint.");
     info!(target: LOG_TARGET, endpoint = %gql_endpoint, "Serving Graphql playground.");
     info!(target: LOG_TARGET, url = %explorer_url, "Serving World Explorer.");
+    info!(target: LOG_TARGET, path = %artifacts_path, "Serving ERC artifacts at path");
 
     if args.explorer {
         if let Err(e) = webbrowser::open(&explorer_url) {
@@ -308,13 +340,15 @@ async fn main() -> anyhow::Result<()> {
     let graphql_server_handle = tokio::spawn(graphql_server);
     let grpc_server_handle = tokio::spawn(grpc_server);
     let libp2p_relay_server_handle = tokio::spawn(async move { libp2p_relay_server.run().await });
+    let artifacts_server_handle = tokio::spawn(artifacts_server);
 
     tokio::select! {
         res = engine_handle => res??,
         res = proxy_server_handle => res??,
         res = graphql_server_handle => res?,
         res = grpc_server_handle => res??,
         res = libp2p_relay_server_handle => res?,
+        res = artifacts_server_handle => res?,
         _ = dojo_utils::signal::wait_signals() => {},
     };
 

crates/dojo-world/Cargo.toml

@@ -11,8 +11,8 @@ anyhow.workspace = true
 async-trait.workspace = true
 cairo-lang-filesystem.workspace = true
 cairo-lang-project.workspace = true
-cairo-lang-starknet.workspace = true
 cairo-lang-starknet-classes.workspace = true
+cairo-lang-starknet.workspace = true
 camino.workspace = true
 convert_case.workspace = true
 dojo-utils = { workspace = true, optional = true }
@@ -22,16 +22,16 @@ serde.workspace = true
 serde_json.workspace = true
 serde_with.workspace = true
 smol_str.workspace = true
-starknet.workspace = true
 starknet-crypto.workspace = true
+starknet.workspace = true
 thiserror.workspace = true
 topological-sort.workspace = true
 tracing.workspace = true
 
 cainome.workspace = true
 dojo-types = { path = "../dojo-types", optional = true }
 http = { workspace = true, optional = true }
-ipfs-api-backend-hyper = { git = "https://github.com/ferristseng/rust-ipfs-api", rev = "af2c17f7b19ef5b9898f458d97a90055c3605633", features = [ "with-hyper-rustls" ], optional = true }
+ipfs-api-backend-hyper = { workspace = true, optional = true }
 scarb = { workspace = true, optional = true }
 tokio = { version = "1.32.0", features = [ "time" ], default-features = false, optional = true }
 toml.workspace = true

crates/sozo/ops/Cargo.toml

@@ -15,10 +15,10 @@ cairo-lang-defs.workspace = true
 cairo-lang-filesystem.workspace = true
 cairo-lang-plugins.workspace = true
 cairo-lang-project.workspace = true
-cairo-lang-sierra.workspace = true
 cairo-lang-sierra-to-casm.workspace = true
-cairo-lang-starknet.workspace = true
+cairo-lang-sierra.workspace = true
 cairo-lang-starknet-classes.workspace = true
+cairo-lang-starknet.workspace = true
 cairo-lang-test-plugin.workspace = true
 cairo-lang-utils.workspace = true
 camino.workspace = true
@@ -36,16 +36,16 @@ num-bigint = "0.4.6"
 num-traits.workspace = true
 reqwest.workspace = true
 rpassword.workspace = true
-scarb.workspace = true
 scarb-ui.workspace = true
+scarb.workspace = true
 semver.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 serde_with.workspace = true
 smol_str.workspace = true
 sozo-walnut = { workspace = true, optional = true }
-starknet.workspace = true
 starknet-crypto.workspace = true
+starknet.workspace = true
 thiserror.workspace = true
 tokio.workspace = true
 toml.workspace = true
@@ -58,7 +58,7 @@ katana-runner = { workspace = true, optional = true }
 [dev-dependencies]
 assert_fs.workspace = true
 dojo-test-utils = { workspace = true, features = [ "build-examples" ] }
-ipfs-api-backend-hyper = { git = "https://github.com/ferristseng/rust-ipfs-api", rev = "af2c17f7b19ef5b9898f458d97a90055c3605633", features = [ "with-hyper-rustls" ] }
+ipfs-api-backend-hyper.workspace = true
 katana-runner.workspace = true
 
 [features]

crates/torii/core/Cargo.toml

@@ -16,8 +16,10 @@ bitflags = "2.6.0"
 cainome.workspace = true
 chrono.workspace = true
 crypto-bigint.workspace = true
+data-url.workspace = true
 dojo-types.workspace = true
 dojo-world = { workspace = true, features = [ "contracts", "manifest" ] }
+# fluent-uri.workspace = true
 futures-channel = "0.3.0"
 futures-util.workspace = true
 hashlink.workspace = true
@@ -33,6 +35,7 @@ starknet.workspace = true
 thiserror.workspace = true
 tokio = { version = "1.32.0", features = [ "sync" ], default-features = true }
 # tokio-stream = "0.1.11"
+ipfs-api-backend-hyper.workspace = true
 tokio-util.workspace = true
 toml.workspace = true
 tracing.workspace = true

crates/torii/core/src/engine.rs

@@ -259,8 +259,9 @@ impl<P: Provider + Send + Sync + std::fmt::Debug + 'static> Engine<P> {
 
                             match self.process(fetch_result).await {
                                 Ok(_) => {
-                                    self.db.execute().await?;
+                                    self.db.flush().await?;
                                     self.db.apply_cache_diff().await?;
+                                    self.db.execute().await?;
                                 },
                                 Err(e) => {
                                     error!(target: LOG_TARGET, error = %e, "Processing fetched data.");
@@ -627,8 +628,7 @@ impl<P: Provider + Send + Sync + std::fmt::Debug + 'static> Engine<P> {
 
             unique_contracts.insert(event.from_address);
 
-            Self::process_event(
-                self,
+            self.process_event(
                 block_number,
                 block_timestamp,
                 &event_id,
@@ -688,8 +688,7 @@ impl<P: Provider + Send + Sync + std::fmt::Debug + 'static> Engine<P> {
                 let event_id =
                     format!("{:#064x}:{:#x}:{:#04x}", block_number, *transaction_hash, event_idx);
 
-                Self::process_event(
-                    self,
+                self.process_event(
                     block_number,
                     block_timestamp,
                     &event_id,
@@ -701,8 +700,7 @@ impl<P: Provider + Send + Sync + std::fmt::Debug + 'static> Engine<P> {
             }
 
             if self.config.flags.contains(IndexingFlags::TRANSACTIONS) {
-                Self::process_transaction(
-                    self,
+                self.process_transaction(
                     block_number,
                     block_timestamp,
                     *transaction_hash,