ref: move storage to its own crate

Cargo.toml

@@ -5,35 +5,33 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [workspace]
-members = ["state-reconstruct-fetcher", "state-reconstruct-fetcher/blobscan-client"]
+members = [
+  "state-reconstruct-fetcher",
+  "state-reconstruct-fetcher/blobscan-client",
+  "state-reconstruct-storage",
+]
 
 [dependencies]
 async-trait = "0.1.74"
 bincode = "1"
 blake2 = "0.10.6"
 blobscan-client = { path = "./state-reconstruct-fetcher/blobscan-client" }
-bytes = "1.5"
 chrono = "0.4.31"
 clap = { version = "4.4.7", features = ["derive", "env"] }
 ethers = "1.0.2"
 eyre = "0.6.8"
-flate2 = "1.0.28"
 hex = "0.4.3"
 indexmap = { version = "2.0.2" }
 primitive-types = "0.12.2"
-prost = "0.12"
 rocksdb = "0.21"
 serde = { version = "1.0.189", features = ["derive"] }
 serde_json = { version = "1.0.107", features = ["std"] }
 state-reconstruct-fetcher = { path = "./state-reconstruct-fetcher" }
+state-reconstruct-storage = { path = "./state-reconstruct-storage" }
 thiserror = "1.0.50"
 tikv-jemallocator = "0.5"
 tokio = { version = "1.33.0", features = ["macros"] }
 tracing = "0.1.40"
 tracing-subscriber = "0.3.17"
-zkevm_opcode_defs = { git = "https://github.com/matter-labs/era-zkevm_opcode_defs.git" }
 zksync_merkle_tree = { git = "https://github.com/matter-labs/zksync-era.git" }
 zksync_storage = { git = "https://github.com/matter-labs/zksync-era.git" }
-
-[build-dependencies]
-prost-build = "0.12"

src/processor/snapshot/exporter.rs

@@ -2,19 +2,20 @@ use std::path::{Path, PathBuf};
 
 use ethers::types::{U256, U64};
 use eyre::Result;
-
-use super::{
-    database::{self, SnapshotDB},
-    types::{SnapshotFactoryDependency, SnapshotHeader},
-    DEFAULT_DB_PATH, SNAPSHOT_FACTORY_DEPS_FILE_NAME_SUFFIX, SNAPSHOT_HEADER_FILE_NAME,
-};
-use crate::processor::snapshot::types::{
-    Proto, SnapshotFactoryDependencies, SnapshotStorageLogsChunk, SnapshotStorageLogsChunkMetadata,
+use state_reconstruct_storage::{
+    types::{
+        Proto, SnapshotFactoryDependencies, SnapshotFactoryDependency, SnapshotHeader,
+        SnapshotStorageLogsChunk, SnapshotStorageLogsChunkMetadata,
+    },
+    InnerDB, FACTORY_DEPS, INDEX_TO_KEY_MAP,
 };
 
+use super::{DEFAULT_DB_PATH, SNAPSHOT_HEADER_FILE_NAME};
+use crate::processor::snapshot::SNAPSHOT_FACTORY_DEPS_FILE_NAME_SUFFIX;
+
 pub struct SnapshotExporter {
     basedir: PathBuf,
-    database: SnapshotDB,
+    database: InnerDB,
 }
 
 impl SnapshotExporter {
@@ -24,7 +25,7 @@ impl SnapshotExporter {
             None => PathBuf::from(DEFAULT_DB_PATH),
         };
 
-        let database = SnapshotDB::new_read_only(db_path)?;
+        let database = InnerDB::new_read_only(db_path)?;
         Ok(Self {
             basedir: basedir.to_path_buf(),
             database,
@@ -61,7 +62,7 @@ impl SnapshotExporter {
     fn export_factory_deps(&self, header: &mut SnapshotHeader) -> Result<()> {
         tracing::info!("Exporting factory dependencies...");
 
-        let storage_logs = self.database.cf_handle(database::FACTORY_DEPS).unwrap();
+        let storage_logs = self.database.cf_handle(FACTORY_DEPS).unwrap();
         let mut iterator = self
             .database
             .iterator_cf(storage_logs, rocksdb::IteratorMode::Start);
@@ -93,7 +94,7 @@ impl SnapshotExporter {
         let num_logs = self.database.get_last_repeated_key_index()?;
         tracing::info!("Found {num_logs} logs.");
 
-        let index_to_key_map = self.database.cf_handle(database::INDEX_TO_KEY_MAP).unwrap();
+        let index_to_key_map = self.database.cf_handle(INDEX_TO_KEY_MAP).unwrap();
         let mut iterator = self
             .database
             .iterator_cf(index_to_key_map, rocksdb::IteratorMode::Start);

src/processor/snapshot/importer.rs

@@ -5,13 +5,14 @@ use std::{
 };
 
 use eyre::Result;
-use state_reconstruct_fetcher::{constants::storage::INNER_DB_NAME, database::InnerDB};
-use tokio::sync::Mutex;
-
-use super::{
+use state_reconstruct_fetcher::constants::storage::INNER_DB_NAME;
+use state_reconstruct_storage::{
     types::{Proto, SnapshotFactoryDependencies, SnapshotHeader, SnapshotStorageLogsChunk},
-    SNAPSHOT_FACTORY_DEPS_FILE_NAME_SUFFIX, SNAPSHOT_HEADER_FILE_NAME,
+    InnerDB,
 };
+use tokio::sync::Mutex;
+
+use super::{SNAPSHOT_FACTORY_DEPS_FILE_NAME_SUFFIX, SNAPSHOT_HEADER_FILE_NAME};
 use crate::processor::tree::tree_wrapper::TreeWrapper;
 
 pub struct SnapshotImporter {

src/processor/snapshot/mod.rs

@@ -1,11 +1,7 @@
 use std::{fs, path::PathBuf, str::FromStr};
 
-pub mod database;
 pub mod exporter;
 pub mod importer;
-pub mod types;
-
-mod bytecode;
 
 use async_trait::async_trait;
 use blake2::{Blake2s256, Digest};
@@ -15,25 +11,21 @@ use state_reconstruct_fetcher::{
     constants::{ethereum, storage},
     types::CommitBlock,
 };
+use state_reconstruct_storage::{
+    bytecode,
+    types::{MiniblockNumber, SnapshotFactoryDependency, SnapshotStorageLog},
+    InnerDB,
+};
 use tokio::sync::mpsc;
 
-use self::{
-    database::SnapshotDB,
-    types::{SnapshotFactoryDependency, SnapshotStorageLog},
-};
 use super::Processor;
-use crate::processor::snapshot::types::MiniblockNumber;
 
 pub const DEFAULT_DB_PATH: &str = "snapshot_db";
 pub const SNAPSHOT_HEADER_FILE_NAME: &str = "snapshot-header.json";
 pub const SNAPSHOT_FACTORY_DEPS_FILE_NAME_SUFFIX: &str = "factory_deps.proto.gzip";
 
-pub mod protobuf {
-    include!(concat!(env!("OUT_DIR"), "/protobuf.rs"));
-}
-
 pub struct SnapshotBuilder {
-    database: SnapshotDB,
+    database: InnerDB,
 }
 
 impl SnapshotBuilder {
@@ -43,7 +35,7 @@ impl SnapshotBuilder {
             None => PathBuf::from(DEFAULT_DB_PATH),
         };
 
-        let mut database = SnapshotDB::new(db_path).unwrap();
+        let mut database = InnerDB::new(db_path).unwrap();
 
         let idx = database
             .get_last_repeated_key_index()
@@ -136,7 +128,7 @@ impl Processor for SnapshotBuilder {
 
 // TODO: Can this be made somewhat generic?
 /// Attempts to reconstruct the genesis state from a CSV file.
-fn reconstruct_genesis_state(database: &mut SnapshotDB, path: &str) -> Result<()> {
+fn reconstruct_genesis_state(database: &mut InnerDB, path: &str) -> Result<()> {
     fn cleanup_encoding(input: &'_ str) -> &'_ str {
         input
             .strip_prefix("E'\\\\x")
@@ -244,7 +236,7 @@ mod tests {
     use std::fs;
 
     use indexmap::IndexMap;
-    use state_reconstruct_fetcher::types::PackingType;
+    use state_reconstruct_storage::{InnerDB, PackingType};
 
     use super::*;
 
@@ -279,7 +271,7 @@ mod tests {
             builder.run(rx).await;
         }
 
-        let db = SnapshotDB::new(PathBuf::from(db_dir.clone())).unwrap();
+        let db = InnerDB::new(PathBuf::from(db_dir.clone())).unwrap();
 
         let key = U256::from_dec_str("1234").unwrap();
         let Some(log) = db.get_storage_log(&key).unwrap() else {

src/processor/tree/mod.rs

@@ -8,10 +8,10 @@ use ethers::types::H256;
 use eyre::Result;
 use state_reconstruct_fetcher::{
     constants::storage::INNER_DB_NAME,
-    database::InnerDB,
     metrics::{PerfMetric, METRICS_TRACING_TARGET},
     types::CommitBlock,
 };
+use state_reconstruct_storage::InnerDB;
 use tokio::{
     sync::{mpsc, Mutex},
     time::Instant,

src/processor/tree/tree_wrapper.rs

@@ -3,18 +3,14 @@ use std::{collections::HashMap, fs, num::NonZeroU32, path::Path, str::FromStr, s
 use blake2::{Blake2s256, Digest};
 use ethers::types::{Address, H256, U256, U64};
 use eyre::Result;
-use state_reconstruct_fetcher::{
-    constants::storage::INITAL_STATE_PATH,
-    database::InnerDB,
-    types::{CommitBlock, PackingType},
-};
+use state_reconstruct_fetcher::{constants::storage::INITAL_STATE_PATH, types::CommitBlock};
+use state_reconstruct_storage::{types::SnapshotStorageLogsChunk, InnerDB, PackingType};
 use thiserror::Error;
 use tokio::sync::Mutex;
 use zksync_merkle_tree::{Database, MerkleTree, RocksDBWrapper, TreeEntry};
 use zksync_storage::{RocksDB, RocksDBOptions};
 
 use super::RootHash;
-use crate::processor::snapshot::types::SnapshotStorageLogsChunk;
 
 #[derive(Error, Debug)]
 pub enum TreeError {

state-reconstruct-fetcher/Cargo.toml

@@ -23,3 +23,4 @@ rocksdb = "0.21.0"
 hex = "0.4.3"
 chrono = "0.4.31"
 zkevm_circuits = { git = "https://github.com/matter-labs/era-zkevm_circuits", branch = "v1.4.1" }
+state-reconstruct-storage = { path = "../state-reconstruct-storage" }