diff --git a/Cargo.lock b/Cargo.lock index 0d0c24c6..90e1d2c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1073,7 +1073,16 @@ version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" dependencies = [ - "dirs-sys", + "dirs-sys 0.3.7", +] + +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys 0.4.1", ] [[package]] @@ -1087,6 +1096,18 @@ dependencies = [ "winapi", ] +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + [[package]] name = "displaydoc" version = "0.2.4" @@ -1999,6 +2020,7 @@ dependencies = [ "bs58", "chrono", "console", + "dirs 5.0.1", "env_logger", "futures", "glob", @@ -2436,6 +2458,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "os_str_bytes" version = "6.6.1" @@ -3480,7 +3508,7 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ccc8076840c4da029af4f87e4e8daeb0fca6b87bbb02e10cb60b791450e11e4" dependencies = [ - "dirs", + "dirs 4.0.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 7fe5d422..e1e2e295 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ borsh = "0.10.3" bs58 = "0.4.0" chrono = "0.4.31" console = "0.15.7" +dirs = "5.0.1" env_logger = "0.9.3" futures = "0.3.29" glob = "0.3.1" @@ -29,6 +30,7 @@ once_cell = "1.19.0" phf = { version = "0.10", features = ["macros"] } ratelimit = "0.4.4" rayon = "1.8.0" +regex = "1.10.2" reqwest = { version = "0.11.23", features = ["json"] } retry = "1.3.1" serde = { version = "1.0.193", features = ["derive"] } @@ -45,9 +47,3 @@ spl-token = "3.5.0" structopt = "0.3.26" thiserror = "1.0.51" tokio = "1.35.1" -regex = "1.10.2" - -[features] - -[dev-dependencies] -regex = "1.10.2" diff --git a/src/helius/data.rs b/src/helius/data.rs deleted file mode 100644 index 436192f8..00000000 --- a/src/helius/data.rs +++ /dev/null @@ -1,19 +0,0 @@ -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Serialize, Deserialize)] -pub struct HeliusResponse { - pub helius_result: Vec, -} - -#[derive(Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct HeliusResult { - pub result: Vec, - pub pagination_token: String, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct Asset { - pub mint: String, - pub name: String, -} diff --git a/src/helius/methods.rs b/src/helius/methods.rs deleted file mode 100644 index cc679671..00000000 --- a/src/helius/methods.rs +++ /dev/null @@ -1,77 +0,0 @@ -use std::fs::File; - -use crate::{ - helius::HeliusResult, - snapshot::{GetMintsArgs, Method}, - spinner::create_spinner, -}; - -use anyhow::Result; -use reqwest::Url; -use serde_json::json; - -use super::Asset; - -pub async fn get_mints(args: GetMintsArgs) -> Result<()> { - let GetMintsArgs { - address, - method, - api_key, - output, - indexer, - } = args; - - let mut url = Url::parse("https://api.helius.xyz/v1/mintlist")?; - url.set_query(Some(&format!("api-key={api_key}"))); - - let mut assets: Vec = Vec::new(); - let client = reqwest::Client::new(); - - let mut pagination_token = None; - - let query = match method { - Method::Creator => json!({ - "firstVerifiedCreators": [address.to_string()], - "verifiedCollectionAddresses": [] - } - ), - Method::Collection => json!( { - "firstVerifiedCreators": [], - "verifiedCollectionAddresses": [address.to_string()] - } - ), - }; - - let spinner = create_spinner("Getting assets..."); - loop { - let body = json!( - { - "query": query, - "options": { - "limit": 10000, - "paginationToken": pagination_token - } - } - ); - - let response = client.post(url.clone()).json(&body).send().await?; - let res: HeliusResult = response.json().await?; - - assets.extend(res.result); - - if res.pagination_token.is_empty() { - break; - } - pagination_token = Some(res.pagination_token); - } - spinner.finish(); - - let mut mints: Vec = assets.iter().map(|asset| asset.mint.clone()).collect(); - mints.sort_unstable(); - - let prefix = address[0..6].to_string(); - let f = File::create(format!("{output}/{prefix}_{method}_mints_{indexer}.json"))?; - serde_json::to_writer_pretty(f, &mints)?; - - Ok(()) -} diff --git a/src/helius/mod.rs b/src/helius/mod.rs deleted file mode 100644 index 850cd585..00000000 --- a/src/helius/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -mod data; -mod methods; - -pub use data::*; -pub use methods::*; diff --git a/src/lib.rs b/src/lib.rs index 866d0d29..f40b1cbd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,12 +11,12 @@ pub mod derive; pub mod errors; pub mod extend_program; pub mod find; -pub mod helius; pub mod limiter; pub mod mint; pub mod opt; pub mod parse; pub mod process_subcommands; +pub mod setup; pub mod sign; pub mod snapshot; pub mod spinner; diff --git a/src/main.rs b/src/main.rs index e3d9e078..f2342b16 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,6 +15,7 @@ use metaboss::constants::*; use metaboss::opt::*; use metaboss::parse::parse_solana_config; use metaboss::process_subcommands::*; +use metaboss::snapshot::process_snapshot; #[tokio::main] async fn main() -> Result<()> { @@ -90,7 +91,7 @@ async fn main() -> Result<()> { Command::Sign { sign_subcommands } => process_sign(&client, sign_subcommands)?, Command::Snapshot { snapshot_subcommands, - } => process_snapshot(client, snapshot_subcommands).await?, + } => process_snapshot(client, rpc, snapshot_subcommands).await?, Command::Transfer { transfer_subcommands, } => process_transfer(client, transfer_subcommands)?, diff --git a/src/opt.rs b/src/opt.rs index c62d1a0e..aa745a3e 100644 --- a/src/opt.rs +++ b/src/opt.rs @@ -4,8 +4,11 @@ use solana_program::pubkey::Pubkey; use structopt::StructOpt; use crate::{ - check::CheckSubcommands, collections::GetCollectionItemsMethods, constants::DEFAULT_RATE_LIMIT, - data::Indexers, mint::Supply, + check::CheckSubcommands, + collections::GetCollectionItemsMethods, + constants::DEFAULT_RATE_LIMIT, + mint::Supply, + snapshot::{HolderGroupKey, MintsGroupKey}, }; #[derive(Debug, StructOpt)] @@ -1179,9 +1182,58 @@ pub enum SignSubcommands { #[derive(Debug, StructOpt)] pub enum SnapshotSubcommands { - /// Snapshot all current holders of NFTs by candy_machine_id / creator or update_authority - #[structopt(name = "holders")] + /// Get all current holders of NFTs by group Holders { + /// Pubkey of the group to find holders for. + group_value: Pubkey, + + /// Type of group to filter by: mint, mcc, fvca. + #[structopt(short, long)] + group_key: HolderGroupKey, + + /// Path to directory to save output file + #[structopt(short, long, default_value = ".")] + output: PathBuf, + }, + /// Get all mint accounts by various group types + Mints { + /// Pubkey of the group to find holders for. + group_value: Pubkey, + + /// Type of group to filter by: authority, mcc, creator. + #[structopt(short, long)] + group_key: MintsGroupKey, + + /// For creators, which position to check as the verified creator. + /// Defaults to 0, for the First Verified Creator Address. + #[structopt(short = "p", long, default_value = "0")] + creator_position: usize, + + /// Path to directory to save output file + #[structopt(short, long, default_value = ".")] + output: PathBuf, + }, + /// Get all mint accounts by First Verified Creator Address + Fvca { + /// First verified creator address. + creator: Option, + + /// Path to directory to save output file + #[structopt(short, long, default_value = ".")] + output: PathBuf, + }, + /// Get all mint accounts by Metaplex Certified Collection key + Mcc { + /// Collection parent mint address. + mcc_id: Pubkey, + + /// Path to directory to save output file + #[structopt(short, long, default_value = ".")] + output: PathBuf, + }, + /// Get all current holders of NFTs by legacy gPA calls + #[structopt(name = "holders-gpa")] + HoldersGpa { /// Update authority to filter accounts by. #[structopt(short, long)] update_authority: Option, @@ -1214,39 +1266,9 @@ pub enum SnapshotSubcommands { #[structopt(short, long, default_value = ".")] output: String, }, - /// Snapshot holders from an indexer. - #[structopt(name = "indexed-holders")] - IndexedHolders { - /// Indexer to use for getting collection items. See docs. - #[structopt(short, long, default_value = "the_index_io")] - indexer: Indexers, - - /// API key for the indexer. - #[structopt(short, long)] - api_key: String, - - /// First verified creator. - #[structopt(short, long)] - creator: String, - - /// Path to directory to save output files. - #[structopt(short, long, default_value = ".")] - output: String, - }, - ///Snapshot all candy machine config and state accounts for a given update_authority - #[structopt(name = "cm-accounts")] - CMAccounts { - /// Update authority to filter accounts by. - #[structopt(short, long)] - update_authority: String, - - /// Path to directory to save output files. - #[structopt(short, long, default_value = ".")] - output: String, - }, - /// Snapshot all mint accounts for a given candy_machine_id / creatoro or update authority - #[structopt(name = "mints")] - Mints { + /// Get all mint accounts using legacy getProgramAccounts call + #[structopt(name = "mints-gpa")] + MintsGpa { /// Creator to filter accounts by (for CM v2 use --v2, for CM v3 use --v3 if candy_machine account is passed) #[structopt(short, long)] creator: Option, @@ -1275,63 +1297,7 @@ pub enum SnapshotSubcommands { #[structopt(short, long, default_value = ".")] output: String, }, - /// Snapshot mints from an indexer. - #[structopt(name = "indexed-mints")] - IndexedMints { - /// Indexer to use for getting collection items. See docs. - #[structopt(short, long, default_value = "the_index_io")] - indexer: Indexers, - - /// API key for the indexer. - #[structopt(short, long)] - api_key: String, - - /// First verified creator. - #[structopt(short, long)] - creator: String, - - /// Path to directory to save output file - #[structopt(short, long, default_value = ".")] - output: String, - }, - /// Get NFT mints by creator from various indexers. - #[structopt(name = "mints-by-creator")] - MintsByCreator { - /// Indexer to use for getting collection items. See docs. - #[structopt(short, long, default_value = "helius")] - indexer: Indexers, - - /// API key for the indexer. - #[structopt(short, long)] - api_key: String, - - /// First verified creator address. - #[structopt(short = "c", long)] - address: String, - - /// Path to directory to save output file - #[structopt(short, long, default_value = ".")] - output: String, - }, - /// Get NFT mints by collection from various indexers. - #[structopt(name = "mints-by-collection")] - MintsByCollection { - /// Indexer to use for getting collection items. See docs. - #[structopt(short, long, default_value = "helius")] - indexer: Indexers, - - /// API key for the indexer. - #[structopt(short, long)] - api_key: String, - - /// Collection parent mint address. - #[structopt(short = "c", long)] - address: String, - - /// Path to directory to save output file - #[structopt(short, long, default_value = ".")] - output: String, - }, + /// Get all print edition mint accounts for a given master edition mint Prints { /// Master edition mint address. #[structopt(short = "m", long)] diff --git a/src/process_subcommands.rs b/src/process_subcommands.rs index 1f5b780a..2b9167a5 100644 --- a/src/process_subcommands.rs +++ b/src/process_subcommands.rs @@ -36,12 +36,6 @@ use crate::mint::{ use crate::opt::*; use crate::parse::{is_only_one_option, parse_errors_code, parse_errors_file}; use crate::sign::{sign_all, sign_one}; -use crate::snapshot::{ - snapshot_cm_accounts, snapshot_holders, snapshot_indexed_holders, snapshot_indexed_mints, - snapshot_mints, snapshot_mints_by_collection, snapshot_mints_by_creator, - snapshot_print_editions, GetMintsArgs, Method, NftsByCreatorArgs, SnapshotHoldersArgs, - SnapshotMintsArgs, SnapshotPrintEditionsArgs, -}; use crate::transfer::process_transfer_asset; use crate::unverify::{ unverify_creator, unverify_creator_all, UnverifyCreatorAllArgs, UnverifyCreatorArgs, @@ -761,128 +755,6 @@ pub fn process_sign(client: &RpcClient, commands: SignSubcommands) -> Result<()> } } -pub async fn process_snapshot(client: RpcClient, commands: SnapshotSubcommands) -> Result<()> { - match commands { - SnapshotSubcommands::Holders { - update_authority, - creator, - position, - mint_accounts_file, - v2, - v3, - allow_unverified, - output, - } => snapshot_holders( - &client, - SnapshotHoldersArgs { - update_authority, - creator, - position, - mint_accounts_file, - v2, - v3, - allow_unverified, - output, - }, - ), - SnapshotSubcommands::IndexedHolders { - indexer, - api_key, - creator, - output, - } => { - snapshot_indexed_holders(NftsByCreatorArgs { - creator, - api_key, - indexer, - output, - }) - .await - } - SnapshotSubcommands::CMAccounts { - update_authority, - output, - } => snapshot_cm_accounts(&client, &update_authority, &output), - SnapshotSubcommands::Mints { - creator, - position, - update_authority, - v2, - v3, - allow_unverified, - output, - } => snapshot_mints( - &client, - SnapshotMintsArgs { - creator, - position, - update_authority, - v2, - v3, - allow_unverified, - output, - }, - ), - SnapshotSubcommands::IndexedMints { - indexer, - api_key, - creator, - output, - } => { - snapshot_indexed_mints(NftsByCreatorArgs { - creator, - api_key, - indexer, - output, - }) - .await - } - SnapshotSubcommands::MintsByCreator { - indexer, - api_key, - address, - output, - } => { - snapshot_mints_by_creator(GetMintsArgs { - indexer, - api_key, - method: Method::Creator, - address, - output, - }) - .await - } - SnapshotSubcommands::MintsByCollection { - indexer, - api_key, - address, - output, - } => { - snapshot_mints_by_collection(GetMintsArgs { - indexer, - api_key, - method: Method::Collection, - address, - output, - }) - .await - } - SnapshotSubcommands::Prints { - master_mint, - creator, - output, - } => { - snapshot_print_editions(SnapshotPrintEditionsArgs { - client, - master_mint, - creator, - output, - }) - .await - } - } -} - pub fn process_transfer(client: RpcClient, commands: TransferSubcommands) -> Result<()> { match commands { TransferSubcommands::Asset { diff --git a/src/setup.rs b/src/setup.rs new file mode 100644 index 00000000..138c1cb7 --- /dev/null +++ b/src/setup.rs @@ -0,0 +1,154 @@ +use anyhow::{anyhow, Result}; +use dirs::home_dir; +use reqwest::Client; +use serde::{Deserialize, Serialize}; +use solana_client::rpc_client::RpcClient; +use solana_sdk::{ + commitment_config::CommitmentConfig, + signature::{read_keypair_file, Keypair}, +}; + +use std::{fs::File, path::PathBuf, str::FromStr}; + +#[derive(Debug, PartialEq, Eq, Clone, Copy, Deserialize, Serialize)] +pub enum ClientType { + Standard, + DAS, +} + +pub enum ClientLike { + RpcClient(RpcClient), + DasClient(Client), +} + +#[derive(Debug, Deserialize, Serialize)] +struct SolanaConfig { + pub json_rpc_url: String, + pub keypair_path: String, + pub commitment: String, +} + +pub struct CliConfig { + pub client: ClientLike, + pub keypair: Option, + pub rpc_url: String, +} + +#[derive(Debug)] +pub struct CliConfigBuilder { + pub json_rpc_url: Option, + pub keypair_path: Option, + pub commitment: Option, + pub client_type: ClientType, +} + +// impl Default for ClientType { +// fn default() -> Self { +// Self::Standard +// } +// } + +impl CliConfigBuilder { + pub fn new(client_type: ClientType) -> Self { + Self { + json_rpc_url: None, + keypair_path: None, + commitment: None, + client_type, + } + } + pub fn rpc_url(mut self, json_rpc_url: String) -> Self { + self.json_rpc_url = Some(json_rpc_url); + self + } + pub fn keypair_path(mut self, keypair_path: PathBuf) -> Self { + self.keypair_path = Some(keypair_path); + self + } + pub fn commitment(mut self, commitment: String) -> Self { + self.commitment = Some(commitment); + self + } + + pub fn build(&self) -> Result { + let rpc_url = self + .json_rpc_url + .clone() + .ok_or_else(|| anyhow!("No rpc url provided"))?; + + let commitment = match self.commitment.clone() { + Some(commitment) => CommitmentConfig::from_str(&commitment)?, + None => CommitmentConfig::confirmed(), + }; + + let client = match self.client_type { + ClientType::Standard => { + ClientLike::RpcClient(RpcClient::new_with_commitment(rpc_url.clone(), commitment)) + } + ClientType::DAS => ClientLike::DasClient(Client::new()), + }; + + let keypair = if let Some(keypair_path) = &self.keypair_path { + let keypair = read_keypair_file(keypair_path) + .map_err(|_| anyhow!("Unable to read keypair file"))?; + + Some(keypair) + } else { + None + }; + + Ok(CliConfig { + client, + keypair, + rpc_url, + }) + } +} + +impl CliConfig { + pub fn new( + keypair_path: Option, + rpc_url: Option, + client_type: ClientType, + ) -> Result { + let mut builder = CliConfigBuilder::new(client_type); + let solana_config = parse_solana_config(); + + if let Some(config) = solana_config { + builder = builder + .rpc_url(config.json_rpc_url) + .keypair_path(config.keypair_path.into()) + .commitment(config.commitment); + } + + if let Some(keypair_path) = keypair_path { + builder = builder.keypair_path(keypair_path); + } + + if let Some(rpc_url) = rpc_url { + builder = builder.rpc_url(rpc_url); + } + + let config = builder.build()?; + + Ok(config) + } +} + +fn parse_solana_config() -> Option { + let home_path = home_dir().expect("Couldn't find home dir"); + + let solana_config_path = home_path + .join(".config") + .join("solana") + .join("cli") + .join("config.yml"); + + let config_file = File::open(solana_config_path).ok(); + + if let Some(config_file) = config_file { + let config: SolanaConfig = serde_yaml::from_reader(config_file).ok()?; + return Some(config); + } + None +} diff --git a/src/snapshot/das_api.rs b/src/snapshot/das_api.rs new file mode 100644 index 00000000..e9e78eaa --- /dev/null +++ b/src/snapshot/das_api.rs @@ -0,0 +1,465 @@ +use std::{fmt::Display, fs::File, path::PathBuf, str::FromStr}; + +use anyhow::Result; +use metaboss_lib::derive::derive_metadata_pda; +use reqwest::header::HeaderMap; +use serde_json::{json, Value}; +use solana_program::pubkey::Pubkey; +use solana_sdk::signer::Signer; +use spl_associated_token_account::get_associated_token_address; + +use crate::{ + setup::{CliConfig, ClientLike, ClientType}, + spinner::create_spinner, +}; + +use super::{DasResponse, Holder, Item}; + +#[derive(Debug)] +pub enum HolderGroupKey { + Mint, + Fvca, + Mcc, +} + +impl FromStr for HolderGroupKey { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "mint" => Ok(HolderGroupKey::Mint), + "fvca" => Ok(HolderGroupKey::Fvca), + "mcc" => Ok(HolderGroupKey::Mcc), + _ => Err(format!("Invalid group key: {}", s)), + } + } +} + +impl Display for HolderGroupKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + HolderGroupKey::Mint => write!(f, "mint"), + HolderGroupKey::Fvca => write!(f, "fvca"), + HolderGroupKey::Mcc => write!(f, "mcc"), + } + } +} + +pub struct HoldersArgs { + pub rpc_url: String, + pub group_key: HolderGroupKey, + pub group_value: Pubkey, + pub output: PathBuf, +} + +struct Query { + method: String, + params: Value, + fvca_filter: bool, +} + +pub async fn snapshot_holders(args: HoldersArgs) -> Result<()> { + let config = CliConfig::new(None, Some(args.rpc_url), ClientType::DAS)?; + + let query = match args.group_key { + HolderGroupKey::Mint => todo!(), + HolderGroupKey::Fvca => Query { + method: "getAssetsByCreator".to_string(), + params: json!({ + "creatorAddress": args.group_value.to_string(), + "onlyVerified": true, + "page": 1, + "limit": 1000 + }), + fvca_filter: true, + }, + HolderGroupKey::Mcc => Query { + method: "getAssetsByGroup".to_string(), + params: json!({ + "groupKey": "collection", + "groupValue": args.group_value.to_string(), + "page": 1, + "limit": 1000 + }), + fvca_filter: false, + }, + }; + + let mut headers = HeaderMap::new(); + headers.insert("Content-Type", "application/json".parse().unwrap()); + + let client = match config.client { + ClientLike::DasClient(client) => client, + _ => panic!("Wrong client type"), + }; + + let mut holders = Vec::new(); + let mut page = 1; + + let mut body = json!( + { + "jsonrpc": "2.0", + "id": 1, + "method": query.method, + "params": query.params, + }); + + let fvca_filter = |item: &Item| { + item.creators.first().is_some() + && item.creators.first().unwrap().address.to_string() == args.group_value.to_string() + }; + + let spinner = create_spinner("Getting assets..."); + loop { + let response = client + .post(config.rpc_url.clone()) + .headers(headers.clone()) + .json(&body) + .send() + .await?; + + let res: DasResponse = response.json().await?; + + if res.result.items.is_empty() { + break; + } + + page += 1; + body["params"]["page"] = json!(page); + + res.result + .items + .iter() + .filter(|item| { + if query.fvca_filter { + fvca_filter(item) + } else { + true + } + }) + .for_each(|item| { + let mint_address = item.id.clone(); + let metadata_pubkey = + derive_metadata_pda(&Pubkey::from_str(mint_address.as_str()).unwrap()); + let owner_address = item.ownership.owner.clone(); + let ata_pubkey = get_associated_token_address( + &Pubkey::from_str(&owner_address).unwrap(), + &Pubkey::from_str(&mint_address).unwrap(), + ); + + holders.push(Holder { + owner_wallet: owner_address, + mint_account: item.id.clone(), + metadata_account: metadata_pubkey.to_string(), + associated_token_address: ata_pubkey.to_string(), + }); + }); + } + spinner.finish(); + + holders.sort(); + + // Write to file + let file = File::create(format!( + "{}_{}_holders.json", + args.group_value, args.group_key + ))?; + serde_json::to_writer_pretty(file, &holders)?; + + Ok(()) +} + +#[derive(Debug)] +pub enum MintsGroupKey { + Authority, + Creator, + Mcc, +} + +impl FromStr for MintsGroupKey { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "authority" => Ok(MintsGroupKey::Authority), + "creator" => Ok(MintsGroupKey::Creator), + "mcc" => Ok(MintsGroupKey::Mcc), + _ => Err(format!("Invalid group key: {}", s)), + } + } +} + +impl Display for MintsGroupKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + MintsGroupKey::Authority => write!(f, "authority"), + MintsGroupKey::Creator => write!(f, "creator"), + MintsGroupKey::Mcc => write!(f, "mcc"), + } + } +} + +pub struct MintsArgs { + pub rpc_url: String, + pub group_key: MintsGroupKey, + pub group_value: Pubkey, + pub creator_position: usize, + pub output: PathBuf, +} + +pub async fn snapshot_mints(args: MintsArgs) -> Result<()> { + let config = CliConfig::new(None, Some(args.rpc_url), ClientType::DAS)?; + + let query = match args.group_key { + MintsGroupKey::Authority => Query { + method: "getAssetsByAuthority".to_string(), + params: json!({ + "authorityAddress": args.group_value.to_string(), + "page": 1, + "limit": 1000 + }), + fvca_filter: false, + }, + MintsGroupKey::Creator => Query { + method: "getAssetsByCreator".to_string(), + params: json!({ + "creatorAddress": args.group_value.to_string(), + "onlyVerified": true, + "page": 1, + "limit": 1000 + }), + fvca_filter: true, + }, + MintsGroupKey::Mcc => Query { + method: "getAssetsByGroup".to_string(), + params: json!({ + "groupKey": "collection", + "groupValue": args.group_value.to_string(), + "page": 1, + "limit": 1000 + }), + fvca_filter: false, + }, + }; + + let mut headers = HeaderMap::new(); + headers.insert("Content-Type", "application/json".parse().unwrap()); + + let client = match config.client { + ClientLike::DasClient(client) => client, + _ => panic!("Wrong client type"), + }; + + let mut mints = Vec::new(); + let mut page = 1; + + let mut body = json!( + { + "jsonrpc": "2.0", + "id": 1, + "method": query.method, + "params": query.params, + }); + + let verified_creator_filter = |item: &Item| { + item.creators.get(args.creator_position).is_some() + && item + .creators + .get(args.creator_position) + .unwrap() + .address + .to_string() + == args.group_value.to_string() + }; + + let spinner = create_spinner("Getting assets..."); + loop { + let response = client + .post(config.rpc_url.clone()) + .headers(headers.clone()) + .json(&body) + .send() + .await?; + + let res: DasResponse = response.json().await?; + + if res.result.items.is_empty() { + break; + } + + page += 1; + body["params"]["page"] = json!(page); + + res.result + .items + .iter() + .filter(|item| { + if query.fvca_filter { + verified_creator_filter(item) + } else { + true + } + }) + .for_each(|item| { + mints.push(item.id.clone()); + }); + } + spinner.finish(); + + mints.sort(); + + // Write to file + let file = File::create(format!( + "{}_{}_mints.json", + args.group_value, args.group_key + ))?; + serde_json::to_writer_pretty(file, &mints)?; + + Ok(()) +} + +pub struct FcvaArgs { + pub rpc_url: String, + pub creator: Option, + pub output: PathBuf, +} + +pub async fn fcva_mints(args: FcvaArgs) -> Result<()> { + let config = CliConfig::new(None, Some(args.rpc_url), ClientType::DAS)?; + + // Prioritize creator from args, then config, then fail. + let creator = if let Some(creator) = args.creator { + creator.to_string() + } else if let Some(creator) = config.keypair { + creator.pubkey().to_string() + } else { + panic!("No creator provided"); + }; + + let mut headers = HeaderMap::new(); + headers.insert("Content-Type", "application/json".parse().unwrap()); + + let client = match config.client { + ClientLike::DasClient(client) => client, + _ => panic!("Wrong client type"), + }; + + let mut mints = Vec::new(); + let mut page = 1; + let spinner = create_spinner("Getting assets..."); + loop { + let body = json!( + { + "jsonrpc": "2.0", + "id": 1, + "method": "getAssetsByCreator", + "params": { + "creatorAddress": creator, + "onlyVerified": true, + "page": page, + "limit": 1000 + }, + }); + + let response = client + .post(config.rpc_url.clone()) + .headers(headers.clone()) + .json(&body) + .send() + .await?; + + let res: DasResponse = response.json().await?; + + if res.result.items.is_empty() { + break; + } + + page += 1; + + res.result + .items + .iter() + .filter(|item| { + item.creators.first().is_some() + && item.creators.first().unwrap().address.to_string() == creator + }) + .for_each(|item| { + mints.push(item.id.clone()); + }); + } + spinner.finish(); + + mints.sort(); + + // Write to file + let file = File::create(format!("{}_mints.json", creator))?; + serde_json::to_writer_pretty(file, &mints)?; + + Ok(()) +} + +pub struct MccArgs { + pub rpc_url: String, + pub mcc_id: Pubkey, + pub output: PathBuf, +} + +pub async fn mcc_mints(args: MccArgs) -> Result<()> { + let config = CliConfig::new(None, Some(args.rpc_url), ClientType::DAS)?; + + let mcc_id = args.mcc_id.to_string(); + + let mut headers = HeaderMap::new(); + headers.insert("Content-Type", "application/json".parse().unwrap()); + + let client = match config.client { + ClientLike::DasClient(client) => client, + _ => panic!("Wrong client type"), + }; + + let mut mints: Vec = Vec::new(); + let mut page = 1; + let spinner = create_spinner("Getting assets..."); + loop { + let body = json!( + { + "jsonrpc": "2.0", + "id": 1, + "method": "getAssetsByGroup", + "params": { + "groupKey": "collection", + "groupValue": mcc_id, + "page": page, + "limit": 1000 + }, + }); + + let response = client + .post(config.rpc_url.clone()) + .headers(headers.clone()) + .json(&body) + .send() + .await?; + + let res: DasResponse = response.json().await?; + + if res.result.items.is_empty() { + break; + } + + page += 1; + + res.result.items.iter().for_each(|item| { + mints.push(item.id.clone()); + }); + } + spinner.finish_and_clear(); + + mints.sort(); + + // Write to file + let file = File::create(format!("{}_mints.json", mcc_id))?; + serde_json::to_writer_pretty(file, &mints)?; + + Ok(()) +} diff --git a/src/snapshot/data.rs b/src/snapshot/data.rs index f934214a..1f383489 100644 --- a/src/snapshot/data.rs +++ b/src/snapshot/data.rs @@ -29,7 +29,7 @@ pub struct CandyMachineAccount { pub data_len: usize, } -pub struct SnapshotMintsArgs { +pub struct SnapshotMintsGpaArgs { pub creator: Option, pub position: usize, pub update_authority: Option, @@ -39,7 +39,7 @@ pub struct SnapshotMintsArgs { pub output: String, } -pub struct SnapshotHoldersArgs { +pub struct SnapshotHoldersGpaArgs { pub creator: Option, pub position: usize, pub update_authority: Option, @@ -49,3 +49,68 @@ pub struct SnapshotHoldersArgs { pub allow_unverified: bool, pub output: String, } + +use mpl_token_metadata::types::Creator; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DasResponse { + id: u32, + jsonrpc: String, + pub result: DasResult, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct DasResult { + pub total: u32, + pub limit: u32, + pub page: u32, + pub items: Vec, +} +#[derive(Debug, Serialize, Deserialize)] +pub struct ByCreatorResult { + pub total: u32, + pub limit: u32, + pub page: u32, + pub items: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Item { + pub interface: String, + pub id: String, + pub content: Value, + pub authorities: Vec, + pub compression: Value, + pub grouping: Value, + pub royalty: Value, + pub creators: Vec, + pub ownership: Ownership, + pub supply: Value, + pub mutable: bool, + pub burnt: bool, + pub inscription: Option, + pub spl20: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Inscription { + pub order: u32, + pub size: u32, + pub content_type: String, + pub encoding: String, + pub validation_hash: String, + pub inscription_data_account: String, + pub authority: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Ownership { + pub delegate: Option, + pub delegated: bool, + pub frozen: bool, + pub owner: String, + pub ownership_model: String, +} diff --git a/src/snapshot/indexer_methods.rs b/src/snapshot/indexer_methods.rs index 4c4afad5..97830d62 100644 --- a/src/snapshot/indexer_methods.rs +++ b/src/snapshot/indexer_methods.rs @@ -1,7 +1,6 @@ use std::fmt::Display; -use crate::{data::Indexers, helius, theindexio}; -use anyhow::Result; +use crate::data::Indexers; #[derive(Debug, Clone)] pub struct NftsByCreatorArgs { @@ -42,22 +41,3 @@ pub struct GetMintsArgs { pub indexer: Indexers, pub output: String, } - -pub async fn snapshot_mints_by_creator(args: GetMintsArgs) -> Result<()> { - match args.indexer { - Indexers::Helius => { - helius::get_mints(args).await?; - } - Indexers::TheIndexIO => { - theindexio::get_mints(args).await?; - } - } - Ok(()) -} - -pub async fn snapshot_mints_by_collection(args: GetMintsArgs) -> Result<()> { - match args.indexer { - Indexers::Helius => helius::get_mints(args).await, - Indexers::TheIndexIO => theindexio::get_mints(args).await, - } -} diff --git a/src/snapshot/methods.rs b/src/snapshot/methods.rs index a0c2ab89..1e780b76 100644 --- a/src/snapshot/methods.rs +++ b/src/snapshot/methods.rs @@ -14,7 +14,7 @@ use crate::theindexio; use crate::theindexio::GPAResult; use crate::{constants::*, decode::get_metadata_pda}; -pub fn snapshot_mints(client: &RpcClient, args: SnapshotMintsArgs) -> Result<()> { +pub fn snapshot_mints_gpa(client: RpcClient, args: SnapshotMintsGpaArgs) -> Result<()> { if !is_only_one_option(&args.creator, &args.update_authority) { return Err(anyhow!( "Please specify either a candy machine id or an update authority, but not both." @@ -32,7 +32,7 @@ pub fn snapshot_mints(client: &RpcClient, args: SnapshotMintsArgs) -> Result<()> }; let mut mint_addresses = get_mint_accounts( - client, + &client, &args.creator, args.position, args.update_authority, @@ -134,30 +134,30 @@ pub fn get_mint_accounts( Ok(mint_accounts) } -pub fn snapshot_holders(client: &RpcClient, args: SnapshotHoldersArgs) -> Result<()> { +pub fn snapshot_holders_gpa(client: RpcClient, args: SnapshotHoldersGpaArgs) -> Result<()> { let use_rate_limit = *USE_RATE_LIMIT.read().unwrap(); let handle = create_default_rate_limiter(); let spinner = create_spinner("Getting accounts..."); let accounts = if let Some(ref update_authority) = args.update_authority { - get_mints_by_update_authority(client, update_authority)? + get_mints_by_update_authority(&client, update_authority)? } else if let Some(ref creator) = args.creator { // Support v2 & v3 cm ids let creator_pubkey = Pubkey::from_str(creator).expect("Failed to parse pubkey from creator!"); if args.v2 { let cmv2_creator = derive_cmv2_pda(&creator_pubkey); - get_cm_creator_accounts(client, &cmv2_creator.to_string(), args.position)? + get_cm_creator_accounts(&client, &cmv2_creator.to_string(), args.position)? } else if args.v3 { let cmv3_creator = derive_cmv3_pda(&creator_pubkey); - get_cm_creator_accounts(client, &cmv3_creator.to_string(), args.position)? + get_cm_creator_accounts(&client, &cmv3_creator.to_string(), args.position)? } else { - get_cm_creator_accounts(client, creator, args.position)? + get_cm_creator_accounts(&client, creator, args.position)? } } else if let Some(ref mint_accounts_file) = args.mint_accounts_file { let file = File::open(mint_accounts_file)?; let mint_accounts: Vec = serde_json::from_reader(&file)?; - get_mint_account_infos(client, mint_accounts)? + get_mint_account_infos(&client, mint_accounts)? } else { return Err(anyhow!( "Must specify either --update-authority or --candy-machine-id or --mint-accounts-file" @@ -195,7 +195,7 @@ pub fn snapshot_holders(client: &RpcClient, args: SnapshotHoldersArgs) -> Result let token_accounts = match retry( Exponential::from_millis_with_factor(250, 2.0).take(3), - || get_holder_token_accounts(client, metadata.mint.to_string()), + || get_holder_token_accounts(&client, metadata.mint.to_string()), ) { Ok(token_accounts) => token_accounts, Err(_) => { @@ -491,12 +491,8 @@ fn get_mints_by_update_authority( Ok(accounts) } -pub fn snapshot_cm_accounts( - client: &RpcClient, - update_authority: &str, - output: &str, -) -> Result<()> { - let accounts = get_cm_accounts_by_update_authority(client, update_authority)?; +pub fn snapshot_cm_accounts(client: RpcClient, update_authority: &str, output: &str) -> Result<()> { + let accounts = get_cm_accounts_by_update_authority(&client, update_authority)?; let mut config_accounts = Vec::new(); let mut candy_machine_accounts = Vec::new(); diff --git a/src/snapshot/mod.rs b/src/snapshot/mod.rs index b391a82c..9c6df0cd 100644 --- a/src/snapshot/mod.rs +++ b/src/snapshot/mod.rs @@ -27,12 +27,16 @@ pub use std::{ sync::{Arc, Mutex}, }; +mod das_api; mod data; mod indexer_methods; mod methods; mod print_editions; +mod process; +pub use das_api::*; pub use data::*; pub use indexer_methods::*; pub use methods::*; pub use print_editions::*; +pub use process::*; diff --git a/src/snapshot/process.rs b/src/snapshot/process.rs new file mode 100644 index 00000000..f230b9ea --- /dev/null +++ b/src/snapshot/process.rs @@ -0,0 +1,115 @@ +use anyhow::Result; + +use solana_client::rpc_client::RpcClient; + +use crate::opt::SnapshotSubcommands; + +use super::*; + +pub async fn process_snapshot( + client: RpcClient, + rpc_url: String, + commands: SnapshotSubcommands, +) -> Result<()> { + match commands { + SnapshotSubcommands::Holders { + group_key, + group_value, + output, + } => { + snapshot_holders(HoldersArgs { + rpc_url, + group_key, + group_value, + output, + }) + .await + } + SnapshotSubcommands::Mints { + group_key, + group_value, + creator_position, + output, + } => { + snapshot_mints(MintsArgs { + rpc_url, + group_key, + group_value, + creator_position, + output, + }) + .await + } + SnapshotSubcommands::Fvca { creator, output } => { + fcva_mints(FcvaArgs { + rpc_url, + creator, + output, + }) + .await + } + SnapshotSubcommands::Mcc { mcc_id, output } => { + mcc_mints(MccArgs { + rpc_url, + mcc_id, + output, + }) + .await + } + SnapshotSubcommands::MintsGpa { + creator, + position, + update_authority, + v2, + v3, + allow_unverified, + output, + } => snapshot_mints_gpa( + client, + SnapshotMintsGpaArgs { + creator, + position, + update_authority, + v2, + v3, + allow_unverified, + output, + }, + ), + SnapshotSubcommands::HoldersGpa { + update_authority, + creator, + position, + mint_accounts_file, + v2, + v3, + allow_unverified, + output, + } => snapshot_holders_gpa( + client, + SnapshotHoldersGpaArgs { + update_authority, + creator, + position, + mint_accounts_file, + v2, + v3, + allow_unverified, + output, + }, + ), + SnapshotSubcommands::Prints { + master_mint, + creator, + output, + } => { + snapshot_print_editions(SnapshotPrintEditionsArgs { + client, + master_mint, + creator, + output, + }) + .await + } + } +}