diff --git a/src/buck.rs b/src/buck.rs index ab90a105..daf14f0a 100644 --- a/src/buck.rs +++ b/src/buck.rs @@ -448,6 +448,38 @@ impl Serialize for HttpArchive { } } +#[derive(Debug)] +pub struct ExtractArchive { + pub name: Name, + pub src: BuckPath, + pub strip_prefix: String, + pub sub_targets: BTreeSet, + pub visibility: Visibility, + pub sort_key: Name, +} + +impl Serialize for ExtractArchive { + fn serialize(&self, ser: S) -> Result { + let Self { + name, + src, + strip_prefix, + sub_targets, + visibility, + sort_key: _, + } = self; + let mut map = ser.serialize_map(None)?; + map.serialize_entry("name", name)?; + map.serialize_entry("src", src)?; + map.serialize_entry("strip_prefix", strip_prefix)?; + if !sub_targets.is_empty() { + map.serialize_entry("sub_targets", sub_targets)?; + } + map.serialize_entry("visibility", visibility)?; + map.end() + } +} + #[derive(Debug)] pub struct GitFetch { pub name: Name, @@ -1025,6 +1057,7 @@ impl Serialize for PrebuiltCxxLibrary { pub enum Rule { Alias(Alias), Filegroup(Filegroup), + ExtractArchive(ExtractArchive), HttpArchive(HttpArchive), GitFetch(GitFetch), Binary(RustBinary), @@ -1066,6 +1099,7 @@ fn rule_sort_key(rule: &Rule) -> impl Ord + '_ { // Make the alias rule come before the actual rule. Note that aliases // emitted by reindeer are always to a target within the same package. Rule::Alias(Alias { actual, .. }) => RuleSortKey::Other(actual, 0), + Rule::ExtractArchive(ExtractArchive { sort_key, .. }) => RuleSortKey::Other(sort_key, 1), Rule::HttpArchive(HttpArchive { sort_key, .. }) => RuleSortKey::Other(sort_key, 1), Rule::GitFetch(GitFetch { name, .. }) => RuleSortKey::GitFetch(name), Rule::Filegroup(_) @@ -1091,6 +1125,7 @@ impl Rule { Rule::Alias(Alias { name, .. }) | Rule::Filegroup(Filegroup { name, .. }) | Rule::HttpArchive(HttpArchive { name, .. }) + | Rule::ExtractArchive(ExtractArchive { name, .. }) | Rule::GitFetch(GitFetch { name, .. }) | Rule::Binary(RustBinary { common: @@ -1143,6 +1178,9 @@ impl Rule { Rule::Filegroup(filegroup) => { FunctionCall::new(&config.filegroup, filegroup).serialize(Serializer) } + Rule::ExtractArchive(compressed_crate) => { + FunctionCall::new(&config.extract_archive, compressed_crate).serialize(Serializer) + } Rule::HttpArchive(http_archive) => { FunctionCall::new(&config.http_archive, http_archive).serialize(Serializer) } diff --git a/src/buckify.rs b/src/buckify.rs index 83ebbc69..e9b29820 100644 --- a/src/buckify.rs +++ b/src/buckify.rs @@ -33,6 +33,7 @@ use crate::buck; use crate::buck::Alias; use crate::buck::BuckPath; use crate::buck::Common; +use crate::buck::ExtractArchive; use crate::buck::Filegroup; use crate::buck::GitFetch; use crate::buck::HttpArchive; @@ -56,6 +57,7 @@ use crate::cargo::Source; use crate::cargo::TargetReq; use crate::collection::SetOrMap; use crate::config::Config; +use crate::config::VendorConfig; use crate::fixups::ExportSources; use crate::fixups::Fixups; use crate::glob::Globs; @@ -219,7 +221,7 @@ fn generate_rules<'scope>( for rule in rules { let _ = rule_tx.send(Ok(rule)); } - if context.config.vendor.is_none() { + if context.config.vendor.is_not_source() { deps.push((pkg, TargetReq::Sources)); } } @@ -258,10 +260,22 @@ fn generate_nonvendored_sources_archive<'scope>( match &lockfile_package.source { Source::Local => Ok(None), - Source::CratesIo => generate_http_archive(context, pkg, lockfile_package).map(Some), + Source::CratesIo => match context.config.vendor { + VendorConfig::Off => generate_http_archive(context, pkg, lockfile_package).map(Some), + VendorConfig::LocalRegistry => { + generate_extract_archive(context, pkg, lockfile_package).map(Some) + } + VendorConfig::Source(_) => unreachable!(), + }, Source::Git { repo, commit_hash, .. - } => generate_git_fetch(repo, commit_hash).map(Some), + } => match context.config.vendor { + VendorConfig::Off => generate_git_fetch(repo, commit_hash).map(Some), + VendorConfig::LocalRegistry => { + generate_extract_archive(context, pkg, lockfile_package).map(Some) + } + VendorConfig::Source(_) => unreachable!(), + }, Source::Unrecognized(_) => { bail!( "`vendor = false` mode is supported only with exclusively crates.io and https git dependencies. \"{}\" {} is coming from some other source", @@ -272,6 +286,25 @@ fn generate_nonvendored_sources_archive<'scope>( } } +fn generate_extract_archive<'scope>( + _context: &'scope RuleContext<'scope>, + pkg: &'scope Manifest, + _lockfile_package: &LockfilePackage, +) -> anyhow::Result { + let vendordir = "vendor"; + Ok(Rule::ExtractArchive(ExtractArchive { + name: Name(format!("{}-{}.crate", pkg.name, pkg.version)), + src: BuckPath(PathBuf::from(format!( + "{vendordir}/{}-{}.crate", + pkg.name, pkg.version + ))), + strip_prefix: format!("{}-{}", pkg.name, pkg.version), + sub_targets: BTreeSet::new(), // populated later after all fixups are constructed + visibility: Visibility::Private, + sort_key: Name(format!("{}-{}", pkg.name, pkg.version)), + })) +} + fn generate_http_archive<'scope>( context: &'scope RuleContext<'scope>, pkg: &'scope Manifest, @@ -414,8 +447,10 @@ fn generate_target_rules<'scope>( let manifest_dir = pkg.manifest_dir(); let mapped_manifest_dir = - if context.config.vendor.is_some() || matches!(pkg.source, Source::Local) { + if context.config.vendor.is_source() || matches!(pkg.source, Source::Local) { relative_path(&paths.third_party_dir, manifest_dir) + } else if let VendorConfig::LocalRegistry = context.config.vendor { + PathBuf::from(format!("{}-{}.crate", pkg.name, pkg.version)) } else if let Source::Git { repo, .. } = &pkg.source { let git_fetch = short_name_for_git_repo(repo)?; let repository_root = find_repository_root(manifest_dir)?; @@ -428,7 +463,7 @@ fn generate_target_rules<'scope>( let edition = tgt.edition.unwrap_or(pkg.edition); let mut licenses = BTreeSet::new(); - if config.vendor.is_none() { + if !config.vendor.is_source() { // The `licenses` attribute takes `attrs.source()` which is the file // containing the custom license text. For `vendor = false` mode, we // don't have such a file on disk, and we don't have a Buck label either @@ -458,7 +493,7 @@ fn generate_target_rules<'scope>( // filename, or a list of globs. // If we're configured to get precise sources and we're using 2018+ edition source, then // parse the crate to see what files are actually used. - let mut srcs = if (config.vendor.is_some() || matches!(pkg.source, Source::Local)) + let mut srcs = if (config.vendor.is_source() || matches!(pkg.source, Source::Local)) && fixups.precise_srcs() && edition >= Edition::Rust2018 { @@ -504,7 +539,7 @@ fn generate_target_rules<'scope>( ) .context("rustc_flags")?; - if config.vendor.is_some() || matches!(pkg.source, Source::Local) { + if config.vendor.is_source() || matches!(pkg.source, Source::Local) { unzip_platform( config, &mut base, @@ -516,6 +551,10 @@ fn generate_target_rules<'scope>( fixups.compute_srcs(srcs)?, ) .context("srcs")?; + } else if let VendorConfig::LocalRegistry = config.vendor { + let http_archive_target = format!(":{}-{}.crate", pkg.name, pkg.version); + base.srcs + .insert(BuckPath(PathBuf::from(http_archive_target))); } else if let Source::Git { repo, .. } = &pkg.source { let short_name = short_name_for_git_repo(repo)?; let git_fetch_target = format!(":{}.git", short_name); @@ -870,7 +909,7 @@ fn generate_target_rules<'scope>( // For non-disk sources (i.e. non-vendor mode git_fetch and // http_archive), `srcs` and `exclude` are ignored because // we can't look at the files to match globs. - let srcs = if config.vendor.is_some() || matches!(pkg.source, Source::Local) { + let srcs = if config.vendor.is_source() || matches!(pkg.source, Source::Local) { // e.g. {"src/lib.rs": "vendor/foo-1.0.0/src/lib.rs"} let mut globs = Globs::new(srcs, exclude).context("export sources")?; let srcs = globs @@ -884,6 +923,14 @@ fn generate_target_rules<'scope>( globs.check_all_globs_used()?; } srcs + } else if let VendorConfig::LocalRegistry = config.vendor { + // e.g. {":foo-1.0.0.git": "foo-1.0.0"} + let http_archive_target = format!(":{}-{}.crate", pkg.name, pkg.version); + [( + BuckPath(mapped_manifest_dir.clone()), + SubtargetOrPath::Path(BuckPath(PathBuf::from(http_archive_target))), + )] + .into() } else if let Source::Git { repo, .. } = &pkg.source { // e.g. {":foo-123.git": "foo-123"} let short_name = short_name_for_git_repo(repo)?; @@ -985,7 +1032,7 @@ fn buckify_for_universe( // Fill in all http_archive rules with all the sub_targets which got // mentioned by fixups. - if config.vendor.is_none() { + if !config.vendor.is_source() { let mut need_subtargets = HashMap::>::new(); let mut insert = |subtarget_or_path: &SubtargetOrPath| { if let SubtargetOrPath::Subtarget(subtarget) = subtarget_or_path { @@ -1026,10 +1073,18 @@ fn buckify_for_universe( rules = rules .into_iter() .map(|mut rule| { - if let Rule::HttpArchive(rule) = &mut rule { - if let Some(need_subtargets) = need_subtargets.remove(&rule.name) { - rule.sub_targets = need_subtargets; + match &mut rule { + Rule::HttpArchive(rule) => { + if let Some(need_subtargets) = need_subtargets.remove(&rule.name) { + rule.sub_targets = need_subtargets; + } + } + Rule::ExtractArchive(rule) => { + if let Some(need_subtargets) = need_subtargets.remove(&rule.name) { + rule.sub_targets = need_subtargets; + } } + _ => (), } rule }) diff --git a/src/cargo.rs b/src/cargo.rs index fc26d796..575fe24c 100644 --- a/src/cargo.rs +++ b/src/cargo.rs @@ -61,7 +61,7 @@ pub fn cargo_get_lockfile_and_metadata( let cargo_home; let lockfile; - if config.vendor.is_none() { + if config.vendor.is_not_source() { cargo_home = None; // Whether or not there is a Cargo.lock already, do not read it yet. diff --git a/src/config.rs b/src/config.rs index 5197af5c..2fd500d8 100644 --- a/src/config.rs +++ b/src/config.rs @@ -77,11 +77,8 @@ pub struct Config { #[serde(default)] pub buck: BuckConfig, - #[serde( - default = "default_vendor_config", - deserialize_with = "deserialize_vendor_config" - )] - pub vendor: Option, + #[serde(default, deserialize_with = "deserialize_vendor_config")] + pub vendor: VendorConfig, #[serde(default)] pub audit: AuditConfig, @@ -131,6 +128,8 @@ pub struct BuckConfig { /// Rule name for http_archive #[serde(default)] pub http_archive: StringWithDefault, + #[serde(default)] + pub extract_archive: StringWithDefault, /// Rule name for git_fetch #[serde(default)] pub git_fetch: StringWithDefault, @@ -153,9 +152,26 @@ pub struct BuckConfig { pub buildscript_genrule: StringWithDefault, } +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub enum VendorConfig { + Off, + LocalRegistry, + Source(VendorSourceConfig), +} +impl VendorConfig { + pub(crate) fn is_source(&self) -> bool { + matches!(self, Self::Source(_)) + } + + pub(crate) fn is_not_source(&self) -> bool { + !self.is_source() + } +} + #[derive(Debug, Default, Clone, Deserialize)] #[serde(deny_unknown_fields)] -pub struct VendorConfig { +pub struct VendorSourceConfig { /// List of .gitignore files to use to filter checksum files, relative to /// this config file. #[serde(default)] @@ -165,6 +181,12 @@ pub struct VendorConfig { pub checksum_exclude: HashSet, } +impl Default for VendorConfig { + fn default() -> Self { + VendorConfig::Source(Default::default()) + } +} + #[derive(Debug, Default, Clone, Deserialize)] #[serde(deny_unknown_fields)] pub struct AuditConfig { @@ -247,10 +269,6 @@ impl From for StringWithDefault { } } -fn default_vendor_config() -> Option { - Some(VendorConfig::default()) -} - fn default_platforms() -> HashMap { const DEFAULT_PLATFORMS_TOML: &str = include_str!("default_platforms.toml"); @@ -270,14 +288,14 @@ fn default_universes() -> BTreeMap { map } -fn deserialize_vendor_config<'de, D>(deserializer: D) -> Result, D::Error> +fn deserialize_vendor_config<'de, D>(deserializer: D) -> Result where D: Deserializer<'de>, { struct VendorConfigVisitor; impl<'de> Visitor<'de> for VendorConfigVisitor { - type Value = Option; + type Value = VendorConfig; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { formatter.write_str("[vendor] section, or `vendor = false`") @@ -289,14 +307,30 @@ where { // `vendor = true`: default configuration with vendoring. // `vendor = false`: do not vendor. - Ok(value.then(VendorConfig::default)) + Ok(if value { + VendorConfig::default() + } else { + VendorConfig::Off + }) + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + if v == "local-registry" { + Ok(VendorConfig::LocalRegistry) + } else { + Err(E::custom("unknown vendor type")) + } } fn visit_map(self, map: M) -> Result where M: MapAccess<'de>, { - VendorConfig::deserialize(MapAccessDeserializer::new(map)).map(Some) + VendorSourceConfig::deserialize(MapAccessDeserializer::new(map)) + .map(VendorConfig::Source) } } diff --git a/src/fixups.rs b/src/fixups.rs index 1b129b56..98c82697 100644 --- a/src/fixups.rs +++ b/src/fixups.rs @@ -42,6 +42,7 @@ use crate::cargo::NodeDepKind; use crate::cargo::Source; use crate::collection::SetOrMap; use crate::config::Config; +use crate::config::VendorConfig; use crate::glob::Globs; use crate::glob::SerializableGlobSet as GlobSet; use crate::glob::NO_EXCLUDE; @@ -166,7 +167,7 @@ impl<'meta> Fixups<'meta> { &self, relative_to_manifest_dir: &Path, ) -> anyhow::Result { - if self.config.vendor.is_some() || matches!(self.package.source, Source::Local) { + if self.config.vendor.is_source() || matches!(self.package.source, Source::Local) { // Path to vendored file looks like "vendor/foo-1.0.0/src/lib.rs" let manifest_dir = relative_path(&self.third_party_dir, self.manifest_dir); let path = manifest_dir.join(relative_to_manifest_dir); @@ -311,7 +312,7 @@ impl<'meta> Fixups<'meta> { }; for fix in fixes { - if self.config.vendor.is_none() { + if self.config.vendor.is_not_source() { if let Source::Git { repo, .. } = &self.package.source { // Cxx_library fixups only work if the sources are vendored // or from an http_archive. They do not work with sources @@ -875,13 +876,18 @@ impl<'meta> Fixups<'meta> { for cargo_env in config.cargo_env.iter() { let v = match cargo_env { CargoEnv::CARGO_MANIFEST_DIR => { - if self.config.vendor.is_some() + if self.config.vendor.is_source() || matches!(self.package.source, Source::Local) { StringOrPath::Path(BuckPath(relative_path( &self.third_party_dir, self.manifest_dir, ))) + } else if let VendorConfig::LocalRegistry = self.config.vendor { + StringOrPath::String(format!( + "{}-{}.crate", + self.package.name, self.package.version, + )) } else if let Source::Git { repo, .. } = &self.package.source { let short_name = short_name_for_git_repo(repo)?; StringOrPath::String(short_name.to_owned()) @@ -936,7 +942,7 @@ impl<'meta> Fixups<'meta> { // This function is only used in vendoring mode, so it's guaranteed that // manifest_dir is a subdirectory of third_party_dir. - assert!(self.config.vendor.is_some() || matches!(self.package.source, Source::Local)); + assert!(self.config.vendor.is_source() || matches!(self.package.source, Source::Local)); let manifest_rel = relative_path(&self.third_party_dir, self.manifest_dir); let srcs_globs: Vec = srcs diff --git a/src/main.rs b/src/main.rs index a22c9860..96c4f5fc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -25,6 +25,8 @@ use std::path::PathBuf; use clap::Parser; use clap::Subcommand; +use crate::config::VendorConfig; + mod audit_sec; mod buck; mod buckify; @@ -145,10 +147,15 @@ fn try_main() -> anyhow::Result<()> { } SubCommand::Buckify { stdout } => { - if config.vendor.is_some() && !vendor::is_vendored(&paths)? { - // If you ran `reindeer buckify` without `reindeer vendor`, then - // default to generating non-vendored targets. - config.vendor = None; + match &config.vendor { + VendorConfig::LocalRegistry | VendorConfig::Source(..) + if !vendor::is_vendored(&config, &paths)? => + { + // If you ran `reindeer buckify` without `reindeer vendor`, then + // default to generating non-vendored targets. + config.vendor = VendorConfig::Off; + } + _ => {} } buckify::buckify(&config, &args, &paths, *stdout)?; } diff --git a/src/remap.rs b/src/remap.rs index e939aa4a..8ec88c5c 100644 --- a/src/remap.rs +++ b/src/remap.rs @@ -11,13 +11,13 @@ use std::path::PathBuf; use serde::Deserialize; use serde::Serialize; -#[derive(Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize)] pub struct RemapConfig { #[serde(rename = "source", default)] pub sources: Map, } -#[derive(Serialize, Deserialize, Default)] +#[derive(Debug, Serialize, Deserialize, Default)] pub struct RemapSource { pub directory: Option, pub git: Option, @@ -26,4 +26,6 @@ pub struct RemapSource { pub tag: Option, #[serde(rename = "replace-with")] pub replace_with: Option, + #[serde(rename = "local-registry")] + pub local_registry: Option, } diff --git a/src/vendor.rs b/src/vendor.rs index 3abde03f..1ed19f41 100644 --- a/src/vendor.rs +++ b/src/vendor.rs @@ -21,6 +21,7 @@ use crate::buckify::relative_path; use crate::cargo; use crate::config::Config; use crate::config::VendorConfig; +use crate::config::VendorSourceConfig; use crate::remap::RemapConfig; use crate::Args; use crate::Paths; @@ -41,35 +42,71 @@ pub(crate) fn cargo_vendor( ) -> anyhow::Result<()> { let vendordir = Path::new("vendor"); // relative to third_party_dir - let mut cmdline = vec![ - "vendor", - "--manifest-path", - paths.manifest_path.to_str().unwrap(), - vendordir.to_str().unwrap(), - "--versioned-dirs", - ]; - if no_delete { - cmdline.push("--no-delete"); - } + match &config.vendor { + VendorConfig::LocalRegistry => { + let mut cmdline = vec![ + "local-registry", + "-s", + paths.lockfile_path.to_str().unwrap(), + vendordir.to_str().unwrap(), + "--git", + ]; + if no_delete { + cmdline.push("--no-delete"); + } + log::info!("Running cargo {:?}", cmdline); + let _ = cargo::run_cargo( + config, + Some(&paths.cargo_home), + &paths.third_party_dir, + args, + &cmdline, + )?; + let cargoconfig = format!( + r#"[source.crates-io] +registry = 'sparse+https://index.crates.io/' +replace-with = 'local-registry' + +[source.local-registry] +local-registry = {vendordir:?} +"# + ); + fs::write(paths.cargo_home.join("config.toml"), &cargoconfig)?; + if !cargoconfig.is_empty() { + assert!(is_vendored(config, paths)?); + } + } + VendorConfig::Source(source_config) => { + let mut cmdline = vec![ + "vendor", + "--manifest-path", + paths.manifest_path.to_str().unwrap(), + vendordir.to_str().unwrap(), + "--versioned-dirs", + ]; + if no_delete { + cmdline.push("--no-delete"); + } - fs::create_dir_all(&paths.cargo_home)?; + fs::create_dir_all(&paths.cargo_home)?; - log::info!("Running cargo {:?}", cmdline); - let cargoconfig = cargo::run_cargo( - config, - Some(&paths.cargo_home), - &paths.third_party_dir, - args, - &cmdline, - )?; + log::info!("Running cargo {:?}", cmdline); + let cargoconfig = cargo::run_cargo( + config, + Some(&paths.cargo_home), + &paths.third_party_dir, + args, + &cmdline, + )?; - fs::write(paths.cargo_home.join("config.toml"), &cargoconfig)?; - if !cargoconfig.is_empty() { - assert!(is_vendored(paths)?); - } + fs::write(paths.cargo_home.join("config.toml"), &cargoconfig)?; + if !cargoconfig.is_empty() { + assert!(is_vendored(config, paths)?); + } - if let Some(vendor_config) = &config.vendor { - filter_checksum_files(&paths.third_party_dir, vendordir, vendor_config)?; + filter_checksum_files(&paths.third_party_dir, vendordir, source_config)?; + } + _ => (), } if audit_sec { @@ -79,7 +116,7 @@ pub(crate) fn cargo_vendor( Ok(()) } -pub(crate) fn is_vendored(paths: &Paths) -> anyhow::Result { +pub(crate) fn is_vendored(config: &Config, paths: &Paths) -> anyhow::Result { // .cargo/config.toml is Cargo's preferred name for the config, but .cargo/config // is the older name so it takes priority if present. let mut cargo_config_path = paths.cargo_home.join("config"); @@ -106,8 +143,17 @@ pub(crate) fn is_vendored(paths: &Paths) -> anyhow::Result { let remap_config: RemapConfig = toml::from_str(&content) .context(format!("Failed to parse {}", cargo_config_path.display()))?; - match remap_config.sources.get("vendored-sources") { - Some(vendored_sources) => Ok(vendored_sources.directory.is_some()), + let source_name = match config.vendor { + VendorConfig::LocalRegistry => "local-registry", + VendorConfig::Source(_) => "vendored-sources", + _ => return Ok(false), + }; + match remap_config.sources.get(source_name) { + Some(source) => match config.vendor { + VendorConfig::LocalRegistry => Ok(source.local_registry.is_some()), + VendorConfig::Source(_) => Ok(source.directory.is_some()), + VendorConfig::Off => Ok(false), + }, None => Ok(false), } } @@ -115,7 +161,7 @@ pub(crate) fn is_vendored(paths: &Paths) -> anyhow::Result { fn filter_checksum_files( third_party_dir: &Path, vendordir: &Path, - config: &VendorConfig, + config: &VendorSourceConfig, ) -> anyhow::Result<()> { if config.checksum_exclude.is_empty() && config.gitignore_checksum_exclude.is_empty() { return Ok(());