Add experimental vendor_filterer support (#77)

Co-authored-by: Martin Sirringhaus <>

bulk-updater/src/operations.rs

@@ -287,6 +287,7 @@ pub fn attempt_cargo_update_before_revendor(
         let mut accept_risks: Vec<String> = Vec::new();
         let mut src: String = String::new();
         let mut tag: Option<String> = None;
+        let mut filter = false; // Still experimental, so default=false for now
         let outdir = package_path.to_path_buf();
         for param in params.iter() {
             if let Some(name) = param.name.clone() {
@@ -317,6 +318,11 @@ pub fn attempt_cargo_update_before_revendor(
                         accept_risks.push(String::from(text));
                     }
                 };
+                if name == "filter" {
+                    if let Some(val) = param.text.as_ref().and_then(|t| t.parse::<bool>().ok()) {
+                        filter = val;
+                    }
+                };
             }
         }
         if compression.is_empty() || src.is_empty() {
@@ -346,6 +352,7 @@ pub fn attempt_cargo_update_before_revendor(
             outdir,
             color: colorize,
             i_accept_the_risk: accept_risks,
+            filter,
         };
         srcpath
             .run_vendor(&new_opts)

cargo/src/cli.rs

@@ -53,6 +53,8 @@ pub struct Opts {
     pub cargotoml: Vec<PathBuf>,
     #[arg(long, default_value_t = true, action = clap::ArgAction::Set, help = "Update dependencies or not")]
     pub update: bool,
+    #[arg(long, default_value_t = false, action = clap::ArgAction::Set, help = "EXPERIMENTAL: Reduce vendor-tarball size by filtering out non-Linux dependencies.")]
+    pub filter: bool,
     #[arg(long, help = "Where to output vendor.tar* and cargo_config")]
     pub outdir: PathBuf,
     #[arg(

cargo/src/utils/mod.rs

@@ -228,7 +228,13 @@ pub fn process_src(args: &Opts, prjdir: &Path) -> Result<(), OBSCargoError> {
     process_reports(reports)?;
 
     if hasdeps {
-        vendor(prjdir, &cargo_config, &first_manifest, &manifest_files)?;
+        vendor(
+            prjdir,
+            &cargo_config,
+            &first_manifest,
+            &manifest_files,
+            args.filter,
+        )?;
 
         // Finally, compress everything together.
         let compression: &Compression = &args.compression;

cargo/src/vendor.rs

@@ -52,27 +52,60 @@ pub fn vendor(
     cargo_config: impl AsRef<Path>,
     manifest_path: impl AsRef<Path>,
     extra_manifest_paths: &[impl AsRef<Path>],
+    filter: bool,
 ) -> Result<(), OBSCargoError> {
-    let mut vendor_options: Vec<OsString> = vec![
-        "-vv".into(),
-        "--manifest-path".into(),
-        manifest_path.as_ref().into(),
-    ];
+    let mut vendor_options: Vec<OsString> =
+        vec!["--manifest-path".into(), manifest_path.as_ref().into()];
 
+    let mut using_sync = false;
     for ex_path in extra_manifest_paths {
         vendor_options.push("--sync".into());
         vendor_options.push(ex_path.as_ref().into());
+        using_sync = true;
     }
 
+    // cargo-vendor-filterer doesn't yet support `--sync` (will have it in the next release, hopefully).
+    // In the meantime, or, if filtering is explicitly disabled, use the normal vendor-subcommand.
+    let using_vendor_filterer = filter && !using_sync;
+    if filter && !using_vendor_filterer {
+        warn!(
+            "⚠️ Cannot use 'filter' in this crate yet. Falling back to regular, unfiltered vendoring."
+        );
+    }
+
+    let cargo_subcommand = if using_vendor_filterer {
+        info!("Filter set to true. Only vendoring crates for platforms *-unknown-linux-gnu and wasm32-*");
+        vendor_options.push("--platform=*-unknown-linux-gnu".into());
+        // Some crates compile their plugins to WASM, so we need those dependencies as well.
+        // Conservatively adding them everywhere, even if they are not needed everywhere.
+        // But the impact should be small.
+        vendor_options.push("--platform=wasm32-wasi".into());
+        vendor_options.push("--platform=wasm32-unknown-unknown".into());
+        // We are conservative here and vendor all possible features, even
+        // if they are not used in the spec. But we can't know.
+        // Maybe make this configurable?
+        vendor_options.push("--all-features=true".into());
+        // vendor-filterer could theoretically also create the tarballs for us,
+        // with using `--format=tar.zstd` for example. But we need to include
+        // additional files and it also doesn't support all compression-schemes.
+        vendor_options.push("--format=dir".into());
+        "vendor-filterer"
+    } else {
+        // cargo-vendor-filterer doesn't support `-vv`
+        vendor_options.push("-vv".into());
+        "vendor"
+    };
+
     debug!(?vendor_options);
 
-    let cargo_vendor_output = cargo_command("vendor", &vendor_options, &prjdir).map_err(|e| {
-        error!(err = %e);
-        OBSCargoError::new(
-            OBSCargoErrorKind::VendorError,
-            "Unable to execute cargo".to_string(),
-        )
-    })?;
+    let cargo_vendor_output =
+        cargo_command(cargo_subcommand, &vendor_options, &prjdir).map_err(|e| {
+            error!(err = %e);
+            OBSCargoError::new(
+                OBSCargoErrorKind::VendorError,
+                "Unable to execute cargo".to_string(),
+            )
+        })?;
 
     if let Some(p_path) = cargo_config.as_ref().parent() {
         fs::create_dir_all(p_path).map_err(|err| {

cargo_vendor.service

@@ -31,4 +31,9 @@
   <parameter name="cargotoml">
     <description>Specify a Cargo.toml to use. Can be specified multiple times. Default will autodetect.</description>
   </parameter>
+  <parameter name="filter">
+    <description>EXPERIMENTAL: Reduce vendor-tarball size by filtering out non-Linux dependencies.</description>
+    <allowedvalue>false</allowedvalue>
+    <allowedvalue>true</allowedvalue>
+  </parameter>
 </service>