Skip to content

Commit

Permalink
Update RustCLI processing to handle virtual manifests / skip over ven…
Browse files Browse the repository at this point in the history 
…dor packages (#1015)

* Update RustCLI processing to handle virtual manifests and skip over vendor packages

* update tests for new logic

* update detector version

* resolve comment
  • Loading branch information
@FernandoRojo
FernandoRojo authored Feb 24, 2024
1 parent c2cfd4d commit 0b8a2e6
Show file tree
Hide file tree
Showing 3 changed files with 659 additions and 636 deletions.
12 changes: 10 additions & 2 deletions src/Microsoft.ComponentDetection.Detectors/rust/Contracts/CargoMetadata.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -345,7 +345,7 @@ public partial class DepKind

public enum Kind { Build, Dev };

public enum CrateType { Bench, Bin, CustomBuild, Example, Lib, ProcMacro, Test };
public enum CrateType { Bench, Bin, CustomBuild, Example, Lib, ProcMacro, Test, RLib, DyLib, CdyLib, StaticLib };

public partial class CargoMetadata
{
Expand Down Expand Up @@ -453,8 +453,16 @@ public override CrateType Read(ref Utf8JsonReader reader, Type typeToConvert, Js
return CrateType.ProcMacro;
case "test":
return CrateType.Test;
case "rlib":
return CrateType.RLib;
case "dylib":
return CrateType.DyLib;
case "cdylib":
return CrateType.CdyLib;
case "staticlib":
return CrateType.StaticLib;
}
throw new Exception("Cannot unmarshal type CrateType");
throw new Exception($"Cannot unmarshal type CrateType - {value}");
}

public override void Write(Utf8JsonWriter writer, CrateType value, JsonSerializerOptions options)
Expand Down
74 changes: 49 additions & 25 deletions src/Microsoft.ComponentDetection.Detectors/rust/RustCliDetector.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ public RustCliDetector(
public override IEnumerable<ComponentType> SupportedComponentTypes => new[] { ComponentType.Cargo };

/// <inheritdoc />
public override int Version => 2;
public override int Version => 3;

/// <inheritdoc />
public override IList<string> SearchPatterns { get; } = new[] { "Cargo.toml" };
Expand Down Expand Up @@ -99,9 +99,9 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID

if (cliResult.ExitCode != 0)
{
this.Logger.LogWarning("`cargo metadata` failed with {Location}. Ensure the Cargo.lock is up to date.", processRequest.ComponentStream.Location);
this.Logger.LogWarning("`cargo metadata` failed while processing {Location}. with error: {Error}", processRequest.ComponentStream.Location, cliResult.StdErr);
record.DidRustCliCommandFail = true;
record.WasRustFallbackStrategyUsed = true;
record.WasRustFallbackStrategyUsed = ShouldFallbackFromError(cliResult.StdErr);
record.RustCliCommandError = cliResult.StdErr;
record.FallbackReason = "`cargo metadata` failed";
}
Expand All @@ -118,22 +118,28 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID
string.IsNullOrWhiteSpace(x.License) ? null : x.License));

var root = metadata.Resolve.Root;
HashSet<string> visitedDependencies = new();

// A cargo.toml can be used to declare a workspace and not a package (A Virtual Manifest).
// In this case, the root will be null as it will not be pulling in dependencies itself.
// https://doc.rust-lang.org/cargo/reference/workspaces.html#virtual-workspace
if (root == null)
{
this.Logger.LogWarning("Virtual Manifest: {Location}, falling back to cargo.lock parsing", processRequest.ComponentStream.Location);
record.DidRustCliCommandFail = true;
record.WasRustFallbackStrategyUsed = true;
record.FallbackReason = "Virtual Manifest";
}
this.Logger.LogWarning("Virtual Manifest: {Location}", processRequest.ComponentStream.Location);

HashSet<string> visitedDependencies = new();
if (!record.WasRustFallbackStrategyUsed)
foreach (var dep in metadata.Resolve.Nodes)
{
var componentKey = $"{dep.Id}";
if (!visitedDependencies.Contains(componentKey))
{
visitedDependencies.Add(componentKey);
this.TraverseAndRecordComponents(processRequest.SingleFileComponentRecorder, componentStream.Location, graph, dep.Id, null, null, packages, visitedDependencies, explicitlyReferencedDependency: true, isTomlRoot: true);
}
}
}
else
{
this.TraverseAndRecordComponents(processRequest.SingleFileComponentRecorder, componentStream.Location, graph, root, null, null, packages, visitedDependencies);
this.TraverseAndRecordComponents(processRequest.SingleFileComponentRecorder, componentStream.Location, graph, root, null, null, packages, visitedDependencies, explicitlyReferencedDependency: true, isTomlRoot: true);
}
}
}
Expand Down Expand Up @@ -169,13 +175,17 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID

private static Dictionary<string, Node> BuildGraph(CargoMetadata cargoMetadata) => cargoMetadata.Resolve.Nodes.ToDictionary(x => x.Id);

private static (string Name, string Version) ParseNameAndVersion(string nameAndVersion)
private static bool IsLocalPackage(CargoPackage package) => package.Source == null;

private static bool ShouldFallbackFromError(string error)
{
var parts = nameAndVersion.Split(' ');
return (parts[0], parts[1]);
}
if (error.Contains("current package believes it's in a workspace", StringComparison.OrdinalIgnoreCase))
{
return false;
}

private static bool IsLocalPackage(CargoPackage package) => package.Source == null;
return true;
}

private static bool ParseDependency(string dependency, out string packageName, out string version, out string source)
{
Expand Down Expand Up @@ -205,38 +215,48 @@ private void TraverseAndRecordComponents(
Dep depInfo,
IReadOnlyDictionary<string, (string Authors, string License)> packagesMetadata,
ISet<string> visitedDependencies,
bool explicitlyReferencedDependency = false)
bool explicitlyReferencedDependency = false,
bool isTomlRoot = false)
{
try
{
var isDevelopmentDependency = depInfo?.DepKinds.Any(x => x.Kind is Kind.Dev) ?? false;
var (name, version) = ParseNameAndVersion(id);
if (!ParseDependency(id, out var name, out var version, out var source))
{
// Could not parse the dependency string
this.Logger.LogWarning("Failed to parse dependency '{Id}'", id);
return;
}

var (authors, license) = packagesMetadata.TryGetValue($"{name} {version}", out var package)
? package
: (null, null);

var detectedComponent = new DetectedComponent(new CargoComponent(name, version, authors, license));

recorder.RegisterUsage(
detectedComponent,
explicitlyReferencedDependency,
isDevelopmentDependency: isDevelopmentDependency,
parentComponentId: parent?.Component.Id);

if (!graph.TryGetValue(id, out var node))
{
this.Logger.LogWarning("Could not find {Id} at {Location} in cargo metadata output", id, location);
return;
}

var shouldRegister = !isTomlRoot && !source.StartsWith("path+file");
if (shouldRegister)
{
recorder.RegisterUsage(
detectedComponent,
explicitlyReferencedDependency,
isDevelopmentDependency: isDevelopmentDependency,
parentComponentId: parent?.Component.Id);
}

foreach (var dep in node.Deps)
{
var componentKey = $"{detectedComponent.Component.Id}{dep.Pkg}";
if (!visitedDependencies.Contains(componentKey))
{
visitedDependencies.Add(componentKey);
this.TraverseAndRecordComponents(recorder, location, graph, dep.Pkg, detectedComponent, dep, packagesMetadata, visitedDependencies, parent == null);
this.TraverseAndRecordComponents(recorder, location, graph, dep.Pkg, shouldRegister ? detectedComponent : null, dep, packagesMetadata, visitedDependencies, parent == null);
}
}
}
Expand Down Expand Up @@ -281,6 +301,10 @@ private async Task ProcessCargoLockFallbackAsync(IComponentStream cargoTomlFile,
record.FallbackCargoLockFound = false;
return;
}
else
{
this.Logger.LogWarning("Falling back to cargo.lock processing using {CargoTomlLocation}", cargoLockFileStream.Location);
}

record.FallbackCargoLockLocation = cargoLockFileStream.Location;
record.FallbackCargoLockFound = true;
Expand Down
Loading

0 comments on commit 0b8a2e6

Please sign in to comment.