Support development dependencies for the Gradle detector

docs/environment-variables.md

@@ -18,4 +18,21 @@ When set to any value, enables detector experiments, a feature to compare the re
 same ecosystem. The available experiments are found in the [`Experiments\Config`](../src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs)
 folder.
 
+## `CD_GRADLE_DEV_LOCKFILES`
+
+Enables dev-dependency categorization for the Gradle
+detector. Comma-separated list of Gradle lockfiles which contain only
+development dependencies.  Dependencies connected to Gradle
+configurations matching the given regex are considered development
+dependencies. If a lockfile will contain a mix of development and
+production dependencies, see `CD_GRADLE_DEV_CONFIGURATIONS` below.
+
+## `CD_GRADLE_DEV_CONFIGURATIONS`
+
+Enables dev-dependency categorization for the Gradle
+detector. Comma-separated list of Gradle configurations which refer to development dependencies.
+Dependencies connected to Gradle configurations matching
+the given configurations are considered development dependencies. 
+If an entire lockfile will contain only dev dependencies, see `CD_GRADLE_DEV_LOCKFILES` above.
+
 [1]: https://go.dev/ref/mod#go-mod-graph

docs/feature-overview.md

@@ -5,7 +5,7 @@
 | CocoaPods | <ul><li>podfile.lock</li></ul> | - | ❌ | - |
 | Conda (Python) | <ul><li>conda-lock.yml</li><li>*.conda-lock.yml</li></ul> | - | ❌ | ✔ |
 | Linux (Debian, Alpine, Rhel, Centos, Fedora, Ubuntu)| <ul><li>(via [syft](https://github.com/anchore/syft))</li></ul> | - | - | - | - |
-| Gradle | <ul><li>*.lockfile</li></ul> | <ul><li>Gradle 7 or prior using [Single File lock](https://docs.gradle.org/6.8.1/userguide/dependency_locking.html#single_lock_file_per_project)</li></ul> | ❌ | ❌ |
+| Gradle | <ul><li>*.lockfile</li></ul> | <ul><li>Gradle 7 or prior using [Single File lock](https://docs.gradle.org/6.8.1/userguide/dependency_locking.html#single_lock_file_per_project)</li></ul> |  ✔ (requires env var configuration for full effect) | ❌ |
 | Go | <ul><li>*go list -m -json all*</li><li>*go mod graph* (edge information only)</li></ul>Fallback</br><ul><li>go.mod</li><li>go.sum</li></ul> | <ul><li>Go 1.11+ (will fallback if not present)</li></ul> | ❌ | ✔ (root idenditication only for fallback) |
 | Maven | <ul><li>pom.xml</li><li>*mvn dependency:tree -f {pom.xml}*</li></ul> | <ul><li>Maven</li><li>Maven Dependency Plugin (auto-installed with Maven)</li></ul> | ✔ (test dependency scope) | ✔ |
 | NPM | <ul><li>package.json</li><li>package-lock.json</li><li>npm-shrinkwrap.json</li><li>lerna.json</li></ul> | - | ✔ (dev-dependencies in package.json, dev flag in package-lock.json) | ✔ |

src/Microsoft.ComponentDetection.Detectors/gradle/GradleComponentDetector.cs

@@ -3,6 +3,7 @@ namespace Microsoft.ComponentDetection.Detectors.Gradle;
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using Microsoft.ComponentDetection.Contracts;
@@ -12,16 +13,27 @@ namespace Microsoft.ComponentDetection.Detectors.Gradle;
 
 public class GradleComponentDetector : FileComponentDetector, IComponentDetector
 {
+    private const string DevConfigurationsEnvVar = "CD_GRADLE_DEV_CONFIGURATIONS";
+    private const string DevLockfilesEnvVar = "CD_GRADLE_DEV_LOCKFILES";
     private static readonly Regex StartsWithLetterRegex = new Regex("^[A-Za-z]", RegexOptions.Compiled);
 
+    private readonly List<string> devConfigurations;
+    private readonly List<string> devLockfiles;
+
     public GradleComponentDetector(
         IComponentStreamEnumerableFactory componentStreamEnumerableFactory,
         IObservableDirectoryWalkerFactory walkerFactory,
+        IEnvironmentVariableService envVarService,
         ILogger<GradleComponentDetector> logger)
     {
         this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory;
         this.Scanner = walkerFactory;
         this.Logger = logger;
+
+        this.devLockfiles = this.ReadEnvVarStringList(envVarService, DevLockfilesEnvVar);
+        this.devConfigurations = this.ReadEnvVarStringList(envVarService, DevConfigurationsEnvVar);
+        this.Logger.LogDebug("Gradle dev-only lockfiles {Lockfiles}", string.Join(", ", this.devLockfiles));
+        this.Logger.LogDebug("Gradle dev-only configurations {Configurations}", string.Join(", ", this.devConfigurations));
     }
 
     public override string Id { get; } = "Gradle";
@@ -45,8 +57,24 @@ protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDiction
         return Task.CompletedTask;
     }
 
+    private List<string> ReadEnvVarStringList(IEnvironmentVariableService envVarService, string envVar)
+        => (envVarService.GetEnvironmentVariable(envVar) ?? string.Empty).Split(",", StringSplitOptions.RemoveEmptyEntries).ToList();
+
     private void ParseLockfile(ISingleFileComponentRecorder singleFileComponentRecorder, IComponentStream file)
     {
+        // Buildscript and Settings lockfiles are always development dependencies
+        var lockfileRelativePath = Path.GetRelativePath(this.CurrentScanRequest.SourceDirectory.FullName, file.Location);
+        var lockfileIsDevDependency = lockfileRelativePath.EndsWith("buildscript-gradle.lockfile") || lockfileRelativePath.EndsWith("settings-gradle.lockfile") || this.devLockfiles.Contains(lockfileRelativePath);
+
+        if (lockfileIsDevDependency)
+        {
+            this.Logger.LogDebug("Gradle lockfile {Location} contains dev dependencies only", lockfileRelativePath);
+        }
+        else
+        {
+            this.Logger.LogDebug("Gradle lockfile {Location} contains at least some production dependencies", lockfileRelativePath);
+        }
+
         string text;
         using (var reader = new StreamReader(file.Stream))
         {
@@ -68,7 +96,8 @@ private void ParseLockfile(ISingleFileComponentRecorder singleFileComponentRecor
             if (line.Split(":").Length == 3)
             {
                 var detectedMavenComponent = new DetectedComponent(this.CreateMavenComponentFromFileLine(line));
-                singleFileComponentRecorder.RegisterUsage(detectedMavenComponent);
+                var devDependency = lockfileIsDevDependency || this.IsDevDependencyByConfigurations(line);
+                singleFileComponentRecorder.RegisterUsage(detectedMavenComponent, isDevelopmentDependency: devDependency);
             }
         }
     }
@@ -87,4 +116,23 @@ private MavenComponent CreateMavenComponentFromFileLine(string line)
     }
 
     private bool StartsWithLetter(string input) => StartsWithLetterRegex.IsMatch(input);
+
+    private bool IsDevDependencyByConfigurations(string line)
+    {
+        var equalsSeparatorIndex = line.IndexOf('=');
+        if (equalsSeparatorIndex == -1)
+        {
+            // We can't parse out the configuration. Maybe the project is using the one-lockfile-per-configuration format but
+            // this is deprecated in Gradle so we don't support it here, projects should upgrade to one-lockfile-per-project.
+            return false;
+        }
+
+        var configurations = line[(equalsSeparatorIndex + 1)..].Split(",");
+        return configurations.Select(c => this.IsDevDependencyByConfigurationName(c)).All(dev => dev);
+    }
+
+    private bool IsDevDependencyByConfigurationName(string configurationName)
+    {
+        return this.devConfigurations.Contains(configurationName);
+    }
 }

test/Microsoft.ComponentDetection.Detectors.Tests/GradleComponentDetectorTests.cs

@@ -1,6 +1,8 @@
-namespace Microsoft.ComponentDetection.Detectors.Tests;
+namespace Microsoft.ComponentDetection.Detectors.Tests;
 
+using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
 using FluentAssertions;
@@ -10,12 +12,21 @@
 using Microsoft.ComponentDetection.Detectors.Tests.Utilities;
 using Microsoft.ComponentDetection.TestsUtilities;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Moq;
 
 [TestClass]
 [TestCategory("Governance/All")]
 [TestCategory("Governance/ComponentDetection")]
 public class GradleComponentDetectorTests : BaseDetectorTest<GradleComponentDetector>
 {
+    private readonly Mock<IEnvironmentVariableService> envVarService;
+
+    public GradleComponentDetectorTests()
+    {
+        this.envVarService = new Mock<IEnvironmentVariableService>();
+        this.DetectorTestUtility.AddServiceMock(this.envVarService);
+    }
+
     [TestMethod]
     public async Task TestGradleDetectorWithNoFiles_ReturnsSuccessfullyAsync()
     {
@@ -216,4 +227,158 @@ four score and seven bugs ago
             component.Should().NotBeNull();
         }
     }
+
+    [TestMethod]
+    public async Task TestGradleDetector_DevDependenciesByLockfileNameAsync()
+    {
+        var regularLockfile =
+            @"org.springframework:spring-beans:5.0.5.RELEASE
+org.springframework:spring-core:5.0.5.RELEASE";
+
+        var devLockfile1 = @"org.hamcrest:hamcrest-core:2.2
+org.springframework:spring-core:5.0.5.RELEASE";
+
+        var devLockfile2 = @"org.jacoco:org.jacoco.agent:0.8.8";
+
+        var (scanResult, componentRecorder) = await this.DetectorTestUtility
+            .WithFile("settings-gradle.lockfile", devLockfile1)
+            .WithFile("buildscript-gradle.lockfile", devLockfile2)
+            .WithFile("gradle.lockfile", regularLockfile)
+            .ExecuteDetectorAsync();
+
+        scanResult.ResultCode.Should().Be(ProcessingResultCode.Success);
+
+        var discoveredComponents = componentRecorder.GetDetectedComponents().Select(c => (MavenComponent)c.Component).OrderBy(c => c.ArtifactId).ToList();
+        var dependencyGraphs = componentRecorder.GetDependencyGraphsByLocation();
+        var gradleLockfileGraph = dependencyGraphs[dependencyGraphs.Keys.First(k => k.EndsWith(Path.DirectorySeparatorChar + "gradle.lockfile"))];
+        var settingsGradleLockfileGraph = dependencyGraphs[dependencyGraphs.Keys.First(k => k.EndsWith("settings-gradle.lockfile"))];
+        var buildscriptGradleLockfileGraph = dependencyGraphs[dependencyGraphs.Keys.First(k => k.EndsWith("buildscript-gradle.lockfile"))];
+
+        discoveredComponents.Should().HaveCount(4);
+
+        // Dev dependency listed only in settings-gradle.lockfile
+        var component = discoveredComponents[0];
+        component.GroupId.Should().Be("org.hamcrest");
+        component.ArtifactId.Should().Be("hamcrest-core");
+        settingsGradleLockfileGraph.IsDevelopmentDependency(component.Id).Should().BeTrue();
+
+        // Dev dependency listed only in buildscript-gradle.lockfile
+        component = discoveredComponents[1];
+        component.GroupId.Should().Be("org.jacoco");
+        component.ArtifactId.Should().Be("org.jacoco.agent");
+        buildscriptGradleLockfileGraph.IsDevelopmentDependency(component.Id).Should().BeTrue();
+
+        // This should be purely a prod dependency, just a basic confidence test
+        component = discoveredComponents[2];
+        component.GroupId.Should().Be("org.springframework");
+        component.ArtifactId.Should().Be("spring-beans");
+        gradleLockfileGraph.IsDevelopmentDependency(component.Id).Should().BeFalse();
+
+        // This is listed as both a prod and a dev dependency in different files
+        component = discoveredComponents[3];
+        component.GroupId.Should().Be("org.springframework");
+        component.ArtifactId.Should().Be("spring-core");
+        gradleLockfileGraph.IsDevelopmentDependency(component.Id).Should().BeFalse();
+        settingsGradleLockfileGraph.IsDevelopmentDependency(component.Id).Should().BeTrue();
+    }
+
+    [TestMethod]
+    public async Task TestGradleDetector_DevDependenciesByDevLockfileEnvironmentAsync()
+    {
+        var regularLockfile =
+            @"org.springframework:spring-beans:5.0.5.RELEASE
+org.springframework:spring-core:5.0.5.RELEASE";
+
+        var devLockfile1 = @"org.hamcrest:hamcrest-core:2.2
+org.springframework:spring-core:5.0.5.RELEASE";
+
+        var devLockfile2 = @"org.jacoco:org.jacoco.agent:0.8.8";
+
+        this.envVarService.Setup(x => x.GetEnvironmentVariable("CD_GRADLE_DEV_LOCKFILES")).Returns("dev1\\gradle.lockfile,dev2\\gradle.lockfile");
+
+        var (scanResult, componentRecorder) = await this.DetectorTestUtility
+            .WithFile("dev1\\gradle.lockfile", devLockfile1)
+            .WithFile("dev2\\gradle.lockfile", devLockfile2)
+            .WithFile("prod\\gradle.lockfile", regularLockfile)
+            .ExecuteDetectorAsync();
+
+        scanResult.ResultCode.Should().Be(ProcessingResultCode.Success);
+
+        var discoveredComponents = componentRecorder.GetDetectedComponents().Select(c => (MavenComponent)c.Component).OrderBy(c => c.ArtifactId).ToList();
+        var dependencyGraphs = componentRecorder.GetDependencyGraphsByLocation();
+        var gradleLockfileGraph = dependencyGraphs[dependencyGraphs.Keys.First(k => k.EndsWith("prod\\gradle.lockfile"))];
+        var dev1GradleLockfileGraph = dependencyGraphs[dependencyGraphs.Keys.First(k => k.EndsWith("dev1\\gradle.lockfile"))];
+        var dev2GradleLockfileGraph = dependencyGraphs[dependencyGraphs.Keys.First(k => k.EndsWith("dev2\\gradle.lockfile"))];
+
+        discoveredComponents.Should().HaveCount(4);
+
+        // Dev dependency listed only in dev1\gradle.lockfile
+        var component = discoveredComponents[0];
+        component.GroupId.Should().Be("org.hamcrest");
+        component.ArtifactId.Should().Be("hamcrest-core");
+        dev1GradleLockfileGraph.IsDevelopmentDependency(component.Id).Should().BeTrue();
+
+        // Dev dependency listed only in dev2\gradle.lockfile
+        component = discoveredComponents[1];
+        component.GroupId.Should().Be("org.jacoco");
+        component.ArtifactId.Should().Be("org.jacoco.agent");
+        dev2GradleLockfileGraph.IsDevelopmentDependency(component.Id).Should().BeTrue();
+
+        // This should be purely a prod dependency, just a basic confidence test
+        component = discoveredComponents[2];
+        component.GroupId.Should().Be("org.springframework");
+        component.ArtifactId.Should().Be("spring-beans");
+        gradleLockfileGraph.IsDevelopmentDependency(component.Id).Should().BeFalse();
+
+        // This is listed as both a prod and a dev dependency in different files
+        component = discoveredComponents[3];
+        component.GroupId.Should().Be("org.springframework");
+        component.ArtifactId.Should().Be("spring-core");
+        gradleLockfileGraph.IsDevelopmentDependency(component.Id).Should().BeFalse();
+
+        dev1GradleLockfileGraph.IsDevelopmentDependency(component.Id).Should().BeTrue();
+    }
+
+    [TestMethod]
+    public async Task TestGradleDetector_DevDependenciesByDevConfigurationEnvironmentAsync()
+    {
+        var lockfile =
+                    @"org.springframework:spring-beans:5.0.5.RELEASE=assembleRelease
+org.springframework:spring-core:5.0.5.RELEASE=assembleRelease,testDebugUnitTest
+org.hamcrest:hamcrest-core:2.2=testReleaseUnitTest";
+
+        this.envVarService.Setup(x => x.GetEnvironmentVariable("CD_GRADLE_DEV_CONFIGURATIONS")).Returns("testDebugUnitTest,testReleaseUnitTest");
+
+        var (scanResult, componentRecorder) = await this.DetectorTestUtility
+            .WithFile("gradle.lockfile", lockfile)
+            .ExecuteDetectorAsync();
+
+        scanResult.ResultCode.Should().Be(ProcessingResultCode.Success);
+
+        var discoveredComponents = componentRecorder.GetDetectedComponents().Select(c => (MavenComponent)c.Component).OrderBy(c => c.ArtifactId).ToList();
+        var dependencyGraph = componentRecorder.GetDependencyGraphsByLocation().Values.First();
+
+        discoveredComponents.Should().HaveCount(3);
+
+        var component = discoveredComponents[0];
+        component.GroupId.Should().Be("org.hamcrest");
+        component.ArtifactId.Should().Be("hamcrest-core");
+
+        // Purely a dev dependency, only present in a test configuration
+        dependencyGraph.IsDevelopmentDependency(component.Id).Should().BeTrue();
+
+        component = discoveredComponents[1];
+        component.GroupId.Should().Be("org.springframework");
+        component.ArtifactId.Should().Be("spring-beans");
+
+        // Purely a prod dependency, only present in a prod configuration
+        dependencyGraph.IsDevelopmentDependency(component.Id).Should().BeFalse();
+
+        component = discoveredComponents[2];
+        component.GroupId.Should().Be("org.springframework");
+        component.ArtifactId.Should().Be("spring-core");
+
+        // Present in both dev and prod configurations, prod should win
+        dependencyGraph.IsDevelopmentDependency(component.Id).Should().BeFalse();
+    }
 }