Support development dependencies for the Gradle detector

Lack of development dependency detection for Gradle is a problem for
Android teams, especially in the context of Component Governance
alerts. Unfortunately Gradle doesn't provide enough information to
definitively identify dev dependencies in all cases, so manual
configuration is required. This change adds dev dependency
classification through two mechanisms

1. `buildscript-gradle.lockfile` and `settings-gradle.lockfile`
   contain only build-system dependencies, so always classify these as
   development dependencies.
2. Processing based on two new environment variables:
   `GRADLE_PROD_CONFIGURATIONS_REGEX` and
   `GRADLE_DEV_CONFIGURATIONS_REGEX`. Gradle lockfiles indicate which
   Gradle configuration(s) each dependency is required by.
   `GRADLE_PROD_CONFIGURATIONS_REGEX` allows specifying
   production configurations explicitly. All other configurations are
   considered development. Alternately, dev configurations may be
   specified in `GRADLE_DEV_CONFIGURATIONS_REGEX` and all others are
   considered production.

docs/environment-variables.md

@@ -18,4 +18,16 @@ When set to any value, enables detector experiments, a feature to compare the re
 same ecosystem. The available experiments are found in the [`Experiments\Config`](../src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs)
 folder.
 
+## `GRADLE_PROD_CONFIGURATIONS_REGEX`
+
+Enables dev-dependency categorization for the Gradle
+detector. Dependencies connected to Gradle configurations NOT matching
+the given regex are considered development dependencies.
+
+## `GRADLE_DEV_CONFIGURATIONS_REGEX`
+
+Enables dev-dependency categorization for the Gradle
+detector. Dependencies connected to Gradle configurations matching
+the given regex are considered development dependencies. 
+
 [1]: https://go.dev/ref/mod#go-mod-graph

docs/feature-overview.md

@@ -5,7 +5,7 @@
 | CocoaPods | <ul><li>podfile.lock</li></ul> | - | ❌ | - |
 | Conda (Python) | <ul><li>conda-lock.yml</li><li>*.conda-lock.yml</li></ul> | - | ❌ | ✔ |
 | Linux (Debian, Alpine, Rhel, Centos, Fedora, Ubuntu)| <ul><li>(via [syft](https://github.com/anchore/syft))</li></ul> | - | - | - | - |
-| Gradle | <ul><li>*.lockfile</li></ul> | <ul><li>Gradle 7 or prior using [Single File lock](https://docs.gradle.org/6.8.1/userguide/dependency_locking.html#single_lock_file_per_project)</li></ul> | ❌ | ❌ |
+| Gradle | <ul><li>*.lockfile</li></ul> | <ul><li>Gradle 7 or prior using [Single File lock](https://docs.gradle.org/6.8.1/userguide/dependency_locking.html#single_lock_file_per_project)</li></ul> |  ✔ (requires env var configuration for full effect) | ❌ |
 | Go | <ul><li>*go list -m -json all*</li><li>*go mod graph* (edge information only)</li></ul>Fallback</br><ul><li>go.mod</li><li>go.sum</li></ul> | <ul><li>Go 1.11+ (will fallback if not present)</li></ul> | ❌ | ✔ (root idenditication only for fallback) |
 | Maven | <ul><li>pom.xml</li><li>*mvn dependency:tree -f {pom.xml}*</li></ul> | <ul><li>Maven</li><li>Maven Dependency Plugin (auto-installed with Maven)</li></ul> | ✔ (test dependency scope) | ✔ |
 | NPM | <ul><li>package.json</li><li>package-lock.json</li><li>npm-shrinkwrap.json</li><li>lerna.json</li></ul> | - | ✔ (dev-dependencies in package.json, dev flag in package-lock.json) | ✔ |

src/Microsoft.ComponentDetection.Detectors/gradle/GradleComponentDetector.cs

@@ -3,6 +3,7 @@ namespace Microsoft.ComponentDetection.Detectors.Gradle;
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using Microsoft.ComponentDetection.Contracts;
@@ -12,8 +13,13 @@ namespace Microsoft.ComponentDetection.Detectors.Gradle;
 
 public class GradleComponentDetector : FileComponentDetector, IComponentDetector
 {
+    private const string ProdDepsEnvVar = "GRADLE_PROD_CONFIGURATIONS_REGEX";
+    private const string DevDepsEnvVar = "GRADLE_DEV_CONFIGURATIONS_REGEX";
     private static readonly Regex StartsWithLetterRegex = new Regex("^[A-Za-z]", RegexOptions.Compiled);
 
+    private readonly Regex prodDepsRegex;
+    private readonly Regex devDepsRegex;
+
     public GradleComponentDetector(
         IComponentStreamEnumerableFactory componentStreamEnumerableFactory,
         IObservableDirectoryWalkerFactory walkerFactory,
@@ -22,6 +28,17 @@ public GradleComponentDetector(
         this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory;
         this.Scanner = walkerFactory;
         this.Logger = logger;
+        var envVar = Environment.GetEnvironmentVariable(ProdDepsEnvVar);
+        if (!string.IsNullOrEmpty(envVar))
+        {
+            this.prodDepsRegex = new Regex(envVar, RegexOptions.Compiled);
+        }
+
+        envVar = Environment.GetEnvironmentVariable(DevDepsEnvVar);
+        if (!string.IsNullOrEmpty(envVar))
+        {
+            this.devDepsRegex = new Regex(envVar, RegexOptions.Compiled);
+        }
     }
 
     public override string Id { get; } = "Gradle";
@@ -47,6 +64,9 @@ protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDiction
 
     private void ParseLockfile(ISingleFileComponentRecorder singleFileComponentRecorder, IComponentStream file)
     {
+        // Buildscript and Settings lockfiles are always development dependencies
+        var lockfileIsDevDependency = file.Location.EndsWith("buildscript-gradle.lockfile") || file.Location.EndsWith("settings-gradle.lockfile");
+
         string text;
         using (var reader = new StreamReader(file.Stream))
         {
@@ -68,7 +88,8 @@ private void ParseLockfile(ISingleFileComponentRecorder singleFileComponentRecor
             if (line.Split(":").Length == 3)
             {
                 var detectedMavenComponent = new DetectedComponent(this.CreateMavenComponentFromFileLine(line));
-                singleFileComponentRecorder.RegisterUsage(detectedMavenComponent);
+                var devDependency = lockfileIsDevDependency || this.IsDevDependencyByConfigurations(line);
+                singleFileComponentRecorder.RegisterUsage(detectedMavenComponent, isDevelopmentDependency: devDependency);
             }
         }
     }
@@ -87,4 +108,38 @@ private MavenComponent CreateMavenComponentFromFileLine(string line)
     }
 
     private bool StartsWithLetter(string input) => StartsWithLetterRegex.IsMatch(input);
+
+    private bool IsDevDependencyByConfigurations(string line)
+    {
+        if (this.prodDepsRegex == null && this.devDepsRegex == null)
+        {
+            return false; // no regexes configured to check against
+        }
+
+        var equalsSeparatorIndex = line.IndexOf('=');
+        if (equalsSeparatorIndex == -1)
+        {
+            // We can't parse out the configuration. Maybe the project is using the one-lockfile-per-configuration format but
+            // this is deprecated in Gradle so we don't support it here, projects should upgrade to one-lockfile-per-project.
+            return false;
+        }
+
+        var configurations = line[(equalsSeparatorIndex + 1)..].Split(",");
+        return configurations.Select(c => this.IsDevDependencyByConfigurationName(c)).All(dev => dev);
+    }
+
+    private bool IsDevDependencyByConfigurationName(string configurationName)
+    {
+        if (this.devDepsRegex != null && this.devDepsRegex.IsMatch(configurationName))
+        {
+            return true;
+        }
+
+        if (this.prodDepsRegex != null && !this.prodDepsRegex.IsMatch(configurationName))
+        {
+            return true;
+        }
+
+        return false;
+    }
 }

test/Microsoft.ComponentDetection.Detectors.Tests/GradleComponentDetectorTests.cs

@@ -1,5 +1,6 @@
-namespace Microsoft.ComponentDetection.Detectors.Tests;
+namespace Microsoft.ComponentDetection.Detectors.Tests;
 
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
@@ -159,7 +160,7 @@ public async Task TestGradleDetector_SameComponentDifferentLocations_DifferentLo
 
         componentRecorder.ForOneComponent(componentRecorder.GetDetectedComponents().First().Component.Id, x =>
         {
-            Enumerable.Count<string>(x.AllFileLocations).Should().Be(2);
+            Enumerable.Count(x.AllFileLocations).Should().Be(2);
         });
 
         var dependencyGraphs = componentRecorder.GetDependencyGraphsByLocation();
@@ -216,4 +217,157 @@ four score and seven bugs ago
             component.Should().NotBeNull();
         }
     }
+
+    [TestMethod]
+    public async Task TestGradleDetector_DevDependenciesByLockfileNameAsync()
+    {
+        var regularLockfile =
+            @"org.springframework:spring-beans:5.0.5.RELEASE
+org.springframework:spring-core:5.0.5.RELEASE";
+
+        var devLockfile1 = @"org.hamcrest:hamcrest-core:2.2
+org.springframework:spring-core:5.0.5.RELEASE";
+
+        var devLockfile2 = @"org.jacoco:org.jacoco.agent:0.8.8";
+
+        var (scanResult, componentRecorder) = await this.DetectorTestUtility
+            .WithFile("settings-gradle.lockfile", devLockfile1)
+            .WithFile("buildscript-gradle.lockfile", devLockfile2)
+            .WithFile("gradle.lockfile", regularLockfile)
+            .ExecuteDetectorAsync();
+
+        Assert.AreEqual(ProcessingResultCode.Success, scanResult.ResultCode);
+
+        var discoveredComponents = componentRecorder.GetDetectedComponents().Select(c => (MavenComponent)c.Component).OrderBy(c => c.ArtifactId).ToList();
+        var dependencyGraphs = componentRecorder.GetDependencyGraphsByLocation();
+        var gradleLockfileGraph = dependencyGraphs[dependencyGraphs.Keys.First(k => k.EndsWith("\\gradle.lockfile"))];
+        var settingsGradleLockfileGraph = dependencyGraphs[dependencyGraphs.Keys.First(k => k.EndsWith("settings-gradle.lockfile"))];
+        var buildscriptGradleLockfileGraph = dependencyGraphs[dependencyGraphs.Keys.First(k => k.EndsWith("buildscript-gradle.lockfile"))];
+
+        Assert.AreEqual(4, discoveredComponents.Count);
+
+        // Dev dependency listed only in settings-gradle.lockfile
+        var component = discoveredComponents[0];
+        Assert.AreEqual("org.hamcrest", component.GroupId);
+        Assert.AreEqual("hamcrest-core", component.ArtifactId);
+        Assert.IsTrue(settingsGradleLockfileGraph.IsDevelopmentDependency(component.Id));
+
+        // Dev dependency listed only in buildscript-gradle.lockfile
+        component = discoveredComponents[1];
+        Assert.AreEqual("org.jacoco", component.GroupId);
+        Assert.AreEqual("org.jacoco.agent", component.ArtifactId);
+        Assert.IsTrue(buildscriptGradleLockfileGraph.IsDevelopmentDependency(component.Id));
+
+        // This should be purely a prod dependency, just a basic confidence test
+        component = discoveredComponents[2];
+        Assert.AreEqual("org.springframework", component.GroupId);
+        Assert.AreEqual("spring-beans", component.ArtifactId);
+        Assert.IsFalse(gradleLockfileGraph.IsDevelopmentDependency(component.Id));
+
+        // This is listed as both a prod and a dev dependency in different files
+        component = discoveredComponents[3];
+        Assert.AreEqual("org.springframework", component.GroupId);
+        Assert.AreEqual("spring-core", component.ArtifactId);
+        Assert.IsFalse(gradleLockfileGraph.IsDevelopmentDependency(component.Id));
+        Assert.IsTrue(settingsGradleLockfileGraph.IsDevelopmentDependency(component.Id));
+    }
+
+    [TestMethod]
+    public async Task TestGradleDetector_DevDependenciesByDevEnvironmentAsync()
+    {
+        var lockfile =
+                    @"org.springframework:spring-beans:5.0.5.RELEASE=assembleRelease
+org.springframework:spring-core:5.0.5.RELEASE=assembleRelease,testDebugUnitTest
+org.hamcrest:hamcrest-core:2.2=testDebugUnitTest";
+
+        Environment.SetEnvironmentVariable("GRADLE_DEV_CONFIGURATIONS_REGEX", ".*UnitTest");
+        try
+        {
+            var (scanResult, componentRecorder) = await this.DetectorTestUtility
+                .WithFile("gradle.lockfile", lockfile)
+                .ExecuteDetectorAsync();
+
+            Assert.AreEqual(ProcessingResultCode.Success, scanResult.ResultCode);
+
+            var discoveredComponents = componentRecorder.GetDetectedComponents().Select(c => (MavenComponent)c.Component).OrderBy(c => c.ArtifactId).ToList();
+            var dependencyGraph = componentRecorder.GetDependencyGraphsByLocation().Values.First();
+
+            Assert.AreEqual(3, discoveredComponents.Count);
+
+            var component = discoveredComponents[0];
+            Assert.AreEqual("org.hamcrest", component.GroupId);
+            Assert.AreEqual("hamcrest-core", component.ArtifactId);
+
+            // Purely a dev dependency, only present in a test configuration
+            Assert.IsTrue(dependencyGraph.IsDevelopmentDependency(component.Id));
+
+            component = discoveredComponents[1];
+            Assert.AreEqual("org.springframework", component.GroupId);
+            Assert.AreEqual("spring-beans", component.ArtifactId);
+
+            // Purely a prod dependency, only present in a prod configuration
+            Assert.IsFalse(dependencyGraph.IsDevelopmentDependency(component.Id));
+
+            component = discoveredComponents[2];
+            Assert.AreEqual("org.springframework", component.GroupId);
+            Assert.AreEqual("spring-core", component.ArtifactId);
+
+            // Present in both dev and prod configurations, prod should win
+            Assert.IsFalse(dependencyGraph.IsDevelopmentDependency(component.Id));
+        }
+        finally
+        {
+            Environment.SetEnvironmentVariable("GRADLE_DEV_CONFIGURATIONS_REGEX", null);
+        }
+    }
+
+    [TestMethod]
+    public async Task TestGradleDetector_DevDependenciesByProdEnvironmentAsync()
+    {
+        var lockfile =
+                    @"org.springframework:spring-beans:5.0.5.RELEASE=assembleRelease
+org.springframework:spring-core:5.0.5.RELEASE=assembleRelease,testDebugUnitTest
+org.hamcrest:hamcrest-core:2.2=testDebugUnitTest";
+
+        // Only the specific configurations are prod. Everything else is a dev dependency
+        Environment.SetEnvironmentVariable("GRADLE_PROD_CONFIGURATIONS_REGEX", "assembleRelease");
+        try
+        {
+            var (scanResult, componentRecorder) = await this.DetectorTestUtility
+                .WithFile("gradle.lockfile", lockfile)
+                .ExecuteDetectorAsync();
+
+            Assert.AreEqual(ProcessingResultCode.Success, scanResult.ResultCode);
+
+            var discoveredComponents = componentRecorder.GetDetectedComponents().Select(c => (MavenComponent)c.Component).OrderBy(c => c.ArtifactId).ToList();
+            var dependencyGraph = componentRecorder.GetDependencyGraphsByLocation().Values.First();
+
+            Assert.AreEqual(3, discoveredComponents.Count);
+
+            var component = discoveredComponents[0];
+            Assert.AreEqual("org.hamcrest", component.GroupId);
+            Assert.AreEqual("hamcrest-core", component.ArtifactId);
+
+            // Purely a dev dependency, only present in a test configuration
+            Assert.IsTrue(dependencyGraph.IsDevelopmentDependency(component.Id));
+
+            component = discoveredComponents[1];
+            Assert.AreEqual("org.springframework", component.GroupId);
+            Assert.AreEqual("spring-beans", component.ArtifactId);
+
+            // Purely a prod dependency, only present in a prod configuration
+            Assert.IsFalse(dependencyGraph.IsDevelopmentDependency(component.Id));
+
+            component = discoveredComponents[2];
+            Assert.AreEqual("org.springframework", component.GroupId);
+            Assert.AreEqual("spring-core", component.ArtifactId);
+
+            // Present in both dev and prod configurations, prod should win
+            Assert.IsFalse(dependencyGraph.IsDevelopmentDependency(component.Id));
+        }
+        finally
+        {
+            Environment.SetEnvironmentVariable("GRADLE_PROD_CONFIGURATIONS_REGEX", null);
+        }
+    }
 }