Support development dependencies for the Gradle detector

docs/environment-variables.md

@@ -18,4 +18,21 @@ When set to any value, enables detector experiments, a feature to compare the re
 same ecosystem. The available experiments are found in the [`Experiments\Config`](../src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs)
 folder.
 
+## `CD_GRADLE_DEV_LOCKFILES`
+
+Enables dev-dependency categorization for the Gradle
+detector. Comma-separated list of Gradle lockfiles which contain only
+development dependencies.  Dependencies connected to Gradle
+configurations matching the given regex are considered development
+dependencies. If a lockfile will contain a mix of development and
+production dependencies, see `CD_GRADLE_DEV_CONFIGURATIONS` below.
+
+## `CD_GRADLE_DEV_CONFIGURATIONS`
+
+Enables dev-dependency categorization for the Gradle
+detector. Comma-separated list of Gradle configurations which refer to development dependencies.
+Dependencies connected to Gradle configurations matching
+the given configurations are considered development dependencies. 
+If an entire lockfile will contain only dev dependencies, see `CD_GRADLE_DEV_LOCKFILES` above.
+
 [1]: https://go.dev/ref/mod#go-mod-graph

docs/feature-overview.md

@@ -5,7 +5,7 @@
 | CocoaPods | <ul><li>podfile.lock</li></ul> | - | ❌ | - |
 | Conda (Python) | <ul><li>conda-lock.yml</li><li>*.conda-lock.yml</li></ul> | - | ❌ | ✔ |
 | Linux (Debian, Alpine, Rhel, Centos, Fedora, Ubuntu)| <ul><li>(via [syft](https://github.com/anchore/syft))</li></ul> | - | - | - | - |
-| Gradle | <ul><li>*.lockfile</li></ul> | <ul><li>Gradle 7 or prior using [Single File lock](https://docs.gradle.org/6.8.1/userguide/dependency_locking.html#single_lock_file_per_project)</li></ul> | ❌ | ❌ |
+| Gradle | <ul><li>*.lockfile</li></ul> | <ul><li>Gradle 7 or prior using [Single File lock](https://docs.gradle.org/6.8.1/userguide/dependency_locking.html#single_lock_file_per_project)</li></ul> |  ✔ (requires env var configuration for full effect) | ❌ |
 | Go | <ul><li>*go list -m -json all*</li><li>*go mod graph* (edge information only)</li></ul>Fallback</br><ul><li>go.mod</li><li>go.sum</li></ul> | <ul><li>Go 1.11+ (will fallback if not present)</li></ul> | ❌ | ✔ (root idenditication only for fallback) |
 | Maven | <ul><li>pom.xml</li><li>*mvn dependency:tree -f {pom.xml}*</li></ul> | <ul><li>Maven</li><li>Maven Dependency Plugin (auto-installed with Maven)</li></ul> | ✔ (test dependency scope) | ✔ |
 | NPM | <ul><li>package.json</li><li>package-lock.json</li><li>npm-shrinkwrap.json</li><li>lerna.json</li></ul> | - | ✔ (dev-dependencies in package.json, dev flag in package-lock.json) | ✔ |

src/Microsoft.ComponentDetection.Common/EnvironmentVariableService.cs

@@ -1,6 +1,7 @@
 namespace Microsoft.ComponentDetection.Common;
 
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using Microsoft.ComponentDetection.Contracts;
 
@@ -23,6 +24,9 @@ public string GetEnvironmentVariable(string name)
         return caseInsensitiveName != null ? Environment.GetEnvironmentVariable(caseInsensitiveName) : null;
     }
 
+    public List<string> GetListEnvironmentVariable(string name, string delimiter)
+    => (this.GetEnvironmentVariable(name) ?? string.Empty).Split(delimiter, StringSplitOptions.RemoveEmptyEntries).ToList();
+
     public bool IsEnvironmentVariableValueTrue(string name)
     {
         _ = bool.TryParse(this.GetEnvironmentVariable(name), out var result);

src/Microsoft.ComponentDetection.Contracts/IEnvironmentVariableService.cs

@@ -1,5 +1,7 @@
 namespace Microsoft.ComponentDetection.Contracts;
 
+using System.Collections.Generic;
+
 /// <summary>
 /// Wraps some common environment variable operations for easier testability.
 /// </summary>
@@ -19,6 +21,14 @@ public interface IEnvironmentVariableService
     /// <returns>Returns a string of the environment variable value.</returns>
     string GetEnvironmentVariable(string name);
 
+    /// <summary>
+    /// Returns the value of an environment variable which is formatted as a delimited list.
+    /// </summary>
+    /// <param name="name">Name of the environment variable.</param>
+    /// <param name="delimiter">Delimiter separating the items in the list.</param>
+    /// <returns>Returns she parsed environment variable value.</returns>
+    List<string> GetListEnvironmentVariable(string name, string delimiter = ",");
+
     /// <summary>
     /// Returns true if the environment variable value is true.
     /// </summary>

src/Microsoft.ComponentDetection.Detectors/gradle/GradleComponentDetector.cs

@@ -3,6 +3,7 @@ namespace Microsoft.ComponentDetection.Detectors.Gradle;
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using Microsoft.ComponentDetection.Contracts;
@@ -12,16 +13,27 @@ namespace Microsoft.ComponentDetection.Detectors.Gradle;
 
 public class GradleComponentDetector : FileComponentDetector, IComponentDetector
 {
+    private const string DevConfigurationsEnvVar = "CD_GRADLE_DEV_CONFIGURATIONS";
+    private const string DevLockfilesEnvVar = "CD_GRADLE_DEV_LOCKFILES";
     private static readonly Regex StartsWithLetterRegex = new Regex("^[A-Za-z]", RegexOptions.Compiled);
 
+    private readonly List<string> devConfigurations;
+    private readonly List<string> devLockfiles;
+
     public GradleComponentDetector(
         IComponentStreamEnumerableFactory componentStreamEnumerableFactory,
         IObservableDirectoryWalkerFactory walkerFactory,
+        IEnvironmentVariableService envVarService,
         ILogger<GradleComponentDetector> logger)
     {
         this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory;
         this.Scanner = walkerFactory;
         this.Logger = logger;
+
+        this.devLockfiles = envVarService.GetListEnvironmentVariable(DevLockfilesEnvVar) ?? new List<string>();
+        this.devConfigurations = envVarService.GetListEnvironmentVariable(DevConfigurationsEnvVar) ?? new List<string>();
+        this.Logger.LogDebug("Gradle dev-only lockfiles {Lockfiles}", string.Join(", ", this.devLockfiles));
+        this.Logger.LogDebug("Gradle dev-only configurations {Configurations}", string.Join(", ", this.devConfigurations));
     }
 
     public override string Id { get; } = "Gradle";
@@ -32,7 +44,7 @@ public GradleComponentDetector(
 
     public override IEnumerable<ComponentType> SupportedComponentTypes { get; } = new[] { ComponentType.Maven };
 
-    public override int Version { get; } = 2;
+    public override int Version { get; } = 3;
 
     protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary<string, string> detectorArgs)
     {
@@ -68,7 +80,8 @@ private void ParseLockfile(ISingleFileComponentRecorder singleFileComponentRecor
             if (line.Split(":").Length == 3)
             {
                 var detectedMavenComponent = new DetectedComponent(this.CreateMavenComponentFromFileLine(line));
-                singleFileComponentRecorder.RegisterUsage(detectedMavenComponent);
+                var devDependency = this.IsDevDependencyByLockfile(file) || this.IsDevDependencyByConfigurations(line);
+                singleFileComponentRecorder.RegisterUsage(detectedMavenComponent, isDevelopmentDependency: devDependency);
             }
         }
     }
@@ -87,4 +100,44 @@ private MavenComponent CreateMavenComponentFromFileLine(string line)
     }
 
     private bool StartsWithLetter(string input) => StartsWithLetterRegex.IsMatch(input);
+
+    private bool IsDevDependencyByConfigurations(string line)
+    {
+        var equalsSeparatorIndex = line.IndexOf('=');
+        if (equalsSeparatorIndex == -1)
+        {
+            // We can't parse out the configuration. Maybe the project is using the one-lockfile-per-configuration format but
+            // this is deprecated in Gradle so we don't support it here, projects should upgrade to one-lockfile-per-project.
+            return false;
+        }
+
+        var configurations = line[(equalsSeparatorIndex + 1)..].Split(",");
+        return configurations.All(this.IsDevDependencyByConfigurationName);
+    }
+
+    private bool IsDevDependencyByConfigurationName(string configurationName)
+    {
+        return this.devConfigurations.Contains(configurationName);
+    }
+
+    private bool IsDevDependencyByLockfile(IComponentStream file)
+    {
+        // Buildscript and Settings lockfiles are always development dependencies
+        var lockfileName = Path.GetFileName(file.Location);
+        var lockfileRelativePath = Path.GetRelativePath(this.CurrentScanRequest.SourceDirectory.FullName, file.Location);
+        var dev = lockfileName == "buildscript-gradle.lockfile"
+            || lockfileName == "settings-gradle.lockfile"
+            || this.devLockfiles.Contains(lockfileRelativePath);
+
+        if (dev)
+        {
+            this.Logger.LogDebug("Gradle lockfile {Location} contains dev dependencies only", lockfileRelativePath);
+        }
+        else
+        {
+            this.Logger.LogDebug("Gradle lockfile {Location} contains at least some production dependencies", lockfileRelativePath);
+        }
+
+        return dev;
+    }
 }

test/Microsoft.ComponentDetection.Common.Tests/EnvironmentVariableServiceTests.cs

@@ -1,4 +1,4 @@
-namespace Microsoft.ComponentDetection.Common.Tests;
+namespace Microsoft.ComponentDetection.Common.Tests;
 
 using System;
 using FluentAssertions;
@@ -93,4 +93,45 @@ public void IsEnvironmentVariableValueTrue_returnsFalseForInvalidAndNull()
         result2.Should().BeFalse();
         Environment.SetEnvironmentVariable(envVariableKey1, null);
     }
+
+    [TestMethod]
+    public void GetListEnvironmentVariable_returnEmptyIfVariableDoesNotExist()
+    {
+        this.testSubject.GetListEnvironmentVariable("NonExistentVar", ",").Should().BeEmpty();
+    }
+
+    [TestMethod]
+    public void GetListEnvironmentVariable_emptyListIfEmptyVar()
+    {
+        var key = "foo";
+        Environment.SetEnvironmentVariable(key, string.Empty);
+        var result = this.testSubject.GetListEnvironmentVariable(key, ",");
+        result.Should().NotBeNull();
+        result.Should().BeEmpty();
+        Environment.SetEnvironmentVariable(key, null);
+    }
+
+    [TestMethod]
+    public void GetListEnvironmentVariable_singleItem()
+    {
+        var key = "foo";
+        Environment.SetEnvironmentVariable(key, "bar");
+        var result = this.testSubject.GetListEnvironmentVariable(key, ",");
+        result.Should().ContainSingle();
+        result.Should().Contain("bar");
+        Environment.SetEnvironmentVariable(key, null);
+    }
+
+    [TestMethod]
+    public void GetListEnvironmentVariable_multipleItems()
+    {
+        var key = "foo";
+        Environment.SetEnvironmentVariable(key, "bar,baz,qux");
+        var result = this.testSubject.GetListEnvironmentVariable(key, ",");
+        result.Should().HaveCount(3);
+        result.Should().Contain("bar");
+        result.Should().Contain("baz");
+        result.Should().Contain("qux");
+        Environment.SetEnvironmentVariable(key, null);
+    }
 }