Support development dependencies for the Gradle detector (#878)

* Support development dependencies for the Gradle detector

Lack of development dependency detection for Gradle is a problem for
Android teams, especially in the context of Component Governance
alerts. Unfortunately Gradle doesn't provide enough information to
definitively identify dev dependencies in all cases, so manual
configuration is required. This change adds dev dependency
classification through two mechanisms

1. `buildscript-gradle.lockfile` and `settings-gradle.lockfile`
   contain only build-system dependencies, so always classify these as
   development dependencies.
2. Processing based on two new environment variables:
   `GRADLE_PROD_CONFIGURATIONS_REGEX` and
   `GRADLE_DEV_CONFIGURATIONS_REGEX`. Gradle lockfiles indicate which
   Gradle configuration(s) each dependency is required by.
   `GRADLE_PROD_CONFIGURATIONS_REGEX` allows specifying
   production configurations explicitly. All other configurations are
   considered development. Alternately, dev configurations may be
   specified in `GRADLE_DEV_CONFIGURATIONS_REGEX` and all others are
   considered production.

* Changes based on meeting prior to the holidays

* fluent assertions

* Visual studio recommendations

* More fluent assertsions

* Fix test to be cross-platform

* Fix the cross-platform test fix

* Fix code coverage by removing dead code check

* Address code review comments

docs/environment-variables.md

@@ -18,4 +18,21 @@ When set to any value, enables detector experiments, a feature to compare the re
 same ecosystem. The available experiments are found in the [`Experiments\Config`](../src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs)
 folder.
 
+## `CD_GRADLE_DEV_LOCKFILES`
+
+Enables dev-dependency categorization for the Gradle
+detector. Comma-separated list of Gradle lockfiles which contain only
+development dependencies.  Dependencies connected to Gradle
+configurations matching the given regex are considered development
+dependencies. If a lockfile will contain a mix of development and
+production dependencies, see `CD_GRADLE_DEV_CONFIGURATIONS` below.
+
+## `CD_GRADLE_DEV_CONFIGURATIONS`
+
+Enables dev-dependency categorization for the Gradle
+detector. Comma-separated list of Gradle configurations which refer to development dependencies.
+Dependencies connected to Gradle configurations matching
+the given configurations are considered development dependencies. 
+If an entire lockfile will contain only dev dependencies, see `CD_GRADLE_DEV_LOCKFILES` above.
+
 [1]: https://go.dev/ref/mod#go-mod-graph

docs/feature-overview.md

@@ -5,7 +5,7 @@
 | CocoaPods | <ul><li>podfile.lock</li></ul> | - | ❌ | - |
 | Conda (Python) | <ul><li>conda-lock.yml</li><li>*.conda-lock.yml</li></ul> | - | ❌ | ✔ |
 | Linux (Debian, Alpine, Rhel, Centos, Fedora, Ubuntu)| <ul><li>(via [syft](https://github.com/anchore/syft))</li></ul> | - | - | - | - |
-| Gradle | <ul><li>*.lockfile</li></ul> | <ul><li>Gradle 7 or prior using [Single File lock](https://docs.gradle.org/6.8.1/userguide/dependency_locking.html#single_lock_file_per_project)</li></ul> | ❌ | ❌ |
+| Gradle | <ul><li>*.lockfile</li></ul> | <ul><li>Gradle 7 or prior using [Single File lock](https://docs.gradle.org/6.8.1/userguide/dependency_locking.html#single_lock_file_per_project)</li></ul> |  ✔ (requires env var configuration for full effect) | ❌ |
 | Go | <ul><li>*go list -m -json all*</li><li>*go mod graph* (edge information only)</li></ul>Fallback</br><ul><li>go.mod</li><li>go.sum</li></ul> | <ul><li>Go 1.11+ (will fallback if not present)</li></ul> | ❌ | ✔ (root idenditication only for fallback) |
 | Maven | <ul><li>pom.xml</li><li>*mvn dependency:tree -f {pom.xml}*</li></ul> | <ul><li>Maven</li><li>Maven Dependency Plugin (auto-installed with Maven)</li></ul> | ✔ (test dependency scope) | ✔ |
 | NPM | <ul><li>package.json</li><li>package-lock.json</li><li>npm-shrinkwrap.json</li><li>lerna.json</li></ul> | - | ✔ (dev-dependencies in package.json, dev flag in package-lock.json) | ✔ |

src/Microsoft.ComponentDetection.Common/EnvironmentVariableService.cs

@@ -1,6 +1,7 @@
 namespace Microsoft.ComponentDetection.Common;
 
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using Microsoft.ComponentDetection.Contracts;
 
@@ -23,6 +24,9 @@ public string GetEnvironmentVariable(string name)
         return caseInsensitiveName != null ? Environment.GetEnvironmentVariable(caseInsensitiveName) : null;
     }
 
+    public List<string> GetListEnvironmentVariable(string name, string delimiter)
+    => (this.GetEnvironmentVariable(name) ?? string.Empty).Split(delimiter, StringSplitOptions.RemoveEmptyEntries).ToList();
+
     public bool IsEnvironmentVariableValueTrue(string name)
     {
         _ = bool.TryParse(this.GetEnvironmentVariable(name), out var result);

src/Microsoft.ComponentDetection.Contracts/IEnvironmentVariableService.cs

@@ -1,5 +1,7 @@
 namespace Microsoft.ComponentDetection.Contracts;
 
+using System.Collections.Generic;
+
 /// <summary>
 /// Wraps some common environment variable operations for easier testability.
 /// </summary>
@@ -19,6 +21,14 @@ public interface IEnvironmentVariableService
     /// <returns>Returns a string of the environment variable value.</returns>
     string GetEnvironmentVariable(string name);
 
+    /// <summary>
+    /// Returns the value of an environment variable which is formatted as a delimited list.
+    /// </summary>
+    /// <param name="name">Name of the environment variable.</param>
+    /// <param name="delimiter">Delimiter separating the items in the list.</param>
+    /// <returns>Returns she parsed environment variable value.</returns>
+    List<string> GetListEnvironmentVariable(string name, string delimiter = ",");
+
     /// <summary>
     /// Returns true if the environment variable value is true.
     /// </summary>

src/Microsoft.ComponentDetection.Detectors/gradle/GradleComponentDetector.cs

@@ -3,6 +3,7 @@ namespace Microsoft.ComponentDetection.Detectors.Gradle;
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using Microsoft.ComponentDetection.Contracts;
@@ -12,16 +13,27 @@ namespace Microsoft.ComponentDetection.Detectors.Gradle;
 
 public class GradleComponentDetector : FileComponentDetector, IComponentDetector
 {
+    private const string DevConfigurationsEnvVar = "CD_GRADLE_DEV_CONFIGURATIONS";
+    private const string DevLockfilesEnvVar = "CD_GRADLE_DEV_LOCKFILES";
     private static readonly Regex StartsWithLetterRegex = new Regex("^[A-Za-z]", RegexOptions.Compiled);
 
+    private readonly List<string> devConfigurations;
+    private readonly List<string> devLockfiles;
+
     public GradleComponentDetector(
         IComponentStreamEnumerableFactory componentStreamEnumerableFactory,
         IObservableDirectoryWalkerFactory walkerFactory,
+        IEnvironmentVariableService envVarService,
         ILogger<GradleComponentDetector> logger)
     {
         this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory;
         this.Scanner = walkerFactory;
         this.Logger = logger;
+
+        this.devLockfiles = envVarService.GetListEnvironmentVariable(DevLockfilesEnvVar) ?? new List<string>();
+        this.devConfigurations = envVarService.GetListEnvironmentVariable(DevConfigurationsEnvVar) ?? new List<string>();
+        this.Logger.LogDebug("Gradle dev-only lockfiles {Lockfiles}", string.Join(", ", this.devLockfiles));
+        this.Logger.LogDebug("Gradle dev-only configurations {Configurations}", string.Join(", ", this.devConfigurations));
     }
 
     public override string Id { get; } = "Gradle";
@@ -32,7 +44,7 @@ public GradleComponentDetector(
 
     public override IEnumerable<ComponentType> SupportedComponentTypes { get; } = new[] { ComponentType.Maven };
 
-    public override int Version { get; } = 2;
+    public override int Version { get; } = 3;
 
     protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary<string, string> detectorArgs)
     {
@@ -68,7 +80,8 @@ private void ParseLockfile(ISingleFileComponentRecorder singleFileComponentRecor
             if (line.Split(":").Length == 3)
             {
                 var detectedMavenComponent = new DetectedComponent(this.CreateMavenComponentFromFileLine(line));
-                singleFileComponentRecorder.RegisterUsage(detectedMavenComponent);
+                var devDependency = this.IsDevDependencyByLockfile(file) || this.IsDevDependencyByConfigurations(line);
+                singleFileComponentRecorder.RegisterUsage(detectedMavenComponent, isDevelopmentDependency: devDependency);
             }
         }
     }
@@ -87,4 +100,44 @@ private MavenComponent CreateMavenComponentFromFileLine(string line)
     }
 
     private bool StartsWithLetter(string input) => StartsWithLetterRegex.IsMatch(input);
+
+    private bool IsDevDependencyByConfigurations(string line)
+    {
+        var equalsSeparatorIndex = line.IndexOf('=');
+        if (equalsSeparatorIndex == -1)
+        {
+            // We can't parse out the configuration. Maybe the project is using the one-lockfile-per-configuration format but
+            // this is deprecated in Gradle so we don't support it here, projects should upgrade to one-lockfile-per-project.
+            return false;
+        }
+
+        var configurations = line[(equalsSeparatorIndex + 1)..].Split(",");
+        return configurations.All(this.IsDevDependencyByConfigurationName);
+    }
+
+    private bool IsDevDependencyByConfigurationName(string configurationName)
+    {
+        return this.devConfigurations.Contains(configurationName);
+    }
+
+    private bool IsDevDependencyByLockfile(IComponentStream file)
+    {
+        // Buildscript and Settings lockfiles are always development dependencies
+        var lockfileName = Path.GetFileName(file.Location);
+        var lockfileRelativePath = Path.GetRelativePath(this.CurrentScanRequest.SourceDirectory.FullName, file.Location);
+        var dev = lockfileName == "buildscript-gradle.lockfile"
+            || lockfileName == "settings-gradle.lockfile"
+            || this.devLockfiles.Contains(lockfileRelativePath);
+
+        if (dev)
+        {
+            this.Logger.LogDebug("Gradle lockfile {Location} contains dev dependencies only", lockfileRelativePath);
+        }
+        else
+        {
+            this.Logger.LogDebug("Gradle lockfile {Location} contains at least some production dependencies", lockfileRelativePath);
+        }
+
+        return dev;
+    }
 }

test/Microsoft.ComponentDetection.Common.Tests/EnvironmentVariableServiceTests.cs

@@ -1,4 +1,4 @@
-namespace Microsoft.ComponentDetection.Common.Tests;
+namespace Microsoft.ComponentDetection.Common.Tests;
 
 using System;
 using FluentAssertions;
@@ -93,4 +93,45 @@ public void IsEnvironmentVariableValueTrue_returnsFalseForInvalidAndNull()
         result2.Should().BeFalse();
         Environment.SetEnvironmentVariable(envVariableKey1, null);
     }
+
+    [TestMethod]
+    public void GetListEnvironmentVariable_returnEmptyIfVariableDoesNotExist()
+    {
+        this.testSubject.GetListEnvironmentVariable("NonExistentVar", ",").Should().BeEmpty();
+    }
+
+    [TestMethod]
+    public void GetListEnvironmentVariable_emptyListIfEmptyVar()
+    {
+        var key = "foo";
+        Environment.SetEnvironmentVariable(key, string.Empty);
+        var result = this.testSubject.GetListEnvironmentVariable(key, ",");
+        result.Should().NotBeNull();
+        result.Should().BeEmpty();
+        Environment.SetEnvironmentVariable(key, null);
+    }
+
+    [TestMethod]
+    public void GetListEnvironmentVariable_singleItem()
+    {
+        var key = "foo";
+        Environment.SetEnvironmentVariable(key, "bar");
+        var result = this.testSubject.GetListEnvironmentVariable(key, ",");
+        result.Should().ContainSingle();
+        result.Should().Contain("bar");
+        Environment.SetEnvironmentVariable(key, null);
+    }
+
+    [TestMethod]
+    public void GetListEnvironmentVariable_multipleItems()
+    {
+        var key = "foo";
+        Environment.SetEnvironmentVariable(key, "bar,baz,qux");
+        var result = this.testSubject.GetListEnvironmentVariable(key, ",");
+        result.Should().HaveCount(3);
+        result.Should().Contain("bar");
+        result.Should().Contain("baz");
+        result.Should().Contain("qux");
+        Environment.SetEnvironmentVariable(key, null);
+    }
 }