From f337a54fc85b3bda11ad5c6431c2e571f94d842b Mon Sep 17 00:00:00 2001 From: James Oakley Date: Thu, 26 Oct 2023 12:55:55 -0400 Subject: [PATCH] Support development dependencies for the Gradle detector Lack of development dependency detection for Gradle is a problem for Android teams, especially in the context of Component Governance alerts. Unfortunately Gradle doesn't provide enough information to definitively identify dev dependencies in all cases, so manual configuration is required. This change adds dev dependency classification through two mechanisms 1. `buildscript-gradle.lockfile` and `settings-gradle.lockfile` contain only build-system dependencies, so always classify these as development dependencies. 2. Processing based on two new environment variables: `GRADLE_PROD_CONFIGURATIONS_REGEX` and `GRADLE_DEV_CONFIGURATIONS_REGEX`. Gradle lockfiles indicate which Gradle configuration(s) each dependency is required by. `GRADLE_PROD_CONFIGURATIONS_REGEX` allows specifying production configurations explicitly. All other configurations are considered development. Alternately, dev configurations may be specified in `GRADLE_DEV_CONFIGURATIONS_REGEX` and all others are considered production. --- docs/environment-variables.md | 12 ++ docs/feature-overview.md | 2 +- .../gradle/GradleComponentDetector.cs | 57 ++++++- .../GradleComponentDetectorTests.cs | 158 +++++++++++++++++- 4 files changed, 225 insertions(+), 4 deletions(-) diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 74f600e21..ac9c5119d 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -18,4 +18,16 @@ When set to any value, enables detector experiments, a feature to compare the re same ecosystem. The available experiments are found in the [`Experiments\Config`](../src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs) folder. +## `GRADLE_PROD_CONFIGURATIONS_REGEX` + +Enables dev-dependency categorization for the Gradle +detector. Dependencies connected to Gradle configurations NOT matching +the given regex are considered development dependencies. + +## `GRADLE_DEV_CONFIGURATIONS_REGEX` + +Enables dev-dependency categorization for the Gradle +detector. Dependencies connected to Gradle configurations matching +the given regex are considered development dependencies. + [1]: https://go.dev/ref/mod#go-mod-graph diff --git a/docs/feature-overview.md b/docs/feature-overview.md index a56b6c51b..aabee8bf1 100644 --- a/docs/feature-overview.md +++ b/docs/feature-overview.md @@ -5,7 +5,7 @@ | CocoaPods | | - | ❌ | - | | Conda (Python) | | - | ❌ | ✔ | | Linux (Debian, Alpine, Rhel, Centos, Fedora, Ubuntu)| | - | - | - | - | -| Gradle | | | ❌ | ❌ | +| Gradle | | | ✔ (requires env var configuration for full effect) | ❌ | | Go | Fallback
| | ❌ | ✔ (root idenditication only for fallback) | | Maven | | | ✔ (test dependency scope) | ✔ | | NPM | | - | ✔ (dev-dependencies in package.json, dev flag in package-lock.json) | ✔ | diff --git a/src/Microsoft.ComponentDetection.Detectors/gradle/GradleComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/gradle/GradleComponentDetector.cs index 397207a39..07ec9a05e 100644 --- a/src/Microsoft.ComponentDetection.Detectors/gradle/GradleComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/gradle/GradleComponentDetector.cs @@ -3,6 +3,7 @@ namespace Microsoft.ComponentDetection.Detectors.Gradle; using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text.RegularExpressions; using System.Threading.Tasks; using Microsoft.ComponentDetection.Contracts; @@ -12,8 +13,13 @@ namespace Microsoft.ComponentDetection.Detectors.Gradle; public class GradleComponentDetector : FileComponentDetector, IComponentDetector { + private const string ProdDepsEnvVar = "GRADLE_PROD_CONFIGURATIONS_REGEX"; + private const string DevDepsEnvVar = "GRADLE_DEV_CONFIGURATIONS_REGEX"; private static readonly Regex StartsWithLetterRegex = new Regex("^[A-Za-z]", RegexOptions.Compiled); + private readonly Regex prodDepsRegex; + private readonly Regex devDepsRegex; + public GradleComponentDetector( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, IObservableDirectoryWalkerFactory walkerFactory, @@ -22,6 +28,17 @@ public GradleComponentDetector( this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; this.Scanner = walkerFactory; this.Logger = logger; + var envVar = Environment.GetEnvironmentVariable(ProdDepsEnvVar); + if (!string.IsNullOrEmpty(envVar)) + { + this.prodDepsRegex = new Regex(envVar, RegexOptions.Compiled); + } + + envVar = Environment.GetEnvironmentVariable(DevDepsEnvVar); + if (!string.IsNullOrEmpty(envVar)) + { + this.devDepsRegex = new Regex(envVar, RegexOptions.Compiled); + } } public override string Id { get; } = "Gradle"; @@ -47,6 +64,9 @@ protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDiction private void ParseLockfile(ISingleFileComponentRecorder singleFileComponentRecorder, IComponentStream file) { + // Buildscript and Settings lockfiles are always development dependencies + var lockfileIsDevDependency = file.Location.EndsWith("buildscript-gradle.lockfile") || file.Location.EndsWith("settings-gradle.lockfile"); + string text; using (var reader = new StreamReader(file.Stream)) { @@ -68,7 +88,8 @@ private void ParseLockfile(ISingleFileComponentRecorder singleFileComponentRecor if (line.Split(":").Length == 3) { var detectedMavenComponent = new DetectedComponent(this.CreateMavenComponentFromFileLine(line)); - singleFileComponentRecorder.RegisterUsage(detectedMavenComponent); + var devDependency = lockfileIsDevDependency || this.IsDevDependencyByConfigurations(line); + singleFileComponentRecorder.RegisterUsage(detectedMavenComponent, isDevelopmentDependency: devDependency); } } } @@ -87,4 +108,38 @@ private MavenComponent CreateMavenComponentFromFileLine(string line) } private bool StartsWithLetter(string input) => StartsWithLetterRegex.IsMatch(input); + + private bool IsDevDependencyByConfigurations(string line) + { + if (this.prodDepsRegex == null && this.devDepsRegex == null) + { + return false; // no regexes configured to check against + } + + var equalsSeparatorIndex = line.IndexOf('='); + if (equalsSeparatorIndex == -1) + { + // We can't parse out the configuration. Maybe the project is using the one-lockfile-per-configuration format but + // this is deprecated in Gradle so we don't support it here, projects should upgrade to one-lockfile-per-project. + return false; + } + + var configurations = line[(equalsSeparatorIndex + 1)..].Split(","); + return configurations.Select(c => this.IsDevDependencyByConfigurationName(c)).All(dev => dev); + } + + private bool IsDevDependencyByConfigurationName(string configurationName) + { + if (this.devDepsRegex != null && this.devDepsRegex.IsMatch(configurationName)) + { + return true; + } + + if (this.prodDepsRegex != null && !this.prodDepsRegex.IsMatch(configurationName)) + { + return true; + } + + return false; + } } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/GradleComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/GradleComponentDetectorTests.cs index dc67b1c11..58a60575c 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/GradleComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/GradleComponentDetectorTests.cs @@ -1,5 +1,6 @@ -namespace Microsoft.ComponentDetection.Detectors.Tests; +namespace Microsoft.ComponentDetection.Detectors.Tests; +using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; @@ -159,7 +160,7 @@ public async Task TestGradleDetector_SameComponentDifferentLocations_DifferentLo componentRecorder.ForOneComponent(componentRecorder.GetDetectedComponents().First().Component.Id, x => { - Enumerable.Count(x.AllFileLocations).Should().Be(2); + Enumerable.Count(x.AllFileLocations).Should().Be(2); }); var dependencyGraphs = componentRecorder.GetDependencyGraphsByLocation(); @@ -216,4 +217,157 @@ four score and seven bugs ago component.Should().NotBeNull(); } } + + [TestMethod] + public async Task TestGradleDetector_DevDependenciesByLockfileNameAsync() + { + var regularLockfile = + @"org.springframework:spring-beans:5.0.5.RELEASE +org.springframework:spring-core:5.0.5.RELEASE"; + + var devLockfile1 = @"org.hamcrest:hamcrest-core:2.2 +org.springframework:spring-core:5.0.5.RELEASE"; + + var devLockfile2 = @"org.jacoco:org.jacoco.agent:0.8.8"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("settings-gradle.lockfile", devLockfile1) + .WithFile("buildscript-gradle.lockfile", devLockfile2) + .WithFile("gradle.lockfile", regularLockfile) + .ExecuteDetectorAsync(); + + Assert.AreEqual(ProcessingResultCode.Success, scanResult.ResultCode); + + var discoveredComponents = componentRecorder.GetDetectedComponents().Select(c => (MavenComponent)c.Component).OrderBy(c => c.ArtifactId).ToList(); + var dependencyGraphs = componentRecorder.GetDependencyGraphsByLocation(); + var gradleLockfileGraph = dependencyGraphs[dependencyGraphs.Keys.First(k => k.EndsWith("\\gradle.lockfile"))]; + var settingsGradleLockfileGraph = dependencyGraphs[dependencyGraphs.Keys.First(k => k.EndsWith("settings-gradle.lockfile"))]; + var buildscriptGradleLockfileGraph = dependencyGraphs[dependencyGraphs.Keys.First(k => k.EndsWith("buildscript-gradle.lockfile"))]; + + Assert.AreEqual(4, discoveredComponents.Count); + + // Dev dependency listed only in settings-gradle.lockfile + var component = discoveredComponents[0]; + Assert.AreEqual("org.hamcrest", component.GroupId); + Assert.AreEqual("hamcrest-core", component.ArtifactId); + Assert.IsTrue(settingsGradleLockfileGraph.IsDevelopmentDependency(component.Id)); + + // Dev dependency listed only in buildscript-gradle.lockfile + component = discoveredComponents[1]; + Assert.AreEqual("org.jacoco", component.GroupId); + Assert.AreEqual("org.jacoco.agent", component.ArtifactId); + Assert.IsTrue(buildscriptGradleLockfileGraph.IsDevelopmentDependency(component.Id)); + + // This should be purely a prod dependency, just a basic confidence test + component = discoveredComponents[2]; + Assert.AreEqual("org.springframework", component.GroupId); + Assert.AreEqual("spring-beans", component.ArtifactId); + Assert.IsFalse(gradleLockfileGraph.IsDevelopmentDependency(component.Id)); + + // This is listed as both a prod and a dev dependency in different files + component = discoveredComponents[3]; + Assert.AreEqual("org.springframework", component.GroupId); + Assert.AreEqual("spring-core", component.ArtifactId); + Assert.IsFalse(gradleLockfileGraph.IsDevelopmentDependency(component.Id)); + Assert.IsTrue(settingsGradleLockfileGraph.IsDevelopmentDependency(component.Id)); + } + + [TestMethod] + public async Task TestGradleDetector_DevDependenciesByDevEnvironmentAsync() + { + var lockfile = + @"org.springframework:spring-beans:5.0.5.RELEASE=assembleRelease +org.springframework:spring-core:5.0.5.RELEASE=assembleRelease,testDebugUnitTest +org.hamcrest:hamcrest-core:2.2=testDebugUnitTest"; + + Environment.SetEnvironmentVariable("GRADLE_DEV_CONFIGURATIONS_REGEX", ".*UnitTest"); + try + { + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("gradle.lockfile", lockfile) + .ExecuteDetectorAsync(); + + Assert.AreEqual(ProcessingResultCode.Success, scanResult.ResultCode); + + var discoveredComponents = componentRecorder.GetDetectedComponents().Select(c => (MavenComponent)c.Component).OrderBy(c => c.ArtifactId).ToList(); + var dependencyGraph = componentRecorder.GetDependencyGraphsByLocation().Values.First(); + + Assert.AreEqual(3, discoveredComponents.Count); + + var component = discoveredComponents[0]; + Assert.AreEqual("org.hamcrest", component.GroupId); + Assert.AreEqual("hamcrest-core", component.ArtifactId); + + // Purely a dev dependency, only present in a test configuration + Assert.IsTrue(dependencyGraph.IsDevelopmentDependency(component.Id)); + + component = discoveredComponents[1]; + Assert.AreEqual("org.springframework", component.GroupId); + Assert.AreEqual("spring-beans", component.ArtifactId); + + // Purely a prod dependency, only present in a prod configuration + Assert.IsFalse(dependencyGraph.IsDevelopmentDependency(component.Id)); + + component = discoveredComponents[2]; + Assert.AreEqual("org.springframework", component.GroupId); + Assert.AreEqual("spring-core", component.ArtifactId); + + // Present in both dev and prod configurations, prod should win + Assert.IsFalse(dependencyGraph.IsDevelopmentDependency(component.Id)); + } + finally + { + Environment.SetEnvironmentVariable("GRADLE_DEV_CONFIGURATIONS_REGEX", null); + } + } + + [TestMethod] + public async Task TestGradleDetector_DevDependenciesByProdEnvironmentAsync() + { + var lockfile = + @"org.springframework:spring-beans:5.0.5.RELEASE=assembleRelease +org.springframework:spring-core:5.0.5.RELEASE=assembleRelease,testDebugUnitTest +org.hamcrest:hamcrest-core:2.2=testDebugUnitTest"; + + // Only the specific configurations are prod. Everything else is a dev dependency + Environment.SetEnvironmentVariable("GRADLE_PROD_CONFIGURATIONS_REGEX", "assembleRelease"); + try + { + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("gradle.lockfile", lockfile) + .ExecuteDetectorAsync(); + + Assert.AreEqual(ProcessingResultCode.Success, scanResult.ResultCode); + + var discoveredComponents = componentRecorder.GetDetectedComponents().Select(c => (MavenComponent)c.Component).OrderBy(c => c.ArtifactId).ToList(); + var dependencyGraph = componentRecorder.GetDependencyGraphsByLocation().Values.First(); + + Assert.AreEqual(3, discoveredComponents.Count); + + var component = discoveredComponents[0]; + Assert.AreEqual("org.hamcrest", component.GroupId); + Assert.AreEqual("hamcrest-core", component.ArtifactId); + + // Purely a dev dependency, only present in a test configuration + Assert.IsTrue(dependencyGraph.IsDevelopmentDependency(component.Id)); + + component = discoveredComponents[1]; + Assert.AreEqual("org.springframework", component.GroupId); + Assert.AreEqual("spring-beans", component.ArtifactId); + + // Purely a prod dependency, only present in a prod configuration + Assert.IsFalse(dependencyGraph.IsDevelopmentDependency(component.Id)); + + component = discoveredComponents[2]; + Assert.AreEqual("org.springframework", component.GroupId); + Assert.AreEqual("spring-core", component.ArtifactId); + + // Present in both dev and prod configurations, prod should win + Assert.IsFalse(dependencyGraph.IsDevelopmentDependency(component.Id)); + } + finally + { + Environment.SetEnvironmentVariable("GRADLE_PROD_CONFIGURATIONS_REGEX", null); + } + } }