diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/PnpmYamlVersion.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/PnpmYaml.cs similarity index 58% rename from src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/PnpmYamlVersion.cs rename to src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/PnpmYaml.cs index 2ceb4ffc3..8ce9c4e56 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/PnpmYamlVersion.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/PnpmYaml.cs @@ -2,7 +2,10 @@ namespace Microsoft.ComponentDetection.Detectors.Pnpm; using YamlDotNet.Serialization; -public class PnpmYamlVersion +/// +/// Base class for all Pnpm lockfiles. Used for parsing the lockfile version. +/// +public class PnpmYaml { [YamlMember(Alias = "lockfileVersion")] public string LockfileVersion { get; set; } diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/PnpmYamlV5.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/PnpmYamlV5.cs index 47f031a9b..d9fbe9a6f 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/PnpmYamlV5.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/PnpmYamlV5.cs @@ -6,14 +6,11 @@ namespace Microsoft.ComponentDetection.Detectors.Pnpm; /// /// Format for a Pnpm lock file version 5 as defined in https://github.com/pnpm/spec/blob/master/lockfile/5.md. /// -public class PnpmYamlV5 +public class PnpmYamlV5 : PnpmYaml { [YamlMember(Alias = "dependencies")] public Dictionary Dependencies { get; set; } [YamlMember(Alias = "packages")] public Dictionary Packages { get; set; } - - [YamlMember(Alias = "lockfileVersion")] - public string LockfileVersion { get; set; } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/PnpmHasDependenciesV6.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V6/PnpmHasDependenciesV6.cs similarity index 91% rename from src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/PnpmHasDependenciesV6.cs rename to src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V6/PnpmHasDependenciesV6.cs index e04135aa2..c1fd6891d 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/PnpmHasDependenciesV6.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V6/PnpmHasDependenciesV6.cs @@ -3,7 +3,7 @@ namespace Microsoft.ComponentDetection.Detectors.Pnpm; using System.Collections.Generic; using YamlDotNet.Serialization; -public class PnpmHasDependenciesV6 +public class PnpmHasDependenciesV6 : PnpmYaml { [YamlMember(Alias = "dependencies")] public Dictionary Dependencies { get; set; } diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/PnpmYamlV6.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V6/PnpmYamlV6.cs similarity index 89% rename from src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/PnpmYamlV6.cs rename to src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V6/PnpmYamlV6.cs index d9ca7bbf4..f71768081 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/PnpmYamlV6.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V6/PnpmYamlV6.cs @@ -17,7 +17,4 @@ public class PnpmYamlV6 : PnpmHasDependenciesV6 [YamlMember(Alias = "packages")] public Dictionary Packages { get; set; } - - [YamlMember(Alias = "lockfileVersion")] - public string LockfileVersion { get; set; } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/PnpmYamlV6Dependency.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V6/PnpmYamlV6Dependency.cs similarity index 100% rename from src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/PnpmYamlV6Dependency.cs rename to src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V6/PnpmYamlV6Dependency.cs diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V9/PnpmHasDependenciesV9.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V9/PnpmHasDependenciesV9.cs new file mode 100644 index 000000000..ebb91a305 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V9/PnpmHasDependenciesV9.cs @@ -0,0 +1,16 @@ +namespace Microsoft.ComponentDetection.Detectors.Pnpm; + +using System.Collections.Generic; +using YamlDotNet.Serialization; + +public class PnpmHasDependenciesV9 : PnpmYaml +{ + [YamlMember(Alias = "dependencies")] + public Dictionary Dependencies { get; set; } + + [YamlMember(Alias = "devDependencies")] + public Dictionary DevDependencies { get; set; } + + [YamlMember(Alias = "optionalDependencies")] + public Dictionary OptionalDependencies { get; set; } +} diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V9/PnpmYamlV9.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V9/PnpmYamlV9.cs new file mode 100644 index 000000000..44e077ee2 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V9/PnpmYamlV9.cs @@ -0,0 +1,21 @@ +namespace Microsoft.ComponentDetection.Detectors.Pnpm; + +using System.Collections.Generic; +using YamlDotNet.Serialization; + +/// +/// There is still no official docs for the new v9 lock if format, so these parsing contracts are empirically based. +/// Issue tracking v9 specs: https://github.com/pnpm/spec/issues/6 +/// Format should eventually get updated here: https://github.com/pnpm/spec/blob/master/lockfile/6.0.md. +/// +public class PnpmYamlV9 : PnpmHasDependenciesV9 +{ + [YamlMember(Alias = "importers")] + public Dictionary Importers { get; set; } + + [YamlMember(Alias = "packages")] + public Dictionary Packages { get; set; } + + [YamlMember(Alias = "snapshots")] + public Dictionary Snapshots { get; set; } +} diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V9/PnpmYamlV9Dependency.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V9/PnpmYamlV9Dependency.cs new file mode 100644 index 000000000..eb7c35e07 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V9/PnpmYamlV9Dependency.cs @@ -0,0 +1,9 @@ +namespace Microsoft.ComponentDetection.Detectors.Pnpm; + +using YamlDotNet.Serialization; + +public class PnpmYamlV9Dependency +{ + [YamlMember(Alias = "version")] + public string Version { get; set; } +} diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/IPnpmDetector.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/IPnpmDetector.cs index 4c2695dc3..7557c76b3 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pnpm/IPnpmDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pnpm/IPnpmDetector.cs @@ -14,3 +14,13 @@ public interface IPnpmDetector /// Component recorder to which to write the dependency graph. public void RecordDependencyGraphFromFile(string yamlFileContent, ISingleFileComponentRecorder singleFileComponentRecorder); } + +/// +/// Constants used in Pnpm Detectors. +/// +public static class PnpmConstants +{ + public const string PnpmFileDependencyPath = "file:"; + + public const string PnpmLinkDependencyPath = "link:"; +} diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmParsingUtilitiesBase.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmParsingUtilitiesBase.cs new file mode 100644 index 000000000..fbda4d291 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmParsingUtilitiesBase.cs @@ -0,0 +1,52 @@ +namespace Microsoft.ComponentDetection.Detectors.Pnpm; + +using System; +using System.Collections.Generic; +using System.IO; +using Microsoft.ComponentDetection.Contracts; +using YamlDotNet.Serialization; + +public abstract class PnpmParsingUtilitiesBase +where T : PnpmYaml +{ + public T DeserializePnpmYamlFile(string fileContent) + { + var deserializer = new DeserializerBuilder() + .IgnoreUnmatchedProperties() + .Build(); + return deserializer.Deserialize(new StringReader(fileContent)); + } + + public virtual bool IsPnpmPackageDevDependency(Package pnpmPackage) + { + ArgumentNullException.ThrowIfNull(pnpmPackage); + + return string.Equals(bool.TrueString, pnpmPackage.Dev, StringComparison.InvariantCultureIgnoreCase); + } + + public bool IsLocalDependency(KeyValuePair dependency) + { + // Local dependencies are dependencies that live in the file system + // this requires an extra parsing that is not supported yet + return dependency.Key.StartsWith(PnpmConstants.PnpmFileDependencyPath) || dependency.Value.StartsWith(PnpmConstants.PnpmFileDependencyPath) || dependency.Value.StartsWith(PnpmConstants.PnpmLinkDependencyPath); + } + + /// + /// Parse a pnpm path of the form "/package-name/version". + /// + /// a pnpm path of the form "/package-name/version". + /// Data parsed from path. + public abstract DetectedComponent CreateDetectedComponentFromPnpmPath(string pnpmPackagePath); + + public virtual string ReconstructPnpmDependencyPath(string dependencyName, string dependencyVersion) + { + if (dependencyVersion.StartsWith('/')) + { + return dependencyVersion; + } + else + { + return $"/{dependencyName}@{dependencyVersion}"; + } + } +} diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmParsingUtilitiesFactory.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmParsingUtilitiesFactory.cs new file mode 100644 index 000000000..4e0c015bf --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmParsingUtilitiesFactory.cs @@ -0,0 +1,27 @@ +namespace Microsoft.ComponentDetection.Detectors.Pnpm; + +using System.IO; +using YamlDotNet.Serialization; + +public static class PnpmParsingUtilitiesFactory +{ + public static PnpmParsingUtilitiesBase Create() + where T : PnpmYaml + { + return typeof(T).Name switch + { + nameof(PnpmYamlV5) => new PnpmV5ParsingUtilities(), + nameof(PnpmYamlV6) => new PnpmV6ParsingUtilities(), + nameof(PnpmYamlV9) => new PnpmV9ParsingUtilities(), + _ => new PnpmV5ParsingUtilities(), + }; + } + + public static string DeserializePnpmYamlFileVersion(string fileContent) + { + var deserializer = new DeserializerBuilder() + .IgnoreUnmatchedProperties() + .Build(); + return deserializer.Deserialize(new StringReader(fileContent))?.LockfileVersion; + } +} diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmV5ParsingUtilities.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmV5ParsingUtilities.cs new file mode 100644 index 000000000..02015c288 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmV5ParsingUtilities.cs @@ -0,0 +1,58 @@ +namespace Microsoft.ComponentDetection.Detectors.Pnpm; + +using System.Linq; +using global::NuGet.Versioning; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.TypedComponent; + +public class PnpmV5ParsingUtilities : PnpmParsingUtilitiesBase +where T : PnpmYaml +{ + public override DetectedComponent CreateDetectedComponentFromPnpmPath(string pnpmPackagePath) + { + var (parentName, parentVersion) = this.ExtractNameAndVersionFromPnpmPackagePath(pnpmPackagePath); + return new DetectedComponent(new NpmComponent(parentName, parentVersion)); + } + + private (string Name, string Version) ExtractNameAndVersionFromPnpmPackagePath(string pnpmPackagePath) + { + var pnpmComponentDefSections = pnpmPackagePath.Trim('/').Split('/'); + (var packageVersion, var indexVersionIsAt) = this.GetPackageVersion(pnpmComponentDefSections); + if (indexVersionIsAt == -1) + { + // No version = not expected input + return (null, null); + } + + var normalizedPackageName = string.Join("/", pnpmComponentDefSections.Take(indexVersionIsAt).ToArray()); + return (normalizedPackageName, packageVersion); + } + + private (string PackageVersion, int VersionIndex) GetPackageVersion(string[] pnpmComponentDefSections) + { + var indexVersionIsAt = -1; + var packageVersion = string.Empty; + var lastIndex = pnpmComponentDefSections.Length - 1; + + // get version from packages with format /mute-stream/0.0.6 + if (SemanticVersion.TryParse(pnpmComponentDefSections[lastIndex], out var _)) + { + return (pnpmComponentDefSections[lastIndex], lastIndex); + } + + // get version from packages with format /@babel/helper-compilation-targets/7.10.4_@babel+core@7.10.5 + var lastComponentSplit = pnpmComponentDefSections[lastIndex].Split("_"); + if (SemanticVersion.TryParse(lastComponentSplit[0], out var _)) + { + return (lastComponentSplit[0], lastIndex); + } + + // get version from packages with format /sinon-chai/2.8.0/chai@3.5.0+sinon@1.17.7 + if (SemanticVersion.TryParse(pnpmComponentDefSections[lastIndex - 1], out var _)) + { + return (pnpmComponentDefSections[lastIndex - 1], lastIndex - 1); + } + + return (packageVersion, indexVersionIsAt); + } +} diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmV6ParsingUtilities.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmV6ParsingUtilities.cs new file mode 100644 index 000000000..cd83e408d --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmV6ParsingUtilities.cs @@ -0,0 +1,42 @@ +namespace Microsoft.ComponentDetection.Detectors.Pnpm; + +using System; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.TypedComponent; + +public class PnpmV6ParsingUtilities : PnpmParsingUtilitiesBase +where T : PnpmYaml +{ + public override DetectedComponent CreateDetectedComponentFromPnpmPath(string pnpmPackagePath) + { + /* + * The format is documented at https://github.com/pnpm/spec/blob/master/dependency-path.md. + * At the writing it does not seem to reflect changes which were made in lock file format v6: + * See https://github.com/pnpm/spec/issues/5. + */ + + // Strip parenthesized suffices from package. These hold peed dep related information that is unneeded here. + // An example of a dependency path with these: /webpack-cli@4.10.0(webpack-bundle-analyzer@4.10.1)(webpack-dev-server@4.6.0)(webpack@5.89.0) + var fullPackageNameAndVersion = pnpmPackagePath.Split("(")[0]; + + var packageNameParts = fullPackageNameAndVersion.Split("@"); + + // If package name contains `@` this will reconstruct it: + var fullPackageName = string.Join("@", packageNameParts[..^1]); + + // Version is section after last `@`. + var packageVersion = packageNameParts[^1]; + + // Check for leading `/` from pnpm. + if (!fullPackageName.StartsWith('/')) + { + throw new FormatException("Found pnpm dependency path not starting with `/`. This case is currently unhandled."); + } + + // Strip leading `/`. + // It is unclear if real packages could have a name starting with `/`, so avoid `TrimStart` that just in case. + var normalizedPackageName = fullPackageName[1..]; + + return new DetectedComponent(new NpmComponent(normalizedPackageName, packageVersion)); + } +} diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmV9ParsingUtilities.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmV9ParsingUtilities.cs new file mode 100644 index 000000000..3270fdc0e --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmV9ParsingUtilities.cs @@ -0,0 +1,54 @@ +namespace Microsoft.ComponentDetection.Detectors.Pnpm; + +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.TypedComponent; + +public class PnpmV9ParsingUtilities : PnpmParsingUtilitiesBase +where T : PnpmYaml +{ + public override DetectedComponent CreateDetectedComponentFromPnpmPath(string pnpmPackagePath) + { + /* + * The format is documented at https://github.com/pnpm/spec/blob/master/dependency-path.md. + * At the writing it does not seem to reflect changes which were made in lock file format v9: + * See https://github.com/pnpm/spec/issues/5. + * In general, the spec sheet for the v9 lockfile is not published, so parsing of this lockfile was emperically determined. + * see https://github.com/pnpm/spec/issues/6 + */ + + // Strip parenthesized suffices from package. These hold peed dep related information that is unneeded here. + // An example of a dependency path with these: /webpack-cli@4.10.0(webpack-bundle-analyzer@4.10.1)(webpack-dev-server@4.6.0)(webpack@5.89.0) + var fullPackageNameAndVersion = pnpmPackagePath.Split("(")[0]; + + var packageNameParts = fullPackageNameAndVersion.Split("@"); + + // If package name contains `@` this will reconstruct it: + var fullPackageName = string.Join("@", packageNameParts[..^1]); + + // Version is section after last `@`. + var packageVersion = packageNameParts[^1]; + + return new DetectedComponent(new NpmComponent(fullPackageName, packageVersion)); + } + + /// + /// Combine the information from a dependency edge in the dependency graph encoded in the ymal file into a full pnpm dependency path. + /// + /// The name of the dependency, as used as as the dictionary key in the yaml file when referring to the dependency. + /// The final resolved version of the package for this dependency edge. + /// This includes details like which version of specific dependencies were specified as peer dependencies. + /// In some edge cases, such as aliased packages, this version may be an absolute dependency path, but the leading slash has been removed. + /// leaving the "dependencyName" unused, which is checked by whether there is an @ in the version. + /// A pnpm dependency path for the specified version of the named package. + public override string ReconstructPnpmDependencyPath(string dependencyName, string dependencyVersion) + { + if (dependencyVersion.StartsWith('/') || dependencyVersion.Split("(")[0].Contains('@')) + { + return dependencyVersion; + } + else + { + return $"{dependencyName}@{dependencyVersion}"; + } + } +} diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm5Detector.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm5Detector.cs index b91403ffe..46273a00e 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm5Detector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm5Detector.cs @@ -7,21 +7,22 @@ namespace Microsoft.ComponentDetection.Detectors.Pnpm; public class Pnpm5Detector : IPnpmDetector { public const string MajorVersion = "5"; + private readonly PnpmParsingUtilitiesBase pnpmParsingUtilities = PnpmParsingUtilitiesFactory.Create(); public void RecordDependencyGraphFromFile(string yamlFileContent, ISingleFileComponentRecorder singleFileComponentRecorder) { - var yaml = PnpmParsingUtilities.DeserializePnpmYamlV5File(yamlFileContent); + var yaml = this.pnpmParsingUtilities.DeserializePnpmYamlFile(yamlFileContent); - foreach (var packageKeyValue in yaml.Packages ?? Enumerable.Empty>()) + foreach (var packageKeyValue in yaml?.Packages ?? Enumerable.Empty>()) { // Ignore file: as these are local packages. - if (packageKeyValue.Key.StartsWith("file:")) + if (packageKeyValue.Key.StartsWith(PnpmConstants.PnpmFileDependencyPath)) { continue; } - var parentDetectedComponent = PnpmParsingUtilities.CreateDetectedComponentFromPnpmPathV5(pnpmPackagePath: packageKeyValue.Key); - var isDevDependency = packageKeyValue.Value != null && PnpmParsingUtilities.IsPnpmPackageDevDependency(packageKeyValue.Value); + var parentDetectedComponent = this.pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath(pnpmPackagePath: packageKeyValue.Key); + var isDevDependency = packageKeyValue.Value != null && this.pnpmParsingUtilities.IsPnpmPackageDevDependency(packageKeyValue.Value); singleFileComponentRecorder.RegisterUsage(parentDetectedComponent, isDevelopmentDependency: isDevDependency); parentDetectedComponent = singleFileComponentRecorder.GetComponent(parentDetectedComponent.Component.Id); @@ -30,13 +31,13 @@ public void RecordDependencyGraphFromFile(string yamlFileContent, ISingleFileCom foreach (var dependency in packageKeyValue.Value.Dependencies) { // Ignore local packages. - if (PnpmParsingUtilities.IsLocalDependency(dependency)) + if (this.pnpmParsingUtilities.IsLocalDependency(dependency)) { continue; } - var childDetectedComponent = PnpmParsingUtilities.CreateDetectedComponentFromPnpmPathV5( - pnpmPackagePath: PnpmParsingUtilities.CreatePnpmPackagePathFromDependencyV5(dependency.Key, dependency.Value)); + var childDetectedComponent = this.pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath( + pnpmPackagePath: this.CreatePnpmPackagePathFromDependency(dependency.Key, dependency.Value)); // Older code used the root's dev dependency value. We're leaving this null until we do a second pass to look at each components' top level referrers. singleFileComponentRecorder.RegisterUsage(childDetectedComponent, parentComponentId: parentDetectedComponent.Component.Id, isDevelopmentDependency: null); @@ -55,4 +56,9 @@ public void RecordDependencyGraphFromFile(string yamlFileContent, ISingleFileCom } } } + + private string CreatePnpmPackagePathFromDependency(string dependencyName, string dependencyVersion) + { + return dependencyVersion.Contains('/') ? dependencyVersion : $"/{dependencyName}/{dependencyVersion}"; + } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm6Detector.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm6Detector.cs index 0a3e5a99e..98513fc6a 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm6Detector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm6Detector.cs @@ -7,10 +7,11 @@ namespace Microsoft.ComponentDetection.Detectors.Pnpm; public class Pnpm6Detector : IPnpmDetector { public const string MajorVersion = "6"; + private readonly PnpmParsingUtilitiesBase pnpmParsingUtilities = PnpmParsingUtilitiesFactory.Create(); public void RecordDependencyGraphFromFile(string yamlFileContent, ISingleFileComponentRecorder singleFileComponentRecorder) { - var yaml = PnpmParsingUtilities.DeserializePnpmYamlV6File(yamlFileContent); + var yaml = this.pnpmParsingUtilities.DeserializePnpmYamlFile(yamlFileContent); // There may be multiple instance of the same package (even at the same version) in pnpm differentiated by other aspects of the pnpm dependency path. // Therefor all DetectedComponents are tracked by the same full string pnpm uses, the pnpm dependency path, which is used as the key in this dictionary. @@ -24,19 +25,19 @@ public void RecordDependencyGraphFromFile(string yamlFileContent, ISingleFileCom // Ignore "file:" as these are local packages. // Such local packages should only be referenced at the top level (via ProcessDependencyList) which also skips them or from other local packages (which this skips). // There should be no cases where a non-local package references a local package, so skipping them here should not result in failed lookups below when adding all the graph references. - if (pnpmDependencyPath.StartsWith("file:")) + if (pnpmDependencyPath.StartsWith(PnpmConstants.PnpmFileDependencyPath)) { continue; } - var parentDetectedComponent = PnpmParsingUtilities.CreateDetectedComponentFromPnpmPathV6(pnpmDependencyPath: pnpmDependencyPath); + var parentDetectedComponent = this.pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath(pnpmPackagePath: pnpmDependencyPath); components.Add(pnpmDependencyPath, (parentDetectedComponent, package)); // Register the component. // It should get registered again with with additional information (what depended on it) later, // but registering it now ensures nothing is missed due to a limitation in dependency traversal // like skipping local dependencies which might have transitively depended on this. - singleFileComponentRecorder.RegisterUsage(parentDetectedComponent, isDevelopmentDependency: PnpmParsingUtilities.IsPnpmPackageDevDependency(package)); + singleFileComponentRecorder.RegisterUsage(parentDetectedComponent, isDevelopmentDependency: this.pnpmParsingUtilities.IsPnpmPackageDevDependency(package)); } // Now that the `components` dictionary is populated, make a second pass registering all the dependency edges in the graph. @@ -44,7 +45,7 @@ public void RecordDependencyGraphFromFile(string yamlFileContent, ISingleFileCom { foreach (var (name, version) in package.Dependencies ?? Enumerable.Empty>()) { - var pnpmDependencyPath = PnpmParsingUtilities.ReconstructPnpmDependencyPathV6(name, version); + var pnpmDependencyPath = this.pnpmParsingUtilities.ReconstructPnpmDependencyPath(name, version); // If this lookup fails, then pnpmDependencyPath was either parsed incorrectly or constructed incorrectly. var (referenced, _) = components[pnpmDependencyPath]; @@ -77,17 +78,17 @@ private void ProcessDependencyList(ISingleFileComponentRecorder singleFileCompon foreach (var (name, dep) in dependencies ?? Enumerable.Empty>()) { // Ignore "file:" and "link:" as these are local packages. - if (dep.Version.StartsWith("link:") || dep.Version.StartsWith("file:")) + if (dep.Version.StartsWith(PnpmConstants.PnpmLinkDependencyPath) || dep.Version.StartsWith(PnpmConstants.PnpmFileDependencyPath)) { continue; } - var pnpmDependencyPath = PnpmParsingUtilities.ReconstructPnpmDependencyPathV6(name, dep.Version); + var pnpmDependencyPath = this.pnpmParsingUtilities.ReconstructPnpmDependencyPath(name, dep.Version); var (component, package) = components[pnpmDependencyPath]; // Determine isDevelopmentDependency using metadata on package from pnpm rather than from which dependency list this package is under. // This ensures that dependencies which are a direct dev dependency and an indirect non-dev dependency get listed as non-dev. - var isDevelopmentDependency = PnpmParsingUtilities.IsPnpmPackageDevDependency(package); + var isDevelopmentDependency = this.pnpmParsingUtilities.IsPnpmPackageDevDependency(package); singleFileComponentRecorder.RegisterUsage(component, isExplicitReferencedDependency: true, isDevelopmentDependency: isDevelopmentDependency); } diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm9Detector.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm9Detector.cs new file mode 100644 index 000000000..82e1093a8 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm9Detector.cs @@ -0,0 +1,114 @@ +namespace Microsoft.ComponentDetection.Detectors.Pnpm; + +using System.Collections.Generic; +using System.Linq; +using Microsoft.ComponentDetection.Contracts; + +/// +/// There is still no official docs for the new v9 lock format, so these parsing contracts are empirically based. +/// Issue tracking v9 specs: https://github.com/pnpm/spec/issues/6 +/// Format should eventually get updated here: https://github.com/pnpm/spec/blob/master/lockfile/6.0.md. +/// +public class Pnpm9Detector : IPnpmDetector +{ + public const string MajorVersion = "9"; + private readonly PnpmParsingUtilitiesBase pnpmParsingUtilities = PnpmParsingUtilitiesFactory.Create(); + + public void RecordDependencyGraphFromFile(string yamlFileContent, ISingleFileComponentRecorder singleFileComponentRecorder) + { + var yaml = this.pnpmParsingUtilities.DeserializePnpmYamlFile(yamlFileContent); + + // There may be multiple instance of the same package (even at the same version) in pnpm differentiated by other aspects of the pnpm dependency path. + // Therefor all DetectedComponents are tracked by the same full string pnpm uses, the pnpm dependency path, which is used as the key in this dictionary. + // Some documentation about pnpm dependency paths can be found at https://github.com/pnpm/spec/blob/master/dependency-path.md. + var components = new Dictionary(); + + // Create a component for every package referenced in the lock file. + // This includes all directly and transitively referenced dependencies. + foreach (var (pnpmDependencyPath, package) in yaml.Snapshots ?? Enumerable.Empty>()) + { + // Ignore "file:" as these are local packages. + // Such local packages should only be referenced at the top level (via ProcessDependencyList) which also skips them or from other local packages (which this skips). + // There should be no cases where a non-local package references a local package, so skipping them here should not result in failed lookups below when adding all the graph references. + if (pnpmDependencyPath.StartsWith(PnpmConstants.PnpmFileDependencyPath)) + { + continue; + } + + var dependencyPath = pnpmDependencyPath; + if (pnpmDependencyPath.StartsWith('/')) + { + dependencyPath = pnpmDependencyPath[1..]; + } + + var parentDetectedComponent = this.pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath(pnpmPackagePath: dependencyPath); + components.Add(dependencyPath, (parentDetectedComponent, package)); + + // Register the component. + // It should get registered again with with additional information (what depended on it) later, + // but registering it now ensures nothing is missed due to a limitation in dependency traversal + // like skipping local dependencies which might have transitively depended on this. + singleFileComponentRecorder.RegisterUsage(parentDetectedComponent); + } + + // now that the components dictionary is populated, add direct dependencies of the current file/project setting isExplicitReferencedDependency to true + // during this step, recursively processes any indirect dependencies + foreach (var (_, package) in yaml.Importers ?? Enumerable.Empty>()) + { + this.ProcessDependencySets(singleFileComponentRecorder, components, package); + } + } + + private void ProcessDependencySets(ISingleFileComponentRecorder singleFileComponentRecorder, Dictionary components, PnpmHasDependenciesV9 item) + { + this.ProcessDependencyList(singleFileComponentRecorder, components, item.Dependencies, isDevelopmentDependency: false); + this.ProcessDependencyList(singleFileComponentRecorder, components, item.DevDependencies, isDevelopmentDependency: true); + this.ProcessDependencyList(singleFileComponentRecorder, components, item.OptionalDependencies, false); + } + + private void ProcessDependencyList(ISingleFileComponentRecorder singleFileComponentRecorder, Dictionary components, Dictionary dependencies, bool isDevelopmentDependency) + { + foreach (var (name, dep) in dependencies ?? Enumerable.Empty>()) + { + // Ignore "file:" and "link:" as these are local packages. + if (dep.Version.StartsWith(PnpmConstants.PnpmLinkDependencyPath) || dep.Version.StartsWith(PnpmConstants.PnpmFileDependencyPath)) + { + continue; + } + + var pnpmDependencyPath = this.pnpmParsingUtilities.ReconstructPnpmDependencyPath(name, dep.Version); + var (component, package) = components[pnpmDependencyPath]; + + // Lockfile v9 apparently removed the tagging of dev dependencies in the lockfile, so we revert to using the dependency tree to establish dev dependency state. + // At this point, the root dependencies are marked according to which dependency group they are declared in the lockfile itself. + singleFileComponentRecorder.RegisterUsage(component, isExplicitReferencedDependency: true, isDevelopmentDependency: isDevelopmentDependency); + var seenDependencies = new HashSet(); + this.ProcessIndirectDependencies(singleFileComponentRecorder, components, component.Component.Id, package.Dependencies, isDevelopmentDependency, seenDependencies); + } + } + + private void ProcessIndirectDependencies(ISingleFileComponentRecorder singleFileComponentRecorder, Dictionary components, string parentComponentId, Dictionary dependencies, bool isDevDependency, HashSet seenDependencies) + { + // Now that the `components` dictionary is populated, make another pass of all components, registering all the dependency edges in the graph. + foreach (var (name, version) in dependencies ?? Enumerable.Empty>()) + { + // Ignore "file:" and "link:" as these are local packages. + if (version.StartsWith(PnpmConstants.PnpmLinkDependencyPath) || version.StartsWith(PnpmConstants.PnpmFileDependencyPath)) + { + continue; + } + + var pnpmDependencyPath = this.pnpmParsingUtilities.ReconstructPnpmDependencyPath(name, version); + if (seenDependencies.Contains(pnpmDependencyPath)) + { + continue; + } + + // If this lookup fails, then pnpmDependencyPath was either parsed incorrectly or constructed incorrectly. + var (component, package) = components[pnpmDependencyPath]; + singleFileComponentRecorder.RegisterUsage(component, parentComponentId: parentComponentId, isExplicitReferencedDependency: false, isDevelopmentDependency: isDevDependency); + seenDependencies.Add(pnpmDependencyPath); + this.ProcessIndirectDependencies(singleFileComponentRecorder, components, component.Component.Id, package.Dependencies, isDevDependency, seenDependencies); + } + } +} diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/PnpmComponentDetectorFactory.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/PnpmComponentDetectorFactory.cs index a4cd8824b..2bc3e7880 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pnpm/PnpmComponentDetectorFactory.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pnpm/PnpmComponentDetectorFactory.cs @@ -21,7 +21,7 @@ public class PnpmComponentDetectorFactory : FileComponentDetector /// /// The maximum version of the report specification that this detector can handle. /// - private static readonly Version MaxLockfileVersion = new(6, 0); + private static readonly Version MaxLockfileVersion = new(9, 0); public PnpmComponentDetectorFactory( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, @@ -41,7 +41,7 @@ public PnpmComponentDetectorFactory( public override IEnumerable SupportedComponentTypes { get; } = [ComponentType.Npm]; - public override int Version { get; } = 6; + public override int Version { get; } = 7; public override bool NeedsAutomaticRootDependencyCalculation => true; @@ -98,7 +98,7 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID private IPnpmDetector GetPnpmComponentDetector(string fileContent, out string detectedVersion) { - detectedVersion = PnpmParsingUtilities.DeserializePnpmYamlFileVersion(fileContent); + detectedVersion = PnpmParsingUtilitiesFactory.DeserializePnpmYamlFileVersion(fileContent); this.RecordLockfileVersion(detectedVersion); var majorVersion = detectedVersion?.Split(".")[0]; return majorVersion switch @@ -110,6 +110,7 @@ private IPnpmDetector GetPnpmComponentDetector(string fileContent, out string de null => new Pnpm5Detector(), Pnpm5Detector.MajorVersion => new Pnpm5Detector(), Pnpm6Detector.MajorVersion => new Pnpm6Detector(), + Pnpm9Detector.MajorVersion => new Pnpm9Detector(), _ => null, }; } diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/PnpmParsingUtilities.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/PnpmParsingUtilities.cs deleted file mode 100644 index fe107d9ec..000000000 --- a/src/Microsoft.ComponentDetection.Detectors/pnpm/PnpmParsingUtilities.cs +++ /dev/null @@ -1,167 +0,0 @@ -namespace Microsoft.ComponentDetection.Detectors.Pnpm; - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using global::NuGet.Versioning; -using Microsoft.ComponentDetection.Contracts; -using Microsoft.ComponentDetection.Contracts.TypedComponent; -using YamlDotNet.Serialization; - -public static class PnpmParsingUtilities -{ - public static string DeserializePnpmYamlFileVersion(string fileContent) - { - var deserializer = new DeserializerBuilder() - .IgnoreUnmatchedProperties() - .Build(); - return deserializer.Deserialize(new StringReader(fileContent)).LockfileVersion; - } - - public static bool IsPnpmPackageDevDependency(Package pnpmPackage) - { - ArgumentNullException.ThrowIfNull(pnpmPackage); - - return string.Equals(bool.TrueString, pnpmPackage.Dev, StringComparison.InvariantCultureIgnoreCase); - } - - public static PnpmYamlV5 DeserializePnpmYamlV5File(string fileContent) - { - var deserializer = new DeserializerBuilder() - .IgnoreUnmatchedProperties() - .Build(); - return deserializer.Deserialize(new StringReader(fileContent)); - } - - public static PnpmYamlV6 DeserializePnpmYamlV6File(string fileContent) - { - var deserializer = new DeserializerBuilder() - .IgnoreUnmatchedProperties() - .Build(); - return deserializer.Deserialize(new StringReader(fileContent)); - } - - public static bool IsLocalDependency(KeyValuePair dependency) - { - // Local dependencies are dependencies that live in the file system - // this requires an extra parsing that is not supported yet - return dependency.Key.StartsWith("file:") || dependency.Value.StartsWith("file:") || dependency.Value.StartsWith("link:"); - } - - /// - /// Parse a pnpm path of the form "/package-name/version". - /// - /// a pnpm path of the form "/package-name/version". - /// Data parsed from path. - public static DetectedComponent CreateDetectedComponentFromPnpmPathV5(string pnpmPackagePath) - { - var (parentName, parentVersion) = ExtractNameAndVersionFromPnpmPackagePathV5(pnpmPackagePath); - return new DetectedComponent(new NpmComponent(parentName, parentVersion)); - } - - public static string CreatePnpmPackagePathFromDependencyV5(string dependencyName, string dependencyVersion) - { - return dependencyVersion.Contains('/') ? dependencyVersion : $"/{dependencyName}/{dependencyVersion}"; - } - - private static (string Name, string Version) ExtractNameAndVersionFromPnpmPackagePathV5(string pnpmPackagePath) - { - var pnpmComponentDefSections = pnpmPackagePath.Trim('/').Split('/'); - (var packageVersion, var indexVersionIsAt) = GetPackageVersionV5(pnpmComponentDefSections); - if (indexVersionIsAt == -1) - { - // No version = not expected input - return (null, null); - } - - var normalizedPackageName = string.Join("/", pnpmComponentDefSections.Take(indexVersionIsAt).ToArray()); - return (normalizedPackageName, packageVersion); - } - - private static (string PackageVersion, int VersionIndex) GetPackageVersionV5(string[] pnpmComponentDefSections) - { - var indexVersionIsAt = -1; - var packageVersion = string.Empty; - var lastIndex = pnpmComponentDefSections.Length - 1; - - // get version from packages with format /mute-stream/0.0.6 - if (SemanticVersion.TryParse(pnpmComponentDefSections[lastIndex], out var _)) - { - return (pnpmComponentDefSections[lastIndex], lastIndex); - } - - // get version from packages with format /@babel/helper-compilation-targets/7.10.4_@babel+core@7.10.5 - var lastComponentSplit = pnpmComponentDefSections[lastIndex].Split("_"); - if (SemanticVersion.TryParse(lastComponentSplit[0], out var _)) - { - return (lastComponentSplit[0], lastIndex); - } - - // get version from packages with format /sinon-chai/2.8.0/chai@3.5.0+sinon@1.17.7 - if (SemanticVersion.TryParse(pnpmComponentDefSections[lastIndex - 1], out var _)) - { - return (pnpmComponentDefSections[lastIndex - 1], lastIndex - 1); - } - - return (packageVersion, indexVersionIsAt); - } - - /// - /// Parse a pnpm dependency path. - /// - /// A pnpm dependency path of the form "/@optional-scope/package-name@version(optional-ignored-data)(optional-ignored-data)". - /// Data parsed from path. - public static DetectedComponent CreateDetectedComponentFromPnpmPathV6(string pnpmDependencyPath) - { - /* - * The format is documented at https://github.com/pnpm/spec/blob/master/dependency-path.md. - * At the writing it does not seem to reflect changes which were made in lock file format v6: - * See https://github.com/pnpm/spec/issues/5. - */ - - // Strip parenthesized suffices from package. These hold peed dep related information that is unneeded here. - // An example of a dependency path with these: /webpack-cli@4.10.0(webpack-bundle-analyzer@4.10.1)(webpack-dev-server@4.6.0)(webpack@5.89.0) - var fullPackageNameAndVersion = pnpmDependencyPath.Split("(")[0]; - - var packageNameParts = fullPackageNameAndVersion.Split("@"); - - // If package name contains `@` this will reconstruct it: - var fullPackageName = string.Join("@", packageNameParts[..^1]); - - // Version is section after last `@`. - var packageVersion = packageNameParts[^1]; - - // Check for leading `/` from pnpm. - if (!fullPackageName.StartsWith('/')) - { - throw new FormatException("Found pnpm dependency path not starting with `/`. This case is currently unhandled."); - } - - // Strip leading `/`. - // It is unclear if real packages could have a name starting with `/`, so avoid `TrimStart` that just in case. - var normalizedPackageName = fullPackageName[1..]; - - return new DetectedComponent(new NpmComponent(normalizedPackageName, packageVersion)); - } - - /// - /// Combine the information from a dependency edge in the dependency graph encoded in the ymal file into a full pnpm dependency path. - /// - /// The name of the dependency, as used as as the dictionary key in the yaml file when referring to the dependency. - /// The final resolved version of the package for this dependency edge. - /// This includes details like which version of specific dependencies were specified as peer dependencies. - /// In some edge cases, such as aliased packages, this version may be an absolute dependency path (starts with a slash) leaving the "dependencyName" unused. - /// A pnpm dependency path for the specified version of the named package. - public static string ReconstructPnpmDependencyPathV6(string dependencyName, string dependencyVersion) - { - if (dependencyVersion.StartsWith('/')) - { - return dependencyVersion; - } - else - { - return $"/{dependencyName}@{dependencyVersion}"; - } - } -} diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Models/ExperimentDiff.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Models/ExperimentDiff.cs index e0fe879eb..16d9eafc0 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Models/ExperimentDiff.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Models/ExperimentDiff.cs @@ -40,10 +40,23 @@ public ExperimentDiff( var experimentDetectorList = new List(); // Need performance benchmark to see if this is worth parallelization - foreach (var id in newComponentDictionary.Keys.Intersect(oldComponentDictionary.Keys)) + foreach (var newComponentPair in newComponentDictionary) { - var oldComponent = oldComponentDictionary[id]; - var newComponent = newComponentDictionary[id]; + var newComponent = newComponentPair.Value; + var id = newComponentPair.Key; + + if (!oldComponentDictionary.TryGetValue(id, out var oldComponent)) + { + if (newComponent.DevelopmentDependency) + { + developmentDependencyChanges.Add(new DevelopmentDependencyChange( + id, + oldValue: false, + newValue: newComponent.DevelopmentDependency)); + } + + continue; + } if (oldComponent.DevelopmentDependency != newComponent.DevelopmentDependency) { diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/PnpmDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/PnpmDetectorTests.cs index 6f94f34aa..e4c60c077 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/PnpmDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/PnpmDetectorTests.cs @@ -332,10 +332,12 @@ public async Task TestPnpmDetector_DependencyGraphIsCreatedAsync() scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); componentRecorder.GetDetectedComponents().Should().HaveCount(4); - var queryStringComponentId = PnpmParsingUtilities.CreateDetectedComponentFromPnpmPathV5("/query-string/4.3.4").Component.Id; - var objectAssignComponentId = PnpmParsingUtilities.CreateDetectedComponentFromPnpmPathV5("/object-assign/4.1.1").Component.Id; - var strictUriComponentId = PnpmParsingUtilities.CreateDetectedComponentFromPnpmPathV5("/strict-uri-encode/1.1.0").Component.Id; - var testComponentId = PnpmParsingUtilities.CreateDetectedComponentFromPnpmPathV5("/test/1.0.0").Component.Id; + var pnpmParsingUtilities = PnpmParsingUtilitiesFactory.Create(); + + var queryStringComponentId = pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath("/query-string/4.3.4").Component.Id; + var objectAssignComponentId = pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath("/object-assign/4.1.1").Component.Id; + var strictUriComponentId = pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath("/strict-uri-encode/1.1.0").Component.Id; + var testComponentId = pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath("/test/1.0.0").Component.Id; var dependencyGraph = componentRecorder.GetDependencyGraphsByLocation().Values.First(); @@ -380,8 +382,10 @@ public async Task TestPnpmDetector_DependenciesRefeToLocalPaths_DependenciesAreI scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); componentRecorder.GetDetectedComponents().Should().HaveCount(2, "Components that comes from a file (file:* or link:*) should be ignored."); - var queryStringComponentId = PnpmParsingUtilities.CreateDetectedComponentFromPnpmPathV5("/query-string/4.3.4").Component.Id; - var nthcheck = PnpmParsingUtilities.CreateDetectedComponentFromPnpmPathV5("/nth-check/2.0.0").Component.Id; + var pnpmParsingUtilities = PnpmParsingUtilitiesFactory.Create(); + + var queryStringComponentId = pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath("/query-string/4.3.4").Component.Id; + var nthcheck = pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath("/nth-check/2.0.0").Component.Id; var dependencyGraph = componentRecorder.GetDependencyGraphsByLocation().Values.First(); @@ -600,4 +604,208 @@ public async Task TestPnpmDetector_V6_BadLockVersion_EmptyAsync() var detectedComponents = componentRecorder.GetDetectedComponents(); detectedComponents.Should().BeEmpty(); } + + [TestMethod] + public async Task TestPnpmDetector_V9_GoodLockVersion_ParsesDependencies() + { + var yamlFile = @" +lockfileVersion: '9.0' +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false +importers: + .: + dependencies: + sampleDependency: + specifier: ^1.1.1 + version: 1.1.1 + devDependencies: + sampleDevDependency: + specifier: ^2.2.2 + version: 2.2.2 +packages: + sampleDependency@1.1.1: + resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} + sampleDevDependency@2.2.2: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + sampleIndirectDependency@3.3.3: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + +snapshots: + sampleDependency@1.1.1: + dependencies: + sampleIndirectDependency: 3.3.3 + sampleDevDependency@2.2.2: {} + sampleIndirectDependency@3.3.3: {} +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("pnpm-lock.yaml", yamlFile) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().HaveCount(3); + var npmComponents = detectedComponents.Select(x => new { Component = x.Component as NpmComponent, DetectedComponent = x }); + npmComponents.Should().Contain(x => x.Component.Name == "sampleDependency" && x.Component.Version == "1.1.1"); + npmComponents.Should().Contain(x => x.Component.Name == "sampleDevDependency" && x.Component.Version == "2.2.2"); + npmComponents.Should().Contain(x => x.Component.Name == "sampleIndirectDependency" && x.Component.Version == "3.3.3"); + + var noDevDependencyComponent = npmComponents.First(x => x.Component.Name == "sampleDependency"); + var devDependencyComponent = npmComponents.First(x => x.Component.Name == "sampleDevDependency"); + var indirectDependencyComponent = npmComponents.First(x => x.Component.Name == "sampleIndirectDependency"); + + componentRecorder.GetEffectiveDevDependencyValue(noDevDependencyComponent.Component.Id).Should().BeFalse(); + componentRecorder.GetEffectiveDevDependencyValue(devDependencyComponent.Component.Id).Should().BeTrue(); + componentRecorder.AssertAllExplicitlyReferencedComponents( + indirectDependencyComponent.Component.Id, + parentComponent => parentComponent.Name == "sampleDependency"); + } + + [TestMethod] + public async Task TestPnpmDetector_V9_GoodLockVersion_ParsesAliasedDependency() + { + var yamlFile = @" +lockfileVersion: '9.0' +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false +importers: + .: + dependencies: + aliasedSample: + specifier: npm:sampleDependency@1.1.1 + version: sampleDependency@1.1.1 +packages: + sampleDependency@1.1.1: + resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} + sampleIndirectDependency@3.3.3: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + +snapshots: + sampleDependency@1.1.1: + dependencies: + sampleIndirectDependency: 3.3.3 + sampleIndirectDependency@3.3.3: {} +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("pnpm-lock.yaml", yamlFile) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().HaveCount(2); + var npmComponents = detectedComponents.Select(x => new { Component = x.Component as NpmComponent, DetectedComponent = x }); + npmComponents.Should().Contain(x => x.Component.Name == "sampleDependency" && x.Component.Version == "1.1.1"); + npmComponents.Should().Contain(x => x.Component.Name == "sampleIndirectDependency" && x.Component.Version == "3.3.3"); + + var noDevDependencyComponent = npmComponents.First(x => x.Component.Name == "sampleDependency"); + var indirectDependencyComponent = npmComponents.First(x => x.Component.Name == "sampleIndirectDependency"); + + componentRecorder.GetEffectiveDevDependencyValue(noDevDependencyComponent.Component.Id).Should().BeFalse(); + componentRecorder.AssertAllExplicitlyReferencedComponents( + indirectDependencyComponent.Component.Id, + parentComponent => parentComponent.Name == "sampleDependency"); + } + + [TestMethod] + public async Task TestPnpmDetector_V9_GoodLockVersion_SkipsFileAndLinkDependencies() + { + var yamlFile = @" +lockfileVersion: '9.0' +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false +importers: + .: + dependencies: + sampleDependency: + specifier: ^1.1.1 + version: 1.1.1 + SampleFileDependency: + specifier: file://../sampleFile + version: link:../ + SampleLinkDependency: + specifier: workspace:* + version: link:SampleLinkDependency +packages: + sampleDependency@1.1.1: + resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} + sampleIndirectDependency2@2.2.2: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + sampleIndirectDependency@3.3.3: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + +snapshots: + sampleDependency@1.1.1: + dependencies: + sampleIndirectDependency: 3.3.3 + sampleIndirectDependency2: 2.2.2 + 'file://../sampleFile': 'link:../\\' + sampleIndirectDependency2@2.2.2: {} + sampleIndirectDependency@3.3.3: {} + 'file://../sampleFile@link:../': {} +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("pnpm-lock.yaml", yamlFile) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().HaveCount(3); + var npmComponents = detectedComponents.Select(x => new { Component = x.Component as NpmComponent, DetectedComponent = x }); + npmComponents.Should().Contain(x => x.Component.Name == "sampleDependency" && x.Component.Version == "1.1.1"); + npmComponents.Should().Contain(x => x.Component.Name == "sampleIndirectDependency2" && x.Component.Version == "2.2.2"); + npmComponents.Should().Contain(x => x.Component.Name == "sampleIndirectDependency" && x.Component.Version == "3.3.3"); + + var noDevDependencyComponent = npmComponents.First(x => x.Component.Name == "sampleDependency"); + var indirectDependencyComponent2 = npmComponents.First(x => x.Component.Name == "sampleIndirectDependency2"); + var indirectDependencyComponent = npmComponents.First(x => x.Component.Name == "sampleIndirectDependency"); + + componentRecorder.GetEffectiveDevDependencyValue(noDevDependencyComponent.Component.Id).Should().BeFalse(); + componentRecorder.GetEffectiveDevDependencyValue(indirectDependencyComponent2.Component.Id).Should().BeFalse(); + componentRecorder.GetEffectiveDevDependencyValue(indirectDependencyComponent.Component.Id).Should().BeFalse(); + componentRecorder.AssertAllExplicitlyReferencedComponents( + indirectDependencyComponent.Component.Id, + parentComponent => parentComponent.Name == "sampleDependency"); + componentRecorder.AssertAllExplicitlyReferencedComponents( + indirectDependencyComponent2.Component.Id, + parentComponent => parentComponent.Name == "sampleDependency"); + } + + [TestMethod] + public async Task TestPnpmDetector_V9_GoodLockVersion_MissingSnapshotsSuccess() + { + var yamlFile = @" +lockfileVersion: '9.0' +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false +importers: + .: + dependencies: + SampleLinkDependency: + specifier: workspace:* + version: link:SampleLinkDependency +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("pnpm-lock.yaml", yamlFile) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().BeEmpty(); + } } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/PnpmParsingUtilitiesTest.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/PnpmParsingUtilitiesTest.cs index ba508d5a5..6f87db271 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/PnpmParsingUtilitiesTest.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/PnpmParsingUtilitiesTest.cs @@ -35,10 +35,10 @@ public void DeserializePnpmYamlFileV3() registry: 'https://test/registry' shrinkwrapMinorVersion: 7 shrinkwrapVersion: 3"; - - var version = PnpmParsingUtilities.DeserializePnpmYamlFileVersion(yamlFile); + var pnpmParsingUtilities = PnpmParsingUtilitiesFactory.Create(); + var version = PnpmParsingUtilitiesFactory.DeserializePnpmYamlFileVersion(yamlFile); version.Should().BeNull(); // Versions older than 5 report null as they don't use the same version field. - var parsedYaml = PnpmParsingUtilities.DeserializePnpmYamlV5File(yamlFile); + var parsedYaml = pnpmParsingUtilities.DeserializePnpmYamlFile(yamlFile); parsedYaml.Packages.Should().HaveCount(2); parsedYaml.Packages.Should().ContainKey("/query-string/4.3.4"); @@ -58,19 +58,20 @@ public void DeserializePnpmYamlFileV3() [TestMethod] public void CreateDetectedComponentFromPnpmPathV5() { - var detectedComponent1 = PnpmParsingUtilities.CreateDetectedComponentFromPnpmPathV5("/@ms/items-view/0.128.9/react-dom@15.6.2+react@15.6.2"); + var pnpmParsingUtilities = PnpmParsingUtilitiesFactory.Create(); + var detectedComponent1 = pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath("/@ms/items-view/0.128.9/react-dom@15.6.2+react@15.6.2"); detectedComponent1.Should().NotBeNull(); detectedComponent1.Component.Should().NotBeNull(); ((NpmComponent)detectedComponent1.Component).Name.Should().BeEquivalentTo("@ms/items-view"); ((NpmComponent)detectedComponent1.Component).Version.Should().BeEquivalentTo("0.128.9"); - var detectedComponent2 = PnpmParsingUtilities.CreateDetectedComponentFromPnpmPathV5("/@babel/helper-compilation-targets/7.10.4_@babel+core@7.10.5"); + var detectedComponent2 = pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath("/@babel/helper-compilation-targets/7.10.4_@babel+core@7.10.5"); detectedComponent2.Should().NotBeNull(); detectedComponent2.Component.Should().NotBeNull(); ((NpmComponent)detectedComponent2.Component).Name.Should().BeEquivalentTo("@babel/helper-compilation-targets"); ((NpmComponent)detectedComponent2.Component).Version.Should().BeEquivalentTo("7.10.4"); - var detectedComponent3 = PnpmParsingUtilities.CreateDetectedComponentFromPnpmPathV5("/query-string/4.3.4"); + var detectedComponent3 = pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath("/query-string/4.3.4"); detectedComponent3.Should().NotBeNull(); detectedComponent3.Component.Should().NotBeNull(); ((NpmComponent)detectedComponent3.Component).Name.Should().BeEquivalentTo("query-string"); @@ -85,24 +86,26 @@ public void IsPnpmPackageDevDependency() Dev = "true", }; - PnpmParsingUtilities.IsPnpmPackageDevDependency(pnpmPackage).Should().BeTrue(); + var pnpmParsingUtilities = PnpmParsingUtilitiesFactory.Create(); + + pnpmParsingUtilities.IsPnpmPackageDevDependency(pnpmPackage).Should().BeTrue(); pnpmPackage.Dev = "TRUE"; - PnpmParsingUtilities.IsPnpmPackageDevDependency(pnpmPackage).Should().BeTrue(); + pnpmParsingUtilities.IsPnpmPackageDevDependency(pnpmPackage).Should().BeTrue(); pnpmPackage.Dev = "false"; - PnpmParsingUtilities.IsPnpmPackageDevDependency(pnpmPackage).Should().BeFalse(); + pnpmParsingUtilities.IsPnpmPackageDevDependency(pnpmPackage).Should().BeFalse(); pnpmPackage.Dev = "FALSE"; - PnpmParsingUtilities.IsPnpmPackageDevDependency(pnpmPackage).Should().BeFalse(); + pnpmParsingUtilities.IsPnpmPackageDevDependency(pnpmPackage).Should().BeFalse(); pnpmPackage.Dev = string.Empty; - PnpmParsingUtilities.IsPnpmPackageDevDependency(pnpmPackage).Should().BeFalse(); + pnpmParsingUtilities.IsPnpmPackageDevDependency(pnpmPackage).Should().BeFalse(); pnpmPackage.Dev = null; - PnpmParsingUtilities.IsPnpmPackageDevDependency(pnpmPackage).Should().BeFalse(); + pnpmParsingUtilities.IsPnpmPackageDevDependency(pnpmPackage).Should().BeFalse(); - Action action = () => PnpmParsingUtilities.IsPnpmPackageDevDependency(null); + Action action = () => pnpmParsingUtilities.IsPnpmPackageDevDependency(null); action.Should().Throw(); } @@ -123,10 +126,10 @@ public void DeserializePnpmYamlFileV6() /minimist@1.2.8: resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} dev: false"; - - var version = PnpmParsingUtilities.DeserializePnpmYamlFileVersion(yamlFile); + var pnpmParsingUtilities = PnpmParsingUtilitiesFactory.Create(); + var version = PnpmParsingUtilitiesFactory.DeserializePnpmYamlFileVersion(yamlFile); version.Should().Be("6.0"); - var parsedYaml = PnpmParsingUtilities.DeserializePnpmYamlV6File(yamlFile); + var parsedYaml = pnpmParsingUtilities.DeserializePnpmYamlFile(yamlFile); parsedYaml.Packages.Should().ContainSingle(); parsedYaml.Packages.Should().ContainKey("/minimist@1.2.8"); @@ -142,23 +145,25 @@ public void DeserializePnpmYamlFileV6() [TestMethod] public void CreateDetectedComponentFromPnpmPathV6() { + var pnpmParsingUtilities = PnpmParsingUtilitiesFactory.Create(); + // Simple case: no scope, simple version - var simple = PnpmParsingUtilities.CreateDetectedComponentFromPnpmPathV6("/sort-scripts@1.0.1"); + var simple = pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath("/sort-scripts@1.0.1"); ((NpmComponent)simple.Component).Name.Should().BeEquivalentTo("sort-scripts"); ((NpmComponent)simple.Component).Version.Should().BeEquivalentTo("1.0.1"); // With scope: - var scoped = PnpmParsingUtilities.CreateDetectedComponentFromPnpmPathV6("/@babel/eslint-parser@7.23.3"); + var scoped = pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath("/@babel/eslint-parser@7.23.3"); ((NpmComponent)scoped.Component).Name.Should().BeEquivalentTo("@babel/eslint-parser"); ((NpmComponent)scoped.Component).Version.Should().BeEquivalentTo("7.23.3"); // With peer deps: - var withPeerDeps = PnpmParsingUtilities.CreateDetectedComponentFromPnpmPathV6("/mocha-json-output-reporter@2.1.0(mocha@10.2.0)(moment@2.29.4)"); + var withPeerDeps = pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath("/mocha-json-output-reporter@2.1.0(mocha@10.2.0)(moment@2.29.4)"); ((NpmComponent)withPeerDeps.Component).Name.Should().BeEquivalentTo("mocha-json-output-reporter"); ((NpmComponent)withPeerDeps.Component).Version.Should().BeEquivalentTo("2.1.0"); // With everything: - var complex = PnpmParsingUtilities.CreateDetectedComponentFromPnpmPathV6("/@babel/eslint-parser@7.23.3(@babel/core@7.23.3)(eslint@8.55.0)"); + var complex = pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath("/@babel/eslint-parser@7.23.3(@babel/core@7.23.3)(eslint@8.55.0)"); ((NpmComponent)complex.Component).Name.Should().BeEquivalentTo("@babel/eslint-parser"); ((NpmComponent)complex.Component).Version.Should().BeEquivalentTo("7.23.3"); } @@ -166,16 +171,18 @@ public void CreateDetectedComponentFromPnpmPathV6() [TestMethod] public void ReconstructPnpmDependencyPathV6() { + var pnpmParsingUtilities = PnpmParsingUtilitiesFactory.Create(); + // Simple case: no scope, simple version - PnpmParsingUtilities.ReconstructPnpmDependencyPathV6("sort-scripts", "1.0.1").Should().BeEquivalentTo("/sort-scripts@1.0.1"); + pnpmParsingUtilities.ReconstructPnpmDependencyPath("sort-scripts", "1.0.1").Should().BeEquivalentTo("/sort-scripts@1.0.1"); // With scope: - PnpmParsingUtilities.ReconstructPnpmDependencyPathV6("@babel/eslint-parser", "7.23.3").Should().BeEquivalentTo("/@babel/eslint-parser@7.23.3"); + pnpmParsingUtilities.ReconstructPnpmDependencyPath("@babel/eslint-parser", "7.23.3").Should().BeEquivalentTo("/@babel/eslint-parser@7.23.3"); // With peer deps: - PnpmParsingUtilities.ReconstructPnpmDependencyPathV6("mocha-json-output-reporter", "2.1.0(mocha@10.2.0)(moment@2.29.4)").Should().BeEquivalentTo("/mocha-json-output-reporter@2.1.0(mocha@10.2.0)(moment@2.29.4)"); + pnpmParsingUtilities.ReconstructPnpmDependencyPath("mocha-json-output-reporter", "2.1.0(mocha@10.2.0)(moment@2.29.4)").Should().BeEquivalentTo("/mocha-json-output-reporter@2.1.0(mocha@10.2.0)(moment@2.29.4)"); // Absolute path: - PnpmParsingUtilities.ReconstructPnpmDependencyPathV6("events_pkg", "/events@3.3.0").Should().BeEquivalentTo("/events@3.3.0"); + pnpmParsingUtilities.ReconstructPnpmDependencyPath("events_pkg", "/events@3.3.0").Should().BeEquivalentTo("/events@3.3.0"); } } diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentDiffTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentDiffTests.cs index f1a1fffcd..d5d2c4dec 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentDiffTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentDiffTests.cs @@ -139,4 +139,27 @@ public void ExperimentDiff_MultipleIds_ShouldntThrow() action.Should().NotThrow(); } + + [TestMethod] + public void ExperimentDiff_DiffsAddedDevDependenciesMissingInControlGroup() + { + var componentA = ExperimentTestUtils.CreateRandomScannedComponent(); + componentA.IsDevelopmentDependency = true; + + var diff = new ExperimentDiff( + [], + [new ExperimentComponent(componentA)]); + + diff.DevelopmentDependencyChanges.Should().ContainSingle(); + + var change = diff.DevelopmentDependencyChanges.First(); + change.Id.Should().Be(componentA.Component.Id); + change.OldValue.Should().BeFalse(); + change.NewValue.Should().BeTrue(); + + diff.AddedIds.Should().BeEquivalentTo([componentA.Component.Id]); + diff.RemovedIds.Should().BeEmpty(); + diff.AddedRootIds.Should().BeEmpty(); + diff.RemovedRootIds.Should().BeEmpty(); + } }