Add Pip installation report experimental detector (#1129)

* Add PipReport experimental detector

* Don't use primary constructor

* Fix CI break

* Address PR comments

* Update src/Microsoft.ComponentDetection.Detectors/pip/PipReportUtilities.cs

Co-authored-by: Jamie Magee <jamagee@microsoft.com>

* Update src/Microsoft.ComponentDetection.Detectors/pip/PipReportComponentDetector.cs

Co-authored-by: Jamie Magee <jamagee@microsoft.com>

* Log cmd failure

---------

Co-authored-by: Coby Allred <coallred@microsoft.com>
Co-authored-by: Jamie Magee <jamagee@microsoft.com>

src/Microsoft.ComponentDetection.Common/FileUtilityService.cs

@@ -1,6 +1,7 @@
 namespace Microsoft.ComponentDetection.Common;
 
 using System.IO;
+using System.Threading.Tasks;
 using Microsoft.ComponentDetection.Contracts;
 
 /// <inheritdoc />
@@ -18,6 +19,12 @@ public string ReadAllText(FileInfo file)
         return File.ReadAllText(file.FullName);
     }
 
+    /// <inheritdoc />
+    public async Task<string> ReadAllTextAsync(FileInfo file)
+    {
+        return await File.ReadAllTextAsync(file.FullName);
+    }
+
     /// <inheritdoc />
     public bool Exists(string fileName)
     {

src/Microsoft.ComponentDetection.Common/PathUtilityService.cs

@@ -1,4 +1,4 @@
-namespace Microsoft.ComponentDetection.Common;
+namespace Microsoft.ComponentDetection.Common;
 
 using System;
 using System.IO;
@@ -75,4 +75,7 @@ public string ResolvePhysicalPath(string path)
     }
 
     private string ResolvePathFromInfo(FileSystemInfo info) => info.LinkTarget ?? info.FullName;
+
+    public string NormalizePath(string path) =>
+        path?.Replace('\\', Path.DirectorySeparatorChar).Replace('/', Path.DirectorySeparatorChar);
 }

src/Microsoft.ComponentDetection.Common/Telemetry/Records/FailedParsingFileRecord.cs

@@ -0,0 +1,14 @@
+namespace Microsoft.ComponentDetection.Common.Telemetry.Records;
+
+public class FailedParsingFileRecord : BaseDetectionTelemetryRecord
+{
+    public override string RecordName => "FailedParsingFile";
+
+    public string DetectorId { get; set; }
+
+    public string FilePath { get; set; }
+
+    public string ExceptionMessage { get; set; }
+
+    public string StackTrace { get; set; }
+}

src/Microsoft.ComponentDetection.Common/Telemetry/Records/PipReportFailureTelemetryRecord.cs

@@ -0,0 +1,10 @@
+namespace Microsoft.ComponentDetection.Common.Telemetry.Records;
+
+public class PipReportFailureTelemetryRecord : BaseDetectionTelemetryRecord
+{
+    public override string RecordName => "PipReportFailure";
+
+    public int ExitCode { get; set; }
+
+    public string StdErr { get; set; }
+}

src/Microsoft.ComponentDetection.Common/Telemetry/Records/PipReportVersionTelemetryRecord.cs

@@ -0,0 +1,10 @@
+namespace Microsoft.ComponentDetection.Common.Telemetry.Records;
+
+public class PipReportVersionTelemetryRecord : BaseDetectionTelemetryRecord
+{
+    public override string RecordName => "PipReportVersion";
+
+    public string Version { get; set; }
+
+    public string MaxVersion { get; set; }
+}

src/Microsoft.ComponentDetection.Contracts/IFileUtilityService.cs

@@ -1,6 +1,7 @@
-namespace Microsoft.ComponentDetection.Contracts;
+namespace Microsoft.ComponentDetection.Contracts;
 
 using System.IO;
+using System.Threading.Tasks;
 
 /// <summary>
 /// Wraps some common file operations for easier testability. This interface is *only used by the command line driven app*.
@@ -21,6 +22,13 @@ public interface IFileUtilityService
     /// <returns>Returns a string of the file contents.</returns>
     string ReadAllText(FileInfo file);
 
+    /// <summary>
+    /// Returns the contents of the file.
+    /// </summary>
+    /// <param name="file">File to read.</param>
+    /// <returns>Returns a string of the file contents.</returns>
+    Task<string> ReadAllTextAsync(FileInfo file);
+
     /// <summary>
     /// Returns true if the file exists.
     /// </summary>

src/Microsoft.ComponentDetection.Contracts/IPathUtilityService.cs

@@ -1,4 +1,4 @@
-namespace Microsoft.ComponentDetection.Contracts;
+namespace Microsoft.ComponentDetection.Contracts;
 
 /// <summary>
 /// Wraps some common folder operations, shared across command line app and service.
@@ -34,4 +34,11 @@ public interface IPathUtilityService
     /// <param name="fileName">File name without directory.</param>
     /// <returns>Returns true if file name matches a pattern, otherwise false.</returns>
     bool MatchesPattern(string searchPattern, string fileName);
+
+    /// <summary>
+    /// Normalizes the path to the system based separator.
+    /// </summary>
+    /// <param name="path">the path.</param>
+    /// <returns>normalized path.</returns>
+    string NormalizePath(string path);
 }

src/Microsoft.ComponentDetection.Detectors/pip/Contracts/IPipCommandService.cs

@@ -0,0 +1,30 @@
+namespace Microsoft.ComponentDetection.Detectors.Pip;
+
+using System;
+using System.IO;
+using System.Threading.Tasks;
+
+public interface IPipCommandService
+{
+    /// <summary>
+    /// Checks the existence of pip.
+    /// </summary>
+    /// <param name="pipPath">Optional override of the pip.exe absolute path.</param>
+    /// <returns>True if pip is found on the OS PATH.</returns>
+    Task<bool> PipExistsAsync(string pipPath = null);
+
+    /// <summary>
+    /// Retrieves the version of pip from the given path. PythonVersion allows for loose version strings such as "1".
+    /// </summary>
+    /// <param name="pipPath">Optional override of the pip.exe absolute path.</param>
+    /// <returns>Version of pip.</returns>
+    Task<Version> GetPipVersionAsync(string pipPath = null);
+
+    /// <summary>
+    /// Generates a pip installation report for a given setup.py or requirements.txt file.
+    /// </summary>
+    /// <param name="path">Path of the Python requirements file.</param>
+    /// <param name="pipExePath">Optional override of the pip.exe absolute path.</param>
+    /// <returns>See https://pip.pypa.io/en/stable/reference/installation-report/#specification.</returns>
+    Task<(PipInstallationReport Report, FileInfo ReportFile)> GenerateInstallationReportAsync(string path, string pipExePath = null);
+}

src/Microsoft.ComponentDetection.Detectors/pip/Contracts/PipInstallationMetadata.cs

@@ -0,0 +1,79 @@
+namespace Microsoft.ComponentDetection.Detectors.Pip;
+
+using Newtonsoft.Json;
+
+/// <summary>
+/// Metadata for a pip component being installed. See https://packaging.python.org/en/latest/specifications/core-metadata/.
+/// Some fields are not collected here because they are not needed for dependency graph construction.
+/// </summary>
+public sealed record PipInstallationMetadata
+{
+    /// <summary>
+    /// Version of the file format; legal values are "1.0", "1.1", "1.2", "2.1", "2.2", and "2.3"
+    /// as of May 2024.
+    /// </summary>
+    [JsonProperty("metadata_version")]
+    public string MetadataVersion { get; set; }
+
+    /// <summary>
+    /// The name of the distribution.
+    /// </summary>
+    [JsonProperty("name")]
+    public string Name { get; set; }
+
+    /// <summary>
+    /// A string containing the distribution’s version number.
+    /// </summary>
+    [JsonProperty("version")]
+    public string Version { get; set; }
+
+    /// <summary>
+    /// Each entry contains a string naming some other distutils project required by this distribution.
+    /// See https://peps.python.org/pep-0508/ for the format of the strings.
+    /// </summary>
+    [JsonProperty("requires_dist")]
+    public string[] RequiresDist { get; set; }
+
+    /// <summary>
+    /// URL for the distribution’s home page.
+    /// </summary>
+    [JsonProperty("home_page")]
+    public string HomePage { get; set; }
+
+    /// <summary>
+    /// Maintainer’s name at a minimum; additional contact information may be provided.
+    /// </summary>
+    [JsonProperty("maintainer")]
+    public string Maintainer { get; set; }
+
+    /// <summary>
+    /// Maintainer’s e-mail address. It can contain a name and e-mail address in the legal forms for a RFC-822 From: header.
+    /// </summary>
+    [JsonProperty("maintainer_email")]
+    public string MaintainerEmail { get; set; }
+
+    /// <summary>
+    /// Author’s name at a minimum; additional contact information may be provided.
+    /// </summary>
+    [JsonProperty("author")]
+    public string Author { get; set; }
+
+    /// <summary>
+    /// Author’s e-mail address. It can contain a name and e-mail address in the legal forms for a RFC-822 From: header.
+    /// </summary>
+    [JsonProperty("author_email")]
+    public string AuthorEmail { get; set; }
+
+    /// <summary>
+    /// Text indicating the license covering the distribution.
+    /// </summary>
+    [JsonProperty("license")]
+    public string License { get; set; }
+
+    /// <summary>
+    /// Each entry is a string giving a single classification value for the distribution.
+    /// Classifiers are described in PEP 301 https://peps.python.org/pep-0301/.
+    /// </summary>
+    [JsonProperty("classifier")]
+    public string[] Classifier { get; set; }
+}

src/Microsoft.ComponentDetection.Detectors/pip/Contracts/PipInstallationReport.cs

@@ -0,0 +1,34 @@
+namespace Microsoft.ComponentDetection.Detectors.Pip;
+
+using System.Collections.Generic;
+using Newtonsoft.Json;
+
+/// <summary>
+/// See https://pip.pypa.io/en/stable/reference/installation-report/#specification.
+/// </summary>
+public sealed record PipInstallationReport
+{
+    /// <summary>
+    /// Version of the installation report specification. Currently 1, but will be incremented if the format changes.
+    /// </summary>
+    [JsonProperty("version")]
+    public string Version { get; set; }
+
+    /// <summary>
+    /// Version of pip used to produce the report.
+    /// </summary>
+    [JsonProperty("pip_version")]
+    public string PipVersion { get; set; }
+
+    /// <summary>
+    /// Distribution packages (to be) installed.
+    /// </summary>
+    [JsonProperty("install")]
+    public PipInstallationReportItem[] InstallItems { get; set; }
+
+    /// <summary>
+    /// Environment metadata for the report. See https://peps.python.org/pep-0508/#environment-markers.
+    /// </summary>
+    [JsonProperty("environment")]
+    public IDictionary<string, string> Environment { get; set; }
+}

src/Microsoft.ComponentDetection.Detectors/pip/Contracts/PipInstallationReportItem.cs

@@ -0,0 +1,45 @@
+namespace Microsoft.ComponentDetection.Detectors.Pip;
+
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+public sealed record PipInstallationReportItem
+{
+    /// <summary>
+    /// The metadata of the distribution.
+    /// </summary>
+    [JsonProperty("metadata")]
+    public PipInstallationMetadata Metadata { get; set; }
+
+    /// <summary>
+    /// true if the requirement was provided as, or constrained to, a direct URL reference. false if the requirements was provided as a name and version specifier.
+    /// </summary>
+    [JsonProperty("is_direct")]
+    public bool IsDirect { get; set; }
+
+    /// <summary>
+    /// true if the requirement was yanked from the index, but was still selected by pip conform.
+    /// </summary>
+    [JsonProperty("is_yanked")]
+    public bool IsYanked { get; set; }
+
+    /// <summary>
+    /// true if the requirement was explicitly provided by the user, either directly via
+    /// a command line argument or indirectly via a requirements file. false if the requirement
+    /// was installed as a dependency of another requirement.
+    /// </summary>
+    [JsonProperty("requested")]
+    public bool Requested { get; set; }
+
+    /// <summary>
+    /// See https://packaging.python.org/en/latest/specifications/direct-url-data-structure/.
+    /// </summary>
+    [JsonProperty("download_info")]
+    public JObject DownloadInfo { get; set; }
+
+    /// <summary>
+    /// Extras requested by the user.
+    /// </summary>
+    [JsonProperty("requested_extras")]
+    public JObject RequestedExtras { get; set; }
+}

src/Microsoft.ComponentDetection.Detectors/pip/Contracts/PipReportGraphNode.cs

@@ -0,0 +1,25 @@
+namespace Microsoft.ComponentDetection.Detectors.Pip;
+
+using System.Collections.Generic;
+using Microsoft.ComponentDetection.Contracts.TypedComponent;
+
+/// <summary>
+/// Internal state used by PipReportDetector to hold intermediate structure info until the final
+/// combination of dependencies and relationships is determined and can be returned.
+/// </summary>
+public sealed record PipReportGraphNode
+{
+    public PipReportGraphNode(PipComponent value, bool requested)
+    {
+        this.Value = value;
+        this.Requested = requested;
+    }
+
+    public PipComponent Value { get; set; }
+
+    public List<PipReportGraphNode> Children { get; } = new List<PipReportGraphNode>();
+
+    public List<PipReportGraphNode> Parents { get; } = new List<PipReportGraphNode>();
+
+    public bool Requested { get; set; }
+}