From 62c5fd207c957ed4989fb5230bee2859ed140537 Mon Sep 17 00:00:00 2001 From: Kevin Schneider Date: Thu, 29 Feb 2024 09:09:23 +0100 Subject: [PATCH] Refactor domain script into compiled project --- arc-validate-package-registry.sln | 7 + src/AVPRIndex/AVPRIndex.fsproj | 20 +++ src/AVPRIndex/AVPRRepo.fs | 42 +++++ src/AVPRIndex/Domain.fs | 150 ++++++++++++++++++ src/AVPRIndex/Frontmatter.fs | 55 +++++++ src/AVPRIndex/Globals.fs | 6 + src/AVPRIndex/Utils.fs | 12 ++ .../Data/DataInitializer.cs | 10 +- .../Data/arc-validate-package-index.json | 10 +- src/PackageRegistryService/Models/Author.cs | 13 -- .../Models/ValidationPackage.cs | 2 +- .../Models/ValidationPackageIndex.cs | 25 --- .../PackageRegistryService.csproj | 4 + .../Pages/Components/Package.cs | 1 + 14 files changed, 312 insertions(+), 45 deletions(-) create mode 100644 src/AVPRIndex/AVPRIndex.fsproj create mode 100644 src/AVPRIndex/AVPRRepo.fs create mode 100644 src/AVPRIndex/Domain.fs create mode 100644 src/AVPRIndex/Frontmatter.fs create mode 100644 src/AVPRIndex/Globals.fs create mode 100644 src/AVPRIndex/Utils.fs delete mode 100644 src/PackageRegistryService/Models/Author.cs delete mode 100644 src/PackageRegistryService/Models/ValidationPackageIndex.cs diff --git a/arc-validate-package-registry.sln b/arc-validate-package-registry.sln index 0c12ed4..d880fa0 100644 --- a/arc-validate-package-registry.sln +++ b/arc-validate-package-registry.sln @@ -48,6 +48,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "publish-workflow", "publish EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AVPRClient", "src\AVPRClient\AVPRClient.csproj", "{D1FABAC1-D0F2-4F6C-B975-236E9969FB38}" EndProject +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "AVPRIndex", "src\AVPRIndex\AVPRIndex.fsproj", "{78A87C32-8957-468A-AFA6-3DCE8F826500}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -70,6 +72,10 @@ Global {D1FABAC1-D0F2-4F6C-B975-236E9969FB38}.Debug|Any CPU.Build.0 = Debug|Any CPU {D1FABAC1-D0F2-4F6C-B975-236E9969FB38}.Release|Any CPU.ActiveCfg = Release|Any CPU {D1FABAC1-D0F2-4F6C-B975-236E9969FB38}.Release|Any CPU.Build.0 = Release|Any CPU + {78A87C32-8957-468A-AFA6-3DCE8F826500}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {78A87C32-8957-468A-AFA6-3DCE8F826500}.Debug|Any CPU.Build.0 = Debug|Any CPU + {78A87C32-8957-468A-AFA6-3DCE8F826500}.Release|Any CPU.ActiveCfg = Release|Any CPU + {78A87C32-8957-468A-AFA6-3DCE8F826500}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -78,6 +84,7 @@ Global {C4D0490E-F8AF-43E6-9A1F-3844292D6247} = {8061A76B-1B85-4649-A667-6E7979ABA1AE} {809F69D2-1740-425C-890A-566BF83993C6} = {412317F5-7FF4-4455-9C9D-CEAF7FE6E7B2} {D1FABAC1-D0F2-4F6C-B975-236E9969FB38} = {412317F5-7FF4-4455-9C9D-CEAF7FE6E7B2} + {78A87C32-8957-468A-AFA6-3DCE8F826500} = {412317F5-7FF4-4455-9C9D-CEAF7FE6E7B2} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {D95036E6-C4D5-4E93-8474-BA6F74965635} diff --git a/src/AVPRIndex/AVPRIndex.fsproj b/src/AVPRIndex/AVPRIndex.fsproj new file mode 100644 index 0000000..1c7b46b --- /dev/null +++ b/src/AVPRIndex/AVPRIndex.fsproj @@ -0,0 +1,20 @@ + + + + net8.0 + true + + + + + + + + + + + + + + + diff --git a/src/AVPRIndex/AVPRRepo.fs b/src/AVPRIndex/AVPRRepo.fs new file mode 100644 index 0000000..d654c33 --- /dev/null +++ b/src/AVPRIndex/AVPRRepo.fs @@ -0,0 +1,42 @@ +namespace AVPRIndex + +open System +open System.IO +open System.Text.Json +open Domain +open Globals +open Frontmatter + +type AVPRRepo = + + ///! Paths are relative to the root of the project, since the script is executed from the repo root in CI + /// Path is adjustable by passing `RepoRoot` + static member getStagedPackages(?RepoRoot: string) = + + let path = + defaultArg + (RepoRoot |> Option.map (fun p -> Path.Combine(p, STAGING_AREA_RELATIVE_PATH))) + STAGING_AREA_RELATIVE_PATH + + Directory.GetFiles(path, "*.fsx", SearchOption.AllDirectories) + |> Array.map (fun x -> x.Replace('\\',Path.DirectorySeparatorChar).Replace('/',Path.DirectorySeparatorChar)) + |> Array.map (fun p -> + ValidationPackageIndex.create( + repoPath = p.Replace(Path.DirectorySeparatorChar, '/'), // use front slash always here, otherwise the backslash will be escaped with another backslah on windows when writing the json + lastUpdated = Utils.truncateDateTime DateTimeOffset.Now // take local time with offset if file will be changed with this commit + ) + ) + + ///! Paths are relative to the root of the project, since the script is executed from the repo root in CI + /// Path is adjustable by passing `RepoRoot` + static member getIndexedPackages(?RepoRoot: string) = + + let path = + defaultArg + (RepoRoot |> Option.map (fun p -> Path.Combine(p, PACKAGE_INDEX_RELATIVE_PATH))) + PACKAGE_INDEX_RELATIVE_PATH + + path + |> File.ReadAllText + |> JsonSerializer.Deserialize + diff --git a/src/AVPRIndex/Domain.fs b/src/AVPRIndex/Domain.fs new file mode 100644 index 0000000..925a837 --- /dev/null +++ b/src/AVPRIndex/Domain.fs @@ -0,0 +1,150 @@ +namespace AVPRIndex + +open System +open System.IO +open System.Text.Json +open System.Security.Cryptography + +module Domain = + + let jsonSerializerOptions = JsonSerializerOptions(WriteIndented = true) + + type Author() = + member val FullName = "" with get,set + member val Email = "" with get,set + member val Affiliation = "" with get,set + member val AffiliationLink = "" with get,set + + override this.GetHashCode() = + hash ( + this.FullName, + this.Email, + this.Affiliation, + this.AffiliationLink + ) + + override this.Equals(other) = + match other with + | :? Author as a -> + ( + this.FullName, + this.Email, + this.Affiliation, + this.AffiliationLink + ) = ( + a.FullName, + a.Email, + a.Affiliation, + a.AffiliationLink + ) + | _ -> false + + type ValidationPackageMetadata() = + // mandatory fields + member val Name = "" with get,set + member val Description = "" with get,set + member val MajorVersion = 0 with get,set + member val MinorVersion = 0 with get,set + member val PatchVersion = 0 with get,set + // optional fields + member val Publish = false with get,set + member val Authors: Author [] = Array.empty with get,set + member val Tags: string [] = Array.empty with get,set + member val ReleaseNotes = "" with get,set + + override this.GetHashCode() = + hash ( + this.Name, + this.Description, + this.MajorVersion, + this.MinorVersion, + this.PatchVersion, + this.Publish, + this.Authors, + this.Tags, + this.ReleaseNotes + ) + + override this.Equals(other) = + match other with + | :? ValidationPackageMetadata as vpm -> + ( + this.Name, + this.Description, + this.MajorVersion, + this.MinorVersion, + this.PatchVersion, + this.Publish, + this.Authors, + this.Tags, + this.ReleaseNotes + ) = ( + vpm.Name, + vpm.Description, + vpm.MajorVersion, + vpm.MinorVersion, + vpm.PatchVersion, + vpm.Publish, + vpm.Authors, + vpm.Tags, + vpm.ReleaseNotes + ) + | _ -> false + + type ValidationPackageIndex = + { + RepoPath: string + FileName: string + LastUpdated: System.DateTimeOffset + ContentHash: string + Metadata: ValidationPackageMetadata + } with + static member create ( + repoPath: string, + fileName: string, + lastUpdated: System.DateTimeOffset, + contentHash: string, + metadata: ValidationPackageMetadata + ) = + { + RepoPath = repoPath + FileName = fileName + LastUpdated = lastUpdated + ContentHash = contentHash + Metadata = metadata + } + static member create ( + repoPath: string, + lastUpdated: System.DateTimeOffset, + metadata: ValidationPackageMetadata + ) = + + let md5 = MD5.Create() + + ValidationPackageIndex.create( + repoPath = repoPath, + fileName = Path.GetFileNameWithoutExtension(repoPath), + lastUpdated = lastUpdated, + contentHash = (md5.ComputeHash(File.ReadAllBytes(repoPath)) |> Convert.ToHexString), + metadata = metadata + ) + + /// returns true when the two packages will have the same stable identifier (consisting of name and semver from their metadata fields) + static member identityEquals (first: ValidationPackageIndex) (second: ValidationPackageIndex) = + first.Metadata.Name = second.Metadata.Name + && first.Metadata.MajorVersion = second.Metadata.MajorVersion + && first.Metadata.MinorVersion = second.Metadata.MinorVersion + && first.Metadata.PatchVersion = second.Metadata.PatchVersion + + /// returns true when the two packages have the same content hash + static member contentEquals (first: ValidationPackageIndex) (second: ValidationPackageIndex) = + first.ContentHash = second.ContentHash + + static member toJson (i: ValidationPackageIndex) = + JsonSerializer.Serialize(i, jsonSerializerOptions) + + static member printJson (i: ValidationPackageIndex) = + let json = ValidationPackageIndex.toJson i + printfn "" + printfn $"Indexed Package info:{System.Environment.NewLine}{json}" + printfn "" \ No newline at end of file diff --git a/src/AVPRIndex/Frontmatter.fs b/src/AVPRIndex/Frontmatter.fs new file mode 100644 index 0000000..900677e --- /dev/null +++ b/src/AVPRIndex/Frontmatter.fs @@ -0,0 +1,55 @@ +namespace AVPRIndex + + +open Domain +open System +open System.IO +open System.Security.Cryptography +open YamlDotNet.Serialization + +module Frontmatter = + + let frontMatterStart = $"(*{System.Environment.NewLine}---" + let frontMatterEnd = $"---{System.Environment.NewLine}*)" + + let yamlDeserializer = + DeserializerBuilder() + .WithNamingConvention(NamingConventions.PascalCaseNamingConvention.Instance) + .Build() + + type ValidationPackageMetadata with + + static member extractFromScript (scriptPath: string) = + let script = File.ReadAllText(scriptPath) + if script.StartsWith(frontMatterStart, StringComparison.Ordinal) && script.Contains(frontMatterEnd) then + let frontmatter = + script.Substring( + frontMatterStart.Length, + (script.IndexOf(frontMatterEnd, StringComparison.Ordinal) - frontMatterEnd.Length)) + try + let result = + yamlDeserializer.Deserialize(frontmatter) + result + with e as exn -> + printfn $"error parsing package metadata at {scriptPath}. Make sure that all required metadata tags are included." + ValidationPackageMetadata() + else + printfn $"script at {scriptPath} has no correctly formatted frontmatter." + ValidationPackageMetadata() + + type ValidationPackageIndex with + + static member create ( + repoPath: string, + lastUpdated: System.DateTimeOffset + ) = + + let md5 = MD5.Create() + + ValidationPackageIndex.create( + repoPath = repoPath, + fileName = Path.GetFileName(repoPath), + lastUpdated = lastUpdated, + contentHash = (md5.ComputeHash(File.ReadAllBytes(repoPath)) |> Convert.ToHexString), + metadata = ValidationPackageMetadata.extractFromScript(repoPath) + ) \ No newline at end of file diff --git a/src/AVPRIndex/Globals.fs b/src/AVPRIndex/Globals.fs new file mode 100644 index 0000000..c88acfa --- /dev/null +++ b/src/AVPRIndex/Globals.fs @@ -0,0 +1,6 @@ +namespace AVPRIndex + +module Globals = + + let [] STAGING_AREA_RELATIVE_PATH = "src/PackageRegistryService/StagingArea" + let [] PACKAGE_INDEX_RELATIVE_PATH = "src/PackageRegistryService/Data/arc-validate-package-index.json" \ No newline at end of file diff --git a/src/AVPRIndex/Utils.fs b/src/AVPRIndex/Utils.fs new file mode 100644 index 0000000..b2c2dc1 --- /dev/null +++ b/src/AVPRIndex/Utils.fs @@ -0,0 +1,12 @@ +namespace AVPRIndex + +open System + +module Utils = + + let truncateDateTime (date: System.DateTimeOffset) = + DateTimeOffset.ParseExact( + date.ToString("yyyy-MM-dd HH:mm:ss zzzz"), + "yyyy-MM-dd HH:mm:ss zzzz", + System.Globalization.CultureInfo.InvariantCulture + ) \ No newline at end of file diff --git a/src/PackageRegistryService/Data/DataInitializer.cs b/src/PackageRegistryService/Data/DataInitializer.cs index d9ae709..5659bd7 100644 --- a/src/PackageRegistryService/Data/DataInitializer.cs +++ b/src/PackageRegistryService/Data/DataInitializer.cs @@ -5,6 +5,9 @@ using System.Security.Policy; using System.Security.Cryptography; using System.Text.Json; +using AVPRIndex; +using static AVPRIndex.Domain; +using static AVPRIndex.Frontmatter; namespace PackageRegistryService.Data @@ -44,7 +47,7 @@ public static void SeedData(ValidationPackageDb context) ReleaseDate = new(i.LastUpdated.Year, i.LastUpdated.Month, i.LastUpdated.Day), Tags = i.Metadata.Tags, ReleaseNotes = i.Metadata.ReleaseNotes, - Authors = i.Metadata.Authors, + Authors = i.Metadata.Authors }; }); @@ -55,6 +58,11 @@ public static void SeedData(ValidationPackageDb context) .Select((i) => { var content = File.ReadAllBytes($"StagingArea/{i.Metadata.Name}/{i.FileName}"); + var hash = Convert.ToHexString(md5.ComputeHash(content)); + if (hash != i.ContentHash) + { + throw new Exception($"Hash collision for indexed hash vs content hash: {$"StagingArea/{i.Metadata.Name}/{i.FileName}"}"); + } return new PackageContentHash { PackageName = i.Metadata.Name, diff --git a/src/PackageRegistryService/Data/arc-validate-package-index.json b/src/PackageRegistryService/Data/arc-validate-package-index.json index c58bce9..3106921 100644 --- a/src/PackageRegistryService/Data/arc-validate-package-index.json +++ b/src/PackageRegistryService/Data/arc-validate-package-index.json @@ -2,7 +2,7 @@ { "RepoPath": "src/PackageRegistryService/StagingArea/invenio/invenio@1.0.0.fsx", "FileName": "invenio@1.0.0.fsx", - "LastUpdated": "2024-02-28T15:39:08+01:00", + "LastUpdated": "2024-02-28T15:39:07+01:00", "ContentHash": "1A3CB3CC0538782C864EA37545451FD5", "Metadata": { "Name": "invenio", @@ -29,7 +29,7 @@ { "RepoPath": "src/PackageRegistryService/StagingArea/test/test@1.0.0.fsx", "FileName": "test@1.0.0.fsx", - "LastUpdated": "2024-02-28T15:39:08+01:00", + "LastUpdated": "2024-02-28T15:39:07+01:00", "ContentHash": "43BFF4CDCC3F3EBB3CA21B6C1F8AC5BA", "Metadata": { "Name": "test", @@ -46,7 +46,7 @@ { "RepoPath": "src/PackageRegistryService/StagingArea/test/test@1.0.1.fsx", "FileName": "test@1.0.1.fsx", - "LastUpdated": "2024-02-28T15:39:08+01:00", + "LastUpdated": "2024-02-28T15:39:07+01:00", "ContentHash": "0B9CBA89F1CECFAF5EB0BA1CE6A480FA", "Metadata": { "Name": "test", @@ -63,7 +63,7 @@ { "RepoPath": "src/PackageRegistryService/StagingArea/test/test@2.0.0.fsx", "FileName": "test@2.0.0.fsx", - "LastUpdated": "2024-02-28T15:39:08+01:00", + "LastUpdated": "2024-02-28T15:39:07+01:00", "ContentHash": "F819359C06456B62F035F2587FBE1EE2", "Metadata": { "Name": "test", @@ -97,7 +97,7 @@ { "RepoPath": "src/PackageRegistryService/StagingArea/test/test@3.0.0.fsx", "FileName": "test@3.0.0.fsx", - "LastUpdated": "2024-02-28T15:39:08+01:00", + "LastUpdated": "2024-02-28T15:39:07+01:00", "ContentHash": "9E6AB1A9C908DE02F583D9FD0E76D8FA", "Metadata": { "Name": "test", diff --git a/src/PackageRegistryService/Models/Author.cs b/src/PackageRegistryService/Models/Author.cs deleted file mode 100644 index 17c36bd..0000000 --- a/src/PackageRegistryService/Models/Author.cs +++ /dev/null @@ -1,13 +0,0 @@ -using Microsoft.EntityFrameworkCore; -using System.Xml.Linq; - -namespace PackageRegistryService.Models -{ - public class Author - { - public required string FullName { get; set; } - public string Email { get; set; } - public string Affiliation { get; set; } - public string AffiliationLink { get; set; } - } -} diff --git a/src/PackageRegistryService/Models/ValidationPackage.cs b/src/PackageRegistryService/Models/ValidationPackage.cs index 8b7cb96..ae5d360 100644 --- a/src/PackageRegistryService/Models/ValidationPackage.cs +++ b/src/PackageRegistryService/Models/ValidationPackage.cs @@ -62,7 +62,7 @@ public class ValidationPackage /// /// /// - public ICollection? Authors { get; set; } // https://www.learnentityframeworkcore.com/relationships#navigation-properties + public ICollection? Authors { get; set; } // https://www.learnentityframeworkcore.com/relationships#navigation-properties /// /// /// diff --git a/src/PackageRegistryService/Models/ValidationPackageIndex.cs b/src/PackageRegistryService/Models/ValidationPackageIndex.cs deleted file mode 100644 index ddcb5ef..0000000 --- a/src/PackageRegistryService/Models/ValidationPackageIndex.cs +++ /dev/null @@ -1,25 +0,0 @@ -namespace PackageRegistryService.Models -{ - public class ValidationPackageMetadata - { - // mandatory fields - public string Name { get; set; } - public string Description { get; set; } - public int MajorVersion { get; set; } - public int MinorVersion { get; set; } - public int PatchVersion { get; set; } - - //optional fields - public bool Publish { get; set; } - public Author[] Authors { get; set; } - public string[] Tags { get; set; } - public string ReleaseNotes { get; set; } - } - public class ValidationPackageIndex - { - public string RepoPath { get; set; } - public string FileName { get; set; } - public System.DateTimeOffset LastUpdated { get; set; } - public ValidationPackageMetadata Metadata { get; set; } - } -} diff --git a/src/PackageRegistryService/PackageRegistryService.csproj b/src/PackageRegistryService/PackageRegistryService.csproj index 89a3171..b8aad75 100644 --- a/src/PackageRegistryService/PackageRegistryService.csproj +++ b/src/PackageRegistryService/PackageRegistryService.csproj @@ -33,4 +33,8 @@ + + + + diff --git a/src/PackageRegistryService/Pages/Components/Package.cs b/src/PackageRegistryService/Pages/Components/Package.cs index cd0b32c..eb9028c 100644 --- a/src/PackageRegistryService/Pages/Components/Package.cs +++ b/src/PackageRegistryService/Pages/Components/Package.cs @@ -1,5 +1,6 @@ using PackageRegistryService.Models; using System.Text; +using static AVPRIndex.Domain; namespace PackageRegistryService.Pages.Components {