Move staging area into repo root and adapt accordingly:

- add index package tests
- release a new version of the index package with tested functionality exposed
- this commit is quite complicated and will test all current ci pipeline capabilities

.github/workflows/pipeline.yml

@@ -62,13 +62,11 @@ jobs:
             global:
               - '**'
             packages:
-              - src/PackageRegistryService/StagingArea/**
+              - StagingArea/**
             tests:
               - tests/**
             api:
               - src/PackageRegistryServive/**
-              - '!src/PackageRegistryServive/StagingArea/**'
-              - '!src/PackageRegistryServive/Data/**'
             client:
               - src/AVPRClient/**
             index:

README.md

@@ -25,7 +25,7 @@ This repository contains:
 
 ## The package index
 
-This repo runs a [custom pre-commit hook](pre-commit.sh) that will run a [script](./update-index.fsx) automatically add any `.fsx` file in the [staging area](src/PackageRegistryService/StagingArea/) to [the package index](src/PackageRegistryService/Data/arc-validate-package-index.json) when it is commited to the repo.
+This repo runs a [custom pre-commit hook](pre-commit.sh) that will run a [script](./update-index.fsx) automatically add any `.fsx` file in the [staging area](StagingArea/) to [the package index](src/PackageRegistryService/Data/arc-validate-package-index.json) when it is commited to the repo.
 
 ## Automated package testing
 
@@ -68,7 +68,7 @@ install the following prerequisites:
 
 To add a package to the staging area, make sure that you installed the pre-commit hook as described in the [Setup](#setup) section. 
 
-Then, simply add a new `.fsx` file to the [staging area](src/PackageRegistryService/StagingArea/), and commit it to the repo. The pre-commit hook will automatically add the new package to the package index.
+Then, simply add a new `.fsx` file to the [staging area](StagingArea/), and commit it to the repo. The pre-commit hook will automatically add the new package to the package index.
 
 All packages in the staging area are automatically tested on every commit. Additionally, all packages set to `publish: true` in their yml frontmatter will be pushed to the registry service if they pass all tests and are not already present in the registry.
 
@@ -194,7 +194,7 @@ Publishing a package to the registry is a multi-step process:
 
 Suppose you want to develop version 1.0.0 of a package called `my-package`.
 
-1. Add a new blank `my-package@1.0.0.fsx` file to the [staging area](./src/PackageRegistryService/StagingArea/) in the folder `my-package`.
+1. Add a new blank `my-package@1.0.0.fsx` file to the [staging area](./StagingArea/) in the folder `my-package`.
 2. Develop the package, using this repositories CI to perform automates integrity tests on it.
 3. Once the package is ready, add `publish: true` to the yml frontmatter of the package file. This will trigger the CI to build and push the package to the registry.
 4. Once a package is published, it cannot be unpublished or changed. To update a package, create a new script with the same name and a higher version number.

StagingAreaTests/ReferenceObjects.fs

@@ -21,7 +21,7 @@ let all_staged_packages_contents =
     all_staged_packages_paths
     |> Array.map (fun p ->
         p, 
-        p |> (File.ReadAllText >> fun x -> x.ReplaceLineEndings())
+        p |> (File.ReadAllText >> fun x -> x.ReplaceLineEndings("\n"))
     )
 
 let all_staged_packages_metadata =

StagingAreaTests/StagingAreaTests.fsproj

@@ -10,7 +10,7 @@
 
   <ItemGroup>
     <None Include="playground.fsx" />
-    <Content Include="..\src\PackageRegistryService\StagingArea\**" CopyToOutputDirectory="Always" LinkBase="StagingArea" />
+    <Content Include="..\StagingArea\**" CopyToOutputDirectory="Always" LinkBase="StagingArea" />
     <Compile Include="Utils.fs" />
     <Compile Include="ReferenceObjects.fs" />
     <Compile Include="DirectoryContentTests.fs" />
@@ -35,6 +35,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\src\AVPRIndex\AVPRIndex.fsproj" />
+    <ProjectReference Include="..\tests\IndexTests\IndexTests.fsproj" />
   </ItemGroup>
 
 </Project>

StagingAreaTests/Utils.fs

@@ -25,22 +25,6 @@ type Assert with
             script.Contains(Frontmatter.frontMatterEnd)
         )
 
-    static member MetadataValid(m: ValidationPackageMetadata) =
-        //test wether all required fields are present
-        Assert.NotNull(m)
-        Assert.NotNull(m.Name)
-        Assert.NotEqual<string>(m.Name, "")
-        Assert.NotNull(m.Summary)
-        Assert.NotEqual<string>(m.Summary, "")
-        Assert.NotNull(m.Description)
-        Assert.NotEqual<string>(m.Description, "")
-        Assert.NotNull(m.MajorVersion)
-        Assert.True(m.MajorVersion >= 0)
-        Assert.NotNull(m.MinorVersion)
-        Assert.True(m.MinorVersion >= 0)
-        Assert.NotNull(m.PatchVersion)
-        Assert.True(m.PatchVersion >= 0)
-
     static member FileNameValid(path:string) =
         let fileName = Path.GetFileName(path)
         let folderName = Path.GetDirectoryName(path) |> Path.GetFileName

arc-validate-package-registry.sln

@@ -23,10 +23,9 @@ Project("{E53339B2-1760-4266-BCC7-CA923CBCF16C}") = "docker-compose", "docker-co
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".ci", ".ci", "{24F7CF58-94B9-4FB4-8A59-FD329B3C431D}"
 	ProjectSection(SolutionItems) = preProject
-		.github\workflows\build-and-test.yml = .github\workflows\build-and-test.yml
+		.github\workflows\build-and-test-solution.yml = .github\workflows\build-and-test-solution.yml
 		.github\workflows\pipeline.yml = .github\workflows\pipeline.yml
-		.github\workflows\publish-pending-packages.yml = .github\workflows\publish-pending-packages.yml
-		.github\workflows\update-docker-image.yml = .github\workflows\update-docker-image.yml
+		.github\workflows\release-package.yml = .github\workflows\release-package.yml
 	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AVPRClient", "src\AVPRClient\AVPRClient.csproj", "{D1FABAC1-D0F2-4F6C-B975-236E9969FB38}"
@@ -41,6 +40,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "APITests", "tests\APITests\
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "scripts", "scripts", "{C4DBD5F6-4A14-44F4-9E5A-AC6B72AAEB81}"
 	ProjectSection(SolutionItems) = preProject
+		scripts\domain.fsx = scripts\domain.fsx
+		scripts\generate-preview-index.fsx = scripts\generate-preview-index.fsx
+		scripts\pre-publish-checks.fsx = scripts\pre-publish-checks.fsx
 		scripts\publish-pending-packages.fsx = scripts\publish-pending-packages.fsx
 		scripts\update-index.fsx = scripts\update-index.fsx
 	EndProjectSection

scripts/pre-publish-checks.fsx

@@ -16,6 +16,7 @@ let current_preview_index =
         |> Request.send
         |> Response.deserializeJson<ValidationPackageIndex[]>
     with e as exn ->
+        printfn $"Failed to fetch current preview index: {exn.Message}"
         [||]
 
 let all_packages_in_staging_area =

src/AVPRClient/AVPRClient.csproj

@@ -24,13 +24,10 @@
     <PackageReadmeFile>README.md</PackageReadmeFile>
     <PackageVersion>0.0.4</PackageVersion>
   </PropertyGroup>
-
+	
   <ItemGroup>
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <None Include="README.md" Pack="true" PackagePath="\" />
-  </ItemGroup>
-
-  <ItemGroup>
     <ProjectReference Include="..\AVPRIndex\AVPRIndex.fsproj" />
   </ItemGroup>
 

src/AVPRIndex/Frontmatter.fs

@@ -1,6 +1,5 @@
 namespace AVPRIndex
 
-
 open Domain
 open System
 open System.IO
@@ -9,8 +8,26 @@ open YamlDotNet.Serialization
 
 module Frontmatter = 
 
-    let frontMatterStart = $"(*{System.Environment.NewLine}---"
-    let frontMatterEnd = $"---{System.Environment.NewLine}*)"
+    let [<Literal>] frontMatterStart = "(*\n---"
+    let [<Literal>] frontMatterEnd = "---\n*)"
+
+    let containsFrontmatter (str: string) =
+        str.StartsWith(frontMatterStart, StringComparison.Ordinal) && str.Contains(frontMatterEnd)
+
+    let tryExtractFromString (str: string) =
+        let norm = str.ReplaceLineEndings("\n")
+        if containsFrontmatter norm then
+            norm.Substring(
+                frontMatterStart.Length, 
+                (norm.IndexOf(frontMatterEnd, StringComparison.Ordinal) - frontMatterEnd.Length))
+            |> Some
+        else 
+            None
+
+    let extractFromString (str: string) =
+        match tryExtractFromString str with
+        | Some frontmatter -> frontmatter
+        | None -> failwith $"input has no correctly formatted frontmatter."
 
     let yamlDeserializer = 
         DeserializerBuilder()
@@ -19,24 +36,34 @@ module Frontmatter =
 
     type ValidationPackageMetadata with
 
+        static member extractFromString (str: string) =
+            let frontmatter = tryExtractFromString str
+            match frontmatter with
+            | Some frontmatter ->
+                let result = 
+                    yamlDeserializer.Deserialize<ValidationPackageMetadata>(frontmatter)
+                result
+            | None ->
+                failwith $"string has no correctly formatted frontmatter."
+
+        static member tryExtractFromString (str: string) =
+            try 
+                ValidationPackageMetadata.extractFromString str |> Some
+            with e ->
+                printfn $"error parsing package metadata: {e.Message}"
+                None
+
         static member extractFromScript (scriptPath: string) =
-            let script = File.ReadAllText(scriptPath).ReplaceLineEndings()
-            if script.StartsWith(frontMatterStart, StringComparison.Ordinal) && script.Contains(frontMatterEnd) then
-                let frontmatter = 
-                    script.Substring(
-                        frontMatterStart.Length, 
-                        (script.IndexOf(frontMatterEnd, StringComparison.Ordinal) - frontMatterEnd.Length))
-                try 
-                    let result = 
-                        yamlDeserializer.Deserialize<ValidationPackageMetadata>(frontmatter)
-                    result
-                with e as exn -> 
-                    printfn $"error parsing package metadata at {scriptPath}. Make sure that all required metadata tags are included."
-                    printfn $"Error msg: {e.Message}."
-                    ValidationPackageMetadata()
-            else 
-                printfn $"script at {scriptPath} has no correctly formatted frontmatter."
-                ValidationPackageMetadata()
+            scriptPath
+            |> File.ReadAllText
+            |> ValidationPackageMetadata.extractFromString 
+
+        static member tryExtractFromScript (scriptPath: string) =
+            try 
+                ValidationPackageMetadata.extractFromScript scriptPath |> Some
+            with e ->
+                printfn $"error parsing package metadata: {e.Message}"
+                None
 
     type ValidationPackageIndex with
 

src/AVPRIndex/Globals.fs

@@ -2,5 +2,4 @@
 
 module Globals =
 
-    let [<Literal>] STAGING_AREA_RELATIVE_PATH = "src/PackageRegistryService/StagingArea"
-    let [<Literal>] PACKAGE_INDEX_RELATIVE_PATH = "src/PackageRegistryService/Data/arc-validate-package-index.json"
+    let [<Literal>] STAGING_AREA_RELATIVE_PATH = "StagingArea"

src/AVPRIndex/RELEASE_NOTES.md

@@ -1,6 +1,8 @@
 ## v0.0.5
 
 - Add `getPreviewIndex` function that downloads the currently released preview index from the github release.
+- Add `ValidationPackageMetadata.extractFromString` to extract metadata from a string
+- Refactor and expose some Frontmatter parsing functions
 
 ## v0.0.4
 

src/PackageRegistryService/Data/DataInitializer.cs

@@ -8,33 +8,33 @@
 using AVPRIndex;
 using static AVPRIndex.Domain;
 using static AVPRIndex.Frontmatter;
+using System.Reflection;
 
 namespace PackageRegistryService.Data
 
 {
     public class DataInitializer
     {
-        public static List<ValidationPackageIndex> ReadIndex()
-        {
-            var json = File.ReadAllText(@"Data/arc-validate-package-index.json");
-            var index = JsonSerializer.Deserialize<List<ValidationPackageIndex>>(json);
-            return index ?? [];
-        }
         public static void SeedData(ValidationPackageDb context)
         {
             MD5 md5 = MD5.Create();
 
             if (!context.ValidationPackages.Any())
             {
-                var index = ReadIndex();
+                var index = AVPRRepo.getPreviewIndex();
 
                 context.SaveChanges();
 
                 var validationPackages =
                     index
                         .Select((i) =>
                         {
-                            var content = File.ReadAllBytes($"StagingArea/{i.Metadata.Name}/{i.FileName}");
+                            var path = 
+                                Path.Combine(
+                                    Path.GetDirectoryName(Assembly.GetEntryAssembly().Location),
+                                    $"StagingArea/{i.Metadata.Name}/{i.FileName}"
+                                );
+                            var content = File.ReadAllBytes(path);
 
                             return new ValidationPackage
                             {
@@ -58,7 +58,13 @@ public static void SeedData(ValidationPackageDb context)
                     index
                         .Select((i) =>
                         {
-                            var content = File.ReadAllBytes($"StagingArea/{i.Metadata.Name}/{i.FileName}");
+                            var path =
+                                Path.Combine(
+                                    Path.GetDirectoryName(Assembly.GetEntryAssembly().Location),
+                                    $"StagingArea/{i.Metadata.Name}/{i.FileName}"
+                                );
+                            var content = File.ReadAllBytes(path);
+
                             var hash = Convert.ToHexString(md5.ComputeHash(content));
                             if (hash != i.ContentHash)
                             {