Update readme and small migration fix

README.md

@@ -5,77 +5,74 @@ This repository contains:
 - a staging area for authoring official validation packages intended for use with [`arc-validate`](). 
 - a web API for serving validation packages. This API is consumed by `arc-validate` to install and sync validation packages.
 
-## Table of contents
-
-- [Validation package staging area](#validation-package-staging-area)
-  - [The package index](#the-package-index)
-  - [Automated package testing](#automated-package-testing)
-- [Web API (PackageRegistryService)](#web-api-packageregistryservice)
-  - [Local development](#local-development)
-  - [OpenAPI endpoint documentation via Swagger UI](#openapi-endpoint-documentation-via-swagger-ui)
-- [Setup](#setup)
-  - [package indexing](#package-indexing)
-  - [local development](#local-development-1)
-- [How to add packages](#how-to-add-packages)
-  - [Package metadata](#package-metadata)
-    - [Mandatory fields](#mandatory-fields)
-    - [Optional fields](#optional-fields)
+# Table of contents
+
+- [The package index](#the-package-index)
+- [Automated package testing](#automated-package-testing)
+- [Local development](#local-development)
+- [OpenAPI endpoint documentation via Swagger UI](#openapi-endpoint-documentation-via-swagger-ui)
+- [package indexing](#package-indexing)
+- [local development](#local-development-1)
+- [Package metadata](#package-metadata)
+  - [Mandatory fields](#mandatory-fields)
+  - [Optional fields](#optional-fields)
+    - [Objects](#objects)
   - [Package publication workflow](#package-publication-workflow)
   - [Versioning packages](#versioning-packages)
 
 
-## Validation package staging area
+# Validation package staging area
 
-### The package index
+## The package index
 
 This repo runs a [custom pre-commit hook](pre-commit.sh) that will run a [script](./update-index.fsx) automatically add any `.fsx` file in the [staging area](src/PackageRegistryService/StagingArea/) to [the package index](src/PackageRegistryService/Data/arc-validate-package-index.json) when it is commited to the repo.
 
-### Automated package testing
+## Automated package testing
 
 Tests located at [./tests](./tests) are run on every package in the index. Only if all packages pass these tests, the docker container will be built and pushed to the registry.
 
-## Web API (PackageRegistryService)
+# Web API (PackageRegistryService)
 
 The `PackageRegistryService` project located in `/src` is a simple ASP.NET Core (8) web API that serves validation packages and/or associated metadata via a few endpoints.
 
 It is developed specifically for containerization and use in a docker environment. 
 
 The service will eventually be continuously deployed to a public endpoint on the nfdi4plants infrastructure.
 
-### Local development
+## Local development
 
 To run the `PackageRegistryService` locally, ideally use VisualStudio and run the `Docker Compose` project in Debug mode. This will launch the stack defined at [`docker-compose.yml`](docker-compose.yml), which includes:
 
 - the containerized `PackageRegistryService` application 
 - a `postgres` database seeded with the [latest indexed packages](src/PackageRegistryService/Data/arc-validate-package-index.json)
 - an `adminer` instance for database management (will maybe be replaced by pgAdmin in the future)
 
-### OpenAPI endpoint documentation via Swagger UI
+## OpenAPI endpoint documentation via Swagger UI
 
 The `PackageRegistryService` has a built-in Swagger UI endpoint for API documentation. It is served at `/swagger/index.html`.
 
-## Setup
+# Setup
 
-### package indexing
+## package indexing
 
 To install the pre-commit hook needed for automatic package indexing, run either `setup.cmd` or `setup.sh` depending on your platform to install the pre-commit hook.
 
-### local development
+## local development
 
 install the following prerequisites:
 - .NET 8 SDK
 - Docker
 - Docker Compose
 
-## How to add packages
+# How to add packages
 
 To add a package to the staging area, make sure that you installed the pre-commit hook as described in the [Setup](#setup) section. 
 
 Then, simply add a new `.fsx` file to the [staging area](src/PackageRegistryService/StagingArea/), and commit it to the repo. The pre-commit hook will automatically add the new package to the package index.
 
 All packages in the staging area are automatically tested on every commit. Additionally, all packages set to `publish: true` in their yml frontmatter will be pushed to the registry service if they pass all tests and are not already present in the registry.
 
-### Package metadata
+## Package metadata
 
 Package metadata is extracted from **yml frontmatter** at the start of the `.fsx` file, indicated by a multiline comment (`(* ... *)`)containing the frontmatter fenced by `---` at its start and end:
 
@@ -87,15 +84,16 @@ Package metadata is extracted from **yml frontmatter** at the start of the `.fsx
 *)
 ```
 
-#### Mandatory fields
+### Mandatory fields
 
 | Field | Type | Description |
 | --- | --- | --- |
 | Name | string | the name of the package |
 | MajorVersion | int | the major version of the package |
 | MinorVersion | int | the minor version of the package |
 | PatchVersion | int | the patch version of the package |
-| Description | string | a short description of the package |
+| Summary | string | a single sentence description (<=50 words) of the package |
+| Description | string | an unconstrained free text description of the package |
 
 example (only mandatory fields):
 
@@ -106,22 +104,27 @@ Name: my-package
 MajorVersion: 1
 MinorVersion: 0
 PatchVersion: 0
-Description: My package does the thing.
+Summary: My package does the thing.
+Description: |
+  My package does the thing. 
+  It does it very good, it does it very well. 
+  It does it very fast, it does it very swell.
 ---
 *)
 let doSomeValidation () = ()
 doSomeValidation ()
 ```
 
-#### Optional fields
+### Optional fields
 
 | Field | Type | Description |
 | --- | --- | --- |
 | Publish | bool | a boolean value indicating whether the package should be published to the registry. If set to `true`, the package will be built and pushed to the registry. If set to `false` (or not present), the package will be ignored. |
-| Authors | author[] | the authors of the package. Authors have a mandatory **FullName** field and optional **Email**, **Affiliation** and **AffiliationLink** fields |
-| Tags | string[] | a list of tags that describe the package |
+| Authors | author[] | the authors of the package. For more information about mandatory and optional fields in this object, see [Objects > Author](#author) |
+| Tags | string[] | a list of tags with optional ontology annotations that describe the package. For more information about mandatory and optional fields in this object, see [Objects > Tag](#tag)  |
 | ReleaseNotes | string[] | a list of release notes for the package indicating changes from previous versions |
 
+
 example (all fields):
 
 ```fsharp
@@ -131,7 +134,11 @@ Name: my-package
 MajorVersion: 1
 MinorVersion: 0
 PatchVersion: 0
-Description: My package does the thing.
+Summary: My package does the thing.
+Description: |
+  My package does the thing. 
+  It does it very good, it does it very well. 
+  It does it very fast, it does it very swell.
 Publish: true
 Authors:
   - FullName: John Doe
@@ -143,10 +150,11 @@ Authors:
     Affiliation: University of Somewhere
     AffiliationLink: https://somewhere.edu
 Tags:
-  - validation
-  - my-package
-  - thing
-ReleaseNotes: " <- note that these must be a single string, not a list of strings
+  - Name: validation
+  - Name: my-tag
+    TermSourceREF: my-ontology
+    TermAccessionNumber: MO:12345
+ReleaseNotes: |
   - initial release
     - does the thing
     - does it well"
@@ -156,6 +164,30 @@ ReleaseNotes: " <- note that these must be a single string, not a list of string
 let doSomeValidation () = ()
 doSomeValidation ()
 ```
+
+#### Objects
+
+##### Author
+
+Author metadata about the people that create and maintain the package. Note that the
+
+| Field | Type | Description | Mandatory |
+| --- | --- | --- | --- |
+| FullName | string | the full name of the author | yes |
+| Email | string | the email address of the author | no |
+| Affiliation | string | the affiliation (e.g. institution) of the author | no |
+| AffiliationLink | string | a link to the affiliation of the author | no |
+
+##### Tag
+
+Tags can be any string with an optional ontology annotation from a controlled vocabulary:
+
+| Field | Type | Description | Mandatory |
+| --- | --- | --- | --- |
+| Name | string | the name of the tag | yes |
+| TermSourceREF | string | Reference to a controlled vocabulary source | no |
+| TermAccessionNumber | string | Accession in the referenced controlled vocabulary source | no |
+
 ### Package publication workflow
 
 Publishing a package to the registry is a multi-step process:

arc-validate-package-registry.sln

@@ -15,7 +15,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "sln", "sln", "{9F6D799F-2DC
 		README.md = README.md
 		setup.cmd = setup.cmd
 		setup.sh = setup.sh
-		update-index.fsx = update-index.fsx
 	EndProjectSection
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{412317F5-7FF4-4455-9C9D-CEAF7FE6E7B2}"
@@ -48,7 +47,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "publish-workflow", "publish
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AVPRClient", "src\AVPRClient\AVPRClient.csproj", "{D1FABAC1-D0F2-4F6C-B975-236E9969FB38}"
 EndProject
-Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "AVPRIndex", "src\AVPRIndex\AVPRIndex.fsproj", "{78A87C32-8957-468A-AFA6-3DCE8F826500}"
+Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "AVPRIndex", "src\AVPRIndex\AVPRIndex.fsproj", "{78A87C32-8957-468A-AFA6-3DCE8F826500}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution

src/PackageRegistryService/Migrations/20240229124121_AddSummaryAndOntologyTags.cs

@@ -21,12 +21,12 @@ CREATE FUNCTION transform_tags(tags text[]) RETURNS jsonb AS $$
 
             migrationBuilder.Sql(@"
                 ALTER TABLE ""ValidationPackages""
-                ADD COLUMN ""TMP"" jsonb
+                ADD COLUMN ""TMP"" jsonb;
             ");
 
             migrationBuilder.Sql(@"
                 UPDATE ""ValidationPackages""
-                SET ""TMP"" = transform_tags(""Tags"")
+                SET ""TMP"" = transform_tags(""Tags"");
             ");
 
             migrationBuilder.Sql(@"
@@ -53,14 +53,14 @@ protected override void Down(MigrationBuilder migrationBuilder)
 
             migrationBuilder.Sql(@"
                 ALTER TABLE ""ValidationPackages""
-                ADD COLUMN ""TMP"" text[]
+                ADD COLUMN ""TMP"" text[];
             ");
 
             migrationBuilder.Sql(@"
                 UPDATE ""ValidationPackages""
                 SET ""TMP"" = (
                     SELECT array_agg(tag->>'Name') FROM jsonb_array_elements(""Tags"") AS tag
-                )
+                );
             ");
 
             migrationBuilder.Sql(@"