Fix: EOL filtering to only exclude builds with no valid tracks. (#272)

- Fix EOL filtering to only exclude builds with no valid tracks. - Refactor of src/image/prepare_single_image_build_matrix.py. - Upgrade pydantic to 2.8.2 globally. - Respective testing for EOL handeling. - Testing for image trigger file validator. --------- Co-authored-by: clay-lake <clay-lake@localhost>
canonical · Nov 14, 2024 · 2818740 · 2818740
1 parent e1dbca7
commit 2818740
Show file tree

Hide file tree

Showing 17 changed files with 501 additions and 219 deletions.
diff --git a/.github/workflows/Image.yaml b/.github/workflows/Image.yaml
@@ -119,7 +119,7 @@ jobs:
         run: |
           mkdir ${{ env.DATA_DIR }}
 
-          ./src/image/prepare_single_image_build_matrix.py \
+          python3 -m src.image.prepare_single_image_build_matrix \
             --oci-path ${{ steps.validate-image.outputs.img-path }} \
             --revision-data-dir ${{ env.DATA_DIR }}
 
@@ -131,7 +131,6 @@ jobs:
       fail-fast: true
       matrix: ${{ fromJSON(needs.prepare-build.outputs.build-matrix) }}
     steps:
-
       - name: Clone GitHub image repository
         uses: actions/checkout@v4
         with:
@@ -254,7 +253,7 @@ jobs:
 
           mkdir ${{ env.DATA_DIR }}
 
-          ./src/image/prepare_single_image_build_matrix.py \
+          python3 -m src.image.prepare_single_image_build_matrix \
             --oci-path ${{ needs.prepare-build.outputs.oci-img-path }} \
             --revision-data-dir ${{ env.DATA_DIR }} \
             --next-revision ${{ steps.get-next-revision.outputs.revision }} \
@@ -278,17 +277,6 @@ jobs:
           ./src/uploads/swift_lockfile_unlock.sh \
             ${{ needs.prepare-build.outputs.oci-img-name }}
 
-      # The revision files have to be sanitised before merging,
-      # since the `track` field should not be present.
-      - name: Sanitise revision files
-        run: |
-          set -ex
-          for revision_file in `ls ${{ env.DATA_DIR }}`
-          do
-            jq 'del(.track, .base)' ${{ env.DATA_DIR }}/$revision_file > ${{ env.DATA_DIR }}/$revision_file.tmp
-            mv ${{ env.DATA_DIR }}/$revision_file.tmp ${{ env.DATA_DIR }}/$revision_file
-          done
-
       - uses: actions/cache/save@v4
         with:
           path: ${{ steps.prepare-matrix.outputs.revision-data-dir }}
@@ -535,7 +523,7 @@ jobs:
 
             echo "Merge revision $revision_file with requested releases"
 
-            ./src/image/merge_release_info.py \
+            python3 -m src.image.merge_release_info \
               --image-trigger "${{ needs.prepare-build.outputs.oci-img-path }}/image.yaml" \
               --revision-data-file "${{ env.REVISION_DATA_DIR }}/${revision_file}"
           done
@@ -640,7 +628,7 @@ jobs:
         id: get-summary
         run: |
           echo '${{ toJson(needs) }}' > jobs.json
-          ./src/notifications/summarize_workflow_results.py --jobs-file jobs.json
+          python3 -m src.notifications.summarize_workflow_results --jobs-file jobs.json
 
       - name: Get contacts for ${{ needs.prepare-build.outputs.oci-img-name }}
         id: get-contacts

diff --git a/oci/mock-rock/_releases.json b/oci/mock-rock/_releases.json
@@ -9,19 +9,19 @@
         "edge": {
             "target": "latest_beta"
         },
-        "end-of-life": "2025-05-01T00:00:00Z"
+        "end-of-life": "2030-05-01T00:00:00Z"
     },
     "1.0-22.04": {
         "candidate": {
-            "target": "654"
+            "target": "732"
         },
         "beta": {
-            "target": "654"
+            "target": "732"
         },
         "edge": {
-            "target": "654"
+            "target": "732"
         },
-        "end-of-life": "2025-05-01T00:00:00Z"
+        "end-of-life": "2030-05-01T00:00:00Z"
     },
     "test": {
         "beta": {
@@ -30,36 +30,36 @@
         "edge": {
             "target": "test_beta"
         },
-        "end-of-life": "2025-05-01T00:00:00Z"
+        "end-of-life": "2030-05-01T00:00:00Z"
     },
     "1.1-22.04": {
-        "end-of-life": "2025-05-01T00:00:00Z",
+        "end-of-life": "2030-05-01T00:00:00Z",
         "candidate": {
-            "target": "677"
+            "target": "733"
         },
         "beta": {
-            "target": "677"
+            "target": "733"
         },
         "edge": {
-            "target": "677"
+            "target": "733"
         }
     },
     "1-22.04": {
-        "end-of-life": "2025-05-01T00:00:00Z",
+        "end-of-life": "2030-05-01T00:00:00Z",
         "candidate": {
-            "target": "677"
+            "target": "733"
         },
         "beta": {
-            "target": "677"
+            "target": "733"
         },
         "edge": {
-            "target": "677"
+            "target": "733"
         }
     },
     "1.2-22.04": {
-        "end-of-life": "2025-05-01T00:00:00Z",
+        "end-of-life": "2030-05-01T00:00:00Z",
         "beta": {
-            "target": "678"
+            "target": "734"
         },
         "edge": {
             "target": "1.2-22.04_beta"

diff --git a/oci/mock-rock/image.yaml b/oci/mock-rock/image.yaml
@@ -2,35 +2,41 @@ version: 1
 
 release:
   latest:
-    end-of-life: "2025-05-01T00:00:00Z"
+    end-of-life: "2030-05-01T00:00:00Z"
     candidate: 1.2-22.04_beta
   test:
-    end-of-life: "2025-05-01T00:00:00Z"
+    end-of-life: "2030-05-01T00:00:00Z"
     beta: 1.1-22.04_beta
 
 upload:
   - source: "canonical/rocks-toolbox"
     commit: 17916dd5de270e61a6a3fd3f4661a6413a50fd6f
     directory: mock_rock/1.0
     release:
-      1.0-22.04:
+      1.0.0-22.04:
         end-of-life: "2024-05-01T00:00:00Z"
         risks:
           - candidate
           - edge
           - beta
+      1.0-22.04:
+        end-of-life: "2030-05-01T00:00:00Z"
+        risks:
+          - candidate
+          - edge
+          - beta
   - source: "canonical/rocks-toolbox"
     commit: 17916dd5de270e61a6a3fd3f4661a6413a50fd6f
     directory: mock_rock/1.1
     release:
       1.1-22.04:
-        end-of-life: "2025-05-01T00:00:00Z"
+        end-of-life: "2030-05-01T00:00:00Z"
         risks:
           - candidate
           - edge
           - beta
       1-22.04:
-        end-of-life: "2025-05-01T00:00:00Z"
+        end-of-life: "2030-05-01T00:00:00Z"
         risks:
           - candidate
           - edge
@@ -40,6 +46,6 @@ upload:
     directory: mock_rock/1.2
     release:
       1.2-22.04:
-        end-of-life: "2025-05-01T00:00:00Z"
+        end-of-life: "2030-05-01T00:00:00Z"
         risks:
           - beta
diff --git a/src/docs/generate_oci_doc_yaml.py b/src/docs/generate_oci_doc_yaml.py
@@ -3,6 +3,7 @@
 This module contains functions for generating documentation
 for OCI images within the oci-factory
 """
+
 import argparse
 import base64
 import json
@@ -279,7 +280,7 @@ def read_documentation_yaml(doc_file: str) -> Dict:
             try:
                 base_doc_data = DocSchema(
                     **yaml.load(file, Loader=yaml.BaseLoader) or {}
-                ).dict(exclude_none=True)
+                ).model_dump(exclude_none=True)
             except (yaml.YAMLError, pydantic.ValidationError) as exc:
                 msg = f"Error loading the {doc_file} file"
                 raise Exception(msg) from exc

diff --git a/src/docs/requirements.txt b/src/docs/requirements.txt
@@ -1,3 +1,3 @@
 boto3
 pyyaml
-pydantic==1.9.0
+pydantic==2.8.2
diff --git a/src/docs/schema/triggers.py b/src/docs/schema/triggers.py
@@ -4,7 +4,7 @@
 """
 
 from typing import Optional
-from pydantic import BaseModel, Extra, constr, conlist
+from pydantic import BaseModel, constr, conlist, ConfigDict
 
 
 class ConfigMapFile(BaseModel):
@@ -14,22 +14,16 @@ class ConfigMapFile(BaseModel):
     name: str
     link: str
 
-    class Config:
-        """permit to not accept extra parameters"""
-
-        extra = Extra.forbid
+    model_config = ConfigDict(extra="forbid")
 
 
 class Microk8sConfigMap(BaseModel):
     """Schema of the microk8s[configmap] section."""
 
-    name: Optional[str]
-    files: conlist(item_type=ConfigMapFile, min_items=1)
-
-    class Config:
-        """permit to not accept extra parameters"""
+    name: Optional[str] = None
+    files: conlist(item_type=ConfigMapFile, min_length=1)
 
-        extra = Extra.forbid
+    model_config = ConfigDict(extra="forbid")
 
 
 class Microk8sDeploy(BaseModel):
@@ -38,34 +32,25 @@ class Microk8sDeploy(BaseModel):
     link: str
     access: str
 
-    class Config:
-        """permit to not accept extra parameters"""
-
-        extra = Extra.forbid
+    model_config = ConfigDict(extra="forbid")
 
 
 class Microk8sInfo(BaseModel):
     """Schema of the microk8s section."""
 
-    configmap: Optional[Microk8sConfigMap]
+    configmap: Optional[Microk8sConfigMap] = None
     deploy: Microk8sDeploy
 
-    class Config:
-        """permit to not accept extra parameters"""
-
-        extra = Extra.forbid
+    model_config = ConfigDict(extra="forbid")
 
 
 class DockerRunParameters(BaseModel):
     """Schema of the docker section."""
 
-    parameters: conlist(item_type=str, min_items=1)
-    access: Optional[str]
-
-    class Config:
-        """permit to not accept extra parameters"""
+    parameters: conlist(item_type=str, min_length=1)
+    access: Optional[str] = None
 
-        extra = Extra.forbid
+    model_config = ConfigDict(extra="forbid")
 
 
 class Parameter(BaseModel):
@@ -75,36 +60,27 @@ class Parameter(BaseModel):
     value: str
     description: str
 
-    class Config:
-        """permit to not accept extra parameters"""
-
-        extra = Extra.forbid
+    model_config = ConfigDict(extra="forbid")
 
 
 class DebugInfo(BaseModel):
     """Schema of the debug section."""
 
     text: str
 
-    class Config:
-        """permit to not accept extra parameters"""
-
-        extra = Extra.forbid
+    model_config = ConfigDict(extra="forbid")
 
 
 class DocSchema(BaseModel):
     """Schema of the documentation.yaml file, and also validation of the schema"""
 
     version: str
     application: constr(min_length=1, strip_whitespace=True)
-    is_chiselled: Optional[bool]
-    description: constr(min_length=1, strip_whitespace=True)
-    docker: Optional[DockerRunParameters]
-    parameters: Optional[conlist(item_type=Parameter, min_items=1)]
-    debug: Optional[DebugInfo]
-    microk8s: Optional[Microk8sInfo]
-
-    class Config:
-        """permit to not accept extra parameters"""
-
-        extra = Extra.forbid
+    is_chiselled: Optional[bool] = None
+    description: constr(min_length=1, strip_whitespace=True) = None
+    docker: Optional[DockerRunParameters] = None
+    parameters: Optional[conlist(item_type=Parameter, min_length=1)] = None
+    debug: Optional[DebugInfo] = None
+    microk8s: Optional[Microk8sInfo] = None
+
+    model_config = ConfigDict(extra="forbid")
diff --git a/src/image/merge_release_info.py b/src/image/merge_release_info.py
@@ -19,8 +19,8 @@
 import json
 import yaml
 
-from utils.schema.triggers import ImageSchema, KNOWN_RISKS_ORDERED
-from utils.schema.revision_data import RevisionDataSchema
+from .utils.schema.triggers import ImageSchema, KNOWN_RISKS_ORDERED
+from .utils.schema.revision_data import RevisionDataSchema
 
 
 def backfill_higher_risks(channels: dict) -> None: