feat: read manifest from stdin (#60)

README.md

@@ -90,20 +90,19 @@ ubuntu@infra1:~$ maas-anvil cluster list
 ubuntu@infra1:~$ juju run maas-region/0 create-admin username=admin password=pass email=admin@maas.io ssh-import=lp:maasadmin
 ```
 
-# Managing the cluster after initial deployment
+### Managing the cluster after initial deployment
 
-
-## Cluster updates
+#### Cluster updates
 
 You can refresh the cluster by running the `refresh` command:
 
 ```bash
 ubuntu@infra1:~$ maas-anvil refresh
 ```
 
-This allows passing a new manifest file with `--manifest` for updating configuration options.
+This allows passing a new manifest file with `--manifest` for updating configuration options. If `--manifest -` is passed, then the manifest is loaded from stdin.
 
-## Juju permission denied
+#### Juju permission denied
 
 If you get an error message such as:
 
@@ -130,7 +129,7 @@ user: $user
 
 And `juju login` as usual.
 
-### Charm documentation
+## Charm documentation
 
 - MAAS Region: <https://charmhub.io/maas-region>
 - MAAS Region: <https://charmhub.io/maas-agent>

anvil-python/anvil/commands/manifest.py

@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 import logging
 import os
 from pathlib import Path

anvil-python/anvil/commands/refresh.py

@@ -22,11 +22,11 @@
     run_plan,
 )
 from sunbeam.jobs.juju import JujuHelper
-from sunbeam.jobs.manifest import AddManifestStep
+import yaml
 
 from anvil.commands.upgrades.inter_channel import ChannelUpgradeCoordinator
 from anvil.commands.upgrades.intra_channel import LatestInChannelCoordinator
-from anvil.jobs.manifest import Manifest
+from anvil.jobs.manifest import AddManifestStep, Manifest
 from anvil.provider.local.deployment import LocalDeployment
 
 LOG = logging.getLogger(__name__)
@@ -39,7 +39,9 @@
     "--manifest",
     "manifest_path",
     help="Manifest file.",
-    type=click.Path(exists=True, dir_okay=False, path_type=Path),
+    type=click.Path(
+        exists=True, dir_okay=False, path_type=Path, allow_dash=True
+    ),
 )
 @click.option(
     "-u",
@@ -65,10 +67,19 @@ def refresh(
 
     manifest = None
     if manifest_path:
+        try:
+            with click.open_file(manifest_path) as file:  # type: ignore
+                manifest_data = yaml.safe_load(file)
+        except (OSError, yaml.YAMLError) as e:
+            LOG.debug(e)
+            raise click.ClickException(f"Manifest parsing failed: {e!s}")
+
         manifest = Manifest.load(
-            deployment, manifest_file=manifest_path, include_defaults=True
+            deployment,
+            manifest_data=manifest_data or {},
+            include_defaults=True,
         )
-        run_plan([AddManifestStep(client, manifest_path)], console)
+        run_plan([AddManifestStep(client, manifest_data)], console)
 
     if not manifest:
         LOG.debug("Getting latest manifest from cluster db")

anvil-python/anvil/jobs/manifest.py

@@ -25,16 +25,22 @@
 from sunbeam import utils
 from sunbeam.clusterd.client import Client
 from sunbeam.clusterd.service import (
+    ClusterServiceUnavailableException,
     ConfigItemNotFoundException,
     ManifestItemNotFoundException,
 )
 from sunbeam.commands.terraform import TerraformHelper
 from sunbeam.jobs.common import (
+    BaseStep,
+    Result,
+    ResultType,
+    Status,
     read_config,
     update_config,
 )
 from sunbeam.jobs.deployment import Deployment
 from sunbeam.jobs.manifest import (
+    EMPTY_MANIFEST,
     CharmsManifest,
     JujuManifest,
     MissingTerraformInfoException,
@@ -61,7 +67,7 @@ class SoftwareConfig:
     terraform: Dict[str, TerraformManifest] | None = None
 
     """
-    # field_validator supported only in pydantix 2.x
+    # field_validator supported only in pydantic 2.x
     @field_validator("terraform", "mode_after")
     def validate_terraform(cls, terraform):
         if terraform:
@@ -152,33 +158,31 @@ def __init__(
     def load(
         cls,
         deployment: Deployment,
-        manifest_file: Path,
+        manifest_data: Dict[str, Any],
         include_defaults: bool = False,
     ) -> "Manifest":
         """Load the manifest with the provided file input
 
-        If include_defaults is True, load the manifest over the defaut manifest.
+        If include_defaults is True, load the manifest over the default manifest.
         """
         if include_defaults:
-            return cls.load_on_default(deployment, manifest_file)
+            return cls.load_on_default(deployment, manifest_data)
 
         plugin_manager = PluginManager()
-        with manifest_file.open() as file:
-            override = yaml.safe_load(file)
-            return Manifest(
-                deployment,
-                plugin_manager,
-                override.get("deployment", {}),
-                override.get("software", {}),
-            )
+        return Manifest(
+            deployment,
+            plugin_manager,
+            manifest_data.get("deployment", {}),
+            manifest_data.get("software", {}),
+        )
 
     @classmethod
     def load_latest_from_clusterdb(
         cls, deployment: Deployment, include_defaults: bool = False
     ) -> "Manifest":
         """Load the latest manifest from clusterdb
 
-        If include_defaults is True, load the manifest over the defaut manifest.
+        If include_defaults is True, load the manifest over the default manifest.
         values.
         """
         if include_defaults:
@@ -204,24 +208,22 @@ def load_latest_from_clusterdb(
 
     @classmethod
     def load_on_default(
-        cls, deployment: Deployment, manifest_file: Path
+        cls, deployment: Deployment, manifest_data: Dict[str, Any]
     ) -> "Manifest":
         """Load manifest and override the default manifest"""
         plugin_manager = PluginManager()
-        with manifest_file.open() as file:
-            override = yaml.safe_load(file)
-            override_deployment = override.get("deployment") or {}
-            override_software = override.get("software") or {}
-            default_software = SoftwareConfig.get_default_software_as_dict(
-                deployment, plugin_manager
-            )
-            utils.merge_dict(default_software, override_software)
-            return Manifest(
-                deployment,
-                plugin_manager,
-                override_deployment,
-                default_software,
-            )
+        override_deployment = manifest_data.get("deployment") or {}
+        override_software = manifest_data.get("software") or {}
+        default_software = SoftwareConfig.get_default_software_as_dict(
+            deployment, plugin_manager
+        )
+        utils.merge_dict(default_software, override_software)
+        return Manifest(
+            deployment,
+            plugin_manager,
+            override_deployment,
+            default_software,
+        )
 
     @classmethod
     def load_latest_from_clusterdb_on_default(
@@ -453,3 +455,46 @@ def _get_tfvar_names(
                 .items()
                 for charm_attribute, tfvar_name in per_charm_tfvar_map.items()
             ]
+
+
+class AddManifestStep(BaseStep):
+    """Add Manifest file to cluster database"""
+
+    def __init__(self, client: Client, manifest: Dict[str, Any] = {}):
+        super().__init__(
+            "Write Manifest to database", "Writing Manifest to database"
+        )
+        # Write EMPTY_MANIFEST if manifest not provided
+        self.manifest = manifest
+        self.client = client
+        self.manifest_content: Dict[str, Any] = {}
+
+    def is_skip(self, status: Status | None = None) -> Result:
+        """Skip if the user provided manifest and the latest from db are same."""
+        try:
+            self.manifest_content = self.manifest or EMPTY_MANIFEST
+            latest_manifest = self.client.cluster.get_latest_manifest()
+        except ManifestItemNotFoundException:
+            return Result(ResultType.COMPLETED)
+        except (ClusterServiceUnavailableException,) as e:
+            LOG.debug(e)
+            return Result(ResultType.FAILED, str(e))
+
+        if (
+            yaml.safe_load(latest_manifest.get("data"))
+            == self.manifest_content
+        ):
+            return Result(ResultType.SKIPPED)
+
+        return Result(ResultType.COMPLETED)
+
+    def run(self, status: Status | None = None) -> Result:
+        """Write manifest to cluster db"""
+        try:
+            id = self.client.cluster.add_manifest(
+                data=yaml.safe_dump(self.manifest_content)
+            )
+            return Result(ResultType.COMPLETED, id)
+        except Exception as e:
+            LOG.debug(e)
+            return Result(ResultType.FAILED, str(e))

anvil-python/anvil/provider/local/commands.py

@@ -59,7 +59,6 @@
 )
 from sunbeam.jobs.deployment import Deployment
 from sunbeam.jobs.juju import JujuHelper
-from sunbeam.jobs.manifest import AddManifestStep
 from sunbeam.provider.base import ProviderBase
 from sunbeam.provider.local.deployment import LOCAL_TYPE
 import yaml
@@ -96,7 +95,7 @@
     validate_roles,
 )
 from anvil.jobs.juju import CONTROLLER
-from anvil.jobs.manifest import Manifest
+from anvil.jobs.manifest import AddManifestStep, Manifest
 from anvil.provider.local.deployment import LocalDeployment
 from anvil.utils import CatchGroup
 
@@ -150,7 +149,9 @@ def deployment_type(self) -> tuple[str, type[Deployment]]:
     "-m",
     "--manifest",
     help="Manifest file.",
-    type=click.Path(exists=True, dir_okay=False, path_type=Path),
+    type=click.Path(
+        exists=True, dir_okay=False, path_type=Path, allow_dash=True
+    ),
 )
 @click.option(
     "--role",
@@ -182,8 +183,17 @@ def bootstrap(
     # Validate manifest file
     manifest_obj = None
     if manifest:
+        try:
+            with click.open_file(manifest) as file:  # type: ignore
+                manifest_data = yaml.safe_load(file)
+        except (OSError, yaml.YAMLError) as e:
+            LOG.debug(e)
+            raise click.ClickException(f"Manifest parsing failed: {e!s}")
+
         manifest_obj = Manifest.load(
-            deployment, manifest_file=manifest, include_defaults=True
+            deployment,
+            manifest_data=manifest_data or {},
+            include_defaults=True,
         )
     else:
         manifest_obj = Manifest.get_default_manifest(deployment)
@@ -232,7 +242,7 @@ def bootstrap(
         ClusterInitStep(
             client, roles_to_str_list(roles), 0
         ),  # bootstrapped node is always machine 0 in controller model
-        AddManifestStep(client, manifest) if manifest else None,
+        AddManifestStep(client, manifest_data) if manifest else None,
         AddCloudJujuStep(cloud_name, cloud_definition),
         BootstrapJujuStep(
             client,