Allow installing snaps via package_update_upgrade_install module (#4202)

This also includes a major refactoring in how the packagement management
code is handled. Changes include:
* Backwards compatible change to cc_package_update_upgrade_install
  schema to allow explicitly specifying the package manager to use
* Create PackageManager base class that new package manager classes
  can inherit from
* Allow distros to specify the package managers they support with
  generic install code to install from any of the supported
  package managers
* Create new snap.py and apt.py classes for anything snap and APT
  related respectively
* Move all APT functionality out of debian.py and into apt.py and update
  callers appropriately
* Pull the packaging related calls out of child distro classes and into
  `distros/__init__.py` so distro code continues to work once package
  management code is factored out.
* Add and update tests

Note that this currently only affects debian and ubuntu, along with apt
and snap. Migrating other package managers should be straightforward
enough, but can be done in later PRs.

cloudinit/config/cc_package_update_upgrade_install.py

@@ -44,6 +44,12 @@
               - pwgen
               - pastebinit
               - [libpython3.8, 3.8.10-0ubuntu1~20.04.2]
+              - snap:
+                - certbot
+                - [juju, --edge]
+                - [lxd, --channel=5.15/stable]
+              - apt:
+                - mg
             package_update: true
             package_upgrade: true
             package_reboot_if_required: true
@@ -96,7 +102,7 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None:
     pkglist = util.get_cfg_option_list(cfg, "packages", [])
 
     errors = []
-    if update or len(pkglist) or upgrade:
+    if update or upgrade:
         try:
             cloud.distro.update_package_sources()
         except Exception as e:

cloudinit/config/schemas/schema-cloud-config-v1.json

@@ -453,6 +453,21 @@
       },
       "additionalProperties": true
     },
+    "package_item_definition": {
+      "oneOf": [
+        {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "minItems": 2,
+          "maxItems": 2
+        },
+        {
+          "type": "string"
+        }
+      ]
+    },
     "cc_ansible": {
       "type": "object",
       "properties": {
@@ -1924,19 +1939,29 @@
       "properties": {
         "packages": {
           "type": "array",
-          "description": "A list of packages to install. Each entry in the list can be either a package name or a list with two entries, the first being the package name and the second being the specific package version to install.",
+          "description": "An array containing either a package specification, or an object consisting of a package manager key having a package specification value . A package specification can be either a package name or a list with two entries, the first being the package name and the second being the specific package version to install.",
           "items": {
             "oneOf": [
               {
-                "type": "array",
-                "items": {
-                  "type": "string"
+                "type": "object",
+                "properties": {
+                  "apt": {
+                    "type": "array",
+                    "items": {
+                      "$ref": "#/$defs/package_item_definition"
+                    }
+                  },
+                  "snap": {
+                    "type": "array",
+                    "items": {
+                      "$ref": "#/$defs/package_item_definition"
+                    }
+                  }
                 },
-                "minItems": 2,
-                "maxItems": 2
+                "additionalProperties": false
               },
               {
-                "type": "string"
+                "$ref": "#/$defs/package_item_definition"
               }
             ]
           },

cloudinit/distros/__init__.py

@@ -16,11 +16,24 @@
 import stat
 import string
 import urllib.parse
+from collections import defaultdict
 from io import StringIO
-from typing import Any, Mapping, MutableMapping, Optional, Type
+from typing import (
+    Any,
+    Dict,
+    Iterable,
+    List,
+    Mapping,
+    MutableMapping,
+    Optional,
+    Set,
+    Tuple,
+    Type,
+)
 
 import cloudinit.net.netops.iproute2 as iproute2
 from cloudinit import (
+    helpers,
     importer,
     net,
     persistence,
@@ -31,6 +44,8 @@
     util,
 )
 from cloudinit.distros.networking import LinuxNetworking, Networking
+from cloudinit.distros.package_management.package_manager import PackageManager
+from cloudinit.distros.package_management.utils import known_package_managers
 from cloudinit.distros.parsers import hosts
 from cloudinit.features import ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES
 from cloudinit.net import activators, dhcp, eni, network_state, renderers
@@ -126,6 +141,8 @@ def __init__(self, name, cfg, paths):
             dhcp.Udhcpc,
         ]
         self.net_ops = iproute2.Iproute2
+        self._runner = helpers.Runners(paths)
+        self.package_managers: List[PackageManager] = []
 
     def _unpickle(self, ci_pkl_version: int) -> None:
         """Perform deserialization fixes for Distro."""
@@ -139,9 +156,84 @@ def _unpickle(self, ci_pkl_version: int) -> None:
             # missing expected instance state otherwise.
             self.networking = self.networking_cls()
 
-    @abc.abstractmethod
+    def _extract_package_by_manager(
+        self, pkglist: Iterable
+    ) -> Tuple[Dict[Type[PackageManager], Set], Set]:
+        """Transform the generic package list to package by package manager.
+
+        Additionally, include list of generic packages
+        """
+        packages_by_manager = defaultdict(set)
+        generic_packages: Set = set()
+        for entry in pkglist:
+            if isinstance(entry, dict):
+                for package_manager, package_list in entry.items():
+                    for definition in package_list:
+                        if isinstance(definition, list):
+                            definition = tuple(definition)
+                        try:
+                            packages_by_manager[
+                                known_package_managers[package_manager]
+                            ].add(definition)
+                        except KeyError:
+                            LOG.error(
+                                "Cannot install packages under '%s' as it is "
+                                "not a supported package manager!",
+                                package_manager,
+                            )
+            elif isinstance(entry, str):
+                generic_packages.add(entry)
+            else:
+                raise ValueError(
+                    "Invalid 'packages' yaml specification. "
+                    "Check schema definition."
+                )
+        return dict(packages_by_manager), generic_packages
+
     def install_packages(self, pkglist):
-        raise NotImplementedError()
+        error_message = (
+            "Failed to install the following packages: %s. "
+            "See associated package manager logs for more details."
+        )
+        # If an entry hasn't been included with an explicit package name,
+        # add it to a 'generic' list of packages
+        (
+            packages_by_manager,
+            generic_packages,
+        ) = self._extract_package_by_manager(pkglist)
+
+        # First install packages using package manager(s)
+        # supported by the distro
+        uninstalled = []
+        for manager in self.package_managers:
+            to_try = (
+                packages_by_manager.get(manager.__class__, set())
+                | generic_packages
+            )
+            if not to_try:
+                continue
+            uninstalled = manager.install_packages(to_try)
+            failed = {
+                pkg for pkg in uninstalled if pkg not in generic_packages
+            }
+            if failed:
+                LOG.error(error_message, failed)
+            generic_packages = set(uninstalled)
+
+        # Now attempt any specified package managers not explicitly supported
+        # by distro
+        for manager, packages in packages_by_manager.items():
+            if manager.name in [p.name for p in self.package_managers]:
+                # We already installed/attempted these; don't try again
+                continue
+            uninstalled.extend(
+                manager.from_config(self._runner, self._cfg).install_packages(
+                    pkglist=packages
+                )
+            )
+
+        if uninstalled:
+            LOG.error(error_message, uninstalled)
 
     def _write_network(self, settings):
         """Deprecated. Remove if/when arch and gentoo support renderers."""
@@ -202,11 +294,19 @@ def uses_systemd():
 
     @abc.abstractmethod
     def package_command(self, command, args=None, pkgs=None):
+        # Long-term, this method should be removed and callers refactored.
+        # Very few commands are going to be consistent across all package
+        # managers.
         raise NotImplementedError()
 
-    @abc.abstractmethod
     def update_package_sources(self):
-        raise NotImplementedError()
+        for manager in self.package_managers:
+            try:
+                manager.update_package_sources()
+            except Exception as e:
+                LOG.error(
+                    "Failed to update package using %s: %s", manager.name, e
+                )
 
     def get_primary_arch(self):
         arch = os.uname()[4]