fix(apt_configure): disable sources.list if rendering deb822 (canonic…

…al#4699)

If deb822 sources format is used and a preexisting
/etc/apt/sources.list exists, then disable the old sources file.
This prevents user-facing warnings while operating with apt or
apt-get and other possible conflicts.
    
This is safe to do as cloud-init owns the base APT configuration which
typically/historically involved overwriting /etc/apt/sources.list

Fixes canonicalGH-4691
LP: #2045086

cloudinit/config/cc_apt_configure.py

@@ -604,7 +604,6 @@ def get_apt_cfg() -> Dict[str, str]:
             "Dir::Etc::sourceparts", DEFAULT_APT_CFG["Dir::Etc::sourceparts"]
         )
     except ImportError:
-
         try:
             apt_dump, _ = subp.subp(["apt-config", "dump"])
         except subp.ProcessExecutionError:
@@ -680,7 +679,20 @@ def generate_sources_list(cfg, release, mirrors, cloud):
             )
             aptsrc_file = apt_sources_list
     disabled = disable_suites(cfg.get("disable_suites"), rendered, release)
+    disable_apt_sources_list = False
+    if aptsrc_file == apt_sources_deb822 and os.path.exists(apt_sources_list):
+        disable_apt_sources_list = True
     util.write_file(aptsrc_file, disabled, mode=0o644)
+    if disable_apt_sources_list:
+        LOG.warning(
+            "Disabling %s to favor deb822 source format", apt_sources_list
+        )
+        content = pathlib.Path(apt_sources_list).read_bytes()
+        content = b"# disabled by cloud-init\n" + content
+        util.rename(apt_sources_list, f"{apt_sources_list}.disabled")
+        util.write_file(
+            f"{apt_sources_list}.disabled", content, preserve_mode=True
+        )
 
 
 def add_apt_key_raw(key, file_name, hardened=False):

tests/integration_tests/modules/test_apt_functionality.py

@@ -336,6 +336,21 @@ def test_security(self, class_client: IntegrationInstance):
             assert 3 == sources_list.count(sec_deb_line)
             assert 3 == sources_list.count(sec_src_deb_line)
 
+    def test_no_duplicate_apt_sources(self, class_client: IntegrationInstance):
+        r = class_client.execute("apt-get update", use_sudo=True)
+        assert not re.match(
+            r"^W: Target Packages .+ is configured multiple times in", r.stderr
+        )
+
+    def test_disabled_apt_sources(self, class_client: IntegrationInstance):
+        feature_deb822 = is_true(
+            get_feature_flag_value(class_client, "APT_DEB822_SOURCE_LIST_FILE")
+        )
+        if feature_deb822:
+            assert class_client.execute(
+                f"test -f {ORIG_SOURCES_FILE}"
+            ).failed, f"Found unexpected {ORIG_SOURCES_FILE}"
+
 
 DEFAULT_DATA_WITH_URI = _DEFAULT_DATA.format(
     uri='uri: "http://something.random.invalid/ubuntu"'

tests/integration_tests/util.py

@@ -67,6 +67,9 @@ def verify_clean_log(log: str, ignore_deprecations: bool = True):
         # Ubuntu lxd storage
         "thinpool by default on Ubuntu due to LP #1982780",
         "WARNING]: Could not match supplied host pattern, ignoring:",
+        # Old Ubuntu cloud-images contain /etc/apt/sources.list
+        "WARNING]: Disabling /etc/apt/sources.list to favor deb822 source"
+        " format",
     ]
     traceback_texts = []
     if "install canonical-livepatch" in log:

tests/unittests/config/test_cc_apt_configure.py

@@ -3,9 +3,12 @@
 """ Tests for cc_apt_configure module """
 
 import re
+from pathlib import Path
+from unittest import mock
 
 import pytest
 
+from cloudinit import features
 from cloudinit.config import cc_apt_configure
 from cloudinit.config.schema import (
     SchemaValidationError,
@@ -15,6 +18,8 @@
 from tests.unittests.helpers import skipUnlessJsonSchema
 from tests.unittests.util import get_cloud
 
+M_PATH = "cloudinit.config.cc_apt_configure."
+
 
 class TestAPTConfigureSchema:
     @pytest.mark.parametrize(
@@ -266,3 +271,33 @@ def fake_which(cmd):
             install_packages.assert_called_once_with(expected_install)
         else:
             install_packages.assert_not_called()
+
+
+class TestAptConfigure:
+    @mock.patch(M_PATH + "get_apt_cfg")
+    def test_disable_source(self, m_get_apt_cfg, tmpdir):
+        m_get_apt_cfg.return_value = {
+            "sourcelist": f"{tmpdir}/etc/apt/sources.list",
+            "sourceparts": f"{tmpdir}/etc/apt/sources.list.d/",
+        }
+        cloud = get_cloud("ubuntu")
+        features.APT_DEB822_SOURCE_LIST_FILE = True
+        sources_file = tmpdir.join("/etc/apt/sources.list")
+        Path(sources_file).parent.mkdir(parents=True, exist_ok=True)
+        with open(sources_file, "w") as f:
+            f.write("content")
+
+        cfg = {
+            "sources_list": """\
+Types: deb
+URIs: {{mirror}}
+Suites: {{codename}} {{codename}}-updates {{codename}}-backports
+Components: main restricted universe multiverse
+Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg"""
+        }
+        cc_apt_configure.generate_sources_list(cfg, "noble", {}, cloud)
+        assert not Path(f"{tmpdir}/etc/apt/sources.list").exists()
+        assert (
+            Path(f"{tmpdir}/etc/apt/sources.list.disabled").read_text()
+            == "# disabled by cloud-init\ncontent"
+        )