Replace flake8 with ruff (#4314)

Ruff is much faster and replicates everything flake8 can do. Additionally, I added a few more checks that seemed relevant to our project and would not require many changes. Long term, we should be able to replace isort with ruff as well, and maybe eventually pylint, though ruff is not yet able to do everything that pylint can.
canonical · Aug 4, 2023 · e2ae755 · e2ae755
1 parent d41264c
commit e2ae755
Show file tree

Hide file tree

Showing 26 changed files with 100 additions and 98 deletions.
diff --git a/.github/workflows/check_format.yml b/.github/workflows/check_format.yml
@@ -14,7 +14,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        env: [flake8, mypy, pylint, black, isort]
+        env: [ruff, mypy, pylint, black, isort]
         lint-with:
           - {tip-versions: false, os: ubuntu-20.04}
           - {tip-versions: true, os: ubuntu-latest}

diff --git a/Makefile b/Makefile
@@ -21,10 +21,10 @@ all: check
 
 check: check_version test yaml
 
-style-check: flake8
+style-check: lint
 
-flake8:
-	@$(CWD)/tools/run-flake8
+lint:
+	@$(CWD)/tools/run-lint
 
 unittest: clean_pyc
 	$(PYTHON) -m pytest -v tests/unittests cloudinit
@@ -162,7 +162,7 @@ fix_spelling:
 		awk -F ': | -> ' '{printf "sed -i \047s/%s/%s/g\047 %s\n", $$2, $$3, $$1}' | \
 		sh
 
-.PHONY: all check test flake8 clean rpm srpm deb deb-src yaml
+.PHONY: all check test lint clean rpm srpm deb deb-src yaml
 .PHONY: check_version clean_pyc
 .PHONY: unittest style-check fix_spelling render-template benchmark-generator
 .PHONY: clean_pytest clean_packaging check_spelling clean_release doc
diff --git a/cloudinit/cmd/devel/make_mime.py b/cloudinit/cmd/devel/make_mime.py
@@ -32,9 +32,10 @@ def create_mime_message(files):
         )
         content_type = sub_message.get_content_type().lower()
         if content_type not in get_content_types():
-            msg = (
-                "content type %r for attachment %s " "may be incorrect!"
-            ) % (content_type, i + 1)
+            msg = ("content type %r for attachment %s may be incorrect!") % (
+                content_type,
+                i + 1,
+            )
             errors.append(msg)
         sub_messages.append(sub_message)
     combined_message = MIMEMultipart()

diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py
@@ -170,7 +170,7 @@ def attempt_cmdline_url(path, network=True, cmdline=None) -> Tuple[int, str]:
     except KeyError:
         return (logging.DEBUG, "No kernel command line url found.")
 
-    path_is_local = url.startswith("file://") or url.startswith("/")
+    path_is_local = url.startswith(("file://", "/"))
 
     if path_is_local and os.path.exists(path):
         if network:

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
@@ -928,7 +928,7 @@ def _get_key_files():
         key_files = [APT_LOCAL_KEYS] if os.path.isfile(APT_LOCAL_KEYS) else []
 
         for file in os.listdir(APT_TRUSTED_GPG_DIR):
-            if file.endswith(".gpg") or file.endswith(".asc"):
+            if file.endswith((".gpg", ".asc")):
                 key_files.append(APT_TRUSTED_GPG_DIR + file)
         return key_files if key_files else ""
 

diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py
@@ -74,7 +74,7 @@ def is_applicable(self) -> bool:
         """
         if self._files:
             for item in shlex.split(self._cmdline):
-                if item.startswith("ip=") or item.startswith("ip6="):
+                if item.startswith(("ip=", "ip6=")):
                     return True
             if os.path.exists(_OPEN_ISCSI_INTERFACE_FILE):
                 # iBft can configure networking without ip=

diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py
@@ -503,7 +503,7 @@ def get_latest_lease(lease_d=None):
             if fname.startswith("dhclient6"):
                 # avoid files that start with dhclient6 assuming dhcpv6.
                 continue
-            if not (fname.endswith(".lease") or fname.endswith(".leases")):
+            if not (fname.endswith((".lease", ".leases"))):
                 continue
 
             abs_path = os.path.join(lease_d, fname)

diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py
@@ -1049,7 +1049,7 @@ def _supported_vlan_names(rdev, vid):
 
 
 def available(target=None):
-    if not util.system_info()["variant"] in KNOWN_DISTROS:
+    if util.system_info()["variant"] not in KNOWN_DISTROS:
         return False
     if available_sysconfig(target):
         return True

diff --git a/cloudinit/sources/DataSourceAkamai.py b/cloudinit/sources/DataSourceAkamai.py
@@ -54,7 +54,6 @@ class MetadataAvailabilityResult(Enum):
 
 
 class DataSourceAkamai(sources.DataSource):
-
     dsname = "Akamai"
     local_stage = False
 
@@ -275,21 +274,18 @@ def _fetch_metadata(self, use_v6: bool = False) -> bool:
                     "Metadata-Token": token,
                 },
             )
-            self.userdata_raw = str(userdata)  # type: ignore
+            self.userdata_raw = str(userdata)
             try:
-                self.userdata_raw = b64decode(
-                    self.userdata_raw  # type: ignore
-                ).decode()
+                self.userdata_raw = b64decode(self.userdata_raw).decode()
             except binascii.Error as e:
                 LOG.warning("Failed to base64 decode userdata due to %s", e)
         except url_helper.UrlError as e:
             # we failed to retrieve data with an exception; log the error and
             # return false, indicating that we should retry using a different
             # network if possible
             LOG.warning(
-                "Failed to retrieve metadata using IPv%s due to %s" "6"
-                if use_v6
-                else "4",
+                "Failed to retrieve metadata using IPv%s due to %s",
+                "6" if use_v6 else "4",
                 e,
             )
             return False

diff --git a/cloudinit/sources/DataSourceVMware.py b/cloudinit/sources/DataSourceVMware.py
@@ -551,7 +551,7 @@ def guestinfo_get_value(key, vmware_rpctool=VMWARE_RPCTOOL):
         util.logexc(
             LOG,
             "Unexpected error while trying to get "
-            + "guestinfo value for key %s",
+            "guestinfo value for key %s",
             key,
         )
 
@@ -593,7 +593,7 @@ def guestinfo_set_value(key, value, vmware_rpctool=VMWARE_RPCTOOL):
         util.logexc(
             LOG,
             "Unexpected error while trying to set "
-            + "guestinfo key=%s to value=%s",
+            "guestinfo key=%s to value=%s",
             key,
             value,
         )
@@ -609,7 +609,7 @@ def guestinfo_redact_keys(keys, vmware_rpctool=VMWARE_RPCTOOL):
     """
     if not keys:
         return
-    if not type(keys) in (list, tuple):
+    if type(keys) not in (list, tuple):
         keys = [keys]
     for key in keys:
         key_name = get_guestinfo_key_name(key)

diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
@@ -288,7 +288,7 @@ def __init__(self, sys_cfg, distro: Distro, paths: Paths, ud_proc=None):
         self.paths = paths
         self.userdata = None
         self.metadata: dict = {}
-        self.userdata_raw = None
+        self.userdata_raw: Optional[str] = None
         self.vendordata = None
         self.vendordata2 = None
         self.vendordata_raw = None

diff --git a/cloudinit/util.py b/cloudinit/util.py
@@ -1406,7 +1406,7 @@ def find_devs_with_netbsd(
     for dev in out.stdout.split():
         if label or _type:
             mscdlabel_out, _ = subp.subp(["mscdlabel", dev], rcs=[0, 1])
-        if label and not ('label "%s"' % label) in mscdlabel_out:
+        if label and ('label "%s"' % label) not in mscdlabel_out:
             continue
         if _type == "iso9660" and "ISO filesystem" not in mscdlabel_out:
             continue
@@ -1444,15 +1444,9 @@ def find_devs_with_dragonflybsd(
     ]
 
     if criteria == "TYPE=iso9660":
-        devlist = [
-            i for i in devlist if i.startswith("cd") or i.startswith("acd")
-        ]
+        devlist = [i for i in devlist if i.startswith(("cd", "acd"))]
     elif criteria in ["LABEL=CONFIG-2", "TYPE=vfat"]:
-        devlist = [
-            i
-            for i in devlist
-            if not (i.startswith("cd") or i.startswith("acd"))
-        ]
+        devlist = [i for i in devlist if not (i.startswith(("cd", "acd")))]
     elif criteria:
         LOG.debug("Unexpected criteria: %s", criteria)
     return ["/dev/" + i for i in devlist]
@@ -2968,7 +2962,7 @@ def get_installed_packages(target=None):
             (state, pkg, _) = line.split(None, 2)
         except ValueError:
             continue
-        if state.startswith("hi") or state.startswith("ii"):
+        if state.startswith(("hi", "ii")):
             pkgs_inst.add(re.sub(":.*", "", pkg))
 
     return pkgs_inst

diff --git a/doc/rtd/conf.py b/doc/rtd/conf.py
@@ -76,8 +76,8 @@
     "light_logo": "logo.png",
     "dark_logo": "logo-dark-mode.png",
     "light_css_variables": {
-        "font-stack": "Ubuntu, -apple-system, Segoe UI, Roboto, Oxygen, Cantarell, Fira Sans, Droid Sans, Helvetica Neue, sans-serif",
-        "font-stack--monospace": "Ubuntu Mono variable, Ubuntu Mono, Consolas, Monaco, Courier, monospace",
+        "font-stack": "Ubuntu, -apple-system, Segoe UI, Roboto, Oxygen, Cantarell, Fira Sans, Droid Sans, Helvetica Neue, sans-serif",  # noqa: E501
+        "font-stack--monospace": "Ubuntu Mono variable, Ubuntu Mono, Consolas, Monaco, Courier, monospace",  # noqa: E501
         "color-foreground-primary": "#111",
         "color-foreground-secondary": "var(--color-foreground-primary)",
         "color-foreground-muted": "#333",

diff --git a/pyproject.toml b/pyproject.toml
@@ -15,24 +15,46 @@ skip = ["cloudinit/cmd/main.py", ".tox", "packages", "tools"]
 follow_imports = "silent"
 warn_unused_ignores = "true"
 warn_redundant_casts = "true"
-exclude=[]
+exclude = []
 
 [[tool.mypy.overrides]]
 module = [
-  "apport.*",
-  "BaseHTTPServer",
-  "cloudinit.feature_overrides",
-  "configobj",
-  "debconf",
-  "httplib",
-  "jsonpatch",
-  "netifaces",
-  "paramiko.*",
-  "pip.*",
-  "pycloudlib.*",
-  "responses",
-  "serial",
-  "tests.integration_tests.user_settings",
-  "uaclient.*"
+    "apport.*",
+    "BaseHTTPServer",
+    "cloudinit.feature_overrides",
+    "configobj",
+    "debconf",
+    "httplib",
+    "jsonpatch",
+    "netifaces",
+    "paramiko.*",
+    "pip.*",
+    "pycloudlib.*",
+    "responses",
+    "serial",
+    "tests.integration_tests.user_settings",
+    "uaclient.*",
 ]
 ignore_missing_imports = true
+
+[tool.ruff]
+target-version = "py37"
+line-length = 79
+# E, W, and F make up the entirety of default flake8
+select = [
+    "E",   # pycodestyle errors
+    "W",   # pycodestyle warnings
+    "F",   # pyflakes
+    "CPY", # flake8-copyright
+    "T10", # flake8-debugger
+    "ISC", # flake8-implicit-str-concat
+    "ICN", # flake8-import-conventions
+    "G",   # flake8-logging-format
+    "PIE", # flake8-pie
+    "Q",   # flake8-quotes
+]
+ignore = [
+    "E731", # Do not assign a `lambda` expression, use a `def`
+]
+[tool.ruff.per-file-ignores]
+"cloudinit/cmd/main.py" = ["E402"]
diff --git a/setup.py b/setup.py
@@ -60,7 +60,7 @@ def render_tmpl(template, mode=None):
     that files are different outside of the debian directory."""
 
     # newer versions just use install.
-    if not ("install" in sys.argv):
+    if "install" not in sys.argv:
         return template
 
     tmpl_ext = ".tmpl"

diff --git a/tests/integration_tests/modules/test_jinja_templating.py b/tests/integration_tests/modules/test_jinja_templating.py
@@ -93,7 +93,7 @@ def test_invalid_etc_cloud_substitution(client: IntegrationInstance):
     )
     client.write_to_file("/etc/cloud/cloud.cfg.d/50-no-var.cfg", no_var_part)
 
-    normal_part = "bootcmd:\n" " - echo hi > /var/tmp/bootcmd_output\n"
+    normal_part = "bootcmd:\n - echo hi > /var/tmp/bootcmd_output\n"
     client.write_to_file("/etc/cloud/cloud.cfg.d/60-normal.cfg", normal_part)
 
     client.execute("cloud-init clean --logs")

diff --git a/tests/integration_tests/test_upgrade.py b/tests/integration_tests/test_upgrade.py
@@ -143,12 +143,12 @@ def test_clean_boot_of_upgraded_package(session_cloud: IntegrationCloud):
         pre_analyze_totals = [
             x
             for x in pre_cloud_analyze.splitlines()
-            if x.startswith("Finished stage") or x.startswith("Total Time")
+            if x.startswith(("Finished stage", "Total Time"))
         ]
         post_analyze_totals = [
             x
             for x in post_cloud_analyze.splitlines()
-            if x.startswith("Finished stage") or x.startswith("Total Time")
+            if x.startswith(("Finished stage", "Total Time"))
         ]
 
         # pylint: disable=logging-format-interpolation

diff --git a/tests/unittests/cmd/test_query.py b/tests/unittests/cmd/test_query.py
@@ -35,7 +35,6 @@ def setup_mocks(mocker):
 
 @mock.patch(M_PATH + "addLogHandlerCLI", lambda *args: "")
 class TestQuery:
-
     Args = namedtuple(
         "Args",
         "debug dump_all format instance_data list_keys user_data vendor_data"
@@ -531,7 +530,7 @@ def test_handle_args_list_keys_sorts_nested_keys_when_varname(
         instance_data = tmpdir.join("instance-data")
         instance_data.write(
             '{"v1": {"v1_1": "val1.1", "v1_2": "val1.2"}, "v2":'
-            + ' {"v2_2": "val2.2"}, "top": "gun"}'
+            ' {"v2_2": "val2.2"}, "top": "gun"}'
         )
         expected = "v1_1\nv1_2\n"
         args = self.Args(
@@ -557,7 +556,7 @@ def test_handle_args_list_keys_errors_when_varname_is_not_a_dict(
         instance_data = tmpdir.join("instance-data")
         instance_data.write(
             '{"v1": {"v1_1": "val1.1", "v1_2": "val1.2"}, "v2": '
-            + '{"v2_2": "val2.2"}, "top": "gun"}'
+            '{"v2_2": "val2.2"}, "top": "gun"}'
         )
         expected_error = "--list-keys provided but 'top' is not a dict"
         args = self.Args(

diff --git a/tests/unittests/config/test_cc_disk_setup.py b/tests/unittests/config/test_cc_disk_setup.py
@@ -247,14 +247,14 @@ def test_with_cmd(self, subp, *args):
 
         self.assertIn(
             "extra_opts "
-            + "ignored because cmd was specified: mkfs -t ext4 -L with_cmd "
-            + "/dev/xdb1",
+            "ignored because cmd was specified: mkfs -t ext4 -L with_cmd "
+            "/dev/xdb1",
             self.logs.getvalue(),
         )
         self.assertIn(
             "overwrite "
-            + "ignored because cmd was specified: mkfs -t ext4 -L with_cmd "
-            + "/dev/xdb1",
+            "ignored because cmd was specified: mkfs -t ext4 -L with_cmd "
+            "/dev/xdb1",
             self.logs.getvalue(),
         )
 

diff --git a/tests/unittests/config/test_cc_ntp.py b/tests/unittests/config/test_cc_ntp.py
@@ -39,7 +39,6 @@
 
 
 class TestNtp(FilesystemMockingTestCase):
-
     with_logs = True
 
     def setUp(self):
@@ -349,8 +348,8 @@ def test_ntp_handler_real_distro_ntp_templates(self):
                     )
                     expected_content = (
                         "# cloud-init generated file\n"
-                        + "# See timesyncd.conf(5) for details.\n\n"
-                        + "[Time]\nNTP=%s %s \n"
+                        "# See timesyncd.conf(5) for details.\n\n"
+                        "[Time]\nNTP=%s %s \n"
                         % (expected_servers, expected_pools)
                     )
                     self.assertEqual(expected_content, content)

diff --git a/tests/unittests/config/test_cc_puppet.py b/tests/unittests/config/test_cc_puppet.py
@@ -77,7 +77,6 @@ def test_enable_fallback_on_failure(self, m_subp):
 
 @mock.patch("cloudinit.config.cc_puppet._manage_puppet_services")
 class TestPuppetHandle(CiTestCase):
-
     with_logs = True
 
     def setUp(self):
@@ -464,7 +463,7 @@ class TestInstallPuppetAio:
                 [mock.call([mock.ANY, "--cleanup"], capture=False)],
                 [
                     mock.call(
-                        url="https://raw.githubusercontent.com/puppetlabs/install-puppet/main/install.sh",  # noqa: 501
+                        url="https://raw.githubusercontent.com/puppetlabs/install-puppet/main/install.sh",  # noqa: E501
                         retries=5,
                     )
                 ],

diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py
@@ -381,7 +381,7 @@ def test_validateconfig_schema_emits_warning_on_missing_jsonschema(
     ):
         """Warning from validate_cloudconfig_schema when missing jsonschema."""
         schema = {"properties": {"p1": {"type": "string"}}}
-        with mock.patch.dict("sys.modules", **{"jsonschema": ImportError()}):
+        with mock.patch.dict("sys.modules", jsonschema=ImportError()):
             validate_cloudconfig_schema({"p1": -1}, schema, strict=True)
         assert "Ignoring schema validation. jsonschema is not present" in (
             caplog.text