From a2f3cc1f4a96d7f66c571c215164b8133e3c23b2 Mon Sep 17 00:00:00 2001 From: Daniel Madej Date: Tue, 10 Sep 2024 10:39:39 +0200 Subject: [PATCH 1/3] Mtab check optional There are situations when /etc/mtab is not present in the system (e.g. in certain container images). This blocks stop/remove operations. With making this check optional the duty of checking mounts falls to kernel. Test modified to check operations with and without mtab. Signed-off-by: Daniel Madej --- casadm/cas_lib.c | 71 +++++++++---------- test/functional/api/cas/cli_messages.py | 17 +++-- .../test_fault_injection_with_mounted_core.py | 67 ++++++++++++++++- 3 files changed, 113 insertions(+), 42 deletions(-) diff --git a/casadm/cas_lib.c b/casadm/cas_lib.c index 27cc0ba16..66720185f 100644 --- a/casadm/cas_lib.c +++ b/casadm/cas_lib.c @@ -45,8 +45,8 @@ #define CORE_ADD_MAX_TIMEOUT 30 -int is_cache_mounted(int cache_id); -int is_core_mounted(int cache_id, int core_id); +bool cache_mounts_detected(int cache_id); +bool core_mounts_detected(int cache_id, int core_id); /* KCAS_IOCTL_CACHE_CHECK_DEVICE wrapper */ int _check_cache_device(const char *device_path, @@ -1119,7 +1119,7 @@ int stop_cache(uint16_t cache_id, int flush) int status; /* Don't stop instance with mounted filesystem */ - if (is_cache_mounted(cache_id) == FAILURE) + if (cache_mounts_detected(cache_id)) return FAILURE; fd = open_ctrl_device(); @@ -1803,58 +1803,60 @@ int add_core(unsigned int cache_id, unsigned int core_id, const char *core_devic return SUCCESS; } -int _check_if_mounted(int cache_id, int core_id) +bool _device_mounts_detected(int cache_id, int core_id) { FILE *mtab; struct mntent *mstruct; char dev_buf[80]; - int difference = 0, error = 0; + int no_match = 0, error = 0, cmplen = 0; if (core_id >= 0) { /* verify if specific core is mounted */ - snprintf(dev_buf, sizeof(dev_buf), "/dev/cas%d-%d", cache_id, core_id); + cmplen = snprintf(dev_buf, sizeof(dev_buf), "/dev/cas%d-%d", cache_id, core_id); } else { - /* verify if any core from given cache is mounted */ - snprintf(dev_buf, sizeof(dev_buf), "/dev/cas%d-", cache_id); + /* verify if any core from given cache is mounted + do not compare terminating NULL for cache */ + cmplen = snprintf(dev_buf, sizeof(dev_buf), "/dev/cas%d-", cache_id) - 1; } mtab = setmntent("/etc/mtab", "r"); - if (!mtab) - { - cas_printf(LOG_ERR, "Error while accessing /etc/mtab\n"); - return FAILURE; + if (!mtab) { + /* if /etc/mtab not found then the kernel will check for mounts */ + return false; } while ((mstruct = getmntent(mtab)) != NULL) { - error = strcmp_s(mstruct->mnt_fsname, PATH_MAX, dev_buf, &difference); + error = strcmp_s(mstruct->mnt_fsname, cmplen, dev_buf, &no_match); /* mstruct->mnt_fsname is /dev/... block device path, not a mountpoint */ if (error != EOK) - return FAILURE; - if (!difference) { - if (core_id<0) { - cas_printf(LOG_ERR, - "Can't stop cache instance %d. Device %s is mounted!\n", - cache_id, mstruct->mnt_fsname); - } else { - cas_printf(LOG_ERR, - "Can't remove core %d from cache %d." - " Device %s is mounted!\n", - core_id, cache_id, mstruct->mnt_fsname); - } - return FAILURE; + return false; + if (no_match) + continue; + + if (core_id < 0) { + cas_printf(LOG_ERR, + "Can't stop cache instance %d. Device %s is mounted!\n", + cache_id, mstruct->mnt_fsname); + } else { + cas_printf(LOG_ERR, + "Can't remove core %d from cache %d." + " Device %s is mounted!\n", + core_id, cache_id, mstruct->mnt_fsname); } + + return true; } - return SUCCESS; + return false; } -int is_cache_mounted(int cache_id) +bool cache_mounts_detected(int cache_id) { - return _check_if_mounted(cache_id, -1); + return _device_mounts_detected(cache_id, -1); } -int is_core_mounted(int cache_id, int core_id) +bool core_mounts_detected(int cache_id, int core_id) { - return _check_if_mounted(cache_id, core_id); + return _device_mounts_detected(cache_id, core_id); } int remove_core(unsigned int cache_id, unsigned int core_id, @@ -1864,7 +1866,7 @@ int remove_core(unsigned int cache_id, unsigned int core_id, struct kcas_remove_core cmd; /* don't even attempt ioctl if filesystem is mounted */ - if (SUCCESS != is_core_mounted(cache_id, core_id)) { + if (core_mounts_detected(cache_id, core_id)) { return FAILURE; } @@ -1929,11 +1931,6 @@ int remove_inactive_core(unsigned int cache_id, unsigned int core_id, int fd = 0; struct kcas_remove_inactive cmd; - /* don't even attempt ioctl if filesystem is mounted */ - if (SUCCESS != is_core_mounted(cache_id, core_id)) { - return FAILURE; - } - fd = open_ctrl_device(); if (fd == -1) return FAILURE; diff --git a/test/functional/api/cas/cli_messages.py b/test/functional/api/cas/cli_messages.py index 7a1cc92e4..b275737ac 100644 --- a/test/functional/api/cas/cli_messages.py +++ b/test/functional/api/cas/cli_messages.py @@ -1,6 +1,6 @@ # # Copyright(c) 2019-2022 Intel Corporation -# Copyright(c) 2024 Huawei Technologies Co., Ltd. +# Copyright(c) 2024 Huawei Technologies # SPDX-License-Identifier: BSD-3-Clause # @@ -84,11 +84,20 @@ ] remove_mounted_core = [ - r"Can\'t remove core \d+ from cache \d+\. Device /dev/cas\d+-\d+ is mounted\!" + r"Can\'t remove core \d+ from cache \d+\. Device /dev/cas\d+-\d+(p\d+|) is mounted\!" +] + +remove_mounted_core_kernel = [ + r"Error while removing core device \d+ from cache instance \d+", + r"Device opens or mount are pending to this cache", ] stop_cache_mounted_core = [ - r"Error while removing cache \d+", + r"Can\'t stop cache instance \d+\. Device /dev/cas\d+-\d+(p\d+|) is mounted\!" +] + +stop_cache_mounted_core_kernel = [ + r"Error while stopping cache \d+", r"Device opens or mount are pending to this cache", ] @@ -242,7 +251,7 @@ def __check_string_msg(text: str, expected_messages, negate=False): msg_ok = False elif matches and negate: TestRun.LOGGER.error( - f"Message is incorrect, expected to not find: {msg}\n " f"actual: {text}." + f"Message is incorrect, expected to not find: {msg}\n actual: {text}." ) msg_ok = False return msg_ok diff --git a/test/functional/tests/fault_injection/test_fault_injection_with_mounted_core.py b/test/functional/tests/fault_injection/test_fault_injection_with_mounted_core.py index 80987e7de..8abf76326 100644 --- a/test/functional/tests/fault_injection/test_fault_injection_with_mounted_core.py +++ b/test/functional/tests/fault_injection/test_fault_injection_with_mounted_core.py @@ -1,5 +1,6 @@ # # Copyright(c) 2019-2022 Intel Corporation +# Copyright(c) 2024 Huawei Technologies # SPDX-License-Identifier: BSD-3-Clause # @@ -10,13 +11,14 @@ from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan from test_tools import fs_utils from test_tools.disk_utils import Filesystem +from test_utils.filesystem.file import File +from test_utils.filesystem.symlink import Symlink from test_utils.size import Size, Unit mount_point = "/mnt/cas" test_file_path = f"{mount_point}/test_file" - @pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand])) @pytest.mark.require_disk("core", DiskTypeLowerThan("cache")) def test_load_cache_with_mounted_core(): @@ -89,6 +91,7 @@ def test_stop_cache_with_mounted_partition(): - No system crash. - Unable to stop cache when partition is mounted. - Unable to remove core when partition is mounted. + - casadm displays proper message. """ with TestRun.step("Prepare cache and core devices. Start CAS."): cache_dev = TestRun.disks['cache'] @@ -104,6 +107,10 @@ def test_stop_cache_with_mounted_partition(): core = cache.add_core(core_part) core.mount(mount_point) + with TestRun.step("Ensure /etc/mtab exists."): + if not fs_utils.check_if_file_exists("/etc/mtab"): + Symlink.create_symlink("/proc/self/mounts", "/etc/mtab") + with TestRun.step("Try to remove core from cache."): output = TestRun.executor.run_expect_fail(cli.remove_core_cmd(cache_id=str(cache.cache_id), core_id=str(core.core_id))) @@ -119,3 +126,61 @@ def test_stop_cache_with_mounted_partition(): with TestRun.step("Stop cache."): casadm.stop_all_caches() + +@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand])) +@pytest.mark.require_disk("core", DiskTypeLowerThan("cache")) +def test_stop_cache_with_mounted_partition_no_mtab(): + """ + title: Test for removing core and stopping cache when casadm is unable to check mounts. + description: | + Negative test of the ability of CAS to remove core and stop cache while core + is still mounted and casadm is unable to check mounts. + pass_criteria: + - No system crash. + - Unable to stop cache when partition is mounted. + - Unable to remove core when partition is mounted. + - casadm displays proper message informing that mount check was performed by kernel module + """ + with TestRun.step("Prepare cache and core devices. Start CAS."): + cache_dev = TestRun.disks['cache'] + cache_dev.create_partitions([Size(1, Unit.GibiByte)]) + cache_part = cache_dev.partitions[0] + core_dev = TestRun.disks['core'] + core_dev.create_partitions([Size(4, Unit.GibiByte)]) + core_part = core_dev.partitions[0] + cache = casadm.start_cache([cache_part], force=True) + + with TestRun.step("Add core device with xfs filesystem and mount it."): + core_part.create_filesystem(Filesystem.xfs) + core = cache.add_core(core_part) + core.mount(mount_point) + + with TestRun.step("Move /etc/mtab"): + if fs_utils.check_if_file_exists("/etc/mtab"): + mtab = File("/etc/mtab") + else: + mtab = Symlink.create_symlink("/proc/self/mounts", "/etc/mtab") + mtab.move("/tmp") + + with TestRun.step("Try to remove core from cache."): + output = TestRun.executor.run_expect_fail(cli.remove_core_cmd(cache_id=str(cache.cache_id), + core_id=str(core.core_id))) + cli_messages.check_stderr_msg(output, cli_messages.remove_mounted_core_kernel) + + with TestRun.step("Try to stop CAS."): + output = TestRun.executor.run_expect_fail(cli.stop_cmd(cache_id=str(cache.cache_id))) + cli_messages.check_stderr_msg(output, cli_messages.stop_cache_mounted_core_kernel) + + with TestRun.step("Unmount core device."): + core.unmount() + + with TestRun.step("Remove core."): + core.remove_core() + + with TestRun.step("Re-add core."): + cache.add_core(core_part) + + with TestRun.step("Stop cache."): + cache.stop() + + mtab.move("/etc") From f11f14d31a5ed252b5fad3fc50a6436d173297e6 Mon Sep 17 00:00:00 2001 From: Daniel Madej Date: Tue, 10 Sep 2024 17:33:32 +0200 Subject: [PATCH 2/3] Refactor mounted device checks Calling functions now print error messages. All the mounted devices are printed (not just the first one). Signed-off-by: Daniel Madej --- casadm/cas_lib.c | 82 +++++++++++++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 33 deletions(-) diff --git a/casadm/cas_lib.c b/casadm/cas_lib.c index 66720185f..c314d7cc3 100644 --- a/casadm/cas_lib.c +++ b/casadm/cas_lib.c @@ -45,8 +45,8 @@ #define CORE_ADD_MAX_TIMEOUT 30 -bool cache_mounts_detected(int cache_id); -bool core_mounts_detected(int cache_id, int core_id); +bool device_mounts_detected(const char *pattern, int cmplen); +void print_mounted_devices(const char *pattern, int cmplen); /* KCAS_IOCTL_CACHE_CHECK_DEVICE wrapper */ int _check_cache_device(const char *device_path, @@ -1119,8 +1119,16 @@ int stop_cache(uint16_t cache_id, int flush) int status; /* Don't stop instance with mounted filesystem */ - if (cache_mounts_detected(cache_id)) + int cmplen = 0; + char pattern[80]; + + /* verify if any core (or core partition) for this cache is mounted */ + cmplen = snprintf(pattern, sizeof(pattern), "/dev/cas%d-", cache_id) - 1; + if (device_mounts_detected(pattern, cmplen)) { + cas_printf(LOG_ERR, "Can't stop cache instance %d due to mounted devices:\n", cache_id); + print_mounted_devices(pattern, cmplen); return FAILURE; + } fd = open_ctrl_device(); if (fd == -1) @@ -1803,20 +1811,11 @@ int add_core(unsigned int cache_id, unsigned int core_id, const char *core_devic return SUCCESS; } -bool _device_mounts_detected(int cache_id, int core_id) +bool device_mounts_detected(const char *pattern, int cmplen) { FILE *mtab; struct mntent *mstruct; - char dev_buf[80]; - int no_match = 0, error = 0, cmplen = 0; - if (core_id >= 0) { - /* verify if specific core is mounted */ - cmplen = snprintf(dev_buf, sizeof(dev_buf), "/dev/cas%d-%d", cache_id, core_id); - } else { - /* verify if any core from given cache is mounted - do not compare terminating NULL for cache */ - cmplen = snprintf(dev_buf, sizeof(dev_buf), "/dev/cas%d-", cache_id) - 1; - } + int no_match = 0, error = 0; mtab = setmntent("/etc/mtab", "r"); if (!mtab) { @@ -1825,38 +1824,39 @@ bool _device_mounts_detected(int cache_id, int core_id) } while ((mstruct = getmntent(mtab)) != NULL) { - error = strcmp_s(mstruct->mnt_fsname, cmplen, dev_buf, &no_match); + error = strcmp_s(mstruct->mnt_fsname, cmplen, pattern, &no_match); /* mstruct->mnt_fsname is /dev/... block device path, not a mountpoint */ if (error != EOK) return false; if (no_match) continue; - if (core_id < 0) { - cas_printf(LOG_ERR, - "Can't stop cache instance %d. Device %s is mounted!\n", - cache_id, mstruct->mnt_fsname); - } else { - cas_printf(LOG_ERR, - "Can't remove core %d from cache %d." - " Device %s is mounted!\n", - core_id, cache_id, mstruct->mnt_fsname); - } - return true; } return false; } -bool cache_mounts_detected(int cache_id) +void print_mounted_devices(const char *pattern, int cmplen) { - return _device_mounts_detected(cache_id, -1); -} + FILE *mtab; + struct mntent *mstruct; + int no_match = 0, error = 0; -bool core_mounts_detected(int cache_id, int core_id) -{ - return _device_mounts_detected(cache_id, core_id); + mtab = setmntent("/etc/mtab", "r"); + if (!mtab) { + /* should exist, but if /etc/mtab not found we cannot print mounted devices */ + return; + } + + while ((mstruct = getmntent(mtab)) != NULL) { + error = strcmp_s(mstruct->mnt_fsname, cmplen, pattern, &no_match); + /* mstruct->mnt_fsname is /dev/... block device path, not a mountpoint */ + if (error != EOK || no_match) + continue; + + cas_printf(LOG_ERR, "%s\n", mstruct->mnt_fsname); + } } int remove_core(unsigned int cache_id, unsigned int core_id, @@ -1866,7 +1866,23 @@ int remove_core(unsigned int cache_id, unsigned int core_id, struct kcas_remove_core cmd; /* don't even attempt ioctl if filesystem is mounted */ - if (core_mounts_detected(cache_id, core_id)) { + bool mounts_detected = false; + int cmplen = 0; + char pattern[80]; + + /* verify if specific core is mounted */ + cmplen = snprintf(pattern, sizeof(pattern), "/dev/cas%d-%d", cache_id, core_id); + mounts_detected = device_mounts_detected(pattern, cmplen) + if (!mounts_detected) { + /* verify if any partition of the core is mounted */ + cmplen = snprintf(pattern, sizeof(pattern), "/dev/cas%d-%dp", cache_id, core_id) - 1; + mounts_detected = device_mounts_detected(pattern, cmplen); + } + if (mounts_detected) { + cas_printf(LOG_ERR, "Can't remove core %d from " + "cache %d due to mounted devices:\n", + core_id, cache_id); + print_mounted_devices(pattern, cmplen); return FAILURE; } From 0c0b10535ed335b8acdd67443163deab214fe493 Mon Sep 17 00:00:00 2001 From: Daniel Madej Date: Thu, 12 Sep 2024 12:47:46 +0200 Subject: [PATCH 3/3] [tests] Update CLI messages and test Signed-off-by: Daniel Madej --- test/functional/api/cas/cli_messages.py | 4 +- .../test_fault_injection_with_mounted_core.py | 61 ++++++++++++++----- 2 files changed, 47 insertions(+), 18 deletions(-) diff --git a/test/functional/api/cas/cli_messages.py b/test/functional/api/cas/cli_messages.py index b275737ac..fc3104bcb 100644 --- a/test/functional/api/cas/cli_messages.py +++ b/test/functional/api/cas/cli_messages.py @@ -84,7 +84,7 @@ ] remove_mounted_core = [ - r"Can\'t remove core \d+ from cache \d+\. Device /dev/cas\d+-\d+(p\d+|) is mounted\!" + r"Can\'t remove core \d+ from cache \d+ due to mounted devices:" ] remove_mounted_core_kernel = [ @@ -93,7 +93,7 @@ ] stop_cache_mounted_core = [ - r"Can\'t stop cache instance \d+\. Device /dev/cas\d+-\d+(p\d+|) is mounted\!" + r"Can\'t stop cache instance \d+ due to mounted devices:" ] stop_cache_mounted_core_kernel = [ diff --git a/test/functional/tests/fault_injection/test_fault_injection_with_mounted_core.py b/test/functional/tests/fault_injection/test_fault_injection_with_mounted_core.py index 8abf76326..d8ac83436 100644 --- a/test/functional/tests/fault_injection/test_fault_injection_with_mounted_core.py +++ b/test/functional/tests/fault_injection/test_fault_injection_with_mounted_core.py @@ -9,13 +9,13 @@ from api.cas import casadm, casadm_parser, cli, cli_messages from core.test_run import TestRun from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan -from test_tools import fs_utils +from test_tools import fs_utils, disk_utils from test_tools.disk_utils import Filesystem from test_utils.filesystem.file import File from test_utils.filesystem.symlink import Symlink from test_utils.size import Size, Unit -mount_point = "/mnt/cas" +mount_point, mount_point2 = "/mnt/cas", "/mnt/cas2" test_file_path = f"{mount_point}/test_file" @@ -81,6 +81,7 @@ def test_load_cache_with_mounted_core(): @pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand])) @pytest.mark.require_disk("core", DiskTypeLowerThan("cache")) +@pytest.mark.require_disk("core2", DiskTypeLowerThan("cache")) def test_stop_cache_with_mounted_partition(): """ title: Fault injection test for removing core and stopping cache with mounted core. @@ -93,35 +94,63 @@ def test_stop_cache_with_mounted_partition(): - Unable to remove core when partition is mounted. - casadm displays proper message. """ - with TestRun.step("Prepare cache and core devices. Start CAS."): + with TestRun.step("Prepare cache device."): cache_dev = TestRun.disks['cache'] cache_dev.create_partitions([Size(1, Unit.GibiByte)]) cache_part = cache_dev.partitions[0] - core_dev = TestRun.disks['core'] - core_dev.create_partitions([Size(4, Unit.GibiByte)]) - core_part = core_dev.partitions[0] + + with TestRun.step("Prepare 2 core devices."): + core_dev, core_dev2 = TestRun.disks['core'], TestRun.disks['core2'] + + with TestRun.step("Start cache."): cache = casadm.start_cache(cache_part, force=True) - with TestRun.step("Add core device with xfs filesystem and mount it."): - core_part.create_filesystem(Filesystem.xfs) - core = cache.add_core(core_part) - core.mount(mount_point) + with TestRun.step("Add core devices to cache."): + core = cache.add_core(core_dev) + core2 = cache.add_core(core_dev2) + + with TestRun.step("Create partitions on one exported object."): + core.block_size = Size(disk_utils.get_block_size(core.get_device_id())) + disk_utils.create_partitions(core, 2 * [Size(4, Unit.GibiByte)]) + fs_part = core.partitions[0] + + with TestRun.step("Create xfs filesystems on one exported object partition " + "and on the non-partitioned exported object."): + fs_part.create_filesystem(Filesystem.xfs) + core2.create_filesystem(Filesystem.xfs) + + with TestRun.step("Mount created filesystems."): + fs_part.mount(mount_point) + core2.mount(mount_point2) with TestRun.step("Ensure /etc/mtab exists."): if not fs_utils.check_if_file_exists("/etc/mtab"): Symlink.create_symlink("/proc/self/mounts", "/etc/mtab") - with TestRun.step("Try to remove core from cache."): + with TestRun.step("Try to remove the core with partitions from cache."): output = TestRun.executor.run_expect_fail(cli.remove_core_cmd(cache_id=str(cache.cache_id), core_id=str(core.core_id))) - cli_messages.check_stderr_msg(output, cli_messages.remove_mounted_core) + messages = cli_messages.remove_mounted_core.copy() + messages.append(fs_part.path) + cli_messages.check_stderr_msg(output, messages) + + with TestRun.step("Try to remove the core without partitions from cache."): + output = TestRun.executor.run_expect_fail(cli.remove_core_cmd(cache_id=str(cache.cache_id), + core_id=str(core2.core_id))) + messages = cli_messages.remove_mounted_core.copy() + messages.append(core2.path) + cli_messages.check_stderr_msg(output, messages) with TestRun.step("Try to stop CAS."): output = TestRun.executor.run_expect_fail(cli.stop_cmd(cache_id=str(cache.cache_id))) - cli_messages.check_stderr_msg(output, cli_messages.stop_cache_mounted_core) + messages = cli_messages.stop_cache_mounted_core.copy() + messages.append(fs_part.path) + messages.append(core2.path) + cli_messages.check_stderr_msg(output, messages) - with TestRun.step("Unmount core device."): - core.unmount() + with TestRun.step("Unmount core devices."): + fs_part.unmount() + core2.unmount() with TestRun.step("Stop cache."): casadm.stop_all_caches() @@ -148,7 +177,7 @@ def test_stop_cache_with_mounted_partition_no_mtab(): core_dev = TestRun.disks['core'] core_dev.create_partitions([Size(4, Unit.GibiByte)]) core_part = core_dev.partitions[0] - cache = casadm.start_cache([cache_part], force=True) + cache = casadm.start_cache(cache_part, force=True) with TestRun.step("Add core device with xfs filesystem and mount it."): core_part.create_filesystem(Filesystem.xfs)