From 650579a6e551b80ce11f3c1f4fe8e5bffd93e49a Mon Sep 17 00:00:00 2001 From: Bryan Gurney Date: Fri, 15 Nov 2019 10:35:10 -0500 Subject: [PATCH 001/109] Remove redundant parentheses from bda.rs option --- src/engine/strat_engine/backstore/metadata/bda.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engine/strat_engine/backstore/metadata/bda.rs b/src/engine/strat_engine/backstore/metadata/bda.rs index 069deb390b..19c84116e5 100644 --- a/src/engine/strat_engine/backstore/metadata/bda.rs +++ b/src/engine/strat_engine/backstore/metadata/bda.rs @@ -41,7 +41,7 @@ const STRAT_SIGBLOCK_VERSION: u8 = 1; /// Get a Stratis pool UUID and device UUID from any device. /// If there is an error while obtaining these values return the error. /// If the device does not appear to be a Stratis device, return None. -pub fn device_identifiers(f: &mut F) -> StratisResult> +pub fn device_identifiers(f: &mut F) -> StratisResult> where F: Read + Seek + SyncAll, { From 107117ec82a7cfd093073d5e92bea938f53b11d1 Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Tue, 29 Oct 2019 11:03:28 -0400 Subject: [PATCH 002/109] Fix for unnecessary clone as checked by clippy --- src/engine/strat_engine/tests/real.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engine/strat_engine/tests/real.rs b/src/engine/strat_engine/tests/real.rs index c864c36da9..15e1f1f063 100644 --- a/src/engine/strat_engine/tests/real.rs +++ b/src/engine/strat_engine/tests/real.rs @@ -181,7 +181,7 @@ fn make_linear_test_dev(devnode: &Path, start: Sectors, length: Sectors) -> Line get_dm(), DmName::new(&format!("stratis_test_{}", Uuid::new_v4())).expect("valid format"), None, - table.clone(), + table, ) .unwrap() } From ee7e3c6254f27999c5f2ee86872685df50f2e5ff Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 18 Nov 2019 08:54:42 -0500 Subject: [PATCH 003/109] Factor out generic D-Bus transformation The transformation from the result to the D-Bus value in the hash is generic, so factor it out into a separate closure passed as argument to map. Signed-off-by: mulhern --- src/dbus_api/blockdev.rs | 22 +++++++++------- src/dbus_api/filesystem.rs | 29 +++++++++++---------- src/dbus_api/pool.rs | 53 ++++++++++++++++++-------------------- 3 files changed, 52 insertions(+), 52 deletions(-) diff --git a/src/dbus_api/blockdev.rs b/src/dbus_api/blockdev.rs index 680480f22a..8a133d6b6c 100644 --- a/src/dbus_api/blockdev.rs +++ b/src/dbus_api/blockdev.rs @@ -200,19 +200,21 @@ fn get_properties_shared( let return_value: HashMap>)> = properties .unique() .filter_map(|prop| match prop.as_str() { - consts::BLOCKDEV_TOTAL_SIZE_PROP => { - let bd_size_result = blockdev_operation(m.tree, object_path.get_name(), |_, bd| { + consts::BLOCKDEV_TOTAL_SIZE_PROP => Some(( + prop, + blockdev_operation(m.tree, object_path.get_name(), |_, bd| { Ok((u128::from(*bd.size()) * devicemapper::SECTOR_SIZE as u128).to_string()) - }); - let (bd_size_success, bd_size_prop) = match bd_size_result { - Ok(bd_size) => (true, Variant(Box::new(bd_size) as Box)), - Err(e) => (false, Variant(Box::new(e) as Box)), - }; - - Some((prop, (bd_size_success, bd_size_prop))) - } + }), + )), _ => None, }) + .map(|(key, result)| { + let (success, value) = match result { + Ok(value) => (true, Variant(Box::new(value) as Box)), + Err(e) => (false, Variant(Box::new(e) as Box)), + }; + (key, (success, value)) + }) .collect(); Ok(vec![return_message.append1(return_value)]) diff --git a/src/dbus_api/filesystem.rs b/src/dbus_api/filesystem.rs index 128c28c333..cbbde4e344 100644 --- a/src/dbus_api/filesystem.rs +++ b/src/dbus_api/filesystem.rs @@ -42,22 +42,23 @@ fn get_properties_shared( let return_value: HashMap>)> = properties .unique() .filter_map(|prop| match prop.as_str() { - consts::FILESYSTEM_USED_PROP => { - let fs_used_result = - filesystem_operation(m.tree, object_path.get_name(), |(_, _, fs)| { - fs.used() - .map(|u| (*u).to_string()) - .map_err(|e| e.to_string()) - }); - let (fs_used_success, fs_used_prop) = match fs_used_result { - Ok(fs_used) => (true, Variant(Box::new(fs_used) as Box)), - Err(e) => (false, Variant(Box::new(e) as Box)), - }; - - Some((prop, (fs_used_success, fs_used_prop))) - } + consts::FILESYSTEM_USED_PROP => Some(( + prop, + filesystem_operation(m.tree, object_path.get_name(), |(_, _, fs)| { + fs.used() + .map(|u| (*u).to_string()) + .map_err(|e| e.to_string()) + }), + )), _ => None, }) + .map(|(key, result)| { + let (success, value) = match result { + Ok(value) => (true, Variant(Box::new(value) as Box)), + Err(e) => (false, Variant(Box::new(e) as Box)), + }; + (key, (success, value)) + }) .collect(); Ok(vec![return_message.append1(return_value)]) diff --git a/src/dbus_api/pool.rs b/src/dbus_api/pool.rs index 2eaa887f20..a0917a999b 100644 --- a/src/dbus_api/pool.rs +++ b/src/dbus_api/pool.rs @@ -387,36 +387,33 @@ fn get_properties_shared( let return_value: HashMap>)> = properties .unique() .filter_map(|prop| match prop.as_str() { - consts::POOL_TOTAL_SIZE_PROP => { - let total_size_result = - pool_operation(m.tree, object_path.get_name(), |(_, _, pool)| { - Ok((u128::from(*pool.total_physical_size()) - * devicemapper::SECTOR_SIZE as u128) - .to_string()) - }); - let (total_size_success, total_size_prop) = match total_size_result { - Ok(size) => (true, Variant(Box::new(size) as Box)), - Err(e) => (false, Variant(Box::new(e) as Box)), - }; - Some((prop, (total_size_success, total_size_prop))) - } - consts::POOL_TOTAL_USED_PROP => { - let total_used_result = - pool_operation(m.tree, object_path.get_name(), |(_, _, pool)| { - pool.total_physical_used() - .map_err(|e| e.to_string()) - .map(|size| { - (u128::from(*size) * devicemapper::SECTOR_SIZE as u128).to_string() - }) - }); - let (total_used_success, total_used_prop) = match total_used_result { - Ok(size) => (true, Variant(Box::new(size) as Box)), - Err(e) => (false, Variant(Box::new(e) as Box)), - }; - Some((prop, (total_used_success, total_used_prop))) - } + consts::POOL_TOTAL_SIZE_PROP => Some(( + prop, + pool_operation(m.tree, object_path.get_name(), |(_, _, pool)| { + Ok((u128::from(*pool.total_physical_size()) + * devicemapper::SECTOR_SIZE as u128) + .to_string()) + }), + )), + consts::POOL_TOTAL_USED_PROP => Some(( + prop, + pool_operation(m.tree, object_path.get_name(), |(_, _, pool)| { + pool.total_physical_used() + .map_err(|e| e.to_string()) + .map(|size| { + (u128::from(*size) * devicemapper::SECTOR_SIZE as u128).to_string() + }) + }), + )), _ => None, }) + .map(|(key, result)| { + let (success, value) = match result { + Ok(value) => (true, Variant(Box::new(value) as Box)), + Err(e) => (false, Variant(Box::new(e) as Box)), + }; + (key, (success, value)) + }) .collect(); Ok(vec![return_message.append1(return_value)]) From 27dd7c04ea6adce002b42939c7e4a36b61731dc8 Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 18 Nov 2019 10:00:22 -0500 Subject: [PATCH 004/109] Factor out mapping from result to D-Bus value into a separate function Tidier that way. Signed-off-by: mulhern --- src/dbus_api/blockdev.rs | 10 ++-------- src/dbus_api/filesystem.rs | 10 ++-------- src/dbus_api/pool.rs | 10 ++-------- src/dbus_api/util.rs | 20 ++++++++++++++++++++ 4 files changed, 26 insertions(+), 24 deletions(-) diff --git a/src/dbus_api/blockdev.rs b/src/dbus_api/blockdev.rs index 8a133d6b6c..49e72f98dc 100644 --- a/src/dbus_api/blockdev.rs +++ b/src/dbus_api/blockdev.rs @@ -22,7 +22,7 @@ use crate::{ types::{DbusContext, DbusErrorEnum, OPContext, TData}, util::{ engine_to_dbus_err_tuple, get_next_arg, get_parent, get_uuid, make_object_path, - msg_code_ok, msg_string_ok, tuple_to_option, + msg_code_ok, msg_string_ok, result_to_tuple, tuple_to_option, }, }, engine::{BlockDev, BlockDevTier, DevUuid, MaybeDbusPath, RenameAction}, @@ -208,13 +208,7 @@ fn get_properties_shared( )), _ => None, }) - .map(|(key, result)| { - let (success, value) = match result { - Ok(value) => (true, Variant(Box::new(value) as Box)), - Err(e) => (false, Variant(Box::new(e) as Box)), - }; - (key, (success, value)) - }) + .map(|(key, result)| result_to_tuple(key, result)) .collect(); Ok(vec![return_message.append1(return_value)]) diff --git a/src/dbus_api/filesystem.rs b/src/dbus_api/filesystem.rs index cbbde4e344..0e9500393f 100644 --- a/src/dbus_api/filesystem.rs +++ b/src/dbus_api/filesystem.rs @@ -22,7 +22,7 @@ use crate::{ types::{DbusContext, DbusErrorEnum, OPContext, TData}, util::{ engine_to_dbus_err_tuple, get_next_arg, get_parent, get_uuid, make_object_path, - msg_code_ok, msg_string_ok, + msg_code_ok, msg_string_ok, result_to_tuple, }, }, engine::{ @@ -52,13 +52,7 @@ fn get_properties_shared( )), _ => None, }) - .map(|(key, result)| { - let (success, value) = match result { - Ok(value) => (true, Variant(Box::new(value) as Box)), - Err(e) => (false, Variant(Box::new(e) as Box)), - }; - (key, (success, value)) - }) + .map(|(key, result)| result_to_tuple(key, result)) .collect(); Ok(vec![return_message.append1(return_value)]) diff --git a/src/dbus_api/pool.rs b/src/dbus_api/pool.rs index a0917a999b..32f1c68278 100644 --- a/src/dbus_api/pool.rs +++ b/src/dbus_api/pool.rs @@ -25,7 +25,7 @@ use crate::{ types::{DbusContext, DbusErrorEnum, OPContext, TData}, util::{ engine_to_dbus_err_tuple, get_next_arg, get_uuid, make_object_path, msg_code_ok, - msg_string_ok, + msg_string_ok, result_to_tuple, }, }, engine::{ @@ -407,13 +407,7 @@ fn get_properties_shared( )), _ => None, }) - .map(|(key, result)| { - let (success, value) = match result { - Ok(value) => (true, Variant(Box::new(value) as Box)), - Err(e) => (false, Variant(Box::new(e) as Box)), - }; - (key, (success, value)) - }) + .map(|(key, result)| result_to_tuple(key, result)) .collect(); Ok(vec![return_message.append1(return_value)]) diff --git a/src/dbus_api/util.rs b/src/dbus_api/util.rs index d8172c88d8..8a84466fed 100644 --- a/src/dbus_api/util.rs +++ b/src/dbus_api/util.rs @@ -31,6 +31,26 @@ pub fn tuple_to_option(value: (bool, T)) -> Option { } } +/// Map a result obtained for the FetchProperties interface to a pair of +/// a key and a value. The key is the property key, and therefore the key +/// of the item returned. The value is a tuple. An error in the result +/// argument yields a false in the return value, indicating that the value +/// returned is a string representation of the error encountered in +/// obtaining the value, and not the value requested. +pub fn result_to_tuple( + key: String, + result: Result, +) -> (String, (bool, Variant>)) +where + T: RefArg + 'static, +{ + let (success, value) = match result { + Ok(value) => (true, Variant(Box::new(value) as Box)), + Err(e) => (false, Variant(Box::new(e) as Box)), + }; + (key, (success, value)) +} + /// Get the next argument off the bus pub fn get_next_arg<'a, T>(iter: &mut Iter<'a>, loc: u16) -> Result where From e1eca1d3402b9c3bfd42d738f29d1d8cb2215efa Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Mon, 4 Nov 2019 16:36:42 -0500 Subject: [PATCH 005/109] Add tests for FetchProperties --- .../dbus/blockdev/test_fetch_properties.py | 61 +++++++++++++++ .../dbus/filesystem/test_fetch_properties.py | 71 ++++++++++++++++++ .../tests/dbus/filesystem/test_properties.py | 8 -- .../tests/dbus/pool/test_fetch_properties.py | 74 +++++++++++++++++++ 4 files changed, 206 insertions(+), 8 deletions(-) create mode 100644 tests/client-dbus/tests/dbus/blockdev/test_fetch_properties.py create mode 100644 tests/client-dbus/tests/dbus/filesystem/test_fetch_properties.py create mode 100644 tests/client-dbus/tests/dbus/pool/test_fetch_properties.py diff --git a/tests/client-dbus/tests/dbus/blockdev/test_fetch_properties.py b/tests/client-dbus/tests/dbus/blockdev/test_fetch_properties.py new file mode 100644 index 0000000000..25a7c4f5f9 --- /dev/null +++ b/tests/client-dbus/tests/dbus/blockdev/test_fetch_properties.py @@ -0,0 +1,61 @@ +# Copyright 2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Test accessing properties of a blockdev using FetchProperties interface. +""" + +from stratisd_client_dbus import FetchProperties +from stratisd_client_dbus import Manager +from stratisd_client_dbus import get_object + +from stratisd_client_dbus._constants import TOP_OBJECT + +from .._misc import SimTestCase + + +class FetchPropertiesTestCase(SimTestCase): + """ + Set up a pool with a name. + """ + + _POOLNAME = "fetchprops" + + def setUp(self): + """ + Start the stratisd daemon with the simulator. + """ + super().setUp() + proxy = get_object(TOP_OBJECT) + ((_, (_, self._bd_object_paths)), _, _) = Manager.Methods.CreatePool( + proxy, + { + "name": self._POOLNAME, + "redundancy": (True, 0), + "devices": ["/dev/one", "/dev/two", "/dev/red", "/dev/blue"], + }, + ) + Manager.Methods.ConfigureSimulator(proxy, {"denominator": 8}) + + def testFetchSizeProperty(self): + """ + Test FetchProperties for blockdev property, TotalPhysicalSize + """ + one_blockdev = get_object(self._bd_object_paths[0]) + + (size_success, size) = FetchProperties.Methods.GetProperties( + one_blockdev, {"properties": ["TotalPhysicalSize"]} + )["TotalPhysicalSize"] + + self.assertEqual(size_success, True) + self.assertTrue(size.isnumeric()) diff --git a/tests/client-dbus/tests/dbus/filesystem/test_fetch_properties.py b/tests/client-dbus/tests/dbus/filesystem/test_fetch_properties.py new file mode 100644 index 0000000000..8e66129a9f --- /dev/null +++ b/tests/client-dbus/tests/dbus/filesystem/test_fetch_properties.py @@ -0,0 +1,71 @@ +# Copyright 2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Test accessing properties of a filesystem using FetchProperties interface. +""" + +from stratisd_client_dbus import FetchProperties +from stratisd_client_dbus import Manager +from stratisd_client_dbus import Pool +from stratisd_client_dbus import get_object + +from stratisd_client_dbus._constants import TOP_OBJECT + +from .._misc import SimTestCase +from .._misc import device_name_list + +_DEVICE_STRATEGY = device_name_list() + + +class FetchPropertiesTestCase(SimTestCase): + """ + Set up a pool with a name and a filesystem. + """ + + _POOLNAME = "fetchprops" + _FSNAME = "fs" + + def setUp(self): + """ + Start the stratisd daemon with the simulator. + """ + super().setUp() + proxy = get_object(TOP_OBJECT) + ((_, (pool_object_path, _)), _, _) = Manager.Methods.CreatePool( + proxy, + { + "name": self._POOLNAME, + "redundancy": (True, 0), + "devices": _DEVICE_STRATEGY(), + }, + ) + pool_object = get_object(pool_object_path) + ((_, created), _, _) = Pool.Methods.CreateFilesystems( + pool_object, {"specs": [self._FSNAME]} + ) + self._filesystem_object_path = created[0][0] + Manager.Methods.ConfigureSimulator(proxy, {"denominator": 8}) + + def testFetchUsedProperty(self): + """ + Test FetchProperties for filesystem property, Used + """ + filesystem = get_object(self._filesystem_object_path) + + (used_success, used) = FetchProperties.Methods.GetProperties( + filesystem, {"properties": ["Used"]} + )["Used"] + + self.assertEqual(used_success, True) + self.assertTrue(used.isnumeric()) diff --git a/tests/client-dbus/tests/dbus/filesystem/test_properties.py b/tests/client-dbus/tests/dbus/filesystem/test_properties.py index eec3c8e01b..44a49ee392 100644 --- a/tests/client-dbus/tests/dbus/filesystem/test_properties.py +++ b/tests/client-dbus/tests/dbus/filesystem/test_properties.py @@ -15,7 +15,6 @@ Test accessing properties of a filesystem. """ -from stratisd_client_dbus import FetchProperties from stratisd_client_dbus import Filesystem from stratisd_client_dbus import Manager from stratisd_client_dbus import Pool @@ -79,13 +78,6 @@ def testProps(self): # I think this is also always true self.assertEqual(len(created), 20) - (used_success, used) = FetchProperties.Methods.GetProperties( - filesystem, {"properties": ["Used"]} - )["Used"] - - self.assertEqual(used_success, True) - self.assertEqual(used, "12345678") - devnode = Filesystem.Properties.Devnode.Get(filesystem) self.assertEqual(devnode, "/stratis/deadpool/fs") diff --git a/tests/client-dbus/tests/dbus/pool/test_fetch_properties.py b/tests/client-dbus/tests/dbus/pool/test_fetch_properties.py new file mode 100644 index 0000000000..02a9e467d6 --- /dev/null +++ b/tests/client-dbus/tests/dbus/pool/test_fetch_properties.py @@ -0,0 +1,74 @@ +# Copyright 2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Test accessing properties of a pool using FetchProperties interface. +""" + +from stratisd_client_dbus import FetchProperties +from stratisd_client_dbus import Manager +from stratisd_client_dbus import get_object + +from stratisd_client_dbus._constants import TOP_OBJECT + +from .._misc import SimTestCase +from .._misc import device_name_list + +_DEVICE_STRATEGY = device_name_list() + + +class FetchPropertiesTestCase(SimTestCase): + """ + Set up a pool with a name. + """ + + _POOLNAME = "fetchprops" + + def setUp(self): + """ + Start the stratisd daemon with the simulator. + """ + super().setUp() + proxy = get_object(TOP_OBJECT) + ((_, (pool_object_path, _)), _, _) = Manager.Methods.CreatePool( + proxy, + { + "name": self._POOLNAME, + "redundancy": (True, 0), + "devices": _DEVICE_STRATEGY(), + }, + ) + self._pool_object = get_object(pool_object_path) + Manager.Methods.ConfigureSimulator(proxy, {"denominator": 8}) + + def testFetchSizeProperty(self): + """ + Test FetchProperties for pool property, TotalPhysicalSize + """ + (size_success, size) = FetchProperties.Methods.GetProperties( + self._pool_object, {"properties": ["TotalPhysicalSize"]} + )["TotalPhysicalSize"] + + self.assertEqual(size_success, True) + self.assertTrue(size.isnumeric()) + + def testFetchUsedSizeProperty(self): + """ + Test FetchProperties for pool property, TotalPhysicalUsed + """ + (size_success, size) = FetchProperties.Methods.GetProperties( + self._pool_object, {"properties": ["TotalPhysicalUsed"]} + )["TotalPhysicalUsed"] + + self.assertEqual(size_success, True) + self.assertTrue(size.isnumeric()) From 210dcb6c9825cbdb66d05129fcd8315b41e7721a Mon Sep 17 00:00:00 2001 From: mulhern Date: Thu, 21 Nov 2019 09:26:44 -0500 Subject: [PATCH 006/109] Increase recommended development toolchain to 1.39 Signed-off-by: mulhern --- .travis.yml | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index d1917dde09..68d870350e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,12 +24,12 @@ matrix: # MANDATORY CHECKS USING CURRENT DEVELOPMENT COMPILER # rustfmt - - rust: 1.38.0 + - rust: 1.39.0 before_script: - rustup component add rustfmt env: TASK=fmt-travis # clippy - - rust: 1.38.0 + - rust: 1.39.0 before_script: - rustup component add clippy env: TASK=clippy diff --git a/README.md b/README.md index fed838f244..7615e00fc1 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ mailing list, if preferred. ### Setting up for development #### Development Compiler -The version of the compiler recommended for development is 1.38. Other +The version of the compiler recommended for development is 1.39. Other versions of the compiler may disagree with the CI tasks on some points, so should be avoided. From d0567f57cef1009aa7e3f54c86942adac941af30 Mon Sep 17 00:00:00 2001 From: mulhern Date: Thu, 21 Nov 2019 09:38:46 -0500 Subject: [PATCH 007/109] Do not specify xenial distribution It is now the default distribution. Signed-off-by: mulhern --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 68d870350e..d0c8ee09a1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -75,8 +75,7 @@ matrix: # MANDATORY PYTHON CHECKS # Run pylint, Python linter, on any Python test code - - dist: xenial - language: python + - language: python python: "3.7" install: pip3 install -r tests/client-dbus/requirements.txt before_script: From da610e55e8f44ed773cd481380aa002cb04c65f1 Mon Sep 17 00:00:00 2001 From: Bryan Gurney Date: Fri, 22 Nov 2019 11:21:35 -0500 Subject: [PATCH 008/109] stratisd: increase minimum rust version to 1.38 Signed-off-by: Bryan Gurney --- .travis.yml | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index d0c8ee09a1..d86d3cf5d1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,10 +36,10 @@ matrix: # MANDATORY TESTING USING LOWEST SUPPORTED COMPILER # tests - - rust: 1.37.0 + - rust: 1.38.0 env: TASK=test # release - - rust: 1.37.0 + - rust: 1.38.0 env: TASK=release diff --git a/README.md b/README.md index 7615e00fc1..a72aae6e01 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ versions of the compiler may disagree with the CI tasks on some points, so should be avoided. #### Building -Stratisd requires Rust 1.37+ and Cargo to build. These may be available via +Stratisd requires Rust 1.38+ and Cargo to build. These may be available via your distribution's package manager. If not, [Rustup](https://www.rustup.rs/) is available to install and update the Rust toolchain. Once toolchain and other dependencies are in place, run `make build` to build, and then run the From 78ce2e7374ac7c219e90b5e415ab8c29f8d6abfb Mon Sep 17 00:00:00 2001 From: Grace Chin Date: Mon, 18 Nov 2019 13:32:43 -0500 Subject: [PATCH 009/109] Clean up variable scopes Instance variables that can be local variables have been turned into local variables, which improves code cleanliness and performance. Local variables that do not need to exist have been removed. --- .../tests/dbus/blockdev/test_properties.py | 4 ++-- .../tests/dbus/filesystem/test_properties.py | 16 ++++++------- .../tests/dbus/filesystem/test_rename.py | 12 +++++----- .../tests/dbus/manager/test_create.py | 12 ++++++---- .../tests/dbus/manager/test_destroy.py | 4 ++-- .../tests/dbus/manager/test_stratis.py | 4 ++-- .../tests/dbus/pool/test_create_filesystem.py | 23 ++++++++++++------- .../tests/dbus/pool/test_create_snapshot.py | 17 +++++++------- .../dbus/pool/test_destroy_filesystem.py | 22 +++++++++++------- 9 files changed, 65 insertions(+), 49 deletions(-) diff --git a/tests/client-dbus/tests/dbus/blockdev/test_properties.py b/tests/client-dbus/tests/dbus/blockdev/test_properties.py index 18dd9eac7f..4242c93f70 100644 --- a/tests/client-dbus/tests/dbus/blockdev/test_properties.py +++ b/tests/client-dbus/tests/dbus/blockdev/test_properties.py @@ -41,9 +41,9 @@ def setUp(self): Start the stratisd daemon with the simulator. """ super().setUp() - self._proxy = get_object(TOP_OBJECT) + proxy = get_object(TOP_OBJECT) ((_, (_, self._blockdev_object_paths)), _, _) = Manager.Methods.CreatePool( - self._proxy, + proxy, { "name": self._POOLNAME, "redundancy": (True, 0), diff --git a/tests/client-dbus/tests/dbus/filesystem/test_properties.py b/tests/client-dbus/tests/dbus/filesystem/test_properties.py index 44a49ee392..9a7eb0d11d 100644 --- a/tests/client-dbus/tests/dbus/filesystem/test_properties.py +++ b/tests/client-dbus/tests/dbus/filesystem/test_properties.py @@ -34,28 +34,28 @@ class SetNameTestCase(SimTestCase): """ _POOLNAME = "deadpool" + _FSNAME = "fs" def setUp(self): """ Start the stratisd daemon with the simulator. """ super().setUp() - self._fs_name = "fs" - self._proxy = get_object(TOP_OBJECT) - ((_, (self._pool_object_path, _)), _, _) = Manager.Methods.CreatePool( - self._proxy, + proxy = get_object(TOP_OBJECT) + ((_, (pool_object_path, _)), _, _) = Manager.Methods.CreatePool( + proxy, { "name": self._POOLNAME, "redundancy": (True, 0), "devices": _DEVICE_STRATEGY(), }, ) - self._pool_object = get_object(self._pool_object_path) + pool_object = get_object(pool_object_path) ((_, created), _, _) = Pool.Methods.CreateFilesystems( - self._pool_object, {"specs": [self._fs_name]} + pool_object, {"specs": [self._FSNAME]} ) self._filesystem_object_path = created[0][0] - Manager.Methods.ConfigureSimulator(self._proxy, {"denominator": 8}) + Manager.Methods.ConfigureSimulator(proxy, {"denominator": 8}) def testProps(self): """ @@ -64,7 +64,7 @@ def testProps(self): filesystem = get_object(self._filesystem_object_path) name = Filesystem.Properties.Name.Get(filesystem) - self.assertEqual(self._fs_name, name) + self.assertEqual(self._FSNAME, name) uuid = Filesystem.Properties.Uuid.Get(filesystem) diff --git a/tests/client-dbus/tests/dbus/filesystem/test_rename.py b/tests/client-dbus/tests/dbus/filesystem/test_rename.py index 6b04fec551..deb3417dd3 100644 --- a/tests/client-dbus/tests/dbus/filesystem/test_rename.py +++ b/tests/client-dbus/tests/dbus/filesystem/test_rename.py @@ -37,15 +37,15 @@ class SetNameTestCase(SimTestCase): """ _POOLNAME = "deadpool" + _FSNAME = "fs" def setUp(self): """ Start the stratisd daemon with the simulator. """ super().setUp() - self._fs_name = "fs" self._proxy = get_object(TOP_OBJECT) - ((_, (self._pool_object_path, _)), _, _) = Manager.Methods.CreatePool( + ((_, (pool_object_path, _)), _, _) = Manager.Methods.CreatePool( self._proxy, { "name": self._POOLNAME, @@ -53,9 +53,9 @@ def setUp(self): "devices": _DEVICE_STRATEGY(), }, ) - self._pool_object = get_object(self._pool_object_path) + pool_object = get_object(pool_object_path) ((_, created), _, _) = Pool.Methods.CreateFilesystems( - self._pool_object, {"specs": [self._fs_name]} + pool_object, {"specs": [self._FSNAME]} ) self._filesystem_object_path = created[0][0] Manager.Methods.ConfigureSimulator(self._proxy, {"denominator": 8}) @@ -66,7 +66,7 @@ def testNullMapping(self): """ filesystem = get_object(self._filesystem_object_path) ((is_some, result), rc, _) = Filesystem.Methods.SetName( - filesystem, {"name": self._fs_name} + filesystem, {"name": self._FSNAME} ) self.assertEqual(rc, StratisdErrors.OK) @@ -90,6 +90,6 @@ def testNewName(self): self.assertEqual(self._filesystem_object_path, fs_object_path) fs_object_path = next( - filesystems(props={"Name": self._fs_name}).search(managed_objects), None + filesystems(props={"Name": self._FSNAME}).search(managed_objects), None ) self.assertIsNone(fs_object_path) diff --git a/tests/client-dbus/tests/dbus/manager/test_create.py b/tests/client-dbus/tests/dbus/manager/test_create.py index 56394a4433..3d37ce1b1a 100644 --- a/tests/client-dbus/tests/dbus/manager/test_create.py +++ b/tests/client-dbus/tests/dbus/manager/test_create.py @@ -51,10 +51,9 @@ def testCreate(self): If rc is OK, then pool must exist. """ - devs = self._devs ((_, (poolpath, devnodes)), rc, _) = Manager.Methods.CreatePool( self._proxy, - {"name": self._POOLNAME, "redundancy": (True, 0), "devices": devs}, + {"name": self._POOLNAME, "redundancy": (True, 0), "devices": self._devs}, ) managed_objects = ObjectManager.Methods.GetManagedObjects(self._proxy, {}) @@ -68,7 +67,7 @@ def testCreate(self): (pool, _) = result self.assertEqual(pool, poolpath) self.assertEqual(len(all_pools), 1) - self.assertLessEqual(len(devnodes), len(devs)) + self.assertLessEqual(len(devnodes), len(self._devs)) else: self.assertIsNone(result) self.assertEqual(len(all_pools), 0) @@ -77,10 +76,13 @@ def testCreateBadRAID(self): """ Creation should always fail if RAID value is wrong. """ - devs = _DEVICE_STRATEGY() (_, rc, _) = Manager.Methods.CreatePool( self._proxy, - {"name": self._POOLNAME, "redundancy": (True, 1), "devices": devs}, + { + "name": self._POOLNAME, + "redundancy": (True, 1), + "devices": _DEVICE_STRATEGY(), + }, ) self.assertEqual(rc, StratisdErrors.ERROR) diff --git a/tests/client-dbus/tests/dbus/manager/test_destroy.py b/tests/client-dbus/tests/dbus/manager/test_destroy.py index 87b0354664..d893a76f6a 100644 --- a/tests/client-dbus/tests/dbus/manager/test_destroy.py +++ b/tests/client-dbus/tests/dbus/manager/test_destroy.py @@ -118,7 +118,7 @@ class Destroy3TestCase(SimTestCase): """ _POOLNAME = "deadpool" - _VOLNAME = "vol" + _FSNAME = "vol" def setUp(self): """ @@ -135,7 +135,7 @@ def setUp(self): "devices": _DEVICE_STRATEGY(), }, ) - Pool.Methods.CreateFilesystems(get_object(poolpath), {"specs": [self._VOLNAME]}) + Pool.Methods.CreateFilesystems(get_object(poolpath), {"specs": [self._FSNAME]}) Manager.Methods.ConfigureSimulator(self._proxy, {"denominator": 8}) def testExecution(self): diff --git a/tests/client-dbus/tests/dbus/manager/test_stratis.py b/tests/client-dbus/tests/dbus/manager/test_stratis.py index 0d8b940ecb..433fe1c5ea 100644 --- a/tests/client-dbus/tests/dbus/manager/test_stratis.py +++ b/tests/client-dbus/tests/dbus/manager/test_stratis.py @@ -36,8 +36,8 @@ def setUp(self): Start the stratisd daemon with the simulator. """ super().setUp() - self._proxy = get_object(TOP_OBJECT) - Manager.Methods.ConfigureSimulator(self._proxy, {"denominator": 8}) + proxy = get_object(TOP_OBJECT) + Manager.Methods.ConfigureSimulator(proxy, {"denominator": 8}) def testStratisVersion(self): """ diff --git a/tests/client-dbus/tests/dbus/pool/test_create_filesystem.py b/tests/client-dbus/tests/dbus/pool/test_create_filesystem.py index fe58c24537..e4e5314ba3 100644 --- a/tests/client-dbus/tests/dbus/pool/test_create_filesystem.py +++ b/tests/client-dbus/tests/dbus/pool/test_create_filesystem.py @@ -38,6 +38,7 @@ class CreateFSTestCase(SimTestCase): """ _POOLNAME = "deadpool" + _FSNAME = "fs" def setUp(self): """ @@ -45,10 +46,13 @@ def setUp(self): """ super().setUp() self._proxy = get_object(TOP_OBJECT) - self._devs = _DEVICE_STRATEGY() ((_, (poolpath, _)), _, _) = Manager.Methods.CreatePool( self._proxy, - {"name": self._POOLNAME, "redundancy": (True, 0), "devices": self._devs}, + { + "name": self._POOLNAME, + "redundancy": (True, 0), + "devices": _DEVICE_STRATEGY(), + }, ) self._pool_object = get_object(poolpath) Manager.Methods.ConfigureSimulator(self._proxy, {"denominator": 8}) @@ -102,7 +106,7 @@ class CreateFSTestCase1(SimTestCase): """ _POOLNAME = "deadpool" - _VOLNAME = "thunk" + _FSNAME = "thunk" def setUp(self): """ @@ -110,13 +114,16 @@ def setUp(self): """ super().setUp() self._proxy = get_object(TOP_OBJECT) - self._devs = _DEVICE_STRATEGY() ((_, (poolpath, _)), _, _) = Manager.Methods.CreatePool( self._proxy, - {"name": self._POOLNAME, "redundancy": (True, 0), "devices": self._devs}, + { + "name": self._POOLNAME, + "redundancy": (True, 0), + "devices": _DEVICE_STRATEGY(), + }, ) self._pool_object = get_object(poolpath) - Pool.Methods.CreateFilesystems(self._pool_object, {"specs": [self._VOLNAME]}) + Pool.Methods.CreateFilesystems(self._pool_object, {"specs": [self._FSNAME]}) Manager.Methods.ConfigureSimulator(self._proxy, {"denominator": 8}) def testCreate(self): @@ -127,7 +134,7 @@ def testCreate(self): should be created. """ ((is_some, result), rc, _) = Pool.Methods.CreateFilesystems( - self._pool_object, {"specs": [self._VOLNAME]} + self._pool_object, {"specs": [self._FSNAME]} ) self.assertEqual(rc, StratisdErrors.OK) @@ -170,7 +177,7 @@ def testCreateWithConflict(self): and the command should succeed. """ ((is_some, result), rc, _) = Pool.Methods.CreateFilesystems( - self._pool_object, {"specs": [self._VOLNAME, "newname"]} + self._pool_object, {"specs": [self._FSNAME, "newname"]} ) self.assertEqual(rc, StratisdErrors.OK) diff --git a/tests/client-dbus/tests/dbus/pool/test_create_snapshot.py b/tests/client-dbus/tests/dbus/pool/test_create_snapshot.py index dae664d48f..d6c355a24b 100644 --- a/tests/client-dbus/tests/dbus/pool/test_create_snapshot.py +++ b/tests/client-dbus/tests/dbus/pool/test_create_snapshot.py @@ -36,7 +36,7 @@ class CreateSnapshotTestCase(SimTestCase): """ _POOLNAME = "deadpool" - _VOLNAME = "some_fs" + _FSNAME = "some_fs" _SNAPSHOTNAME = "ss_fs" def setUp(self): @@ -45,24 +45,25 @@ def setUp(self): """ super().setUp() self._proxy = get_object(TOP_OBJECT) - self._devs = _DEVICE_STRATEGY() ((_, (poolpath, _)), _, _) = Manager.Methods.CreatePool( self._proxy, - {"name": self._POOLNAME, "redundancy": (True, 0), "devices": self._devs}, + { + "name": self._POOLNAME, + "redundancy": (True, 0), + "devices": _DEVICE_STRATEGY(), + }, ) self._pool_object = get_object(poolpath) Manager.Methods.ConfigureSimulator(self._proxy, {"denominator": 8}) ((_, fs_objects), rc, _) = Pool.Methods.CreateFilesystems( - self._pool_object, {"specs": [self._VOLNAME]} + self._pool_object, {"specs": [self._FSNAME]} ) self.assertEqual(rc, StratisdErrors.OK) - fs_object_path = fs_objects[0][0] - self.assertNotEqual(fs_object_path, "/") - - self._fs_object_path = fs_object_path + self._fs_object_path = fs_objects[0][0] + self.assertNotEqual(self._fs_object_path, "/") def testCreate(self): """ diff --git a/tests/client-dbus/tests/dbus/pool/test_destroy_filesystem.py b/tests/client-dbus/tests/dbus/pool/test_destroy_filesystem.py index 4da60e9c55..309ece371a 100644 --- a/tests/client-dbus/tests/dbus/pool/test_destroy_filesystem.py +++ b/tests/client-dbus/tests/dbus/pool/test_destroy_filesystem.py @@ -43,10 +43,13 @@ def setUp(self): """ super().setUp() self._proxy = get_object(TOP_OBJECT) - self._devs = _DEVICE_STRATEGY() ((_, (poolpath, _)), _, _) = Manager.Methods.CreatePool( self._proxy, - {"name": self._POOLNAME, "redundancy": (True, 0), "devices": self._devs}, + { + "name": self._POOLNAME, + "redundancy": (True, 0), + "devices": _DEVICE_STRATEGY(), + }, ) self._pool_object = get_object(poolpath) Manager.Methods.ConfigureSimulator(self._proxy, {"denominator": 8}) @@ -92,7 +95,7 @@ class DestroyFSTestCase1(SimTestCase): """ _POOLNAME = "deadpool" - _VOLNAME = "thunk" + _FSNAME = "thunk" def setUp(self): """ @@ -100,14 +103,17 @@ def setUp(self): """ super().setUp() self._proxy = get_object(TOP_OBJECT) - self._devs = _DEVICE_STRATEGY() - ((_, (self._poolpath, _)), _, _) = Manager.Methods.CreatePool( + ((_, (poolpath, _)), _, _) = Manager.Methods.CreatePool( self._proxy, - {"name": self._POOLNAME, "redundancy": (True, 0), "devices": self._devs}, + { + "name": self._POOLNAME, + "redundancy": (True, 0), + "devices": _DEVICE_STRATEGY(), + }, ) - self._pool_object = get_object(self._poolpath) + self._pool_object = get_object(poolpath) ((_, self._filesystems), _, _) = Pool.Methods.CreateFilesystems( - self._pool_object, {"specs": [(self._VOLNAME, "", None)]} + self._pool_object, {"specs": [(self._FSNAME, "", None)]} ) Manager.Methods.ConfigureSimulator(self._proxy, {"denominator": 8}) From 841f797433c2e64d1ca7046e0f33c25f411ab645 Mon Sep 17 00:00:00 2001 From: mulhern Date: Wed, 9 Oct 2019 17:08:14 -0400 Subject: [PATCH 010/109] Move util module to udev module Only the last method, get_stratis_block_devices() does non-udev things. It is only invoked in one place, so it is unnecessary to put it in a shared spot. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/blockdevmgr.rs | 2 +- src/engine/strat_engine/backstore/device.rs | 2 +- src/engine/strat_engine/backstore/mod.rs | 2 +- src/engine/strat_engine/backstore/setup.rs | 2 +- src/engine/strat_engine/backstore/{util.rs => udev.rs} | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) rename src/engine/strat_engine/backstore/{util.rs => udev.rs} (99%) diff --git a/src/engine/strat_engine/backstore/blockdevmgr.rs b/src/engine/strat_engine/backstore/blockdevmgr.rs index cd724557d9..e0fb645f31 100644 --- a/src/engine/strat_engine/backstore/blockdevmgr.rs +++ b/src/engine/strat_engine/backstore/blockdevmgr.rs @@ -27,7 +27,7 @@ use crate::{ blockdev::StratBlockDev, device::{identify, resolve_devices, DevOwnership}, metadata::{disown_device, BlockdevSize, MDADataSize, BDA}, - util::hw_lookup, + udev::hw_lookup, }, device::blkdev_size, serde_structs::{BaseBlockDevSave, BaseDevSave, Recordable}, diff --git a/src/engine/strat_engine/backstore/device.rs b/src/engine/strat_engine/backstore/device.rs index d924be3757..61415f679b 100644 --- a/src/engine/strat_engine/backstore/device.rs +++ b/src/engine/strat_engine/backstore/device.rs @@ -10,7 +10,7 @@ use devicemapper::{devnode_to_devno, Device}; use crate::{ engine::{ - strat_engine::backstore::{metadata::device_identifiers, util::get_udev_block_device}, + strat_engine::backstore::{metadata::device_identifiers, udev::get_udev_block_device}, types::{DevUuid, PoolUuid}, }, stratis::{ErrorEnum, StratisError, StratisResult}, diff --git a/src/engine/strat_engine/backstore/mod.rs b/src/engine/strat_engine/backstore/mod.rs index 92025196eb..755bcaec39 100644 --- a/src/engine/strat_engine/backstore/mod.rs +++ b/src/engine/strat_engine/backstore/mod.rs @@ -13,7 +13,7 @@ mod metadata; mod range_alloc; mod setup; mod shared; -mod util; +mod udev; pub use self::{ backstore::Backstore, diff --git a/src/engine/strat_engine/backstore/setup.rs b/src/engine/strat_engine/backstore/setup.rs index a5e095bb51..72a08086ad 100644 --- a/src/engine/strat_engine/backstore/setup.rs +++ b/src/engine/strat_engine/backstore/setup.rs @@ -22,7 +22,7 @@ use crate::{ backstore::{ blockdev::StratBlockDev, metadata::{device_identifiers, BDA}, - util::get_stratis_block_devices, + udev::get_stratis_block_devices, }, device::blkdev_size, serde_structs::{BackstoreSave, BaseBlockDevSave, PoolSave}, diff --git a/src/engine/strat_engine/backstore/util.rs b/src/engine/strat_engine/backstore/udev.rs similarity index 99% rename from src/engine/strat_engine/backstore/util.rs rename to src/engine/strat_engine/backstore/udev.rs index 1c9bb33066..b30ad411d9 100644 --- a/src/engine/strat_engine/backstore/util.rs +++ b/src/engine/strat_engine/backstore/udev.rs @@ -2,7 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. -// Utilities to support Stratis. +//! udev-related methods use std::{ collections::HashMap, fs, From 49bdc745a103db1c6820e5e6bdac6f7407260cb0 Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 22 Oct 2019 08:56:16 -0400 Subject: [PATCH 011/109] Move get_stratis_block_devices into setup module It is only used there. Make it private, but make udev method get_all_empty_devices() public so that it can be accessed by get_stratis_block_devices(). Signed-off-by: mulhern --- src/engine/strat_engine/backstore/setup.rs | 44 +++++++++++++++++++++- src/engine/strat_engine/backstore/udev.rs | 44 +--------------------- 2 files changed, 45 insertions(+), 43 deletions(-) diff --git a/src/engine/strat_engine/backstore/setup.rs b/src/engine/strat_engine/backstore/setup.rs index 72a08086ad..31f49fc719 100644 --- a/src/engine/strat_engine/backstore/setup.rs +++ b/src/engine/strat_engine/backstore/setup.rs @@ -12,6 +12,7 @@ use std::{ }; use chrono::{DateTime, Utc}; +use libudev; use serde_json; use devicemapper::{devnode_to_devno, Device, Sectors}; @@ -21,8 +22,9 @@ use crate::{ strat_engine::{ backstore::{ blockdev::StratBlockDev, + device::identify, metadata::{device_identifiers, BDA}, - udev::get_stratis_block_devices, + udev::get_all_empty_devices, }, device::blkdev_size, serde_structs::{BackstoreSave, BaseBlockDevSave, PoolSave}, @@ -32,6 +34,46 @@ use crate::{ stratis::{ErrorEnum, StratisError, StratisResult}, }; +/// Retrieve all the block devices on the system that have a Stratis signature. +fn get_stratis_block_devices() -> StratisResult> { + let context = libudev::Context::new()?; + let mut enumerator = libudev::Enumerator::new(&context)?; + enumerator.match_subsystem("block")?; + enumerator.match_property("ID_FS_TYPE", "stratis")?; + + let devices: Vec = enumerator + .scan_devices()? + .filter(|dev| dev.is_initialized()) + .filter(|dev| { + dev.property_value("DM_MULTIPATH_DEVICE_PATH") + .map_or(true, |v| v != "1") + }) + .filter_map(|i| i.devnode().map(|d| d.into())) + .collect(); + + if devices.is_empty() { + // We have found no Stratis devices, possible reasons are: + // 1. We really don't have any + // 2. We have some, but libblkid is too old to support Stratis, thus we appear empty + // 3. We ran this code at early boot before we have any udev db entries which are complete + // or are complete but fall into reasons 1 & 2 above + // + // In this case we will get all the block devices which have complete udev db block device + // entries and appear "empty" and go out to disk and check them! + + Ok(get_all_empty_devices()? + .into_iter() + .filter(|x| { + identify(x) + .map(|ownership| ownership.stratis_identifiers().is_some()) + .unwrap_or(false) + }) + .collect()) + } else { + Ok(devices) + } +} + /// Find all Stratis devices. /// /// Returns a map of pool uuids to a map of devices to devnodes for each pool. diff --git a/src/engine/strat_engine/backstore/udev.rs b/src/engine/strat_engine/backstore/udev.rs index b30ad411d9..5c50767f08 100644 --- a/src/engine/strat_engine/backstore/udev.rs +++ b/src/engine/strat_engine/backstore/udev.rs @@ -11,7 +11,7 @@ use std::{ use libudev; -use crate::{engine::strat_engine::backstore::device::identify, stratis::StratisResult}; +use crate::stratis::StratisResult; /// Takes a libudev device entry and returns the properties as a HashMap. fn device_as_map(device: &libudev::Device) -> HashMap { @@ -54,7 +54,7 @@ pub fn hw_lookup(dev_node_search: &Path) -> StratisResult> { /// Collect paths for all the block devices which are not individual multipath paths and which /// appear to be empty from a udev perspective. -fn get_all_empty_devices() -> StratisResult> { +pub fn get_all_empty_devices() -> StratisResult> { let context = libudev::Context::new()?; let mut enumerator = libudev::Enumerator::new(&context)?; enumerator.match_subsystem("block")?; @@ -72,43 +72,3 @@ fn get_all_empty_devices() -> StratisResult> { .filter_map(|i| i.devnode().map(|d| d.into())) .collect()) } - -/// Retrieve all the block devices on the system that have a Stratis signature. -pub fn get_stratis_block_devices() -> StratisResult> { - let context = libudev::Context::new()?; - let mut enumerator = libudev::Enumerator::new(&context)?; - enumerator.match_subsystem("block")?; - enumerator.match_property("ID_FS_TYPE", "stratis")?; - - let devices: Vec = enumerator - .scan_devices()? - .filter(|dev| dev.is_initialized()) - .filter(|dev| { - dev.property_value("DM_MULTIPATH_DEVICE_PATH") - .map_or(true, |v| v != "1") - }) - .filter_map(|i| i.devnode().map(|d| d.into())) - .collect(); - - if devices.is_empty() { - // We have found no Stratis devices, possible reasons are: - // 1. We really don't have any - // 2. We have some, but libblkid is too old to support Stratis, thus we appear empty - // 3. We ran this code at early boot before we have any udev db entries which are complete - // or are complete but fall into reasons 1 & 2 above - // - // In this case we will get all the block devices which have complete udev db block device - // entries and appear "empty" and go out to disk and check them! - - Ok(get_all_empty_devices()? - .into_iter() - .filter(|x| { - identify(x) - .map(|ownership| ownership.stratis_identifiers().is_some()) - .unwrap_or(false) - }) - .collect()) - } else { - Ok(devices) - } -} From d80d627c122eec89049dd25059a9fe8f77857834 Mon Sep 17 00:00:00 2001 From: mulhern Date: Wed, 23 Oct 2019 10:10:33 -0400 Subject: [PATCH 012/109] Make a udev enumerator for matching block devices Use it wherever appropriate. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/udev.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/engine/strat_engine/backstore/udev.rs b/src/engine/strat_engine/backstore/udev.rs index 5c50767f08..1ecd1cf434 100644 --- a/src/engine/strat_engine/backstore/udev.rs +++ b/src/engine/strat_engine/backstore/udev.rs @@ -13,6 +13,14 @@ use libudev; use crate::stratis::StratisResult; +/// Make an enumerator for enumerating block devices. Return an error if there +/// was any udev-related error. +fn block_enumerator(context: &libudev::Context) -> libudev::Result { + let mut enumerator = libudev::Enumerator::new(context)?; + enumerator.match_subsystem("block")?; + Ok(enumerator) +} + /// Takes a libudev device entry and returns the properties as a HashMap. fn device_as_map(device: &libudev::Device) -> HashMap { let rc: HashMap<_, _> = device @@ -32,8 +40,7 @@ pub fn get_udev_block_device( dev_node_search: &Path, ) -> StratisResult>> { let context = libudev::Context::new()?; - let mut enumerator = libudev::Enumerator::new(&context)?; - enumerator.match_subsystem("block")?; + let mut enumerator = block_enumerator(&context)?; // Get canonical form to ensure we do correct lookup in udev db let canonical = fs::canonicalize(dev_node_search)?; @@ -56,8 +63,7 @@ pub fn hw_lookup(dev_node_search: &Path) -> StratisResult> { /// appear to be empty from a udev perspective. pub fn get_all_empty_devices() -> StratisResult> { let context = libudev::Context::new()?; - let mut enumerator = libudev::Enumerator::new(&context)?; - enumerator.match_subsystem("block")?; + let mut enumerator = block_enumerator(&context)?; Ok(enumerator .scan_devices()? From 77228cdfeef07c40c8f1af4710428c968c568aeb Mon Sep 17 00:00:00 2001 From: mulhern Date: Wed, 23 Oct 2019 10:25:21 -0400 Subject: [PATCH 013/109] Remove get_all_empty_devices It is only used in one place, and the code is a bit clearer if it is only in that one place. Also, it is slightly more efficient, in that it iterates once over the list of devices obtained from udev, rather than making that list into a Vec, and then reiterating over the Vec. Move the first udev context and enumerator that occurs in the same method into a separate scope, for clarity. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/setup.rs | 50 ++++++++++++++-------- src/engine/strat_engine/backstore/udev.rs | 28 +----------- 2 files changed, 33 insertions(+), 45 deletions(-) diff --git a/src/engine/strat_engine/backstore/setup.rs b/src/engine/strat_engine/backstore/setup.rs index 31f49fc719..0368da9bce 100644 --- a/src/engine/strat_engine/backstore/setup.rs +++ b/src/engine/strat_engine/backstore/setup.rs @@ -24,7 +24,7 @@ use crate::{ blockdev::StratBlockDev, device::identify, metadata::{device_identifiers, BDA}, - udev::get_all_empty_devices, + udev::block_enumerator, }, device::blkdev_size, serde_structs::{BackstoreSave, BaseBlockDevSave, PoolSave}, @@ -36,20 +36,22 @@ use crate::{ /// Retrieve all the block devices on the system that have a Stratis signature. fn get_stratis_block_devices() -> StratisResult> { - let context = libudev::Context::new()?; - let mut enumerator = libudev::Enumerator::new(&context)?; - enumerator.match_subsystem("block")?; - enumerator.match_property("ID_FS_TYPE", "stratis")?; + let devices = { + let context = libudev::Context::new()?; + let mut enumerator = libudev::Enumerator::new(&context)?; + enumerator.match_subsystem("block")?; + enumerator.match_property("ID_FS_TYPE", "stratis")?; - let devices: Vec = enumerator - .scan_devices()? - .filter(|dev| dev.is_initialized()) - .filter(|dev| { - dev.property_value("DM_MULTIPATH_DEVICE_PATH") - .map_or(true, |v| v != "1") - }) - .filter_map(|i| i.devnode().map(|d| d.into())) - .collect(); + enumerator + .scan_devices()? + .filter(|dev| dev.is_initialized()) + .filter(|dev| { + dev.property_value("DM_MULTIPATH_DEVICE_PATH") + .map_or(true, |v| v != "1") + }) + .filter_map(|i| i.devnode().map(|d| d.into())) + .collect::>() + }; if devices.is_empty() { // We have found no Stratis devices, possible reasons are: @@ -60,11 +62,21 @@ fn get_stratis_block_devices() -> StratisResult> { // // In this case we will get all the block devices which have complete udev db block device // entries and appear "empty" and go out to disk and check them! - - Ok(get_all_empty_devices()? - .into_iter() - .filter(|x| { - identify(x) + let context = libudev::Context::new()?; + let mut enumerator = block_enumerator(&context)?; + Ok(enumerator + .scan_devices()? + .filter(|dev| dev.is_initialized()) + .filter(|dev| { + dev.property_value("DM_MULTIPATH_DEVICE_PATH") + .map_or(true, |v| v != "1") + && !((dev.property_value("ID_PART_TABLE_TYPE").is_some() + && dev.property_value("ID_PART_ENTRY_DISK").is_none()) + || dev.property_value("ID_FS_USAGE").is_some()) + }) + .filter_map(|dev| dev.devnode().map(|d| d.to_owned())) + .filter(|devnode| { + identify(devnode) .map(|ownership| ownership.stratis_identifiers().is_some()) .unwrap_or(false) }) diff --git a/src/engine/strat_engine/backstore/udev.rs b/src/engine/strat_engine/backstore/udev.rs index 1ecd1cf434..8013a43a9d 100644 --- a/src/engine/strat_engine/backstore/udev.rs +++ b/src/engine/strat_engine/backstore/udev.rs @@ -3,11 +3,7 @@ // file, You can obtain one at http://mozilla.org/MPL/2.0/. //! udev-related methods -use std::{ - collections::HashMap, - fs, - path::{Path, PathBuf}, -}; +use std::{collections::HashMap, fs, path::Path}; use libudev; @@ -15,7 +11,7 @@ use crate::stratis::StratisResult; /// Make an enumerator for enumerating block devices. Return an error if there /// was any udev-related error. -fn block_enumerator(context: &libudev::Context) -> libudev::Result { +pub fn block_enumerator(context: &libudev::Context) -> libudev::Result { let mut enumerator = libudev::Enumerator::new(context)?; enumerator.match_subsystem("block")?; Ok(enumerator) @@ -58,23 +54,3 @@ pub fn hw_lookup(dev_node_search: &Path) -> StratisResult> { let dev = get_udev_block_device(dev_node_search)?; Ok(dev.and_then(|dev| dev.get("ID_WWN").cloned())) } - -/// Collect paths for all the block devices which are not individual multipath paths and which -/// appear to be empty from a udev perspective. -pub fn get_all_empty_devices() -> StratisResult> { - let context = libudev::Context::new()?; - let mut enumerator = block_enumerator(&context)?; - - Ok(enumerator - .scan_devices()? - .filter(|dev| dev.is_initialized()) - .filter(|dev| { - dev.property_value("DM_MULTIPATH_DEVICE_PATH") - .map_or(true, |v| v != "1") - && !((dev.property_value("ID_PART_TABLE_TYPE").is_some() - && dev.property_value("ID_PART_ENTRY_DISK").is_none()) - || dev.property_value("ID_FS_USAGE").is_some()) - }) - .filter_map(|i| i.devnode().map(|d| d.into())) - .collect()) -} From f8676a04da0b7bf7bdaf73568b2ca57c2f42024e Mon Sep 17 00:00:00 2001 From: mulhern Date: Wed, 23 Oct 2019 11:34:06 -0400 Subject: [PATCH 014/109] Add a method to generate a Stratis enumerator Use it where appropriate. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/setup.rs | 6 ++---- src/engine/strat_engine/backstore/udev.rs | 9 +++++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/engine/strat_engine/backstore/setup.rs b/src/engine/strat_engine/backstore/setup.rs index 0368da9bce..012ea564eb 100644 --- a/src/engine/strat_engine/backstore/setup.rs +++ b/src/engine/strat_engine/backstore/setup.rs @@ -24,7 +24,7 @@ use crate::{ blockdev::StratBlockDev, device::identify, metadata::{device_identifiers, BDA}, - udev::block_enumerator, + udev::{block_enumerator, stratis_enumerator}, }, device::blkdev_size, serde_structs::{BackstoreSave, BaseBlockDevSave, PoolSave}, @@ -38,9 +38,7 @@ use crate::{ fn get_stratis_block_devices() -> StratisResult> { let devices = { let context = libudev::Context::new()?; - let mut enumerator = libudev::Enumerator::new(&context)?; - enumerator.match_subsystem("block")?; - enumerator.match_property("ID_FS_TYPE", "stratis")?; + let mut enumerator = stratis_enumerator(&context)?; enumerator .scan_devices()? diff --git a/src/engine/strat_engine/backstore/udev.rs b/src/engine/strat_engine/backstore/udev.rs index 8013a43a9d..8f02b16755 100644 --- a/src/engine/strat_engine/backstore/udev.rs +++ b/src/engine/strat_engine/backstore/udev.rs @@ -17,6 +17,15 @@ pub fn block_enumerator(context: &libudev::Context) -> libudev::Result libudev::Result { + let mut enumerator = libudev::Enumerator::new(context)?; + enumerator.match_subsystem("block")?; + enumerator.match_property("ID_FS_TYPE", "stratis")?; + Ok(enumerator) +} + /// Takes a libudev device entry and returns the properties as a HashMap. fn device_as_map(device: &libudev::Device) -> HashMap { let rc: HashMap<_, _> = device From 0372a7890c9855e4a223a3fb7d879cfd2ce3595b Mon Sep 17 00:00:00 2001 From: mulhern Date: Wed, 23 Oct 2019 12:16:56 -0400 Subject: [PATCH 015/109] Add a udev property getter and some methods that use it. Use the is_multipath_member() and the is_unclaimed() method in the source. It is not possible to use the new methods in the other places where the code checks for multipath membership or unclaimedness, those are currently defined in a different way. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/setup.rs | 15 ++----- src/engine/strat_engine/backstore/udev.rs | 48 +++++++++++++++++++++- 2 files changed, 49 insertions(+), 14 deletions(-) diff --git a/src/engine/strat_engine/backstore/setup.rs b/src/engine/strat_engine/backstore/setup.rs index 012ea564eb..7405ab6ebd 100644 --- a/src/engine/strat_engine/backstore/setup.rs +++ b/src/engine/strat_engine/backstore/setup.rs @@ -24,7 +24,7 @@ use crate::{ blockdev::StratBlockDev, device::identify, metadata::{device_identifiers, BDA}, - udev::{block_enumerator, stratis_enumerator}, + udev::{block_enumerator, is_multipath_member, is_unclaimed, stratis_enumerator}, }, device::blkdev_size, serde_structs::{BackstoreSave, BaseBlockDevSave, PoolSave}, @@ -43,10 +43,7 @@ fn get_stratis_block_devices() -> StratisResult> { enumerator .scan_devices()? .filter(|dev| dev.is_initialized()) - .filter(|dev| { - dev.property_value("DM_MULTIPATH_DEVICE_PATH") - .map_or(true, |v| v != "1") - }) + .filter(|dev| !is_multipath_member(dev).unwrap_or(true)) .filter_map(|i| i.devnode().map(|d| d.into())) .collect::>() }; @@ -65,13 +62,7 @@ fn get_stratis_block_devices() -> StratisResult> { Ok(enumerator .scan_devices()? .filter(|dev| dev.is_initialized()) - .filter(|dev| { - dev.property_value("DM_MULTIPATH_DEVICE_PATH") - .map_or(true, |v| v != "1") - && !((dev.property_value("ID_PART_TABLE_TYPE").is_some() - && dev.property_value("ID_PART_ENTRY_DISK").is_none()) - || dev.property_value("ID_FS_USAGE").is_some()) - }) + .filter(|dev| !is_multipath_member(dev).unwrap_or(true) && is_unclaimed(dev)) .filter_map(|dev| dev.devnode().map(|d| d.to_owned())) .filter(|devnode| { identify(devnode) diff --git a/src/engine/strat_engine/backstore/udev.rs b/src/engine/strat_engine/backstore/udev.rs index 8f02b16755..16e189acfe 100644 --- a/src/engine/strat_engine/backstore/udev.rs +++ b/src/engine/strat_engine/backstore/udev.rs @@ -3,11 +3,11 @@ // file, You can obtain one at http://mozilla.org/MPL/2.0/. //! udev-related methods -use std::{collections::HashMap, fs, path::Path}; +use std::{collections::HashMap, ffi::OsStr, fs, path::Path}; use libudev; -use crate::stratis::StratisResult; +use crate::stratis::{StratisError, StratisResult}; /// Make an enumerator for enumerating block devices. Return an error if there /// was any udev-related error. @@ -26,6 +26,50 @@ pub fn stratis_enumerator(context: &libudev::Context) -> libudev::Result>( + device: &libudev::Device, + property_name: T, +) -> Option> +where + T: std::fmt::Display, +{ + device + .property_value(&property_name) + .map(|value| match value.to_str() { + Some(value) => Ok(value.into()), + None => Err(StratisError::Error(format!( + "Unable to convert udev property value with key {} belonging to device {} to a string", + property_name, + device.devnode().map_or("".into(), |x| x.to_string_lossy().into_owned()) + ))), + }) +} + +/// Returns true if udev indicates that the device is a multipath member +/// device, else false. Returns an error on a failure to interpret the +/// value. +pub fn is_multipath_member(device: &libudev::Device) -> StratisResult { + match get_udev_property(device, "DM_MULTIPATH_DEVICE_PATH") { + None => Ok(false), + Some(Ok(value)) => Ok(value == "1"), + Some(Err(err)) => Err(err), + } +} + +/// If the expression is true, then it seems that no other system is +/// known to udev to claim this device. +/// Note from mulhern: I have no idea myself why this particular expression +/// should be correct. I was told that the original source was dlehman. +pub fn is_unclaimed(device: &libudev::Device) -> bool { + (get_udev_property(device, "ID_PART_TABLE_TYPE").is_none() + || get_udev_property(device, "ID_PART_ENTRY_DISK").is_some()) + && get_udev_property(device, "ID_FS_USAGE").is_none() +} + /// Takes a libudev device entry and returns the properties as a HashMap. fn device_as_map(device: &libudev::Device) -> HashMap { let rc: HashMap<_, _> = device From 18620bd909746be9dedf6baf1e061629dc6ae674 Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 25 Nov 2019 10:48:15 -0500 Subject: [PATCH 016/109] Move udev-related identifying into udev module Also, remove signature function. It's misleading, because it makes it look like stratis actually looked at those properties to determine the device ownership, which it didn't. Also, it is a refactoring burden. Add a FIXME on the DevOwnership::Theirs constructor. Where it is matched against in blockdevmgr::initialize say something truer to its real meaning. The empty() function now gone was equivalent to !is_multipath_member(&device) && is_unclaimed(&device). Note that the checks in the decide_ownership() method are in a slightly different order from previous, but I think in an equivalent order. I did my best to document what I believe may be the ordering constraints. Note that the identify method identifies a multipath member device as Theirs which is a very rough classification, since a multipath member that belongs to Stratis is also ours. I haven't tried to address that. Signed-off-by: mulhern --- .../strat_engine/backstore/blockdevmgr.rs | 6 +- src/engine/strat_engine/backstore/device.rs | 134 +++++++++--------- src/engine/strat_engine/backstore/udev.rs | 48 +++++++ 3 files changed, 116 insertions(+), 72 deletions(-) diff --git a/src/engine/strat_engine/backstore/blockdevmgr.rs b/src/engine/strat_engine/backstore/blockdevmgr.rs index e0fb645f31..d4e819d752 100644 --- a/src/engine/strat_engine/backstore/blockdevmgr.rs +++ b/src/engine/strat_engine/backstore/blockdevmgr.rs @@ -399,11 +399,11 @@ fn initialize( }; match ownership { DevOwnership::Unowned => add_devs.push((dev, (devnode, dev_size, f))), - DevOwnership::Theirs(signature) => { + DevOwnership::Theirs(info) => { let err_str = format!( - "Device {} has an existing signature {}", + "Device {} appears to be already claimed by another, reason: {}", devnode.display(), - signature + info ); return Err(StratisError::Engine(ErrorEnum::Invalid, err_str)); } diff --git a/src/engine/strat_engine/backstore/device.rs b/src/engine/strat_engine/backstore/device.rs index 61415f679b..6746969b84 100644 --- a/src/engine/strat_engine/backstore/device.rs +++ b/src/engine/strat_engine/backstore/device.rs @@ -10,7 +10,10 @@ use devicemapper::{devnode_to_devno, Device}; use crate::{ engine::{ - strat_engine::backstore::{metadata::device_identifiers, udev::get_udev_block_device}, + strat_engine::backstore::{ + metadata::device_identifiers, + udev::{block_enumerator, decide_ownership, UdevOwnership}, + }, types::{DevUuid, PoolUuid}, }, stratis::{ErrorEnum, StratisError, StratisResult}, @@ -40,7 +43,9 @@ pub fn resolve_devices<'a>(paths: &'a [&Path]) -> StratisResult) -> bool { - device - .get("DM_MULTIPATH_DEVICE_PATH") - .map_or(true, |v| v != "1") - && !((device.contains_key("ID_PART_TABLE_TYPE") - && !device.contains_key("ID_PART_ENTRY_DISK")) - || device.contains_key("ID_FS_USAGE")) -} - -/// Generate some kind of human readable text about what's on a device. -fn signature(device: &HashMap) -> String { - if empty(device) { - String::from("empty") - } else { - device - .iter() - .filter(|&(k, _)| k.contains("ID_FS_") || k.contains("ID_PART_TABLE_")) - .map(|(k, v)| format!("{}={}", k, v)) - .collect::>() - .join(" ") - } -} - /// Determine what a block device is used for. pub fn identify(devnode: &Path) -> StratisResult { - if let Some(device) = get_udev_block_device(devnode)? { - if empty(&device) { - // The device is either really empty or we are running on a distribution that hasn't - // picked up the latest libblkid, lets read down to the device and find out for sure. - // TODO: At some point in the future we can remove this and just return Unowned. - if let Some((pool_uuid, device_uuid)) = - device_identifiers(&mut OpenOptions::new().read(true).open(&devnode)?)? - { - Ok(DevOwnership::Ours(pool_uuid, device_uuid)) - } else { - Ok(DevOwnership::Unowned) + let canonical = devnode.canonicalize()?; + + let context = libudev::Context::new()?; + let mut enumerator = block_enumerator(&context)?; + if let Some(udev_decision) = enumerator + .scan_devices()? + .filter(|dev| dev.is_initialized()) + .find(|x| x.devnode().map_or(false, |d| canonical == d)) + .map(|d| decide_ownership(&d)) + { + match udev_decision { + Ok(decision) => { + match decision { + UdevOwnership::Unowned => { + // FIXME: It is possible that Stratis is running in + // an old environment without the necessary version of + // libblkid that would cause udev database to be + // populated with Stratis information. So, if the + // device appears unowned, attempt to read information + // from Stratis metadata. We believe that this block + // can be removed once Stratis is certainly runnng + // with libblkid 2.32 or above. + // https://github.com/stratis-storage/stratisd/issues/1656 + if let Some((pool_uuid, device_uuid)) = + device_identifiers(&mut OpenOptions::new().read(true).open(&devnode)?)? + { + Ok(DevOwnership::Ours(pool_uuid, device_uuid)) + } else { + Ok(DevOwnership::Unowned) + } + } + UdevOwnership::MultipathMember => { + Ok(DevOwnership::Theirs("multipath member".into())) + } + UdevOwnership::Stratis => { + // Udev information does not include pool UUID and + // device UUID so read these from Stratis metadata. + if let Some((pool_uuid, device_uuid)) = + device_identifiers(&mut OpenOptions::new().read(true).open(&devnode)?)? + { + Ok(DevOwnership::Ours(pool_uuid, device_uuid)) + } else { + // FIXME: if udev says stratis but no stratis + // idenfiers on device, likely they were there + // recently, and udev has not yet caught up. It's + // just as likely that this device is unclaimed as + // that it belongs to some other entity. + Ok(DevOwnership::Theirs( + "Udev db says stratis, disk meta says no".into(), + )) + } + } + UdevOwnership::Theirs => Ok(DevOwnership::Theirs( + "udev properties for this device did not indicate that the devices was unowned".into() + )), + } } - } else if device - .get("DM_MULTIPATH_DEVICE_PATH") - .map_or(false, |v| v == "1") - { - Ok(DevOwnership::Theirs(String::from("multipath path"))) - } else if device.contains_key("ID_FS_TYPE") && device["ID_FS_TYPE"] == "stratis" { - // Device is ours, but we don't get everything we need from udev db, lets go to disk. - if let Some((pool_uuid, device_uuid)) = - device_identifiers(&mut OpenOptions::new().read(true).open(&devnode)?)? - { - Ok(DevOwnership::Ours(pool_uuid, device_uuid)) - } else { - // In this case the udev db says it's ours, but our check says otherwise. We should - // trust ourselves. Should we raise an error here? - Ok(DevOwnership::Theirs(String::from( - "Udev db says stratis, disk meta says no", - ))) - } - } else { - Ok(DevOwnership::Theirs(signature(&device))) + Err(err) => Err(err), } } else { Err(StratisError::Engine( @@ -138,18 +144,8 @@ mod test { /// Verify that the device is not stratis by creating a device with XFS fs. fn test_other_ownership(paths: &[&Path]) { cmd::create_fs(paths[0], None).unwrap(); - cmd::udev_settle().unwrap(); - - match identify(paths[0]).unwrap() { - DevOwnership::Theirs(identity) => { - assert!(identity.contains("ID_FS_USAGE=filesystem")); - assert!(identity.contains("ID_FS_TYPE=xfs")); - assert!(identity.contains("ID_FS_UUID")); - } - // This must fail, and will give a helpful error message - id => assert_matches!(id, DevOwnership::Theirs(_)), - } + assert_matches!(identify(paths[0]).unwrap(), DevOwnership::Theirs(_)) } /// Verify that identify() recognizes a blank device as unowned diff --git a/src/engine/strat_engine/backstore/udev.rs b/src/engine/strat_engine/backstore/udev.rs index 16e189acfe..398af3801e 100644 --- a/src/engine/strat_engine/backstore/udev.rs +++ b/src/engine/strat_engine/backstore/udev.rs @@ -70,6 +70,54 @@ pub fn is_unclaimed(device: &libudev::Device) -> bool { && get_udev_property(device, "ID_FS_USAGE").is_none() } +/// Return true if the device is identified by udev as belonging to Stratis. +/// Return an error if a udev property value could not be converted. +pub fn is_stratis(device: &libudev::Device) -> StratisResult { + match get_udev_property(device, "ID_FS_TYPE") { + None => Ok(false), + Some(Ok(value)) => Ok(value == "stratis"), + Some(Err(err)) => Err(err), + } +} + +/// An enum to encode udev classification of a device +#[derive(Debug, Eq, PartialEq)] +pub enum UdevOwnership { + MultipathMember, + Stratis, + Theirs, + Unowned, +} + +/// Decide the ownership of a device based on udev information. +/// +/// Always check multipath member status first. The theory is that a multipath +/// member may also appear to belong to Stratis, but it should not be identified +/// as a Stratis device. +/// +/// Note that the designation Theirs, i.e., the device already belongs to some +/// other entity is the default designation. This seems counterintuitive, but +/// it is the unclaimed designation that has a boolean expression on udev +/// properties associated with it. +pub fn decide_ownership(device: &libudev::Device) -> StratisResult { + // We believe that it is possible to be a multipath member and also to + // be identified as a Stratis device. The designations are not mutually + // exclusive, but the multipath member device must not be used by Stratis. + if is_multipath_member(device)? { + return Ok(UdevOwnership::MultipathMember); + } + + // We believe that the following designations are mutually exclusive, i.e. + // it is not possible to be a Stratis device and also to appear unowned. + Ok(if is_stratis(device)? { + UdevOwnership::Stratis + } else if is_unclaimed(device) { + UdevOwnership::Unowned + } else { + UdevOwnership::Theirs + }) +} + /// Takes a libudev device entry and returns the properties as a HashMap. fn device_as_map(device: &libudev::Device) -> HashMap { let rc: HashMap<_, _> = device From 59532ad9c3dffe584053fdd3de0b7ef75963d236 Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 25 Nov 2019 11:48:09 -0500 Subject: [PATCH 017/109] Factor code that obtains DevOwnership out of identify() method Signed-off-by: mulhern --- src/engine/strat_engine/backstore/device.rs | 101 ++++++++++---------- 1 file changed, 52 insertions(+), 49 deletions(-) diff --git a/src/engine/strat_engine/backstore/device.rs b/src/engine/strat_engine/backstore/device.rs index 6746969b84..fa3aa3657e 100644 --- a/src/engine/strat_engine/backstore/device.rs +++ b/src/engine/strat_engine/backstore/device.rs @@ -55,6 +55,57 @@ impl DevOwnership { _ => None, } } + + /// Given a udev assignment of ownership and the devnode for the device + /// in question, do some additional work to determine DevOwnership. + pub fn from_udev_ownership( + ownership: &UdevOwnership, + devnode: &Path, + ) -> StratisResult { + match ownership { + UdevOwnership::Unowned => { + // FIXME: It is possible that Stratis is running in + // an old environment without the necessary version of + // libblkid that would cause udev database to be + // populated with Stratis information. So, if the + // device appears unowned, attempt to read information + // from Stratis metadata. We believe that this block + // can be removed once Stratis is certainly runnng + // with libblkid 2.32 or above. + // https://github.com/stratis-storage/stratisd/issues/1656 + if let Some((pool_uuid, device_uuid)) = + device_identifiers(&mut OpenOptions::new().read(true).open(&devnode)?)? + { + Ok(DevOwnership::Ours(pool_uuid, device_uuid)) + } else { + Ok(DevOwnership::Unowned) + } + } + UdevOwnership::MultipathMember => Ok(DevOwnership::Theirs("multipath member".into())), + UdevOwnership::Stratis => { + // Udev information does not include pool UUID and + // device UUID so read these from Stratis metadata. + if let Some((pool_uuid, device_uuid)) = + device_identifiers(&mut OpenOptions::new().read(true).open(&devnode)?)? + { + Ok(DevOwnership::Ours(pool_uuid, device_uuid)) + } else { + // FIXME: if udev says stratis but no stratis + // idenfiers on device, likely they were there + // recently, and udev has not yet caught up. It's + // just as likely that this device is unclaimed as + // that it belongs to some other entity. + Ok(DevOwnership::Theirs( + "Udev db says stratis, disk meta says no".into(), + )) + } + } + UdevOwnership::Theirs => Ok(DevOwnership::Theirs( + "udev properties for this device did not indicate that the device was unowned" + .into(), + )), + } + } } /// Determine what a block device is used for. @@ -69,55 +120,7 @@ pub fn identify(devnode: &Path) -> StratisResult { .find(|x| x.devnode().map_or(false, |d| canonical == d)) .map(|d| decide_ownership(&d)) { - match udev_decision { - Ok(decision) => { - match decision { - UdevOwnership::Unowned => { - // FIXME: It is possible that Stratis is running in - // an old environment without the necessary version of - // libblkid that would cause udev database to be - // populated with Stratis information. So, if the - // device appears unowned, attempt to read information - // from Stratis metadata. We believe that this block - // can be removed once Stratis is certainly runnng - // with libblkid 2.32 or above. - // https://github.com/stratis-storage/stratisd/issues/1656 - if let Some((pool_uuid, device_uuid)) = - device_identifiers(&mut OpenOptions::new().read(true).open(&devnode)?)? - { - Ok(DevOwnership::Ours(pool_uuid, device_uuid)) - } else { - Ok(DevOwnership::Unowned) - } - } - UdevOwnership::MultipathMember => { - Ok(DevOwnership::Theirs("multipath member".into())) - } - UdevOwnership::Stratis => { - // Udev information does not include pool UUID and - // device UUID so read these from Stratis metadata. - if let Some((pool_uuid, device_uuid)) = - device_identifiers(&mut OpenOptions::new().read(true).open(&devnode)?)? - { - Ok(DevOwnership::Ours(pool_uuid, device_uuid)) - } else { - // FIXME: if udev says stratis but no stratis - // idenfiers on device, likely they were there - // recently, and udev has not yet caught up. It's - // just as likely that this device is unclaimed as - // that it belongs to some other entity. - Ok(DevOwnership::Theirs( - "Udev db says stratis, disk meta says no".into(), - )) - } - } - UdevOwnership::Theirs => Ok(DevOwnership::Theirs( - "udev properties for this device did not indicate that the devices was unowned".into() - )), - } - } - Err(err) => Err(err), - } + udev_decision.and_then(|decision| DevOwnership::from_udev_ownership(&decision, &canonical)) } else { Err(StratisError::Engine( ErrorEnum::NotFound, From 63ab73e7c7433109cd6bf192ffce0d5b6a1769ba Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 25 Nov 2019 12:41:22 -0500 Subject: [PATCH 018/109] Do not call identify() method in find_all() method The drawback of doing this was confusion due to multiple redundant checks for the same properties, i.e., checking is_multipath_member and is_unclaimed both in find_all() and in identify(), and also iterating twice through all the udev devices, essentially making the function quadratic instead of linear. Use the two component methods decide_ownership(), which already handles checking for multipath membership and unclaimedness via udev, and DevOwnership::from_udev_ownership() which does the remainder of the actions in identify. This is intended to be semantics preserving: * All devices that don't have a devnode are excluded. * All devices considered multipath member devices are excluded. * All devices considered unclaimed are included. * If there is an error in getting the information, the device is just ignored. Fix up the comment that describes the code so that it describes what the code actually does. Note: This is a slight change, because the previously existing code would have excluded from its search a device that appeared to be a Stratis device, but the current search will not. This would have happened if in the interval between the first udev enumeration and the second, some devices properly labeled Stratis had actually appeared. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/setup.rs | 37 +++++++++++++--------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/src/engine/strat_engine/backstore/setup.rs b/src/engine/strat_engine/backstore/setup.rs index 7405ab6ebd..deadba26b9 100644 --- a/src/engine/strat_engine/backstore/setup.rs +++ b/src/engine/strat_engine/backstore/setup.rs @@ -22,9 +22,11 @@ use crate::{ strat_engine::{ backstore::{ blockdev::StratBlockDev, - device::identify, + device::DevOwnership, metadata::{device_identifiers, BDA}, - udev::{block_enumerator, is_multipath_member, is_unclaimed, stratis_enumerator}, + udev::{ + block_enumerator, decide_ownership, is_multipath_member, stratis_enumerator, + }, }, device::blkdev_size, serde_structs::{BackstoreSave, BaseBlockDevSave, PoolSave}, @@ -49,25 +51,30 @@ fn get_stratis_block_devices() -> StratisResult> { }; if devices.is_empty() { - // We have found no Stratis devices, possible reasons are: - // 1. We really don't have any - // 2. We have some, but libblkid is too old to support Stratis, thus we appear empty - // 3. We ran this code at early boot before we have any udev db entries which are complete - // or are complete but fall into reasons 1 & 2 above + // No Stratis devices have been found, possible reasons are: + // 1. There are none + // 2. There are some but libblkid version is less than 2.32, so + // Stratis devices are not recognized by udev. + // 3. There are many incomplete udev entries, because this code is + // being run before the udev database is populated. // - // In this case we will get all the block devices which have complete udev db block device - // entries and appear "empty" and go out to disk and check them! + // Try again to find Stratis block devices, but this time enumerate + // all block devices, not just all the ones that can be identified + // as Stratis blockdevs by udev, and then scrutinize each one + // using various methods. let context = libudev::Context::new()?; let mut enumerator = block_enumerator(&context)?; Ok(enumerator .scan_devices()? .filter(|dev| dev.is_initialized()) - .filter(|dev| !is_multipath_member(dev).unwrap_or(true) && is_unclaimed(dev)) - .filter_map(|dev| dev.devnode().map(|d| d.to_owned())) - .filter(|devnode| { - identify(devnode) - .map(|ownership| ownership.stratis_identifiers().is_some()) - .unwrap_or(false) + .filter_map(|dev| { + dev.devnode().and_then(|devnode| { + decide_ownership(&dev) + .and_then(|decision| DevOwnership::from_udev_ownership(&decision, devnode)) + .ok() + .and_then(|ownership| ownership.stratis_identifiers()) + .map(|_| devnode.into()) + }) }) .collect()) } else { From 2a3dca95963a8d6f21fb3ba96fdb244f614eaa26 Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 26 Nov 2019 09:42:35 -0500 Subject: [PATCH 019/109] Get the ownership designation and the hardware id at the same time This way, all devices belonging to the udev database are iterated through just once for each block device passed to udev_initialize, instead of twice. Also, the missing udev entry possibility is handled in the one place that the udev devices are iterated over, by returning an error if it can't be found, as before. Previously, with the lookup of hw_id, it ignored a missing udev entry, which made sense, because it had sort of proved that one existed, by finding it at an earlier stage in the method. But, that now leaves the only possibility of failure as a failure to decode the hw id properly, so I'll just log that as a warn. This takes care of the TODO and is slightly better than what the previous code would do: in the unlikely event of an error decoding the ID_WWN it would return a lossy string. Allow type_complexity on dev_info, it is an internal method. Get rid of some obsoleted methods. Signed-off-by: mulhern --- .../strat_engine/backstore/blockdevmgr.rs | 93 ++++++++++++++----- src/engine/strat_engine/backstore/udev.rs | 42 +-------- 2 files changed, 74 insertions(+), 61 deletions(-) diff --git a/src/engine/strat_engine/backstore/blockdevmgr.rs b/src/engine/strat_engine/backstore/blockdevmgr.rs index d4e819d752..a3664cd7a4 100644 --- a/src/engine/strat_engine/backstore/blockdevmgr.rs +++ b/src/engine/strat_engine/backstore/blockdevmgr.rs @@ -25,9 +25,9 @@ use crate::{ strat_engine::{ backstore::{ blockdev::StratBlockDev, - device::{identify, resolve_devices, DevOwnership}, + device::{resolve_devices, DevOwnership}, metadata::{disown_device, BlockdevSize, MDADataSize, BDA}, - udev::hw_lookup, + udev::{block_enumerator, decide_ownership, get_udev_property}, }, device::blkdev_size, serde_structs::{BaseBlockDevSave, BaseDevSave, Recordable}, @@ -367,14 +367,48 @@ fn initialize( /// Get device information, returns an error if problem with obtaining /// that information. /// Returns a tuple with the device's path, its size in bytes, - /// its signature as determined by calling device::identify(), - /// and an open File handle, all of which are needed later. - fn dev_info(devnode: &Path) -> StratisResult<(&Path, Bytes, DevOwnership, File)> { + /// its DevOwnership classification, its optional hw_id, + /// and an open File handle. + #[allow(clippy::type_complexity)] + fn dev_info( + devnode: &Path, + ) -> StratisResult<( + &Path, + Bytes, + DevOwnership, + Option>, + File, + )> { let f = OpenOptions::new().read(true).write(true).open(&devnode)?; let dev_size = blkdev_size(&f)?; - let ownership = identify(devnode)?; - Ok((devnode, dev_size, ownership, f)) + let canonical = devnode.canonicalize()?; + + let context = libudev::Context::new()?; + let mut enumerator = block_enumerator(&context)?; + if let Some((ownership, hw_id)) = enumerator + .scan_devices()? + .filter(|dev| dev.is_initialized()) + .find(|x| x.devnode().map_or(false, |d| canonical == d)) + .map(|d| { + ( + decide_ownership(&d).and_then(|decision| { + DevOwnership::from_udev_ownership(&decision, &canonical) + }), + get_udev_property(&d, "ID_WWN"), + ) + }) + { + Ok((devnode, dev_size, ownership?, hw_id, f)) + } else { + Err(StratisError::Engine( + ErrorEnum::NotFound, + format!( + "Block device {} not found in the udev database", + devnode.display() + ), + )) + } } /// Filter devices for admission to pool based on dev_infos. @@ -385,20 +419,36 @@ fn initialize( dev_infos: I, pool_uuid: PoolUuid, owned_devs: &HashSet, - ) -> StratisResult> + ) -> StratisResult< + Vec<( + Device, + (&'a Path, Bytes, Option>, File), + )>, + > where - I: Iterator)>, + I: Iterator< + Item = ( + Device, + StratisResult<( + &'a Path, + Bytes, + DevOwnership, + Option>, + File, + )>, + ), + >, { let mut add_devs = Vec::new(); for (dev, dev_result) in dev_infos { - let (devnode, dev_size, ownership, f) = dev_result?; + let (devnode, dev_size, ownership, hw_id, f) = dev_result?; if dev_size < MIN_DEV_SIZE { let error_message = format!("{} too small, minimum {}", devnode.display(), MIN_DEV_SIZE); return Err(StratisError::Engine(ErrorEnum::Invalid, error_message)); }; match ownership { - DevOwnership::Unowned => add_devs.push((dev, (devnode, dev_size, f))), + DevOwnership::Unowned => add_devs.push((dev, (devnode, dev_size, hw_id, f))), DevOwnership::Theirs(info) => { let err_str = format!( "Device {} appears to be already claimed by another, reason: {}", @@ -435,7 +485,7 @@ fn initialize( let add_devs = filter_devs(dev_infos, pool_uuid, owned_devs)?; let mut bds: Vec = Vec::new(); - for (dev, (devnode, dev_size, mut f)) in add_devs { + for (dev, (devnode, dev_size, hw_id, mut f)) in add_devs { let bda = BDA::initialize( &mut f, pool_uuid, @@ -445,14 +495,15 @@ fn initialize( Utc::now().timestamp() as u64, ); if let Ok(bda) = bda { - let hw_id = match hw_lookup(devnode) { - Ok(id) => id, - // TODO: Consider logging if no hardware ID obtained. If - // logging distinguish between non-existant ID, which is a - // normal situation, and failure to obtain the ID due to an - // error in decoding the ID or failure to locate the udev - // entry for this devnode, which is not normal. - Err(_) => None, + let hw_id = match hw_id { + Some(Ok(hw_id)) => Some(hw_id), + Some(Err(_)) => { + warn!("Value for ID_WWN for device {} obtained from the udev database could not be decoded; inserting device into pool with UUID {} anyway", + devnode.display(), + pool_uuid.to_simple_ref()); + None + } + None => None, }; // FIXME: The expect is only provisionally true. @@ -503,7 +554,7 @@ mod tests { use uuid::Uuid; use crate::engine::strat_engine::{ - backstore::{find_all, get_metadata}, + backstore::{device::identify, find_all, get_metadata}, cmd, tests::{loopbacked, real}, }; diff --git a/src/engine/strat_engine/backstore/udev.rs b/src/engine/strat_engine/backstore/udev.rs index 398af3801e..93e9425431 100644 --- a/src/engine/strat_engine/backstore/udev.rs +++ b/src/engine/strat_engine/backstore/udev.rs @@ -3,7 +3,7 @@ // file, You can obtain one at http://mozilla.org/MPL/2.0/. //! udev-related methods -use std::{collections::HashMap, ffi::OsStr, fs, path::Path}; +use std::ffi::OsStr; use libudev; @@ -30,7 +30,7 @@ pub fn stratis_enumerator(context: &libudev::Context) -> libudev::Result>( +pub fn get_udev_property>( device: &libudev::Device, property_name: T, ) -> Option> @@ -117,41 +117,3 @@ pub fn decide_ownership(device: &libudev::Device) -> StratisResult HashMap { - let rc: HashMap<_, _> = device - .properties() - .map(|i| { - ( - String::from(i.name().to_string_lossy()), - String::from(i.value().to_string_lossy()), - ) - }) - .collect(); - rc -} - -/// Common function used to retrieve the udev db entry for a block device as a HashMap when found -pub fn get_udev_block_device( - dev_node_search: &Path, -) -> StratisResult>> { - let context = libudev::Context::new()?; - let mut enumerator = block_enumerator(&context)?; - - // Get canonical form to ensure we do correct lookup in udev db - let canonical = fs::canonicalize(dev_node_search)?; - - let result = enumerator - .scan_devices()? - .filter(|dev| dev.is_initialized()) - .find(|x| x.devnode().map_or(false, |d| canonical == d)) - .map(|dev| device_as_map(&dev)); - Ok(result) -} - -/// Lookup the WWN from the udev db using the device node eg. /dev/sda -pub fn hw_lookup(dev_node_search: &Path) -> StratisResult> { - let dev = get_udev_block_device(dev_node_search)?; - Ok(dev.and_then(|dev| dev.get("ID_WWN").cloned())) -} From 4f0978c2b9abfab2df071f564a5d70a51ef3d8e1 Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 26 Nov 2019 10:34:04 -0500 Subject: [PATCH 020/109] Hoist udev module out of backstore into strat_engine Signed-off-by: mulhern --- src/engine/strat_engine/backstore/blockdevmgr.rs | 2 +- src/engine/strat_engine/backstore/device.rs | 4 ++-- src/engine/strat_engine/backstore/mod.rs | 1 - src/engine/strat_engine/backstore/setup.rs | 4 +--- src/engine/strat_engine/mod.rs | 1 + src/engine/strat_engine/{backstore => }/udev.rs | 0 6 files changed, 5 insertions(+), 7 deletions(-) rename src/engine/strat_engine/{backstore => }/udev.rs (100%) diff --git a/src/engine/strat_engine/backstore/blockdevmgr.rs b/src/engine/strat_engine/backstore/blockdevmgr.rs index a3664cd7a4..ba19bab142 100644 --- a/src/engine/strat_engine/backstore/blockdevmgr.rs +++ b/src/engine/strat_engine/backstore/blockdevmgr.rs @@ -27,10 +27,10 @@ use crate::{ blockdev::StratBlockDev, device::{resolve_devices, DevOwnership}, metadata::{disown_device, BlockdevSize, MDADataSize, BDA}, - udev::{block_enumerator, decide_ownership, get_udev_property}, }, device::blkdev_size, serde_structs::{BaseBlockDevSave, BaseDevSave, Recordable}, + udev::{block_enumerator, decide_ownership, get_udev_property}, }, types::{DevUuid, PoolUuid}, }, diff --git a/src/engine/strat_engine/backstore/device.rs b/src/engine/strat_engine/backstore/device.rs index fa3aa3657e..55a2a8a8d5 100644 --- a/src/engine/strat_engine/backstore/device.rs +++ b/src/engine/strat_engine/backstore/device.rs @@ -10,8 +10,8 @@ use devicemapper::{devnode_to_devno, Device}; use crate::{ engine::{ - strat_engine::backstore::{ - metadata::device_identifiers, + strat_engine::{ + backstore::metadata::device_identifiers, udev::{block_enumerator, decide_ownership, UdevOwnership}, }, types::{DevUuid, PoolUuid}, diff --git a/src/engine/strat_engine/backstore/mod.rs b/src/engine/strat_engine/backstore/mod.rs index 755bcaec39..9e6e4489dd 100644 --- a/src/engine/strat_engine/backstore/mod.rs +++ b/src/engine/strat_engine/backstore/mod.rs @@ -13,7 +13,6 @@ mod metadata; mod range_alloc; mod setup; mod shared; -mod udev; pub use self::{ backstore::Backstore, diff --git a/src/engine/strat_engine/backstore/setup.rs b/src/engine/strat_engine/backstore/setup.rs index deadba26b9..51e3da343c 100644 --- a/src/engine/strat_engine/backstore/setup.rs +++ b/src/engine/strat_engine/backstore/setup.rs @@ -24,12 +24,10 @@ use crate::{ blockdev::StratBlockDev, device::DevOwnership, metadata::{device_identifiers, BDA}, - udev::{ - block_enumerator, decide_ownership, is_multipath_member, stratis_enumerator, - }, }, device::blkdev_size, serde_structs::{BackstoreSave, BaseBlockDevSave, PoolSave}, + udev::{block_enumerator, decide_ownership, is_multipath_member, stratis_enumerator}, }, types::{BlockDevTier, DevUuid, PoolUuid}, }, diff --git a/src/engine/strat_engine/mod.rs b/src/engine/strat_engine/mod.rs index d9915c5cd0..7606fcb21b 100644 --- a/src/engine/strat_engine/mod.rs +++ b/src/engine/strat_engine/mod.rs @@ -13,6 +13,7 @@ mod names; mod pool; mod serde_structs; mod thinpool; +mod udev; pub use self::engine::StratEngine; diff --git a/src/engine/strat_engine/backstore/udev.rs b/src/engine/strat_engine/udev.rs similarity index 100% rename from src/engine/strat_engine/backstore/udev.rs rename to src/engine/strat_engine/udev.rs From 7ffc0ace83d0c005bb5a7931c2787e92d3570629 Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 26 Nov 2019 10:43:46 -0500 Subject: [PATCH 021/109] Remove use of identify from block_evaluate This is the last use of identify() in source, so downgrade it to a test method. Make the assignment to stratis_identifiers the same as before, to minimize distracting changes in the remainder of the block_evaluate method. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/device.rs | 9 ++++--- src/engine/strat_engine/backstore/mod.rs | 2 +- src/engine/strat_engine/engine.rs | 28 +++++++++++++++++++-- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/engine/strat_engine/backstore/device.rs b/src/engine/strat_engine/backstore/device.rs index 55a2a8a8d5..c176322d53 100644 --- a/src/engine/strat_engine/backstore/device.rs +++ b/src/engine/strat_engine/backstore/device.rs @@ -10,15 +10,15 @@ use devicemapper::{devnode_to_devno, Device}; use crate::{ engine::{ - strat_engine::{ - backstore::metadata::device_identifiers, - udev::{block_enumerator, decide_ownership, UdevOwnership}, - }, + strat_engine::{backstore::metadata::device_identifiers, udev::UdevOwnership}, types::{DevUuid, PoolUuid}, }, stratis::{ErrorEnum, StratisError, StratisResult}, }; +#[cfg(test)] +use crate::engine::strat_engine::udev::{block_enumerator, decide_ownership}; + /// Resolve a list of Paths of some sort to a set of unique Devices. /// Return an IOError if there was a problem resolving any particular device. /// The set of devices maps each device to one of the paths passed. @@ -109,6 +109,7 @@ impl DevOwnership { } /// Determine what a block device is used for. +#[cfg(test)] pub fn identify(devnode: &Path) -> StratisResult { let canonical = devnode.canonicalize()?; diff --git a/src/engine/strat_engine/backstore/mod.rs b/src/engine/strat_engine/backstore/mod.rs index 9e6e4489dd..ce5745aeed 100644 --- a/src/engine/strat_engine/backstore/mod.rs +++ b/src/engine/strat_engine/backstore/mod.rs @@ -17,7 +17,7 @@ mod shared; pub use self::{ backstore::Backstore, blockdev::StratBlockDev, - device::identify, + device::DevOwnership, metadata::MDADataSize, setup::{find_all, get_metadata}, }; diff --git a/src/engine/strat_engine/engine.rs b/src/engine/strat_engine/engine.rs index c6b1b7eee6..4a9baecdc6 100644 --- a/src/engine/strat_engine/engine.rs +++ b/src/engine/strat_engine/engine.rs @@ -8,6 +8,8 @@ use std::{ path::{Path, PathBuf}, }; +use libudev; + use devicemapper::{Device, DmNameBuf}; #[cfg(test)] @@ -20,11 +22,12 @@ use crate::{ event::get_engine_listener_list, shared::create_pool_idempotent_or_err, strat_engine::{ - backstore::{find_all, get_metadata, identify}, + backstore::{find_all, get_metadata, DevOwnership}, cmd::verify_binaries, dm::{get_dm, get_dm_init}, names::validate_name, pool::{check_metadata, StratPool}, + udev::{block_enumerator, decide_ownership}, }, structures::Table, types::{CreateAction, DeleteAction, RenameAction}, @@ -206,7 +209,28 @@ impl Engine for StratEngine { device: Device, dev_node: PathBuf, ) -> StratisResult> { - let stratis_identifiers = identify(&dev_node)?.stratis_identifiers(); + let canonical = dev_node.canonicalize()?; + + let context = libudev::Context::new()?; + let mut enumerator = block_enumerator(&context)?; + let stratis_identifiers = if let Some(ownership) = enumerator + .scan_devices()? + .filter(|dev| dev.is_initialized()) + .find(|x| x.devnode().map_or(false, |d| canonical == d)) + .map(|d| { + decide_ownership(&d) + .and_then(|decision| DevOwnership::from_udev_ownership(&decision, &canonical)) + }) { + ownership?.stratis_identifiers() + } else { + return Err(StratisError::Engine( + ErrorEnum::NotFound, + format!( + "Block device {} not found in the udev database", + dev_node.display() + ), + )); + }; let pool_uuid = if let Some((pool_uuid, device_uuid)) = stratis_identifiers { if self.pools.contains_uuid(pool_uuid) { // We can get udev events for devices that are already in the pool. Lets check From 384c1347a51f17c2064dc1e549a7c301ac747512 Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 26 Nov 2019 11:15:17 -0500 Subject: [PATCH 022/109] Add a block_device_apply method to the udev module Use where appropriate. Entirely remove the identify() method but substitute applications of block_device_apply where appropriate in testing. Signed-off-by: mulhern --- .../strat_engine/backstore/blockdevmgr.rs | 47 ++++++----- src/engine/strat_engine/backstore/device.rs | 80 ------------------- src/engine/strat_engine/engine.rs | 22 ++--- src/engine/strat_engine/udev.rs | 25 +++++- 4 files changed, 56 insertions(+), 118 deletions(-) diff --git a/src/engine/strat_engine/backstore/blockdevmgr.rs b/src/engine/strat_engine/backstore/blockdevmgr.rs index ba19bab142..3045e80fcf 100644 --- a/src/engine/strat_engine/backstore/blockdevmgr.rs +++ b/src/engine/strat_engine/backstore/blockdevmgr.rs @@ -30,7 +30,7 @@ use crate::{ }, device::blkdev_size, serde_structs::{BaseBlockDevSave, BaseDevSave, Recordable}, - udev::{block_enumerator, decide_ownership, get_udev_property}, + udev::{block_device_apply, decide_ownership, get_udev_property}, }, types::{DevUuid, PoolUuid}, }, @@ -382,23 +382,13 @@ fn initialize( let f = OpenOptions::new().read(true).write(true).open(&devnode)?; let dev_size = blkdev_size(&f)?; - let canonical = devnode.canonicalize()?; - - let context = libudev::Context::new()?; - let mut enumerator = block_enumerator(&context)?; - if let Some((ownership, hw_id)) = enumerator - .scan_devices()? - .filter(|dev| dev.is_initialized()) - .find(|x| x.devnode().map_or(false, |d| canonical == d)) - .map(|d| { - ( - decide_ownership(&d).and_then(|decision| { - DevOwnership::from_udev_ownership(&decision, &canonical) - }), - get_udev_property(&d, "ID_WWN"), - ) - }) - { + if let Some((ownership, hw_id)) = block_device_apply(devnode, |d| { + ( + decide_ownership(d) + .and_then(|decision| DevOwnership::from_udev_ownership(&decision, devnode)), + get_udev_property(d, "ID_WWN"), + ) + })? { Ok((devnode, dev_size, ownership?, hw_id, f)) } else { Err(StratisError::Engine( @@ -554,9 +544,10 @@ mod tests { use uuid::Uuid; use crate::engine::strat_engine::{ - backstore::{device::identify, find_all, get_metadata}, + backstore::{find_all, get_metadata}, cmd, tests::{loopbacked, real}, + udev::{block_device_apply, decide_ownership}, }; use crate::engine::strat_engine::backstore::metadata::device_identifiers; @@ -620,9 +611,23 @@ mod tests { ); for (i, path) in paths.iter().enumerate() { if i == index { - assert_matches!(identify(path).unwrap(), DevOwnership::Theirs(_)); + assert_matches!( + block_device_apply(path, |d| decide_ownership(d) + .and_then(|decision| DevOwnership::from_udev_ownership(&decision, path))) + .unwrap() + .unwrap() + .unwrap(), + DevOwnership::Theirs(_) + ); } else { - assert_matches!(identify(path).unwrap(), DevOwnership::Unowned); + assert_matches!( + block_device_apply(path, |d| decide_ownership(d) + .and_then(|decision| DevOwnership::from_udev_ownership(&decision, path))) + .unwrap() + .unwrap() + .unwrap(), + DevOwnership::Unowned + ); } } diff --git a/src/engine/strat_engine/backstore/device.rs b/src/engine/strat_engine/backstore/device.rs index c176322d53..5c95f63bda 100644 --- a/src/engine/strat_engine/backstore/device.rs +++ b/src/engine/strat_engine/backstore/device.rs @@ -16,9 +16,6 @@ use crate::{ stratis::{ErrorEnum, StratisError, StratisResult}, }; -#[cfg(test)] -use crate::engine::strat_engine::udev::{block_enumerator, decide_ownership}; - /// Resolve a list of Paths of some sort to a set of unique Devices. /// Return an IOError if there was a problem resolving any particular device. /// The set of devices maps each device to one of the paths passed. @@ -107,80 +104,3 @@ impl DevOwnership { } } } - -/// Determine what a block device is used for. -#[cfg(test)] -pub fn identify(devnode: &Path) -> StratisResult { - let canonical = devnode.canonicalize()?; - - let context = libudev::Context::new()?; - let mut enumerator = block_enumerator(&context)?; - if let Some(udev_decision) = enumerator - .scan_devices()? - .filter(|dev| dev.is_initialized()) - .find(|x| x.devnode().map_or(false, |d| canonical == d)) - .map(|d| decide_ownership(&d)) - { - udev_decision.and_then(|decision| DevOwnership::from_udev_ownership(&decision, &canonical)) - } else { - Err(StratisError::Engine( - ErrorEnum::NotFound, - format!( - "We expected to find the block device {:?} \ - in the udev db", - devnode - ), - )) - } -} - -#[cfg(test)] -mod test { - use std::path::Path; - - use crate::engine::strat_engine::{ - cmd, - tests::{loopbacked, real}, - }; - - use super::*; - - /// Verify that the device is not stratis by creating a device with XFS fs. - fn test_other_ownership(paths: &[&Path]) { - cmd::create_fs(paths[0], None).unwrap(); - cmd::udev_settle().unwrap(); - assert_matches!(identify(paths[0]).unwrap(), DevOwnership::Theirs(_)) - } - - /// Verify that identify() recognizes a blank device as unowned - fn test_empty(paths: &[&Path]) { - cmd::udev_settle().unwrap(); - assert_matches!(identify(paths[0]).unwrap(), DevOwnership::Unowned); - } - - #[test] - pub fn loop_test_device_other_ownership() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Range(1, 3, None), - test_other_ownership, - ); - } - - #[test] - pub fn real_test_device_other_ownership() { - real::test_with_spec( - &real::DeviceLimits::AtLeast(1, None, None), - test_other_ownership, - ); - } - - #[test] - pub fn loop_test_device_empty() { - loopbacked::test_with_spec(&loopbacked::DeviceLimits::Range(1, 3, None), test_empty); - } - - #[test] - pub fn real_test_device_empty() { - real::test_with_spec(&real::DeviceLimits::AtLeast(1, None, None), test_empty); - } -} diff --git a/src/engine/strat_engine/engine.rs b/src/engine/strat_engine/engine.rs index 4a9baecdc6..509a9d9165 100644 --- a/src/engine/strat_engine/engine.rs +++ b/src/engine/strat_engine/engine.rs @@ -8,8 +8,6 @@ use std::{ path::{Path, PathBuf}, }; -use libudev; - use devicemapper::{Device, DmNameBuf}; #[cfg(test)] @@ -27,7 +25,7 @@ use crate::{ dm::{get_dm, get_dm_init}, names::validate_name, pool::{check_metadata, StratPool}, - udev::{block_enumerator, decide_ownership}, + udev::{block_device_apply, decide_ownership}, }, structures::Table, types::{CreateAction, DeleteAction, RenameAction}, @@ -209,19 +207,11 @@ impl Engine for StratEngine { device: Device, dev_node: PathBuf, ) -> StratisResult> { - let canonical = dev_node.canonicalize()?; - - let context = libudev::Context::new()?; - let mut enumerator = block_enumerator(&context)?; - let stratis_identifiers = if let Some(ownership) = enumerator - .scan_devices()? - .filter(|dev| dev.is_initialized()) - .find(|x| x.devnode().map_or(false, |d| canonical == d)) - .map(|d| { - decide_ownership(&d) - .and_then(|decision| DevOwnership::from_udev_ownership(&decision, &canonical)) - }) { - ownership?.stratis_identifiers() + let stratis_identifiers = if let Some(ownership) = block_device_apply(&dev_node, |d| { + decide_ownership(d) + .and_then(|decision| DevOwnership::from_udev_ownership(&decision, &dev_node)) + })? { + ownership?.stratis_identifiers() } else { return Err(StratisError::Engine( ErrorEnum::NotFound, diff --git a/src/engine/strat_engine/udev.rs b/src/engine/strat_engine/udev.rs index 93e9425431..f27c0bfc3e 100644 --- a/src/engine/strat_engine/udev.rs +++ b/src/engine/strat_engine/udev.rs @@ -3,7 +3,7 @@ // file, You can obtain one at http://mozilla.org/MPL/2.0/. //! udev-related methods -use std::ffi::OsStr; +use std::{ffi::OsStr, path::Path}; use libudev; @@ -117,3 +117,26 @@ pub fn decide_ownership(device: &libudev::Device) -> StratisResult(devnode: &Path, f: F) -> StratisResult> +where + F: FnOnce(&libudev::Device) -> U, +{ + let canonical = devnode.canonicalize()?; + + let context = libudev::Context::new()?; + let mut enumerator = block_enumerator(&context)?; + + Ok(enumerator + .scan_devices()? + .filter(|dev| dev.is_initialized()) + .find(|x| x.devnode().map_or(false, |d| canonical == d)) + .map(|d| f(&d))) +} From 11ad1b26c97ec3bcdabbe5d31252181cd47d19bc Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 26 Nov 2019 18:15:04 -0500 Subject: [PATCH 023/109] Make a few functions private Might as well not make them public if unnecessary. We can make them public in future with very little trouble. Signed-off-by: mulhern --- src/engine/strat_engine/udev.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/engine/strat_engine/udev.rs b/src/engine/strat_engine/udev.rs index f27c0bfc3e..f19427784c 100644 --- a/src/engine/strat_engine/udev.rs +++ b/src/engine/strat_engine/udev.rs @@ -64,7 +64,7 @@ pub fn is_multipath_member(device: &libudev::Device) -> StratisResult { /// known to udev to claim this device. /// Note from mulhern: I have no idea myself why this particular expression /// should be correct. I was told that the original source was dlehman. -pub fn is_unclaimed(device: &libudev::Device) -> bool { +fn is_unclaimed(device: &libudev::Device) -> bool { (get_udev_property(device, "ID_PART_TABLE_TYPE").is_none() || get_udev_property(device, "ID_PART_ENTRY_DISK").is_some()) && get_udev_property(device, "ID_FS_USAGE").is_none() @@ -72,7 +72,7 @@ pub fn is_unclaimed(device: &libudev::Device) -> bool { /// Return true if the device is identified by udev as belonging to Stratis. /// Return an error if a udev property value could not be converted. -pub fn is_stratis(device: &libudev::Device) -> StratisResult { +fn is_stratis(device: &libudev::Device) -> StratisResult { match get_udev_property(device, "ID_FS_TYPE") { None => Ok(false), Some(Ok(value)) => Ok(value == "stratis"), From b0689b858fe235aa3afcc1c6082307b1e1235433 Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 26 Nov 2019 18:21:37 -0500 Subject: [PATCH 024/109] Improve some error messages slightly Signed-off-by: mulhern --- src/engine/strat_engine/backstore/blockdevmgr.rs | 2 +- src/engine/strat_engine/engine.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/engine/strat_engine/backstore/blockdevmgr.rs b/src/engine/strat_engine/backstore/blockdevmgr.rs index 3045e80fcf..f5dc6a4cd9 100644 --- a/src/engine/strat_engine/backstore/blockdevmgr.rs +++ b/src/engine/strat_engine/backstore/blockdevmgr.rs @@ -394,7 +394,7 @@ fn initialize( Err(StratisError::Engine( ErrorEnum::NotFound, format!( - "Block device {} not found in the udev database", + "Could not determine ownership of block device {} because it could not be found in the udev database", devnode.display() ), )) diff --git a/src/engine/strat_engine/engine.rs b/src/engine/strat_engine/engine.rs index 509a9d9165..f90023ffa1 100644 --- a/src/engine/strat_engine/engine.rs +++ b/src/engine/strat_engine/engine.rs @@ -216,7 +216,7 @@ impl Engine for StratEngine { return Err(StratisError::Engine( ErrorEnum::NotFound, format!( - "Block device {} not found in the udev database", + "Could not determine ownership of block device {} because it could not be found in the udev database", dev_node.display() ), )); From 16a3103393d514048b97bcb825a809ddf581bc14 Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 26 Nov 2019 18:40:42 -0500 Subject: [PATCH 025/109] Improve the header comment on get_stratis_block_devices Signed-off-by: mulhern --- src/engine/strat_engine/backstore/setup.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/engine/strat_engine/backstore/setup.rs b/src/engine/strat_engine/backstore/setup.rs index 51e3da343c..366299e360 100644 --- a/src/engine/strat_engine/backstore/setup.rs +++ b/src/engine/strat_engine/backstore/setup.rs @@ -34,7 +34,15 @@ use crate::{ stratis::{ErrorEnum, StratisError, StratisResult}, }; -/// Retrieve all the block devices on the system that have a Stratis signature. +/// Retrieve all Stratis block devices that should be made use of by the +/// Stratis engine. This excludes Stratis block devices that are multipath +/// members. +/// +/// Includes a fallback path, which is used if no Stratis block devices are +/// found using the obvious udev property- and enumerator-based approach. +/// This fallback path is more expensive, because it must search all block +/// devices via udev, and may also attempt to directly read Stratis +/// metadata from the device. fn get_stratis_block_devices() -> StratisResult> { let devices = { let context = libudev::Context::new()?; From 9b2319128c0624daa89aab0fa30ef19e8503d261 Mon Sep 17 00:00:00 2001 From: mulhern Date: Wed, 27 Nov 2019 11:19:17 -0500 Subject: [PATCH 026/109] Add a log entry to show if forced to use fallback search Signed-off-by: mulhern --- src/engine/strat_engine/backstore/setup.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/engine/strat_engine/backstore/setup.rs b/src/engine/strat_engine/backstore/setup.rs index 366299e360..7a9defd3c5 100644 --- a/src/engine/strat_engine/backstore/setup.rs +++ b/src/engine/strat_engine/backstore/setup.rs @@ -68,6 +68,9 @@ fn get_stratis_block_devices() -> StratisResult> { // all block devices, not just all the ones that can be identified // as Stratis blockdevs by udev, and then scrutinize each one // using various methods. + + info!("Could not identify any Stratis devices by a udev search for devices with ID_FS_TYPE=\"stratis\"; using fallback search mechanism"); + let context = libudev::Context::new()?; let mut enumerator = block_enumerator(&context)?; Ok(enumerator From 08b6a8dd1fbb0d5cc378b74a7c4d092c4452009d Mon Sep 17 00:00:00 2001 From: mulhern Date: Wed, 27 Nov 2019 13:05:23 -0500 Subject: [PATCH 027/109] Added a FIXME but decided not to address one outstanding problem Signed-off-by: mulhern --- src/engine/strat_engine/backstore/setup.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/engine/strat_engine/backstore/setup.rs b/src/engine/strat_engine/backstore/setup.rs index 7a9defd3c5..f9c227c5a8 100644 --- a/src/engine/strat_engine/backstore/setup.rs +++ b/src/engine/strat_engine/backstore/setup.rs @@ -97,6 +97,13 @@ fn get_stratis_block_devices() -> StratisResult> { pub fn find_all() -> StratisResult>> { let mut pool_map = HashMap::new(); + // FIXME: If get_stratis_block_devices() has to go to fallback mechanism, + // it has already read the device identifiers for every device it returns. + // It seems a waste to just read them all again in that case. Likely + // get_stratis_block_devices() and this method should be consolidated in + // some creative way, or get_stratis_block_devices() should be required + // to return the device identifiers from the device as well as the devnode, + // regardless of which mechanism was used. for devnode in get_stratis_block_devices()? { match devnode_to_devno(&devnode)? { None => continue, From 7ed22203be567a4e12a0a6dbb61256e3fdfdc43b Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 3 Dec 2019 09:09:09 -0500 Subject: [PATCH 028/109] Add an additional info-level log message Signed-off-by: mulhern --- src/engine/strat_engine/backstore/setup.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/engine/strat_engine/backstore/setup.rs b/src/engine/strat_engine/backstore/setup.rs index f9c227c5a8..bc814161dc 100644 --- a/src/engine/strat_engine/backstore/setup.rs +++ b/src/engine/strat_engine/backstore/setup.rs @@ -44,6 +44,7 @@ use crate::{ /// devices via udev, and may also attempt to directly read Stratis /// metadata from the device. fn get_stratis_block_devices() -> StratisResult> { + info!("Beginning initial search for Stratis block devices"); let devices = { let context = libudev::Context::new()?; let mut enumerator = stratis_enumerator(&context)?; From 130401cd9847ca96e382cc741dbc899a79a529ad Mon Sep 17 00:00:00 2001 From: mulhern Date: Wed, 4 Dec 2019 10:04:24 -0500 Subject: [PATCH 029/109] Use isort before black to sort imports Require isort 4.3.4 because it's the version available via dnf on Fedora 30. Use black-compatible isort configuration specified in the online black documentation. Add an additional section specification to group and order imports. Use headings for each category of import except the local folder. Signed-off-by: mulhern --- tests/client-dbus/.isort.cfg | 21 ++++++++++++++++ tests/client-dbus/Makefile | 2 ++ tests/client-dbus/check.py | 1 + tests/client-dbus/requirements.txt | 1 + tests/client-dbus/setup.py | 3 +++ .../src/stratisd_client_dbus/_connection.py | 1 + .../stratisd_client_dbus/_implementation.py | 6 ++--- .../_stratisd_constants.py | 1 + tests/client-dbus/tests/dbus/_misc.py | 2 ++ .../dbus/blockdev/test_fetch_properties.py | 6 ++--- .../tests/dbus/blockdev/test_properties.py | 10 ++++---- .../dbus/filesystem/test_fetch_properties.py | 10 +++----- .../tests/dbus/filesystem/test_properties.py | 10 +++----- .../tests/dbus/filesystem/test_rename.py | 21 ++++++++-------- .../tests/dbus/manager/test_create.py | 17 ++++++------- .../tests/dbus/manager/test_destroy.py | 23 +++++++++--------- .../tests/dbus/manager/test_object_path.py | 7 +++--- .../tests/dbus/manager/test_stratis.py | 7 +++--- .../tests/dbus/pool/test_add_cache_devs.py | 21 ++++++++-------- .../tests/dbus/pool/test_add_data_devs.py | 21 ++++++++-------- .../tests/dbus/pool/test_create_filesystem.py | 20 +++++++++------- .../tests/dbus/pool/test_create_snapshot.py | 19 ++++++++------- .../dbus/pool/test_destroy_filesystem.py | 19 ++++++++------- .../tests/dbus/pool/test_fetch_properties.py | 9 +++---- .../tests/dbus/pool/test_rename.py | 19 ++++++++------- tests/client-dbus/tests/misc/test_startup.py | 2 +- tests/client-dbus/tests/udev/_dm.py | 1 + tests/client-dbus/tests/udev/_loopback.py | 1 + tests/client-dbus/tests/udev/_stratis_id.py | 2 ++ tests/client-dbus/tests/udev/test_udev.py | 24 +++++++++---------- 30 files changed, 168 insertions(+), 139 deletions(-) create mode 100644 tests/client-dbus/.isort.cfg diff --git a/tests/client-dbus/.isort.cfg b/tests/client-dbus/.isort.cfg new file mode 100644 index 0000000000..68dc298099 --- /dev/null +++ b/tests/client-dbus/.isort.cfg @@ -0,0 +1,21 @@ +[settings] +multi_line_output=3 +include_trailing_comma=True +force_grid_wrap=0 +use_parentheses=True +line_length=88 + +sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCAL,LOCALFOLDER +default_section=THIRDPARTY + +import_heading_future=isort: FUTURE +import_heading_stdlib=isort: STDLIB +import_heading_thirdparty=isort: THIRDPARTY +import_heading_firstparty=isort: FIRSTPARTY +import_heading_local=isort: LOCAL + +# All items above should be the same for every +# Stratis project. The items below vary with +# each project. +known_first_party=dbus_client_gen,dbus_python_client_gen +known_local=stratisd_client_dbus diff --git a/tests/client-dbus/Makefile b/tests/client-dbus/Makefile index 641f2b29f4..fbc53c5073 100644 --- a/tests/client-dbus/Makefile +++ b/tests/client-dbus/Makefile @@ -15,10 +15,12 @@ misc-tests: .PHONY: fmt fmt: + isort --recursive check.py setup.py src tests black . .PHONY: fmt-travis fmt-travis: + isort --recursive --diff --check-only check.py setup.py src tests black . --check .PHONY: udev-tests diff --git a/tests/client-dbus/check.py b/tests/client-dbus/check.py index 46763bfc1b..263f14b74b 100755 --- a/tests/client-dbus/check.py +++ b/tests/client-dbus/check.py @@ -1,5 +1,6 @@ #!/usr/bin/python +# isort: STDLIB import argparse import subprocess import sys diff --git a/tests/client-dbus/requirements.txt b/tests/client-dbus/requirements.txt index c9849b3016..f7fe3acde7 100644 --- a/tests/client-dbus/requirements.txt +++ b/tests/client-dbus/requirements.txt @@ -1,2 +1,3 @@ tox==3.2.1 black==18.9b0 +isort==4.3.4 diff --git a/tests/client-dbus/setup.py b/tests/client-dbus/setup.py index 28e4377e7e..c95f8c1829 100644 --- a/tests/client-dbus/setup.py +++ b/tests/client-dbus/setup.py @@ -1,5 +1,8 @@ +# isort: STDLIB import os import sys + +# isort: THIRDPARTY import setuptools if sys.version_info[0] < 3: diff --git a/tests/client-dbus/src/stratisd_client_dbus/_connection.py b/tests/client-dbus/src/stratisd_client_dbus/_connection.py index 1b52d8ed25..f6054efc00 100644 --- a/tests/client-dbus/src/stratisd_client_dbus/_connection.py +++ b/tests/client-dbus/src/stratisd_client_dbus/_connection.py @@ -15,6 +15,7 @@ Miscellaneous helpful methods. """ +# isort: THIRDPARTY import dbus from ._constants import SERVICE diff --git a/tests/client-dbus/src/stratisd_client_dbus/_implementation.py b/tests/client-dbus/src/stratisd_client_dbus/_implementation.py index 8b70a50a85..e82015de48 100644 --- a/tests/client-dbus/src/stratisd_client_dbus/_implementation.py +++ b/tests/client-dbus/src/stratisd_client_dbus/_implementation.py @@ -15,11 +15,11 @@ Classes to implement dbus interface. """ +# isort: STDLIB import xml.etree.ElementTree as ET -from dbus_client_gen import managed_object_class -from dbus_client_gen import mo_query_builder - +# isort: FIRSTPARTY +from dbus_client_gen import managed_object_class, mo_query_builder from dbus_python_client_gen import make_class from ._data import SPECS diff --git a/tests/client-dbus/src/stratisd_client_dbus/_stratisd_constants.py b/tests/client-dbus/src/stratisd_client_dbus/_stratisd_constants.py index e85cc47378..6baa5a4d5f 100644 --- a/tests/client-dbus/src/stratisd_client_dbus/_stratisd_constants.py +++ b/tests/client-dbus/src/stratisd_client_dbus/_stratisd_constants.py @@ -15,6 +15,7 @@ Representing stratisd contants. """ +# isort: STDLIB from enum import IntEnum diff --git a/tests/client-dbus/tests/dbus/_misc.py b/tests/client-dbus/tests/dbus/_misc.py index 78b54dc250..7257e5fa5e 100644 --- a/tests/client-dbus/tests/dbus/_misc.py +++ b/tests/client-dbus/tests/dbus/_misc.py @@ -15,6 +15,7 @@ Miscellaneous methods to support testing. """ +# isort: STDLIB import os import random import string @@ -22,6 +23,7 @@ import time import unittest +# isort: THIRDPARTY import psutil _STRATISD = os.environ["STRATISD"] diff --git a/tests/client-dbus/tests/dbus/blockdev/test_fetch_properties.py b/tests/client-dbus/tests/dbus/blockdev/test_fetch_properties.py index 25a7c4f5f9..972dab1163 100644 --- a/tests/client-dbus/tests/dbus/blockdev/test_fetch_properties.py +++ b/tests/client-dbus/tests/dbus/blockdev/test_fetch_properties.py @@ -15,10 +15,8 @@ Test accessing properties of a blockdev using FetchProperties interface. """ -from stratisd_client_dbus import FetchProperties -from stratisd_client_dbus import Manager -from stratisd_client_dbus import get_object - +# isort: LOCAL +from stratisd_client_dbus import FetchProperties, Manager, get_object from stratisd_client_dbus._constants import TOP_OBJECT from .._misc import SimTestCase diff --git a/tests/client-dbus/tests/dbus/blockdev/test_properties.py b/tests/client-dbus/tests/dbus/blockdev/test_properties.py index 4242c93f70..6ec98d1854 100644 --- a/tests/client-dbus/tests/dbus/blockdev/test_properties.py +++ b/tests/client-dbus/tests/dbus/blockdev/test_properties.py @@ -15,16 +15,14 @@ Test accessing properties of a blockdev. """ +# isort: STDLIB from random import choice -from stratisd_client_dbus import Blockdev -from stratisd_client_dbus import Manager -from stratisd_client_dbus import get_object - +# isort: LOCAL +from stratisd_client_dbus import Blockdev, Manager, get_object from stratisd_client_dbus._constants import TOP_OBJECT -from .._misc import SimTestCase -from .._misc import device_name_list +from .._misc import SimTestCase, device_name_list _DEVICE_STRATEGY = device_name_list(1) diff --git a/tests/client-dbus/tests/dbus/filesystem/test_fetch_properties.py b/tests/client-dbus/tests/dbus/filesystem/test_fetch_properties.py index 8e66129a9f..966949abb8 100644 --- a/tests/client-dbus/tests/dbus/filesystem/test_fetch_properties.py +++ b/tests/client-dbus/tests/dbus/filesystem/test_fetch_properties.py @@ -15,15 +15,11 @@ Test accessing properties of a filesystem using FetchProperties interface. """ -from stratisd_client_dbus import FetchProperties -from stratisd_client_dbus import Manager -from stratisd_client_dbus import Pool -from stratisd_client_dbus import get_object - +# isort: LOCAL +from stratisd_client_dbus import FetchProperties, Manager, Pool, get_object from stratisd_client_dbus._constants import TOP_OBJECT -from .._misc import SimTestCase -from .._misc import device_name_list +from .._misc import SimTestCase, device_name_list _DEVICE_STRATEGY = device_name_list() diff --git a/tests/client-dbus/tests/dbus/filesystem/test_properties.py b/tests/client-dbus/tests/dbus/filesystem/test_properties.py index 9a7eb0d11d..36d7a584d2 100644 --- a/tests/client-dbus/tests/dbus/filesystem/test_properties.py +++ b/tests/client-dbus/tests/dbus/filesystem/test_properties.py @@ -15,15 +15,11 @@ Test accessing properties of a filesystem. """ -from stratisd_client_dbus import Filesystem -from stratisd_client_dbus import Manager -from stratisd_client_dbus import Pool -from stratisd_client_dbus import get_object - +# isort: LOCAL +from stratisd_client_dbus import Filesystem, Manager, Pool, get_object from stratisd_client_dbus._constants import TOP_OBJECT -from .._misc import SimTestCase -from .._misc import device_name_list +from .._misc import SimTestCase, device_name_list _DEVICE_STRATEGY = device_name_list() diff --git a/tests/client-dbus/tests/dbus/filesystem/test_rename.py b/tests/client-dbus/tests/dbus/filesystem/test_rename.py index deb3417dd3..fb1c8eb5c9 100644 --- a/tests/client-dbus/tests/dbus/filesystem/test_rename.py +++ b/tests/client-dbus/tests/dbus/filesystem/test_rename.py @@ -15,18 +15,19 @@ Test renaming a filesystem. """ -from stratisd_client_dbus import Filesystem -from stratisd_client_dbus import Manager -from stratisd_client_dbus import ObjectManager -from stratisd_client_dbus import Pool -from stratisd_client_dbus import StratisdErrors -from stratisd_client_dbus import filesystems -from stratisd_client_dbus import get_object - +# isort: LOCAL +from stratisd_client_dbus import ( + Filesystem, + Manager, + ObjectManager, + Pool, + StratisdErrors, + filesystems, + get_object, +) from stratisd_client_dbus._constants import TOP_OBJECT -from .._misc import SimTestCase -from .._misc import device_name_list +from .._misc import SimTestCase, device_name_list _DEVICE_STRATEGY = device_name_list() diff --git a/tests/client-dbus/tests/dbus/manager/test_create.py b/tests/client-dbus/tests/dbus/manager/test_create.py index 3d37ce1b1a..7dfbcd6d60 100644 --- a/tests/client-dbus/tests/dbus/manager/test_create.py +++ b/tests/client-dbus/tests/dbus/manager/test_create.py @@ -15,16 +15,17 @@ Test 'CreatePool'. """ -from stratisd_client_dbus import Manager -from stratisd_client_dbus import StratisdErrors -from stratisd_client_dbus import ObjectManager -from stratisd_client_dbus import get_object -from stratisd_client_dbus import pools - +# isort: LOCAL +from stratisd_client_dbus import ( + Manager, + ObjectManager, + StratisdErrors, + get_object, + pools, +) from stratisd_client_dbus._constants import TOP_OBJECT -from .._misc import SimTestCase -from .._misc import device_name_list +from .._misc import SimTestCase, device_name_list _DEVICE_STRATEGY = device_name_list() diff --git a/tests/client-dbus/tests/dbus/manager/test_destroy.py b/tests/client-dbus/tests/dbus/manager/test_destroy.py index d893a76f6a..712f030dab 100644 --- a/tests/client-dbus/tests/dbus/manager/test_destroy.py +++ b/tests/client-dbus/tests/dbus/manager/test_destroy.py @@ -15,19 +15,20 @@ Test DestroyPool. """ -from stratisd_client_dbus import Manager -from stratisd_client_dbus import ObjectManager -from stratisd_client_dbus import Pool -from stratisd_client_dbus import MOBlockDev -from stratisd_client_dbus import StratisdErrors -from stratisd_client_dbus import blockdevs -from stratisd_client_dbus import get_object -from stratisd_client_dbus import pools - +# isort: LOCAL +from stratisd_client_dbus import ( + Manager, + MOBlockDev, + ObjectManager, + Pool, + StratisdErrors, + blockdevs, + get_object, + pools, +) from stratisd_client_dbus._constants import TOP_OBJECT -from .._misc import SimTestCase -from .._misc import device_name_list +from .._misc import SimTestCase, device_name_list _DEVICE_STRATEGY = device_name_list() diff --git a/tests/client-dbus/tests/dbus/manager/test_object_path.py b/tests/client-dbus/tests/dbus/manager/test_object_path.py index 3171e7f9fa..8c0d5f8d59 100644 --- a/tests/client-dbus/tests/dbus/manager/test_object_path.py +++ b/tests/client-dbus/tests/dbus/manager/test_object_path.py @@ -14,13 +14,14 @@ """ Test object path methods. """ +# isort: THIRDPARTY import dbus +# isort: FIRSTPARTY from dbus_python_client_gen import DPClientInvocationError -from stratisd_client_dbus import Manager -from stratisd_client_dbus import ObjectManager -from stratisd_client_dbus import get_object +# isort: LOCAL +from stratisd_client_dbus import Manager, ObjectManager, get_object from .._misc import SimTestCase diff --git a/tests/client-dbus/tests/dbus/manager/test_stratis.py b/tests/client-dbus/tests/dbus/manager/test_stratis.py index 433fe1c5ea..e3c1129a59 100644 --- a/tests/client-dbus/tests/dbus/manager/test_stratis.py +++ b/tests/client-dbus/tests/dbus/manager/test_stratis.py @@ -15,12 +15,11 @@ Test 'stratisd'. """ +# isort: FIRSTPARTY from dbus_python_client_gen import DPClientInvalidArgError -from stratisd_client_dbus import Manager -from stratisd_client_dbus import ObjectManager -from stratisd_client_dbus import get_object - +# isort: LOCAL +from stratisd_client_dbus import Manager, ObjectManager, get_object from stratisd_client_dbus._constants import TOP_OBJECT from .._misc import SimTestCase diff --git a/tests/client-dbus/tests/dbus/pool/test_add_cache_devs.py b/tests/client-dbus/tests/dbus/pool/test_add_cache_devs.py index efb79a27a1..0f378d7e9a 100644 --- a/tests/client-dbus/tests/dbus/pool/test_add_cache_devs.py +++ b/tests/client-dbus/tests/dbus/pool/test_add_cache_devs.py @@ -15,18 +15,19 @@ Test adding blockdevs to a pool. """ -from stratisd_client_dbus import Manager -from stratisd_client_dbus import ObjectManager -from stratisd_client_dbus import Pool -from stratisd_client_dbus import StratisdErrors -from stratisd_client_dbus import blockdevs -from stratisd_client_dbus import get_object -from stratisd_client_dbus import pools - +# isort: LOCAL +from stratisd_client_dbus import ( + Manager, + ObjectManager, + Pool, + StratisdErrors, + blockdevs, + get_object, + pools, +) from stratisd_client_dbus._constants import TOP_OBJECT -from .._misc import SimTestCase -from .._misc import device_name_list +from .._misc import SimTestCase, device_name_list _DEVICE_STRATEGY = device_name_list(1) diff --git a/tests/client-dbus/tests/dbus/pool/test_add_data_devs.py b/tests/client-dbus/tests/dbus/pool/test_add_data_devs.py index 78e871cec2..a82165d6c7 100644 --- a/tests/client-dbus/tests/dbus/pool/test_add_data_devs.py +++ b/tests/client-dbus/tests/dbus/pool/test_add_data_devs.py @@ -15,18 +15,19 @@ Test adding blockdevs to a pool. """ -from stratisd_client_dbus import Manager -from stratisd_client_dbus import ObjectManager -from stratisd_client_dbus import Pool -from stratisd_client_dbus import StratisdErrors -from stratisd_client_dbus import blockdevs -from stratisd_client_dbus import get_object -from stratisd_client_dbus import pools - +# isort: LOCAL +from stratisd_client_dbus import ( + Manager, + ObjectManager, + Pool, + StratisdErrors, + blockdevs, + get_object, + pools, +) from stratisd_client_dbus._constants import TOP_OBJECT -from .._misc import SimTestCase -from .._misc import device_name_list +from .._misc import SimTestCase, device_name_list _DEVICE_STRATEGY = device_name_list(1) diff --git a/tests/client-dbus/tests/dbus/pool/test_create_filesystem.py b/tests/client-dbus/tests/dbus/pool/test_create_filesystem.py index e4e5314ba3..51c536bdfe 100644 --- a/tests/client-dbus/tests/dbus/pool/test_create_filesystem.py +++ b/tests/client-dbus/tests/dbus/pool/test_create_filesystem.py @@ -15,19 +15,21 @@ Test creating a filesystem in a pool. """ +# isort: STDLIB import unittest -from stratisd_client_dbus import Manager -from stratisd_client_dbus import ObjectManager -from stratisd_client_dbus import Pool -from stratisd_client_dbus import StratisdErrors -from stratisd_client_dbus import filesystems -from stratisd_client_dbus import get_object - +# isort: LOCAL +from stratisd_client_dbus import ( + Manager, + ObjectManager, + Pool, + StratisdErrors, + filesystems, + get_object, +) from stratisd_client_dbus._constants import TOP_OBJECT -from .._misc import SimTestCase -from .._misc import device_name_list +from .._misc import SimTestCase, device_name_list _DEVICE_STRATEGY = device_name_list() diff --git a/tests/client-dbus/tests/dbus/pool/test_create_snapshot.py b/tests/client-dbus/tests/dbus/pool/test_create_snapshot.py index d6c355a24b..55140b07b4 100644 --- a/tests/client-dbus/tests/dbus/pool/test_create_snapshot.py +++ b/tests/client-dbus/tests/dbus/pool/test_create_snapshot.py @@ -15,17 +15,18 @@ Test creating a snapshot """ -from stratisd_client_dbus import Manager -from stratisd_client_dbus import ObjectManager -from stratisd_client_dbus import Pool -from stratisd_client_dbus import StratisdErrors -from stratisd_client_dbus import filesystems -from stratisd_client_dbus import get_object - +# isort: LOCAL +from stratisd_client_dbus import ( + Manager, + ObjectManager, + Pool, + StratisdErrors, + filesystems, + get_object, +) from stratisd_client_dbus._constants import TOP_OBJECT -from .._misc import SimTestCase -from .._misc import device_name_list +from .._misc import SimTestCase, device_name_list _DEVICE_STRATEGY = device_name_list() diff --git a/tests/client-dbus/tests/dbus/pool/test_destroy_filesystem.py b/tests/client-dbus/tests/dbus/pool/test_destroy_filesystem.py index 309ece371a..d41b7949c2 100644 --- a/tests/client-dbus/tests/dbus/pool/test_destroy_filesystem.py +++ b/tests/client-dbus/tests/dbus/pool/test_destroy_filesystem.py @@ -15,17 +15,18 @@ Test destroying a filesystem in a pool. """ -from stratisd_client_dbus import Manager -from stratisd_client_dbus import ObjectManager -from stratisd_client_dbus import Pool -from stratisd_client_dbus import StratisdErrors -from stratisd_client_dbus import filesystems -from stratisd_client_dbus import get_object - +# isort: LOCAL +from stratisd_client_dbus import ( + Manager, + ObjectManager, + Pool, + StratisdErrors, + filesystems, + get_object, +) from stratisd_client_dbus._constants import TOP_OBJECT -from .._misc import SimTestCase -from .._misc import device_name_list +from .._misc import SimTestCase, device_name_list _DEVICE_STRATEGY = device_name_list() diff --git a/tests/client-dbus/tests/dbus/pool/test_fetch_properties.py b/tests/client-dbus/tests/dbus/pool/test_fetch_properties.py index 02a9e467d6..c6459854d9 100644 --- a/tests/client-dbus/tests/dbus/pool/test_fetch_properties.py +++ b/tests/client-dbus/tests/dbus/pool/test_fetch_properties.py @@ -15,14 +15,11 @@ Test accessing properties of a pool using FetchProperties interface. """ -from stratisd_client_dbus import FetchProperties -from stratisd_client_dbus import Manager -from stratisd_client_dbus import get_object - +# isort: LOCAL +from stratisd_client_dbus import FetchProperties, Manager, get_object from stratisd_client_dbus._constants import TOP_OBJECT -from .._misc import SimTestCase -from .._misc import device_name_list +from .._misc import SimTestCase, device_name_list _DEVICE_STRATEGY = device_name_list() diff --git a/tests/client-dbus/tests/dbus/pool/test_rename.py b/tests/client-dbus/tests/dbus/pool/test_rename.py index d0f00f51f7..3d7ea74a24 100644 --- a/tests/client-dbus/tests/dbus/pool/test_rename.py +++ b/tests/client-dbus/tests/dbus/pool/test_rename.py @@ -15,17 +15,18 @@ Test renaming a pool. """ -from stratisd_client_dbus import Manager -from stratisd_client_dbus import ObjectManager -from stratisd_client_dbus import Pool -from stratisd_client_dbus import StratisdErrors -from stratisd_client_dbus import get_object -from stratisd_client_dbus import pools - +# isort: LOCAL +from stratisd_client_dbus import ( + Manager, + ObjectManager, + Pool, + StratisdErrors, + get_object, + pools, +) from stratisd_client_dbus._constants import TOP_OBJECT -from .._misc import SimTestCase -from .._misc import device_name_list +from .._misc import SimTestCase, device_name_list _DEVICE_STRATEGY = device_name_list() diff --git a/tests/client-dbus/tests/misc/test_startup.py b/tests/client-dbus/tests/misc/test_startup.py index 3927292349..83d32b7d40 100644 --- a/tests/client-dbus/tests/misc/test_startup.py +++ b/tests/client-dbus/tests/misc/test_startup.py @@ -17,11 +17,11 @@ """ +# isort: STDLIB import os import subprocess import unittest - _STRATISD = os.environ["STRATISD"] diff --git a/tests/client-dbus/tests/udev/_dm.py b/tests/client-dbus/tests/udev/_dm.py index 42c36a1376..f99d83da77 100644 --- a/tests/client-dbus/tests/udev/_dm.py +++ b/tests/client-dbus/tests/udev/_dm.py @@ -15,6 +15,7 @@ Functionality pertaining to device mapper """ +# isort: STDLIB import os import subprocess diff --git a/tests/client-dbus/tests/udev/_loopback.py b/tests/client-dbus/tests/udev/_loopback.py index 2e680e8092..c12070d42e 100644 --- a/tests/client-dbus/tests/udev/_loopback.py +++ b/tests/client-dbus/tests/udev/_loopback.py @@ -15,6 +15,7 @@ Class to handle loop back devices. """ +# isort: STDLIB import os import subprocess import tempfile diff --git a/tests/client-dbus/tests/udev/_stratis_id.py b/tests/client-dbus/tests/udev/_stratis_id.py index c2a4bf5c75..077137a69a 100755 --- a/tests/client-dbus/tests/udev/_stratis_id.py +++ b/tests/client-dbus/tests/udev/_stratis_id.py @@ -16,9 +16,11 @@ """ Identify a device as a Stratis block device. """ +# isort: STDLIB import struct import sys from collections import namedtuple + from ._crc32 import crc BS = 512 diff --git a/tests/client-dbus/tests/udev/test_udev.py b/tests/client-dbus/tests/udev/test_udev.py index a0841394a7..c88351e661 100644 --- a/tests/client-dbus/tests/udev/test_udev.py +++ b/tests/client-dbus/tests/udev/test_udev.py @@ -15,26 +15,24 @@ Used to test behavior of the udev device discovery mechanism. """ -import unittest - -import subprocess -import time +# isort: STDLIB +import os import random import string -import os -import pyudev +import subprocess +import time +import unittest -from stratisd_client_dbus import Manager -from stratisd_client_dbus import ObjectManager -from stratisd_client_dbus import get_object -from stratisd_client_dbus import pools -from stratisd_client_dbus import Pool +# isort: THIRDPARTY +import pyudev +# isort: LOCAL +from stratisd_client_dbus import Manager, ObjectManager, Pool, get_object, pools from stratisd_client_dbus._constants import TOP_OBJECT +from ._dm import _get_stratis_devices, remove_stratis_setup from ._loopback import LoopBackDevices -from ._dm import remove_stratis_setup, _get_stratis_devices -from ._stratis_id import stratis_signature, dump_stratis_signature_area +from ._stratis_id import dump_stratis_signature_area, stratis_signature _STRATISD = os.environ["STRATISD"] From bf8b17834150662d5ffdc4725486f866ed8a7416 Mon Sep 17 00:00:00 2001 From: mulhern Date: Wed, 4 Dec 2019 13:21:15 -0500 Subject: [PATCH 030/109] Ensure that the devices found only correspond to the pools constructed During work on PR https://github.com/stratis-storage/stratisd/pull/1672 intermittent errors were experienced in test_setup. The reported error was: UUIDs of devices found () did not correspond with UUIDs specified in the metadata for this group of devices (223b6299d883449e95dff1d2ddaa05c7) I believe that the cause of this error was that find_all() would find the devices for both the pool made by the test _and_ for a pool corresponding to some devices left around from a previous test. Sometimes the devices corresponding to the test pool were placed first in the result of find_all() and hence were passed to the setup() function, and the test succeeded. Other times the device left over was found first and passed to the setup() function, and so the test failed. These changes to the test ensure that a problem like that would be detected immediately if it recurred. Signed-off-by: mulhern --- .../strat_engine/backstore/backstore.rs | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/engine/strat_engine/backstore/backstore.rs b/src/engine/strat_engine/backstore/backstore.rs index 37020fdf38..0ab2190570 100644 --- a/src/engine/strat_engine/backstore/backstore.rs +++ b/src/engine/strat_engine/backstore/backstore.rs @@ -585,6 +585,8 @@ impl Recordable for Backstore { #[cfg(test)] mod tests { + use std::collections::HashSet; + use uuid::Uuid; use devicemapper::{CacheDevStatus, DataBlocks, IEC}; @@ -818,9 +820,15 @@ mod tests { let backstore_save = backstore.record(); cmd::udev_settle().unwrap(); + let map = find_all().unwrap(); - let map = &map[&pool_uuid]; - let mut backstore = Backstore::setup(pool_uuid, &backstore_save, map, Utc::now()).unwrap(); + assert_eq!( + map.keys().collect::>(), + vec![pool_uuid].iter().collect::>() + ); + + let mut backstore = + Backstore::setup(pool_uuid, &backstore_save, &map[&pool_uuid], Utc::now()).unwrap(); invariant(&backstore); let backstore_save2 = backstore.record(); @@ -830,9 +838,15 @@ mod tests { backstore.teardown().unwrap(); cmd::udev_settle().unwrap(); + let map = find_all().unwrap(); - let map = &map[&pool_uuid]; - let mut backstore = Backstore::setup(pool_uuid, &backstore_save, map, Utc::now()).unwrap(); + assert_eq!( + map.keys().collect::>(), + vec![pool_uuid].iter().collect::>() + ); + + let mut backstore = + Backstore::setup(pool_uuid, &backstore_save, &map[&pool_uuid], Utc::now()).unwrap(); invariant(&backstore); let backstore_save2 = backstore.record(); From 31f954447110c1098a77c0ed015ab78bc272cf3a Mon Sep 17 00:00:00 2001 From: mulhern Date: Thu, 5 Dec 2019 08:55:12 -0500 Subject: [PATCH 031/109] Remove some unnecessary implicit-hasher clippy allows I think these were necessary once, because the methods were exposed in the public interface in order to enable testing using a different framework from the one that we are using now. Now that the methods are not exposed in the public interface, the lint doesn't apply, so we might as well get rid of the allows. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/setup.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/engine/strat_engine/backstore/setup.rs b/src/engine/strat_engine/backstore/setup.rs index bc814161dc..52fb105e60 100644 --- a/src/engine/strat_engine/backstore/setup.rs +++ b/src/engine/strat_engine/backstore/setup.rs @@ -125,7 +125,6 @@ pub fn find_all() -> StratisResult>> /// Get the most recent metadata from a set of Devices for a given pool UUID. /// Returns None if no metadata found for this pool. -#[allow(clippy::implicit_hasher)] pub fn get_metadata( pool_uuid: PoolUuid, devnodes: &HashMap, @@ -195,7 +194,6 @@ pub fn get_metadata( /// are the devs that support the cache tier. /// Precondition: Every device in devnodes has already been determined to /// belong to the pool with the specified pool uuid. -#[allow(clippy::implicit_hasher)] pub fn get_blockdevs( pool_uuid: PoolUuid, backstore_save: &BackstoreSave, From 35d01950800cbc8bf9487bfd99c9652060a98ea2 Mon Sep 17 00:00:00 2001 From: mulhern Date: Thu, 5 Dec 2019 08:59:56 -0500 Subject: [PATCH 032/109] Remove a collapsible-if clippy allow clippy does not seem to be enforcing its opinions as strongly as formerly. Signed-off-by: mulhern --- src/engine/strat_engine/thinpool/thinpool.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/engine/strat_engine/thinpool/thinpool.rs b/src/engine/strat_engine/thinpool/thinpool.rs index 357f946f7d..19b5540066 100644 --- a/src/engine/strat_engine/thinpool/thinpool.rs +++ b/src/engine/strat_engine/thinpool/thinpool.rs @@ -1161,7 +1161,6 @@ fn setup_metadev( meta_segments: Vec<(Sectors, Sectors)>, spare_segments: Vec<(Sectors, Sectors)>, ) -> StratisResult<(LinearDev, Vec<(Sectors, Sectors)>, Vec<(Sectors, Sectors)>)> { - #![allow(clippy::collapsible_if)] let (dm_name, dm_uuid) = format_flex_ids(pool_uuid, FlexRole::ThinMeta); let mut meta_dev = LinearDev::setup( get_dm(), From 196ef25ab388b732b1308d0c6eb2f77cf10bdbd9 Mon Sep 17 00:00:00 2001 From: mulhern Date: Thu, 5 Dec 2019 09:02:56 -0500 Subject: [PATCH 033/109] Remove a borrowed-box clippy allow clippy understands now why the box needs to be borrowed. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/shared.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/engine/strat_engine/backstore/shared.rs b/src/engine/strat_engine/backstore/shared.rs index 3f8f11b1aa..605b4c91e4 100644 --- a/src/engine/strat_engine/backstore/shared.rs +++ b/src/engine/strat_engine/backstore/shared.rs @@ -27,7 +27,6 @@ use crate::{ // the value from a function, and there is no other way for the function to // be returned from a closure. // In future, it may be possible to address this better with FnBox. -#[allow(clippy::borrowed_box)] pub fn metadata_to_segment( uuid_to_devno: &HashMap, base_dev_save: &BaseDevSave, From 682e0cba815e54352214beea8e6bf2c0f9f91d8c Mon Sep 17 00:00:00 2001 From: mulhern Date: Thu, 5 Dec 2019 11:00:00 -0500 Subject: [PATCH 034/109] Require black version 19.3b0 Signed-off-by: mulhern --- tests/client-dbus/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/client-dbus/requirements.txt b/tests/client-dbus/requirements.txt index f7fe3acde7..97d96ded43 100644 --- a/tests/client-dbus/requirements.txt +++ b/tests/client-dbus/requirements.txt @@ -1,3 +1,3 @@ tox==3.2.1 -black==18.9b0 +black==19.3b0 isort==4.3.4 From bf7a323466e99cb981cb1c0f7c5e3271e252f9c9 Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 3 Dec 2019 11:38:58 -0500 Subject: [PATCH 035/109] Make path_to_mount_filesystem a method of the Filesystem trait This provides better encapsulation and allows moving the devlinks module down into the strat_engine module. Expect something more general in the test than absolute equality between path names. Signed-off-by: mulhern --- src/dbus_api/filesystem.rs | 14 ++++++-------- src/engine/engine.rs | 3 +++ src/engine/sim_engine/filesystem.rs | 6 +++++- src/engine/strat_engine/thinpool/filesystem.rs | 5 +++++ .../tests/dbus/filesystem/test_properties.py | 6 ++++-- 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/dbus_api/filesystem.rs b/src/dbus_api/filesystem.rs index 0e9500393f..048edfa4ae 100644 --- a/src/dbus_api/filesystem.rs +++ b/src/dbus_api/filesystem.rs @@ -25,9 +25,7 @@ use crate::{ msg_code_ok, msg_string_ok, result_to_tuple, }, }, - engine::{ - filesystem_mount_path, Filesystem, FilesystemUuid, MaybeDbusPath, Name, RenameAction, - }, + engine::{Filesystem, FilesystemUuid, MaybeDbusPath, Name, RenameAction}, }; fn get_properties_shared( @@ -284,11 +282,11 @@ fn get_filesystem_devnode( i: &mut IterAppend, p: &PropInfo, TData>, ) -> Result<(), MethodErr> { - get_filesystem_property(i, p, |(pool_name, fs_name, _)| { - Ok(format!( - "{}", - filesystem_mount_path(pool_name, fs_name).display() - )) + get_filesystem_property(i, p, |(pool_name, fs_name, fs)| { + Ok(fs + .path_to_mount_filesystem(&pool_name, &fs_name) + .display() + .to_string()) }) } diff --git a/src/engine/engine.rs b/src/engine/engine.rs index 2adc04616c..e6ef6a9a56 100644 --- a/src/engine/engine.rs +++ b/src/engine/engine.rs @@ -30,6 +30,9 @@ pub trait Filesystem: Debug { /// When the filesystem was created. fn created(&self) -> DateTime; + /// path to mount the filesystem by + fn path_to_mount_filesystem(&self, pool_name: &str, fs_name: &str) -> PathBuf; + /// The amount of data stored on the filesystem, including overhead. fn used(&self) -> StratisResult; diff --git a/src/engine/sim_engine/filesystem.rs b/src/engine/sim_engine/filesystem.rs index 10c08df8d9..bb1a3218dc 100644 --- a/src/engine/sim_engine/filesystem.rs +++ b/src/engine/sim_engine/filesystem.rs @@ -11,7 +11,7 @@ use std::path::PathBuf; use devicemapper::Bytes; use crate::{ - engine::{types::MaybeDbusPath, Filesystem}, + engine::{devlinks, types::MaybeDbusPath, Filesystem}, stratis::StratisResult, }; @@ -43,6 +43,10 @@ impl Filesystem for SimFilesystem { self.created } + fn path_to_mount_filesystem(&self, pool_name: &str, fs_name: &str) -> PathBuf { + devlinks::filesystem_mount_path(pool_name, fs_name) + } + fn used(&self) -> StratisResult { Ok(Bytes(12_345_678)) } diff --git a/src/engine/strat_engine/thinpool/filesystem.rs b/src/engine/strat_engine/thinpool/filesystem.rs index ea4e817e5d..53a5538df7 100644 --- a/src/engine/strat_engine/thinpool/filesystem.rs +++ b/src/engine/strat_engine/thinpool/filesystem.rs @@ -26,6 +26,7 @@ use tempfile; use crate::{ engine::{ + devlinks, engine::Filesystem, strat_engine::{ cmd::{create_fs, set_uuid, udev_settle, xfs_growfs}, @@ -314,6 +315,10 @@ impl Filesystem for StratFilesystem { self.created } + fn path_to_mount_filesystem(&self, pool_name: &str, fs_name: &str) -> PathBuf { + devlinks::filesystem_mount_path(pool_name, fs_name) + } + fn used(&self) -> StratisResult { match self.thin_dev.status(get_dm())? { ThinStatus::Working(wk_status) => Ok(wk_status.nr_mapped_sectors.bytes()), diff --git a/tests/client-dbus/tests/dbus/filesystem/test_properties.py b/tests/client-dbus/tests/dbus/filesystem/test_properties.py index 36d7a584d2..aec14f0759 100644 --- a/tests/client-dbus/tests/dbus/filesystem/test_properties.py +++ b/tests/client-dbus/tests/dbus/filesystem/test_properties.py @@ -15,6 +15,9 @@ Test accessing properties of a filesystem. """ +# isort: STDLIB +from os.path import isabs + # isort: LOCAL from stratisd_client_dbus import Filesystem, Manager, Pool, get_object from stratisd_client_dbus._constants import TOP_OBJECT @@ -75,5 +78,4 @@ def testProps(self): self.assertEqual(len(created), 20) devnode = Filesystem.Properties.Devnode.Get(filesystem) - - self.assertEqual(devnode, "/stratis/deadpool/fs") + self.assertTrue(isabs(devnode)) From 759b1697d5a8cd16ba3e41c028629b5b55d46e0b Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 3 Dec 2019 11:22:23 -0500 Subject: [PATCH 036/109] Move devlinks module down into the strat_engine module For better encapsulation. Signed-off-by: mulhern --- src/engine/mod.rs | 2 -- src/engine/sim_engine/filesystem.rs | 4 ++-- src/engine/{ => strat_engine}/devlinks.rs | 0 src/engine/strat_engine/engine.rs | 2 +- src/engine/strat_engine/mod.rs | 1 + src/engine/strat_engine/pool.rs | 3 +-- src/engine/strat_engine/thinpool/filesystem.rs | 2 +- src/engine/strat_engine/thinpool/thinpool.rs | 2 +- 8 files changed, 7 insertions(+), 9 deletions(-) rename src/engine/{ => strat_engine}/devlinks.rs (100%) diff --git a/src/engine/mod.rs b/src/engine/mod.rs index 8736d3fbd8..dedd41ad69 100644 --- a/src/engine/mod.rs +++ b/src/engine/mod.rs @@ -3,7 +3,6 @@ // file, You can obtain one at http://mozilla.org/MPL/2.0/. pub use self::{ - devlinks::filesystem_mount_path, engine::{BlockDev, Engine, Filesystem, Pool}, event::{get_engine_listener_list_mut, EngineEvent, EngineListener}, sim_engine::SimEngine, @@ -18,7 +17,6 @@ pub use self::{ #[macro_use] mod macros; -mod devlinks; #[allow(clippy::module_inception)] mod engine; mod event; diff --git a/src/engine/sim_engine/filesystem.rs b/src/engine/sim_engine/filesystem.rs index bb1a3218dc..519d4f1ae6 100644 --- a/src/engine/sim_engine/filesystem.rs +++ b/src/engine/sim_engine/filesystem.rs @@ -11,7 +11,7 @@ use std::path::PathBuf; use devicemapper::Bytes; use crate::{ - engine::{devlinks, types::MaybeDbusPath, Filesystem}, + engine::{types::MaybeDbusPath, Filesystem}, stratis::StratisResult, }; @@ -44,7 +44,7 @@ impl Filesystem for SimFilesystem { } fn path_to_mount_filesystem(&self, pool_name: &str, fs_name: &str) -> PathBuf { - devlinks::filesystem_mount_path(pool_name, fs_name) + vec!["/somepath", pool_name, fs_name].iter().collect() } fn used(&self) -> StratisResult { diff --git a/src/engine/devlinks.rs b/src/engine/strat_engine/devlinks.rs similarity index 100% rename from src/engine/devlinks.rs rename to src/engine/strat_engine/devlinks.rs diff --git a/src/engine/strat_engine/engine.rs b/src/engine/strat_engine/engine.rs index f90023ffa1..33e0a538bd 100644 --- a/src/engine/strat_engine/engine.rs +++ b/src/engine/strat_engine/engine.rs @@ -15,13 +15,13 @@ use crate::engine::strat_engine::cleanup::teardown_pools; use crate::{ engine::{ - devlinks, engine::Eventable, event::get_engine_listener_list, shared::create_pool_idempotent_or_err, strat_engine::{ backstore::{find_all, get_metadata, DevOwnership}, cmd::verify_binaries, + devlinks, dm::{get_dm, get_dm_init}, names::validate_name, pool::{check_metadata, StratPool}, diff --git a/src/engine/strat_engine/mod.rs b/src/engine/strat_engine/mod.rs index 7606fcb21b..91dce4fa54 100644 --- a/src/engine/strat_engine/mod.rs +++ b/src/engine/strat_engine/mod.rs @@ -7,6 +7,7 @@ mod backstore; mod cleanup; mod cmd; mod device; +mod devlinks; mod dm; mod engine; mod names; diff --git a/src/engine/strat_engine/pool.rs b/src/engine/strat_engine/pool.rs index b0396fcd1a..eace09b45b 100644 --- a/src/engine/strat_engine/pool.rs +++ b/src/engine/strat_engine/pool.rs @@ -490,10 +490,9 @@ mod tests { use devicemapper::{Bytes, IEC, SECTOR_SIZE}; use crate::engine::{ - devlinks, strat_engine::{ backstore::{find_all, get_metadata}, - cmd, + cmd, devlinks, tests::{loopbacked, real}, }, types::{EngineAction, PoolExtendState, PoolState, Redundancy}, diff --git a/src/engine/strat_engine/thinpool/filesystem.rs b/src/engine/strat_engine/thinpool/filesystem.rs index 53a5538df7..95d2e55c18 100644 --- a/src/engine/strat_engine/thinpool/filesystem.rs +++ b/src/engine/strat_engine/thinpool/filesystem.rs @@ -26,10 +26,10 @@ use tempfile; use crate::{ engine::{ - devlinks, engine::Filesystem, strat_engine::{ cmd::{create_fs, set_uuid, udev_settle, xfs_growfs}, + devlinks, dm::get_dm, names::{format_thin_ids, ThinRole}, serde_structs::FilesystemSave, diff --git a/src/engine/strat_engine/thinpool/thinpool.rs b/src/engine/strat_engine/thinpool/thinpool.rs index 19b5540066..05dafe8539 100644 --- a/src/engine/strat_engine/thinpool/thinpool.rs +++ b/src/engine/strat_engine/thinpool/thinpool.rs @@ -21,13 +21,13 @@ use devicemapper::{ use crate::{ engine::{ - devlinks, engine::Filesystem, event::{get_engine_listener_list, EngineEvent}, strat_engine::{ backstore::Backstore, cmd::{thin_check, thin_repair, udev_settle}, device::wipe_sectors, + devlinks, dm::get_dm, names::{ format_flex_ids, format_thin_ids, format_thinpool_ids, FlexRole, ThinPoolRole, From 8e00f8535925eedbc28be5f3ba546c91863d7fa0 Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Fri, 6 Dec 2019 09:54:33 -0500 Subject: [PATCH 037/109] Smarter error handling for thinpool suspend/resume --- src/engine/strat_engine/pool.rs | 14 ++++++++++++-- src/engine/strat_engine/thinpool/thinpool.rs | 15 +++++++++++++-- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/engine/strat_engine/pool.rs b/src/engine/strat_engine/pool.rs index eace09b45b..c29910ce3e 100644 --- a/src/engine/strat_engine/pool.rs +++ b/src/engine/strat_engine/pool.rs @@ -315,9 +315,19 @@ impl Pool for StratPool { // If adding cache devices, must suspend the pool, since the cache // must be augmeneted with the new devices. self.thin_pool.suspend()?; - let bdev_info = self.backstore.add_cachedevs(pool_uuid, paths)?; - self.thin_pool.set_device(self.backstore.device().expect("Since thin pool exists, space must have been allocated from the backstore, so backstore must have a cap device"))?; + let bdev_info_res = self + .backstore + .add_cachedevs(pool_uuid, paths) + .and_then(|bdi| { + self.thin_pool + .set_device(self.backstore.device().expect( + "Since thin pool exists, space must have been allocated \ + from the backstore, so backstore must have a cap device", + )) + .and(Ok(bdi)) + }); self.thin_pool.resume()?; + let bdev_info = bdev_info_res?; Ok(SetCreateAction::new(bdev_info)) } else { // If just adding data devices, no need to suspend the pool. diff --git a/src/engine/strat_engine/thinpool/thinpool.rs b/src/engine/strat_engine/thinpool/thinpool.rs index 05dafe8539..0a91a1a990 100644 --- a/src/engine/strat_engine/thinpool/thinpool.rs +++ b/src/engine/strat_engine/thinpool/thinpool.rs @@ -1039,8 +1039,19 @@ impl ThinPool { pub fn suspend(&mut self) -> StratisResult<()> { // thindevs automatically suspended when thinpool is suspended self.thin_pool.suspend(get_dm(), true)?; - self.mdv.suspend()?; - Ok(()) + // If MDV suspend fails, resume the thin pool and return the error + if let Err(err) = self.mdv.suspend() { + self.thin_pool.resume(get_dm()).map_err(|e| { + StratisError::Error(format!( + "Suspending the MDV failed: {}. MDV suspend clean up action \ + of resuming the thin pool also failed: {}.", + err, e + )) + })?; + Err(err) + } else { + Ok(()) + } } /// Resume the thinpool From f027b824a352a0dcf3c9d82b497bb9ccc35e115a Mon Sep 17 00:00:00 2001 From: mulhern Date: Fri, 6 Dec 2019 10:07:20 -0500 Subject: [PATCH 038/109] Consolidate find_all and get_stratis_block_devices The advantages of this are: * Devices are only iterated through once, rather than being collected and re-iterated through. Note that this slightly changes semantics. If there were some block devices identified as Stratis devices by the udev enumerator, but these devices did not have stratis metadata or devnodes, the fallback path will be pursued. Previously, it would have been skipped. The justification for this change is that that would be a very wierd and unusual situation, so it doesn't hurt to follow the fallback path in that case. * The fallback path is easier to understand. * If the fallback path is followed, only one attempt is made to read the Stratis header from each block device found, which is better than two attempts for some. * There are now some useful warning messages and these messages are appropriate for the context in which they are emitted. * Gets rid of stratis_enumerator method; it made the code a bit busier without any real advantage * Logs at the warn level instead of just returning with an error from the find_all method if there is an error reading the Stratis header on the standard path. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/setup.rs | 129 +++++++++++++-------- src/engine/strat_engine/udev.rs | 9 -- 2 files changed, 80 insertions(+), 58 deletions(-) diff --git a/src/engine/strat_engine/backstore/setup.rs b/src/engine/strat_engine/backstore/setup.rs index 52fb105e60..20491ce059 100644 --- a/src/engine/strat_engine/backstore/setup.rs +++ b/src/engine/strat_engine/backstore/setup.rs @@ -22,42 +22,71 @@ use crate::{ strat_engine::{ backstore::{ blockdev::StratBlockDev, - device::DevOwnership, metadata::{device_identifiers, BDA}, }, device::blkdev_size, serde_structs::{BackstoreSave, BaseBlockDevSave, PoolSave}, - udev::{block_enumerator, decide_ownership, is_multipath_member, stratis_enumerator}, + udev::{block_enumerator, decide_ownership, is_multipath_member, UdevOwnership}, }, types::{BlockDevTier, DevUuid, PoolUuid}, }, stratis::{ErrorEnum, StratisError, StratisResult}, }; -/// Retrieve all Stratis block devices that should be made use of by the -/// Stratis engine. This excludes Stratis block devices that are multipath -/// members. +/// Retrieve all block devices that should be made use of by the +/// Stratis engine. This excludes Stratis block devices that appear to be +/// multipath members. /// /// Includes a fallback path, which is used if no Stratis block devices are /// found using the obvious udev property- and enumerator-based approach. /// This fallback path is more expensive, because it must search all block -/// devices via udev, and may also attempt to directly read Stratis -/// metadata from the device. -fn get_stratis_block_devices() -> StratisResult> { +/// devices via udev rather than just all Stratis block devices. +/// +/// Returns a map of pool uuids to a map of devices to devnodes for each pool. +pub fn find_all() -> StratisResult>> { info!("Beginning initial search for Stratis block devices"); - let devices = { - let context = libudev::Context::new()?; - let mut enumerator = stratis_enumerator(&context)?; + let pool_map = { + let mut pool_map = HashMap::new(); - enumerator + let context = libudev::Context::new()?; + let mut enumerator = block_enumerator(&context)?; + enumerator.match_property("ID_FS_TYPE", "stratis")?; + for devnode in enumerator .scan_devices()? .filter(|dev| dev.is_initialized()) .filter(|dev| !is_multipath_member(dev).unwrap_or(true)) - .filter_map(|i| i.devnode().map(|d| d.into())) - .collect::>() + .filter_map(|i| i.devnode().map(|d| d.to_path_buf())) + { + if let Some(devno) = devnode_to_devno(&devnode)? { + if let Some((pool_uuid, _)) = match device_identifiers( + &mut OpenOptions::new().read(true).open(&devnode)?, + ) { + Ok(ids) => ids, + Err(err) => { + warn!("udev identified device {} as a Stratis block device, but there was an error when reading the Stratis header: {}", + devnode.display(), + err); + None + } + } { + pool_map + .entry(pool_uuid) + .or_insert_with(HashMap::new) + .insert(Device::from(devno), devnode); + } else { + warn!("udev identified device {} as a Stratis block device but there appeared to be no Stratis metadata on the device", + devnode.display()) + } + } else { + warn!("udev identified device {} as a Stratis block device but its device number could not be found", + devnode.display()) + } + } + + pool_map }; - if devices.is_empty() { + if pool_map.is_empty() { // No Stratis devices have been found, possible reasons are: // 1. There are none // 2. There are some but libblkid version is less than 2.32, so @@ -67,60 +96,62 @@ fn get_stratis_block_devices() -> StratisResult> { // // Try again to find Stratis block devices, but this time enumerate // all block devices, not just all the ones that can be identified - // as Stratis blockdevs by udev, and then scrutinize each one - // using various methods. + // as Stratis blockdevs by udev, and return only those that appear + // unclaimed or appear to be claimed by Stratis (and not + // multipath members). info!("Could not identify any Stratis devices by a udev search for devices with ID_FS_TYPE=\"stratis\"; using fallback search mechanism"); + let mut pool_map = HashMap::new(); + let context = libudev::Context::new()?; let mut enumerator = block_enumerator(&context)?; - Ok(enumerator + for devnode in enumerator .scan_devices()? .filter(|dev| dev.is_initialized()) .filter_map(|dev| { dev.devnode().and_then(|devnode| { decide_ownership(&dev) - .and_then(|decision| DevOwnership::from_udev_ownership(&decision, devnode)) .ok() - .and_then(|ownership| ownership.stratis_identifiers()) - .map(|_| devnode.into()) + .and_then(|decision| match decision { + UdevOwnership::Stratis | UdevOwnership::Unowned => { + Some(devnode.to_path_buf()) + } + _ => None, + }) }) }) - .collect()) - } else { - Ok(devices) - } -} - -/// Find all Stratis devices. -/// -/// Returns a map of pool uuids to a map of devices to devnodes for each pool. -pub fn find_all() -> StratisResult>> { - let mut pool_map = HashMap::new(); - - // FIXME: If get_stratis_block_devices() has to go to fallback mechanism, - // it has already read the device identifiers for every device it returns. - // It seems a waste to just read them all again in that case. Likely - // get_stratis_block_devices() and this method should be consolidated in - // some creative way, or get_stratis_block_devices() should be required - // to return the device identifiers from the device as well as the devnode, - // regardless of which mechanism was used. - for devnode in get_stratis_block_devices()? { - match devnode_to_devno(&devnode)? { - None => continue, - Some(devno) => { - if let Some((pool_uuid, _)) = - device_identifiers(&mut OpenOptions::new().read(true).open(&devnode)?)? - { + { + if let Some(devno) = devnode_to_devno(&devnode)? { + if let Some((pool_uuid, _)) = match device_identifiers( + &mut OpenOptions::new().read(true).open(&devnode)?, + ) { + Ok(ids) => ids, + // FIXME: Refine error return in StaticHeader::setup(), + // so it can be used to distinguish between signficant + // and insignficant errors and then use that ability to + // distinguish here between different levels of severity. + Err(err) => { + debug!("Encountered an error while trying to get Stratis device identifiers from block device {}: {}", + devnode.display(), + err); + None + } + } { pool_map .entry(pool_uuid) .or_insert_with(HashMap::new) - .insert(Device::from(devno), devnode); + .insert(Device::from(devno), devnode.to_path_buf()); } + } else { + warn!("udev identified device {} as a block device but its device number could not be found", + devnode.display()) } } + Ok(pool_map) + } else { + Ok(pool_map) } - Ok(pool_map) } /// Get the most recent metadata from a set of Devices for a given pool UUID. diff --git a/src/engine/strat_engine/udev.rs b/src/engine/strat_engine/udev.rs index f19427784c..123804eaff 100644 --- a/src/engine/strat_engine/udev.rs +++ b/src/engine/strat_engine/udev.rs @@ -17,15 +17,6 @@ pub fn block_enumerator(context: &libudev::Context) -> libudev::Result libudev::Result { - let mut enumerator = libudev::Enumerator::new(context)?; - enumerator.match_subsystem("block")?; - enumerator.match_property("ID_FS_TYPE", "stratis")?; - Ok(enumerator) -} - /// Get a udev property with the given name for the given device. /// Returns None if no udev property found for the given property name. /// Returns an error if the value of the property can not be converted to From aa6174d51bf931885c523562e905266e554da6e4 Mon Sep 17 00:00:00 2001 From: Bryan Gurney Date: Mon, 9 Dec 2019 17:14:35 -0500 Subject: [PATCH 039/109] Refine description of stratisd.service in systemd config Signed-off-by: Bryan Gurney --- stratisd.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stratisd.service b/stratisd.service index 4219e98f9d..8653f9e999 100644 --- a/stratisd.service +++ b/stratisd.service @@ -1,5 +1,5 @@ [Unit] -Description=A daemon that manages a pool of block devices to create flexible file systems +Description=Stratis daemon Documentation=man:stratisd(8) Before=local-fs-pre.target DefaultDependencies=no From 665d3a4c12d3c623fa85148ebb4e673c4a870ab4 Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 10 Dec 2019 10:30:40 -0500 Subject: [PATCH 040/109] Add somewhat ad-hoc method to discover udev-based ownership The benefit to this method is the very clear errors returned, which are determined by the context in which this method is expected to be called. Signed-off-by: mulhern --- .../strat_engine/backstore/blockdevmgr.rs | 18 ++++----- src/engine/strat_engine/engine.rs | 19 ++-------- src/engine/strat_engine/udev.rs | 38 +++++++++++++++++++ 3 files changed, 51 insertions(+), 24 deletions(-) diff --git a/src/engine/strat_engine/backstore/blockdevmgr.rs b/src/engine/strat_engine/backstore/blockdevmgr.rs index f5dc6a4cd9..dae77b944a 100644 --- a/src/engine/strat_engine/backstore/blockdevmgr.rs +++ b/src/engine/strat_engine/backstore/blockdevmgr.rs @@ -547,7 +547,7 @@ mod tests { backstore::{find_all, get_metadata}, cmd, tests::{loopbacked, real}, - udev::{block_device_apply, decide_ownership}, + udev::block_device_ownership, }; use crate::engine::strat_engine::backstore::metadata::device_identifiers; @@ -612,19 +612,19 @@ mod tests { for (i, path) in paths.iter().enumerate() { if i == index { assert_matches!( - block_device_apply(path, |d| decide_ownership(d) - .and_then(|decision| DevOwnership::from_udev_ownership(&decision, path))) - .unwrap() - .unwrap() + DevOwnership::from_udev_ownership( + &block_device_ownership(path).unwrap().unwrap().unwrap(), + path + ) .unwrap(), DevOwnership::Theirs(_) ); } else { assert_matches!( - block_device_apply(path, |d| decide_ownership(d) - .and_then(|decision| DevOwnership::from_udev_ownership(&decision, path))) - .unwrap() - .unwrap() + DevOwnership::from_udev_ownership( + &block_device_ownership(path).unwrap().unwrap().unwrap(), + path + ) .unwrap(), DevOwnership::Unowned ); diff --git a/src/engine/strat_engine/engine.rs b/src/engine/strat_engine/engine.rs index 33e0a538bd..dfb3bfcb4c 100644 --- a/src/engine/strat_engine/engine.rs +++ b/src/engine/strat_engine/engine.rs @@ -25,7 +25,7 @@ use crate::{ dm::{get_dm, get_dm_init}, names::validate_name, pool::{check_metadata, StratPool}, - udev::{block_device_apply, decide_ownership}, + udev::block_device_ownership, }, structures::Table, types::{CreateAction, DeleteAction, RenameAction}, @@ -207,20 +207,9 @@ impl Engine for StratEngine { device: Device, dev_node: PathBuf, ) -> StratisResult> { - let stratis_identifiers = if let Some(ownership) = block_device_apply(&dev_node, |d| { - decide_ownership(d) - .and_then(|decision| DevOwnership::from_udev_ownership(&decision, &dev_node)) - })? { - ownership?.stratis_identifiers() - } else { - return Err(StratisError::Engine( - ErrorEnum::NotFound, - format!( - "Could not determine ownership of block device {} because it could not be found in the udev database", - dev_node.display() - ), - )); - }; + let stratis_identifiers = + DevOwnership::from_udev_ownership(&block_device_ownership(&dev_node)???, &dev_node)? + .stratis_identifiers(); let pool_uuid = if let Some((pool_uuid, device_uuid)) = stratis_identifiers { if self.pools.contains_uuid(pool_uuid) { // We can get udev events for devices that are already in the pool. Lets check diff --git a/src/engine/strat_engine/udev.rs b/src/engine/strat_engine/udev.rs index 123804eaff..fcfb899d8c 100644 --- a/src/engine/strat_engine/udev.rs +++ b/src/engine/strat_engine/udev.rs @@ -109,6 +109,44 @@ pub fn decide_ownership(device: &libudev::Device) -> StratisResult StratisResult>> { + block_device_apply(devnode, |d| decide_ownership(d)) + .map_err(|err| { + StratisError::Error(format!( + "Could not determine ownership of block device {} because of a udev failure: {}", + devnode.display(), + err + )) + }) + .map(|option_ownership| { + option_ownership.ok_or_else(|| { + StratisError::Error(format!( + "Could not determine ownership of block device {} because it could not be found in the udev database", + devnode.display(), + )) + }) + .map(|error_ownership| { + error_ownership.map_err(|err| { + StratisError::Error(format!( + "Could not determine ownership of block device {} because of an error processing udev information: {}", + devnode.display(), + err + )) + }) + }) + }) +} + /// Locate a udev block device with the specified devnode and apply a function /// to that device, returning the result. /// This approach is necessitated by the libudev lifetimes, which do not allow From c9a575cf50e2a2ad44ea39234b64f4af66655a33 Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 10 Dec 2019 11:10:55 -0500 Subject: [PATCH 041/109] Filter based on udev ownership before getting the devnode in fallback This makes the similarities between the regular path and the fallback path in find_all() much clearer. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/setup.rs | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/engine/strat_engine/backstore/setup.rs b/src/engine/strat_engine/backstore/setup.rs index 20491ce059..60745ad07f 100644 --- a/src/engine/strat_engine/backstore/setup.rs +++ b/src/engine/strat_engine/backstore/setup.rs @@ -109,18 +109,15 @@ pub fn find_all() -> StratisResult>> for devnode in enumerator .scan_devices()? .filter(|dev| dev.is_initialized()) - .filter_map(|dev| { - dev.devnode().and_then(|devnode| { - decide_ownership(&dev) - .ok() - .and_then(|decision| match decision { - UdevOwnership::Stratis | UdevOwnership::Unowned => { - Some(devnode.to_path_buf()) - } - _ => None, - }) - }) + .filter(|dev| { + decide_ownership(dev) + .map(|decision| match decision { + UdevOwnership::Stratis | UdevOwnership::Unowned => true, + _ => false, + }) + .unwrap_or(false) }) + .filter_map(|i| i.devnode().map(|d| d.to_path_buf())) { if let Some(devno) = devnode_to_devno(&devnode)? { if let Some((pool_uuid, _)) = match device_identifiers( From f1d28011003a9c11244a0c3f3e473546c85b5f76 Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 10 Dec 2019 11:24:07 -0500 Subject: [PATCH 042/109] Log error when there is a failure to find udev information in find_all This brings this method better into line with other methods. Note that because find_all() is just trying to grab as many devices as it can find and be sort of certain of it ignores problematic devices in situations where block_device_ownership returns an error. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/setup.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/engine/strat_engine/backstore/setup.rs b/src/engine/strat_engine/backstore/setup.rs index 60745ad07f..97aa0602de 100644 --- a/src/engine/strat_engine/backstore/setup.rs +++ b/src/engine/strat_engine/backstore/setup.rs @@ -54,7 +54,12 @@ pub fn find_all() -> StratisResult>> for devnode in enumerator .scan_devices()? .filter(|dev| dev.is_initialized()) - .filter(|dev| !is_multipath_member(dev).unwrap_or(true)) + .filter(|dev| !is_multipath_member(dev) + .map_err(|err| { + warn!("Could not certainly determine whether a device was a multipath member because of an error processing udev information, discarded the device from the set of devices to process, for safety: {}", + err); + }) + .unwrap_or(true)) .filter_map(|i| i.devnode().map(|d| d.to_path_buf())) { if let Some(devno) = devnode_to_devno(&devnode)? { @@ -111,6 +116,10 @@ pub fn find_all() -> StratisResult>> .filter(|dev| dev.is_initialized()) .filter(|dev| { decide_ownership(dev) + .map_err(|err| { + warn!("Could not determine ownership of a udev block device because of an error processing udev information, discarded the device from the set of devices to process, for safety: {}", + err); + }) .map(|decision| match decision { UdevOwnership::Stratis | UdevOwnership::Unowned => true, _ => false, From 73f2a602d526464002f6573e84128c7948f5e30b Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Thu, 5 Dec 2019 11:46:04 -0500 Subject: [PATCH 043/109] Move idempotent return type to pool level Everywhere else in the code, we return idempotent data types at the layer right below the D-Bus layer (such as Pool, Engine, etc.). This moves the idempotent data type for renaming actions up one level from the backstore to make it consistent with all of the other return types. It does not change the Engine interface and is merely an internal change to make it clearer where idempotence is handled. It also makes the backstore API consistent. --- src/engine/macros.rs | 70 +++++++++++++------ src/engine/sim_engine/engine.rs | 2 +- src/engine/sim_engine/pool.rs | 2 +- .../strat_engine/backstore/backstore.rs | 19 +++-- src/engine/strat_engine/engine.rs | 2 +- src/engine/strat_engine/pool.rs | 29 +++++--- src/engine/strat_engine/thinpool/thinpool.rs | 27 ++++--- 7 files changed, 105 insertions(+), 46 deletions(-) diff --git a/src/engine/macros.rs b/src/engine/macros.rs index a7321b9eb3..872892b833 100644 --- a/src/engine/macros.rs +++ b/src/engine/macros.rs @@ -30,18 +30,18 @@ macro_rules! get_mut_pool { }; } -macro_rules! rename_filesystem_pre { - ($s:ident; $uuid:ident; $new_name:ident) => {{ - let old_name = match $s.filesystems.get_by_uuid($uuid) { +macro_rules! rename_pre { + ($s:expr; $uuid:ident; $new_name:ident; $not_found:expr; $same:expr) => {{ + let old_name = match $s.get_by_uuid($uuid) { Some((name, _)) => name, - None => return Ok(RenameAction::NoSource), + None => return $not_found, }; if &*old_name == $new_name { - return Ok(RenameAction::Identity); + return $same; } - if $s.filesystems.contains_name($new_name) { + if $s.contains_name($new_name) { return Err(StratisError::Engine( ErrorEnum::AlreadyExists, $new_name.into(), @@ -51,25 +51,51 @@ macro_rules! rename_filesystem_pre { }}; } -macro_rules! rename_pool_pre { +macro_rules! rename_filesystem_pre { ($s:ident; $uuid:ident; $new_name:ident) => {{ - let old_name = match $s.pools.get_by_uuid($uuid) { - Some((name, _)) => name, - None => return Ok(RenameAction::NoSource), - }; + rename_pre!( + $s.filesystems; + $uuid; + $new_name; + Err(StratisError::Engine( + ErrorEnum::NotFound, + format!("Filesystem not found with UUID of {}", $uuid), + )); + Ok(None) + ) + }} +} - if &*old_name == $new_name { - return Ok(RenameAction::Identity); - } +macro_rules! rename_pre_idem { + ($s:expr; $uuid:ident; $new_name:ident) => {{ + rename_pre!( + $s; + $uuid; + $new_name; + Ok(RenameAction::NoSource); + Ok(RenameAction::Identity) + ) + }} +} - if $s.pools.contains_name($new_name) { - return Err(StratisError::Engine( - ErrorEnum::AlreadyExists, - $new_name.into(), - )); - } - old_name - }}; +macro_rules! rename_filesystem_pre_idem { + ($s:ident; $uuid:ident; $new_name:ident) => {{ + rename_pre_idem!( + $s.filesystems; + $uuid; + $new_name + ) + }} +} + +macro_rules! rename_pool_pre_idem { + ($s:ident; $uuid:ident; $new_name:ident) => {{ + rename_pre_idem!( + $s.pools; + $uuid; + $new_name + ) + }} } macro_rules! set_blockdev_user_info { diff --git a/src/engine/sim_engine/engine.rs b/src/engine/sim_engine/engine.rs index a47c984fdb..6f8a3bc4b2 100644 --- a/src/engine/sim_engine/engine.rs +++ b/src/engine/sim_engine/engine.rs @@ -94,7 +94,7 @@ impl Engine for SimEngine { uuid: PoolUuid, new_name: &str, ) -> StratisResult> { - rename_pool_pre!(self; uuid; new_name); + rename_pool_pre_idem!(self; uuid; new_name); let (_, pool) = self .pools diff --git a/src/engine/sim_engine/pool.rs b/src/engine/sim_engine/pool.rs index bec1bc5d4c..495004de18 100644 --- a/src/engine/sim_engine/pool.rs +++ b/src/engine/sim_engine/pool.rs @@ -164,7 +164,7 @@ impl Pool for SimPool { uuid: FilesystemUuid, new_name: &str, ) -> StratisResult> { - rename_filesystem_pre!(self; uuid; new_name); + rename_filesystem_pre_idem!(self; uuid; new_name); let (_, filesystem) = self .filesystems diff --git a/src/engine/strat_engine/backstore/backstore.rs b/src/engine/strat_engine/backstore/backstore.rs index 0ab2190570..4ac7d316d5 100644 --- a/src/engine/strat_engine/backstore/backstore.rs +++ b/src/engine/strat_engine/backstore/backstore.rs @@ -30,7 +30,6 @@ use crate::{ names::{format_backstore_ids, CacheRole}, serde_structs::{BackstoreSave, CapSave, Recordable}, }, - types::RenameAction, BlockDevTier, DevUuid, PoolUuid, }, stratis::{ErrorEnum, StratisError, StratisResult}, @@ -553,18 +552,28 @@ impl Backstore { /// Set user info field on the specified blockdev. /// May return an error if there is no blockdev for the given UUID. + /// + /// * Ok(Some(uuid)) provides the uuid of the changed blockdev + /// * Ok(None) is returned if the blockdev was unchanged + /// * Err(StratisError::Engine(ErrorEnum::NotFound, _)) is returned if the UUID + /// does not correspond to a blockdev pub fn set_blockdev_user_info( &mut self, uuid: DevUuid, user_info: Option<&str>, - ) -> RenameAction { + ) -> StratisResult> { self.get_mut_blockdev_by_uuid(uuid).map_or_else( - || RenameAction::NoSource, + || { + Err(StratisError::Engine( + ErrorEnum::NotFound, + format!("Blockdev with a UUID of {} was not found", uuid), + )) + }, |(_, b)| { if b.set_user_info(user_info) { - RenameAction::Renamed(uuid) + Ok(Some(uuid)) } else { - RenameAction::Identity + Ok(None) } }, ) diff --git a/src/engine/strat_engine/engine.rs b/src/engine/strat_engine/engine.rs index dfb3bfcb4c..3ea5873506 100644 --- a/src/engine/strat_engine/engine.rs +++ b/src/engine/strat_engine/engine.rs @@ -305,7 +305,7 @@ impl Engine for StratEngine { new_name: &str, ) -> StratisResult> { validate_name(new_name)?; - let old_name = rename_pool_pre!(self; uuid; new_name); + let old_name = rename_pool_pre_idem!(self; uuid; new_name); let (_, mut pool) = self .pools diff --git a/src/engine/strat_engine/pool.rs b/src/engine/strat_engine/pool.rs index c29910ce3e..080b9c41b1 100644 --- a/src/engine/strat_engine/pool.rs +++ b/src/engine/strat_engine/pool.rs @@ -25,8 +25,8 @@ use crate::{ thinpool::{ThinPool, ThinPoolSizeParams, DATA_BLOCK_SIZE}, }, types::{ - BlockDevTier, CreateAction, DevUuid, EngineAction, FilesystemUuid, MaybeDbusPath, Name, - PoolUuid, Redundancy, RenameAction, SetCreateAction, SetDeleteAction, + BlockDevTier, CreateAction, DevUuid, FilesystemUuid, MaybeDbusPath, Name, PoolUuid, + Redundancy, RenameAction, SetCreateAction, SetDeleteAction, }, }, stratis::{ErrorEnum, StratisError, StratisResult}, @@ -360,8 +360,7 @@ impl Pool for StratPool { ) -> StratisResult> { let mut removed = Vec::new(); for &uuid in fs_uuids { - let changed = self.thin_pool.destroy_filesystem(pool_name, uuid)?; - if changed.is_changed() { + if let Some(uuid) = self.thin_pool.destroy_filesystem(pool_name, uuid)? { removed.push(uuid); } } @@ -376,7 +375,17 @@ impl Pool for StratPool { new_name: &str, ) -> StratisResult> { validate_name(new_name)?; - self.thin_pool.rename_filesystem(pool_name, uuid, new_name) + match self.thin_pool.rename_filesystem(pool_name, uuid, new_name) { + Ok(Some(uuid)) => Ok(RenameAction::Renamed(uuid)), + Ok(None) => Ok(RenameAction::Identity), + Err(e) => { + if let StratisError::Engine(ErrorEnum::NotFound, _) = e { + Ok(RenameAction::NoSource) + } else { + Err(e) + } + } + } } fn snapshot_filesystem( @@ -471,10 +480,14 @@ impl Pool for StratPool { user_info: Option<&str>, ) -> StratisResult> { let result = self.backstore.set_blockdev_user_info(uuid, user_info); - if let RenameAction::Renamed(_) = result { - self.write_metadata(pool_name)?; + match result { + Ok(Some(uuid)) => { + self.write_metadata(pool_name)?; + Ok(RenameAction::Renamed(uuid)) + } + Ok(None) => Ok(RenameAction::Identity), + Err(_) => Ok(RenameAction::NoSource), } - Ok(result) } fn set_dbus_path(&mut self, path: MaybeDbusPath) { diff --git a/src/engine/strat_engine/thinpool/thinpool.rs b/src/engine/strat_engine/thinpool/thinpool.rs index 0a91a1a990..fe1d048715 100644 --- a/src/engine/strat_engine/thinpool/thinpool.rs +++ b/src/engine/strat_engine/thinpool/thinpool.rs @@ -42,8 +42,8 @@ use crate::{ }, structures::Table, types::{ - DeleteAction, FilesystemUuid, FreeSpaceState, MaybeDbusPath, Name, PoolExtendState, - PoolState, PoolUuid, RenameAction, + FilesystemUuid, FreeSpaceState, MaybeDbusPath, Name, PoolExtendState, PoolState, + PoolUuid, }, }, stratis::{ErrorEnum, StratisError, StratisResult}, @@ -957,11 +957,15 @@ impl ThinPool { /// Destroy a filesystem within the thin pool. Destroy metadata and /// devlinks information associated with the thinpool. If there is a /// failure to destroy the filesystem, retain it, and return an error. + /// + /// * Ok(Some(uuid)) provides the uuid of the destroyed filesystem + /// * Ok(None) is returned if the filesystem did not exist + /// * Err(_) is returned if the filesystem could not be destroyed pub fn destroy_filesystem( &mut self, pool_name: &str, uuid: FilesystemUuid, - ) -> StratisResult> { + ) -> StratisResult> { match self.filesystems.remove_by_uuid(uuid) { Some((fs_name, mut fs)) => match fs.destroy(&self.thin_pool) { Ok(_) => { @@ -973,14 +977,14 @@ impl ThinPool { err); } devlinks::filesystem_removed(pool_name, &fs_name); - Ok(DeleteAction::Deleted(uuid)) + Ok(Some(uuid)) } Err(err) => { self.filesystems.insert(fs_name, uuid, fs); Err(err) } }, - None => Ok(DeleteAction::Identity), + None => Ok(None), } } @@ -995,12 +999,19 @@ impl ThinPool { } /// Rename a filesystem within the thin pool. + /// + /// * Ok(Some(uuid)) provides the uuid of the renamed filesystem + /// * Ok(None) is returned if the source and target filesystem names are the same + /// * Err(StratisError::Engine(ErrorEnum::NotFound, _)) is returned if the source + /// filesystem name does not exist + /// * Err(StratisError::Engine(ErrorEnum::AlreadyExists, _)) is returned if the target + /// filesystem name already exists pub fn rename_filesystem( &mut self, pool_name: &str, uuid: FilesystemUuid, new_name: &str, - ) -> StratisResult> { + ) -> StratisResult> { let old_name = rename_filesystem_pre!(self; uuid; new_name); let new_name = Name::new(new_name.to_owned()); @@ -1021,7 +1032,7 @@ impl ThinPool { }); self.filesystems.insert(new_name.clone(), uuid, filesystem); devlinks::filesystem_renamed(pool_name, &old_name, &new_name); - Ok(RenameAction::Renamed(uuid)) + Ok(Some(uuid)) } } @@ -1536,7 +1547,7 @@ mod tests { .unwrap(); let action = pool.rename_filesystem(pool_name, fs_uuid, name2).unwrap(); - assert_matches!(action, RenameAction::Renamed(_)); + assert_matches!(action, Some(_)); let flexdevs: FlexDevsSave = pool.record(); let thinpoolsave: ThinPoolDevSave = pool.record(); pool.teardown().unwrap(); From edf73f7fd95fa3a5a4f9b7cbae745b1de42bd748 Mon Sep 17 00:00:00 2001 From: mulhern Date: Wed, 11 Dec 2019 12:15:52 -0500 Subject: [PATCH 044/109] Do not request non-determinism from the simulator unless explicitly using it Non-determinism was only implemented in the simulator in a few places. Given that the simulator is unlikely to become non-deterministic in more places than it is already, only request non-determinism when associated tests explicitly use it, by, e.g., setting up a different set of assertions depending on whether an operation succeeds or fails. Otherwise, assume that the test is intended to be deterministic. Remove a setUp method made redundant by removal of call to ConfigureSimulator. Signed-off-by: mulhern --- .../tests/dbus/blockdev/test_fetch_properties.py | 1 - .../tests/dbus/filesystem/test_fetch_properties.py | 1 - .../client-dbus/tests/dbus/filesystem/test_properties.py | 1 - tests/client-dbus/tests/dbus/filesystem/test_rename.py | 1 - tests/client-dbus/tests/dbus/manager/test_create.py | 2 -- tests/client-dbus/tests/dbus/manager/test_destroy.py | 4 ---- tests/client-dbus/tests/dbus/manager/test_stratis.py | 9 --------- .../tests/dbus/pool/test_create_filesystem.py | 2 -- .../client-dbus/tests/dbus/pool/test_create_snapshot.py | 1 - .../tests/dbus/pool/test_destroy_filesystem.py | 2 -- .../client-dbus/tests/dbus/pool/test_fetch_properties.py | 1 - tests/client-dbus/tests/dbus/pool/test_rename.py | 1 - 12 files changed, 26 deletions(-) diff --git a/tests/client-dbus/tests/dbus/blockdev/test_fetch_properties.py b/tests/client-dbus/tests/dbus/blockdev/test_fetch_properties.py index 972dab1163..3409a648e8 100644 --- a/tests/client-dbus/tests/dbus/blockdev/test_fetch_properties.py +++ b/tests/client-dbus/tests/dbus/blockdev/test_fetch_properties.py @@ -43,7 +43,6 @@ def setUp(self): "devices": ["/dev/one", "/dev/two", "/dev/red", "/dev/blue"], }, ) - Manager.Methods.ConfigureSimulator(proxy, {"denominator": 8}) def testFetchSizeProperty(self): """ diff --git a/tests/client-dbus/tests/dbus/filesystem/test_fetch_properties.py b/tests/client-dbus/tests/dbus/filesystem/test_fetch_properties.py index 966949abb8..b249a2ab26 100644 --- a/tests/client-dbus/tests/dbus/filesystem/test_fetch_properties.py +++ b/tests/client-dbus/tests/dbus/filesystem/test_fetch_properties.py @@ -51,7 +51,6 @@ def setUp(self): pool_object, {"specs": [self._FSNAME]} ) self._filesystem_object_path = created[0][0] - Manager.Methods.ConfigureSimulator(proxy, {"denominator": 8}) def testFetchUsedProperty(self): """ diff --git a/tests/client-dbus/tests/dbus/filesystem/test_properties.py b/tests/client-dbus/tests/dbus/filesystem/test_properties.py index aec14f0759..175c6b9be6 100644 --- a/tests/client-dbus/tests/dbus/filesystem/test_properties.py +++ b/tests/client-dbus/tests/dbus/filesystem/test_properties.py @@ -54,7 +54,6 @@ def setUp(self): pool_object, {"specs": [self._FSNAME]} ) self._filesystem_object_path = created[0][0] - Manager.Methods.ConfigureSimulator(proxy, {"denominator": 8}) def testProps(self): """ diff --git a/tests/client-dbus/tests/dbus/filesystem/test_rename.py b/tests/client-dbus/tests/dbus/filesystem/test_rename.py index fb1c8eb5c9..14454097ab 100644 --- a/tests/client-dbus/tests/dbus/filesystem/test_rename.py +++ b/tests/client-dbus/tests/dbus/filesystem/test_rename.py @@ -59,7 +59,6 @@ def setUp(self): pool_object, {"specs": [self._FSNAME]} ) self._filesystem_object_path = created[0][0] - Manager.Methods.ConfigureSimulator(self._proxy, {"denominator": 8}) def testNullMapping(self): """ diff --git a/tests/client-dbus/tests/dbus/manager/test_create.py b/tests/client-dbus/tests/dbus/manager/test_create.py index 7dfbcd6d60..6e485b956b 100644 --- a/tests/client-dbus/tests/dbus/manager/test_create.py +++ b/tests/client-dbus/tests/dbus/manager/test_create.py @@ -109,7 +109,6 @@ def setUp(self): "devices": _DEVICE_STRATEGY(), }, ) - Manager.Methods.ConfigureSimulator(self._proxy, {"denominator": 8}) def testCreateDifferentBlockdevs(self): """ @@ -162,7 +161,6 @@ def setUp(self): "devices": self._blockdevs, }, ) - Manager.Methods.ConfigureSimulator(self._proxy, {"denominator": 8}) def testCreateSameBlockdevs(self): """ diff --git a/tests/client-dbus/tests/dbus/manager/test_destroy.py b/tests/client-dbus/tests/dbus/manager/test_destroy.py index 712f030dab..63440eed57 100644 --- a/tests/client-dbus/tests/dbus/manager/test_destroy.py +++ b/tests/client-dbus/tests/dbus/manager/test_destroy.py @@ -48,7 +48,6 @@ def setUp(self): """ super().setUp() self._proxy = get_object(TOP_OBJECT) - Manager.Methods.ConfigureSimulator(self._proxy, {"denominator": 8}) def testExecution(self): """ @@ -85,7 +84,6 @@ def setUp(self): self._proxy, {"name": self._POOLNAME, "redundancy": (True, 0), "devices": self._devices}, ) - Manager.Methods.ConfigureSimulator(self._proxy, {"denominator": 8}) def testExecution(self): """ @@ -137,7 +135,6 @@ def setUp(self): }, ) Pool.Methods.CreateFilesystems(get_object(poolpath), {"specs": [self._FSNAME]}) - Manager.Methods.ConfigureSimulator(self._proxy, {"denominator": 8}) def testExecution(self): """ @@ -172,7 +169,6 @@ def setUp(self): self._proxy, {"name": self._POOLNAME, "redundancy": (True, 0), "devices": []}, ) - Manager.Methods.ConfigureSimulator(self._proxy, {"denominator": 8}) def testExecution(self): """ diff --git a/tests/client-dbus/tests/dbus/manager/test_stratis.py b/tests/client-dbus/tests/dbus/manager/test_stratis.py index e3c1129a59..eec6fb60b0 100644 --- a/tests/client-dbus/tests/dbus/manager/test_stratis.py +++ b/tests/client-dbus/tests/dbus/manager/test_stratis.py @@ -30,14 +30,6 @@ class StratisTestCase(SimTestCase): Test meta information about stratisd. """ - def setUp(self): - """ - Start the stratisd daemon with the simulator. - """ - super().setUp() - proxy = get_object(TOP_OBJECT) - Manager.Methods.ConfigureSimulator(proxy, {"denominator": 8}) - def testStratisVersion(self): """ Getting version should succeed. @@ -60,7 +52,6 @@ def setUp(self): """ super().setUp() self._proxy = get_object(TOP_OBJECT) - Manager.Methods.ConfigureSimulator(self._proxy, {"denominator": 8}) def testArguments(self): """ diff --git a/tests/client-dbus/tests/dbus/pool/test_create_filesystem.py b/tests/client-dbus/tests/dbus/pool/test_create_filesystem.py index 51c536bdfe..f4420765cb 100644 --- a/tests/client-dbus/tests/dbus/pool/test_create_filesystem.py +++ b/tests/client-dbus/tests/dbus/pool/test_create_filesystem.py @@ -57,7 +57,6 @@ def setUp(self): }, ) self._pool_object = get_object(poolpath) - Manager.Methods.ConfigureSimulator(self._proxy, {"denominator": 8}) def testCreate(self): """ @@ -126,7 +125,6 @@ def setUp(self): ) self._pool_object = get_object(poolpath) Pool.Methods.CreateFilesystems(self._pool_object, {"specs": [self._FSNAME]}) - Manager.Methods.ConfigureSimulator(self._proxy, {"denominator": 8}) def testCreate(self): """ diff --git a/tests/client-dbus/tests/dbus/pool/test_create_snapshot.py b/tests/client-dbus/tests/dbus/pool/test_create_snapshot.py index 55140b07b4..f689fd60ea 100644 --- a/tests/client-dbus/tests/dbus/pool/test_create_snapshot.py +++ b/tests/client-dbus/tests/dbus/pool/test_create_snapshot.py @@ -55,7 +55,6 @@ def setUp(self): }, ) self._pool_object = get_object(poolpath) - Manager.Methods.ConfigureSimulator(self._proxy, {"denominator": 8}) ((_, fs_objects), rc, _) = Pool.Methods.CreateFilesystems( self._pool_object, {"specs": [self._FSNAME]} diff --git a/tests/client-dbus/tests/dbus/pool/test_destroy_filesystem.py b/tests/client-dbus/tests/dbus/pool/test_destroy_filesystem.py index d41b7949c2..154e7d6004 100644 --- a/tests/client-dbus/tests/dbus/pool/test_destroy_filesystem.py +++ b/tests/client-dbus/tests/dbus/pool/test_destroy_filesystem.py @@ -53,7 +53,6 @@ def setUp(self): }, ) self._pool_object = get_object(poolpath) - Manager.Methods.ConfigureSimulator(self._proxy, {"denominator": 8}) def testDestroyNone(self): """ @@ -116,7 +115,6 @@ def setUp(self): ((_, self._filesystems), _, _) = Pool.Methods.CreateFilesystems( self._pool_object, {"specs": [(self._FSNAME, "", None)]} ) - Manager.Methods.ConfigureSimulator(self._proxy, {"denominator": 8}) def testDestroyOne(self): """ diff --git a/tests/client-dbus/tests/dbus/pool/test_fetch_properties.py b/tests/client-dbus/tests/dbus/pool/test_fetch_properties.py index c6459854d9..a681bc46f7 100644 --- a/tests/client-dbus/tests/dbus/pool/test_fetch_properties.py +++ b/tests/client-dbus/tests/dbus/pool/test_fetch_properties.py @@ -46,7 +46,6 @@ def setUp(self): }, ) self._pool_object = get_object(pool_object_path) - Manager.Methods.ConfigureSimulator(proxy, {"denominator": 8}) def testFetchSizeProperty(self): """ diff --git a/tests/client-dbus/tests/dbus/pool/test_rename.py b/tests/client-dbus/tests/dbus/pool/test_rename.py index 3d7ea74a24..371de78d81 100644 --- a/tests/client-dbus/tests/dbus/pool/test_rename.py +++ b/tests/client-dbus/tests/dbus/pool/test_rename.py @@ -53,7 +53,6 @@ def setUp(self): }, ) self._pool_object = get_object(self._pool_object_path) - Manager.Methods.ConfigureSimulator(self._proxy, {"denominator": 8}) def testNullMapping(self): """ From 18511361c9963a935b8437f9076d611b49b4f23e Mon Sep 17 00:00:00 2001 From: mulhern Date: Wed, 11 Dec 2019 12:20:04 -0500 Subject: [PATCH 045/109] Fix a test header comment Signed-off-by: mulhern --- tests/client-dbus/tests/dbus/manager/test_create.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/client-dbus/tests/dbus/manager/test_create.py b/tests/client-dbus/tests/dbus/manager/test_create.py index 6e485b956b..33b5020058 100644 --- a/tests/client-dbus/tests/dbus/manager/test_create.py +++ b/tests/client-dbus/tests/dbus/manager/test_create.py @@ -164,8 +164,8 @@ def setUp(self): def testCreateSameBlockdevs(self): """ - Create should fail trying to create new pool with same name - and different blockdevs from previous. + Create should succeed trying to create new pool with same name + and same blockdevs as previous. """ pools1 = pools().search( ObjectManager.Methods.GetManagedObjects(self._proxy, {}) From 51ad4dc69fdd8ed8d82fd9492dddea9d62220522 Mon Sep 17 00:00:00 2001 From: mulhern Date: Wed, 11 Dec 2019 12:22:55 -0500 Subject: [PATCH 046/109] Require _DEVICE_STRATEGY to always create at least 1 device Otherwise, testCreateDifferentBlockdevs may randomly fail if setUp created a pool w/ 0 blockdevs and then the second CreatePool command, which should cause a conflict, also randomly uses 0 blockdevs to create the pool. In that case, even though the two lists of devices are random, since they are both empty, the devices do not conflict and the second CreatePool call succeeds, causing the test to fail. Signed-off-by: mulhern --- tests/client-dbus/tests/dbus/manager/test_create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/client-dbus/tests/dbus/manager/test_create.py b/tests/client-dbus/tests/dbus/manager/test_create.py index 33b5020058..905960120e 100644 --- a/tests/client-dbus/tests/dbus/manager/test_create.py +++ b/tests/client-dbus/tests/dbus/manager/test_create.py @@ -27,7 +27,7 @@ from .._misc import SimTestCase, device_name_list -_DEVICE_STRATEGY = device_name_list() +_DEVICE_STRATEGY = device_name_list(1) class Create2TestCase(SimTestCase): From 86e359600e384028ae6b7cf9e6f0291997e4b546 Mon Sep 17 00:00:00 2001 From: mulhern Date: Fri, 13 Dec 2019 15:10:31 -0500 Subject: [PATCH 047/109] Use "omitting" instead of "discarded" in log entries in find_all() "discarded" suggests that something might actually have been done to the device, "omitting" sounds less active. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/setup.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/engine/strat_engine/backstore/setup.rs b/src/engine/strat_engine/backstore/setup.rs index 97aa0602de..e4689631b8 100644 --- a/src/engine/strat_engine/backstore/setup.rs +++ b/src/engine/strat_engine/backstore/setup.rs @@ -56,7 +56,7 @@ pub fn find_all() -> StratisResult>> .filter(|dev| dev.is_initialized()) .filter(|dev| !is_multipath_member(dev) .map_err(|err| { - warn!("Could not certainly determine whether a device was a multipath member because of an error processing udev information, discarded the device from the set of devices to process, for safety: {}", + warn!("Could not certainly determine whether a device was a multipath member because of an error processing udev information, omitting the device from the set of devices to process, for safety: {}", err); }) .unwrap_or(true)) @@ -117,7 +117,7 @@ pub fn find_all() -> StratisResult>> .filter(|dev| { decide_ownership(dev) .map_err(|err| { - warn!("Could not determine ownership of a udev block device because of an error processing udev information, discarded the device from the set of devices to process, for safety: {}", + warn!("Could not determine ownership of a udev block device because of an error processing udev information, omitting the device from the set of devices to process, for safety: {}", err); }) .map(|decision| match decision { From 068b6b1e3cf8c31cda205eecd8a0ea9ace658cff Mon Sep 17 00:00:00 2001 From: mulhern Date: Fri, 13 Dec 2019 15:12:35 -0500 Subject: [PATCH 048/109] Log existence of uninitialized devices in find_all() Log at different levels, because their significance is different depending on how the devices are being searched for. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/setup.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/engine/strat_engine/backstore/setup.rs b/src/engine/strat_engine/backstore/setup.rs index e4689631b8..b33f583e98 100644 --- a/src/engine/strat_engine/backstore/setup.rs +++ b/src/engine/strat_engine/backstore/setup.rs @@ -53,7 +53,13 @@ pub fn find_all() -> StratisResult>> enumerator.match_property("ID_FS_TYPE", "stratis")?; for devnode in enumerator .scan_devices()? - .filter(|dev| dev.is_initialized()) + .filter(|dev| { + let initialized = dev.is_initialized(); + if !initialized { + warn!("Found a udev entry for a device identified as a Stratis device, but udev also identified it as uninitialized, omitting the device from the set of devices to process, for safety"); + }; + initialized + }) .filter(|dev| !is_multipath_member(dev) .map_err(|err| { warn!("Could not certainly determine whether a device was a multipath member because of an error processing udev information, omitting the device from the set of devices to process, for safety: {}", @@ -113,7 +119,13 @@ pub fn find_all() -> StratisResult>> let mut enumerator = block_enumerator(&context)?; for devnode in enumerator .scan_devices()? - .filter(|dev| dev.is_initialized()) + .filter(|dev| { + let initialized = dev.is_initialized(); + if !initialized { + debug!("Found a udev entry for a device identified as a block device, but udev also identified it as uninitialized, omitting the device from the set of devices to process, for safety"); + }; + initialized + }) .filter(|dev| { decide_ownership(dev) .map_err(|err| { From e45877c2d65437d0e97b5192efff06214d04bd92 Mon Sep 17 00:00:00 2001 From: mulhern Date: Fri, 13 Dec 2019 15:14:05 -0500 Subject: [PATCH 049/109] In find_all() warn on missing device node Whether a device is believed to be a Stratis device or it is believed just to be a block device, if it doesn't have a device node that ought to be considered a bit of a surprise. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/setup.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/engine/strat_engine/backstore/setup.rs b/src/engine/strat_engine/backstore/setup.rs index b33f583e98..bfc0b1f26b 100644 --- a/src/engine/strat_engine/backstore/setup.rs +++ b/src/engine/strat_engine/backstore/setup.rs @@ -66,7 +66,12 @@ pub fn find_all() -> StratisResult>> err); }) .unwrap_or(true)) - .filter_map(|i| i.devnode().map(|d| d.to_path_buf())) + .filter_map(|i| i.devnode() + .map(|d| d.to_path_buf()) + .or_else(||{ + warn!("udev identified a device as a Stratis device, but the udev entry for the device had no device node, omitting the the device from the set of devices to process"); + None + })) { if let Some(devno) = devnode_to_devno(&devnode)? { if let Some((pool_uuid, _)) = match device_identifiers( @@ -138,7 +143,12 @@ pub fn find_all() -> StratisResult>> }) .unwrap_or(false) }) - .filter_map(|i| i.devnode().map(|d| d.to_path_buf())) + .filter_map(|i| i.devnode() + .map(|d| d.to_path_buf()) + .or_else(||{ + warn!("udev identified a device as a block device, but the udev entry for the device had no device node, omitting the the device from the set of devices to process"); + None + })) { if let Some(devno) = devnode_to_devno(&devnode)? { if let Some((pool_uuid, _)) = match device_identifiers( From 848a90e406ff3e592995fbc5ca46999e3bc663bb Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 16 Dec 2019 13:16:38 -0500 Subject: [PATCH 050/109] Fix up broken indices in arguments to get_next_arg() Document get_next_arg(). Signed-off-by: mulhern --- src/dbus_api/api.rs | 2 +- src/dbus_api/pool.rs | 4 ++-- src/dbus_api/util.rs | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/dbus_api/api.rs b/src/dbus_api/api.rs index 9064a6c4e1..15416fd60c 100644 --- a/src/dbus_api/api.rs +++ b/src/dbus_api/api.rs @@ -36,7 +36,7 @@ fn create_pool(m: &MethodInfo, TData>) -> MethodResult { let name: &str = get_next_arg(&mut iter, 0)?; let redundancy: (bool, u16) = get_next_arg(&mut iter, 1)?; - let devs: Array<&str, _> = get_next_arg(&mut iter, 3)?; + let devs: Array<&str, _> = get_next_arg(&mut iter, 2)?; let blockdevs = devs.map(|x| Path::new(x)).collect::>(); diff --git a/src/dbus_api/pool.rs b/src/dbus_api/pool.rs index 32f1c68278..8fb81d51f5 100644 --- a/src/dbus_api/pool.rs +++ b/src/dbus_api/pool.rs @@ -176,7 +176,7 @@ fn snapshot_filesystem(m: &MethodInfo, TData>) -> MethodResult { let mut iter = message.iter_init(); let filesystem: dbus::Path<'static> = get_next_arg(&mut iter, 0)?; - let snapshot_name: &str = get_next_arg(&mut iter, 0)?; + let snapshot_name: &str = get_next_arg(&mut iter, 1)?; let dbus_context = m.tree.get_data(); let object_path = m.path.get_name(); @@ -223,7 +223,7 @@ fn add_blockdevs(m: &MethodInfo, TData>, tier: BlockDevTier) -> Meth let message: &Message = m.msg; let mut iter = message.iter_init(); - let devs: Array<&str, _> = get_next_arg(&mut iter, 1)?; + let devs: Array<&str, _> = get_next_arg(&mut iter, 0)?; let dbus_context = m.tree.get_data(); let object_path = m.path.get_name(); diff --git a/src/dbus_api/util.rs b/src/dbus_api/util.rs index 8a84466fed..a8a123b0a0 100644 --- a/src/dbus_api/util.rs +++ b/src/dbus_api/util.rs @@ -51,7 +51,8 @@ where (key, (success, value)) } -/// Get the next argument off the bus +/// Get the next argument off the bus. loc is the index of the location of +/// the argument in the iterator, and is used solely for error-reporting. pub fn get_next_arg<'a, T>(iter: &mut Iter<'a>, loc: u16) -> Result where T: dbus::arg::Get<'a> + dbus::arg::Arg, From 5b47cd18e065582f0830d8a34dab2bba597adf87 Mon Sep 17 00:00:00 2001 From: Bryan Gurney Date: Fri, 13 Dec 2019 15:57:14 -0500 Subject: [PATCH 051/109] Fully qualify non-prelude macro data types Signed-off-by: Bryan Gurney --- src/dbus_api/api.rs | 2 +- src/dbus_api/macros.rs | 15 +++++++++--- src/engine/macros.rs | 23 +++++++++++-------- src/engine/sim_engine/engine.rs | 2 +- .../strat_engine/backstore/metadata/bda.rs | 2 +- src/engine/strat_engine/engine.rs | 2 +- 6 files changed, 29 insertions(+), 17 deletions(-) diff --git a/src/dbus_api/api.rs b/src/dbus_api/api.rs index 15416fd60c..f2d5df8dab 100644 --- a/src/dbus_api/api.rs +++ b/src/dbus_api/api.rs @@ -21,7 +21,7 @@ use crate::{ consts, filesystem::create_dbus_filesystem, pool::create_dbus_pool, - types::{DbusContext, DbusErrorEnum, DeferredAction, TData}, + types::{DbusContext, DeferredAction, TData}, util::{ engine_to_dbus_err_tuple, get_next_arg, msg_code_ok, msg_string_ok, tuple_to_option, }, diff --git a/src/dbus_api/macros.rs b/src/dbus_api/macros.rs index 1a360ffe6a..40805efdd9 100644 --- a/src/dbus_api/macros.rs +++ b/src/dbus_api/macros.rs @@ -10,7 +10,10 @@ macro_rules! get_data { data } else { let message = format!("no data for object path {}", $path.get_name()); - let (rc, rs) = (DbusErrorEnum::INTERNAL_ERROR as u16, message); + let (rc, rs) = ( + $crate::dbus_api::types::DbusErrorEnum::INTERNAL_ERROR as u16, + message, + ); return Ok(vec![$message.append3($default, rc, rs)]); } }; @@ -24,7 +27,10 @@ macro_rules! get_parent { parent } else { let message = format!("no path for object path {}", $data.parent); - let (rc, rs) = (DbusErrorEnum::INTERNAL_ERROR as u16, message); + let (rc, rs) = ( + $crate::dbus_api::types::DbusErrorEnum::INTERNAL_ERROR as u16, + message, + ); return Ok(vec![$message.append3($default, rc, rs)]); } }; @@ -37,7 +43,10 @@ macro_rules! get_mut_pool { pool } else { let message = format!("engine does not know about pool with uuid {}", $uuid); - let (rc, rs) = (DbusErrorEnum::INTERNAL_ERROR as u16, message); + let (rc, rs) = ( + $crate::dbus_api::types::DbusErrorEnum::INTERNAL_ERROR as u16, + message, + ); return Ok(vec![$message.append3($default, rc, rs)]); } }; diff --git a/src/engine/macros.rs b/src/engine/macros.rs index 872892b833..c451544c0a 100644 --- a/src/engine/macros.rs +++ b/src/engine/macros.rs @@ -5,10 +5,13 @@ macro_rules! calculate_redundancy { ($redundancy:ident) => { match $redundancy { - None | Some(0) => Redundancy::NONE, + None | Some(0) => $crate::engine::Redundancy::NONE, Some(n) => { let message = format!("code {} does not correspond to any redundancy", n); - return Err(StratisError::Engine(ErrorEnum::Error, message)); + return Err($crate::stratis::StratisError::Engine( + $crate::stratis::ErrorEnum::Error, + message, + )); } } }; @@ -18,7 +21,7 @@ macro_rules! get_pool { ($s:ident; $uuid:ident) => { $s.pools .get_by_uuid($uuid) - .map(|(name, p)| (name.clone(), p as &dyn Pool)) + .map(|(name, p)| (name.clone(), p as &dyn $crate::engine::Pool)) }; } @@ -26,7 +29,7 @@ macro_rules! get_mut_pool { ($s:ident; $uuid:ident) => { $s.pools .get_mut_by_uuid($uuid) - .map(|(name, p)| (name.clone(), p as &mut dyn Pool)) + .map(|(name, p)| (name.clone(), p as &mut dyn $crate::engine::Pool)) }; } @@ -42,8 +45,8 @@ macro_rules! rename_pre { } if $s.contains_name($new_name) { - return Err(StratisError::Engine( - ErrorEnum::AlreadyExists, + return Err($crate::stratis::StratisError::Engine( + $crate::stratis::ErrorEnum::AlreadyExists, $new_name.into(), )); } @@ -57,8 +60,8 @@ macro_rules! rename_filesystem_pre { $s.filesystems; $uuid; $new_name; - Err(StratisError::Engine( - ErrorEnum::NotFound, + Err($crate::stratis::StratisError::Engine( + $crate::stratis::ErrorEnum::NotFound, format!("Filesystem not found with UUID of {}", $uuid), )); Ok(None) @@ -72,8 +75,8 @@ macro_rules! rename_pre_idem { $s; $uuid; $new_name; - Ok(RenameAction::NoSource); - Ok(RenameAction::Identity) + Ok($crate::engine::RenameAction::NoSource); + Ok($crate::engine::RenameAction::Identity) ) }} } diff --git a/src/engine/sim_engine/engine.rs b/src/engine/sim_engine/engine.rs index 6f8a3bc4b2..146903705c 100644 --- a/src/engine/sim_engine/engine.rs +++ b/src/engine/sim_engine/engine.rs @@ -18,7 +18,7 @@ use crate::{ shared::create_pool_idempotent_or_err, sim_engine::{pool::SimPool, randomization::Randomizer}, structures::Table, - types::{CreateAction, DeleteAction, Name, PoolUuid, Redundancy, RenameAction}, + types::{CreateAction, DeleteAction, Name, PoolUuid, RenameAction}, }, stratis::{ErrorEnum, StratisError, StratisResult}, }; diff --git a/src/engine/strat_engine/backstore/metadata/bda.rs b/src/engine/strat_engine/backstore/metadata/bda.rs index 19c84116e5..5af5e1704a 100644 --- a/src/engine/strat_engine/backstore/metadata/bda.rs +++ b/src/engine/strat_engine/backstore/metadata/bda.rs @@ -59,7 +59,7 @@ where // Transform a constant in sectors to a constant in bytes macro_rules! bytes { ($number:expr) => { - $number * SECTOR_SIZE + $number * devicemapper::SECTOR_SIZE }; } diff --git a/src/engine/strat_engine/engine.rs b/src/engine/strat_engine/engine.rs index 3ea5873506..497ab33e53 100644 --- a/src/engine/strat_engine/engine.rs +++ b/src/engine/strat_engine/engine.rs @@ -29,7 +29,7 @@ use crate::{ }, structures::Table, types::{CreateAction, DeleteAction, RenameAction}, - Engine, EngineEvent, Name, Pool, PoolUuid, Redundancy, + Engine, EngineEvent, Name, Pool, PoolUuid, }, stratis::{ErrorEnum, StratisError, StratisResult}, }; From 7ddc6fa8d8515b4516d1e2b11bf83e60ec6c50f0 Mon Sep 17 00:00:00 2001 From: mulhern Date: Wed, 18 Dec 2019 09:31:26 -0500 Subject: [PATCH 052/109] Introduce local methods for device number and pool uuid in find_all() Interpret the results of these two methods appropriately for their context, i.e., either on the Stratis path or on the fallback path. find_all() can now only fail on an initial udev error, so make that as clear as possible. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/setup.rs | 145 ++++++++++++++------- 1 file changed, 97 insertions(+), 48 deletions(-) diff --git a/src/engine/strat_engine/backstore/setup.rs b/src/engine/strat_engine/backstore/setup.rs index bfc0b1f26b..0ff6633013 100644 --- a/src/engine/strat_engine/backstore/setup.rs +++ b/src/engine/strat_engine/backstore/setup.rs @@ -42,8 +42,68 @@ use crate::{ /// This fallback path is more expensive, because it must search all block /// devices via udev rather than just all Stratis block devices. /// +/// Omits any device that appears problematic in some way. +/// +/// Return an error only on a failure to construct or scan with a udev +/// enumerator. +/// /// Returns a map of pool uuids to a map of devices to devnodes for each pool. -pub fn find_all() -> StratisResult>> { +pub fn find_all() -> libudev::Result>> { + // A wrapper for obtaining the device number as a devicemapper Device + // which interprets both an error in obtaining the value and no value + // as an error, which it is in this context. + fn devnode_to_devno_wrapper(devnode: &Path) -> Result { + devnode_to_devno(devnode) + .map_err(|err| { + format!( + "encountered an error while trying to obtain device number for {}: {}", + devnode.display(), + err + ) + }) + .and_then(|maybe_devno| { + maybe_devno + .ok_or_else(|| { + format!("no device number found for device {}", devnode.display()) + }) + .map(Device::from) + }) + } + + // A wrapper around the metadata module's device_identifers method + // which also handles failure to open a device for reading. + // Returns an error if the device could not be opened for reading. + // Returns Ok(Err(...)) if there was an error while reading the + // Stratis identifiers from the device. + // Returns Ok(Ok(None)) if the identifers did not appear to be on + // the device. + fn device_identifiers_wrapper( + devnode: &Path, + ) -> Result, String>, String> { + OpenOptions::new() + .read(true) + .open(devnode) + .as_mut() + .map_err(|err| { + format!( + "device {} could not be opened for reading: {}", + devnode.display(), + err + ) + }) + .map(|f| { + device_identifiers(f) + .map_err(|err| { + format!( + "encountered an error while reading Stratis header for device {}: {}", + devnode.display(), + err + ) + }) + .map(|maybe_ids| maybe_ids.map(|(pool_uuid, _)| pool_uuid)) + }) + } + info!("Beginning initial search for Stratis block devices"); let pool_map = { let mut pool_map = HashMap::new(); @@ -73,30 +133,22 @@ pub fn find_all() -> StratisResult>> None })) { - if let Some(devno) = devnode_to_devno(&devnode)? { - if let Some((pool_uuid, _)) = match device_identifiers( - &mut OpenOptions::new().read(true).open(&devnode)?, - ) { - Ok(ids) => ids, - Err(err) => { - warn!("udev identified device {} as a Stratis block device, but there was an error when reading the Stratis header: {}", - devnode.display(), - err); + if let Some((pool_uuid, devno)) = match (devnode_to_devno_wrapper(&devnode), device_identifiers_wrapper(&devnode)) { + (Err(err), _) | (_, Err(err)) | (_, Ok(Err(err)))=> { + warn!("udev identified device {} as a Stratis device but {}, omitting the device from the set of devices to process", + devnode.display(), + err); None - } - } { - pool_map - .entry(pool_uuid) - .or_insert_with(HashMap::new) - .insert(Device::from(devno), devnode); - } else { - warn!("udev identified device {} as a Stratis block device but there appeared to be no Stratis metadata on the device", - devnode.display()) } - } else { - warn!("udev identified device {} as a Stratis block device but its device number could not be found", - devnode.display()) - } + (_, Ok(Ok(None))) => { + warn!("udev identified device {} as a Stratis device but there appeared to be no Stratis metadata on the device, omitting the device from the set of devices to process", + devnode.display()); + None + } + (Ok(devno), Ok(Ok(Some(pool_uuid)))) => Some((pool_uuid, devno)), + } { + pool_map.entry(pool_uuid).or_insert_with(HashMap::new).insert(devno, devnode); + } else {}; } pool_map @@ -146,35 +198,32 @@ pub fn find_all() -> StratisResult>> .filter_map(|i| i.devnode() .map(|d| d.to_path_buf()) .or_else(||{ - warn!("udev identified a device as a block device, but the udev entry for the device had no device node, omitting the the device from the set of devices to process"); + warn!("udev identified a device as a block device, but the udev entry for the device had no device node, omitting the device from the set of devices to process"); None })) { - if let Some(devno) = devnode_to_devno(&devnode)? { - if let Some((pool_uuid, _)) = match device_identifiers( - &mut OpenOptions::new().read(true).open(&devnode)?, - ) { - Ok(ids) => ids, - // FIXME: Refine error return in StaticHeader::setup(), - // so it can be used to distinguish between signficant - // and insignficant errors and then use that ability to - // distinguish here between different levels of severity. - Err(err) => { - debug!("Encountered an error while trying to get Stratis device identifiers from block device {}: {}", - devnode.display(), - err); - None - } - } { - pool_map - .entry(pool_uuid) - .or_insert_with(HashMap::new) - .insert(Device::from(devno), devnode.to_path_buf()); + if let Some((pool_uuid, devno)) = match (devnode_to_devno_wrapper(&devnode), device_identifiers_wrapper(&devnode)) { + (Err(err), _) | (_, Err(err)) => { + warn!("udev identified device {} as a block device but {}, omitting the device from the set of devices to process", + devnode.display(), + err); + None } - } else { - warn!("udev identified device {} as a block device but its device number could not be found", - devnode.display()) - } + // FIXME: Refine error return in StaticHeader::setup(), + // so it can be used to distinguish between signficant + // and insignficant errors and then use that ability to + // distinguish here between different levels of severity. + (_, Ok(Err(err))) => { + debug!("udev identified device {} as a block device but {}, omitting the device from the set of devices to process", + devnode.display(), + err); + None + } + (_, Ok(Ok(None))) => None, + (Ok(devno), Ok(Ok(Some(pool_uuid)))) => Some((pool_uuid, devno)), + } { + pool_map.entry(pool_uuid).or_insert_with(HashMap::new).insert(devno, devnode); + } else {}; } Ok(pool_map) } else { From 979e47e2a358cca79b44a69dda8770970b9ab9bb Mon Sep 17 00:00:00 2001 From: mulhern Date: Wed, 18 Dec 2019 12:41:12 -0500 Subject: [PATCH 053/109] Do both HashMap accumulating operations as a fold It makes the structure of both clearer. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/setup.rs | 99 ++++++++++------------ 1 file changed, 47 insertions(+), 52 deletions(-) diff --git a/src/engine/strat_engine/backstore/setup.rs b/src/engine/strat_engine/backstore/setup.rs index 0ff6633013..8c0d75eb80 100644 --- a/src/engine/strat_engine/backstore/setup.rs +++ b/src/engine/strat_engine/backstore/setup.rs @@ -106,12 +106,11 @@ pub fn find_all() -> libudev::Result> info!("Beginning initial search for Stratis block devices"); let pool_map = { - let mut pool_map = HashMap::new(); - let context = libudev::Context::new()?; let mut enumerator = block_enumerator(&context)?; enumerator.match_property("ID_FS_TYPE", "stratis")?; - for devnode in enumerator + + enumerator .scan_devices()? .filter(|dev| { let initialized = dev.is_initialized(); @@ -127,31 +126,28 @@ pub fn find_all() -> libudev::Result> }) .unwrap_or(true)) .filter_map(|i| i.devnode() - .map(|d| d.to_path_buf()) + .and_then(|devnode| match (devnode_to_devno_wrapper(devnode), device_identifiers_wrapper(devnode)) { + (Err(err), _) | (_, Err(err)) | (_, Ok(Err(err)))=> { + warn!("udev identified device {} as a Stratis device but {}, omitting the device from the set of devices to process", + devnode.display(), + err); + None + } + (_, Ok(Ok(None))) => { + warn!("udev identified device {} as a Stratis device but there appeared to be no Stratis metadata on the device, omitting the device from the set of devices to process", + devnode.display()); + None + } + (Ok(devno), Ok(Ok(Some(pool_uuid)))) => Some((pool_uuid, devno, devnode.to_path_buf())), + }) .or_else(||{ warn!("udev identified a device as a Stratis device, but the udev entry for the device had no device node, omitting the the device from the set of devices to process"); None })) - { - if let Some((pool_uuid, devno)) = match (devnode_to_devno_wrapper(&devnode), device_identifiers_wrapper(&devnode)) { - (Err(err), _) | (_, Err(err)) | (_, Ok(Err(err)))=> { - warn!("udev identified device {} as a Stratis device but {}, omitting the device from the set of devices to process", - devnode.display(), - err); - None - } - (_, Ok(Ok(None))) => { - warn!("udev identified device {} as a Stratis device but there appeared to be no Stratis metadata on the device, omitting the device from the set of devices to process", - devnode.display()); - None - } - (Ok(devno), Ok(Ok(Some(pool_uuid)))) => Some((pool_uuid, devno)), - } { - pool_map.entry(pool_uuid).or_insert_with(HashMap::new).insert(devno, devnode); - } else {}; - } - - pool_map + .fold(HashMap::new(), |mut acc, (pool_uuid, device, devnode)| { + acc.entry(pool_uuid).or_insert_with(HashMap::new).insert(device, devnode); + acc + }) }; if pool_map.is_empty() { @@ -170,11 +166,10 @@ pub fn find_all() -> libudev::Result> info!("Could not identify any Stratis devices by a udev search for devices with ID_FS_TYPE=\"stratis\"; using fallback search mechanism"); - let mut pool_map = HashMap::new(); - let context = libudev::Context::new()?; let mut enumerator = block_enumerator(&context)?; - for devnode in enumerator + + let pool_map = enumerator .scan_devices()? .filter(|dev| { let initialized = dev.is_initialized(); @@ -196,35 +191,35 @@ pub fn find_all() -> libudev::Result> .unwrap_or(false) }) .filter_map(|i| i.devnode() - .map(|d| d.to_path_buf()) + .and_then(|devnode| match (devnode_to_devno_wrapper(devnode), device_identifiers_wrapper(devnode)) { + (Err(err), _) | (_, Err(err)) => { + warn!("udev identified device {} as a block device but {}, omitting the device from the set of devices to process", + devnode.display(), + err); + None + } + // FIXME: Refine error return in + // StaticHeader::setup(), so it can be used to + // distinguish between signficant and insignficant + // errors and then use that ability to distinguish + // here between different levels of severity. + (_, Ok(Err(err))) => { + debug!("udev identified device {} as a block device but {}, omitting the device from the set of devices to process", + devnode.display(), + err); + None + } + (_, Ok(Ok(None))) => None, + (Ok(devno), Ok(Ok(Some(pool_uuid)))) => Some((pool_uuid, devno, devnode.to_path_buf())), + }) .or_else(||{ warn!("udev identified a device as a block device, but the udev entry for the device had no device node, omitting the device from the set of devices to process"); None })) - { - if let Some((pool_uuid, devno)) = match (devnode_to_devno_wrapper(&devnode), device_identifiers_wrapper(&devnode)) { - (Err(err), _) | (_, Err(err)) => { - warn!("udev identified device {} as a block device but {}, omitting the device from the set of devices to process", - devnode.display(), - err); - None - } - // FIXME: Refine error return in StaticHeader::setup(), - // so it can be used to distinguish between signficant - // and insignficant errors and then use that ability to - // distinguish here between different levels of severity. - (_, Ok(Err(err))) => { - debug!("udev identified device {} as a block device but {}, omitting the device from the set of devices to process", - devnode.display(), - err); - None - } - (_, Ok(Ok(None))) => None, - (Ok(devno), Ok(Ok(Some(pool_uuid)))) => Some((pool_uuid, devno)), - } { - pool_map.entry(pool_uuid).or_insert_with(HashMap::new).insert(devno, devnode); - } else {}; - } + .fold(HashMap::new(), |mut acc, (pool_uuid, device, devnode)| { + acc.entry(pool_uuid).or_insert_with(HashMap::new).insert(device, devnode); + acc + }); Ok(pool_map) } else { Ok(pool_map) From bb307203791577494caaa1890729dce0e44311a9 Mon Sep 17 00:00:00 2001 From: mulhern Date: Wed, 18 Dec 2019 13:13:30 -0500 Subject: [PATCH 054/109] Use match expressions in order to dedent in some closures Signed-off-by: mulhern --- src/engine/strat_engine/backstore/setup.rs | 92 ++++++++++++---------- 1 file changed, 49 insertions(+), 43 deletions(-) diff --git a/src/engine/strat_engine/backstore/setup.rs b/src/engine/strat_engine/backstore/setup.rs index 8c0d75eb80..5eee70a1f3 100644 --- a/src/engine/strat_engine/backstore/setup.rs +++ b/src/engine/strat_engine/backstore/setup.rs @@ -125,25 +125,28 @@ pub fn find_all() -> libudev::Result> err); }) .unwrap_or(true)) - .filter_map(|i| i.devnode() - .and_then(|devnode| match (devnode_to_devno_wrapper(devnode), device_identifiers_wrapper(devnode)) { - (Err(err), _) | (_, Err(err)) | (_, Ok(Err(err)))=> { - warn!("udev identified device {} as a Stratis device but {}, omitting the device from the set of devices to process", - devnode.display(), - err); - None - } - (_, Ok(Ok(None))) => { - warn!("udev identified device {} as a Stratis device but there appeared to be no Stratis metadata on the device, omitting the device from the set of devices to process", - devnode.display()); - None - } - (Ok(devno), Ok(Ok(Some(pool_uuid)))) => Some((pool_uuid, devno, devnode.to_path_buf())), - }) - .or_else(||{ - warn!("udev identified a device as a Stratis device, but the udev entry for the device had no device node, omitting the the device from the set of devices to process"); + .filter_map(|i| match i.devnode() { + Some(devnode) => { + match (devnode_to_devno_wrapper(devnode), device_identifiers_wrapper(devnode)) { + (Err(err), _) | (_, Err(err)) | (_, Ok(Err(err)))=> { + warn!("udev identified device {} as a Stratis device but {}, omitting the device from the set of devices to process", + devnode.display(), + err); None - })) + } + (_, Ok(Ok(None))) => { + warn!("udev identified device {} as a Stratis device but there appeared to be no Stratis metadata on the device, omitting the device from the set of devices to process", + devnode.display()); + None + } + (Ok(devno), Ok(Ok(Some(pool_uuid)))) => Some((pool_uuid, devno, devnode.to_path_buf())), + } + } + None => { + warn!("udev identified a device as a Stratis device, but the udev entry for the device had no device node, omitting the the device from the set of devices to process"); + None + } + }) .fold(HashMap::new(), |mut acc, (pool_uuid, device, devnode)| { acc.entry(pool_uuid).or_insert_with(HashMap::new).insert(device, devnode); acc @@ -190,32 +193,35 @@ pub fn find_all() -> libudev::Result> }) .unwrap_or(false) }) - .filter_map(|i| i.devnode() - .and_then(|devnode| match (devnode_to_devno_wrapper(devnode), device_identifiers_wrapper(devnode)) { - (Err(err), _) | (_, Err(err)) => { - warn!("udev identified device {} as a block device but {}, omitting the device from the set of devices to process", - devnode.display(), - err); - None - } - // FIXME: Refine error return in - // StaticHeader::setup(), so it can be used to - // distinguish between signficant and insignficant - // errors and then use that ability to distinguish - // here between different levels of severity. - (_, Ok(Err(err))) => { - debug!("udev identified device {} as a block device but {}, omitting the device from the set of devices to process", - devnode.display(), - err); - None - } - (_, Ok(Ok(None))) => None, - (Ok(devno), Ok(Ok(Some(pool_uuid)))) => Some((pool_uuid, devno, devnode.to_path_buf())), - }) - .or_else(||{ - warn!("udev identified a device as a block device, but the udev entry for the device had no device node, omitting the device from the set of devices to process"); + .filter_map(|i| match i.devnode() { + Some(devnode) => { + match (devnode_to_devno_wrapper(devnode), device_identifiers_wrapper(devnode)) { + (Err(err), _) | (_, Err(err)) => { + warn!("udev identified device {} as a block device but {}, omitting the device from the set of devices to process", + devnode.display(), + err); + None + } + // FIXME: Refine error return in StaticHeader::setup(), + // so it can be used to distinguish between signficant + // and insignficant errors and then use that ability to + // distinguish here between different levels of + // severity. + (_, Ok(Err(err))) => { + debug!("udev identified device {} as a block device but {}, omitting the device from the set of devices to process", + devnode.display(), + err); None - })) + } + (_, Ok(Ok(None))) => None, + (Ok(devno), Ok(Ok(Some(pool_uuid)))) => Some((pool_uuid, devno, devnode.to_path_buf())), + } + } + None => { + warn!("udev identified a device as a block device, but the udev entry for the device had no device node, omitting the device from the set of devices to process"); + None + } + }) .fold(HashMap::new(), |mut acc, (pool_uuid, device, devnode)| { acc.entry(pool_uuid).or_insert_with(HashMap::new).insert(device, devnode); acc From d1b9417a861e4304957774f2ffd3817fe0bf6597 Mon Sep 17 00:00:00 2001 From: mulhern Date: Fri, 20 Dec 2019 12:35:26 -0500 Subject: [PATCH 055/109] Fix a clippy error that is new in Rust 1.40.0 It was introduced in PR #1759; the beta clippy task failed as it should, but I failed to notice this happening. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/setup.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engine/strat_engine/backstore/setup.rs b/src/engine/strat_engine/backstore/setup.rs index 5eee70a1f3..95a3e99bca 100644 --- a/src/engine/strat_engine/backstore/setup.rs +++ b/src/engine/strat_engine/backstore/setup.rs @@ -79,7 +79,7 @@ pub fn find_all() -> libudev::Result> // the device. fn device_identifiers_wrapper( devnode: &Path, - ) -> Result, String>, String> { + ) -> Result, String>, String> { OpenOptions::new() .read(true) .open(devnode) From 8b037ed64d8ae81363ff7bf258e9f305230525df Mon Sep 17 00:00:00 2001 From: mulhern Date: Fri, 20 Dec 2019 17:28:15 -0500 Subject: [PATCH 056/109] Remove beta fmt task rustfmt has been keeping to its stable guarantee for a couple of releases and it seems like overkill to run this for just one extra line. Signed-off-by: mulhern --- .travis.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index d86d3cf5d1..736a674294 100644 --- a/.travis.yml +++ b/.travis.yml @@ -100,11 +100,6 @@ matrix: # * It should be an advisory, and should not gate our development. - rust: stable env: TASK=audit - # Run fmt on rust beta, to observe any upcoming formatting changes - - rust: beta - before_script: - - rustup component add rustfmt - env: TASK=fmt-travis branches: only: master From 9c1fa8c93439469e5faeb1fd1211df2d81ae7fb2 Mon Sep 17 00:00:00 2001 From: mulhern Date: Fri, 20 Dec 2019 14:54:21 -0500 Subject: [PATCH 057/109] Do not make beta clippy task an allowed failure The plan is to it mandatory most of the time, and to allow it to fail only immediately after a new Rust release if there is a failure then and until the failure is fixed. Signed-off-by: mulhern --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 736a674294..07f0589806 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,8 +16,6 @@ language: rust matrix: fast_finish: true allow_failures: - # Allow beta tasks to fail - - rust: beta # Allow audit task to fail - env: TASK=audit include: From 6d72d39d3757fdc65e29f1a2b736f47425405642 Mon Sep 17 00:00:00 2001 From: mulhern Date: Fri, 20 Dec 2019 16:19:53 -0500 Subject: [PATCH 058/109] Distinguish always allowed failures from intermittently allowed ones Just group the intermittently allowed failures. Signed-off-by: mulhern --- .travis.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 07f0589806..117f0fe022 100644 --- a/.travis.yml +++ b/.travis.yml @@ -87,12 +87,18 @@ matrix: - cd tests/client-dbus env: TASK=fmt-travis - # ALLOWED FAILURES - # Run clippy on rust beta, in order to be good Rustaceans + + # INTERMITTENTLY ALLOWED FAILURES + # Allowed if a failure occurs after a new Rust release until the + # failure is addressed. + # Run clippy on rust beta, in order to be good Rustaceans. - rust: beta before_script: - rustup component add clippy env: TASK=clippy + + + # ALLOWED FAILURES # Run audit on Rust stable. Make it an allowed failure, because: # * It takes 9 minutes, the longest of any task. # * It should be an advisory, and should not gate our development. From 8b5ea2739f386ec73a84850e94a6735d1b941d7c Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 23 Dec 2019 16:38:46 -0500 Subject: [PATCH 059/109] Get the device number from udev in find_all It makes no sense to use a devicemapper function which stats the device in find_all, but get the device number from the udev entry in the event-based part of the device discovery mechanism. I choose to consistently use udev in both contexts. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/setup.rs | 35 ++++++++-------------- 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/src/engine/strat_engine/backstore/setup.rs b/src/engine/strat_engine/backstore/setup.rs index 95a3e99bca..9ea64ddab3 100644 --- a/src/engine/strat_engine/backstore/setup.rs +++ b/src/engine/strat_engine/backstore/setup.rs @@ -15,7 +15,7 @@ use chrono::{DateTime, Utc}; use libudev; use serde_json; -use devicemapper::{devnode_to_devno, Device, Sectors}; +use devicemapper::{Device, Sectors}; use crate::{ engine::{ @@ -50,24 +50,13 @@ use crate::{ /// Returns a map of pool uuids to a map of devices to devnodes for each pool. pub fn find_all() -> libudev::Result>> { // A wrapper for obtaining the device number as a devicemapper Device - // which interprets both an error in obtaining the value and no value - // as an error, which it is in this context. - fn devnode_to_devno_wrapper(devnode: &Path) -> Result { - devnode_to_devno(devnode) - .map_err(|err| { - format!( - "encountered an error while trying to obtain device number for {}: {}", - devnode.display(), - err - ) - }) - .and_then(|maybe_devno| { - maybe_devno - .ok_or_else(|| { - format!("no device number found for device {}", devnode.display()) - }) - .map(Device::from) - }) + // which interprets absence of the value as an error, which it is in this + // context. + fn device_to_devno_wrapper(device: &libudev::Device) -> Result { + device + .devnum() + .ok_or_else(|| "udev entry did not contain a device number".into()) + .map(Device::from) } // A wrapper around the metadata module's device_identifers method @@ -125,9 +114,9 @@ pub fn find_all() -> libudev::Result> err); }) .unwrap_or(true)) - .filter_map(|i| match i.devnode() { + .filter_map(|dev| match dev.devnode() { Some(devnode) => { - match (devnode_to_devno_wrapper(devnode), device_identifiers_wrapper(devnode)) { + match (device_to_devno_wrapper(&dev), device_identifiers_wrapper(devnode)) { (Err(err), _) | (_, Err(err)) | (_, Ok(Err(err)))=> { warn!("udev identified device {} as a Stratis device but {}, omitting the device from the set of devices to process", devnode.display(), @@ -193,9 +182,9 @@ pub fn find_all() -> libudev::Result> }) .unwrap_or(false) }) - .filter_map(|i| match i.devnode() { + .filter_map(|dev| match dev.devnode() { Some(devnode) => { - match (devnode_to_devno_wrapper(devnode), device_identifiers_wrapper(devnode)) { + match (device_to_devno_wrapper(&dev), device_identifiers_wrapper(devnode)) { (Err(err), _) | (_, Err(err)) => { warn!("udev identified device {} as a block device but {}, omitting the device from the set of devices to process", devnode.display(), From ee67fd88a72c8bde2e1ab56af952b61cdb48525b Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Thu, 2 Jan 2020 12:20:59 -0500 Subject: [PATCH 060/109] Restructure dbus_api module This commit breaks the growing number of methods into their own submodules. The structure is as follows: * The first version in which a method appears is the interface submodule in which the method should be added * In the given interface submodule: * Methods generation objects to set up the API should go in api.rs * Methods that do the backend work with the engine for D-Bus methods should go in methods.rs * Methods that fetch properties via the D-Bus property mechanism (NOT the FetchProperties stratisd-specific interface) should go in props.rs * Methods that are shared across multiple interfaces should go in the shared.rs module next to all of the interface submodules --- src/dbus_api/api.rs | 306 ---------------- src/dbus_api/api/manager_2_0/api.rs | 59 ++++ src/dbus_api/api/manager_2_0/methods.rs | 148 ++++++++ src/dbus_api/api/manager_2_0/mod.rs | 3 + src/dbus_api/api/manager_2_0/props.rs | 16 + src/dbus_api/api/mod.rs | 39 ++ src/dbus_api/blockdev.rs | 334 ------------------ src/dbus_api/blockdev/blockdev_2_0/api.rs | 83 +++++ src/dbus_api/blockdev/blockdev_2_0/methods.rs | 71 ++++ src/dbus_api/blockdev/blockdev_2_0/mod.rs | 3 + src/dbus_api/blockdev/blockdev_2_0/props.rs | 75 ++++ .../blockdev/fetch_properties_2_0/api.rs | 34 ++ .../blockdev/fetch_properties_2_0/methods.rs | 62 ++++ .../blockdev/fetch_properties_2_0/mod.rs | 2 + src/dbus_api/blockdev/mod.rs | 62 ++++ src/dbus_api/blockdev/shared.rs | 56 +++ src/dbus_api/connection.rs | 110 ++++++ src/dbus_api/filesystem.rs | 308 ---------------- .../filesystem/fetch_properties_2_0/api.rs | 34 ++ .../fetch_properties_2_0/methods.rs | 64 ++++ .../filesystem/fetch_properties_2_0/mod.rs | 2 + src/dbus_api/filesystem/filesystem_2_0/api.rs | 65 ++++ .../filesystem/filesystem_2_0/methods.rs | 66 ++++ src/dbus_api/filesystem/filesystem_2_0/mod.rs | 3 + .../filesystem/filesystem_2_0/props.rs | 64 ++++ src/dbus_api/filesystem/mod.rs | 59 ++++ src/dbus_api/filesystem/shared.rs | 57 +++ src/dbus_api/mod.rs | 3 +- src/dbus_api/pool/fetch_properties_2_0/api.rs | 31 ++ .../pool/fetch_properties_2_0/methods.rs | 73 ++++ src/dbus_api/pool/fetch_properties_2_0/mod.rs | 2 + src/dbus_api/pool/mod.rs | 62 ++++ src/dbus_api/pool/pool_2_0/api.rs | 105 ++++++ .../{pool.rs => pool/pool_2_0/methods.rs} | 278 +-------------- src/dbus_api/pool/pool_2_0/mod.rs | 3 + src/dbus_api/pool/pool_2_0/props.rs | 39 ++ src/dbus_api/pool/shared.rs | 39 ++ 37 files changed, 1606 insertions(+), 1214 deletions(-) delete mode 100644 src/dbus_api/api.rs create mode 100644 src/dbus_api/api/manager_2_0/api.rs create mode 100644 src/dbus_api/api/manager_2_0/methods.rs create mode 100644 src/dbus_api/api/manager_2_0/mod.rs create mode 100644 src/dbus_api/api/manager_2_0/props.rs create mode 100644 src/dbus_api/api/mod.rs delete mode 100644 src/dbus_api/blockdev.rs create mode 100644 src/dbus_api/blockdev/blockdev_2_0/api.rs create mode 100644 src/dbus_api/blockdev/blockdev_2_0/methods.rs create mode 100644 src/dbus_api/blockdev/blockdev_2_0/mod.rs create mode 100644 src/dbus_api/blockdev/blockdev_2_0/props.rs create mode 100644 src/dbus_api/blockdev/fetch_properties_2_0/api.rs create mode 100644 src/dbus_api/blockdev/fetch_properties_2_0/methods.rs create mode 100644 src/dbus_api/blockdev/fetch_properties_2_0/mod.rs create mode 100644 src/dbus_api/blockdev/mod.rs create mode 100644 src/dbus_api/blockdev/shared.rs create mode 100644 src/dbus_api/connection.rs delete mode 100644 src/dbus_api/filesystem.rs create mode 100644 src/dbus_api/filesystem/fetch_properties_2_0/api.rs create mode 100644 src/dbus_api/filesystem/fetch_properties_2_0/methods.rs create mode 100644 src/dbus_api/filesystem/fetch_properties_2_0/mod.rs create mode 100644 src/dbus_api/filesystem/filesystem_2_0/api.rs create mode 100644 src/dbus_api/filesystem/filesystem_2_0/methods.rs create mode 100644 src/dbus_api/filesystem/filesystem_2_0/mod.rs create mode 100644 src/dbus_api/filesystem/filesystem_2_0/props.rs create mode 100644 src/dbus_api/filesystem/mod.rs create mode 100644 src/dbus_api/filesystem/shared.rs create mode 100644 src/dbus_api/pool/fetch_properties_2_0/api.rs create mode 100644 src/dbus_api/pool/fetch_properties_2_0/methods.rs create mode 100644 src/dbus_api/pool/fetch_properties_2_0/mod.rs create mode 100644 src/dbus_api/pool/mod.rs create mode 100644 src/dbus_api/pool/pool_2_0/api.rs rename src/dbus_api/{pool.rs => pool/pool_2_0/methods.rs} (52%) create mode 100644 src/dbus_api/pool/pool_2_0/mod.rs create mode 100644 src/dbus_api/pool/pool_2_0/props.rs create mode 100644 src/dbus_api/pool/shared.rs diff --git a/src/dbus_api/api.rs b/src/dbus_api/api.rs deleted file mode 100644 index f2d5df8dab..0000000000 --- a/src/dbus_api/api.rs +++ /dev/null @@ -1,306 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. - -use std::{cell::RefCell, path::Path, rc::Rc, vec::Vec}; - -use dbus::{ - self, - arg::{Array, IterAppend}, - tree::{ - Access, EmitsChangedSignal, Factory, MTFn, MethodErr, MethodInfo, MethodResult, PropInfo, - Tree, - }, - BusType, Connection, ConnectionItem, Message, NameFlag, -}; -use libc; - -use crate::{ - dbus_api::{ - blockdev::create_dbus_blockdev, - consts, - filesystem::create_dbus_filesystem, - pool::create_dbus_pool, - types::{DbusContext, DeferredAction, TData}, - util::{ - engine_to_dbus_err_tuple, get_next_arg, msg_code_ok, msg_string_ok, tuple_to_option, - }, - }, - engine::{CreateAction, DeleteAction, Engine, Pool, PoolUuid}, - stratis::VERSION, -}; - -fn create_pool(m: &MethodInfo, TData>) -> MethodResult { - let message: &Message = m.msg; - let mut iter = message.iter_init(); - - let name: &str = get_next_arg(&mut iter, 0)?; - let redundancy: (bool, u16) = get_next_arg(&mut iter, 1)?; - let devs: Array<&str, _> = get_next_arg(&mut iter, 2)?; - - let blockdevs = devs.map(|x| Path::new(x)).collect::>(); - - let object_path = m.path.get_name(); - let dbus_context = m.tree.get_data(); - let mut engine = dbus_context.engine.borrow_mut(); - let result = engine.create_pool(name, &blockdevs, tuple_to_option(redundancy)); - - let return_message = message.method_return(); - - let default_return: (bool, (dbus::Path<'static>, Vec>)) = - (false, (dbus::Path::default(), Vec::new())); - - let msg = match result { - Ok(pool_uuid_action) => { - let results = match pool_uuid_action { - CreateAction::Created(uuid) => { - let (_, pool) = get_mut_pool!(engine; uuid; default_return; return_message); - - let pool_object_path: dbus::Path = - create_dbus_pool(dbus_context, object_path.clone(), uuid, pool); - - let bd_paths = pool - .blockdevs_mut() - .into_iter() - .map(|(uuid, bd)| { - create_dbus_blockdev(dbus_context, pool_object_path.clone(), uuid, bd) - }) - .collect::>(); - (true, (pool_object_path, bd_paths)) - } - CreateAction::Identity => default_return, - }; - return_message.append3(results, msg_code_ok(), msg_string_ok()) - } - Err(x) => { - let (rc, rs) = engine_to_dbus_err_tuple(&x); - return_message.append3(default_return, rc, rs) - } - }; - Ok(vec![msg]) -} - -fn destroy_pool(m: &MethodInfo, TData>) -> MethodResult { - let message: &Message = m.msg; - let mut iter = message.iter_init(); - - let pool_path: dbus::Path<'static> = get_next_arg(&mut iter, 0)?; - - let dbus_context = m.tree.get_data(); - - let default_return = (false, uuid_to_string!(PoolUuid::nil())); - let return_message = message.method_return(); - - let pool_uuid = match m - .tree - .get(&pool_path) - .and_then(|op| op.get_data().as_ref()) - .map(|d| d.uuid) - { - Some(uuid) => uuid, - None => { - return Ok(vec![return_message.append3( - default_return, - msg_code_ok(), - msg_string_ok(), - )]); - } - }; - - let msg = match dbus_context.engine.borrow_mut().destroy_pool(pool_uuid) { - Ok(DeleteAction::Deleted(uuid)) => { - dbus_context - .actions - .borrow_mut() - .push_remove(&pool_path, m.tree); - return_message.append3( - (true, uuid_to_string!(uuid)), - msg_code_ok(), - msg_string_ok(), - ) - } - Ok(DeleteAction::Identity) => { - return_message.append3(default_return, msg_code_ok(), msg_string_ok()) - } - Err(err) => { - let (rc, rs) = engine_to_dbus_err_tuple(&err); - return_message.append3(default_return, rc, rs) - } - }; - Ok(vec![msg]) -} - -fn get_version(i: &mut IterAppend, _p: &PropInfo, TData>) -> Result<(), MethodErr> { - i.append(VERSION); - Ok(()) -} - -fn configure_simulator(m: &MethodInfo, TData>) -> MethodResult { - let message = m.msg; - let mut iter = message.iter_init(); - - let denominator: u32 = get_next_arg(&mut iter, 0)?; - - let dbus_context = m.tree.get_data(); - let result = dbus_context - .engine - .borrow_mut() - .configure_simulator(denominator); - - let return_message = message.method_return(); - - let msg = match result { - Ok(_) => return_message.append2(msg_code_ok(), msg_string_ok()), - Err(err) => { - let (rc, rs) = engine_to_dbus_err_tuple(&err); - return_message.append2(rc, rs) - } - }; - Ok(vec![msg]) -} - -fn get_base_tree<'a>(dbus_context: DbusContext) -> (Tree, TData>, dbus::Path<'a>) { - let f = Factory::new_fn(); - - let base_tree = f.tree(dbus_context); - - let create_pool_method = f - .method("CreatePool", (), create_pool) - .in_arg(("name", "s")) - .in_arg(("redundancy", "(bq)")) - .in_arg(("devices", "as")) - // In order from left to right: - // b: true if a pool was created and object paths were returned - // o: Object path for Pool - // a(o): Array of object paths for block devices - // - // Rust representation: (bool, (dbus::Path, Vec)) - .out_arg(("result", "(b(oao))")) - .out_arg(("return_code", "q")) - .out_arg(("return_string", "s")); - - let destroy_pool_method = f - .method("DestroyPool", (), destroy_pool) - .in_arg(("pool", "o")) - // In order from left to right: - // b: true if a valid UUID is returned - otherwise no action was performed - // s: String representation of pool UUID that was destroyed - // - // Rust representation: (bool, String) - .out_arg(("result", "(bs)")) - .out_arg(("return_code", "q")) - .out_arg(("return_string", "s")); - - let configure_simulator_method = f - .method("ConfigureSimulator", (), configure_simulator) - .in_arg(("denominator", "u")) - .out_arg(("return_code", "q")) - .out_arg(("return_string", "s")); - - let version_property = f - .property::<&str, _>("Version", ()) - .access(Access::Read) - .emits_changed(EmitsChangedSignal::Const) - .on_get(get_version); - - let obj_path = f - .object_path(consts::STRATIS_BASE_PATH, None) - .introspectable() - .object_manager() - .add( - f.interface(consts::MANAGER_INTERFACE_NAME, ()) - .add_m(create_pool_method) - .add_m(destroy_pool_method) - .add_m(configure_simulator_method) - .add_p(version_property), - ); - - let path = obj_path.get_name().to_owned(); - (base_tree.add(obj_path), path) -} - -/// Returned data from when you connect a stratis engine to dbus. -pub struct DbusConnectionData { - pub connection: Rc>, - pub tree: Tree, TData>, - pub path: dbus::Path<'static>, - pub context: DbusContext, -} - -impl DbusConnectionData { - /// Connect a stratis engine to dbus. - pub fn connect(engine: Rc>) -> Result { - let c = Connection::get_private(BusType::System)?; - let (tree, object_path) = get_base_tree(DbusContext::new(engine)); - let dbus_context = tree.get_data().clone(); - tree.set_registered(&c, true)?; - c.register_name( - consts::STRATIS_BASE_SERVICE, - NameFlag::ReplaceExisting as u32, - )?; - Ok(DbusConnectionData { - connection: Rc::new(RefCell::new(c)), - tree, - path: object_path, - context: dbus_context, - }) - } - - /// Given the UUID of a pool, register all the pertinent information with dbus. - pub fn register_pool(&mut self, pool_uuid: PoolUuid, pool: &mut dyn Pool) { - let pool_path = create_dbus_pool(&self.context, self.path.clone(), pool_uuid, pool); - for (_, fs_uuid, fs) in pool.filesystems_mut() { - create_dbus_filesystem(&self.context, pool_path.clone(), fs_uuid, fs); - } - for (uuid, bd) in pool.blockdevs_mut() { - create_dbus_blockdev(&self.context, pool_path.clone(), uuid, bd); - } - - self.process_deferred_actions() - } - - /// Update the dbus tree with deferred adds and removes. - fn process_deferred_actions(&mut self) { - let mut actions = self.context.actions.borrow_mut(); - for action in actions.drain() { - match action { - DeferredAction::Add(path) => { - self.connection - .borrow_mut() - .register_object_path(path.get_name()) - .expect("Must succeed since object paths are unique"); - self.tree.insert(path); - } - DeferredAction::Remove(path) => { - self.connection.borrow_mut().unregister_object_path(&path); - self.tree.remove(&path); - } - } - } - } - - /// Handle any client dbus requests - pub fn handle(&mut self, fds: &[libc::pollfd]) { - for pfd in fds.iter().filter(|pfd| pfd.revents != 0) { - let items: Vec = self - .connection - .borrow() - .watch_handle(pfd.fd, dbus::WatchEvent::from_revents(pfd.revents)) - .collect(); - - for item in items { - if let ConnectionItem::MethodCall(ref msg) = item { - if let Some(v) = self.tree.handle(msg) { - // Probably the wisest is to ignore any send errors here - - // maybe the remote has disconnected during our processing. - for m in v { - let _ = self.connection.borrow_mut().send(m); - } - } - - self.process_deferred_actions(); - } - } - } - } -} diff --git a/src/dbus_api/api/manager_2_0/api.rs b/src/dbus_api/api/manager_2_0/api.rs new file mode 100644 index 0000000000..71cb75dc64 --- /dev/null +++ b/src/dbus_api/api/manager_2_0/api.rs @@ -0,0 +1,59 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use dbus::{ + self, + tree::{Access, EmitsChangedSignal, Factory, MTFn, Method, Property}, +}; + +use crate::dbus_api::{ + api::manager_2_0::{ + methods::{configure_simulator, create_pool, destroy_pool}, + props::get_version, + }, + types::TData, +}; + +pub fn create_pool_method(f: &Factory, TData>) -> Method, TData> { + f.method("CreatePool", (), create_pool) + .in_arg(("name", "s")) + .in_arg(("redundancy", "(bq)")) + .in_arg(("devices", "as")) + // In order from left to right: + // b: true if a pool was created and object paths were returned + // o: Object path for Pool + // a(o): Array of object paths for block devices + // + // Rust representation: (bool, (dbus::Path, Vec)) + .out_arg(("result", "(b(oao))")) + .out_arg(("return_code", "q")) + .out_arg(("return_string", "s")) +} + +pub fn destroy_pool_method(f: &Factory, TData>) -> Method, TData> { + f.method("DestroyPool", (), destroy_pool) + .in_arg(("pool", "o")) + // In order from left to right: + // b: true if a valid UUID is returned - otherwise no action was performed + // s: String representation of pool UUID that was destroyed + // + // Rust representation: (bool, String) + .out_arg(("result", "(bs)")) + .out_arg(("return_code", "q")) + .out_arg(("return_string", "s")) +} + +pub fn configure_simulator_method(f: &Factory, TData>) -> Method, TData> { + f.method("ConfigureSimulator", (), configure_simulator) + .in_arg(("denominator", "u")) + .out_arg(("return_code", "q")) + .out_arg(("return_string", "s")) +} + +pub fn version_property(f: &Factory, TData>) -> Property, TData> { + f.property::<&str, _>("Version", ()) + .access(Access::Read) + .emits_changed(EmitsChangedSignal::Const) + .on_get(get_version) +} diff --git a/src/dbus_api/api/manager_2_0/methods.rs b/src/dbus_api/api/manager_2_0/methods.rs new file mode 100644 index 0000000000..ed8d05c0f1 --- /dev/null +++ b/src/dbus_api/api/manager_2_0/methods.rs @@ -0,0 +1,148 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use std::{path::Path, vec::Vec}; + +use dbus::{ + self, + arg::Array, + tree::{MTFn, MethodInfo, MethodResult}, + Message, +}; + +use crate::{ + dbus_api::{ + blockdev::create_dbus_blockdev, + pool::create_dbus_pool, + types::TData, + util::{ + engine_to_dbus_err_tuple, get_next_arg, msg_code_ok, msg_string_ok, tuple_to_option, + }, + }, + engine::{CreateAction, DeleteAction, PoolUuid}, +}; + +pub fn create_pool(m: &MethodInfo, TData>) -> MethodResult { + let message: &Message = m.msg; + let mut iter = message.iter_init(); + + let name: &str = get_next_arg(&mut iter, 0)?; + let redundancy: (bool, u16) = get_next_arg(&mut iter, 1)?; + let devs: Array<&str, _> = get_next_arg(&mut iter, 2)?; + + let blockdevs = devs.map(|x| Path::new(x)).collect::>(); + + let object_path = m.path.get_name(); + let dbus_context = m.tree.get_data(); + let mut engine = dbus_context.engine.borrow_mut(); + let result = engine.create_pool(name, &blockdevs, tuple_to_option(redundancy)); + + let return_message = message.method_return(); + + let default_return: (bool, (dbus::Path<'static>, Vec>)) = + (false, (dbus::Path::default(), Vec::new())); + + let msg = match result { + Ok(pool_uuid_action) => { + let results = match pool_uuid_action { + CreateAction::Created(uuid) => { + let (_, pool) = get_mut_pool!(engine; uuid; default_return; return_message); + + let pool_object_path: dbus::Path = + create_dbus_pool(dbus_context, object_path.clone(), uuid, pool); + + let bd_paths = pool + .blockdevs_mut() + .into_iter() + .map(|(uuid, bd)| { + create_dbus_blockdev(dbus_context, pool_object_path.clone(), uuid, bd) + }) + .collect::>(); + (true, (pool_object_path, bd_paths)) + } + CreateAction::Identity => default_return, + }; + return_message.append3(results, msg_code_ok(), msg_string_ok()) + } + Err(x) => { + let (rc, rs) = engine_to_dbus_err_tuple(&x); + return_message.append3(default_return, rc, rs) + } + }; + Ok(vec![msg]) +} + +pub fn destroy_pool(m: &MethodInfo, TData>) -> MethodResult { + let message: &Message = m.msg; + let mut iter = message.iter_init(); + + let pool_path: dbus::Path<'static> = get_next_arg(&mut iter, 0)?; + + let dbus_context = m.tree.get_data(); + + let default_return = (false, uuid_to_string!(PoolUuid::nil())); + let return_message = message.method_return(); + + let pool_uuid = match m + .tree + .get(&pool_path) + .and_then(|op| op.get_data().as_ref()) + .map(|d| d.uuid) + { + Some(uuid) => uuid, + None => { + return Ok(vec![return_message.append3( + default_return, + msg_code_ok(), + msg_string_ok(), + )]); + } + }; + + let msg = match dbus_context.engine.borrow_mut().destroy_pool(pool_uuid) { + Ok(DeleteAction::Deleted(uuid)) => { + dbus_context + .actions + .borrow_mut() + .push_remove(&pool_path, m.tree); + return_message.append3( + (true, uuid_to_string!(uuid)), + msg_code_ok(), + msg_string_ok(), + ) + } + Ok(DeleteAction::Identity) => { + return_message.append3(default_return, msg_code_ok(), msg_string_ok()) + } + Err(err) => { + let (rc, rs) = engine_to_dbus_err_tuple(&err); + return_message.append3(default_return, rc, rs) + } + }; + Ok(vec![msg]) +} + +pub fn configure_simulator(m: &MethodInfo, TData>) -> MethodResult { + let message = m.msg; + let mut iter = message.iter_init(); + + let denominator: u32 = get_next_arg(&mut iter, 0)?; + + let dbus_context = m.tree.get_data(); + let result = dbus_context + .engine + .borrow_mut() + .configure_simulator(denominator); + + let return_message = message.method_return(); + + let msg = match result { + Ok(_) => return_message.append2(msg_code_ok(), msg_string_ok()), + Err(err) => { + let (rc, rs) = engine_to_dbus_err_tuple(&err); + return_message.append2(rc, rs) + } + }; + Ok(vec![msg]) +} diff --git a/src/dbus_api/api/manager_2_0/mod.rs b/src/dbus_api/api/manager_2_0/mod.rs new file mode 100644 index 0000000000..333cf7108f --- /dev/null +++ b/src/dbus_api/api/manager_2_0/mod.rs @@ -0,0 +1,3 @@ +pub mod api; +pub mod methods; +pub mod props; diff --git a/src/dbus_api/api/manager_2_0/props.rs b/src/dbus_api/api/manager_2_0/props.rs new file mode 100644 index 0000000000..5c73d9f216 --- /dev/null +++ b/src/dbus_api/api/manager_2_0/props.rs @@ -0,0 +1,16 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use dbus::{ + self, + arg::IterAppend, + tree::{MTFn, MethodErr, PropInfo}, +}; + +use crate::{dbus_api::types::TData, stratis::VERSION}; + +pub fn get_version(i: &mut IterAppend, _p: &PropInfo, TData>) -> Result<(), MethodErr> { + i.append(VERSION); + Ok(()) +} diff --git a/src/dbus_api/api/mod.rs b/src/dbus_api/api/mod.rs new file mode 100644 index 0000000000..0106a68a32 --- /dev/null +++ b/src/dbus_api/api/mod.rs @@ -0,0 +1,39 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use dbus::{ + self, + tree::{Factory, MTFn, Tree}, +}; + +use crate::dbus_api::{ + api::manager_2_0::api::{ + configure_simulator_method, create_pool_method, destroy_pool_method, version_property, + }, + consts, + types::{DbusContext, TData}, +}; + +pub mod manager_2_0; + +pub fn get_base_tree<'a>(dbus_context: DbusContext) -> (Tree, TData>, dbus::Path<'a>) { + let f = Factory::new_fn(); + + let base_tree = f.tree(dbus_context); + + let obj_path = f + .object_path(consts::STRATIS_BASE_PATH, None) + .introspectable() + .object_manager() + .add( + f.interface(consts::MANAGER_INTERFACE_NAME, ()) + .add_m(create_pool_method(&f)) + .add_m(destroy_pool_method(&f)) + .add_m(configure_simulator_method(&f)) + .add_p(version_property(&f)), + ); + + let path = obj_path.get_name().to_owned(); + (base_tree.add(obj_path), path) +} diff --git a/src/dbus_api/blockdev.rs b/src/dbus_api/blockdev.rs deleted file mode 100644 index 49e72f98dc..0000000000 --- a/src/dbus_api/blockdev.rs +++ /dev/null @@ -1,334 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. - -use std::collections::HashMap; - -use dbus::{ - self, - arg::{Array, IterAppend, RefArg, Variant}, - tree::{ - Access, EmitsChangedSignal, Factory, MTFn, MethodErr, MethodInfo, MethodResult, PropInfo, - Tree, - }, - Message, Path, -}; -use itertools::Itertools; -use uuid::Uuid; - -use crate::{ - dbus_api::{ - consts, - types::{DbusContext, DbusErrorEnum, OPContext, TData}, - util::{ - engine_to_dbus_err_tuple, get_next_arg, get_parent, get_uuid, make_object_path, - msg_code_ok, msg_string_ok, result_to_tuple, tuple_to_option, - }, - }, - engine::{BlockDev, BlockDevTier, DevUuid, MaybeDbusPath, RenameAction}, -}; - -pub fn create_dbus_blockdev<'a>( - dbus_context: &DbusContext, - parent: dbus::Path<'static>, - uuid: Uuid, - blockdev: &mut dyn BlockDev, -) -> dbus::Path<'a> { - let f = Factory::new_fn(); - - let set_userid_method = f - .method("SetUserInfo", (), set_user_info) - .in_arg(("id", "(bs)")) - // b: false if no change to the user info - // s: UUID of the changed device - // - // Rust representation: (bool, String) - .out_arg(("changed", "(bs)")) - .out_arg(("return_code", "q")) - .out_arg(("return_string", "s")); - - let devnode_property = f - .property::<&str, _>("Devnode", ()) - .access(Access::Read) - .emits_changed(EmitsChangedSignal::Const) - .on_get(get_blockdev_devnode); - - let hardware_info_property = f - .property::<(bool, &str), _>("HardwareInfo", ()) - .access(Access::Read) - .emits_changed(EmitsChangedSignal::Const) - .on_get(get_blockdev_hardware_info); - - let user_info_property = f - .property::<(bool, &str), _>("UserInfo", ()) - .access(Access::Read) - .emits_changed(EmitsChangedSignal::False) - .on_get(get_blockdev_user_info); - - let initialization_time_property = f - .property::("InitializationTime", ()) - .access(Access::Read) - .emits_changed(EmitsChangedSignal::Const) - .on_get(get_blockdev_initialization_time); - - let pool_property = f - .property::<&dbus::Path, _>("Pool", ()) - .access(Access::Read) - .emits_changed(EmitsChangedSignal::Const) - .on_get(get_parent); - - let uuid_property = f - .property::<&str, _>("Uuid", ()) - .access(Access::Read) - .emits_changed(EmitsChangedSignal::Const) - .on_get(get_uuid); - - let tier_property = f - .property::("Tier", ()) - .access(Access::Read) - .emits_changed(EmitsChangedSignal::False) - .on_get(get_blockdev_tier); - - let get_all_properties_method = f - .method("GetAllProperties", (), get_all_properties) - // a{s(bv)}: Dictionary of property names to tuples - // In the tuple: - // b: Indicates whether the property value fetched was successful - // v: If b is true, represents the value for the given property - // If b is false, represents the error returned when fetching the property - .out_arg(("results", "a{s(bv)}")); - - let get_properties_method = f - .method("GetProperties", (), get_properties) - .in_arg(("properties", "as")) - // a{s(bv)}: Dictionary of property names to tuples - // In the tuple: - // b: Indicates whether the property value fetched was successful - // v: If b is true, represents the value for the given property - // If b is false, represents the error returned when fetching the property - .out_arg(("results", "a{s(bv)}")); - - let object_name = make_object_path(dbus_context); - - let object_path = f - .object_path(object_name, Some(OPContext::new(parent, uuid))) - .introspectable() - .add( - f.interface(consts::BLOCKDEV_INTERFACE_NAME, ()) - .add_m(set_userid_method) - .add_p(devnode_property) - .add_p(hardware_info_property) - .add_p(initialization_time_property) - .add_p(pool_property) - .add_p(tier_property) - .add_p(user_info_property) - .add_p(uuid_property), - ) - .add( - f.interface(consts::PROPERTY_FETCH_INTERFACE_NAME, ()) - .add_m(get_all_properties_method) - .add_m(get_properties_method), - ); - - let path = object_path.get_name().to_owned(); - dbus_context.actions.borrow_mut().push_add(object_path); - blockdev.set_dbus_path(MaybeDbusPath(Some(path.clone()))); - path -} - -fn set_user_info(m: &MethodInfo, TData>) -> MethodResult { - let message: &Message = m.msg; - let mut iter = message.iter_init(); - - let new_id_spec: (bool, &str) = get_next_arg(&mut iter, 0)?; - - let dbus_context = m.tree.get_data(); - let object_path = m.path.get_name(); - let return_message = message.method_return(); - let default_return = (false, uuid_to_string!(DevUuid::nil())); - - let blockdev_path = m - .tree - .get(object_path) - .expect("implicit argument must be in tree"); - let blockdev_data = get_data!(blockdev_path; default_return; return_message); - - let pool_path = get_parent!(m; blockdev_data; default_return; return_message); - let pool_uuid = get_data!(pool_path; default_return; return_message).uuid; - - let mut engine = dbus_context.engine.borrow_mut(); - let (pool_name, pool) = get_mut_pool!(engine; pool_uuid; default_return; return_message); - - let result = - pool.set_blockdev_user_info(&pool_name, blockdev_data.uuid, tuple_to_option(new_id_spec)); - - let msg = match result { - Ok(RenameAction::NoSource) => { - let error_message = format!( - "pool doesn't know about block device {}", - blockdev_data.uuid - ); - let (rc, rs) = (DbusErrorEnum::INTERNAL_ERROR as u16, error_message); - return_message.append3(default_return, rc, rs) - } - Ok(RenameAction::Renamed(uuid)) => return_message.append3( - (true, uuid_to_string!(uuid)), - msg_code_ok(), - msg_string_ok(), - ), - Ok(RenameAction::Identity) => { - return_message.append3(default_return, msg_code_ok(), msg_string_ok()) - } - Err(err) => { - let (rc, rs) = engine_to_dbus_err_tuple(&err); - return_message.append3(default_return, rc, rs) - } - }; - - Ok(vec![msg]) -} - -fn get_properties_shared( - m: &MethodInfo, TData>, - properties: &mut dyn Iterator, -) -> MethodResult { - let message: &Message = m.msg; - let object_path = &m.path; - - let return_message = message.method_return(); - - let return_value: HashMap>)> = properties - .unique() - .filter_map(|prop| match prop.as_str() { - consts::BLOCKDEV_TOTAL_SIZE_PROP => Some(( - prop, - blockdev_operation(m.tree, object_path.get_name(), |_, bd| { - Ok((u128::from(*bd.size()) * devicemapper::SECTOR_SIZE as u128).to_string()) - }), - )), - _ => None, - }) - .map(|(key, result)| result_to_tuple(key, result)) - .collect(); - - Ok(vec![return_message.append1(return_value)]) -} - -fn get_all_properties(m: &MethodInfo, TData>) -> MethodResult { - get_properties_shared( - m, - &mut vec![consts::BLOCKDEV_TOTAL_SIZE_PROP] - .into_iter() - .map(|s| s.to_string()), - ) -} - -fn get_properties(m: &MethodInfo, TData>) -> MethodResult { - let message: &Message = m.msg; - let mut iter = message.iter_init(); - let mut properties: Array = get_next_arg(&mut iter, 0)?; - get_properties_shared(m, &mut properties) -} - -/// Perform an operation on a `BlockDev` object for a given -/// DBus implicit argument that is a block device -fn blockdev_operation( - tree: &Tree, TData>, - object_path: &Path<'static>, - closure: F, -) -> Result -where - F: Fn(BlockDevTier, &dyn BlockDev) -> Result, - R: dbus::arg::Append, -{ - let dbus_context = tree.get_data(); - - let blockdev_path = tree - .get(object_path) - .expect("tree must contain implicit argument"); - - let blockdev_data = blockdev_path - .get_data() - .as_ref() - .ok_or_else(|| format!("no data for object path {}", object_path))?; - - let pool_path = tree - .get(&blockdev_data.parent) - .ok_or_else(|| format!("no path for parent object path {}", &blockdev_data.parent))?; - - let pool_uuid = pool_path - .get_data() - .as_ref() - .ok_or_else(|| format!("no data for object path {}", object_path))? - .uuid; - - let engine = dbus_context.engine.borrow(); - let (_, pool) = engine - .get_pool(pool_uuid) - .ok_or_else(|| format!("no pool corresponding to uuid {}", &pool_uuid))?; - let (tier, blockdev) = pool - .get_blockdev(blockdev_data.uuid) - .ok_or_else(|| format!("no blockdev with uuid {}", blockdev_data.uuid))?; - closure(tier, blockdev) -} - -/// Get a blockdev property and place it on the D-Bus. The property is -/// found by means of the getter method which takes a reference to a -/// blockdev and obtains the property from the blockdev. -fn get_blockdev_property( - i: &mut IterAppend, - p: &PropInfo, TData>, - getter: F, -) -> Result<(), MethodErr> -where - F: Fn(BlockDevTier, &dyn BlockDev) -> Result, - R: dbus::arg::Append, -{ - i.append( - blockdev_operation(p.tree, p.path.get_name(), getter) - .map_err(|ref e| MethodErr::failed(e))?, - ); - Ok(()) -} - -/// Get the devnode for an object path. -fn get_blockdev_devnode( - i: &mut IterAppend, - p: &PropInfo, TData>, -) -> Result<(), MethodErr> { - get_blockdev_property(i, p, |_, p| Ok(format!("{}", p.devnode().display()))) -} - -fn get_blockdev_hardware_info( - i: &mut IterAppend, - p: &PropInfo, TData>, -) -> Result<(), MethodErr> { - get_blockdev_property(i, p, |_, p| { - Ok(p.hardware_info() - .map_or_else(|| (false, "".to_owned()), |val| (true, val.to_owned()))) - }) -} - -fn get_blockdev_user_info( - i: &mut IterAppend, - p: &PropInfo, TData>, -) -> Result<(), MethodErr> { - get_blockdev_property(i, p, |_, p| { - Ok(p.user_info() - .map_or_else(|| (false, "".to_owned()), |val| (true, val.to_owned()))) - }) -} - -fn get_blockdev_initialization_time( - i: &mut IterAppend, - p: &PropInfo, TData>, -) -> Result<(), MethodErr> { - get_blockdev_property(i, p, |_, p| Ok(p.initialization_time().timestamp() as u64)) -} - -fn get_blockdev_tier( - i: &mut IterAppend, - p: &PropInfo, TData>, -) -> Result<(), MethodErr> { - get_blockdev_property(i, p, |t, _| Ok(t as u16)) -} diff --git a/src/dbus_api/blockdev/blockdev_2_0/api.rs b/src/dbus_api/blockdev/blockdev_2_0/api.rs new file mode 100644 index 0000000000..1c0c720fd6 --- /dev/null +++ b/src/dbus_api/blockdev/blockdev_2_0/api.rs @@ -0,0 +1,83 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use dbus::{ + self, + tree::{Access, EmitsChangedSignal, Factory, MTFn, Method, Property}, +}; + +use crate::dbus_api::{ + blockdev::blockdev_2_0::{ + methods::set_user_info, + props::{ + get_blockdev_devnode, get_blockdev_hardware_info, get_blockdev_initialization_time, + get_blockdev_tier, get_blockdev_user_info, + }, + }, + types::TData, + util::{get_parent, get_uuid}, +}; + +pub fn set_userid_method(f: &Factory, TData>) -> Method, TData> { + f.method("SetUserInfo", (), set_user_info) + .in_arg(("id", "(bs)")) + // b: false if no change to the user info + // s: UUID of the changed device + // + // Rust representation: (bool, String) + .out_arg(("changed", "(bs)")) + .out_arg(("return_code", "q")) + .out_arg(("return_string", "s")) +} + +pub fn devnode_property(f: &Factory, TData>) -> Property, TData> { + f.property::<&str, _>("Devnode", ()) + .access(Access::Read) + .emits_changed(EmitsChangedSignal::Const) + .on_get(get_blockdev_devnode) +} + +pub fn hardware_info_property(f: &Factory, TData>) -> Property, TData> { + f.property::<(bool, &str), _>("HardwareInfo", ()) + .access(Access::Read) + .emits_changed(EmitsChangedSignal::Const) + .on_get(get_blockdev_hardware_info) +} + +pub fn user_info_property(f: &Factory, TData>) -> Property, TData> { + f.property::<(bool, &str), _>("UserInfo", ()) + .access(Access::Read) + .emits_changed(EmitsChangedSignal::False) + .on_get(get_blockdev_user_info) +} + +pub fn initialization_time_property( + f: &Factory, TData>, +) -> Property, TData> { + f.property::("InitializationTime", ()) + .access(Access::Read) + .emits_changed(EmitsChangedSignal::Const) + .on_get(get_blockdev_initialization_time) +} + +pub fn pool_property(f: &Factory, TData>) -> Property, TData> { + f.property::<&dbus::Path, _>("Pool", ()) + .access(Access::Read) + .emits_changed(EmitsChangedSignal::Const) + .on_get(get_parent) +} + +pub fn uuid_property(f: &Factory, TData>) -> Property, TData> { + f.property::<&str, _>("Uuid", ()) + .access(Access::Read) + .emits_changed(EmitsChangedSignal::Const) + .on_get(get_uuid) +} + +pub fn tier_property(f: &Factory, TData>) -> Property, TData> { + f.property::("Tier", ()) + .access(Access::Read) + .emits_changed(EmitsChangedSignal::False) + .on_get(get_blockdev_tier) +} diff --git a/src/dbus_api/blockdev/blockdev_2_0/methods.rs b/src/dbus_api/blockdev/blockdev_2_0/methods.rs new file mode 100644 index 0000000000..7923e1ecd5 --- /dev/null +++ b/src/dbus_api/blockdev/blockdev_2_0/methods.rs @@ -0,0 +1,71 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use dbus::{ + self, + tree::{MTFn, MethodInfo, MethodResult}, + Message, +}; + +use crate::{ + dbus_api::{ + types::{DbusErrorEnum, TData}, + util::{ + engine_to_dbus_err_tuple, get_next_arg, msg_code_ok, msg_string_ok, tuple_to_option, + }, + }, + engine::{DevUuid, RenameAction}, +}; + +pub fn set_user_info(m: &MethodInfo, TData>) -> MethodResult { + let message: &Message = m.msg; + let mut iter = message.iter_init(); + + let new_id_spec: (bool, &str) = get_next_arg(&mut iter, 0)?; + + let dbus_context = m.tree.get_data(); + let object_path = m.path.get_name(); + let return_message = message.method_return(); + let default_return = (false, uuid_to_string!(DevUuid::nil())); + + let blockdev_path = m + .tree + .get(object_path) + .expect("implicit argument must be in tree"); + let blockdev_data = get_data!(blockdev_path; default_return; return_message); + + let pool_path = get_parent!(m; blockdev_data; default_return; return_message); + let pool_uuid = get_data!(pool_path; default_return; return_message).uuid; + + let mut engine = dbus_context.engine.borrow_mut(); + let (pool_name, pool) = get_mut_pool!(engine; pool_uuid; default_return; return_message); + + let result = + pool.set_blockdev_user_info(&pool_name, blockdev_data.uuid, tuple_to_option(new_id_spec)); + + let msg = match result { + Ok(RenameAction::NoSource) => { + let error_message = format!( + "pool doesn't know about block device {}", + blockdev_data.uuid + ); + let (rc, rs) = (DbusErrorEnum::INTERNAL_ERROR as u16, error_message); + return_message.append3(default_return, rc, rs) + } + Ok(RenameAction::Renamed(uuid)) => return_message.append3( + (true, uuid_to_string!(uuid)), + msg_code_ok(), + msg_string_ok(), + ), + Ok(RenameAction::Identity) => { + return_message.append3(default_return, msg_code_ok(), msg_string_ok()) + } + Err(err) => { + let (rc, rs) = engine_to_dbus_err_tuple(&err); + return_message.append3(default_return, rc, rs) + } + }; + + Ok(vec![msg]) +} diff --git a/src/dbus_api/blockdev/blockdev_2_0/mod.rs b/src/dbus_api/blockdev/blockdev_2_0/mod.rs new file mode 100644 index 0000000000..333cf7108f --- /dev/null +++ b/src/dbus_api/blockdev/blockdev_2_0/mod.rs @@ -0,0 +1,3 @@ +pub mod api; +pub mod methods; +pub mod props; diff --git a/src/dbus_api/blockdev/blockdev_2_0/props.rs b/src/dbus_api/blockdev/blockdev_2_0/props.rs new file mode 100644 index 0000000000..77bcba0aea --- /dev/null +++ b/src/dbus_api/blockdev/blockdev_2_0/props.rs @@ -0,0 +1,75 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use dbus::{ + self, + arg::IterAppend, + tree::{MTFn, MethodErr, PropInfo}, +}; + +use crate::{ + dbus_api::{blockdev::shared::blockdev_operation, types::TData}, + engine::{BlockDev, BlockDevTier}, +}; + +/// Get a blockdev property and place it on the D-Bus. The property is +/// found by means of the getter method which takes a reference to a +/// blockdev and obtains the property from the blockdev. +fn get_blockdev_property( + i: &mut IterAppend, + p: &PropInfo, TData>, + getter: F, +) -> Result<(), MethodErr> +where + F: Fn(BlockDevTier, &dyn BlockDev) -> Result, + R: dbus::arg::Append, +{ + i.append( + blockdev_operation(p.tree, p.path.get_name(), getter) + .map_err(|ref e| MethodErr::failed(e))?, + ); + Ok(()) +} + +/// Get the devnode for an object path. +pub fn get_blockdev_devnode( + i: &mut IterAppend, + p: &PropInfo, TData>, +) -> Result<(), MethodErr> { + get_blockdev_property(i, p, |_, p| Ok(format!("{}", p.devnode().display()))) +} + +pub fn get_blockdev_hardware_info( + i: &mut IterAppend, + p: &PropInfo, TData>, +) -> Result<(), MethodErr> { + get_blockdev_property(i, p, |_, p| { + Ok(p.hardware_info() + .map_or_else(|| (false, "".to_owned()), |val| (true, val.to_owned()))) + }) +} + +pub fn get_blockdev_user_info( + i: &mut IterAppend, + p: &PropInfo, TData>, +) -> Result<(), MethodErr> { + get_blockdev_property(i, p, |_, p| { + Ok(p.user_info() + .map_or_else(|| (false, "".to_owned()), |val| (true, val.to_owned()))) + }) +} + +pub fn get_blockdev_initialization_time( + i: &mut IterAppend, + p: &PropInfo, TData>, +) -> Result<(), MethodErr> { + get_blockdev_property(i, p, |_, p| Ok(p.initialization_time().timestamp() as u64)) +} + +pub fn get_blockdev_tier( + i: &mut IterAppend, + p: &PropInfo, TData>, +) -> Result<(), MethodErr> { + get_blockdev_property(i, p, |t, _| Ok(t as u16)) +} diff --git a/src/dbus_api/blockdev/fetch_properties_2_0/api.rs b/src/dbus_api/blockdev/fetch_properties_2_0/api.rs new file mode 100644 index 0000000000..fbe44d4a0f --- /dev/null +++ b/src/dbus_api/blockdev/fetch_properties_2_0/api.rs @@ -0,0 +1,34 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use dbus::{ + self, + tree::{Factory, MTFn, Method}, +}; + +use crate::dbus_api::{ + blockdev::fetch_properties_2_0::methods::{get_all_properties, get_properties}, + types::TData, +}; + +pub fn get_all_properties_method(f: &Factory, TData>) -> Method, TData> { + f.method("GetAllProperties", (), get_all_properties) + // a{s(bv)}: Dictionary of property names to tuples + // In the tuple: + // b: Indicates whether the property value fetched was successful + // v: If b is true, represents the value for the given property + // If b is false, represents the error returned when fetching the property + .out_arg(("results", "a{s(bv)}")) +} + +pub fn get_properties_method(f: &Factory, TData>) -> Method, TData> { + f.method("GetProperties", (), get_properties) + .in_arg(("properties", "as")) + // a{s(bv)}: Dictionary of property names to tuples + // In the tuple: + // b: Indicates whether the property value fetched was successful + // v: If b is true, represents the value for the given property + // If b is false, represents the error returned when fetching the property + .out_arg(("results", "a{s(bv)}")) +} diff --git a/src/dbus_api/blockdev/fetch_properties_2_0/methods.rs b/src/dbus_api/blockdev/fetch_properties_2_0/methods.rs new file mode 100644 index 0000000000..0389cc45e3 --- /dev/null +++ b/src/dbus_api/blockdev/fetch_properties_2_0/methods.rs @@ -0,0 +1,62 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use std::collections::HashMap; + +use dbus::{ + self, + arg::{Array, RefArg, Variant}, + tree::{MTFn, MethodInfo, MethodResult}, + Message, +}; +use itertools::Itertools; + +use crate::dbus_api::{ + blockdev::shared::blockdev_operation, + consts, + types::TData, + util::{get_next_arg, result_to_tuple}, +}; + +pub fn get_all_properties(m: &MethodInfo, TData>) -> MethodResult { + get_properties_shared( + m, + &mut vec![consts::BLOCKDEV_TOTAL_SIZE_PROP] + .into_iter() + .map(|s| s.to_string()), + ) +} + +pub fn get_properties(m: &MethodInfo, TData>) -> MethodResult { + let message: &Message = m.msg; + let mut iter = message.iter_init(); + let mut properties: Array = get_next_arg(&mut iter, 0)?; + get_properties_shared(m, &mut properties) +} + +fn get_properties_shared( + m: &MethodInfo, TData>, + properties: &mut dyn Iterator, +) -> MethodResult { + let message: &Message = m.msg; + let object_path = &m.path; + + let return_message = message.method_return(); + + let return_value: HashMap>)> = properties + .unique() + .filter_map(|prop| match prop.as_str() { + consts::BLOCKDEV_TOTAL_SIZE_PROP => Some(( + prop, + blockdev_operation(m.tree, object_path.get_name(), |_, bd| { + Ok((u128::from(*bd.size()) * devicemapper::SECTOR_SIZE as u128).to_string()) + }), + )), + _ => None, + }) + .map(|(key, result)| result_to_tuple(key, result)) + .collect(); + + Ok(vec![return_message.append1(return_value)]) +} diff --git a/src/dbus_api/blockdev/fetch_properties_2_0/mod.rs b/src/dbus_api/blockdev/fetch_properties_2_0/mod.rs new file mode 100644 index 0000000000..9ba05c1db4 --- /dev/null +++ b/src/dbus_api/blockdev/fetch_properties_2_0/mod.rs @@ -0,0 +1,2 @@ +pub mod api; +pub mod methods; diff --git a/src/dbus_api/blockdev/mod.rs b/src/dbus_api/blockdev/mod.rs new file mode 100644 index 0000000000..2663629994 --- /dev/null +++ b/src/dbus_api/blockdev/mod.rs @@ -0,0 +1,62 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use dbus::{self, tree::Factory}; +use uuid::Uuid; + +use crate::{ + dbus_api::{ + blockdev::{ + blockdev_2_0::api::{ + devnode_property, hardware_info_property, initialization_time_property, + pool_property, set_userid_method, tier_property, user_info_property, uuid_property, + }, + fetch_properties_2_0::api::{get_all_properties_method, get_properties_method}, + }, + consts, + types::{DbusContext, OPContext}, + util::make_object_path, + }, + engine::{BlockDev, MaybeDbusPath}, +}; + +mod blockdev_2_0; +mod fetch_properties_2_0; +mod shared; + +pub fn create_dbus_blockdev<'a>( + dbus_context: &DbusContext, + parent: dbus::Path<'static>, + uuid: Uuid, + blockdev: &mut dyn BlockDev, +) -> dbus::Path<'a> { + let f = Factory::new_fn(); + + let object_name = make_object_path(dbus_context); + + let object_path = f + .object_path(object_name, Some(OPContext::new(parent, uuid))) + .introspectable() + .add( + f.interface(consts::BLOCKDEV_INTERFACE_NAME, ()) + .add_m(set_userid_method(&f)) + .add_p(devnode_property(&f)) + .add_p(hardware_info_property(&f)) + .add_p(initialization_time_property(&f)) + .add_p(pool_property(&f)) + .add_p(tier_property(&f)) + .add_p(user_info_property(&f)) + .add_p(uuid_property(&f)), + ) + .add( + f.interface(consts::PROPERTY_FETCH_INTERFACE_NAME, ()) + .add_m(get_all_properties_method(&f)) + .add_m(get_properties_method(&f)), + ); + + let path = object_path.get_name().to_owned(); + dbus_context.actions.borrow_mut().push_add(object_path); + blockdev.set_dbus_path(MaybeDbusPath(Some(path.clone()))); + path +} diff --git a/src/dbus_api/blockdev/shared.rs b/src/dbus_api/blockdev/shared.rs new file mode 100644 index 0000000000..edecce59d0 --- /dev/null +++ b/src/dbus_api/blockdev/shared.rs @@ -0,0 +1,56 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use dbus::{ + self, + tree::{MTFn, Tree}, + Path, +}; + +use crate::{ + dbus_api::types::TData, + engine::{BlockDev, BlockDevTier}, +}; + +/// Perform an operation on a `BlockDev` object for a given +/// DBus implicit argument that is a block device +pub fn blockdev_operation( + tree: &Tree, TData>, + object_path: &Path<'static>, + closure: F, +) -> Result +where + F: Fn(BlockDevTier, &dyn BlockDev) -> Result, + R: dbus::arg::Append, +{ + let dbus_context = tree.get_data(); + + let blockdev_path = tree + .get(object_path) + .expect("tree must contain implicit argument"); + + let blockdev_data = blockdev_path + .get_data() + .as_ref() + .ok_or_else(|| format!("no data for object path {}", object_path))?; + + let pool_path = tree + .get(&blockdev_data.parent) + .ok_or_else(|| format!("no path for parent object path {}", &blockdev_data.parent))?; + + let pool_uuid = pool_path + .get_data() + .as_ref() + .ok_or_else(|| format!("no data for object path {}", object_path))? + .uuid; + + let engine = dbus_context.engine.borrow(); + let (_, pool) = engine + .get_pool(pool_uuid) + .ok_or_else(|| format!("no pool corresponding to uuid {}", &pool_uuid))?; + let (tier, blockdev) = pool + .get_blockdev(blockdev_data.uuid) + .ok_or_else(|| format!("no blockdev with uuid {}", blockdev_data.uuid))?; + closure(tier, blockdev) +} diff --git a/src/dbus_api/connection.rs b/src/dbus_api/connection.rs new file mode 100644 index 0000000000..911e1cdb84 --- /dev/null +++ b/src/dbus_api/connection.rs @@ -0,0 +1,110 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use std::{cell::RefCell, rc::Rc, vec::Vec}; + +use dbus::{ + self, + tree::{MTFn, Tree}, + BusType, Connection, ConnectionItem, NameFlag, +}; +use libc; + +use crate::{ + dbus_api::{ + api::get_base_tree, + blockdev::create_dbus_blockdev, + consts, + filesystem::create_dbus_filesystem, + pool::create_dbus_pool, + types::{DbusContext, DeferredAction, TData}, + }, + engine::{Engine, Pool, PoolUuid}, +}; + +/// Returned data from when you connect a stratis engine to dbus. +pub struct DbusConnectionData { + pub connection: Rc>, + pub tree: Tree, TData>, + pub path: dbus::Path<'static>, + pub context: DbusContext, +} + +impl DbusConnectionData { + /// Connect a stratis engine to dbus. + pub fn connect(engine: Rc>) -> Result { + let c = Connection::get_private(BusType::System)?; + let (tree, object_path) = get_base_tree(DbusContext::new(engine)); + let dbus_context = tree.get_data().clone(); + tree.set_registered(&c, true)?; + c.register_name( + consts::STRATIS_BASE_SERVICE, + NameFlag::ReplaceExisting as u32, + )?; + Ok(DbusConnectionData { + connection: Rc::new(RefCell::new(c)), + tree, + path: object_path, + context: dbus_context, + }) + } + + /// Given the UUID of a pool, register all the pertinent information with dbus. + pub fn register_pool(&mut self, pool_uuid: PoolUuid, pool: &mut dyn Pool) { + let pool_path = create_dbus_pool(&self.context, self.path.clone(), pool_uuid, pool); + for (_, fs_uuid, fs) in pool.filesystems_mut() { + create_dbus_filesystem(&self.context, pool_path.clone(), fs_uuid, fs); + } + for (uuid, bd) in pool.blockdevs_mut() { + create_dbus_blockdev(&self.context, pool_path.clone(), uuid, bd); + } + + self.process_deferred_actions() + } + + /// Update the dbus tree with deferred adds and removes. + fn process_deferred_actions(&mut self) { + let mut actions = self.context.actions.borrow_mut(); + for action in actions.drain() { + match action { + DeferredAction::Add(path) => { + self.connection + .borrow_mut() + .register_object_path(path.get_name()) + .expect("Must succeed since object paths are unique"); + self.tree.insert(path); + } + DeferredAction::Remove(path) => { + self.connection.borrow_mut().unregister_object_path(&path); + self.tree.remove(&path); + } + } + } + } + + /// Handle any client dbus requests + pub fn handle(&mut self, fds: &[libc::pollfd]) { + for pfd in fds.iter().filter(|pfd| pfd.revents != 0) { + let items: Vec = self + .connection + .borrow() + .watch_handle(pfd.fd, dbus::WatchEvent::from_revents(pfd.revents)) + .collect(); + + for item in items { + if let ConnectionItem::MethodCall(ref msg) = item { + if let Some(v) = self.tree.handle(msg) { + // Probably the wisest is to ignore any send errors here - + // maybe the remote has disconnected during our processing. + for m in v { + let _ = self.connection.borrow_mut().send(m); + } + } + + self.process_deferred_actions(); + } + } + } + } +} diff --git a/src/dbus_api/filesystem.rs b/src/dbus_api/filesystem.rs deleted file mode 100644 index 048edfa4ae..0000000000 --- a/src/dbus_api/filesystem.rs +++ /dev/null @@ -1,308 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. - -use std::collections::HashMap; - -use chrono::SecondsFormat; -use dbus::{ - self, - arg::{Array, IterAppend, RefArg, Variant}, - tree::{ - Access, EmitsChangedSignal, Factory, MTFn, MethodErr, MethodInfo, MethodResult, PropInfo, - Tree, - }, - Message, Path, -}; -use itertools::Itertools; - -use crate::{ - dbus_api::{ - consts, - types::{DbusContext, DbusErrorEnum, OPContext, TData}, - util::{ - engine_to_dbus_err_tuple, get_next_arg, get_parent, get_uuid, make_object_path, - msg_code_ok, msg_string_ok, result_to_tuple, - }, - }, - engine::{Filesystem, FilesystemUuid, MaybeDbusPath, Name, RenameAction}, -}; - -fn get_properties_shared( - m: &MethodInfo, TData>, - properties: &mut dyn Iterator, -) -> MethodResult { - let message: &Message = m.msg; - let object_path = &m.path; - - let return_message = message.method_return(); - - let return_value: HashMap>)> = properties - .unique() - .filter_map(|prop| match prop.as_str() { - consts::FILESYSTEM_USED_PROP => Some(( - prop, - filesystem_operation(m.tree, object_path.get_name(), |(_, _, fs)| { - fs.used() - .map(|u| (*u).to_string()) - .map_err(|e| e.to_string()) - }), - )), - _ => None, - }) - .map(|(key, result)| result_to_tuple(key, result)) - .collect(); - - Ok(vec![return_message.append1(return_value)]) -} - -fn get_all_properties(m: &MethodInfo, TData>) -> MethodResult { - get_properties_shared( - m, - &mut vec![consts::FILESYSTEM_USED_PROP] - .into_iter() - .map(|s| s.to_string()), - ) -} - -fn get_properties(m: &MethodInfo, TData>) -> MethodResult { - let message: &Message = m.msg; - let mut iter = message.iter_init(); - let mut properties: Array = get_next_arg(&mut iter, 0)?; - get_properties_shared(m, &mut properties) -} - -pub fn create_dbus_filesystem<'a>( - dbus_context: &DbusContext, - parent: dbus::Path<'static>, - uuid: FilesystemUuid, - filesystem: &mut dyn Filesystem, -) -> dbus::Path<'a> { - let f = Factory::new_fn(); - - let rename_method = f - .method("SetName", (), rename_filesystem) - .in_arg(("name", "s")) - // b: true if UUID of changed resource has been returned - // s: UUID of changed resource - // - // Rust representation: (bool, String) - .out_arg(("result", "(bs)")) - .out_arg(("return_code", "q")) - .out_arg(("return_string", "s")); - - let devnode_property = f - .property::<&str, _>("Devnode", ()) - .access(Access::Read) - .emits_changed(EmitsChangedSignal::Const) - .on_get(get_filesystem_devnode); - - let name_property = f - .property::<&str, _>(consts::FILESYSTEM_NAME_PROP, ()) - .access(Access::Read) - .emits_changed(EmitsChangedSignal::True) - .on_get(get_filesystem_name); - - let pool_property = f - .property::<&dbus::Path, _>("Pool", ()) - .access(Access::Read) - .emits_changed(EmitsChangedSignal::Const) - .on_get(get_parent); - - let uuid_property = f - .property::<&str, _>("Uuid", ()) - .access(Access::Read) - .emits_changed(EmitsChangedSignal::Const) - .on_get(get_uuid); - - let created_property = f - .property::<&str, _>("Created", ()) - .access(Access::Read) - .emits_changed(EmitsChangedSignal::Const) - .on_get(get_filesystem_created); - - let get_all_properties_method = f - .method("GetAllProperties", (), get_all_properties) - // a{s(bv)}: Dictionary of property names to tuples - // In the tuple: - // b: Indicates whether the property value fetched was successful - // v: If b is true, represents the value for the given property - // If b is false, represents the error returned when fetching the property - .out_arg(("results", "a{s(bv)}")); - - let get_properties_method = f - .method("GetProperties", (), get_properties) - .in_arg(("properties", "as")) - // a{s(bv)}: Dictionary of property names to tuples - // In the tuple: - // b: Indicates whether the property value fetched was successful - // v: If b is true, represents the value for the given property - // If b is false, represents the error returned when fetching the property - .out_arg(("results", "a{s(bv)}")); - - let object_name = make_object_path(dbus_context); - - let object_path = f - .object_path(object_name, Some(OPContext::new(parent, uuid))) - .introspectable() - .add( - f.interface(consts::FILESYSTEM_INTERFACE_NAME, ()) - .add_m(rename_method) - .add_p(devnode_property) - .add_p(name_property) - .add_p(pool_property) - .add_p(uuid_property) - .add_p(created_property), - ) - .add( - f.interface(consts::PROPERTY_FETCH_INTERFACE_NAME, ()) - .add_m(get_all_properties_method) - .add_m(get_properties_method), - ); - - let path = object_path.get_name().to_owned(); - dbus_context.actions.borrow_mut().push_add(object_path); - filesystem.set_dbus_path(MaybeDbusPath(Some(path.clone()))); - path -} - -fn rename_filesystem(m: &MethodInfo, TData>) -> MethodResult { - let message: &Message = m.msg; - let mut iter = message.iter_init(); - - let new_name: &str = get_next_arg(&mut iter, 0)?; - - let dbus_context = m.tree.get_data(); - let object_path = m.path.get_name(); - let return_message = message.method_return(); - let default_return = (false, uuid_to_string!(FilesystemUuid::nil())); - - let filesystem_path = m - .tree - .get(object_path) - .expect("implicit argument must be in tree"); - let filesystem_data = get_data!(filesystem_path; default_return; return_message); - - let pool_path = get_parent!(m; filesystem_data; default_return; return_message); - let pool_uuid = get_data!(pool_path; default_return; return_message).uuid; - - let mut engine = dbus_context.engine.borrow_mut(); - let (pool_name, pool) = get_mut_pool!(engine; pool_uuid; default_return; return_message); - - let msg = match pool.rename_filesystem(&pool_name, filesystem_data.uuid, new_name) { - Ok(RenameAction::NoSource) => { - let error_message = format!( - "pool {} doesn't know about filesystem {}", - pool_uuid, filesystem_data.uuid - ); - let (rc, rs) = (DbusErrorEnum::INTERNAL_ERROR as u16, error_message); - return_message.append3(default_return, rc, rs) - } - Ok(RenameAction::Identity) => { - return_message.append3(default_return, msg_code_ok(), msg_string_ok()) - } - Ok(RenameAction::Renamed(uuid)) => return_message.append3( - (true, uuid_to_string!(uuid)), - msg_code_ok(), - msg_string_ok(), - ), - Err(err) => { - let (rc, rs) = engine_to_dbus_err_tuple(&err); - return_message.append3(default_return, rc, rs) - } - }; - - Ok(vec![msg]) -} - -/// Get execute a given closure providing a filesystem object and return -/// the calculated value -fn filesystem_operation( - tree: &Tree, TData>, - object_path: &Path<'static>, - closure: F, -) -> Result -where - F: Fn((Name, Name, &dyn Filesystem)) -> Result, - R: dbus::arg::Append, -{ - let dbus_context = tree.get_data(); - - let filesystem_path = tree - .get(object_path) - .expect("tree must contain implicit argument"); - - let filesystem_data = filesystem_path - .get_data() - .as_ref() - .ok_or_else(|| format!("no data for object path {}", object_path))?; - - let pool_path = tree - .get(&filesystem_data.parent) - .ok_or_else(|| format!("no path for parent object path {}", &filesystem_data.parent))?; - - let pool_uuid = pool_path - .get_data() - .as_ref() - .ok_or_else(|| format!("no data for object path {}", object_path))? - .uuid; - - let engine = dbus_context.engine.borrow(); - let (pool_name, pool) = engine - .get_pool(pool_uuid) - .ok_or_else(|| format!("no pool corresponding to uuid {}", &pool_uuid))?; - let filesystem_uuid = filesystem_data.uuid; - let (fs_name, fs) = pool - .get_filesystem(filesystem_uuid) - .ok_or_else(|| format!("no name for filesystem with uuid {}", &filesystem_uuid))?; - closure((pool_name, fs_name, fs)) -} - -/// Get a filesystem property and place it on the D-Bus. The property is -/// found by means of the getter method which takes a reference to a -/// Filesystem and obtains the property from the filesystem. -fn get_filesystem_property( - i: &mut IterAppend, - p: &PropInfo, TData>, - getter: F, -) -> Result<(), MethodErr> -where - F: Fn((Name, Name, &dyn Filesystem)) -> Result, - R: dbus::arg::Append, -{ - i.append( - filesystem_operation(p.tree, p.path.get_name(), getter) - .map_err(|ref e| MethodErr::failed(e))?, - ); - Ok(()) -} - -/// Get the devnode for an object path. -fn get_filesystem_devnode( - i: &mut IterAppend, - p: &PropInfo, TData>, -) -> Result<(), MethodErr> { - get_filesystem_property(i, p, |(pool_name, fs_name, fs)| { - Ok(fs - .path_to_mount_filesystem(&pool_name, &fs_name) - .display() - .to_string()) - }) -} - -fn get_filesystem_name( - i: &mut IterAppend, - p: &PropInfo, TData>, -) -> Result<(), MethodErr> { - get_filesystem_property(i, p, |(_, fs_name, _)| Ok(fs_name.to_owned())) -} - -/// Get the creation date and time in rfc3339 format. -fn get_filesystem_created( - i: &mut IterAppend, - p: &PropInfo, TData>, -) -> Result<(), MethodErr> { - get_filesystem_property(i, p, |(_, _, fs)| { - Ok(fs.created().to_rfc3339_opts(SecondsFormat::Secs, true)) - }) -} diff --git a/src/dbus_api/filesystem/fetch_properties_2_0/api.rs b/src/dbus_api/filesystem/fetch_properties_2_0/api.rs new file mode 100644 index 0000000000..0924cab655 --- /dev/null +++ b/src/dbus_api/filesystem/fetch_properties_2_0/api.rs @@ -0,0 +1,34 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use dbus::{ + self, + tree::{Factory, MTFn, Method}, +}; + +use crate::dbus_api::{ + filesystem::fetch_properties_2_0::methods::{get_all_properties, get_properties}, + types::TData, +}; + +pub fn get_all_properties_method(f: &Factory, TData>) -> Method, TData> { + f.method("GetAllProperties", (), get_all_properties) + // a{s(bv)}: Dictionary of property names to tuples + // In the tuple: + // b: Indicates whether the property value fetched was successful + // v: If b is true, represents the value for the given property + // If b is false, represents the error returned when fetching the property + .out_arg(("results", "a{s(bv)}")) +} + +pub fn get_properties_method(f: &Factory, TData>) -> Method, TData> { + f.method("GetProperties", (), get_properties) + .in_arg(("properties", "as")) + // a{s(bv)}: Dictionary of property names to tuples + // In the tuple: + // b: Indicates whether the property value fetched was successful + // v: If b is true, represents the value for the given property + // If b is false, represents the error returned when fetching the property + .out_arg(("results", "a{s(bv)}")) +} diff --git a/src/dbus_api/filesystem/fetch_properties_2_0/methods.rs b/src/dbus_api/filesystem/fetch_properties_2_0/methods.rs new file mode 100644 index 0000000000..1bbdd592c2 --- /dev/null +++ b/src/dbus_api/filesystem/fetch_properties_2_0/methods.rs @@ -0,0 +1,64 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use std::collections::HashMap; + +use dbus::{ + self, + arg::{Array, RefArg, Variant}, + tree::{MTFn, MethodInfo, MethodResult}, + Message, +}; +use itertools::Itertools; + +use crate::dbus_api::{ + consts, + filesystem::shared::filesystem_operation, + types::TData, + util::{get_next_arg, result_to_tuple}, +}; + +fn get_properties_shared( + m: &MethodInfo, TData>, + properties: &mut dyn Iterator, +) -> MethodResult { + let message: &Message = m.msg; + let object_path = &m.path; + + let return_message = message.method_return(); + + let return_value: HashMap>)> = properties + .unique() + .filter_map(|prop| match prop.as_str() { + consts::FILESYSTEM_USED_PROP => Some(( + prop, + filesystem_operation(m.tree, object_path.get_name(), |(_, _, fs)| { + fs.used() + .map(|u| (*u).to_string()) + .map_err(|e| e.to_string()) + }), + )), + _ => None, + }) + .map(|(key, result)| result_to_tuple(key, result)) + .collect(); + + Ok(vec![return_message.append1(return_value)]) +} + +pub fn get_all_properties(m: &MethodInfo, TData>) -> MethodResult { + get_properties_shared( + m, + &mut vec![consts::FILESYSTEM_USED_PROP] + .into_iter() + .map(|s| s.to_string()), + ) +} + +pub fn get_properties(m: &MethodInfo, TData>) -> MethodResult { + let message: &Message = m.msg; + let mut iter = message.iter_init(); + let mut properties: Array = get_next_arg(&mut iter, 0)?; + get_properties_shared(m, &mut properties) +} diff --git a/src/dbus_api/filesystem/fetch_properties_2_0/mod.rs b/src/dbus_api/filesystem/fetch_properties_2_0/mod.rs new file mode 100644 index 0000000000..9ba05c1db4 --- /dev/null +++ b/src/dbus_api/filesystem/fetch_properties_2_0/mod.rs @@ -0,0 +1,2 @@ +pub mod api; +pub mod methods; diff --git a/src/dbus_api/filesystem/filesystem_2_0/api.rs b/src/dbus_api/filesystem/filesystem_2_0/api.rs new file mode 100644 index 0000000000..4d6d3527b2 --- /dev/null +++ b/src/dbus_api/filesystem/filesystem_2_0/api.rs @@ -0,0 +1,65 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use dbus::{ + self, + tree::{Access, EmitsChangedSignal, Factory, MTFn, Method, Property}, +}; + +use crate::dbus_api::{ + consts, + filesystem::filesystem_2_0::{ + methods::rename_filesystem, + props::{get_filesystem_created, get_filesystem_devnode, get_filesystem_name}, + }, + types::TData, + util::{get_parent, get_uuid}, +}; + +pub fn rename_method(f: &Factory, TData>) -> Method, TData> { + f.method("SetName", (), rename_filesystem) + .in_arg(("name", "s")) + // b: true if UUID of changed resource has been returned + // s: UUID of changed resource + // + // Rust representation: (bool, String) + .out_arg(("result", "(bs)")) + .out_arg(("return_code", "q")) + .out_arg(("return_string", "s")) +} + +pub fn devnode_property(f: &Factory, TData>) -> Property, TData> { + f.property::<&str, _>("Devnode", ()) + .access(Access::Read) + .emits_changed(EmitsChangedSignal::Const) + .on_get(get_filesystem_devnode) +} + +pub fn name_property(f: &Factory, TData>) -> Property, TData> { + f.property::<&str, _>(consts::FILESYSTEM_NAME_PROP, ()) + .access(Access::Read) + .emits_changed(EmitsChangedSignal::True) + .on_get(get_filesystem_name) +} + +pub fn pool_property(f: &Factory, TData>) -> Property, TData> { + f.property::<&dbus::Path, _>("Pool", ()) + .access(Access::Read) + .emits_changed(EmitsChangedSignal::Const) + .on_get(get_parent) +} + +pub fn uuid_property(f: &Factory, TData>) -> Property, TData> { + f.property::<&str, _>("Uuid", ()) + .access(Access::Read) + .emits_changed(EmitsChangedSignal::Const) + .on_get(get_uuid) +} + +pub fn created_property(f: &Factory, TData>) -> Property, TData> { + f.property::<&str, _>("Created", ()) + .access(Access::Read) + .emits_changed(EmitsChangedSignal::Const) + .on_get(get_filesystem_created) +} diff --git a/src/dbus_api/filesystem/filesystem_2_0/methods.rs b/src/dbus_api/filesystem/filesystem_2_0/methods.rs new file mode 100644 index 0000000000..32b3bbc9f2 --- /dev/null +++ b/src/dbus_api/filesystem/filesystem_2_0/methods.rs @@ -0,0 +1,66 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use dbus::{ + self, + tree::{MTFn, MethodInfo, MethodResult}, + Message, +}; + +use crate::{ + dbus_api::{ + types::{DbusErrorEnum, TData}, + util::{engine_to_dbus_err_tuple, get_next_arg, msg_code_ok, msg_string_ok}, + }, + engine::{FilesystemUuid, RenameAction}, +}; + +pub fn rename_filesystem(m: &MethodInfo, TData>) -> MethodResult { + let message: &Message = m.msg; + let mut iter = message.iter_init(); + + let new_name: &str = get_next_arg(&mut iter, 0)?; + + let dbus_context = m.tree.get_data(); + let object_path = m.path.get_name(); + let return_message = message.method_return(); + let default_return = (false, uuid_to_string!(FilesystemUuid::nil())); + + let filesystem_path = m + .tree + .get(object_path) + .expect("implicit argument must be in tree"); + let filesystem_data = get_data!(filesystem_path; default_return; return_message); + + let pool_path = get_parent!(m; filesystem_data; default_return; return_message); + let pool_uuid = get_data!(pool_path; default_return; return_message).uuid; + + let mut engine = dbus_context.engine.borrow_mut(); + let (pool_name, pool) = get_mut_pool!(engine; pool_uuid; default_return; return_message); + + let msg = match pool.rename_filesystem(&pool_name, filesystem_data.uuid, new_name) { + Ok(RenameAction::NoSource) => { + let error_message = format!( + "pool {} doesn't know about filesystem {}", + pool_uuid, filesystem_data.uuid + ); + let (rc, rs) = (DbusErrorEnum::INTERNAL_ERROR as u16, error_message); + return_message.append3(default_return, rc, rs) + } + Ok(RenameAction::Identity) => { + return_message.append3(default_return, msg_code_ok(), msg_string_ok()) + } + Ok(RenameAction::Renamed(uuid)) => return_message.append3( + (true, uuid_to_string!(uuid)), + msg_code_ok(), + msg_string_ok(), + ), + Err(err) => { + let (rc, rs) = engine_to_dbus_err_tuple(&err); + return_message.append3(default_return, rc, rs) + } + }; + + Ok(vec![msg]) +} diff --git a/src/dbus_api/filesystem/filesystem_2_0/mod.rs b/src/dbus_api/filesystem/filesystem_2_0/mod.rs new file mode 100644 index 0000000000..333cf7108f --- /dev/null +++ b/src/dbus_api/filesystem/filesystem_2_0/mod.rs @@ -0,0 +1,3 @@ +pub mod api; +pub mod methods; +pub mod props; diff --git a/src/dbus_api/filesystem/filesystem_2_0/props.rs b/src/dbus_api/filesystem/filesystem_2_0/props.rs new file mode 100644 index 0000000000..3478b1124c --- /dev/null +++ b/src/dbus_api/filesystem/filesystem_2_0/props.rs @@ -0,0 +1,64 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use chrono::SecondsFormat; +use dbus::{ + self, + arg::IterAppend, + tree::{MTFn, MethodErr, PropInfo}, +}; + +use crate::{ + dbus_api::{filesystem::shared::filesystem_operation, types::TData}, + engine::{Filesystem, Name}, +}; + +/// Get a filesystem property and place it on the D-Bus. The property is +/// found by means of the getter method which takes a reference to a +/// Filesystem and obtains the property from the filesystem. +fn get_filesystem_property( + i: &mut IterAppend, + p: &PropInfo, TData>, + getter: F, +) -> Result<(), MethodErr> +where + F: Fn((Name, Name, &dyn Filesystem)) -> Result, + R: dbus::arg::Append, +{ + i.append( + filesystem_operation(p.tree, p.path.get_name(), getter) + .map_err(|ref e| MethodErr::failed(e))?, + ); + Ok(()) +} + +/// Get the devnode for an object path. +pub fn get_filesystem_devnode( + i: &mut IterAppend, + p: &PropInfo, TData>, +) -> Result<(), MethodErr> { + get_filesystem_property(i, p, |(pool_name, fs_name, fs)| { + Ok(fs + .path_to_mount_filesystem(&pool_name, &fs_name) + .display() + .to_string()) + }) +} + +pub fn get_filesystem_name( + i: &mut IterAppend, + p: &PropInfo, TData>, +) -> Result<(), MethodErr> { + get_filesystem_property(i, p, |(_, fs_name, _)| Ok(fs_name.to_owned())) +} + +/// Get the creation date and time in rfc3339 format. +pub fn get_filesystem_created( + i: &mut IterAppend, + p: &PropInfo, TData>, +) -> Result<(), MethodErr> { + get_filesystem_property(i, p, |(_, _, fs)| { + Ok(fs.created().to_rfc3339_opts(SecondsFormat::Secs, true)) + }) +} diff --git a/src/dbus_api/filesystem/mod.rs b/src/dbus_api/filesystem/mod.rs new file mode 100644 index 0000000000..f5af163f08 --- /dev/null +++ b/src/dbus_api/filesystem/mod.rs @@ -0,0 +1,59 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use dbus::{self, tree::Factory}; + +use crate::{ + dbus_api::{ + consts, + filesystem::{ + fetch_properties_2_0::api::{get_all_properties_method, get_properties_method}, + filesystem_2_0::api::{ + created_property, devnode_property, name_property, pool_property, rename_method, + uuid_property, + }, + }, + types::{DbusContext, OPContext}, + util::make_object_path, + }, + engine::{Filesystem, FilesystemUuid, MaybeDbusPath}, +}; + +mod fetch_properties_2_0; +mod filesystem_2_0; +mod shared; + +pub fn create_dbus_filesystem<'a>( + dbus_context: &DbusContext, + parent: dbus::Path<'static>, + uuid: FilesystemUuid, + filesystem: &mut dyn Filesystem, +) -> dbus::Path<'a> { + let f = Factory::new_fn(); + + let object_name = make_object_path(dbus_context); + + let object_path = f + .object_path(object_name, Some(OPContext::new(parent, uuid))) + .introspectable() + .add( + f.interface(consts::FILESYSTEM_INTERFACE_NAME, ()) + .add_m(rename_method(&f)) + .add_p(devnode_property(&f)) + .add_p(name_property(&f)) + .add_p(pool_property(&f)) + .add_p(uuid_property(&f)) + .add_p(created_property(&f)), + ) + .add( + f.interface(consts::PROPERTY_FETCH_INTERFACE_NAME, ()) + .add_m(get_all_properties_method(&f)) + .add_m(get_properties_method(&f)), + ); + + let path = object_path.get_name().to_owned(); + dbus_context.actions.borrow_mut().push_add(object_path); + filesystem.set_dbus_path(MaybeDbusPath(Some(path.clone()))); + path +} diff --git a/src/dbus_api/filesystem/shared.rs b/src/dbus_api/filesystem/shared.rs new file mode 100644 index 0000000000..63e0453e15 --- /dev/null +++ b/src/dbus_api/filesystem/shared.rs @@ -0,0 +1,57 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use dbus::{ + self, + tree::{MTFn, Tree}, + Path, +}; + +use crate::{ + dbus_api::types::TData, + engine::{Filesystem, Name}, +}; + +/// Get execute a given closure providing a filesystem object and return +/// the calculated value +pub fn filesystem_operation( + tree: &Tree, TData>, + object_path: &Path<'static>, + closure: F, +) -> Result +where + F: Fn((Name, Name, &dyn Filesystem)) -> Result, + R: dbus::arg::Append, +{ + let dbus_context = tree.get_data(); + + let filesystem_path = tree + .get(object_path) + .expect("tree must contain implicit argument"); + + let filesystem_data = filesystem_path + .get_data() + .as_ref() + .ok_or_else(|| format!("no data for object path {}", object_path))?; + + let pool_path = tree + .get(&filesystem_data.parent) + .ok_or_else(|| format!("no path for parent object path {}", &filesystem_data.parent))?; + + let pool_uuid = pool_path + .get_data() + .as_ref() + .ok_or_else(|| format!("no data for object path {}", object_path))? + .uuid; + + let engine = dbus_context.engine.borrow(); + let (pool_name, pool) = engine + .get_pool(pool_uuid) + .ok_or_else(|| format!("no pool corresponding to uuid {}", &pool_uuid))?; + let filesystem_uuid = filesystem_data.uuid; + let (fs_name, fs) = pool + .get_filesystem(filesystem_uuid) + .ok_or_else(|| format!("no name for filesystem with uuid {}", &filesystem_uuid))?; + closure((pool_name, fs_name, fs)) +} diff --git a/src/dbus_api/mod.rs b/src/dbus_api/mod.rs index ca3d798ed6..2d375d723d 100644 --- a/src/dbus_api/mod.rs +++ b/src/dbus_api/mod.rs @@ -7,6 +7,7 @@ mod macros; mod api; mod blockdev; +mod connection; mod consts; mod event_handler; mod filesystem; @@ -14,4 +15,4 @@ mod pool; mod types; mod util; -pub use self::{api::DbusConnectionData, event_handler::EventHandler}; +pub use self::{connection::DbusConnectionData, event_handler::EventHandler}; diff --git a/src/dbus_api/pool/fetch_properties_2_0/api.rs b/src/dbus_api/pool/fetch_properties_2_0/api.rs new file mode 100644 index 0000000000..d4bbc2c5cd --- /dev/null +++ b/src/dbus_api/pool/fetch_properties_2_0/api.rs @@ -0,0 +1,31 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use dbus::tree::{Factory, MTFn, Method}; + +use crate::dbus_api::{ + pool::fetch_properties_2_0::methods::{get_all_properties, get_properties}, + types::TData, +}; + +pub fn get_all_properties_method(f: &Factory, TData>) -> Method, TData> { + f.method("GetAllProperties", (), get_all_properties) + // a{s(bv)}: Dictionary of property names to tuples + // In the tuple: + // b: Indicates whether the property value fetched was successful + // v: If b is true, represents the value for the given property + // If b is false, represents the error returned when fetching the property + .out_arg(("results", "a{s(bv)}")) +} + +pub fn get_properties_method(f: &Factory, TData>) -> Method, TData> { + f.method("GetProperties", (), get_properties) + .in_arg(("properties", "as")) + // a{s(bv)}: Dictionary of property names to tuples + // In the tuple: + // b: Indicates whether the property value fetched was successful + // v: If b is true, represents the value for the given property + // If b is false, represents the error returned when fetching the property + .out_arg(("results", "a{s(bv)}")) +} diff --git a/src/dbus_api/pool/fetch_properties_2_0/methods.rs b/src/dbus_api/pool/fetch_properties_2_0/methods.rs new file mode 100644 index 0000000000..c40daabb70 --- /dev/null +++ b/src/dbus_api/pool/fetch_properties_2_0/methods.rs @@ -0,0 +1,73 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use std::collections::HashMap; + +use dbus::{ + arg::{Array, RefArg, Variant}, + tree::{MTFn, MethodInfo, MethodResult}, + Message, +}; +use itertools::Itertools; + +use crate::dbus_api::{ + consts, + pool::shared::pool_operation, + types::TData, + util::{get_next_arg, result_to_tuple}, +}; + +fn get_properties_shared( + m: &MethodInfo, TData>, + properties: &mut dyn Iterator, +) -> MethodResult { + let message: &Message = m.msg; + let object_path = &m.path; + + let return_message = message.method_return(); + + let return_value: HashMap>)> = properties + .unique() + .filter_map(|prop| match prop.as_str() { + consts::POOL_TOTAL_SIZE_PROP => Some(( + prop, + pool_operation(m.tree, object_path.get_name(), |(_, _, pool)| { + Ok((u128::from(*pool.total_physical_size()) + * devicemapper::SECTOR_SIZE as u128) + .to_string()) + }), + )), + consts::POOL_TOTAL_USED_PROP => Some(( + prop, + pool_operation(m.tree, object_path.get_name(), |(_, _, pool)| { + pool.total_physical_used() + .map_err(|e| e.to_string()) + .map(|size| { + (u128::from(*size) * devicemapper::SECTOR_SIZE as u128).to_string() + }) + }), + )), + _ => None, + }) + .map(|(key, result)| result_to_tuple(key, result)) + .collect(); + + Ok(vec![return_message.append1(return_value)]) +} + +pub fn get_all_properties(m: &MethodInfo, TData>) -> MethodResult { + get_properties_shared( + m, + &mut vec![consts::POOL_TOTAL_SIZE_PROP, consts::POOL_TOTAL_USED_PROP] + .into_iter() + .map(|s| s.to_string()), + ) +} + +pub fn get_properties(m: &MethodInfo, TData>) -> MethodResult { + let message: &Message = m.msg; + let mut iter = message.iter_init(); + let mut properties: Array = get_next_arg(&mut iter, 0)?; + get_properties_shared(m, &mut properties) +} diff --git a/src/dbus_api/pool/fetch_properties_2_0/mod.rs b/src/dbus_api/pool/fetch_properties_2_0/mod.rs new file mode 100644 index 0000000000..9ba05c1db4 --- /dev/null +++ b/src/dbus_api/pool/fetch_properties_2_0/mod.rs @@ -0,0 +1,2 @@ +pub mod api; +pub mod methods; diff --git a/src/dbus_api/pool/mod.rs b/src/dbus_api/pool/mod.rs new file mode 100644 index 0000000000..a65e7a5afe --- /dev/null +++ b/src/dbus_api/pool/mod.rs @@ -0,0 +1,62 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use dbus::{self, tree::Factory}; + +use crate::{ + dbus_api::{ + consts, + pool::{ + fetch_properties_2_0::api::{get_all_properties_method, get_properties_method}, + pool_2_0::api::{ + add_blockdevs_method, add_cachedevs_method, create_filesystems_method, + destroy_filesystems_method, name_property, rename_method, + snapshot_filesystem_method, uuid_property, + }, + }, + types::{DbusContext, OPContext}, + util::make_object_path, + }, + engine::{MaybeDbusPath, Pool, PoolUuid}, +}; + +mod fetch_properties_2_0; +mod pool_2_0; +mod shared; + +pub fn create_dbus_pool<'a>( + dbus_context: &DbusContext, + parent: dbus::Path<'static>, + uuid: PoolUuid, + pool: &mut dyn Pool, +) -> dbus::Path<'a> { + let f = Factory::new_fn(); + + let object_name = make_object_path(dbus_context); + + let object_path = f + .object_path(object_name, Some(OPContext::new(parent, uuid))) + .introspectable() + .add( + f.interface(consts::POOL_INTERFACE_NAME, ()) + .add_m(create_filesystems_method(&f)) + .add_m(destroy_filesystems_method(&f)) + .add_m(snapshot_filesystem_method(&f)) + .add_m(add_blockdevs_method(&f)) + .add_m(add_cachedevs_method(&f)) + .add_m(rename_method(&f)) + .add_p(name_property(&f)) + .add_p(uuid_property(&f)), + ) + .add( + f.interface(consts::PROPERTY_FETCH_INTERFACE_NAME, ()) + .add_m(get_all_properties_method(&f)) + .add_m(get_properties_method(&f)), + ); + + let path = object_path.get_name().to_owned(); + dbus_context.actions.borrow_mut().push_add(object_path); + pool.set_dbus_path(MaybeDbusPath(Some(path.clone()))); + path +} diff --git a/src/dbus_api/pool/pool_2_0/api.rs b/src/dbus_api/pool/pool_2_0/api.rs new file mode 100644 index 0000000000..c2381c2cbe --- /dev/null +++ b/src/dbus_api/pool/pool_2_0/api.rs @@ -0,0 +1,105 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use dbus::tree::{Access, EmitsChangedSignal, Factory, MTFn, Method, Property}; + +use crate::dbus_api::{ + consts, + pool::pool_2_0::{ + methods::{ + add_cachedevs, add_datadevs, create_filesystems, destroy_filesystems, rename_pool, + snapshot_filesystem, + }, + props::get_pool_name, + }, + types::TData, + util::get_uuid, +}; + +pub fn create_filesystems_method(f: &Factory, TData>) -> Method, TData> { + f.method("CreateFilesystems", (), create_filesystems) + .in_arg(("specs", "as")) + // b: true if filesystems were created + // a(os): Array of tuples with object paths and names + // + // Rust representation: (bool, Vec<(dbus::Path, String)>) + .out_arg(("results", "(ba(os))")) + .out_arg(("return_code", "q")) + .out_arg(("return_string", "s")) +} + +pub fn destroy_filesystems_method(f: &Factory, TData>) -> Method, TData> { + f.method("DestroyFilesystems", (), destroy_filesystems) + .in_arg(("filesystems", "ao")) + // b: true if filesystems were destroyed + // as: Array of UUIDs of destroyed filesystems + // + // Rust representation: (bool, Vec) + .out_arg(("results", "(bas)")) + .out_arg(("return_code", "q")) + .out_arg(("return_string", "s")) +} + +pub fn snapshot_filesystem_method(f: &Factory, TData>) -> Method, TData> { + f.method("SnapshotFilesystem", (), snapshot_filesystem) + .in_arg(("origin", "o")) + .in_arg(("snapshot_name", "s")) + // b: false if no new snapshot was created + // s: Object path of new snapshot + // + // Rust representation: (bool, String) + .out_arg(("result", "(bo)")) + .out_arg(("return_code", "q")) + .out_arg(("return_string", "s")) +} + +pub fn add_blockdevs_method(f: &Factory, TData>) -> Method, TData> { + f.method("AddDataDevs", (), add_datadevs) + .in_arg(("devices", "as")) + // b: Indicates if any data devices were added + // ao: Array of object paths of created data devices + // + // Rust representation: (bool, Vec) + .out_arg(("results", "(bao)")) + .out_arg(("return_code", "q")) + .out_arg(("return_string", "s")) +} + +pub fn add_cachedevs_method(f: &Factory, TData>) -> Method, TData> { + f.method("AddCacheDevs", (), add_cachedevs) + .in_arg(("devices", "as")) + // b: Indicates if any cache devices were added + // ao: Array of object paths of created cache devices + // + // Rust representation: (bool, Vec) + .out_arg(("results", "(bao)")) + .out_arg(("return_code", "q")) + .out_arg(("return_string", "s")) +} + +pub fn rename_method(f: &Factory, TData>) -> Method, TData> { + f.method("SetName", (), rename_pool) + .in_arg(("name", "s")) + // b: false if no pool was renamed + // s: UUID of renamed pool + // + // Rust representation: (bool, String) + .out_arg(("result", "(bs)")) + .out_arg(("return_code", "q")) + .out_arg(("return_string", "s")) +} + +pub fn name_property(f: &Factory, TData>) -> Property, TData> { + f.property::<&str, _>(consts::POOL_NAME_PROP, ()) + .access(Access::Read) + .emits_changed(EmitsChangedSignal::True) + .on_get(get_pool_name) +} + +pub fn uuid_property(f: &Factory, TData>) -> Property, TData> { + f.property::<&str, _>("Uuid", ()) + .access(Access::Read) + .emits_changed(EmitsChangedSignal::Const) + .on_get(get_uuid) +} diff --git a/src/dbus_api/pool.rs b/src/dbus_api/pool/pool_2_0/methods.rs similarity index 52% rename from src/dbus_api/pool.rs rename to src/dbus_api/pool/pool_2_0/methods.rs index 8fb81d51f5..0025374f44 100644 --- a/src/dbus_api/pool.rs +++ b/src/dbus_api/pool/pool_2_0/methods.rs @@ -2,39 +2,26 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. -use std::{collections::HashMap, path::Path, vec::Vec}; +use std::{collections::HashMap, path::Path}; use dbus::{ - self, - arg::{Array, IterAppend, RefArg, Variant}, - tree::{ - Access, EmitsChangedSignal, Factory, MTFn, MethodErr, MethodInfo, MethodResult, PropInfo, - Tree, - }, + arg::Array, + tree::{MTFn, MethodInfo, MethodResult}, Message, }; -use itertools::Itertools; - use devicemapper::Sectors; use crate::{ dbus_api::{ blockdev::create_dbus_blockdev, - consts, filesystem::create_dbus_filesystem, - types::{DbusContext, DbusErrorEnum, OPContext, TData}, - util::{ - engine_to_dbus_err_tuple, get_next_arg, get_uuid, make_object_path, msg_code_ok, - msg_string_ok, result_to_tuple, - }, - }, - engine::{ - BlockDevTier, CreateAction, EngineAction, FilesystemUuid, MaybeDbusPath, Name, Pool, - PoolUuid, RenameAction, + types::{DbusErrorEnum, TData}, + util::{engine_to_dbus_err_tuple, get_next_arg, msg_code_ok, msg_string_ok}, }, + engine::{BlockDevTier, CreateAction, EngineAction, FilesystemUuid, PoolUuid, RenameAction}, }; -fn create_filesystems(m: &MethodInfo, TData>) -> MethodResult { +pub fn create_filesystems(m: &MethodInfo, TData>) -> MethodResult { let message: &Message = m.msg; let mut iter = message.iter_init(); @@ -109,7 +96,7 @@ fn create_filesystems(m: &MethodInfo, TData>) -> MethodResult { )]) } -fn destroy_filesystems(m: &MethodInfo, TData>) -> MethodResult { +pub fn destroy_filesystems(m: &MethodInfo, TData>) -> MethodResult { let message: &Message = m.msg; let mut iter = message.iter_init(); @@ -171,7 +158,7 @@ fn destroy_filesystems(m: &MethodInfo, TData>) -> MethodResult { Ok(vec![msg]) } -fn snapshot_filesystem(m: &MethodInfo, TData>) -> MethodResult { +pub fn snapshot_filesystem(m: &MethodInfo, TData>) -> MethodResult { let message: &Message = m.msg; let mut iter = message.iter_init(); @@ -219,7 +206,7 @@ fn snapshot_filesystem(m: &MethodInfo, TData>) -> MethodResult { Ok(vec![msg]) } -fn add_blockdevs(m: &MethodInfo, TData>, tier: BlockDevTier) -> MethodResult { +pub fn add_blockdevs(m: &MethodInfo, TData>, tier: BlockDevTier) -> MethodResult { let message: &Message = m.msg; let mut iter = message.iter_init(); @@ -273,15 +260,15 @@ fn add_blockdevs(m: &MethodInfo, TData>, tier: BlockDevTier) -> Meth Ok(vec![msg]) } -fn add_datadevs(m: &MethodInfo, TData>) -> MethodResult { +pub fn add_datadevs(m: &MethodInfo, TData>) -> MethodResult { add_blockdevs(m, BlockDevTier::Data) } -fn add_cachedevs(m: &MethodInfo, TData>) -> MethodResult { +pub fn add_cachedevs(m: &MethodInfo, TData>) -> MethodResult { add_blockdevs(m, BlockDevTier::Cache) } -fn rename_pool(m: &MethodInfo, TData>) -> MethodResult { +pub fn rename_pool(m: &MethodInfo, TData>) -> MethodResult { let message: &Message = m.msg; let mut iter = message.iter_init(); @@ -323,242 +310,3 @@ fn rename_pool(m: &MethodInfo, TData>) -> MethodResult { }; Ok(vec![msg]) } - -fn pool_operation( - tree: &Tree, TData>, - object_path: &dbus::Path<'static>, - closure: F, -) -> Result -where - F: Fn((Name, PoolUuid, &dyn Pool)) -> Result, - R: dbus::arg::Append, -{ - let dbus_context = tree.get_data(); - - let pool_path = tree - .get(object_path) - .expect("implicit argument must be in tree"); - - let pool_uuid = pool_path - .get_data() - .as_ref() - .ok_or_else(|| format!("no data for object path {}", object_path))? - .uuid; - - let engine = dbus_context.engine.borrow(); - let (pool_name, pool) = engine - .get_pool(pool_uuid) - .ok_or_else(|| format!("no pool corresponding to uuid {}", &pool_uuid))?; - - closure((pool_name, pool_uuid, pool)) -} - -/// Get a pool property and place it on the D-Bus. The property is -/// found by means of the getter method which takes a reference to a -/// Pool and obtains the property from the pool. -fn get_pool_property( - i: &mut IterAppend, - p: &PropInfo, TData>, - getter: F, -) -> Result<(), MethodErr> -where - F: Fn((Name, PoolUuid, &dyn Pool)) -> Result, - R: dbus::arg::Append, -{ - i.append( - pool_operation(p.tree, p.path.get_name(), getter).map_err(|ref e| MethodErr::failed(e))?, - ); - Ok(()) -} - -fn get_pool_name(i: &mut IterAppend, p: &PropInfo, TData>) -> Result<(), MethodErr> { - get_pool_property(i, p, |(name, _, _)| Ok(name.to_owned())) -} - -fn get_properties_shared( - m: &MethodInfo, TData>, - properties: &mut dyn Iterator, -) -> MethodResult { - let message: &Message = m.msg; - let object_path = &m.path; - - let return_message = message.method_return(); - - let return_value: HashMap>)> = properties - .unique() - .filter_map(|prop| match prop.as_str() { - consts::POOL_TOTAL_SIZE_PROP => Some(( - prop, - pool_operation(m.tree, object_path.get_name(), |(_, _, pool)| { - Ok((u128::from(*pool.total_physical_size()) - * devicemapper::SECTOR_SIZE as u128) - .to_string()) - }), - )), - consts::POOL_TOTAL_USED_PROP => Some(( - prop, - pool_operation(m.tree, object_path.get_name(), |(_, _, pool)| { - pool.total_physical_used() - .map_err(|e| e.to_string()) - .map(|size| { - (u128::from(*size) * devicemapper::SECTOR_SIZE as u128).to_string() - }) - }), - )), - _ => None, - }) - .map(|(key, result)| result_to_tuple(key, result)) - .collect(); - - Ok(vec![return_message.append1(return_value)]) -} - -fn get_all_properties(m: &MethodInfo, TData>) -> MethodResult { - get_properties_shared( - m, - &mut vec![consts::POOL_TOTAL_SIZE_PROP, consts::POOL_TOTAL_USED_PROP] - .into_iter() - .map(|s| s.to_string()), - ) -} - -fn get_properties(m: &MethodInfo, TData>) -> MethodResult { - let message: &Message = m.msg; - let mut iter = message.iter_init(); - let mut properties: Array = get_next_arg(&mut iter, 0)?; - get_properties_shared(m, &mut properties) -} - -pub fn create_dbus_pool<'a>( - dbus_context: &DbusContext, - parent: dbus::Path<'static>, - uuid: PoolUuid, - pool: &mut dyn Pool, -) -> dbus::Path<'a> { - let f = Factory::new_fn(); - - let create_filesystems_method = f - .method("CreateFilesystems", (), create_filesystems) - .in_arg(("specs", "as")) - // b: true if filesystems were created - // a(os): Array of tuples with object paths and names - // - // Rust representation: (bool, Vec<(dbus::Path, String)>) - .out_arg(("results", "(ba(os))")) - .out_arg(("return_code", "q")) - .out_arg(("return_string", "s")); - - let destroy_filesystems_method = f - .method("DestroyFilesystems", (), destroy_filesystems) - .in_arg(("filesystems", "ao")) - // b: true if filesystems were destroyed - // as: Array of UUIDs of destroyed filesystems - // - // Rust representation: (bool, Vec) - .out_arg(("results", "(bas)")) - .out_arg(("return_code", "q")) - .out_arg(("return_string", "s")); - - let add_blockdevs_method = f - .method("AddDataDevs", (), add_datadevs) - .in_arg(("devices", "as")) - // b: Indicates if any data devices were added - // ao: Array of object paths of created data devices - // - // Rust representation: (bool, Vec) - .out_arg(("results", "(bao)")) - .out_arg(("return_code", "q")) - .out_arg(("return_string", "s")); - - let add_cachedevs_method = f - .method("AddCacheDevs", (), add_cachedevs) - .in_arg(("devices", "as")) - // b: Indicates if any cache devices were added - // ao: Array of object paths of created cache devices - // - // Rust representation: (bool, Vec) - .out_arg(("results", "(bao)")) - .out_arg(("return_code", "q")) - .out_arg(("return_string", "s")); - - let rename_method = f - .method("SetName", (), rename_pool) - .in_arg(("name", "s")) - // b: false if no pool was renamed - // s: UUID of renamed pool - // - // Rust representation: (bool, String) - .out_arg(("result", "(bs)")) - .out_arg(("return_code", "q")) - .out_arg(("return_string", "s")); - - let snapshot_method = f - .method("SnapshotFilesystem", (), snapshot_filesystem) - .in_arg(("origin", "o")) - .in_arg(("snapshot_name", "s")) - // b: false if no new snapshot was created - // s: Object path of new snapshot - // - // Rust representation: (bool, String) - .out_arg(("result", "(bo)")) - .out_arg(("return_code", "q")) - .out_arg(("return_string", "s")); - - let name_property = f - .property::<&str, _>(consts::POOL_NAME_PROP, ()) - .access(Access::Read) - .emits_changed(EmitsChangedSignal::True) - .on_get(get_pool_name); - - let uuid_property = f - .property::<&str, _>("Uuid", ()) - .access(Access::Read) - .emits_changed(EmitsChangedSignal::Const) - .on_get(get_uuid); - - let get_all_properties_method = f - .method("GetAllProperties", (), get_all_properties) - // a{s(bv)}: Dictionary of property names to tuples - // In the tuple: - // b: Indicates whether the property value fetched was successful - // v: If b is true, represents the value for the given property - // If b is false, represents the error returned when fetching the property - .out_arg(("results", "a{s(bv)}")); - - let get_properties_method = f - .method("GetProperties", (), get_properties) - .in_arg(("properties", "as")) - // a{s(bv)}: Dictionary of property names to tuples - // In the tuple: - // b: Indicates whether the property value fetched was successful - // v: If b is true, represents the value for the given property - // If b is false, represents the error returned when fetching the property - .out_arg(("results", "a{s(bv)}")); - - let object_name = make_object_path(dbus_context); - - let object_path = f - .object_path(object_name, Some(OPContext::new(parent, uuid))) - .introspectable() - .add( - f.interface(consts::POOL_INTERFACE_NAME, ()) - .add_m(create_filesystems_method) - .add_m(destroy_filesystems_method) - .add_m(snapshot_method) - .add_m(add_blockdevs_method) - .add_m(add_cachedevs_method) - .add_m(rename_method) - .add_p(name_property) - .add_p(uuid_property), - ) - .add( - f.interface(consts::PROPERTY_FETCH_INTERFACE_NAME, ()) - .add_m(get_all_properties_method) - .add_m(get_properties_method), - ); - - let path = object_path.get_name().to_owned(); - dbus_context.actions.borrow_mut().push_add(object_path); - pool.set_dbus_path(MaybeDbusPath(Some(path.clone()))); - path -} diff --git a/src/dbus_api/pool/pool_2_0/mod.rs b/src/dbus_api/pool/pool_2_0/mod.rs new file mode 100644 index 0000000000..333cf7108f --- /dev/null +++ b/src/dbus_api/pool/pool_2_0/mod.rs @@ -0,0 +1,3 @@ +pub mod api; +pub mod methods; +pub mod props; diff --git a/src/dbus_api/pool/pool_2_0/props.rs b/src/dbus_api/pool/pool_2_0/props.rs new file mode 100644 index 0000000000..6f0f98b303 --- /dev/null +++ b/src/dbus_api/pool/pool_2_0/props.rs @@ -0,0 +1,39 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use dbus::{ + self, + arg::IterAppend, + tree::{MTFn, MethodErr, PropInfo}, +}; + +use crate::{ + dbus_api::{pool::shared::pool_operation, types::TData}, + engine::{Name, Pool, PoolUuid}, +}; + +/// Get a pool property and place it on the D-Bus. The property is +/// found by means of the getter method which takes a reference to a +/// Pool and obtains the property from the pool. +fn get_pool_property( + i: &mut IterAppend, + p: &PropInfo, TData>, + getter: F, +) -> Result<(), MethodErr> +where + F: Fn((Name, PoolUuid, &dyn Pool)) -> Result, + R: dbus::arg::Append, +{ + i.append( + pool_operation(p.tree, p.path.get_name(), getter).map_err(|ref e| MethodErr::failed(e))?, + ); + Ok(()) +} + +pub fn get_pool_name( + i: &mut IterAppend, + p: &PropInfo, TData>, +) -> Result<(), MethodErr> { + get_pool_property(i, p, |(name, _, _)| Ok(name.to_owned())) +} diff --git a/src/dbus_api/pool/shared.rs b/src/dbus_api/pool/shared.rs new file mode 100644 index 0000000000..3ae59aff70 --- /dev/null +++ b/src/dbus_api/pool/shared.rs @@ -0,0 +1,39 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use dbus::tree::{MTFn, Tree}; + +use crate::{ + dbus_api::types::TData, + engine::{Name, Pool, PoolUuid}, +}; + +pub fn pool_operation( + tree: &Tree, TData>, + object_path: &dbus::Path<'static>, + closure: F, +) -> Result +where + F: Fn((Name, PoolUuid, &dyn Pool)) -> Result, + R: dbus::arg::Append, +{ + let dbus_context = tree.get_data(); + + let pool_path = tree + .get(object_path) + .expect("implicit argument must be in tree"); + + let pool_uuid = pool_path + .get_data() + .as_ref() + .ok_or_else(|| format!("no data for object path {}", object_path))? + .uuid; + + let engine = dbus_context.engine.borrow(); + let (pool_name, pool) = engine + .get_pool(pool_uuid) + .ok_or_else(|| format!("no pool corresponding to uuid {}", &pool_uuid))?; + + closure((pool_name, pool_uuid, pool)) +} From 43c3f3088cd11c7637b01c5a9dee02014a034556 Mon Sep 17 00:00:00 2001 From: Bryan Gurney Date: Tue, 7 Jan 2020 11:19:43 -0500 Subject: [PATCH 061/109] Increase recommended development toolchain to 1.40 Signed-off-by: Bryan Gurney --- .travis.yml | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 117f0fe022..c0cf805cf2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,12 +22,12 @@ matrix: # MANDATORY CHECKS USING CURRENT DEVELOPMENT COMPILER # rustfmt - - rust: 1.39.0 + - rust: 1.40.0 before_script: - rustup component add rustfmt env: TASK=fmt-travis # clippy - - rust: 1.39.0 + - rust: 1.40.0 before_script: - rustup component add clippy env: TASK=clippy diff --git a/README.md b/README.md index a72aae6e01..ed191ee884 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ mailing list, if preferred. ### Setting up for development #### Development Compiler -The version of the compiler recommended for development is 1.39. Other +The version of the compiler recommended for development is 1.40. Other versions of the compiler may disagree with the CI tasks on some points, so should be avoided. From 680e601f705234ad904d7862a208186af180e4f9 Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 23 Dec 2019 15:51:33 -0500 Subject: [PATCH 062/109] Move some engine-specific actions out of bin/stratisd.rs Do this by making a new method, handle_event, which is part of the Engine API and making block_evaluate private. git is wierd, and rather than show the move of block_evaluate, it chooses instead to show a move of create_pool, which wasn't really the moved method. Signed-off-by: mulhern --- src/bin/stratisd.rs | 22 +-------- src/engine/engine.rs | 16 +++---- src/engine/sim_engine/engine.rs | 14 ++---- src/engine/strat_engine/engine.rs | 77 +++++++++++++++++++++---------- 4 files changed, 64 insertions(+), 65 deletions(-) diff --git a/src/bin/stratisd.rs b/src/bin/stratisd.rs index 4955d2bc87..13cb832a54 100644 --- a/src/bin/stratisd.rs +++ b/src/bin/stratisd.rs @@ -13,7 +13,6 @@ use std::{ fs::{File, OpenOptions}, io::{Read, Write}, os::unix::io::{AsRawFd, RawFd}, - path::PathBuf, process::exit, rc::Rc, }; @@ -34,8 +33,6 @@ use nix::{ use timerfd::{SetTimeFlags, TimerFd, TimerState}; use uuid::Uuid; -use devicemapper::Device; - #[cfg(feature = "dbus_enabled")] use libstratis::{ dbus_api::{DbusConnectionData, EventHandler}, @@ -265,23 +262,8 @@ impl<'a> UdevMonitor<'a> { /// data structures if so. fn handle_events(&mut self, engine: &mut dyn Engine, dbus_support: &mut MaybeDbusSupport) { while let Some(event) = self.socket.receive_event() { - if event.event_type() == libudev::EventType::Add - || event.event_type() == libudev::EventType::Change - { - let device = event.device(); - let new_pool_uuid = device.devnode().and_then(|devnode| { - device.devnum().and_then(|devnum| { - engine - .block_evaluate(Device::from(devnum), PathBuf::from(devnode)) - .unwrap_or(None) - }) - }); - if let Some(pool_uuid) = new_pool_uuid { - let (_, pool) = engine - .get_mut_pool(pool_uuid) - .expect("block_evaluate() returned a pool UUID, pool must be available"); - dbus_support.register_pool(pool_uuid, pool); - } + if let Some((pool_uuid, pool)) = engine.handle_event(&event) { + dbus_support.register_pool(pool_uuid, pool); } } } diff --git a/src/engine/engine.rs b/src/engine/engine.rs index e6ef6a9a56..65f7671868 100644 --- a/src/engine/engine.rs +++ b/src/engine/engine.rs @@ -11,7 +11,7 @@ use std::{ use chrono::{DateTime, Utc}; use uuid::Uuid; -use devicemapper::{Bytes, Device, Sectors}; +use devicemapper::{Bytes, Sectors}; use crate::{ engine::types::{ @@ -196,14 +196,12 @@ pub trait Engine: Debug { redundancy: Option, ) -> StratisResult>; - /// Evaluate a device node & devicemapper::Device to see if it's a valid - /// stratis device. If all the devices are present in the pool and the pool isn't already - /// up and running, it will get setup and the pool uuid will be returned. - fn block_evaluate( - &mut self, - device: Device, - dev_node: PathBuf, - ) -> StratisResult>; + /// Handle a libudev event. + /// If the handling action resulted in pool creation, return the pool + /// and its UUID. + /// + /// Precondition: the subsystem of the device evented on is "block". + fn handle_event(&mut self, event: &libudev::Event) -> Option<(PoolUuid, &mut dyn Pool)>; /// Destroy a pool. /// Ensures that the pool of the given UUID is absent on completion. diff --git a/src/engine/sim_engine/engine.rs b/src/engine/sim_engine/engine.rs index 146903705c..be4e9ec560 100644 --- a/src/engine/sim_engine/engine.rs +++ b/src/engine/sim_engine/engine.rs @@ -6,12 +6,10 @@ use std::{ cell::RefCell, collections::{hash_map::RandomState, HashSet}, iter::FromIterator, - path::{Path, PathBuf}, + path::Path, rc::Rc, }; -use devicemapper::Device; - use crate::{ engine::{ engine::{Engine, Eventable, Pool}, @@ -60,14 +58,8 @@ impl Engine for SimEngine { } } - fn block_evaluate( - &mut self, - device: Device, - dev_node: PathBuf, - ) -> StratisResult> { - assert_ne!(dev_node, PathBuf::from("/")); - assert_ne!(libc::dev_t::from(device), 0); - Ok(None) + fn handle_event(&mut self, _event: &libudev::Event) -> Option<(PoolUuid, &mut dyn Pool)> { + None } fn destroy_pool(&mut self, uuid: PoolUuid) -> StratisResult> { diff --git a/src/engine/strat_engine/engine.rs b/src/engine/strat_engine/engine.rs index 497ab33e53..513f68b8a6 100644 --- a/src/engine/strat_engine/engine.rs +++ b/src/engine/strat_engine/engine.rs @@ -8,6 +8,8 @@ use std::{ path::{Path, PathBuf}, }; +use libudev; + use devicemapper::{Device, DmNameBuf}; #[cfg(test)] @@ -169,31 +171,6 @@ impl StratEngine { pub fn teardown(self) -> StratisResult<()> { teardown_pools(self.pools) } -} - -impl Engine for StratEngine { - fn create_pool( - &mut self, - name: &str, - blockdev_paths: &[&Path], - redundancy: Option, - ) -> StratisResult> { - let redundancy = calculate_redundancy!(redundancy); - - validate_name(name)?; - - match self.pools.get_by_name(name) { - Some((_, pool)) => create_pool_idempotent_or_err(pool, name, blockdev_paths), - None => { - let (uuid, pool) = StratPool::initialize(name, blockdev_paths, redundancy)?; - - let name = Name::new(name.to_owned()); - devlinks::pool_added(&name); - self.pools.insert(name, uuid, pool); - Ok(CreateAction::Created(uuid)) - } - } - } /// Evaluate a device node & devicemapper::Device to see if it's a valid /// stratis device. If all the devices are present in the pool and the pool isn't already @@ -272,6 +249,56 @@ impl Engine for StratEngine { }; Ok(pool_uuid) } +} + +impl Engine for StratEngine { + fn handle_event(&mut self, event: &libudev::Event) -> Option<(PoolUuid, &mut dyn Pool)> { + let event_type = event.event_type(); + if event_type == libudev::EventType::Add || event_type == libudev::EventType::Change { + let device = event.device(); + device + .devnode() + .and_then(|devnode| { + device.devnum().and_then(|devnum| { + self.block_evaluate(Device::from(devnum), PathBuf::from(devnode)) + .unwrap_or(None) + }) + }) + .map(move |pool_uuid| { + ( + pool_uuid, + self.get_mut_pool(pool_uuid) + .expect("block_evaluate() returned a pool UUID, pool must be available") + .1, + ) + }) + } else { + None + } + } + + fn create_pool( + &mut self, + name: &str, + blockdev_paths: &[&Path], + redundancy: Option, + ) -> StratisResult> { + let redundancy = calculate_redundancy!(redundancy); + + validate_name(name)?; + + match self.pools.get_by_name(name) { + Some((_, pool)) => create_pool_idempotent_or_err(pool, name, blockdev_paths), + None => { + let (uuid, pool) = StratPool::initialize(name, blockdev_paths, redundancy)?; + + let name = Name::new(name.to_owned()); + devlinks::pool_added(&name); + self.pools.insert(name, uuid, pool); + Ok(CreateAction::Created(uuid)) + } + } + } fn destroy_pool(&mut self, uuid: PoolUuid) -> StratisResult> { if let Some((_, pool)) = self.pools.get_by_uuid(uuid) { From 4338c6785dcd56c0c9e5a6458eadd24f8c9135a4 Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 23 Dec 2019 10:42:25 -0500 Subject: [PATCH 063/109] Remove dependency in tests on block_device_ownership It's somewhat inappropriate, because block_device_ownership is especially designed to be only called if a udev event on the device has occurred and this is not what is occurring in the test. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/blockdevmgr.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/engine/strat_engine/backstore/blockdevmgr.rs b/src/engine/strat_engine/backstore/blockdevmgr.rs index dae77b944a..c4412d128f 100644 --- a/src/engine/strat_engine/backstore/blockdevmgr.rs +++ b/src/engine/strat_engine/backstore/blockdevmgr.rs @@ -547,7 +547,6 @@ mod tests { backstore::{find_all, get_metadata}, cmd, tests::{loopbacked, real}, - udev::block_device_ownership, }; use crate::engine::strat_engine::backstore::metadata::device_identifiers; @@ -613,7 +612,10 @@ mod tests { if i == index { assert_matches!( DevOwnership::from_udev_ownership( - &block_device_ownership(path).unwrap().unwrap().unwrap(), + &block_device_apply(path, |d| decide_ownership(d)) + .unwrap() + .unwrap() + .unwrap(), path ) .unwrap(), @@ -622,7 +624,10 @@ mod tests { } else { assert_matches!( DevOwnership::from_udev_ownership( - &block_device_ownership(path).unwrap().unwrap().unwrap(), + &block_device_apply(path, |d| decide_ownership(d)) + .unwrap() + .unwrap() + .unwrap(), path ) .unwrap(), From 0f9ac4d72ac50448343f5ff250c033fb18509b90 Mon Sep 17 00:00:00 2001 From: mulhern Date: Fri, 3 Jan 2020 12:11:59 -0500 Subject: [PATCH 064/109] Explode find_all into some separate functions Signed-off-by: mulhern --- src/engine/strat_engine/backstore/setup.rs | 308 +++++++++++---------- 1 file changed, 158 insertions(+), 150 deletions(-) diff --git a/src/engine/strat_engine/backstore/setup.rs b/src/engine/strat_engine/backstore/setup.rs index 9ea64ddab3..2adb2680ae 100644 --- a/src/engine/strat_engine/backstore/setup.rs +++ b/src/engine/strat_engine/backstore/setup.rs @@ -33,6 +33,161 @@ use crate::{ stratis::{ErrorEnum, StratisError, StratisResult}, }; +// A wrapper for obtaining the device number as a devicemapper Device +// which interprets absence of the value as an error, which it is in this +// context. +fn device_to_devno_wrapper(device: &libudev::Device) -> Result { + device + .devnum() + .ok_or_else(|| "udev entry did not contain a device number".into()) + .map(Device::from) +} + +// A wrapper around the metadata module's device_identifers method +// which also handles failure to open a device for reading. +// Returns an error if the device could not be opened for reading. +// Returns Ok(Err(...)) if there was an error while reading the +// Stratis identifiers from the device. +// Returns Ok(Ok(None)) if the identifers did not appear to be on +// the device. +fn device_identifiers_wrapper(devnode: &Path) -> Result, String>, String> { + OpenOptions::new() + .read(true) + .open(devnode) + .as_mut() + .map_err(|err| { + format!( + "device {} could not be opened for reading: {}", + devnode.display(), + err + ) + }) + .map(|f| { + device_identifiers(f) + .map_err(|err| { + format!( + "encountered an error while reading Stratis header for device {}: {}", + devnode.display(), + err + ) + }) + .map(|maybe_ids| maybe_ids.map(|(pool_uuid, _)| pool_uuid)) + }) +} + +// Use udev to identify all block devices and return the subset of those +// that have Stratis signatures. +fn find_all_block_devices_with_stratis_signatures( +) -> libudev::Result>> { + let context = libudev::Context::new()?; + let mut enumerator = block_enumerator(&context)?; + + let pool_map = enumerator.scan_devices()? + .filter(|dev| { + let initialized = dev.is_initialized(); + if !initialized { + debug!("Found a udev entry for a device identified as a block device, but udev also identified it as uninitialized, omitting the device from the set of devices to process, for safety"); + }; + initialized + }) + .filter(|dev| { + decide_ownership(dev) + .map_err(|err| { + warn!("Could not determine ownership of a udev block device because of an error processing udev information, omitting the device from the set of devices to process, for safety: {}", + err); + }) + .map(|decision| match decision { + UdevOwnership::Stratis | UdevOwnership::Unowned => true, + _ => false, + }) + .unwrap_or(false) + }) + .filter_map(|dev| match dev.devnode() { + Some(devnode) => { + match (device_to_devno_wrapper(&dev), device_identifiers_wrapper(devnode)) { + (Err(err), _) | (_, Err(err)) => { + warn!("udev identified device {} as a block device but {}, omitting the device from the set of devices to process", + devnode.display(), + err); + None + } + // FIXME: Refine error return in StaticHeader::setup(), + // so it can be used to distinguish between signficant + // and insignficant errors and then use that ability to + // distinguish here between different levels of + // severity. + (_, Ok(Err(err))) => { + debug!("udev identified device {} as a block device but {}, omitting the device from the set of devices to process", + devnode.display(), + err); + None + } + (_, Ok(Ok(None))) => None, + (Ok(devno), Ok(Ok(Some(pool_uuid)))) => Some((pool_uuid, devno, devnode.to_path_buf())), + } + } + None => { + warn!("udev identified a device as a block device, but the udev entry for the device had no device node, omitting the device from the set of devices to process"); + None + } + }) + .fold(HashMap::new(), |mut acc, (pool_uuid, device, devnode)| { + acc.entry(pool_uuid).or_insert_with(HashMap::new).insert(device, devnode); + acc + }); + + Ok(pool_map) +} + +// Find all devices identified by udev as Stratis devices. +fn find_all_stratis_devices() -> libudev::Result>> { + let context = libudev::Context::new()?; + let mut enumerator = block_enumerator(&context)?; + enumerator.match_property("ID_FS_TYPE", "stratis")?; + + let pool_map = enumerator.scan_devices()? + .filter(|dev| { + let initialized = dev.is_initialized(); + if !initialized { + warn!("Found a udev entry for a device identified as a Stratis device, but udev also identified it as uninitialized, omitting the device from the set of devices to process, for safety"); + }; + initialized + }) + .filter(|dev| !is_multipath_member(dev) + .map_err(|err| { + warn!("Could not certainly determine whether a device was a multipath member because of an error processing udev information, omitting the device from the set of devices to process, for safety: {}", + err); + }) + .unwrap_or(true)) + .filter_map(|dev| match dev.devnode() { + Some(devnode) => { + match (device_to_devno_wrapper(&dev), device_identifiers_wrapper(devnode)) { + (Err(err), _) | (_, Err(err)) | (_, Ok(Err(err)))=> { + warn!("udev identified device {} as a Stratis device but {}, omitting the device from the set of devices to process", + devnode.display(), + err); + None + } + (_, Ok(Ok(None))) => { + warn!("udev identified device {} as a Stratis device but there appeared to be no Stratis metadata on the device, omitting the device from the set of devices to process", + devnode.display()); + None + } + (Ok(devno), Ok(Ok(Some(pool_uuid)))) => Some((pool_uuid, devno, devnode.to_path_buf())), + } + } + None => { + warn!("udev identified a device as a Stratis device, but the udev entry for the device had no device node, omitting the the device from the set of devices to process"); + None + } + }) + .fold(HashMap::new(), |mut acc, (pool_uuid, device, devnode)| { + acc.entry(pool_uuid).or_insert_with(HashMap::new).insert(device, devnode); + acc + }); + Ok(pool_map) +} + /// Retrieve all block devices that should be made use of by the /// Stratis engine. This excludes Stratis block devices that appear to be /// multipath members. @@ -49,98 +204,8 @@ use crate::{ /// /// Returns a map of pool uuids to a map of devices to devnodes for each pool. pub fn find_all() -> libudev::Result>> { - // A wrapper for obtaining the device number as a devicemapper Device - // which interprets absence of the value as an error, which it is in this - // context. - fn device_to_devno_wrapper(device: &libudev::Device) -> Result { - device - .devnum() - .ok_or_else(|| "udev entry did not contain a device number".into()) - .map(Device::from) - } - - // A wrapper around the metadata module's device_identifers method - // which also handles failure to open a device for reading. - // Returns an error if the device could not be opened for reading. - // Returns Ok(Err(...)) if there was an error while reading the - // Stratis identifiers from the device. - // Returns Ok(Ok(None)) if the identifers did not appear to be on - // the device. - fn device_identifiers_wrapper( - devnode: &Path, - ) -> Result, String>, String> { - OpenOptions::new() - .read(true) - .open(devnode) - .as_mut() - .map_err(|err| { - format!( - "device {} could not be opened for reading: {}", - devnode.display(), - err - ) - }) - .map(|f| { - device_identifiers(f) - .map_err(|err| { - format!( - "encountered an error while reading Stratis header for device {}: {}", - devnode.display(), - err - ) - }) - .map(|maybe_ids| maybe_ids.map(|(pool_uuid, _)| pool_uuid)) - }) - } - info!("Beginning initial search for Stratis block devices"); - let pool_map = { - let context = libudev::Context::new()?; - let mut enumerator = block_enumerator(&context)?; - enumerator.match_property("ID_FS_TYPE", "stratis")?; - - enumerator - .scan_devices()? - .filter(|dev| { - let initialized = dev.is_initialized(); - if !initialized { - warn!("Found a udev entry for a device identified as a Stratis device, but udev also identified it as uninitialized, omitting the device from the set of devices to process, for safety"); - }; - initialized - }) - .filter(|dev| !is_multipath_member(dev) - .map_err(|err| { - warn!("Could not certainly determine whether a device was a multipath member because of an error processing udev information, omitting the device from the set of devices to process, for safety: {}", - err); - }) - .unwrap_or(true)) - .filter_map(|dev| match dev.devnode() { - Some(devnode) => { - match (device_to_devno_wrapper(&dev), device_identifiers_wrapper(devnode)) { - (Err(err), _) | (_, Err(err)) | (_, Ok(Err(err)))=> { - warn!("udev identified device {} as a Stratis device but {}, omitting the device from the set of devices to process", - devnode.display(), - err); - None - } - (_, Ok(Ok(None))) => { - warn!("udev identified device {} as a Stratis device but there appeared to be no Stratis metadata on the device, omitting the device from the set of devices to process", - devnode.display()); - None - } - (Ok(devno), Ok(Ok(Some(pool_uuid)))) => Some((pool_uuid, devno, devnode.to_path_buf())), - } - } - None => { - warn!("udev identified a device as a Stratis device, but the udev entry for the device had no device node, omitting the the device from the set of devices to process"); - None - } - }) - .fold(HashMap::new(), |mut acc, (pool_uuid, device, devnode)| { - acc.entry(pool_uuid).or_insert_with(HashMap::new).insert(device, devnode); - acc - }) - }; + let pool_map = find_all_stratis_devices()?; if pool_map.is_empty() { // No Stratis devices have been found, possible reasons are: @@ -152,70 +217,13 @@ pub fn find_all() -> libudev::Result> // // Try again to find Stratis block devices, but this time enumerate // all block devices, not just all the ones that can be identified - // as Stratis blockdevs by udev, and return only those that appear + // as Stratis blockdevs by udev, and process only those that appear // unclaimed or appear to be claimed by Stratis (and not // multipath members). info!("Could not identify any Stratis devices by a udev search for devices with ID_FS_TYPE=\"stratis\"; using fallback search mechanism"); - let context = libudev::Context::new()?; - let mut enumerator = block_enumerator(&context)?; - - let pool_map = enumerator - .scan_devices()? - .filter(|dev| { - let initialized = dev.is_initialized(); - if !initialized { - debug!("Found a udev entry for a device identified as a block device, but udev also identified it as uninitialized, omitting the device from the set of devices to process, for safety"); - }; - initialized - }) - .filter(|dev| { - decide_ownership(dev) - .map_err(|err| { - warn!("Could not determine ownership of a udev block device because of an error processing udev information, omitting the device from the set of devices to process, for safety: {}", - err); - }) - .map(|decision| match decision { - UdevOwnership::Stratis | UdevOwnership::Unowned => true, - _ => false, - }) - .unwrap_or(false) - }) - .filter_map(|dev| match dev.devnode() { - Some(devnode) => { - match (device_to_devno_wrapper(&dev), device_identifiers_wrapper(devnode)) { - (Err(err), _) | (_, Err(err)) => { - warn!("udev identified device {} as a block device but {}, omitting the device from the set of devices to process", - devnode.display(), - err); - None - } - // FIXME: Refine error return in StaticHeader::setup(), - // so it can be used to distinguish between signficant - // and insignficant errors and then use that ability to - // distinguish here between different levels of - // severity. - (_, Ok(Err(err))) => { - debug!("udev identified device {} as a block device but {}, omitting the device from the set of devices to process", - devnode.display(), - err); - None - } - (_, Ok(Ok(None))) => None, - (Ok(devno), Ok(Ok(Some(pool_uuid)))) => Some((pool_uuid, devno, devnode.to_path_buf())), - } - } - None => { - warn!("udev identified a device as a block device, but the udev entry for the device had no device node, omitting the device from the set of devices to process"); - None - } - }) - .fold(HashMap::new(), |mut acc, (pool_uuid, device, devnode)| { - acc.entry(pool_uuid).or_insert_with(HashMap::new).insert(device, devnode); - acc - }); - Ok(pool_map) + find_all_block_devices_with_stratis_signatures() } else { Ok(pool_map) } From 7719b27c34ecfeba0fce31cc6189b77ae5cc1b67 Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 6 Jan 2020 09:39:52 -0500 Subject: [PATCH 065/109] Move all device identification into a separate module Signed-off-by: mulhern --- src/engine/strat_engine/backstore/identify.rs | 219 ++++++++++++++++++ src/engine/strat_engine/backstore/mod.rs | 8 +- src/engine/strat_engine/backstore/setup.rs | 203 +--------------- 3 files changed, 223 insertions(+), 207 deletions(-) create mode 100644 src/engine/strat_engine/backstore/identify.rs diff --git a/src/engine/strat_engine/backstore/identify.rs b/src/engine/strat_engine/backstore/identify.rs new file mode 100644 index 0000000000..a068ae11fd --- /dev/null +++ b/src/engine/strat_engine/backstore/identify.rs @@ -0,0 +1,219 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Discover or identify devices that may belong to Stratis. + +use std::{ + collections::HashMap, + fs::OpenOptions, + path::{Path, PathBuf}, +}; + +use libudev; + +use devicemapper::Device; + +use crate::engine::{ + strat_engine::{ + backstore::metadata::device_identifiers, + udev::{block_enumerator, decide_ownership, is_multipath_member, UdevOwnership}, + }, + types::PoolUuid, +}; + +// A wrapper for obtaining the device number as a devicemapper Device +// which interprets absence of the value as an error, which it is in this +// context. +fn device_to_devno_wrapper(device: &libudev::Device) -> Result { + device + .devnum() + .ok_or_else(|| "udev entry did not contain a device number".into()) + .map(Device::from) +} + +// A wrapper around the metadata module's device_identifers method +// which also handles failure to open a device for reading. +// Returns an error if the device could not be opened for reading. +// Returns Ok(Err(...)) if there was an error while reading the +// Stratis identifiers from the device. +// Returns Ok(Ok(None)) if the identifers did not appear to be on +// the device. +fn device_identifiers_wrapper(devnode: &Path) -> Result, String>, String> { + OpenOptions::new() + .read(true) + .open(devnode) + .as_mut() + .map_err(|err| { + format!( + "device {} could not be opened for reading: {}", + devnode.display(), + err + ) + }) + .map(|f| { + device_identifiers(f) + .map_err(|err| { + format!( + "encountered an error while reading Stratis header for device {}: {}", + devnode.display(), + err + ) + }) + .map(|maybe_ids| maybe_ids.map(|(pool_uuid, _)| pool_uuid)) + }) +} + +// Use udev to identify all block devices and return the subset of those +// that have Stratis signatures. +fn find_all_block_devices_with_stratis_signatures( +) -> libudev::Result>> { + let context = libudev::Context::new()?; + let mut enumerator = block_enumerator(&context)?; + + let pool_map = enumerator.scan_devices()? + .filter(|dev| { + let initialized = dev.is_initialized(); + if !initialized { + debug!("Found a udev entry for a device identified as a block device, but udev also identified it as uninitialized, omitting the device from the set of devices to process, for safety"); + }; + initialized + }) + .filter(|dev| { + decide_ownership(dev) + .map_err(|err| { + warn!("Could not determine ownership of a udev block device because of an error processing udev information, omitting the device from the set of devices to process, for safety: {}", + err); + }) + .map(|decision| match decision { + UdevOwnership::Stratis | UdevOwnership::Unowned => true, + _ => false, + }) + .unwrap_or(false) + }) + .filter_map(|dev| match dev.devnode() { + Some(devnode) => { + match (device_to_devno_wrapper(&dev), device_identifiers_wrapper(devnode)) { + (Err(err), _) | (_, Err(err)) => { + warn!("udev identified device {} as a block device but {}, omitting the device from the set of devices to process", + devnode.display(), + err); + None + } + // FIXME: Refine error return in StaticHeader::setup(), + // so it can be used to distinguish between signficant + // and insignficant errors and then use that ability to + // distinguish here between different levels of + // severity. + (_, Ok(Err(err))) => { + debug!("udev identified device {} as a block device but {}, omitting the device from the set of devices to process", + devnode.display(), + err); + None + } + (_, Ok(Ok(None))) => None, + (Ok(devno), Ok(Ok(Some(pool_uuid)))) => Some((pool_uuid, devno, devnode.to_path_buf())), + } + } + None => { + warn!("udev identified a device as a block device, but the udev entry for the device had no device node, omitting the device from the set of devices to process"); + None + } + }) + .fold(HashMap::new(), |mut acc, (pool_uuid, device, devnode)| { + acc.entry(pool_uuid).or_insert_with(HashMap::new).insert(device, devnode); + acc + }); + + Ok(pool_map) +} + +// Find all devices identified by udev as Stratis devices. +fn find_all_stratis_devices() -> libudev::Result>> { + let context = libudev::Context::new()?; + let mut enumerator = block_enumerator(&context)?; + enumerator.match_property("ID_FS_TYPE", "stratis")?; + + let pool_map = enumerator.scan_devices()? + .filter(|dev| { + let initialized = dev.is_initialized(); + if !initialized { + warn!("Found a udev entry for a device identified as a Stratis device, but udev also identified it as uninitialized, omitting the device from the set of devices to process, for safety"); + }; + initialized + }) + .filter(|dev| !is_multipath_member(dev) + .map_err(|err| { + warn!("Could not certainly determine whether a device was a multipath member because of an error processing udev information, omitting the device from the set of devices to process, for safety: {}", + err); + }) + .unwrap_or(true)) + .filter_map(|dev| match dev.devnode() { + Some(devnode) => { + match (device_to_devno_wrapper(&dev), device_identifiers_wrapper(devnode)) { + (Err(err), _) | (_, Err(err)) | (_, Ok(Err(err)))=> { + warn!("udev identified device {} as a Stratis device but {}, omitting the device from the set of devices to process", + devnode.display(), + err); + None + } + (_, Ok(Ok(None))) => { + warn!("udev identified device {} as a Stratis device but there appeared to be no Stratis metadata on the device, omitting the device from the set of devices to process", + devnode.display()); + None + } + (Ok(devno), Ok(Ok(Some(pool_uuid)))) => Some((pool_uuid, devno, devnode.to_path_buf())), + } + } + None => { + warn!("udev identified a device as a Stratis device, but the udev entry for the device had no device node, omitting the the device from the set of devices to process"); + None + } + }) + .fold(HashMap::new(), |mut acc, (pool_uuid, device, devnode)| { + acc.entry(pool_uuid).or_insert_with(HashMap::new).insert(device, devnode); + acc + }); + Ok(pool_map) +} + +/// Retrieve all block devices that should be made use of by the +/// Stratis engine. This excludes Stratis block devices that appear to be +/// multipath members. +/// +/// Includes a fallback path, which is used if no Stratis block devices are +/// found using the obvious udev property- and enumerator-based approach. +/// This fallback path is more expensive, because it must search all block +/// devices via udev rather than just all Stratis block devices. +/// +/// Omits any device that appears problematic in some way. +/// +/// Return an error only on a failure to construct or scan with a udev +/// enumerator. +/// +/// Returns a map of pool uuids to a map of devices to devnodes for each pool. +pub fn find_all() -> libudev::Result>> { + info!("Beginning initial search for Stratis block devices"); + let pool_map = find_all_stratis_devices()?; + + if pool_map.is_empty() { + // No Stratis devices have been found, possible reasons are: + // 1. There are none + // 2. There are some but libblkid version is less than 2.32, so + // Stratis devices are not recognized by udev. + // 3. There are many incomplete udev entries, because this code is + // being run before the udev database is populated. + // + // Try again to find Stratis block devices, but this time enumerate + // all block devices, not just all the ones that can be identified + // as Stratis blockdevs by udev, and process only those that appear + // unclaimed or appear to be claimed by Stratis (and not + // multipath members). + + info!("Could not identify any Stratis devices by a udev search for devices with ID_FS_TYPE=\"stratis\"; using fallback search mechanism"); + + find_all_block_devices_with_stratis_signatures() + } else { + Ok(pool_map) + } +} diff --git a/src/engine/strat_engine/backstore/mod.rs b/src/engine/strat_engine/backstore/mod.rs index ce5745aeed..ede1426842 100644 --- a/src/engine/strat_engine/backstore/mod.rs +++ b/src/engine/strat_engine/backstore/mod.rs @@ -9,15 +9,13 @@ mod blockdevmgr; mod cache_tier; mod data_tier; mod device; +mod identify; mod metadata; mod range_alloc; mod setup; mod shared; pub use self::{ - backstore::Backstore, - blockdev::StratBlockDev, - device::DevOwnership, - metadata::MDADataSize, - setup::{find_all, get_metadata}, + backstore::Backstore, blockdev::StratBlockDev, device::DevOwnership, identify::find_all, + metadata::MDADataSize, setup::get_metadata, }; diff --git a/src/engine/strat_engine/backstore/setup.rs b/src/engine/strat_engine/backstore/setup.rs index 2adb2680ae..e1c5df865c 100644 --- a/src/engine/strat_engine/backstore/setup.rs +++ b/src/engine/strat_engine/backstore/setup.rs @@ -12,7 +12,6 @@ use std::{ }; use chrono::{DateTime, Utc}; -use libudev; use serde_json; use devicemapper::{Device, Sectors}; @@ -20,215 +19,15 @@ use devicemapper::{Device, Sectors}; use crate::{ engine::{ strat_engine::{ - backstore::{ - blockdev::StratBlockDev, - metadata::{device_identifiers, BDA}, - }, + backstore::{blockdev::StratBlockDev, metadata::BDA}, device::blkdev_size, serde_structs::{BackstoreSave, BaseBlockDevSave, PoolSave}, - udev::{block_enumerator, decide_ownership, is_multipath_member, UdevOwnership}, }, types::{BlockDevTier, DevUuid, PoolUuid}, }, stratis::{ErrorEnum, StratisError, StratisResult}, }; -// A wrapper for obtaining the device number as a devicemapper Device -// which interprets absence of the value as an error, which it is in this -// context. -fn device_to_devno_wrapper(device: &libudev::Device) -> Result { - device - .devnum() - .ok_or_else(|| "udev entry did not contain a device number".into()) - .map(Device::from) -} - -// A wrapper around the metadata module's device_identifers method -// which also handles failure to open a device for reading. -// Returns an error if the device could not be opened for reading. -// Returns Ok(Err(...)) if there was an error while reading the -// Stratis identifiers from the device. -// Returns Ok(Ok(None)) if the identifers did not appear to be on -// the device. -fn device_identifiers_wrapper(devnode: &Path) -> Result, String>, String> { - OpenOptions::new() - .read(true) - .open(devnode) - .as_mut() - .map_err(|err| { - format!( - "device {} could not be opened for reading: {}", - devnode.display(), - err - ) - }) - .map(|f| { - device_identifiers(f) - .map_err(|err| { - format!( - "encountered an error while reading Stratis header for device {}: {}", - devnode.display(), - err - ) - }) - .map(|maybe_ids| maybe_ids.map(|(pool_uuid, _)| pool_uuid)) - }) -} - -// Use udev to identify all block devices and return the subset of those -// that have Stratis signatures. -fn find_all_block_devices_with_stratis_signatures( -) -> libudev::Result>> { - let context = libudev::Context::new()?; - let mut enumerator = block_enumerator(&context)?; - - let pool_map = enumerator.scan_devices()? - .filter(|dev| { - let initialized = dev.is_initialized(); - if !initialized { - debug!("Found a udev entry for a device identified as a block device, but udev also identified it as uninitialized, omitting the device from the set of devices to process, for safety"); - }; - initialized - }) - .filter(|dev| { - decide_ownership(dev) - .map_err(|err| { - warn!("Could not determine ownership of a udev block device because of an error processing udev information, omitting the device from the set of devices to process, for safety: {}", - err); - }) - .map(|decision| match decision { - UdevOwnership::Stratis | UdevOwnership::Unowned => true, - _ => false, - }) - .unwrap_or(false) - }) - .filter_map(|dev| match dev.devnode() { - Some(devnode) => { - match (device_to_devno_wrapper(&dev), device_identifiers_wrapper(devnode)) { - (Err(err), _) | (_, Err(err)) => { - warn!("udev identified device {} as a block device but {}, omitting the device from the set of devices to process", - devnode.display(), - err); - None - } - // FIXME: Refine error return in StaticHeader::setup(), - // so it can be used to distinguish between signficant - // and insignficant errors and then use that ability to - // distinguish here between different levels of - // severity. - (_, Ok(Err(err))) => { - debug!("udev identified device {} as a block device but {}, omitting the device from the set of devices to process", - devnode.display(), - err); - None - } - (_, Ok(Ok(None))) => None, - (Ok(devno), Ok(Ok(Some(pool_uuid)))) => Some((pool_uuid, devno, devnode.to_path_buf())), - } - } - None => { - warn!("udev identified a device as a block device, but the udev entry for the device had no device node, omitting the device from the set of devices to process"); - None - } - }) - .fold(HashMap::new(), |mut acc, (pool_uuid, device, devnode)| { - acc.entry(pool_uuid).or_insert_with(HashMap::new).insert(device, devnode); - acc - }); - - Ok(pool_map) -} - -// Find all devices identified by udev as Stratis devices. -fn find_all_stratis_devices() -> libudev::Result>> { - let context = libudev::Context::new()?; - let mut enumerator = block_enumerator(&context)?; - enumerator.match_property("ID_FS_TYPE", "stratis")?; - - let pool_map = enumerator.scan_devices()? - .filter(|dev| { - let initialized = dev.is_initialized(); - if !initialized { - warn!("Found a udev entry for a device identified as a Stratis device, but udev also identified it as uninitialized, omitting the device from the set of devices to process, for safety"); - }; - initialized - }) - .filter(|dev| !is_multipath_member(dev) - .map_err(|err| { - warn!("Could not certainly determine whether a device was a multipath member because of an error processing udev information, omitting the device from the set of devices to process, for safety: {}", - err); - }) - .unwrap_or(true)) - .filter_map(|dev| match dev.devnode() { - Some(devnode) => { - match (device_to_devno_wrapper(&dev), device_identifiers_wrapper(devnode)) { - (Err(err), _) | (_, Err(err)) | (_, Ok(Err(err)))=> { - warn!("udev identified device {} as a Stratis device but {}, omitting the device from the set of devices to process", - devnode.display(), - err); - None - } - (_, Ok(Ok(None))) => { - warn!("udev identified device {} as a Stratis device but there appeared to be no Stratis metadata on the device, omitting the device from the set of devices to process", - devnode.display()); - None - } - (Ok(devno), Ok(Ok(Some(pool_uuid)))) => Some((pool_uuid, devno, devnode.to_path_buf())), - } - } - None => { - warn!("udev identified a device as a Stratis device, but the udev entry for the device had no device node, omitting the the device from the set of devices to process"); - None - } - }) - .fold(HashMap::new(), |mut acc, (pool_uuid, device, devnode)| { - acc.entry(pool_uuid).or_insert_with(HashMap::new).insert(device, devnode); - acc - }); - Ok(pool_map) -} - -/// Retrieve all block devices that should be made use of by the -/// Stratis engine. This excludes Stratis block devices that appear to be -/// multipath members. -/// -/// Includes a fallback path, which is used if no Stratis block devices are -/// found using the obvious udev property- and enumerator-based approach. -/// This fallback path is more expensive, because it must search all block -/// devices via udev rather than just all Stratis block devices. -/// -/// Omits any device that appears problematic in some way. -/// -/// Return an error only on a failure to construct or scan with a udev -/// enumerator. -/// -/// Returns a map of pool uuids to a map of devices to devnodes for each pool. -pub fn find_all() -> libudev::Result>> { - info!("Beginning initial search for Stratis block devices"); - let pool_map = find_all_stratis_devices()?; - - if pool_map.is_empty() { - // No Stratis devices have been found, possible reasons are: - // 1. There are none - // 2. There are some but libblkid version is less than 2.32, so - // Stratis devices are not recognized by udev. - // 3. There are many incomplete udev entries, because this code is - // being run before the udev database is populated. - // - // Try again to find Stratis block devices, but this time enumerate - // all block devices, not just all the ones that can be identified - // as Stratis blockdevs by udev, and process only those that appear - // unclaimed or appear to be claimed by Stratis (and not - // multipath members). - - info!("Could not identify any Stratis devices by a udev search for devices with ID_FS_TYPE=\"stratis\"; using fallback search mechanism"); - - find_all_block_devices_with_stratis_signatures() - } else { - Ok(pool_map) - } -} - /// Get the most recent metadata from a set of Devices for a given pool UUID. /// Returns None if no metadata found for this pool. pub fn get_metadata( From 944f688ad36342e776f39fa0280ad37ade8d09f6 Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 6 Jan 2020 11:18:06 -0500 Subject: [PATCH 066/109] Have device_identifiers_wrapper return both device identifiers The device UUID is useful in some contexts soon to be introduced. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/identify.rs | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/engine/strat_engine/backstore/identify.rs b/src/engine/strat_engine/backstore/identify.rs index a068ae11fd..6a161ee8a7 100644 --- a/src/engine/strat_engine/backstore/identify.rs +++ b/src/engine/strat_engine/backstore/identify.rs @@ -19,7 +19,7 @@ use crate::engine::{ backstore::metadata::device_identifiers, udev::{block_enumerator, decide_ownership, is_multipath_member, UdevOwnership}, }, - types::PoolUuid, + types::{DevUuid, PoolUuid}, }; // A wrapper for obtaining the device number as a devicemapper Device @@ -39,7 +39,9 @@ fn device_to_devno_wrapper(device: &libudev::Device) -> Result { // Stratis identifiers from the device. // Returns Ok(Ok(None)) if the identifers did not appear to be on // the device. -fn device_identifiers_wrapper(devnode: &Path) -> Result, String>, String> { +fn device_identifiers_wrapper( + devnode: &Path, +) -> Result, String>, String> { OpenOptions::new() .read(true) .open(devnode) @@ -52,15 +54,13 @@ fn device_identifiers_wrapper(devnode: &Path) -> Result, ) }) .map(|f| { - device_identifiers(f) - .map_err(|err| { - format!( - "encountered an error while reading Stratis header for device {}: {}", - devnode.display(), - err - ) - }) - .map(|maybe_ids| maybe_ids.map(|(pool_uuid, _)| pool_uuid)) + device_identifiers(f).map_err(|err| { + format!( + "encountered an error while reading Stratis header for device {}: {}", + devnode.display(), + err + ) + }) }) } @@ -112,7 +112,7 @@ fn find_all_block_devices_with_stratis_signatures( None } (_, Ok(Ok(None))) => None, - (Ok(devno), Ok(Ok(Some(pool_uuid)))) => Some((pool_uuid, devno, devnode.to_path_buf())), + (Ok(devno), Ok(Ok(Some((pool_uuid, _))))) => Some((pool_uuid, devno, devnode.to_path_buf())), } } None => { @@ -162,7 +162,7 @@ fn find_all_stratis_devices() -> libudev::Result Some((pool_uuid, devno, devnode.to_path_buf())), + (Ok(devno), Ok(Ok(Some((pool_uuid, _))))) => Some((pool_uuid, devno, devnode.to_path_buf())), } } None => { From 12111fffdab379f17c2ebdf292f797eb0e312a76 Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 6 Jan 2020 10:09:41 -0500 Subject: [PATCH 067/109] Introduce a new method, identify_block_device, in the identify module Use it in block_evaluate and discard any newly introduced dead code. Have block_evaluate take a udev device as an argument. This makes the udev lookup constant time instead of linear in the number of block devices. Previously, the device node was discovered from the udev database entry, and then the entries for all block devices in the udev database were searched in order to locate a block device that had the specified device node in order to locate the relevant udev database entry. Simplify block_evaluate's return type. It never returns an error, logging any internal errors instead. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/device.rs | 7 -- src/engine/strat_engine/backstore/identify.rs | 85 +++++++++++++++++++ src/engine/strat_engine/backstore/mod.rs | 8 +- src/engine/strat_engine/engine.rs | 48 ++++------- src/engine/strat_engine/udev.rs | 38 --------- 5 files changed, 106 insertions(+), 80 deletions(-) diff --git a/src/engine/strat_engine/backstore/device.rs b/src/engine/strat_engine/backstore/device.rs index 5c95f63bda..177490cb54 100644 --- a/src/engine/strat_engine/backstore/device.rs +++ b/src/engine/strat_engine/backstore/device.rs @@ -46,13 +46,6 @@ pub enum DevOwnership { } impl DevOwnership { - pub fn stratis_identifiers(&self) -> Option<(PoolUuid, DevUuid)> { - match self { - DevOwnership::Ours(pool_uuid, dev_uuid) => Some((*pool_uuid, *dev_uuid)), - _ => None, - } - } - /// Given a udev assignment of ownership and the devnode for the device /// in question, do some additional work to determine DevOwnership. pub fn from_udev_ownership( diff --git a/src/engine/strat_engine/backstore/identify.rs b/src/engine/strat_engine/backstore/identify.rs index 6a161ee8a7..4d005069fc 100644 --- a/src/engine/strat_engine/backstore/identify.rs +++ b/src/engine/strat_engine/backstore/identify.rs @@ -177,6 +177,91 @@ fn find_all_stratis_devices() -> libudev::Result Option<(PoolUuid, DevUuid, Device, PathBuf)> { + let initialized = dev.is_initialized(); + if !initialized { + debug!("Found a udev entry for a device identified as a block device, but udev also identified it as uninitialized, disregarding the device"); + return None; + }; + + match decide_ownership(dev) { + Err(err) => { + warn!("Could not determine ownership of a udev block device because of an error processing udev information, disregarding the device: {}", + err); + None + } + Ok(ownership) => match ownership { + UdevOwnership::Stratis => match dev.devnode() { + Some(devnode) => { + match ( + device_to_devno_wrapper(dev), + device_identifiers_wrapper(devnode), + ) { + (Err(err), _) | (_, Err(err)) | (_, Ok(Err(err))) => { + warn!("udev identified device {} as a Stratis device but {}, disregarding the device", + devnode.display(), + err); + None + } + (_, Ok(Ok(None))) => { + warn!("udev identified device {} as a Stratis device but there appeared to be no Stratis metadata on the device, disregarding the device", + devnode.display()); + None + } + (Ok(devno), Ok(Ok(Some((pool_uuid, device_uuid))))) => { + Some((pool_uuid, device_uuid, devno, devnode.to_path_buf())) + } + } + } + None => { + warn!("udev identified a device as a Stratis device, but the udev entry for the device had no device node, disregarding device"); + None + } + }, + UdevOwnership::Unowned => match dev.devnode() { + Some(devnode) => { + match ( + device_to_devno_wrapper(dev), + device_identifiers_wrapper(devnode), + ) { + (Err(err), _) | (_, Err(err)) => { + warn!("udev identified device {} as a block device but {}, disregarding device", + devnode.display(), + err); + None + } + // FIXME: Refine error return in StaticHeader::setup(), + // so it can be used to distinguish between signficant + // and insignficant errors and then use that ability to + // distinguish here between different levels of + // severity. + (_, Ok(Err(err))) => { + debug!("udev identified device {} as a block device but {}, disregarding device", + devnode.display(), + err); + None + } + (_, Ok(Ok(None))) => None, + (Ok(devno), Ok(Ok(Some((pool_uuid, device_uuid))))) => { + Some((pool_uuid, device_uuid, devno, devnode.to_path_buf())) + } + } + } + None => { + warn!("udev identified a device as a block device, but the udev entry for the device had no device node, omitting the device from the set of devices to process"); + None + } + }, + _ => None, + }, + } +} + /// Retrieve all block devices that should be made use of by the /// Stratis engine. This excludes Stratis block devices that appear to be /// multipath members. diff --git a/src/engine/strat_engine/backstore/mod.rs b/src/engine/strat_engine/backstore/mod.rs index ede1426842..03ac37031a 100644 --- a/src/engine/strat_engine/backstore/mod.rs +++ b/src/engine/strat_engine/backstore/mod.rs @@ -16,6 +16,10 @@ mod setup; mod shared; pub use self::{ - backstore::Backstore, blockdev::StratBlockDev, device::DevOwnership, identify::find_all, - metadata::MDADataSize, setup::get_metadata, + backstore::Backstore, + blockdev::StratBlockDev, + device::DevOwnership, + identify::{find_all, identify_block_device}, + metadata::MDADataSize, + setup::get_metadata, }; diff --git a/src/engine/strat_engine/engine.rs b/src/engine/strat_engine/engine.rs index 513f68b8a6..aab0133eea 100644 --- a/src/engine/strat_engine/engine.rs +++ b/src/engine/strat_engine/engine.rs @@ -21,13 +21,12 @@ use crate::{ event::get_engine_listener_list, shared::create_pool_idempotent_or_err, strat_engine::{ - backstore::{find_all, get_metadata, DevOwnership}, + backstore::{find_all, get_metadata, identify_block_device}, cmd::verify_binaries, devlinks, dm::{get_dm, get_dm_init}, names::validate_name, pool::{check_metadata, StratPool}, - udev::block_device_ownership, }, structures::Table, types::{CreateAction, DeleteAction, RenameAction}, @@ -172,22 +171,15 @@ impl StratEngine { teardown_pools(self.pools) } - /// Evaluate a device node & devicemapper::Device to see if it's a valid - /// stratis device. If all the devices are present in the pool and the pool isn't already + /// Given a udev database entry, process the entry. + /// + /// If all the devices are present in the pool and the pool isn't already /// up and running, it will get setup and the pool uuid will be returned. /// - /// Returns an error if the status of the block device can not be evaluated. /// Logs a warning if the block devices appears to be a Stratis block /// device and no pool is set up. - fn block_evaluate( - &mut self, - device: Device, - dev_node: PathBuf, - ) -> StratisResult> { - let stratis_identifiers = - DevOwnership::from_udev_ownership(&block_device_ownership(&dev_node)???, &dev_node)? - .stratis_identifiers(); - let pool_uuid = if let Some((pool_uuid, device_uuid)) = stratis_identifiers { + fn block_evaluate(&mut self, device: &libudev::Device) -> Option { + if let Some((pool_uuid, device_uuid, device, dev_node)) = identify_block_device(device) { if self.pools.contains_uuid(pool_uuid) { // We can get udev events for devices that are already in the pool. Lets check // to see if this block device is already in this existing pool. If it is, then all @@ -246,8 +238,7 @@ impl StratEngine { } } else { None - }; - Ok(pool_uuid) + } } } @@ -255,23 +246,14 @@ impl Engine for StratEngine { fn handle_event(&mut self, event: &libudev::Event) -> Option<(PoolUuid, &mut dyn Pool)> { let event_type = event.event_type(); if event_type == libudev::EventType::Add || event_type == libudev::EventType::Change { - let device = event.device(); - device - .devnode() - .and_then(|devnode| { - device.devnum().and_then(|devnum| { - self.block_evaluate(Device::from(devnum), PathBuf::from(devnode)) - .unwrap_or(None) - }) - }) - .map(move |pool_uuid| { - ( - pool_uuid, - self.get_mut_pool(pool_uuid) - .expect("block_evaluate() returned a pool UUID, pool must be available") - .1, - ) - }) + self.block_evaluate(event.device()).map(move |pool_uuid| { + ( + pool_uuid, + self.get_mut_pool(pool_uuid) + .expect("block_evaluate() returned a pool UUID, pool must be available") + .1, + ) + }) } else { None } diff --git a/src/engine/strat_engine/udev.rs b/src/engine/strat_engine/udev.rs index fcfb899d8c..123804eaff 100644 --- a/src/engine/strat_engine/udev.rs +++ b/src/engine/strat_engine/udev.rs @@ -109,44 +109,6 @@ pub fn decide_ownership(device: &libudev::Device) -> StratisResult StratisResult>> { - block_device_apply(devnode, |d| decide_ownership(d)) - .map_err(|err| { - StratisError::Error(format!( - "Could not determine ownership of block device {} because of a udev failure: {}", - devnode.display(), - err - )) - }) - .map(|option_ownership| { - option_ownership.ok_or_else(|| { - StratisError::Error(format!( - "Could not determine ownership of block device {} because it could not be found in the udev database", - devnode.display(), - )) - }) - .map(|error_ownership| { - error_ownership.map_err(|err| { - StratisError::Error(format!( - "Could not determine ownership of block device {} because of an error processing udev information: {}", - devnode.display(), - err - )) - }) - }) - }) -} - /// Locate a udev block device with the specified devnode and apply a function /// to that device, returning the result. /// This approach is necessitated by the libudev lifetimes, which do not allow From ba54e8666f029c76fd4eebea989ac0efa0c0fdf7 Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 6 Jan 2020 12:09:38 -0500 Subject: [PATCH 068/109] Abstract the action of checking a device after udev processing Use it in all methods where this makes sense, according to the context. Fix a small bug in find_all_block_devices_with_stratis_signatures. Account for the somewhat slight possibility that udev has belatedly identified the device as a Stratis device. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/identify.rs | 203 +++++++----------- 1 file changed, 82 insertions(+), 121 deletions(-) diff --git a/src/engine/strat_engine/backstore/identify.rs b/src/engine/strat_engine/backstore/identify.rs index 4d005069fc..265d09134f 100644 --- a/src/engine/strat_engine/backstore/identify.rs +++ b/src/engine/strat_engine/backstore/identify.rs @@ -64,6 +64,75 @@ fn device_identifiers_wrapper( }) } +/// Process a device which udev information indicates is a Stratis device. +fn process_stratis_device(dev: &libudev::Device) -> Option<(PoolUuid, DevUuid, Device, PathBuf)> { + match dev.devnode() { + Some(devnode) => { + match ( + device_to_devno_wrapper(dev), + device_identifiers_wrapper(devnode), + ) { + (Err(err), _) | (_, Err(err)) | (_, Ok(Err(err))) => { + warn!("udev identified device {} as a Stratis device but {}, disregarding the device", + devnode.display(), + err); + None + } + (_, Ok(Ok(None))) => { + warn!("udev identified device {} as a Stratis device but there appeared to be no Stratis metadata on the device, disregarding the device", + devnode.display()); + None + } + (Ok(devno), Ok(Ok(Some((pool_uuid, device_uuid))))) => { + Some((pool_uuid, device_uuid, devno, devnode.to_path_buf())) + } + } + } + None => { + warn!("udev identified a device as a Stratis device, but the udev entry for the device had no device node, disregarding device"); + None + } + } +} + +/// Process a device which udev information indicates is unowned. +fn process_unowned_device(dev: &libudev::Device) -> Option<(PoolUuid, DevUuid, Device, PathBuf)> { + match dev.devnode() { + Some(devnode) => { + match ( + device_to_devno_wrapper(dev), + device_identifiers_wrapper(devnode), + ) { + (Err(err), _) | (_, Err(err)) => { + warn!("udev identified device {} as a block device but {}, disregarding the device", + devnode.display(), + err); + None + } + // FIXME: Refine error return in StaticHeader::setup(), + // so it can be used to distinguish between signficant + // and insignficant errors and then use that ability to + // distinguish here between different levels of + // severity. + (_, Ok(Err(err))) => { + debug!("udev identified device {} as a block device but {}, disregarding the device", + devnode.display(), + err); + None + } + (_, Ok(Ok(None))) => None, + (Ok(devno), Ok(Ok(Some((pool_uuid, device_uuid))))) => { + Some((pool_uuid, device_uuid, devno, devnode.to_path_buf())) + } + } + } + None => { + warn!("udev identified a device as a block device, but the udev entry for the device had no device node, disregarding the device"); + None + } + } +} + // Use udev to identify all block devices and return the subset of those // that have Stratis signatures. fn find_all_block_devices_with_stratis_signatures( @@ -79,48 +148,20 @@ fn find_all_block_devices_with_stratis_signatures( }; initialized }) - .filter(|dev| { - decide_ownership(dev) + .filter_map(|dev| { + decide_ownership(&dev) .map_err(|err| { warn!("Could not determine ownership of a udev block device because of an error processing udev information, omitting the device from the set of devices to process, for safety: {}", err); }) - .map(|decision| match decision { - UdevOwnership::Stratis | UdevOwnership::Unowned => true, - _ => false, - }) - .unwrap_or(false) - }) - .filter_map(|dev| match dev.devnode() { - Some(devnode) => { - match (device_to_devno_wrapper(&dev), device_identifiers_wrapper(devnode)) { - (Err(err), _) | (_, Err(err)) => { - warn!("udev identified device {} as a block device but {}, omitting the device from the set of devices to process", - devnode.display(), - err); - None - } - // FIXME: Refine error return in StaticHeader::setup(), - // so it can be used to distinguish between signficant - // and insignficant errors and then use that ability to - // distinguish here between different levels of - // severity. - (_, Ok(Err(err))) => { - debug!("udev identified device {} as a block device but {}, omitting the device from the set of devices to process", - devnode.display(), - err); - None - } - (_, Ok(Ok(None))) => None, - (Ok(devno), Ok(Ok(Some((pool_uuid, _))))) => Some((pool_uuid, devno, devnode.to_path_buf())), - } - } - None => { - warn!("udev identified a device as a block device, but the udev entry for the device had no device node, omitting the device from the set of devices to process"); - None - } + .map(|decision| match decision { + UdevOwnership::Stratis => process_stratis_device(&dev), + UdevOwnership::Unowned => process_unowned_device(&dev), + _ => None, + }) + .unwrap_or(None) }) - .fold(HashMap::new(), |mut acc, (pool_uuid, device, devnode)| { + .fold(HashMap::new(), |mut acc, (pool_uuid, _, device, devnode)| { acc.entry(pool_uuid).or_insert_with(HashMap::new).insert(device, devnode); acc }); @@ -148,29 +189,8 @@ fn find_all_stratis_devices() -> libudev::Result { - match (device_to_devno_wrapper(&dev), device_identifiers_wrapper(devnode)) { - (Err(err), _) | (_, Err(err)) | (_, Ok(Err(err)))=> { - warn!("udev identified device {} as a Stratis device but {}, omitting the device from the set of devices to process", - devnode.display(), - err); - None - } - (_, Ok(Ok(None))) => { - warn!("udev identified device {} as a Stratis device but there appeared to be no Stratis metadata on the device, omitting the device from the set of devices to process", - devnode.display()); - None - } - (Ok(devno), Ok(Ok(Some((pool_uuid, _))))) => Some((pool_uuid, devno, devnode.to_path_buf())), - } - } - None => { - warn!("udev identified a device as a Stratis device, but the udev entry for the device had no device node, omitting the the device from the set of devices to process"); - None - } - }) - .fold(HashMap::new(), |mut acc, (pool_uuid, device, devnode)| { + .filter_map(|dev| process_stratis_device(&dev)) + .fold(HashMap::new(), |mut acc, (pool_uuid, _, device, devnode)| { acc.entry(pool_uuid).or_insert_with(HashMap::new).insert(device, devnode); acc }); @@ -196,67 +216,8 @@ pub fn identify_block_device( None } Ok(ownership) => match ownership { - UdevOwnership::Stratis => match dev.devnode() { - Some(devnode) => { - match ( - device_to_devno_wrapper(dev), - device_identifiers_wrapper(devnode), - ) { - (Err(err), _) | (_, Err(err)) | (_, Ok(Err(err))) => { - warn!("udev identified device {} as a Stratis device but {}, disregarding the device", - devnode.display(), - err); - None - } - (_, Ok(Ok(None))) => { - warn!("udev identified device {} as a Stratis device but there appeared to be no Stratis metadata on the device, disregarding the device", - devnode.display()); - None - } - (Ok(devno), Ok(Ok(Some((pool_uuid, device_uuid))))) => { - Some((pool_uuid, device_uuid, devno, devnode.to_path_buf())) - } - } - } - None => { - warn!("udev identified a device as a Stratis device, but the udev entry for the device had no device node, disregarding device"); - None - } - }, - UdevOwnership::Unowned => match dev.devnode() { - Some(devnode) => { - match ( - device_to_devno_wrapper(dev), - device_identifiers_wrapper(devnode), - ) { - (Err(err), _) | (_, Err(err)) => { - warn!("udev identified device {} as a block device but {}, disregarding device", - devnode.display(), - err); - None - } - // FIXME: Refine error return in StaticHeader::setup(), - // so it can be used to distinguish between signficant - // and insignficant errors and then use that ability to - // distinguish here between different levels of - // severity. - (_, Ok(Err(err))) => { - debug!("udev identified device {} as a block device but {}, disregarding device", - devnode.display(), - err); - None - } - (_, Ok(Ok(None))) => None, - (Ok(devno), Ok(Ok(Some((pool_uuid, device_uuid))))) => { - Some((pool_uuid, device_uuid, devno, devnode.to_path_buf())) - } - } - } - None => { - warn!("udev identified a device as a block device, but the udev entry for the device had no device node, omitting the device from the set of devices to process"); - None - } - }, + UdevOwnership::Stratis => process_stratis_device(dev), + UdevOwnership::Unowned => process_unowned_device(dev), _ => None, }, } From 63c5b6c3048b503acd18bbfe4c823bef3366e2c9 Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 6 Jan 2020 16:36:40 -0500 Subject: [PATCH 069/109] Make is_multipath_member private Rely solely on publicly exposed decide_ownership method. It allows for better encapsulation, it's a bit clearer and it makes the actions that occur for the stratis enumeration and for the block device enumeration more consistent. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/identify.rs | 19 ++++++++++++++----- src/engine/strat_engine/udev.rs | 2 +- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/engine/strat_engine/backstore/identify.rs b/src/engine/strat_engine/backstore/identify.rs index 265d09134f..648d11986d 100644 --- a/src/engine/strat_engine/backstore/identify.rs +++ b/src/engine/strat_engine/backstore/identify.rs @@ -17,7 +17,7 @@ use devicemapper::Device; use crate::engine::{ strat_engine::{ backstore::metadata::device_identifiers, - udev::{block_enumerator, decide_ownership, is_multipath_member, UdevOwnership}, + udev::{block_enumerator, decide_ownership, UdevOwnership}, }, types::{DevUuid, PoolUuid}, }; @@ -183,13 +183,22 @@ fn find_all_stratis_devices() -> libudev::Result process_stratis_device(&dev), + UdevOwnership::MultipathMember => None, + _ => { + warn!("udev enumeration identified this device as a Stratis block device but on further examination it appears not to belong to Stratis"); + None + } + }) + .unwrap_or(None) + }) .fold(HashMap::new(), |mut acc, (pool_uuid, _, device, devnode)| { acc.entry(pool_uuid).or_insert_with(HashMap::new).insert(device, devnode); acc diff --git a/src/engine/strat_engine/udev.rs b/src/engine/strat_engine/udev.rs index 123804eaff..90a144a619 100644 --- a/src/engine/strat_engine/udev.rs +++ b/src/engine/strat_engine/udev.rs @@ -43,7 +43,7 @@ where /// Returns true if udev indicates that the device is a multipath member /// device, else false. Returns an error on a failure to interpret the /// value. -pub fn is_multipath_member(device: &libudev::Device) -> StratisResult { +fn is_multipath_member(device: &libudev::Device) -> StratisResult { match get_udev_property(device, "DM_MULTIPATH_DEVICE_PATH") { None => Ok(false), Some(Ok(value)) => Ok(value == "1"), From ee52727fb15e6a700f0b2b0a70165a4f6cee5aa6 Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 6 Jan 2020 16:54:26 -0500 Subject: [PATCH 070/109] Have block_evaluate return the pool and the pool UUID This allows moving an expect to a position directly after the change that makes it logically necessary, rather than in another method entirely. Signed-off-by: mulhern --- src/engine/strat_engine/engine.rs | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/engine/strat_engine/engine.rs b/src/engine/strat_engine/engine.rs index aab0133eea..7671f76c47 100644 --- a/src/engine/strat_engine/engine.rs +++ b/src/engine/strat_engine/engine.rs @@ -174,11 +174,12 @@ impl StratEngine { /// Given a udev database entry, process the entry. /// /// If all the devices are present in the pool and the pool isn't already - /// up and running, it will get setup and the pool uuid will be returned. + /// up and running, it will get setup and the newly created pool and UUID + /// will be returned. /// /// Logs a warning if the block devices appears to be a Stratis block /// device and no pool is set up. - fn block_evaluate(&mut self, device: &libudev::Device) -> Option { + fn block_evaluate(&mut self, device: &libudev::Device) -> Option<(PoolUuid, &mut dyn Pool)> { if let Some((pool_uuid, device_uuid, device, dev_node)) = identify_block_device(device) { if self.pools.contains_uuid(pool_uuid) { // We can get udev events for devices that are already in the pool. Lets check @@ -227,7 +228,12 @@ impl StratEngine { match setup_pool(pool_uuid, &devices, &self.pools) { Ok((pool_name, pool)) => { self.pools.insert(pool_name, pool_uuid, pool); - Some(pool_uuid) + Some(( + pool_uuid, + self.get_mut_pool(pool_uuid) + .expect("pool was just inserted") + .1, + )) } Err(err) => { warn!("no pool set up, reason: {:?}", err); @@ -246,14 +252,7 @@ impl Engine for StratEngine { fn handle_event(&mut self, event: &libudev::Event) -> Option<(PoolUuid, &mut dyn Pool)> { let event_type = event.event_type(); if event_type == libudev::EventType::Add || event_type == libudev::EventType::Change { - self.block_evaluate(event.device()).map(move |pool_uuid| { - ( - pool_uuid, - self.get_mut_pool(pool_uuid) - .expect("block_evaluate() returned a pool UUID, pool must be available") - .1, - ) - }) + self.block_evaluate(event.device()) } else { None } From d5e4fcf7d7b892905e1509a0125692945f5c6a14 Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 7 Jan 2020 08:54:54 -0500 Subject: [PATCH 071/109] Use identify_block_device in the body of a find method Use it in the body of find_all_block_devices_with_stratis_signatures. The context in which this method is called by block_evaluate is the same as is established by find_all_block_devices_with_stratis_signatures. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/identify.rs | 37 ++++++------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/src/engine/strat_engine/backstore/identify.rs b/src/engine/strat_engine/backstore/identify.rs index 648d11986d..75c1bf3b35 100644 --- a/src/engine/strat_engine/backstore/identify.rs +++ b/src/engine/strat_engine/backstore/identify.rs @@ -140,31 +140,18 @@ fn find_all_block_devices_with_stratis_signatures( let context = libudev::Context::new()?; let mut enumerator = block_enumerator(&context)?; - let pool_map = enumerator.scan_devices()? - .filter(|dev| { - let initialized = dev.is_initialized(); - if !initialized { - debug!("Found a udev entry for a device identified as a block device, but udev also identified it as uninitialized, omitting the device from the set of devices to process, for safety"); - }; - initialized - }) - .filter_map(|dev| { - decide_ownership(&dev) - .map_err(|err| { - warn!("Could not determine ownership of a udev block device because of an error processing udev information, omitting the device from the set of devices to process, for safety: {}", - err); - }) - .map(|decision| match decision { - UdevOwnership::Stratis => process_stratis_device(&dev), - UdevOwnership::Unowned => process_unowned_device(&dev), - _ => None, - }) - .unwrap_or(None) - }) - .fold(HashMap::new(), |mut acc, (pool_uuid, _, device, devnode)| { - acc.entry(pool_uuid).or_insert_with(HashMap::new).insert(device, devnode); - acc - }); + let pool_map = enumerator + .scan_devices()? + .filter_map(|dev| identify_block_device(&dev)) + .fold( + HashMap::new(), + |mut acc, (pool_uuid, _, device, devnode)| { + acc.entry(pool_uuid) + .or_insert_with(HashMap::new) + .insert(device, devnode); + acc + }, + ); Ok(pool_map) } From 5f9dcca03d1f6fc1f151c3e7c2e24be846becb0b Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 7 Jan 2020 09:05:15 -0500 Subject: [PATCH 072/109] Make a method called identify_stratis_device This method has no public use yet, but is analogous to the identify_block_device method. Use this method in find_all_stratis_devices in the same way that identify_block_device is used in find_all_block_devices_with_stratis_signatures. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/identify.rs | 67 +++++++++++-------- 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/src/engine/strat_engine/backstore/identify.rs b/src/engine/strat_engine/backstore/identify.rs index 75c1bf3b35..524f15c973 100644 --- a/src/engine/strat_engine/backstore/identify.rs +++ b/src/engine/strat_engine/backstore/identify.rs @@ -162,37 +162,48 @@ fn find_all_stratis_devices() -> libudev::Result process_stratis_device(&dev), - UdevOwnership::MultipathMember => None, - _ => { - warn!("udev enumeration identified this device as a Stratis block device but on further examination it appears not to belong to Stratis"); - None - } - }) - .unwrap_or(None) - }) - .fold(HashMap::new(), |mut acc, (pool_uuid, _, device, devnode)| { - acc.entry(pool_uuid).or_insert_with(HashMap::new).insert(device, devnode); - acc - }); + let pool_map = enumerator + .scan_devices()? + .filter_map(|dev| identify_stratis_device(&dev)) + .fold( + HashMap::new(), + |mut acc, (pool_uuid, _, device, devnode)| { + acc.entry(pool_uuid) + .or_insert_with(HashMap::new) + .insert(device, devnode); + acc + }, + ); Ok(pool_map) } +// Identify a device that udev enumeration has already picked up as a Stratis +// device. Return None if the device does not, after all, appear to be a Stratis +// device. Log anything unusual at an appropriate level. +fn identify_stratis_device(dev: &libudev::Device) -> Option<(PoolUuid, DevUuid, Device, PathBuf)> { + let initialized = dev.is_initialized(); + if !initialized { + warn!("Found a udev entry for a device identified as a Stratis device, but udev also identified it as uninitialized, disregarding the device"); + return None; + }; + + match decide_ownership(dev) { + Err(err) => { + warn!("Could not determine ownership of a block device identified as a Stratis device by udev because of an error processing udev information, disregarding the device: {}", + err); + None + } + Ok(ownership) => match ownership { + UdevOwnership::Stratis => process_stratis_device(dev), + UdevOwnership::MultipathMember => None, + _ => { + warn!("udev enumeration identified this device as a Stratis block device but on further examination it appears not to belong to Stratis"); + None + } + }, + } +} + /// Identify a block device in the context where a udev event has been /// captured for some block device. Return None if the device does not /// appear to be a Stratis device. Log at an appropriate level on all errors. From 6319429424769c7079412a1a967ef0481fab35c9 Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Thu, 9 Jan 2020 10:22:52 -0500 Subject: [PATCH 073/109] Add missing license header --- src/engine/shared.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/engine/shared.rs b/src/engine/shared.rs index 6f70255619..937ac8b903 100644 --- a/src/engine/shared.rs +++ b/src/engine/shared.rs @@ -1,3 +1,7 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + use std::{ collections::{hash_map::RandomState, HashSet}, path::{Path, PathBuf}, From c08f413b98e40d8a66aac96a7cc614c88f1e9c64 Mon Sep 17 00:00:00 2001 From: mulhern Date: Fri, 10 Jan 2020 09:59:43 -0500 Subject: [PATCH 074/109] Remove some unnecessary publicity This seems to have been made unnecessary by the introduction of the BlockDev interface in 77f3e23cc08749ae4f5e35d85e2f60cc666538. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/blockdev.rs | 2 +- src/engine/strat_engine/backstore/blockdevmgr.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/engine/strat_engine/backstore/blockdev.rs b/src/engine/strat_engine/backstore/blockdev.rs index e228e93472..bb4f3b5b9e 100644 --- a/src/engine/strat_engine/backstore/blockdev.rs +++ b/src/engine/strat_engine/backstore/blockdev.rs @@ -28,7 +28,7 @@ use crate::{ #[derive(Debug)] pub struct StratBlockDev { dev: Device, - pub(super) devnode: PathBuf, + devnode: PathBuf, bda: BDA, used: RangeAllocator, user_info: Option, diff --git a/src/engine/strat_engine/backstore/blockdevmgr.rs b/src/engine/strat_engine/backstore/blockdevmgr.rs index c4412d128f..543631f76f 100644 --- a/src/engine/strat_engine/backstore/blockdevmgr.rs +++ b/src/engine/strat_engine/backstore/blockdevmgr.rs @@ -520,7 +520,7 @@ pub fn wipe_blockdevs(blockdevs: &[StratBlockDev]) -> StratisResult<()> { let mut unerased_devnodes = Vec::new(); for bd in blockdevs { - let bd_devnode = bd.devnode.to_owned(); + let bd_devnode = bd.devnode(); bd.disown() .unwrap_or_else(|_| unerased_devnodes.push(bd_devnode)); } From 0cb3d4ed94e149fdd2c5e4568c4521b097c45d1e Mon Sep 17 00:00:00 2001 From: mulhern Date: Fri, 10 Jan 2020 10:07:21 -0500 Subject: [PATCH 075/109] Be more functional in calculating the unerased devnodes On the principal of least strength. Signed-off-by: mulhern --- src/engine/strat_engine/backstore/blockdevmgr.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/engine/strat_engine/backstore/blockdevmgr.rs b/src/engine/strat_engine/backstore/blockdevmgr.rs index 543631f76f..37d0aee87e 100644 --- a/src/engine/strat_engine/backstore/blockdevmgr.rs +++ b/src/engine/strat_engine/backstore/blockdevmgr.rs @@ -517,13 +517,13 @@ fn initialize( /// Return an error if any of the blockdevs could not be wiped. /// If an error occurs while wiping a blockdev, attempt to wipe all remaining. pub fn wipe_blockdevs(blockdevs: &[StratBlockDev]) -> StratisResult<()> { - let mut unerased_devnodes = Vec::new(); - - for bd in blockdevs { - let bd_devnode = bd.devnode(); - bd.disown() - .unwrap_or_else(|_| unerased_devnodes.push(bd_devnode)); - } + let unerased_devnodes: Vec<_> = blockdevs + .iter() + .filter_map(|bd| match bd.disown() { + Err(_) => Some(bd.devnode()), + _ => None, + }) + .collect(); if unerased_devnodes.is_empty() { Ok(()) From cc341ff5fa18e5a5c2bdabfd2d01f4895351d454 Mon Sep 17 00:00:00 2001 From: mulhern Date: Thu, 9 Jan 2020 17:08:37 -0500 Subject: [PATCH 076/109] Adjust dbus-api names and permissions Make methods and props modules private in their respective parent modules. Instead of using version-specific method names, distinguish api methods by the parent of their api module. This eliminates a kind of redundancy, where the method name and the interface name both include the version number, and that number is always the same. By specifying an api method by its parent module name, the definition of the objects and the interfaces they support, always in mod.rs, become more uniform, and a bit clearer. So, when a second version of the same interface is added, it is easier to pick out precisely the origin of each api method, i.e., whether it is an override of a method from a previous interface or not. The methods in methods.rs and props.rs are used only to support the methods in api.rs. So, if these modules are made private, then any methods that are truly shared between interfaces won't find themselves in these methods, but instead in the appropriate shared.rs. Signed-off-by: mulhern --- src/dbus_api/api/manager_2_0/mod.rs | 10 +++++-- src/dbus_api/api/mod.rs | 11 +++----- src/dbus_api/blockdev/blockdev_2_0/mod.rs | 11 ++++++-- .../blockdev/fetch_properties_2_0/mod.rs | 6 ++-- src/dbus_api/blockdev/mod.rs | 27 +++++++----------- .../filesystem/fetch_properties_2_0/mod.rs | 6 ++-- src/dbus_api/filesystem/filesystem_2_0/mod.rs | 10 +++++-- src/dbus_api/filesystem/mod.rs | 23 ++++++--------- src/dbus_api/pool/fetch_properties_2_0/mod.rs | 6 ++-- src/dbus_api/pool/mod.rs | 28 +++++++------------ src/dbus_api/pool/pool_2_0/mod.rs | 12 ++++++-- 11 files changed, 75 insertions(+), 75 deletions(-) diff --git a/src/dbus_api/api/manager_2_0/mod.rs b/src/dbus_api/api/manager_2_0/mod.rs index 333cf7108f..98ea410e9f 100644 --- a/src/dbus_api/api/manager_2_0/mod.rs +++ b/src/dbus_api/api/manager_2_0/mod.rs @@ -1,3 +1,7 @@ -pub mod api; -pub mod methods; -pub mod props; +mod api; +mod methods; +mod props; + +pub use api::{ + configure_simulator_method, create_pool_method, destroy_pool_method, version_property, +}; diff --git a/src/dbus_api/api/mod.rs b/src/dbus_api/api/mod.rs index 0106a68a32..5153c6b2ef 100644 --- a/src/dbus_api/api/mod.rs +++ b/src/dbus_api/api/mod.rs @@ -8,9 +8,6 @@ use dbus::{ }; use crate::dbus_api::{ - api::manager_2_0::api::{ - configure_simulator_method, create_pool_method, destroy_pool_method, version_property, - }, consts, types::{DbusContext, TData}, }; @@ -28,10 +25,10 @@ pub fn get_base_tree<'a>(dbus_context: DbusContext) -> (Tree, TData> .object_manager() .add( f.interface(consts::MANAGER_INTERFACE_NAME, ()) - .add_m(create_pool_method(&f)) - .add_m(destroy_pool_method(&f)) - .add_m(configure_simulator_method(&f)) - .add_p(version_property(&f)), + .add_m(manager_2_0::create_pool_method(&f)) + .add_m(manager_2_0::destroy_pool_method(&f)) + .add_m(manager_2_0::configure_simulator_method(&f)) + .add_p(manager_2_0::version_property(&f)), ); let path = obj_path.get_name().to_owned(); diff --git a/src/dbus_api/blockdev/blockdev_2_0/mod.rs b/src/dbus_api/blockdev/blockdev_2_0/mod.rs index 333cf7108f..84b678b9f3 100644 --- a/src/dbus_api/blockdev/blockdev_2_0/mod.rs +++ b/src/dbus_api/blockdev/blockdev_2_0/mod.rs @@ -1,3 +1,8 @@ -pub mod api; -pub mod methods; -pub mod props; +mod api; +mod methods; +mod props; + +pub use api::{ + devnode_property, hardware_info_property, initialization_time_property, pool_property, + set_userid_method, tier_property, user_info_property, uuid_property, +}; diff --git a/src/dbus_api/blockdev/fetch_properties_2_0/mod.rs b/src/dbus_api/blockdev/fetch_properties_2_0/mod.rs index 9ba05c1db4..4409247246 100644 --- a/src/dbus_api/blockdev/fetch_properties_2_0/mod.rs +++ b/src/dbus_api/blockdev/fetch_properties_2_0/mod.rs @@ -1,2 +1,4 @@ -pub mod api; -pub mod methods; +mod api; +mod methods; + +pub use api::{get_all_properties_method, get_properties_method}; diff --git a/src/dbus_api/blockdev/mod.rs b/src/dbus_api/blockdev/mod.rs index 2663629994..05bca15e42 100644 --- a/src/dbus_api/blockdev/mod.rs +++ b/src/dbus_api/blockdev/mod.rs @@ -7,13 +7,6 @@ use uuid::Uuid; use crate::{ dbus_api::{ - blockdev::{ - blockdev_2_0::api::{ - devnode_property, hardware_info_property, initialization_time_property, - pool_property, set_userid_method, tier_property, user_info_property, uuid_property, - }, - fetch_properties_2_0::api::{get_all_properties_method, get_properties_method}, - }, consts, types::{DbusContext, OPContext}, util::make_object_path, @@ -40,19 +33,19 @@ pub fn create_dbus_blockdev<'a>( .introspectable() .add( f.interface(consts::BLOCKDEV_INTERFACE_NAME, ()) - .add_m(set_userid_method(&f)) - .add_p(devnode_property(&f)) - .add_p(hardware_info_property(&f)) - .add_p(initialization_time_property(&f)) - .add_p(pool_property(&f)) - .add_p(tier_property(&f)) - .add_p(user_info_property(&f)) - .add_p(uuid_property(&f)), + .add_m(blockdev_2_0::set_userid_method(&f)) + .add_p(blockdev_2_0::devnode_property(&f)) + .add_p(blockdev_2_0::hardware_info_property(&f)) + .add_p(blockdev_2_0::initialization_time_property(&f)) + .add_p(blockdev_2_0::pool_property(&f)) + .add_p(blockdev_2_0::tier_property(&f)) + .add_p(blockdev_2_0::user_info_property(&f)) + .add_p(blockdev_2_0::uuid_property(&f)), ) .add( f.interface(consts::PROPERTY_FETCH_INTERFACE_NAME, ()) - .add_m(get_all_properties_method(&f)) - .add_m(get_properties_method(&f)), + .add_m(fetch_properties_2_0::get_all_properties_method(&f)) + .add_m(fetch_properties_2_0::get_properties_method(&f)), ); let path = object_path.get_name().to_owned(); diff --git a/src/dbus_api/filesystem/fetch_properties_2_0/mod.rs b/src/dbus_api/filesystem/fetch_properties_2_0/mod.rs index 9ba05c1db4..4409247246 100644 --- a/src/dbus_api/filesystem/fetch_properties_2_0/mod.rs +++ b/src/dbus_api/filesystem/fetch_properties_2_0/mod.rs @@ -1,2 +1,4 @@ -pub mod api; -pub mod methods; +mod api; +mod methods; + +pub use api::{get_all_properties_method, get_properties_method}; diff --git a/src/dbus_api/filesystem/filesystem_2_0/mod.rs b/src/dbus_api/filesystem/filesystem_2_0/mod.rs index 333cf7108f..60cccf9702 100644 --- a/src/dbus_api/filesystem/filesystem_2_0/mod.rs +++ b/src/dbus_api/filesystem/filesystem_2_0/mod.rs @@ -1,3 +1,7 @@ -pub mod api; -pub mod methods; -pub mod props; +mod api; +mod methods; +mod props; + +pub use api::{ + created_property, devnode_property, name_property, pool_property, rename_method, uuid_property, +}; diff --git a/src/dbus_api/filesystem/mod.rs b/src/dbus_api/filesystem/mod.rs index f5af163f08..e9f253cab7 100644 --- a/src/dbus_api/filesystem/mod.rs +++ b/src/dbus_api/filesystem/mod.rs @@ -7,13 +7,6 @@ use dbus::{self, tree::Factory}; use crate::{ dbus_api::{ consts, - filesystem::{ - fetch_properties_2_0::api::{get_all_properties_method, get_properties_method}, - filesystem_2_0::api::{ - created_property, devnode_property, name_property, pool_property, rename_method, - uuid_property, - }, - }, types::{DbusContext, OPContext}, util::make_object_path, }, @@ -39,17 +32,17 @@ pub fn create_dbus_filesystem<'a>( .introspectable() .add( f.interface(consts::FILESYSTEM_INTERFACE_NAME, ()) - .add_m(rename_method(&f)) - .add_p(devnode_property(&f)) - .add_p(name_property(&f)) - .add_p(pool_property(&f)) - .add_p(uuid_property(&f)) - .add_p(created_property(&f)), + .add_m(filesystem_2_0::rename_method(&f)) + .add_p(filesystem_2_0::devnode_property(&f)) + .add_p(filesystem_2_0::name_property(&f)) + .add_p(filesystem_2_0::pool_property(&f)) + .add_p(filesystem_2_0::uuid_property(&f)) + .add_p(filesystem_2_0::created_property(&f)), ) .add( f.interface(consts::PROPERTY_FETCH_INTERFACE_NAME, ()) - .add_m(get_all_properties_method(&f)) - .add_m(get_properties_method(&f)), + .add_m(fetch_properties_2_0::get_all_properties_method(&f)) + .add_m(fetch_properties_2_0::get_properties_method(&f)), ); let path = object_path.get_name().to_owned(); diff --git a/src/dbus_api/pool/fetch_properties_2_0/mod.rs b/src/dbus_api/pool/fetch_properties_2_0/mod.rs index 9ba05c1db4..4409247246 100644 --- a/src/dbus_api/pool/fetch_properties_2_0/mod.rs +++ b/src/dbus_api/pool/fetch_properties_2_0/mod.rs @@ -1,2 +1,4 @@ -pub mod api; -pub mod methods; +mod api; +mod methods; + +pub use api::{get_all_properties_method, get_properties_method}; diff --git a/src/dbus_api/pool/mod.rs b/src/dbus_api/pool/mod.rs index a65e7a5afe..7be0d7eab4 100644 --- a/src/dbus_api/pool/mod.rs +++ b/src/dbus_api/pool/mod.rs @@ -7,14 +7,6 @@ use dbus::{self, tree::Factory}; use crate::{ dbus_api::{ consts, - pool::{ - fetch_properties_2_0::api::{get_all_properties_method, get_properties_method}, - pool_2_0::api::{ - add_blockdevs_method, add_cachedevs_method, create_filesystems_method, - destroy_filesystems_method, name_property, rename_method, - snapshot_filesystem_method, uuid_property, - }, - }, types::{DbusContext, OPContext}, util::make_object_path, }, @@ -40,19 +32,19 @@ pub fn create_dbus_pool<'a>( .introspectable() .add( f.interface(consts::POOL_INTERFACE_NAME, ()) - .add_m(create_filesystems_method(&f)) - .add_m(destroy_filesystems_method(&f)) - .add_m(snapshot_filesystem_method(&f)) - .add_m(add_blockdevs_method(&f)) - .add_m(add_cachedevs_method(&f)) - .add_m(rename_method(&f)) - .add_p(name_property(&f)) - .add_p(uuid_property(&f)), + .add_m(pool_2_0::create_filesystems_method(&f)) + .add_m(pool_2_0::destroy_filesystems_method(&f)) + .add_m(pool_2_0::snapshot_filesystem_method(&f)) + .add_m(pool_2_0::add_blockdevs_method(&f)) + .add_m(pool_2_0::add_cachedevs_method(&f)) + .add_m(pool_2_0::rename_method(&f)) + .add_p(pool_2_0::name_property(&f)) + .add_p(pool_2_0::uuid_property(&f)), ) .add( f.interface(consts::PROPERTY_FETCH_INTERFACE_NAME, ()) - .add_m(get_all_properties_method(&f)) - .add_m(get_properties_method(&f)), + .add_m(fetch_properties_2_0::get_all_properties_method(&f)) + .add_m(fetch_properties_2_0::get_properties_method(&f)), ); let path = object_path.get_name().to_owned(); diff --git a/src/dbus_api/pool/pool_2_0/mod.rs b/src/dbus_api/pool/pool_2_0/mod.rs index 333cf7108f..092d60c0de 100644 --- a/src/dbus_api/pool/pool_2_0/mod.rs +++ b/src/dbus_api/pool/pool_2_0/mod.rs @@ -1,3 +1,9 @@ -pub mod api; -pub mod methods; -pub mod props; +mod api; +mod methods; +mod props; + +pub use api::{ + add_blockdevs_method, add_cachedevs_method, create_filesystems_method, + destroy_filesystems_method, name_property, rename_method, snapshot_filesystem_method, + uuid_property, +}; From a47fa399b08b2c8b150035e7b532a408cf237c85 Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 13 Jan 2020 11:11:01 -0500 Subject: [PATCH 077/109] Just return None if the pool for this device is already set up Simply assume that the pool is fine and already includes the device in this case. It might not, but the plan is to check this in a different way. Also, get rid of get_strat_blockdev which was only used in the eliminated code. Signed-off-by: mulhern --- src/engine/strat_engine/engine.rs | 37 +------------------------------ src/engine/strat_engine/pool.rs | 9 +++----- 2 files changed, 4 insertions(+), 42 deletions(-) diff --git a/src/engine/strat_engine/engine.rs b/src/engine/strat_engine/engine.rs index 7671f76c47..9efee41d7b 100644 --- a/src/engine/strat_engine/engine.rs +++ b/src/engine/strat_engine/engine.rs @@ -180,43 +180,8 @@ impl StratEngine { /// Logs a warning if the block devices appears to be a Stratis block /// device and no pool is set up. fn block_evaluate(&mut self, device: &libudev::Device) -> Option<(PoolUuid, &mut dyn Pool)> { - if let Some((pool_uuid, device_uuid, device, dev_node)) = identify_block_device(device) { + if let Some((pool_uuid, _, device, dev_node)) = identify_block_device(device) { if self.pools.contains_uuid(pool_uuid) { - // We can get udev events for devices that are already in the pool. Lets check - // to see if this block device is already in this existing pool. If it is, then all - // is well. If it's not then we have what is documented below. - // - // TODO: Handle the case where we have found a device for an already active pool - // ref. https://github.com/stratis-storage/stratisd/issues/748 - - let (name, pool) = self - .pools - .get_by_uuid(pool_uuid) - .expect("pools.contains_uuid(pool_uuid)"); - - match pool.get_strat_blockdev(device_uuid) { - None => { - error!( - "we have a block device {:?} with pool {}, uuid = {} device uuid = {} \ - which believes it belongs in this pool, but existing active pool has \ - no knowledge of it", - dev_node, name, pool_uuid, device_uuid - ); - } - Some((_tier, block_dev)) => { - // Make sure that this block device and existing block device refer to the - // same physical device that's already in the pool - if device != *block_dev.device() { - error!( - "we have a block device with the same uuid as one already in the \ - pool, but the one in the pool has device number {:}, \ - while the one just found has device number {:}", - block_dev.device(), - device, - ); - } - } - } None } else { let mut devices = self diff --git a/src/engine/strat_engine/pool.rs b/src/engine/strat_engine/pool.rs index 080b9c41b1..5eaac32d8a 100644 --- a/src/engine/strat_engine/pool.rs +++ b/src/engine/strat_engine/pool.rs @@ -19,7 +19,7 @@ use crate::{ engine::{ engine::{BlockDev, Filesystem, Pool}, strat_engine::{ - backstore::{Backstore, MDADataSize, StratBlockDev}, + backstore::{Backstore, MDADataSize}, names::validate_name, serde_structs::{FlexDevsSave, PoolSave, Recordable}, thinpool::{ThinPool, ThinPoolSizeParams, DATA_BLOCK_SIZE}, @@ -271,10 +271,6 @@ impl StratPool { thinpool_dev: self.thin_pool.record(), } } - - pub fn get_strat_blockdev(&self, uuid: DevUuid) -> Option<(BlockDevTier, &StratBlockDev)> { - self.backstore.get_blockdev_by_uuid(uuid) - } } impl Pool for StratPool { @@ -463,7 +459,8 @@ impl Pool for StratPool { } fn get_blockdev(&self, uuid: DevUuid) -> Option<(BlockDevTier, &dyn BlockDev)> { - self.get_strat_blockdev(uuid) + self.backstore + .get_blockdev_by_uuid(uuid) .map(|(t, b)| (t, b as &dyn BlockDev)) } From 0672f2ee7cf5bcd1cd645b6a6019217050a45579 Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 13 Jan 2020 11:14:26 -0500 Subject: [PATCH 078/109] Use standard Option method and_then It's there for the using. Signed-off-by: mulhern --- src/engine/strat_engine/engine.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/engine/strat_engine/engine.rs b/src/engine/strat_engine/engine.rs index 9efee41d7b..f6d1074636 100644 --- a/src/engine/strat_engine/engine.rs +++ b/src/engine/strat_engine/engine.rs @@ -180,7 +180,7 @@ impl StratEngine { /// Logs a warning if the block devices appears to be a Stratis block /// device and no pool is set up. fn block_evaluate(&mut self, device: &libudev::Device) -> Option<(PoolUuid, &mut dyn Pool)> { - if let Some((pool_uuid, _, device, dev_node)) = identify_block_device(device) { + identify_block_device(device).and_then(move |(pool_uuid, _, device, dev_node)| { if self.pools.contains_uuid(pool_uuid) { None } else { @@ -207,9 +207,7 @@ impl StratEngine { } } } - } else { - None - } + }) } } From 142514f1bdb3bd41e8c3716ffb7677a09817f2b0 Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 13 Jan 2020 11:43:22 -0500 Subject: [PATCH 079/109] Remove an unnecessary expect Signed-off-by: mulhern --- src/engine/strat_engine/engine.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/engine/strat_engine/engine.rs b/src/engine/strat_engine/engine.rs index f6d1074636..3514499753 100644 --- a/src/engine/strat_engine/engine.rs +++ b/src/engine/strat_engine/engine.rs @@ -187,8 +187,7 @@ impl StratEngine { let mut devices = self .incomplete_pools .remove(&pool_uuid) - .or_else(|| Some(HashMap::new())) - .expect("We just retrieved or created a HashMap"); + .unwrap_or_else(HashMap::new); devices.insert(device, dev_node); match setup_pool(pool_uuid, &devices, &self.pools) { Ok((pool_name, pool)) => { From 25bffab8dad2f86770a283d41b61dce21f430a67 Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 13 Jan 2020 11:59:38 -0500 Subject: [PATCH 080/109] Get rid of an unnecessary expect On general principles. Signed-off-by: mulhern --- src/engine/strat_engine/engine.rs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/engine/strat_engine/engine.rs b/src/engine/strat_engine/engine.rs index 3514499753..1bbf472160 100644 --- a/src/engine/strat_engine/engine.rs +++ b/src/engine/strat_engine/engine.rs @@ -192,19 +192,14 @@ impl StratEngine { match setup_pool(pool_uuid, &devices, &self.pools) { Ok((pool_name, pool)) => { self.pools.insert(pool_name, pool_uuid, pool); - Some(( - pool_uuid, - self.get_mut_pool(pool_uuid) - .expect("pool was just inserted") - .1, - )) } Err(err) => { warn!("no pool set up, reason: {:?}", err); self.incomplete_pools.insert(pool_uuid, devices); - None } - } + }; + self.get_mut_pool(pool_uuid) + .map(|(_, pool)| (pool_uuid, pool)) } }) } From 193933fac425c8bb09d7d137d4f5a21a7c22cea7 Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 13 Jan 2020 12:15:16 -0500 Subject: [PATCH 081/109] Use an internal method to get the pool It's clearer than using the method from the Engine API, which exists really for providing an interface to the D-Bus. Signed-off-by: mulhern --- src/engine/strat_engine/engine.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/engine/strat_engine/engine.rs b/src/engine/strat_engine/engine.rs index 1bbf472160..01562fa591 100644 --- a/src/engine/strat_engine/engine.rs +++ b/src/engine/strat_engine/engine.rs @@ -198,8 +198,9 @@ impl StratEngine { self.incomplete_pools.insert(pool_uuid, devices); } }; - self.get_mut_pool(pool_uuid) - .map(|(_, pool)| (pool_uuid, pool)) + self.pools + .get_mut_by_uuid(pool_uuid) + .map(|(_, pool)| (pool_uuid, pool as &mut dyn Pool)) } }) } From 7affdab8cb91c695e76190d940ce2bdbc3a1ed7d Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 13 Jan 2020 12:49:00 -0500 Subject: [PATCH 082/109] Make a private method called try_setup_pool Use it in the two places where it should be used. Signed-off-by: mulhern --- src/engine/strat_engine/engine.rs | 50 ++++++++++++++----------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/src/engine/strat_engine/engine.rs b/src/engine/strat_engine/engine.rs index 01562fa591..9da2b84482 100644 --- a/src/engine/strat_engine/engine.rs +++ b/src/engine/strat_engine/engine.rs @@ -138,33 +138,35 @@ impl StratEngine { devlinks::setup_dev_path()?; - let pools = find_all()?; - - let mut table = Table::default(); - let mut incomplete_pools = HashMap::new(); - for (pool_uuid, devices) in pools { - match setup_pool(pool_uuid, &devices, &table) { - Ok((pool_name, pool)) => { - table.insert(pool_name, pool_uuid, pool); - } - Err(err) => { - warn!("no pool set up, reason: {:?}", err); - incomplete_pools.insert(pool_uuid, devices); - } - } - } - - let engine = StratEngine { - pools: table, - incomplete_pools, + let mut engine = StratEngine { + pools: Table::default(), + incomplete_pools: HashMap::new(), watched_dev_last_event_nrs: HashMap::new(), }; + for (pool_uuid, devices) in find_all()? { + engine.try_setup_pool(pool_uuid, devices); + } + devlinks::cleanup_devlinks(engine.pools().iter()); Ok(engine) } + // Given a set of devices, try to set up a pool. If the setup fails, + // insert the devices into incomplete_pools. + fn try_setup_pool(&mut self, pool_uuid: PoolUuid, devices: HashMap) { + match setup_pool(pool_uuid, &devices, &self.pools) { + Ok((pool_name, pool)) => { + self.pools.insert(pool_name, pool_uuid, pool); + } + Err(err) => { + warn!("no pool set up, reason: {:?}", err); + self.incomplete_pools.insert(pool_uuid, devices); + } + } + } + /// Teardown Stratis, preparatory to a shutdown. #[cfg(test)] pub fn teardown(self) -> StratisResult<()> { @@ -189,15 +191,7 @@ impl StratEngine { .remove(&pool_uuid) .unwrap_or_else(HashMap::new); devices.insert(device, dev_node); - match setup_pool(pool_uuid, &devices, &self.pools) { - Ok((pool_name, pool)) => { - self.pools.insert(pool_name, pool_uuid, pool); - } - Err(err) => { - warn!("no pool set up, reason: {:?}", err); - self.incomplete_pools.insert(pool_uuid, devices); - } - }; + self.try_setup_pool(pool_uuid, devices); self.pools .get_mut_by_uuid(pool_uuid) .map(|(_, pool)| (pool_uuid, pool as &mut dyn Pool)) From de5a76c02dc766c2b8586d091ddbd4bceae329d8 Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 14 Jan 2020 09:28:06 -0500 Subject: [PATCH 083/109] Make check_metadata private to the pool module For better encapsulation. Signed-off-by: mulhern --- src/engine/strat_engine/engine.rs | 16 +++------------- src/engine/strat_engine/pool.rs | 4 +++- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/src/engine/strat_engine/engine.rs b/src/engine/strat_engine/engine.rs index 9da2b84482..1c86283b91 100644 --- a/src/engine/strat_engine/engine.rs +++ b/src/engine/strat_engine/engine.rs @@ -26,7 +26,7 @@ use crate::{ devlinks, dm::{get_dm, get_dm_init}, names::validate_name, - pool::{check_metadata, StratPool}, + pool::StratPool, }, structures::Table, types::{CreateAction, DeleteAction, RenameAction}, @@ -76,25 +76,15 @@ fn setup_pool( return Err(StratisError::Engine(ErrorEnum::AlreadyExists, err_msg)); } - check_metadata(&metadata) + StratPool::setup(pool_uuid, devices, timestamp, &metadata) .or_else(|e| { let err_msg = format!( - "inconsistent metadata for {}: reason: {:?}", + "failed to set up pool for {}: reason: {:?}", info_string(), e ); Err(StratisError::Engine(ErrorEnum::Error, err_msg)) }) - .and_then(|_| { - StratPool::setup(pool_uuid, devices, timestamp, &metadata).or_else(|e| { - let err_msg = format!( - "failed to set up pool for {}: reason: {:?}", - info_string(), - e - ); - Err(StratisError::Engine(ErrorEnum::Error, err_msg)) - }) - }) .and_then(|(pool_name, pool)| { devlinks::setup_pool_devlinks(&pool_name, &pool); Ok((pool_name, pool)) diff --git a/src/engine/strat_engine/pool.rs b/src/engine/strat_engine/pool.rs index 5eaac32d8a..197fab2c1a 100644 --- a/src/engine/strat_engine/pool.rs +++ b/src/engine/strat_engine/pool.rs @@ -69,7 +69,7 @@ fn next_index(flex_devs: &FlexDevsSave) -> Sectors { /// Check the metadata of an individual pool for consistency. /// Precondition: This method is called only when setting up a pool, which /// ensures that the flex devs metadata lists are all non-empty. -pub fn check_metadata(metadata: &PoolSave) -> StratisResult<()> { +fn check_metadata(metadata: &PoolSave) -> StratisResult<()> { let flex_devs = &metadata.flex_devs; let next = next_index(flex_devs); let allocated_from_cap = metadata.backstore.cap.allocs[0].1; @@ -195,6 +195,8 @@ impl StratPool { timestamp: DateTime, metadata: &PoolSave, ) -> StratisResult<(Name, StratPool)> { + check_metadata(metadata)?; + let mut backstore = Backstore::setup(uuid, &metadata.backstore, devnodes, timestamp)?; let mut thinpool = ThinPool::setup( uuid, From 10958084b247a48fc990adbeb2cc847f75a5f29e Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 14 Jan 2020 10:21:16 -0500 Subject: [PATCH 084/109] Move setup_pool into try_pool_setup It serves no purpose as an independent method. Signed-off-by: mulhern --- src/engine/strat_engine/engine.rs | 108 +++++++++++++++--------------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/src/engine/strat_engine/engine.rs b/src/engine/strat_engine/engine.rs index 1c86283b91..d98dd9e271 100644 --- a/src/engine/strat_engine/engine.rs +++ b/src/engine/strat_engine/engine.rs @@ -37,60 +37,6 @@ use crate::{ const REQUIRED_DM_MINOR_VERSION: u32 = 37; -/// Setup a pool from constituent devices in the context of some already -/// setup pools. Return an error on anything that prevents the pool -/// being set up. -/// Precondition: every device in devices has already been determined to belong -/// to the pool with pool_uuid. -fn setup_pool( - pool_uuid: PoolUuid, - devices: &HashMap, - pools: &Table, -) -> StratisResult<(Name, StratPool)> { - // FIXME: In this method, various errors are assembled from various - // sources and combined into strings, so that they - // can be printed as log messages if necessary. Instead, some kind of - // error-chaining should be used here and if it is necessary - // to log the error, the log code should be able to reduce the error - // chain to something that can be sensibly logged. - let info_string = || { - let dev_paths = devices - .values() - .map(|p| p.to_str().expect("Unix is utf-8")) - .collect::>() - .join(" ,"); - format!("(pool UUID: {}, devnodes: {})", pool_uuid, dev_paths) - }; - - let (timestamp, metadata) = get_metadata(pool_uuid, devices)?.ok_or_else(|| { - let err_msg = format!("no metadata found for {}", info_string()); - StratisError::Engine(ErrorEnum::NotFound, err_msg) - })?; - - if pools.contains_name(&metadata.name) { - let err_msg = format!( - "pool with name \"{}\" set up; metadata specifies same name for {}", - &metadata.name, - info_string() - ); - return Err(StratisError::Engine(ErrorEnum::AlreadyExists, err_msg)); - } - - StratPool::setup(pool_uuid, devices, timestamp, &metadata) - .or_else(|e| { - let err_msg = format!( - "failed to set up pool for {}: reason: {:?}", - info_string(), - e - ); - Err(StratisError::Engine(ErrorEnum::Error, err_msg)) - }) - .and_then(|(pool_name, pool)| { - devlinks::setup_pool_devlinks(&pool_name, &pool); - Ok((pool_name, pool)) - }) -} - #[derive(Debug)] pub struct StratEngine { pools: Table, @@ -146,6 +92,60 @@ impl StratEngine { // Given a set of devices, try to set up a pool. If the setup fails, // insert the devices into incomplete_pools. fn try_setup_pool(&mut self, pool_uuid: PoolUuid, devices: HashMap) { + /// Setup a pool from constituent devices in the context of some already + /// setup pools. Return an error on anything that prevents the pool + /// being set up. + /// Precondition: every device in devices has already been determined to belong + /// to the pool with pool_uuid. + fn setup_pool( + pool_uuid: PoolUuid, + devices: &HashMap, + pools: &Table, + ) -> StratisResult<(Name, StratPool)> { + // FIXME: In this method, various errors are assembled from various + // sources and combined into strings, so that they + // can be printed as log messages if necessary. Instead, some kind of + // error-chaining should be used here and if it is necessary + // to log the error, the log code should be able to reduce the error + // chain to something that can be sensibly logged. + let info_string = || { + let dev_paths = devices + .values() + .map(|p| p.to_str().expect("Unix is utf-8")) + .collect::>() + .join(" ,"); + format!("(pool UUID: {}, devnodes: {})", pool_uuid, dev_paths) + }; + + let (timestamp, metadata) = get_metadata(pool_uuid, devices)?.ok_or_else(|| { + let err_msg = format!("no metadata found for {}", info_string()); + StratisError::Engine(ErrorEnum::NotFound, err_msg) + })?; + + if pools.contains_name(&metadata.name) { + let err_msg = format!( + "pool with name \"{}\" set up; metadata specifies same name for {}", + &metadata.name, + info_string() + ); + return Err(StratisError::Engine(ErrorEnum::AlreadyExists, err_msg)); + } + + StratPool::setup(pool_uuid, devices, timestamp, &metadata) + .or_else(|e| { + let err_msg = format!( + "failed to set up pool for {}: reason: {:?}", + info_string(), + e + ); + Err(StratisError::Engine(ErrorEnum::Error, err_msg)) + }) + .and_then(|(pool_name, pool)| { + devlinks::setup_pool_devlinks(&pool_name, &pool); + Ok((pool_name, pool)) + }) + } + match setup_pool(pool_uuid, &devices, &self.pools) { Ok((pool_name, pool)) => { self.pools.insert(pool_name, pool_uuid, pool); From 5d98ac25897f140393bf59bb4d7d19f096f43b0a Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 14 Jan 2020 10:52:48 -0500 Subject: [PATCH 085/109] Move devlinks setup to the part where the action is done In an effort to gather up the actions in a separate place from the error checking. Signed-off-by: mulhern --- src/engine/strat_engine/engine.rs | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/engine/strat_engine/engine.rs b/src/engine/strat_engine/engine.rs index d98dd9e271..cfbd779b29 100644 --- a/src/engine/strat_engine/engine.rs +++ b/src/engine/strat_engine/engine.rs @@ -131,23 +131,19 @@ impl StratEngine { return Err(StratisError::Engine(ErrorEnum::AlreadyExists, err_msg)); } - StratPool::setup(pool_uuid, devices, timestamp, &metadata) - .or_else(|e| { - let err_msg = format!( - "failed to set up pool for {}: reason: {:?}", - info_string(), - e - ); - Err(StratisError::Engine(ErrorEnum::Error, err_msg)) - }) - .and_then(|(pool_name, pool)| { - devlinks::setup_pool_devlinks(&pool_name, &pool); - Ok((pool_name, pool)) - }) + StratPool::setup(pool_uuid, devices, timestamp, &metadata).or_else(|e| { + let err_msg = format!( + "failed to set up pool for {}: reason: {:?}", + info_string(), + e + ); + Err(StratisError::Engine(ErrorEnum::Error, err_msg)) + }) } match setup_pool(pool_uuid, &devices, &self.pools) { Ok((pool_name, pool)) => { + devlinks::setup_pool_devlinks(&pool_name, &pool); self.pools.insert(pool_name, pool_uuid, pool); } Err(err) => { From 0f49f1350077cd1f7fd6cecf702b4b56a9d31335 Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 14 Jan 2020 11:25:37 -0500 Subject: [PATCH 086/109] Use setup_pool methods to distinguish among reasons for no setup Some of them can arise through normal operation, and do not represent any actual problem. Signed-off-by: mulhern --- src/engine/strat_engine/engine.rs | 86 +++++++++++++++---------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/src/engine/strat_engine/engine.rs b/src/engine/strat_engine/engine.rs index cfbd779b29..f280cf99b3 100644 --- a/src/engine/strat_engine/engine.rs +++ b/src/engine/strat_engine/engine.rs @@ -92,62 +92,62 @@ impl StratEngine { // Given a set of devices, try to set up a pool. If the setup fails, // insert the devices into incomplete_pools. fn try_setup_pool(&mut self, pool_uuid: PoolUuid, devices: HashMap) { - /// Setup a pool from constituent devices in the context of some already - /// setup pools. Return an error on anything that prevents the pool - /// being set up. - /// Precondition: every device in devices has already been determined to belong - /// to the pool with pool_uuid. + // Setup a pool from constituent devices in the context of some already + // setup pools. + // Return None if the pool's metadata was not found. This is a + // legitimate non-error condition, which may result if only a subset + // of the pool's devices are in the set of devices being used. + // Return an error on all other errors. Note that any one of these + // errors could represent a temporary condition, that could be changed + // by finding another device. So it is reasonable to treat them all + // as loggable at the warning level, but not at the error level. + // Precondition: every device in devices has already been determined to belong + // to the pool with pool_uuid. fn setup_pool( pool_uuid: PoolUuid, devices: &HashMap, pools: &Table, - ) -> StratisResult<(Name, StratPool)> { - // FIXME: In this method, various errors are assembled from various - // sources and combined into strings, so that they - // can be printed as log messages if necessary. Instead, some kind of - // error-chaining should be used here and if it is necessary - // to log the error, the log code should be able to reduce the error - // chain to something that can be sensibly logged. - let info_string = || { - let dev_paths = devices - .values() - .map(|p| p.to_str().expect("Unix is utf-8")) - .collect::>() - .join(" ,"); - format!("(pool UUID: {}, devnodes: {})", pool_uuid, dev_paths) + ) -> Result, String> { + let (timestamp, metadata) = match get_metadata(pool_uuid, devices) { + Err(err) => return Err(format!( + "There was an error encountered when reading the metadata for the devices found for pool with UUID {}: {}", + pool_uuid.to_simple_ref(), + err)), + Ok(None) => return Ok(None), + Ok(Some((timestamp, metadata))) => (timestamp, metadata), }; - let (timestamp, metadata) = get_metadata(pool_uuid, devices)?.ok_or_else(|| { - let err_msg = format!("no metadata found for {}", info_string()); - StratisError::Engine(ErrorEnum::NotFound, err_msg) - })?; - - if pools.contains_name(&metadata.name) { - let err_msg = format!( - "pool with name \"{}\" set up; metadata specifies same name for {}", - &metadata.name, - info_string() - ); - return Err(StratisError::Engine(ErrorEnum::AlreadyExists, err_msg)); + if let Some((uuid, _)) = pools.get_by_name(&metadata.name) { + return Err(format!( + "There is a pool name conflict. The devices currently being processed have been identified as belonging to the pool with UUID {} and name {}, but a pool with the same name and UUID {} is already active", + pool_uuid.to_simple_ref(), + &metadata.name, + uuid.to_simple_ref())); } - StratPool::setup(pool_uuid, devices, timestamp, &metadata).or_else(|e| { - let err_msg = format!( - "failed to set up pool for {}: reason: {:?}", - info_string(), - e - ); - Err(StratisError::Engine(ErrorEnum::Error, err_msg)) - }) + StratPool::setup(pool_uuid, devices, timestamp, &metadata) + .map_err(|err| { + format!( + "An attempt to set up pool with UUID {} from the assembled devices failed: {}", + pool_uuid.to_simple_ref(), + err + ) + }) + .map(Some) + } + + let result = setup_pool(pool_uuid, &devices, &self.pools); + + if let Err(err) = &result { + warn!("{}", err); } - match setup_pool(pool_uuid, &devices, &self.pools) { - Ok((pool_name, pool)) => { + match result { + Ok(Some((pool_name, pool))) => { devlinks::setup_pool_devlinks(&pool_name, &pool); self.pools.insert(pool_name, pool_uuid, pool); } - Err(err) => { - warn!("no pool set up, reason: {:?}", err); + _ => { self.incomplete_pools.insert(pool_uuid, devices); } } From 5b5d3e30a5ab7bb3e39b067ca9667d78f1f7613f Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 14 Jan 2020 18:36:04 -0500 Subject: [PATCH 087/109] Revise block_evaluate method documentation Make it more like a precise specification and less like a description of what it usually does. Signed-off-by: mulhern --- src/engine/strat_engine/engine.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/engine/strat_engine/engine.rs b/src/engine/strat_engine/engine.rs index f280cf99b3..f58aaa31c5 100644 --- a/src/engine/strat_engine/engine.rs +++ b/src/engine/strat_engine/engine.rs @@ -161,12 +161,8 @@ impl StratEngine { /// Given a udev database entry, process the entry. /// - /// If all the devices are present in the pool and the pool isn't already - /// up and running, it will get setup and the newly created pool and UUID - /// will be returned. - /// - /// Logs a warning if the block devices appears to be a Stratis block - /// device and no pool is set up. + /// If a new pool is created as a result of the processing, return + /// the newly created pool and its UUID, otherwise return None. fn block_evaluate(&mut self, device: &libudev::Device) -> Option<(PoolUuid, &mut dyn Pool)> { identify_block_device(device).and_then(move |(pool_uuid, _, device, dev_node)| { if self.pools.contains_uuid(pool_uuid) { From a3e280624e9bbb676c108a562124f88258536c37 Mon Sep 17 00:00:00 2001 From: Bryan Gurney Date: Thu, 16 Jan 2020 15:08:16 -0500 Subject: [PATCH 088/109] stratisd: increase minimum rust version to 1.39 Signed-off-by: Bryan Gurney --- .travis.yml | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index c0cf805cf2..6a35085980 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,10 +34,10 @@ matrix: # MANDATORY TESTING USING LOWEST SUPPORTED COMPILER # tests - - rust: 1.38.0 + - rust: 1.39.0 env: TASK=test # release - - rust: 1.38.0 + - rust: 1.39.0 env: TASK=release diff --git a/README.md b/README.md index ed191ee884..60d1e63482 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ versions of the compiler may disagree with the CI tasks on some points, so should be avoided. #### Building -Stratisd requires Rust 1.38+ and Cargo to build. These may be available via +Stratisd requires Rust 1.39+ and Cargo to build. These may be available via your distribution's package manager. If not, [Rustup](https://www.rustup.rs/) is available to install and update the Rust toolchain. Once toolchain and other dependencies are in place, run `make build` to build, and then run the From 2c6b0ed4afbc08009dcd0222173f13fd14472bf7 Mon Sep 17 00:00:00 2001 From: Leah Leshchinsky Date: Thu, 16 Jan 2020 15:43:45 -0500 Subject: [PATCH 089/109] Remove unnecessary clippy allows for new_ret_no_self Clippy has become less strict with this lint. Signed-off-by: Leah Leshchinsky --- src/engine/sim_engine/blockdev.rs | 1 - src/engine/sim_engine/pool.rs | 1 - src/engine/strat_engine/backstore/blockdev.rs | 1 - src/engine/strat_engine/backstore/cache_tier.rs | 1 - src/engine/strat_engine/backstore/range_alloc.rs | 1 - src/engine/strat_engine/thinpool/thinpool.rs | 1 - 6 files changed, 6 deletions(-) diff --git a/src/engine/sim_engine/blockdev.rs b/src/engine/sim_engine/blockdev.rs index 2b09d0a57a..6f8eee42d7 100644 --- a/src/engine/sim_engine/blockdev.rs +++ b/src/engine/sim_engine/blockdev.rs @@ -60,7 +60,6 @@ impl BlockDev for SimDev { impl SimDev { /// Generates a new device from any devnode. - #[allow(clippy::new_ret_no_self)] pub fn new(rdm: Rc>, devnode: &Path) -> (Uuid, SimDev) { ( Uuid::new_v4(), diff --git a/src/engine/sim_engine/pool.rs b/src/engine/sim_engine/pool.rs index 495004de18..1fccbc8d71 100644 --- a/src/engine/sim_engine/pool.rs +++ b/src/engine/sim_engine/pool.rs @@ -43,7 +43,6 @@ pub struct SimPool { } impl SimPool { - #[allow(clippy::new_ret_no_self)] pub fn new( rdm: &Rc>, paths: &[&Path], diff --git a/src/engine/strat_engine/backstore/blockdev.rs b/src/engine/strat_engine/backstore/blockdev.rs index bb4f3b5b9e..f528af4534 100644 --- a/src/engine/strat_engine/backstore/blockdev.rs +++ b/src/engine/strat_engine/backstore/blockdev.rs @@ -52,7 +52,6 @@ impl StratBlockDev { /// on the device is simply invisible to the blockdev. Consequently, it /// is invisible to the engine, and is not part of the total size value /// reported on the D-Bus. - #[allow(clippy::new_ret_no_self)] pub fn new( dev: Device, devnode: PathBuf, diff --git a/src/engine/strat_engine/backstore/cache_tier.rs b/src/engine/strat_engine/backstore/cache_tier.rs index e87d99de22..d3bb2c98a6 100644 --- a/src/engine/strat_engine/backstore/cache_tier.rs +++ b/src/engine/strat_engine/backstore/cache_tier.rs @@ -143,7 +143,6 @@ impl CacheTier { /// sub-device too big. /// /// WARNING: metadata changing event - #[allow(clippy::new_ret_no_self)] pub fn new(mut block_mgr: BlockDevMgr) -> StratisResult { let avail_space = block_mgr.avail_space(); diff --git a/src/engine/strat_engine/backstore/range_alloc.rs b/src/engine/strat_engine/backstore/range_alloc.rs index 04dcf00a3b..be281f666f 100644 --- a/src/engine/strat_engine/backstore/range_alloc.rs +++ b/src/engine/strat_engine/backstore/range_alloc.rs @@ -20,7 +20,6 @@ pub struct RangeAllocator { impl RangeAllocator { /// Create a new RangeAllocator with the specified (offset, length) /// ranges marked as used. - #[allow(clippy::new_ret_no_self)] pub fn new( limit: BlockdevSize, initial_used: &[(Sectors, Sectors)], diff --git a/src/engine/strat_engine/thinpool/thinpool.rs b/src/engine/strat_engine/thinpool/thinpool.rs index fe1d048715..c419a88fe5 100644 --- a/src/engine/strat_engine/thinpool/thinpool.rs +++ b/src/engine/strat_engine/thinpool/thinpool.rs @@ -235,7 +235,6 @@ pub struct ThinPool { impl ThinPool { /// Make a new thin pool. - #[allow(clippy::new_ret_no_self)] pub fn new( pool_uuid: PoolUuid, thin_pool_size: &ThinPoolSizeParams, From ab39719b3e308409c0092b1249ceada6827c3d0b Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 21 Jan 2020 15:50:30 -0500 Subject: [PATCH 090/109] Omit global gcc-multilib requirement It's redundant, because it is required explicitly in the one task where it is needed. Signed-off-by: mulhern --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6a35085980..060682fe9f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,8 +7,6 @@ addons: - libdbus-1-dev - libudev-dev - libdbus-glib-1-dev - # needed for i686-unknown-linux-gnu target - - gcc-multilib - python3-pyudev language: rust From 7163c4363b9e9dbabe8fec1887f8e87637b38bd7 Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 21 Jan 2020 18:24:14 -0500 Subject: [PATCH 091/109] Add name fields to matrix Otherwise, it is easy to become confused when looking at the Travis display. Remove all comments that are redundant, including the ones that are also obsoleted, like the one that mentions yapf. Signed-off-by: mulhern --- .travis.yml | 59 +++++++++++++++++++++++++++-------------------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/.travis.yml b/.travis.yml index 060682fe9f..7c94b8cd36 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,66 +19,66 @@ matrix: include: # MANDATORY CHECKS USING CURRENT DEVELOPMENT COMPILER - # rustfmt - - rust: 1.40.0 + - name: "format Rust source using current development toolchain" + rust: 1.40.0 before_script: - rustup component add rustfmt env: TASK=fmt-travis - # clippy - - rust: 1.40.0 + - name: "lint Rust source using current development toolchain" + rust: 1.40.0 before_script: - rustup component add clippy env: TASK=clippy # MANDATORY TESTING USING LOWEST SUPPORTED COMPILER - # tests - - rust: 1.39.0 + - name: "run Rust unit tests on lowest supported toolchain" + rust: 1.39.0 env: TASK=test - # release - - rust: 1.39.0 + - name: "build release on lowest supported toolchain" + rust: 1.39.0 env: TASK=release # MANDATORY TESTING ON STABLE - # compile - - rust: stable + - name: "build using stable toolchain" + rust: stable env: TASK=build TARGET=x86_64-unknown-linux-gnu - # compile with no defaults enabled - - rust: stable + - name: "build without defaults using stable toolchain" + rust: stable env: TASK=build-no-default TARGET=x86_64-unknown-linux-gnu - # compile on a 32-bit system - - rust: stable + - name: "build Rust source on a 32-bit system using stable toolchain" + rust: stable env: TASK=build TARGET=i686-unknown-linux-gnu PKG_CONFIG_ALLOW_CROSS=1 PKG_CONFIG_PATH=/usr/lib/i386-linux-gnu/pkgconfig/ install: - rustup target add $TARGET - sudo dpkg --add-architecture i386 - sudo apt-get update - sudo apt-get install -y gcc-multilib libdbus-1-dev:i386 libdbus-glib-1-dev:i386 libglib2.0-dev:i386 libudev-dev:i386 - # Build docs - - rust: stable + - name: "build Rust docs using stable toolchain" + rust: stable env: TASK=docs-travis - # test - - rust: stable + - name: "run Rust unit tests using stable toolchain" + rust: stable env: TASK=test - # destructive tests that can be run on Travis - - rust: stable + - name: "run Rust destructive unit tests using stable toolchain" + rust: stable sudo: required script: sudo PATH=${TRAVIS_HOME}/.cargo/bin:$PATH make -f Makefile test-travis - # release - - rust: stable + - name: "build release using stable toolchain" + rust: stable env: TASK=release # MANDATORY PYTHON CHECKS - # Run pylint, Python linter, on any Python test code - - language: python + - name: "lint Python test code on Python 3.7" + language: python python: "3.7" install: pip3 install -r tests/client-dbus/requirements.txt before_script: - cd tests/client-dbus env: TASK=lint - # Format any Python test code using yapf - - language: python + - name: "format Python test code on Python 3.7" + language: python python: "3.7" install: pip3 install -r tests/client-dbus/requirements.txt before_script: @@ -89,8 +89,8 @@ matrix: # INTERMITTENTLY ALLOWED FAILURES # Allowed if a failure occurs after a new Rust release until the # failure is addressed. - # Run clippy on rust beta, in order to be good Rustaceans. - - rust: beta + - name: "lint Rust source using beta toolchain" + rust: beta before_script: - rustup component add clippy env: TASK=clippy @@ -100,7 +100,8 @@ matrix: # Run audit on Rust stable. Make it an allowed failure, because: # * It takes 9 minutes, the longest of any task. # * It should be an advisory, and should not gate our development. - - rust: stable + - name: "run Rust audit task using stable toolchain" + rust: stable env: TASK=audit branches: From 91baf274c83d33ca7481063db779da78956f130e Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 21 Jan 2020 18:56:35 -0500 Subject: [PATCH 092/109] Remove requirements.txt Require task dependencies in the task where they are required. This way, dependencies that are only required by one task are not still installed for all tasks. Signed-off-by: mulhern --- .travis.yml | 4 ++-- tests/client-dbus/requirements.txt | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) delete mode 100644 tests/client-dbus/requirements.txt diff --git a/.travis.yml b/.travis.yml index 7c94b8cd36..1c6ec0e58f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -73,14 +73,14 @@ matrix: - name: "lint Python test code on Python 3.7" language: python python: "3.7" - install: pip3 install -r tests/client-dbus/requirements.txt + install: pip3 install tox==3.2.1 before_script: - cd tests/client-dbus env: TASK=lint - name: "format Python test code on Python 3.7" language: python python: "3.7" - install: pip3 install -r tests/client-dbus/requirements.txt + install: pip3 install black==19.3b0 isort==4.3.4 before_script: - cd tests/client-dbus env: TASK=fmt-travis diff --git a/tests/client-dbus/requirements.txt b/tests/client-dbus/requirements.txt deleted file mode 100644 index 97d96ded43..0000000000 --- a/tests/client-dbus/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -tox==3.2.1 -black==19.3b0 -isort==4.3.4 From 45d6352baba632f7e23a90a9ffc878ba585152f9 Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 21 Jan 2020 18:59:21 -0500 Subject: [PATCH 093/109] Get rid of use of TARGET variable The reason these tasks don't fail is that the specified target corresponds with the default target. Let's just let the toolchain pick the default target. Signed-off-by: mulhern --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1c6ec0e58f..25661089a1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,10 +42,10 @@ matrix: # MANDATORY TESTING ON STABLE - name: "build using stable toolchain" rust: stable - env: TASK=build TARGET=x86_64-unknown-linux-gnu + env: TASK=build - name: "build without defaults using stable toolchain" rust: stable - env: TASK=build-no-default TARGET=x86_64-unknown-linux-gnu + env: TASK=build-no-default - name: "build Rust source on a 32-bit system using stable toolchain" rust: stable env: TASK=build TARGET=i686-unknown-linux-gnu PKG_CONFIG_ALLOW_CROSS=1 PKG_CONFIG_PATH=/usr/lib/i386-linux-gnu/pkgconfig/ From c81ca16e22312d1f453280690445153ed141122b Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 21 Jan 2020 19:15:27 -0500 Subject: [PATCH 094/109] Omit requirement of python3-pyudev That requirement is handled by tox. Signed-off-by: mulhern --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 25661089a1..86c3184d10 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,6 @@ addons: - libdbus-1-dev - libudev-dev - libdbus-glib-1-dev - - python3-pyudev language: rust From 1801855e1194d5c20d533e4c152d69aad0142006 Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 21 Jan 2020 19:30:58 -0500 Subject: [PATCH 095/109] Just use "pip" not "pip3" Travis should be able to figure out which is meant, since we're running on Python 3.7. Signed-off-by: mulhern --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 86c3184d10..342518a189 100644 --- a/.travis.yml +++ b/.travis.yml @@ -72,14 +72,14 @@ matrix: - name: "lint Python test code on Python 3.7" language: python python: "3.7" - install: pip3 install tox==3.2.1 + install: pip install tox==3.2.1 before_script: - cd tests/client-dbus env: TASK=lint - name: "format Python test code on Python 3.7" language: python python: "3.7" - install: pip3 install black==19.3b0 isort==4.3.4 + install: pip install black==19.3b0 isort==4.3.4 before_script: - cd tests/client-dbus env: TASK=fmt-travis From cd02c6dbdb402258eb6da76070de06802c15a257 Mon Sep 17 00:00:00 2001 From: mulhern Date: Wed, 22 Jan 2020 12:23:57 -0500 Subject: [PATCH 096/109] Run build-tests instead of just build for cross-compilation target This target builds the tests as well as the source, and so provides better coverage. Signed-off-by: mulhern --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 342518a189..32e9b4f174 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,9 +45,9 @@ matrix: - name: "build without defaults using stable toolchain" rust: stable env: TASK=build-no-default - - name: "build Rust source on a 32-bit system using stable toolchain" + - name: "build Rust source and tests on a 32-bit system using stable toolchain" rust: stable - env: TASK=build TARGET=i686-unknown-linux-gnu PKG_CONFIG_ALLOW_CROSS=1 PKG_CONFIG_PATH=/usr/lib/i386-linux-gnu/pkgconfig/ + env: TASK=build-tests TARGET=i686-unknown-linux-gnu PKG_CONFIG_ALLOW_CROSS=1 PKG_CONFIG_PATH=/usr/lib/i386-linux-gnu/pkgconfig/ install: - rustup target add $TARGET - sudo dpkg --add-architecture i386 From 6dd444fbf6fc4a4290109c3e9e444984454a46cd Mon Sep 17 00:00:00 2001 From: mulhern Date: Thu, 23 Jan 2020 17:40:18 -0500 Subject: [PATCH 097/109] No longer use tox on Travis It is discouraged, since Travis makes its own virtual environment. Introduce the notion of a recommended development Python interpreter; analogous to recommded development Rust toolchain. Signed-off-by: mulhern --- .travis.yml | 14 +++++++------- tests/client-dbus/Makefile | 4 ++-- tests/client-dbus/tox.ini | 13 ------------- 3 files changed, 9 insertions(+), 22 deletions(-) delete mode 100644 tests/client-dbus/tox.ini diff --git a/.travis.yml b/.travis.yml index 32e9b4f174..eeb6f26c8b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -68,17 +68,17 @@ matrix: env: TASK=release - # MANDATORY PYTHON CHECKS - - name: "lint Python test code on Python 3.7" + # MANDATORY PYTHON CHECKS ON RECOMMENDED DEVELOPMENT INTERPRETER + - name: "lint Python test code on recommended development interpreter" language: python - python: "3.7" - install: pip install tox==3.2.1 + python: "3.7.6" + install: pip install pylint==2.3.1 dbus-client-gen==0.4 dbus-python-client-gen==0.7 psutil==5.4.3 pyudev==0.21.0 before_script: - cd tests/client-dbus - env: TASK=lint - - name: "format Python test code on Python 3.7" + script: PYTHONPATH=./src make -f Makefile lint + - name: "format Python test code on recommended development interpreter" language: python - python: "3.7" + python: "3.7.6" install: pip install black==19.3b0 isort==4.3.4 before_script: - cd tests/client-dbus diff --git a/tests/client-dbus/Makefile b/tests/client-dbus/Makefile index fbc53c5073..daa87573df 100644 --- a/tests/client-dbus/Makefile +++ b/tests/client-dbus/Makefile @@ -1,9 +1,9 @@ PYTEST_OPTS = --verbose -TOX=tox .PHONY: lint lint: - $(TOX) -c tox.ini -e lint + ./check.py src/stratisd_client_dbus + ./check.py tests .PHONY: dbus-tests dbus-tests: diff --git a/tests/client-dbus/tox.ini b/tests/client-dbus/tox.ini deleted file mode 100644 index 42c2aa13d8..0000000000 --- a/tests/client-dbus/tox.ini +++ /dev/null @@ -1,13 +0,0 @@ -[tox] -envlist=lint - -[testenv:lint] -deps = - dbus-python - psutil - pylint - pytest>=2.8 - pyudev -commands = - ./check.py src/stratisd_client_dbus - ./check.py tests From 08ec460a2237426f41b34c305443afffb4af11a8 Mon Sep 17 00:00:00 2001 From: mulhern Date: Fri, 24 Jan 2020 10:14:10 -0500 Subject: [PATCH 098/109] Add changelog matter from previous releases to CHANGES.txt Because sometimes users are running older versions, and it would be nice to know at a glance what bugs their version could be expected to have. It is also sometimes helpful to have a more complete record, generally. Signed-off-by: mulhern --- CHANGES.txt | 132 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index 001dbc283e..9a6b5d512a 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -125,3 +125,135 @@ New minimum Rust crate requirements: https://github.com/stratis-storage/stratisd/pull/1566 https://github.com/stratis-storage/stratisd/pull/1565 https://github.com/stratis-storage/stratisd/pull/1563 + + +stratisd 1.0.5 +============== +Recommended Rust toolchain version: 1.33 +Lowest supported Rust toolchain version: 1.31 +Python auto-formatter: yapf (0.21.0) + +New minimum Rust crate requirements: + - devicemapper: 0.28 + - libmount: 0.1.13 + - nix: 0.14 + +- Fix an error in the calculation of the maximum size of variable length + metadata that a single block device can store: + https://github.com/stratis-storage/stratisd/pull/1524 + +- Make a note of some code defects that would cause a bug if variable + length metadata were very large, as might occur if a pool contained very + many devices: + https://github.com/stratis-storage/stratisd/pull/1521 + +- Clarify the error message that stratisd displays if an external application + that it depends on is missing: + https://github.com/stratis-storage/stratisd/pull/1547 + +- Use nested imports in all Rust source: + https://github.com/stratis-storage/stratisd/pull/1517 + +- Metadata refactoring to improve encapsulation and clarity and to use + types to distinguish among the sizes of different metadata regions: + https://github.com/stratis-storage/stratisd/pull/1554 + https://github.com/stratis-storage/stratisd/pull/1549 + https://github.com/stratis-storage/stratisd/pull/1546 + https://github.com/stratis-storage/stratisd/pull/1545 + https://github.com/stratis-storage/stratisd/pull/1541 + https://github.com/stratis-storage/stratisd/pull/1534 + https://github.com/stratis-storage/stratisd/pull/1522 + https://github.com/stratis-storage/stratisd/pull/1516 + +- Tidies and Maintenance: + https://github.com/stratis-storage/stratisd/pull/1560 + https://github.com/stratis-storage/stratisd/pull/1555 + https://github.com/stratis-storage/stratisd/pull/1553 + https://github.com/stratis-storage/stratisd/pull/1552 + https://github.com/stratis-storage/stratisd/pull/1550 + https://github.com/stratis-storage/stratisd/pull/1537 + https://github.com/stratis-storage/stratisd/pull/1536 + https://github.com/stratis-storage/stratisd/pull/1535 + https://github.com/stratis-storage/stratisd/pull/1532 + https://github.com/stratis-storage/stratisd/pull/1530 + https://github.com/stratis-storage/stratisd/pull/1528 + https://github.com/stratis-storage/stratisd/pull/1525 + https://github.com/stratis-storage/stratisd/pull/1515 + https://github.com/stratis-storage/stratisd/pull/1508 + https://github.com/stratis-storage/stratisd/pull/1507 + + +stratisd 1.0.4 +============== +Recommended Rust toolchain version: 1.33 +Lowest supported Rust toolchain version: 1.31 +Python auto-formatter: yapf (0.21.0) + +New minimum Rust crate requirements: + - devicemapper: 0.27.0 + - libc: 0.2.47 + - nix: 0.13 + +- Fix a bug where stratisd was not writing to Stratis filesystem metadata + properly: + https://github.com/stratis-storage/stratisd/pull/1480 + +- Require new version of devicemapper that contains fixes to devicemapper + status parsing code, and handle the newly available values appropriately: + https://github.com/stratis-storage/stratisd/pull/1461 + +- Set RUST_BACKTRACE to 1 in the systemd service file in order to ensure that + a stack trace is generated if stratisd panics: + https://github.com/stratis-storage/stratisd/pull/1479 + +- Use edition 2018: + https://github.com/stratis-storage/stratisd/pull/1501 + +- Do not combine the errors that might result from reading one or the other + of the Stratis static headers from a device: + https://github.com/stratis-storage/stratisd/pull/1439 + +- Tidies and Maintenance: + https://github.com/stratis-storage/stratisd/pull/1510 + https://github.com/stratis-storage/stratisd/pull/1505 + https://github.com/stratis-storage/stratisd/pull/1504 + https://github.com/stratis-storage/stratisd/pull/1503 + https://github.com/stratis-storage/stratisd/pull/1500 + https://github.com/stratis-storage/stratisd/pull/1498 + https://github.com/stratis-storage/stratisd/pull/1497 + https://github.com/stratis-storage/stratisd/pull/1493 + https://github.com/stratis-storage/stratisd/pull/1492 + https://github.com/stratis-storage/stratisd/pull/1488 + https://github.com/stratis-storage/stratisd/pull/1484 + https://github.com/stratis-storage/stratisd/pull/1477 + https://github.com/stratis-storage/stratisd/pull/1469 + https://github.com/stratis-storage/stratisd/pull/1459 + https://github.com/stratis-storage/stratisd/pull/1452 + https://github.com/stratis-storage/stratisd/pull/1451 + https://github.com/stratis-storage/stratisd/pull/1450 + https://github.com/stratis-storage/stratisd/pull/1442 + https://github.com/stratis-storage/stratisd/pull/1441 + https://github.com/stratis-storage/stratisd/pull/1436 + https://github.com/stratis-storage/stratisd/pull/1435 + https://github.com/stratis-storage/stratisd/pull/1434 + https://github.com/stratis-storage/stratisd/pull/1432 + https://github.com/stratis-storage/stratisd/pull/1431 + https://github.com/stratis-storage/stratisd/pull/1427 + https://github.com/stratis-storage/stratisd/pull/1425 + https://github.com/stratis-storage/stratisd/pull/1423 + https://github.com/stratis-storage/stratisd/pull/1420 + https://github.com/stratis-storage/stratisd/pull/1418 + https://github.com/stratis-storage/stratisd/pull/1416 + https://github.com/stratis-storage/stratisd/pull/1415 + https://github.com/stratis-storage/stratisd/pull/1414 + https://github.com/stratis-storage/stratisd/pull/1413 + https://github.com/stratis-storage/stratisd/pull/1412 + https://github.com/stratis-storage/stratisd/pull/1411 + https://github.com/stratis-storage/stratisd/pull/1409 + https://github.com/stratis-storage/stratisd/pull/1407 + https://github.com/stratis-storage/stratisd/pull/1406 + https://github.com/stratis-storage/stratisd/pull/1405 + https://github.com/stratis-storage/stratisd/pull/1404 + https://github.com/stratis-storage/stratisd/pull/1403 + https://github.com/stratis-storage/stratisd/pull/1391 + https://github.com/stratis-storage/stratisd/pull/1345 From dc94e81b5d706ca3cc2a27c3a15806f71be9b264 Mon Sep 17 00:00:00 2001 From: mulhern Date: Thu, 30 Jan 2020 15:08:01 -0500 Subject: [PATCH 099/109] Temporarily allow beta lint task to fail To give us leisure to fix all the errors. Signed-off-by: mulhern --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index eeb6f26c8b..44fb0bc696 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,8 @@ language: rust matrix: fast_finish: true allow_failures: + # Temporarily allow beta lint task to fail + - rust: beta # Allow audit task to fail - env: TASK=audit include: From 56a63358c514c254eb8d4dbc23f677142b8aaca6 Mon Sep 17 00:00:00 2001 From: Bryan Gurney Date: Fri, 31 Jan 2020 13:43:53 -0500 Subject: [PATCH 100/109] Add stratisd startup message Signed-off-by: Bryan Gurney --- src/bin/stratisd.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bin/stratisd.rs b/src/bin/stratisd.rs index 13cb832a54..8bbed2d88e 100644 --- a/src/bin/stratisd.rs +++ b/src/bin/stratisd.rs @@ -340,6 +340,7 @@ fn run(matches: &ArgMatches, buff_log: &buff_log::Handle) -> let mut udev_monitor = UdevMonitor::create(&context)?; let engine: Rc> = { + info!("stratis daemon version {} started", VERSION); if matches.is_present("sim") { info!("Using SimEngine"); Rc::new(RefCell::new(SimEngine::default())) From 4e5374037a9bb3e8332d051df32987bfd096aa8e Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Wed, 22 Jan 2020 20:53:07 +0100 Subject: [PATCH 101/109] Move StaticHeader into its own file --- .../strat_engine/backstore/metadata/bda.rs | 616 +---------------- .../strat_engine/backstore/metadata/mod.rs | 12 +- .../backstore/metadata/static_header.rs | 629 ++++++++++++++++++ 3 files changed, 649 insertions(+), 608 deletions(-) create mode 100644 src/engine/strat_engine/backstore/metadata/static_header.rs diff --git a/src/engine/strat_engine/backstore/metadata/bda.rs b/src/engine/strat_engine/backstore/metadata/bda.rs index 5af5e1704a..5788dcf43f 100644 --- a/src/engine/strat_engine/backstore/metadata/bda.rs +++ b/src/engine/strat_engine/backstore/metadata/bda.rs @@ -2,80 +2,32 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. -use std::{ - fmt, - io::{self, Read, Seek, SeekFrom}, - str::from_utf8, -}; +use std::io::{Read, Seek}; -use byteorder::{ByteOrder, LittleEndian}; use chrono::{DateTime, Utc}; -use crc::crc32; use uuid::Uuid; -use devicemapper::{Sectors, IEC, SECTOR_SIZE}; - use crate::{ engine::{ strat_engine::{ backstore::metadata::{ mda, - sizes::{ - static_header_size, BDAExtendedSize, BlockdevSize, MDADataSize, MDASize, - ReservedSize, STATIC_HEADER_SIZE, - }, + sizes::{BDAExtendedSize, BlockdevSize, MDADataSize, STATIC_HEADER_SIZE}, + static_header::{MetadataLocation, StaticHeader}, }, device::SyncAll, }, types::{DevUuid, PoolUuid}, }, - stratis::{ErrorEnum, StratisError, StratisResult}, + stratis::StratisResult, }; -const RESERVED_SECTORS: Sectors = Sectors(3 * IEC::Mi / (SECTOR_SIZE as u64)); // = 3 MiB - -const STRAT_MAGIC: &[u8] = b"!Stra0tis\x86\xff\x02^\x41rh"; - -const STRAT_SIGBLOCK_VERSION: u8 = 1; - -/// Get a Stratis pool UUID and device UUID from any device. -/// If there is an error while obtaining these values return the error. -/// If the device does not appear to be a Stratis device, return None. -pub fn device_identifiers(f: &mut F) -> StratisResult> -where - F: Read + Seek + SyncAll, -{ - StaticHeader::setup(f).map(|sh| sh.map(|sh| (sh.pool_uuid, sh.dev_uuid))) -} - -/// Remove Stratis identifying information from device. -pub fn disown_device(f: &mut F) -> StratisResult<()> -where - F: Seek + SyncAll, -{ - StaticHeader::wipe(f) -} - -// Transform a constant in sectors to a constant in bytes -macro_rules! bytes { - ($number:expr) => { - $number * devicemapper::SECTOR_SIZE - }; -} - #[derive(Debug)] pub struct BDA { header: StaticHeader, regions: mda::MDARegions, } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum MetadataLocation { - Both, - First, - Second, -} - impl BDA { /// Initialize a blockdev with a Stratis BDA. pub fn initialize( @@ -188,408 +140,19 @@ impl BDA { } } -#[derive(Eq, PartialEq)] -pub struct StaticHeader { - blkdev_size: BlockdevSize, - pool_uuid: PoolUuid, - dev_uuid: DevUuid, - mda_size: MDASize, - reserved_size: ReservedSize, - flags: u64, - /// Seconds portion of DateTime value. - initialization_time: u64, -} - -impl StaticHeader { - fn new( - pool_uuid: PoolUuid, - dev_uuid: DevUuid, - mda_size: MDASize, - blkdev_size: BlockdevSize, - initialization_time: u64, - ) -> StaticHeader { - StaticHeader { - blkdev_size, - pool_uuid, - dev_uuid, - mda_size, - reserved_size: ReservedSize::new(RESERVED_SECTORS), - flags: 0, - initialization_time, - } - } - - /// Read the data at both signature block locations. - /// - /// Return the data from each location as an array of bytes - /// or an error if the read fails. The values are returned - /// in the same order in which they occur on the device. - /// - /// Read the contents of each signature block separately, - /// as this increases the probability that at least one read - /// will not fail. - fn read( - f: &mut F, - ) -> ( - io::Result<[u8; bytes!(static_header_size::SIGBLOCK_SECTORS)]>, - io::Result<[u8; bytes!(static_header_size::SIGBLOCK_SECTORS)]>, - ) - where - F: Read + Seek, - { - let mut buf_loc_1 = [0u8; bytes!(static_header_size::SIGBLOCK_SECTORS)]; - let mut buf_loc_2 = [0u8; bytes!(static_header_size::SIGBLOCK_SECTORS)]; - - fn read_sector_at_offset(f: &mut F, offset: usize, mut buf: &mut [u8]) -> io::Result<()> - where - F: Read + Seek, - { - f.seek(SeekFrom::Start(offset as u64)) - .and_then(|_| f.read_exact(&mut buf)) - } - - ( - read_sector_at_offset( - f, - bytes!(static_header_size::FIRST_SIGBLOCK_START_SECTORS), - &mut buf_loc_1, - ) - .map(|_| buf_loc_1), - read_sector_at_offset( - f, - bytes!(static_header_size::SECOND_SIGBLOCK_START_SECTORS), - &mut buf_loc_2, - ) - .map(|_| buf_loc_2), - ) - } - - // Writes signature_block according to the value of which. - // If first location is specified, write zeroes to empty regions in the - // first 8 sectors. If the second location is specified, writes zeroes to empty - // regions in the second 8 sectors. - fn write(&self, f: &mut F, which: MetadataLocation) -> io::Result<()> - where - F: Seek + SyncAll, - { - let signature_block = self.sigblock_to_buf(); - let zeroed = [0u8; bytes!(static_header_size::POST_SIGBLOCK_PADDING_SECTORS)]; - f.seek(SeekFrom::Start(0))?; - - // Write to a static header region in the static header. - fn write_region(f: &mut F, signature_block: &[u8], zeroed: &[u8]) -> io::Result<()> - where - F: Seek + SyncAll, - { - f.write_all(&zeroed[..bytes!(static_header_size::PRE_SIGBLOCK_PADDING_SECTORS)])?; - f.write_all(signature_block)?; - f.write_all(&zeroed[..bytes!(static_header_size::POST_SIGBLOCK_PADDING_SECTORS)])?; - f.sync_all()?; - Ok(()) - }; - - if which == MetadataLocation::Both || which == MetadataLocation::First { - write_region(f, &signature_block, &zeroed)?; - } else { - f.seek(SeekFrom::Start( - bytes!(static_header_size::SIGBLOCK_REGION_SECTORS) as u64, - ))?; - } - - if which == MetadataLocation::Both || which == MetadataLocation::Second { - write_region(f, &signature_block, &zeroed)?; - } - Ok(()) - } - - pub fn bda_extended_size(&self) -> BDAExtendedSize { - BDAExtendedSize::new(self.mda_size.bda_size().sectors() + self.reserved_size.sectors()) - } - - /// Try to find a valid StaticHeader on a device. - /// Return the latest copy that validates as a Stratis BDA, however verify both - /// copies and if one validates but one does not, re-write the one that is incorrect. If both - /// copies are valid, but one is newer than the other, rewrite the older one to match. - /// Return None if it's not a Stratis device. - /// Return an error if the metadata seems to indicate that the device is - /// a Stratis device, but no well-formed signature block could be read. - /// Return an error if neither sigblock location can be read. - /// Return an error if the sigblocks differ in some unaccountable way. - /// Returns an error if a write intended to repair an ill-formed, - /// unreadable, or stale signature block failed. - fn setup(f: &mut F) -> StratisResult> - where - F: Read + Seek + SyncAll, - { - // Action taken when one sigblock is interpreted as invalid. - // - // If the other sigblock is interpreted as a Stratis header, attempts repair - // of the invalid sigblock, returning an error if that fails, otherwise returning - // the valid sigblock. - // - // In all other cases, return the error associated with the invalid sigblock. - fn ok_err_static_header_handling( - f: &mut F, - maybe_sh: Option, - sh_error: StratisError, - repair_location: MetadataLocation, - ) -> StratisResult> - where - F: Read + Seek + SyncAll, - { - if let Some(sh) = maybe_sh { - write_header(f, sh, repair_location) - } else { - Err(sh_error) - } - } - - // Action taken when there was an I/O error reading the other sigblock. - // - // * If this sigblock region is interpreted as having no siglblock, it returns None. - // * If this sigblock region has a valid sigblock, attempts repair of the other - // sigblock region with the valid sigblock, returning the valid sigblock - // if the repair succeeds, otherwise returning an error. - // * If this sigblock appears to be invalid, return the error encountered when - // reading the sigblock. - fn copy_ok_err_handling( - f: &mut F, - maybe_sh: StratisResult>, - repair_location: MetadataLocation, - ) -> StratisResult> - where - F: Read + Seek + SyncAll, - { - match maybe_sh { - Ok(loc) => { - if let Some(ref sh) = loc { - sh.write(f, repair_location)?; - } - Ok(loc) - } - Err(e) => Err(e), - } - } - - fn write_header( - f: &mut F, - sh: StaticHeader, - repair_location: MetadataLocation, - ) -> StratisResult> - where - F: Read + Seek + SyncAll, - { - sh.write(f, repair_location)?; - Ok(Some(sh)) - } - - let (maybe_buf_1, maybe_buf_2) = StaticHeader::read(f); - match ( - maybe_buf_1.map(|buf| StaticHeader::sigblock_from_buf(&buf)), - maybe_buf_2.map(|buf| StaticHeader::sigblock_from_buf(&buf)), - ) { - // We read both copies without an IO error. - (Ok(buf_loc_1), Ok(buf_loc_2)) => match (buf_loc_1, buf_loc_2) { - (Ok(loc_1), Ok(loc_2)) => match (loc_1, loc_2) { - (Some(loc_1), Some(loc_2)) => { - if loc_1 == loc_2 { - Ok(Some(loc_1)) - } else if loc_1.initialization_time == loc_2.initialization_time { - // Inexplicable disagreement among static headers - let err_str = - "Appeared to be a Stratis device, but signature blocks disagree."; - Err(StratisError::Engine(ErrorEnum::Invalid, err_str.into())) - } else if loc_1.initialization_time > loc_2.initialization_time { - // If the first header block is newer, overwrite second with - // contents of first. - write_header(f, loc_1, MetadataLocation::Second) - } else { - // The second header block must be newer, so overwrite first - // with contents of second. - write_header(f, loc_2, MetadataLocation::First) - } - } - (None, None) => Ok(None), - (Some(loc_1), None) => write_header(f, loc_1, MetadataLocation::Second), - (None, Some(loc_2)) => write_header(f, loc_2, MetadataLocation::First), - }, - (Ok(loc_1), Err(loc_2)) => { - ok_err_static_header_handling(f, loc_1, loc_2, MetadataLocation::Second) - } - (Err(loc_1), Ok(loc_2)) => { - ok_err_static_header_handling(f, loc_2, loc_1, MetadataLocation::First) - } - (Err(_), Err(_)) => { - let err_str = "Appeared to be a Stratis device, but no valid sigblock found"; - Err(StratisError::Engine(ErrorEnum::Invalid, err_str.into())) - } - }, - // Copy 1 read OK, 2 resulted in an IO error - (Ok(buf_loc_1), Err(_)) => copy_ok_err_handling(f, buf_loc_1, MetadataLocation::Second), - // Copy 2 read OK, 1 resulted in IO Error - (Err(_), Ok(buf_loc_2)) => copy_ok_err_handling(f, buf_loc_2, MetadataLocation::First), - (Err(_), Err(_)) => { - // Unable to read the device at all. - let err_str = "Unable to read data at sigblock locations."; - Err(StratisError::Engine(ErrorEnum::Invalid, err_str.into())) - } - } - } - - /// Generate a buf suitable for writing to blockdev - fn sigblock_to_buf(&self) -> [u8; bytes!(static_header_size::SIGBLOCK_SECTORS)] { - let mut buf = [0u8; bytes!(static_header_size::SIGBLOCK_SECTORS)]; - buf[4..20].clone_from_slice(STRAT_MAGIC); - LittleEndian::write_u64(&mut buf[20..28], *self.blkdev_size.sectors()); - buf[28] = STRAT_SIGBLOCK_VERSION; - buf[32..64].clone_from_slice(self.pool_uuid.to_simple_ref().to_string().as_bytes()); - buf[64..96].clone_from_slice(self.dev_uuid.to_simple_ref().to_string().as_bytes()); - LittleEndian::write_u64(&mut buf[96..104], *self.mda_size.sectors()); - LittleEndian::write_u64(&mut buf[104..112], *self.reserved_size.sectors()); - LittleEndian::write_u64(&mut buf[120..128], self.initialization_time); - - let hdr_crc = - crc32::checksum_castagnoli(&buf[4..bytes!(static_header_size::SIGBLOCK_SECTORS)]); - LittleEndian::write_u32(&mut buf[..4], hdr_crc); - buf - } - - /// Parse a buffer to a StaticHeader. - /// Return None if no stratis magic number found. - /// Return an error if stored checksum and calculated checksum do not - /// match. - /// Return an error if the version number is not expected. - fn sigblock_from_buf(buf: &[u8]) -> StratisResult> { - assert_eq!(buf.len(), bytes!(static_header_size::SIGBLOCK_SECTORS)); - - if &buf[4..20] != STRAT_MAGIC { - return Ok(None); - } - - let crc = crc32::checksum_castagnoli(&buf[4..bytes!(static_header_size::SIGBLOCK_SECTORS)]); - if crc != LittleEndian::read_u32(&buf[..4]) { - return Err(StratisError::Engine( - ErrorEnum::Invalid, - "header CRC invalid".into(), - )); - } - - let blkdev_size = BlockdevSize::new(Sectors(LittleEndian::read_u64(&buf[20..28]))); - - let version = buf[28]; - if version != STRAT_SIGBLOCK_VERSION { - return Err(StratisError::Engine( - ErrorEnum::Invalid, - format!("Unknown sigblock version: {}", version), - )); - } - - let pool_uuid = Uuid::parse_str(from_utf8(&buf[32..64])?)?; - let dev_uuid = Uuid::parse_str(from_utf8(&buf[64..96])?)?; - - let mda_size = MDASize(Sectors(LittleEndian::read_u64(&buf[96..104]))); - - Ok(Some(StaticHeader { - pool_uuid, - dev_uuid, - blkdev_size, - mda_size, - reserved_size: ReservedSize::new(Sectors(LittleEndian::read_u64(&buf[104..112]))), - flags: 0, - initialization_time: LittleEndian::read_u64(&buf[120..128]), - })) - } - - /// Zero out the entire static header region on the designated file. - pub fn wipe(f: &mut F) -> StratisResult<()> - where - F: Seek + SyncAll, - { - let zeroed = [0u8; bytes!(static_header_size::STATIC_HEADER_SECTORS)]; - f.seek(SeekFrom::Start(0))?; - f.write_all(&zeroed)?; - f.sync_all()?; - Ok(()) - } -} - -impl fmt::Debug for StaticHeader { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.debug_struct("StaticHeader") - .field("blkdev_size", &self.blkdev_size) - .field("pool_uuid", &self.pool_uuid.to_simple_ref()) - .field("dev_uuid", &self.dev_uuid.to_simple_ref()) - .field("mda_size", &self.mda_size) - .field("reserved_size", &self.reserved_size) - .field("flags", &self.flags) - .field("initialization_time", &self.initialization_time) - .finish() - } -} - #[cfg(test)] mod tests { use std::io::Cursor; - use proptest::{collection::vec, num, option, prelude::BoxedStrategy, strategy::Strategy}; - use uuid::Uuid; + use proptest::{collection::vec, num}; - use devicemapper::{Bytes, Sectors, IEC}; + use crate::engine::strat_engine::backstore::metadata::{ + sizes::static_header_size, + static_header::{tests::random_static_header, tests::static_header_strategy}, + }; use super::*; - /// Return a static header with random block device and MDA size. - /// The block device is less than the minimum, for efficiency in testing. - fn random_static_header(blkdev_size: u64, mda_size_factor: u32) -> StaticHeader { - let pool_uuid = Uuid::new_v4(); - let dev_uuid = Uuid::new_v4(); - let mda_size = MDADataSize::new( - MDADataSize::default().bytes() + Bytes(u64::from(mda_size_factor * 4)), - ) - .region_size() - .mda_size(); - let blkdev_size = (Bytes(IEC::Mi) + Sectors(blkdev_size).bytes()).sectors(); - StaticHeader::new( - pool_uuid, - dev_uuid, - mda_size, - BlockdevSize::new(blkdev_size), - Utc::now().timestamp() as u64, - ) - } - - /// Make a static header strategy - fn static_header_strategy() -> BoxedStrategy { - (0..64u64, 0..64u32) - .prop_map(|(b, m)| random_static_header(b, m)) - .boxed() - } - - proptest! { - #[test] - /// Construct an arbitrary StaticHeader object. - /// Verify that the "memory buffer" is unowned. - /// Initialize a static header. - /// Verify that Stratis buffer validates. - /// Wipe the static header. - /// Verify that the buffer is again unowned. - fn test_ownership(ref sh in static_header_strategy()) { - let buf_size = *sh.mda_size.sectors().bytes() as usize + bytes!(static_header_size::STATIC_HEADER_SECTORS); - let mut buf = Cursor::new(vec![0; buf_size]); - prop_assert!(StaticHeader::setup(&mut buf).unwrap().is_none()); - - sh.write(&mut buf, MetadataLocation::Both).unwrap(); - - prop_assert!(StaticHeader::setup(&mut buf) - .unwrap() - .map(|new_sh| new_sh.pool_uuid == sh.pool_uuid && new_sh.dev_uuid == sh.dev_uuid) - .unwrap_or(false)); - - StaticHeader::wipe(&mut buf).unwrap(); - prop_assert!(StaticHeader::setup(&mut buf).unwrap().is_none()); - } - } - proptest! { #[test] /// Construct an arbitrary StaticHeader object. @@ -692,165 +255,4 @@ mod tests { } } - - proptest! { - #[test] - /// Construct an arbitrary StaticHeader object. - /// Write it to a buffer, read it out and make sure you get the same thing. - fn static_header(ref sh1 in static_header_strategy()) { - let buf = sh1.sigblock_to_buf(); - let sh2 = StaticHeader::sigblock_from_buf(&buf).unwrap().unwrap(); - prop_assert_eq!(sh1.pool_uuid, sh2.pool_uuid); - prop_assert_eq!(sh1.dev_uuid, sh2.dev_uuid); - prop_assert_eq!(sh1.blkdev_size, sh2.blkdev_size); - prop_assert_eq!(sh1.mda_size, sh2.mda_size); - prop_assert_eq!(sh1.reserved_size, sh2.reserved_size); - prop_assert_eq!(sh1.flags, sh2.flags); - prop_assert_eq!(sh1.initialization_time, sh2.initialization_time); - } - } - - proptest! { - #[test] - /// Verify correct reading of the static header if only one of - /// the two static headers is corrupted. Verify expected behavior - /// if both are corrupted, which varies depending on whether the - /// Stratis magic number or some other part of the header is corrupted. - fn test_corrupted_sigblock_recovery(primary in option::of(0..bytes!(static_header_size::SIGBLOCK_SECTORS)), - secondary in option::of(0..bytes!(static_header_size::SIGBLOCK_SECTORS))) { - - // Corrupt a byte at the specified position. - fn corrupt_byte(f: &mut F, position: u64) -> io::Result<()> - where - F: Read + Seek + SyncAll, - { - let mut byte_to_corrupt = [0; 1]; - f.seek(SeekFrom::Start(position)) - .and_then(|_| f.read_exact(&mut byte_to_corrupt))?; - - byte_to_corrupt[0] = !byte_to_corrupt[0]; - - f.seek(SeekFrom::Start(position)) - .and_then(|_| f.write_all(&byte_to_corrupt)) - .and_then(|_| f.sync_all()) - } - - let sh = random_static_header(10000, 4); - let buf_size = bytes!(static_header_size::STATIC_HEADER_SECTORS); - - let mut reference_buf = Cursor::new(vec![0; buf_size]); - sh.write(&mut reference_buf, MetadataLocation::Both).unwrap(); - - let mut buf = Cursor::new(vec![0; buf_size]); - sh.write(&mut buf, MetadataLocation::Both).unwrap(); - - if let Some(index) = primary { - corrupt_byte( - &mut buf, - (bytes!(static_header_size::FIRST_SIGBLOCK_START_SECTORS) + index) as u64, - ) - .unwrap(); - } - - if let Some(index) = secondary { - corrupt_byte( - &mut buf, - (bytes!(static_header_size::SECOND_SIGBLOCK_START_SECTORS) + index) as u64, - ) - .unwrap(); - } - - let setup_result = StaticHeader::setup(&mut buf); - - match (primary, secondary) { - (Some(p_index), Some(s_index)) => { - match (p_index, s_index) { - // Both magic locations are corrupted, conclusion is - // that this is not a Stratis static header. - (4..=19, 4..=19) => { - prop_assert!(setup_result.is_ok()); - prop_assert_eq!(setup_result.unwrap(), None); - } - // Both sigblocks were corrupted, but at least one - // was recognized as a Stratis sigblock. - _ => { - prop_assert!(setup_result.is_err()); - } - } - // No healing was attempted. - prop_assert_ne!(reference_buf.get_ref(), buf.get_ref()); - - } - // Only one header was corrupted, so the other was healed. - _ => { - prop_assert!(setup_result.is_ok()); - prop_assert_eq!(setup_result.unwrap(), Some(sh)); - prop_assert_eq!(reference_buf.get_ref(), buf.get_ref()); - } - } - } - } - - #[test] - /// Test that the newer sigblock is copied to the older sigblock's location - /// if the sigblock's timestamps don't match but they are otherwise - /// identical. - fn test_rewrite_older_sigblock() { - let sh = random_static_header(10000, 4); - - let ts = Utc::now().timestamp() as u64; - let sh_older = - StaticHeader::new(sh.pool_uuid, sh.dev_uuid, sh.mda_size, sh.blkdev_size, ts); - let sh_newer = StaticHeader::new( - sh.pool_uuid, - sh.dev_uuid, - sh.mda_size, - sh.blkdev_size, - ts + 1, - ); - assert_ne!(sh_older, sh_newer); - - let buf_size = bytes!(static_header_size::STATIC_HEADER_SECTORS); - - let mut reference_buf = Cursor::new(vec![0; buf_size]); - sh_newer - .write(&mut reference_buf, MetadataLocation::Both) - .unwrap(); - - // Test that StaticHeader::setup succeeds by writing the older - // signature block to the specified older location and the newer - // sigblock to the specified newer location and then calling - // StaticHeader::setup. StaticHeader::setup should return without - // error with the newer sigblock. As a side-effect, it should have - // overwritten the location of the older sigblock with the value of - // the newer sigblock. - let test_rewrite = |sh_older: &StaticHeader, - sh_newer: &StaticHeader, - older_location: MetadataLocation, - newer_location: MetadataLocation| { - let mut buf = Cursor::new(vec![0; buf_size]); - sh_older.write(&mut buf, older_location).unwrap(); - sh_newer.write(&mut buf, newer_location).unwrap(); - assert_ne!(buf.get_ref(), reference_buf.get_ref()); - assert_eq!( - StaticHeader::setup(&mut buf).unwrap().as_ref(), - Some(sh_newer) - ); - assert_eq!(buf.get_ref(), reference_buf.get_ref()); - }; - - test_rewrite( - &sh_older, - &sh_newer, - MetadataLocation::First, - MetadataLocation::Second, - ); - - test_rewrite( - &sh_older, - &sh_newer, - MetadataLocation::Second, - MetadataLocation::First, - ); - } } diff --git a/src/engine/strat_engine/backstore/metadata/mod.rs b/src/engine/strat_engine/backstore/metadata/mod.rs index b8dde8b7ae..897abee4a1 100644 --- a/src/engine/strat_engine/backstore/metadata/mod.rs +++ b/src/engine/strat_engine/backstore/metadata/mod.rs @@ -2,11 +2,21 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. +// Transform a constant in sectors to a constant in bytes +#[macro_export] +macro_rules! bytes { + ($number:expr) => { + $number * devicemapper::SECTOR_SIZE + }; +} + mod bda; mod mda; mod sizes; +mod static_header; pub use self::{ - bda::{device_identifiers, disown_device, BDA}, + bda::BDA, sizes::{BDAExtendedSize, BlockdevSize, MDADataSize}, + static_header::{device_identifiers, disown_device}, }; diff --git a/src/engine/strat_engine/backstore/metadata/static_header.rs b/src/engine/strat_engine/backstore/metadata/static_header.rs new file mode 100644 index 0000000000..e50994e2aa --- /dev/null +++ b/src/engine/strat_engine/backstore/metadata/static_header.rs @@ -0,0 +1,629 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use std::{ + fmt, + io::{self, Read, Seek, SeekFrom}, + str::from_utf8, +}; + +use byteorder::{ByteOrder, LittleEndian}; +use crc::crc32; +use uuid::Uuid; + +use devicemapper::{Sectors, IEC, SECTOR_SIZE}; + +use crate::{ + engine::{ + strat_engine::{ + backstore::metadata::sizes::{ + static_header_size, BDAExtendedSize, BlockdevSize, MDASize, ReservedSize, + }, + device::SyncAll, + }, + types::{DevUuid, PoolUuid}, + }, + stratis::{ErrorEnum, StratisError, StratisResult}, +}; + +const RESERVED_SECTORS: Sectors = Sectors(3 * IEC::Mi / (SECTOR_SIZE as u64)); // = 3 MiB + +const STRAT_MAGIC: &[u8] = b"!Stra0tis\x86\xff\x02^\x41rh"; + +const STRAT_SIGBLOCK_VERSION: u8 = 1; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum MetadataLocation { + Both, + First, + Second, +} + +/// Get a Stratis pool UUID and device UUID from any device. +/// If there is an error while obtaining these values return the error. +/// If the device does not appear to be a Stratis device, return None. +pub fn device_identifiers(f: &mut F) -> StratisResult> +where + F: Read + Seek + SyncAll, +{ + StaticHeader::setup(f).map(|sh| sh.map(|sh| (sh.pool_uuid, sh.dev_uuid))) +} + +/// Remove Stratis identifying information from device. +pub fn disown_device(f: &mut F) -> StratisResult<()> +where + F: Seek + SyncAll, +{ + StaticHeader::wipe(f) +} + +#[derive(Eq, PartialEq)] +pub struct StaticHeader { + pub blkdev_size: BlockdevSize, + pub pool_uuid: PoolUuid, + pub dev_uuid: DevUuid, + pub mda_size: MDASize, + pub reserved_size: ReservedSize, + pub flags: u64, + /// Seconds portion of DateTime value. + pub initialization_time: u64, +} + +impl StaticHeader { + pub fn new( + pool_uuid: PoolUuid, + dev_uuid: DevUuid, + mda_size: MDASize, + blkdev_size: BlockdevSize, + initialization_time: u64, + ) -> StaticHeader { + StaticHeader { + blkdev_size, + pool_uuid, + dev_uuid, + mda_size, + reserved_size: ReservedSize::new(RESERVED_SECTORS), + flags: 0, + initialization_time, + } + } + + /// Read the data at both signature block locations. + /// + /// Return the data from each location as an array of bytes + /// or an error if the read fails. The values are returned + /// in the same order in which they occur on the device. + /// + /// Read the contents of each signature block separately, + /// as this increases the probability that at least one read + /// will not fail. + fn read( + f: &mut F, + ) -> ( + io::Result<[u8; bytes!(static_header_size::SIGBLOCK_SECTORS)]>, + io::Result<[u8; bytes!(static_header_size::SIGBLOCK_SECTORS)]>, + ) + where + F: Read + Seek, + { + let mut buf_loc_1 = [0u8; bytes!(static_header_size::SIGBLOCK_SECTORS)]; + let mut buf_loc_2 = [0u8; bytes!(static_header_size::SIGBLOCK_SECTORS)]; + + fn read_sector_at_offset(f: &mut F, offset: usize, mut buf: &mut [u8]) -> io::Result<()> + where + F: Read + Seek, + { + f.seek(SeekFrom::Start(offset as u64)) + .and_then(|_| f.read_exact(&mut buf)) + } + + ( + read_sector_at_offset( + f, + bytes!(static_header_size::FIRST_SIGBLOCK_START_SECTORS), + &mut buf_loc_1, + ) + .map(|_| buf_loc_1), + read_sector_at_offset( + f, + bytes!(static_header_size::SECOND_SIGBLOCK_START_SECTORS), + &mut buf_loc_2, + ) + .map(|_| buf_loc_2), + ) + } + + // Writes signature_block according to the value of which. + // If first location is specified, write zeroes to empty regions in the + // first 8 sectors. If the second location is specified, writes zeroes to empty + // regions in the second 8 sectors. + pub fn write(&self, f: &mut F, which: MetadataLocation) -> io::Result<()> + where + F: Seek + SyncAll, + { + let signature_block = self.sigblock_to_buf(); + let zeroed = [0u8; bytes!(static_header_size::POST_SIGBLOCK_PADDING_SECTORS)]; + f.seek(SeekFrom::Start(0))?; + + // Write to a static header region in the static header. + fn write_region(f: &mut F, signature_block: &[u8], zeroed: &[u8]) -> io::Result<()> + where + F: Seek + SyncAll, + { + f.write_all(&zeroed[..bytes!(static_header_size::PRE_SIGBLOCK_PADDING_SECTORS)])?; + f.write_all(signature_block)?; + f.write_all(&zeroed[..bytes!(static_header_size::POST_SIGBLOCK_PADDING_SECTORS)])?; + f.sync_all()?; + Ok(()) + }; + + if which == MetadataLocation::Both || which == MetadataLocation::First { + write_region(f, &signature_block, &zeroed)?; + } else { + f.seek(SeekFrom::Start( + bytes!(static_header_size::SIGBLOCK_REGION_SECTORS) as u64, + ))?; + } + + if which == MetadataLocation::Both || which == MetadataLocation::Second { + write_region(f, &signature_block, &zeroed)?; + } + Ok(()) + } + + pub fn bda_extended_size(&self) -> BDAExtendedSize { + BDAExtendedSize::new(self.mda_size.bda_size().sectors() + self.reserved_size.sectors()) + } + + /// Try to find a valid StaticHeader on a device. + /// Return the latest copy that validates as a Stratis BDA, however verify both + /// copies and if one validates but one does not, re-write the one that is incorrect. If both + /// copies are valid, but one is newer than the other, rewrite the older one to match. + /// Return None if it's not a Stratis device. + /// Return an error if the metadata seems to indicate that the device is + /// a Stratis device, but no well-formed signature block could be read. + /// Return an error if neither sigblock location can be read. + /// Return an error if the sigblocks differ in some unaccountable way. + /// Returns an error if a write intended to repair an ill-formed, + /// unreadable, or stale signature block failed. + pub fn setup(f: &mut F) -> StratisResult> + where + F: Read + Seek + SyncAll, + { + // Action taken when one sigblock is interpreted as invalid. + // + // If the other sigblock is interpreted as a Stratis header, attempts repair + // of the invalid sigblock, returning an error if that fails, otherwise returning + // the valid sigblock. + // + // In all other cases, return the error associated with the invalid sigblock. + fn ok_err_static_header_handling( + f: &mut F, + maybe_sh: Option, + sh_error: StratisError, + repair_location: MetadataLocation, + ) -> StratisResult> + where + F: Read + Seek + SyncAll, + { + if let Some(sh) = maybe_sh { + write_header(f, sh, repair_location) + } else { + Err(sh_error) + } + } + + // Action taken when there was an I/O error reading the other sigblock. + // + // * If this sigblock region is interpreted as having no siglblock, it returns None. + // * If this sigblock region has a valid sigblock, attempts repair of the other + // sigblock region with the valid sigblock, returning the valid sigblock + // if the repair succeeds, otherwise returning an error. + // * If this sigblock appears to be invalid, return the error encountered when + // reading the sigblock. + fn copy_ok_err_handling( + f: &mut F, + maybe_sh: StratisResult>, + repair_location: MetadataLocation, + ) -> StratisResult> + where + F: Read + Seek + SyncAll, + { + match maybe_sh { + Ok(loc) => { + if let Some(ref sh) = loc { + sh.write(f, repair_location)?; + } + Ok(loc) + } + Err(e) => Err(e), + } + } + + fn write_header( + f: &mut F, + sh: StaticHeader, + repair_location: MetadataLocation, + ) -> StratisResult> + where + F: Read + Seek + SyncAll, + { + sh.write(f, repair_location)?; + Ok(Some(sh)) + } + + let (maybe_buf_1, maybe_buf_2) = StaticHeader::read(f); + match ( + maybe_buf_1.map(|buf| StaticHeader::sigblock_from_buf(&buf)), + maybe_buf_2.map(|buf| StaticHeader::sigblock_from_buf(&buf)), + ) { + // We read both copies without an IO error. + (Ok(buf_loc_1), Ok(buf_loc_2)) => match (buf_loc_1, buf_loc_2) { + (Ok(loc_1), Ok(loc_2)) => match (loc_1, loc_2) { + (Some(loc_1), Some(loc_2)) => { + if loc_1 == loc_2 { + Ok(Some(loc_1)) + } else if loc_1.initialization_time == loc_2.initialization_time { + // Inexplicable disagreement among static headers + let err_str = + "Appeared to be a Stratis device, but signature blocks disagree."; + Err(StratisError::Engine(ErrorEnum::Invalid, err_str.into())) + } else if loc_1.initialization_time > loc_2.initialization_time { + // If the first header block is newer, overwrite second with + // contents of first. + write_header(f, loc_1, MetadataLocation::Second) + } else { + // The second header block must be newer, so overwrite first + // with contents of second. + write_header(f, loc_2, MetadataLocation::First) + } + } + (None, None) => Ok(None), + (Some(loc_1), None) => write_header(f, loc_1, MetadataLocation::Second), + (None, Some(loc_2)) => write_header(f, loc_2, MetadataLocation::First), + }, + (Ok(loc_1), Err(loc_2)) => { + ok_err_static_header_handling(f, loc_1, loc_2, MetadataLocation::Second) + } + (Err(loc_1), Ok(loc_2)) => { + ok_err_static_header_handling(f, loc_2, loc_1, MetadataLocation::First) + } + (Err(_), Err(_)) => { + let err_str = "Appeared to be a Stratis device, but no valid sigblock found"; + Err(StratisError::Engine(ErrorEnum::Invalid, err_str.into())) + } + }, + // Copy 1 read OK, 2 resulted in an IO error + (Ok(buf_loc_1), Err(_)) => copy_ok_err_handling(f, buf_loc_1, MetadataLocation::Second), + // Copy 2 read OK, 1 resulted in IO Error + (Err(_), Ok(buf_loc_2)) => copy_ok_err_handling(f, buf_loc_2, MetadataLocation::First), + (Err(_), Err(_)) => { + // Unable to read the device at all. + let err_str = "Unable to read data at sigblock locations."; + Err(StratisError::Engine(ErrorEnum::Invalid, err_str.into())) + } + } + } + + /// Generate a buf suitable for writing to blockdev + fn sigblock_to_buf(&self) -> [u8; bytes!(static_header_size::SIGBLOCK_SECTORS)] { + let mut buf = [0u8; bytes!(static_header_size::SIGBLOCK_SECTORS)]; + buf[4..20].clone_from_slice(STRAT_MAGIC); + LittleEndian::write_u64(&mut buf[20..28], *self.blkdev_size.sectors()); + buf[28] = STRAT_SIGBLOCK_VERSION; + buf[32..64].clone_from_slice(self.pool_uuid.to_simple_ref().to_string().as_bytes()); + buf[64..96].clone_from_slice(self.dev_uuid.to_simple_ref().to_string().as_bytes()); + LittleEndian::write_u64(&mut buf[96..104], *self.mda_size.sectors()); + LittleEndian::write_u64(&mut buf[104..112], *self.reserved_size.sectors()); + LittleEndian::write_u64(&mut buf[120..128], self.initialization_time); + + let hdr_crc = + crc32::checksum_castagnoli(&buf[4..bytes!(static_header_size::SIGBLOCK_SECTORS)]); + LittleEndian::write_u32(&mut buf[..4], hdr_crc); + buf + } + + /// Parse a buffer to a StaticHeader. + /// Return None if no stratis magic number found. + /// Return an error if stored checksum and calculated checksum do not + /// match. + /// Return an error if the version number is not expected. + fn sigblock_from_buf(buf: &[u8]) -> StratisResult> { + assert_eq!(buf.len(), bytes!(static_header_size::SIGBLOCK_SECTORS)); + + if &buf[4..20] != STRAT_MAGIC { + return Ok(None); + } + + let crc = crc32::checksum_castagnoli(&buf[4..bytes!(static_header_size::SIGBLOCK_SECTORS)]); + if crc != LittleEndian::read_u32(&buf[..4]) { + return Err(StratisError::Engine( + ErrorEnum::Invalid, + "header CRC invalid".into(), + )); + } + + let blkdev_size = BlockdevSize::new(Sectors(LittleEndian::read_u64(&buf[20..28]))); + + let version = buf[28]; + if version != STRAT_SIGBLOCK_VERSION { + return Err(StratisError::Engine( + ErrorEnum::Invalid, + format!("Unknown sigblock version: {}", version), + )); + } + + let pool_uuid = Uuid::parse_str(from_utf8(&buf[32..64])?)?; + let dev_uuid = Uuid::parse_str(from_utf8(&buf[64..96])?)?; + + let mda_size = MDASize(Sectors(LittleEndian::read_u64(&buf[96..104]))); + + Ok(Some(StaticHeader { + pool_uuid, + dev_uuid, + blkdev_size, + mda_size, + reserved_size: ReservedSize::new(Sectors(LittleEndian::read_u64(&buf[104..112]))), + flags: 0, + initialization_time: LittleEndian::read_u64(&buf[120..128]), + })) + } + + /// Zero out the entire static header region on the designated file. + pub fn wipe(f: &mut F) -> StratisResult<()> + where + F: Seek + SyncAll, + { + let zeroed = [0u8; bytes!(static_header_size::STATIC_HEADER_SECTORS)]; + f.seek(SeekFrom::Start(0))?; + f.write_all(&zeroed)?; + f.sync_all()?; + Ok(()) + } +} + +impl fmt::Debug for StaticHeader { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("StaticHeader") + .field("blkdev_size", &self.blkdev_size) + .field("pool_uuid", &self.pool_uuid.to_simple_ref()) + .field("dev_uuid", &self.dev_uuid.to_simple_ref()) + .field("mda_size", &self.mda_size) + .field("reserved_size", &self.reserved_size) + .field("flags", &self.flags) + .field("initialization_time", &self.initialization_time) + .finish() + } +} + +#[cfg(test)] +pub mod tests { + use std::io::Cursor; + + use proptest::{option, prelude::BoxedStrategy, strategy::Strategy}; + use uuid::Uuid; + + use chrono::Utc; + + use devicemapper::{Bytes, Sectors, IEC}; + + use crate::engine::strat_engine::backstore::metadata::sizes::{ + static_header_size, MDADataSize, + }; + + use super::*; + + proptest! { + #[test] + /// Construct an arbitrary StaticHeader object. + /// Verify that the "memory buffer" is unowned. + /// Initialize a static header. + /// Verify that Stratis buffer validates. + /// Wipe the static header. + /// Verify that the buffer is again unowned. + fn test_ownership(ref sh in static_header_strategy()) { + let buf_size = *sh.mda_size.sectors().bytes() as usize + bytes!(static_header_size::STATIC_HEADER_SECTORS); + let mut buf = Cursor::new(vec![0; buf_size]); + prop_assert!(StaticHeader::setup(&mut buf).unwrap().is_none()); + + sh.write(&mut buf, MetadataLocation::Both).unwrap(); + + prop_assert!(StaticHeader::setup(&mut buf) + .unwrap() + .map(|new_sh| new_sh.pool_uuid == sh.pool_uuid && new_sh.dev_uuid == sh.dev_uuid) + .unwrap_or(false)); + + StaticHeader::wipe(&mut buf).unwrap(); + prop_assert!(StaticHeader::setup(&mut buf).unwrap().is_none()); + } + } + + /// Return a static header with random block device and MDA size. + /// The block device is less than the minimum, for efficiency in testing. + pub fn random_static_header(blkdev_size: u64, mda_size_factor: u32) -> StaticHeader { + let pool_uuid = Uuid::new_v4(); + let dev_uuid = Uuid::new_v4(); + let mda_size = MDADataSize::new( + MDADataSize::default().bytes() + Bytes(u64::from(mda_size_factor * 4)), + ) + .region_size() + .mda_size(); + let blkdev_size = (Bytes(IEC::Mi) + Sectors(blkdev_size).bytes()).sectors(); + StaticHeader::new( + pool_uuid, + dev_uuid, + mda_size, + BlockdevSize::new(blkdev_size), + Utc::now().timestamp() as u64, + ) + } + + /// Make a static header strategy + pub fn static_header_strategy() -> BoxedStrategy { + (0..64u64, 0..64u32) + .prop_map(|(b, m)| random_static_header(b, m)) + .boxed() + } + + proptest! { + #[test] + /// Verify correct reading of the static header if only one of + /// the two static headers is corrupted. Verify expected behavior + /// if both are corrupted, which varies depending on whether the + /// Stratis magic number or some other part of the header is corrupted. + fn test_corrupted_sigblock_recovery(primary in option::of(0..bytes!(static_header_size::SIGBLOCK_SECTORS)), + secondary in option::of(0..bytes!(static_header_size::SIGBLOCK_SECTORS))) { + + // Corrupt a byte at the specified position. + fn corrupt_byte(f: &mut F, position: u64) -> io::Result<()> + where + F: Read + Seek + SyncAll, + { + let mut byte_to_corrupt = [0; 1]; + f.seek(SeekFrom::Start(position)) + .and_then(|_| f.read_exact(&mut byte_to_corrupt))?; + + byte_to_corrupt[0] = !byte_to_corrupt[0]; + + f.seek(SeekFrom::Start(position)) + .and_then(|_| f.write_all(&byte_to_corrupt)) + .and_then(|_| f.sync_all()) + } + + let sh = random_static_header(10000, 4); + let buf_size = bytes!(static_header_size::STATIC_HEADER_SECTORS); + + let mut reference_buf = Cursor::new(vec![0; buf_size]); + sh.write(&mut reference_buf, MetadataLocation::Both).unwrap(); + + let mut buf = Cursor::new(vec![0; buf_size]); + sh.write(&mut buf, MetadataLocation::Both).unwrap(); + + if let Some(index) = primary { + corrupt_byte( + &mut buf, + (bytes!(static_header_size::FIRST_SIGBLOCK_START_SECTORS) + index) as u64, + ) + .unwrap(); + } + + if let Some(index) = secondary { + corrupt_byte( + &mut buf, + (bytes!(static_header_size::SECOND_SIGBLOCK_START_SECTORS) + index) as u64, + ) + .unwrap(); + } + + let setup_result = StaticHeader::setup(&mut buf); + + match (primary, secondary) { + (Some(p_index), Some(s_index)) => { + match (p_index, s_index) { + // Both magic locations are corrupted, conclusion is + // that this is not a Stratis static header. + (4..=19, 4..=19) => { + prop_assert!(setup_result.is_ok()); + prop_assert_eq!(setup_result.unwrap(), None); + } + // Both sigblocks were corrupted, but at least one + // was recognized as a Stratis sigblock. + _ => { + prop_assert!(setup_result.is_err()); + } + } + // No healing was attempted. + prop_assert_ne!(reference_buf.get_ref(), buf.get_ref()); + + } + // Only one header was corrupted, so the other was healed. + _ => { + prop_assert!(setup_result.is_ok()); + prop_assert_eq!(setup_result.unwrap(), Some(sh)); + prop_assert_eq!(reference_buf.get_ref(), buf.get_ref()); + } + } + } + } + + proptest! { + #[test] + /// Construct an arbitrary StaticHeader object. + /// Write it to a buffer, read it out and make sure you get the same thing. + fn static_header(ref sh1 in static_header_strategy()) { + let buf = sh1.sigblock_to_buf(); + let sh2 = StaticHeader::sigblock_from_buf(&buf).unwrap().unwrap(); + prop_assert_eq!(sh1.pool_uuid, sh2.pool_uuid); + prop_assert_eq!(sh1.dev_uuid, sh2.dev_uuid); + prop_assert_eq!(sh1.blkdev_size, sh2.blkdev_size); + prop_assert_eq!(sh1.mda_size, sh2.mda_size); + prop_assert_eq!(sh1.reserved_size, sh2.reserved_size); + prop_assert_eq!(sh1.flags, sh2.flags); + prop_assert_eq!(sh1.initialization_time, sh2.initialization_time); + } + } + + #[test] + /// Test that the newer sigblock is copied to the older sigblock's location + /// if the sigblock's timestamps don't match but they are otherwise + /// identical. + fn test_rewrite_older_sigblock() { + let sh = random_static_header(10000, 4); + + let ts = Utc::now().timestamp() as u64; + let sh_older = + StaticHeader::new(sh.pool_uuid, sh.dev_uuid, sh.mda_size, sh.blkdev_size, ts); + let sh_newer = StaticHeader::new( + sh.pool_uuid, + sh.dev_uuid, + sh.mda_size, + sh.blkdev_size, + ts + 1, + ); + assert_ne!(sh_older, sh_newer); + + let buf_size = bytes!(static_header_size::STATIC_HEADER_SECTORS); + + let mut reference_buf = Cursor::new(vec![0; buf_size]); + sh_newer + .write(&mut reference_buf, MetadataLocation::Both) + .unwrap(); + + // Test that StaticHeader::setup succeeds by writing the older + // signature block to the specified older location and the newer + // sigblock to the specified newer location and then calling + // StaticHeader::setup. StaticHeader::setup should return without + // error with the newer sigblock. As a side-effect, it should have + // overwritten the location of the older sigblock with the value of + // the newer sigblock. + let test_rewrite = |sh_older: &StaticHeader, + sh_newer: &StaticHeader, + older_location: MetadataLocation, + newer_location: MetadataLocation| { + let mut buf = Cursor::new(vec![0; buf_size]); + sh_older.write(&mut buf, older_location).unwrap(); + sh_newer.write(&mut buf, newer_location).unwrap(); + assert_ne!(buf.get_ref(), reference_buf.get_ref()); + assert_eq!( + StaticHeader::setup(&mut buf).unwrap().as_ref(), + Some(sh_newer) + ); + assert_eq!(buf.get_ref(), reference_buf.get_ref()); + }; + + test_rewrite( + &sh_older, + &sh_newer, + MetadataLocation::First, + MetadataLocation::Second, + ); + + test_rewrite( + &sh_older, + &sh_newer, + MetadataLocation::Second, + MetadataLocation::First, + ); + } +} From d7aef582652c8b50bc850d483876b4ad46d58f7a Mon Sep 17 00:00:00 2001 From: mulhern Date: Thu, 6 Feb 2020 16:39:16 -0500 Subject: [PATCH 102/109] Allow the use of Error::description() in macro-generated code But remove it once the version of error_chain is released. Note, again, that we would like to remove our dependency on error_chain. Signed-off-by: mulhern --- src/engine/strat_engine/tests/util.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/engine/strat_engine/tests/util.rs b/src/engine/strat_engine/tests/util.rs index c858158809..11c22893de 100644 --- a/src/engine/strat_engine/tests/util.rs +++ b/src/engine/strat_engine/tests/util.rs @@ -15,6 +15,9 @@ use crate::engine::strat_engine::{ }; mod cleanup_errors { + // FIXME: It should be possible to remove this allow when the next + // version of error_chain is released. + #![allow(deprecated)] use libmount; use nix; use std; From d4cbf9a9d4794e6a19b917a61d29c6f54eaeafba Mon Sep 17 00:00:00 2001 From: mulhern Date: Thu, 6 Feb 2020 17:25:33 -0500 Subject: [PATCH 103/109] No longer implement Error::description in StratisError It is substantially the same as the the Display implementation and it is now hard-deprecated. It is used by the D-Bus translation function, change that use to to_string(), i.e., use the Display implementation instead. We do not support the Rust API of libstratis, so the function can be removed, rather than just deprecated. Signed-off-by: mulhern --- src/dbus_api/util.rs | 4 +--- src/stratis/errors.rs | 17 ----------------- 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/src/dbus_api/util.rs b/src/dbus_api/util.rs index a8a123b0a0..98f30631cf 100644 --- a/src/dbus_api/util.rs +++ b/src/dbus_api/util.rs @@ -2,8 +2,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. -use std::error::Error; - use dbus::{ self, arg::{ArgType, Iter, IterAppend, RefArg, Variant}, @@ -97,7 +95,7 @@ pub fn engine_to_dbus_err_tuple(err: &StratisError) -> (u16, String) { }; let description = match *err { StratisError::DM(DmError::Core(ref err)) => err.to_string(), - ref err => err.description().to_owned(), + ref err => err.to_string(), }; (error as u16, description) } diff --git a/src/stratis/errors.rs b/src/stratis/errors.rs index 1372768efa..95e89f7aa2 100644 --- a/src/stratis/errors.rs +++ b/src/stratis/errors.rs @@ -63,23 +63,6 @@ impl fmt::Display for StratisError { } impl Error for StratisError { - fn description(&self) -> &str { - match *self { - StratisError::Error(ref s) => s, - StratisError::Engine(_, ref msg) => msg, - StratisError::Io(ref err) => err.description(), - StratisError::Nix(ref err) => err.description(), - StratisError::Uuid(_) => "Uuid::ParseError", - StratisError::Utf8(ref err) => err.description(), - StratisError::Serde(ref err) => err.description(), - StratisError::DM(ref err) => err.description(), - - #[cfg(feature = "dbus_enabled")] - StratisError::Dbus(ref err) => err.message().unwrap_or("D-Bus Error"), - StratisError::Udev(ref err) => Error::description(err), - } - } - fn cause(&self) -> Option<&dyn Error> { match *self { StratisError::Error(_) | StratisError::Engine(_, _) => None, From 6a3c69bc4aaf2dbd8e4b830b2496a4d74972f2d7 Mon Sep 17 00:00:00 2001 From: mulhern Date: Thu, 6 Feb 2020 17:31:16 -0500 Subject: [PATCH 104/109] No longer allow beta task to fail All errors have been addressed. Signed-off-by: mulhern --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 44fb0bc696..eeb6f26c8b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,8 +13,6 @@ language: rust matrix: fast_finish: true allow_failures: - # Temporarily allow beta lint task to fail - - rust: beta # Allow audit task to fail - env: TASK=audit include: From 48dcb27158f4f9cc5084a20b8ca43ed498c01417 Mon Sep 17 00:00:00 2001 From: mulhern Date: Thu, 6 Feb 2020 13:01:39 -0500 Subject: [PATCH 105/109] Make more commonalities among the various FetchProperties implementations Make the code entirely the same by defining a constant in each file and then use a macro for the shared code. Signed-off-by: mulhern --- .../blockdev/fetch_properties_2_0/methods.rs | 25 ++++------------- .../fetch_properties_2_0/methods.rs | 25 ++++------------- src/dbus_api/macros.rs | 26 ++++++++++++++++++ .../pool/fetch_properties_2_0/methods.rs | 27 ++++--------------- 4 files changed, 41 insertions(+), 62 deletions(-) diff --git a/src/dbus_api/blockdev/fetch_properties_2_0/methods.rs b/src/dbus_api/blockdev/fetch_properties_2_0/methods.rs index 0389cc45e3..16cf6b6c66 100644 --- a/src/dbus_api/blockdev/fetch_properties_2_0/methods.rs +++ b/src/dbus_api/blockdev/fetch_properties_2_0/methods.rs @@ -6,34 +6,17 @@ use std::collections::HashMap; use dbus::{ self, - arg::{Array, RefArg, Variant}, + arg::{RefArg, Variant}, tree::{MTFn, MethodInfo, MethodResult}, Message, }; use itertools::Itertools; use crate::dbus_api::{ - blockdev::shared::blockdev_operation, - consts, - types::TData, - util::{get_next_arg, result_to_tuple}, + blockdev::shared::blockdev_operation, consts, types::TData, util::result_to_tuple, }; -pub fn get_all_properties(m: &MethodInfo, TData>) -> MethodResult { - get_properties_shared( - m, - &mut vec![consts::BLOCKDEV_TOTAL_SIZE_PROP] - .into_iter() - .map(|s| s.to_string()), - ) -} - -pub fn get_properties(m: &MethodInfo, TData>) -> MethodResult { - let message: &Message = m.msg; - let mut iter = message.iter_init(); - let mut properties: Array = get_next_arg(&mut iter, 0)?; - get_properties_shared(m, &mut properties) -} +const ALL_PROPERTIES: [&str; 1] = [consts::BLOCKDEV_TOTAL_SIZE_PROP]; fn get_properties_shared( m: &MethodInfo, TData>, @@ -60,3 +43,5 @@ fn get_properties_shared( Ok(vec![return_message.append1(return_value)]) } + +properties_footer!(); diff --git a/src/dbus_api/filesystem/fetch_properties_2_0/methods.rs b/src/dbus_api/filesystem/fetch_properties_2_0/methods.rs index 1bbdd592c2..04eeb0d835 100644 --- a/src/dbus_api/filesystem/fetch_properties_2_0/methods.rs +++ b/src/dbus_api/filesystem/fetch_properties_2_0/methods.rs @@ -6,19 +6,18 @@ use std::collections::HashMap; use dbus::{ self, - arg::{Array, RefArg, Variant}, + arg::{RefArg, Variant}, tree::{MTFn, MethodInfo, MethodResult}, Message, }; use itertools::Itertools; use crate::dbus_api::{ - consts, - filesystem::shared::filesystem_operation, - types::TData, - util::{get_next_arg, result_to_tuple}, + consts, filesystem::shared::filesystem_operation, types::TData, util::result_to_tuple, }; +const ALL_PROPERTIES: [&str; 1] = [consts::FILESYSTEM_USED_PROP]; + fn get_properties_shared( m: &MethodInfo, TData>, properties: &mut dyn Iterator, @@ -47,18 +46,4 @@ fn get_properties_shared( Ok(vec![return_message.append1(return_value)]) } -pub fn get_all_properties(m: &MethodInfo, TData>) -> MethodResult { - get_properties_shared( - m, - &mut vec![consts::FILESYSTEM_USED_PROP] - .into_iter() - .map(|s| s.to_string()), - ) -} - -pub fn get_properties(m: &MethodInfo, TData>) -> MethodResult { - let message: &Message = m.msg; - let mut iter = message.iter_init(); - let mut properties: Array = get_next_arg(&mut iter, 0)?; - get_properties_shared(m, &mut properties) -} +properties_footer!(); diff --git a/src/dbus_api/macros.rs b/src/dbus_api/macros.rs index 40805efdd9..00b7a26d2c 100644 --- a/src/dbus_api/macros.rs +++ b/src/dbus_api/macros.rs @@ -58,3 +58,29 @@ macro_rules! uuid_to_string { $uuid.to_simple_ref().to_string() }; } + +macro_rules! properties_footer { + () => { + pub fn get_all_properties( + m: &dbus::tree::MethodInfo< + dbus::tree::MTFn<$crate::dbus_api::types::TData>, + $crate::dbus_api::types::TData, + >, + ) -> dbus::tree::MethodResult { + get_properties_shared(m, &mut ALL_PROPERTIES.iter().map(|&s| s.to_string())) + } + + pub fn get_properties( + m: &dbus::tree::MethodInfo< + dbus::tree::MTFn<$crate::dbus_api::types::TData>, + $crate::dbus_api::types::TData, + >, + ) -> dbus::tree::MethodResult { + let message: &dbus::Message = m.msg; + let mut iter = message.iter_init(); + let mut properties: dbus::arg::Array = + $crate::dbus_api::util::get_next_arg(&mut iter, 0)?; + get_properties_shared(m, &mut properties) + } + }; +} diff --git a/src/dbus_api/pool/fetch_properties_2_0/methods.rs b/src/dbus_api/pool/fetch_properties_2_0/methods.rs index c40daabb70..e74a2f37fa 100644 --- a/src/dbus_api/pool/fetch_properties_2_0/methods.rs +++ b/src/dbus_api/pool/fetch_properties_2_0/methods.rs @@ -5,18 +5,15 @@ use std::collections::HashMap; use dbus::{ - arg::{Array, RefArg, Variant}, + arg::{RefArg, Variant}, tree::{MTFn, MethodInfo, MethodResult}, Message, }; use itertools::Itertools; -use crate::dbus_api::{ - consts, - pool::shared::pool_operation, - types::TData, - util::{get_next_arg, result_to_tuple}, -}; +use crate::dbus_api::{consts, pool::shared::pool_operation, types::TData, util::result_to_tuple}; + +const ALL_PROPERTIES: [&str; 2] = [consts::POOL_TOTAL_SIZE_PROP, consts::POOL_TOTAL_USED_PROP]; fn get_properties_shared( m: &MethodInfo, TData>, @@ -56,18 +53,4 @@ fn get_properties_shared( Ok(vec![return_message.append1(return_value)]) } -pub fn get_all_properties(m: &MethodInfo, TData>) -> MethodResult { - get_properties_shared( - m, - &mut vec![consts::POOL_TOTAL_SIZE_PROP, consts::POOL_TOTAL_USED_PROP] - .into_iter() - .map(|s| s.to_string()), - ) -} - -pub fn get_properties(m: &MethodInfo, TData>) -> MethodResult { - let message: &Message = m.msg; - let mut iter = message.iter_init(); - let mut properties: Array = get_next_arg(&mut iter, 0)?; - get_properties_shared(m, &mut properties) -} +properties_footer!(); From 6cb8b63950267053bc36fd8ee25b1f80019d9e15 Mon Sep 17 00:00:00 2001 From: mulhern Date: Thu, 6 Feb 2020 13:31:59 -0500 Subject: [PATCH 106/109] Map to the Variant type for each individual match arm Otherwise, in the general case, the whole match-expression will fail to type-check because the types of the individual match arms will be different. Signed-off-by: mulhern --- src/dbus_api/blockdev/fetch_properties_2_0/methods.rs | 3 +-- src/dbus_api/filesystem/fetch_properties_2_0/methods.rs | 3 +-- src/dbus_api/pool/fetch_properties_2_0/methods.rs | 5 ++--- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/dbus_api/blockdev/fetch_properties_2_0/methods.rs b/src/dbus_api/blockdev/fetch_properties_2_0/methods.rs index 16cf6b6c66..8d2125ad61 100644 --- a/src/dbus_api/blockdev/fetch_properties_2_0/methods.rs +++ b/src/dbus_api/blockdev/fetch_properties_2_0/methods.rs @@ -30,7 +30,7 @@ fn get_properties_shared( let return_value: HashMap>)> = properties .unique() .filter_map(|prop| match prop.as_str() { - consts::BLOCKDEV_TOTAL_SIZE_PROP => Some(( + consts::BLOCKDEV_TOTAL_SIZE_PROP => Some(result_to_tuple( prop, blockdev_operation(m.tree, object_path.get_name(), |_, bd| { Ok((u128::from(*bd.size()) * devicemapper::SECTOR_SIZE as u128).to_string()) @@ -38,7 +38,6 @@ fn get_properties_shared( )), _ => None, }) - .map(|(key, result)| result_to_tuple(key, result)) .collect(); Ok(vec![return_message.append1(return_value)]) diff --git a/src/dbus_api/filesystem/fetch_properties_2_0/methods.rs b/src/dbus_api/filesystem/fetch_properties_2_0/methods.rs index 04eeb0d835..33efb179c5 100644 --- a/src/dbus_api/filesystem/fetch_properties_2_0/methods.rs +++ b/src/dbus_api/filesystem/fetch_properties_2_0/methods.rs @@ -30,7 +30,7 @@ fn get_properties_shared( let return_value: HashMap>)> = properties .unique() .filter_map(|prop| match prop.as_str() { - consts::FILESYSTEM_USED_PROP => Some(( + consts::FILESYSTEM_USED_PROP => Some(result_to_tuple( prop, filesystem_operation(m.tree, object_path.get_name(), |(_, _, fs)| { fs.used() @@ -40,7 +40,6 @@ fn get_properties_shared( )), _ => None, }) - .map(|(key, result)| result_to_tuple(key, result)) .collect(); Ok(vec![return_message.append1(return_value)]) diff --git a/src/dbus_api/pool/fetch_properties_2_0/methods.rs b/src/dbus_api/pool/fetch_properties_2_0/methods.rs index e74a2f37fa..666ed09d0e 100644 --- a/src/dbus_api/pool/fetch_properties_2_0/methods.rs +++ b/src/dbus_api/pool/fetch_properties_2_0/methods.rs @@ -27,7 +27,7 @@ fn get_properties_shared( let return_value: HashMap>)> = properties .unique() .filter_map(|prop| match prop.as_str() { - consts::POOL_TOTAL_SIZE_PROP => Some(( + consts::POOL_TOTAL_SIZE_PROP => Some(result_to_tuple( prop, pool_operation(m.tree, object_path.get_name(), |(_, _, pool)| { Ok((u128::from(*pool.total_physical_size()) @@ -35,7 +35,7 @@ fn get_properties_shared( .to_string()) }), )), - consts::POOL_TOTAL_USED_PROP => Some(( + consts::POOL_TOTAL_USED_PROP => Some(result_to_tuple( prop, pool_operation(m.tree, object_path.get_name(), |(_, _, pool)| { pool.total_physical_used() @@ -47,7 +47,6 @@ fn get_properties_shared( )), _ => None, }) - .map(|(key, result)| result_to_tuple(key, result)) .collect(); Ok(vec![return_message.append1(return_value)]) From df8acd7c8ce93e4a173b2dd24d7c66f8610d80e2 Mon Sep 17 00:00:00 2001 From: mulhern Date: Sat, 8 Feb 2020 13:19:56 -0500 Subject: [PATCH 107/109] Simplify result_to_tuple so it doesn't take key argument This makes it more generally useful, should that be required. Signed-off-by: mulhern --- .../blockdev/fetch_properties_2_0/methods.rs | 30 +++++++++------- .../fetch_properties_2_0/methods.rs | 16 +++++---- .../pool/fetch_properties_2_0/methods.rs | 36 +++++++++++-------- src/dbus_api/util.rs | 12 +++---- 4 files changed, 54 insertions(+), 40 deletions(-) diff --git a/src/dbus_api/blockdev/fetch_properties_2_0/methods.rs b/src/dbus_api/blockdev/fetch_properties_2_0/methods.rs index 8d2125ad61..9464b5a283 100644 --- a/src/dbus_api/blockdev/fetch_properties_2_0/methods.rs +++ b/src/dbus_api/blockdev/fetch_properties_2_0/methods.rs @@ -27,18 +27,24 @@ fn get_properties_shared( let return_message = message.method_return(); - let return_value: HashMap>)> = properties - .unique() - .filter_map(|prop| match prop.as_str() { - consts::BLOCKDEV_TOTAL_SIZE_PROP => Some(result_to_tuple( - prop, - blockdev_operation(m.tree, object_path.get_name(), |_, bd| { - Ok((u128::from(*bd.size()) * devicemapper::SECTOR_SIZE as u128).to_string()) - }), - )), - _ => None, - }) - .collect(); + let return_value: HashMap>)> = + properties + .unique() + .filter_map(|prop| match prop.as_str() { + consts::BLOCKDEV_TOTAL_SIZE_PROP => Some(( + prop, + result_to_tuple(blockdev_operation( + m.tree, + object_path.get_name(), + |_, bd| { + Ok((u128::from(*bd.size()) * devicemapper::SECTOR_SIZE as u128) + .to_string()) + }, + )), + )), + _ => None, + }) + .collect(); Ok(vec![return_message.append1(return_value)]) } diff --git a/src/dbus_api/filesystem/fetch_properties_2_0/methods.rs b/src/dbus_api/filesystem/fetch_properties_2_0/methods.rs index 33efb179c5..9aee017811 100644 --- a/src/dbus_api/filesystem/fetch_properties_2_0/methods.rs +++ b/src/dbus_api/filesystem/fetch_properties_2_0/methods.rs @@ -30,13 +30,17 @@ fn get_properties_shared( let return_value: HashMap>)> = properties .unique() .filter_map(|prop| match prop.as_str() { - consts::FILESYSTEM_USED_PROP => Some(result_to_tuple( + consts::FILESYSTEM_USED_PROP => Some(( prop, - filesystem_operation(m.tree, object_path.get_name(), |(_, _, fs)| { - fs.used() - .map(|u| (*u).to_string()) - .map_err(|e| e.to_string()) - }), + result_to_tuple(filesystem_operation( + m.tree, + object_path.get_name(), + |(_, _, fs)| { + fs.used() + .map(|u| (*u).to_string()) + .map_err(|e| e.to_string()) + }, + )), )), _ => None, }) diff --git a/src/dbus_api/pool/fetch_properties_2_0/methods.rs b/src/dbus_api/pool/fetch_properties_2_0/methods.rs index 666ed09d0e..c73f3ea479 100644 --- a/src/dbus_api/pool/fetch_properties_2_0/methods.rs +++ b/src/dbus_api/pool/fetch_properties_2_0/methods.rs @@ -27,23 +27,31 @@ fn get_properties_shared( let return_value: HashMap>)> = properties .unique() .filter_map(|prop| match prop.as_str() { - consts::POOL_TOTAL_SIZE_PROP => Some(result_to_tuple( + consts::POOL_TOTAL_SIZE_PROP => Some(( prop, - pool_operation(m.tree, object_path.get_name(), |(_, _, pool)| { - Ok((u128::from(*pool.total_physical_size()) - * devicemapper::SECTOR_SIZE as u128) - .to_string()) - }), + result_to_tuple(pool_operation( + m.tree, + object_path.get_name(), + |(_, _, pool)| { + Ok((u128::from(*pool.total_physical_size()) + * devicemapper::SECTOR_SIZE as u128) + .to_string()) + }, + )), )), - consts::POOL_TOTAL_USED_PROP => Some(result_to_tuple( + consts::POOL_TOTAL_USED_PROP => Some(( prop, - pool_operation(m.tree, object_path.get_name(), |(_, _, pool)| { - pool.total_physical_used() - .map_err(|e| e.to_string()) - .map(|size| { - (u128::from(*size) * devicemapper::SECTOR_SIZE as u128).to_string() - }) - }), + result_to_tuple(pool_operation( + m.tree, + object_path.get_name(), + |(_, _, pool)| { + pool.total_physical_used() + .map_err(|e| e.to_string()) + .map(|size| { + (u128::from(*size) * devicemapper::SECTOR_SIZE as u128).to_string() + }) + }, + )), )), _ => None, }) diff --git a/src/dbus_api/util.rs b/src/dbus_api/util.rs index 98f30631cf..7e90ba7d31 100644 --- a/src/dbus_api/util.rs +++ b/src/dbus_api/util.rs @@ -29,16 +29,12 @@ pub fn tuple_to_option(value: (bool, T)) -> Option { } } -/// Map a result obtained for the FetchProperties interface to a pair of -/// a key and a value. The key is the property key, and therefore the key -/// of the item returned. The value is a tuple. An error in the result +/// Map a result obtained for the FetchProperties interface to a value used +/// to represent an option. An error in the result /// argument yields a false in the return value, indicating that the value /// returned is a string representation of the error encountered in /// obtaining the value, and not the value requested. -pub fn result_to_tuple( - key: String, - result: Result, -) -> (String, (bool, Variant>)) +pub fn result_to_tuple(result: Result) -> (bool, Variant>) where T: RefArg + 'static, { @@ -46,7 +42,7 @@ where Ok(value) => (true, Variant(Box::new(value) as Box)), Err(e) => (false, Variant(Box::new(e) as Box)), }; - (key, (success, value)) + (success, value) } /// Get the next argument off the bus. loc is the index of the location of From b0f936abc0754d5e844754a67b9d6245cef38aea Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 25 Nov 2019 09:02:06 -0500 Subject: [PATCH 108/109] Update changelog Signed-off-by: mulhern --- CHANGES.txt | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index 9a6b5d512a..3ddabbc1f9 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,83 @@ +stratisd 2.0.1 +============== +Recommended Rust toolchain version: 1.40 +Lowest supported Rust toolchain version: 1.39 +Python import sorter: isort (4.3.4) +Python auto-formatter: black (19.3b0) + +- Avoid returning from function while pool is suspended: + https://github.com/stratis-storage/stratisd/issues/1730 + https://github.com/stratis-storage/stratisd/pull/1734 + +- Refine description of stratisd.service in systemd configuration: + https://github.com/stratis-storage/stratisd/issues/1647 + https://github.com/stratis-storage/stratisd/pull/1738 + +- Fix a few places where the index of an incorrect D-Bus argument would be + misidentified in a D-Bus error message: + https://github.com/stratis-storage/stratisd/pull/1756 + +- Add changelog matter from two previous releases to CHANGES.txt: + https://github.com/stratis-storage/stratisd/pull/1793 + +- Add some additional logging for significant events: + https://github.com/stratis-storage/stratisd/pull/1797 + +- Restructure dbus_api module to better support multiple versioned + interfaces: + https://github.com/stratis-storage/stratisd/pull/1804 + https://github.com/stratis-storage/stratisd/pull/1776 + https://github.com/stratis-storage/stratisd/pull/1770 + +- Refactor device discovery mechanism: + https://github.com/stratis-storage/stratisd/pull/1779 + https://github.com/stratis-storage/stratisd/pull/1767 + https://github.com/stratis-storage/stratisd/pull/1765 + https://github.com/stratis-storage/stratisd/pull/1759 + https://github.com/stratis-storage/stratisd/pull/1750 + https://github.com/stratis-storage/stratisd/pull/1739 + https://github.com/stratis-storage/stratisd/pull/1736 + https://github.com/stratis-storage/stratisd/pull/1725 + https://github.com/stratis-storage/stratisd/pull/1723 + +- Refactor idempotency implementation so that it is handled as close to + entry points to the engine as possible: + https://github.com/stratis-storage/stratisd/pull/1743 + +- Refactor metadata handling for better encapsulation: + https://github.com/stratis-storage/stratisd/pull/1792 + +- Fully qualify all non-prelude data types in macros: + https://github.com/stratis-storage/stratisd/issues/1748 + https://github.com/stratis-storage/stratisd/pull/1758 + +- Tidies and Maintenance: + https://github.com/stratis-storage/stratisd/pull/1803 + https://github.com/stratis-storage/stratisd/pull/1796 + https://github.com/stratis-storage/stratisd/pull/1791 + https://github.com/stratis-storage/stratisd/pull/1786 + https://github.com/stratis-storage/stratisd/pull/1785 + https://github.com/stratis-storage/stratisd/pull/1777 + https://github.com/stratis-storage/stratisd/pull/1774 + https://github.com/stratis-storage/stratisd/pull/1772 + https://github.com/stratis-storage/stratisd/pull/1763 + https://github.com/stratis-storage/stratisd/pull/1762 + https://github.com/stratis-storage/stratisd/pull/1758 + https://github.com/stratis-storage/stratisd/pull/1745 + https://github.com/stratis-storage/stratisd/pull/1728 + https://github.com/stratis-storage/stratisd/pull/1726 + https://github.com/stratis-storage/stratisd/pull/1724 + https://github.com/stratis-storage/stratisd/pull/1721 + https://github.com/stratis-storage/stratisd/pull/1715 + https://github.com/stratis-storage/stratisd/pull/1714 + https://github.com/stratis-storage/stratisd/pull/1713 + https://github.com/stratis-storage/stratisd/pull/1712 + https://github.com/stratis-storage/stratisd/pull/1709 + https://github.com/stratis-storage/stratisd/pull/1707 + https://github.com/stratis-storage/stratisd/pull/1704 + https://github.com/stratis-storage/stratisd/pull/1701 + + stratisd 2.0.0 ============== Recommended Rust toolchain version: 1.38 From c0201c46cedb36970f4e090063ee55a0de652bd2 Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 10 Feb 2020 16:07:37 -0500 Subject: [PATCH 109/109] version 2.0.1 Signed-off-by: mulhern --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b2d8bfefb8..3e942f6a5f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -284,7 +284,7 @@ dependencies = [ [[package]] name = "libstratis" -version = "2.0.0" +version = "2.0.1" dependencies = [ "byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index 99d43ac5ab..ab5b1c2386 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "libstratis" -version = "2.0.0" +version = "2.0.1" authors = ["Stratis Developers "] edition = "2018"