<NME> pool.rs
<BEF> // This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

use std::{collections::HashMap, path::Path, vec::Vec};

use chrono::{DateTime, Utc};
use serde_json::{Map, Value};

use devicemapper::{Bytes, DmNameBuf, Sectors};
use stratisd_proc_macros::strat_pool_impl_gen;

use crate::{
    engine::{
use devicemapper::consts::SECTOR_SIZE;
use devicemapper::Device;
use devicemapper::DM;
use devicemapper::{Sectors, Segment};
use devicemapper::LinearDev;
use devicemapper::{ThinDevId, ThinPoolStatus, ThinPoolWorkingStatus};

            liminal::{DeviceInfo, DeviceSet, LInfo},
            metadata::MDADataSize,
            serde_structs::{FlexDevsSave, PoolSave, Recordable},
            thinpool::{StratFilesystem, ThinPool, ThinPoolSizeParams, DATA_BLOCK_SIZE},
        },
        types::{
            ActionAvailability, BlockDevTier, Clevis, Compare, CreateAction, DeleteAction, DevUuid,
            EncryptionInfo, FilesystemUuid, GrowAction, Key, KeyDescription, Name, PoolDiff,
            PoolEncryptionInfo, PoolUuid, RegenAction, RenameAction, SetCreateAction,
            SetDeleteAction, StratFilesystemDiff, StratPoolDiff,
        },
    },
    stratis::{StratisError, StratisResult},
};


pub use super::thinpool::{DATA_BLOCK_SIZE, DATA_LOWATER};

const INITIAL_META_SIZE: Sectors = Sectors(16 * Mi / SECTOR_SIZE as u64);
pub const INITIAL_DATA_SIZE: Sectors = Sectors(768 * Mi / SECTOR_SIZE as u64);
const INITIAL_MDV_SIZE: Sectors = Sectors(16 * Mi / SECTOR_SIZE as u64);

#[derive(Debug)]
fn next_index(flex_devs: &FlexDevsSave) -> Sectors {
    let expect_msg = "Setting up rather than initializing a pool, so each flex dev must have been allocated at least some segments.";
    [
        flex_devs
            .meta_dev
            .last()
            .unwrap_or_else(|| panic!("{}", expect_msg)),
        flex_devs
            .thin_meta_dev
            .last()
            .unwrap_or_else(|| panic!("{}", expect_msg)),
        flex_devs
            .thin_data_dev
            .last()
            .unwrap_or_else(|| panic!("{}", expect_msg)),
        flex_devs
            .thin_meta_dev_spare
            .last()
            .unwrap_or_else(|| panic!("{}", expect_msg)),
    ]
    .iter()
    .max_by_key(|x| x.0)
    .map(|&&(start, length)| start + length)
    .expect("iterator is non-empty")
}

/// Check the metadata of an individual pool for consistency.
/// Precondition: This method is called only when setting up a pool, which
/// ensures that the flex devs metadata lists are all non-empty.
fn check_metadata(metadata: &PoolSave) -> StratisResult<()> {
    let flex_devs = &metadata.flex_devs;
    let next = next_index(flex_devs);
    let allocated_from_cap = metadata.backstore.cap.allocs[0].1;

    if allocated_from_cap != next {
        let err_msg = format!(
            "{} used in thinpool, but {} allocated from backstore cap device",
            next, allocated_from_cap
        }

        let meta_regions = block_mgr
            .alloc_space(INITIAL_META_SIZE)
            .expect("blockmgr must not fail, already checked for space");

        let meta_spare_regions = block_mgr
            .alloc_space(INITIAL_META_SIZE)
            .expect("blockmgr must not fail, already checked for space");

        let data_regions = block_mgr
            .alloc_space(INITIAL_DATA_SIZE)
            .expect("blockmgr must not fail, already checked for space");

        // When constructing a thin-pool, Stratis reserves the first N
        if total_allocated != next {
            let err_msg = format!(
                "{} used in thinpool, but {} given up by cache for pool {}",
                total_allocated, next, metadata.name
            );
            return Err(StratisError::Msg(err_msg));
        }
        let meta_dev = try!(LinearDev::new(&format_flex_name(&pool_uuid, FlexRole::ThinMeta),
                                           dm,
                                           meta_regions));
        try!(wipe_sectors(&try!(meta_dev.devnode()), Sectors(0), INITIAL_META_SIZE));

        let data_dev = try!(LinearDev::new(&format_flex_name(&pool_uuid, FlexRole::ThinData),
                                           dm,
            .map(|x| x.length)
            .sum::<Sectors>();

        if total_allocated == Sectors(0) {
            let err_msg = format!(
                "no segments allocated to the cap device for pool {}",
                metadata.name
            );
            return Err(StratisError::Msg(err_msg));
        }

        if next > total_allocated {
            let err_msg = format!(
                "{} allocated to cap device, but {} allocated to flex devs",
                next, total_allocated
            );
            return Err(StratisError::Msg(err_msg));
        }
    }

    Ok(())
}

#[derive(Debug)]
pub struct StratPool {
    backstore: Backstore,
    thin_pool: ThinPool,
    action_avail: ActionAvailability,
    metadata_size: Sectors,
}

#[strat_pool_impl_gen]
impl StratPool {
    /// Initialize a Stratis Pool.
    /// 1. Initialize the block devices specified by paths.
    /// 2. Set up thinpool device to back filesystems.
    /// Precondition: p.is_absolute() is true for all p in paths
    pub fn initialize(
        name: &str,
        devices: UnownedDevices,
        encryption_info: Option<&EncryptionInfo>,
    ) -> StratisResult<(PoolUuid, StratPool)> {
        let pool_uuid = PoolUuid::new_v4();

        // FIXME: Initializing with the minimum MDA size is not necessarily
        // enough. If there are enough devices specified, more space will be
        // required.
        let mut backstore =
            Backstore::initialize(pool_uuid, devices, MDADataSize::default(), encryption_info)?;

        let thinpool = ThinPool::new(
            pool_uuid,
            match ThinPoolSizeParams::new(backstore.datatier_usable_size()) {
                Ok(ref params) => params,
                Err(e) => {
                    let _ = backstore.destroy();
                    return Err(e);
                }
            },
            DATA_BLOCK_SIZE,
            &mut backstore,
        );

        let thinpool = match thinpool {
            Ok(thinpool) => thinpool,
            Err(err) => {
                let _ = backstore.destroy();
                return Err(err);
            }
        };

        let metadata_size = backstore.datatier_metadata_size();
        let mut pool = StratPool {
            backstore,
            thin_pool: thinpool,
            action_avail: ActionAvailability::Full,
            metadata_size,
        };

        pool.write_metadata(&Name::new(name.to_owned()))?;

        Ok((pool_uuid, pool))
    }

    /// Setup a StratPool using its UUID and the list of devnodes it has.
    /// Precondition: every device in devnodes has already been determined
    /// to belong to the pool with the specified uuid.
    /// Precondition: A metadata verification step has already been run.
    ///
    /// Precondition:
    ///   * key_description.is_some() -> every StratBlockDev in datadevs has a
    ///   key description and that key description == key_description
    ///   * key_description.is_none() -> no StratBlockDev in datadevs has a
    ///   key description.
    ///   * no StratBlockDev in cachdevs has a key description
    pub fn setup(
        uuid: PoolUuid,
        datadevs: Vec<StratBlockDev>,
        cachedevs: Vec<StratBlockDev>,
        timestamp: DateTime<Utc>,
        metadata: &PoolSave,
        action_avail: ActionAvailability,
    ) -> StratisResult<(Name, StratPool)> {
        check_metadata(metadata)?;

        let mut backstore =
            Backstore::setup(uuid, &metadata.backstore, datadevs, cachedevs, timestamp)?;
        let pool_name = &metadata.name;

        let mut thinpool = ThinPool::setup(
            pool_name,
            uuid,
            &metadata.thinpool_dev,
            &metadata.flex_devs,
            &backstore,
        )?;

        // TODO: Remove in stratisd 4.0
        let mut needs_save = metadata.thinpool_dev.fs_limit.is_none()
            || metadata.thinpool_dev.feature_args.is_none();

        needs_save |= thinpool.check(uuid, &mut backstore)?.0;

        let metadata_size = backstore.datatier_metadata_size();
        let mut pool = StratPool {
            backstore,
            thin_pool: thinpool,
            action_avail,
            metadata_size,
        };

        // Change the pool to started at this point not that the pool has been set up.
        needs_save |= !metadata.started.unwrap_or(false);

        if needs_save {
            pool.write_metadata(pool_name)?;
        }

        Ok((Name::new(pool_name.to_owned()), pool))
    }

    /// Send a synthetic udev change event to every filesystem on the given pool.
    pub fn udev_pool_change(&self, pool_name: &str) {
        for (name, uuid, fs) in self.thin_pool.filesystems() {
            fs.udev_fs_change(pool_name, uuid, &name);
        }
    }

    /// Write current metadata to pool members.
    #[pool_mutating_action("NoPoolChanges")]
    pub fn write_metadata(&mut self, name: &str) -> StratisResult<()> {
        let data = serde_json::to_string(&self.record(name))?;
        self.backstore.save_state(data.as_bytes())
    }

    /// Teardown a pool.
    #[cfg(test)]
    pub fn teardown(&mut self) -> StratisResult<()> {
        self.thin_pool.teardown()?;
        self.backstore.teardown()
    /// Minimum initial size for a pool.
    pub fn min_initial_size() -> Sectors {
        // One extra meta for spare
        (INITIAL_META_SIZE * 2u64) + INITIAL_DATA_SIZE + INITIAL_MDV_SIZE
    }

    /// Write current metadata to pool members.
        self.thin_pool.get_eventing_dev_names(pool_uuid)
    }

    /// Called when a DM device in this pool has generated an event. This method
    /// handles checking pools.
    #[pool_mutating_action("NoPoolChanges")]
    pub fn event_on(&mut self, pool_uuid: PoolUuid, pool_name: &Name) -> StratisResult<PoolDiff> {
        let cached = self.cached();
        let (changed, thin_pool) = self.thin_pool.check(pool_uuid, &mut self.backstore)?;
        let pool = cached.diff(&self.dump(()));
        if changed {
            self.write_metadata(pool_name)?;
        }
        Ok(PoolDiff { thin_pool, pool })
    }

    /// Called when a DM device in this pool has generated an event. This method
    /// handles checking filesystems.
    #[pool_mutating_action("NoPoolChanges")]
    pub fn fs_event_on(
        &mut self,
        pool_uuid: PoolUuid,
    ) -> StratisResult<HashMap<FilesystemUuid, StratFilesystemDiff>> {
        self.thin_pool.check_fs(pool_uuid, &self.backstore)
    }

    pub fn record(&self, name: &str) -> PoolSave {
        PoolSave {
            name: name.to_owned(),
            backstore: self.backstore.record(),
            flex_devs: self.thin_pool.record(),
            thinpool_dev: self.thin_pool.record(),
            started: Some(true),
        }
    }

    fn datadevs_encrypted(&self) -> bool {
        self.backstore.data_tier_is_encrypted()
    }

    pub fn get_strat_blockdev(&self, uuid: DevUuid) -> Option<(BlockDevTier, &StratBlockDev)> {
        self.backstore.get_blockdev_by_uuid(uuid)
    }

    #[pool_mutating_action("NoPoolChanges")]
    pub fn get_mut_strat_blockdev(
        &mut self,
        uuid: DevUuid,
    ) -> StratisResult<Option<(BlockDevTier, &mut StratBlockDev)>> {
        Ok(self.backstore.get_mut_blockdev_by_uuid(uuid))
    }

    #[pool_mutating_action("NoPoolChanges")]
    pub fn blockdevs_mut(
        &mut self,
    ) -> StratisResult<Vec<(DevUuid, BlockDevTier, &mut StratBlockDev)>> {
        Ok(self.backstore.blockdevs_mut())
    }

    /// Destroy the pool.
    /// Precondition: All filesystems belonging to this pool must be
    /// unmounted.
    ///
    /// This method is not a mutating action as the pool should be allowed
    /// to be destroyed even if the metadata is inconsistent.
    pub fn destroy(&mut self) -> StratisResult<()> {
        self.thin_pool.teardown()?;
        self.backstore.destroy()?;
        Ok(())
    }

    /// Check the limit of filesystems on a pool and return an error if it has been passed.
    fn check_fs_limit(&self, new_fs: usize) -> StratisResult<()> {
        let fs_limit = self.fs_limit();
        if convert_int!(fs_limit, u64, usize)? < self.filesystems().len() + new_fs {
            Err(StratisError::Msg(format!("The pool limit of {} filesystems has already been reached; increase the filesystem limit on the pool to continue", fs_limit)))
        } else {
            Ok(())
        }
    }

    /// Stop a pool, consuming it and converting it into a set of devices to be
    /// set up again later.
    pub fn stop(&mut self, pool_name: &Name) -> StratisResult<DeviceSet> {
        self.thin_pool.teardown()?;
        let mut data = self.record(pool_name);
        data.started = Some(false);
        let json = serde_json::to_string(&data)?;
        self.backstore.save_state(json.as_bytes())?;
        self.backstore.teardown()?;
        Ok(self
            .backstore
            .blockdevs()
            .into_iter()
            .map(|(uuid, _, bd)| (uuid, LInfo::from(DeviceInfo::from(bd))))
            .collect::<DeviceSet>())
    }

    #[cfg(test)]
    #[pool_mutating_action("NoRequests")]
    #[pool_rollback]
    pub fn return_rollback_failure(&mut self) -> StratisResult<()> {
        Err(StratisError::RollbackError {
            causal_error: Box::new(StratisError::Msg("Causal error".to_string())),
            rollback_error: Box::new(StratisError::Msg("Rollback error".to_string())),
            level: ActionAvailability::NoRequests,
        })
    }

    #[cfg(test)]
    #[pool_mutating_action("NoRequests")]
    #[pool_rollback]
    pub fn return_rollback_failure_chain(&mut self) -> StratisResult<()> {
        Err(StratisError::Chained(
            "Chained error".to_string(),
            Box::new(StratisError::RollbackError {
                causal_error: Box::new(StratisError::Msg("Causal error".to_string())),
                rollback_error: Box::new(StratisError::Msg("Rollback error".to_string())),
                level: ActionAvailability::NoRequests,
            }),
        ))
    }

    /// Verifies that the filesystem operation to be performed is allowed to perform
    /// overprovisioning if it is determined to be the end result.
    fn check_overprov(&self, increase: Sectors) -> StratisResult<()> {
        if !self.thin_pool.overprov_enabled()
            && self
                .thin_pool
                .filesystems()
                .iter()
                .map(|(_, _, fs)| fs.thindev_size())
                .sum::<Sectors>()
                + increase
                > self.thin_pool.total_fs_limit(&self.backstore)
        {
            Err(StratisError::Msg(format!(
                "Overprovisioning is disabled on this pool and increasing total filesystems size by {} would result in overprovisioning",
                increase
            )))
        } else {
            Ok(())
        }
    }
}

impl<'a> Into<Value> for &'a StratPool {
    // Precondition: (&ThinPool).into() pattern matches Value::Object(_)
    // Precondition: (&Backstore).into() pattern matches Value::Object(_)
    fn into(self) -> Value {
        let mut map: Map<_, _> =
            if let Value::Object(map) = <&ThinPool as Into<Value>>::into(&self.thin_pool) {
                map.into_iter()
            } else {
                unreachable!("ThinPool conversion returns a JSON object")
            }
            .collect();
        map.extend(
            if let Value::Object(map) = <&Backstore as Into<Value>>::into(&self.backstore) {
                map.into_iter()
            } else {
                unreachable!("Backstore conversion returns a JSON object")
            },
        );
        map.insert(
            "available_actions".to_string(),
            Value::from(self.action_avail.to_string()),
        );
        map.insert("fs_limit".to_string(), Value::from(self.fs_limit()));
        Value::from(map)
    }
}

#[strat_pool_impl_gen]
impl Pool for StratPool {
    type Filesystem = StratFilesystem;
    type BlockDev = StratBlockDev;

    #[pool_mutating_action("NoRequests")]
    fn init_cache(
        &mut self,
        pool_uuid: PoolUuid,
        pool_name: &str,
        blockdevs: &[&Path],
    ) -> StratisResult<SetCreateAction<DevUuid>> {
        validate_paths(blockdevs)?;

        if self.is_encrypted() {
            return Err(StratisError::Msg(
                "Use of a cache is not supported with an encrypted pool".to_string(),
            ));
        }

        let devices = ProcessedPathInfos::try_from(blockdevs)?;
        let (stratis_devices, unowned_devices) = devices.unpack();
        let (this_pool, other_pools) = stratis_devices.partition(pool_uuid);

        other_pools.error_on_not_empty()?;

        let (in_pool, out_pool): (Vec<_>, Vec<_>) = this_pool
            .iter()
            .map(|(dev_uuid, _)| {
                self.backstore
                    .get_blockdev_by_uuid(*dev_uuid)
                    .map(|(tier, _)| (*dev_uuid, tier))
            })
            .partition(|v| v.is_some());

        if !out_pool.is_empty() {
            let error_message = format!(
                    "Devices ({}) appear to be already in use by this pool which has UUID {} but this pool has no record of them",
                    out_pool
                    .iter()
                    .map(|opt| this_pool.get(&opt.expect("was looked up").0).expect("partitioned from this_pool").devnode.display().to_string())
                    .collect::<Vec<_>>()
                    .join(", "),
                    pool_uuid
                );
            return Err(StratisError::Msg(error_message));
        };

        let (datadevs, cachedevs): (Vec<_>, Vec<_>) = in_pool
            .iter()
            .map(|opt| opt.expect("in_pool devices are Some"))
            .partition(|(_, tier)| *tier == BlockDevTier::Data);

        if !datadevs.is_empty() {
            let error_message = format!(
                "Devices ({}) appear to be already in use by this pool which has UUID {} in the data tier",
                datadevs
                    .iter()
                    .map(|(uuid, _)| this_pool.get(uuid).expect("partitioned from this_pool").devnode.display().to_string())
                    .collect::<Vec<_>>()
                    .join(", "),
                pool_uuid
            );
            return Err(StratisError::Msg(error_message));
        };

        if !self.has_cache() {
            if unowned_devices.is_empty() {
                return Err(StratisError::Msg(
                    "At least one device is reqired to initialize a cache.".to_string(),
                ));
            }

            self.thin_pool.suspend()?;
            let devices_result = self
                .backstore
                .init_cache(pool_uuid, unowned_devices)
                .and_then(|bdi| {
                    self.thin_pool
                        .set_device(self.backstore.device().expect(
                            "Since thin pool exists, space must have been allocated \
                             from the backstore, so backstore must have a cap device",
                        ))
                        .and(Ok(bdi))
                });
            self.thin_pool.resume()?;
            let devices = devices_result?;
            self.write_metadata(pool_name)?;
            Ok(SetCreateAction::new(devices))
        } else {
            init_cache_idempotent_or_err(
                &cachedevs
                    .iter()
                    .map(|(uuid, _)| {
                        this_pool
                            .get(uuid)
                            .expect("partitioned from this_pool")
                            .devnode
                            .as_path()
                    })
                    .chain(
                        unowned_devices
                            .unpack()
                            .iter()
                            .map(|info| info.devnode.as_path()),
                    )
                    .collect::<Vec<_>>(),
                self.backstore
                    .cachedevs()
                    .into_iter()
                    .map(|(_, bd)| bd.physical_path().to_owned()),
            )
        }
    }

    #[pool_mutating_action("NoRequests")]
    #[pool_rollback]
    fn bind_clevis(
        &mut self,
        pin: &str,
        clevis_info: &Value,
    ) -> StratisResult<CreateAction<Clevis>> {
        let changed = self.backstore.bind_clevis(pin, clevis_info)?;
        if changed {
            Ok(CreateAction::Created(Clevis))
        } else {
            Ok(CreateAction::Identity)
        }
    }

    #[pool_mutating_action("NoRequests")]
    #[pool_rollback]
    fn unbind_clevis(&mut self) -> StratisResult<DeleteAction<Clevis>> {
        let changed = self.backstore.unbind_clevis()?;
        if changed {
            Ok(DeleteAction::Deleted(Clevis))
        } else {
            Ok(DeleteAction::Identity)
        }
    }

    #[pool_mutating_action("NoRequests")]
    #[pool_rollback]
    fn bind_keyring(
        &mut self,
        key_description: &KeyDescription,
    ) -> StratisResult<CreateAction<Key>> {
        let changed = self.backstore.bind_keyring(key_description)?;
        if changed {
            Ok(CreateAction::Created(Key))
        } else {
            Ok(CreateAction::Identity)
        }
    }

    #[pool_mutating_action("NoRequests")]
    #[pool_rollback]
    fn unbind_keyring(&mut self) -> StratisResult<DeleteAction<Key>> {
        let changed = self.backstore.unbind_keyring()?;
        if changed {
            Ok(DeleteAction::Deleted(Key))
        } else {
            Ok(DeleteAction::Identity)
        }
    }

    #[pool_mutating_action("NoRequests")]
    #[pool_rollback]
    fn rebind_keyring(
        &mut self,
        new_key_desc: &KeyDescription,
    ) -> StratisResult<RenameAction<Key>> {
        match self.backstore.rebind_keyring(new_key_desc)? {
            Some(true) => Ok(RenameAction::Renamed(Key)),
            Some(false) => Ok(RenameAction::Identity),
            None => Ok(RenameAction::NoSource),
        }
    }

    #[pool_mutating_action("NoRequests")]
    #[pool_rollback]
    fn rebind_clevis(&mut self) -> StratisResult<RegenAction> {
        self.backstore.rebind_clevis().map(|_| RegenAction)
    }

    #[pool_mutating_action("NoRequests")]
    fn create_filesystems<'a>(
        &mut self,
        pool_name: &str,
        pool_uuid: PoolUuid,
        specs: &[(&'a str, Option<Bytes>)],
    ) -> StratisResult<SetCreateAction<(&'a str, FilesystemUuid, Sectors)>> {
        self.check_fs_limit(specs.len())?;

        let spec_map = validate_filesystem_size_specs(specs)?;

        let increase = spec_map.iter().map(|(_, size)| *size).sum::<Sectors>();
        self.check_overprov(increase)?;

        spec_map.iter().fold(Ok(()), |res, (name, size)| {
            res.and_then(|()| validate_name(name))
                .and_then(|()| {
                    if let Some((_, fs)) = self.thin_pool.get_filesystem_by_name(name) {
                        if fs.thindev_size() == *size {
                            Ok(())
                        } else {
                            Err(StratisError::Msg(format!(
                                "Size {} of filesystem {} to be created conflicts with size {} for existing filesystem",
                                size,
                                name,
                                fs.thindev_size()
                            )))
                        }
                    } else {
                        Ok(())
                    }
                })
        })?;

        // TODO: Roll back on filesystem initialization failure.
        let mut result = Vec::new();
        for (name, size) in spec_map {
            if self.thin_pool.get_mut_filesystem_by_name(name).is_none() {
                let fs_uuid = self
                    .thin_pool
                    .create_filesystem(pool_name, pool_uuid, name, size)?;
                result.push((name, fs_uuid, size));
            }
        }

        Ok(SetCreateAction::new(result))
    }

    #[pool_mutating_action("NoRequests")]
    fn add_blockdevs(
        &mut self,
        pool_uuid: PoolUuid,
        pool_name: &str,
        paths: &[&Path],
        tier: BlockDevTier,
    ) -> StratisResult<(SetCreateAction<DevUuid>, Option<PoolDiff>)> {
        validate_paths(paths)?;

        let bdev_info = if tier == BlockDevTier::Cache && !self.has_cache() {
            return Err(StratisError::Msg(
                format!(
                    "No cache has been initialized for pool with UUID {} and name {}; it is therefore impossible to add additional devices to the cache",
                    pool_uuid,
                    pool_name
                )
            ));
        } else {
            let devices = ProcessedPathInfos::try_from(paths)?;
            let (stratis_devices, unowned_devices) = devices.unpack();
            let (this_pool, other_pools) = stratis_devices.partition(pool_uuid);

            other_pools.error_on_not_empty()?;

            let (in_pool, out_pool): (Vec<_>, Vec<_>) = this_pool
                .iter()
                .map(|(dev_uuid, _)| {
                    self.backstore
                        .get_blockdev_by_uuid(*dev_uuid)
                        .map(|(tier, _)| (*dev_uuid, tier))
                })
                .partition(|v| v.is_some());

            if !out_pool.is_empty() {
                let error_message = format!(
                    "Devices ({}) appear to be already in use by this pool which has UUID {} but this pool has no record of them",
                    out_pool
                    .iter()
                    .map(|opt| this_pool.get(&opt.expect("was looked up").0).expect("partitioned from this_pool").devnode.display().to_string())
                    .collect::<Vec<_>>()
                    .join(", "),
                    pool_uuid
                );
                return Err(StratisError::Msg(error_message));
            };

            let (datadevs, cachedevs): (Vec<_>, Vec<_>) = in_pool
                .iter()
                .map(|opt| opt.expect("in_pool devices are Some"))
                .partition(|(_, tier)| *tier == BlockDevTier::Data);

            if tier == BlockDevTier::Cache {
                // If adding cache devices, must suspend the pool; the cache
                // must be augmented with the new devices.
                if !datadevs.is_empty() {
                    let error_message = format!(
                        "Devices ({}) appear to be already in use by this pool which has UUID {}, but in the data tier not the cache tier",
                        datadevs
                        .iter()
                        .map(|(uuid, _)| this_pool.get(uuid).expect("partitioned from this_pool").devnode.display().to_string())
                        .collect::<Vec<_>>()
                        .join(", "),
                        pool_uuid
                    );
                    return Err(StratisError::Msg(error_message));
                };

                if unowned_devices.is_empty() {
                    return Ok((SetCreateAction::new(vec![]), None));
                }

                self.thin_pool.suspend()?;
                let bdev_info_res = self.backstore.add_cachedevs(pool_uuid, unowned_devices);
                self.thin_pool.resume()?;
                let bdev_info = bdev_info_res?;
                Ok((SetCreateAction::new(bdev_info), None))
            } else {
                if !cachedevs.is_empty() {
                    let error_message = format!(
                        "Devices ({}) appear to be already in use by this pool which has UUID {}, but in the cache tier not the data tier",
                        cachedevs
                        .iter()
                        .map(|(uuid, _)| this_pool.get(uuid).expect("partitioned from this_pool").devnode.display().to_string())
                        .collect::<Vec<_>>()
                        .join(", "),
                        pool_uuid
                    );
                    return Err(StratisError::Msg(error_message));
                };

                if unowned_devices.is_empty() {
                    return Ok((SetCreateAction::new(vec![]), None));
                }

                let cached = self.cached();

                // If just adding data devices, no need to suspend the pool.
                // No action will be taken on the DM devices.
                let bdev_info = self.backstore.add_datadevs(pool_uuid, unowned_devices)?;
                self.thin_pool.set_queue_mode();

                // Adding data devices does not change the state of the thin
                // pool at all. However, if the thin pool is in a state
                // where it would request an allocation from the backstore the
                // addition of the new data devs may have changed its context
                // so that it can satisfy the allocation request where
                // previously it could not. Run check() in case that is true.
                let check = match self.thin_pool.check(pool_uuid, &mut self.backstore) {
                    Ok((_, thin_pool)) => {
                        let pool = cached.diff(&self.dump(()));
                        Some(PoolDiff { thin_pool, pool })
                    }
                    Err(e) => {
                        warn!("Failed to check the thin pool for status changes; some information may not be able to reported to the IPC layer: {}", e);
                        None
                    }
                };
                Ok((SetCreateAction::new(bdev_info), check))
            }
        };
        self.write_metadata(pool_name)?;
        bdev_info
    }

    #[pool_mutating_action("NoRequests")]
    fn destroy_filesystems(
        &mut self,
        pool_name: &str,
        fs_uuids: &[FilesystemUuid],
    ) -> StratisResult<SetDeleteAction<FilesystemUuid>> {
        let mut removed = Vec::new();
        for &uuid in fs_uuids {
            if let Some(uuid) = self.thin_pool.destroy_filesystem(pool_name, uuid)? {
                removed.push(uuid);
            }
        }

        Ok(SetDeleteAction::new(removed))
    }

    #[pool_mutating_action("NoRequests")]
    fn rename_filesystem(
        &mut self,
        pool_name: &str,
        uuid: FilesystemUuid,
        new_name: &str,
    ) -> StratisResult<RenameAction<FilesystemUuid>> {
        validate_name(new_name)?;
        match self
            .thin_pool
            .rename_filesystem(pool_name, uuid, new_name)?
        {
            Some(true) => Ok(RenameAction::Renamed(uuid)),
            Some(false) => Ok(RenameAction::Identity),
            None => Ok(RenameAction::NoSource),
        }
    }

    #[pool_mutating_action("NoRequests")]
    fn snapshot_filesystem<'a>(
        &'a mut self,
        pool_name: &str,
        pool_uuid: PoolUuid,
        origin_uuid: FilesystemUuid,
        snapshot_name: &str,
    ) -> StratisResult<CreateAction<(FilesystemUuid, &'a mut Self::Filesystem)>> {
        self.check_fs_limit(1)?;

        validate_name(snapshot_name)?;
        self.check_overprov(
            self.thin_pool
                .get_filesystem_by_uuid(origin_uuid)
                .ok_or_else(|| {
                    StratisError::Msg(format!(
                        "Filesystem with UUID {} could not be found",
                        origin_uuid
                    ))
                })?
                .1
                .thindev_size(),
        )?;

        if self
            .thin_pool
            .get_filesystem_by_name(snapshot_name)
            .is_some()
        {
            return Ok(CreateAction::Identity);
        }

        self.thin_pool
            .snapshot_filesystem(pool_name, pool_uuid, origin_uuid, snapshot_name)
            .map(CreateAction::Created)
    }

    fn total_physical_size(&self) -> Sectors {
        self.backstore.datatier_size()
    }

    fn total_allocated_size(&self) -> Sectors {
        self.backstore.datatier_allocated_size() + self.metadata_size
    }

    fn total_physical_used(&self) -> Option<Sectors> {
        // TODO: note that with the addition of another layer, the
        // calculation of the amount of physical spaced used by means
        // of adding the amount used by Stratis metadata to the amount used
        // by the pool abstraction will be invalid. In the event of, e.g.,
        // software RAID, the amount will be far too low to be useful, in the
        // event of, e.g, VDO, the amount will be far too large to be useful.
        self.thin_pool
            .total_physical_used()
            .map(|u| u + self.metadata_size)
    }

    fn filesystems(&self) -> Vec<(Name, FilesystemUuid, &Self::Filesystem)> {
        self.thin_pool
            .filesystems()
            .into_iter()
            .map(|(n, u, f)| (n, u, f))
            .collect()
    }

    fn get_filesystem(&self, uuid: FilesystemUuid) -> Option<(Name, &Self::Filesystem)> {
        self.thin_pool
            .get_filesystem_by_uuid(uuid)
            .map(|(name, fs)| (name, fs))
    }

    fn get_filesystem_by_name(
        &self,
        fs_name: &Name,
    ) -> Option<(FilesystemUuid, &Self::Filesystem)> {
        self.thin_pool
            .get_filesystem_by_name(fs_name)
            .map(|(uuid, fs)| (uuid, fs))
    }

    fn blockdevs(&self) -> Vec<(DevUuid, BlockDevTier, &Self::BlockDev)> {
        self.backstore.blockdevs()
    }

    fn get_blockdev(&self, uuid: DevUuid) -> Option<(BlockDevTier, &Self::BlockDev)> {
        self.get_strat_blockdev(uuid)
    }

    #[pool_mutating_action("NoRequests")]
    fn get_mut_blockdev(
        &mut self,
        uuid: DevUuid,
    ) -> StratisResult<Option<(BlockDevTier, &mut Self::BlockDev)>> {
        self.get_mut_strat_blockdev(uuid)
    }

    #[pool_mutating_action("NoRequests")]
    fn set_blockdev_user_info(
        &mut self,
        pool_name: &str,
        uuid: DevUuid,
        user_info: Option<&str>,
    ) -> StratisResult<RenameAction<DevUuid>> {
        let result = self.backstore.set_blockdev_user_info(uuid, user_info);
        match result {
            Ok(Some(uuid)) => {
                self.write_metadata(pool_name)?;
                Ok(RenameAction::Renamed(uuid))
            }
            Ok(None) => Ok(RenameAction::Identity),
            Err(_) => Ok(RenameAction::NoSource),
        }
    }

    fn has_cache(&self) -> bool {
        self.backstore.has_cache()
    }

    fn is_encrypted(&self) -> bool {
        self.datadevs_encrypted()
    }

    fn encryption_info(&self) -> Option<PoolEncryptionInfo> {
        self.backstore.data_tier_encryption_info()
    }

    fn avail_actions(&self) -> ActionAvailability {
        self.action_avail.clone()
    }

    fn fs_limit(&self) -> u64 {
        self.thin_pool.fs_limit()
    }

    #[pool_mutating_action("NoPoolChanges")]
    fn set_fs_limit(
        &mut self,
        pool_name: &Name,
        pool_uuid: PoolUuid,
        new_limit: u64,
    ) -> StratisResult<()> {
        let (should_save, res) =
            self.thin_pool
                .set_fs_limit(pool_uuid, &mut self.backstore, new_limit);
        if should_save {
            self.write_metadata(pool_name)?;
        }
        res
    }

    fn overprov_enabled(&self) -> bool {
        self.thin_pool.overprov_enabled()
    }

    #[pool_mutating_action("NoPoolChanges")]
    fn set_overprov_mode(&mut self, pool_name: &Name, enabled: bool) -> StratisResult<()> {
        let (should_save, res) = self.thin_pool.set_overprov_mode(&self.backstore, enabled);
        if should_save {
            self.write_metadata(pool_name)?;
        }
        res
    }

    fn out_of_alloc_space(&self) -> bool {
        self.thin_pool.out_of_alloc_space()
    }

    #[pool_mutating_action("NoRequests")]
    fn grow_physical(
        &mut self,
        name: &Name,
        pool_uuid: PoolUuid,
        device: DevUuid,
    ) -> StratisResult<GrowAction<(PoolUuid, DevUuid)>> {
        let changed = self.backstore.grow(device)?;
        if changed {
            if self.thin_pool.set_queue_mode() {
                self.write_metadata(name)?;
            }
            Ok(GrowAction::Grown((pool_uuid, device)))
        } else {
            Ok(GrowAction::Identity)
        }
    }
}

pub struct StratPoolState {
    metadata_size: Bytes,
    out_of_alloc_space: bool,
}

impl StateDiff for StratPoolState {
    type Diff = StratPoolDiff;

    fn diff(&self, other: &Self) -> Self::Diff {
        StratPoolDiff {
            metadata_size: self.metadata_size.compare(&other.metadata_size),
            out_of_alloc_space: self.out_of_alloc_space.compare(&other.out_of_alloc_space),
        }
    }
}

impl<'a> DumpState<'a> for StratPool {
    type State = StratPoolState;
    type DumpInput = ();

    fn cached(&self) -> Self::State {
        StratPoolState {
            metadata_size: self.metadata_size.bytes(),
            out_of_alloc_space: self.thin_pool.out_of_alloc_space(),
        }
    }

    fn dump(&mut self, _: ()) -> Self::State {
        self.metadata_size = self.backstore.datatier_metadata_size();
        StratPoolState {
            metadata_size: self.metadata_size.bytes(),
            out_of_alloc_space: self.thin_pool.out_of_alloc_space(),
        }
    }
}

#[cfg(test)]
mod tests {
    use std::{
        fs::OpenOptions,
        io::{BufWriter, Read, Write},
    };

    use nix::mount::{mount, umount, MsFlags};

    use devicemapper::{Bytes, IEC, SECTOR_SIZE};

    use crate::engine::{
        engine::{BlockDev, Filesystem},
        strat_engine::{
            cmd::udev_settle,
            tests::{loopbacked, real},
            thinpool::ThinPoolStatusDigest,
        },
        types::{EngineAction, LockKey},
        Engine, StratEngine,
    };

    use super::*;

    fn invariant(pool: &StratPool, pool_name: &str) {
        check_metadata(&pool.record(&Name::new(pool_name.into()))).unwrap();
        assert!(!(pool.is_encrypted() && pool.backstore.has_cache()));
        if pool.avail_actions() == ActionAvailability::NoRequests {
            assert!(
                pool.encryption_info().is_some()
                    && pool
                        .encryption_info()
                        .map(|ei| { ei.is_inconsistent() })
                        .unwrap_or(false)
            );
        } else if pool.avail_actions() == ActionAvailability::Full {
            assert!(!pool
                .encryption_info()
                .map(|ei| ei.is_inconsistent())
                .unwrap_or(false));
        }
        assert!(pool
            .backstore
            .blockdevs()
            .iter()
            .all(|(_, _, bd)| bd.metadata_path().is_absolute()))
    }

    /// Test that initializing a cache causes metadata to be updated. Verify
    /// that data written before the cache was initialized can be read
    /// afterwards.
    fn test_add_cachedevs(paths: &[&Path]) {
        assert!(paths.len() > 1);

        let (paths1, paths2) = paths.split_at(paths.len() / 2);

        let devices2 = ProcessedPathInfos::try_from(paths2).unwrap();
        let (stratis_devices, unowned_devices2) = devices2.unpack();
        stratis_devices.error_on_not_empty().unwrap();

        let name = "stratis-test-pool";
        let (uuid, mut pool) = StratPool::initialize(name, unowned_devices2, None).unwrap();
        invariant(&pool, name);

        let metadata1 = pool.record(name);
        assert_matches!(metadata1.backstore.cache_tier, None);

        let (_, fs_uuid, _) = pool
            .create_filesystems(name, uuid, &[("stratis-filesystem", None)])
            .unwrap()
            .changed()
            .and_then(|mut fs| fs.pop())
            .unwrap();
        invariant(&pool, name);

        let tmp_dir = tempfile::Builder::new()
            .prefix("stratis_testing")
            .tempdir()
            .unwrap();
        let new_file = tmp_dir.path().join("stratis_test.txt");
        let bytestring = b"some bytes";
        {
            let (_, fs) = pool.get_filesystem(fs_uuid).unwrap();
            mount(
                Some(&fs.devnode()),
                tmp_dir.path(),
                Some("xfs"),
                MsFlags::empty(),
                None as Option<&str>,
            )
            .unwrap();
            OpenOptions::new()
                .create(true)
                .write(true)
                .open(&new_file)
                .unwrap()
                .write_all(bytestring)
                .unwrap();
        }

        pool.init_cache(uuid, name, paths1).unwrap();
        invariant(&pool, name);

        let metadata2 = pool.record(name);
        assert!(metadata2.backstore.cache_tier.is_some());

        let mut buf = [0u8; 10];
        {
            OpenOptions::new()
                .read(true)
                .open(&new_file)
                .unwrap()
                .read_exact(&mut buf)
                .unwrap();
        }
        assert_eq!(&buf, bytestring);
        umount(tmp_dir.path()).unwrap();
        pool.teardown().unwrap();
    }

    #[test]
    fn loop_test_add_cachedevs() {
        loopbacked::test_with_spec(
            &loopbacked::DeviceLimits::Range(3, 4, None),
            test_add_cachedevs,
        );
    }

    #[test]
    fn real_test_add_cachedevs() {
        real::test_with_spec(
            &real::DeviceLimits::AtLeast(2, None, None),
            test_add_cachedevs,
        );
    }

    // Verify that it is possible to add datadevs after a cache is initialized.
    fn test_add_cachedevs_and_datadevs(paths: &[&Path]) {
        assert!(paths.len() > 2);

        let (cache_path, data_paths) = paths.split_at(1);
        let (data_path, data_paths) = data_paths.split_at(1);

        let devices = ProcessedPathInfos::try_from(data_path).unwrap();
        let (stratis_devices, unowned_devices) = devices.unpack();
        stratis_devices.error_on_not_empty().unwrap();

        let name = "stratis-test-pool";
        let (uuid, mut pool) = StratPool::initialize(name, unowned_devices, None).unwrap();
        invariant(&pool, name);

        pool.init_cache(uuid, name, cache_path).unwrap();
        invariant(&pool, name);

        pool.add_blockdevs(uuid, name, data_paths, BlockDevTier::Data)
            .unwrap();

        pool.teardown().unwrap();
    }

    #[test]
    fn loop_test_add_cachedevs_and_datadevs() {
        loopbacked::test_with_spec(
            &loopbacked::DeviceLimits::Range(3, 4, None),
            test_add_cachedevs_and_datadevs,
        );
    }

    #[test]
    fn real_test_add_cachedevs_and_datadevs() {
        real::test_with_spec(
            &real::DeviceLimits::AtLeast(3, None, None),
            test_add_cachedevs_and_datadevs,
        );
    }

    /// Verify that adding additional blockdevs will cause a pool that is
    /// out of space to be extended.
    fn test_add_datadevs(paths: &[&Path]) {
        assert!(paths.len() > 1);

        let (paths1, paths2) = paths.split_at(1);

        let devices1 = ProcessedPathInfos::try_from(paths1).unwrap();
        let (stratis_devices, unowned_devices1) = devices1.unpack();
        stratis_devices.error_on_not_empty().unwrap();

        let name = "stratis-test-pool";
        let (pool_uuid, mut pool) = StratPool::initialize(name, unowned_devices1, None).unwrap();
        invariant(&pool, name);

        let fs_name = "stratis_test_filesystem";
        let (_, fs_uuid, _) = pool
            .create_filesystems(name, pool_uuid, &[(fs_name, None)])
            .unwrap()
            .changed()
            .and_then(|mut fs| fs.pop())
            .expect("just created one");

        let devnode = pool.get_filesystem(fs_uuid).unwrap().1.devnode();

        {
            let buffer_length = IEC::Mi;
            let mut f = BufWriter::with_capacity(
                convert_test!(buffer_length, u64, usize),
                OpenOptions::new().write(true).open(devnode).unwrap(),
            );

            let buf = &[1u8; SECTOR_SIZE];

            let mut amount_written = Sectors(0);
            let buffer_length = Bytes::from(buffer_length).sectors();
            while matches!(pool.thin_pool.state(), Some(ThinPoolStatusDigest::Good)) {
                f.write_all(buf).unwrap();
                amount_written += Sectors(1);
                // Run check roughly every time the buffer is cleared.
                // Running it more often is pointless as the pool is guaranteed
                // not to see any effects unless the buffer is cleared.
                if amount_written % buffer_length == Sectors(1) {
                    pool.event_on(pool_uuid, &Name::new(name.to_string()))
                        .unwrap();
                }
            }

            pool.add_blockdevs(pool_uuid, name, paths2, BlockDevTier::Data)
                .unwrap();

            match pool.thin_pool.state() {
                Some(ThinPoolStatusDigest::Good) => (),
                _ => panic!("thin pool status should be back to working"),
            }
        }
    }

    #[test]
    fn loop_test_add_datadevs() {
        loopbacked::test_with_spec(
            &loopbacked::DeviceLimits::Range(2, 3, Some(Bytes::from(IEC::Gi * 4).sectors())),
            test_add_datadevs,
        );
    }

    #[test]
    fn real_test_add_datadevs() {
        real::test_with_spec(
            &real::DeviceLimits::AtLeast(
                2,
                Some(Bytes::from(IEC::Gi * 2).sectors()),
                Some(Bytes::from(IEC::Gi * 4).sectors()),
            ),
            test_add_datadevs,
        );
    }

    /// Test that rollback errors are properly detected an maintenance mode
    /// is set accordingly.
    fn test_maintenance_mode(paths: &[&Path]) {
        assert!(paths.len() > 1);

        let name = "stratis-test-pool";

        let devices = ProcessedPathInfos::try_from(paths).unwrap();
        let (stratis_devices, unowned_devices) = devices.unpack();
        stratis_devices.error_on_not_empty().unwrap();

        let (_, mut pool) = StratPool::initialize(name, unowned_devices, None).unwrap();
        invariant(&pool, name);

        assert_eq!(pool.action_avail, ActionAvailability::Full);
        assert!(pool.return_rollback_failure().is_err());
        assert_eq!(pool.action_avail, ActionAvailability::NoRequests);

        pool.destroy().unwrap();
        udev_settle().unwrap();

        let name = "stratis-test-pool";

        let devices = ProcessedPathInfos::try_from(paths).unwrap();
        let (stratis_devices, unowned_devices) = devices.unpack();
        stratis_devices.error_on_not_empty().unwrap();

        let (_, mut pool) = StratPool::initialize(name, unowned_devices, None).unwrap();
        invariant(&pool, name);

        assert_eq!(pool.action_avail, ActionAvailability::Full);
        assert!(pool.return_rollback_failure_chain().is_err());
        assert_eq!(pool.action_avail, ActionAvailability::NoRequests);
    }

    #[test]
    fn loop_test_maintenance_mode() {
        loopbacked::test_with_spec(
            &loopbacked::DeviceLimits::Range(2, 3, Some(Bytes::from(IEC::Gi * 4).sectors())),
            test_maintenance_mode,
        );
    }

    #[test]
    fn real_test_mainenance_mode() {
        real::test_with_spec(
            &real::DeviceLimits::AtLeast(
                2,
                Some(Bytes::from(IEC::Gi * 2).sectors()),
                Some(Bytes::from(IEC::Gi * 4).sectors()),
            ),
            test_maintenance_mode,
        );
    }

    /// Test overprovisioning mode disabled and enabled assuring that the appropriate
    /// checks and behavior are in place.
    fn test_overprov(paths: &[&Path]) {
        assert!(paths.len() == 1);

        let pool_name = "pool";

        let devices = ProcessedPathInfos::try_from(paths).unwrap();
        let (stratis_devices, unowned_devices) = devices.unpack();
        stratis_devices.error_on_not_empty().unwrap();

        let (pool_uuid, mut pool) =
            StratPool::initialize(pool_name, unowned_devices, None).unwrap();

        let (_, fs_uuid, _) = pool
            .create_filesystems(
                pool_name,
                pool_uuid,
                &[(
                    "stratis_test_filesystem",
                    Some(pool.backstore.datatier_usable_size().bytes() * 2u64),
                )],
            )
            .unwrap()
            .changed()
            .unwrap()
            .pop()
            .unwrap();
        udev_settle().unwrap();
        assert!(pool
            .set_overprov_mode(&Name::new(pool_name.to_string()), false)
            .is_err());
        pool.destroy_filesystems(pool_name, &[fs_uuid]).unwrap();

        pool.set_overprov_mode(&Name::new(pool_name.to_string()), false)
            .unwrap();
        assert!(pool
            .create_filesystems(
                pool_name,
                pool_uuid,
                &[(
                    "stratis_test_filesystem",
                    Some(pool.backstore.datatier_usable_size().bytes() * 2u64)
                )],
            )
            .is_err());

        let mut initial_fs_size = pool.backstore.datatier_usable_size().bytes() * 2u64 / 3u64;
        initial_fs_size = initial_fs_size.sectors().bytes();
        let half_init_size = initial_fs_size / 2u64 + Bytes(1);
        let (_, fs_uuid, _) = pool
            .create_filesystems(
                pool_name,
                pool_uuid,
                &[("stratis_test_filesystem", Some(initial_fs_size))],
            )
            .unwrap()
            .changed()
            .unwrap()
            .pop()
            .unwrap();

        let tmp_dir = tempfile::Builder::new()
            .prefix("stratis_testing")
            .tempdir()
            .unwrap();
        let new_file = tmp_dir.path().join("stratis_test.txt");
        let sector = &[0; 512];

        {
            let (_, fs) = pool.get_filesystem(fs_uuid).unwrap();
            mount(
                Some(&fs.devnode()),
                tmp_dir.path(),
                Some("xfs"),
                MsFlags::empty(),
                None as Option<&str>,
            )
            .unwrap();
        }

        let mut f = OpenOptions::new()
            .create(true)
            .write(true)
            .open(&new_file)
            .unwrap();
        let mut written = Sectors(0);
        while written.bytes() < half_init_size {
            f.write_all(sector).unwrap();
            written += Sectors(1);
        }
        let diffs = pool.fs_event_on(pool_uuid).unwrap();
        assert!(diffs.get(&fs_uuid).unwrap().size.is_changed());

        let (_, fs) = pool.get_filesystem(fs_uuid).unwrap();
        assert!(fs.thindev_size() < initial_fs_size.sectors() * 2u64);
    }

    #[test]
    fn loop_test_overprov() {
        loopbacked::test_with_spec(
            &loopbacked::DeviceLimits::Exactly(1, Some(Sectors(10 * IEC::Mi))),
            test_overprov,
        );
    }

    #[test]
    fn real_test_overprov() {
        real::test_with_spec(
            &real::DeviceLimits::Exactly(1, Some(Sectors(10 * IEC::Mi)), None),
            test_overprov,
        );
    }

    /// Set up for testing physical device growth.
    fn test_grow_physical_pre_grow(paths: &[&Path]) {
        let pool_name = Name::new("pool".to_string());
        let engine = StratEngine::initialize().unwrap();
        let pool_uuid = test_async!(engine.create_pool(&*pool_name, paths, None))
            .unwrap()
            .changed()
            .unwrap();
        let mut guard = test_async!(engine.get_mut_pool(LockKey::Uuid(pool_uuid))).unwrap();
        let (_, _, pool) = guard.as_mut_tuple();

        let (_, fs_uuid, _) = pool
            .create_filesystems(&*pool_name, pool_uuid, &[("stratis_test_filesystem", None)])
            .unwrap()
            .changed()
            .unwrap()
            .pop()
            .unwrap();

        let tmp_dir = tempfile::Builder::new()
            .prefix("stratis_testing")
            .tempdir()
            .unwrap();
        let new_file = tmp_dir.path().join("stratis_test.txt");
        let write_block = &[0; 512_000];

        {
            let (_, fs) = pool.get_filesystem(fs_uuid).unwrap();
            mount(
                Some(&fs.devnode()),
                tmp_dir.path(),
                Some("xfs"),
                MsFlags::empty(),
                None as Option<&str>,
            )
            .unwrap();
        }

        let mut f = OpenOptions::new()
            .create(true)
            .write(true)
            .open(&new_file)
            .unwrap();
        while !pool.out_of_alloc_space() {
            f.write_all(write_block).unwrap();
            f.sync_all().unwrap();
            pool.event_on(pool_uuid, &pool_name).unwrap();
        }
    }

    /// Test that growing a physical device succeeds, the device has doubled in size,
    /// and that the pool registers new available allocation space if it is out of space
    /// at the time of device growth.
    fn test_grow_physical_post_grow(_: &[&Path]) {
        let engine = StratEngine::initialize().unwrap();

        let mut pools = test_async!(engine.pools_mut());
        assert!(pools.len() == 1);
        let (pool_name, pool_uuid, pool) = pools.iter_mut().next().unwrap();

        let (dev_uuid, size) = {
            let blockdevs = pool.blockdevs();
            let (dev_uuid, _, dev) = blockdevs.first().unwrap();
            (*dev_uuid, dev.size())
        };

        assert!(pool.out_of_alloc_space());
        pool.grow_physical(pool_name, *pool_uuid, dev_uuid)
            .unwrap()
            .changed()
            .unwrap();
        let (_, dev) = pool.get_blockdev(dev_uuid).unwrap();
        assert_eq!(dev.size(), 2u64 * size);
        assert!(!pool.out_of_alloc_space());
    }

    #[test]
    fn loop_test_grow_physical() {
        loopbacked::test_device_grow_with_spec(
            &loopbacked::DeviceLimits::Exactly(2, Some(Sectors(10 * IEC::Mi))),
            test_grow_physical_pre_grow,
            test_grow_physical_post_grow,
        );
    }
}

<MSG> Make initial sizes *Blocks instead of Sectors

The more specfic type should make things clearer.

Signed-off-by: mulhern <7b51bcf507bcd7afb72bf8663752c0ddbeb517f6@redhat.com>

<DFF> @@ -15,7 +15,7 @@ use uuid::Uuid;
 use devicemapper::consts::SECTOR_SIZE;
 use devicemapper::Device;
 use devicemapper::DM;
-use devicemapper::{Sectors, Segment};
+use devicemapper::{DataBlocks, MetaBlocks, Sectors, Segment};
 use devicemapper::LinearDev;
 use devicemapper::{ThinDevId, ThinPoolStatus, ThinPoolWorkingStatus};
 
@@ -37,8 +37,8 @@ use super::thinpool::{META_LOWATER, ThinPool};
 
 pub use super::thinpool::{DATA_BLOCK_SIZE, DATA_LOWATER};
 
-const INITIAL_META_SIZE: Sectors = Sectors(16 * Mi / SECTOR_SIZE as u64);
-pub const INITIAL_DATA_SIZE: Sectors = Sectors(768 * Mi / SECTOR_SIZE as u64);
+const INITIAL_META_SIZE: MetaBlocks = MetaBlocks(4096);
+pub const INITIAL_DATA_SIZE: DataBlocks = DataBlocks(768);
 const INITIAL_MDV_SIZE: Sectors = Sectors(16 * Mi / SECTOR_SIZE as u64);
 
 #[derive(Debug)]
@@ -83,15 +83,15 @@ impl StratPool {
         }
 
         let meta_regions = block_mgr
-            .alloc_space(INITIAL_META_SIZE)
+            .alloc_space(INITIAL_META_SIZE.sectors())
             .expect("blockmgr must not fail, already checked for space");
 
         let meta_spare_regions = block_mgr
-            .alloc_space(INITIAL_META_SIZE)
+            .alloc_space(INITIAL_META_SIZE.sectors())
             .expect("blockmgr must not fail, already checked for space");
 
         let data_regions = block_mgr
-            .alloc_space(INITIAL_DATA_SIZE)
+            .alloc_space(*INITIAL_DATA_SIZE * DATA_BLOCK_SIZE)
             .expect("blockmgr must not fail, already checked for space");
 
         // When constructing a thin-pool, Stratis reserves the first N
@@ -105,7 +105,9 @@ impl StratPool {
         let meta_dev = try!(LinearDev::new(&format_flex_name(&pool_uuid, FlexRole::ThinMeta),
                                            dm,
                                            meta_regions));
-        try!(wipe_sectors(&try!(meta_dev.devnode()), Sectors(0), INITIAL_META_SIZE));
+        try!(wipe_sectors(&try!(meta_dev.devnode()),
+                          Sectors(0),
+                          INITIAL_META_SIZE.sectors()));
 
         let data_dev = try!(LinearDev::new(&format_flex_name(&pool_uuid, FlexRole::ThinData),
                                            dm,
@@ -272,7 +274,8 @@ impl StratPool {
     /// Minimum initial size for a pool.
     pub fn min_initial_size() -> Sectors {
         // One extra meta for spare
-        (INITIAL_META_SIZE * 2u64) + INITIAL_DATA_SIZE + INITIAL_MDV_SIZE
+        (INITIAL_META_SIZE.sectors() * 2u64) + *INITIAL_DATA_SIZE * DATA_BLOCK_SIZE +
+        INITIAL_MDV_SIZE
     }
 
     /// Write current metadata to pool members.

{'commit': 'diff --git a/src/engine/strat_engine/pool.rs b/src/engine/strat_engine/pool.rs\nindex fc58ad22..be7e4e67 100644\n--- a/src/engine/strat_engine/pool.rs\n+++ b/src/engine/strat_engine/pool.rs\n@@ -15,7 +15,7 @@ use uuid::Uuid;\n use devicemapper::consts::SECTOR_SIZE;\n use devicemapper::Device;\n use devicemapper::DM;\n-use devicemapper::{Sectors, Segment};\n+use devicemapper::{DataBlocks, MetaBlocks, Sectors, Segment};\n use devicemapper::LinearDev;\n use devicemapper::{ThinDevId, ThinPoolStatus, ThinPoolWorkingStatus};\n \n@@ -37,8 +37,8 @@ use super::thinpool::{META_LOWATER, ThinPool};\n \n pub use super::thinpool::{DATA_BLOCK_SIZE, DATA_LOWATER};\n \n-const INITIAL_META_SIZE: Sectors = Sectors(16 * Mi / SECTOR_SIZE as u64);\n-pub const INITIAL_DATA_SIZE: Sectors = Sectors(768 * Mi / SECTOR_SIZE as u64);\n+const INITIAL_META_SIZE: MetaBlocks = MetaBlocks(4096);\n+pub const INITIAL_DATA_SIZE: DataBlocks = DataBlocks(768);\n const INITIAL_MDV_SIZE: Sectors = Sectors(16 * Mi / SECTOR_SIZE as u64);\n \n #[derive(Debug)]\n@@ -83,15 +83,15 @@ impl StratPool {\n         }\n \n         let meta_regions = block_mgr\n-            .alloc_space(INITIAL_META_SIZE)\n+            .alloc_space(INITIAL_META_SIZE.sectors())\n             .expect("blockmgr must not fail, already checked for space");\n \n         let meta_spare_regions = block_mgr\n-            .alloc_space(INITIAL_META_SIZE)\n+            .alloc_space(INITIAL_META_SIZE.sectors())\n             .expect("blockmgr must not fail, already checked for space");\n \n         let data_regions = block_mgr\n-            .alloc_space(INITIAL_DATA_SIZE)\n+            .alloc_space(*INITIAL_DATA_SIZE * DATA_BLOCK_SIZE)\n             .expect("blockmgr must not fail, already checked for space");\n \n         // When constructing a thin-pool, Stratis reserves the first N\n@@ -105,7 +105,9 @@ impl StratPool {\n         let meta_dev = try!(LinearDev::new(&format_flex_name(&pool_uuid, FlexRole::ThinMeta),\n                                            dm,\n                                            meta_regions));\n-        try!(wipe_sectors(&try!(meta_dev.devnode()), Sectors(0), INITIAL_META_SIZE));\n+        try!(wipe_sectors(&try!(meta_dev.devnode()),\n+                          Sectors(0),\n+                          INITIAL_META_SIZE.sectors()));\n \n         let data_dev = try!(LinearDev::new(&format_flex_name(&pool_uuid, FlexRole::ThinData),\n                                            dm,\n@@ -272,7 +274,8 @@ impl StratPool {\n     /// Minimum initial size for a pool.\n     pub fn min_initial_size() -> Sectors {\n         // One extra meta for spare\n-        (INITIAL_META_SIZE * 2u64) + INITIAL_DATA_SIZE + INITIAL_MDV_SIZE\n+        (INITIAL_META_SIZE.sectors() * 2u64) + *INITIAL_DATA_SIZE * DATA_BLOCK_SIZE +\n+        INITIAL_MDV_SIZE\n     }\n \n     /// Write current metadata to pool members.\ndiff --git a/tests/util/pool_tests.rs b/tests/util/pool_tests.rs\nindex e8c40285..f171c675 100644\n--- a/tests/util/pool_tests.rs\n+++ b/tests/util/pool_tests.rs\n@@ -45,7 +45,7 @@ pub fn test_thinpool_expand(paths: &[&Path]) -> () {\n     {\n         let mut f = OpenOptions::new().write(true).open(devnode).unwrap();\n         // Write 1 more sector than is initially allocated to a pool\n-        let write_size = INITIAL_DATA_SIZE + Sectors(1);\n+        let write_size = *INITIAL_DATA_SIZE * DATA_BLOCK_SIZE + Sectors(1);\n         let buf = &[1u8; SECTOR_SIZE];\n         for i in 0..*write_size {\n             f.write_all(buf).unwrap();\n@@ -53,7 +53,7 @@ pub fn test_thinpool_expand(paths: &[&Path]) -> () {\n             // the amount of free space in pool has decreased to the DATA_LOWATER value.\n             // TODO: Actually handle DM events and possibly call extend() directly,\n             // depending on the specificity of the events.\n-            if i == *(INITIAL_DATA_SIZE - Sectors(*DATA_LOWATER * *DATA_BLOCK_SIZE)) {\n+            if i == *(*(INITIAL_DATA_SIZE - DATA_LOWATER) * DATA_BLOCK_SIZE) {\n                 pool.check();\n             }\n         }\n', 'commit_subject': 'Make initial sizes *Blocks instead of Sectors', 'repo_name': 'stratis-storage/stratisd', 'committer': {'name': 'mulhern', 'email': '7b51bcf507bcd7afb72bf8663752c0ddbeb517f6@redhat.com', 'time_sec': '1500406335', 'tz_offset': '-240', 'date': {'seconds': '1500406335'}}, 'commit_tree': '8c346194b64f69db0b51bd9d5b62165cd24becd9', 'commit_message': 'Make initial sizes *Blocks instead of Sectors\n\nThe more specfic type should make things clearer.\n\nSigned-off-by: mulhern <7b51bcf507bcd7afb72bf8663752c0ddbeb517f6@redhat.com>\n', 'language_name': 'Python', 'license': 'mpl-2.0', 'hunks': '@@ -15,7 +15,7 @@ use uuid::Uuid;\n use devicemapper::consts::SECTOR_SIZE;\n use devicemapper::Device;\n use devicemapper::DM;\n-use devicemapper::{Sectors, Segment};\n+use devicemapper::{DataBlocks, MetaBlocks, Sectors, Segment};\n use devicemapper::LinearDev;\n use devicemapper::{ThinDevId, ThinPoolStatus, ThinPoolWorkingStatus};\n \n@@ -37,8 +37,8 @@ use super::thinpool::{META_LOWATER, ThinPool};\n \n pub use super::thinpool::{DATA_BLOCK_SIZE, DATA_LOWATER};\n \n-const INITIAL_META_SIZE: Sectors = Sectors(16 * Mi / SECTOR_SIZE as u64);\n-pub const INITIAL_DATA_SIZE: Sectors = Sectors(768 * Mi / SECTOR_SIZE as u64);\n+const INITIAL_META_SIZE: MetaBlocks = MetaBlocks(4096);\n+pub const INITIAL_DATA_SIZE: DataBlocks = DataBlocks(768);\n const INITIAL_MDV_SIZE: Sectors = Sectors(16 * Mi / SECTOR_SIZE as u64);\n \n #[derive(Debug)]\n@@ -83,15 +83,15 @@ impl StratPool {\n         }\n \n         let meta_regions = block_mgr\n-            .alloc_space(INITIAL_META_SIZE)\n+            .alloc_space(INITIAL_META_SIZE.sectors())\n             .expect("blockmgr must not fail, already checked for space");\n \n         let meta_spare_regions = block_mgr\n-            .alloc_space(INITIAL_META_SIZE)\n+            .alloc_space(INITIAL_META_SIZE.sectors())\n             .expect("blockmgr must not fail, already checked for space");\n \n         let data_regions = block_mgr\n-            .alloc_space(INITIAL_DATA_SIZE)\n+            .alloc_space(*INITIAL_DATA_SIZE * DATA_BLOCK_SIZE)\n             .expect("blockmgr must not fail, already checked for space");\n \n         // When constructing a thin-pool, Stratis reserves the first N\n@@ -105,7 +105,9 @@ impl StratPool {\n         let meta_dev = try!(LinearDev::new(&format_flex_name(&pool_uuid, FlexRole::ThinMeta),\n                                            dm,\n                                            meta_regions));\n-        try!(wipe_sectors(&try!(meta_dev.devnode()), Sectors(0), INITIAL_META_SIZE));\n+        try!(wipe_sectors(&try!(meta_dev.devnode()),\n+                          Sectors(0),\n+                          INITIAL_META_SIZE.sectors()));\n \n         let data_dev = try!(LinearDev::new(&format_flex_name(&pool_uuid, FlexRole::ThinData),\n                                            dm,\n@@ -272,7 +274,8 @@ impl StratPool {\n     /// Minimum initial size for a pool.\n     pub fn min_initial_size() -> Sectors {\n         // One extra meta for spare\n-        (INITIAL_META_SIZE * 2u64) + INITIAL_DATA_SIZE + INITIAL_MDV_SIZE\n+        (INITIAL_META_SIZE.sectors() * 2u64) + *INITIAL_DATA_SIZE * DATA_BLOCK_SIZE +\n+        INITIAL_MDV_SIZE\n     }\n \n     /// Write current metadata to pool members.\n', 'addition_count': 11, 'deletion_count': 8, 'src_file': 'a/src/engine/strat_engine/pool.rs', 'tgt_file': 'b/src/engine/strat_engine/pool.rs', 'file_extension': '.rs', 'before_file': '// This Source Code Form is subject to the terms of the Mozilla Public\n// License, v. 2.0. If a copy of the MPL was not distributed with this\n// file, You can obtain one at http://mozilla.org/MPL/2.0/.\n\nuse std::{collections::HashMap, path::Path, vec::Vec};\n\nuse chrono::{DateTime, Utc};\nuse serde_json::{Map, Value};\n\nuse devicemapper::{Bytes, DmNameBuf, Sectors};\nuse stratisd_proc_macros::strat_pool_impl_gen;\n\nuse crate::{\n    engine::{\nuse devicemapper::consts::SECTOR_SIZE;\nuse devicemapper::Device;\nuse devicemapper::DM;\nuse devicemapper::{Sectors, Segment};\nuse devicemapper::LinearDev;\nuse devicemapper::{ThinDevId, ThinPoolStatus, ThinPoolWorkingStatus};\n\n            liminal::{DeviceInfo, DeviceSet, LInfo},\n            metadata::MDADataSize,\n            serde_structs::{FlexDevsSave, PoolSave, Recordable},\n            thinpool::{StratFilesystem, ThinPool, ThinPoolSizeParams, DATA_BLOCK_SIZE},\n        },\n        types::{\n            ActionAvailability, BlockDevTier, Clevis, Compare, CreateAction, DeleteAction, DevUuid,\n            EncryptionInfo, FilesystemUuid, GrowAction, Key, KeyDescription, Name, PoolDiff,\n            PoolEncryptionInfo, PoolUuid, RegenAction, RenameAction, SetCreateAction,\n            SetDeleteAction, StratFilesystemDiff, StratPoolDiff,\n        },\n    },\n    stratis::{StratisError, StratisResult},\n};\n\n\npub use super::thinpool::{DATA_BLOCK_SIZE, DATA_LOWATER};\n\nconst INITIAL_META_SIZE: Sectors = Sectors(16 * Mi / SECTOR_SIZE as u64);\npub const INITIAL_DATA_SIZE: Sectors = Sectors(768 * Mi / SECTOR_SIZE as u64);\nconst INITIAL_MDV_SIZE: Sectors = Sectors(16 * Mi / SECTOR_SIZE as u64);\n\n#[derive(Debug)]\nfn next_index(flex_devs: &FlexDevsSave) -> Sectors {\n    let expect_msg = "Setting up rather than initializing a pool, so each flex dev must have been allocated at least some segments.";\n    [\n        flex_devs\n            .meta_dev\n            .last()\n            .unwrap_or_else(|| panic!("{}", expect_msg)),\n        flex_devs\n            .thin_meta_dev\n            .last()\n            .unwrap_or_else(|| panic!("{}", expect_msg)),\n        flex_devs\n            .thin_data_dev\n            .last()\n            .unwrap_or_else(|| panic!("{}", expect_msg)),\n        flex_devs\n            .thin_meta_dev_spare\n            .last()\n            .unwrap_or_else(|| panic!("{}", expect_msg)),\n    ]\n    .iter()\n    .max_by_key(|x| x.0)\n    .map(|&&(start, length)| start + length)\n    .expect("iterator is non-empty")\n}\n\n/// Check the metadata of an individual pool for consistency.\n/// Precondition: This method is called only when setting up a pool, which\n/// ensures that the flex devs metadata lists are all non-empty.\nfn check_metadata(metadata: &PoolSave) -> StratisResult<()> {\n    let flex_devs = &metadata.flex_devs;\n    let next = next_index(flex_devs);\n    let allocated_from_cap = metadata.backstore.cap.allocs[0].1;\n\n    if allocated_from_cap != next {\n        let err_msg = format!(\n            "{} used in thinpool, but {} allocated from backstore cap device",\n            next, allocated_from_cap\n        }\n\n        let meta_regions = block_mgr\n            .alloc_space(INITIAL_META_SIZE)\n            .expect("blockmgr must not fail, already checked for space");\n\n        let meta_spare_regions = block_mgr\n            .alloc_space(INITIAL_META_SIZE)\n            .expect("blockmgr must not fail, already checked for space");\n\n        let data_regions = block_mgr\n            .alloc_space(INITIAL_DATA_SIZE)\n            .expect("blockmgr must not fail, already checked for space");\n\n        // When constructing a thin-pool, Stratis reserves the first N\n        if total_allocated != next {\n            let err_msg = format!(\n                "{} used in thinpool, but {} given up by cache for pool {}",\n                total_allocated, next, metadata.name\n            );\n            return Err(StratisError::Msg(err_msg));\n        }\n        let meta_dev = try!(LinearDev::new(&format_flex_name(&pool_uuid, FlexRole::ThinMeta),\n                                           dm,\n                                           meta_regions));\n        try!(wipe_sectors(&try!(meta_dev.devnode()), Sectors(0), INITIAL_META_SIZE));\n\n        let data_dev = try!(LinearDev::new(&format_flex_name(&pool_uuid, FlexRole::ThinData),\n                                           dm,\n            .map(|x| x.length)\n            .sum::<Sectors>();\n\n        if total_allocated == Sectors(0) {\n            let err_msg = format!(\n                "no segments allocated to the cap device for pool {}",\n                metadata.name\n            );\n            return Err(StratisError::Msg(err_msg));\n        }\n\n        if next > total_allocated {\n            let err_msg = format!(\n                "{} allocated to cap device, but {} allocated to flex devs",\n                next, total_allocated\n            );\n            return Err(StratisError::Msg(err_msg));\n        }\n    }\n\n    Ok(())\n}\n\n#[derive(Debug)]\npub struct StratPool {\n    backstore: Backstore,\n    thin_pool: ThinPool,\n    action_avail: ActionAvailability,\n    metadata_size: Sectors,\n}\n\n#[strat_pool_impl_gen]\nimpl StratPool {\n    /// Initialize a Stratis Pool.\n    /// 1. Initialize the block devices specified by paths.\n    /// 2. Set up thinpool device to back filesystems.\n    /// Precondition: p.is_absolute() is true for all p in paths\n    pub fn initialize(\n        name: &str,\n        devices: UnownedDevices,\n        encryption_info: Option<&EncryptionInfo>,\n    ) -> StratisResult<(PoolUuid, StratPool)> {\n        let pool_uuid = PoolUuid::new_v4();\n\n        // FIXME: Initializing with the minimum MDA size is not necessarily\n        // enough. If there are enough devices specified, more space will be\n        // required.\n        let mut backstore =\n            Backstore::initialize(pool_uuid, devices, MDADataSize::default(), encryption_info)?;\n\n        let thinpool = ThinPool::new(\n            pool_uuid,\n            match ThinPoolSizeParams::new(backstore.datatier_usable_size()) {\n                Ok(ref params) => params,\n                Err(e) => {\n                    let _ = backstore.destroy();\n                    return Err(e);\n                }\n            },\n            DATA_BLOCK_SIZE,\n            &mut backstore,\n        );\n\n        let thinpool = match thinpool {\n            Ok(thinpool) => thinpool,\n            Err(err) => {\n                let _ = backstore.destroy();\n                return Err(err);\n            }\n        };\n\n        let metadata_size = backstore.datatier_metadata_size();\n        let mut pool = StratPool {\n            backstore,\n            thin_pool: thinpool,\n            action_avail: ActionAvailability::Full,\n            metadata_size,\n        };\n\n        pool.write_metadata(&Name::new(name.to_owned()))?;\n\n        Ok((pool_uuid, pool))\n    }\n\n    /// Setup a StratPool using its UUID and the list of devnodes it has.\n    /// Precondition: every device in devnodes has already been determined\n    /// to belong to the pool with the specified uuid.\n    /// Precondition: A metadata verification step has already been run.\n    ///\n    /// Precondition:\n    ///   * key_description.is_some() -> every StratBlockDev in datadevs has a\n    ///   key description and that key description == key_description\n    ///   * key_description.is_none() -> no StratBlockDev in datadevs has a\n    ///   key description.\n    ///   * no StratBlockDev in cachdevs has a key description\n    pub fn setup(\n        uuid: PoolUuid,\n        datadevs: Vec<StratBlockDev>,\n        cachedevs: Vec<StratBlockDev>,\n        timestamp: DateTime<Utc>,\n        metadata: &PoolSave,\n        action_avail: ActionAvailability,\n    ) -> StratisResult<(Name, StratPool)> {\n        check_metadata(metadata)?;\n\n        let mut backstore =\n            Backstore::setup(uuid, &metadata.backstore, datadevs, cachedevs, timestamp)?;\n        let pool_name = &metadata.name;\n\n        let mut thinpool = ThinPool::setup(\n            pool_name,\n            uuid,\n            &metadata.thinpool_dev,\n            &metadata.flex_devs,\n            &backstore,\n        )?;\n\n        // TODO: Remove in stratisd 4.0\n        let mut needs_save = metadata.thinpool_dev.fs_limit.is_none()\n            || metadata.thinpool_dev.feature_args.is_none();\n\n        needs_save |= thinpool.check(uuid, &mut backstore)?.0;\n\n        let metadata_size = backstore.datatier_metadata_size();\n        let mut pool = StratPool {\n            backstore,\n            thin_pool: thinpool,\n            action_avail,\n            metadata_size,\n        };\n\n        // Change the pool to started at this point not that the pool has been set up.\n        needs_save |= !metadata.started.unwrap_or(false);\n\n        if needs_save {\n            pool.write_metadata(pool_name)?;\n        }\n\n        Ok((Name::new(pool_name.to_owned()), pool))\n    }\n\n    /// Send a synthetic udev change event to every filesystem on the given pool.\n    pub fn udev_pool_change(&self, pool_name: &str) {\n        for (name, uuid, fs) in self.thin_pool.filesystems() {\n            fs.udev_fs_change(pool_name, uuid, &name);\n        }\n    }\n\n    /// Write current metadata to pool members.\n    #[pool_mutating_action("NoPoolChanges")]\n    pub fn write_metadata(&mut self, name: &str) -> StratisResult<()> {\n        let data = serde_json::to_string(&self.record(name))?;\n        self.backstore.save_state(data.as_bytes())\n    }\n\n    /// Teardown a pool.\n    #[cfg(test)]\n    pub fn teardown(&mut self) -> StratisResult<()> {\n        self.thin_pool.teardown()?;\n        self.backstore.teardown()\n    /// Minimum initial size for a pool.\n    pub fn min_initial_size() -> Sectors {\n        // One extra meta for spare\n        (INITIAL_META_SIZE * 2u64) + INITIAL_DATA_SIZE + INITIAL_MDV_SIZE\n    }\n\n    /// Write current metadata to pool members.\n        self.thin_pool.get_eventing_dev_names(pool_uuid)\n    }\n\n    /// Called when a DM device in this pool has generated an event. This method\n    /// handles checking pools.\n    #[pool_mutating_action("NoPoolChanges")]\n    pub fn event_on(&mut self, pool_uuid: PoolUuid, pool_name: &Name) -> StratisResult<PoolDiff> {\n        let cached = self.cached();\n        let (changed, thin_pool) = self.thin_pool.check(pool_uuid, &mut self.backstore)?;\n        let pool = cached.diff(&self.dump(()));\n        if changed {\n            self.write_metadata(pool_name)?;\n        }\n        Ok(PoolDiff { thin_pool, pool })\n    }\n\n    /// Called when a DM device in this pool has generated an event. This method\n    /// handles checking filesystems.\n    #[pool_mutating_action("NoPoolChanges")]\n    pub fn fs_event_on(\n        &mut self,\n        pool_uuid: PoolUuid,\n    ) -> StratisResult<HashMap<FilesystemUuid, StratFilesystemDiff>> {\n        self.thin_pool.check_fs(pool_uuid, &self.backstore)\n    }\n\n    pub fn record(&self, name: &str) -> PoolSave {\n        PoolSave {\n            name: name.to_owned(),\n            backstore: self.backstore.record(),\n            flex_devs: self.thin_pool.record(),\n            thinpool_dev: self.thin_pool.record(),\n            started: Some(true),\n        }\n    }\n\n    fn datadevs_encrypted(&self) -> bool {\n        self.backstore.data_tier_is_encrypted()\n    }\n\n    pub fn get_strat_blockdev(&self, uuid: DevUuid) -> Option<(BlockDevTier, &StratBlockDev)> {\n        self.backstore.get_blockdev_by_uuid(uuid)\n    }\n\n    #[pool_mutating_action("NoPoolChanges")]\n    pub fn get_mut_strat_blockdev(\n        &mut self,\n        uuid: DevUuid,\n    ) -> StratisResult<Option<(BlockDevTier, &mut StratBlockDev)>> {\n        Ok(self.backstore.get_mut_blockdev_by_uuid(uuid))\n    }\n\n    #[pool_mutating_action("NoPoolChanges")]\n    pub fn blockdevs_mut(\n        &mut self,\n    ) -> StratisResult<Vec<(DevUuid, BlockDevTier, &mut StratBlockDev)>> {\n        Ok(self.backstore.blockdevs_mut())\n    }\n\n    /// Destroy the pool.\n    /// Precondition: All filesystems belonging to this pool must be\n    /// unmounted.\n    ///\n    /// This method is not a mutating action as the pool should be allowed\n    /// to be destroyed even if the metadata is inconsistent.\n    pub fn destroy(&mut self) -> StratisResult<()> {\n        self.thin_pool.teardown()?;\n        self.backstore.destroy()?;\n        Ok(())\n    }\n\n    /// Check the limit of filesystems on a pool and return an error if it has been passed.\n    fn check_fs_limit(&self, new_fs: usize) -> StratisResult<()> {\n        let fs_limit = self.fs_limit();\n        if convert_int!(fs_limit, u64, usize)? < self.filesystems().len() + new_fs {\n            Err(StratisError::Msg(format!("The pool limit of {} filesystems has already been reached; increase the filesystem limit on the pool to continue", fs_limit)))\n        } else {\n            Ok(())\n        }\n    }\n\n    /// Stop a pool, consuming it and converting it into a set of devices to be\n    /// set up again later.\n    pub fn stop(&mut self, pool_name: &Name) -> StratisResult<DeviceSet> {\n        self.thin_pool.teardown()?;\n        let mut data = self.record(pool_name);\n        data.started = Some(false);\n        let json = serde_json::to_string(&data)?;\n        self.backstore.save_state(json.as_bytes())?;\n        self.backstore.teardown()?;\n        Ok(self\n            .backstore\n            .blockdevs()\n            .into_iter()\n            .map(|(uuid, _, bd)| (uuid, LInfo::from(DeviceInfo::from(bd))))\n            .collect::<DeviceSet>())\n    }\n\n    #[cfg(test)]\n    #[pool_mutating_action("NoRequests")]\n    #[pool_rollback]\n    pub fn return_rollback_failure(&mut self) -> StratisResult<()> {\n        Err(StratisError::RollbackError {\n            causal_error: Box::new(StratisError::Msg("Causal error".to_string())),\n            rollback_error: Box::new(StratisError::Msg("Rollback error".to_string())),\n            level: ActionAvailability::NoRequests,\n        })\n    }\n\n    #[cfg(test)]\n    #[pool_mutating_action("NoRequests")]\n    #[pool_rollback]\n    pub fn return_rollback_failure_chain(&mut self) -> StratisResult<()> {\n        Err(StratisError::Chained(\n            "Chained error".to_string(),\n            Box::new(StratisError::RollbackError {\n                causal_error: Box::new(StratisError::Msg("Causal error".to_string())),\n                rollback_error: Box::new(StratisError::Msg("Rollback error".to_string())),\n                level: ActionAvailability::NoRequests,\n            }),\n        ))\n    }\n\n    /// Verifies that the filesystem operation to be performed is allowed to perform\n    /// overprovisioning if it is determined to be the end result.\n    fn check_overprov(&self, increase: Sectors) -> StratisResult<()> {\n        if !self.thin_pool.overprov_enabled()\n            && self\n                .thin_pool\n                .filesystems()\n                .iter()\n                .map(|(_, _, fs)| fs.thindev_size())\n                .sum::<Sectors>()\n                + increase\n                > self.thin_pool.total_fs_limit(&self.backstore)\n        {\n            Err(StratisError::Msg(format!(\n                "Overprovisioning is disabled on this pool and increasing total filesystems size by {} would result in overprovisioning",\n                increase\n            )))\n        } else {\n            Ok(())\n        }\n    }\n}\n\nimpl<\'a> Into<Value> for &\'a StratPool {\n    // Precondition: (&ThinPool).into() pattern matches Value::Object(_)\n    // Precondition: (&Backstore).into() pattern matches Value::Object(_)\n    fn into(self) -> Value {\n        let mut map: Map<_, _> =\n            if let Value::Object(map) = <&ThinPool as Into<Value>>::into(&self.thin_pool) {\n                map.into_iter()\n            } else {\n                unreachable!("ThinPool conversion returns a JSON object")\n            }\n            .collect();\n        map.extend(\n            if let Value::Object(map) = <&Backstore as Into<Value>>::into(&self.backstore) {\n                map.into_iter()\n            } else {\n                unreachable!("Backstore conversion returns a JSON object")\n            },\n        );\n        map.insert(\n            "available_actions".to_string(),\n            Value::from(self.action_avail.to_string()),\n        );\n        map.insert("fs_limit".to_string(), Value::from(self.fs_limit()));\n        Value::from(map)\n    }\n}\n\n#[strat_pool_impl_gen]\nimpl Pool for StratPool {\n    type Filesystem = StratFilesystem;\n    type BlockDev = StratBlockDev;\n\n    #[pool_mutating_action("NoRequests")]\n    fn init_cache(\n        &mut self,\n        pool_uuid: PoolUuid,\n        pool_name: &str,\n        blockdevs: &[&Path],\n    ) -> StratisResult<SetCreateAction<DevUuid>> {\n        validate_paths(blockdevs)?;\n\n        if self.is_encrypted() {\n            return Err(StratisError::Msg(\n                "Use of a cache is not supported with an encrypted pool".to_string(),\n            ));\n        }\n\n        let devices = ProcessedPathInfos::try_from(blockdevs)?;\n        let (stratis_devices, unowned_devices) = devices.unpack();\n        let (this_pool, other_pools) = stratis_devices.partition(pool_uuid);\n\n        other_pools.error_on_not_empty()?;\n\n        let (in_pool, out_pool): (Vec<_>, Vec<_>) = this_pool\n            .iter()\n            .map(|(dev_uuid, _)| {\n                self.backstore\n                    .get_blockdev_by_uuid(*dev_uuid)\n                    .map(|(tier, _)| (*dev_uuid, tier))\n            })\n            .partition(|v| v.is_some());\n\n        if !out_pool.is_empty() {\n            let error_message = format!(\n                    "Devices ({}) appear to be already in use by this pool which has UUID {} but this pool has no record of them",\n                    out_pool\n                    .iter()\n                    .map(|opt| this_pool.get(&opt.expect("was looked up").0).expect("partitioned from this_pool").devnode.display().to_string())\n                    .collect::<Vec<_>>()\n                    .join(", "),\n                    pool_uuid\n                );\n            return Err(StratisError::Msg(error_message));\n        };\n\n        let (datadevs, cachedevs): (Vec<_>, Vec<_>) = in_pool\n            .iter()\n            .map(|opt| opt.expect("in_pool devices are Some"))\n            .partition(|(_, tier)| *tier == BlockDevTier::Data);\n\n        if !datadevs.is_empty() {\n            let error_message = format!(\n                "Devices ({}) appear to be already in use by this pool which has UUID {} in the data tier",\n                datadevs\n                    .iter()\n                    .map(|(uuid, _)| this_pool.get(uuid).expect("partitioned from this_pool").devnode.display().to_string())\n                    .collect::<Vec<_>>()\n                    .join(", "),\n                pool_uuid\n            );\n            return Err(StratisError::Msg(error_message));\n        };\n\n        if !self.has_cache() {\n            if unowned_devices.is_empty() {\n                return Err(StratisError::Msg(\n                    "At least one device is reqired to initialize a cache.".to_string(),\n                ));\n            }\n\n            self.thin_pool.suspend()?;\n            let devices_result = self\n                .backstore\n                .init_cache(pool_uuid, unowned_devices)\n                .and_then(|bdi| {\n                    self.thin_pool\n                        .set_device(self.backstore.device().expect(\n                            "Since thin pool exists, space must have been allocated \\\n                             from the backstore, so backstore must have a cap device",\n                        ))\n                        .and(Ok(bdi))\n                });\n            self.thin_pool.resume()?;\n            let devices = devices_result?;\n            self.write_metadata(pool_name)?;\n            Ok(SetCreateAction::new(devices))\n        } else {\n            init_cache_idempotent_or_err(\n                &cachedevs\n                    .iter()\n                    .map(|(uuid, _)| {\n                        this_pool\n                            .get(uuid)\n                            .expect("partitioned from this_pool")\n                            .devnode\n                            .as_path()\n                    })\n                    .chain(\n                        unowned_devices\n                            .unpack()\n                            .iter()\n                            .map(|info| info.devnode.as_path()),\n                    )\n                    .collect::<Vec<_>>(),\n                self.backstore\n                    .cachedevs()\n                    .into_iter()\n                    .map(|(_, bd)| bd.physical_path().to_owned()),\n            )\n        }\n    }\n\n    #[pool_mutating_action("NoRequests")]\n    #[pool_rollback]\n    fn bind_clevis(\n        &mut self,\n        pin: &str,\n        clevis_info: &Value,\n    ) -> StratisResult<CreateAction<Clevis>> {\n        let changed = self.backstore.bind_clevis(pin, clevis_info)?;\n        if changed {\n            Ok(CreateAction::Created(Clevis))\n        } else {\n            Ok(CreateAction::Identity)\n        }\n    }\n\n    #[pool_mutating_action("NoRequests")]\n    #[pool_rollback]\n    fn unbind_clevis(&mut self) -> StratisResult<DeleteAction<Clevis>> {\n        let changed = self.backstore.unbind_clevis()?;\n        if changed {\n            Ok(DeleteAction::Deleted(Clevis))\n        } else {\n            Ok(DeleteAction::Identity)\n        }\n    }\n\n    #[pool_mutating_action("NoRequests")]\n    #[pool_rollback]\n    fn bind_keyring(\n        &mut self,\n        key_description: &KeyDescription,\n    ) -> StratisResult<CreateAction<Key>> {\n        let changed = self.backstore.bind_keyring(key_description)?;\n        if changed {\n            Ok(CreateAction::Created(Key))\n        } else {\n            Ok(CreateAction::Identity)\n        }\n    }\n\n    #[pool_mutating_action("NoRequests")]\n    #[pool_rollback]\n    fn unbind_keyring(&mut self) -> StratisResult<DeleteAction<Key>> {\n        let changed = self.backstore.unbind_keyring()?;\n        if changed {\n            Ok(DeleteAction::Deleted(Key))\n        } else {\n            Ok(DeleteAction::Identity)\n        }\n    }\n\n    #[pool_mutating_action("NoRequests")]\n    #[pool_rollback]\n    fn rebind_keyring(\n        &mut self,\n        new_key_desc: &KeyDescription,\n    ) -> StratisResult<RenameAction<Key>> {\n        match self.backstore.rebind_keyring(new_key_desc)? {\n            Some(true) => Ok(RenameAction::Renamed(Key)),\n            Some(false) => Ok(RenameAction::Identity),\n            None => Ok(RenameAction::NoSource),\n        }\n    }\n\n    #[pool_mutating_action("NoRequests")]\n    #[pool_rollback]\n    fn rebind_clevis(&mut self) -> StratisResult<RegenAction> {\n        self.backstore.rebind_clevis().map(|_| RegenAction)\n    }\n\n    #[pool_mutating_action("NoRequests")]\n    fn create_filesystems<\'a>(\n        &mut self,\n        pool_name: &str,\n        pool_uuid: PoolUuid,\n        specs: &[(&\'a str, Option<Bytes>)],\n    ) -> StratisResult<SetCreateAction<(&\'a str, FilesystemUuid, Sectors)>> {\n        self.check_fs_limit(specs.len())?;\n\n        let spec_map = validate_filesystem_size_specs(specs)?;\n\n        let increase = spec_map.iter().map(|(_, size)| *size).sum::<Sectors>();\n        self.check_overprov(increase)?;\n\n        spec_map.iter().fold(Ok(()), |res, (name, size)| {\n            res.and_then(|()| validate_name(name))\n                .and_then(|()| {\n                    if let Some((_, fs)) = self.thin_pool.get_filesystem_by_name(name) {\n                        if fs.thindev_size() == *size {\n                            Ok(())\n                        } else {\n                            Err(StratisError::Msg(format!(\n                                "Size {} of filesystem {} to be created conflicts with size {} for existing filesystem",\n                                size,\n                                name,\n                                fs.thindev_size()\n                            )))\n                        }\n                    } else {\n                        Ok(())\n                    }\n                })\n        })?;\n\n        // TODO: Roll back on filesystem initialization failure.\n        let mut result = Vec::new();\n        for (name, size) in spec_map {\n            if self.thin_pool.get_mut_filesystem_by_name(name).is_none() {\n                let fs_uuid = self\n                    .thin_pool\n                    .create_filesystem(pool_name, pool_uuid, name, size)?;\n                result.push((name, fs_uuid, size));\n            }\n        }\n\n        Ok(SetCreateAction::new(result))\n    }\n\n    #[pool_mutating_action("NoRequests")]\n    fn add_blockdevs(\n        &mut self,\n        pool_uuid: PoolUuid,\n        pool_name: &str,\n        paths: &[&Path],\n        tier: BlockDevTier,\n    ) -> StratisResult<(SetCreateAction<DevUuid>, Option<PoolDiff>)> {\n        validate_paths(paths)?;\n\n        let bdev_info = if tier == BlockDevTier::Cache && !self.has_cache() {\n            return Err(StratisError::Msg(\n                format!(\n                    "No cache has been initialized for pool with UUID {} and name {}; it is therefore impossible to add additional devices to the cache",\n                    pool_uuid,\n                    pool_name\n                )\n            ));\n        } else {\n            let devices = ProcessedPathInfos::try_from(paths)?;\n            let (stratis_devices, unowned_devices) = devices.unpack();\n            let (this_pool, other_pools) = stratis_devices.partition(pool_uuid);\n\n            other_pools.error_on_not_empty()?;\n\n            let (in_pool, out_pool): (Vec<_>, Vec<_>) = this_pool\n                .iter()\n                .map(|(dev_uuid, _)| {\n                    self.backstore\n                        .get_blockdev_by_uuid(*dev_uuid)\n                        .map(|(tier, _)| (*dev_uuid, tier))\n                })\n                .partition(|v| v.is_some());\n\n            if !out_pool.is_empty() {\n                let error_message = format!(\n                    "Devices ({}) appear to be already in use by this pool which has UUID {} but this pool has no record of them",\n                    out_pool\n                    .iter()\n                    .map(|opt| this_pool.get(&opt.expect("was looked up").0).expect("partitioned from this_pool").devnode.display().to_string())\n                    .collect::<Vec<_>>()\n                    .join(", "),\n                    pool_uuid\n                );\n                return Err(StratisError::Msg(error_message));\n            };\n\n            let (datadevs, cachedevs): (Vec<_>, Vec<_>) = in_pool\n                .iter()\n                .map(|opt| opt.expect("in_pool devices are Some"))\n                .partition(|(_, tier)| *tier == BlockDevTier::Data);\n\n            if tier == BlockDevTier::Cache {\n                // If adding cache devices, must suspend the pool; the cache\n                // must be augmented with the new devices.\n                if !datadevs.is_empty() {\n                    let error_message = format!(\n                        "Devices ({}) appear to be already in use by this pool which has UUID {}, but in the data tier not the cache tier",\n                        datadevs\n                        .iter()\n                        .map(|(uuid, _)| this_pool.get(uuid).expect("partitioned from this_pool").devnode.display().to_string())\n                        .collect::<Vec<_>>()\n                        .join(", "),\n                        pool_uuid\n                    );\n                    return Err(StratisError::Msg(error_message));\n                };\n\n                if unowned_devices.is_empty() {\n                    return Ok((SetCreateAction::new(vec![]), None));\n                }\n\n                self.thin_pool.suspend()?;\n                let bdev_info_res = self.backstore.add_cachedevs(pool_uuid, unowned_devices);\n                self.thin_pool.resume()?;\n                let bdev_info = bdev_info_res?;\n                Ok((SetCreateAction::new(bdev_info), None))\n            } else {\n                if !cachedevs.is_empty() {\n                    let error_message = format!(\n                        "Devices ({}) appear to be already in use by this pool which has UUID {}, but in the cache tier not the data tier",\n                        cachedevs\n                        .iter()\n                        .map(|(uuid, _)| this_pool.get(uuid).expect("partitioned from this_pool").devnode.display().to_string())\n                        .collect::<Vec<_>>()\n                        .join(", "),\n                        pool_uuid\n                    );\n                    return Err(StratisError::Msg(error_message));\n                };\n\n                if unowned_devices.is_empty() {\n                    return Ok((SetCreateAction::new(vec![]), None));\n                }\n\n                let cached = self.cached();\n\n                // If just adding data devices, no need to suspend the pool.\n                // No action will be taken on the DM devices.\n                let bdev_info = self.backstore.add_datadevs(pool_uuid, unowned_devices)?;\n                self.thin_pool.set_queue_mode();\n\n                // Adding data devices does not change the state of the thin\n                // pool at all. However, if the thin pool is in a state\n                // where it would request an allocation from the backstore the\n                // addition of the new data devs may have changed its context\n                // so that it can satisfy the allocation request where\n                // previously it could not. Run check() in case that is true.\n                let check = match self.thin_pool.check(pool_uuid, &mut self.backstore) {\n                    Ok((_, thin_pool)) => {\n                        let pool = cached.diff(&self.dump(()));\n                        Some(PoolDiff { thin_pool, pool })\n                    }\n                    Err(e) => {\n                        warn!("Failed to check the thin pool for status changes; some information may not be able to reported to the IPC layer: {}", e);\n                        None\n                    }\n                };\n                Ok((SetCreateAction::new(bdev_info), check))\n            }\n        };\n        self.write_metadata(pool_name)?;\n        bdev_info\n    }\n\n    #[pool_mutating_action("NoRequests")]\n    fn destroy_filesystems(\n        &mut self,\n        pool_name: &str,\n        fs_uuids: &[FilesystemUuid],\n    ) -> StratisResult<SetDeleteAction<FilesystemUuid>> {\n        let mut removed = Vec::new();\n        for &uuid in fs_uuids {\n            if let Some(uuid) = self.thin_pool.destroy_filesystem(pool_name, uuid)? {\n                removed.push(uuid);\n            }\n        }\n\n        Ok(SetDeleteAction::new(removed))\n    }\n\n    #[pool_mutating_action("NoRequests")]\n    fn rename_filesystem(\n        &mut self,\n        pool_name: &str,\n        uuid: FilesystemUuid,\n        new_name: &str,\n    ) -> StratisResult<RenameAction<FilesystemUuid>> {\n        validate_name(new_name)?;\n        match self\n            .thin_pool\n            .rename_filesystem(pool_name, uuid, new_name)?\n        {\n            Some(true) => Ok(RenameAction::Renamed(uuid)),\n            Some(false) => Ok(RenameAction::Identity),\n            None => Ok(RenameAction::NoSource),\n        }\n    }\n\n    #[pool_mutating_action("NoRequests")]\n    fn snapshot_filesystem<\'a>(\n        &\'a mut self,\n        pool_name: &str,\n        pool_uuid: PoolUuid,\n        origin_uuid: FilesystemUuid,\n        snapshot_name: &str,\n    ) -> StratisResult<CreateAction<(FilesystemUuid, &\'a mut Self::Filesystem)>> {\n        self.check_fs_limit(1)?;\n\n        validate_name(snapshot_name)?;\n        self.check_overprov(\n            self.thin_pool\n                .get_filesystem_by_uuid(origin_uuid)\n                .ok_or_else(|| {\n                    StratisError::Msg(format!(\n                        "Filesystem with UUID {} could not be found",\n                        origin_uuid\n                    ))\n                })?\n                .1\n                .thindev_size(),\n        )?;\n\n        if self\n            .thin_pool\n            .get_filesystem_by_name(snapshot_name)\n            .is_some()\n        {\n            return Ok(CreateAction::Identity);\n        }\n\n        self.thin_pool\n            .snapshot_filesystem(pool_name, pool_uuid, origin_uuid, snapshot_name)\n            .map(CreateAction::Created)\n    }\n\n    fn total_physical_size(&self) -> Sectors {\n        self.backstore.datatier_size()\n    }\n\n    fn total_allocated_size(&self) -> Sectors {\n        self.backstore.datatier_allocated_size() + self.metadata_size\n    }\n\n    fn total_physical_used(&self) -> Option<Sectors> {\n        // TODO: note that with the addition of another layer, the\n        // calculation of the amount of physical spaced used by means\n        // of adding the amount used by Stratis metadata to the amount used\n        // by the pool abstraction will be invalid. In the event of, e.g.,\n        // software RAID, the amount will be far too low to be useful, in the\n        // event of, e.g, VDO, the amount will be far too large to be useful.\n        self.thin_pool\n            .total_physical_used()\n            .map(|u| u + self.metadata_size)\n    }\n\n    fn filesystems(&self) -> Vec<(Name, FilesystemUuid, &Self::Filesystem)> {\n        self.thin_pool\n            .filesystems()\n            .into_iter()\n            .map(|(n, u, f)| (n, u, f))\n            .collect()\n    }\n\n    fn get_filesystem(&self, uuid: FilesystemUuid) -> Option<(Name, &Self::Filesystem)> {\n        self.thin_pool\n            .get_filesystem_by_uuid(uuid)\n            .map(|(name, fs)| (name, fs))\n    }\n\n    fn get_filesystem_by_name(\n        &self,\n        fs_name: &Name,\n    ) -> Option<(FilesystemUuid, &Self::Filesystem)> {\n        self.thin_pool\n            .get_filesystem_by_name(fs_name)\n            .map(|(uuid, fs)| (uuid, fs))\n    }\n\n    fn blockdevs(&self) -> Vec<(DevUuid, BlockDevTier, &Self::BlockDev)> {\n        self.backstore.blockdevs()\n    }\n\n    fn get_blockdev(&self, uuid: DevUuid) -> Option<(BlockDevTier, &Self::BlockDev)> {\n        self.get_strat_blockdev(uuid)\n    }\n\n    #[pool_mutating_action("NoRequests")]\n    fn get_mut_blockdev(\n        &mut self,\n        uuid: DevUuid,\n    ) -> StratisResult<Option<(BlockDevTier, &mut Self::BlockDev)>> {\n        self.get_mut_strat_blockdev(uuid)\n    }\n\n    #[pool_mutating_action("NoRequests")]\n    fn set_blockdev_user_info(\n        &mut self,\n        pool_name: &str,\n        uuid: DevUuid,\n        user_info: Option<&str>,\n    ) -> StratisResult<RenameAction<DevUuid>> {\n        let result = self.backstore.set_blockdev_user_info(uuid, user_info);\n        match result {\n            Ok(Some(uuid)) => {\n                self.write_metadata(pool_name)?;\n                Ok(RenameAction::Renamed(uuid))\n            }\n            Ok(None) => Ok(RenameAction::Identity),\n            Err(_) => Ok(RenameAction::NoSource),\n        }\n    }\n\n    fn has_cache(&self) -> bool {\n        self.backstore.has_cache()\n    }\n\n    fn is_encrypted(&self) -> bool {\n        self.datadevs_encrypted()\n    }\n\n    fn encryption_info(&self) -> Option<PoolEncryptionInfo> {\n        self.backstore.data_tier_encryption_info()\n    }\n\n    fn avail_actions(&self) -> ActionAvailability {\n        self.action_avail.clone()\n    }\n\n    fn fs_limit(&self) -> u64 {\n        self.thin_pool.fs_limit()\n    }\n\n    #[pool_mutating_action("NoPoolChanges")]\n    fn set_fs_limit(\n        &mut self,\n        pool_name: &Name,\n        pool_uuid: PoolUuid,\n        new_limit: u64,\n    ) -> StratisResult<()> {\n        let (should_save, res) =\n            self.thin_pool\n                .set_fs_limit(pool_uuid, &mut self.backstore, new_limit);\n        if should_save {\n            self.write_metadata(pool_name)?;\n        }\n        res\n    }\n\n    fn overprov_enabled(&self) -> bool {\n        self.thin_pool.overprov_enabled()\n    }\n\n    #[pool_mutating_action("NoPoolChanges")]\n    fn set_overprov_mode(&mut self, pool_name: &Name, enabled: bool) -> StratisResult<()> {\n        let (should_save, res) = self.thin_pool.set_overprov_mode(&self.backstore, enabled);\n        if should_save {\n            self.write_metadata(pool_name)?;\n        }\n        res\n    }\n\n    fn out_of_alloc_space(&self) -> bool {\n        self.thin_pool.out_of_alloc_space()\n    }\n\n    #[pool_mutating_action("NoRequests")]\n    fn grow_physical(\n        &mut self,\n        name: &Name,\n        pool_uuid: PoolUuid,\n        device: DevUuid,\n    ) -> StratisResult<GrowAction<(PoolUuid, DevUuid)>> {\n        let changed = self.backstore.grow(device)?;\n        if changed {\n            if self.thin_pool.set_queue_mode() {\n                self.write_metadata(name)?;\n            }\n            Ok(GrowAction::Grown((pool_uuid, device)))\n        } else {\n            Ok(GrowAction::Identity)\n        }\n    }\n}\n\npub struct StratPoolState {\n    metadata_size: Bytes,\n    out_of_alloc_space: bool,\n}\n\nimpl StateDiff for StratPoolState {\n    type Diff = StratPoolDiff;\n\n    fn diff(&self, other: &Self) -> Self::Diff {\n        StratPoolDiff {\n            metadata_size: self.metadata_size.compare(&other.metadata_size),\n            out_of_alloc_space: self.out_of_alloc_space.compare(&other.out_of_alloc_space),\n        }\n    }\n}\n\nimpl<\'a> DumpState<\'a> for StratPool {\n    type State = StratPoolState;\n    type DumpInput = ();\n\n    fn cached(&self) -> Self::State {\n        StratPoolState {\n            metadata_size: self.metadata_size.bytes(),\n            out_of_alloc_space: self.thin_pool.out_of_alloc_space(),\n        }\n    }\n\n    fn dump(&mut self, _: ()) -> Self::State {\n        self.metadata_size = self.backstore.datatier_metadata_size();\n        StratPoolState {\n            metadata_size: self.metadata_size.bytes(),\n            out_of_alloc_space: self.thin_pool.out_of_alloc_space(),\n        }\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use std::{\n        fs::OpenOptions,\n        io::{BufWriter, Read, Write},\n    };\n\n    use nix::mount::{mount, umount, MsFlags};\n\n    use devicemapper::{Bytes, IEC, SECTOR_SIZE};\n\n    use crate::engine::{\n        engine::{BlockDev, Filesystem},\n        strat_engine::{\n            cmd::udev_settle,\n            tests::{loopbacked, real},\n            thinpool::ThinPoolStatusDigest,\n        },\n        types::{EngineAction, LockKey},\n        Engine, StratEngine,\n    };\n\n    use super::*;\n\n    fn invariant(pool: &StratPool, pool_name: &str) {\n        check_metadata(&pool.record(&Name::new(pool_name.into()))).unwrap();\n        assert!(!(pool.is_encrypted() && pool.backstore.has_cache()));\n        if pool.avail_actions() == ActionAvailability::NoRequests {\n            assert!(\n                pool.encryption_info().is_some()\n                    && pool\n                        .encryption_info()\n                        .map(|ei| { ei.is_inconsistent() })\n                        .unwrap_or(false)\n            );\n        } else if pool.avail_actions() == ActionAvailability::Full {\n            assert!(!pool\n                .encryption_info()\n                .map(|ei| ei.is_inconsistent())\n                .unwrap_or(false));\n        }\n        assert!(pool\n            .backstore\n            .blockdevs()\n            .iter()\n            .all(|(_, _, bd)| bd.metadata_path().is_absolute()))\n    }\n\n    /// Test that initializing a cache causes metadata to be updated. Verify\n    /// that data written before the cache was initialized can be read\n    /// afterwards.\n    fn test_add_cachedevs(paths: &[&Path]) {\n        assert!(paths.len() > 1);\n\n        let (paths1, paths2) = paths.split_at(paths.len() / 2);\n\n        let devices2 = ProcessedPathInfos::try_from(paths2).unwrap();\n        let (stratis_devices, unowned_devices2) = devices2.unpack();\n        stratis_devices.error_on_not_empty().unwrap();\n\n        let name = "stratis-test-pool";\n        let (uuid, mut pool) = StratPool::initialize(name, unowned_devices2, None).unwrap();\n        invariant(&pool, name);\n\n        let metadata1 = pool.record(name);\n        assert_matches!(metadata1.backstore.cache_tier, None);\n\n        let (_, fs_uuid, _) = pool\n            .create_filesystems(name, uuid, &[("stratis-filesystem", None)])\n            .unwrap()\n            .changed()\n            .and_then(|mut fs| fs.pop())\n            .unwrap();\n        invariant(&pool, name);\n\n        let tmp_dir = tempfile::Builder::new()\n            .prefix("stratis_testing")\n            .tempdir()\n            .unwrap();\n        let new_file = tmp_dir.path().join("stratis_test.txt");\n        let bytestring = b"some bytes";\n        {\n            let (_, fs) = pool.get_filesystem(fs_uuid).unwrap();\n            mount(\n                Some(&fs.devnode()),\n                tmp_dir.path(),\n                Some("xfs"),\n                MsFlags::empty(),\n                None as Option<&str>,\n            )\n            .unwrap();\n            OpenOptions::new()\n                .create(true)\n                .write(true)\n                .open(&new_file)\n                .unwrap()\n                .write_all(bytestring)\n                .unwrap();\n        }\n\n        pool.init_cache(uuid, name, paths1).unwrap();\n        invariant(&pool, name);\n\n        let metadata2 = pool.record(name);\n        assert!(metadata2.backstore.cache_tier.is_some());\n\n        let mut buf = [0u8; 10];\n        {\n            OpenOptions::new()\n                .read(true)\n                .open(&new_file)\n                .unwrap()\n                .read_exact(&mut buf)\n                .unwrap();\n        }\n        assert_eq!(&buf, bytestring);\n        umount(tmp_dir.path()).unwrap();\n        pool.teardown().unwrap();\n    }\n\n    #[test]\n    fn loop_test_add_cachedevs() {\n        loopbacked::test_with_spec(\n            &loopbacked::DeviceLimits::Range(3, 4, None),\n            test_add_cachedevs,\n        );\n    }\n\n    #[test]\n    fn real_test_add_cachedevs() {\n        real::test_with_spec(\n            &real::DeviceLimits::AtLeast(2, None, None),\n            test_add_cachedevs,\n        );\n    }\n\n    // Verify that it is possible to add datadevs after a cache is initialized.\n    fn test_add_cachedevs_and_datadevs(paths: &[&Path]) {\n        assert!(paths.len() > 2);\n\n        let (cache_path, data_paths) = paths.split_at(1);\n        let (data_path, data_paths) = data_paths.split_at(1);\n\n        let devices = ProcessedPathInfos::try_from(data_path).unwrap();\n        let (stratis_devices, unowned_devices) = devices.unpack();\n        stratis_devices.error_on_not_empty().unwrap();\n\n        let name = "stratis-test-pool";\n        let (uuid, mut pool) = StratPool::initialize(name, unowned_devices, None).unwrap();\n        invariant(&pool, name);\n\n        pool.init_cache(uuid, name, cache_path).unwrap();\n        invariant(&pool, name);\n\n        pool.add_blockdevs(uuid, name, data_paths, BlockDevTier::Data)\n            .unwrap();\n\n        pool.teardown().unwrap();\n    }\n\n    #[test]\n    fn loop_test_add_cachedevs_and_datadevs() {\n        loopbacked::test_with_spec(\n            &loopbacked::DeviceLimits::Range(3, 4, None),\n            test_add_cachedevs_and_datadevs,\n        );\n    }\n\n    #[test]\n    fn real_test_add_cachedevs_and_datadevs() {\n        real::test_with_spec(\n            &real::DeviceLimits::AtLeast(3, None, None),\n            test_add_cachedevs_and_datadevs,\n        );\n    }\n\n    /// Verify that adding additional blockdevs will cause a pool that is\n    /// out of space to be extended.\n    fn test_add_datadevs(paths: &[&Path]) {\n        assert!(paths.len() > 1);\n\n        let (paths1, paths2) = paths.split_at(1);\n\n        let devices1 = ProcessedPathInfos::try_from(paths1).unwrap();\n        let (stratis_devices, unowned_devices1) = devices1.unpack();\n        stratis_devices.error_on_not_empty().unwrap();\n\n        let name = "stratis-test-pool";\n        let (pool_uuid, mut pool) = StratPool::initialize(name, unowned_devices1, None).unwrap();\n        invariant(&pool, name);\n\n        let fs_name = "stratis_test_filesystem";\n        let (_, fs_uuid, _) = pool\n            .create_filesystems(name, pool_uuid, &[(fs_name, None)])\n            .unwrap()\n            .changed()\n            .and_then(|mut fs| fs.pop())\n            .expect("just created one");\n\n        let devnode = pool.get_filesystem(fs_uuid).unwrap().1.devnode();\n\n        {\n            let buffer_length = IEC::Mi;\n            let mut f = BufWriter::with_capacity(\n                convert_test!(buffer_length, u64, usize),\n                OpenOptions::new().write(true).open(devnode).unwrap(),\n            );\n\n            let buf = &[1u8; SECTOR_SIZE];\n\n            let mut amount_written = Sectors(0);\n            let buffer_length = Bytes::from(buffer_length).sectors();\n            while matches!(pool.thin_pool.state(), Some(ThinPoolStatusDigest::Good)) {\n                f.write_all(buf).unwrap();\n                amount_written += Sectors(1);\n                // Run check roughly every time the buffer is cleared.\n                // Running it more often is pointless as the pool is guaranteed\n                // not to see any effects unless the buffer is cleared.\n                if amount_written % buffer_length == Sectors(1) {\n                    pool.event_on(pool_uuid, &Name::new(name.to_string()))\n                        .unwrap();\n                }\n            }\n\n            pool.add_blockdevs(pool_uuid, name, paths2, BlockDevTier::Data)\n                .unwrap();\n\n            match pool.thin_pool.state() {\n                Some(ThinPoolStatusDigest::Good) => (),\n                _ => panic!("thin pool status should be back to working"),\n            }\n        }\n    }\n\n    #[test]\n    fn loop_test_add_datadevs() {\n        loopbacked::test_with_spec(\n            &loopbacked::DeviceLimits::Range(2, 3, Some(Bytes::from(IEC::Gi * 4).sectors())),\n            test_add_datadevs,\n        );\n    }\n\n    #[test]\n    fn real_test_add_datadevs() {\n        real::test_with_spec(\n            &real::DeviceLimits::AtLeast(\n                2,\n                Some(Bytes::from(IEC::Gi * 2).sectors()),\n                Some(Bytes::from(IEC::Gi * 4).sectors()),\n            ),\n            test_add_datadevs,\n        );\n    }\n\n    /// Test that rollback errors are properly detected an maintenance mode\n    /// is set accordingly.\n    fn test_maintenance_mode(paths: &[&Path]) {\n        assert!(paths.len() > 1);\n\n        let name = "stratis-test-pool";\n\n        let devices = ProcessedPathInfos::try_from(paths).unwrap();\n        let (stratis_devices, unowned_devices) = devices.unpack();\n        stratis_devices.error_on_not_empty().unwrap();\n\n        let (_, mut pool) = StratPool::initialize(name, unowned_devices, None).unwrap();\n        invariant(&pool, name);\n\n        assert_eq!(pool.action_avail, ActionAvailability::Full);\n        assert!(pool.return_rollback_failure().is_err());\n        assert_eq!(pool.action_avail, ActionAvailability::NoRequests);\n\n        pool.destroy().unwrap();\n        udev_settle().unwrap();\n\n        let name = "stratis-test-pool";\n\n        let devices = ProcessedPathInfos::try_from(paths).unwrap();\n        let (stratis_devices, unowned_devices) = devices.unpack();\n        stratis_devices.error_on_not_empty().unwrap();\n\n        let (_, mut pool) = StratPool::initialize(name, unowned_devices, None).unwrap();\n        invariant(&pool, name);\n\n        assert_eq!(pool.action_avail, ActionAvailability::Full);\n        assert!(pool.return_rollback_failure_chain().is_err());\n        assert_eq!(pool.action_avail, ActionAvailability::NoRequests);\n    }\n\n    #[test]\n    fn loop_test_maintenance_mode() {\n        loopbacked::test_with_spec(\n            &loopbacked::DeviceLimits::Range(2, 3, Some(Bytes::from(IEC::Gi * 4).sectors())),\n            test_maintenance_mode,\n        );\n    }\n\n    #[test]\n    fn real_test_mainenance_mode() {\n        real::test_with_spec(\n            &real::DeviceLimits::AtLeast(\n                2,\n                Some(Bytes::from(IEC::Gi * 2).sectors()),\n                Some(Bytes::from(IEC::Gi * 4).sectors()),\n            ),\n            test_maintenance_mode,\n        );\n    }\n\n    /// Test overprovisioning mode disabled and enabled assuring that the appropriate\n    /// checks and behavior are in place.\n    fn test_overprov(paths: &[&Path]) {\n        assert!(paths.len() == 1);\n\n        let pool_name = "pool";\n\n        let devices = ProcessedPathInfos::try_from(paths).unwrap();\n        let (stratis_devices, unowned_devices) = devices.unpack();\n        stratis_devices.error_on_not_empty().unwrap();\n\n        let (pool_uuid, mut pool) =\n            StratPool::initialize(pool_name, unowned_devices, None).unwrap();\n\n        let (_, fs_uuid, _) = pool\n            .create_filesystems(\n                pool_name,\n                pool_uuid,\n                &[(\n                    "stratis_test_filesystem",\n                    Some(pool.backstore.datatier_usable_size().bytes() * 2u64),\n                )],\n            )\n            .unwrap()\n            .changed()\n            .unwrap()\n            .pop()\n            .unwrap();\n        udev_settle().unwrap();\n        assert!(pool\n            .set_overprov_mode(&Name::new(pool_name.to_string()), false)\n            .is_err());\n        pool.destroy_filesystems(pool_name, &[fs_uuid]).unwrap();\n\n        pool.set_overprov_mode(&Name::new(pool_name.to_string()), false)\n            .unwrap();\n        assert!(pool\n            .create_filesystems(\n                pool_name,\n                pool_uuid,\n                &[(\n                    "stratis_test_filesystem",\n                    Some(pool.backstore.datatier_usable_size().bytes() * 2u64)\n                )],\n            )\n            .is_err());\n\n        let mut initial_fs_size = pool.backstore.datatier_usable_size().bytes() * 2u64 / 3u64;\n        initial_fs_size = initial_fs_size.sectors().bytes();\n        let half_init_size = initial_fs_size / 2u64 + Bytes(1);\n        let (_, fs_uuid, _) = pool\n            .create_filesystems(\n                pool_name,\n                pool_uuid,\n                &[("stratis_test_filesystem", Some(initial_fs_size))],\n            )\n            .unwrap()\n            .changed()\n            .unwrap()\n            .pop()\n            .unwrap();\n\n        let tmp_dir = tempfile::Builder::new()\n            .prefix("stratis_testing")\n            .tempdir()\n            .unwrap();\n        let new_file = tmp_dir.path().join("stratis_test.txt");\n        let sector = &[0; 512];\n\n        {\n            let (_, fs) = pool.get_filesystem(fs_uuid).unwrap();\n            mount(\n                Some(&fs.devnode()),\n                tmp_dir.path(),\n                Some("xfs"),\n                MsFlags::empty(),\n                None as Option<&str>,\n            )\n            .unwrap();\n        }\n\n        let mut f = OpenOptions::new()\n            .create(true)\n            .write(true)\n            .open(&new_file)\n            .unwrap();\n        let mut written = Sectors(0);\n        while written.bytes() < half_init_size {\n            f.write_all(sector).unwrap();\n            written += Sectors(1);\n        }\n        let diffs = pool.fs_event_on(pool_uuid).unwrap();\n        assert!(diffs.get(&fs_uuid).unwrap().size.is_changed());\n\n        let (_, fs) = pool.get_filesystem(fs_uuid).unwrap();\n        assert!(fs.thindev_size() < initial_fs_size.sectors() * 2u64);\n    }\n\n    #[test]\n    fn loop_test_overprov() {\n        loopbacked::test_with_spec(\n            &loopbacked::DeviceLimits::Exactly(1, Some(Sectors(10 * IEC::Mi))),\n            test_overprov,\n        );\n    }\n\n    #[test]\n    fn real_test_overprov() {\n        real::test_with_spec(\n            &real::DeviceLimits::Exactly(1, Some(Sectors(10 * IEC::Mi)), None),\n            test_overprov,\n        );\n    }\n\n    /// Set up for testing physical device growth.\n    fn test_grow_physical_pre_grow(paths: &[&Path]) {\n        let pool_name = Name::new("pool".to_string());\n        let engine = StratEngine::initialize().unwrap();\n        let pool_uuid = test_async!(engine.create_pool(&*pool_name, paths, None))\n            .unwrap()\n            .changed()\n            .unwrap();\n        let mut guard = test_async!(engine.get_mut_pool(LockKey::Uuid(pool_uuid))).unwrap();\n        let (_, _, pool) = guard.as_mut_tuple();\n\n        let (_, fs_uuid, _) = pool\n            .create_filesystems(&*pool_name, pool_uuid, &[("stratis_test_filesystem", None)])\n            .unwrap()\n            .changed()\n            .unwrap()\n            .pop()\n            .unwrap();\n\n        let tmp_dir = tempfile::Builder::new()\n            .prefix("stratis_testing")\n            .tempdir()\n            .unwrap();\n        let new_file = tmp_dir.path().join("stratis_test.txt");\n        let write_block = &[0; 512_000];\n\n        {\n            let (_, fs) = pool.get_filesystem(fs_uuid).unwrap();\n            mount(\n                Some(&fs.devnode()),\n                tmp_dir.path(),\n                Some("xfs"),\n                MsFlags::empty(),\n                None as Option<&str>,\n            )\n            .unwrap();\n        }\n\n        let mut f = OpenOptions::new()\n            .create(true)\n            .write(true)\n            .open(&new_file)\n            .unwrap();\n        while !pool.out_of_alloc_space() {\n            f.write_all(write_block).unwrap();\n            f.sync_all().unwrap();\n            pool.event_on(pool_uuid, &pool_name).unwrap();\n        }\n    }\n\n    /// Test that growing a physical device succeeds, the device has doubled in size,\n    /// and that the pool registers new available allocation space if it is out of space\n    /// at the time of device growth.\n    fn test_grow_physical_post_grow(_: &[&Path]) {\n        let engine = StratEngine::initialize().unwrap();\n\n        let mut pools = test_async!(engine.pools_mut());\n        assert!(pools.len() == 1);\n        let (pool_name, pool_uuid, pool) = pools.iter_mut().next().unwrap();\n\n        let (dev_uuid, size) = {\n            let blockdevs = pool.blockdevs();\n            let (dev_uuid, _, dev) = blockdevs.first().unwrap();\n            (*dev_uuid, dev.size())\n        };\n\n        assert!(pool.out_of_alloc_space());\n        pool.grow_physical(pool_name, *pool_uuid, dev_uuid)\n            .unwrap()\n            .changed()\n            .unwrap();\n        let (_, dev) = pool.get_blockdev(dev_uuid).unwrap();\n        assert_eq!(dev.size(), 2u64 * size);\n        assert!(!pool.out_of_alloc_space());\n    }\n\n    #[test]\n    fn loop_test_grow_physical() {\n        loopbacked::test_device_grow_with_spec(\n            &loopbacked::DeviceLimits::Exactly(2, Some(Sectors(10 * IEC::Mi))),\n            test_grow_physical_pre_grow,\n            test_grow_physical_post_grow,\n        );\n    }\n}\n', 'message': 'Make initial sizes *Blocks instead of Sectors\n\nThe more specfic type should make things clearer.\n\nSigned-off-by: mulhern <7b51bcf507bcd7afb72bf8663752c0ddbeb517f6@redhat.com>\n', 'author_name': 'mulhern', 'lang': 'rs'}