Add device information to integration tests

iml-system-docker-tests/src/lib.rs

@@ -41,7 +41,7 @@ pub async fn run_fs_test<S: std::hash::BuildHasher>(
 
     docker::deploy_iml_stack().await?;
 
-    vagrant::setup_deploy_docker_servers(&config, server_map).await?;
+    vagrant::setup_deploy_docker_servers(&config, docker_setup, server_map).await?;
 
     vagrant::create_fs(fs_type, &config).await?;
 

iml-system-test-utils/Cargo.toml

@@ -11,5 +11,6 @@ iml-cmd = { version = "0.2.0", path = "../iml-cmd" }
 iml-fs = { version = "0.2", path = "../iml-fs" }
 iml-systemd = { version = "0.2.0", path = "../iml-systemd" }
 iml-wire-types = { version = "0.2.0", path = "../iml-wire-types" }
+serde_json = "1.0"
 tokio = {version="0.2", features=["process", "macros", "fs"]}
 thiserror = "1.0"

iml-system-test-utils/src/iml.rs

@@ -4,7 +4,7 @@
 
 use crate::get_local_server_names;
 use iml_cmd::{CheckedCommandExt, CmdError};
-use std::collections::HashMap;
+use std::{collections::HashMap, process::Output};
 use tokio::{fs, process::Command};
 
 pub const IML_DOCKER_PATH: &str = "/etc/iml-docker";
@@ -27,6 +27,16 @@ pub async fn list_servers() -> Result<Command, CmdError> {
     Ok(x)
 }
 
+pub async fn list_devices() -> Result<Output, CmdError> {
+    let mut x = iml().await?;
+    x.arg("devices")
+        .arg("list")
+        .arg("-d")
+        .arg("json")
+        .checked_output()
+        .await
+}
+
 pub async fn server_add<S: std::hash::BuildHasher>(
     host_map: &HashMap<String, &[&str], S>,
 ) -> Result<(), CmdError> {

iml-system-test-utils/src/ssh.rs

@@ -5,6 +5,7 @@
 use futures::future::try_join_all;
 use iml_cmd::{CheckedChildExt, CheckedCommandExt, CmdError};
 use std::{
+    collections::BTreeSet,
     process::{Output, Stdio},
     str,
     time::{SystemTime, UNIX_EPOCH},
@@ -199,3 +200,27 @@ pub async fn create_iml_diagnostics<'a, 'b>(
     )
     .await
 }
+
+pub async fn get_host_bindings<'a, 'b>(hosts: &'b [&'a str]) -> Result<BTreeSet<String>, CmdError> {
+    let bindings: Vec<(&str, Output)> =
+        ssh_exec_parallel(hosts, "cat /etc/multipath/bindings").await?;
+
+    let wwids = bindings
+        .into_iter()
+        .map(|(_, out)| {
+            str::from_utf8(&out.stdout)
+                .expect("Couldn't parse multipath bindings file.")
+                .to_string()
+        })
+        .filter(|x| !x.starts_with('#'))
+        .map(|out| {
+            out.lines()
+                .map(|line| line.split(' ').last().expect("Couldn't parse WWID").into())
+                .collect::<BTreeSet<String>>()
+        })
+        .fold(BTreeSet::<String>::new(), |acc, xs| {
+            acc.union(&xs).cloned().collect()
+        });
+
+    Ok(wwids)
+}

iml-system-test-utils/src/vagrant.rs

@@ -8,9 +8,12 @@ use crate::{
 };
 use futures::future::try_join_all;
 use iml_cmd::{CheckedCommandExt, CmdError};
+use iml_wire_types::Volume;
 use std::{
     collections::{BTreeSet, HashMap},
-    env, str,
+    env,
+    process::Output,
+    str,
     time::Duration,
 };
 use tokio::{
@@ -320,13 +323,81 @@ pub async fn setup_iml_install(
     Ok(())
 }
 
+pub fn parse_devices(output: &Output) -> BTreeSet<String> {
+    let data_str = str::from_utf8(&output.stdout).expect("Couldn't parse devices information.");
+    let volumes: Vec<Volume> =
+        serde_json::from_str(data_str).expect("Couldn't serialize devices information.");
+
+    let labels: BTreeSet<String> = volumes.into_iter().map(|v| v.label).collect();
+
+    labels
+}
+
+pub async fn get_iml_devices(config: &ClusterConfig) -> Result<BTreeSet<String>, CmdError> {
+    let output = run_vm_command(config.manager, "iml devices list -d json")
+        .await?
+        .checked_output()
+        .await?;
+
+    Ok(parse_devices(&output))
+}
+
+pub async fn get_iml_docker_devices() -> Result<BTreeSet<String>, CmdError> {
+    let output = iml::list_devices().await?;
+
+    Ok(parse_devices(&output))
+}
+
+pub async fn get_devices(
+    config: &ClusterConfig,
+    setup_config: &SetupConfigType,
+) -> Result<BTreeSet<String>, CmdError> {
+    match setup_config {
+        SetupConfigType::RpmSetup(_) => get_iml_devices(config).await,
+        SetupConfigType::DockerSetup(_) => get_iml_docker_devices().await,
+    }
+}
+
+pub async fn wait_for_all_devices(
+    max_tries: i32,
+    config: &ClusterConfig,
+    setup_config: &SetupConfigType,
+) -> Result<(), CmdError> {
+    let mut count = 1;
+    let wwids: BTreeSet<String> = ssh::get_host_bindings(&config.storage_servers()[..]).await?;
+    println!("Comparing wwids from api to bindings: {:?}", wwids);
+
+    let iml_devices: BTreeSet<String> = get_devices(config, setup_config).await?;
+    let mut all_volumes_accounted_for = wwids.is_subset(&iml_devices);
+
+    println!("Comparing iml devices to bindings files.");
+    println!("iml_devices: {:?}", iml_devices);
+    println!("binding wwids: {:?}", wwids);
+
+    while !all_volumes_accounted_for && count < max_tries {
+        delay_for(Duration::from_secs(5)).await;
+
+        let iml_devices: BTreeSet<String> = get_devices(config, setup_config).await?;
+        all_volumes_accounted_for = wwids.is_subset(&iml_devices);
+        count += 1;
+
+        println!("Comparing iml devices to bindings files.");
+        println!("iml_devices: {:?}", iml_devices);
+        println!("binding wwids: {:?}", wwids);
+    }
+
+    Ok(())
+}
+
 pub async fn setup_deploy_servers<S: std::hash::BuildHasher>(
     config: &ClusterConfig,
     setup_config: &SetupConfigType,
     server_map: HashMap<String, &[&str], S>,
 ) -> Result<(), CmdError> {
     setup_iml_install(&config.all(), &setup_config, &config).await?;
 
+    wait_for_all_devices(10, config, setup_config).await?;
+
     for (profile, hosts) in server_map {
         run_vm_command(
             config.manager,
@@ -353,8 +424,11 @@ pub async fn setup_deploy_servers<S: std::hash::BuildHasher>(
 
 pub async fn add_docker_servers<S: std::hash::BuildHasher>(
     config: &ClusterConfig,
+    setup_config: &SetupConfigType,
     server_map: &HashMap<String, &[&str], S>,
 ) -> Result<(), CmdError> {
+    wait_for_all_devices(10, config, setup_config).await?;
+
     iml::server_add(&server_map).await?;
 
     halt()
@@ -378,6 +452,7 @@ pub async fn add_docker_servers<S: std::hash::BuildHasher>(
 
 pub async fn setup_deploy_docker_servers<S: std::hash::BuildHasher>(
     config: &ClusterConfig,
+    setup_config: &SetupConfigType,
     server_map: HashMap<String, &[&str], S>,
 ) -> Result<(), CmdError> {
     let server_set: BTreeSet<_> = server_map.values().cloned().flatten().collect();
@@ -393,7 +468,7 @@ pub async fn setup_deploy_docker_servers<S: std::hash::BuildHasher>(
 
     configure_docker_network(server_set).await?;
 
-    add_docker_servers(&config, &server_map).await?;
+    add_docker_servers(&config, setup_config, &server_map).await?;
 
     Ok(())
 }