ptest: Start of utility to run tests in parallel
The travis config allows multiple tests to run in parallel. Run a small
program that does the same thing.
Signed-off-by: David Brown <david.brown@linaro.org>
diff --git a/ptest/src/main.rs b/ptest/src/main.rs
new file mode 100644
index 0000000..1ef4979
--- /dev/null
+++ b/ptest/src/main.rs
@@ -0,0 +1,291 @@
+//! Parallel testing.
+//!
+//! mcuboot simulator is strictly single threaded, as there is a lock around running the C startup
+//! code, because it contains numerous global variables.
+//!
+//! To help speed up testing, the Travis configuration defines all of the configurations that can
+//! be run in parallel. Fortunately, cargo works well this way, and these can be run by simply
+//! using subprocess for each particular thread.
+
+use chrono::Local;
+use failure::format_err;
+use log::{debug, error, warn};
+use regex::Regex;
+use std::{
+ collections::HashSet,
+ fs::{self, OpenOptions},
+ io::{ErrorKind, stdout, Write},
+ process::{Command, Output},
+ result,
+ sync::{
+ Arc,
+ Mutex,
+ },
+ thread,
+ time::Duration,
+};
+use std_semaphore::Semaphore;
+use yaml_rust::{
+ Yaml,
+ YamlLoader,
+};
+
+type Result<T> = result::Result<T, failure::Error>;
+
+fn main() -> Result<()> {
+ env_logger::init();
+
+ let travis_text = fs::read_to_string("../.travis.yml")?;
+ let travis = YamlLoader::load_from_str(&travis_text)?;
+
+ let ncpus = num_cpus::get();
+ let limiter = Arc::new(Semaphore::new(ncpus as isize));
+
+ let matrix = Matrix::from_yaml(&travis)?;
+
+ let mut children = vec![];
+ let state = State::new(matrix.envs.len());
+ let st2 = state.clone();
+ let _status = thread::spawn(move || {
+ loop {
+ thread::sleep(Duration::new(15, 0));
+ st2.lock().unwrap().status();
+ }
+ });
+ for env in matrix.envs {
+ let state = state.clone();
+ let limiter = limiter.clone();
+
+ let child = thread::spawn(move || {
+ let _run = limiter.access();
+ state.lock().unwrap().start(&env);
+ let out = env.run();
+ state.lock().unwrap().done(&env, out);
+ });
+ children.push(child);
+ }
+
+ for child in children {
+ child.join().unwrap();
+ }
+
+ println!("");
+
+ Ok(())
+}
+
+/// State, for printing status.
+struct State {
+ running: HashSet<String>,
+ done: HashSet<String>,
+ total: usize,
+}
+
+impl State {
+ fn new(total: usize) -> Arc<Mutex<State>> {
+ Arc::new(Mutex::new(State {
+ running: HashSet::new(),
+ done: HashSet::new(),
+ total: total,
+ }))
+ }
+
+ fn start(&mut self, fs: &FeatureSet) {
+ let key = fs.textual();
+ if self.running.contains(&key) || self.done.contains(&key) {
+ warn!("Duplicate: {:?}", key);
+ }
+ debug!("Starting: {} ({} running)", key, self.running.len() + 1);
+ self.running.insert(key);
+ self.status();
+ }
+
+ fn done(&mut self, fs: &FeatureSet, output: Result<Option<Output>>) {
+ let key = fs.textual();
+ self.running.remove(&key);
+ self.done.insert(key.clone());
+ match output {
+ Ok(None) => {
+ // println!("Success {} ({} running)", key, self.running.len());
+ }
+ Ok(Some(output)) => {
+ // Write the output into a file.
+ let mut count = 1;
+ let (mut fd, logname) = loop {
+ let name = format!("./failure-{:04}.log", count);
+ count += 1;
+ match OpenOptions::new()
+ .create_new(true)
+ .write(true)
+ .open(&name)
+ {
+ Ok(file) => break (file, name),
+ Err(ref err) if err.kind() == ErrorKind::AlreadyExists => continue,
+ Err(err) => {
+ error!("Unable to write log file to current directory: {:?}", err);
+ return;
+ }
+ }
+ };
+ writeln!(&mut fd, "Test failure {}", key).unwrap();
+ writeln!(&mut fd, "time: {}", Local::now().to_rfc3339()).unwrap();
+ writeln!(&mut fd, "----------------------------------------").unwrap();
+ writeln!(&mut fd, "stdout:").unwrap();
+ fd.write_all(&output.stdout).unwrap();
+ writeln!(&mut fd, "----------------------------------------").unwrap();
+ writeln!(&mut fd, "\nstderr:").unwrap();
+ fd.write_all(&output.stderr).unwrap();
+ error!("Failure {} log:{:?} ({} running)", key, logname,
+ self.running.len());
+ }
+ Err(err) => {
+ error!("Unable to run test {:?} ({:?})", key, err);
+ }
+ }
+ self.status();
+ }
+
+ fn status(&self) {
+ let running = self.running.len();
+ let done = self.done.len();
+ print!(" {} running ({}/{}/{} done)\r", running, done, running + done, self.total);
+ stdout().flush().unwrap();
+ }
+}
+
+/// The extracted configurations from the travis config
+#[derive(Debug)]
+struct Matrix {
+ envs: Vec<FeatureSet>,
+}
+
+#[derive(Debug, Eq, Hash, PartialEq)]
+struct FeatureSet {
+ // The environment variable to set.
+ env: String,
+ // The successive values to set it to.
+ values: Vec<String>,
+}
+
+impl Matrix {
+ fn from_yaml(yaml: &[Yaml]) -> Result<Matrix> {
+ let mut envs = vec![];
+
+ let mut all_tests = HashSet::new();
+
+ for y in yaml {
+ let m = match lookup_matrix(y) {
+ Some (m) => m,
+ None => continue,
+ };
+ for elt in m {
+ if lookup_os(elt) == Some("linux") {
+ debug!("yaml: {:?}", lookup_env(elt));
+ let env = match lookup_env(elt) {
+ Some (env) => env,
+ None => continue,
+ };
+
+ let fset = FeatureSet::decode(env)?;
+ debug!("fset: {:?}", fset);
+
+ if false {
+ // Respect the groupings in the `.travis.yml` file.
+ envs.push(fset);
+ } else {
+ // Break each test up so we can run more in
+ // parallel.
+ let env = fset.env.clone();
+ for val in fset.values {
+ if !all_tests.contains(&val) {
+ all_tests.insert(val.clone());
+ envs.push(FeatureSet {
+ env: env.clone(),
+ values: vec![val],
+ });
+ } else {
+ warn!("Duplicate: {:?}: {:?}", env, val);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ Ok(Matrix {
+ envs: envs,
+ })
+ }
+}
+
+impl FeatureSet {
+ fn decode(text: &str) -> Result<FeatureSet> {
+ let re = Regex::new(r#"^([A-Z_]+)="(.*)"$"#)?;
+
+ match re.captures(text) {
+ None => Err(format_err!("Invalid line: {:?}", text)),
+ Some(cap) => {
+ let ename = &cap[1];
+ let sep = if ename == "SINGLE_FEATURES" { ' ' } else { ',' };
+ let values: Vec<_> = cap[2]
+ .split(sep)
+ .map(|s| s.to_string())
+ .collect();
+ debug!("name={:?} values={:?}", ename, values);
+ Ok(FeatureSet {
+ env: ename.to_string(),
+ values: values,
+ })
+ }
+ }
+ }
+
+ /// Run a test for this given feature set. Output is captured and will be returned if there is
+ /// an error. Each will be run successively, and the first failure will be returned.
+ /// Otherwise, it returns None, which means everything worked.
+ fn run(&self) -> Result<Option<Output>> {
+ for v in &self.values {
+ let output = Command::new("bash")
+ .arg("./scripts/run_tests.sh")
+ .current_dir("..")
+ .env(&self.env, v)
+ .output()?;
+ if !output.status.success() {
+ return Ok(Some(output));
+ }
+ }
+ return Ok(None);
+ }
+
+ /// Convert this feature set into a textual representation
+ fn textual(&self) -> String {
+ use std::fmt::Write;
+
+ let mut buf = String::new();
+
+ write!(&mut buf, "{}:", self.env).unwrap();
+ for v in &self.values {
+ write!(&mut buf, " {}", v).unwrap();
+ }
+
+ buf
+ }
+}
+
+fn lookup_matrix(y: &Yaml) -> Option<&Vec<Yaml>> {
+ let matrix = Yaml::String("matrix".to_string());
+ let include = Yaml::String("include".to_string());
+ y.as_hash()?.get(&matrix)?.as_hash()?.get(&include)?.as_vec()
+}
+
+fn lookup_os(y: &Yaml) -> Option<&str> {
+ let os = Yaml::String("os".to_string());
+
+ y.as_hash()?.get(&os)?.as_str()
+}
+
+fn lookup_env(y: &Yaml) -> Option<&str> {
+ let env = Yaml::String("env".to_string());
+
+ y.as_hash()?.get(&env)?.as_str()
+}