Stop integration tests after 5 minutes and print their log files
for better debuggability on CI and to avoid infinite runs hogging machines.
@@ -21,11 +21,14 @@ use crate::{
errors::{Error, ErrorKind, Result},
};
use crate::{cli::TestMode, job};
+use async_std::io::WriteExt;
+use async_std::prelude::FutureExt;
use globwalk::{FileType, GlobWalkerBuilder};
use metadata::Meta;
use std::collections::HashMap;
use std::io::Write;
use std::path::{Path, PathBuf};
+use std::time::Duration;
use tag::TagFilter;
use tremor_common::file;
use tremor_common::time::nanotime;
@@ -40,6 +43,8 @@ pub mod stats;
pub mod tag;
mod unit;
+const INTEGRATION_TEST_TIMEOUT: Duration = Duration::from_secs(5);
+
async fn suite_bench(
root: &Path,
config: &TestConfig,
@@ -178,7 +183,33 @@ async fn run_integration(
status::tags(&tags, Some(&matched), Some(&config.excludes))?;
// Run integration tests
- let test_report = process::run_process("integration", base, root, &tags).await?;
+ let test_report = match process::run_process("integration", base, root, &tags)
+ .timeout(INTEGRATION_TEST_TIMEOUT)
+ .await
+ {
+ Err(_) => {
+ // timeout
+ // walk the entire dir and print all *.log files for debuggability
+ error!("Timeout running integration test {}", root.display());
+ let walker = GlobWalkerBuilder::new(root, "*.log")
+ .file_type(FileType::FILE)
+ .build()
+ .map_err(|e| {
+ format!("Unable to walk test path for printing log files: {}", e)
+ })?;
+ let mut stderr = async_std::io::stderr();
+ for log_file in walker.filter_map(std::result::Result::ok) {
+ let path = log_file.path();
+ let mut log_fd = tremor_common::asy::file::open(path).await?;
+ stderr
+ .write_all(format!("\n\t{}\n", path.display()).as_bytes())
+ .await?;
+ async_std::io::copy(&mut log_fd, &mut stderr).await?;
+ }
+ return Err(format!("Timeout running test {}", root.display()).into());
+ }
+ Ok(res) => res?,
+ };
// Restore cwd
file::set_current_dir(&cwd)?;
