Retry coverage recording
@@ -13,7 +13,7 @@ use coverage::cache::ModuleCache;
use coverage::code::{CmdFilter, CmdFilterDef};
use onefuzz::expand::Expand;
use onefuzz::syncdir::SyncedDir;
-use onefuzz_telemetry::{Event::coverage_data, EventData};
+use onefuzz_telemetry::{warn, Event::coverage_data, EventData};
use serde::de::DeserializeOwned;
use storage_queue::{Message, QueueClient};
use tokio::fs;
@@ -25,6 +25,7 @@ use crate::tasks::config::CommonConfig;
use crate::tasks::generic::input_poller::{CallbackImpl, InputPoller, Processor};
use crate::tasks::heartbeat::{HeartbeatSender, TaskHeartbeatClient};
+const MAX_COVERAGE_RECORDING_ATTEMPTS: usize = 2;
const COVERAGE_FILE: &str = "coverage.json";
const MODULE_CACHE_FILE: &str = "module-cache.json";
@@ -188,6 +189,40 @@ impl<'a> TaskContext<'a> {
}
pub async fn record_input(&mut self, input: &Path) -> Result<()> {
+ let attempts = MAX_COVERAGE_RECORDING_ATTEMPTS;
+
+ for attempt in 1..=attempts {
+ let result = self.try_record_input(input).await;
+
+ if let Err(err) = &result {
+ // Recording failed, check if we can retry.
+ if attempt < attempts {
+ // We will retry, but warn to capture the error if we succeed.
+ warn!(
+ "error recording coverage for input = {}: {:?}",
+ input.display(),
+ err
+ );
+ } else {
+ // Final attempt, do not retry.
+ return result.with_context(|| {
+ format_err!(
+ "failed to record coverage for input = {} after {} attempts",
+ input.display(),
+ attempts
+ )
+ });
+ }
+ } else {
+ // We successfully recorded the coverage for `input`, so stop.
+ break;
+ }
+ }
+
+ Ok(())
+ }
+
+ async fn try_record_input(&mut self, input: &Path) -> Result<()> {
let coverage = self.record_impl(input).await?;
self.coverage.merge_max(&coverage);
