change the way we calculate baselines
We want to also include the outliers.
@@ -12,7 +12,7 @@ from providers.types import DataParser
# that were not caught while gathering baselines. This provides
# slightly better reliability, while not affecting regression
# detection.
-DELTA_EXTRA_MARGIN = 4
+DELTA_EXTRA_MARGIN = 6
# pylint: disable=R0903
@@ -33,8 +33,14 @@ class SnapshotRestoreDataParser(DataParser):
def calculate_baseline(self, data: List[float]) -> dict:
"""Return the target and delta values, given a list of data points."""
avg = statistics.mean(data)
- stddev = statistics.stdev(data)
+ min_ = min(data)
+ max_ = max(data)
+
+ min_delta = 100 * abs(avg - min_) / avg
+ max_delta = 100 * abs(avg - max_) / avg
+ delta = max(max_delta, min_delta)
+
return {
"target": round(avg, 3),
- "delta_percentage": math.ceil(3 * stddev / avg * 100) + DELTA_EXTRA_MARGIN,
+ "delta_percentage": math.ceil(delta) + DELTA_EXTRA_MARGIN,
}
