make tests fail when criteria not found
When criteria is not available, performance tests pass without any
errors. This commit makes tests fail in such situation. It lets us know
that performance baselines are missing.
@@ -7,7 +7,8 @@ from abc import ABC, abstractmethod
from numbers import Number
from typing import Any, Callable
from collections import defaultdict
-from framework.utils import ExceptionAggregator
+from framework.utils import ExceptionAggregator, get_kernel_version
+from framework.utils_cpuid import get_cpu_model_name, get_instance_type
from .criteria import CriteriaException
from .metadata import Provider as MetadataProvider
@@ -141,6 +142,16 @@ class Consumer(ABC):
self._failure_aggregator.add_row(fail_msg)
if fail_fast:
raise self._failure_aggregator
+ else:
+ self._statistics[ms_name][st_def.name]["outcome"] = "FAILED"
+ fail_msg = (
+ f"'{ms_name}/{st_def.name}': Criteria not found for "
+ f"{get_instance_type()} / {get_cpu_model_name()} / "
+ f"kernel {get_kernel_version(1)}."
+ )
+ self._failure_aggregator.add_row(fail_msg)
+ if fail_fast:
+ raise self._failure_aggregator
self._reset()
