Dump problematic Nomad task events
@@ -40,16 +40,21 @@ def _dump_nomad_consul_agent_logs(artifacts_dir: Path) -> None:
class NomadAllocation:
allocation_id: str
allocation_name: str
+ status: str
tasks: List[NomadTask]
def __init__(self, input: Dict[str, Any]) -> None:
self.allocation_id = input["ID"]
self.allocation_name = input["Name"]
+ self.status = input["ClientStatus"]
# Remove tasks we don't super care about
task_names = [
t for t in input["TaskStates"].keys() if not t.startswith("connect-proxy")
]
- self.tasks = [NomadTask(parent=self, name=t) for t in task_names]
+ self.tasks = [
+ NomadTask(parent=self, name=t, events=input["TaskStates"][t]["Events"])
+ for t in task_names
+ ]
def short_alloc_id(self) -> str:
return self.allocation_id.split("-")[0]
@@ -59,6 +64,7 @@ class NomadAllocation:
class NomadTask:
name: str
parent: NomadAllocation = dataclasses.field(repr=False)
+ events: List[dict]
def get_logs(self, nomad_client: Nomad, type: OutOrErr) -> Optional[str]:
try:
@@ -72,6 +78,12 @@ class NomadTask:
LOGGER.info(f"Couldn't get logs for {self.name}")
return None
+ def get_events(self) -> str:
+ if self.parent.status not in ["running", "completed"]:
+ event_list = [event["DisplayMessage"] for event in self.events]
+ return "\n".join(event_list)
+ return ""
+
def _get_nomad_job_names(nomad_client: Nomad) -> List[str]:
# Filter out the Parameterized Batch jobs, because those don't themselves have logs;
@@ -113,6 +125,14 @@ def _write_nomad_logs(
for alloc in allocs:
for task in alloc.tasks:
+ # publish task events
+ events = task.get_events()
+ if events:
+ filename = f"{task.name}.events.{alloc.short_alloc_id()}.log"
+ with (write_to_dir / filename).open("w") as file:
+ file.write(events)
+
+ # publish logs
for output_type in OUTPUT_TYPES:
logs = task.get_logs(nomad_client, output_type)
if not logs:
