message: track allocated and total ram use metrics
diff: @@ -45,7 +45,7 @@ use datasize::DataSize;
  use futures::{future::BoxFuture, FutureExt};
  use jemalloc_ctl::{epoch as jemalloc_epoch, stats::allocated as jemalloc_allocated};
  use once_cell::sync::Lazy;
  -use prometheus::{self, Histogram, HistogramOpts, IntCounter, Registry};
  +use prometheus::{self, Histogram, HistogramOpts, IntCounter, IntGauge, Registry};
  use quanta::IntoNanoseconds;
  use serde::Serialize;
  use tokio::time::{Duration, Instant};
  @@ -265,6 +265,12 @@ struct RunnerMetrics {
  /// Handle to the metrics registry, in case we need to unregister.
  registry: Registry,
  +
  + /// Total allocated RAM in bytes, as reported by jemalloc.
  + allocated_ram_bytes: IntGauge,
  +
  + /// Total system RAM in bytes, as reported by sys-info.
  + total_ram_bytes: IntGauge,
  }
  impl RunnerMetrics {
  @@ -301,13 +307,21 @@ impl RunnerMetrics {
  ]),
  )?;
  + let allocated_ram_bytes =
  + IntGauge::new("allocated_ram_bytes", "total allocated ram in bytes")?;
  + let total_ram_bytes = IntGauge::new("total_ram_bytes", "total system ram in bytes")?;
  +
  registry.register(Box::new(events.clone()))?;
  registry.register(Box::new(event_dispatch_duration.clone()))?;
  + registry.register(Box::new(allocated_ram_bytes.clone()))?;
  + registry.register(Box::new(total_ram_bytes.clone()))?;
  Ok(RunnerMetrics {
  events,
  event_dispatch_duration,
  registry: registry.clone(),
  + allocated_ram_bytes,
  + total_ram_bytes,
  })
  }
  }
  @@ -320,6 +334,12 @@ impl Drop for RunnerMetrics {
  self.registry
  .unregister(Box::new(self.event_dispatch_duration.clone()))
  .expect("did not expect deregistering event_dispatch_duration to fail");
  + self.registry
  + .unregister(Box::new(self.allocated_ram_bytes.clone()))
  + .expect("did not expect deregistering allocated_ram_bytes to fail");
  + self.registry
  + .unregister(Box::new(self.total_ram_bytes.clone()))
  + .expect("did not expect deregistering total_ram_bytes to fail");
  }
  }
  @@ -425,6 +445,8 @@ where
  if let Some(AllocatedMem { allocated, total }) = Self::get_allocated_memory() {
  debug!(%allocated, %total, "memory allocated");
  + self.metrics.allocated_ram_bytes.set(allocated as i64);
  + self.metrics.total_ram_bytes.set(total as i64);
  if let Some(threshold_mb) = *MEM_DUMP_THRESHOLD_MB {
  let threshold_bytes = threshold_mb * 1024 * 1024;
  if allocated >= threshold_bytes && self.last_queue_dump.is_none() {
  