feat: emit http_request_limit_rejected metric
Add the "http_request_limit_rejected" metric that is incremented once
each time a request is dropped due to the simultaneous request service
protection limit being exceeded.
This metric will allow us to effectively alert on router saturation, and
the increase rate/second will help inform us in how much capacity needs
adding.
@@ -227,6 +227,7 @@ pub struct HttpDelegate<D, T = SystemProvider> {
write_metric_tables: U64Counter,
write_metric_body_size: U64Counter,
delete_metric_body_size: U64Counter,
+ request_limit_rejected: U64Counter,
}
impl<D> HttpDelegate<D, SystemProvider> {
@@ -271,6 +272,12 @@ impl<D> HttpDelegate<D, SystemProvider> {
"cumulative byte size of successfully routed (decompressed) delete requests",
)
.recorder(&[]);
+ let request_limit_rejected = metrics
+ .register_metric::<U64Counter>(
+ "http_request_limit_rejected",
+ "number of HTTP requests rejected due to exceeding parallel request limit",
+ )
+ .recorder(&[]);
Self {
max_request_bytes,
@@ -282,6 +289,7 @@ impl<D> HttpDelegate<D, SystemProvider> {
write_metric_tables,
write_metric_body_size,
delete_metric_body_size,
+ request_limit_rejected,
}
}
}
@@ -305,6 +313,7 @@ where
Ok(p) => p,
Err(TryAcquireError::NoPermits) => {
error!("simultaneous request limit exceeded - dropping request");
+ self.request_limit_rejected.inc(1);
return Err(Error::RequestLimit);
}
Err(e) => panic!("request limiter error: {}", e),
@@ -520,9 +529,10 @@ mod tests {
.expect("failed to get observer")
.fetch();
- assert!(counter > 0, "metric {} did not record any values", name);
if let Some(want) = value {
assert_eq!(want, counter, "metric does not have expected value");
+ } else {
+ assert!(counter > 0, "metric {} did not record any values", name);
}
}
@@ -1177,6 +1187,8 @@ mod tests {
// immediate drop of any subsequent requests.
//
+ assert_metric_hit(&*metrics, "http_request_limit_rejected", Some(0));
+
// Retain this tx handle for the second request and use it to prove the
// request dropped before anything was read from the body - the request
// should error _before_ anything is sent over tx, and subsequently
@@ -1200,6 +1212,9 @@ mod tests {
.expect_err("second request should be rejected");
assert_matches!(err, Error::RequestLimit);
+ // Ensure the "rejected requests" metric was incremented
+ assert_metric_hit(&*metrics, "http_request_limit_rejected", Some(1));
+
// Prove the dropped request body is not being read:
body_2_tx
.send(Ok("wat"))
@@ -1232,5 +1247,8 @@ mod tests {
.with_timeout_panic(Duration::from_secs(1))
.await
.expect("empty write should succeed");
+
+ // And the request rejected metric must remain unchanged
+ assert_metric_hit(&*metrics, "http_request_limit_rejected", Some(1));
}
}
