# Source: firehose-exporter
---

- type: replace
  path: /instance_groups/name=prometheus2/jobs/name=prometheus2/properties/prometheus/custom_rules?/-
  value:
    name: CCFailedJobCount
    rules:
      - record: firehose_value_metric_cc_failed_job_count_total:avg30m
        expr: avg_over_time(firehose_value_metric_cc_failed_job_count_total[30m])

      - &alert
        alert: CCFailedJobCount_Warning
        expr: max(delta(firehose_value_metric_cc_failed_job_count_total:avg30m[30m])) > 3
        labels:
          severity: warning
          notify: email
        annotations:
          summary: "Cloud Controller API failed job count"
          description: "Amount of failed jobs in Cloud Controller API grew considerably in the last hour: {{ $value | printf \"%.0f\" }}. See logit.io: '@source.component:cloud_controller_worker AND @level:ERROR'"

      - <<: *alert
        alert: CCFailedJobCount_Critical
        expr: max(delta(firehose_value_metric_cc_failed_job_count_total:avg30m[30m])) > 5
        labels:
          severity: critical
          notify: email

Migrate "Cloud Controller API failed job count" monitor to Prometheus
Migrate "Cloud Controller API failed job count" monitor to Prometheus

Original DataDog monitor:
change(max(last_1m),last_30m):max:cf.cc.failed_job_count.total{deployment:prod}.rollup(avg, 30) > 5
