agent-company
1a1a109aa3
Validate Flux manifests / kustomize-build (pull_request) Failing after 17s
- Fix APIDataStale 'for' duration from 5m to 30m (issue spec requires 30m) - Add APIDataCriticallyStale alert at 60-day threshold (5184000s) with for: 1h and severity: critical Both alerts fire on api_data_freshness_seconds metric for all three API jobs. Existing APIHighErrorRate and APIHighLatency alerts are unmodified. kustomize build flux/ passes. Closes leeworks-agents/api-company#88
111 lines
4.7 KiB
YAML
111 lines
4.7 KiB
YAML
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: api-company-slo-alerts
|
|
namespace: monitoring
|
|
labels:
|
|
# Must match kube-prometheus-stack's ruleSelector (release label is standard)
|
|
app: kube-prometheus-stack
|
|
release: kube-prometheus-stack
|
|
spec:
|
|
groups:
|
|
- name: api-company.slo
|
|
interval: 1m
|
|
rules:
|
|
# -------------------------------------------------------------------
|
|
# APIHighErrorRate — warning: >5% 5xx over 5 min
|
|
# -------------------------------------------------------------------
|
|
- alert: APIHighErrorRate
|
|
expr: |
|
|
(
|
|
sum by (job) (rate(api_requests_total{status=~"5..", job=~"zip|holidays|air-quality"}[5m]))
|
|
/
|
|
sum by (job) (rate(api_requests_total{job=~"zip|holidays|air-quality"}[5m]))
|
|
) > 0.05
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
team: api-company
|
|
annotations:
|
|
summary: "High 5xx error rate on {{ $labels.job }}"
|
|
description: "{{ $labels.job }} 5xx error rate is {{ $value | humanizePercentage }} over the last 5 minutes (threshold: 5%)."
|
|
|
|
# -------------------------------------------------------------------
|
|
# APIHighErrorRate — critical: >20% 5xx over 5 min
|
|
# -------------------------------------------------------------------
|
|
- alert: APIHighErrorRate
|
|
expr: |
|
|
(
|
|
sum by (job) (rate(api_requests_total{status=~"5..", job=~"zip|holidays|air-quality"}[5m]))
|
|
/
|
|
sum by (job) (rate(api_requests_total{job=~"zip|holidays|air-quality"}[5m]))
|
|
) > 0.20
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
team: api-company
|
|
annotations:
|
|
summary: "Critical 5xx error rate on {{ $labels.job }}"
|
|
description: "{{ $labels.job }} 5xx error rate is {{ $value | humanizePercentage }} over the last 5 minutes (threshold: 20%)."
|
|
|
|
# -------------------------------------------------------------------
|
|
# APIHighLatency — P95 > 2 s over 5 min
|
|
# -------------------------------------------------------------------
|
|
- alert: APIHighLatency
|
|
expr: |
|
|
histogram_quantile(
|
|
0.95,
|
|
sum by (job, le) (rate(api_response_duration_seconds_bucket{job=~"zip|holidays|air-quality"}[5m]))
|
|
) > 2
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
team: api-company
|
|
annotations:
|
|
summary: "High P95 latency on {{ $labels.job }}"
|
|
description: "{{ $labels.job }} P95 response time is {{ $value | humanizeDuration }} (threshold: 2s)."
|
|
|
|
# -------------------------------------------------------------------
|
|
# APIDataStale — data freshness > 30 days
|
|
# -------------------------------------------------------------------
|
|
- alert: APIDataStale
|
|
expr: |
|
|
api_data_freshness_seconds{job=~"zip|holidays|air-quality"} > 2592000
|
|
for: 30m
|
|
labels:
|
|
severity: warning
|
|
team: api-company
|
|
annotations:
|
|
summary: "Stale dataset on {{ $labels.job }} ({{ $labels.dataset }})"
|
|
description: "{{ $labels.job }} dataset '{{ $labels.dataset }}' has not been re-seeded in {{ $value | humanizeDuration }} (threshold: 30 days). Re-seed required."
|
|
|
|
# -------------------------------------------------------------------
|
|
# APIDataCriticallyStale — data freshness > 60 days
|
|
# -------------------------------------------------------------------
|
|
- alert: APIDataCriticallyStale
|
|
expr: |
|
|
api_data_freshness_seconds{job=~"zip|holidays|air-quality"} > 5184000
|
|
for: 1h
|
|
labels:
|
|
severity: critical
|
|
team: api-company
|
|
annotations:
|
|
summary: "API data is critically stale on {{ $labels.job }}"
|
|
description: "{{ $labels.job }} data has not been re-seeded in more than 60 days ({{ $value | humanizeDuration }})"
|
|
|
|
# -------------------------------------------------------------------
|
|
# APIDown — any API job absent for 2 min
|
|
# -------------------------------------------------------------------
|
|
- alert: APIDown
|
|
expr: |
|
|
absent(up{job=~"zip|holidays|air-quality"} == 1)
|
|
or
|
|
up{job=~"zip|holidays|air-quality"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
team: api-company
|
|
annotations:
|
|
summary: "API service {{ $labels.job }} is down"
|
|
description: "Prometheus target {{ $labels.job }} has been unreachable for more than 2 minutes."
|