AI-Engineer
1bbcb8e28a
Validate Flux manifests / kustomize-build (pull_request) Failing after 17s
- flux/monitoring/prometheusrule-apis.yaml: extend all 5 SLO alert rules to include vin-decoder in the job regex filter - flux/monitoring/grafana-dashboard-apis.yaml: update all PromQL expressions to include vin-decoder in the api regex filter; update dashboard description - flux/monitoring/vin-decoder-dashboard.yaml: new dedicated Grafana dashboard for VIN Decoder showing request rate by route, 5xx error rate, P50/P95/P99 latency by route, and data freshness gauge - flux/monitoring/kustomization.yaml: add vin-decoder-dashboard.yaml - flux/vin-decoder/servicemonitor.yaml: Prometheus ServiceMonitor for auto-discovery of /metrics on the vin-decoder service - flux/vin-decoder/kustomization.yaml: add servicemonitor.yaml Closes leeworks-agents/api-company#129
111 lines
4.8 KiB
YAML
111 lines
4.8 KiB
YAML
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: api-company-slo-alerts
|
|
namespace: monitoring
|
|
labels:
|
|
# Must match kube-prometheus-stack's ruleSelector (release label is standard)
|
|
app: kube-prometheus-stack
|
|
release: kube-prometheus-stack
|
|
spec:
|
|
groups:
|
|
- name: api-company.slo
|
|
interval: 1m
|
|
rules:
|
|
# -------------------------------------------------------------------
|
|
# APIHighErrorRate — warning: >5% 5xx over 5 min
|
|
# -------------------------------------------------------------------
|
|
- alert: APIHighErrorRate
|
|
expr: |
|
|
(
|
|
sum by (job) (rate(api_requests_total{status=~"5..", job=~"zip|holidays|air-quality|vin-decoder"}[5m]))
|
|
/
|
|
sum by (job) (rate(api_requests_total{job=~"zip|holidays|air-quality|vin-decoder"}[5m]))
|
|
) > 0.05
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
team: api-company
|
|
annotations:
|
|
summary: "High 5xx error rate on {{ $labels.job }}"
|
|
description: "{{ $labels.job }} 5xx error rate is {{ $value | humanizePercentage }} over the last 5 minutes (threshold: 5%)."
|
|
|
|
# -------------------------------------------------------------------
|
|
# APIHighErrorRate — critical: >20% 5xx over 5 min
|
|
# -------------------------------------------------------------------
|
|
- alert: APIHighErrorRate
|
|
expr: |
|
|
(
|
|
sum by (job) (rate(api_requests_total{status=~"5..", job=~"zip|holidays|air-quality|vin-decoder"}[5m]))
|
|
/
|
|
sum by (job) (rate(api_requests_total{job=~"zip|holidays|air-quality|vin-decoder"}[5m]))
|
|
) > 0.20
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
team: api-company
|
|
annotations:
|
|
summary: "Critical 5xx error rate on {{ $labels.job }}"
|
|
description: "{{ $labels.job }} 5xx error rate is {{ $value | humanizePercentage }} over the last 5 minutes (threshold: 20%)."
|
|
|
|
# -------------------------------------------------------------------
|
|
# APIHighLatency — P95 > 2 s over 5 min
|
|
# -------------------------------------------------------------------
|
|
- alert: APIHighLatency
|
|
expr: |
|
|
histogram_quantile(
|
|
0.95,
|
|
sum by (job, le) (rate(api_response_duration_seconds_bucket{job=~"zip|holidays|air-quality|vin-decoder"}[5m]))
|
|
) > 2
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
team: api-company
|
|
annotations:
|
|
summary: "High P95 latency on {{ $labels.job }}"
|
|
description: "{{ $labels.job }} P95 response time is {{ $value | humanizeDuration }} (threshold: 2s)."
|
|
|
|
# -------------------------------------------------------------------
|
|
# APIDataStale — data freshness > 30 days
|
|
# -------------------------------------------------------------------
|
|
- alert: APIDataStale
|
|
expr: |
|
|
api_data_freshness_seconds{job=~"zip|holidays|air-quality|vin-decoder"} > 2592000
|
|
for: 30m
|
|
labels:
|
|
severity: warning
|
|
team: api-company
|
|
annotations:
|
|
summary: "Stale dataset on {{ $labels.job }} ({{ $labels.dataset }})"
|
|
description: "{{ $labels.job }} dataset '{{ $labels.dataset }}' has not been re-seeded in {{ $value | humanizeDuration }} (threshold: 30 days). Re-seed required."
|
|
|
|
# -------------------------------------------------------------------
|
|
# APIDataCriticallyStale — data freshness > 60 days
|
|
# -------------------------------------------------------------------
|
|
- alert: APIDataCriticallyStale
|
|
expr: |
|
|
api_data_freshness_seconds{job=~"zip|holidays|air-quality|vin-decoder"} > 5184000
|
|
for: 1h
|
|
labels:
|
|
severity: critical
|
|
team: api-company
|
|
annotations:
|
|
summary: "API data is critically stale on {{ $labels.job }}"
|
|
description: "{{ $labels.job }} data has not been re-seeded in more than 60 days ({{ $value | humanizeDuration }})"
|
|
|
|
# -------------------------------------------------------------------
|
|
# APIDown — any API job absent for 2 min
|
|
# -------------------------------------------------------------------
|
|
- alert: APIDown
|
|
expr: |
|
|
absent(up{job=~"zip|holidays|air-quality|vin-decoder"} == 1)
|
|
or
|
|
up{job=~"zip|holidays|air-quality|vin-decoder"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
team: api-company
|
|
annotations:
|
|
summary: "API service {{ $labels.job }} is down"
|
|
description: "Prometheus target {{ $labels.job }} has been unreachable for more than 2 minutes."
|