agent-company
c748e8d074
- flux/monitoring/grafana-dashboard-apis.yaml: Grafana dashboard ConfigMap labelled grafana_dashboard=1, covering request rate, P50/P95/P99 latency, 5xx error rate, and data freshness for all three APIs. Closes leeworks-agents/api-company#22 - flux/monitoring/prometheusrule-apis.yaml: PrometheusRule with four alert rules: APIHighErrorRate (warning >5%, critical >20%), APIHighLatency (P95 >2s), APIDataStale (>30 days), APIDown (up==0 for 2 min). All labelled severity + team: api-company. Closes leeworks-agents/api-company#23 - docs/rapidapi-listings.md: Full RapidAPI listing copy for zip-enrichment, holidays, and air-quality — API name, tagline (<=120 chars), short description (<=300 chars), long description, category, plan table (Basic $9/Pro $19/ Ultra $49), endpoint descriptions, and keywords. Closes leeworks-agents/api-company#24 - flux/monitoring/kustomization.yaml: added grafana-dashboard-apis.yaml and prometheusrule-apis.yaml to resources list.
97 lines
4.1 KiB
YAML
97 lines
4.1 KiB
YAML
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: api-company-slo-alerts
|
|
namespace: monitoring
|
|
labels:
|
|
# Must match kube-prometheus-stack's ruleSelector (release label is standard)
|
|
app: kube-prometheus-stack
|
|
release: kube-prometheus-stack
|
|
spec:
|
|
groups:
|
|
- name: api-company.slo
|
|
interval: 1m
|
|
rules:
|
|
# -------------------------------------------------------------------
|
|
# APIHighErrorRate — warning: >5% 5xx over 5 min
|
|
# -------------------------------------------------------------------
|
|
- alert: APIHighErrorRate
|
|
expr: |
|
|
(
|
|
sum by (job) (rate(api_requests_total{status=~"5..", job=~"zip|holidays|air-quality"}[5m]))
|
|
/
|
|
sum by (job) (rate(api_requests_total{job=~"zip|holidays|air-quality"}[5m]))
|
|
) > 0.05
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
team: api-company
|
|
annotations:
|
|
summary: "High 5xx error rate on {{ $labels.job }}"
|
|
description: "{{ $labels.job }} 5xx error rate is {{ $value | humanizePercentage }} over the last 5 minutes (threshold: 5%)."
|
|
|
|
# -------------------------------------------------------------------
|
|
# APIHighErrorRate — critical: >20% 5xx over 5 min
|
|
# -------------------------------------------------------------------
|
|
- alert: APIHighErrorRate
|
|
expr: |
|
|
(
|
|
sum by (job) (rate(api_requests_total{status=~"5..", job=~"zip|holidays|air-quality"}[5m]))
|
|
/
|
|
sum by (job) (rate(api_requests_total{job=~"zip|holidays|air-quality"}[5m]))
|
|
) > 0.20
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
team: api-company
|
|
annotations:
|
|
summary: "Critical 5xx error rate on {{ $labels.job }}"
|
|
description: "{{ $labels.job }} 5xx error rate is {{ $value | humanizePercentage }} over the last 5 minutes (threshold: 20%)."
|
|
|
|
# -------------------------------------------------------------------
|
|
# APIHighLatency — P95 > 2 s over 5 min
|
|
# -------------------------------------------------------------------
|
|
- alert: APIHighLatency
|
|
expr: |
|
|
histogram_quantile(
|
|
0.95,
|
|
sum by (job, le) (rate(api_response_duration_seconds_bucket{job=~"zip|holidays|air-quality"}[5m]))
|
|
) > 2
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
team: api-company
|
|
annotations:
|
|
summary: "High P95 latency on {{ $labels.job }}"
|
|
description: "{{ $labels.job }} P95 response time is {{ $value | humanizeDuration }} (threshold: 2s)."
|
|
|
|
# -------------------------------------------------------------------
|
|
# APIDataStale — data freshness > 30 days
|
|
# -------------------------------------------------------------------
|
|
- alert: APIDataStale
|
|
expr: |
|
|
api_data_freshness_seconds{job=~"zip|holidays|air-quality"} > 2592000
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
team: api-company
|
|
annotations:
|
|
summary: "Stale dataset on {{ $labels.job }} ({{ $labels.dataset }})"
|
|
description: "{{ $labels.job }} dataset '{{ $labels.dataset }}' has not been re-seeded in {{ $value | humanizeDuration }} (threshold: 30 days). Re-seed required."
|
|
|
|
# -------------------------------------------------------------------
|
|
# APIDown — any API job absent for 2 min
|
|
# -------------------------------------------------------------------
|
|
- alert: APIDown
|
|
expr: |
|
|
absent(up{job=~"zip|holidays|air-quality"} == 1)
|
|
or
|
|
up{job=~"zip|holidays|air-quality"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
team: api-company
|
|
annotations:
|
|
summary: "API service {{ $labels.job }} is down"
|
|
description: "Prometheus target {{ $labels.job }} has been unreachable for more than 2 minutes."
|