feat: Grafana dashboard ConfigMap, PrometheusRule SLO alerts, RapidAPI listing copy
- flux/monitoring/grafana-dashboard-apis.yaml: Grafana dashboard ConfigMap labelled grafana_dashboard=1, covering request rate, P50/P95/P99 latency, 5xx error rate, and data freshness for all three APIs. Closes leeworks-agents/api-company#22 - flux/monitoring/prometheusrule-apis.yaml: PrometheusRule with four alert rules: APIHighErrorRate (warning >5%, critical >20%), APIHighLatency (P95 >2s), APIDataStale (>30 days), APIDown (up==0 for 2 min). All labelled severity + team: api-company. Closes leeworks-agents/api-company#23 - docs/rapidapi-listings.md: Full RapidAPI listing copy for zip-enrichment, holidays, and air-quality — API name, tagline (<=120 chars), short description (<=300 chars), long description, category, plan table (Basic $9/Pro $19/ Ultra $49), endpoint descriptions, and keywords. Closes leeworks-agents/api-company#24 - flux/monitoring/kustomization.yaml: added grafana-dashboard-apis.yaml and prometheusrule-apis.yaml to resources list.
This commit is contained in:
agent-company
@@ -0,0 +1,190 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: grafana-dashboard-apis
|
||||
namespace: monitoring
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
data:
|
||||
api-dashboard.json: |
|
||||
{
|
||||
"annotations": { "list": [] },
|
||||
"description": "Request rate, latency, error rate, and data freshness for zip-enrichment, holidays, and air-quality APIs",
|
||||
"editable": true,
|
||||
"graphTooltip": 1,
|
||||
"panels": [
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"id": 1,
|
||||
"title": "Request Rate",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "palette-classic" },
|
||||
"custom": { "axisLabel": "requests/sec", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never" },
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 },
|
||||
"id": 2,
|
||||
"options": {
|
||||
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
|
||||
"tooltip": { "mode": "multi" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (api, route) (rate(api_requests_total{api=~\"zip-enrichment|holidays|air-quality\"}[5m]))",
|
||||
"legendFormat": "{{api}} {{route}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Request Rate by API / Route",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "palette-classic" },
|
||||
"custom": { "axisLabel": "error fraction", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 0.05 },
|
||||
{ "color": "red", "value": 0.20 }
|
||||
]
|
||||
},
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 },
|
||||
"id": 3,
|
||||
"options": {
|
||||
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
|
||||
"tooltip": { "mode": "multi" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (api) (rate(api_requests_total{api=~\"zip-enrichment|holidays|air-quality\",status=~\"5..\"}[5m])) / sum by (api) (rate(api_requests_total{api=~\"zip-enrichment|holidays|air-quality\"}[5m]))",
|
||||
"legendFormat": "{{api}} 5xx error rate",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "5xx Error Rate by API",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 },
|
||||
"id": 4,
|
||||
"title": "Latency P50 / P95 / P99",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "palette-classic" },
|
||||
"custom": { "axisLabel": "seconds", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 1.0 },
|
||||
{ "color": "red", "value": 2.0 }
|
||||
]
|
||||
},
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 10 },
|
||||
"id": 5,
|
||||
"options": {
|
||||
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
|
||||
"tooltip": { "mode": "multi" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, sum by (api, le) (rate(api_response_duration_seconds_bucket{api=~\"zip-enrichment|holidays|air-quality\"}[5m])))",
|
||||
"legendFormat": "P50 {{api}}",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum by (api, le) (rate(api_response_duration_seconds_bucket{api=~\"zip-enrichment|holidays|air-quality\"}[5m])))",
|
||||
"legendFormat": "P95 {{api}}",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum by (api, le) (rate(api_response_duration_seconds_bucket{api=~\"zip-enrichment|holidays|air-quality\"}[5m])))",
|
||||
"legendFormat": "P99 {{api}}",
|
||||
"refId": "C"
|
||||
}
|
||||
],
|
||||
"title": "Response Latency P50 / P95 / P99 by API",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 },
|
||||
"id": 6,
|
||||
"title": "Data Freshness",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "thresholds" },
|
||||
"mappings": [],
|
||||
"max": 2592000,
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 1296000 },
|
||||
{ "color": "red", "value": 2592000 }
|
||||
]
|
||||
},
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 19 },
|
||||
"id": 7,
|
||||
"options": {
|
||||
"orientation": "horizontal",
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "api_data_freshness_seconds{api=~\"zip-enrichment|holidays|air-quality\"}",
|
||||
"legendFormat": "{{api}} ({{dataset}})",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Data Freshness — alert threshold at 30 days (2592000 s)",
|
||||
"type": "gauge"
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 38,
|
||||
"tags": ["api-company", "leeworks"],
|
||||
"templating": { "list": [] },
|
||||
"time": { "from": "now-3h", "to": "now" },
|
||||
"timepicker": {},
|
||||
"timezone": "browser",
|
||||
"title": "leeworks.dev API Metrics",
|
||||
"uid": "leeworks-api-metrics",
|
||||
"version": 1
|
||||
}
|
||||
@@ -6,3 +6,5 @@ resources:
|
||||
- helmrelease.yaml
|
||||
- gatus-helmrepository.yaml
|
||||
- gatus-helmrelease.yaml
|
||||
- grafana-dashboard-apis.yaml
|
||||
- prometheusrule-apis.yaml
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: api-company-slo-alerts
|
||||
namespace: monitoring
|
||||
labels:
|
||||
# Must match kube-prometheus-stack's ruleSelector (release label is standard)
|
||||
app: kube-prometheus-stack
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
groups:
|
||||
- name: api-company.slo
|
||||
interval: 1m
|
||||
rules:
|
||||
# -------------------------------------------------------------------
|
||||
# APIHighErrorRate — warning: >5% 5xx over 5 min
|
||||
# -------------------------------------------------------------------
|
||||
- alert: APIHighErrorRate
|
||||
expr: |
|
||||
(
|
||||
sum by (job) (rate(api_requests_total{status=~"5..", job=~"zip|holidays|air-quality"}[5m]))
|
||||
/
|
||||
sum by (job) (rate(api_requests_total{job=~"zip|holidays|air-quality"}[5m]))
|
||||
) > 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
team: api-company
|
||||
annotations:
|
||||
summary: "High 5xx error rate on {{ $labels.job }}"
|
||||
description: "{{ $labels.job }} 5xx error rate is {{ $value | humanizePercentage }} over the last 5 minutes (threshold: 5%)."
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# APIHighErrorRate — critical: >20% 5xx over 5 min
|
||||
# -------------------------------------------------------------------
|
||||
- alert: APIHighErrorRate
|
||||
expr: |
|
||||
(
|
||||
sum by (job) (rate(api_requests_total{status=~"5..", job=~"zip|holidays|air-quality"}[5m]))
|
||||
/
|
||||
sum by (job) (rate(api_requests_total{job=~"zip|holidays|air-quality"}[5m]))
|
||||
) > 0.20
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
team: api-company
|
||||
annotations:
|
||||
summary: "Critical 5xx error rate on {{ $labels.job }}"
|
||||
description: "{{ $labels.job }} 5xx error rate is {{ $value | humanizePercentage }} over the last 5 minutes (threshold: 20%)."
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# APIHighLatency — P95 > 2 s over 5 min
|
||||
# -------------------------------------------------------------------
|
||||
- alert: APIHighLatency
|
||||
expr: |
|
||||
histogram_quantile(
|
||||
0.95,
|
||||
sum by (job, le) (rate(api_response_duration_seconds_bucket{job=~"zip|holidays|air-quality"}[5m]))
|
||||
) > 2
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
team: api-company
|
||||
annotations:
|
||||
summary: "High P95 latency on {{ $labels.job }}"
|
||||
description: "{{ $labels.job }} P95 response time is {{ $value | humanizeDuration }} (threshold: 2s)."
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# APIDataStale — data freshness > 30 days
|
||||
# -------------------------------------------------------------------
|
||||
- alert: APIDataStale
|
||||
expr: |
|
||||
api_data_freshness_seconds{job=~"zip|holidays|air-quality"} > 2592000
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
team: api-company
|
||||
annotations:
|
||||
summary: "Stale dataset on {{ $labels.job }} ({{ $labels.dataset }})"
|
||||
description: "{{ $labels.job }} dataset '{{ $labels.dataset }}' has not been re-seeded in {{ $value | humanizeDuration }} (threshold: 30 days). Re-seed required."
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# APIDown — any API job absent for 2 min
|
||||
# -------------------------------------------------------------------
|
||||
- alert: APIDown
|
||||
expr: |
|
||||
absent(up{job=~"zip|holidays|air-quality"} == 1)
|
||||
or
|
||||
up{job=~"zip|holidays|air-quality"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
team: api-company
|
||||
annotations:
|
||||
summary: "API service {{ $labels.job }} is down"
|
||||
description: "Prometheus target {{ $labels.job }} has been unreachable for more than 2 minutes."
|
||||
Reference in New Issue
Block a user