feat: extend monitoring to include VIN Decoder API

- flux/monitoring/prometheusrule-apis.yaml: extend all 5 SLO alert rules to include vin-decoder in the job regex filter - flux/monitoring/grafana-dashboard-apis.yaml: update all PromQL expressions to include vin-decoder in the api regex filter; update dashboard description - flux/monitoring/vin-decoder-dashboard.yaml: new dedicated Grafana dashboard for VIN Decoder showing request rate by route, 5xx error rate, P50/P95/P99 latency by route, and data freshness gauge - flux/monitoring/kustomization.yaml: add vin-decoder-dashboard.yaml - flux/vin-decoder/servicemonitor.yaml: Prometheus ServiceMonitor for auto-discovery of /metrics on the vin-decoder service - flux/vin-decoder/kustomization.yaml: add servicemonitor.yaml Closes leeworks-agents/api-company#129
2026-05-31 20:11:03 +00:00
parent cd41724f39
commit 1bbcb8e28a
6 changed files with 188 additions and 10 deletions
@@ -9,7 +9,7 @@ data:
  api-dashboard.json: |
    {
      "annotations": { "list": [] },
-      "description": "Request rate, latency, error rate, and data freshness for zip-enrichment, holidays, and air-quality APIs",
+      "description": "Request rate, latency, error rate, and data freshness for zip-enrichment, holidays, air-quality, and vin-decoder APIs",
      "editable": true,
      "graphTooltip": 1,
      "panels": [
@@ -8,3 +8,4 @@ resources:
  - gatus-helmrelease.yaml
  - grafana-dashboard-apis.yaml
  - prometheusrule-apis.yaml
+  - vin-decoder-dashboard.yaml
@@ -18,9 +18,9 @@ spec:
        - alert: APIHighErrorRate
          expr: |
            (
-              sum by (job) (rate(api_requests_total{status=~"5..", job=~"zip|holidays|air-quality"}[5m]))
+              sum by (job) (rate(api_requests_total{status=~"5..", job=~"zip|holidays|air-quality|vin-decoder"}[5m]))
              /
-              sum by (job) (rate(api_requests_total{job=~"zip|holidays|air-quality"}[5m]))
+              sum by (job) (rate(api_requests_total{job=~"zip|holidays|air-quality|vin-decoder"}[5m]))
            ) > 0.05
          for: 5m
          labels:
@@ -36,9 +36,9 @@ spec:
        - alert: APIHighErrorRate
          expr: |
            (
-              sum by (job) (rate(api_requests_total{status=~"5..", job=~"zip|holidays|air-quality"}[5m]))
+              sum by (job) (rate(api_requests_total{status=~"5..", job=~"zip|holidays|air-quality|vin-decoder"}[5m]))
              /
-              sum by (job) (rate(api_requests_total{job=~"zip|holidays|air-quality"}[5m]))
+              sum by (job) (rate(api_requests_total{job=~"zip|holidays|air-quality|vin-decoder"}[5m]))
            ) > 0.20
          for: 5m
          labels:
@@ -55,7 +55,7 @@ spec:
          expr: |
            histogram_quantile(
              0.95,
-              sum by (job, le) (rate(api_response_duration_seconds_bucket{job=~"zip|holidays|air-quality"}[5m]))
+              sum by (job, le) (rate(api_response_duration_seconds_bucket{job=~"zip|holidays|air-quality|vin-decoder"}[5m]))
            ) > 2
          for: 5m
          labels:
@@ -70,7 +70,7 @@ spec:
        # -------------------------------------------------------------------
        - alert: APIDataStale
          expr: |
-            api_data_freshness_seconds{job=~"zip|holidays|air-quality"} > 2592000
+            api_data_freshness_seconds{job=~"zip|holidays|air-quality|vin-decoder"} > 2592000
          for: 30m
          labels:
            severity: warning
@@ -84,7 +84,7 @@ spec:
        # -------------------------------------------------------------------
        - alert: APIDataCriticallyStale
          expr: |
-            api_data_freshness_seconds{job=~"zip|holidays|air-quality"} > 5184000
+            api_data_freshness_seconds{job=~"zip|holidays|air-quality|vin-decoder"} > 5184000
          for: 1h
          labels:
            severity: critical
@@ -98,9 +98,9 @@ spec:
        # -------------------------------------------------------------------
        - alert: APIDown
          expr: |
-            absent(up{job=~"zip|holidays|air-quality"} == 1)
+            absent(up{job=~"zip|holidays|air-quality|vin-decoder"} == 1)
            or
-            up{job=~"zip|holidays|air-quality"} == 0
+            up{job=~"zip|holidays|air-quality|vin-decoder"} == 0
          for: 2m
          labels:
            severity: critical
@@ -0,0 +1,161 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-dashboard-vin-decoder
+  namespace: monitoring
+  labels:
+    grafana_dashboard: "1"
+data:
+  vin-decoder-dashboard.json: |
+    {
+      "annotations": { "list": [] },
+      "description": "Request rate, latency, error rate, and cache hit ratio for the VIN Decoder API",
+      "editable": true,
+      "graphTooltip": 1,
+      "panels": [
+        {
+          "collapsed": false,
+          "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+          "id": 1,
+          "title": "VIN Decoder — Request Rate",
+          "type": "row"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "palette-classic" },
+              "custom": { "axisLabel": "requests/sec", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never" },
+              "unit": "reqps"
+            },
+            "overrides": []
+          },
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 },
+          "id": 2,
+          "options": {
+            "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
+            "tooltip": { "mode": "multi" }
+          },
+          "targets": [
+            {
+              "expr": "sum by (route) (rate(api_requests_total{api=\"vin-decoder\"}[5m]))",
+              "legendFormat": "{{route}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Request Rate by Route",
+          "type": "timeseries"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "palette-classic" },
+              "custom": { "axisLabel": "", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never" },
+              "unit": "percentunit"
+            },
+            "overrides": []
+          },
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 },
+          "id": 3,
+          "options": {
+            "legend": { "calcs": ["mean", "last"], "displayMode": "table", "placement": "bottom" },
+            "tooltip": { "mode": "multi" }
+          },
+          "targets": [
+            {
+              "expr": "sum(rate(api_requests_total{api=\"vin-decoder\",status_code=~\"5..\"}[5m])) / sum(rate(api_requests_total{api=\"vin-decoder\"}[5m]))",
+              "legendFormat": "5xx error rate",
+              "refId": "A"
+            }
+          ],
+          "title": "5xx Error Rate",
+          "type": "timeseries"
+        },
+        {
+          "collapsed": false,
+          "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 },
+          "id": 10,
+          "title": "VIN Decoder — Latency",
+          "type": "row"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "palette-classic" },
+              "custom": { "axisLabel": "seconds", "drawStyle": "line", "fillOpacity": 5, "lineWidth": 1, "showPoints": "never" },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": { "h": 8, "w": 24, "x": 0, "y": 10 },
+          "id": 4,
+          "options": {
+            "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
+            "tooltip": { "mode": "multi" }
+          },
+          "targets": [
+            {
+              "expr": "histogram_quantile(0.50, sum by (route, le) (rate(api_response_duration_seconds_bucket{api=\"vin-decoder\"}[5m])))",
+              "legendFormat": "P50 {{route}}",
+              "refId": "A"
+            },
+            {
+              "expr": "histogram_quantile(0.95, sum by (route, le) (rate(api_response_duration_seconds_bucket{api=\"vin-decoder\"}[5m])))",
+              "legendFormat": "P95 {{route}}",
+              "refId": "B"
+            },
+            {
+              "expr": "histogram_quantile(0.99, sum by (route, le) (rate(api_response_duration_seconds_bucket{api=\"vin-decoder\"}[5m])))",
+              "legendFormat": "P99 {{route}}",
+              "refId": "C"
+            }
+          ],
+          "title": "Response Latency P50 / P95 / P99 by Route",
+          "type": "timeseries"
+        },
+        {
+          "collapsed": false,
+          "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 },
+          "id": 20,
+          "title": "VIN Decoder — Cache & Data Freshness",
+          "type": "row"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "thresholds" },
+              "thresholds": {
+                "steps": [
+                  { "color": "green", "value": null },
+                  { "color": "yellow", "value": 2592000 },
+                  { "color": "red", "value": 5184000 }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 },
+          "id": 5,
+          "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "textMode": "auto", "colorMode": "background" },
+          "targets": [
+            {
+              "expr": "api_data_freshness_seconds{api=\"vin-decoder\"}",
+              "legendFormat": "data freshness",
+              "refId": "A"
+            }
+          ],
+          "title": "Data Freshness (seconds since last NHTSA fetch)",
+          "type": "stat"
+        }
+      ],
+      "refresh": "1m",
+      "schemaVersion": 38,
+      "tags": ["vin-decoder", "api-company"],
+      "title": "VIN Decoder API",
+      "uid": "vin-decoder-api",
+      "version": 1
+    }