feat: extend monitoring to include VIN Decoder API

- flux/monitoring/prometheusrule-apis.yaml: extend all 5 SLO alert
  rules to include vin-decoder in the job regex filter
- flux/monitoring/grafana-dashboard-apis.yaml: update all PromQL
  expressions to include vin-decoder in the api regex filter;
  update dashboard description
- flux/monitoring/vin-decoder-dashboard.yaml: new dedicated Grafana
  dashboard for VIN Decoder showing request rate by route, 5xx error
  rate, P50/P95/P99 latency by route, and data freshness gauge
- flux/monitoring/kustomization.yaml: add vin-decoder-dashboard.yaml
- flux/vin-decoder/servicemonitor.yaml: Prometheus ServiceMonitor
  for auto-discovery of /metrics on the vin-decoder service
- flux/vin-decoder/kustomization.yaml: add servicemonitor.yaml

Closes leeworks-agents/api-company#129

flux/monitoring/grafana-dashboard-apis.yaml

@@ -9,7 +9,7 @@ data:
   api-dashboard.json: |
     {
       "annotations": { "list": [] },
-      "description": "Request rate, latency, error rate, and data freshness for zip-enrichment, holidays, and air-quality APIs",
+      "description": "Request rate, latency, error rate, and data freshness for zip-enrichment, holidays, air-quality, and vin-decoder APIs",
       "editable": true,
       "graphTooltip": 1,
       "panels": [

flux/monitoring/kustomization.yaml

@@ -8,3 +8,4 @@ resources:
   - gatus-helmrelease.yaml
   - grafana-dashboard-apis.yaml
   - prometheusrule-apis.yaml
+  - vin-decoder-dashboard.yaml

flux/monitoring/prometheusrule-apis.yaml

@@ -18,9 +18,9 @@ spec:
         - alert: APIHighErrorRate
           expr: |
             (
-              sum by (job) (rate(api_requests_total{status=~"5..", job=~"zip|holidays|air-quality"}[5m]))
+              sum by (job) (rate(api_requests_total{status=~"5..", job=~"zip|holidays|air-quality|vin-decoder"}[5m]))
               /
-              sum by (job) (rate(api_requests_total{job=~"zip|holidays|air-quality"}[5m]))
+              sum by (job) (rate(api_requests_total{job=~"zip|holidays|air-quality|vin-decoder"}[5m]))
             ) > 0.05
           for: 5m
           labels:
@@ -36,9 +36,9 @@ spec:
         - alert: APIHighErrorRate
           expr: |
             (
-              sum by (job) (rate(api_requests_total{status=~"5..", job=~"zip|holidays|air-quality"}[5m]))
+              sum by (job) (rate(api_requests_total{status=~"5..", job=~"zip|holidays|air-quality|vin-decoder"}[5m]))
               /
-              sum by (job) (rate(api_requests_total{job=~"zip|holidays|air-quality"}[5m]))
+              sum by (job) (rate(api_requests_total{job=~"zip|holidays|air-quality|vin-decoder"}[5m]))
             ) > 0.20
           for: 5m
           labels:
@@ -55,7 +55,7 @@ spec:
           expr: |
             histogram_quantile(
               0.95,
-              sum by (job, le) (rate(api_response_duration_seconds_bucket{job=~"zip|holidays|air-quality"}[5m]))
+              sum by (job, le) (rate(api_response_duration_seconds_bucket{job=~"zip|holidays|air-quality|vin-decoder"}[5m]))
             ) > 2
           for: 5m
           labels:
@@ -70,7 +70,7 @@ spec:
         # -------------------------------------------------------------------
         - alert: APIDataStale
           expr: |
-            api_data_freshness_seconds{job=~"zip|holidays|air-quality"} > 2592000
+            api_data_freshness_seconds{job=~"zip|holidays|air-quality|vin-decoder"} > 2592000
           for: 30m
           labels:
             severity: warning
@@ -84,7 +84,7 @@ spec:
         # -------------------------------------------------------------------
         - alert: APIDataCriticallyStale
           expr: |
-            api_data_freshness_seconds{job=~"zip|holidays|air-quality"} > 5184000
+            api_data_freshness_seconds{job=~"zip|holidays|air-quality|vin-decoder"} > 5184000
           for: 1h
           labels:
             severity: critical
@@ -98,9 +98,9 @@ spec:
         # -------------------------------------------------------------------
         - alert: APIDown
           expr: |
-            absent(up{job=~"zip|holidays|air-quality"} == 1)
+            absent(up{job=~"zip|holidays|air-quality|vin-decoder"} == 1)
             or
-            up{job=~"zip|holidays|air-quality"} == 0
+            up{job=~"zip|holidays|air-quality|vin-decoder"} == 0
           for: 2m
           labels:
             severity: critical

flux/monitoring/vin-decoder-dashboard.yaml

@@ -0,0 +1,161 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-dashboard-vin-decoder
+  namespace: monitoring
+  labels:
+    grafana_dashboard: "1"
+data:
+  vin-decoder-dashboard.json: |
+    {
+      "annotations": { "list": [] },
+      "description": "Request rate, latency, error rate, and cache hit ratio for the VIN Decoder API",
+      "editable": true,
+      "graphTooltip": 1,
+      "panels": [
+        {
+          "collapsed": false,
+          "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+          "id": 1,
+          "title": "VIN Decoder — Request Rate",
+          "type": "row"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "palette-classic" },
+              "custom": { "axisLabel": "requests/sec", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never" },
+              "unit": "reqps"
+            },
+            "overrides": []
+          },
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 },
+          "id": 2,
+          "options": {
+            "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
+            "tooltip": { "mode": "multi" }
+          },
+          "targets": [
+            {
+              "expr": "sum by (route) (rate(api_requests_total{api=\"vin-decoder\"}[5m]))",
+              "legendFormat": "{{route}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Request Rate by Route",
+          "type": "timeseries"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "palette-classic" },
+              "custom": { "axisLabel": "", "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never" },
+              "unit": "percentunit"
+            },
+            "overrides": []
+          },
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 },
+          "id": 3,
+          "options": {
+            "legend": { "calcs": ["mean", "last"], "displayMode": "table", "placement": "bottom" },
+            "tooltip": { "mode": "multi" }
+          },
+          "targets": [
+            {
+              "expr": "sum(rate(api_requests_total{api=\"vin-decoder\",status_code=~\"5..\"}[5m])) / sum(rate(api_requests_total{api=\"vin-decoder\"}[5m]))",
+              "legendFormat": "5xx error rate",
+              "refId": "A"
+            }
+          ],
+          "title": "5xx Error Rate",
+          "type": "timeseries"
+        },
+        {
+          "collapsed": false,
+          "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 },
+          "id": 10,
+          "title": "VIN Decoder — Latency",
+          "type": "row"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "palette-classic" },
+              "custom": { "axisLabel": "seconds", "drawStyle": "line", "fillOpacity": 5, "lineWidth": 1, "showPoints": "never" },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": { "h": 8, "w": 24, "x": 0, "y": 10 },
+          "id": 4,
+          "options": {
+            "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
+            "tooltip": { "mode": "multi" }
+          },
+          "targets": [
+            {
+              "expr": "histogram_quantile(0.50, sum by (route, le) (rate(api_response_duration_seconds_bucket{api=\"vin-decoder\"}[5m])))",
+              "legendFormat": "P50 {{route}}",
+              "refId": "A"
+            },
+            {
+              "expr": "histogram_quantile(0.95, sum by (route, le) (rate(api_response_duration_seconds_bucket{api=\"vin-decoder\"}[5m])))",
+              "legendFormat": "P95 {{route}}",
+              "refId": "B"
+            },
+            {
+              "expr": "histogram_quantile(0.99, sum by (route, le) (rate(api_response_duration_seconds_bucket{api=\"vin-decoder\"}[5m])))",
+              "legendFormat": "P99 {{route}}",
+              "refId": "C"
+            }
+          ],
+          "title": "Response Latency P50 / P95 / P99 by Route",
+          "type": "timeseries"
+        },
+        {
+          "collapsed": false,
+          "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 },
+          "id": 20,
+          "title": "VIN Decoder — Cache & Data Freshness",
+          "type": "row"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "thresholds" },
+              "thresholds": {
+                "steps": [
+                  { "color": "green", "value": null },
+                  { "color": "yellow", "value": 2592000 },
+                  { "color": "red", "value": 5184000 }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 },
+          "id": 5,
+          "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "textMode": "auto", "colorMode": "background" },
+          "targets": [
+            {
+              "expr": "api_data_freshness_seconds{api=\"vin-decoder\"}",
+              "legendFormat": "data freshness",
+              "refId": "A"
+            }
+          ],
+          "title": "Data Freshness (seconds since last NHTSA fetch)",
+          "type": "stat"
+        }
+      ],
+      "refresh": "1m",
+      "schemaVersion": 38,
+      "tags": ["vin-decoder", "api-company"],
+      "title": "VIN Decoder API",
+      "uid": "vin-decoder-api",
+      "version": 1
+    }

flux/vin-decoder/kustomization.yaml

@@ -4,3 +4,4 @@ resources:
   - namespace.yaml
   - externalsecret.yaml
   - helmrelease.yaml
+  - servicemonitor.yaml

flux/vin-decoder/servicemonitor.yaml

@@ -0,0 +1,15 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: vin-decoder
+  namespace: vin-decoder
+  labels:
+    release: kube-prometheus-stack
+spec:
+  selector:
+    matchLabels:
+      app: vin-decoder
+  endpoints:
+    - port: http
+      path: /metrics
+      interval: 30s