Compare commits
12 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 9370d2c898 | |||
| 7cfcd0f46b | |||
| 2f249a9e18 | |||
| cea4658b91 | |||
| c5a8f9f58c | |||
| 8e64a9e2ac | |||
| 348baf96bb | |||
| cb9909172b | |||
| 38cddd6dfc | |||
| cd0ed86571 | |||
| 1a1a109aa3 | |||
| b616e11bdf |
@@ -0,0 +1,58 @@
|
|||||||
|
# Validate and publish OpenAPI specs to RapidAPI when apis/*/openapi.yaml changes on main.
|
||||||
|
# Requires secrets (configured once RapidAPI listings are live):
|
||||||
|
# RAPIDAPI_PLATFORM_KEY, RAPIDAPI_ZIP_API_ID, RAPIDAPI_ZIP_VERSION_ID,
|
||||||
|
# RAPIDAPI_HOLIDAYS_API_ID, RAPIDAPI_HOLIDAYS_VERSION_ID,
|
||||||
|
# RAPIDAPI_AQI_API_ID, RAPIDAPI_AQI_VERSION_ID
|
||||||
|
#
|
||||||
|
# If RAPIDAPI_PLATFORM_KEY is not set the publish step exits 0 with a skip message.
|
||||||
|
|
||||||
|
name: Validate and Publish OpenAPI Specs
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
paths:
|
||||||
|
- 'apis/*/openapi.yaml'
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
publish:
|
||||||
|
name: Lint and publish specs
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Checkout (with history for diff)
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
fetch-depth: 2
|
||||||
|
|
||||||
|
- name: Set up Node.js 20
|
||||||
|
uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: '20'
|
||||||
|
|
||||||
|
- name: Install Redocly CLI
|
||||||
|
run: npm install -g @redocly/cli@latest
|
||||||
|
|
||||||
|
- name: Lint OpenAPI specs
|
||||||
|
run: |
|
||||||
|
echo "Linting all OpenAPI specs..."
|
||||||
|
npx @redocly/cli lint apis/*/openapi.yaml
|
||||||
|
|
||||||
|
- name: Detect changed specs
|
||||||
|
id: changed
|
||||||
|
run: |
|
||||||
|
changed=$(git diff --name-only HEAD~1 HEAD | grep 'openapi\.yaml' || true)
|
||||||
|
echo "Changed specs: ${changed:-none}"
|
||||||
|
echo "files=${changed}" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
|
- name: Publish specs to RapidAPI
|
||||||
|
env:
|
||||||
|
RAPIDAPI_KEY: ${{ secrets.RAPIDAPI_PLATFORM_KEY }}
|
||||||
|
RAPIDAPI_ZIP_API_ID: ${{ secrets.RAPIDAPI_ZIP_API_ID }}
|
||||||
|
RAPIDAPI_ZIP_VERSION_ID: ${{ secrets.RAPIDAPI_ZIP_VERSION_ID }}
|
||||||
|
RAPIDAPI_HOLIDAYS_API_ID: ${{ secrets.RAPIDAPI_HOLIDAYS_API_ID }}
|
||||||
|
RAPIDAPI_HOLIDAYS_VERSION_ID: ${{ secrets.RAPIDAPI_HOLIDAYS_VERSION_ID }}
|
||||||
|
RAPIDAPI_AQI_API_ID: ${{ secrets.RAPIDAPI_AQI_API_ID }}
|
||||||
|
RAPIDAPI_AQI_VERSION_ID: ${{ secrets.RAPIDAPI_AQI_VERSION_ID }}
|
||||||
|
run: node scripts/publish-openapi.js
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
# Smoke test: confirms Gitea Act Runner is online and accepting jobs.
|
||||||
|
# Run manually via workflow_dispatch after runner is registered (#77).
|
||||||
|
# Closes leeworks-agents/api-company#96
|
||||||
|
|
||||||
|
name: Runner Smoke Test
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
smoke:
|
||||||
|
name: Smoke Test
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Confirm runner is online
|
||||||
|
run: |
|
||||||
|
echo "Runner is online!"
|
||||||
|
echo "Job ID: $GITHUB_JOB"
|
||||||
|
echo "Runner OS: $(uname -a)"
|
||||||
|
echo "Date: $(date -u)"
|
||||||
|
echo "Smoke test PASSED"
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
# Tooling Versions
|
||||||
|
|
||||||
|
> Phase 0 checklist item — documents the local agent container tooling and confirms git remote config.
|
||||||
|
> Live cluster verification (kubectl/flux against the running cluster) requires operator confirmation.
|
||||||
|
|
||||||
|
## Agent Container Tooling
|
||||||
|
|
||||||
|
| Tool | Version | Source |
|
||||||
|
|------|---------|--------|
|
||||||
|
| Node.js | v22.22.2 | `node --version` |
|
||||||
|
| Python 3 | 3.11.2 | `python3 --version` |
|
||||||
|
| Helm | v3.21.0 | `helm version --short` |
|
||||||
|
| Flux CLI | v2.4.0 | `flux version --client` |
|
||||||
|
| kustomize | v5.6.0 | `kustomize version` |
|
||||||
|
| kubectl | available | `kubectl` (cluster context not set in agent container — live commands require kubeconfig from operator) |
|
||||||
|
|
||||||
|
## Git Remote Config
|
||||||
|
|
||||||
|
```
|
||||||
|
origin ssh://git@gitea.leeworks.dev/leeworks-agents/api-company (fetch)
|
||||||
|
origin ssh://git@gitea.leeworks.dev/leeworks-agents/api-company (push)
|
||||||
|
upstream ssh://git@gitea.leeworks.dev/0xWheatyz/api-company (fetch)
|
||||||
|
upstream ssh://git@gitea.leeworks.dev/0xWheatyz/api-company (push)
|
||||||
|
```
|
||||||
|
|
||||||
|
Remote `origin` confirmed pointing to `gitea.leeworks.dev/leeworks-agents/api-company`.
|
||||||
|
|
||||||
|
> Note: `0xWheatyz/api-company` upstream repo does not yet exist on Gitea — tracked by issue #47.
|
||||||
|
|
||||||
|
## Stack Summary
|
||||||
|
|
||||||
|
### Runtime choices
|
||||||
|
|
||||||
|
| API Service | Runtime | Rationale |
|
||||||
|
|-------------|---------|-----------|
|
||||||
|
| zip-enrichment | Node.js (Fastify) | Lightweight, fast JSON serialisation; large ZIP dataset fits SQLite well |
|
||||||
|
| holidays | Node.js (Fastify) | Simple lookup API; Fastify handles high req/s with low memory |
|
||||||
|
| air-quality | Python (FastAPI) | AQI ingestion benefits from Python data-science ecosystem |
|
||||||
|
| docs-site | Astro (static) | Zero-runtime static site; built by CI, served from container |
|
||||||
|
|
||||||
|
### Helm chart registry
|
||||||
|
|
||||||
|
All HelmReleases use charts sourced from public Helm chart repositories declared in `flux/` as `HelmRepository` resources:
|
||||||
|
|
||||||
|
- `flux/gitea-runner/helmrepository.yaml` — Gitea Act Runner chart
|
||||||
|
- `flux/monitoring/` — `kube-prometheus-stack` and Gatus charts
|
||||||
|
- `flux/zip-enrichment/`, `flux/holidays/`, `flux/air-quality/` — per-API service charts
|
||||||
|
|
||||||
|
The in-cluster container registry is `registry.leeworks.dev` (Gitea built-in packages/container registry, documented in `docs/registry.md`).
|
||||||
|
|
||||||
|
### Flux version and bootstrap
|
||||||
|
|
||||||
|
- **Flux CLI**: v2.4.0 (available in agent container for manifest authoring)
|
||||||
|
- **Flux controllers**: bootstrapped into `0xWheatyz/Talos` cluster (FluxCD v2.x)
|
||||||
|
- `flux-system` namespace managed by Talos GitOps repo
|
||||||
|
- Flux watches `0xWheatyz/Talos` → `testing1/first-cluster/cluster/`
|
||||||
|
- `api-company-source` GitRepository + Kustomization adds this repo to Flux (issues #2, #90)
|
||||||
|
- **Bootstrap reference**: `0xWheatyz/Talos` repo — see Talos cluster documentation
|
||||||
|
|
||||||
|
## Live Verification (Operator)
|
||||||
|
|
||||||
|
The following commands require a valid `kubeconfig` (not available in agent container):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
kubectl config current-context # confirm cluster context
|
||||||
|
kubectl version --client # kubectl client version
|
||||||
|
flux check # Flux controller health
|
||||||
|
flux get sources git api-company # GitRepository READY status
|
||||||
|
flux get kustomizations api-company # Kustomization READY status
|
||||||
|
helm version # Helm client version
|
||||||
|
kustomize version # kustomize version
|
||||||
|
```
|
||||||
|
|
||||||
|
_Reference: MASTER_BUILD_PROMPT.md §Phase 0 step 2 — Closes leeworks-agents/api-company#94_
|
||||||
@@ -1,7 +1,8 @@
|
|||||||
# This manifest is FOR REFERENCE — the live version must be committed to
|
# This manifest is FOR REFERENCE — the live version must be committed to
|
||||||
# 0xWheatyz/Talos at testing1/first-cluster/cluster/flux/api-company/
|
# 0xWheatyz/Talos at testing1/first-cluster/cluster/flux/api-company-source/
|
||||||
#
|
#
|
||||||
# See leeworks-agents/api-company#2
|
# See leeworks-agents/api-company#2
|
||||||
|
# See leeworks-agents/api-company#97
|
||||||
|
|
||||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||||
kind: Kustomization
|
kind: Kustomization
|
||||||
@@ -17,3 +18,8 @@ spec:
|
|||||||
prune: true
|
prune: true
|
||||||
wait: true
|
wait: true
|
||||||
timeout: 5m
|
timeout: 5m
|
||||||
|
postBuild:
|
||||||
|
substituteFrom:
|
||||||
|
- kind: Secret
|
||||||
|
name: grafana-admin
|
||||||
|
optional: false
|
||||||
|
|||||||
@@ -2,5 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
|
|||||||
kind: Kustomization
|
kind: Kustomization
|
||||||
resources:
|
resources:
|
||||||
- namespace.yaml
|
- namespace.yaml
|
||||||
- helmrepository.yaml
|
|
||||||
- helmrelease.yaml
|
- helmrelease.yaml
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
kind: Kustomization
|
kind: Kustomization
|
||||||
resources:
|
resources:
|
||||||
|
- bedag-helmrepository.yaml
|
||||||
- external-secrets
|
- external-secrets
|
||||||
- gitea-runner
|
- gitea-runner
|
||||||
- monitoring
|
- monitoring
|
||||||
|
|||||||
@@ -71,7 +71,7 @@ spec:
|
|||||||
- alert: APIDataStale
|
- alert: APIDataStale
|
||||||
expr: |
|
expr: |
|
||||||
api_data_freshness_seconds{job=~"zip|holidays|air-quality"} > 2592000
|
api_data_freshness_seconds{job=~"zip|holidays|air-quality"} > 2592000
|
||||||
for: 5m
|
for: 30m
|
||||||
labels:
|
labels:
|
||||||
severity: warning
|
severity: warning
|
||||||
team: api-company
|
team: api-company
|
||||||
@@ -79,6 +79,20 @@ spec:
|
|||||||
summary: "Stale dataset on {{ $labels.job }} ({{ $labels.dataset }})"
|
summary: "Stale dataset on {{ $labels.job }} ({{ $labels.dataset }})"
|
||||||
description: "{{ $labels.job }} dataset '{{ $labels.dataset }}' has not been re-seeded in {{ $value | humanizeDuration }} (threshold: 30 days). Re-seed required."
|
description: "{{ $labels.job }} dataset '{{ $labels.dataset }}' has not been re-seeded in {{ $value | humanizeDuration }} (threshold: 30 days). Re-seed required."
|
||||||
|
|
||||||
|
# -------------------------------------------------------------------
|
||||||
|
# APIDataCriticallyStale — data freshness > 60 days
|
||||||
|
# -------------------------------------------------------------------
|
||||||
|
- alert: APIDataCriticallyStale
|
||||||
|
expr: |
|
||||||
|
api_data_freshness_seconds{job=~"zip|holidays|air-quality"} > 5184000
|
||||||
|
for: 1h
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
team: api-company
|
||||||
|
annotations:
|
||||||
|
summary: "API data is critically stale on {{ $labels.job }}"
|
||||||
|
description: "{{ $labels.job }} data has not been re-seeded in more than 60 days ({{ $value | humanizeDuration }})"
|
||||||
|
|
||||||
# -------------------------------------------------------------------
|
# -------------------------------------------------------------------
|
||||||
# APIDown — any API job absent for 2 min
|
# APIDown — any API job absent for 2 min
|
||||||
# -------------------------------------------------------------------
|
# -------------------------------------------------------------------
|
||||||
|
|||||||
@@ -0,0 +1,134 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
/**
|
||||||
|
* publish-openapi.js
|
||||||
|
*
|
||||||
|
* Uploads each API's openapi.yaml to the RapidAPI Platform API.
|
||||||
|
* Environment variables required per API (skips silently if not set):
|
||||||
|
* RAPIDAPI_KEY — RapidAPI Platform API bearer token
|
||||||
|
* RAPIDAPI_ZIP_API_ID — API ID for ZIP Enrichment on RapidAPI
|
||||||
|
* RAPIDAPI_ZIP_VERSION_ID — Version ID for ZIP Enrichment
|
||||||
|
* RAPIDAPI_HOLIDAYS_API_ID
|
||||||
|
* RAPIDAPI_HOLIDAYS_VERSION_ID
|
||||||
|
* RAPIDAPI_AQI_API_ID
|
||||||
|
* RAPIDAPI_AQI_VERSION_ID
|
||||||
|
*
|
||||||
|
* Usage: node scripts/publish-openapi.js
|
||||||
|
*/
|
||||||
|
|
||||||
|
const fs = require('fs');
|
||||||
|
const path = require('path');
|
||||||
|
const https = require('https');
|
||||||
|
|
||||||
|
const RAPIDAPI_KEY = process.env.RAPIDAPI_KEY || process.env.RAPIDAPI_PLATFORM_KEY;
|
||||||
|
|
||||||
|
if (!RAPIDAPI_KEY) {
|
||||||
|
console.log('⚠️ RAPIDAPI_KEY / RAPIDAPI_PLATFORM_KEY not set — skipping all spec uploads.');
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
const APIS = [
|
||||||
|
{
|
||||||
|
name: 'zip-enrichment',
|
||||||
|
specPath: path.join(__dirname, '..', 'apis', 'zip-enrichment', 'openapi.yaml'),
|
||||||
|
apiId: process.env.RAPIDAPI_ZIP_API_ID,
|
||||||
|
versionId: process.env.RAPIDAPI_ZIP_VERSION_ID,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'holidays',
|
||||||
|
specPath: path.join(__dirname, '..', 'apis', 'holidays', 'openapi.yaml'),
|
||||||
|
apiId: process.env.RAPIDAPI_HOLIDAYS_API_ID,
|
||||||
|
versionId: process.env.RAPIDAPI_HOLIDAYS_VERSION_ID,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'air-quality',
|
||||||
|
specPath: path.join(__dirname, '..', 'apis', 'air-quality', 'openapi.yaml'),
|
||||||
|
apiId: process.env.RAPIDAPI_AQI_API_ID,
|
||||||
|
versionId: process.env.RAPIDAPI_AQI_VERSION_ID,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build a multipart/form-data body from a file buffer.
|
||||||
|
* Returns { body: Buffer, boundary: string }
|
||||||
|
*/
|
||||||
|
function buildMultipart(fieldName, filename, fileBuffer, contentType = 'application/yaml') {
|
||||||
|
const boundary = '----FormBoundary' + Math.random().toString(36).slice(2);
|
||||||
|
const CRLF = '\r\n';
|
||||||
|
const parts = [
|
||||||
|
Buffer.from(
|
||||||
|
`--${boundary}${CRLF}` +
|
||||||
|
`Content-Disposition: form-data; name="${fieldName}"; filename="${filename}"${CRLF}` +
|
||||||
|
`Content-Type: ${contentType}${CRLF}${CRLF}`
|
||||||
|
),
|
||||||
|
fileBuffer,
|
||||||
|
Buffer.from(`${CRLF}--${boundary}--${CRLF}`),
|
||||||
|
];
|
||||||
|
return { body: Buffer.concat(parts), boundary };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Upload a spec file to RapidAPI Platform API.
|
||||||
|
* Returns a promise that resolves with the response status code.
|
||||||
|
*/
|
||||||
|
function uploadSpec(api) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
if (!api.apiId || !api.versionId) {
|
||||||
|
console.log(`⏭️ Skipping ${api.name}: API ID or Version ID not configured.`);
|
||||||
|
return resolve(null);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!fs.existsSync(api.specPath)) {
|
||||||
|
console.log(`⏭️ Skipping ${api.name}: spec file not found at ${api.specPath}`);
|
||||||
|
return resolve(null);
|
||||||
|
}
|
||||||
|
|
||||||
|
const fileBuffer = fs.readFileSync(api.specPath);
|
||||||
|
const { body, boundary } = buildMultipart('spec', 'openapi.yaml', fileBuffer);
|
||||||
|
|
||||||
|
const options = {
|
||||||
|
hostname: 'platformapi1.p.rapidapi.com',
|
||||||
|
path: `/v1/apis/${api.apiId}/versions/${api.versionId}`,
|
||||||
|
method: 'PUT',
|
||||||
|
headers: {
|
||||||
|
'Authorization': `Bearer ${RAPIDAPI_KEY}`,
|
||||||
|
'X-RapidAPI-Key': RAPIDAPI_KEY,
|
||||||
|
'Content-Type': `multipart/form-data; boundary=${boundary}`,
|
||||||
|
'Content-Length': body.length,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const req = https.request(options, (res) => {
|
||||||
|
let data = '';
|
||||||
|
res.on('data', (chunk) => { data += chunk; });
|
||||||
|
res.on('end', () => {
|
||||||
|
if (res.statusCode >= 200 && res.statusCode < 300) {
|
||||||
|
console.log(`✓ Published ${api.name} spec to RapidAPI (HTTP ${res.statusCode})`);
|
||||||
|
resolve(res.statusCode);
|
||||||
|
} else {
|
||||||
|
reject(new Error(`Failed to publish ${api.name}: HTTP ${res.statusCode} — ${data}`));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
req.on('error', reject);
|
||||||
|
req.write(body);
|
||||||
|
req.end();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
let hasError = false;
|
||||||
|
for (const api of APIS) {
|
||||||
|
try {
|
||||||
|
await uploadSpec(api);
|
||||||
|
} catch (err) {
|
||||||
|
console.error(`✗ ${err.message}`);
|
||||||
|
hasError = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (hasError) {
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
main();
|
||||||
Reference in New Issue
Block a user