Merge pull request 'Add rate limit stats to admin panel' (#1682) from feature/1675-rate-limit-admin into main

Merge PR #1682

ROADMAP.md

@@ -81,57 +81,50 @@ Items that have been implemented and merged into main.
 - ~~OpenAPI client generation.~~ TypeScript API client auto-generated from
   FastAPI spec with CI freshness check.
 
+### Resilience
+
+- ~~`_jobs` dict is in-memory only.~~ Database-backed job persistence
+  implemented using `db.list_jobs()` and `mark_stale_jobs_failed()`. The
+  in-memory `_jobs` dict has been removed.
+
+### Test coverage (P1/P2)
+
+- ~~Export endpoint tests.~~ Tests added for CSV and PDF export endpoints.
+- ~~Tracked company admin endpoint tests.~~ Tests added for `/admin/tracked`
+  CRUD endpoints and scheduler integration.
+- ~~Webhook integration tests.~~ Tests added for retry logic, Slack/Discord
+  payload format, and multi-URL dispatch.
+- ~~S3/MinIO storage backend tests.~~ Unit tests added for the S3 backend
+  (read, write, exists, delete, error handling).
+- ~~`analyze_single_patent` auto-download path tests.~~ Tests added for the
+  auto-download fallback (cache lookup, PDF download, FileNotFoundError).
+
+### Code quality
+
+- ~~Scheduler creates its own DatabaseClient.~~ Refactored to use the
+  application-level pooled `get_db_client()`.
+
 ---
 
 ## P1 -- High Priority
 
-These items address correctness, reliability, and coverage gaps that should be
+No outstanding P1 items. All previously listed items have been completed and
-resolved before broader production use.
+moved to the Completed section above.
-
-### Resilience
-
-- **`_jobs` dict is in-memory only.** Job state is lost on API restart.
-  Persist job status in PostgreSQL or Redis so async batch results survive
-  restarts.
-
-### Test coverage gaps
-
-- **Export endpoint tests.** The CSV and PDF export endpoints (`/export/`)
-  lack test coverage. Add tests covering auth, success, 404, and edge cases.
-  *(Issue #1655)*
-- **Tracked company admin endpoint tests.** The `/admin/tracked` CRUD
-  endpoints and scheduler integration lack test coverage. *(Issue #1656)*
 
 ---
 
 ## P2 -- Medium Priority
 
-Improvements to reliability, test coverage, and code quality.
+Improvements to the API surface.
-
-### Test coverage
-
-- **Webhook integration tests.** The retry logic, Slack/Discord payload
-  format, and multi-URL dispatch in `webhooks.py` need test coverage.
-  *(Issue #1657)*
-- **S3/MinIO storage backend tests.** `storage.py` has local filesystem tests
-  but no unit tests for the S3 backend (read, write, exists, delete,
-  error handling). *(Issue #1660)*
-- **`analyze_single_patent` auto-download path tests.** The auto-download
-  fallback (cache lookup, PDF download, FileNotFoundError) in
-  `analyzer.py` lacks test coverage. *(Issue #1661)*
-
-### Code quality
-
-- **Scheduler creates its own DatabaseClient.** `scheduler.py` bypasses the
-  application-level pooled client, creating a new connection on every tick.
-  Refactor to use `get_db_client()`. *(Issue #1658)*
 
 ### API improvements
 
-- **API pagination.** The `/analyze/batch` and `/jobs` endpoints could benefit
+- **API pagination.** The `/analyze/batch` endpoint needs cursor-based
-  from cursor-based pagination for large result sets.
+  pagination for large result sets. The `/jobs` endpoint already has cursor
+  pagination. *(Issue #1669)*
 - **Request validation improvements.** Add stricter input validation for
   company names (disallow special characters, enforce length limits).
+  *(Issue #1670)*
 
 ---
 

SPARC/api.py

@@ -12,10 +12,10 @@ from typing import TYPE_CHECKING, Annotated, List
 if TYPE_CHECKING:
     from SPARC.database import DatabaseClient
 
-from fastapi import BackgroundTasks, Depends, FastAPI, HTTPException, Query, Request
+from fastapi import BackgroundTasks, Depends, FastAPI, HTTPException, Path, Query, Request
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse, StreamingResponse
-from pydantic import BaseModel, EmailStr, Field
+from pydantic import BaseModel, EmailStr, Field, StringConstraints
 from slowapi import Limiter
 from slowapi.errors import RateLimitExceeded
 from slowapi.util import get_remote_address
@@ -36,6 +36,16 @@ from SPARC.auth import (
 )
 from SPARC.types import BatchAnalysisResult, CompanyAnalysisResult
 
+# Validated company name type: 2-100 chars, alphanumeric + spaces/hyphens/ampersands/periods only.
+CompanyName = Annotated[
+    str,
+    StringConstraints(
+        min_length=2,
+        max_length=100,
+        pattern=r"^[a-zA-Z0-9][a-zA-Z0-9 \-&.]*$",
+    ),
+]
+
 
 # Pydantic models for API
 class CompanyAnalysisResponse(BaseModel):
@@ -72,7 +82,7 @@ class CompanyAnalysisRequest(BaseModel):
 class BatchAnalysisRequest(BaseModel):
     """Request model for batch company analysis."""
 
-    companies: list[str] = Field(
+    companies: list[CompanyName] = Field(
         ..., min_length=1, max_length=20, description="List of company names to analyze"
     )
     max_workers: int = Field(
@@ -235,10 +245,37 @@ app = FastAPI(
 limiter = Limiter(key_func=get_remote_address)
 app.state.limiter = limiter
 
+# In-memory rate limit statistics
+_rate_limit_stats: dict[str, dict] = {}
+
+
+def _track_rate_limit_request(endpoint: str, ip: str, rejected: bool = False) -> None:
+    """Record a request against a rate-limited endpoint."""
+    key = endpoint
+    if key not in _rate_limit_stats:
+        _rate_limit_stats[key] = {
+            "endpoint": endpoint,
+            "total_requests": 0,
+            "rejected_requests": 0,
+            "by_ip": {},
+        }
+    _rate_limit_stats[key]["total_requests"] += 1
+    if rejected:
+        _rate_limit_stats[key]["rejected_requests"] += 1
+    ip_stats = _rate_limit_stats[key].setdefault("by_ip", {})
+    if ip not in ip_stats:
+        ip_stats[ip] = {"total": 0, "rejected": 0}
+    ip_stats[ip]["total"] += 1
+    if rejected:
+        ip_stats[ip]["rejected"] += 1
+
 
 @app.exception_handler(RateLimitExceeded)
 async def rate_limit_handler(request: Request, exc: RateLimitExceeded):
     """Return 429 with Retry-After header when rate limit is exceeded."""
+    endpoint = request.url.path
+    ip = get_remote_address(request)
+    _track_rate_limit_request(endpoint, ip, rejected=True)
     retry_after = getattr(exc, "retry_after", 60)
     return JSONResponse(
         status_code=429,
@@ -267,6 +304,7 @@ async def register(request: Request, body: RegisterRequest):
 
     The first registered user automatically becomes an admin.
     """
+    _track_rate_limit_request("/auth/register", get_remote_address(request))
     db = get_db_client()
 
     # First user becomes admin
@@ -297,6 +335,7 @@ async def register(request: Request, body: RegisterRequest):
 @limiter.limit("10/minute")
 async def login(request: Request, body: LoginRequest):
     """Authenticate user and return JWT tokens."""
+    _track_rate_limit_request("/auth/login", get_remote_address(request))
     db = get_db_client()
 
     user = db.authenticate_user(body.email, body.password)
@@ -423,7 +462,7 @@ async def delete_user(
 class TrackCompanyRequest(BaseModel):
     """Request to add a company to tracking."""
 
-    company_name: str = Field(..., min_length=1, max_length=255)
+    company_name: CompanyName = Field(...)
 
 
 @app.get("/admin/tracked", tags=["Admin"])
@@ -450,7 +489,7 @@ async def add_tracked_company(
 
 @app.delete("/admin/tracked/{company_name}", tags=["Admin"])
 async def remove_tracked_company(
-    company_name: str,
+    company_name: Annotated[str, Path(min_length=2, max_length=100, pattern=r"^[a-zA-Z0-9][a-zA-Z0-9 \-&.]*$")],
     _: UserResponse = Depends(get_current_admin),
 ):
     """Remove a company from the tracked list (admin only)."""
@@ -461,6 +500,36 @@ async def remove_tracked_company(
     return {"message": f"Stopped tracking {company_name}"}
 
 
+@app.get("/admin/rate-limits", tags=["Admin"])
+async def get_rate_limit_stats(
+    _: UserResponse = Depends(get_current_admin),
+):
+    """Get rate limit status and usage statistics (admin only).
+
+    Returns current rate limit configuration and request statistics
+    for all rate-limited endpoints.
+
+    Returns:
+        List of rate limit stats per endpoint with total/rejected counts
+    """
+    rate_limits_config = {
+        "/auth/register": {"limit": "5/minute"},
+        "/auth/login": {"limit": "10/minute"},
+    }
+
+    results = []
+    for endpoint, conf in rate_limits_config.items():
+        stats = _rate_limit_stats.get(endpoint, {})
+        results.append({
+            "endpoint": endpoint,
+            "limit": conf["limit"],
+            "total_requests": stats.get("total_requests", 0),
+            "rejected_requests": stats.get("rejected_requests", 0),
+        })
+
+    return {"rate_limits": results}
+
+
 @app.get("/admin/alerts", tags=["Admin"])
 async def list_alerts(
     limit: int = Query(default=50, ge=1, le=200),
@@ -608,7 +677,7 @@ async def get_analytics_trends(
 
 @app.get("/export/{company_name}", tags=["Export"])
 async def export_company_csv(
-    company_name: str,
+    company_name: Annotated[str, Path(min_length=2, max_length=100, pattern=r"^[a-zA-Z0-9][a-zA-Z0-9 \-&.]*$")],
     _: UserResponse = Depends(get_current_user),
 ):
     """Export analysis results for a company as a CSV file.
@@ -660,7 +729,7 @@ async def export_company_csv(
 
 @app.get("/export/{company_name}/pdf", tags=["Export"])
 async def export_company_pdf(
-    company_name: str,
+    company_name: Annotated[str, Path(min_length=2, max_length=100, pattern=r"^[a-zA-Z0-9][a-zA-Z0-9 \-&.]*$")],
     _: UserResponse = Depends(get_current_user),
 ):
     """Export analysis results for a company as a formatted PDF report.
@@ -834,7 +903,7 @@ async def health_check():
     tags=["Analysis"],
 )
 async def analyze_company(
-    company_name: str,
+    company_name: Annotated[str, Path(min_length=2, max_length=100, pattern=r"^[a-zA-Z0-9][a-zA-Z0-9 \-&.]*$")],
     model: str | None = Query(default=None, description="LLM model to use (e.g. 'openai/gpt-4o'). Defaults to server config."),
     _: UserResponse = Depends(get_current_user),
 ):
@@ -864,7 +933,7 @@ async def analyze_company(
 )
 async def analyze_single_patent(
     patent_id: str,
-    company_name: str = Query(description="Company name for analysis context"),
+    company_name: Annotated[str, Query(min_length=2, max_length=100, pattern=r"^[a-zA-Z0-9][a-zA-Z0-9 \-&.]*$", description="Company name for analysis context")],
     _: UserResponse = Depends(get_current_user),
 ):
     """Analyze a single patent by its publication ID.

tests/test_company_name_validation.py

@@ -0,0 +1,157 @@
+"""Tests for company name input validation on analysis endpoints."""
+
+from datetime import datetime
+from unittest.mock import Mock
+
+import pytest
+from fastapi.testclient import TestClient
+
+from SPARC.api import app
+from SPARC.types import CompanyAnalysisResult
+
+
+@pytest.fixture
+def client():
+    """Create test client."""
+    return TestClient(app)
+
+
+@pytest.fixture
+def mock_analyzer(mocker):
+    """Mock the global analyzer so valid requests succeed."""
+    mock = Mock()
+    mock._analyze_company_safe.return_value = CompanyAnalysisResult(
+        company_name="nvidia",
+        analysis="Test analysis",
+        patent_count=1,
+        success=True,
+        timestamp=datetime.now(),
+    )
+    mocker.patch("SPARC.api._analyzer", mock)
+    return mock
+
+
+class TestCompanyNameValidation:
+    """Test that company names are validated on analysis endpoints."""
+
+    # --- Too short ---
+
+    def test_single_char_rejected(self, client, mock_analyzer):
+        """A one-character company name should be rejected."""
+        response = client.get("/analyze/X")
+        assert response.status_code == 422
+
+    # --- Too long ---
+
+    def test_over_100_chars_rejected(self, client, mock_analyzer):
+        """A company name longer than 100 characters should be rejected."""
+        long_name = "A" * 101
+        response = client.get(f"/analyze/{long_name}")
+        assert response.status_code == 422
+
+    # --- Special characters ---
+
+    @pytest.mark.parametrize(
+        "bad_name",
+        [
+            "nvidia!",
+            "intel@corp",
+            "test#company",
+            "foo$bar",
+            "a%b",
+            "x^y",
+            "semi;colon",
+            "drop'table",
+            'say"hello',
+            "path/traversal",
+            "back\\slash",
+            "pipe|char",
+            "star*glob",
+            "question?mark",
+            "<script>",
+            "curly{brace}",
+            "equal=sign",
+            "plus+plus",
+            "comma,separated",
+        ],
+    )
+    def test_special_chars_rejected(self, client, mock_analyzer, bad_name):
+        """Company names with disallowed special characters should be rejected."""
+        response = client.get(f"/analyze/{bad_name}")
+        assert response.status_code == 422
+
+    # --- Valid names ---
+
+    @pytest.mark.parametrize(
+        "valid_name",
+        [
+            "nvidia",
+            "Intel",
+            "TSMC",
+            "Texas Instruments",
+            "Johnson-Johnson",
+            "AT&T",
+            "St. Jude Medical",
+            "3M",
+            "21st Century Fox",
+            "ab",  # minimum length
+            "A" * 100,  # maximum length
+        ],
+    )
+    def test_valid_names_accepted(self, client, mock_analyzer, valid_name):
+        """Valid company names should be accepted (200, not 422)."""
+        response = client.get(f"/analyze/{valid_name}")
+        # Should not be a validation error; 200 or other non-422 status is fine
+        assert response.status_code != 422
+
+    # --- Batch endpoint validation ---
+
+    def test_batch_too_short_rejected(self, client, mock_analyzer):
+        """Batch endpoint should reject company names that are too short."""
+        response = client.post(
+            "/analyze/batch",
+            json={"companies": ["X"]},
+        )
+        assert response.status_code == 422
+
+    def test_batch_too_long_rejected(self, client, mock_analyzer):
+        """Batch endpoint should reject company names that are too long."""
+        response = client.post(
+            "/analyze/batch",
+            json={"companies": ["A" * 101]},
+        )
+        assert response.status_code == 422
+
+    def test_batch_special_chars_rejected(self, client, mock_analyzer):
+        """Batch endpoint should reject company names with special chars."""
+        response = client.post(
+            "/analyze/batch",
+            json={"companies": ["nvidia!", "intel"]},
+        )
+        assert response.status_code == 422
+
+    def test_batch_valid_names_accepted(self, client, mock_analyzer):
+        """Batch endpoint should accept valid company names."""
+        response = client.post(
+            "/analyze/batch",
+            json={"companies": ["nvidia", "Intel", "AT&T"]},
+        )
+        assert response.status_code != 422
+
+    # --- Name must start with alphanumeric ---
+
+    def test_leading_space_rejected(self, client, mock_analyzer):
+        """Company name starting with a space should be rejected."""
+        response = client.post(
+            "/analyze/batch",
+            json={"companies": [" nvidia"]},
+        )
+        assert response.status_code == 422
+
+    def test_leading_hyphen_rejected(self, client, mock_analyzer):
+        """Company name starting with a hyphen should be rejected."""
+        response = client.post(
+            "/analyze/batch",
+            json={"companies": ["-nvidia"]},
+        )
+        assert response.status_code == 422

tests/test_rate_limit_admin.py

@@ -0,0 +1,109 @@
+"""Tests for the /admin/rate-limits endpoint."""
+
+from unittest.mock import patch
+
+import pytest
+from fastapi.testclient import TestClient
+
+from SPARC import api
+from SPARC.api import app
+from SPARC.auth import UserResponse
+
+
+@pytest.fixture
+def client():
+    """Create test client."""
+    return TestClient(app)
+
+
+@pytest.fixture(autouse=True)
+def reset_stats():
+    """Reset rate limit stats between tests."""
+    api._rate_limit_stats.clear()
+    yield
+    api._rate_limit_stats.clear()
+
+
+def _mock_admin():
+    """Return a mock admin user."""
+    return UserResponse(id=1, email="admin@test.com", role="admin", created_at="2025-01-01T00:00:00")
+
+
+def _mock_user():
+    """Return a mock non-admin user."""
+    return UserResponse(id=2, email="user@test.com", role="user", created_at="2025-01-01T00:00:00")
+
+
+class TestRateLimitAdminEndpoint:
+    """Test GET /admin/rate-limits."""
+
+    def test_admin_can_access(self, client):
+        """Admin users should be able to access the rate-limits endpoint."""
+        app.dependency_overrides[api.get_current_admin] = _mock_admin
+        try:
+            response = client.get("/admin/rate-limits")
+            assert response.status_code == 200
+            data = response.json()
+            assert "rate_limits" in data
+            assert isinstance(data["rate_limits"], list)
+        finally:
+            app.dependency_overrides.clear()
+
+    def test_non_admin_rejected(self, client):
+        """Non-admin users should get 403."""
+        # Without overriding the dependency, it should fail auth
+        response = client.get("/admin/rate-limits")
+        assert response.status_code in (401, 403)
+
+    def test_returns_configured_endpoints(self, client):
+        """Should list all rate-limited endpoints."""
+        app.dependency_overrides[api.get_current_admin] = _mock_admin
+        try:
+            response = client.get("/admin/rate-limits")
+            assert response.status_code == 200
+            data = response.json()
+            endpoints = [rl["endpoint"] for rl in data["rate_limits"]]
+            assert "/auth/register" in endpoints
+            assert "/auth/login" in endpoints
+        finally:
+            app.dependency_overrides.clear()
+
+    def test_empty_state_shows_zero_counts(self, client):
+        """When no requests have been made, counts should be zero."""
+        app.dependency_overrides[api.get_current_admin] = _mock_admin
+        try:
+            response = client.get("/admin/rate-limits")
+            data = response.json()
+            for rl in data["rate_limits"]:
+                assert rl["total_requests"] == 0
+                assert rl["rejected_requests"] == 0
+        finally:
+            app.dependency_overrides.clear()
+
+    def test_tracks_requests(self, client):
+        """After making requests, the stats should reflect them."""
+        api._track_rate_limit_request("/auth/login", "127.0.0.1")
+        api._track_rate_limit_request("/auth/login", "127.0.0.1")
+        api._track_rate_limit_request("/auth/login", "192.168.1.1", rejected=True)
+
+        app.dependency_overrides[api.get_current_admin] = _mock_admin
+        try:
+            response = client.get("/admin/rate-limits")
+            data = response.json()
+            login_stats = next(rl for rl in data["rate_limits"] if rl["endpoint"] == "/auth/login")
+            assert login_stats["total_requests"] == 3
+            assert login_stats["rejected_requests"] == 1
+        finally:
+            app.dependency_overrides.clear()
+
+    def test_includes_limit_config(self, client):
+        """Each endpoint entry should include the rate limit config string."""
+        app.dependency_overrides[api.get_current_admin] = _mock_admin
+        try:
+            response = client.get("/admin/rate-limits")
+            data = response.json()
+            for rl in data["rate_limits"]:
+                assert "limit" in rl
+                assert isinstance(rl["limit"], str)
+        finally:
+            app.dependency_overrides.clear()