From 2d5c0e34d76fd56402807fa80b69ae336bb08c1e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Jul 2026 18:36:16 +0000 Subject: [PATCH] Add local model backends: per-spawn dropdown pointing claude at alternative endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operators can register Anthropic-API-compatible endpoints (a local Qwen/Llama behind LiteLLM or claude-code-router, an LLM gateway) on the dashboard's Claude -> Models tab and pick one from a Model dropdown when spawning an agent. The agent still launches as the same claude binary with the same hooks, skills, connectors, plugins, and gates — only its ANTHROPIC_BASE_URL / ANTHROPIC_MODEL / ANTHROPIC_AUTH_TOKEN env differs — and it stays pinned to its backend across resumes. No selection keeps the worker's Claude subscription untouched. - claude_models table (+ agents.model_id pin), migration 0012 - control.models resolves a row into the launch env (API keys Fernet-encrypted at rest, decrypted only in the control container; placeholder key when none is stored so the subscription OAuth token never reaches a local endpoint) - /claude/models CRUD (admin-gated writes, key never returned), spawn route + worker + CLI (--model) pass the selection through, fail-fast on missing or disabled backends - dashboard: Models tab, spawn-form dropdown, model badge in the agents table - docs/local-models.md: why bare OpenAI-compatible servers break tool calling with Qwen-Coder, and working vLLM/LiteLLM/llama.cpp stacks Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01DzDofD7gP63WpeLG8vEdZu --- README.md | 15 +- docs/local-models.md | 104 ++++++ .../components/sections/AgentsSection.tsx | 33 +- .../components/sections/ClaudeSection.tsx | 171 ++++++++- frontend/components/store.tsx | 78 +++- frontend/lib/api.ts | 18 + src/handler/api/routes/agents.py | 13 + src/handler/api/routes/claude.py | 91 +++++ src/handler/api/schemas.py | 72 +++- src/handler/api/static/404.html | 2 +- src/handler/api/static/404/index.html | 2 +- .../_buildManifest.js | 0 .../_ssgManifest.js | 0 .../static/chunks/171-0a6dd93b551f1ca6.js | 1 - .../static/chunks/171-7eacb8f2c45d0186.js | 1 + .../app/agents/page-65cc70b94cb9abd3.js | 1 + .../app/agents/page-ed5717b0ac0347b8.js | 1 - .../app/claude/page-235e137771d99edc.js | 1 + .../app/claude/page-6e97b9d68ff8eca2.js | 1 - src/handler/api/static/activity/index.html | 2 +- src/handler/api/static/activity/index.txt | 6 +- src/handler/api/static/agents/index.html | 2 +- src/handler/api/static/agents/index.txt | 6 +- src/handler/api/static/approvals/index.html | 2 +- src/handler/api/static/approvals/index.txt | 6 +- src/handler/api/static/claude/index.html | 2 +- src/handler/api/static/claude/index.txt | 6 +- src/handler/api/static/index.html | 2 +- src/handler/api/static/index.txt | 6 +- src/handler/api/static/login/index.html | 2 +- src/handler/api/static/login/index.txt | 4 +- .../api/static/repositories/index.html | 2 +- src/handler/api/static/repositories/index.txt | 6 +- src/handler/api/static/schedules/index.html | 2 +- src/handler/api/static/schedules/index.txt | 6 +- src/handler/api/static/servers/index.html | 2 +- src/handler/api/static/servers/index.txt | 6 +- src/handler/api/static/shared/index.html | 2 +- src/handler/api/static/shared/index.txt | 6 +- src/handler/control/cli.py | 17 + src/handler/control/models.py | 79 ++++ src/handler/control/spawn.py | 22 +- src/handler/control/worker.py | 1 + src/handler/db/repository.py | 61 ++++ src/handler/db/tables.py | 26 ++ .../migrations/versions/0012_claude_models.py | 51 +++ tests/test_claude_models.py | 339 ++++++++++++++++++ 47 files changed, 1219 insertions(+), 60 deletions(-) create mode 100644 docs/local-models.md rename src/handler/api/static/_next/static/{1cIeZLB5pMn-RK3Yuxqs0 => KmcUQ9fxOry3HfFR_U9HV}/_buildManifest.js (100%) rename src/handler/api/static/_next/static/{1cIeZLB5pMn-RK3Yuxqs0 => KmcUQ9fxOry3HfFR_U9HV}/_ssgManifest.js (100%) delete mode 100644 src/handler/api/static/_next/static/chunks/171-0a6dd93b551f1ca6.js create mode 100644 src/handler/api/static/_next/static/chunks/171-7eacb8f2c45d0186.js create mode 100644 src/handler/api/static/_next/static/chunks/app/agents/page-65cc70b94cb9abd3.js delete mode 100644 src/handler/api/static/_next/static/chunks/app/agents/page-ed5717b0ac0347b8.js create mode 100644 src/handler/api/static/_next/static/chunks/app/claude/page-235e137771d99edc.js delete mode 100644 src/handler/api/static/_next/static/chunks/app/claude/page-6e97b9d68ff8eca2.js create mode 100644 src/handler/control/models.py create mode 100644 src/handler/migrations/versions/0012_claude_models.py create mode 100644 tests/test_claude_models.py diff --git a/README.md b/README.md index 660b453..1eaf364 100644 --- a/README.md +++ b/README.md @@ -253,8 +253,16 @@ What the dashboard can now do (all state-changing actions require `ADMIN_TOKEN`) the audit log of what the dashboard triggered. The UI polls `GET /commands/{id}` for live status. - **Claude** — the management page for the Claude Code install agents run on. The account - login lives here (see below), plus web-managed **skills**, **MCP connectors**, - **plugins**, and **permission overrides**. These are plain DB rows the control container + login lives here (see below), plus web-managed **model backends**, **skills**, + **MCP connectors**, **plugins**, and **permission overrides**. Model backends are + Anthropic-API-compatible endpoints (a local Qwen/Llama behind LiteLLM or + claude-code-router, an LLM gateway) offered in the spawn form's **Model** dropdown next + to the Claude subscription: the same `claude` binary is pointed at the endpoint via + `ANTHROPIC_BASE_URL`/`ANTHROPIC_MODEL` env at launch, so hooks, skills, connectors, and + gates apply unchanged, and the agent stays pinned to its backend across resumes. API + keys are stored encrypted (`HANDLER_SECRET_KEY`) and never returned. See + [`docs/local-models.md`](docs/local-models.md) for working local stacks (and why bare + OpenAI-compatible servers break tool calling). These are plain DB rows the control container applies at every launch: skills sync to each worker's user-level `~/.claude/skills` (marker-file managed, so hand-installed skills survive), enabled connectors become the run's `--mcp-config` file (nothing lands in the repo tree), and plugins/permissions fold @@ -271,7 +279,7 @@ The command queue is exposed over HTTP as `POST …/agents/spawn`, `POST …/age `POST …/approvals`, `POST …/forge-init`, `POST …/poll-ci`, `POST …/sync`, `POST /login/start`, `POST /login/submit`, and `GET /commands[/{id}]`; hosts as `/hosts`; schedules as `/schedules` + `/projects/{id}/schedules`; project mutation as -`PATCH`/`DELETE /projects/{id}`; Claude management as `/claude/skills`, +`PATCH`/`DELETE /projects/{id}`; Claude management as `/claude/models`, `/claude/skills`, `/claude/connectors`, `/claude/plugins` (CRUD), and `GET`/`PUT /claude/permissions` (reads with the normal token, writes admin-gated). Run the worker with `handler worker` (the control image's default command). @@ -310,6 +318,7 @@ operators at a shell): ```bash handler spawn --project leeworks-api --name junior --role junior --worktree feat/auth --task "add login" +# [--model qwen3-coder] # run on a registered model backend (Claude page → Models) handler list [--project leeworks-api] handler attach --project leeworks-api --name junior handler kill --project leeworks-api --name junior diff --git a/docs/local-models.md b/docs/local-models.md new file mode 100644 index 0000000..3ef07fe --- /dev/null +++ b/docs/local-models.md @@ -0,0 +1,104 @@ +# Local model backends (Qwen-Coder & friends) + +Handler can run agents on locally-hosted models without changing anything about how an +agent works: it is still the same `claude` binary with the same generated +`settings.json`, hooks, skills, MCP connectors, plugins, and permission gates. The only +thing a **model backend** changes is the environment of that one agent's process: + +| Variable | From | +|---|---| +| `ANTHROPIC_BASE_URL` | the backend's `base_url` | +| `ANTHROPIC_AUTH_TOKEN` | the backend's stored API key (decrypted at launch; a placeholder when none is stored, so the subscription OAuth token is never sent to a local endpoint) | +| `ANTHROPIC_MODEL` | the backend's `model` | +| `ANTHROPIC_SMALL_FAST_MODEL` | `small_fast_model`, falling back to `model` | +| `CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC` | `1` (skip sidecar calls a local endpoint won't serve; override via the row's env map) | + +Register backends on the dashboard's **Claude → Models** tab (or `POST /claude/models`), +then pick one from the **Model** dropdown when spawning an agent. No selection = the +worker's logged-in Claude subscription, exactly as before. The agent is *pinned* to its +backend: resumes come back up on the same one, and deleting a backend makes resumes of +its agents fail loudly rather than silently falling back to the subscription. + +## Why "tool calling not working" happens with Qwen-Coder + +Claude Code speaks the **Anthropic Messages API** (`POST /v1/messages`): it sends tool +definitions in Anthropic's schema and expects structured `tool_use` content blocks back. +Local servers — Ollama, llama.cpp's `llama-server`, LM Studio, vLLM's default OpenAI +mode — speak the **OpenAI Chat Completions API** instead. Point `ANTHROPIC_BASE_URL` at +one of those and the request either 404s or, with a naive translator in between, the +model's tool calls come back as *plain text* (Qwen emits its own XML-ish +`` format) that Claude Code can't execute. That is the whole failure: the +model is fine, the dialect in the middle is wrong. + +Two things must both be true: + +1. **The endpoint must serve the Anthropic Messages API**, translating to whatever your + server speaks. +2. **The inference server must parse the model's native tool-call format into + structured tool calls** — for Qwen that means a Qwen-aware parser/template, not the + default one. + +## Working stacks + +### Recommended: vLLM (Qwen tool parser) + LiteLLM (Anthropic translation) + +vLLM parses Qwen's tool-call format natively when told to: + +```bash +# Qwen3-Coder +vllm serve Qwen/Qwen3-Coder-30B-A3B-Instruct \ + --enable-auto-tool-choice \ + --tool-call-parser qwen3_coder \ + --port 8000 + +# Qwen2.5-Coder uses the hermes parser instead: +# --tool-call-parser hermes +``` + +LiteLLM in front exposes the Anthropic `/v1/messages` endpoint: + +```yaml +# litellm-config.yaml +model_list: + - model_name: qwen3-coder-30b + litellm_params: + model: hosted_vllm/Qwen/Qwen3-Coder-30B-A3B-Instruct + api_base: http://127.0.0.1:8000/v1 +general_settings: + master_key: sk-local-anything +``` + +```bash +litellm --config litellm-config.yaml --port 4000 +``` + +Then register the backend in Handler: base URL `http://:4000`, model +`qwen3-coder-30b`, API key `sk-local-anything`. + +### llama.cpp / Ollama + +- `llama-server` needs `--jinja` (and, for Qwen, a chat template with tool support — + recent official Qwen GGUFs ship one; older community quants often don't, which is + another common source of "tools don't work"). +- Ollama supports OpenAI-style tool calling for models whose Modelfile template declares + it; check `ollama show --template` mentions `.Tools` before blaming the proxy. +- Either way, they still only speak OpenAI-dialect — keep LiteLLM (use + `ollama_chat/`, not `ollama/`, for tool support) or + [claude-code-router](https://github.com/musistudio/claude-code-router) in front as + the Anthropic translator. + +## Expectations and tips for small models + +- **Keep the harness light.** Handler's agents run tool-heavy (hooks, MCP connectors, + skills). A 7B model will fumble that loop; Qwen3-Coder-30B-class models handle it + reasonably. Disable connectors the agent doesn't need and keep tasks small and + concrete. +- **Raise timeouts, cap output.** The row's env map is the escape hatch: + `API_TIMEOUT_MS=600000`, `CLAUDE_CODE_MAX_OUTPUT_TOKENS=8192` are sensible for a + local 30B. +- **The gates don't relax.** The Stop/PreToolUse hooks still block un-tested, + un-pushed work regardless of which model produced it — that's the point of keeping + the same binary. +- **The subscription is untouched.** The web login, credential sync, and every agent + spawned without a model selection keep working exactly as before; backends are purely + additive. diff --git a/frontend/components/sections/AgentsSection.tsx b/frontend/components/sections/AgentsSection.tsx index 19c4b61..960e689 100644 --- a/frontend/components/sections/AgentsSection.tsx +++ b/frontend/components/sections/AgentsSection.tsx @@ -25,6 +25,7 @@ const emptySpawn = { worktree: "", subdir: "", task: "", + model_id: "", }; export function AgentsSection() { @@ -35,6 +36,22 @@ export function AgentsSection() { () => s.projects.map((p) => ({ value: p.id, label: p.id })), [s.projects], ); + /* The seamless switch: Claude subscription by default, plus every enabled backend + * registered on the Claude page's Models tab. Same binary, hooks, and skills either + * way — only the ANTHROPIC_* env of the launched process differs. */ + const modelOpts = useMemo( + () => [ + { value: "", label: "Claude (subscription)" }, + ...s.claudeModels + .filter((m) => m.enabled) + .map((m) => ({ value: String(m.id), label: `${m.name} (${m.model})` })), + ], + [s.claudeModels], + ); + const modelName = useMemo(() => { + const byId = new Map(s.claudeModels.map((m) => [m.id, m.name])); + return (id: number | null | undefined) => (id == null ? null : byId.get(id) ?? `#${id}`); + }, [s.claudeModels]); const agents = useMemo( () => s.agents.filter((a) => a.project_id === s.selectedProjectId), [s.agents, s.selectedProjectId], @@ -87,6 +104,12 @@ export function AgentsSection() { ) : ( setForm({ ...form, subdir: v })} placeholder="api" /> )} +