mirror of
https://github.com/0xWheatyz/handler.git
synced 2026-08-31 11:26:25 +00:00
feat(control,api,ui): worker liveness + run event stream (phase 3)
- worker: heartbeat every loop pass (workers registry); reaper pass
every ~15s marks a silent worker's running runs crashed and flips
agents stuck in 'working' to crashed (paused/blocked keep their
still-accurate status). Idempotent via finish_run's running-guard;
any surviving worker can reap; no auto-requeue (half-done runs may
have pushed). Dead workers' registry rows are dropped once settled.
- api: GET /projects/{p}/agents/{name}/events - the persisted
stream-json event log, oldest-first, cursor-paged by row id;
AgentOut exposes session_id/worker_id
- frontend: Run events panel in the run detail (assistant text, tool
chips, result footer with cost/turns, runner notices, raw lines),
cursor-appended on the existing 5s poll; 'Crashed' filter + danger
badge; crashed agents show their frozen last frame ('last output
before crash'); static export regenerated
Suite 290 -> 296 green; next build clean.
This commit is contained in:
@@ -144,11 +144,12 @@ export function AgentsSection() {
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
{a.status === "working" && a.last_output?.trim() && (
|
||||
{(a.status === "working" || a.status === "crashed") && a.last_output?.trim() && (
|
||||
<tr>
|
||||
<td colSpan={6} style={{ paddingTop: 0 }}>
|
||||
<div className="faint" style={{ fontSize: "var(--text-xs)", marginBottom: 4 }}>
|
||||
live output{a.output_at ? ` · ${timeAgo(a.output_at)}` : ""}
|
||||
{a.status === "crashed" ? "last output before crash" : "live output"}
|
||||
{a.output_at ? ` · ${timeAgo(a.output_at)}` : ""}
|
||||
</div>
|
||||
<pre
|
||||
className="mono"
|
||||
|
||||
@@ -7,13 +7,14 @@ import { useMemo, useState } from "react";
|
||||
import { useDashboard } from "@/components/store";
|
||||
import { Badge, Button, Callout, Stat, StatusBadge, Tabs, Textarea } from "@/components/ui";
|
||||
import { fmtFull, shortSha, statusTone, timeAgo } from "@/lib/format";
|
||||
import type { Agent } from "@/lib/api";
|
||||
import type { Agent, AgentEvent } from "@/lib/api";
|
||||
|
||||
const FILTERS = [
|
||||
{ value: "all", label: "All" },
|
||||
{ value: "needs", label: "Needs Input" },
|
||||
{ value: "working", label: "Working" },
|
||||
{ value: "done", label: "Done" },
|
||||
{ value: "crashed", label: "Crashed" },
|
||||
];
|
||||
|
||||
function matches(filter: string, status: string): boolean {
|
||||
@@ -21,6 +22,7 @@ function matches(filter: string, status: string): boolean {
|
||||
if (filter === "needs") return status === "paused_for_input";
|
||||
if (filter === "working") return status === "working" || status === "running";
|
||||
if (filter === "done") return status === "done" || status === "completed";
|
||||
if (filter === "crashed") return status === "crashed" || status === "blocked";
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -239,6 +241,40 @@ function RunDetail() {
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Headless run event stream (empty for legacy tmux agents) */}
|
||||
{agent?.session_id && (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 10 }}>
|
||||
<div className="eyebrow">
|
||||
Run events
|
||||
{agent.worker_id ? (
|
||||
<span className="faint mono" style={{ fontSize: "var(--text-xs)", marginLeft: 8 }}>
|
||||
on {agent.worker_id}
|
||||
</span>
|
||||
) : null}
|
||||
</div>
|
||||
{s.events.length === 0 ? (
|
||||
<div className="empty">No events yet.</div>
|
||||
) : (
|
||||
<div
|
||||
style={{
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
gap: 6,
|
||||
maxHeight: 420,
|
||||
overflow: "auto",
|
||||
padding: "10px 12px",
|
||||
background: "var(--surface-2, rgba(0,0,0,0.25))",
|
||||
borderRadius: 6,
|
||||
}}
|
||||
>
|
||||
{s.events.map((e) => (
|
||||
<EventLine key={e.id} e={e} />
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Log */}
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 10 }}>
|
||||
<div className="eyebrow">Log · newest first</div>
|
||||
@@ -304,3 +340,99 @@ function RunDetail() {
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
/* One stream-json event, rendered by type: assistant text as prose, tool calls as chips,
|
||||
* the result as a cost/turns footer, worker notices as callouts, raw lines verbatim. */
|
||||
function EventLine({ e }: { e: AgentEvent }) {
|
||||
const p = (e.payload ?? {}) as Record<string, any>;
|
||||
const xs = { fontSize: "var(--text-xs)" } as const;
|
||||
|
||||
if (e.type === "system") {
|
||||
return (
|
||||
<div className="faint mono" style={xs}>
|
||||
▸ session {p.subtype ?? "event"}
|
||||
{p.session_id ? ` · ${String(p.session_id).slice(0, 8)}` : ""}
|
||||
{Array.isArray(p.tools) ? ` · ${p.tools.length} tools` : ""}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
if (e.type === "assistant") {
|
||||
const content = p.message?.content;
|
||||
const blocks: any[] = Array.isArray(content) ? content : [];
|
||||
const text = blocks
|
||||
.filter((b) => b?.type === "text" && b.text)
|
||||
.map((b) => b.text)
|
||||
.join("\n");
|
||||
const tools = blocks.filter((b) => b?.type === "tool_use");
|
||||
return (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 4 }}>
|
||||
{text && (
|
||||
<div style={{ fontSize: "var(--text-sm)", whiteSpace: "pre-wrap" }}>{text}</div>
|
||||
)}
|
||||
{tools.length > 0 && (
|
||||
<div className="hstack" style={{ gap: 6, flexWrap: "wrap" }}>
|
||||
{tools.map((t, i) => (
|
||||
<Badge key={i} tone="info">
|
||||
{t.name}
|
||||
{t.input ? `: ${oneLine(t.input)}` : ""}
|
||||
</Badge>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
if (e.type === "result") {
|
||||
const err = Boolean(p.is_error);
|
||||
return (
|
||||
<div className="hstack" style={{ gap: 8, flexWrap: "wrap" }}>
|
||||
<Badge tone={err ? "danger" : "success"}>{err ? "run errored" : "run finished"}</Badge>
|
||||
<span className="faint mono" style={xs}>
|
||||
{p.num_turns != null ? `${p.num_turns} turns` : ""}
|
||||
{p.total_cost_usd != null ? ` · $${Number(p.total_cost_usd).toFixed(4)}` : ""}
|
||||
</span>
|
||||
{typeof p.result === "string" && p.result && (
|
||||
<span className="muted" style={{ ...xs, whiteSpace: "pre-wrap", width: "100%" }}>
|
||||
{p.result}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
if (e.type === "worker") {
|
||||
return (
|
||||
<Callout tone="danger">
|
||||
{p.notice ?? "runner notice"}
|
||||
{p.stderr_tail ? (
|
||||
<pre className="mono" style={{ ...xs, margin: "6px 0 0", whiteSpace: "pre-wrap" }}>
|
||||
{p.stderr_tail}
|
||||
</pre>
|
||||
) : null}
|
||||
</Callout>
|
||||
);
|
||||
}
|
||||
if (e.type === "raw") {
|
||||
return (
|
||||
<div className="faint mono" style={{ ...xs, whiteSpace: "pre-wrap" }}>
|
||||
{typeof p.line === "string" ? p.line.trimEnd() : JSON.stringify(p)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
// user (tool results) and anything future: a quiet one-liner, nothing lost, no noise.
|
||||
return (
|
||||
<div className="faint mono" style={xs}>
|
||||
▸ {e.type}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/* Compact single-line preview of a tool_use input object. */
|
||||
function oneLine(input: unknown): string {
|
||||
const s =
|
||||
typeof input === "string"
|
||||
? input
|
||||
: (input as Record<string, unknown>)?.command
|
||||
? String((input as Record<string, unknown>).command)
|
||||
: JSON.stringify(input);
|
||||
return s.length > 80 ? `${s.slice(0, 77)}…` : s;
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
AuthError,
|
||||
createClient,
|
||||
type Agent,
|
||||
type AgentEvent,
|
||||
type ApiError,
|
||||
type Approval,
|
||||
type Checkmark,
|
||||
@@ -83,6 +84,9 @@ interface StoreValue {
|
||||
log: LogEntry[];
|
||||
logOffset: number;
|
||||
pageLog: (dir: 1 | -1) => void;
|
||||
/* Headless run event stream for the selected run, oldest-first, appended by cursor
|
||||
* polls (empty for legacy tmux agents). */
|
||||
events: AgentEvent[];
|
||||
|
||||
approvals: Approval[];
|
||||
hosts: Host[];
|
||||
@@ -204,6 +208,9 @@ export function DashboardProvider({
|
||||
const [checkmarkMissing, setCheckmarkMissing] = useState(false);
|
||||
const [log, setLog] = useState<LogEntry[]>([]);
|
||||
const [logOffset, setLogOffset] = useState(0);
|
||||
const [events, setEvents] = useState<AgentEvent[]>([]);
|
||||
const eventsRef = useRef<AgentEvent[]>([]);
|
||||
eventsRef.current = events;
|
||||
const [approvals, setApprovals] = useState<Approval[]>([]);
|
||||
const [hosts, setHosts] = useState<Host[]>([]);
|
||||
const [commands, setCommands] = useState<Command[]>([]);
|
||||
@@ -282,6 +289,17 @@ export function DashboardProvider({
|
||||
} catch (e) {
|
||||
swallow(e);
|
||||
}
|
||||
try {
|
||||
// Cursor poll: only events newer than what we already hold come back.
|
||||
const cur = eventsRef.current;
|
||||
const after = cur.length ? cur[cur.length - 1].id : 0;
|
||||
const fresh = await clientRef.current.api<AgentEvent[]>(
|
||||
`${path}/events?after_id=${after}&limit=500`,
|
||||
);
|
||||
if (fresh.length) setEvents((prev) => [...prev, ...fresh]);
|
||||
} catch (e) {
|
||||
swallow(e);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const loadApprovals = useCallback(async (projectId: string) => {
|
||||
@@ -407,6 +425,8 @@ export function DashboardProvider({
|
||||
setCheckmark(null);
|
||||
setCheckmarkMissing(false);
|
||||
setLog([]);
|
||||
setEvents([]);
|
||||
eventsRef.current = [];
|
||||
void loadRun(projectId, name);
|
||||
},
|
||||
[loadRun],
|
||||
@@ -925,6 +945,7 @@ export function DashboardProvider({
|
||||
log,
|
||||
logOffset,
|
||||
pageLog,
|
||||
events,
|
||||
approvals,
|
||||
hosts,
|
||||
commands,
|
||||
|
||||
+20
-2
@@ -30,10 +30,28 @@ export interface Agent {
|
||||
working_dir: string;
|
||||
status: string;
|
||||
role?: string | null;
|
||||
/* Latest tmux pane-tail snapshot from the worker, so the UI can show what a running
|
||||
* agent is actually doing (and expose one wedged on an interactive prompt). */
|
||||
/* Latest output snapshot from the worker: the tmux pane tail for legacy agents, the
|
||||
* latest assistant text for headless runs. For a crashed agent this is the evidence
|
||||
* frame — the last thing the process said. */
|
||||
last_output?: string | null;
|
||||
output_at?: string | null;
|
||||
/* Headless runner: claude session UUID (null = legacy tmux agent) + supervising worker. */
|
||||
session_id?: string | null;
|
||||
worker_id?: string | null;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
/* One persisted stream-json event of a headless run (GET .../events, cursor-paged by id).
|
||||
* `type` mirrors the stream (system/assistant/user/result) plus `worker` (runner notices)
|
||||
* and `raw` (unparseable line kept verbatim). */
|
||||
export interface AgentEvent {
|
||||
id: number;
|
||||
agent_id: number;
|
||||
run_id: number;
|
||||
session_id?: string | null;
|
||||
seq: number;
|
||||
type: string;
|
||||
payload?: Record<string, unknown> | null;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ export function statusTone(status: string | null | undefined): Tone {
|
||||
case "blocked":
|
||||
case "rejected":
|
||||
case "error":
|
||||
case "crashed":
|
||||
return "danger";
|
||||
case "pending":
|
||||
case "queued":
|
||||
|
||||
@@ -15,7 +15,15 @@ from sqlalchemy.exc import IntegrityError
|
||||
from ...config import get_settings
|
||||
from ...db import repository as repo
|
||||
from ..deps import db_conn, require_admin, require_auth
|
||||
from ..schemas import AgentIn, AgentOut, CheckmarkOut, CommandOut, LogEntryOut, SpawnIn
|
||||
from ..schemas import (
|
||||
AgentEventOut,
|
||||
AgentIn,
|
||||
AgentOut,
|
||||
CheckmarkOut,
|
||||
CommandOut,
|
||||
LogEntryOut,
|
||||
SpawnIn,
|
||||
)
|
||||
from .common import resolve_agent
|
||||
|
||||
router = APIRouter(
|
||||
@@ -123,6 +131,23 @@ def get_checkmark(project: str, name: str, conn: Connection = Depends(db_conn))
|
||||
return checkmark
|
||||
|
||||
|
||||
@router.get("/{name}/events", response_model=list[AgentEventOut])
|
||||
def get_events(
|
||||
project: str,
|
||||
name: str,
|
||||
after_id: int = Query(0, ge=0),
|
||||
limit: int = Query(200, ge=1, le=1000),
|
||||
conn: Connection = Depends(db_conn),
|
||||
) -> list[dict]:
|
||||
"""The headless run event stream, oldest-first, cursor-paged by row id.
|
||||
|
||||
The UI polls with ``after_id`` = the largest id it has seen, so each poll returns
|
||||
only new events (an empty list for a legacy tmux agent or an idle one).
|
||||
"""
|
||||
agent = resolve_agent(conn, project, name)
|
||||
return repo.list_agent_events(conn, agent["id"], after_id=after_id, limit=limit)
|
||||
|
||||
|
||||
@router.get("/{name}/log", response_model=list[LogEntryOut])
|
||||
def get_log(
|
||||
project: str,
|
||||
|
||||
@@ -128,10 +128,14 @@ class AgentOut(BaseModel):
|
||||
working_dir: str
|
||||
status: str
|
||||
role: Role | None = None
|
||||
# Latest tmux pane-tail snapshot (worker poll loop) so the UI can show what a running
|
||||
# agent is doing — including one wedged on an interactive prompt no one can answer.
|
||||
# Latest output snapshot so the UI can show what a running agent is doing: the tmux
|
||||
# pane tail for legacy agents, the latest assistant text for headless runs.
|
||||
last_output: str | None = None
|
||||
output_at: datetime | None = None
|
||||
# Headless runner: the claude session UUID (null = legacy tmux agent) and the worker
|
||||
# container supervising (or last to supervise) this agent's runs.
|
||||
session_id: str | None = None
|
||||
worker_id: str | None = None
|
||||
created_at: datetime
|
||||
|
||||
|
||||
@@ -306,6 +310,25 @@ class LogEntryOut(BaseModel):
|
||||
ci_checked_at: datetime | None = None
|
||||
|
||||
|
||||
class AgentEventOut(BaseModel):
|
||||
"""One persisted stream-json event of a headless run (the UI's live log panel).
|
||||
|
||||
``type`` mirrors the stream's top-level type (system/assistant/user/result), plus
|
||||
``worker`` (runner-generated notices) and ``raw`` (unparseable line, kept verbatim).
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
id: int
|
||||
agent_id: int
|
||||
run_id: int
|
||||
session_id: str | None = None
|
||||
seq: int
|
||||
type: str
|
||||
payload: dict | None = None
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class AnswerIn(BaseModel):
|
||||
answer: str
|
||||
# If omitted, the answer targets the agent's latest open question.
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -1 +0,0 @@
|
||||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[185],{3156:function(n,e,u){Promise.resolve().then(u.t.bind(u,7960,23))},7960:function(){}},function(n){n.O(0,[587,971,117,744],function(){return n(n.s=3156)}),_N_E=n.O()}]);
|
||||
@@ -0,0 +1 @@
|
||||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[185],{2385:function(n,e,u){Promise.resolve().then(u.t.bind(u,7960,23))},7960:function(){}},function(n){n.O(0,[587,971,117,744],function(){return n(n.s=2385)}),_N_E=n.O()}]);
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+1
-1
@@ -1 +1 @@
|
||||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[744],{6994:function(e,n,t){Promise.resolve().then(t.t.bind(t,2846,23)),Promise.resolve().then(t.t.bind(t,9107,23)),Promise.resolve().then(t.t.bind(t,1060,23)),Promise.resolve().then(t.t.bind(t,4707,23)),Promise.resolve().then(t.t.bind(t,80,23)),Promise.resolve().then(t.t.bind(t,6423,23))}},function(e){var n=function(n){return e(e.s=n)};e.O(0,[971,117],function(){return n(4278),n(6994)}),_N_E=e.O()}]);
|
||||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[744],{2730:function(e,n,t){Promise.resolve().then(t.t.bind(t,2846,23)),Promise.resolve().then(t.t.bind(t,9107,23)),Promise.resolve().then(t.t.bind(t,1060,23)),Promise.resolve().then(t.t.bind(t,4707,23)),Promise.resolve().then(t.t.bind(t,80,23)),Promise.resolve().then(t.t.bind(t,6423,23))}},function(e){var n=function(n){return e(e.s=n)};e.O(0,[971,117],function(){return n(4278),n(2730)}),_N_E=e.O()}]);
|
||||
@@ -1 +1 @@
|
||||
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="stylesheet" href="/_next/static/css/bbd6ee1e7265be74.css" data-precedence="next"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/webpack-29957205745576aa.js"/><script src="/_next/static/chunks/fd9d1056-e47440ca368281f5.js" async=""></script><script src="/_next/static/chunks/117-d9d1ed00b6332a85.js" async=""></script><script src="/_next/static/chunks/main-app-8321800782c5a11e.js" async=""></script><script src="/_next/static/chunks/app/page-a511d608db773c67.js" async=""></script><title>Handler · Claude Activity</title><meta name="description" content="Monitor and manage Claude Code agents across projects."/><link rel="icon" href="/icon.svg?bab509b45a421e43" type="image/svg+xml" sizes="any"/><script src="/_next/static/chunks/polyfills-42372ed130431b0a.js" noModule=""></script></head><body><div class="gate"><form class="gate-card"><div class="gate-brand"><span class="logo" style="width:26px;height:26px;border-radius:7px"></span>Claude Monitor</div><p class="muted" style="font-size:var(--text-sm);margin:0">Paste your API token to continue. Management actions require the admin token; read-only views work with the plain auth token.</p><input class="input" type="password" autoComplete="current-password" placeholder="API token" autofocus="" value=""/><button class="btn btn-primary" type="submit">Continue</button></form></div><script src="/_next/static/chunks/webpack-29957205745576aa.js" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/_next/static/css/bbd6ee1e7265be74.css\",\"style\"]\n"])</script><script>self.__next_f.push([1,"2:I[2846,[],\"\"]\n4:I[9107,[],\"ClientPageRoot\"]\n5:I[9859,[\"931\",\"static/chunks/app/page-a511d608db773c67.js\"],\"default\",1]\n6:I[4707,[],\"\"]\n7:I[6423,[],\"\"]\n9:I[1060,[],\"\"]\na:[]\n0:[\"$\",\"$L2\",null,{\"buildId\":\"tQnVLKX-Dr8H3frHzPrCa\",\"assetPrefix\":\"\",\"urlParts\":[\"\",\"\"],\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[[\"$L3\",[\"$\",\"$L4\",null,{\"props\":{\"params\":{},\"searchParams\":{}},\"Component\":\"$5\"}],null],null],null]},[[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/bbd6ee1e7265be74.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\"}]],[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"children\":[\"$\",\"$L6\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L7\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyl"])</script><script>self.__next_f.push([1,"es\":[]}]}]}]],null],null],\"couldBeIntercepted\":false,\"initialHead\":[null,\"$L8\"],\"globalErrorComponent\":\"$9\",\"missingSlots\":\"$Wa\"}]\n"])</script><script>self.__next_f.push([1,"8:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"Handler · Claude Activity\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"Monitor and manage Claude Code agents across projects.\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/icon.svg?bab509b45a421e43\",\"type\":\"image/svg+xml\",\"sizes\":\"any\"}]]\n3:null\n"])</script></body></html>
|
||||
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="stylesheet" href="/_next/static/css/bbd6ee1e7265be74.css" data-precedence="next"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/webpack-29957205745576aa.js"/><script src="/_next/static/chunks/fd9d1056-e47440ca368281f5.js" async=""></script><script src="/_next/static/chunks/117-d9d1ed00b6332a85.js" async=""></script><script src="/_next/static/chunks/main-app-cb8ae0c3c2bb8b85.js" async=""></script><script src="/_next/static/chunks/app/page-0637b8cf149a88ec.js" async=""></script><title>Handler · Claude Activity</title><meta name="description" content="Monitor and manage Claude Code agents across projects."/><link rel="icon" href="/icon.svg?bab509b45a421e43" type="image/svg+xml" sizes="any"/><script src="/_next/static/chunks/polyfills-42372ed130431b0a.js" noModule=""></script></head><body><div class="gate"><form class="gate-card"><div class="gate-brand"><span class="logo" style="width:26px;height:26px;border-radius:7px"></span>Claude Monitor</div><p class="muted" style="font-size:var(--text-sm);margin:0">Paste your API token to continue. Management actions require the admin token; read-only views work with the plain auth token.</p><input class="input" type="password" autoComplete="current-password" placeholder="API token" autofocus="" value=""/><button class="btn btn-primary" type="submit">Continue</button></form></div><script src="/_next/static/chunks/webpack-29957205745576aa.js" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/_next/static/css/bbd6ee1e7265be74.css\",\"style\"]\n"])</script><script>self.__next_f.push([1,"2:I[2846,[],\"\"]\n4:I[9107,[],\"ClientPageRoot\"]\n5:I[9859,[\"931\",\"static/chunks/app/page-0637b8cf149a88ec.js\"],\"default\",1]\n6:I[4707,[],\"\"]\n7:I[6423,[],\"\"]\n9:I[1060,[],\"\"]\na:[]\n0:[\"$\",\"$L2\",null,{\"buildId\":\"ga21jhjKhYsutf8F3vo_-\",\"assetPrefix\":\"\",\"urlParts\":[\"\",\"\"],\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[[\"$L3\",[\"$\",\"$L4\",null,{\"props\":{\"params\":{},\"searchParams\":{}},\"Component\":\"$5\"}],null],null],null]},[[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/bbd6ee1e7265be74.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\"}]],[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"children\":[\"$\",\"$L6\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L7\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyl"])</script><script>self.__next_f.push([1,"es\":[]}]}]}]],null],null],\"couldBeIntercepted\":false,\"initialHead\":[null,\"$L8\"],\"globalErrorComponent\":\"$9\",\"missingSlots\":\"$Wa\"}]\n"])</script><script>self.__next_f.push([1,"8:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"Handler · Claude Activity\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"Monitor and manage Claude Code agents across projects.\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/icon.svg?bab509b45a421e43\",\"type\":\"image/svg+xml\",\"sizes\":\"any\"}]]\n3:null\n"])</script></body></html>
|
||||
@@ -1,7 +1,7 @@
|
||||
2:I[9107,[],"ClientPageRoot"]
|
||||
3:I[9859,["931","static/chunks/app/page-a511d608db773c67.js"],"default",1]
|
||||
3:I[9859,["931","static/chunks/app/page-0637b8cf149a88ec.js"],"default",1]
|
||||
4:I[4707,[],""]
|
||||
5:I[6423,[],""]
|
||||
0:["tQnVLKX-Dr8H3frHzPrCa",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/bbd6ee1e7265be74.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
|
||||
0:["ga21jhjKhYsutf8F3vo_-",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/bbd6ee1e7265be74.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
|
||||
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"Handler · Claude Activity"}],["$","meta","3",{"name":"description","content":"Monitor and manage Claude Code agents across projects."}],["$","link","4",{"rel":"icon","href":"/icon.svg?bab509b45a421e43","type":"image/svg+xml","sizes":"any"}]]
|
||||
1:null
|
||||
|
||||
@@ -298,6 +298,63 @@ def _run_one(command: dict) -> None:
|
||||
repo.finish_command(conn, command["id"], "failed", error=str(exc))
|
||||
|
||||
|
||||
def heartbeat(worker_id: str) -> None:
|
||||
"""Refresh this worker's registry row — the reaper's proof-of-life."""
|
||||
with connection() as conn:
|
||||
active = len(repo.list_running_runs(conn, worker_id=worker_id))
|
||||
repo.upsert_worker_heartbeat(
|
||||
conn,
|
||||
worker_id,
|
||||
hostname=socket.gethostname(),
|
||||
pid=os.getpid(),
|
||||
max_runs=get_settings().max_concurrent_runs,
|
||||
active_runs=active,
|
||||
)
|
||||
|
||||
|
||||
def reap_dead_workers(now: datetime | None = None) -> int:
|
||||
"""Settle the record for workers that stopped heartbeating: their ``running`` runs
|
||||
become ``crashed``, and agents those runs left in ``working`` become ``crashed`` too.
|
||||
|
||||
Any surviving worker may reap any dead one — ``finish_run``'s still-running guard
|
||||
makes concurrent reapers idempotent. Deliberately NO auto-requeue: a half-done run
|
||||
may have already pushed commits, so re-running it isn't safe to assume; the operator
|
||||
sees the crashed badge (with the frozen last_output as evidence) and resumes
|
||||
explicitly. Agents in ``paused_for_input``/``blocked`` keep their status — those are
|
||||
still accurate descriptions of what the agent needs. Returns runs reaped.
|
||||
"""
|
||||
now = now or datetime.now(UTC)
|
||||
cutoff = now - timedelta(seconds=get_settings().worker_stale_after)
|
||||
reaped = 0
|
||||
with connection() as conn:
|
||||
stale = repo.list_stale_workers(conn, cutoff)
|
||||
for dead in stale:
|
||||
with connection() as conn:
|
||||
for run in repo.list_running_runs(conn, worker_id=dead["id"]):
|
||||
if not repo.finish_run(conn, run["id"], "crashed"):
|
||||
continue # another reaper won the race
|
||||
agent = repo.get_agent_by_id(conn, run["agent_id"])
|
||||
if agent is not None and agent["status"] == "working":
|
||||
repo.set_agent_status(conn, agent["id"], "crashed")
|
||||
repo.insert_agent_event(
|
||||
conn,
|
||||
run["agent_id"],
|
||||
run["id"],
|
||||
seq=0, # runner-generated, outside the supervisor's line counter
|
||||
type="worker",
|
||||
payload={
|
||||
"notice": "worker died mid-run; run marked crashed",
|
||||
"worker_id": dead["id"],
|
||||
},
|
||||
session_id=run["session_id"],
|
||||
)
|
||||
reaped += 1
|
||||
# Row served its purpose; dropping it keeps the registry to live workers
|
||||
# (and stops every future pass re-scanning long-dead ids).
|
||||
repo.delete_worker(conn, dead["id"])
|
||||
return reaped
|
||||
|
||||
|
||||
def fire_due_schedules(now: datetime | None = None) -> int:
|
||||
"""Enqueue a spawn command for every schedule whose ``next_run_at`` has passed.
|
||||
|
||||
@@ -419,10 +476,11 @@ def run(
|
||||
ci_interval: float = 30.0,
|
||||
capture_interval: float = 2.0,
|
||||
credsync_interval: float = 30.0,
|
||||
reap_interval: float = 15.0,
|
||||
iterations: int | None = None,
|
||||
) -> None:
|
||||
"""The control-container main loop: drain the command queue, snapshot live agent
|
||||
output, sync claude credentials, and sweep CI periodically.
|
||||
output, sync claude credentials, heartbeat + reap dead workers, and sweep CI.
|
||||
|
||||
``iterations`` bounds the loop for tests; production runs unbounded. Sleeps
|
||||
``poll_interval`` only when a pass found no commands, so bursts drain promptly.
|
||||
@@ -431,8 +489,19 @@ def run(
|
||||
last_ci = 0.0
|
||||
last_capture = 0.0
|
||||
last_credsync = 0.0
|
||||
last_reap = 0.0
|
||||
count = 0
|
||||
while iterations is None or count < iterations:
|
||||
try:
|
||||
heartbeat(worker_id)
|
||||
except Exception: # noqa: BLE001 - a heartbeat hiccup must not kill the worker
|
||||
pass
|
||||
if reap_interval > 0 and time.monotonic() - last_reap >= reap_interval:
|
||||
try:
|
||||
reap_dead_workers()
|
||||
except Exception: # noqa: BLE001 - a reap hiccup must not kill the worker
|
||||
pass
|
||||
last_reap = time.monotonic()
|
||||
try:
|
||||
fire_due_schedules()
|
||||
except Exception: # noqa: BLE001 - a schedule hiccup must not kill the worker
|
||||
|
||||
@@ -727,6 +727,12 @@ def list_stale_workers(conn: Connection, cutoff: datetime) -> list[dict]:
|
||||
return [dict(r._mapping) for r in rows]
|
||||
|
||||
|
||||
def delete_worker(conn: Connection, worker_id: str) -> bool:
|
||||
"""Drop a (dead) worker's registry row after its runs have been settled."""
|
||||
result = conn.execute(workers.delete().where(workers.c.id == worker_id))
|
||||
return result.rowcount > 0
|
||||
|
||||
|
||||
def create_run(
|
||||
conn: Connection, agent_id: int, session_id: str, worker_id: str, kind: str
|
||||
) -> dict:
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Worker heartbeat + reaper: a worker that stops heartbeating gets its running runs
|
||||
(and their still-'working' agents) marked crashed by any surviving worker — the positive
|
||||
liveness that replaces tmux scraping. No auto-requeue: half-done runs may have pushed."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from handler.control import worker
|
||||
from handler.db import repository as repo
|
||||
from handler.db.engine import get_engine
|
||||
|
||||
|
||||
def _seed_run(worker_id, agent_name="a", agent_status="working"):
|
||||
with get_engine().begin() as conn:
|
||||
if repo.get_project(conn, "p") is None:
|
||||
repo.create_project(conn, "p", "/tmp/p")
|
||||
agent = repo.create_agent(conn, "p", agent_name, f"/tmp/p/{agent_name}",
|
||||
status=agent_status)
|
||||
run = repo.create_run(conn, agent["id"], f"sid-{agent_name}", worker_id, "spawn")
|
||||
return agent, run
|
||||
|
||||
|
||||
def _hb(worker_id, age_seconds=0.0):
|
||||
with get_engine().begin() as conn:
|
||||
repo.upsert_worker_heartbeat(conn, worker_id, hostname="h", pid=1)
|
||||
if age_seconds:
|
||||
from handler.db.tables import workers as workers_table
|
||||
|
||||
conn.execute(
|
||||
workers_table.update()
|
||||
.where(workers_table.c.id == worker_id)
|
||||
.values(heartbeat_at=datetime.now(UTC) - timedelta(seconds=age_seconds))
|
||||
)
|
||||
|
||||
|
||||
def test_reaper_marks_stale_workers_runs_crashed(env):
|
||||
agent, run = _seed_run("w-dead")
|
||||
_hb("w-dead", age_seconds=120) # stale: default worker_stale_after is 60s
|
||||
|
||||
assert worker.reap_dead_workers() == 1
|
||||
|
||||
with get_engine().begin() as conn:
|
||||
assert repo.get_run(conn, run["id"])["status"] == "crashed"
|
||||
assert repo.get_agent_by_id(conn, agent["id"])["status"] == "crashed"
|
||||
events = repo.list_agent_events(conn, agent["id"])
|
||||
# The dead worker's registry row is dropped once settled.
|
||||
assert repo.list_stale_workers(conn, datetime.now(UTC) + timedelta(days=1)) == []
|
||||
assert events[0]["type"] == "worker"
|
||||
assert events[0]["payload"]["worker_id"] == "w-dead"
|
||||
|
||||
|
||||
def test_reaper_leaves_fresh_workers_alone(env):
|
||||
agent, run = _seed_run("w-live")
|
||||
_hb("w-live") # fresh heartbeat
|
||||
|
||||
assert worker.reap_dead_workers() == 0
|
||||
with get_engine().begin() as conn:
|
||||
assert repo.get_run(conn, run["id"])["status"] == "running"
|
||||
assert repo.get_agent_by_id(conn, agent["id"])["status"] == "working"
|
||||
|
||||
|
||||
def test_reaper_preserves_paused_agent_status(env):
|
||||
"""paused_for_input still accurately describes what the agent needs — only agents
|
||||
stuck in 'working' get flipped to crashed."""
|
||||
agent, run = _seed_run("w-dead2", agent_name="paused", agent_status="paused_for_input")
|
||||
_hb("w-dead2", age_seconds=120)
|
||||
|
||||
assert worker.reap_dead_workers() == 1
|
||||
with get_engine().begin() as conn:
|
||||
assert repo.get_run(conn, run["id"])["status"] == "crashed"
|
||||
assert repo.get_agent_by_id(conn, agent["id"])["status"] == "paused_for_input"
|
||||
|
||||
|
||||
def test_reaper_idempotent_against_races(env):
|
||||
agent, run = _seed_run("w-dead3", agent_name="raced")
|
||||
_hb("w-dead3", age_seconds=120)
|
||||
assert worker.reap_dead_workers() == 1
|
||||
# A second reaper (or the same one next pass) finds nothing left to settle.
|
||||
assert worker.reap_dead_workers() == 0
|
||||
with get_engine().begin() as conn:
|
||||
assert len(repo.list_agent_events(conn, agent["id"])) == 1
|
||||
|
||||
|
||||
def test_heartbeat_registers_worker(env):
|
||||
worker.heartbeat("w-me")
|
||||
with get_engine().begin() as conn:
|
||||
rows = repo.list_stale_workers(conn, datetime.now(UTC) + timedelta(seconds=1))
|
||||
assert [w["id"] for w in rows] == ["w-me"]
|
||||
assert rows[0]["hostname"]
|
||||
assert rows[0]["active_runs"] == 0
|
||||
|
||||
|
||||
def test_events_route_serves_stream(env, client, auth):
|
||||
agent, run = _seed_run("w-api", agent_name="api-agent")
|
||||
with get_engine().begin() as conn:
|
||||
for seq in range(1, 4):
|
||||
repo.insert_agent_event(
|
||||
conn, agent["id"], run["id"], seq=seq, type="assistant",
|
||||
payload={"n": seq}, session_id=run["session_id"],
|
||||
)
|
||||
|
||||
resp = client.get("/projects/p/agents/api-agent/events", headers=auth)
|
||||
assert resp.status_code == 200
|
||||
events = resp.json()
|
||||
assert [e["seq"] for e in events] == [1, 2, 3]
|
||||
assert events[0]["type"] == "assistant"
|
||||
|
||||
# Cursor paging: only events after the given id come back.
|
||||
resp = client.get(
|
||||
f"/projects/p/agents/api-agent/events?after_id={events[1]['id']}", headers=auth
|
||||
)
|
||||
assert [e["seq"] for e in resp.json()] == [3]
|
||||
|
||||
resp = client.get("/projects/p/agents/missing/events", headers=auth)
|
||||
assert resp.status_code == 404
|
||||
Reference in New Issue
Block a user