feat(db,control): dormant headless-runner schema, stream parser, and fixtures (phase 1)

Groundwork for replacing tmux-TUI agent runs with worker-owned
'claude -p --output-format stream-json' subprocesses (Postgres as the
single source of truth; no shared files between workers):

- migration 0008: workers (heartbeat registry), agent_runs (one row per
  headless invocation), agent_events (persisted stream-json event log),
  session_archives (tar.gz'd claude session per agent for cross-worker
  --resume); agents gains session_id/worker_id, commands gains
  target_worker; 'crashed' joins the agent status vocabulary
- control/headless.py: munged-path helper, argv builders, tolerant
  stream parser, archive/materialize round-trip, RunSupervisor + launch
  (nothing calls it yet - runner config still defaults to tmux)
- repository: run/event/archive/worker accessors; claim_next_command
  learns target_worker pinning and type exclusion for full slots
- tests/fixtures/fake_claude.py: scripted stream-json stand-in binary
- scripts/validate_claude_headless.sh: manual real-binary validation
  checklist (resume-on-clean-HOME linchpin, hook behavior under -p)

No behavior change; suite 245 -> 270 green.
This commit is contained in:
2026-07-21 22:41:46 -04:00
parent 40390c0568
commit f3acc57015
9 changed files with 1496 additions and 4 deletions
+163
View File
@@ -0,0 +1,163 @@
#!/usr/bin/env python3
"""A stand-in ``claude`` binary for headless-runner tests.
Wired in via the existing ``claude_bin`` setting (the same seam the tmux fakes used).
Parses the real headless argv, emits a scripted ``--output-format stream-json`` stream on
stdout, and writes a genuine session transcript + sidecar under
``$HOME/.claude/projects/<munged-cwd>/`` so archive/materialize/resume paths exercise the
real filesystem layout. Behavior is selected with ``FAKE_CLAUDE_MODE``:
- ``success`` (default): init + assistant + result, exit 0.
- ``error``: init + assistant + one garbage line, then exit 2 with no result event.
- ``hang``: init, then sleep forever (the kill/cancel/reaper tests SIGTERM it).
- ``slow``: like success with a pause between events (concurrency tests).
- ``resume-fail``: a ``--resume`` invocation exits 1 before any assistant event
(exercises the context re-injection fallback).
On ``--resume`` (outside resume-fail) the transcript materialized by the worker MUST
already exist at the expected path — missing means cross-worker materialization broke,
so the fake fails loudly, exit 3.
"""
from __future__ import annotations
import json
import os
import signal
import sys
import time
from pathlib import Path
def _parse_argv(argv: list[str]) -> dict:
opts = {
"print": False,
"verbose": False,
"output_format": None,
"session_id": None,
"resume": None,
"settings": None,
"budget": None,
"prompt": None,
}
i = 0
while i < len(argv):
arg = argv[i]
if arg == "-p":
opts["print"] = True
elif arg == "--verbose":
opts["verbose"] = True
elif arg == "--output-format":
i += 1
opts["output_format"] = argv[i]
elif arg == "--session-id":
i += 1
opts["session_id"] = argv[i]
elif arg in ("--resume", "-r"):
i += 1
opts["resume"] = argv[i]
elif arg == "--settings":
i += 1
opts["settings"] = argv[i]
elif arg == "--max-budget-usd":
i += 1
opts["budget"] = argv[i]
elif arg == "--":
opts["prompt"] = " ".join(argv[i + 1 :])
break
else:
opts["prompt"] = arg
i += 1
return opts
def _munged(cwd: str) -> str:
return cwd.replace("/", "-").replace(".", "-")
def _emit(event: dict) -> None:
sys.stdout.write(json.dumps(event) + "\n")
sys.stdout.flush()
def _write_transcript(session_id: str, prompt: str) -> None:
base = Path(os.path.expanduser("~")) / ".claude" / "projects" / _munged(os.getcwd())
base.mkdir(parents=True, exist_ok=True)
jsonl = base / f"{session_id}.jsonl"
with jsonl.open("a") as fh:
fh.write(json.dumps({"type": "user", "prompt": prompt}) + "\n")
fh.write(json.dumps({"type": "assistant", "text": f"handled: {prompt}"}) + "\n")
sidecar = base / session_id / "tool-results"
sidecar.mkdir(parents=True, exist_ok=True)
(sidecar / "result-1.txt").write_text("fake tool output\n")
def main() -> int:
signal.signal(signal.SIGTERM, signal.SIG_DFL)
mode = os.environ.get("FAKE_CLAUDE_MODE", "success")
opts = _parse_argv(sys.argv[1:])
if not opts["print"] or opts["output_format"] != "stream-json":
sys.stderr.write("fake_claude: expected -p --output-format stream-json\n")
return 64
session_id = opts["session_id"] or opts["resume"] or "fake-session"
prompt = opts["prompt"] or ""
if mode == "resume-fail" and opts["resume"]:
sys.stderr.write("fake_claude: no conversation found to resume\n")
return 1
if opts["resume"]:
base = Path(os.path.expanduser("~")) / ".claude" / "projects" / _munged(os.getcwd())
if not (base / f"{session_id}.jsonl").exists():
sys.stderr.write(f"fake_claude: transcript missing at {base}\n")
return 3
_emit(
{
"type": "system",
"subtype": "init",
"session_id": session_id,
"cwd": os.getcwd(),
"tools": ["Bash", "Read", "Edit"],
}
)
if mode == "hang":
time.sleep(3600)
return 0
if mode == "slow":
time.sleep(float(os.environ.get("FAKE_CLAUDE_SLOW_SECONDS", "1.0")))
_emit(
{
"type": "assistant",
"session_id": session_id,
"message": {
"role": "assistant",
"content": [{"type": "text", "text": f"working on: {prompt}"}],
},
}
)
_write_transcript(session_id, prompt)
if mode == "error":
sys.stdout.write("this is not json\n")
sys.stdout.flush()
return 2
_emit(
{
"type": "result",
"subtype": "success",
"session_id": session_id,
"is_error": False,
"num_turns": 1,
"total_cost_usd": 0.01,
"result": f"done: {prompt}",
}
)
return 0
if __name__ == "__main__":
sys.exit(main())
+178
View File
@@ -0,0 +1,178 @@
"""Pure-helper tests for the headless runner: stream parsing, path munging, argv
construction, and the session archive round-trip. No subprocess is launched here —
the process-level tests live in test_headless_run.py (phase 2)."""
from __future__ import annotations
import io
import tarfile
from handler.control import headless
# ------------------------------------------------------------------ parse_stream_line
def test_parse_valid_event_types():
for etype in ("system", "assistant", "user", "result", "hook"):
parsed_type, payload = headless.parse_stream_line(f'{{"type": "{etype}", "x": 1}}\n')
assert parsed_type == etype
assert payload == {"type": etype, "x": 1}
def test_parse_unknown_type_keeps_its_name():
parsed_type, payload = headless.parse_stream_line('{"type": "telemetry", "n": 2}')
assert parsed_type == "telemetry"
assert payload["n"] == 2
def test_parse_garbage_becomes_raw():
parsed_type, payload = headless.parse_stream_line("this is not json\n")
assert parsed_type == "raw"
assert payload == {"line": "this is not json\n"}
def test_parse_non_dict_json_becomes_raw():
parsed_type, payload = headless.parse_stream_line('["a", "b"]')
assert parsed_type == "raw"
def test_parse_missing_type_becomes_raw():
parsed_type, payload = headless.parse_stream_line('{"message": "no type field"}')
assert parsed_type == "raw"
assert payload == {"message": "no type field"}
def test_parse_blank_line_becomes_raw():
parsed_type, _ = headless.parse_stream_line(" \n")
assert parsed_type == "raw"
# -------------------------------------------------------------------- assistant_text
def test_assistant_text_joins_text_blocks():
payload = {
"type": "assistant",
"message": {
"content": [
{"type": "text", "text": "first"},
{"type": "tool_use", "name": "Bash", "input": {}},
{"type": "text", "text": "second"},
]
},
}
assert headless.assistant_text(payload) == "first\nsecond"
def test_assistant_text_none_for_pure_tool_use():
payload = {
"type": "assistant",
"message": {"content": [{"type": "tool_use", "name": "Bash", "input": {}}]},
}
assert headless.assistant_text(payload) is None
def test_assistant_text_string_content():
assert headless.assistant_text({"message": {"content": "plain"}}) == "plain"
def test_assistant_text_missing_message():
assert headless.assistant_text({"type": "assistant"}) is None
# ---------------------------------------------------------------- munged_project_dir
def test_munge_matches_recorded_real_examples():
# Recorded from a real ~/.claude/projects/ (see plan): '/' and '.' both map to '-'.
assert headless.munged_project_dir("/root/handler") == "-root-handler"
assert (
headless.munged_project_dir("/root/Talos/.claude/worktrees/mise-tooling")
== "-root-Talos--claude-worktrees-mise-tooling"
)
# ------------------------------------------------------------------- argv builders
def test_spawn_argv_shape(env):
argv = headless.build_spawn_argv("do the task", "/wd/.claude/settings.json", "sid-1")
assert argv[0] == "claude"
assert "-p" in argv and "--verbose" in argv
assert argv[argv.index("--output-format") + 1] == "stream-json"
assert argv[argv.index("--session-id") + 1] == "sid-1"
assert argv[argv.index("--settings") + 1] == "/wd/.claude/settings.json"
assert "--max-budget-usd" not in argv # default budget is 0 = off
assert argv[-2:] == ["--", "do the task"]
assert "--resume" not in argv
def test_resume_argv_shape(env):
argv = headless.build_resume_argv("sid-2", "the answer", "/wd/.claude/settings.json")
assert argv[argv.index("--resume") + 1] == "sid-2"
assert argv[-2:] == ["--", "the answer"]
assert "--session-id" not in argv
def test_spawn_argv_includes_budget_when_set(env, monkeypatch):
monkeypatch.setenv("RUN_BUDGET_USD", "2.5")
from handler import config
config.get_settings.cache_clear()
argv = headless.build_spawn_argv("t", "/s.json", "sid")
assert argv[argv.index("--max-budget-usd") + 1] == "2.5"
config.get_settings.cache_clear()
# ------------------------------------------------------------- archive round-trip
def _write_fake_session(home, working_dir: str, session_id: str) -> None:
base = home / ".claude" / "projects" / headless.munged_project_dir(working_dir)
base.mkdir(parents=True)
(base / f"{session_id}.jsonl").write_text('{"type": "user", "prompt": "hi"}\n')
sidecar = base / session_id / "tool-results"
sidecar.mkdir(parents=True)
(sidecar / "r1.txt").write_text("tool output")
def test_archive_and_materialize_round_trip(env, tmp_path, monkeypatch):
working_dir = "/projects/demo"
_write_fake_session(tmp_path, working_dir, "sid-rt")
data = headless.archive_session(working_dir, "sid-rt")
assert data is not None
# Materialize onto a *different* worker: a fresh HOME with no session state.
other_home = tmp_path / "other-worker"
other_home.mkdir()
monkeypatch.setenv("HOME", str(other_home))
headless.materialize_session(working_dir, data)
base = other_home / ".claude" / "projects" / headless.munged_project_dir(working_dir)
assert (base / "sid-rt.jsonl").read_text() == '{"type": "user", "prompt": "hi"}\n'
assert (base / "sid-rt" / "tool-results" / "r1.txt").read_text() == "tool output"
def test_archive_none_when_no_session(env):
assert headless.archive_session("/projects/none", "missing-sid") is None
def test_archive_refuses_oversize(env, tmp_path):
working_dir = "/projects/big"
_write_fake_session(tmp_path, working_dir, "sid-big")
assert headless.archive_session(working_dir, "sid-big", max_bytes=10) is None
def test_archive_contains_only_session_members(env, tmp_path):
working_dir = "/projects/demo2"
_write_fake_session(tmp_path, working_dir, "sid-a")
# A sibling session must not leak into sid-a's archive.
base = tmp_path / ".claude" / "projects" / headless.munged_project_dir(working_dir)
(base / "sid-other.jsonl").write_text("{}\n")
data = headless.archive_session(working_dir, "sid-a")
with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tar:
names = tar.getnames()
assert all(n == "sid-a.jsonl" or n.startswith("sid-a") for n in names)
+174
View File
@@ -0,0 +1,174 @@
"""Repository coverage for the headless-runner tables (workers / agent_runs /
agent_events / session_archives) and the new claim filters. The ``env`` fixture runs the
real ``alembic upgrade head``, so migration 0008 itself is under test here too."""
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from handler.db import repository as repo
def _agent(conn, name="a1"):
repo.create_project(conn, "p", "/projects/p")
return repo.create_agent(conn, "p", name, "/projects/p")
# ------------------------------------------------------------------------ agent_runs
def test_run_lifecycle(conn):
agent = _agent(conn)
run = repo.create_run(conn, agent["id"], "sid-1", "worker-a", "spawn")
assert run["status"] == "running"
assert run["kind"] == "spawn"
assert run["cancel_requested"] is False
assert repo.finish_run(conn, run["id"], "completed", exit_code=0, result={"ok": True})
done = repo.get_run(conn, run["id"])
assert done["status"] == "completed"
assert done["exit_code"] == 0
assert done["result"] == {"ok": True}
assert done["finished_at"] is not None
def test_finish_run_only_once(conn):
"""The supervisor's verdict and a racing reaper can't clobber each other."""
agent = _agent(conn)
run = repo.create_run(conn, agent["id"], "sid", "w", "spawn")
assert repo.finish_run(conn, run["id"], "crashed") is True
assert repo.finish_run(conn, run["id"], "completed", exit_code=0) is False
assert repo.get_run(conn, run["id"])["status"] == "crashed"
def test_cancel_request_roundtrip(conn):
agent = _agent(conn)
run = repo.create_run(conn, agent["id"], "sid", "w", "resume")
assert repo.get_cancel_requested(conn, run["id"]) is False
assert repo.request_run_cancel(conn, run["id"]) is True
assert repo.get_cancel_requested(conn, run["id"]) is True
# A finished run can't be re-flagged.
repo.finish_run(conn, run["id"], "canceled")
assert repo.request_run_cancel(conn, run["id"]) is False
def test_list_running_runs_scoped_by_worker(conn):
agent = _agent(conn)
r1 = repo.create_run(conn, agent["id"], "s1", "worker-a", "spawn")
r2 = repo.create_run(conn, agent["id"], "s2", "worker-b", "spawn")
repo.finish_run(conn, r1["id"], "completed")
running = repo.list_running_runs(conn)
assert [r["id"] for r in running] == [r2["id"]]
assert repo.list_running_runs(conn, worker_id="worker-a") == []
assert [r["id"] for r in repo.list_running_runs(conn, worker_id="worker-b")] == [r2["id"]]
def test_latest_run_and_agent_session(conn):
agent = _agent(conn)
repo.create_run(conn, agent["id"], "s1", "w", "spawn")
latest = repo.create_run(conn, agent["id"], "s1", "w", "resume")
assert repo.get_latest_run(conn, agent["id"])["id"] == latest["id"]
repo.set_agent_session(conn, agent["id"], "s1", "worker-a")
updated = repo.get_agent_by_id(conn, agent["id"])
assert updated["session_id"] == "s1"
assert updated["worker_id"] == "worker-a"
def test_agent_status_crashed_allowed(conn):
"""Migration 0008 widened ck_agents_status — 'crashed' must insert cleanly."""
agent = _agent(conn)
repo.set_agent_status(conn, agent["id"], "crashed")
assert repo.get_agent_by_id(conn, agent["id"])["status"] == "crashed"
# ---------------------------------------------------------------------- agent_events
def test_events_cursor_pagination(conn):
agent = _agent(conn)
run = repo.create_run(conn, agent["id"], "sid", "w", "spawn")
for seq in range(1, 4):
repo.insert_agent_event(
conn, agent["id"], run["id"], seq=seq, type="assistant",
payload={"n": seq}, session_id="sid",
)
first = repo.list_agent_events(conn, agent["id"], limit=2)
assert [e["payload"]["n"] for e in first] == [1, 2]
rest = repo.list_agent_events(conn, agent["id"], after_id=first[-1]["id"])
assert [e["payload"]["n"] for e in rest] == [3]
assert repo.list_agent_events(conn, agent["id"], after_id=rest[-1]["id"]) == []
# ------------------------------------------------------------------ session_archives
def test_session_archive_upsert_replaces(conn):
agent = _agent(conn)
repo.upsert_session_archive(conn, agent["id"], "s1", b"v1")
repo.upsert_session_archive(conn, agent["id"], "s2", b"v2-longer")
row = repo.get_session_archive(conn, agent["id"])
assert row["session_id"] == "s2"
assert bytes(row["archive"]) == b"v2-longer"
assert row["bytes"] == len(b"v2-longer")
def test_session_archive_missing(conn):
agent = _agent(conn)
assert repo.get_session_archive(conn, agent["id"]) is None
# ------------------------------------------------------------------------- workers
def test_worker_heartbeat_upsert_and_staleness(conn):
repo.upsert_worker_heartbeat(conn, "worker-a", hostname="h1", pid=42, max_runs=4)
repo.upsert_worker_heartbeat(conn, "worker-a", hostname="h1", pid=42, active_runs=2)
future = datetime.now(UTC) + timedelta(seconds=1)
stale = repo.list_stale_workers(conn, cutoff=future)
assert [w["id"] for w in stale] == ["worker-a"]
assert stale[0]["active_runs"] == 2
past = datetime.now(UTC) - timedelta(minutes=5)
assert repo.list_stale_workers(conn, cutoff=past) == []
# ------------------------------------------------------------------- claim filters
def test_claim_respects_target_worker(conn):
repo.create_project(conn, "p", "/projects/p")
pinned = repo.enqueue_command(conn, "login_submit", payload={"code": "x"},
target_worker="worker-b")
# worker-a can't see worker-b's pinned command...
assert repo.claim_next_command(conn, "worker-a") is None
# ...but worker-b claims it.
claimed = repo.claim_next_command(conn, "worker-b")
assert claimed["id"] == pinned["id"]
assert claimed["status"] == "running"
def test_claim_excludes_types_when_slots_full(conn):
repo.create_project(conn, "p", "/projects/p")
spawn_cmd = repo.enqueue_command(conn, "spawn", project_id="p", agent_name="a")
sync_cmd = repo.enqueue_command(conn, "sync", project_id="p")
# With run-starting types excluded, the older spawn is skipped for the sync.
claimed = repo.claim_next_command(
conn, "w", types_excluded=("spawn", "resume", "mise_init")
)
assert claimed["id"] == sync_cmd["id"]
# The spawn stays queued for a worker with a free slot.
assert repo.claim_next_command(conn, "w2")["id"] == spawn_cmd["id"]
def test_delete_agent_cascades_headless_rows(conn):
agent = _agent(conn)
run = repo.create_run(conn, agent["id"], "sid", "w", "spawn")
repo.insert_agent_event(conn, agent["id"], run["id"], seq=1, type="system", payload={})
repo.upsert_session_archive(conn, agent["id"], "sid", b"data")
assert repo.delete_agent(conn, "p", agent["name"]) is True
assert repo.get_latest_run(conn, agent["id"]) is None
assert repo.list_agent_events(conn, agent["id"]) == []
assert repo.get_session_archive(conn, agent["id"]) is None