fix(control,ui): close review findings - heartbeat starvation, resume race, stale UI writes

Fixes from the post-migration code review (3 major, 4 minor):

- worker: heartbeat between every drained command so a long queue can't
  starve proof-of-life into a false reap; worker_stale_after default
  60s -> 300s (one slow sync/login command must not look like a crash)
- repository.create_run: enforces one running run per agent atomically
  (agent-row FOR UPDATE on Postgres; SQLite's single writer suffices) -
  two workers claiming resumes for the same agent can no longer both
  launch claude on one session; resume surfaces the loss loudly
- headless._settle: upload the final session archive BEFORE marking the
  run finished - a resume claimed the instant a run leaves 'running'
  materializes from session_archives, and the old order let it race an
  incomplete archive into needless context re-injection (found as a
  test flake, real in production)
- store.tsx: generation token drops in-flight loadRun writes after the
  user switches runs (run A's events/log/checkmark no longer land on
  run B), plus id-keyed dedup on event appends from overlapping polls
- credsync: credential files written 0600 from the first byte
- headless: seq counter locked (reader thread + supervisor both emit
  events); proc.stdout closed after reader join
- login: submit pins to the latest CLAIMED login_start (a still-running
  one previously pinned to the wrong worker)

Suite 296 green (new: create_run conflict coverage); reaper tests track
the new staleness default.
This commit is contained in:
2026-07-21 23:41:36 -04:00
parent 1517e4dca8
commit 478d178542
18 changed files with 145 additions and 57 deletions
+3 -3
View File
@@ -36,7 +36,7 @@ def _hb(worker_id, age_seconds=0.0):
def test_reaper_marks_stale_workers_runs_crashed(env):
agent, run = _seed_run("w-dead")
_hb("w-dead", age_seconds=120) # stale: default worker_stale_after is 60s
_hb("w-dead", age_seconds=600) # stale: default worker_stale_after is 300s
assert worker.reap_dead_workers() == 1
@@ -64,7 +64,7 @@ def test_reaper_preserves_paused_agent_status(env):
"""paused_for_input still accurately describes what the agent needs — only agents
stuck in 'working' get flipped to crashed."""
agent, run = _seed_run("w-dead2", agent_name="paused", agent_status="paused_for_input")
_hb("w-dead2", age_seconds=120)
_hb("w-dead2", age_seconds=600)
assert worker.reap_dead_workers() == 1
with get_engine().begin() as conn:
@@ -74,7 +74,7 @@ def test_reaper_preserves_paused_agent_status(env):
def test_reaper_idempotent_against_races(env):
agent, run = _seed_run("w-dead3", agent_name="raced")
_hb("w-dead3", age_seconds=120)
_hb("w-dead3", age_seconds=600)
assert worker.reap_dead_workers() == 1
# A second reaper (or the same one next pass) finds nothing left to settle.
assert worker.reap_dead_workers() == 0
+14 -2
View File
@@ -55,17 +55,29 @@ def test_cancel_request_roundtrip(conn):
def test_list_running_runs_scoped_by_worker(conn):
agent = _agent(conn)
r1 = repo.create_run(conn, agent["id"], "s1", "worker-a", "spawn")
r2 = repo.create_run(conn, agent["id"], "s2", "worker-b", "spawn")
repo.finish_run(conn, r1["id"], "completed")
r2 = repo.create_run(conn, agent["id"], "s2", "worker-b", "spawn")
running = repo.list_running_runs(conn)
assert [r["id"] for r in running] == [r2["id"]]
assert repo.list_running_runs(conn, worker_id="worker-a") == []
assert [r["id"] for r in repo.list_running_runs(conn, worker_id="worker-b")] == [r2["id"]]
def test_create_run_refuses_concurrent_run_for_agent(conn):
"""One running run per agent, atomically — two workers racing a resume must not both
launch a claude process on the same session."""
import pytest
agent = _agent(conn)
repo.create_run(conn, agent["id"], "s1", "worker-a", "spawn")
with pytest.raises(repo.RunConflictError):
repo.create_run(conn, agent["id"], "s1", "worker-b", "resume")
def test_latest_run_and_agent_session(conn):
agent = _agent(conn)
repo.create_run(conn, agent["id"], "s1", "w", "spawn")
first = repo.create_run(conn, agent["id"], "s1", "w", "spawn")
repo.finish_run(conn, first["id"], "completed")
latest = repo.create_run(conn, agent["id"], "s1", "w", "resume")
assert repo.get_latest_run(conn, agent["id"])["id"] == latest["id"]
+12 -8
View File
@@ -22,22 +22,26 @@ def headless_env(env, monkeypatch):
config.get_settings.cache_clear()
def _seed(conn_count_running_for=None):
def _seed(extra_agents=("b",)):
with get_engine().begin() as conn:
repo.create_project(conn, "p", "/tmp/p")
agent = repo.create_agent(conn, "p", "a", "/tmp/p/a")
for name in extra_agents:
repo.create_agent(conn, "p", name, f"/tmp/p/{name}")
return agent
def _running_run(agent_id, worker_id):
def _running_run(agent_name, worker_id):
with get_engine().begin() as conn:
return repo.create_run(conn, agent_id, f"sid-{worker_id}", worker_id, "spawn")
agent = repo.get_agent_by_name(conn, "p", agent_name)
return repo.create_run(conn, agent["id"], f"sid-{agent_name}", worker_id, "spawn")
def test_full_worker_skips_run_commands_but_processes_others(headless_env, monkeypatch):
agent = _seed()
_running_run(agent["id"], "w-full")
_running_run(agent["id"], "w-full") # 2 running == MAX_CONCURRENT_RUNS
# 2 running (one per agent — one running run per agent) == MAX_CONCURRENT_RUNS
_running_run("a", "w-full")
_running_run("b", "w-full")
spawned = {}
monkeypatch.setattr(
@@ -66,9 +70,9 @@ def test_full_worker_skips_run_commands_but_processes_others(headless_env, monke
def test_slot_frees_when_run_finishes(headless_env, monkeypatch):
agent = _seed()
run1 = _running_run(agent["id"], "w1")
_running_run(agent["id"], "w1")
_seed()
run1 = _running_run("a", "w1")
_running_run("b", "w1")
assert worker._full_slot_exclusions("w1") == worker._RUN_COMMANDS
with get_engine().begin() as conn: