mirror of
https://github.com/0xWheatyz/handler.git
synced 2026-08-30 04:36:24 +00:00
fix(control,ui): close review findings - heartbeat starvation, resume race, stale UI writes
Fixes from the post-migration code review (3 major, 4 minor): - worker: heartbeat between every drained command so a long queue can't starve proof-of-life into a false reap; worker_stale_after default 60s -> 300s (one slow sync/login command must not look like a crash) - repository.create_run: enforces one running run per agent atomically (agent-row FOR UPDATE on Postgres; SQLite's single writer suffices) - two workers claiming resumes for the same agent can no longer both launch claude on one session; resume surfaces the loss loudly - headless._settle: upload the final session archive BEFORE marking the run finished - a resume claimed the instant a run leaves 'running' materializes from session_archives, and the old order let it race an incomplete archive into needless context re-injection (found as a test flake, real in production) - store.tsx: generation token drops in-flight loadRun writes after the user switches runs (run A's events/log/checkmark no longer land on run B), plus id-keyed dedup on event appends from overlapping polls - credsync: credential files written 0600 from the first byte - headless: seq counter locked (reader thread + supervisor both emit events); proc.stdout closed after reader join - login: submit pins to the latest CLAIMED login_start (a still-running one previously pinned to the wrong worker) Suite 296 green (new: create_run conflict coverage); reaper tests track the new staleness default.
This commit is contained in:
@@ -36,7 +36,7 @@ def _hb(worker_id, age_seconds=0.0):
|
||||
|
||||
def test_reaper_marks_stale_workers_runs_crashed(env):
|
||||
agent, run = _seed_run("w-dead")
|
||||
_hb("w-dead", age_seconds=120) # stale: default worker_stale_after is 60s
|
||||
_hb("w-dead", age_seconds=600) # stale: default worker_stale_after is 300s
|
||||
|
||||
assert worker.reap_dead_workers() == 1
|
||||
|
||||
@@ -64,7 +64,7 @@ def test_reaper_preserves_paused_agent_status(env):
|
||||
"""paused_for_input still accurately describes what the agent needs — only agents
|
||||
stuck in 'working' get flipped to crashed."""
|
||||
agent, run = _seed_run("w-dead2", agent_name="paused", agent_status="paused_for_input")
|
||||
_hb("w-dead2", age_seconds=120)
|
||||
_hb("w-dead2", age_seconds=600)
|
||||
|
||||
assert worker.reap_dead_workers() == 1
|
||||
with get_engine().begin() as conn:
|
||||
@@ -74,7 +74,7 @@ def test_reaper_preserves_paused_agent_status(env):
|
||||
|
||||
def test_reaper_idempotent_against_races(env):
|
||||
agent, run = _seed_run("w-dead3", agent_name="raced")
|
||||
_hb("w-dead3", age_seconds=120)
|
||||
_hb("w-dead3", age_seconds=600)
|
||||
assert worker.reap_dead_workers() == 1
|
||||
# A second reaper (or the same one next pass) finds nothing left to settle.
|
||||
assert worker.reap_dead_workers() == 0
|
||||
|
||||
@@ -55,17 +55,29 @@ def test_cancel_request_roundtrip(conn):
|
||||
def test_list_running_runs_scoped_by_worker(conn):
|
||||
agent = _agent(conn)
|
||||
r1 = repo.create_run(conn, agent["id"], "s1", "worker-a", "spawn")
|
||||
r2 = repo.create_run(conn, agent["id"], "s2", "worker-b", "spawn")
|
||||
repo.finish_run(conn, r1["id"], "completed")
|
||||
r2 = repo.create_run(conn, agent["id"], "s2", "worker-b", "spawn")
|
||||
running = repo.list_running_runs(conn)
|
||||
assert [r["id"] for r in running] == [r2["id"]]
|
||||
assert repo.list_running_runs(conn, worker_id="worker-a") == []
|
||||
assert [r["id"] for r in repo.list_running_runs(conn, worker_id="worker-b")] == [r2["id"]]
|
||||
|
||||
|
||||
def test_create_run_refuses_concurrent_run_for_agent(conn):
|
||||
"""One running run per agent, atomically — two workers racing a resume must not both
|
||||
launch a claude process on the same session."""
|
||||
import pytest
|
||||
|
||||
agent = _agent(conn)
|
||||
repo.create_run(conn, agent["id"], "s1", "worker-a", "spawn")
|
||||
with pytest.raises(repo.RunConflictError):
|
||||
repo.create_run(conn, agent["id"], "s1", "worker-b", "resume")
|
||||
|
||||
|
||||
def test_latest_run_and_agent_session(conn):
|
||||
agent = _agent(conn)
|
||||
repo.create_run(conn, agent["id"], "s1", "w", "spawn")
|
||||
first = repo.create_run(conn, agent["id"], "s1", "w", "spawn")
|
||||
repo.finish_run(conn, first["id"], "completed")
|
||||
latest = repo.create_run(conn, agent["id"], "s1", "w", "resume")
|
||||
assert repo.get_latest_run(conn, agent["id"])["id"] == latest["id"]
|
||||
|
||||
|
||||
@@ -22,22 +22,26 @@ def headless_env(env, monkeypatch):
|
||||
config.get_settings.cache_clear()
|
||||
|
||||
|
||||
def _seed(conn_count_running_for=None):
|
||||
def _seed(extra_agents=("b",)):
|
||||
with get_engine().begin() as conn:
|
||||
repo.create_project(conn, "p", "/tmp/p")
|
||||
agent = repo.create_agent(conn, "p", "a", "/tmp/p/a")
|
||||
for name in extra_agents:
|
||||
repo.create_agent(conn, "p", name, f"/tmp/p/{name}")
|
||||
return agent
|
||||
|
||||
|
||||
def _running_run(agent_id, worker_id):
|
||||
def _running_run(agent_name, worker_id):
|
||||
with get_engine().begin() as conn:
|
||||
return repo.create_run(conn, agent_id, f"sid-{worker_id}", worker_id, "spawn")
|
||||
agent = repo.get_agent_by_name(conn, "p", agent_name)
|
||||
return repo.create_run(conn, agent["id"], f"sid-{agent_name}", worker_id, "spawn")
|
||||
|
||||
|
||||
def test_full_worker_skips_run_commands_but_processes_others(headless_env, monkeypatch):
|
||||
agent = _seed()
|
||||
_running_run(agent["id"], "w-full")
|
||||
_running_run(agent["id"], "w-full") # 2 running == MAX_CONCURRENT_RUNS
|
||||
# 2 running (one per agent — one running run per agent) == MAX_CONCURRENT_RUNS
|
||||
_running_run("a", "w-full")
|
||||
_running_run("b", "w-full")
|
||||
|
||||
spawned = {}
|
||||
monkeypatch.setattr(
|
||||
@@ -66,9 +70,9 @@ def test_full_worker_skips_run_commands_but_processes_others(headless_env, monke
|
||||
|
||||
|
||||
def test_slot_frees_when_run_finishes(headless_env, monkeypatch):
|
||||
agent = _seed()
|
||||
run1 = _running_run(agent["id"], "w1")
|
||||
_running_run(agent["id"], "w1")
|
||||
_seed()
|
||||
run1 = _running_run("a", "w1")
|
||||
_running_run("b", "w1")
|
||||
assert worker._full_slot_exclusions("w1") == worker._RUN_COMMANDS
|
||||
|
||||
with get_engine().begin() as conn:
|
||||
|
||||
Reference in New Issue
Block a user