Files
handler/tests/test_worker_concurrency.py
T
0xWheatyz 650f376934 feat(control): flag-gated headless runner with cross-worker resume (phase 2)
Wires the phase-1 headless machinery behind runner=headless (default
stays tmux; legacy agents, session_id null, keep the tmux paths):

- spawn: branches tmux vs headless.launch; extracts _agent_env (shared
  with resume - a headless resume is a new process needing identity/
  credential env); headless spawns require a task (no idle-REPL mode),
  enforced at spawn and as a 400 in the API
- resume: headless path materializes the session archive from the DB
  onto whichever worker claimed the command, then claude -p --resume;
  falls back to a fresh session with DB-re-injected context (visible
  worker event) when no transcript survives anywhere; refuses while a
  run is live. Undeliverable resumes now raise -> command FAILED,
  fixing silent input loss on both runners
- kill: headless path flags cancel_requested; the owning supervisor
  SIGTERMs its own child (cross-worker safe)
- worker: stable per-container ids, DB-driven run slots (full workers
  skip claiming spawn/resume/mise_init, leaving them for less-loaded
  workers), credsync refresh in the main loop
- settings_gen: permissions block (defaultMode + allowlist) for
  headless runs - -p auto-denies anything that would prompt; hooks
  remain the hard gate
- credsync + migration 0009 (runtime_secrets): login publishes the
  Fernet-encrypted claude credential bundle; every worker materializes
  it (merge-safe for local trust state); login_submit pinned to the
  login_start worker via commands.target_worker

Suite 270 -> 290 green, including the cross-worker resume linchpin
(clean-HOME materialize + --resume against the fake binary).
2026-07-21 22:56:02 -04:00

82 lines
2.9 KiB
Python

"""Slot-aware command claiming: a worker at max_concurrent_runs must leave run-starting
commands queued (for a less-loaded worker) while still processing everything else. Slot
accounting is DB-driven — this worker's ``running`` agent_runs rows — so it needs no
in-memory registry and is exercised here without real subprocesses."""
from __future__ import annotations
import pytest
from handler.control import spawn, worker
from handler.db import repository as repo
from handler.db.engine import get_engine
@pytest.fixture
def headless_env(env, monkeypatch):
from handler import config
monkeypatch.setenv("RUNNER", "headless")
monkeypatch.setenv("MAX_CONCURRENT_RUNS", "2")
config.get_settings.cache_clear()
yield env
config.get_settings.cache_clear()
def _seed(conn_count_running_for=None):
with get_engine().begin() as conn:
repo.create_project(conn, "p", "/tmp/p")
agent = repo.create_agent(conn, "p", "a", "/tmp/p/a")
return agent
def _running_run(agent_id, worker_id):
with get_engine().begin() as conn:
return repo.create_run(conn, agent_id, f"sid-{worker_id}", worker_id, "spawn")
def test_full_worker_skips_run_commands_but_processes_others(headless_env, monkeypatch):
agent = _seed()
_running_run(agent["id"], "w-full")
_running_run(agent["id"], "w-full") # 2 running == MAX_CONCURRENT_RUNS
spawned = {}
monkeypatch.setattr(
spawn, "spawn",
lambda project_id, name, **kw: spawned.update(name=name, **kw)
or {"id": 1, "name": name, "working_dir": "/tmp/p/x", "forge_note": None},
)
with get_engine().begin() as conn:
spawn_cmd = repo.enqueue_command(conn, "spawn", project_id="p", agent_name="x")
kill_cmd = repo.enqueue_command(conn, "kill", project_id="p", agent_name="a")
monkeypatch.setattr(spawn, "kill", lambda p, n: None)
# The full worker processes the kill but leaves the spawn queued.
assert worker.drain("w-full") == 1
with get_engine().begin() as conn:
assert repo.get_command(conn, kill_cmd["id"])["status"] == "done"
assert repo.get_command(conn, spawn_cmd["id"])["status"] == "queued"
assert spawned == {}
# A worker with free slots picks the spawn up.
assert worker.drain("w-free") == 1
with get_engine().begin() as conn:
assert repo.get_command(conn, spawn_cmd["id"])["status"] == "done"
assert spawned["name"] == "x"
assert spawned["worker_id"] == "w-free"
def test_slot_frees_when_run_finishes(headless_env, monkeypatch):
agent = _seed()
run1 = _running_run(agent["id"], "w1")
_running_run(agent["id"], "w1")
assert worker._full_slot_exclusions("w1") == worker._RUN_COMMANDS
with get_engine().begin() as conn:
repo.finish_run(conn, run1["id"], "completed", exit_code=0)
assert worker._full_slot_exclusions("w1") == ()
def test_tmux_runner_never_excludes(env):
assert worker._full_slot_exclusions("w") == ()