diff --git a/.env.example b/.env.example index fc11d91..ab22cfe 100644 --- a/.env.example +++ b/.env.example @@ -32,6 +32,12 @@ AUTH_TOKEN=change-me-to-a-long-random-string # Base directory under which per-project roots and agent worktrees live (isolation). PROJECTS_ROOT=/var/lib/handler/projects +# Web search provider for the agents' web_search tool (pi harness). Resolution order: +# SearXNG instance -> Brave Search API -> unset = DuckDuckGo HTML fallback (zero-config, +# rate-limited). web_fetch needs no provider. +# SEARXNG_URL=http://searxng.lan:8080 +# BRAVE_SEARCH_API_KEY= + # Binary overrides (defaults shown). Point at fakes in tests/CI. # CLAUDE_BIN=claude # PI_BIN=pi diff --git a/README.md b/README.md index 0840fb8..a10bc36 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,7 @@ Configuration is entirely environment-driven (see [`.env.example`](.env.example) | `SHARED_CONTEXT_WRITE_TOKEN` | Higher-trust token gating `PUT /shared/context/:key` | falls back to `AUTH_TOKEN` | | `ADMIN_TOKEN` | Gates the web control surface (enqueue commands, project/host CRUD, credential edits) | falls back to `AUTH_TOKEN` | | `WEBHOOK_URL` | Generic target for the `Notification` hook (ntfy, Slack, …) | unset → no-op | +| `SEARXNG_URL` / `BRAVE_SEARCH_API_KEY` | Provider for the agents' `web_search` tool (pi harness) | unset → DuckDuckGo fallback | | `HANDLER_SECRET_KEY` | Fernet key encrypting git-server tokens + SSH keys at rest (set the same value on API and control) | unset → secret store disabled | | `PROJECTS_ROOT` | Base dir for per-project roots / worktrees / auto-clones | `./projects` | | `CLAUDE_BIN` / `PI_BIN` / `MISE_BIN` / `TMUX_BIN` / `FORGE_BIN` / `GIT_BIN` | Binary overrides | `claude` / `pi` / `mise` / `tmux` / `forge` / `git` | diff --git a/docs/local-models.md b/docs/local-models.md index 258f38f..d9ed519 100644 --- a/docs/local-models.md +++ b/docs/local-models.md @@ -143,6 +143,10 @@ completion gate never sees generated files: normal answer/resume flow. Memory recall is injected at session start, and the memory tools (`memory_search/get/save/link`) are registered directly — pi has no MCP by design, so the bridge shells to `python -m handler.mcpserver --call ` instead. + The bridge also registers **`web_search` / `web_fetch`** (`python -m handler.webtool`): + pi ships no web tools and claude's live server-side at Anthropic, so these are + handler-owned — fetch is plain HTTP + HTML-to-text with no provider needed, and search + resolves `SEARXNG_URL` → `BRAVE_SEARCH_API_KEY` → a zero-config DuckDuckGo fallback. - **`APPEND_SYSTEM.md`** — the handler conventions (completion contract, ask_operator, memory usage) appended to pi's system prompt. @@ -158,8 +162,8 @@ whichever worker claims the resume, continued by launching pi again on the same ### What differs from the claude harness - **MCP connectors and plugins don't apply** — pi has no MCP client or plugin system. - The bundled memory server is bridged as native tools; other connectors are - claude-harness-only for now. + The bundled memory server and the web tools are bridged as native tools; other + connectors are claude-harness-only for now. - **Permission modes don't apply** — pi has no permission system. The hard gates (PreToolUse-equivalent blocking, Stop gate) are enforced by the bridge, which is the layer handler actually relies on for claude too. diff --git a/src/handler/config.py b/src/handler/config.py index 263df30..44f5baf 100644 --- a/src/handler/config.py +++ b/src/handler/config.py @@ -35,6 +35,14 @@ class Settings(BaseSettings): # Optional generic webhook target for the Notification hook. No-op when unset. webhook_url: str | None = None + # ---- Web tools (handler.webtool): the agents' web_search/web_fetch, exposed to + # pi-harness agents via the bridge extension. Search provider resolution order: + # SEARXNG_URL (self-hosted metasearch, format=json enabled) -> BRAVE_SEARCH_API_KEY + # (Brave Search API) -> neither = DuckDuckGo's HTML endpoint (zero-config fallback, + # rate-limited and markup-brittle; fine for occasional lookups). + searxng_url: str | None = None + brave_search_api_key: str | None = None + # Symmetric key (Fernet, urlsafe-base64) for the DB-backed secret store: git-server # tokens and SSH private keys are encrypted with it at rest. Generate one with # ``python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"``. diff --git a/src/handler/control/pi_bridge.ts b/src/handler/control/pi_bridge.ts index 03a8b66..79241ec 100644 --- a/src/handler/control/pi_bridge.ts +++ b/src/handler/control/pi_bridge.ts @@ -19,7 +19,8 @@ * It also registers the memory tools (memory_search/get/save/link) that claude agents * reach over MCP, by shelling to `python -m handler.mcpserver --call ` — pi has * no MCP by design, and a subprocess inheriting the agent env is the same trust model - * the MCP server used anyway. + * the MCP server used anyway — plus web_search/web_fetch (`python -m handler.webtool`), + * because pi ships no web tools and claude's live server-side at Anthropic. * * Identity and configuration arrive via the spawn environment, exactly like hooks: * HANDLER_AGENT_ID / HANDLER_PROJECT_ID / HANDLER_AGENT_NAME / DATABASE_URL, plus @@ -57,19 +58,27 @@ function runHook(event: string, payload: Record, timeoutMs = HO } } -function callMemory(tool: string, args: Record): string { - const res = spawnSync(PYTHON, ["-m", "handler.mcpserver", "--call", tool], { +function callPython(moduleArgs: string[], toolName: string, args: Record): string { + const res = spawnSync(PYTHON, moduleArgs, { input: JSON.stringify(args ?? {}), encoding: "utf8", timeout: 60_000, }); if (res.error || res.status !== 0) { const detail = res.error ? String(res.error) : (res.stderr || "").slice(-2000); - throw new Error(`${tool} failed: ${detail}`); + throw new Error(`${toolName} failed: ${detail}`); } return (res.stdout || "").trim() || "{}"; } +function callMemory(tool: string, args: Record): string { + return callPython(["-m", "handler.mcpserver", "--call", tool], tool, args); +} + +function callWeb(tool: string, args: Record): string { + return callPython(["-m", "handler.webtool", tool], tool, args); +} + function permissionDeny(out: any): string | null { const spec = out?.hookSpecificOutput; if (spec?.permissionDecision === "deny") { @@ -254,4 +263,42 @@ export default function (pi: ExtensionAPI) { }, }); } + + // ---- web tools (handler.webtool — pi ships none, claude's are Anthropic-server-side) + const webTools: Array<{ name: string; label: string; description: string; parameters: any }> = [ + { + name: "web_search", + label: "Web search", + description: + "Search the web. Returns titles, URLs, and snippets; follow up with web_fetch " + + "to read a promising result in full. Provider is operator-configured " + + "(SearXNG / Brave / DuckDuckGo fallback).", + parameters: Type.Object({ + query: Type.String({ description: "The search query" }), + limit: Type.Optional(Type.Integer({ minimum: 1, maximum: 10 })), + }), + }, + { + name: "web_fetch", + label: "Web fetch", + description: + "Fetch a URL and return its readable text (HTML is stripped; other content " + + "types come back as-is, truncated). Use for docs, changelogs, issues, articles.", + parameters: Type.Object({ + url: Type.String({ description: "The http(s) URL to fetch" }), + max_chars: Type.Optional( + Type.Integer({ minimum: 1000, maximum: 100000, description: "Text cap (default 20000)" }), + ), + }), + }, + ]; + for (const tool of webTools) { + pi.registerTool({ + ...tool, + async execute(_toolCallId: string, params: Record) { + const text = callWeb(tool.name, params ?? {}); + return { content: [{ type: "text", text }], details: {} }; + }, + }); + } } diff --git a/src/handler/webtool/__init__.py b/src/handler/webtool/__init__.py new file mode 100644 index 0000000..36616a9 --- /dev/null +++ b/src/handler/webtool/__init__.py @@ -0,0 +1,253 @@ +"""Self-hosted web tools for agents: ``web_search`` and ``web_fetch``. + +pi deliberately ships no web tools, and claude's built-in WebSearch/WebFetch are +Anthropic-server-side — they don't exist when the binary is pointed at a local +endpoint. This module is the handler-owned replacement: plain HTTP from the worker +container, no new capability an agent's bash + curl didn't already have, just a +structured tool the model can actually use well. + +Search is bring-your-own-provider, resolved in order: + +1. ``SEARXNG_URL`` — a SearXNG instance (self-hosted metasearch; set the base URL, + ``format=json`` must be enabled in its settings). +2. ``BRAVE_SEARCH_API_KEY`` — the Brave Search API. +3. Neither set — DuckDuckGo's HTML endpoint, parsed. Zero-config but rate-limited and + markup-brittle; fine for occasional agent lookups, configure a real provider for + heavy use. + +Fetch needs no provider: GET the URL, strip the HTML to readable text, cap the size. +Both are exposed to pi via the bridge extension (``python -m handler.webtool ``, +JSON args on stdin — the same seam shape as ``handler.mcpserver --call``). +""" + +from __future__ import annotations + +import html as html_lib +import json +import re +import urllib.parse + +import httpx + +_TIMEOUT = 20.0 +_UA = "Mozilla/5.0 (X11; Linux x86_64) handler-agent/1.0" +_MAX_FETCH_BYTES = 2 * 1024 * 1024 +_DEFAULT_FETCH_CHARS = 20_000 +_MAX_RESULTS = 10 + +TOOLS = ("web_search", "web_fetch") + + +class WebToolError(Exception): + """A tool-level failure the caller renders back to the model in-band.""" + + +# ---- html -> text ---------------------------------------------------------------------- + +_DROP_BLOCKS = re.compile( + r"<(script|style|noscript|svg|head)\b.*?", re.IGNORECASE | re.DOTALL +) +_BLOCK_TAGS = re.compile( + r"]*>", + re.IGNORECASE, +) +_TAGS = re.compile(r"<[^>]+>") +_TITLE = re.compile(r"]*>(.*?)", re.IGNORECASE | re.DOTALL) + + +def html_to_text(markup: str) -> tuple[str | None, str]: + """(title, readable text) from an HTML document — regex-grade readability, which is + the right weight here: agents want the words, not a perfect DOM.""" + title_match = _TITLE.search(markup) + title = html_lib.unescape(title_match.group(1)).strip() if title_match else None + body = _DROP_BLOCKS.sub(" ", markup) + body = _BLOCK_TAGS.sub("\n", body) + body = _TAGS.sub(" ", body) + body = html_lib.unescape(body) + lines = [" ".join(line.split()) for line in body.splitlines()] + text = "\n".join(line for line in lines if line) + return title, text + + +# ---- search providers -------------------------------------------------------------------- + + +def _searxng_search(base_url: str, query: str, limit: int) -> list[dict]: + resp = httpx.get( + f"{base_url.rstrip('/')}/search", + params={"q": query, "format": "json"}, + headers={"User-Agent": _UA}, + timeout=_TIMEOUT, + follow_redirects=True, + ) + resp.raise_for_status() + results = resp.json().get("results") or [] + return [ + { + "title": r.get("title") or "", + "url": r.get("url") or "", + "snippet": r.get("content") or "", + } + for r in results[:limit] + ] + + +def _brave_search(api_key: str, query: str, limit: int) -> list[dict]: + resp = httpx.get( + "https://api.search.brave.com/res/v1/web/search", + params={"q": query, "count": limit}, + headers={"X-Subscription-Token": api_key, "Accept": "application/json"}, + timeout=_TIMEOUT, + ) + resp.raise_for_status() + results = ((resp.json().get("web") or {}).get("results")) or [] + return [ + { + "title": r.get("title") or "", + "url": r.get("url") or "", + "snippet": r.get("description") or "", + } + for r in results[:limit] + ] + + +_DDG_RESULT = re.compile( + r']+class="result__a"[^>]+href="(?P[^"]+)"[^>]*>(?P.*?)</a>', + re.IGNORECASE | re.DOTALL, +) +_DDG_SNIPPET = re.compile( + r'<a[^>]+class="result__snippet"[^>]*>(?P<snippet>.*?)</a>', re.IGNORECASE | re.DOTALL +) + + +def _ddg_url(href: str) -> str: + """DDG's result hrefs are redirect links carrying the real URL in ``uddg``.""" + parsed = urllib.parse.urlparse(href, scheme="https") + if "duckduckgo.com" in (parsed.netloc or "") and parsed.path.startswith("/l/"): + target = urllib.parse.parse_qs(parsed.query).get("uddg") + if target: + return target[0] + return urllib.parse.urlunparse(parsed) + + +def _ddg_search(query: str, limit: int) -> list[dict]: + resp = httpx.get( + "https://html.duckduckgo.com/html/", + params={"q": query}, + headers={"User-Agent": _UA}, + timeout=_TIMEOUT, + follow_redirects=True, + ) + resp.raise_for_status() + markup = resp.text + snippets = [ + " ".join(html_lib.unescape(_TAGS.sub(" ", m.group("snippet"))).split()) + for m in _DDG_SNIPPET.finditer(markup) + ] + results = [] + for i, m in enumerate(_DDG_RESULT.finditer(markup)): + if len(results) >= limit: + break + title = " ".join(html_lib.unescape(_TAGS.sub(" ", m.group("title"))).split()) + results.append( + { + "title": title, + "url": _ddg_url(html_lib.unescape(m.group("href"))), + "snippet": snippets[i] if i < len(snippets) else "", + } + ) + return results + + +# ---- tools --------------------------------------------------------------------------------- + + +def web_search(args: dict) -> dict: + from ..config import get_settings + + query = (args.get("query") or "").strip() + if not query: + raise WebToolError("query is required") + limit = min(int(args.get("limit") or 5), _MAX_RESULTS) + s = get_settings() + try: + if s.searxng_url: + provider = "searxng" + results = _searxng_search(s.searxng_url, query, limit) + elif s.brave_search_api_key: + provider = "brave" + results = _brave_search(s.brave_search_api_key, query, limit) + else: + provider = "duckduckgo" + results = _ddg_search(query, limit) + except httpx.HTTPError as exc: + raise WebToolError(f"search failed ({exc.__class__.__name__}): {exc}") from exc + return {"provider": provider, "query": query, "results": results} + + +def web_fetch(args: dict) -> dict: + url = (args.get("url") or "").strip() + if not url.startswith(("http://", "https://")): + raise WebToolError("url must be an http(s) URL") + max_chars = min(int(args.get("max_chars") or _DEFAULT_FETCH_CHARS), 100_000) + try: + with httpx.stream( + "GET", + url, + headers={"User-Agent": _UA}, + timeout=_TIMEOUT, + follow_redirects=True, + ) as resp: + status = resp.status_code + final_url = str(resp.url) + content_type = resp.headers.get("content-type", "") + raw = b"" + for chunk in resp.iter_bytes(): + raw += chunk + if len(raw) >= _MAX_FETCH_BYTES: + break + except httpx.HTTPError as exc: + raise WebToolError(f"fetch failed ({exc.__class__.__name__}): {exc}") from exc + body = raw.decode("utf-8", "replace") + if "html" in content_type.lower() or "<html" in body[:2000].lower(): + title, text = html_to_text(body) + else: + title, text = None, body + truncated = len(text) > max_chars + return { + "url": final_url, + "status": status, + "content_type": content_type, + "title": title, + "truncated": truncated, + "text": text[:max_chars], + } + + +def call_tool(name: str, args: dict) -> dict: + handlers = {"web_search": web_search, "web_fetch": web_fetch} + if name not in handlers: + raise WebToolError(f"unknown tool '{name}'") + return handlers[name](args) + + +def main(argv: list[str]) -> int: + """CLI seam: ``python -m handler.webtool <tool>`` with JSON args on stdin.""" + import sys + + if len(argv) != 1 or argv[0] not in TOOLS: + print(f"usage: python -m handler.webtool {{{'|'.join(TOOLS)}}}", file=sys.stderr) + return 2 + raw = sys.stdin.read() + try: + args = json.loads(raw) if raw.strip() else {} + except ValueError: + print("invalid JSON arguments on stdin", file=sys.stderr) + return 2 + try: + payload = call_tool(argv[0], args if isinstance(args, dict) else {}) + except WebToolError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + print(json.dumps(payload, ensure_ascii=False)) + return 0 diff --git a/src/handler/webtool/__main__.py b/src/handler/webtool/__main__.py new file mode 100644 index 0000000..9413147 --- /dev/null +++ b/src/handler/webtool/__main__.py @@ -0,0 +1,10 @@ +"""``python -m handler.webtool <web_search|web_fetch>`` — JSON args in, JSON result out.""" + +from __future__ import annotations + +import sys + +from . import main + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tests/test_pi_harness.py b/tests/test_pi_harness.py index bd370e8..99a96e8 100644 --- a/tests/test_pi_harness.py +++ b/tests/test_pi_harness.py @@ -107,8 +107,10 @@ def test_write_config_renders_provider_and_bridge(pi_env, tmp_path): assert "handler.mcpserver" in text assert "ask_operator" in text # Tool parity: the bridge activates pi's full built-in set (grep/find/ls are off - # by default) alongside its own tools. + # by default) alongside its own tools, and registers the handler web tools. assert "setActiveTools" in text + assert "handler.webtool" in text + assert "web_search" in text and "web_fetch" in text assert (base / "APPEND_SYSTEM.md").read_text().strip() diff --git a/tests/test_webtool.py b/tests/test_webtool.py new file mode 100644 index 0000000..11ab0b2 --- /dev/null +++ b/tests/test_webtool.py @@ -0,0 +1,147 @@ +"""The agents' web tools (``handler.webtool``): provider selection, result shaping, +HTML-to-text, the fetch cap, and the stdin/stdout CLI seam the pi bridge shells to. +All HTTP is mocked with respx — no live network.""" + +from __future__ import annotations + +import io +import json + +import httpx +import pytest +import respx + +from handler import webtool + +DDG_HTML = """ +<html><body> +<a rel="nofollow" class="result__a" + href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com%2Fdocs&rut=abc">Example + <b>Docs</b></a> +<a class="result__snippet" href="#">The official <b>docs</b> for Example.</a> +<a rel="nofollow" class="result__a" href="https://other.example.org/page">Other page</a> +<a class="result__snippet" href="#">Another snippet.</a> +</body></html> +""" + + +@pytest.fixture +def clean_settings(env): + """The env fixture already resets the settings cache; just be explicit that the + provider env vars are unset unless a test sets them.""" + return env + + +@respx.mock +def test_search_falls_back_to_duckduckgo(clean_settings): + respx.get("https://html.duckduckgo.com/html/").mock( + return_value=httpx.Response(200, text=DDG_HTML) + ) + out = webtool.web_search({"query": "example docs"}) + assert out["provider"] == "duckduckgo" + assert out["results"][0]["title"] == "Example Docs" + # The redirect wrapper is unwrapped to the real target URL. + assert out["results"][0]["url"] == "https://example.com/docs" + assert "official docs" in out["results"][0]["snippet"] + assert out["results"][1]["url"] == "https://other.example.org/page" + + +@respx.mock +def test_search_prefers_searxng_when_configured(clean_settings, monkeypatch): + from handler import config + + monkeypatch.setenv("SEARXNG_URL", "http://searx.lan:8080") + config.get_settings.cache_clear() + respx.get("http://searx.lan:8080/search").mock( + return_value=httpx.Response( + 200, + json={ + "results": [ + {"title": "T", "url": "https://t.example", "content": "snippet"}, + ] + }, + ) + ) + out = webtool.web_search({"query": "q", "limit": 3}) + assert out["provider"] == "searxng" + assert out["results"] == [ + {"title": "T", "url": "https://t.example", "snippet": "snippet"} + ] + + +@respx.mock +def test_search_uses_brave_with_key(clean_settings, monkeypatch): + from handler import config + + monkeypatch.setenv("BRAVE_SEARCH_API_KEY", "brave-key") + config.get_settings.cache_clear() + route = respx.get("https://api.search.brave.com/res/v1/web/search").mock( + return_value=httpx.Response( + 200, + json={"web": {"results": [{"title": "B", "url": "https://b", "description": "d"}]}}, + ) + ) + out = webtool.web_search({"query": "q"}) + assert out["provider"] == "brave" + assert route.calls[0].request.headers["X-Subscription-Token"] == "brave-key" + assert out["results"][0]["snippet"] == "d" + + +def test_search_requires_query(clean_settings): + with pytest.raises(webtool.WebToolError): + webtool.web_search({"query": " "}) + + +@respx.mock +def test_search_provider_error_is_tool_error(clean_settings): + respx.get("https://html.duckduckgo.com/html/").mock(side_effect=httpx.ConnectError) + with pytest.raises(webtool.WebToolError, match="search failed"): + webtool.web_search({"query": "q"}) + + +@respx.mock +def test_fetch_strips_html_and_caps_text(clean_settings): + page = ( + "<html><head><title>My & Page" + "

Header

Hello world.

" + ) + respx.get("https://example.com/a").mock( + return_value=httpx.Response(200, text=page, headers={"content-type": "text/html"}) + ) + out = webtool.web_fetch({"url": "https://example.com/a"}) + assert out["title"] == "My & Page" + assert "Header" in out["text"] and "Hello world" in out["text"] + assert "evil" not in out["text"] + + out = webtool.web_fetch({"url": "https://example.com/a", "max_chars": 1000}) + assert len(out["text"]) <= 1000 + + +@respx.mock +def test_fetch_passes_plain_text_through(clean_settings): + respx.get("https://example.com/raw").mock( + return_value=httpx.Response(200, text="plain body", headers={"content-type": "text/plain"}) + ) + out = webtool.web_fetch({"url": "https://example.com/raw"}) + assert out["text"] == "plain body" + assert out["title"] is None + + +def test_fetch_rejects_non_http(clean_settings): + with pytest.raises(webtool.WebToolError, match="http"): + webtool.web_fetch({"url": "file:///etc/passwd"}) + + +@respx.mock +def test_cli_seam_round_trip(clean_settings, monkeypatch, capsys): + respx.get("https://html.duckduckgo.com/html/").mock( + return_value=httpx.Response(200, text=DDG_HTML) + ) + monkeypatch.setattr("sys.stdin", io.StringIO(json.dumps({"query": "example"}))) + assert webtool.main(["web_search"]) == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["provider"] == "duckduckgo" + + monkeypatch.setattr("sys.stdin", io.StringIO("{}")) + assert webtool.main(["web_search"]) == 1 # missing query -> tool error, exit 1 + assert webtool.main(["nope"]) == 2