feat: tool normalizers, Exploit-DB grounding, orchestrator, report, triage, CLI
This commit is contained in:
@@ -0,0 +1,2 @@
|
||||
"""Tool wrappers. Each module runs one recon tool with its native structured-output
|
||||
flag and normalizes the result into the unified schema."""
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Common subprocess runner: explicit timeouts, output capture, fail-soft records.
|
||||
|
||||
Every tool wrapper goes through ``run_tool``. A tool that crashes, times out, or
|
||||
returns nothing never aborts the run — the caller records a StageResult and moves on.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import shutil
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
|
||||
log = logging.getLogger("recon_triage.tools")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolRun:
|
||||
"""Result of one external tool invocation."""
|
||||
|
||||
cmd: list[str]
|
||||
returncode: int | None
|
||||
stdout: str
|
||||
stderr: str
|
||||
timed_out: bool = False
|
||||
not_found: bool = False
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
return not self.timed_out and not self.not_found and self.returncode == 0
|
||||
|
||||
@property
|
||||
def empty(self) -> bool:
|
||||
return not self.stdout.strip()
|
||||
|
||||
|
||||
def tool_available(binary: str) -> bool:
|
||||
return shutil.which(binary) is not None
|
||||
|
||||
|
||||
def run_tool(
|
||||
cmd: list[str],
|
||||
*,
|
||||
timeout: float = 120.0,
|
||||
input_text: str | None = None,
|
||||
) -> ToolRun:
|
||||
"""Run ``cmd`` with a hard timeout, capturing stdout/stderr. Never raises for
|
||||
process-level failures — they are reported on the returned ToolRun."""
|
||||
binary = cmd[0]
|
||||
if not tool_available(binary):
|
||||
log.warning("Tool not found on PATH: %s", binary)
|
||||
return ToolRun(cmd=cmd, returncode=None, stdout="", stderr="not found", not_found=True)
|
||||
log.info("Running: %s", " ".join(cmd))
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
cmd,
|
||||
input=input_text,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
check=False,
|
||||
)
|
||||
return ToolRun(
|
||||
cmd=cmd,
|
||||
returncode=proc.returncode,
|
||||
stdout=proc.stdout or "",
|
||||
stderr=proc.stderr or "",
|
||||
)
|
||||
except subprocess.TimeoutExpired as e:
|
||||
log.warning("Tool timed out after %ss: %s", timeout, binary)
|
||||
out = e.stdout.decode() if isinstance(e.stdout, bytes) else (e.stdout or "")
|
||||
err = e.stderr.decode() if isinstance(e.stderr, bytes) else (e.stderr or "")
|
||||
return ToolRun(cmd=cmd, returncode=None, stdout=out, stderr=err, timed_out=True)
|
||||
except OSError as e: # pragma: no cover - defensive
|
||||
log.warning("Tool execution error for %s: %s", binary, e)
|
||||
return ToolRun(cmd=cmd, returncode=None, stdout="", stderr=str(e), not_found=True)
|
||||
|
||||
|
||||
def parse_jsonl(text: str) -> list[dict]:
|
||||
"""Parse JSONL (one JSON object per line), skipping blank/garbage lines."""
|
||||
import json
|
||||
|
||||
out: list[dict] = []
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
log.debug("Skipping non-JSON line: %s", line[:120])
|
||||
continue
|
||||
if isinstance(obj, dict):
|
||||
out.append(obj)
|
||||
return out
|
||||
@@ -0,0 +1,45 @@
|
||||
"""dnsx — resolve/validate hosts, record A/AAAA/CNAME. Native output: ``-json``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from .base import parse_jsonl, run_tool
|
||||
|
||||
BINARY = "dnsx"
|
||||
|
||||
|
||||
@dataclass
|
||||
class DnsRecord:
|
||||
host: str
|
||||
a: list[str] = field(default_factory=list)
|
||||
aaaa: list[str] = field(default_factory=list)
|
||||
cname: list[str] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def ips(self) -> list[str]:
|
||||
return list(self.a) + list(self.aaaa)
|
||||
|
||||
|
||||
def normalize(raw: str) -> list[DnsRecord]:
|
||||
"""Map dnsx -json lines to DnsRecord objects."""
|
||||
out: list[DnsRecord] = []
|
||||
for rec in parse_jsonl(raw):
|
||||
host = (rec.get("host") or "").strip().lower()
|
||||
if not host:
|
||||
continue
|
||||
out.append(
|
||||
DnsRecord(
|
||||
host=host,
|
||||
a=[str(x) for x in (rec.get("a") or [])],
|
||||
aaaa=[str(x) for x in (rec.get("aaaa") or [])],
|
||||
cname=[str(x).lower() for x in (rec.get("cname") or [])],
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def run(hosts: list[str], *, timeout: float = 120.0) -> tuple[list[DnsRecord], object]:
|
||||
cmd = [BINARY, "-json", "-a", "-aaaa", "-cname", "-resp"]
|
||||
res = run_tool(cmd, timeout=timeout, input_text="\n".join(hosts) + "\n")
|
||||
return normalize(res.stdout), res
|
||||
@@ -0,0 +1,90 @@
|
||||
"""httpx — HTTP probing: status, title, tech, webserver, TLS. Output: ``-json -td``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from ..schema import HttpInfo, TlsInfo
|
||||
from .base import parse_jsonl, run_tool
|
||||
|
||||
BINARY = "httpx"
|
||||
|
||||
|
||||
@dataclass
|
||||
class HttpProbe:
|
||||
host: str | None
|
||||
ip: str | None
|
||||
port: int | None
|
||||
info: HttpInfo
|
||||
|
||||
|
||||
def _port_of(rec: dict) -> int | None:
|
||||
port = rec.get("port")
|
||||
try:
|
||||
return int(port)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _tls_of(rec: dict) -> TlsInfo | None:
|
||||
tls = rec.get("tls")
|
||||
if not isinstance(tls, dict):
|
||||
return None
|
||||
issuer = tls.get("issuer_cn") or tls.get("issuer_common_name")
|
||||
if not issuer:
|
||||
org = tls.get("issuer_org")
|
||||
if isinstance(org, list) and org:
|
||||
issuer = org[0]
|
||||
elif isinstance(org, str):
|
||||
issuer = org
|
||||
return TlsInfo(
|
||||
subject_cn=tls.get("subject_cn") or tls.get("subject_common_name"),
|
||||
issuer=issuer,
|
||||
not_after=tls.get("not_after"),
|
||||
)
|
||||
|
||||
|
||||
def normalize(raw: str) -> list[HttpProbe]:
|
||||
"""Map httpx -json lines into HttpProbe records keyed by host/ip/port."""
|
||||
out: list[HttpProbe] = []
|
||||
for rec in parse_jsonl(raw):
|
||||
tech = rec.get("tech") or rec.get("technologies") or []
|
||||
if isinstance(tech, str):
|
||||
tech = [tech]
|
||||
info = HttpInfo(
|
||||
url=rec.get("url"),
|
||||
status=rec.get("status_code") or rec.get("status-code"),
|
||||
title=rec.get("title"),
|
||||
webserver=rec.get("webserver"),
|
||||
technologies=[str(t) for t in tech],
|
||||
tls=_tls_of(rec),
|
||||
)
|
||||
out.append(
|
||||
HttpProbe(
|
||||
host=(rec.get("input") or rec.get("host") or "").lower() or None,
|
||||
ip=rec.get("host") if _looks_like_ip(rec.get("host")) else rec.get("a", [None])[0] if isinstance(rec.get("a"), list) else None,
|
||||
port=_port_of(rec),
|
||||
info=info,
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _looks_like_ip(v) -> bool:
|
||||
import ipaddress
|
||||
|
||||
if not isinstance(v, str):
|
||||
return False
|
||||
try:
|
||||
ipaddress.ip_address(v)
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def run(targets: list[str], *, timeout: float = 180.0, rate_limit: int | None = None) -> tuple[list[HttpProbe], object]:
|
||||
cmd = [BINARY, "-json", "-td", "-silent", "-title", "-tech-detect", "-web-server", "-status-code"]
|
||||
if rate_limit:
|
||||
cmd += ["-rate-limit", str(rate_limit)]
|
||||
res = run_tool(cmd, timeout=timeout, input_text="\n".join(targets) + "\n")
|
||||
return normalize(res.stdout), res
|
||||
@@ -0,0 +1,68 @@
|
||||
"""naabu — port discovery. Native output: ``-json``.
|
||||
|
||||
Defaults to ``-scan-type connect`` (TCP connect) so it runs unprivileged with no
|
||||
added capabilities.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from .base import parse_jsonl, run_tool
|
||||
|
||||
BINARY = "naabu"
|
||||
|
||||
|
||||
@dataclass
|
||||
class PortHit:
|
||||
host: str | None
|
||||
ip: str | None
|
||||
port: int
|
||||
protocol: str = "tcp"
|
||||
|
||||
|
||||
@dataclass
|
||||
class NaabuResult:
|
||||
hits: list[PortHit] = field(default_factory=list)
|
||||
|
||||
|
||||
def normalize(raw: str) -> list[PortHit]:
|
||||
"""Map naabu -json lines to PortHit records."""
|
||||
out: list[PortHit] = []
|
||||
for rec in parse_jsonl(raw):
|
||||
port = rec.get("port")
|
||||
# naabu may emit port as int or nested object depending on version.
|
||||
if isinstance(port, dict):
|
||||
port_num = port.get("Port") or port.get("port")
|
||||
else:
|
||||
port_num = port
|
||||
try:
|
||||
port_num = int(port_num)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
out.append(
|
||||
PortHit(
|
||||
host=(rec.get("host") or None),
|
||||
ip=(rec.get("ip") or None),
|
||||
port=port_num,
|
||||
protocol=str(rec.get("protocol") or "tcp").lower(),
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def run(
|
||||
targets: list[str],
|
||||
*,
|
||||
timeout: float = 300.0,
|
||||
rate_limit: int | None = None,
|
||||
ports: str | None = None,
|
||||
) -> tuple[list[PortHit], object]:
|
||||
# connect scan keeps the container unprivileged (no NET_RAW needed).
|
||||
cmd = [BINARY, "-json", "-scan-type", "connect", "-silent"]
|
||||
if ports:
|
||||
cmd += ["-p", ports]
|
||||
if rate_limit:
|
||||
cmd += ["-rate", str(rate_limit)]
|
||||
res = run_tool(cmd, timeout=timeout, input_text="\n".join(targets) + "\n")
|
||||
return normalize(res.stdout), res
|
||||
@@ -0,0 +1,113 @@
|
||||
"""nmap — service + version detection. Native output: ``-sV -oX -`` (XML).
|
||||
|
||||
``product`` + ``version`` per service is the key signal feeding Exploit-DB matching,
|
||||
so it is captured precisely. Uses ``-sT`` connect scan to stay unprivileged.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import xmltodict
|
||||
|
||||
from ..schema import Port, Service
|
||||
from .base import run_tool
|
||||
|
||||
log = logging.getLogger("recon_triage.tools.nmap")
|
||||
|
||||
BINARY = "nmap"
|
||||
|
||||
|
||||
@dataclass
|
||||
class NmapHost:
|
||||
ip: str | None = None
|
||||
hostnames: list[str] = field(default_factory=list)
|
||||
ports: list[Port] = field(default_factory=list)
|
||||
|
||||
|
||||
def _as_list(value) -> list:
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
return [value]
|
||||
|
||||
|
||||
def _parse_service(svc: dict | None) -> Service | None:
|
||||
if not svc:
|
||||
return None
|
||||
cpe = svc.get("cpe")
|
||||
if isinstance(cpe, list):
|
||||
cpe = cpe[0] if cpe else None
|
||||
if isinstance(cpe, dict): # xmltodict text node
|
||||
cpe = cpe.get("#text")
|
||||
return Service(
|
||||
name=svc.get("@name"),
|
||||
product=svc.get("@product"),
|
||||
version=svc.get("@version"),
|
||||
cpe=cpe,
|
||||
)
|
||||
|
||||
|
||||
def normalize(raw_xml: str) -> list[NmapHost]:
|
||||
"""Parse nmap XML into NmapHost records with precise product/version."""
|
||||
if not raw_xml.strip():
|
||||
return []
|
||||
try:
|
||||
doc = xmltodict.parse(raw_xml)
|
||||
except Exception as e: # pragma: no cover - malformed XML
|
||||
log.warning("Failed to parse nmap XML: %s", e)
|
||||
return []
|
||||
|
||||
run = (doc or {}).get("nmaprun") or {}
|
||||
hosts_out: list[NmapHost] = []
|
||||
for host in _as_list(run.get("host")):
|
||||
ip = None
|
||||
for addr in _as_list(host.get("address")):
|
||||
if addr.get("@addrtype") in ("ipv4", "ipv6"):
|
||||
ip = addr.get("@addr")
|
||||
break
|
||||
hostnames = []
|
||||
hn = host.get("hostnames")
|
||||
if isinstance(hn, dict):
|
||||
for h in _as_list(hn.get("hostname")):
|
||||
if h.get("@name"):
|
||||
hostnames.append(h["@name"].lower())
|
||||
|
||||
ports_out: list[Port] = []
|
||||
ports_node = host.get("ports")
|
||||
if isinstance(ports_node, dict):
|
||||
for p in _as_list(ports_node.get("port")):
|
||||
state = (p.get("state") or {}).get("@state", "open")
|
||||
try:
|
||||
number = int(p.get("@portid"))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
ports_out.append(
|
||||
Port(
|
||||
number=number,
|
||||
protocol=p.get("@protocol", "tcp"),
|
||||
state=state,
|
||||
service=_parse_service(p.get("service")),
|
||||
)
|
||||
)
|
||||
hosts_out.append(NmapHost(ip=ip, hostnames=hostnames, ports=ports_out))
|
||||
return hosts_out
|
||||
|
||||
|
||||
def run(
|
||||
targets: list[str],
|
||||
ports: list[int] | None = None,
|
||||
*,
|
||||
timeout: float = 600.0,
|
||||
unprivileged: bool = True,
|
||||
) -> tuple[list[NmapHost], object]:
|
||||
cmd = [BINARY, "-sV", "-oX", "-", "-Pn"]
|
||||
if unprivileged:
|
||||
cmd.append("-sT") # TCP connect scan; no raw sockets required
|
||||
if ports:
|
||||
cmd += ["-p", ",".join(str(p) for p in sorted(set(ports)))]
|
||||
cmd += list(targets)
|
||||
res = run_tool(cmd, timeout=timeout)
|
||||
return normalize(res.stdout), res
|
||||
@@ -0,0 +1,59 @@
|
||||
"""nuclei — template-based exposure/misconfig/CVE checks. Output: ``-jsonl``.
|
||||
|
||||
Gated behind ``--enable-nuclei`` (off by default in V0): it needs network for
|
||||
template fetch and meaningful runtime. Templates cache to a runtime volume.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from ..schema import NucleiFinding, Severity
|
||||
from .base import parse_jsonl, run_tool
|
||||
|
||||
BINARY = "nuclei"
|
||||
|
||||
_SEVERITIES = {s.value for s in Severity}
|
||||
|
||||
|
||||
def _severity(value) -> Severity:
|
||||
v = str(value or "").lower().strip()
|
||||
return Severity(v) if v in _SEVERITIES else Severity.UNKNOWN
|
||||
|
||||
|
||||
def normalize(raw: str) -> list[NucleiFinding]:
|
||||
"""Map nuclei -jsonl lines into NucleiFinding records."""
|
||||
out: list[NucleiFinding] = []
|
||||
for rec in parse_jsonl(raw):
|
||||
info = rec.get("info") or {}
|
||||
ref = info.get("reference") or []
|
||||
if isinstance(ref, str):
|
||||
ref = [ref]
|
||||
template_id = rec.get("template-id") or rec.get("templateID") or rec.get("template_id")
|
||||
if not template_id:
|
||||
continue
|
||||
out.append(
|
||||
NucleiFinding(
|
||||
template_id=str(template_id),
|
||||
name=info.get("name"),
|
||||
severity=_severity(info.get("severity")),
|
||||
matched_at=rec.get("matched-at") or rec.get("matched_at"),
|
||||
host=rec.get("host"),
|
||||
reference=[str(r) for r in ref if r],
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def run(
|
||||
targets: list[str],
|
||||
*,
|
||||
timeout: float = 600.0,
|
||||
rate_limit: int | None = None,
|
||||
templates_dir: str | None = None,
|
||||
) -> tuple[list[NucleiFinding], object]:
|
||||
cmd = [BINARY, "-jsonl", "-silent", "-disable-update-check"]
|
||||
if templates_dir:
|
||||
cmd += ["-update-template-dir", templates_dir, "-templates", templates_dir]
|
||||
if rate_limit:
|
||||
cmd += ["-rate-limit", str(rate_limit)]
|
||||
res = run_tool(cmd, timeout=timeout, input_text="\n".join(targets) + "\n")
|
||||
return normalize(res.stdout), res
|
||||
@@ -0,0 +1,59 @@
|
||||
"""searchsploit — Exploit-DB grounding. Native output: ``--json``.
|
||||
|
||||
This is the anti-hallucination layer: we ONLY emit EDB-IDs/paths/titles that
|
||||
searchsploit actually returned. Nothing is fabricated or inferred. The DB ships in
|
||||
the image, so this requires no network at runtime.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from ..schema import ExploitDBMatch
|
||||
from .base import run_tool
|
||||
|
||||
log = logging.getLogger("recon_triage.tools.searchsploit")
|
||||
|
||||
BINARY = "searchsploit"
|
||||
|
||||
|
||||
def normalize(raw_json: str, query: str | None = None) -> list[ExploitDBMatch]:
|
||||
"""Parse ``searchsploit --json`` output into ExploitDBMatch candidates.
|
||||
|
||||
Only ``RESULTS_EXPLOIT`` rows are emitted, each marked verified=False.
|
||||
"""
|
||||
if not raw_json.strip():
|
||||
return []
|
||||
try:
|
||||
doc = json.loads(raw_json)
|
||||
except json.JSONDecodeError:
|
||||
log.warning("searchsploit returned non-JSON output; ignoring")
|
||||
return []
|
||||
|
||||
rows = doc.get("RESULTS_EXPLOIT") or []
|
||||
out: list[ExploitDBMatch] = []
|
||||
for row in rows:
|
||||
edb_id = row.get("EDB-ID") or row.get("Codes")
|
||||
title = row.get("Title")
|
||||
path = row.get("Path")
|
||||
if not edb_id or not title:
|
||||
continue
|
||||
out.append(
|
||||
ExploitDBMatch(
|
||||
edb_id=str(edb_id),
|
||||
title=str(title),
|
||||
path=str(path or ""),
|
||||
url=row.get("URL"),
|
||||
query=query,
|
||||
verified=False,
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def run(term: str, *, timeout: float = 60.0) -> tuple[list[ExploitDBMatch], object]:
|
||||
# --json gives structured results; the DB is local so no network is used.
|
||||
cmd = [BINARY, "--json", term]
|
||||
res = run_tool(cmd, timeout=timeout)
|
||||
return normalize(res.stdout, query=term), res
|
||||
@@ -0,0 +1,27 @@
|
||||
"""subfinder — passive subdomain enumeration. Native output: ``-silent -oJ`` (JSONL)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .base import parse_jsonl, run_tool
|
||||
|
||||
BINARY = "subfinder"
|
||||
|
||||
|
||||
def normalize(raw: str) -> list[str]:
|
||||
"""Extract unique hostnames from subfinder JSONL output."""
|
||||
hosts: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for rec in parse_jsonl(raw):
|
||||
host = (rec.get("host") or rec.get("input") or "").strip().lower()
|
||||
if host and host not in seen:
|
||||
seen.add(host)
|
||||
hosts.append(host)
|
||||
return hosts
|
||||
|
||||
|
||||
def run(domain: str, *, timeout: float = 120.0, rate_limit: int | None = None) -> tuple[list[str], object]:
|
||||
cmd = [BINARY, "-silent", "-oJ", "-d", domain]
|
||||
if rate_limit:
|
||||
cmd += ["-rate-limit", str(rate_limit)]
|
||||
res = run_tool(cmd, timeout=timeout)
|
||||
return normalize(res.stdout), res
|
||||
Reference in New Issue
Block a user