feat: project scaffold, unified Pydantic schema, scope gating
This commit is contained in:
@@ -0,0 +1,9 @@
|
||||
"""recon-triage: defensive recon & triage pipeline for authorized security testing only.
|
||||
|
||||
This package performs reconnaissance and triage ONLY. It never runs, generates,
|
||||
downloads, or executes exploits. It enumerates assets, normalizes tool output into
|
||||
a single schema, grounds findings against Exploit-DB via searchsploit, and emits a
|
||||
prioritized report. A human decides what to do next.
|
||||
"""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
@@ -0,0 +1,202 @@
|
||||
"""Unified schema for recon-triage — the single source of truth for all output.
|
||||
|
||||
Every tool normalizer maps into this model tree, and the report/LLM stages consume
|
||||
it. The JSON Schema is exported to ``schemas/`` so the contract is inspectable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
|
||||
class StageStatus(str, Enum):
|
||||
"""Per-tool stage outcome. The pipeline fails soft: a tool that crashes, times
|
||||
out, or returns nothing is recorded here and the run continues."""
|
||||
|
||||
OK = "ok"
|
||||
EMPTY = "empty"
|
||||
FAILED = "failed"
|
||||
SKIPPED = "skipped"
|
||||
|
||||
|
||||
class Severity(str, Enum):
|
||||
INFO = "info"
|
||||
LOW = "low"
|
||||
MEDIUM = "medium"
|
||||
HIGH = "high"
|
||||
CRITICAL = "critical"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
# Ordering used by the deterministic fallback ranking (higher = more urgent).
|
||||
SEVERITY_RANK: dict[str, int] = {
|
||||
Severity.CRITICAL.value: 5,
|
||||
Severity.HIGH.value: 4,
|
||||
Severity.MEDIUM.value: 3,
|
||||
Severity.LOW.value: 2,
|
||||
Severity.INFO.value: 1,
|
||||
Severity.UNKNOWN.value: 0,
|
||||
}
|
||||
|
||||
|
||||
class _Model(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", use_enum_values=True)
|
||||
|
||||
|
||||
class ExploitDBMatch(_Model):
|
||||
"""An Exploit-DB candidate returned verbatim by ``searchsploit``.
|
||||
|
||||
These are ALWAYS unverified references. We never fabricate or infer EDB-IDs or
|
||||
paths; only what searchsploit actually returned is emitted.
|
||||
"""
|
||||
|
||||
edb_id: str = Field(description="Exploit-DB id, e.g. '50383'.")
|
||||
title: str
|
||||
path: str = Field(description="Local path within the Exploit-DB checkout.")
|
||||
url: str | None = Field(default=None, description="exploit-db.com URL.")
|
||||
query: str | None = Field(default=None, description="The searchsploit query that produced this hit.")
|
||||
source: str = Field(default="exploit-db", frozen=True)
|
||||
verified: bool = Field(default=False, description="Always false: candidate reference only.")
|
||||
|
||||
|
||||
class TlsInfo(_Model):
|
||||
subject_cn: str | None = None
|
||||
issuer: str | None = None
|
||||
not_after: str | None = None
|
||||
|
||||
|
||||
class HttpInfo(_Model):
|
||||
url: str | None = None
|
||||
status: int | None = None
|
||||
title: str | None = None
|
||||
webserver: str | None = None
|
||||
technologies: list[str] = Field(default_factory=list)
|
||||
tls: TlsInfo | None = None
|
||||
|
||||
|
||||
class Service(_Model):
|
||||
name: str | None = Field(default=None, description="Service name, e.g. 'http'.")
|
||||
product: str | None = Field(default=None, description="Product, e.g. 'Apache httpd'.")
|
||||
version: str | None = Field(default=None, description="Version string, e.g. '2.4.49'.")
|
||||
cpe: str | None = None
|
||||
http: HttpInfo | None = None
|
||||
exploitdb_candidates: list[ExploitDBMatch] = Field(default_factory=list)
|
||||
|
||||
def query_term(self) -> str | None:
|
||||
"""Build the searchsploit query: 'product version', falling back to product."""
|
||||
if not self.product:
|
||||
return None
|
||||
if self.version:
|
||||
return f"{self.product} {self.version}".strip()
|
||||
return self.product.strip()
|
||||
|
||||
|
||||
class Port(_Model):
|
||||
number: int = Field(ge=0, le=65535)
|
||||
protocol: str = Field(default="tcp")
|
||||
state: str = Field(default="open")
|
||||
service: Service | None = None
|
||||
|
||||
|
||||
class Host(_Model):
|
||||
hostname: str | None = None
|
||||
ips: list[str] = Field(default_factory=list)
|
||||
cnames: list[str] = Field(default_factory=list)
|
||||
ports: list[Port] = Field(default_factory=list)
|
||||
|
||||
def key(self) -> str:
|
||||
return self.hostname or (self.ips[0] if self.ips else "unknown")
|
||||
|
||||
|
||||
class NucleiFinding(_Model):
|
||||
template_id: str
|
||||
name: str | None = None
|
||||
severity: Severity = Severity.UNKNOWN
|
||||
matched_at: str | None = None
|
||||
host: str | None = None
|
||||
reference: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class StageResult(_Model):
|
||||
name: str
|
||||
status: StageStatus = StageStatus.SKIPPED
|
||||
detail: str | None = None
|
||||
started_at: str | None = None
|
||||
finished_at: str | None = None
|
||||
|
||||
|
||||
class ScopeUsed(_Model):
|
||||
in_scope_domains: list[str] = Field(default_factory=list)
|
||||
in_scope_cidrs: list[str] = Field(default_factory=list)
|
||||
out_of_scope: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ToolVersion(_Model):
|
||||
name: str
|
||||
version: str | None = None
|
||||
|
||||
|
||||
class TriageItem(_Model):
|
||||
"""One prioritized finding from the LLM stage. All references must point at ids
|
||||
present in the ReconReport — see triage.llm for the acceptance gate."""
|
||||
|
||||
summary: str
|
||||
rationale: str
|
||||
severity: Severity = Severity.UNKNOWN
|
||||
evidence_refs: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="ids/keys present in the ReconReport (host:port, edb_id, template_id).",
|
||||
)
|
||||
suggested_next_step: str | None = Field(
|
||||
default=None,
|
||||
description="Must reference a real edb_id / nuclei template / host:port from input.",
|
||||
)
|
||||
confidence: float = Field(default=0.0, ge=0.0, le=1.0)
|
||||
|
||||
|
||||
class TriageReport(_Model):
|
||||
prioritized_findings: list[TriageItem] = Field(default_factory=list)
|
||||
model: str | None = None
|
||||
generated_by: str = Field(default="deterministic", description="'llm' or 'deterministic'.")
|
||||
|
||||
|
||||
class ReconReport(_Model):
|
||||
"""Top-level run artifact. Schema-valid instances are written to report.json."""
|
||||
|
||||
run_id: str
|
||||
target: str | None = None
|
||||
started_at: str | None = None
|
||||
finished_at: str | None = None
|
||||
tool_versions: list[ToolVersion] = Field(default_factory=list)
|
||||
scope: ScopeUsed = Field(default_factory=ScopeUsed)
|
||||
stages: list[StageResult] = Field(default_factory=list)
|
||||
hosts: list[Host] = Field(default_factory=list)
|
||||
nuclei_findings: list[NucleiFinding] = Field(default_factory=list)
|
||||
triage: TriageReport | None = None
|
||||
notice: str = Field(
|
||||
default="Authorized use only. Recon-and-triage scope: this report enumerates and "
|
||||
"references findings; it does not run or generate exploits.",
|
||||
frozen=True,
|
||||
)
|
||||
|
||||
|
||||
def export_json_schema(out_dir: str | Path) -> Path:
|
||||
"""Write the ReconReport + TriageReport JSON Schemas to ``out_dir``."""
|
||||
out = Path(out_dir)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
recon_path = out / "recon_report.schema.json"
|
||||
triage_path = out / "triage_report.schema.json"
|
||||
recon_path.write_text(json.dumps(ReconReport.model_json_schema(), indent=2) + "\n")
|
||||
triage_path.write_text(json.dumps(TriageReport.model_json_schema(), indent=2) + "\n")
|
||||
return recon_path
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
import sys
|
||||
|
||||
target = sys.argv[1] if len(sys.argv) > 1 else "schemas"
|
||||
print(export_json_schema(target))
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Scope enforcement — mandatory gate. Anything not explicitly in scope is dropped.
|
||||
|
||||
A ``scope.yaml`` declares in-scope domains and CIDRs plus optional out-of-scope
|
||||
exclusions. The tool refuses to touch anything not in scope: out-of-scope input is
|
||||
dropped with a logged warning, never scanned.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from .schema import ScopeUsed
|
||||
|
||||
log = logging.getLogger("recon_triage.scope")
|
||||
|
||||
|
||||
def _norm_domain(d: str) -> str:
|
||||
return d.strip().lower().rstrip(".")
|
||||
|
||||
|
||||
def _is_ip(value: str) -> bool:
|
||||
try:
|
||||
ipaddress.ip_address(value)
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
@dataclass
|
||||
class Scope:
|
||||
"""Loaded scope rules with in/out gating.
|
||||
|
||||
Domain matching includes subdomains (``example.com`` matches ``api.example.com``).
|
||||
Out-of-scope rules take precedence over in-scope rules.
|
||||
"""
|
||||
|
||||
domains: list[str] = field(default_factory=list)
|
||||
cidrs: list[str] = field(default_factory=list)
|
||||
out_of_scope: list[str] = field(default_factory=list)
|
||||
|
||||
@classmethod
|
||||
def load(cls, path: str | Path) -> Scope:
|
||||
data = yaml.safe_load(Path(path).read_text()) or {}
|
||||
return cls.from_dict(data)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> Scope:
|
||||
domains = [_norm_domain(d) for d in (data.get("in_scope_domains") or []) if d]
|
||||
cidrs = [str(c).strip() for c in (data.get("in_scope_cidrs") or []) if c]
|
||||
out = [str(o).strip().lower().rstrip(".") for o in (data.get("out_of_scope") or []) if o]
|
||||
# Validate CIDRs early; drop invalid with a warning rather than crashing.
|
||||
valid_cidrs = []
|
||||
for c in cidrs:
|
||||
try:
|
||||
ipaddress.ip_network(c, strict=False)
|
||||
valid_cidrs.append(c)
|
||||
except ValueError:
|
||||
log.warning("Ignoring invalid CIDR in scope: %s", c)
|
||||
return cls(domains=domains, cidrs=valid_cidrs, out_of_scope=out)
|
||||
|
||||
# -- matching -----------------------------------------------------------
|
||||
|
||||
def _domain_in(self, host: str, rules: list[str]) -> bool:
|
||||
host = _norm_domain(host)
|
||||
for rule in rules:
|
||||
if host == rule or host.endswith("." + rule):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _ip_in_cidrs(self, ip: str) -> bool:
|
||||
try:
|
||||
addr = ipaddress.ip_address(ip)
|
||||
except ValueError:
|
||||
return False
|
||||
for c in self.cidrs:
|
||||
try:
|
||||
if addr in ipaddress.ip_network(c, strict=False):
|
||||
return True
|
||||
except ValueError:
|
||||
continue
|
||||
return False
|
||||
|
||||
def _ip_out(self, ip: str) -> bool:
|
||||
for rule in self.out_of_scope:
|
||||
if rule == ip:
|
||||
return True
|
||||
# out_of_scope may also carry CIDRs
|
||||
try:
|
||||
if ipaddress.ip_address(ip) in ipaddress.ip_network(rule, strict=False):
|
||||
return True
|
||||
except ValueError:
|
||||
continue
|
||||
return False
|
||||
|
||||
def is_in_scope(self, value: str) -> bool:
|
||||
"""True iff ``value`` (a hostname or IP) is in scope and not excluded."""
|
||||
value = value.strip()
|
||||
if not value:
|
||||
return False
|
||||
if _is_ip(value):
|
||||
if self._ip_out(value):
|
||||
return False
|
||||
return self._ip_in_cidrs(value)
|
||||
# hostname
|
||||
if self._domain_in(value, self.out_of_scope):
|
||||
return False
|
||||
return self._domain_in(value, self.domains)
|
||||
|
||||
def filter(self, values: list[str]) -> list[str]:
|
||||
"""Return only in-scope values; log a warning for each dropped one."""
|
||||
kept: list[str] = []
|
||||
for v in values:
|
||||
if self.is_in_scope(v):
|
||||
kept.append(v)
|
||||
else:
|
||||
log.warning("Dropping out-of-scope target: %s", v)
|
||||
return kept
|
||||
|
||||
def as_used(self) -> ScopeUsed:
|
||||
return ScopeUsed(
|
||||
in_scope_domains=list(self.domains),
|
||||
in_scope_cidrs=list(self.cidrs),
|
||||
out_of_scope=list(self.out_of_scope),
|
||||
)
|
||||
Reference in New Issue
Block a user