diff --git a/recon-triage/Dockerfile b/recon-triage/Dockerfile new file mode 100644 index 0000000..9e1ab6b --- /dev/null +++ b/recon-triage/Dockerfile @@ -0,0 +1,116 @@ +# syntax=docker/dockerfile:1 +# +# recon-triage — Docker-only build. Multi-stage: +# go-builder : compiles pinned ProjectDiscovery Go tools (static binaries) +# test : runs the fully-offline pytest suite (used by `make test`) +# runtime : minimal Alpine image, non-root, with nmap + exploitdb + the app +# +# Alpine is used throughout (musl). All Python deps (pydantic v2 included) ship +# musllinux wheels, so no Debian fallback is needed. + +# --------------------------------------------------------------------------- +# Stage 1: build Go recon tools. Pinned for reproducibility. +# --------------------------------------------------------------------------- +FROM golang:1.23-alpine AS go-builder + +# libpcap-dev is required to compile naabu; git for go module fetches. +RUN apk add --no-cache git build-base libpcap-dev + +ENV CGO_ENABLED=1 GOFLAGS=-buildvcs=false + +# Pinned tool versions (see README "Tool versions"). +ARG SUBFINDER_VERSION=v2.6.6 +ARG DNSX_VERSION=v1.2.1 +ARG NAABU_VERSION=v2.3.1 +ARG HTTPX_VERSION=v1.6.9 +ARG NUCLEI_VERSION=v3.3.5 + +RUN go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@${SUBFINDER_VERSION} \ + && go install -v github.com/projectdiscovery/dnsx/cmd/dnsx@${DNSX_VERSION} \ + && go install -v github.com/projectdiscovery/naabu/v2/cmd/naabu@${NAABU_VERSION} \ + && go install -v github.com/projectdiscovery/httpx/cmd/httpx@${HTTPX_VERSION} \ + && go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@${NUCLEI_VERSION} + +# --------------------------------------------------------------------------- +# Stage 1b: fetch Exploit-DB + searchsploit at BUILD time (release-independent; +# the Alpine `exploitdb` package is not in every release's stable repo). The DB +# ships inside the image, so searchsploit needs NO network at runtime. +# --------------------------------------------------------------------------- +FROM alpine:3.20 AS exploitdb-builder +ARG EXPLOITDB_REF=main +RUN apk add --no-cache git \ + && git clone --depth 1 --branch ${EXPLOITDB_REF} \ + https://gitlab.com/exploit-database/exploitdb.git /opt/exploitdb \ + && rm -rf /opt/exploitdb/.git + +# --------------------------------------------------------------------------- +# Stage 2: offline test image. No Go tools, no network at runtime — the suite +# is driven entirely by committed fixtures and an injected searchsploit stub. +# --------------------------------------------------------------------------- +FROM python:3.12-alpine AS test + +WORKDIR /app +RUN python -m venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" + +COPY pyproject.toml README.md ./ +COPY src ./src +RUN pip install --no-cache-dir -e ".[dev]" + +COPY tests ./tests +# Fully offline: -p no:cacheprovider keeps it hermetic. +RUN python -m pytest -q -p no:cacheprovider + +# --------------------------------------------------------------------------- +# Stage 3: runtime image. Non-root, connect-scan by default (no caps needed). +# --------------------------------------------------------------------------- +FROM python:3.12-alpine AS runtime + +# Runtime packages: +# nmap service/version detection +# libpcap naabu runtime dependency +# bash, coreutils searchsploit is a bash script using standard text utilities +# ca-certificates TLS roots for passive sources / optional LLM endpoint +RUN apk add --no-cache \ + nmap nmap-scripts \ + libpcap \ + ca-certificates \ + bash coreutils + +# Copy compiled recon tools from the builder. +COPY --from=go-builder /go/bin/subfinder /usr/local/bin/subfinder +COPY --from=go-builder /go/bin/dnsx /usr/local/bin/dnsx +COPY --from=go-builder /go/bin/naabu /usr/local/bin/naabu +COPY --from=go-builder /go/bin/httpx /usr/local/bin/httpx +COPY --from=go-builder /go/bin/nuclei /usr/local/bin/nuclei + +# Exploit-DB checkout (DB shipped in the image). searchsploit resolves its DB path +# relative to the real script location, so a symlink onto PATH is sufficient. +COPY --from=exploitdb-builder /opt/exploitdb /opt/exploitdb +RUN ln -sf /opt/exploitdb/searchsploit /usr/local/bin/searchsploit \ + && searchsploit --json apache >/dev/null 2>&1 || true + +# Install the app into an isolated venv. +RUN python -m venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" +WORKDIR /app +COPY pyproject.toml README.md ./ +COPY src ./src +RUN pip install --no-cache-dir . + +# Export the JSON Schema into the image for inspection. +COPY tests/fixtures ./tests/fixtures +RUN python -m recon_triage.schema /app/schemas + +# Non-root user. Connect scans need no elevated capabilities. +RUN addgroup -S app && adduser -S -G app -h /home/app app \ + && mkdir -p /data/out /home/app/nuclei-templates \ + && chown -R app:app /data /home/app /app/schemas +USER app + +# nuclei templates cache to a mountable volume (not baked into the image). +ENV NUCLEI_TEMPLATES_DIR=/home/app/nuclei-templates +ENV HOME=/home/app + +ENTRYPOINT ["recon-triage"] +CMD ["--help"] diff --git a/recon-triage/Makefile b/recon-triage/Makefile new file mode 100644 index 0000000..f719775 --- /dev/null +++ b/recon-triage/Makefile @@ -0,0 +1,61 @@ +# recon-triage — Docker-only workflow. No host-side tool installs required. +# +# AUTHORIZED USE ONLY. Recon & triage scope: this tool enumerates and references +# findings; it never runs or generates exploits. + +IMAGE ?= recon-triage:latest +TEST_IMAGE ?= recon-triage:test +SCOPE ?= scope.yaml +TARGET ?= example.com +OUT ?= $(CURDIR)/out +FIXTURES ?= tests/fixtures + +DOCKER ?= docker + +.PHONY: help build test replay scan lint schema clean + +help: + @echo "recon-triage — Docker-only targets:" + @echo " make build Build the runtime image" + @echo " make test Build the test stage and run pytest (fully offline)" + @echo " make replay Run the fixture demo -> ./out (no network)" + @echo " make scan TARGET=t SCOPE=s Live recon (requires ./\$$SCOPE)" + @echo " make lint Run ruff inside the test image" + @echo " make schema Export JSON Schema to ./schemas" + @echo " make clean Remove ./out" + +# Build the final runtime image from a clean checkout (no host tool installs). +build: + $(DOCKER) build --target runtime -t $(IMAGE) . + +# Build ONLY the test stage and run the suite. The pytest run happens during the +# image build (RUN pytest) and is fully offline — fixtures + injected stubs. +test: + $(DOCKER) build --target test -t $(TEST_IMAGE) . + +# Offline demo: full normalize -> ground -> report on committed fixtures. +replay: build + @mkdir -p $(OUT) + $(DOCKER) run --rm \ + -v $(CURDIR)/$(FIXTURES):/app/tests/fixtures:ro \ + -v $(OUT):/data/out \ + $(IMAGE) replay --fixtures /app/tests/fixtures --out /data/out --scope /app/tests/fixtures/scope.yaml + +# Live scan. Connect scans run unprivileged — no --cap-add needed. +# For faster SYN scans you MAY add: --cap-add=NET_RAW --cap-add=NET_ADMIN (optional). +scan: build + @mkdir -p $(OUT) + $(DOCKER) run --rm \ + -v $(CURDIR)/$(SCOPE):/data/scope.yaml:ro \ + -v $(OUT):/data/out \ + $(IMAGE) scan --scope /data/scope.yaml --target $(TARGET) --out /data/out + +lint: + $(DOCKER) build --target test -t $(TEST_IMAGE) . + $(DOCKER) run --rm $(TEST_IMAGE) ruff check src tests + +schema: + $(DOCKER) run --rm -v $(CURDIR)/schemas:/out $(IMAGE) schema --out /out + +clean: + rm -rf $(OUT) diff --git a/recon-triage/README.md b/recon-triage/README.md new file mode 100644 index 0000000..074afaf --- /dev/null +++ b/recon-triage/README.md @@ -0,0 +1,223 @@ +# recon-triage + +**Defensive bug-bounty reconnaissance & triage — authorized security testing only.** + +`recon-triage` runs a fixed, fail-soft pipeline of open-source recon tools against +**in-scope** targets, normalizes every tool's output into **one unified JSON schema**, +grounds findings against **Exploit-DB** via `searchsploit`, and produces a prioritized +triage report (`report.json` + `report.md`). An optional, fully-optional LLM stage +enriches the ranking — but the build and every run work with **no model available**. + +> ## ⚠️ Authorized use only — recon & triage scope +> This tool performs **reconnaissance and triage only**. It **never runs, generates, +> downloads, or executes exploits**. It enumerates assets, standardizes output, and +> *cites* candidate references (e.g. unverified Exploit-DB IDs). A human decides what +> to do next. You are responsible for ensuring every target is explicitly in scope and +> that you are authorized to test it. + +--- + +## Pipeline + +``` +scope.yaml ─▶ subfinder ─▶ dnsx ─▶ naabu ─▶ nmap -sV ─▶ httpx ─▶ [nuclei] ─▶ searchsploit ─▶ report.{json,md} + (passive) (resolve) (ports) (versions) (http) (opt-in) (Exploit-DB) (+ optional LLM triage) +``` + +Every stage **fails soft**: a tool that crashes, times out, or returns nothing is +recorded as `failed`/`empty` in the report and the run continues. + +| Stage | Tool | Structured output | +|---|---|---| +| 1 | `subfinder` | `-silent -oJ` (JSONL) | +| 2 | `dnsx` | `-json` (A/AAAA/CNAME) | +| 3 | `naabu` | `-json -scan-type connect` (unprivileged) | +| 4 | `nmap` | `-sV -sT -oX -` (product + **version**) | +| 5 | `httpx` | `-json -td` (status/title/tech/TLS) | +| 6 | `nuclei` | `-jsonl` — **off by default**, enable with `--enable-nuclei` | +| 7 | `searchsploit` | `--json` — **Exploit-DB grounding** | + +`nmap`'s `product` + `version` per service is the key signal feeding Exploit-DB +matching, so it is captured precisely. + +## Exploit-DB grounding (the anti-hallucination layer) + +For each detected service with a product, the tool builds a query (`"Apache httpd +2.4.49"`, falling back to just the product) and runs `searchsploit --json`. Each hit +is attached as an `ExploitDBMatch` with `verified=false`. **Nothing is fabricated or +inferred — only what `searchsploit` actually returned is emitted.** The Exploit-DB +ships inside the image, so this needs **no network at runtime**. + +--- + +## Quick start (Docker only — no host tool installs) + +```bash +# 1. Build the runtime image +make build + +# 2. Run the full offline demo on committed fixtures -> ./out (no network, no scan) +make replay +cat out/report.md + +# 3. Run the offline test suite (schema, normalizers, scope, Exploit-DB matcher) +make test + +# 4. Live scan an in-scope target (copy and edit the example scope first) +cp scope.example.yaml scope.yaml # edit to YOUR authorized scope +make scan TARGET=example.com SCOPE=scope.yaml +``` + +A reviewer with **only Docker installed** can run all of the above. Nothing is +installed on the host. + +### CLI + +``` +recon-triage scan --scope scope.yaml --target example.com --out /data/out \ + [--enable-nuclei] [--passive-only] [--rate-limit N] [--timeout S] +recon-triage replay --fixtures tests/fixtures --out /data/out [--scope scope.yaml] +recon-triage schema --out schemas +``` + +`replay` runs the full **normalize → ground → report** path on canned fixtures with +**zero network** — it's the offline demo and the basis of the test suite. + +--- + +## Scope enforcement (mandatory) + +`scope.yaml` declares in-scope domains + CIDRs and optional exclusions. Anything not +in scope is **dropped with a logged warning, never scanned**. Out-of-scope rules take +precedence; domain rules match subdomains. + +```yaml +in_scope_domains: + - example.com # also matches api.example.com +in_scope_cidrs: + - 93.184.216.0/24 +out_of_scope: + - internal-only.example.com + - 93.184.216.200/30 +``` + +See `scope.example.yaml`. Your real `scope.yaml` is git-ignored. + +--- + +## Running unprivileged & capabilities + +The container runs as a **non-root** `app` user and defaults to **TCP connect scans** +(`naabu -scan-type connect`, `nmap -sT`), so it works with **no added capabilities**: + +```bash +docker run --rm -v "$PWD/scope.yaml:/data/scope.yaml:ro" -v "$PWD/out:/data/out" \ + recon-triage:latest scan --scope /data/scope.yaml --target example.com --out /data/out +``` + +Optional: for faster SYN scans you may grant raw-socket capabilities. This is **never +required**: + +```bash +docker run --rm --cap-add=NET_RAW --cap-add=NET_ADMIN ... recon-triage:latest scan ... +``` + +--- + +## Optional LLM triage + +Configured purely by environment. **If `LLM_BASE_URL` is unset, the LLM stage is +skipped** and a deterministic severity-based ranking is used instead — the report is +always grounded and ranked. + +| Env var | Example | Meaning | +|---|---|---| +| `LLM_BASE_URL` | `http://ollama:11434/v1` | OpenAI-compatible endpoint (enables the stage) | +| `LLM_MODEL` | `qwen2.5:7b-instruct` | model name | +| `LLM_API_KEY` | `not-needed` | dummy ok for local | + +The model is instructed to use **only** the provided ReconReport, cite findings by id, +and is **forbidden from inventing CVEs, EDB-IDs, paths, or tools**. Output is validated +against the `TriageReport` schema (one retry on invalid JSON, then deterministic +fallback). **Acceptance gate:** every `suggested_next_step` / evidence reference must +point at an identifier present in the input report; anything else is stripped. + +Enable a local model via Compose: + +```bash +docker compose --profile llm up -d ollama +docker compose --profile llm run --rm app-llm \ + scan --scope /data/scope.yaml --target example.com --out /data/out +``` + +Without the profile, `docker compose run --rm app scan ...` runs the pipeline LLM-free. + +--- + +## Output + +Written to the mounted `/data/out` volume: + +- `report.json` — schema-valid `ReconReport` (+ `TriageReport` if triage ran). +- `report.md` — operator-facing summary: hosts → services (with versions) → Exploit-DB + candidates → nuclei findings → triage priorities. +- `schemas/` — exported JSON Schema of the contract. + +The unified schema (Pydantic v2) lives in `src/recon_triage/schema.py` and is the +single source of truth; `schemas/*.schema.json` is exported on build. + +--- + +## Tool versions (pinned) + +| Tool | Version | Source | +|---|---|---| +| subfinder | `v2.6.6` | `go install` (build arg `SUBFINDER_VERSION`) | +| dnsx | `v1.2.1` | `go install` (`DNSX_VERSION`) | +| naabu | `v2.3.1` | `go install` (`NAABU_VERSION`) | +| httpx | `v1.6.9` | `go install` (`HTTPX_VERSION`) | +| nuclei | `v3.3.5` | `go install` (`NUCLEI_VERSION`) | +| nmap | distro | `apk add nmap` | +| searchsploit + Exploit-DB | `main` | shallow `git clone` of gitlab.com/exploit-database/exploitdb at build time (pin via build arg `EXPLOITDB_REF`) | + +> The Exploit-DB checkout is fetched at **build** time and shipped inside the image, +> so `searchsploit` needs **no network at runtime**. (The Alpine `exploitdb` package +> is not in every release's stable repo, so the upstream git checkout is used for +> reproducibility.) + +Override any Go tool version at build time, e.g. `docker build --build-arg +NAABU_VERSION=v2.3.2 ...`. Base image is **Alpine** (musl) throughout; all Python +dependencies, including Pydantic v2, ship musllinux wheels so no Debian fallback is +needed. + +nuclei templates are **not baked** into the image — they cache to a mounted volume +(`nuclei-templates`) at runtime when `--enable-nuclei` is used. + +--- + +## Development & testing + +The test suite is **fully offline** — schema, every normalizer (fed from committed +fixtures in `tests/fixtures/`), the Exploit-DB matcher (via an injected `searchsploit` +stub), scope gating, and the end-to-end `replay` path. `make test` runs it inside the +`test` build stage with **zero external calls**. + +``` +make test # docker: build test stage + run pytest (offline) +make lint # ruff +``` + +## Project layout + +``` +src/recon_triage/ + schema.py Pydantic v2 models (single source of truth) + JSON-schema export + scope.py scope load + in/out-of-scope gating + orchestrator.py stage sequencing, host merging, failure isolation, replay path + tools/ one wrapper per tool (native structured output -> schema) + grounding/exploitdb.py searchsploit grounding (never fabricates) + triage/llm.py optional OpenAI-compatible triage + validation + fallback + triage/ranking.py deterministic severity ranking (always available) + report/markdown.py operator-facing report +tests/ offline suite + committed fixtures +``` diff --git a/recon-triage/docker-compose.yml b/recon-triage/docker-compose.yml new file mode 100644 index 0000000..63d1ca0 --- /dev/null +++ b/recon-triage/docker-compose.yml @@ -0,0 +1,52 @@ +# recon-triage compose. +# +# Default (LLM-free): docker compose run --rm app scan --scope /data/scope.yaml --target example.com --out /data/out +# With local LLM: docker compose --profile llm up -d ollama +# docker compose --profile llm run --rm app-llm scan ... +# +# The default `app` service leaves LLM_BASE_URL unset, so the triage stage uses +# deterministic ranking. The `app-llm` service (only created with the `llm` +# profile) starts Ollama and wires LLM_BASE_URL=http://ollama:11434/v1. + +x-app-base: &app-base + build: + context: . + target: runtime + image: recon-triage:latest + user: "app" # recon/triage only; connect scans need no caps + volumes: + - ./scope.yaml:/data/scope.yaml:ro + - ./out:/data/out + - nuclei-templates:/home/app/nuclei-templates + command: ["--help"] + +services: + app: + <<: *app-base + environment: + NUCLEI_TEMPLATES_DIR: /home/app/nuclei-templates + # LLM_BASE_URL intentionally unset -> deterministic triage. + + app-llm: + <<: *app-base + profiles: ["llm"] + depends_on: + - ollama + environment: + NUCLEI_TEMPLATES_DIR: /home/app/nuclei-templates + LLM_BASE_URL: http://ollama:11434/v1 + LLM_MODEL: ${LLM_MODEL:-qwen2.5:7b-instruct} + LLM_API_KEY: ${LLM_API_KEY:-not-needed} + + # Optional local LLM. Only created with the `llm` profile. + ollama: + image: ollama/ollama:latest + profiles: ["llm"] + ports: + - "11434:11434" + volumes: + - ollama:/root/.ollama + +volumes: + nuclei-templates: + ollama: