129 lines
5.8 KiB
Text
129 lines
5.8 KiB
Text
// ═══════════════════════════════════════════════════════════════════════════
|
|
// recon-subfinder
|
|
// Subdomain enumeration + dnsx pre-resolve + diff.
|
|
// Writes all-resolved-latest.txt consumed by recon-httpx.
|
|
// Recommended schedule: every 2 days (0 23 */2 * *)
|
|
//
|
|
// CHANGE vs original: pointer reset removed — httpx runs as infinite queue
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
pipeline {
|
|
agent any
|
|
options {
|
|
timestamps()
|
|
disableConcurrentBuilds()
|
|
timeout(time: 2, unit: 'HOURS')
|
|
}
|
|
triggers { cron('0 23 */2 * *') }
|
|
parameters {
|
|
text(
|
|
name: 'DOMAINS',
|
|
defaultValue: 'example.com',
|
|
description: 'One root domain per line. Only domains you are authorised to test.'
|
|
)
|
|
string(name: 'RESOLVERS', defaultValue: '1.1.1.1,1.0.0.1,8.8.8.8,8.8.4.4,9.9.9.9,149.112.112.112', description: 'DNS resolvers')
|
|
string(name: 'DNSX_THREADS', defaultValue: '50', description: 'dnsx thread count')
|
|
string(name: 'DNSX_RATE_LIMIT',defaultValue: '500', description: 'dnsx max queries/sec')
|
|
}
|
|
environment {
|
|
STATE_BASE = '/var/jenkins_home/recon-state'
|
|
RUN_DIR = 'current-run'
|
|
}
|
|
stages {
|
|
stage('Prepare workspace') {
|
|
steps {
|
|
sh '''#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
rm -rf "$RUN_DIR" && mkdir -p "$RUN_DIR/results"
|
|
printf '%s\n' "${DOMAINS}" \
|
|
| sed 's/\r$//' | grep -vE '^[[:space:]]*($|#)' \
|
|
| sed 's/^[[:space:]]*//;s/[[:space:]]*$//' \
|
|
| sort -u > "$RUN_DIR/domains.clean.txt"
|
|
echo "[*] Domains:"; cat "$RUN_DIR/domains.clean.txt"
|
|
[ -s "$RUN_DIR/domains.clean.txt" ] || { echo "[!] No domains."; exit 1; }
|
|
'''
|
|
}
|
|
}
|
|
stage('Subfinder') {
|
|
steps {
|
|
sh '''#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
command -v subfinder >/dev/null 2>&1 || { echo "[!] subfinder not found"; exit 1; }
|
|
RESULT_BASE="$RUN_DIR/results"
|
|
SF_STATE="$STATE_BASE/subfinder/domains"
|
|
mkdir -p "$SF_STATE"
|
|
while IFS= read -r DOMAIN; do
|
|
[ -z "$DOMAIN" ] && continue
|
|
SAFE="$(echo "$DOMAIN" | sed 's/[^a-zA-Z0-9._-]/_/g')"
|
|
RDIR="$RESULT_BASE/$SAFE"; SDIR="$SF_STATE/$SAFE"
|
|
mkdir -p "$RDIR" "$SDIR"
|
|
subfinder -d "$DOMAIN" -silent -r "$RESOLVERS" | sort -u > "$RDIR/subdomains-raw.txt"
|
|
awk -v d="$DOMAIN" 'BEGIN{d=tolower(d)}{x=tolower($0);s="."d;if(x==d||substr(x,length(x)-length(s)+1)==s)print}' \
|
|
"$RDIR/subdomains-raw.txt" | sort -u > "$RDIR/subdomains-current.txt"
|
|
if [ ! -f "$SDIR/subdomains-last.txt" ]; then
|
|
cp "$RDIR/subdomains-current.txt" "$SDIR/subdomains-last.txt"
|
|
cp "$RDIR/subdomains-current.txt" "$RDIR/subdomains-new.txt"
|
|
else
|
|
comm -13 "$SDIR/subdomains-last.txt" "$RDIR/subdomains-current.txt" > "$RDIR/subdomains-new.txt"
|
|
cp "$RDIR/subdomains-current.txt" "$SDIR/subdomains-last.txt"
|
|
fi
|
|
echo "[*] $DOMAIN — current=$(wc -l < "$RDIR/subdomains-current.txt") new=$(wc -l < "$RDIR/subdomains-new.txt")"
|
|
done < "$RUN_DIR/domains.clean.txt"
|
|
find "$RESULT_BASE" -name "subdomains-current.txt" -print0 | xargs -0 cat | sort -u > "$RUN_DIR/all-subdomains.txt"
|
|
echo "[*] Total: $(wc -l < "$RUN_DIR/all-subdomains.txt")"
|
|
'''
|
|
}
|
|
}
|
|
stage('Scope check') {
|
|
steps {
|
|
sh '''#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
PATTERN="$(awk '{d=tolower($0);printf "(^%s$|\\.%s$)|",d,d}' "$RUN_DIR/domains.clean.txt" | sed "s/|$//")"
|
|
grep -ivE "$PATTERN" "$RUN_DIR/all-subdomains.txt" > "$RUN_DIR/out-of-scope.txt" || : > "$RUN_DIR/out-of-scope.txt"
|
|
[ -s "$RUN_DIR/out-of-scope.txt" ] && { echo "[!] Out-of-scope:"; cat "$RUN_DIR/out-of-scope.txt"; exit 1; }
|
|
echo "[*] Scope check passed."
|
|
'''
|
|
}
|
|
}
|
|
stage('dnsx') {
|
|
steps {
|
|
sh '''#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
if ! command -v dnsx >/dev/null 2>&1; then
|
|
echo "[!] dnsx not found — using raw list."; cp "$RUN_DIR/all-subdomains.txt" "$RUN_DIR/all-resolved.txt"; exit 0
|
|
fi
|
|
TOTAL=$(wc -l < "$RUN_DIR/all-subdomains.txt")
|
|
dnsx -l "$RUN_DIR/all-subdomains.txt" -silent -r "$RESOLVERS" -a -aaaa \
|
|
-threads "${DNSX_THREADS}" -rl "${DNSX_RATE_LIMIT}" -retry 3 \
|
|
| sort -u > "$RUN_DIR/all-resolved.txt" || true
|
|
AFTER=$(wc -l < "$RUN_DIR/all-resolved.txt")
|
|
[ "$AFTER" -lt 10 ] && [ "$TOTAL" -gt 100 ] && cp "$RUN_DIR/all-subdomains.txt" "$RUN_DIR/all-resolved.txt"
|
|
echo "[*] Resolved: $(wc -l < "$RUN_DIR/all-resolved.txt") Dropped: $(( TOTAL - $(wc -l < "$RUN_DIR/all-resolved.txt") ))"
|
|
'''
|
|
}
|
|
}
|
|
stage('Publish state') {
|
|
steps {
|
|
sh '''#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
SF_DIR="$STATE_BASE/subfinder"
|
|
mkdir -p "$SF_DIR/history"
|
|
cp "$RUN_DIR/all-subdomains.txt" "$SF_DIR/all-subdomains-latest.txt"
|
|
cp "$RUN_DIR/all-subdomains.txt" "$SF_DIR/history/subdomains-build-${BUILD_NUMBER}.txt"
|
|
cp "$RUN_DIR/all-resolved.txt" "$SF_DIR/all-resolved-latest.txt"
|
|
cp "$RUN_DIR/all-resolved.txt" "$SF_DIR/history/resolved-build-${BUILD_NUMBER}.txt"
|
|
echo "[*] Subdomains: $(wc -l < "$SF_DIR/all-subdomains-latest.txt")"
|
|
echo "[*] Resolved: $(wc -l < "$SF_DIR/all-resolved-latest.txt")"
|
|
echo "[*] httpx pointer NOT reset — continues from current position."
|
|
'''
|
|
}
|
|
}
|
|
stage('Archive') {
|
|
steps { archiveArtifacts artifacts: 'current-run/**/*.txt', fingerprint: true }
|
|
}
|
|
}
|
|
post {
|
|
always { echo "[*] recon-subfinder finished." }
|
|
failure { echo "[!] recon-subfinder FAILED." }
|
|
}
|
|
}
|