531 lines
No EOL
22 KiB
Groovy
531 lines
No EOL
22 KiB
Groovy
// ═══════════════════════════════════════════════════════════════════════════
|
||
// recon-httpx
|
||
// Probes ONE chunk of resolved hosts per run (pointer-file rotation).
|
||
// Reads from all-resolved-latest.txt written by recon-subfinder.
|
||
// No per-chunk DNS resolution needed — subfinder already did it.
|
||
//
|
||
// Recommended schedule: every 2–4 hours (e.g. H */3 * * *)
|
||
// ═══════════════════════════════════════════════════════════════════════════
|
||
|
||
pipeline {
|
||
agent any
|
||
|
||
options {
|
||
timestamps()
|
||
disableConcurrentBuilds()
|
||
timeout(time: 1, unit: 'HOURS')
|
||
}
|
||
|
||
triggers {
|
||
cron('''20,50 0-22 * * *''')
|
||
}
|
||
|
||
parameters {
|
||
string(
|
||
name: 'CHUNK_SIZE',
|
||
defaultValue: '500',
|
||
description: 'Max hosts to probe per run'
|
||
)
|
||
string(
|
||
name: 'RESOLVERS',
|
||
defaultValue: '1.1.1.1,1.0.0.1,8.8.8.8,8.8.4.4,9.9.9.9,149.112.112.112',
|
||
description: 'Comma-separated DNS resolvers'
|
||
)
|
||
string(
|
||
name: 'HTTPX_THREADS',
|
||
defaultValue: '25',
|
||
description: 'httpx thread count'
|
||
)
|
||
string(
|
||
name: 'HTTPX_TIMEOUT',
|
||
defaultValue: '10',
|
||
description: 'httpx timeout in seconds'
|
||
)
|
||
string(
|
||
name: 'GREP_PATTERNS',
|
||
defaultValue: 'admin|administrator|adminpanel|admin-panel|admin_panel|admincp|cpanel|webadmin|superadmin|siteadmin|login|signin|sign-in|sign_in|logon|log-in|log_in|sso|oauth|openid|saml|ldap|kerberos|portal|dashboard|control-panel|controlpanel|manage|management|manager|console|panel|wp-admin|wp-login|phpmyadmin|adminer|dbadmin|phpinfo|server-status|server-info|auth|authenticate|authentication|authorize|authorization|access-control|rbac|acl|staging|stage|preprod|pre-prod|pre_prod|uat|sit|qa|qe|integration|testing-env|dev|develop|development|sandbox|local|localhost|internal|intranet|corp|corpnet|test|testing|testenv|test-env|demo|poc|pilot|beta|alpha|canary|nightly|feature|hotfix|release|deploy|build|preview|review-app|draft|api|api-v1|api-v2|api-v3|rest|restapi|graphql|grpc|rpc|soap|xmlrpc|endpoint|gateway|proxy|backend|service|microservice|webhook|callback|listener|api_key|apikey|access_token|bearer|client_secret|client_id|consumer_key|consumer_secret|swagger|openapi|api-docs|apidoc|redoc|api-explorer|wsdl|schema|raml|jenkins|bamboo|teamcity|travis|circle-ci|circleci|drone|github-actions|actions|pipeline|ci|cicd|ci-cd|gitlab|github|bitbucket|gitea|gogs|sourcegraph|gerrit|phabricator|arcanist|jira|confluence|trello|notion|asana|monday|basecamp|linear|clickup|sonarqube|nexus|artifactory|harbor|registry|docker|kubernetes|k8s|helm|kustomize|rancher|portainer|grafana|kibana|prometheus|elasticsearch|logstash|datadog|splunk|graylog|loki|jaeger|zipkin|sentry|s3|ec2|lambda|cloudfront|cloudwatch|sqs|sns|rds|eks|ecs|iam|route53|cognito|amplify|blob|keyvault|devops|aks|appservice|servicebus|cosmosdb|functions|frontdoor|bigquery|firebase|cloudsql|gke|pubsub|appengine|cloudrun|firestore|terraform|ansible|puppet|chef|vagrant|packer|pulumi|cdk|vpn|bastion|jump|nat|firewall|loadbalancer|alb|nlb|elb|cdn|reverse-proxy|haproxy|nginx|traefik|actuator|health|healthcheck|status|ping|metrics|monitoring|uptime|alive|readiness|liveness|mysql|postgres|redis|mongo|cassandra|mssql|oracle|mariadb|sqlite|dynamodb|memcached|couchdb|influxdb|ftp|sftp|smb|nfs|smtp|imap|pop3|rdp|vnc|telnet|ssh|rsync|.env|env.local|env.dev|env.prod|env.staging|env.backup|env.bak|dotenv|config|config.json|config.yml|config.yaml|settings|application.properties|application.yml|web.config|appsettings|.git|gitconfig|htaccess|htpasswd|bash_history|zsh_history|fish_history|netrc|npmrc|pypirc|backup|bak|old|copy|tmp|temp|archive|dump|db.sql|database.sql|restore|snapshot|secret|secrets|password|passwd|credentials|creds|private|private_key|id_rsa|id_ed25519|pem|pfx|p12|keystore|truststore|jks|token|tokens|jwt|session|cookie|auth.json|serviceaccount|service-account|mail|webmail|owa|exchange|autodiscover|postfix|roundcube|dovecot|zimbra|sendgrid|mailgun|remote|pulse|citrix|anyconnect|f5|juniper|ivanti|globalprotect|openvpn|wireguard|sharepoint|teams|office365|onedrive|gsuite|workspace|zoom|webex|slack|mattermost|crm|erp|sap|salesforce|hubspot|zendesk|freshdesk|servicenow|dynamics|workday|bamboohr|upload|uploads|files|download|downloads|media|images|static|assets|public|private|documents|error|exception|stacktrace|stack_trace|traceback|debug|verbose|warning|trace|fatal|TODO|FIXME|HACK|XXX|BUG|TEMP|DEPRECATED|REMOVE|NOCOMMIT|WORKAROUND|user|users|account|accounts|member|members|profile|profiles|customer|customers|subscriber|subscribers|export|csv|xls|xlsx|report|extract|bulk|dump|import|stripe|paypal|braintree|square|adyen|checkout|payment|billing|invoice|receipt|refund|wallet|crypto|bitcoin|ethereum|solana|blockchain|web3|metamask|BEGIN RSA PRIVATE KEY|BEGIN OPENSSH PRIVATE KEY|BEGIN EC PRIVATE KEY|BEGIN PGP PRIVATE KEY|sk_live|pk_live|sk_test|rk_live|whsec|AKIA|ghp_|gho_|glpat|ssh-rsa|ssh-ed25519|ecdsa-sha2|mongod|mysqld|postgres|redis-server|memcached|rabbitmq|kafka|zookeeper|consul|vault|etcd|readonly|read-only|writeonly|internal-api|private-api|hidden|undocumented|legacy|deprecated|password=|passwd=|secret=|token=|apikey=|api_key=|auth=|key=|pass=|pwd=',
|
||
description: 'Pipe-separated grep patterns for interesting findings'
|
||
)
|
||
}
|
||
|
||
environment {
|
||
STATE_BASE = '/var/jenkins_home/recon-state'
|
||
RUN_DIR = 'current-run'
|
||
// recon-subfinder writes resolved hosts here — this is our input
|
||
RESOLVED_SRC = '/var/jenkins_home/recon-state/subfinder/all-resolved-latest.txt'
|
||
}
|
||
|
||
stages {
|
||
|
||
// ── 1. Prepare ────────────────────────────────────────────────────
|
||
stage('Prepare workspace') {
|
||
steps {
|
||
sh '''#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
|
||
echo "[*] Preparing clean workspace..."
|
||
rm -rf "$RUN_DIR"
|
||
mkdir -p "$RUN_DIR/results"
|
||
|
||
if [ ! -s "$RESOLVED_SRC" ]; then
|
||
# Fallback: try raw subdomain list (pre-dnsx setup)
|
||
RAW_SRC="$STATE_BASE/subfinder/all-subdomains-latest.txt"
|
||
if [ -s "$RAW_SRC" ]; then
|
||
echo "[!] No resolved list found — falling back to raw subdomain list."
|
||
echo "[!] Run recon-subfinder with dnsx installed for best results."
|
||
RESOLVED_SRC="$RAW_SRC"
|
||
else
|
||
echo "[!] No subfinder state found. Run recon-subfinder first."
|
||
exit 1
|
||
fi
|
||
fi
|
||
|
||
TOTAL=$(wc -l < "$RESOLVED_SRC")
|
||
echo "[*] Input list: $RESOLVED_SRC"
|
||
echo "[*] Total hosts: $TOTAL"
|
||
echo "[*] Chunk size: ${CHUNK_SIZE}"
|
||
'''
|
||
}
|
||
}
|
||
|
||
// ── 2. Select chunk via pointer file ─────────────────────────────
|
||
stage('Select chunk') {
|
||
steps {
|
||
sh '''#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
|
||
# Use resolved list if available, fall back to raw
|
||
INPUT="$RESOLVED_SRC"
|
||
[ ! -s "$INPUT" ] && INPUT="$STATE_BASE/subfinder/all-subdomains-latest.txt"
|
||
|
||
POINTER="$STATE_BASE/httpx/chunk-pointer.txt"
|
||
mkdir -p "$STATE_BASE/httpx"
|
||
|
||
TOTAL=$(wc -l < "$INPUT")
|
||
CHUNK_SIZE="${CHUNK_SIZE:-300}"
|
||
|
||
OFFSET=0
|
||
[ -f "$POINTER" ] && OFFSET=$(cat "$POINTER" | tr -d '[:space:]')
|
||
[[ "$OFFSET" =~ ^[0-9]+$ ]] || OFFSET=0
|
||
[ "$OFFSET" -ge "$TOTAL" ] && OFFSET=0
|
||
|
||
echo "[*] Total: $TOTAL | Chunk size: $CHUNK_SIZE | Offset: $OFFSET"
|
||
|
||
START=$(( OFFSET + 1 ))
|
||
{ tail -n "+${START}" "$INPUT" | head -n "$CHUNK_SIZE" \
|
||
> "$RUN_DIR/chunk.txt"; } || true
|
||
|
||
ACTUAL=$(wc -l < "$RUN_DIR/chunk.txt")
|
||
echo "[*] Chunk: $ACTUAL hosts (lines $START – $(( OFFSET + ACTUAL )))"
|
||
|
||
NEW_OFFSET=$(( OFFSET + ACTUAL ))
|
||
[ "$NEW_OFFSET" -ge "$TOTAL" ] && NEW_OFFSET=0
|
||
echo "$NEW_OFFSET" > "$POINTER"
|
||
echo "[*] Pointer advanced to $NEW_OFFSET"
|
||
|
||
echo "$OFFSET" > "$RUN_DIR/chunk-offset.txt"
|
||
echo "$ACTUAL" > "$RUN_DIR/chunk-actual.txt"
|
||
echo "$TOTAL" > "$RUN_DIR/host-total.txt"
|
||
'''
|
||
}
|
||
}
|
||
|
||
// ── 3. httpx probe ────────────────────────────────────────────────
|
||
stage('httpx – probe chunk') {
|
||
steps {
|
||
sh '''#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
|
||
command -v httpx >/dev/null 2>&1 || {
|
||
echo "[!] httpx not found. Skipping probe."
|
||
touch "$RUN_DIR/results/httpx-live.txt"
|
||
touch "$RUN_DIR/results/httpx-live.jsonl"
|
||
exit 0
|
||
}
|
||
|
||
CHUNK="$RUN_DIR/chunk.txt"
|
||
RDIR="$RUN_DIR/results"
|
||
|
||
if [ ! -s "$CHUNK" ]; then
|
||
echo "[!] Chunk is empty — nothing to probe."
|
||
touch "$RDIR/httpx-live.txt" "$RDIR/httpx-live.jsonl"
|
||
exit 0
|
||
fi
|
||
|
||
echo "[*] Probing $(wc -l < "$CHUNK") hosts..."
|
||
|
||
httpx -l "$CHUNK" \
|
||
-silent -sc -title -td -location -cl -rt -fr \
|
||
-nc \
|
||
-r "$RESOLVERS" \
|
||
-t "$HTTPX_THREADS" \
|
||
-timeout "$HTTPX_TIMEOUT" \
|
||
-retries 1 \
|
||
-H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" \
|
||
-rl 50 \
|
||
| sort -u > "$RDIR/httpx-live.txt" || true
|
||
|
||
httpx -l "$CHUNK" \
|
||
-silent -json -sc -title -td -location -cl -rt -fr \
|
||
-r "$RESOLVERS" \
|
||
-t "$HTTPX_THREADS" \
|
||
-timeout "$HTTPX_TIMEOUT" \
|
||
-retries 1 \
|
||
-H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" \
|
||
-rl 50 \
|
||
> "$RDIR/httpx-live.jsonl" || true
|
||
|
||
echo "[*] Live services found: $(wc -l < "$RDIR/httpx-live.txt")"
|
||
'''
|
||
}
|
||
}
|
||
|
||
// ── 4. Diff ───────────────────────────────────────────────────────
|
||
// Cumulative state stores: URL TAB status TAB title TAB tech
|
||
// Three categories: NEW / CHANGED / REMOVED
|
||
stage('httpx – diff') {
|
||
steps {
|
||
sh '''#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
|
||
RDIR="$RUN_DIR/results"
|
||
HDIR="$STATE_BASE/httpx"
|
||
mkdir -p "$HDIR/history"
|
||
|
||
LIVE="$RDIR/httpx-live.txt"
|
||
OLD_STATE="$HDIR/httpx-state-cumulative.txt"
|
||
NEW_TXT="$RDIR/httpx-new.txt"
|
||
CHANGED_TXT="$RDIR/httpx-changed.txt"
|
||
CHANGED_URLS="$RDIR/httpx-changed-urls.txt"
|
||
REM_TXT="$RDIR/httpx-removed.txt"
|
||
DIFF="$RDIR/httpx-diff.txt"
|
||
|
||
# initialise so variables are always bound
|
||
: > "$NEW_TXT"
|
||
: > "$CHANGED_TXT"
|
||
: > "$CHANGED_URLS"
|
||
: > "$REM_TXT"
|
||
|
||
OFFSET=$(cat "$RUN_DIR/chunk-offset.txt")
|
||
ACTUAL=$(cat "$RUN_DIR/chunk-actual.txt")
|
||
TOTAL=$(cat "$RUN_DIR/host-total.txt")
|
||
|
||
# ── Parse httpx text line → TAB-separated: URL status title tech ──────
|
||
# httpx format: URL [status] [title] [content-length] [error/redirect] [response-time] [tech]
|
||
# status = bracket 1 (always a number)
|
||
# title = first non-empty bracket that is NOT a number and NOT response-time
|
||
# tech = last bracket that is NOT a number and NOT response-time
|
||
parse_state() {
|
||
local infile="$1"
|
||
local outfile="$2"
|
||
: > "$outfile"
|
||
while IFS= read -r ln; do
|
||
[ -z "$ln" ] && continue
|
||
url="$(echo "$ln" | cut -d' ' -f1)"
|
||
rest="${ln#"$url"}"
|
||
status=""; title=""; tech=""
|
||
count=0
|
||
tmp="$rest"
|
||
while true; do
|
||
case "$tmp" in
|
||
*"["*) tmp="${tmp#*[}" ;;
|
||
*) break ;;
|
||
esac
|
||
val="${tmp%%]*}"
|
||
tmp="${tmp#"$val]"}"
|
||
[ -z "$val" ] && continue
|
||
count=$(( count + 1 ))
|
||
# bracket 1 is always status
|
||
if [ "$count" -eq 1 ]; then
|
||
status="$val"
|
||
continue
|
||
fi
|
||
# skip response-time (ends in ms or digits+s)
|
||
case "$val" in
|
||
*ms) continue ;;
|
||
*[0-9]s) continue ;;
|
||
esac
|
||
# skip pure numbers and comma-separated numbers (content-length, status codes)
|
||
case "$val" in
|
||
*[!0-9,]*) : ;;
|
||
*) continue ;;
|
||
esac
|
||
# first remaining non-empty non-number non-time value = title
|
||
if [ -z "$title" ]; then
|
||
title="$val"
|
||
fi
|
||
# keep updating tech — last one wins
|
||
tech="$val"
|
||
done
|
||
# if title and tech are same (only one real value found) — it's tech not title
|
||
if [ "$title" = "$tech" ]; then
|
||
title=""
|
||
fi
|
||
printf '%s\t%s\t%s\t%s\n' "$url" "$status" "$title" "$tech"
|
||
done < "$infile" | sort -u >> "$outfile"
|
||
}
|
||
|
||
LIVE_STATE="$RDIR/httpx-live-state.txt"
|
||
parse_state "$LIVE" "$LIVE_STATE"
|
||
|
||
LIVE_URLS="$RDIR/httpx-live-urls.txt"
|
||
cut -f1 "$LIVE_STATE" | sort -u > "$LIVE_URLS"
|
||
|
||
if [ ! -f "$OLD_STATE" ]; then
|
||
echo "[*] No cumulative baseline — creating one."
|
||
cp "$LIVE_STATE" "$OLD_STATE"
|
||
cp "$LIVE" "$NEW_TXT"
|
||
: > "$CHANGED_TXT"
|
||
: > "$REM_TXT"
|
||
{
|
||
echo "========================================"
|
||
echo " recon-httpx — Initial Baseline"
|
||
echo "========================================"
|
||
echo "Job: ${JOB_NAME} #${BUILD_NUMBER}"
|
||
echo "Timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||
echo "Chunk: lines ${OFFSET}-$(( OFFSET + ACTUAL - 1 )) / $TOTAL"
|
||
echo ""
|
||
echo "Live services ($(wc -l < "$LIVE_URLS")):"
|
||
cat "$LIVE"
|
||
} > "$DIFF"
|
||
else
|
||
OLD_URLS="$RDIR/httpx-old-urls.txt"
|
||
cut -f1 "$OLD_STATE" | sort -u > "$OLD_URLS"
|
||
|
||
# ── NEW: URL never seen before ────────────────────────────────────
|
||
NEW_URLS="$RDIR/httpx-new-urls.txt"
|
||
comm -13 "$OLD_URLS" "$LIVE_URLS" > "$NEW_URLS"
|
||
|
||
if [ -s "$NEW_URLS" ]; then
|
||
grep -Ff "$NEW_URLS" "$LIVE" | sort -u > "$NEW_TXT" || : > "$NEW_TXT"
|
||
else
|
||
: > "$NEW_TXT"
|
||
fi
|
||
|
||
# ── CHANGED: URL known but state different ────────────────────────
|
||
KNOWN_URLS="$RDIR/httpx-known-urls.txt"
|
||
comm -12 "$OLD_URLS" "$LIVE_URLS" > "$KNOWN_URLS"
|
||
|
||
: > "$CHANGED_TXT"
|
||
CHANGED_URLS="$RDIR/httpx-changed-urls.txt"
|
||
: > "$CHANGED_URLS"
|
||
|
||
if [ -s "$KNOWN_URLS" ]; then
|
||
while IFS= read -r url; do
|
||
OLD_LINE="$(grep -F "$url " "$OLD_STATE" | head -1 || true)"
|
||
NEW_LINE="$(grep -F "$url " "$LIVE_STATE" | head -1 || true)"
|
||
if [ -n "$OLD_LINE" ] && [ -n "$NEW_LINE" ] && [ "$OLD_LINE" != "$NEW_LINE" ]; then
|
||
OS="$(echo "$OLD_LINE" | cut -f2)"
|
||
NS="$(echo "$NEW_LINE" | cut -f2)"
|
||
OT="$(echo "$OLD_LINE" | cut -f3)"
|
||
NT="$(echo "$NEW_LINE" | cut -f3)"
|
||
OK="$(echo "$OLD_LINE" | cut -f4)"
|
||
NK="$(echo "$NEW_LINE" | cut -f4)"
|
||
echo "$url" >> "$CHANGED_TXT"
|
||
echo "$url" >> "$CHANGED_URLS"
|
||
[ "$OS" != "$NS" ] && echo " status: $OS -> $NS" >> "$CHANGED_TXT"
|
||
[ "$OT" != "$NT" ] && echo " title: $OT -> $NT" >> "$CHANGED_TXT"
|
||
[ "$OK" != "$NK" ] && echo " tech: $OK -> $NK" >> "$CHANGED_TXT"
|
||
fi
|
||
done < "$KNOWN_URLS"
|
||
fi
|
||
|
||
# ── REMOVED: was live in this chunk last time, now gone ───────────
|
||
PREV_CHUNK_LIVE="$RDIR/httpx-prev-chunk-live-urls.txt"
|
||
: > "$PREV_CHUNK_LIVE"
|
||
while IFS= read -r host; do
|
||
grep -xF "https://${host}" "$OLD_URLS" >> "$PREV_CHUNK_LIVE" || true
|
||
grep -xF "http://${host}" "$OLD_URLS" >> "$PREV_CHUNK_LIVE" || true
|
||
done < "$RUN_DIR/chunk.txt"
|
||
sort -u "$PREV_CHUNK_LIVE" -o "$PREV_CHUNK_LIVE"
|
||
comm -23 "$PREV_CHUNK_LIVE" "$LIVE_URLS" > "$REM_TXT" || : > "$REM_TXT"
|
||
|
||
# ── Counts ────────────────────────────────────────────────────────
|
||
NEW_COUNT=$(wc -l < "$NEW_URLS")
|
||
CHANGED_COUNT=$(wc -l < "$CHANGED_URLS")
|
||
REM_COUNT=$(wc -l < "$REM_TXT")
|
||
|
||
# ── Diff report ───────────────────────────────────────────────────
|
||
{
|
||
echo "========================================"
|
||
echo " recon-httpx — Diff Report"
|
||
echo "========================================"
|
||
echo "Job: ${JOB_NAME} #${BUILD_NUMBER}"
|
||
echo "Timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||
echo "Chunk: lines ${OFFSET}-$(( OFFSET + ACTUAL - 1 )) / $TOTAL"
|
||
echo "Probed: $ACTUAL | Live: $(wc -l < "$LIVE_URLS")"
|
||
echo ""
|
||
echo "=== NEW hosts (${NEW_COUNT}) ==="
|
||
cat "$NEW_TXT"
|
||
echo ""
|
||
echo "=== CHANGED hosts (${CHANGED_COUNT}) ==="
|
||
cat "$CHANGED_TXT"
|
||
echo ""
|
||
echo "=== REMOVED from this chunk (${REM_COUNT}) ==="
|
||
cat "$REM_TXT"
|
||
} > "$DIFF"
|
||
|
||
# ── Update cumulative state ───────────────────────────────────────
|
||
URLS_TO_REMOVE="$RDIR/httpx-urls-to-remove.txt"
|
||
{
|
||
cat "$NEW_URLS"
|
||
cat "$CHANGED_URLS"
|
||
cat "$REM_TXT"
|
||
} | sort -u > "$URLS_TO_REMOVE"
|
||
|
||
if [ -s "$URLS_TO_REMOVE" ]; then
|
||
grep -vFf "$URLS_TO_REMOVE" "$OLD_STATE" > "$RDIR/httpx-state-kept.txt" || true
|
||
else
|
||
cp "$OLD_STATE" "$RDIR/httpx-state-kept.txt"
|
||
fi
|
||
|
||
{
|
||
cat "$RDIR/httpx-state-kept.txt"
|
||
grep -Ff "$NEW_URLS" "$LIVE_STATE" 2>/dev/null || true
|
||
grep -Ff "$CHANGED_URLS" "$LIVE_STATE" 2>/dev/null || true
|
||
} | sort -u > "$OLD_STATE"
|
||
fi
|
||
|
||
cp "$LIVE" "$HDIR/history/build-${BUILD_NUMBER}.txt"
|
||
|
||
# nuclei input: new + changed URLs only (not full httpx lines)
|
||
{
|
||
cut -f1 "$RDIR/httpx-new-urls.txt" 2>/dev/null || true
|
||
cat "$CHANGED_URLS" 2>/dev/null || true
|
||
} | sort -u > "$HDIR/httpx-last-new.txt"
|
||
|
||
# ── Daily digest — accumulates NEW + CHANGED across all runs today ───
|
||
# Resets automatically when date changes
|
||
TODAY="$(date -u +%Y-%m-%d)"
|
||
DIGEST="$HDIR/daily-digest.txt"
|
||
DIGEST_DATE="$HDIR/daily-digest-date.txt"
|
||
|
||
# Reset digest if it's a new day
|
||
if [ -f "$DIGEST_DATE" ]; then
|
||
LAST_DATE="$(cat "$DIGEST_DATE")"
|
||
if [ "$LAST_DATE" != "$TODAY" ]; then
|
||
: > "$DIGEST"
|
||
echo "$TODAY" > "$DIGEST_DATE"
|
||
fi
|
||
else
|
||
: > "$DIGEST"
|
||
echo "$TODAY" > "$DIGEST_DATE"
|
||
fi
|
||
|
||
# Append new findings to digest
|
||
if [ -s "$NEW_TXT" ] || [ -s "$CHANGED_TXT" ]; then
|
||
{
|
||
echo ""
|
||
echo "--- Build #${BUILD_NUMBER} $(date -u +%H:%M:%SZ) ---"
|
||
if [ -s "$NEW_TXT" ]; then
|
||
echo "[NEW]"
|
||
cat "$NEW_TXT"
|
||
fi
|
||
if [ -s "$CHANGED_TXT" ]; then
|
||
echo "[CHANGED]"
|
||
cat "$CHANGED_TXT"
|
||
fi
|
||
} >> "$DIGEST"
|
||
fi
|
||
|
||
NEW_COUNT=$(wc -l < "$NEW_TXT")
|
||
CHANGED_COUNT=$(wc -l < "$CHANGED_URLS" 2>/dev/null || echo 0)
|
||
REM_COUNT=$(wc -l < "$REM_TXT")
|
||
|
||
{
|
||
echo "job=${JOB_NAME}"
|
||
echo "build=${BUILD_NUMBER}"
|
||
echo "timestamp=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||
echo "offset=${OFFSET}"
|
||
echo "chunk_size=${ACTUAL}"
|
||
echo "total_hosts=${TOTAL}"
|
||
echo "live=$(wc -l < "$LIVE_URLS")"
|
||
echo "new=${NEW_COUNT}"
|
||
echo "changed=${CHANGED_COUNT}"
|
||
echo "removed=${REM_COUNT}"
|
||
} > "$HDIR/metadata.txt"
|
||
|
||
echo
|
||
echo "[*] httpx diff complete."
|
||
echo " Live: $(wc -l < "$LIVE_URLS")"
|
||
echo " New: ${NEW_COUNT}"
|
||
echo " Changed: ${CHANGED_COUNT}"
|
||
echo " Removed: ${REM_COUNT}"
|
||
'''
|
||
}
|
||
}
|
||
|
||
// ── 5. Grep – interesting findings ───────────────────────────────
|
||
stage('grep – interesting findings') {
|
||
steps {
|
||
sh '''#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
|
||
LIVE="$RUN_DIR/results/httpx-live.txt"
|
||
JSON="$RUN_DIR/results/httpx-live.jsonl"
|
||
RDIR="$RUN_DIR/results"
|
||
PATTERNS="${GREP_PATTERNS:-admin|login|staging|dev|test|api}"
|
||
|
||
echo "[*] Scanning for interesting patterns..."
|
||
|
||
GREP_TXT="$RDIR/grep-interesting.txt"
|
||
if [ -s "$LIVE" ]; then
|
||
grep -iE "$PATTERNS" "$LIVE" | sort -u > "$GREP_TXT" || : > "$GREP_TXT"
|
||
else
|
||
: > "$GREP_TXT"
|
||
fi
|
||
|
||
GREP_JSON="$RDIR/grep-interesting.jsonl"
|
||
if [ -s "$JSON" ] && command -v jq >/dev/null 2>&1; then
|
||
jq -c --arg p "$PATTERNS" \
|
||
'select(
|
||
(.url // "" | test($p; "i")) or
|
||
(.title // "" | test($p; "i")) or
|
||
((.tech // []) | map(.) | join(" ") | test($p; "i"))
|
||
)' "$JSON" | sort -u > "$GREP_JSON" || : > "$GREP_JSON"
|
||
else
|
||
grep -iE "$PATTERNS" "$JSON" 2>/dev/null | sort -u > "$GREP_JSON" \
|
||
|| : > "$GREP_JSON"
|
||
fi
|
||
|
||
GREP_SUBS="$RDIR/grep-interesting-subdomains.txt"
|
||
grep -iE "$PATTERNS" "$RUN_DIR/chunk.txt" | sort -u > "$GREP_SUBS" \
|
||
|| : > "$GREP_SUBS"
|
||
|
||
echo "[*] Text hits: $(wc -l < "$GREP_TXT")"
|
||
echo "[*] JSON hits: $(wc -l < "$GREP_JSON")"
|
||
echo "[*] Subdomain hits: $(wc -l < "$GREP_SUBS")"
|
||
|
||
if [ -s "$GREP_TXT" ]; then
|
||
echo ""
|
||
echo "[*] Top 30 interesting (text):"
|
||
head -n 30 "$GREP_TXT"
|
||
fi
|
||
'''
|
||
}
|
||
}
|
||
|
||
// ── 6. Archive ────────────────────────────────────────────────────
|
||
stage('Archive results') {
|
||
steps {
|
||
archiveArtifacts artifacts: [
|
||
'current-run/**/*.txt',
|
||
'current-run/**/*.jsonl'
|
||
].join(','), fingerprint: true
|
||
}
|
||
}
|
||
}
|
||
|
||
post {
|
||
success {
|
||
build job: 'recon-nuclei',
|
||
parameters: [
|
||
booleanParam(name: 'SCAN_NEW_ONLY', value: true)
|
||
],
|
||
wait: false,
|
||
propagate: false
|
||
}
|
||
always { echo "[*] recon-httpx finished." }
|
||
failure { echo "[!] recon-httpx FAILED." }
|
||
}
|
||
} |