// ═══════════════════════════════════════════════════════════════════════════
//  recon-httpx
//  Probes ONE chunk of resolved hosts per run (pointer-file rotation).
//  Reads from all-resolved-latest.txt written by recon-subfinder.
//
//  CHANGES vs previous version:
//  - parse_state() State-Fix: alle Chunk-Hosts werden neu geschrieben
//  - NEWs + CHANGEDs werden in nuclei-queue.txt akkumuliert
//  - Blacklist-Filter vor Queue-Eintrag
//  - nuclei wird nicht mehr direkt getriggert
//  - HTTPX_BLACKLIST: hosts vor httpx-Probe herausfiltern
//
//  Recommended schedule: every 30 min (H/2 * * * *)
// ═══════════════════════════════════════════════════════════════════════════

pipeline {
    agent any

    options {
        timestamps()
        disableConcurrentBuilds()
        timeout(time: 1, unit: 'HOURS')
    }

    triggers {
        cron('20,50 0-22 * * *')
    }

    parameters {
        string(
            name: 'CHUNK_SIZE',
            defaultValue: '300',
            description: 'Max hosts to probe per run'
        )
        string(
            name: 'RESOLVERS',
            defaultValue: '1.1.1.1,1.0.0.1,8.8.8.8,8.8.4.4,9.9.9.9,149.112.112.112',
            description: 'Comma-separated DNS resolvers'
        )
        string(
            name: 'HTTPX_THREADS',
            defaultValue: '25',
            description: 'httpx thread count'
        )
        string(
            name: 'HTTPX_TIMEOUT',
            defaultValue: '10',
            description: 'httpx timeout in seconds'
        )
        string(
            name: 'GREP_PATTERNS',
            defaultValue: 'admin|administrator|adminpanel|admin-panel|admin_panel|admincp|cpanel|webadmin|superadmin|siteadmin|login|signin|sign-in|sign_in|logon|log-in|log_in|sso|oauth|openid|saml|ldap|kerberos|portal|dashboard|control-panel|controlpanel|manage|management|manager|console|panel|wp-admin|wp-login|phpmyadmin|adminer|dbadmin|phpinfo|server-status|server-info|auth|authenticate|authentication|authorize|authorization|access-control|rbac|acl|staging|stage|preprod|pre-prod|pre_prod|uat|sit|qa|qe|integration|testing-env|dev|develop|development|sandbox|local|localhost|internal|intranet|corp|corpnet|test|testing|testenv|test-env|demo|poc|pilot|beta|alpha|canary|nightly|feature|hotfix|release|deploy|build|preview|review-app|draft|api|api-v1|api-v2|api-v3|rest|restapi|graphql|grpc|rpc|soap|xmlrpc|endpoint|gateway|proxy|backend|service|microservice|webhook|callback|listener|swagger|openapi|api-docs|jenkins|grafana|kibana|prometheus|elasticsearch|gitlab|github|bitbucket|sonarqube|nexus|artifactory|harbor|docker|kubernetes|rancher|portainer|splunk|datadog|sentry|s3|ec2|lambda|cloudfront|terraform|ansible|vpn|bastion|jump|firewall|loadbalancer|.env|config|secret|secrets|password|passwd|credentials|backup|dump|token|jwt|private|id_rsa|mysql|postgres|redis|mongo|ftp|sftp|smtp|rdp|vnc|ssh|actuator|health|healthcheck|metrics|monitoring',
            description: 'Pipe-separated grep patterns for interesting findings'
        )
        text(
            name: 'HTTPX_BLACKLIST',
            defaultValue: '',
            description: 'Domains/wildcards to skip entirely — not probed by httpx. One per line. Wildcards: *.example.com'
        )
        text(
            name: 'NUCLEI_BLACKLIST',
            defaultValue: '''\
*.hubspot.com
*.twilio.com''',
            description: 'Domains/wildcards to exclude from nuclei queue. One per line. Wildcards: *.example.com'
        )
    }

    environment {
        STATE_BASE   = '/var/jenkins_home/recon-state'
        RUN_DIR      = 'current-run'
        RESOLVED_SRC = '/var/jenkins_home/recon-state/subfinder/all-resolved-latest.txt'
    }

    stages {

        // ── 1. Prepare ────────────────────────────────────────────────────
        stage('Prepare workspace') {
            steps {
                sh '''#!/usr/bin/env bash
set -euo pipefail

echo "[*] Preparing clean workspace..."
rm -rf "$RUN_DIR"
mkdir -p "$RUN_DIR/results"

if [ ! -s "$RESOLVED_SRC" ]; then
    RAW_SRC="$STATE_BASE/subfinder/all-subdomains-latest.txt"
    if [ -s "$RAW_SRC" ]; then
        echo "[!] No resolved list found — falling back to raw subdomain list."
        RESOLVED_SRC="$RAW_SRC"
    else
        echo "[!] No subfinder state found. Run recon-subfinder first."
        exit 1
    fi
fi

TOTAL=$(wc -l < "$RESOLVED_SRC")
echo "[*] Input list:  $RESOLVED_SRC"
echo "[*] Total hosts: $TOTAL"
echo "[*] Chunk size:  ${CHUNK_SIZE}"
'''
            }
        }

        // ── 2. Select chunk ───────────────────────────────────────────────
        stage('Select chunk') {
            steps {
                sh '''#!/usr/bin/env bash
set -euo pipefail

INPUT="$RESOLVED_SRC"
[ ! -s "$INPUT" ] && INPUT="$STATE_BASE/subfinder/all-subdomains-latest.txt"

POINTER="$STATE_BASE/httpx/chunk-pointer.txt"
mkdir -p "$STATE_BASE/httpx"

TOTAL=$(wc -l < "$INPUT")
CHUNK_SIZE="${CHUNK_SIZE:-300}"

OFFSET=0
[ -f "$POINTER" ] && OFFSET=$(cat "$POINTER" | tr -d '[:space:]')
[[ "$OFFSET" =~ ^[0-9]+$ ]] || OFFSET=0
[ "$OFFSET" -ge "$TOTAL" ] && OFFSET=0

echo "[*] Total: $TOTAL | Chunk size: $CHUNK_SIZE | Offset: $OFFSET"

START=$(( OFFSET + 1 ))
{ tail -n "+${START}" "$INPUT" | head -n "$CHUNK_SIZE" \
    > "$RUN_DIR/chunk.txt"; } || true

ACTUAL=$(wc -l < "$RUN_DIR/chunk.txt")
echo "[*] Chunk: $ACTUAL hosts (lines $START to $(( OFFSET + ACTUAL )))"

NEW_OFFSET=$(( OFFSET + ACTUAL ))
[ "$NEW_OFFSET" -ge "$TOTAL" ] && NEW_OFFSET=0
echo "$NEW_OFFSET" > "$POINTER"
echo "[*] Pointer advanced to $NEW_OFFSET"

# ── HTTPX_BLACKLIST filter — remove hosts before probing ─────────────
HTTPX_BL_FILE="$RUN_DIR/httpx-blacklist.txt"
printf '%s\n' "${HTTPX_BLACKLIST}" \
    | sed 's/\r$//' \
    | grep -vE '^[[:space:]]*($|#)' \
    | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' \
    | sort -u > "$HTTPX_BL_FILE" || true

if [ -s "$HTTPX_BL_FILE" ]; then
    BEFORE=$(wc -l < "$RUN_DIR/chunk.txt")
    FILTERED_CHUNK="$RUN_DIR/chunk-filtered.txt"
    : > "$FILTERED_CHUNK"
    while IFS= read -r host; do
        [ -z "$host" ] && continue
        blocked=false
        while IFS= read -r rule; do
            [ -z "$rule" ] && continue
            case "$rule" in
                *.*)
                    suffix="${rule#*.}"
                    host_lower="$(echo "$host" | tr '[:upper:]' '[:lower:]')"
                    suffix_lower="$(echo "$suffix" | tr '[:upper:]' '[:lower:]')"
                    if [ "$host_lower" = "$suffix_lower" ]; then blocked=true; break; fi
                    case "$host_lower" in
                        *."$suffix_lower") blocked=true; break ;;
                    esac
                    ;;
                *)
                    host_lower="$(echo "$host" | tr '[:upper:]' '[:lower:]')"
                    rule_lower="$(echo "$rule" | tr '[:upper:]' '[:lower:]')"
                    if [ "$host_lower" = "$rule_lower" ]; then blocked=true; break; fi
                    ;;
            esac
        done < "$HTTPX_BL_FILE"
        [ "$blocked" = "false" ] && echo "$host" >> "$FILTERED_CHUNK"
    done < "$RUN_DIR/chunk.txt"
    mv "$FILTERED_CHUNK" "$RUN_DIR/chunk.txt"
    AFTER=$(wc -l < "$RUN_DIR/chunk.txt")
    echo "[*] HTTPX_BLACKLIST: removed $(( BEFORE - AFTER )) hosts, $AFTER remaining"
else
    echo "[*] HTTPX_BLACKLIST: empty — no hosts filtered"
fi

ACTUAL=$(wc -l < "$RUN_DIR/chunk.txt")
echo "$OFFSET" > "$RUN_DIR/chunk-offset.txt"
echo "$ACTUAL" > "$RUN_DIR/chunk-actual.txt"
echo "$TOTAL"  > "$RUN_DIR/host-total.txt"
'''
            }
        }

        // ── 3. httpx probe ────────────────────────────────────────────────
        stage('httpx – probe chunk') {
            steps {
                sh '''#!/usr/bin/env bash
set -euo pipefail

command -v httpx >/dev/null 2>&1 || {
    echo "[!] httpx not found. Skipping probe."
    touch "$RUN_DIR/results/httpx-live.txt"
    touch "$RUN_DIR/results/httpx-live.jsonl"
    exit 0
}

CHUNK="$RUN_DIR/chunk.txt"
RDIR="$RUN_DIR/results"

if [ ! -s "$CHUNK" ]; then
    echo "[!] Chunk is empty — nothing to probe."
    touch "$RDIR/httpx-live.txt" "$RDIR/httpx-live.jsonl"
    exit 0
fi

echo "[*] Probing $(wc -l < "$CHUNK") hosts..."

httpx -l "$CHUNK" \
    -silent -sc -title -td -location -cl -rt -fr \
    -nc \
    -r "$RESOLVERS" \
    -t "$HTTPX_THREADS" \
    -timeout "$HTTPX_TIMEOUT" \
    -retries 1 \
    -H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" \
    -rl 50 \
    | sort -u > "$RDIR/httpx-live.txt" || true

httpx -l "$CHUNK" \
    -silent -json -sc -title -td -location -cl -rt -fr \
    -r "$RESOLVERS" \
    -t "$HTTPX_THREADS" \
    -timeout "$HTTPX_TIMEOUT" \
    -retries 1 \
    -H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" \
    -rl 50 \
    > "$RDIR/httpx-live.jsonl" || true

echo "[*] Live services found: $(wc -l < "$RDIR/httpx-live.txt")"
'''
            }
        }

        // ── 4. Diff ───────────────────────────────────────────────────────
        //  STATE-FIX: alle Hosts des aktuellen Chunks werden aus dem alten
        //  State entfernt und mit dem frischen Live-State neu geschrieben.
        //  Verhindert dass kaputte Einträge (z.B. CL als Titel) ewig bleiben.
        stage('httpx – diff') {
            steps {
                sh '''#!/usr/bin/env bash
set -euo pipefail

RDIR="$RUN_DIR/results"
HDIR="$STATE_BASE/httpx"
mkdir -p "$HDIR/history"

LIVE="$RDIR/httpx-live.txt"
OLD_STATE="$HDIR/httpx-state-cumulative.txt"
NEW_TXT="$RDIR/httpx-new.txt"
CHANGED_TXT="$RDIR/httpx-changed.txt"
CHANGED_URLS="$RDIR/httpx-changed-urls.txt"
REM_TXT="$RDIR/httpx-removed.txt"
DIFF="$RDIR/httpx-diff.txt"

: > "$NEW_TXT"
: > "$CHANGED_TXT"
: > "$CHANGED_URLS"
: > "$REM_TXT"

OFFSET=$(cat "$RUN_DIR/chunk-offset.txt")
ACTUAL=$(cat "$RUN_DIR/chunk-actual.txt")
TOTAL=$(cat "$RUN_DIR/host-total.txt")

# ── Parse httpx text line → TAB-separated: URL status title tech ──────
parse_state() {
    local infile="$1"
    local outfile="$2"
    : > "$outfile"
    while IFS= read -r ln; do
        [ -z "$ln" ] && continue
        url="$(echo "$ln" | cut -d' ' -f1)"
        rest="${ln#"$url"}"
        status=""; title=""; tech=""
        count=0
        tmp="$rest"
        while true; do
            case "$tmp" in
                *"["*) tmp="${tmp#*[}" ;;
                *) break ;;
            esac
            val="${tmp%%]*}"
            tmp="${tmp#"$val]"}"
            [ -z "$val" ] && continue
            count=$(( count + 1 ))
            if [ "$count" -eq 1 ]; then
                status="$val"
                continue
            fi
            case "$val" in
                *ms) continue ;;
                *[0-9]s) continue ;;
            esac
            case "$val" in
                *[!0-9,]*) : ;;
                *) continue ;;
            esac
            if [ -z "$title" ]; then
                title="$val"
            fi
            tech="$val"
        done
        if [ "$title" = "$tech" ]; then
            title=""
        fi
        printf '%s\t%s\t%s\t%s\n' "$url" "$status" "$title" "$tech"
    done < "$infile" | sort -u >> "$outfile"
}

LIVE_STATE="$RDIR/httpx-live-state.txt"
parse_state "$LIVE" "$LIVE_STATE"

LIVE_URLS="$RDIR/httpx-live-urls.txt"
cut -f1 "$LIVE_STATE" | sort -u > "$LIVE_URLS"

if [ ! -f "$OLD_STATE" ]; then
    echo "[*] No cumulative baseline — creating one."
    cp "$LIVE_STATE" "$OLD_STATE"
    cp "$LIVE" "$NEW_TXT"
    : > "$CHANGED_TXT"
    : > "$REM_TXT"
    {
        echo "========================================"
        echo "  recon-httpx — Initial Baseline"
        echo "========================================"
        echo "Job:       ${JOB_NAME}  #${BUILD_NUMBER}"
        echo "Timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
        echo "Chunk:     lines ${OFFSET}-$(( OFFSET + ACTUAL - 1 )) / $TOTAL"
        echo ""
        echo "Live services ($(wc -l < "$LIVE_URLS")):"
        cat "$LIVE"
    } > "$DIFF"
else
    OLD_URLS="$RDIR/httpx-old-urls.txt"
    cut -f1 "$OLD_STATE" | sort -u > "$OLD_URLS"

    # ── NEW ───────────────────────────────────────────────────────────
    NEW_URLS="$RDIR/httpx-new-urls.txt"
    comm -13 "$OLD_URLS" "$LIVE_URLS" > "$NEW_URLS"

    if [ -s "$NEW_URLS" ]; then
        grep -Ff "$NEW_URLS" "$LIVE" | sort -u > "$NEW_TXT" || : > "$NEW_TXT"
    else
        : > "$NEW_TXT"
    fi

    # ── CHANGED ───────────────────────────────────────────────────────
    KNOWN_URLS="$RDIR/httpx-known-urls.txt"
    comm -12 "$OLD_URLS" "$LIVE_URLS" > "$KNOWN_URLS"

    : > "$CHANGED_TXT"
    : > "$CHANGED_URLS"

    if [ -s "$KNOWN_URLS" ]; then
        while IFS= read -r url; do
            OLD_LINE="$(grep -F "$url	" "$OLD_STATE" | head -1 || true)"
            NEW_LINE="$(grep -F "$url	" "$LIVE_STATE" | head -1 || true)"
            if [ -n "$OLD_LINE" ] && [ -n "$NEW_LINE" ] && [ "$OLD_LINE" != "$NEW_LINE" ]; then
                OS="$(echo "$OLD_LINE" | cut -f2)"
                NS="$(echo "$NEW_LINE" | cut -f2)"
                OT="$(echo "$OLD_LINE" | cut -f3)"
                NT="$(echo "$NEW_LINE" | cut -f3)"
                OK="$(echo "$OLD_LINE" | cut -f4)"
                NK="$(echo "$NEW_LINE" | cut -f4)"
                echo "$url" >> "$CHANGED_TXT"
                echo "$url" >> "$CHANGED_URLS"
                [ "$OS" != "$NS" ] && echo "  status: $OS -> $NS" >> "$CHANGED_TXT"
                [ "$OT" != "$NT" ] && echo "  title:  $OT -> $NT" >> "$CHANGED_TXT"
                [ "$OK" != "$NK" ] && echo "  tech:   $OK -> $NK" >> "$CHANGED_TXT"
            fi
        done < "$KNOWN_URLS"
    fi

    # ── REMOVED ───────────────────────────────────────────────────────
    PREV_CHUNK_LIVE="$RDIR/httpx-prev-chunk-live-urls.txt"
    : > "$PREV_CHUNK_LIVE"
    while IFS= read -r host; do
        grep -xF "https://${host}" "$OLD_URLS" >> "$PREV_CHUNK_LIVE" || true
        grep -xF "http://${host}"  "$OLD_URLS" >> "$PREV_CHUNK_LIVE" || true
    done < "$RUN_DIR/chunk.txt"
    sort -u "$PREV_CHUNK_LIVE" -o "$PREV_CHUNK_LIVE"
    comm -23 "$PREV_CHUNK_LIVE" "$LIVE_URLS" > "$REM_TXT" || : > "$REM_TXT"

    NEW_COUNT=$(wc -l < "$NEW_URLS")
    CHANGED_COUNT=$(wc -l < "$CHANGED_URLS")
    REM_COUNT=$(wc -l < "$REM_TXT")

    {
        echo "========================================"
        echo "  recon-httpx — Diff Report"
        echo "========================================"
        echo "Job:       ${JOB_NAME}  #${BUILD_NUMBER}"
        echo "Timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
        echo "Chunk:     lines ${OFFSET}-$(( OFFSET + ACTUAL - 1 )) / $TOTAL"
        echo "Probed:    $ACTUAL  |  Live: $(wc -l < "$LIVE_URLS")"
        echo ""
        echo "=== NEW hosts (${NEW_COUNT}) ==="
        cat "$NEW_TXT"
        echo ""
        echo "=== CHANGED hosts (${CHANGED_COUNT}) ==="
        cat "$CHANGED_TXT"
        echo ""
        echo "=== REMOVED from this chunk (${REM_COUNT}) ==="
        cat "$REM_TXT"
    } > "$DIFF"

    # ── STATE-FIX: alle Chunk-Hosts aus State entfernen und neu schreiben
    # Entferne alle URLs des aktuellen Chunks aus dem alten State
    # (nicht nur NEW+CHANGED+REMOVED) damit kaputte Einträge sich heilen
    CHUNK_URLS="$RDIR/httpx-chunk-urls.txt"
    : > "$CHUNK_URLS"
    while IFS= read -r host; do
        echo "https://${host}" >> "$CHUNK_URLS"
        echo "http://${host}"  >> "$CHUNK_URLS"
    done < "$RUN_DIR/chunk.txt"
    sort -u "$CHUNK_URLS" -o "$CHUNK_URLS"

    grep -vFf "$CHUNK_URLS" "$OLD_STATE" > "$RDIR/httpx-state-kept.txt" || true

    {
        cat "$RDIR/httpx-state-kept.txt"
        cat "$LIVE_STATE"
    } | sort -u > "$OLD_STATE"
fi

cp "$LIVE" "$HDIR/history/build-${BUILD_NUMBER}.txt"

# ── Daily digest ──────────────────────────────────────────────────────
TODAY="$(date -u +%Y-%m-%d)"
DIGEST="$HDIR/daily-digest.txt"
DIGEST_DATE="$HDIR/daily-digest-date.txt"

if [ -f "$DIGEST_DATE" ]; then
    LAST_DATE="$(cat "$DIGEST_DATE")"
    if [ "$LAST_DATE" != "$TODAY" ]; then
        : > "$DIGEST"
        echo "$TODAY" > "$DIGEST_DATE"
    fi
else
    : > "$DIGEST"
    echo "$TODAY" > "$DIGEST_DATE"
fi

if [ -s "$NEW_TXT" ] || [ -s "$CHANGED_TXT" ]; then
    {
        echo ""
        echo "--- Build #${BUILD_NUMBER} $(date -u +%H:%M:%SZ) ---"
        if [ -s "$NEW_TXT" ]; then
            echo "[NEW]"
            cat "$NEW_TXT"
        fi
        if [ -s "$CHANGED_TXT" ]; then
            echo "[CHANGED]"
            cat "$CHANGED_TXT"
        fi
    } >> "$DIGEST"
fi

NEW_COUNT=$(wc -l < "$NEW_TXT")
CHANGED_COUNT=$(wc -l < "$CHANGED_URLS" 2>/dev/null || echo 0)
REM_COUNT=$(wc -l < "$REM_TXT")

{
    echo "job=${JOB_NAME}"
    echo "build=${BUILD_NUMBER}"
    echo "timestamp=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
    echo "offset=${OFFSET}"
    echo "chunk_size=${ACTUAL}"
    echo "total_hosts=${TOTAL}"
    echo "live=$(wc -l < "$LIVE_URLS")"
    echo "new=${NEW_COUNT}"
    echo "changed=${CHANGED_COUNT}"
    echo "removed=${REM_COUNT}"
} > "$HDIR/metadata.txt"

echo
echo "[*] httpx diff complete."
echo "    Live:    $(wc -l < "$LIVE_URLS")"
echo "    New:     ${NEW_COUNT}"
echo "    Changed: ${CHANGED_COUNT}"
echo "    Removed: ${REM_COUNT}"
'''
            }
        }

        // ── 5. Grep – interesting findings ───────────────────────────────
        stage('grep – interesting findings') {
            steps {
                sh '''#!/usr/bin/env bash
set -euo pipefail

LIVE="$RUN_DIR/results/httpx-live.txt"
JSON="$RUN_DIR/results/httpx-live.jsonl"
RDIR="$RUN_DIR/results"
PATTERNS="${GREP_PATTERNS:-admin|login|staging|dev|test|api}"

echo "[*] Scanning for interesting patterns..."

GREP_TXT="$RDIR/grep-interesting.txt"
if [ -s "$LIVE" ]; then
    grep -iE "$PATTERNS" "$LIVE" | sort -u > "$GREP_TXT" || : > "$GREP_TXT"
else
    : > "$GREP_TXT"
fi

GREP_JSON="$RDIR/grep-interesting.jsonl"
if [ -s "$JSON" ] && command -v jq >/dev/null 2>&1; then
    jq -c --arg p "$PATTERNS" \
        'select(
            (.url         // "" | test($p; "i")) or
            (.title       // "" | test($p; "i")) or
            ((.tech // []) | map(.) | join(" ") | test($p; "i"))
        )' "$JSON" | sort -u > "$GREP_JSON" || : > "$GREP_JSON"
else
    grep -iE "$PATTERNS" "$JSON" 2>/dev/null | sort -u > "$GREP_JSON" \
        || : > "$GREP_JSON"
fi

GREP_SUBS="$RDIR/grep-interesting-subdomains.txt"
grep -iE "$PATTERNS" "$RUN_DIR/chunk.txt" | sort -u > "$GREP_SUBS" \
    || : > "$GREP_SUBS"

echo "[*] Text hits:      $(wc -l < "$GREP_TXT")"
echo "[*] JSON hits:      $(wc -l < "$GREP_JSON")"
echo "[*] Subdomain hits: $(wc -l < "$GREP_SUBS")"

if [ -s "$GREP_TXT" ]; then
    echo ""
    echo "[*] Top 30 interesting (text):"
    head -n 30 "$GREP_TXT"
fi
'''
            }
        }

        // ── 6. Queue – NEWs und CHANGEDs in nuclei-queue.txt schreiben ───
        //  - Blacklist-Filter (Wildcards via *.domain.com)
        //  - sort -u verhindert Duplicates
        //  - nuclei-Job liest Queue eigenständig per Cron
        stage('Queue – feed nuclei') {
            steps {
                sh '''#!/usr/bin/env bash
set -euo pipefail

RDIR="$RUN_DIR/results"
NDIR="$STATE_BASE/nuclei"
mkdir -p "$NDIR"

QUEUE="$NDIR/nuclei-queue.txt"
NEW_URLS="$RDIR/httpx-new-urls.txt"
CHANGED_URLS="$RDIR/httpx-changed-urls.txt"

touch "$QUEUE"

# ── Kandidaten: NEWs + CHANGEDs ──────────────────────────────────────
CANDIDATES="$RDIR/nuclei-candidates.txt"
{
    cat "$NEW_URLS"    2>/dev/null || true
    cat "$CHANGED_URLS" 2>/dev/null || true
} | sort -u > "$CANDIDATES"

if [ ! -s "$CANDIDATES" ]; then
    echo "[*] No new/changed hosts — nothing to queue."
    exit 0
fi

echo "[*] Candidates before blacklist: $(wc -l < "$CANDIDATES")"

# ── Blacklist-Filter ──────────────────────────────────────────────────
# Parst NUCLEI_BLACKLIST (eine Regel pro Zeile)
# Wildcards: *.example.com  → matched alle subdomains
# Exakt:     sub.example.com → nur genau dieser host
BLACKLIST_FILE="$RDIR/nuclei-blacklist.txt"
printf '%s\n' "${NUCLEI_BLACKLIST}" \
    | sed 's/\r$//' \
    | grep -vE '^[[:space:]]*($|#)' \
    | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' \
    | sort -u > "$BLACKLIST_FILE" || true

FILTERED="$RDIR/nuclei-filtered.txt"
: > "$FILTERED"

if [ -s "$BLACKLIST_FILE" ]; then
    while IFS= read -r candidate; do
        [ -z "$candidate" ] && continue
        host="${candidate#http://}"; host="${host#https://}"; host="${host%%/*}"; host="${host%%:*}"
        blocked=false
        while IFS= read -r rule; do
            [ -z "$rule" ] && continue
            case "$rule" in
                *.*)
                    suffix="${rule#*.}"
                    host_lower="$(echo "$host" | tr '[:upper:]' '[:lower:]')"
                    suffix_lower="$(echo "$suffix" | tr '[:upper:]' '[:lower:]')"
                    if [ "$host_lower" = "$suffix_lower" ]; then blocked=true; break; fi
                    case "$host_lower" in
                        *."$suffix_lower") blocked=true; break ;;
                    esac
                    ;;
                *)
                    host_lower="$(echo "$host" | tr '[:upper:]' '[:lower:]')"
                    rule_lower="$(echo "$rule" | tr '[:upper:]' '[:lower:]')"
                    if [ "$host_lower" = "$rule_lower" ]; then blocked=true; break; fi
                    ;;
            esac
        done < "$BLACKLIST_FILE"
        if [ "$blocked" = "false" ]; then
            echo "$candidate" >> "$FILTERED"
        fi
    done < "$CANDIDATES"
else
    cp "$CANDIDATES" "$FILTERED"
fi

echo "[*] Candidates after blacklist:  $(wc -l < "$FILTERED")"

if [ ! -s "$FILTERED" ]; then
    echo "[*] All candidates blacklisted — nothing to queue."
    exit 0
fi

# ── In Queue schreiben, Duplicates verhindern ─────────────────────────
ADDED_BEFORE=$(wc -l < "$QUEUE")
{ cat "$QUEUE"; cat "$FILTERED"; } | sort -u > "${QUEUE}.tmp"
mv "${QUEUE}.tmp" "$QUEUE"
ADDED_AFTER=$(wc -l < "$QUEUE")
ADDED=$(( ADDED_AFTER - ADDED_BEFORE ))

echo "[*] Added to queue: $ADDED"
echo "[*] Queue total:    $(wc -l < "$QUEUE")"
'''
            }
        }

        // ── 7. Archive ────────────────────────────────────────────────────
        stage('Archive results') {
            steps {
                archiveArtifacts artifacts: [
                    'current-run/**/*.txt',
                    'current-run/**/*.jsonl'
                ].join(','), fingerprint: true
            }
        }
    }

    post {
        always { echo "[*] recon-httpx finished." }
        failure { echo "[!] recon-httpx FAILED." }
    }
}
