#!/bin/bash
# HSEQ Scout — Full Content Scanner v2
# Scrapet alle 20 bronnen met curl, slaat content hashes op, vergelijkt met vorige run

SOURCES="/root/projects/jg/2026-pbm-HSEQ_SCOUT/working/sources.json"
CACHE="/root/projects/jg/2026-pbm-HSEQ_SCOUT/working/last-check.json"
OUTPUT="/root/projects/jg/2026-pbm-HSEQ_SCOUT/working/scout-result.json"

# Extract sources
mapfile -t SOURCE_LINES < <(python3 -c "
import json
with open('$SOURCES') as f:
    d = json.load(f)
for s in d.get('sources', []):
    if s.get('enabled', True):
        print(f\"{s['id']}||{s['name']}||{s['url']}\")
")

# Load previous hashes
PREV='{}'
[ -f "$CACHE" ] && PREV=$(cat "$CACHE")

echo "Scanning ${#SOURCE_LINES[@]} sources..."

# Process each source
RESULTS_JSON='[]'
NEW_CACHE='{}'

for line in "${SOURCE_LINES[@]}"; do
    IFS='||' read -r sid sname surl <<< "$line"
    [ -z "$surl" ] && continue
    
    # Fetch + hash
    fetched=$(curl -sL --max-time 15 \
        -H "User-Agent: Mozilla/5.0 (compatible; HSEQ-Scout/1.0)" \
        "$surl" 2>/dev/null | \
        python3 -c "
import sys, re, hashlib
html = sys.stdin.read()
if not html or len(html) < 50:
    print('ERROR')
    sys.exit()
text = re.sub(r'<(script|style|nav|footer|header)[^>]*>.*?</\1>', '', html, flags=re.DOTALL|re.IGNORECASE)
text = re.sub(r'<[^>]+>', ' ', text)
text = re.sub(r'\s+', ' ', text).strip()
snippet = text[:500]
h = hashlib.sha256(text[:2000].encode()).hexdigest()[:16]
# Output hash TAB snippet (single tab = clean delimiter)
print(h + '\t' + snippet)
" 2>/dev/null)
    
    hash=$(echo "$fetched" | cut -f1)
    snippet=$(echo "$fetched" | cut -f2-)
    
    if [ -z "$hash" ] || [ "$hash" = "ERROR" ]; then
        hash="error"
        snippet="Fetch failed"
        status="error"
    else
        # Compare with previous
        prev_hash=$(echo "$PREV" | python3 -c "
import json,sys
try:
    d=json.load(sys.stdin)
    print(d.get('$sid',{}).get('hash',''))
except: print('')
" 2>/dev/null)
        
        if [ -z "$prev_hash" ]; then
            status="new"
        elif [ "$hash" = "$prev_hash" ]; then
            status="unchanged"
        else
            status="changed"
        fi
    fi
    
    # Accumulate via python (avoids bash JSON hell)
    RESULTS_JSON=$(python3 -c "
import json, sys
r = json.loads('''$RESULTS_JSON''')
if '$status' != 'unchanged':
    r.append({'id':'$sid','name':'$sname','url':'$surl','status':'$status','hash':'$hash','snippet':$(python3 -c "import json; print(json.dumps('$snippet'[:300]))")})
print(json.dumps(r))
" 2>/dev/null)
    
    NEW_CACHE=$(python3 -c "
import json, sys
d = json.loads('''$NEW_CACHE''')
d['$sid'] = {'hash': '$hash', 'name': '$sname', 'url': '$surl'}
print(json.dumps(d))
" 2>/dev/null)
    
    echo "  [$status] $sid"
done

# Final output
python3 -c "
import json
from datetime import datetime

cache = json.loads('''$NEW_CACHE''')
changes = json.loads('''$RESULTS_JSON''')

result = {
    'scan_date': datetime.now().isoformat(),
    'total_sources': len(cache),
    'changed': len(changes),
    'unchanged': len(cache) - len(changes),
    'changes': changes
}

with open('$OUTPUT', 'w') as f:
    json.dump(result, f, indent=2, ensure_ascii=False)
with open('$CACHE', 'w') as f:
    json.dump(cache, f, indent=2, ensure_ascii=False)

print(json.dumps(result, indent=2, ensure_ascii=False))
"