#!/usr/bin/env python3
"""HSEQ Scout — Full Content Scanner v3
Scrapet alle bronnen, hasht content, vergelijkt met vorige run.
Pure Python — geen bash quoting hell.
"""
import json, hashlib, re, os, sys
from datetime import datetime
from urllib.request import urlopen, Request
from urllib.error import URLError, HTTPError

SOURCES_FILE = "/root/projects/jg/HSEQ/2026-pbm-HSEQ_SCOUT/working/sources.json"
CACHE_FILE   = "/root/projects/jg/HSEQ/2026-pbm-HSEQ_SCOUT/working/last-check.json"
OUTPUT_FILE  = "/root/projects/jg/HSEQ/2026-pbm-HSEQ_SCOUT/working/scout-result.json"

def fetch(url):
    """Fetch URL and return (hash, snippet) or (None, error_msg)."""
    try:
        req = Request(url, headers={
            "User-Agent": "Mozilla/5.0 (compatible; HSEQ-Scout/1.0)",
            "Accept": "text/html,application/xhtml+xml",
            "Accept-Language": "nl,en;q=0.9",
        })
        with urlopen(req, timeout=15) as resp:
            html = resp.read().decode("utf-8", errors="replace")
        if len(html) < 50:
            return None, "Empty response"
        # Strip HTML
        text = re.sub(r'<(script|style|nav|footer|header|aside)[^>]*>.*?</\1>', '', html, flags=re.DOTALL|re.IGNORECASE)
        text = re.sub(r'<[^>]+>', ' ', text)
        text = re.sub(r'\s+', ' ', text).strip()
        if len(text) < 20:
            return None, "No text content"
        snippet = text[:400]
        h = hashlib.sha256(text[:3000].encode()).hexdigest()[:16]
        return h, snippet
    except (URLError, HTTPError, TimeoutError, OSError) as e:
        return None, str(e)[:100]

def main():
    # Load sources
    with open(SOURCES_FILE) as f:
        config = json.load(f)
    sources = [s for s in config.get("sources", []) if s.get("enabled", True)]

    # Load previous cache
    prev = {}
    if os.path.exists(CACHE_FILE):
        try:
            with open(CACHE_FILE) as f:
                prev = json.load(f)
        except:
            pass

    print(f"Scanning {len(sources)} sources (cache: {len(prev)} entries)...", file=sys.stderr)

    changes = []
    new_cache = {}

    for s in sources:
        sid = s["id"]
        sname = s.get("name", sid)
        surl = s["url"]

        h, snippet = fetch(surl)

        if h is None:
            status = "error"
            h = "error"
        elif sid not in prev:
            status = "new"
        elif prev[sid].get("hash") == h:
            status = "unchanged"
        else:
            status = "changed"

        new_cache[sid] = {"hash": h, "name": sname, "url": surl}

        if status != "unchanged":
            changes.append({
                "id": sid, "name": sname, "url": surl,
                "status": status, "hash": h,
                "snippet": snippet or ""
            })

        print(f"  [{status}] {sid}", file=sys.stderr)

    # Build result
    result = {
        "scan_date": datetime.now().isoformat(),
        "total_sources": len(sources),
        "changed": len(changes),
        "unchanged": len(sources) - len(changes),
        "changes": changes
    }

    # Save
    with open(OUTPUT_FILE, "w") as f:
        json.dump(result, f, indent=2, ensure_ascii=False)
    with open(CACHE_FILE, "w") as f:
        json.dump(new_cache, f, indent=2, ensure_ascii=False)

    # Stdout = JSON for the cron LLM to consume
    print(json.dumps(result, indent=2, ensure_ascii=False))

if __name__ == "__main__":
    main()