#!/usr/bin/env python3 """Batch process all source documents and generate structured summaries.""" import os import docx from PyPDF2 import PdfReader OUTDIR = "/root/projects/jg/2026-reach-phoenix-metals/working/bronnen_data" REACH = "/root/Documents/HSEQ_kennis/Interne_Procedures/reach" PM = "/root/Documents/HSEQ_kennis/Phoenixmetals" def read_docx(path): try: doc = docx.Document(path) return "\n".join(p.text for p in doc.paragraphs if p.text.strip()) except Exception as e: return f"ERROR reading DOCX: {e}" def read_pdf(path, max_pages=40): try: reader = PdfReader(path) texts = [] for i, page in enumerate(reader.pages[:max_pages]): t = page.extract_text() if t: texts.append(t) total = len(reader.pages) return "\n".join(texts), total except Exception as e: return f"ERROR reading PDF: {e}", 0 def read_xlsx(path): try: import openpyxl wb = openpyxl.load_workbook(path, data_only=True) rows = [] for ws in wb.worksheets: for row in ws.iter_rows(max_row=100, values_only=True): cells = [str(c) if c is not None else "" for c in row] if any(cells): rows.append(" | ".join(cells)) return "\n".join(rows) except Exception as e: # Try pandas try: import pandas as pd dfs = pd.read_excel(path, sheet_name=None, nrows=100) texts = [] for name, df in dfs.items(): texts.append(f"=== Sheet: {name} ===") texts.append(df.to_string()) return "\n".join(texts) except Exception as e2: return f"ERROR reading XLSX: {e}; {e2}" def write_summary(filename, source_path, file_type, size_kb, content, extraction_hints=""): """Write a structured summary file.""" # Truncate content for the summary content_preview = content[:4000] if len(content) > 4000 else content summary = f"""# {filename.replace('.md','').split('_',1)[-1].replace('_',' ').title()} ā€” Bronsamenvatting ## Bron - Bestand: {source_path} - Type: {file_type} - Grootte: {size_kb} KB ## Kerninhoud {content_preview} ## Relevante data voor Phoenix Metals [Wordt ingevuld na kruisanalyse] ## Key artikelen / paragrafen {extraction_hints} ## Kruisverwijzingen [Wordt ingevuld na kruisanalyse] """ outpath = os.path.join(OUTDIR, filename) with open(outpath, 'w') as f: f.write(summary) print(f" āœ“ {filename} ({len(content)} chars)") # Define all sources sources = [ # REACH sources ("01_reach_basis.md", f"{REACH}/reach basis.docx", "DOCX"), ("02_registration_nl.md", f"{REACH}/registration_nl.pdf", "PDF"), ("03_substance_id_nl.md", f"{REACH}/substance_id_nl.pdf", "PDF"), ("04_svhc_en.md", f"{REACH}/svhc_en.pdf", "PDF"), ("05_sds_nl.md", f"{REACH}/sds_nl.pdf", "PDF"), ("06_guide_chemical_safety_sme.md", f"{REACH}/guide_chemical_safety_sme_nl.pdf", "PDF"), ("07_sip_guide.md", f"{REACH}/practical_guide_how_to_develop_prepare_sip_nl.pdf", "PDF"), ("08_clp_labelling.md", f"{REACH}/clp_labelling_en.pdf", "PDF"), ("09_duurzaamheid.md", f"{REACH}/du_practical_guide_13_nl.pdf", "PDF"), ("10_gezondheidsbeoordeling.md", f"{REACH}/pg_15_qualitative-human_health_assessment_documenting_en.pdf", "PDF"), ("11_qsars.md", f"{REACH}/pg_report_qsars_nl.pdf", "PDF"), ("12_clh_dossier.md", f"{REACH}/how_to_submit_clh_dossier_en.pdf", "PDF"), ("13_data_sharing.md", f"{REACH}/guidance_on_data_sharing_nl.pdf", "PDF"), ("14_autorisatie.md", f"{REACH}/apply_for_authorisation_en.pdf", "PDF"), ("15_clp_basisgids.md", f"{REACH}/Basisgids Chemische Veiligheid_ Begrijpen en Werken met CLP.docx", "DOCX"), # Phoenix Metals sources ("16_pm_bedrijfsomschrijving.md", f"{PM}/phoenix metals.docx", "DOCX"), ("17_pm_voorbereiding.md", f"{PM}/PhoenixMetals voorbereiding.xlsx", "XLSX"), ("18_pm_pva.md", f"{PM}/Draft PvA opstart Phoenix (1).docx", "DOCX"), ("19_pm_haalbaarheid.md", f"{PM}/Business plan - Haalbaarheids studie/Haalbaarheidsstudie Phoenix.pdf", "PDF"), ("20_pm_bfd.md", f"{PM}/algemene omschrijving process/1. Block Flow Diagram.pdf", "PDF"), ("21_pm_pfd.md", f"{PM}/algemene omschrijving process/2. PFD.pdf", "PDF"), ("22_pm_layout.md", f"{PM}/algemene omschrijving process/3. Plant Layout.pdf", "PDF"), ("23_pm_demo_process.md", f"{PM}/algemene omschrijving process/Demo Process VR (1).docx", "DOCX"), ("24_pm_demo_vr.md", f"{PM}/Demonstrator/Demo Process VR.docx", "DOCX"), ("25_pm_demo_layout.md", f"{PM}/Demonstrator/Demo Layout.pdf", "PDF"), ("26_pm_chemicals_master.md", f"{PM}/Gevaarlijke stoffen/Phoenix Metals 2024/Registration Chemicals - MASTER (versie 2.1.0) - beveiligd.xlsx", "XLSX"), ("27_msds_vanadium.md", f"{PM}/Gevaarlijke stoffen/MSDS/vanadium concentrate Versie 1-nl.pdf", "PDF"), ("28_msds_zwavelzuur.md", f"{PM}/Gevaarlijke stoffen/MSDS/Sulfuric acid_ 25-75% Versie 1-en.pdf", "PDF"), ("29_msds_naoh.md", f"{PM}/Gevaarlijke stoffen/MSDS/Sodium hydroxide 50% Versie 1-en.pdf", "PDF"), ("30_pm_rie_gevaarlijk.md", f"{PM}/Gevaarlijke stoffen/Rapportage verdiepende RI&E 2021 gevaarlijke stoffen_V3.0.docx", "DOCX"), ("31_pm_gevaarlijke_stoffen_procedure.md", f"{PM}/Gevaarlijke stoffen/PE-240-01-NL_V1 Aanvraag, beoordeling en goedkeuring van gevaarlijke stoffen.docx", "DOCX"), ("32_pm_hse_policy.md", f"{PM}/HSE policy/Phoenix metals HSE policy 2025/Phoenix Metals HSE policy-V1.docx", "DOCX"), ("33_pm_bnp.md", f"{PM}/Bedrijfsnoodplan/Phoenix metals BNP 2024/Phoenix Metals BNP-V1.docx", "DOCX"), ("34_pm_pbm_matrix.md", f"{PM}/PBM's/Phoenix Metals 2024/PBM matrix/PBM matrix Phoenix metals draft 1.xlsm", "XLSX"), ("35_pm_pbm_beleid.md", f"{PM}/PBM's/Phoenix Metals 2024/PS-201-NL_V1.0 - PBM beleid.docx", "DOCX"), ("36_pm_rie_inventarisatie.md", f"{PM}/RI&E/instrumenten.rie/Inventarisatie PhoenixMetals (1).docx", "DOCX"), ("37_pm_rie_mkb.md", f"{PM}/RI&E/instrumenten.rie/Inventarisatie PhoenixMetals MKB.docx", "DOCX"), ("38_pm_rie_actionplan.md", f"{PM}/RI&E/RIE Action plan and follow up.docx", "DOCX"), ("39_pm_vooroverleg.md", f"{PM}/Vergunning/2025-09-23, verslag vooroverleg.docx", "DOCX"), ("40_pm_haalbaarheid_rapport.md", f"{PM}/Vergunning/Rapport haalbaarheidsstudie V04.docx", "DOCX"), ("41_pm_uitgangspunten.md", f"{PM}/Vergunning/Uitgangspuntendocument Phoenix Metals IJmuiden.xlsx", "XLSX"), ("42_pm_fact_sheet_ash.md", f"{PM}/Info Ash/Fact sheet Ash 2025.docx", "DOCX"), ("43_pm_hse_pilots.md", f"{PM}/Info Ash/HSE voorschriften Pilots.pdf", "PDF"), ("44_pm_legal_compliance.md", f"{PM}/compliance/Legal compliance Phoenix Metals 2025.pptx", "PPTX"), ("45_pm_milieu_wetgeving.md", f"{PM}/compliance/Register Relevante Milieu Wet- en Regelgeving.pdf", "PDF"), ("46_pm_compliance_register.md", f"{PM}/compliance/Environmental Aspects and Compliance Register.xlsx", "XLSX"), ("47_pm_brzo_overzicht.md", f"{PM}/document control/overzicht RELATIE BRZO ELEMENTEN EN VBS 3.xlsx", "XLSX"), ("48_pm_moc.md", f"{PM}/Management of change/Phoenix metals 2024/PE-202-01_V1 Management of Change.docx", "DOCX"), ("49_pm_document_control.md", f"{PM}/document control/PE-210-04-NL_V14 Opstellen wijzigen goedkeuren en archiveren van voorschrijvende documenten van het managementsysteem.pdf", "PDF"), ("50_pm_tra_hulplijst.md", f"{PM}/project risk assessment/HS-F-0003 Taak Risico Analyse met hulplijst.docx", "DOCX"), ("51_pm_tra_procedure.md", f"{PM}/project risk assessment/HS-P-0008 Taak Risico Analyse.docx", "DOCX"), ("52_pm_pssr.md", f"{PM}/PSSR/HS-P-0018 Procedure PSSR.docx", "DOCX"), ("53_pm_training_chemicals.md", f"{PM}/trainingen/Exposure to hazardous chemicals Phoenix Metals 2025.pptx", "PPTX"), ("54_pm_training_ergonomics.md", f"{PM}/trainingen/Ergonomics Phoenix Metals 2025.pptx", "PPTX"), ("55_pm_opleiding_matrix.md", f"{PM}/competence management/Phoenix Metals 2024/Phoenix metals opleiding matrix 2024.xlsx", "XLSX"), ] print(f"Processing {len(sources)} sources...") for filename, path, ftype in sources: print(f"\n[{filename}] {ftype}: {path}") if not os.path.exists(path): print(f" āœ— FILE NOT FOUND") write_summary(filename, path, ftype, "?", "BESTAND NIET GEVONDEN") continue size_kb = os.path.getsize(path) // 1024 if ftype == "DOCX": content = read_docx(path) write_summary(filename, path, ftype, size_kb, content) elif ftype == "PDF": content, pages = read_pdf(path) write_summary(filename, path, ftype, size_kb, content) elif ftype == "XLSX": content = read_xlsx(path) write_summary(filename, path, ftype, size_kb, content) elif ftype == "PPTX": try: from pptx import Presentation prs = Presentation(path) texts = [] for slide in prs.slides: for shape in slide.shapes: if hasattr(shape, "text") and shape.text.strip(): texts.append(shape.text) content = "\n".join(texts) write_summary(filename, path, ftype, size_kb, content) except Exception as e: write_summary(filename, path, ftype, size_kb, f"ERROR reading PPTX: {e}") print("\n\nāœ… All sources processed!")