#!/usr/bin/env python3 # -*- coding: utf-8 -*- import argparse import re import sys import unicodedata from html import unescape from pathlib import Path from bs4 import BeautifulSoup def normalize_text(text: str) -> str: text = unicodedata.normalize("NFC", text) text = text.replace("\r\n", "\n").replace("\r", "\n") text = text.replace("\xa0", " ") text = text.replace("\u200b", "") text = re.sub(r"[ \t]+\n", "\n", text) text = re.sub(r"\n{3,}", "\n\n", text) return text.strip() + "\n" def extract_visible_text_from_html( html_text: str, prefer_pre: bool = True, keep_pre_linebreaks: bool = True, ) -> str: """ Extract visible text from HTML. For many GRETIL pages,

 contains the real text.
    Fallback is .
    """
    soup = BeautifulSoup(html_text, "lxml")

    # Remove non-content tags
    for tag in soup(["script", "style", "noscript"]):
        tag.decompose()

    node = None

    if prefer_pre:
        node = soup.find("pre")

    if node is None:
        node = soup.body

    if node is None:
        node = soup

    if keep_pre_linebreaks and node.name == "pre":
        text = node.get_text("\n")
    else:
        text = node.get_text("\n")

    text = unescape(text)
    text = normalize_text(text)
    return text


def maybe_strip_gretil_boilerplate(text: str) -> str:
    """
    Conservative cleanup only.
    Keeps the main text but removes obvious repeated web-page boilerplate.
    """
    lines = text.splitlines()

    bad_starts = (
        "GRETIL",
        "Göttingen Register of Electronic Texts",
        "Electronic text input",
        "Input by",
        "Proof-reading",
        "Proof reading",
        "Generated from",
        "This file is for reference purposes only",
        "Read the license",
        "Text converted to Unicode",
        "Version",
        "PROJECT",
        "COPYRIGHT",
    )

    cleaned = []
    for line in lines:
        s = line.strip()

        if not s:
            cleaned.append("")
            continue

        skip = False
        for p in bad_starts:
            if s.startswith(p):
                skip = True
                break

        if not skip:
            cleaned.append(line)

    out = "\n".join(cleaned)
    out = re.sub(r"\n{3,}", "\n\n", out)
    return out.strip() + "\n"


def process_one_file(
    src_file: Path,
    src_root: Path,
    dst_root: Path,
    prefer_pre: bool,
    strip_boilerplate: bool,
    output_ext: str,
) -> None:
    try:
        raw_html = src_file.read_text(encoding="utf-8", errors="replace")
    except Exception as e:
        print(f"SKIP read error: {src_file} -> {e}")
        return

    roman_text = extract_visible_text_from_html(
        raw_html,
        prefer_pre=prefer_pre,
        keep_pre_linebreaks=True,
    )

    if strip_boilerplate:
        roman_text = maybe_strip_gretil_boilerplate(roman_text)

    rel = src_file.relative_to(src_root)
    out_dir = dst_root / rel.parent
    out_dir.mkdir(parents=True, exist_ok=True)

    base = src_file.stem
    out_file = out_dir / f"{base}{output_ext}"

    try:
        out_file.write_text(roman_text, encoding="utf-8", newline="\n")
        print(f"OK  : {src_file} -> {out_file}")
    except Exception as e:
        print(f"SKIP write error: {out_file} -> {e}")


def main():
    parser = argparse.ArgumentParser(
        description="Batch-convert local GRETIL HTML files to Roman UTF-8 text files."
    )
    parser.add_argument("src_root", help="Root folder containing .htm/.html files")
    parser.add_argument("dst_root", help="Output root folder for Roman text files")
    parser.add_argument(
        "--prefer-pre",
        action="store_true",
        help="Prefer text inside  if present (recommended for GRETIL)"
    )
    parser.add_argument(
        "--strip-boilerplate",
        action="store_true",
        help="Remove some obvious repeated GRETIL/web-page boilerplate"
    )
    parser.add_argument(
        "--output-ext",
        default=".roman.txt",
        help="Output file extension, default: .roman.txt"
    )

    args = parser.parse_args()

    src_root = Path(args.src_root).resolve()
    dst_root = Path(args.dst_root).resolve()

    if not src_root.exists():
        print(f"ERROR: source folder does not exist: {src_root}")
        sys.exit(1)

    html_files = []
    html_files.extend(src_root.rglob("*.htm"))
    html_files.extend(src_root.rglob("*.html"))
    html_files = sorted(set(html_files))

    if not html_files:
        print(f"ERROR: no .htm/.html files found under: {src_root}")
        sys.exit(1)

    print(f"Found {len(html_files)} HTML files.")
    print(f"Source : {src_root}")
    print(f"Output : {dst_root}")

    for f in html_files:
        process_one_file(
            src_file=f,
            src_root=src_root,
            dst_root=dst_root,
            prefer_pre=args.prefer_pre,
            strip_boilerplate=args.strip_boilerplate,
            output_ext=args.output_ext,
        )

    print("Done.")


if __name__ == "__main__":
    main()