#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import argparse
import re
import sys
import unicodedata
from html import unescape
from pathlib import Path

from bs4 import BeautifulSoup

# Import the transliterator from the earlier script
# That file must be in the same folder as this batch script
try:
    from gretil_iast_to_deva import transliterate_text
except Exception as e:
    print("ERROR: could not import transliterate_text from gretil_iast_to_deva.py")
    print("Keep gretil_iast_to_deva.py in the same folder as this batch script.")
    print(f"Import error: {e}")
    sys.exit(1)


def normalize_text(text: str) -> str:
    text = unicodedata.normalize("NFC", text)
    text = text.replace("\r\n", "\n").replace("\r", "\n")
    text = text.replace("\xa0", " ")
    text = re.sub(r"[ \t]+\n", "\n", text)
    text = re.sub(r"\n{3,}", "\n\n", text)
    return text.strip() + "\n"


def extract_visible_text_from_html(html_text: str, prefer_pre: bool = True) -> str:
    """
    Extract visible text from an HTML file.
    For GRETIL-like files, <pre> is often the best source.
    Fallback: body text.
    """
    soup = BeautifulSoup(html_text, "lxml")

    # Remove non-content tags
    for tag in soup(["script", "style", "noscript"]):
        tag.decompose()

    node = None

    if prefer_pre:
        node = soup.find("pre")

    if node is None:
        node = soup.body

    if node is None:
        node = soup

    text = node.get_text("\n")
    text = unescape(text)
    text = normalize_text(text)
    return text


def maybe_strip_gretil_boilerplate(text: str) -> str:
    """
    Conservative cleanup only.
    This does NOT try to guess the true beginning/end of the work.
    It only removes some repeated web-page boilerplate if present.
    """
    lines = text.splitlines()

    bad_starts = (
        "GRETIL",
        "Göttingen Register of Electronic Texts",
        "Electronic text input",
        "Input by",
        "Proof-reading",
        "Generated from",
        "This file is for reference purposes only",
        "Read the license",
    )

    cleaned = []
    for line in lines:
        s = line.strip()
        if not s:
            cleaned.append("")
            continue

        skip = False
        for p in bad_starts:
            if s.startswith(p):
                skip = True
                break

        if not skip:
            cleaned.append(line)

    out = "\n".join(cleaned)
    out = re.sub(r"\n{3,}", "\n\n", out)
    return out.strip() + "\n"


def process_one_file(
    src_file: Path,
    src_root: Path,
    dst_root: Path,
    prefer_pre: bool,
    strip_gretil_tags: bool,
    devanagari_digits: bool,
    save_roman: bool,
    strip_boilerplate: bool,
) -> None:
    try:
        raw_html = src_file.read_text(encoding="utf-8", errors="replace")
    except Exception as e:
        print(f"SKIP read error: {src_file} -> {e}")
        return

    roman_text = extract_visible_text_from_html(raw_html, prefer_pre=prefer_pre)

    if strip_boilerplate:
        roman_text = maybe_strip_gretil_boilerplate(roman_text)

    dev_text = transliterate_text(
        roman_text,
        strip_gretil_tags=strip_gretil_tags,
        devanagari_digits=devanagari_digits,
    )

    rel = src_file.relative_to(src_root)
    out_dir = dst_root / rel.parent
    out_dir.mkdir(parents=True, exist_ok=True)

    base = src_file.stem

    if save_roman:
        roman_out = out_dir / f"{base}.roman.txt"
        roman_out.write_text(roman_text, encoding="utf-8", newline="\n")

    dev_out = out_dir / f"{base}.dev.txt"
    dev_out.write_text(dev_text, encoding="utf-8", newline="\n")

    print(f"OK  : {src_file} -> {dev_out}")


def main():
    parser = argparse.ArgumentParser(
        description="Batch-convert local GRETIL HTML files to Devanagari text files."
    )
    parser.add_argument("src_root", help="Root folder containing .htm/.html files")
    parser.add_argument("dst_root", help="Output root folder for .dev.txt files")
    parser.add_argument(
        "--prefer-pre",
        action="store_true",
        help="Prefer text inside <pre> if present (recommended for GRETIL)"
    )
    parser.add_argument(
        "--strip-gretil-tags",
        action="store_true",
        help="Remove markers like /AP_291.001ab/"
    )
    parser.add_argument(
        "--devanagari-digits",
        action="store_true",
        help="Convert ASCII digits 0-9 to Devanagari digits"
    )
    parser.add_argument(
        "--save-roman",
        action="store_true",
        help="Also save extracted Roman text as .roman.txt"
    )
    parser.add_argument(
        "--strip-boilerplate",
        action="store_true",
        help="Remove some obvious repeated GRETIL/web-page boilerplate"
    )

    args = parser.parse_args()

    src_root = Path(args.src_root).resolve()
    dst_root = Path(args.dst_root).resolve()

    if not src_root.exists():
        print(f"ERROR: source folder does not exist: {src_root}")
        sys.exit(1)

    html_files = []
    html_files.extend(src_root.rglob("*.htm"))
    html_files.extend(src_root.rglob("*.html"))
    html_files = sorted(set(html_files))

    if not html_files:
        print(f"ERROR: no .htm/.html files found under: {src_root}")
        sys.exit(1)

    print(f"Found {len(html_files)} HTML files.")
    print(f"Source : {src_root}")
    print(f"Output : {dst_root}")

    for f in html_files:
        process_one_file(
            src_file=f,
            src_root=src_root,
            dst_root=dst_root,
            prefer_pre=args.prefer_pre,
            strip_gretil_tags=args.strip_gretil_tags,
            devanagari_digits=args.devanagari_digits,
            save_roman=args.save_roman,
            strip_boilerplate=args.strip_boilerplate,
        )

    print("Done.")


if __name__ == "__main__":
    main()