#!/usr/bin/env python3
"""
find_untranslated_terms.py (v2)

Scan a directory tree of .po files with polib and find msgstr entries that
still contain a given English technical term verbatim (whole word, case
insensitive) in PROSE, even though the corresponding msgid also contains
that term in prose.

This version is markup-aware: Sphinx roles (:class:`Foo`, :func:`bar`,
:mod:`os`, ...), inline code spans (`...`, ``...``), literal/code blocks,
and quoted code-ish snippets are stripped out before matching, so words
that are only appearing as ROLE NAMES or inside CODE (e.g. "class" in
":class:`bool`", "return"/"raise"/"import" inside a doctest) are not
counted as "untranslated prose".

Fixes vs the first version:
  - .git (and other VCS/venv/cache dirs) is excluded from the file walk.
  - Sphinx role markup is stripped, not just backticks, which is why
    "class", "api", "import", "return", "value", "type" etc. were wildly
    inflated before (those words double as role names / Python keywords
    that show up constantly in code, not prose).
  - Full per-term hit lists are always written out (JSON/CSV); the
    console summary intentionally still only *shows* a few examples
    per term, since printing 5000+ lines to a terminal isn't useful --
    use --json/--csv to get everything.

Requires: polib  (pip install polib --break-system-packages)
"""

import argparse
import csv
import json
import re
import sys
from collections import defaultdict
from pathlib import Path

try:
    import polib
except ImportError:
    sys.exit(
        "This script requires polib. Install with: pip install polib --break-system-packages"
    )


TERMS = [
    "heap",
    "type",
    "value",
    "async",
    "wildcard",
    "index",
    "property",
    "bootstrapping",
    "mock",
    "pipe",
    "docstring",
    "object",
    "action",
    "local",
    "escape",
    "raise",
    "return",
    "list",
    "operator",
    "element",
    "import",
    "encoding",
    "global",
    "string",
    "class",
    "module",
    "function",
    "shell",
    "exception",
    "event",
    "coroutine",
    "interface",
    "cache",
    "command line",
    "package",
    "method",
    "widget",
    "symlink",
    "item",
    "generator",
    "loop",
    "runtime",
    "built-in",
    "namespace",
    "syntax",
    "argument",
    "wrapper",
    "load",
    "attribute",
    "thread",
    "api",
    "variable",
    "expression",
    "f-string",
    "callback",
]

EXCLUDE_DIRS = {".git", ".venv", "venv", "node_modules", "__pycache__", ".tox"}

# ---------------------------------------------------------------------------
# Markup stripping (adapted from check_consistency.py's approach)
# ---------------------------------------------------------------------------

# Sphinx/reST roles whose *content* is code/identifiers, not prose.
CODE_ROLES = {
    "func",
    "meth",
    "mod",
    "class",
    "data",
    "const",
    "attr",
    "exc",
    "obj",
    "command",
    "cmdoption",
    "envvar",
    "file",
    "kbd",
    "option",
    "program",
    "regexp",
    "makevar",
    "dunder",
    "module",
    "method",
    "exception",
    "ref",
    "doc",
    "download",
    "env",
    "pep",
    "rfc",
    "issue",
    "source",
    "mimetype",
    "keyword",
    "literal",
    "token",
    "grammar",
    "confval",
    "setting",
}

ROLE_RE = re.compile(r":([\w+-]+):`([^`]*)`")
DOUBLE_BACKTICK_RE = re.compile(r"``.*?``", re.DOTALL)
SINGLE_BACKTICK_RE = re.compile(r"`([^`]*)`")
SUBREF_RE = re.compile(r"\|[\w.-]+\|")
URL_RE = re.compile(r"https?://\S+")
EMPH_RE = re.compile(r"(\*\*?)(.+?)\1")
TARGET_RE = re.compile(r"^(.*)\s$")


def strip_markup(text, keep_prose_roles=True):
    """Remove code spans / role markup; keep the *display text* of prose
    roles like :term:`Foo` or :ref:`title ` since translators do
    translate that part."""

    def role_repl(m):
        role, body = m.group(1).lower(), m.group(2)
        if role in CODE_ROLES:
            return " "
        if not keep_prose_roles:
            return " "
        t = TARGET_RE.match(body)
        return t.group(1) if t else body

    text = DOUBLE_BACKTICK_RE.sub(" ", text)
    text = ROLE_RE.sub(role_repl, text)
    text = SINGLE_BACKTICK_RE.sub(r" \1 ", text)
    text = SUBREF_RE.sub(" ", text)
    text = URL_RE.sub(" ", text)
    text = EMPH_RE.sub(r"\2", text)
    return text


WORD_RE = re.compile(r"[A-Za-z][A-Za-z\-]*[A-Za-z]|[A-Za-z]")
QUOTED_CODE_RE = re.compile(r"'([\w.\-/ ]+)'|\"([\w.\-/ ]+)\"")


def quoted_span_is_code(content):
    words = content.split()
    return (
        len(words) == 1
        or any(c in content for c in ".-_")
        or content.startswith(("import ", "from "))
    )


def strip_quoted_code(text):
    def repl(m):
        content = m.group(1) or m.group(2)
        return " " if quoted_span_is_code(content) else m.group(0)

    return QUOTED_CODE_RE.sub(repl, text)


CODE_LINE_START_RE = re.compile(
    r"^(?:import|from)\s" r"|^[a-z_][\w.]*(?:\[[^\]]*\])?\s*[=(]" r"|^[a-z_][\w.]*\("
)
CODE_STMT_RE = re.compile(
    r"^(?:if|elif|else|for|while|def|class|try|except|finally|with|return"
    r"|raise|break|continue|lambda|async|await)\b.*:\s*(?:#.*)?$"
)


def looks_codeish(line):
    if line.startswith((">>>", "...", "$", "#")):
        return True
    if line.startswith(("'", '"', "[", "{", ">>", "...", "#", "$"))
        or ln.startswith((" ", "\t"))
    )
    return codeish / len(lines) >= 0.8


def standalone_code_entry(msgid):
    lines = [ln for ln in msgid.split("\n") if ln.strip()]
    if not lines:
        return False
    codeish = sum(1 for ln in lines if ln[0] in " \t" or looks_codeish(ln.strip()))
    prose = sum(1 for ln in lines if prose_line(ln))
    return codeish >= max(1, len(lines) // 2) and prose = 0.8


# Bare code fragments embedded directly in prose with NO backticks/roles at
# all, e.g. "...(e.g. list.index()) but functions for other (e.g. len(list))"
# or "using ``list(dictview)``" -- covers dotted-call (name.method(...)),
# bare call (name(...)), and subscript/dunder-ish tokens.
BARE_DOTTED_CALL_RE = re.compile(r"\b[A-Za-z_][\w]*(?:\.[A-Za-z_][\w]*)+\(\)?")
BARE_CALL_RE = re.compile(r"\b[A-Za-z_][\w]*\([^()\s]{0,40}\)")


def strip_bare_code_fragments(text):
    text = BARE_DOTTED_CALL_RE.sub(" ", text)
    text = BARE_CALL_RE.sub(" ", text)
    return text


def clean_prose(msgid):
    """Full pipeline: literal blocks -> markup -> quoted code -> bare code."""
    text = strip_literal_blocks(msgid)
    text = strip_markup(text)
    text = strip_quoted_code(text)
    text = strip_bare_code_fragments(text)
    return text


# ---------------------------------------------------------------------------
# Term matching
# ---------------------------------------------------------------------------


def build_term_pattern(term):
    escaped = re.escape(term)
    # spaces/hyphens interchangeable, optional plural
    parts = [re.escape(p) for p in re.split(r"[\s\-]+", term) if p]
    body = r"[\s\-]+".join(parts)
    return re.compile(
        rf"(?