GitHub Viewer
#!/usr/bin/env python3
"""
find_untranslated_terms.py (v2)
Scan a directory tree of .po files with polib and find msgstr entries that
still contain a given English technical term verbatim (whole word, case
insensitive) in PROSE, even though the corresponding msgid also contains
that term in prose.
This version is markup-aware: Sphinx roles (:class:`Foo`, :func:`bar`,
:mod:`os`, ...), inline code spans (`...`, ``...``), literal/code blocks,
and quoted code-ish snippets are stripped out before matching, so words
that are only appearing as ROLE NAMES or inside CODE (e.g. "class" in
":class:`bool`", "return"/"raise"/"import" inside a doctest) are not
counted as "untranslated prose".
Fixes vs the first version:
- .git (and other VCS/venv/cache dirs) is excluded from the file walk.
- Sphinx role markup is stripped, not just backticks, which is why
"class", "api", "import", "return", "value", "type" etc. were wildly
inflated before (those words double as role names / Python keywords
that show up constantly in code, not prose).
- Full per-term hit lists are always written out (JSON/CSV); the
console summary intentionally still only *shows* a few examples
per term, since printing 5000+ lines to a terminal isn't useful --
use --json/--csv to get everything.
Requires: polib (pip install polib --break-system-packages)
"""
import argparse
import csv
import json
import re
import sys
from collections import defaultdict
from pathlib import Path
try:
import polib
except ImportError:
sys.exit(
"This script requires polib. Install with: pip install polib --break-system-packages"
)
TERMS = [
"heap",
"type",
"value",
"async",
"wildcard",
"index",
"property",
"bootstrapping",
"mock",
"pipe",
"docstring",
"object",
"action",
"local",
"escape",
"raise",
"return",
"list",
"operator",
"element",
"import",
"encoding",
"global",
"string",
"class",
"module",
"function",
"shell",
"exception",
"event",
"coroutine",
"interface",
"cache",
"command line",
"package",
"method",
"widget",
"symlink",
"item",
"generator",
"loop",
"runtime",
"built-in",
"namespace",
"syntax",
"argument",
"wrapper",
"load",
"attribute",
"thread",
"api",
"variable",
"expression",
"f-string",
"callback",
]
EXCLUDE_DIRS = {".git", ".venv", "venv", "node_modules", "__pycache__", ".tox"}
# ---------------------------------------------------------------------------
# Markup stripping (adapted from check_consistency.py's approach)
# ---------------------------------------------------------------------------
# Sphinx/reST roles whose *content* is code/identifiers, not prose.
CODE_ROLES = {
"func",
"meth",
"mod",
"class",
"data",
"const",
"attr",
"exc",
"obj",
"command",
"cmdoption",
"envvar",
"file",
"kbd",
"option",
"program",
"regexp",
"makevar",
"dunder",
"module",
"method",
"exception",
"ref",
"doc",
"download",
"env",
"pep",
"rfc",
"issue",
"source",
"mimetype",
"keyword",
"literal",
"token",
"grammar",
"confval",
"setting",
}
ROLE_RE = re.compile(r":([\w+-]+):`([^`]*)`")
DOUBLE_BACKTICK_RE = re.compile(r"``.*?``", re.DOTALL)
SINGLE_BACKTICK_RE = re.compile(r"`([^`]*)`")
SUBREF_RE = re.compile(r"\|[\w.-]+\|")
URL_RE = re.compile(r"https?://\S+")
EMPH_RE = re.compile(r"(\*\*?)(.+?)\1")
TARGET_RE = re.compile(r"^(.*)\s$")
def strip_markup(text, keep_prose_roles=True):
"""Remove code spans / role markup; keep the *display text* of prose
roles like :term:`Foo` or :ref:`title ` since translators do
translate that part."""
def role_repl(m):
role, body = m.group(1).lower(), m.group(2)
if role in CODE_ROLES:
return " "
if not keep_prose_roles:
return " "
t = TARGET_RE.match(body)
return t.group(1) if t else body
text = DOUBLE_BACKTICK_RE.sub(" ", text)
text = ROLE_RE.sub(role_repl, text)
text = SINGLE_BACKTICK_RE.sub(r" \1 ", text)
text = SUBREF_RE.sub(" ", text)
text = URL_RE.sub(" ", text)
text = EMPH_RE.sub(r"\2", text)
return text
WORD_RE = re.compile(r"[A-Za-z][A-Za-z\-]*[A-Za-z]|[A-Za-z]")
QUOTED_CODE_RE = re.compile(r"'([\w.\-/ ]+)'|\"([\w.\-/ ]+)\"")
def quoted_span_is_code(content):
words = content.split()
return (
len(words) == 1
or any(c in content for c in ".-_")
or content.startswith(("import ", "from "))
)
def strip_quoted_code(text):
def repl(m):
content = m.group(1) or m.group(2)
return " " if quoted_span_is_code(content) else m.group(0)
return QUOTED_CODE_RE.sub(repl, text)
CODE_LINE_START_RE = re.compile(
r"^(?:import|from)\s" r"|^[a-z_][\w.]*(?:\[[^\]]*\])?\s*[=(]" r"|^[a-z_][\w.]*\("
)
CODE_STMT_RE = re.compile(
r"^(?:if|elif|else|for|while|def|class|try|except|finally|with|return"
r"|raise|break|continue|lambda|async|await)\b.*:\s*(?:#.*)?$"
)
def looks_codeish(line):
if line.startswith((">>>", "...", "$", "#")):
return True
if line.startswith(("'", '"', "[", "{", ">>", "...", "#", "$"))
or ln.startswith((" ", "\t"))
)
return codeish / len(lines) >= 0.8
def standalone_code_entry(msgid):
lines = [ln for ln in msgid.split("\n") if ln.strip()]
if not lines:
return False
codeish = sum(1 for ln in lines if ln[0] in " \t" or looks_codeish(ln.strip()))
prose = sum(1 for ln in lines if prose_line(ln))
return codeish >= max(1, len(lines) // 2) and prose = 0.8
# Bare code fragments embedded directly in prose with NO backticks/roles at
# all, e.g. "...(e.g. list.index()) but functions for other (e.g. len(list))"
# or "using ``list(dictview)``" -- covers dotted-call (name.method(...)),
# bare call (name(...)), and subscript/dunder-ish tokens.
BARE_DOTTED_CALL_RE = re.compile(r"\b[A-Za-z_][\w]*(?:\.[A-Za-z_][\w]*)+\(\)?")
BARE_CALL_RE = re.compile(r"\b[A-Za-z_][\w]*\([^()\s]{0,40}\)")
def strip_bare_code_fragments(text):
text = BARE_DOTTED_CALL_RE.sub(" ", text)
text = BARE_CALL_RE.sub(" ", text)
return text
def clean_prose(msgid):
"""Full pipeline: literal blocks -> markup -> quoted code -> bare code."""
text = strip_literal_blocks(msgid)
text = strip_markup(text)
text = strip_quoted_code(text)
text = strip_bare_code_fragments(text)
return text
# ---------------------------------------------------------------------------
# Term matching
# ---------------------------------------------------------------------------
def build_term_pattern(term):
escaped = re.escape(term)
# spaces/hyphens interchangeable, optional plural
parts = [re.escape(p) for p in re.split(r"[\s\-]+", term) if p]
body = r"[\s\-]+".join(parts)
return re.compile(
rf"(?