python-docs-fa/scripts/list_parentheticals.py at 3.15 · python/python-docs-fa · GitHub

Repository navigation

History

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

#!/usr/bin/env python3

"""

scripts/list_parentheticals.py

List every "فارسی (English)" gloss in the .po corpus: msgstr places that

use a Persian rendering and repeat the English term in parentheses right

next to it, e.g.

مدیریت زمینه (context management)

لیترال درون‌یابی‌شده (interpolated literal)

«الگوی as» (as-pattern)

Scope rules:

- only prose msgstrs: code blocks, doctests, mostly-preserved code

listings and untranslated entries are skipped

- the paren must follow a Persian word (guillemets, quotes, diacritics

and whitespace in between are tolerated); a paren hanging off a

stopword like «و (list)» or «را (foo)» is ignored, since the gloss

there belongs to something else

- the paren content must look like an English term: starts with a

letter, only letters/digits/spaces and .+-_&/'#!, at most 6 words

and 40 chars, not ending in '.' (so "(default: None)" and

"(e.g. ...)" fall out automatically)

Two outputs:

1. the full list, grouped by English term and Persian partner, with

file:line refs -> reports/parentheticals.md

2. a coverage view: for each glossed term, how many msgids containing

that term were glossed vs. translated without the gloss (the msgstr

still contains one of the term's Persian partner words). Glosses

are often a first-mention-only convention, so read coverage as a

map of where glosses appear vs. don't -- not as violations.

Usage:

python3 scripts/list_parentheticals.py

python3 scripts/list_parentheticals.py --max-examples 4

"""

import argparse

import json

import re

import sys

from collections import defaultdict

from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent))

from check_consistency import ( # noqa: E402

FARSI_STOPWORDS,

FARSI_WORD_RE,

REPO_ROOT,

load_entries,

mostly_preserved_code,

standalone_code_entry,

)

from triage_consistency import canon_lexeme # noqa: E402

FA_WORD = r"[\u0620-\u064A\u066E-\u066F\u0671-\u06D3\u06EE-\u06FC\u200c]+"

# between the Persian word and '(': marks/ZWNJ/whitespace/guillemets/

# quotes, or (for «الگوی as») a single short Latin word closed by a

# guillemet. Kept strictly bounded: an unbounded (…+)+ here makes the

# regex backtrack catastrophically on long English tails.

GLUE = (

r'(?:[\u064B-\u065F\u0670\u200c\s»«"\']*' r"|\s*[A-Za-z][A-Za-z0-9\-]{0,19}\s*»\s*)"

)

GLOSS_RE = re.compile("(" + FA_WORD + ")" + GLUE + r"\(\s*([^()\n]{1,60})\s*\)")

def is_english_gloss(content: str) -> bool:

c = content.strip()

if not (2 <= len(c) <= 40):

return False

if c.endswith("."):

return False

if re.match(r"(?i)^(e\.g\.|i\.e\.|etc\.?|cf\.?|vs\.?)", c):

return False # abbreviation, not a term gloss

if not re.fullmatch(r"[A-Za-z][A-Za-z0-9 .+\-_&/'#!]*", c):

return False

return 1 <= len(c.split()) <= 6

def scan_glosses(entries):

"""All (fa word, (english), file:line) occurrences in prose msgstrs."""

occs = []

for e in entries:

msgid, msgstr = e["msgid"], e["msgstr"]

if e["code"] or msgstr.strip() == msgid.strip():

continue

if standalone_code_entry(msgid) or mostly_preserved_code(msgid, msgstr):

continue

for m in GLOSS_RE.finditer(msgstr):

fa, en = m.group(1), m.group(2).strip()

if not is_english_gloss(en):

continue

if canon_lexeme(fa) in FARSI_STOPWORDS:

continue

occs.append(

{

"en": en,

"en_key": en.lower(),

"fa": fa,

"file": e["file"],

"line": e["line"],

"ctx": msgstr[max(0, m.start() - 30) : m.end() + 30].replace(

"\n", " "

}

)

return occs

def coverage(entries, terms, partner_lexemes):

if not terms:

return {}

terms_parts = {t: re.split(r"[\s\-]+", t) for t in terms}

by_first = defaultdict(list)

for t, parts in terms_parts.items():

by_first[parts[0].lower()].append(

) # <-- lowercase for case-insensitive lookup

gloss_re = {

t: re.compile(

r"\(\s*" + r"[\s\-]+".join(re.escape(p) for p in parts) + r"s?\s*\)",

re.IGNORECASE,

)

for t, parts in terms_parts.items()

}

stats = {t: {"glossed": [], "no_gloss_fa": [], "no_gloss_other": []} for t in terms}

for e in entries:

msgid, msgstr = e["msgid"], e["msgstr"]

if e["code"] or msgstr.strip() == msgid.strip():

continue

if standalone_code_entry(msgid) or mostly_preserved_code(msgid, msgstr):

continue

if not msgid.strip():

continue

# FIX: tokenize on whitespace only, preserving hyphenated tokens,

# then normalise each token to lowercase for lookup.

raw_tokens = re.findall(r"[A-Za-z][A-Za-z0-9\-']*", msgid)

low = [t.lower() for t in raw_tokens]

n = len(low)

found = set()

for i, tok in enumerate(low):

for term in by_first.get(tok, ()):

parts = terms_parts[term]

np = len(parts)

if np == 1:

found.add(term)

else:

# FIX: compare against hyphen-normalised msgid tokens

# so "as-pattern" in msgid matches parts ["as", "pattern"]

# whether written as "as-pattern" or "as pattern".

msgid_slice = [

re.sub(r"\-+", "-", low[j]) for j in range(i, min(i + np, n))

]

term_parts_norm = [re.sub(r"\-+", "-", p.lower()) for p in parts]

if msgid_slice == term_parts_norm:

found.add(term)

# Also match when the whole hyphenated term appears as one token.

elif np > 1 and low[i] == "-".join(p.lower() for p in parts):

found.add(term)

if not found:

continue

loc = {"file": e["file"], "line": e["line"], "msgid": msgid[:80]}

fa_toks = {canon_lexeme(w) for w in FARSI_WORD_RE.findall(msgstr)}

for t in found:

if gloss_re[t].search(msgstr):

stats[t]["glossed"].append(loc)

elif partner_lexemes.get(t, set()) & fa_toks:

stats[t]["no_gloss_fa"].append(loc)

else:

stats[t]["no_gloss_other"].append(loc)

return stats

def render_markdown(pairs, cov, n_occ, n_files, args):

L = []

A = L.append

n_terms = len(pairs)

n_pairs = sum(len(v) for v in pairs.values())

A("# Parenthetical English glosses — فارسی (English)")

A("")

f"{n_occ} gloss occurrences · {n_terms} English terms · "

f"{n_pairs} Persian–English pairs · {n_files} files."

)

A("")

"Every place where a msgstr uses a Persian rendering and repeats "

"the English term in parentheses next to it (prose msgstrs only; "

"code blocks and doctests are skipped)."

)

A("")

if cov:

A("## Coverage by English term")

A("")

"For each glossed term: how many msgids containing that term "

"carry the gloss, how many use the same Persian rendering "

"*without* the gloss, and how many do neither. Glosses are "

"often a first-mention-only convention, so read this as a map "

"of where they appear vs. don't — not as violations."

)

A("")

A("| --- | ---: | ---: | ---: |")

rows = sorted(

cov.items(),

key=lambda kv: -(len(kv[1]["glossed"]) + len(kv[1]["no_gloss_fa"])),

)

for t, s in rows:

f"| {t} | {len(s['glossed'])} | {len(s['no_gloss_fa'])} "

f"| {len(s['no_gloss_other'])} |"

)

A("")

A("## Full list")

A("")

for en, fas in sorted(

pairs.items(), key=lambda kv: -sum(len(v) for v in kv[1].items())

total = sum(len(v) for v in fas.values())

A(f"### {en} — {total}×")

A("")

for fa, occs in sorted(fas.items(), key=lambda kv: -len(kv[1])):

A(f"- **{fa}** ×{len(occs)}")

for o in occs[: args.max_examples]:

A(f" - {o['file']}:{o['line']} — “...{o['ctx']}...”")

if len(occs) > args.max_examples:

f" - … and {len(occs) - args.max_examples} more "

f"(see JSON report)"

)

A("")

return "\n".join(L)

def main():

ap = argparse.ArgumentParser(

description=__doc__,

formatter_class=argparse.RawDescriptionHelpFormatter,

)

ap.add_argument("--po-dir", default=str(REPO_ROOT))

ap.add_argument(

"--out-md", default=str(REPO_ROOT / "reports" / "parentheticals.md")

)

ap.add_argument(

"--out-json", default=str(REPO_ROOT / "reports" / "parentheticals.json")

)

ap.add_argument(

"--max-examples",

type=int,

default=4,

help="Max occurrences listed per Persian partner",

)

ap.add_argument(

"--max-terms",

type=int,

default=400,

help="Max terms for the coverage table (0 = all; "

"the table is O(entries x terms))",

)

args = ap.parse_args()

print("Loading .po entries ...")

entries, n_files = load_entries(Path(args.po_dir))

print(f" {n_files} files, {len(entries)} translated entries")

print("Scanning for فارسی (English) glosses ...")

occs = scan_glosses(entries)

pairs = defaultdict(lambda: defaultdict(list))

en_display = {}

for o in occs:

pairs[o["en_key"]][o["fa"]].append(o)

en_display.setdefault(o["en_key"], o["en"])

pairs = {k: dict(v) for k, v in pairs.items()}

partner_lexemes = {en: {canon_lexeme(f) for f in fas} for en, fas in pairs.items()}

n_occ = len(occs)

n_gloss_files = len({o["file"] for o in occs})

print(

f" {n_occ} glosses · {len(pairs)} English terms · "

f"{sum(len(v) for v in pairs.values())} pairs · "

f"{n_gloss_files} files"

)

print("Computing per-term coverage ...")

ranked = sorted(

pairs.items(),

key=lambda kv: -sum(len(v) for v in kv[1].values()),

)

cov_terms = [en for en, _ in ranked]

if args.max_terms and len(cov_terms) > args.max_terms:

print(

f" capping coverage to top {args.max_terms} glossed terms "

f"(--max-terms 0 for all)"

)

cov_terms = cov_terms[: args.max_terms]

cov = coverage(entries, set(cov_terms), partner_lexemes)

md = render_markdown(pairs, cov, n_occ, n_gloss_files, args)

Path(args.out_md).write_text(md, encoding="utf-8")

out = {

"summary": {

"gloss_occurrences": n_occ,

"english_terms": len(pairs),

"persian_english_pairs": sum(len(v) for v in pairs.values()),

"files_with_glosses": n_gloss_files,

"pairs": {

en: {

fa: [{k: o[k] for k in ("file", "line", "ctx")} for o in occs]

for fa, occs in fas.items()

}

for en, fas in pairs.items()

"coverage": {t: {k: v for k, v in s.items()} for t, s in cov.items()},

}

Path(args.out_json).write_text(

json.dumps(out, ensure_ascii=False, indent=2), encoding="utf-8"

)

print(f"\nWrote {args.out_md}")

print(f"Wrote {args.out_json}")

print("\nTop glossed English terms:")

for en, fas in sorted(

pairs.items(), key=lambda kv: -sum(len(v) for v in kv[1].items())

)[:15]:

total = sum(len(v) for v in fas.values())

partners = "، ".join(

f"{fa}×{len(occ)}"

for fa, occ in sorted(fas.items(), key=lambda kv: -len(kv[1]))[:3]

)

print(f" {en_display[en]} ({total}×): {partners}")

if cov:

print(

"\nMost often translated without the gloss "

"(same Persian word, no paren):"

)

ranked = sorted(

cov.items(),

key=lambda kv: -len(kv[1]["no_gloss_fa"]),

)[:10]

for t, s in ranked:

print(

f" {t}: {len(s['glossed'])} glossed, "

f"{len(s['no_gloss_fa'])} without"

)

if __name__ == "__main__":

main()

python-docs-fa/scripts/list_parentheticals.py at 3.15 · python/python-docs-fa · GitHub

Repository navigation

Expand file tree

Breadcrumbs

list_parentheticals.py

Latest commit

History

Breadcrumbs

list_parentheticals.py

File metadata and controls