| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
1 parent 5b850fd commit ce83ac0
2 files changed
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -23,11 +23,23 @@ import { encodeTranslationToBuffer } from './i18n-translation-encoder'; | |||
| 23 | 23 | const LOCALIZE_KEYWORD = '$localize'; | |
| 24 | 24 | ||
| 25 | 25 | /** | |
| 26 | - * The maximum number of locales to process concurrently in a single sliding window. | ||
| 27 | - * This caps peak worker memory while maintaining multi-locale batching throughput. | ||
| 26 | + * The baseline number of locales to process concurrently in a single sliding window. | ||
| 27 | + * This caps peak worker memory on low-core machines while maintaining multi-locale batching throughput. | ||
| 28 | 28 | */ | |
| 29 | 29 | const DEFAULT_LOCALE_WINDOW_SIZE = 8; | |
| 30 | 30 | ||
| 31 | + /** | ||
| 32 | + * Minimum byte size threshold for a file to be eligible for multi-batch sharding. | ||
| 33 | + * Files below this threshold (< 100 KB) are processed in a single batch to minimize IPC overhead. | ||
| 34 | + */ | ||
| 35 | + const SMALL_FILE_FLOOR_BYTES = 100 * 1024; | ||
| 36 | + | ||
| 37 | + /** | ||
| 38 | + * Ratio of the maximum file size in a window to consider a file "dominant". | ||
| 39 | + * Files within 70% of the largest file are sharded across all workers for maximum concurrency. | ||
| 40 | + */ | ||
| 41 | + const DOMINANT_FILE_RATIO = 0.7; | ||
| 42 | + | ||
| 31 | 43 | /** | |
| 32 | 44 | * Serializes the translation messages for a locale for transfer to an inliner Worker. | |
| 33 | 45 | * | |
@@ -248,11 +260,13 @@ export class I18nInliner { | |||
| 248 | 260 | (name) => !name.endsWith('.map'), | |
| 249 | 261 | ); | |
| 250 | 262 | ||
| 251 | - // Process locales in sliding windows to cap peak worker memory | ||
| 252 | - for (let i = 0; i < localeList.length; i += DEFAULT_LOCALE_WINDOW_SIZE) { | ||
| 253 | - const windowLocales = localeList.slice(i, i + DEFAULT_LOCALE_WINDOW_SIZE); | ||
| 263 | + // Process locales in sliding windows to cap peak worker memory. | ||
| 264 | + // Ensure the window has at least enough locales to saturate all available workers on high-core machines. | ||
| 265 | + const windowSize = Math.max(DEFAULT_LOCALE_WINDOW_SIZE, this.#workerPool.maxThreads || 1); | ||
| 266 | + for (let i = 0; i < localeList.length; i += windowSize) { | ||
| 267 | + const windowLocales = localeList.slice(i, i + windowSize); | ||
| 254 | 268 | const activeLocales = windowLocales.map((item) => item.locale); | |
| 255 | - const isLastWindow = i + DEFAULT_LOCALE_WINDOW_SIZE >= localeList.length; | ||
| 269 | + const isLastWindow = i + windowSize >= localeList.length; | ||
| 256 | 270 | ||
| 257 | 271 | // Pre-calculate cache key bases and serialized Blobs for each locale in this window | |
| 258 | 272 | const localeCacheBases = new Map<string, string>(); | |
@@ -348,7 +362,6 @@ export class I18nInliner { | |||
| 348 | 362 | if (uncachedByFile.size > 0) { | |
| 349 | 363 | await this.#processUncachedBatches( | |
| 350 | 364 | uncachedByFile, | |
| 351 | - windowLocales.length, | ||
| 352 | 365 | fileResultsByLocale, | |
| 353 | 366 | activeLocales, | |
| 354 | 367 | isLastWindow, | |
@@ -415,28 +428,53 @@ export class I18nInliner { | |||
| 415 | 428 | ||
| 416 | 429 | async #processUncachedBatches( | |
| 417 | 430 | uncachedByFile: Map<string, UncachedLocaleEntry[]>, | |
| 418 | - localeCount: number, | ||
| 419 | 431 | fileResultsByLocale: Map<string, Map<string, TransformedFileResult>>, | |
| 420 | 432 | activeLocales?: string[], | |
| 421 | 433 | isLastWindow = true, | |
| 422 | 434 | generation?: number, | |
| 423 | 435 | ): Promise<void> { | |
| 424 | 436 | const workerCount = this.#workerPool.maxThreads || 1; | |
| 425 | - const targetTaskCount = Math.max(uncachedByFile.size, workerCount * 2); | ||
| 426 | - const localesPerBatch = Math.max( | ||
| 427 | - 1, | ||
| 428 | - Math.ceil(localeCount / (targetTaskCount / (uncachedByFile.size || 1))), | ||
| 429 | - ); | ||
| 430 | 437 | ||
| 431 | - const workerTasks: Promise<void>[] = []; | ||
| 432 | - | ||
| 433 | - for (const [filename, entries] of uncachedByFile) { | ||
| 438 | + // Extract file data and identify the heaviest file size in a single pass | ||
| 439 | + let maxFileSize = 0; | ||
| 440 | + const sortedFiles = Array.from(uncachedByFile, ([filename, entries]) => { | ||
| 434 | 441 | const codeFile = this.#localizeFiles.get(filename); | |
| 435 | 442 | assert(codeFile !== undefined, 'Localize file must exist: ' + filename); | |
| 443 | + const fileSize = codeFile.contents.byteLength; | ||
| 444 | + if (fileSize > maxFileSize) { | ||
| 445 | + maxFileSize = fileSize; | ||
| 446 | + } | ||
| 447 | + | ||
| 448 | + return { filename, entries, codeFile, fileSize }; | ||
| 449 | + }); | ||
| 450 | + | ||
| 451 | + // Sort files descending by byte size (Longest Processing Time First / LPT). | ||
| 452 | + // Heavy files (e.g. main.js) are queued first to saturate all worker threads immediately, | ||
| 453 | + // while small files act as gap fillers near the window barrier to prevent tail stragglers. | ||
| 454 | + sortedFiles.sort((a, b) => b.fileSize - a.fileSize); | ||
| 455 | + | ||
| 456 | + const workerTasks: Promise<void>[] = []; | ||
| 457 | + | ||
| 458 | + for (const { filename, entries, codeFile, fileSize } of sortedFiles) { | ||
| 436 | 459 | const mapFile = this.#localizeFiles.get(filename + '.map'); | |
| 437 | 460 | const codeBlob = new Blob([codeFile.contents]); | |
| 438 | 461 | const mapBlob = mapFile ? new Blob([mapFile.contents]) : undefined; | |
| 439 | 462 | ||
| 463 | + let localesPerBatch: number; | ||
| 464 | + if (uncachedByFile.size === 1) { | ||
| 465 | + // Single file in window: shard across all workers to avoid idle threads | ||
| 466 | + localesPerBatch = Math.max(1, Math.ceil(entries.length / workerCount)); | ||
| 467 | + } else if (fileSize < SMALL_FILE_FLOOR_BYTES) { | ||
| 468 | + // Small chunks (< 100 KB): process all locales in 1 batch to eliminate IPC overhead | ||
| 469 | + localesPerBatch = entries.length; | ||
| 470 | + } else if (fileSize >= maxFileSize * DOMINANT_FILE_RATIO) { | ||
| 471 | + // Dominant file(s): shard across all workers for maximum multi-core parallelism | ||
| 472 | + localesPerBatch = Math.max(1, Math.ceil(entries.length / workerCount)); | ||
| 473 | + } else { | ||
| 474 | + // Intermediate files: moderate sharding | ||
| 475 | + localesPerBatch = Math.max(1, Math.ceil(entries.length / 2)); | ||
| 476 | + } | ||
| 477 | + | ||
| 440 | 478 | const ephemeral = isLastWindow && entries.length <= localesPerBatch; | |
| 441 | 479 | for (let i = 0; i < entries.length; i += localesPerBatch) { | |
| 442 | 480 | const batchEntries = entries.slice(i, i + localesPerBatch); | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -870,4 +870,57 @@ describe('I18nInliner', () => { | |||
| 870 | 870 | const outputText = findFile(results.get('fr')?.outputFiles ?? [], 'main.js').text; | |
| 871 | 871 | expect(outputText).toContain('`Bonjour "${name}\\` with \\${injected} and \\\\backslash!`'); | |
| 872 | 872 | }); | |
| 873 | + | ||
| 874 | + it('correctly inlines when files have varying sizes across multiple workers', async () => { | ||
| 875 | + // Create a large dominant bundle (> 120 KB) and a small chunk (< 10 KB) | ||
| 876 | + const largePadding = '/* padding */ console.log(1);\n'.repeat(4000); | ||
| 877 | + const largeSource = `${GREETING_SOURCE}\n${largePadding}`; | ||
| 878 | + const smallSource = 'console.log($localize`:@@farewell:Goodbye`);'; | ||
| 879 | + | ||
| 880 | + const largeFile = browserFile('main.js', largeSource); | ||
| 881 | + const smallFile = browserFile('chunk.js', smallSource); | ||
| 882 | + | ||
| 883 | + inliner = new I18nInliner( | ||
| 884 | + { missingTranslation: 'error', outputFiles: [largeFile, smallFile] }, | ||
| 885 | + 2, | ||
| 886 | + ); | ||
| 887 | + | ||
| 888 | + const results = await inliner.inlineAll([ | ||
| 889 | + { | ||
| 890 | + locale: 'fr', | ||
| 891 | + translation: { | ||
| 892 | + greeting: translationFor('Bonjour'), | ||
| 893 | + farewell: translationFor('Au revoir'), | ||
| 894 | + }, | ||
| 895 | + }, | ||
| 896 | + { | ||
| 897 | + locale: 'de', | ||
| 898 | + translation: { | ||
| 899 | + greeting: translationFor('Guten Tag'), | ||
| 900 | + farewell: translationFor('Auf Wiedersehen'), | ||
| 901 | + }, | ||
| 902 | + }, | ||
| 903 | + { | ||
| 904 | + locale: 'es', | ||
| 905 | + translation: { | ||
| 906 | + greeting: translationFor('Hola'), | ||
| 907 | + farewell: translationFor('Adios'), | ||
| 908 | + }, | ||
| 909 | + }, | ||
| 910 | + ]); | ||
| 911 | + | ||
| 912 | + expect(results.size).toBe(3); | ||
| 913 | + | ||
| 914 | + const frFiles = results.get('fr')?.outputFiles ?? []; | ||
| 915 | + expect(findFile(frFiles, 'main.js').text).toContain('"Bonjour"'); | ||
| 916 | + expect(findFile(frFiles, 'chunk.js').text).toContain('"Au revoir"'); | ||
| 917 | + | ||
| 918 | + const deFiles = results.get('de')?.outputFiles ?? []; | ||
| 919 | + expect(findFile(deFiles, 'main.js').text).toContain('"Guten Tag"'); | ||
| 920 | + expect(findFile(deFiles, 'chunk.js').text).toContain('"Auf Wiedersehen"'); | ||
| 921 | + | ||
| 922 | + const esFiles = results.get('es')?.outputFiles ?? []; | ||
| 923 | + expect(findFile(esFiles, 'main.js').text).toContain('"Hola"'); | ||
| 924 | + expect(findFile(esFiles, 'chunk.js').text).toContain('"Adios"'); | ||
| 925 | + }); | ||
| 873 | 926 | }); | |
| Back | FazBrowse Home | New Git URL |
0 commit comments