FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

perf(@angular/build): selectively compress large entries in SQLite ca… · angular/angular-cli@4d11a13 · GitHub

  1. angular
  2. angular-cli

Repository navigation

Commit 4d11a13

Browse files
clydin
authored and
dgp1130
committed
perf(@angular/build): selectively compress large entries in SQLite cache store
Transformed JavaScript files from node_modules and other cached assets stored in the persistent SQLite cache store can range from tens to hundreds of kilobytes. Because SQLite rows exceeding the page size spill into overflow b-tree pages, storing large uncompressed entries increases disk I/O and cache file bloat. Serialized entries and binary payloads of 32 KB or greater are now selectively compressed asynchronously using raw DEFLATE at level 1. Performing compression asynchronously offloads CPU-intensive compression work to worker threads, preventing blocking of the main JavaScript event loop during builds. Smaller payloads and incompressible entries remain uncompressed. A dedicated `format` column stores the encoding format (`CacheFormat`), distinguishing between V8 serialized and raw binary (`Uint8Array`) data in both uncompressed and compressed states. This allows raw binary payloads (such as transformed JavaScript files) to be stored and retrieved with zero serialization overhead and zero memory copies. For backward compatibility with existing databases on disk, the database schema migration adds the `format` column with a default value of 0 (`V8`), ensuring pre-existing cache entries continue to deserialize seamlessly.
1 parent ef40c62 commit 4d11a13

2 files changed

Lines changed: 257 additions & 6 deletions

File tree

‎packages/angular/build/src/tools/esbuild/sqlite-cache-store.ts‎

Lines changed: 69 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,31 @@
99
import { mkdirSync, rmSync } from 'node:fs';
1010
import { dirname } from 'node:path';
1111
import { DatabaseSync, StatementSync } from 'node:sqlite';
12+
import { promisify } from 'node:util';
1213
import { deserialize, serialize } from 'node:v8';
14+
import { deflateRaw, inflateRawSync } from 'node:zlib';
1315
import { Cache, PersistentCacheStore } from './cache';
1416

17+
const deflateRawAsync = promisify(deflateRaw);
18+
19+
/**
20+
* Minimum payload size (in bytes) required to attempt compression.
21+
* Smaller payloads generally do not achieve meaningful compression ratios, while larger payloads
22+
* (such as transformed JavaScript modules from node_modules) achieve 70-80% size reduction
23+
* and drastically reduce SQLite overflow pages.
24+
*/
25+
const COMPRESSION_THRESHOLD = 32 * 1024; // 32 KB
26+
27+
/**
28+
* Storage format discriminator values for cache table entries.
29+
*/
30+
const enum CacheFormat {
31+
V8 = 0,
32+
V8Compressed = 1,
33+
RawBinary = 2,
34+
RawBinaryCompressed = 3,
35+
}
36+
1537
/**
1638
* Common SQLite primary result codes.
1739
* @see https://www.sqlite.org/rescode.html
@@ -88,16 +110,26 @@ export class SqliteCacheStore implements PersistentCacheStore<unknown> {
88110
db.exec('PRAGMA temp_store = MEMORY;');
89111
db.exec('PRAGMA mmap_size = 268435456;');
90112
db.exec(
91-
'CREATE TABLE IF NOT EXISTS cache (key TEXT PRIMARY KEY, value BLOB, last_accessed INTEGER NOT NULL) WITHOUT ROWID;',
113+
'CREATE TABLE IF NOT EXISTS cache (' +
114+
'key TEXT PRIMARY KEY, ' +
115+
'value BLOB, ' +
116+
'format INTEGER NOT NULL DEFAULT 0, ' +
117+
'last_accessed INTEGER NOT NULL' +
118+
') WITHOUT ROWID;',
92119
);
120+
try {
121+
db.exec('ALTER TABLE cache ADD COLUMN format INTEGER NOT NULL DEFAULT 0;');
122+
} catch {
123+
// Ignore error if format column already exists
124+
}
93125
db.exec(
94126
'CREATE INDEX IF NOT EXISTS idx_cache_accessed ON cache (last_accessed DESC, key DESC);',
95127
);
96128

97-
this.#getStmt = db.prepare('SELECT value FROM cache WHERE key = ?');
129+
this.#getStmt = db.prepare('SELECT value, format FROM cache WHERE key = ?');
98130
this.#hasStmt = db.prepare('SELECT 1 FROM cache WHERE key = ?');
99131
this.#setStmt = db.prepare(
100-
'INSERT OR REPLACE INTO cache (key, value, last_accessed) VALUES (?, ?, unixepoch())',
132+
'INSERT OR REPLACE INTO cache (key, value, format, last_accessed) VALUES (?, ?, ?, unixepoch())',
101133
);
102134
this.#updateAccessedStmt = db.prepare(
103135
'UPDATE cache SET last_accessed = unixepoch() WHERE key = ?',
@@ -206,14 +238,31 @@ export class SqliteCacheStore implements PersistentCacheStore<unknown> {
206238

207239
try {
208240
// SQLite column types are dynamic, so the stored value is only known at runtime.
209-
const row = this.#getStmt?.get(key) as { value: unknown } | undefined;
241+
const row = this.#getStmt?.get(key) as { value: unknown; format: CacheFormat } | undefined;
210242

211243
if (row) {
212244
this.#queueAccessUpdate(key);
213245

214246
if (row.value instanceof Uint8Array) {
215247
try {
216-
return deserialize(row.value);
248+
switch (row.format) {
249+
case CacheFormat.RawBinary:
250+
return row.value;
251+
case CacheFormat.RawBinaryCompressed: {
252+
const decompressed = inflateRawSync(row.value);
253+
254+
return new Uint8Array(
255+
decompressed.buffer,
256+
decompressed.byteOffset,
257+
decompressed.byteLength,
258+
);
259+
}
260+
case CacheFormat.V8Compressed:
261+
return deserialize(inflateRawSync(row.value));
262+
case CacheFormat.V8:
263+
default:
264+
return deserialize(row.value);
265+
}
217266
} catch {
218267
// Treat corrupt or unparseable cached payloads as a cache miss.
219268
}
@@ -245,7 +294,21 @@ export class SqliteCacheStore implements PersistentCacheStore<unknown> {
245294

246295
try {
247296
this.#pendingAccessedKeys.delete(key);
248-
this.#setStmt?.run(key, serialize(value));
297+
298+
const isBinary = value instanceof Uint8Array;
299+
const data = isBinary ? value : serialize(value);
300+
let format = isBinary ? CacheFormat.RawBinary : CacheFormat.V8;
301+
let payload: Uint8Array = data;
302+
303+
if (data.byteLength >= COMPRESSION_THRESHOLD) {
304+
const compressed = await deflateRawAsync(data, { level: 1 });
305+
if (compressed.byteLength < data.byteLength) {
306+
payload = compressed;
307+
format = isBinary ? CacheFormat.RawBinaryCompressed : CacheFormat.V8Compressed;
308+
}
309+
}
310+
311+
this.#setStmt?.run(key, payload, format);
249312
} catch {
250313
// Writing to cache is non-fatal and should not fail the build.
251314
}

‎packages/angular/build/src/tools/esbuild/sqlite-cache-store_spec.ts‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,194 @@ describe('SqliteCacheStore', () => {
224224
expect(row.last_accessed).toBeGreaterThan(pastTimestamp);
225225
});
226226

227+
it('should selectively compress entries that exceed the 32KB threshold', async () => {
228+
const largeContent =
229+
'export const testFunction = () => { console.log("hello world"); };\n'.repeat(1000);
230+
await store.set('large-key', largeContent);
231+
store.close();
232+
233+
const { DatabaseSync } = await import('node:sqlite');
234+
const directDb = new DatabaseSync(cachePath);
235+
const row = directDb
236+
.prepare('SELECT value, format FROM cache WHERE key = ?')
237+
.get('large-key') as {
238+
value: Uint8Array;
239+
format: number;
240+
};
241+
directDb.close();
242+
243+
// Verify format flag is 1 (CacheFormat.V8Compressed) and compressed size is drastically smaller
244+
expect(row.format).toBe(1);
245+
expect(row.value.byteLength).toBeLessThan(largeContent.length / 5);
246+
247+
const reopenedStore = new SqliteCacheStore(cachePath);
248+
try {
249+
const result = await reopenedStore.get('large-key');
250+
expect(result).toBe(largeContent);
251+
} finally {
252+
reopenedStore.close();
253+
}
254+
});
255+
256+
it('should not compress entries below the 32KB threshold', async () => {
257+
const smallContent = 'export const small = true;';
258+
await store.set('small-key', smallContent);
259+
store.close();
260+
261+
const { DatabaseSync } = await import('node:sqlite');
262+
const directDb = new DatabaseSync(cachePath);
263+
const row = directDb
264+
.prepare('SELECT value, format FROM cache WHERE key = ?')
265+
.get('small-key') as {
266+
value: Uint8Array;
267+
format: number;
268+
};
269+
directDb.close();
270+
271+
// Verify format flag is 0 (CacheFormat.V8)
272+
expect(row.format).toBe(0);
273+
274+
const reopenedStore = new SqliteCacheStore(cachePath);
275+
try {
276+
const result = await reopenedStore.get('small-key');
277+
expect(result).toBe(smallContent);
278+
} finally {
279+
reopenedStore.close();
280+
}
281+
});
282+
283+
it('should not compress entries exceeding the 32KB threshold if compression does not reduce size', async () => {
284+
const { randomBytes } = await import('node:crypto');
285+
// High-entropy random bytes cannot be compressed by DEFLATE
286+
const incompressibleData = new Uint8Array(randomBytes(48 * 1024));
287+
await store.set('incompressible-key', incompressibleData);
288+
store.close();
289+
290+
const { DatabaseSync } = await import('node:sqlite');
291+
const directDb = new DatabaseSync(cachePath);
292+
const row = directDb
293+
.prepare('SELECT value, format FROM cache WHERE key = ?')
294+
.get('incompressible-key') as {
295+
value: Uint8Array;
296+
format: number;
297+
};
298+
directDb.close();
299+
300+
// Verify entry is stored uncompressed as raw binary (format 2 = CacheFormat.RawBinary)
301+
expect(row.format).toBe(2);
302+
303+
const reopenedStore = new SqliteCacheStore(cachePath);
304+
try {
305+
const result = await reopenedStore.get('incompressible-key');
306+
expect(result).toEqual(incompressibleData);
307+
} finally {
308+
reopenedStore.close();
309+
}
310+
});
311+
312+
it('should store raw Uint8Array directly with zero serialization', async () => {
313+
const binaryData = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]);
314+
await store.set('binary-key', binaryData);
315+
store.close();
316+
317+
const { DatabaseSync } = await import('node:sqlite');
318+
const directDb = new DatabaseSync(cachePath);
319+
const row = directDb
320+
.prepare('SELECT value, format FROM cache WHERE key = ?')
321+
.get('binary-key') as {
322+
value: Uint8Array;
323+
format: number;
324+
};
325+
directDb.close();
326+
327+
// Verify format is 2 (CacheFormat.RawBinary) and value is the exact raw bytes
328+
expect(row.format).toBe(2);
329+
expect(row.value).toEqual(binaryData);
330+
331+
const reopenedStore = new SqliteCacheStore(cachePath);
332+
try {
333+
const result = await reopenedStore.get('binary-key');
334+
expect(result).toEqual(binaryData);
335+
} finally {
336+
reopenedStore.close();
337+
}
338+
});
339+
340+
it('should selectively compress large Uint8Array entries', async () => {
341+
const largeBinary = new Uint8Array(Buffer.from('export const x = 1;\n'.repeat(2000)));
342+
await store.set('large-binary-key', largeBinary);
343+
store.close();
344+
345+
const { DatabaseSync } = await import('node:sqlite');
346+
const directDb = new DatabaseSync(cachePath);
347+
const row = directDb
348+
.prepare('SELECT value, format FROM cache WHERE key = ?')
349+
.get('large-binary-key') as {
350+
value: Uint8Array;
351+
format: number;
352+
};
353+
directDb.close();
354+
355+
// Verify format is 3 (CacheFormat.RawBinaryCompressed)
356+
expect(row.format).toBe(3);
357+
expect(row.value.byteLength).toBeLessThan(largeBinary.byteLength / 5);
358+
359+
const reopenedStore = new SqliteCacheStore(cachePath);
360+
try {
361+
const result = await reopenedStore.get('large-binary-key');
362+
expect(result).toEqual(largeBinary);
363+
} finally {
364+
reopenedStore.close();
365+
}
366+
});
367+
368+
it('should seamlessly read legacy entries from databases created without a format column', async () => {
369+
const { serialize } = await import('node:v8');
370+
const { DatabaseSync } = await import('node:sqlite');
371+
372+
// Create a legacy database without a format column
373+
const directDb = new DatabaseSync(cachePath);
374+
directDb.exec(
375+
'CREATE TABLE cache (key TEXT PRIMARY KEY, value BLOB, last_accessed INTEGER NOT NULL) WITHOUT ROWID;',
376+
);
377+
const legacyPayload = serialize('legacy-content');
378+
directDb
379+
.prepare('INSERT INTO cache (key, value, last_accessed) VALUES (?, ?, unixepoch())')
380+
.run('legacy-key', legacyPayload);
381+
directDb.close();
382+
383+
const reopenedStore = new SqliteCacheStore(cachePath);
384+
try {
385+
const result = await reopenedStore.get('legacy-key');
386+
expect(result).toBe('legacy-content');
387+
} finally {
388+
reopenedStore.close();
389+
}
390+
});
391+
392+
it('should treat a corrupted compressed payload as a cache miss', async () => {
393+
// Initialize database
394+
await store.set('init-key', 'init-val');
395+
store.close();
396+
397+
// Insert an entry with format = 1 (V8Compressed) followed by garbage
398+
const { DatabaseSync } = await import('node:sqlite');
399+
const directDb = new DatabaseSync(cachePath);
400+
directDb
401+
.prepare(
402+
'INSERT INTO cache (key, value, format, last_accessed) VALUES (?, ?, 1, unixepoch())',
403+
)
404+
.run('corrupt-compressed-key', new Uint8Array([0xde, 0xad, 0xbe, 0xef]));
405+
directDb.close();
406+
407+
const reopenedStore = new SqliteCacheStore(cachePath);
408+
try {
409+
expect(await reopenedStore.get('corrupt-compressed-key')).toBeUndefined();
410+
} finally {
411+
reopenedStore.close();
412+
}
413+
});
414+
227415
it('should treat a non-binary payload as a cache miss', async () => {
228416
await store.set('text-key', 'value');
229417
store.close();

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL