| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
1 parent 8516602 commit e3258fd
1 file changed
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -4,60 +4,121 @@ | |||
| 4 | 4 | ||
| 5 | 5 | > Stability: 2 - Stable | |
| 6 | 6 | ||
| 7 | - The `zlib` module provides compression functionality implemented using Gzip and | ||
| 8 | - Deflate/Inflate, as well as Brotli. It can be accessed using: | ||
| 7 | + The `zlib` module provides compression functionality implemented using Gzip, | ||
| 8 | + Deflate/Inflate, and Brotli. | ||
| 9 | + | ||
| 10 | + To access it: | ||
| 9 | 11 | ||
| 10 | 12 | ```js | |
| 11 | 13 | const zlib = require('zlib'); | |
| 12 | 14 | ``` | |
| 13 | 15 | ||
| 16 | + Compression and decompression are built around the Node.js [Streams API][]. | ||
| 17 | + | ||
| 14 | 18 | Compressing or decompressing a stream (such as a file) can be accomplished by | |
| 15 | - piping the source stream data through a `zlib` stream into a destination stream: | ||
| 19 | + piping the source stream through a `zlib` `Transform` stream into a destination | ||
| 20 | + stream: | ||
| 16 | 21 | ||
| 17 | 22 | ```js | |
| 18 | - const gzip = zlib.createGzip(); | ||
| 19 | - const fs = require('fs'); | ||
| 20 | - const inp = fs.createReadStream('input.txt'); | ||
| 21 | - const out = fs.createWriteStream('input.txt.gz'); | ||
| 22 | - | ||
| 23 | - inp.pipe(gzip) | ||
| 24 | - .on('error', () => { | ||
| 25 | - // handle error | ||
| 26 | - }) | ||
| 27 | - .pipe(out) | ||
| 28 | - .on('error', () => { | ||
| 29 | - // handle error | ||
| 23 | + const { createGzip } = require('zlib'); | ||
| 24 | + const { pipeline } = require('stream'); | ||
| 25 | + const { | ||
| 26 | + createReadStream, | ||
| 27 | + createWriteStream | ||
| 28 | + } = require('fs'); | ||
| 29 | + | ||
| 30 | + const gzip = createGzip(); | ||
| 31 | + const source = createReadStream('input.txt'); | ||
| 32 | + const destination = createWriteStream('input.txt.gz'); | ||
| 33 | + | ||
| 34 | + pipeline(source, gzip, destination, (err) => { | ||
| 35 | + if (err) { | ||
| 36 | + console.error('An error occurred:', err); | ||
| 37 | + process.exitCode = 1; | ||
| 38 | + } | ||
| 39 | + }); | ||
| 40 | + | ||
| 41 | + // Or, Promisified | ||
| 42 | + | ||
| 43 | + const { promisify } = require('util'); | ||
| 44 | + const pipe = promisify(pipeline); | ||
| 45 | + | ||
| 46 | + async function do_gzip(input, output) { | ||
| 47 | + const gzip = createGzip(); | ||
| 48 | + const source = createReadStream(input); | ||
| 49 | + const destination = createWriteStream(output); | ||
| 50 | + await pipe(source, gzip, destination); | ||
| 51 | + } | ||
| 52 | + | ||
| 53 | + do_gzip('input.txt', 'input.txt.gz') | ||
| 54 | + .catch((err) => { | ||
| 55 | + console.error('An error occurred:', err); | ||
| 56 | + process.exitCode = 1; | ||
| 30 | 57 | }); | |
| 31 | 58 | ``` | |
| 32 | 59 | ||
| 33 | 60 | It is also possible to compress or decompress data in a single step: | |
| 34 | 61 | ||
| 35 | 62 | ```js | |
| 63 | + const { deflate, unzip } = require('zlib'); | ||
| 64 | + | ||
| 36 | 65 | const input = '.................................'; | |
| 37 | - zlib.deflate(input, (err, buffer) => { | ||
| 38 | - if (!err) { | ||
| 39 | - console.log(buffer.toString('base64')); | ||
| 40 | - } else { | ||
| 41 | - // handle error | ||
| 66 | + deflate(input, (err, buffer) => { | ||
| 67 | + if (err) { | ||
| 68 | + console.error('An error occurred:', err); | ||
| 69 | + process.exitCode = 1; | ||
| 42 | 70 | } | |
| 71 | + console.log(buffer.toString('base64')); | ||
| 43 | 72 | }); | |
| 44 | 73 | ||
| 45 | 74 | const buffer = Buffer.from('eJzT0yMAAGTvBe8=', 'base64'); | |
| 46 | - zlib.unzip(buffer, (err, buffer) => { | ||
| 47 | - if (!err) { | ||
| 48 | - console.log(buffer.toString()); | ||
| 49 | - } else { | ||
| 50 | - // handle error | ||
| 75 | + unzip(buffer, (err, buffer) => { | ||
| 76 | + if (err) { | ||
| 77 | + console.error('An error occurred:', err); | ||
| 78 | + process.exitCode = 1; | ||
| 51 | 79 | } | |
| 80 | + console.log(buffer.toString()); | ||
| 52 | 81 | }); | |
| 82 | + | ||
| 83 | + // Or, Promisified | ||
| 84 | + | ||
| 85 | + const { promisify } = require('util'); | ||
| 86 | + const do_unzip = promisify(unzip); | ||
| 87 | + | ||
| 88 | + do_unzip(buffer) | ||
| 89 | + .then((buf) => console.log(buf.toString())) | ||
| 90 | + .catch((err) => { | ||
| 91 | + console.error('An error occurred:', err); | ||
| 92 | + process.exitCode = 1; | ||
| 93 | + }); | ||
| 53 | 94 | ``` | |
| 54 | 95 | ||
| 55 | - ## Threadpool Usage | ||
| 96 | + ## Threadpool Usage and Performance Considerations | ||
| 97 | + | ||
| 98 | + All `zlib` APIs, except those that are explicitly synchronous, use the Node.js | ||
| 99 | + internal threadpool. This can lead to surprising effects and performance | ||
| 100 | + limitations in some applications. | ||
| 56 | 101 | ||
| 57 | - All zlib APIs, except those that are explicitly synchronous, use libuv's | ||
| 58 | - threadpool. This can lead to surprising effects in some applications, such as | ||
| 59 | - subpar performance (which can be mitigated by adjusting the [pool size][]) | ||
| 60 | - and/or unrecoverable and catastrophic memory fragmentation. | ||
| 102 | + Creating and using a large number of zlib objects simultaneously can cause | ||
| 103 | + significant memory fragmentation. | ||
| 104 | + | ||
| 105 | + ```js | ||
| 106 | + const zlib = require('zlib'); | ||
| 107 | + | ||
| 108 | + const payload = Buffer.from('This is some data'); | ||
| 109 | + | ||
| 110 | + // WARNING: DO NOT DO THIS! | ||
| 111 | + for (let i = 0; i < 30000; ++i) { | ||
| 112 | + zlib.deflate(payload, (err, buffer) => {}); | ||
| 113 | + } | ||
| 114 | + ``` | ||
| 115 | + | ||
| 116 | + In the preceding example, 30,000 deflate instances are created concurrently. | ||
| 117 | + Because of how some operating systems handle memory allocation and | ||
| 118 | + deallocation, this may lead to to significant memory fragmentation. | ||
| 119 | + | ||
| 120 | + It is strongly recommended that the results of compression | ||
| 121 | + operations be cached to avoid duplication of effort. | ||
| 61 | 122 | ||
| 62 | 123 | ## Compressing HTTP requests and responses | |
| 63 | 124 | ||
@@ -80,26 +141,35 @@ tradeoffs involved in `zlib` usage. | |||
| 80 | 141 | const zlib = require('zlib'); | |
| 81 | 142 | const http = require('http'); | |
| 82 | 143 | const fs = require('fs'); | |
| 144 | + const { pipeline } = require('stream'); | ||
| 145 | + | ||
| 83 | 146 | const request = http.get({ host: 'example.com', | |
| 84 | 147 | path: '/', | |
| 85 | 148 | port: 80, | |
| 86 | 149 | headers: { 'Accept-Encoding': 'br,gzip,deflate' } }); | |
| 87 | 150 | request.on('response', (response) => { | |
| 88 | 151 | const output = fs.createWriteStream('example.com_index.html'); | |
| 89 | 152 | ||
| 153 | + const onError = (err) => { | ||
| 154 | + if (err) { | ||
| 155 | + console.error('An error occurred:', err); | ||
| 156 | + process.exitCode = 1; | ||
| 157 | + } | ||
| 158 | + }; | ||
| 159 | + | ||
| 90 | 160 | switch (response.headers['content-encoding']) { | |
| 91 | 161 | case 'br': | |
| 92 | - response.pipe(zlib.createBrotliDecompress()).pipe(output); | ||
| 162 | + pipeline(response, zlib.createBrotliDecompress(), output, onError); | ||
| 93 | 163 | break; | |
| 94 | 164 | // Or, just use zlib.createUnzip() to handle both of the following cases: | |
| 95 | 165 | case 'gzip': | |
| 96 | - response.pipe(zlib.createGunzip()).pipe(output); | ||
| 166 | + pipeline(response, zlib.createGunzip(), output, onError); | ||
| 97 | 167 | break; | |
| 98 | 168 | case 'deflate': | |
| 99 | - response.pipe(zlib.createInflate()).pipe(output); | ||
| 169 | + pipeline(response, zlib.createInflate(), outout, onError); | ||
| 100 | 170 | break; | |
| 101 | 171 | default: | |
| 102 | - response.pipe(output); | ||
| 172 | + pipeline(response, output, onError); | ||
| 103 | 173 | break; | |
| 104 | 174 | } | |
| 105 | 175 | }); | |
@@ -112,6 +182,8 @@ request.on('response', (response) => { | |||
| 112 | 182 | const zlib = require('zlib'); | |
| 113 | 183 | const http = require('http'); | |
| 114 | 184 | const fs = require('fs'); | |
| 185 | + const { pipeline } = require('stream'); | ||
| 186 | + | ||
| 115 | 187 | http.createServer((request, response) => { | |
| 116 | 188 | const raw = fs.createReadStream('index.html'); | |
| 117 | 189 | // Store both a compressed and an uncompressed version of the resource. | |
@@ -121,20 +193,32 @@ http.createServer((request, response) => { | |||
| 121 | 193 | acceptEncoding = ''; | |
| 122 | 194 | } | |
| 123 | 195 | ||
| 196 | + const onError = (err) => { | ||
| 197 | + if (err) { | ||
| 198 | + // If an error occurs, there's not much we can do because | ||
| 199 | + // the server has already sent the 200 response code and | ||
| 200 | + // some amount of data has already been sent to the client. | ||
| 201 | + // The best we can do is terminate the response immediately | ||
| 202 | + // and log the error. | ||
| 203 | + response.end(); | ||
| 204 | + console.error('An error occurred:', err); | ||
| 205 | + } | ||
| 206 | + }; | ||
| 207 | + | ||
| 124 | 208 | // Note: This is not a conformant accept-encoding parser. | |
| 125 | 209 | // See https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.3 | |
| 126 | 210 | if (/\bdeflate\b/.test(acceptEncoding)) { | |
| 127 | 211 | response.writeHead(200, { 'Content-Encoding': 'deflate' }); | |
| 128 | - raw.pipe(zlib.createDeflate()).pipe(response); | ||
| 212 | + pipeline(raw, zlib.createDeflate(), response, onError); | ||
| 129 | 213 | } else if (/\bgzip\b/.test(acceptEncoding)) { | |
| 130 | 214 | response.writeHead(200, { 'Content-Encoding': 'gzip' }); | |
| 131 | - raw.pipe(zlib.createGzip()).pipe(response); | ||
| 215 | + pipeline(raw, zlib.createGzip(), response, onError); | ||
| 132 | 216 | } else if (/\bbr\b/.test(acceptEncoding)) { | |
| 133 | 217 | response.writeHead(200, { 'Content-Encoding': 'br' }); | |
| 134 | - raw.pipe(zlib.createBrotliCompress()).pipe(response); | ||
| 218 | + pipeline(raw, zlib.createBrotliCompress(), response, onError); | ||
| 135 | 219 | } else { | |
| 136 | 220 | response.writeHead(200, {}); | |
| 137 | - raw.pipe(response); | ||
| 221 | + pipeline(raw, response, onError); | ||
| 138 | 222 | } | |
| 139 | 223 | }).listen(1337); | |
| 140 | 224 | ``` | |
@@ -154,11 +238,11 @@ zlib.unzip( | |||
| 154 | 238 | // For Brotli, the equivalent is zlib.constants.BROTLI_OPERATION_FLUSH. | |
| 155 | 239 | { finishFlush: zlib.constants.Z_SYNC_FLUSH }, | |
| 156 | 240 | (err, buffer) => { | |
| 157 | - if (!err) { | ||
| 158 | - console.log(buffer.toString()); | ||
| 159 | - } else { | ||
| 160 | - // handle error | ||
| 241 | + if (err) { | ||
| 242 | + console.error('An error occurred:', err); | ||
| 243 | + process.exitCode = 1; | ||
| 161 | 244 | } | |
| 245 | + console.log(buffer.toString()); | ||
| 162 | 246 | }); | |
| 163 | 247 | ``` | |
| 164 | 248 | ||
@@ -234,14 +318,28 @@ HTTP response to the client: | |||
| 234 | 318 | ```js | |
| 235 | 319 | const zlib = require('zlib'); | |
| 236 | 320 | const http = require('http'); | |
| 321 | + const { pipeline } = require('stream'); | ||
| 237 | 322 | ||
| 238 | 323 | http.createServer((request, response) => { | |
| 239 | 324 | // For the sake of simplicity, the Accept-Encoding checks are omitted. | |
| 240 | 325 | response.writeHead(200, { 'content-encoding': 'gzip' }); | |
| 241 | 326 | const output = zlib.createGzip(); | |
| 242 | - output.pipe(response); | ||
| 327 | + let i; | ||
| 328 | + | ||
| 329 | + pipeline(output, response, (err) => { | ||
| 330 | + if (err) { | ||
| 331 | + // If an error occurs, there's not much we can do because | ||
| 332 | + // the server has already sent the 200 response code and | ||
| 333 | + // some amount of data has already been sent to the client. | ||
| 334 | + // The best we can do is terminate the response immediately | ||
| 335 | + // and log the error. | ||
| 336 | + clearInterval(i); | ||
| 337 | + response.end(); | ||
| 338 | + console.error('An error occurred:', err); | ||
| 339 | + } | ||
| 340 | + }); | ||
| 243 | 341 | ||
| 244 | - setInterval(() => { | ||
| 342 | + i = setInterval(() => { | ||
| 245 | 343 | output.write(`The current time is ${Date()}\n`, () => { | |
| 246 | 344 | // The data has been passed to zlib, but the compression algorithm may | |
| 247 | 345 | // have decided to buffer the data for more efficient compression. | |
@@ -399,7 +497,7 @@ changes: | |||
| 399 | 497 | ||
| 400 | 498 | <!--type=misc--> | |
| 401 | 499 | ||
| 402 | - Each zlib-based class takes an `options` object. All options are optional. | ||
| 500 | + Each zlib-based class takes an `options` object. No options are required. | ||
| 403 | 501 | ||
| 404 | 502 | Some options are only relevant when compressing and are | |
| 405 | 503 | ignored by the decompression classes. | |
@@ -1058,6 +1156,6 @@ Decompress a chunk of data with [`Unzip`][]. | |||
| 1058 | 1156 | [Brotli parameters]: #zlib_brotli_constants | |
| 1059 | 1157 | [Memory Usage Tuning]: #zlib_memory_usage_tuning | |
| 1060 | 1158 | [RFC 7932]: https://www.rfc-editor.org/rfc/rfc7932.txt | |
| 1061 | - [pool size]: cli.html#cli_uv_threadpool_size_size | ||
| 1159 | + [Streams API]: stream.md | ||
| 1062 | 1160 | [zlib documentation]: https://zlib.net/manual.html#Constants | |
| 1063 | 1161 | [zlib.createGzip example]: #zlib_zlib | |
| Back | FazBrowse Home | New Git URL |
0 commit comments