Skip to content

Commit e3258fd

Browse files
jasnellcodebytere
authored andcommitted
doc: update zlib doc
Just some general improvements to zlib docs and examples Signed-off-by: James M Snell <jasnell@gmail.com> PR-URL: #31665 Reviewed-By: Anna Henningsen <anna@addaleax.net> Reviewed-By: Luigi Pinca <luigipinca@gmail.com>
1 parent 8516602 commit e3258fd

File tree

1 file changed

+144
-46
lines changed

1 file changed

+144
-46
lines changed

doc/api/zlib.md

+144-46
Original file line numberDiff line numberDiff line change
@@ -4,60 +4,121 @@
44

55
> Stability: 2 - Stable
66
7-
The `zlib` module provides compression functionality implemented using Gzip and
8-
Deflate/Inflate, as well as Brotli. It can be accessed using:
7+
The `zlib` module provides compression functionality implemented using Gzip,
8+
Deflate/Inflate, and Brotli.
9+
10+
To access it:
911

1012
```js
1113
const zlib = require('zlib');
1214
```
1315

16+
Compression and decompression are built around the Node.js [Streams API][].
17+
1418
Compressing or decompressing a stream (such as a file) can be accomplished by
15-
piping the source stream data through a `zlib` stream into a destination stream:
19+
piping the source stream through a `zlib` `Transform` stream into a destination
20+
stream:
1621

1722
```js
18-
const gzip = zlib.createGzip();
19-
const fs = require('fs');
20-
const inp = fs.createReadStream('input.txt');
21-
const out = fs.createWriteStream('input.txt.gz');
22-
23-
inp.pipe(gzip)
24-
.on('error', () => {
25-
// handle error
26-
})
27-
.pipe(out)
28-
.on('error', () => {
29-
// handle error
23+
const { createGzip } = require('zlib');
24+
const { pipeline } = require('stream');
25+
const {
26+
createReadStream,
27+
createWriteStream
28+
} = require('fs');
29+
30+
const gzip = createGzip();
31+
const source = createReadStream('input.txt');
32+
const destination = createWriteStream('input.txt.gz');
33+
34+
pipeline(source, gzip, destination, (err) => {
35+
if (err) {
36+
console.error('An error occurred:', err);
37+
process.exitCode = 1;
38+
}
39+
});
40+
41+
// Or, Promisified
42+
43+
const { promisify } = require('util');
44+
const pipe = promisify(pipeline);
45+
46+
async function do_gzip(input, output) {
47+
const gzip = createGzip();
48+
const source = createReadStream(input);
49+
const destination = createWriteStream(output);
50+
await pipe(source, gzip, destination);
51+
}
52+
53+
do_gzip('input.txt', 'input.txt.gz')
54+
.catch((err) => {
55+
console.error('An error occurred:', err);
56+
process.exitCode = 1;
3057
});
3158
```
3259

3360
It is also possible to compress or decompress data in a single step:
3461

3562
```js
63+
const { deflate, unzip } = require('zlib');
64+
3665
const input = '.................................';
37-
zlib.deflate(input, (err, buffer) => {
38-
if (!err) {
39-
console.log(buffer.toString('base64'));
40-
} else {
41-
// handle error
66+
deflate(input, (err, buffer) => {
67+
if (err) {
68+
console.error('An error occurred:', err);
69+
process.exitCode = 1;
4270
}
71+
console.log(buffer.toString('base64'));
4372
});
4473

4574
const buffer = Buffer.from('eJzT0yMAAGTvBe8=', 'base64');
46-
zlib.unzip(buffer, (err, buffer) => {
47-
if (!err) {
48-
console.log(buffer.toString());
49-
} else {
50-
// handle error
75+
unzip(buffer, (err, buffer) => {
76+
if (err) {
77+
console.error('An error occurred:', err);
78+
process.exitCode = 1;
5179
}
80+
console.log(buffer.toString());
5281
});
82+
83+
// Or, Promisified
84+
85+
const { promisify } = require('util');
86+
const do_unzip = promisify(unzip);
87+
88+
do_unzip(buffer)
89+
.then((buf) => console.log(buf.toString()))
90+
.catch((err) => {
91+
console.error('An error occurred:', err);
92+
process.exitCode = 1;
93+
});
5394
```
5495

55-
## Threadpool Usage
96+
## Threadpool Usage and Performance Considerations
97+
98+
All `zlib` APIs, except those that are explicitly synchronous, use the Node.js
99+
internal threadpool. This can lead to surprising effects and performance
100+
limitations in some applications.
56101

57-
All zlib APIs, except those that are explicitly synchronous, use libuv's
58-
threadpool. This can lead to surprising effects in some applications, such as
59-
subpar performance (which can be mitigated by adjusting the [pool size][])
60-
and/or unrecoverable and catastrophic memory fragmentation.
102+
Creating and using a large number of zlib objects simultaneously can cause
103+
significant memory fragmentation.
104+
105+
```js
106+
const zlib = require('zlib');
107+
108+
const payload = Buffer.from('This is some data');
109+
110+
// WARNING: DO NOT DO THIS!
111+
for (let i = 0; i < 30000; ++i) {
112+
zlib.deflate(payload, (err, buffer) => {});
113+
}
114+
```
115+
116+
In the preceding example, 30,000 deflate instances are created concurrently.
117+
Because of how some operating systems handle memory allocation and
118+
deallocation, this may lead to to significant memory fragmentation.
119+
120+
It is strongly recommended that the results of compression
121+
operations be cached to avoid duplication of effort.
61122

62123
## Compressing HTTP requests and responses
63124

@@ -80,26 +141,35 @@ tradeoffs involved in `zlib` usage.
80141
const zlib = require('zlib');
81142
const http = require('http');
82143
const fs = require('fs');
144+
const { pipeline } = require('stream');
145+
83146
const request = http.get({ host: 'example.com',
84147
path: '/',
85148
port: 80,
86149
headers: { 'Accept-Encoding': 'br,gzip,deflate' } });
87150
request.on('response', (response) => {
88151
const output = fs.createWriteStream('example.com_index.html');
89152

153+
const onError = (err) => {
154+
if (err) {
155+
console.error('An error occurred:', err);
156+
process.exitCode = 1;
157+
}
158+
};
159+
90160
switch (response.headers['content-encoding']) {
91161
case 'br':
92-
response.pipe(zlib.createBrotliDecompress()).pipe(output);
162+
pipeline(response, zlib.createBrotliDecompress(), output, onError);
93163
break;
94164
// Or, just use zlib.createUnzip() to handle both of the following cases:
95165
case 'gzip':
96-
response.pipe(zlib.createGunzip()).pipe(output);
166+
pipeline(response, zlib.createGunzip(), output, onError);
97167
break;
98168
case 'deflate':
99-
response.pipe(zlib.createInflate()).pipe(output);
169+
pipeline(response, zlib.createInflate(), outout, onError);
100170
break;
101171
default:
102-
response.pipe(output);
172+
pipeline(response, output, onError);
103173
break;
104174
}
105175
});
@@ -112,6 +182,8 @@ request.on('response', (response) => {
112182
const zlib = require('zlib');
113183
const http = require('http');
114184
const fs = require('fs');
185+
const { pipeline } = require('stream');
186+
115187
http.createServer((request, response) => {
116188
const raw = fs.createReadStream('index.html');
117189
// Store both a compressed and an uncompressed version of the resource.
@@ -121,20 +193,32 @@ http.createServer((request, response) => {
121193
acceptEncoding = '';
122194
}
123195

196+
const onError = (err) => {
197+
if (err) {
198+
// If an error occurs, there's not much we can do because
199+
// the server has already sent the 200 response code and
200+
// some amount of data has already been sent to the client.
201+
// The best we can do is terminate the response immediately
202+
// and log the error.
203+
response.end();
204+
console.error('An error occurred:', err);
205+
}
206+
};
207+
124208
// Note: This is not a conformant accept-encoding parser.
125209
// See https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.3
126210
if (/\bdeflate\b/.test(acceptEncoding)) {
127211
response.writeHead(200, { 'Content-Encoding': 'deflate' });
128-
raw.pipe(zlib.createDeflate()).pipe(response);
212+
pipeline(raw, zlib.createDeflate(), response, onError);
129213
} else if (/\bgzip\b/.test(acceptEncoding)) {
130214
response.writeHead(200, { 'Content-Encoding': 'gzip' });
131-
raw.pipe(zlib.createGzip()).pipe(response);
215+
pipeline(raw, zlib.createGzip(), response, onError);
132216
} else if (/\bbr\b/.test(acceptEncoding)) {
133217
response.writeHead(200, { 'Content-Encoding': 'br' });
134-
raw.pipe(zlib.createBrotliCompress()).pipe(response);
218+
pipeline(raw, zlib.createBrotliCompress(), response, onError);
135219
} else {
136220
response.writeHead(200, {});
137-
raw.pipe(response);
221+
pipeline(raw, response, onError);
138222
}
139223
}).listen(1337);
140224
```
@@ -154,11 +238,11 @@ zlib.unzip(
154238
// For Brotli, the equivalent is zlib.constants.BROTLI_OPERATION_FLUSH.
155239
{ finishFlush: zlib.constants.Z_SYNC_FLUSH },
156240
(err, buffer) => {
157-
if (!err) {
158-
console.log(buffer.toString());
159-
} else {
160-
// handle error
241+
if (err) {
242+
console.error('An error occurred:', err);
243+
process.exitCode = 1;
161244
}
245+
console.log(buffer.toString());
162246
});
163247
```
164248

@@ -234,14 +318,28 @@ HTTP response to the client:
234318
```js
235319
const zlib = require('zlib');
236320
const http = require('http');
321+
const { pipeline } = require('stream');
237322

238323
http.createServer((request, response) => {
239324
// For the sake of simplicity, the Accept-Encoding checks are omitted.
240325
response.writeHead(200, { 'content-encoding': 'gzip' });
241326
const output = zlib.createGzip();
242-
output.pipe(response);
327+
let i;
328+
329+
pipeline(output, response, (err) => {
330+
if (err) {
331+
// If an error occurs, there's not much we can do because
332+
// the server has already sent the 200 response code and
333+
// some amount of data has already been sent to the client.
334+
// The best we can do is terminate the response immediately
335+
// and log the error.
336+
clearInterval(i);
337+
response.end();
338+
console.error('An error occurred:', err);
339+
}
340+
});
243341

244-
setInterval(() => {
342+
i = setInterval(() => {
245343
output.write(`The current time is ${Date()}\n`, () => {
246344
// The data has been passed to zlib, but the compression algorithm may
247345
// have decided to buffer the data for more efficient compression.
@@ -399,7 +497,7 @@ changes:
399497

400498
<!--type=misc-->
401499

402-
Each zlib-based class takes an `options` object. All options are optional.
500+
Each zlib-based class takes an `options` object. No options are required.
403501

404502
Some options are only relevant when compressing and are
405503
ignored by the decompression classes.
@@ -1058,6 +1156,6 @@ Decompress a chunk of data with [`Unzip`][].
10581156
[Brotli parameters]: #zlib_brotli_constants
10591157
[Memory Usage Tuning]: #zlib_memory_usage_tuning
10601158
[RFC 7932]: https://www.rfc-editor.org/rfc/rfc7932.txt
1061-
[pool size]: cli.html#cli_uv_threadpool_size_size
1159+
[Streams API]: stream.md
10621160
[zlib documentation]: https://zlib.net/manual.html#Constants
10631161
[zlib.createGzip example]: #zlib_zlib

0 commit comments

Comments
 (0)