Skip to content

Commit c4f6698

Browse files
HiroyukiYagihashiaduh95
HiroyukiYagihashi
authored and
aduh95
committed
fs: add support for async iterators to fsPromises.writeFile
Fixes: nodejs#37391 PR-URL: nodejs#37490 Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com> Reviewed-By: James M Snell <jasnell@gmail.com>
1 parent 2bcf3bc commit c4f6698

File tree

4 files changed

+144
-16
lines changed

4 files changed

+144
-16
lines changed

‎doc/api/fs.md

+5-1
Original file line numberDiff line numberDiff line change
@@ -1234,6 +1234,9 @@ All the [caveats][] for `fs.watch()` also apply to `fsPromises.watch()`.
12341234
<!-- YAML
12351235
added: v10.0.0
12361236
changes:
1237+
- version: REPLACEME
1238+
pr-url: https://github.com/nodejs/node/pull/37490
1239+
description: The `data` argument supports `AsyncIterable`, `Iterable` & `Stream`.
12371240
- version: v14.17.0
12381241
pr-url: https://github.com/nodejs/node/pull/35993
12391242
description: The options argument may include an AbortSignal to abort an
@@ -1249,7 +1252,8 @@ changes:
12491252
-->
12501253
12511254
* `file` {string|Buffer|URL|FileHandle} filename or `FileHandle`
1252-
* `data` {string|Buffer|Uint8Array|Object}
1255+
* `data` {string|Buffer|Uint8Array|Object|AsyncIterable|Iterable
1256+
|Stream}
12531257
* `options` {Object|string}
12541258
* `encoding` {string|null} **Default:** `'utf8'`
12551259
* `mode` {integer} **Default:** `0o666`

‎lib/internal/fs/promises.js

+28-8
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ const {
3434
const binding = internalBinding('fs');
3535
const { Buffer } = require('buffer');
3636

37-
const { codes, hideStackFrames } = require('internal/errors');
37+
const { AbortError, codes, hideStackFrames } = require('internal/errors');
3838
const {
3939
ERR_FS_FILE_TOO_LARGE,
4040
ERR_INVALID_ARG_TYPE,
@@ -73,6 +73,7 @@ const {
7373
const pathModule = require('path');
7474
const { promisify } = require('internal/util');
7575
const { watch } = require('internal/fs/watchers');
76+
const { isIterable } = require('internal/streams/utils');
7677

7778
const kHandle = Symbol('kHandle');
7879
const kFd = Symbol('kFd');
@@ -254,8 +255,23 @@ async function fsCall(fn, handle, ...args) {
254255
}
255256
}
256257

257-
async function writeFileHandle(filehandle, data, signal) {
258-
// `data` could be any kind of typed array.
258+
function checkAborted(signal) {
259+
if (signal && signal.aborted)
260+
throw new AbortError();
261+
}
262+
263+
async function writeFileHandle(filehandle, data, signal, encoding) {
264+
checkAborted(signal);
265+
if (isCustomIterable(data)) {
266+
for await (const buf of data) {
267+
checkAborted(signal);
268+
await write(
269+
filehandle, buf, undefined,
270+
isArrayBufferView(buf) ? buf.length : encoding);
271+
checkAborted(signal);
272+
}
273+
return;
274+
}
259275
data = new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
260276
let remaining = data.length;
261277
if (remaining === 0) return;
@@ -403,7 +419,7 @@ async function readv(handle, buffers, position) {
403419
}
404420

405421
async function write(handle, buffer, offset, length, position) {
406-
if (buffer.length === 0)
422+
if (buffer && buffer.length === 0)
407423
return { bytesWritten: 0, buffer };
408424

409425
if (isArrayBufferView(buffer)) {
@@ -645,22 +661,26 @@ async function writeFile(path, data, options) {
645661
options = getOptions(options, { encoding: 'utf8', mode: 0o666, flag: 'w' });
646662
const flag = options.flag || 'w';
647663

648-
if (!isArrayBufferView(data)) {
664+
if (!isArrayBufferView(data) && !isCustomIterable(data)) {
649665
validateStringAfterArrayBufferView(data, 'data');
650666
data = Buffer.from(data, options.encoding || 'utf8');
651667
}
652668

653669
validateAbortSignal(options.signal);
654670
if (path instanceof FileHandle)
655-
return writeFileHandle(path, data, options.signal);
671+
return writeFileHandle(path, data, options.signal, options.encoding);
656672

657673
if (options.signal?.aborted) {
658674
throw lazyDOMException('The operation was aborted', 'AbortError');
659675
}
660676

661677
const fd = await open(path, flag, options.mode);
662-
const { signal } = options;
663-
return PromisePrototypeFinally(writeFileHandle(fd, data, signal), fd.close);
678+
return PromisePrototypeFinally(
679+
writeFileHandle(fd, data, options.signal, options.encoding), fd.close);
680+
}
681+
682+
function isCustomIterable(obj) {
683+
return isIterable(obj) && !isArrayBufferView(obj) && typeof obj !== 'string';
664684
}
665685

666686
async function appendFile(path, data, options) {

‎test/parallel/test-fs-append-file.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ const throwNextTick = (e) => { process.nextTick(() => { throw e; }); };
121121
}
122122

123123
// Test that appendFile does not accept invalid data type (callback API).
124-
[false, 5, {}, [], null, undefined].forEach(async (data) => {
124+
[false, 5, {}, null, undefined].forEach(async (data) => {
125125
const errObj = {
126126
code: 'ERR_INVALID_ARG_TYPE',
127127
message: /"data"|"buffer"/

‎test/parallel/test-fs-promises-writefile.js

+110-6
Original file line numberDiff line numberDiff line change
@@ -8,20 +8,115 @@ const path = require('path');
88
const tmpdir = require('../common/tmpdir');
99
const assert = require('assert');
1010
const tmpDir = tmpdir.path;
11+
const { Readable } = require('stream');
1112

1213
tmpdir.refresh();
1314

1415
const dest = path.resolve(tmpDir, 'tmp.txt');
1516
const otherDest = path.resolve(tmpDir, 'tmp-2.txt');
1617
const buffer = Buffer.from('abc'.repeat(1000));
1718
const buffer2 = Buffer.from('xyz'.repeat(1000));
19+
const stream = Readable.from(['a', 'b', 'c']);
20+
const stream2 = Readable.from(['ümlaut', ' ', 'sechzig']);
21+
const iterable = {
22+
expected: 'abc',
23+
*[Symbol.iterator]() {
24+
yield 'a';
25+
yield 'b';
26+
yield 'c';
27+
}
28+
};
29+
function iterableWith(value) {
30+
return {
31+
*[Symbol.iterator]() {
32+
yield value;
33+
}
34+
};
35+
}
36+
const bufferIterable = {
37+
expected: 'abc',
38+
*[Symbol.iterator]() {
39+
yield Buffer.from('a');
40+
yield Buffer.from('b');
41+
yield Buffer.from('c');
42+
}
43+
};
44+
const asyncIterable = {
45+
expected: 'abc',
46+
async* [Symbol.asyncIterator]() {
47+
yield 'a';
48+
yield 'b';
49+
yield 'c';
50+
}
51+
};
1852

1953
async function doWrite() {
2054
await fsPromises.writeFile(dest, buffer);
2155
const data = fs.readFileSync(dest);
2256
assert.deepStrictEqual(data, buffer);
2357
}
2458

59+
async function doWriteStream() {
60+
await fsPromises.writeFile(dest, stream);
61+
const expected = 'abc';
62+
const data = fs.readFileSync(dest, 'utf-8');
63+
assert.deepStrictEqual(data, expected);
64+
}
65+
66+
async function doWriteStreamWithCancel() {
67+
const controller = new AbortController();
68+
const { signal } = controller;
69+
process.nextTick(() => controller.abort());
70+
assert.rejects(fsPromises.writeFile(otherDest, stream, { signal }), {
71+
name: 'AbortError'
72+
});
73+
}
74+
75+
async function doWriteIterable() {
76+
await fsPromises.writeFile(dest, iterable);
77+
const data = fs.readFileSync(dest, 'utf-8');
78+
assert.deepStrictEqual(data, iterable.expected);
79+
}
80+
81+
async function doWriteInvalidIterable() {
82+
await Promise.all(
83+
[42, 42n, {}, Symbol('42'), true, undefined, null, NaN].map((value) =>
84+
assert.rejects(fsPromises.writeFile(dest, iterableWith(value)), {
85+
code: 'ERR_INVALID_ARG_TYPE',
86+
})
87+
)
88+
);
89+
}
90+
91+
async function doWriteIterableWithEncoding() {
92+
await fsPromises.writeFile(dest, stream2, 'latin1');
93+
const expected = 'ümlaut sechzig';
94+
const data = fs.readFileSync(dest, 'latin1');
95+
assert.deepStrictEqual(data, expected);
96+
}
97+
98+
async function doWriteBufferIterable() {
99+
await fsPromises.writeFile(dest, bufferIterable);
100+
const data = fs.readFileSync(dest, 'utf-8');
101+
assert.deepStrictEqual(data, bufferIterable.expected);
102+
}
103+
104+
async function doWriteAsyncIterable() {
105+
await fsPromises.writeFile(dest, asyncIterable);
106+
const data = fs.readFileSync(dest, 'utf-8');
107+
assert.deepStrictEqual(data, asyncIterable.expected);
108+
}
109+
110+
async function doWriteInvalidValues() {
111+
await Promise.all(
112+
[42, 42n, {}, Symbol('42'), true, undefined, null, NaN].map((value) =>
113+
assert.rejects(fsPromises.writeFile(dest, value), {
114+
code: 'ERR_INVALID_ARG_TYPE',
115+
})
116+
)
117+
);
118+
}
119+
25120
async function doWriteWithCancel() {
26121
const controller = new AbortController();
27122
const { signal } = controller;
@@ -51,9 +146,18 @@ async function doReadWithEncoding() {
51146
assert.deepStrictEqual(data, syncData);
52147
}
53148

54-
doWrite()
55-
.then(doWriteWithCancel)
56-
.then(doAppend)
57-
.then(doRead)
58-
.then(doReadWithEncoding)
59-
.then(common.mustCall());
149+
(async () => {
150+
await doWrite();
151+
await doWriteWithCancel();
152+
await doAppend();
153+
await doRead();
154+
await doReadWithEncoding();
155+
await doWriteStream();
156+
await doWriteStreamWithCancel();
157+
await doWriteIterable();
158+
await doWriteInvalidIterable();
159+
await doWriteIterableWithEncoding();
160+
await doWriteBufferIterable();
161+
await doWriteAsyncIterable();
162+
await doWriteInvalidValues();
163+
})().then(common.mustCall());

0 commit comments

Comments
 (0)