Skip to content

Commit 03412dc

Browse files
committed
stream: update TextEncoderStream to align the latest spec
1 parent 948712b commit 03412dc

File tree

2 files changed

+65
-8
lines changed

2 files changed

+65
-8
lines changed

lib/internal/webstreams/encoding.js

+36-6
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
const {
44
ObjectDefineProperties,
55
Symbol,
6+
String,
7+
Uint8Array,
68
} = primordials;
79

810
const {
@@ -31,6 +33,7 @@ const {
3133
const kHandle = Symbol('kHandle');
3234
const kTransform = Symbol('kTransform');
3335
const kType = Symbol('kType');
36+
const kPendingHighSurrogate = Symbol('kPendingHighSurrogate');
3437

3538
/**
3639
* @typedef {import('./readablestream').ReadableStream} ReadableStream
@@ -49,19 +52,46 @@ function isTextDecoderStream(value) {
4952

5053
class TextEncoderStream {
5154
constructor() {
55+
this[kPendingHighSurrogate] = null;
5256
this[kType] = 'TextEncoderStream';
5357
this[kHandle] = new TextEncoder();
5458
this[kTransform] = new TransformStream({
5559
transform: (chunk, controller) => {
56-
const value = this[kHandle].encode(chunk);
57-
if (value)
60+
// https://encoding.spec.whatwg.org/#encode-and-enqueue-a-chunk
61+
chunk = String(chunk);
62+
let finalChunk = '';
63+
for (let i = 0; i < chunk.length; i++) {
64+
const item = chunk[i];
65+
const codeUnit = item.charCodeAt(0);
66+
if (this[kPendingHighSurrogate] !== null) {
67+
const highSurrogate = this[kPendingHighSurrogate];
68+
this[kPendingHighSurrogate] = null;
69+
if (0xDC00 <= codeUnit && codeUnit <= 0xDFFF) {
70+
finalChunk += highSurrogate + item;
71+
continue;
72+
}
73+
finalChunk += '\uFFFD';
74+
}
75+
if (0xD800 <= codeUnit && codeUnit <= 0xDBFF) {
76+
this[kPendingHighSurrogate] = item;
77+
continue;
78+
}
79+
if (0xDC00 <= codeUnit && codeUnit <= 0xDFFF) {
80+
finalChunk += '\uFFFD';
81+
continue;
82+
}
83+
finalChunk += item;
84+
}
85+
if (finalChunk) {
86+
const value = this[kHandle].encode(finalChunk);
5887
controller.enqueue(value);
88+
}
5989
},
6090
flush: (controller) => {
61-
const value = this[kHandle].encode();
62-
if (value.byteLength > 0)
63-
controller.enqueue(value);
64-
controller.terminate();
91+
// https://encoding.spec.whatwg.org/#encode-and-flush
92+
if (this[kPendingHighSurrogate] !== null) {
93+
controller.enqueue(new Uint8Array([0xEF, 0xBF, 0xBD]));
94+
}
6595
},
6696
});
6797
}

test/wpt/status/encoding.json

+29-2
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,35 @@
4848
"unsupported-encodings.any.js": {
4949
"skip": "decoding-helpers.js needs XMLHttpRequest"
5050
},
51-
"streams/*.js": {
52-
"fail": "No implementation of TextDecoderStream and TextEncoderStream"
51+
"streams/decode-ignore-bom.any.js": {
52+
"requires": ["small-icu"]
53+
},
54+
"streams/realms.window.js": {
55+
"skip": "window is not defined"
56+
},
57+
"streams/decode-attributes.any.js": {
58+
"requires": ["full-icu"]
59+
},
60+
"streams/decode-incomplete-input.any.js": {
61+
"requires": ["small-icu"]
62+
},
63+
"streams/decode-utf8.any.js": {
64+
"requires": ["small-icu"],
65+
"fail": {
66+
"unexpected": [
67+
"promise_test: Unhandled rejection with value: object 'TypeError: Cannot perform Construct on a detached ArrayBuffer'"
68+
]
69+
}
70+
},
71+
"streams/decode-bad-chunks.any.js": {
72+
"fail": {
73+
"unexpected": [
74+
"assert_unreached: Should have rejected: write should reject Reached unreachable code"
75+
]
76+
}
77+
},
78+
"streams/decode-non-utf8.any.js": {
79+
"requires": ["full-icu"]
5380
},
5481
"encodeInto.any.js": {
5582
"requires": ["small-icu"]

0 commit comments

Comments
 (0)