FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

src: implement Windows-1252 encoding support and update related tests · nodejs/node@7484a34 · GitHub

/ node Public

Commit 7484a34

Browse files
mertcanaltin
authored and
aduh95
committed
src: implement Windows-1252 encoding support and update related tests
PR-URL: #60893 Fixes: #60888 Fixes: #59515 Fixes: #56542 Reviewed-By: Matteo Collina <matteo.collina@gmail.com> Reviewed-By: Rafael Gonzaga <rafael.nunu@hotmail.com>
1 parent 49f7a8c commit 7484a34

6 files changed

Lines changed: 121 additions & 44 deletions

File tree

‎lib/internal/encoding.js‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ const kDecoder = Symbol('decoder');
2929
const kEncoder = Symbol('encoder');
3030
const kFatal = Symbol('kFatal');
3131
const kUTF8FastPath = Symbol('kUTF8FastPath');
32-
const kLatin1FastPath = Symbol('kLatin1FastPath');
32+
const kWindows1252FastPath = Symbol('kWindows1252FastPath');
3333
const kIgnoreBOM = Symbol('kIgnoreBOM');
3434

3535
const {
@@ -56,7 +56,7 @@ const {
5656
encodeIntoResults,
5757
encodeUtf8String,
5858
decodeUTF8,
59-
decodeLatin1,
59+
decodeWindows1252,
6060
} = binding;
6161

6262
const { Buffer } = require('buffer');
@@ -421,10 +421,10 @@ function makeTextDecoderICU() {
421421
this[kFatal] = Boolean(options?.fatal);
422422
// Only support fast path for UTF-8.
423423
this[kUTF8FastPath] = enc === 'utf-8';
424-
this[kLatin1FastPath] = enc === 'windows-1252';
424+
this[kWindows1252FastPath] = enc === 'windows-1252';
425425
this[kHandle] = undefined;
426426

427-
if (!this[kUTF8FastPath] && !this[kLatin1FastPath]) {
427+
if (!this[kUTF8FastPath] && !this[kWindows1252FastPath]) {
428428
this.#prepareConverter();
429429
}
430430
}
@@ -441,14 +441,14 @@ function makeTextDecoderICU() {
441441
validateDecoder(this);
442442

443443
this[kUTF8FastPath] &&= !(options?.stream);
444-
this[kLatin1FastPath] &&= !(options?.stream);
444+
this[kWindows1252FastPath] &&= !(options?.stream);
445445

446446
if (this[kUTF8FastPath]) {
447447
return decodeUTF8(input, this[kIgnoreBOM], this[kFatal]);
448448
}
449449

450-
if (this[kLatin1FastPath]) {
451-
return decodeLatin1(input, this[kIgnoreBOM], this[kFatal]);
450+
if (this[kWindows1252FastPath]) {
451+
return decodeWindows1252(input, this[kIgnoreBOM], this[kFatal]);
452452
}
453453

454454
this.#prepareConverter();

‎src/encoding_binding.cc‎

Lines changed: 39 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -221,7 +221,8 @@ void BindingData::CreatePerIsolateProperties(IsolateData* isolate_data,
221221
SetMethodNoSideEffect(isolate, target, "decodeUTF8", DecodeUTF8);
222222
SetMethodNoSideEffect(isolate, target, "toASCII", ToASCII);
223223
SetMethodNoSideEffect(isolate, target, "toUnicode", ToUnicode);
224-
SetMethodNoSideEffect(isolate, target, "decodeLatin1", DecodeLatin1);
224+
SetMethodNoSideEffect(
225+
isolate, target, "decodeWindows1252", DecodeWindows1252);
225226
}
226227

227228
void BindingData::CreatePerContextProperties(Local<Object> target,
@@ -239,10 +240,10 @@ void BindingData::RegisterTimerExternalReferences(
239240
registry->Register(DecodeUTF8);
240241
registry->Register(ToASCII);
241242
registry->Register(ToUnicode);
242-
registry->Register(DecodeLatin1);
243+
registry->Register(DecodeWindows1252);
243244
}
244245

245-
void BindingData::DecodeLatin1(const FunctionCallbackInfo<Value>& args) {
246+
void BindingData::DecodeWindows1252(const FunctionCallbackInfo<Value>& args) {
246247
Environment* env = Environment::GetCurrent(args);
247248

248249
CHECK_GE(args.Length(), 1);
@@ -255,7 +256,6 @@ void BindingData::DecodeLatin1(const FunctionCallbackInfo<Value>& args) {
255256
}
256257

257258
bool ignore_bom = args[1]->IsTrue();
258-
bool has_fatal = args[2]->IsTrue();
259259

260260
ArrayBufferViewContents<uint8_t> buffer(args[0]);
261261
const uint8_t* data = buffer.data();
@@ -270,20 +270,45 @@ void BindingData::DecodeLatin1(const FunctionCallbackInfo<Value>& args) {
270270
return args.GetReturnValue().SetEmptyString();
271271
}
272272

273-
std::string result(length * 2, '\0');
274-
275-
size_t written = simdutf::convert_latin1_to_utf8(
276-
reinterpret_cast<const char*>(data), length, result.data());
273+
// Windows-1252 specific mapping for bytes 128-159
274+
// These differ from Latin-1/ISO-8859-1
275+
static const uint16_t windows1252_mapping[32] = {
276+
0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, // 80-87
277+
0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008D, 0x017D, 0x008F, // 88-8F
278+
0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, // 90-97
279+
0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x009D, 0x017E, 0x0178 // 98-9F
280+
};
281+
282+
std::string result;
283+
result.reserve(length * 3); // Reserve space for UTF-8 output
284+
285+
for (size_t i = 0; i < length; i++) {
286+
uint8_t byte = data[i];
287+
uint32_t codepoint;
288+
289+
// Check if byte is in the special Windows-1252 range (128-159)
290+
if (byte >= 0x80 && byte <= 0x9F) {
291+
codepoint = windows1252_mapping[byte - 0x80];
292+
} else {
293+
// For all other bytes, Windows-1252 is identical to Latin-1
294+
codepoint = byte;
295+
}
277296

278-
if (has_fatal && written == 0) {
279-
return node::THROW_ERR_ENCODING_INVALID_ENCODED_DATA(
280-
env->isolate(), "The encoded data was not valid for encoding latin1");
297+
// Convert codepoint to UTF-8
298+
if (codepoint < 0x80) {
299+
result.push_back(static_cast<char>(codepoint));
300+
} else if (codepoint < 0x800) {
301+
result.push_back(static_cast<char>(0xC0 | (codepoint >> 6)));
302+
result.push_back(static_cast<char>(0x80 | (codepoint & 0x3F)));
303+
} else {
304+
result.push_back(static_cast<char>(0xE0 | (codepoint >> 12)));
305+
result.push_back(static_cast<char>(0x80 | ((codepoint >> 6) & 0x3F)));
306+
result.push_back(static_cast<char>(0x80 | (codepoint & 0x3F)));
307+
}
281308
}
282309

283-
std::string_view view(result.c_str(), written);
284-
285310
Local<Value> ret;
286-
if (ToV8Value(env->context(), view, env->isolate()).ToLocal(&ret)) {
311+
if (ToV8Value(env->context(), result, env->isolate()).ToLocal(&ret)) {
287312
args.GetReturnValue().Set(ret);
288313
}
289314
}

‎src/encoding_binding.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@ class BindingData : public SnapshotableObject {
3131
static void EncodeInto(const v8::FunctionCallbackInfo<v8::Value>& args);
3232
static void EncodeUtf8String(const v8::FunctionCallbackInfo<v8::Value>& args);
3333
static void DecodeUTF8(const v8::FunctionCallbackInfo<v8::Value>& args);
34-
static void DecodeLatin1(const v8::FunctionCallbackInfo<v8::Value>& args);
34+
static void DecodeWindows1252(
35+
const v8::FunctionCallbackInfo<v8::Value>& args);
3536

3637
static void ToASCII(const v8::FunctionCallbackInfo<v8::Value>& args);
3738
static void ToUnicode(const v8::FunctionCallbackInfo<v8::Value>& args);

‎test/parallel/test-internal-encoding-binding.js‎

Lines changed: 26 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -8,41 +8,46 @@ const assert = require('node:assert');
88
const { internalBinding } = require('internal/test/binding');
99
const binding = internalBinding('encoding_binding');
1010

11+
// Windows-1252 specific tests
1112
{
12-
// Valid input
13-
const buf = Uint8Array.from([0xC1, 0xE9, 0xF3]);
14-
assert.strictEqual(binding.decodeLatin1(buf, false, false), 'Áéó');
13+
// Test Windows-1252 special characters in 128-159 range
14+
// These differ from Latin-1
15+
assert.strictEqual(binding.decodeWindows1252(Uint8Array.of(0x80), false, false), '€');
16+
assert.strictEqual(binding.decodeWindows1252(Uint8Array.of(0x82), false, false), '‚');
17+
assert.strictEqual(binding.decodeWindows1252(Uint8Array.of(0x83), false, false), 'ƒ');
18+
assert.strictEqual(binding.decodeWindows1252(Uint8Array.of(0x9F), false, false), 'Ÿ');
1519
}
1620

1721
{
18-
// Empty input
19-
const buf = Uint8Array.from([]);
20-
assert.strictEqual(binding.decodeLatin1(buf, false, false), '');
22+
// Test Windows-1252 characters outside 128-159 range (same as Latin-1)
23+
const buf = Uint8Array.from([0xC1, 0xE9, 0xF3]);
24+
assert.strictEqual(binding.decodeWindows1252(buf, false, false), 'Áéó');
2125
}
2226

2327
{
24-
// Invalid input, but Latin1 has no invalid chars and should never throw.
25-
const buf = new TextEncoder().encode('Invalid Latin1 🧑‍🧑‍🧒‍🧒');
26-
assert.strictEqual(
27-
binding.decodeLatin1(buf, false, false),
28-
'Invalid Latin1 ð\x9F§\x91â\x80\x8Dð\x9F§\x91â\x80\x8Dð\x9F§\x92â\x80\x8Dð\x9F§\x92'
29-
);
28+
// Empty input
29+
const buf = Uint8Array.from([]);
30+
assert.strictEqual(binding.decodeWindows1252(buf, false, false), '');
3031
}
3132

33+
// Windows-1252 specific tests
3234
{
33-
// IgnoreBOM with BOM
34-
const buf = Uint8Array.from([0xFE, 0xFF, 0xC1, 0xE9, 0xF3]);
35-
assert.strictEqual(binding.decodeLatin1(buf, true, false), 'þÿÁéó');
35+
// Test Windows-1252 special characters in 128-159 range
36+
// These differ from Latin-1
37+
assert.strictEqual(binding.decodeWindows1252(Uint8Array.of(0x80), false, false), '€');
38+
assert.strictEqual(binding.decodeWindows1252(Uint8Array.of(0x82), false, false), '‚');
39+
assert.strictEqual(binding.decodeWindows1252(Uint8Array.of(0x83), false, false), 'ƒ');
40+
assert.strictEqual(binding.decodeWindows1252(Uint8Array.of(0x9F), false, false), 'Ÿ');
3641
}
3742

3843
{
39-
// Fatal and InvalidInput, but Latin1 has no invalid chars and should never throw.
40-
const buf = Uint8Array.from([0xFF, 0xFF, 0xFF]);
41-
assert.strictEqual(binding.decodeLatin1(buf, false, true), 'ÿÿÿ');
44+
// Test Windows-1252 characters outside 128-159 range (same as Latin-1)
45+
const buf = Uint8Array.from([0xC1, 0xE9, 0xF3]);
46+
assert.strictEqual(binding.decodeWindows1252(buf, false, false), 'Áéó');
4247
}
4348

4449
{
45-
// IgnoreBOM and Fatal, but Latin1 has no invalid chars and should never throw.
46-
const buf = Uint8Array.from([0xFE, 0xFF, 0xC1, 0xE9, 0xF3]);
47-
assert.strictEqual(binding.decodeLatin1(buf, true, true), 'þÿÁéó');
50+
// Empty input
51+
const buf = Uint8Array.from([]);
52+
assert.strictEqual(binding.decodeWindows1252(buf, false, false), '');
4853
}

‎test/parallel/test-util-text-decoder.js‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,49 @@ test('TextDecoder correctly decodes windows-1252 encoded data', { skip: !common.
1515

1616
assert.strictEqual(decodedString, expectedString);
1717
});
18+
19+
// Test for the difference between Latin1 and Windows-1252 in the 128-159
20+
// range
21+
// Ref: https://github.com/nodejs/node/issues/60888
22+
test('TextDecoder correctly decodes windows-1252 special characters in ' +
23+
'128-159 range', { skip: !common.hasIntl }, () => {
24+
const decoder = new TextDecoder('windows-1252');
25+
26+
// Test specific characters that differ between Latin1 and Windows-1252.
27+
// € Euro sign
28+
assert.strictEqual(decoder.decode(Uint8Array.of(128)).codePointAt(0),
29+
8364);
30+
// ‚ Single low-9 quotation mark
31+
assert.strictEqual(decoder.decode(Uint8Array.of(130)).codePointAt(0),
32+
8218);
33+
// Latin small letter f with hook (ƒ)
34+
assert.strictEqual(decoder.decode(Uint8Array.of(131)).codePointAt(0),
35+
402);
36+
// Ÿ Latin capital letter Y with diaeresis
37+
assert.strictEqual(decoder.decode(Uint8Array.of(159)).codePointAt(0),
38+
376);
39+
40+
// Test the full range to ensure no character is treated as Latin1
41+
// Directly.
42+
const expectedMappings = [
43+
[128, 8364], [129, 129], [130, 8218], [131, 402], [132, 8222],
44+
[133, 8230], [134, 8224], [135, 8225], [136, 710], [137, 8240],
45+
[138, 352], [139, 8249], [140, 338], [141, 141], [142, 381],
46+
[143, 143], [144, 144], [145, 8216], [146, 8217], [147, 8220],
47+
[148, 8221], [149, 8226], [150, 8211], [151, 8212], [152, 732],
48+
[153, 8482], [154, 353], [155, 8250], [156, 339], [157, 157],
49+
[158, 382], [159, 376],
50+
];
51+
52+
for (const [byte, expectedCodePoint] of expectedMappings) {
53+
const result = decoder.decode(Uint8Array.of(byte));
54+
const actualCodePoint = result.codePointAt(0);
55+
assert.strictEqual(
56+
actualCodePoint,
57+
expectedCodePoint,
58+
`Byte 0x${byte.toString(16)} should decode to ` +
59+
`U+${expectedCodePoint.toString(16)} but got ` +
60+
`U+${actualCodePoint.toString(16)}`
61+
);
62+
}
63+
});

‎typings/internalBinding/encoding_binding.d.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,5 +4,5 @@ export interface EncodingBinding {
44
decodeUTF8(buffer: ArrayBufferView | ArrayBuffer | SharedArrayBuffer, ignoreBOM?: boolean, hasFatal?: boolean): string;
55
toASCII(input: string): string;
66
toUnicode(input: string): string;
7-
decodeLatin1(buffer: ArrayBufferView | ArrayBuffer | SharedArrayBuffer, ignoreBOM?: boolean, hasFatal?: boolean): string;
7+
decodeWindows1252(buffer: ArrayBufferView | ArrayBuffer | SharedArrayBuffer, ignoreBOM?: boolean, hasFatal?: boolean): string;
88
}

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL