Skip to content
rachel/workerd
File

Blob: src/workerd/api/node/i18n.c++

8.9 KB
1// Copyright (c) 2017-2022 Cloudflare, Inc.
2// Licensed under the Apache 2.0 license found in the LICENSE file or at:
3//     https://opensource.org/licenses/Apache-2.0
4// Copyright Joyent and Node contributors. All rights reserved. MIT license.
5 
6#include "i18n.h"
7 
8#include "simdutf.h"
9 
10#include <workerd/jsg/exception.h>
11 
12#include <unicode/ucnv.h>
13#include <unicode/uidna.h>
14#include <unicode/urename.h>
15#include <unicode/utypes.h>
16#include <unicode/uvernum.h>
17#include <unicode/uversion.h>
18 
19namespace workerd::api::node {
20 
21namespace i18n {
22 
23namespace {
24 
25// An isolate has a 128mb memory limit.
26constexpr int ISOLATE_LIMIT = 134217728;
27 
28constexpr const char* getEncodingName(Encoding input) {
29  switch (input) {
30    case Encoding::ASCII:
31      return "us-ascii";
32    case Encoding::LATIN1:
33      return "iso8859-1";
34    case Encoding::UTF16LE:
35      return "utf16le";
36    case Encoding::UTF8:
37      return "utf-8";
38    default:
39      KJ_UNREACHABLE;
40  }
41}
42 
43using TranscodeImpl = kj::Function<kj::Maybe<jsg::JsUint8Array>(
44    jsg::Lock& js, kj::ArrayPtr<kj::byte> source, Encoding fromEncoding, Encoding toEncoding)>;
45 
46kj::Maybe<jsg::JsUint8Array> TranscodeDefault(
47    jsg::Lock& js, kj::ArrayPtr<kj::byte> source, Encoding fromEncoding, Encoding toEncoding) {
48  Converter to(toEncoding);
49  auto substitute = kj::str(kj::repeat('?', to.minCharSize()));
50  to.setSubstituteChars(substitute);
51  Converter from(fromEncoding);
52 
53  size_t limit = source.size() * to.maxCharSize();
54  if (limit == 0) {
55    return jsg::JsUint8Array::create(js, 0);
56  }
57  // Workers are limited to 128MB so this isn't actually a realistic concern, but sanity check.
58  JSG_REQUIRE(limit <= ISOLATE_LIMIT, Error, "Source buffer is too large to transcode");
59 
60  auto out = jsg::JsUint8Array::create(js, limit);
61  auto outPtr = out.asArrayPtr().asChars();
62  char* target = outPtr.begin();
63  const char* source_ = source.asChars().begin();
64  UErrorCode status{};
65  ucnv_convertEx(to.conv(), from.conv(), &target, target + limit, &source_, source_ + source.size(),
66      nullptr, nullptr, nullptr, nullptr, true, true, &status);
67  if (U_SUCCESS(status)) {
68    auto written = target - outPtr.begin();
69    if (written < out.size()) {
70      return out.slice(js, written);
71    }
72    return out;
73  }
74 
75  return kj::none;
76}
77 
78kj::Maybe<jsg::JsUint8Array> TranscodeLatin1ToUTF16(
79    jsg::Lock& js, kj::ArrayPtr<kj::byte> source, Encoding fromEncoding, Encoding toEncoding) {
80  auto length_in_chars = source.size() * sizeof(char16_t);
81  // Workers are limited to 128MB so this isn't actually a realistic concern, but sanity check.
82  JSG_REQUIRE(length_in_chars <= ISOLATE_LIMIT, Error, "Source buffer is too large to transcode");
83 
84  if (length_in_chars == 0) {
85    return jsg::JsUint8Array::create(js, 0);
86  }
87 
88  Converter from(fromEncoding);
89  auto destBuf = jsg::JsUint8Array::create(js, length_in_chars);
90  auto destPtr = destBuf.asArrayPtr<char16_t>();
91  auto actual_length =
92      simdutf::convert_latin1_to_utf16(source.asChars().begin(), source.size(), destPtr.begin());
93 
94  // simdutf returns 0 for invalid value.
95  if (actual_length == 0) {
96    return kj::none;
97  }
98 
99  auto written = actual_length * sizeof(char16_t);
100  if (written < destBuf.size()) {
101    return destBuf.slice(js, written);
102  }
103  return destBuf;
104}
105 
106kj::Maybe<jsg::JsUint8Array> TranscodeFromUTF16(
107    jsg::Lock& js, kj::ArrayPtr<kj::byte> source, Encoding fromEncoding, Encoding toEncoding) {
108  Converter to(toEncoding);
109  auto substitute = kj::str(kj::repeat('?', to.minCharSize()));
110  to.setSubstituteChars(substitute);
111 
112  JSG_REQUIRE(
113      source.size() % sizeof(char16_t) == 0, Error, "UTF-16le input size should be multiple of 2");
114  auto utf16_input = kj::arrayPtr<char16_t>(
115      reinterpret_cast<char16_t*>(source.begin()), source.size() / sizeof(char16_t));
116 
117  const auto limit = utf16_input.size() * to.maxCharSize();
118 
119  // Workers are limited to 128MB so this isn't actually a realistic concern, but sanity check.
120  JSG_REQUIRE(limit <= ISOLATE_LIMIT, Error, "Buffer is too large to transcode");
121 
122  if (limit == 0) {
123    return jsg::JsUint8Array::create(js, 0);
124  }
125 
126  auto destBuf = jsg::JsUint8Array::create(js, limit);
127  auto destPtr = destBuf.asArrayPtr().asChars();
128  UErrorCode status{};
129  auto len = ucnv_fromUChars(
130      to.conv(), destPtr.begin(), destPtr.size(), utf16_input.begin(), utf16_input.size(), &status);
131 
132  if (U_SUCCESS(status)) {
133    if (len < destBuf.size()) {
134      destBuf = destBuf.slice(js, len);
135    }
136    return destBuf;
137  }
138 
139  return kj::none;
140}
141 
142kj::Maybe<jsg::JsUint8Array> TranscodeUTF16FromUTF8(
143    jsg::Lock& js, kj::ArrayPtr<kj::byte> source, Encoding fromEncoding, Encoding toEncoding) {
144  size_t expected_utf16_length =
145      simdutf::utf16_length_from_utf8(source.asChars().begin(), source.size());
146  // Workers are limited to 128MB so this isn't actually a realistic concern, but sanity check.
147  JSG_REQUIRE(expected_utf16_length <= ISOLATE_LIMIT, Error,
148      "Expected UTF-16le length is too large to transcode");
149 
150  auto length_in_chars = expected_utf16_length * sizeof(char16_t);
151  if (length_in_chars == 0) {
152    return jsg::JsUint8Array::create(js, 0);
153  }
154 
155  auto destBuf = jsg::JsUint8Array::create(js, length_in_chars);
156  auto destPtr = destBuf.asArrayPtr<char16_t>();
157 
158  size_t actual_length =
159      simdutf::convert_utf8_to_utf16le(source.asChars().begin(), source.size(), destPtr.begin());
160 
161  // simdutf returns 0 for invalid UTF-8 value.
162  if (actual_length == 0) {
163    return kj::none;
164  }
165 
166  JSG_REQUIRE(actual_length == expected_utf16_length, Error, "Expected UTF16 length mismatch");
167 
168  return destBuf;
169}
170 
171kj::Maybe<jsg::JsUint8Array> TranscodeUTF8FromUTF16(
172    jsg::Lock& js, kj::ArrayPtr<kj::byte> source, Encoding fromEncoding, Encoding toEncoding) {
173  JSG_REQUIRE(source.size() % 2 == 0, Error, "UTF-16le input size should be multiple of 2");
174  auto utf16_input =
175      kj::arrayPtr<char16_t>(reinterpret_cast<char16_t*>(source.begin()), source.size() / 2);
176  size_t expected_utf8_length =
177      simdutf::utf8_length_from_utf16le(utf16_input.begin(), utf16_input.size());
178 
179  // Workers are limited to 128MB so this isn't actually a realistic concern, but sanity check.
180  JSG_REQUIRE(expected_utf8_length <= ISOLATE_LIMIT, Error,
181      "Expected UTF-8 length is too large to transcode");
182 
183  if (expected_utf8_length == 0) {
184    return jsg::JsUint8Array::create(js, 0);
185  }
186 
187  auto destBuf = jsg::JsUint8Array::create(js, expected_utf8_length);
188  auto destPtr = destBuf.asArrayPtr().asChars();
189 
190  size_t actual_length =
191      simdutf::convert_utf16le_to_utf8(utf16_input.begin(), utf16_input.size(), destPtr.begin());
192  JSG_REQUIRE(actual_length == expected_utf8_length, Error, "Expected UTF8 length mismatch");
193 
194  // simdutf returns 0 for invalid UTF-8 value.
195  if (actual_length == 0) {
196    return kj::none;
197  }
198 
199  return destBuf;
200}
201 
202}  // namespace
203 
204Converter::Converter(Encoding encoding, kj::StringPtr substitute) {
205  UErrorCode status = U_ZERO_ERROR;
206  auto name = getEncodingName(encoding);
207  auto conv = ucnv_open(name, &status);
208  JSG_REQUIRE(U_SUCCESS(status), Error, "Failed to initialize converter");
209  conv_ = kj::disposeWith<ucnv_close>(conv);
210  setSubstituteChars(substitute);
211}
212 
213UConverter* Converter::conv() const {
214  return const_cast<UConverter*>(conv_.get());
215}
216 
217size_t Converter::maxCharSize() const {
218  KJ_ASSERT_NONNULL(conv_.get());
219  return ucnv_getMaxCharSize(conv_.get());
220}
221 
222size_t Converter::minCharSize() const {
223  KJ_ASSERT_NONNULL(conv_.get());
224  return ucnv_getMinCharSize(conv_.get());
225}
226 
227void Converter::reset() {
228  KJ_ASSERT_NONNULL(conv_.get());
229  ucnv_reset(conv_.get());
230}
231 
232void Converter::setSubstituteChars(kj::StringPtr sub) {
233  KJ_ASSERT_NONNULL(conv_.get());
234  UErrorCode status = U_ZERO_ERROR;
235  if (sub.size() > 0) {
236    ucnv_setSubstChars(conv_.get(), sub.begin(), sub.size(), &status);
237    JSG_REQUIRE(U_SUCCESS(status), Error, "Setting ICU substitute characters failed");
238  }
239}
240 
241jsg::JsUint8Array transcode(
242    jsg::Lock& js, kj::ArrayPtr<kj::byte> source, Encoding fromEncoding, Encoding toEncoding) {
243  TranscodeImpl transcode_function = &TranscodeDefault;
244  switch (fromEncoding) {
245    case Encoding::ASCII:
246    case Encoding::LATIN1:
247      if (toEncoding == Encoding::UTF16LE) {
248        transcode_function = &TranscodeLatin1ToUTF16;
249      }
250      break;
251    case Encoding::UTF8:
252      if (toEncoding == Encoding::UTF16LE) {
253        transcode_function = &TranscodeUTF16FromUTF8;
254      }
255      break;
256    case Encoding::UTF16LE:
257      switch (toEncoding) {
258        case Encoding::UTF16LE:
259          transcode_function = &TranscodeDefault;
260          break;
261        case Encoding::UTF8:
262          transcode_function = &TranscodeUTF8FromUTF16;
263          break;
264        default:
265          transcode_function = &TranscodeFromUTF16;
266      }
267      break;
268    default:
269      JSG_FAIL_REQUIRE(Error, "Invalid encoding passed to transcode");
270  }
271 
272  return JSG_REQUIRE_NONNULL(transcode_function(js, source, fromEncoding, toEncoding), Error,
273      "Unable to transcode buffer");
274}
275 
276}  // namespace i18n
277 
278}  // namespace workerd::api::node