1 /*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "Unicode_test"
18
19 #include <sys/mman.h>
20 #include <unistd.h>
21
22 #include <log/log.h>
23 #include <utils/Unicode.h>
24
25 #include <gtest/gtest.h>
26
27 namespace android {
28
29 class UnicodeTest : public testing::Test {
30 protected:
SetUp()31 virtual void SetUp() {
32 }
33
TearDown()34 virtual void TearDown() {
35 }
36
37 char16_t const * const kSearchString = u"I am a leaf on the wind.";
38
39 constexpr static size_t BUFSIZE = 64; // large enough for all tests
40
TestUTF8toUTF16(std::initializer_list<uint8_t> input,std::initializer_list<char16_t> expect,const char * err_msg_length="",ssize_t expected_length=0)41 void TestUTF8toUTF16(std::initializer_list<uint8_t> input,
42 std::initializer_list<char16_t> expect,
43 const char* err_msg_length = "",
44 ssize_t expected_length = 0) {
45 uint8_t empty_str[] = {};
46 char16_t output[BUFSIZE];
47
48 const size_t inlen = input.size(), outlen = expect.size();
49 ASSERT_LT(outlen, BUFSIZE);
50
51 const uint8_t *input_data = inlen ? std::data(input) : empty_str;
52 ssize_t measured = utf8_to_utf16_length(input_data, inlen);
53 EXPECT_EQ(expected_length ? : (ssize_t)outlen, measured) << err_msg_length;
54
55 utf8_to_utf16(input_data, inlen, output, outlen + 1);
56 for (size_t i = 0; i < outlen; i++) {
57 EXPECT_EQ(std::data(expect)[i], output[i]);
58 }
59 EXPECT_EQ(0, output[outlen]) << "should be null terminated";
60 }
61
TestUTF16toUTF8(std::initializer_list<char16_t> input,std::initializer_list<char> expect,const char * err_msg_length="",ssize_t expected_length=0)62 void TestUTF16toUTF8(std::initializer_list<char16_t> input,
63 std::initializer_list<char> expect,
64 const char* err_msg_length = "",
65 ssize_t expected_length = 0) {
66 char16_t empty_str[] = {};
67 char output[BUFSIZE];
68
69 const size_t inlen = input.size(), outlen = expect.size();
70 ASSERT_LT(outlen, BUFSIZE);
71
72 const char16_t *input_data = inlen ? std::data(input) : empty_str;
73 ssize_t measured = utf16_to_utf8_length(input_data, inlen);
74 EXPECT_EQ(expected_length ? : (ssize_t)outlen, measured) << err_msg_length;
75
76 utf16_to_utf8(input_data, inlen, output, outlen + 1);
77 for (size_t i = 0; i < outlen; i++) {
78 EXPECT_EQ(std::data(expect)[i], output[i]);
79 }
80 EXPECT_EQ(0, output[outlen]) << "should be null terminated";
81 }
82 };
83
TEST_F(UnicodeTest,UTF8toUTF16ZeroLength)84 TEST_F(UnicodeTest, UTF8toUTF16ZeroLength) {
85 TestUTF8toUTF16({}, {},
86 "Zero length input should return zero length output.");
87 }
88
TEST_F(UnicodeTest,UTF8toUTF16ASCII)89 TEST_F(UnicodeTest, UTF8toUTF16ASCII) {
90 TestUTF8toUTF16(
91 { 0x30 }, // U+0030 or ASCII '0'
92 { 0x0030 },
93 "ASCII codepoints should have a length of 1 char16_t");
94 }
95
TEST_F(UnicodeTest,UTF8toUTF16Plane1)96 TEST_F(UnicodeTest, UTF8toUTF16Plane1) {
97 TestUTF8toUTF16(
98 { 0xE2, 0x8C, 0xA3 }, // U+2323 SMILE
99 { 0x2323 },
100 "Plane 1 codepoints should have a length of 1 char16_t");
101 }
102
TEST_F(UnicodeTest,UTF8toUTF16Surrogate)103 TEST_F(UnicodeTest, UTF8toUTF16Surrogate) {
104 TestUTF8toUTF16(
105 { 0xF0, 0x90, 0x80, 0x80 }, // U+10000
106 { 0xD800, 0xDC00 },
107 "Surrogate pairs should have a length of 2 char16_t");
108 }
109
TEST_F(UnicodeTest,UTF8toUTF16TruncatedUTF8)110 TEST_F(UnicodeTest, UTF8toUTF16TruncatedUTF8) {
111 TestUTF8toUTF16(
112 { 0xE2, 0x8C }, // Truncated U+2323 SMILE
113 { }, // Conversion should still work but produce nothing
114 "Truncated UTF-8 should return -1 to indicate invalid",
115 -1);
116 }
117
TEST_F(UnicodeTest,UTF8toUTF16Normal)118 TEST_F(UnicodeTest, UTF8toUTF16Normal) {
119 TestUTF8toUTF16({
120 0x30, // U+0030, 1 UTF-16 character
121 0xC4, 0x80, // U+0100, 1 UTF-16 character
122 0xE2, 0x8C, 0xA3, // U+2323, 1 UTF-16 character
123 0xF0, 0x90, 0x80, 0x80, // U+10000, 2 UTF-16 character
124 }, {
125 0x0030,
126 0x0100,
127 0x2323,
128 0xD800, 0xDC00
129 });
130 }
131
TEST_F(UnicodeTest,UTF8toUTF16Invalid)132 TEST_F(UnicodeTest, UTF8toUTF16Invalid) {
133 // TODO: The current behavior of utf8_to_utf16 is to treat invalid
134 // leading byte (>= 0xf8) as a 4-byte UTF8 sequence, and to treat
135 // invalid trailing byte(s) (i.e. bytes not having MSB set) as if
136 // they are valid and do the normal conversion. However, a better
137 // handling would be to treat invalid sequences as errors, such
138 // cases need to be reported and invalid characters (e.g. U+FFFD)
139 // could be produced at the place of error. Until a fix is ready
140 // and compatibility is not an issue, we will keep testing the
141 // current behavior
142 TestUTF8toUTF16({
143 0xf8, // invalid leading byte
144 0xc4, 0x00, // U+0100 with invalid trailing byte
145 0xe2, 0x0c, 0xa3, // U+2323 with invalid trailing bytes
146 0xf0, 0x10, 0x00, 0x00, // U+10000 with invalid trailing bytes
147 }, {
148 0x4022, // invalid leading byte (>=0xfc) is treated
149 // as valid for 4-byte UTF8 sequence
150 0x000C,
151 0x00A3, // invalid leadnig byte (b'10xxxxxx) is
152 // treated as valid single UTF-8 byte
153 0xD800, // invalid trailing bytes are treated
154 0xDC00, // as valid bytes and follow normal
155 });
156 }
157
TEST_F(UnicodeTest,UTF16toUTF8ZeroLength)158 TEST_F(UnicodeTest, UTF16toUTF8ZeroLength) {
159 // TODO: The current behavior of utf16_to_utf8_length() is that
160 // it returns -1 if the input is a zero length UTF16 string.
161 // This is inconsistent with utf8_to_utf16_length() where a zero
162 // length string returns 0. However, to fix the current behavior,
163 // we could have compatibility issue. Until then, we will keep
164 // testing the current behavior
165 TestUTF16toUTF8({}, {},
166 "Zero length UTF16 input should return length of -1.", -1);
167 }
168
TEST_F(UnicodeTest,UTF16toUTF8ASCII)169 TEST_F(UnicodeTest, UTF16toUTF8ASCII) {
170 TestUTF16toUTF8(
171 { 0x0030 }, // U+0030 or ASCII '0'
172 { '\x30' },
173 "ASCII codepoints in UTF16 should give a length of 1 in UTF8");
174 }
175
TEST_F(UnicodeTest,UTF16toUTF8Plane1)176 TEST_F(UnicodeTest, UTF16toUTF8Plane1) {
177 TestUTF16toUTF8(
178 { 0x2323 }, // U+2323 SMILE
179 { '\xE2', '\x8C', '\xA3' },
180 "Plane 1 codepoints should have a length of 3 char in UTF-8");
181 }
182
TEST_F(UnicodeTest,UTF16toUTF8Surrogate)183 TEST_F(UnicodeTest, UTF16toUTF8Surrogate) {
184 TestUTF16toUTF8(
185 { 0xD800, 0xDC00 }, // U+10000
186 { '\xF0', '\x90', '\x80', '\x80' },
187 "Surrogate pairs should have a length of 4 chars");
188 }
189
TEST_F(UnicodeTest,UTF16toUTF8UnpairedSurrogate)190 TEST_F(UnicodeTest, UTF16toUTF8UnpairedSurrogate) {
191 TestUTF16toUTF8(
192 { 0xD800 }, // U+10000 with high surrogate pair only
193 { }, // Unpaired surrogate should be ignored
194 "A single unpaired high surrogate should have a length of 0 chars");
195
196 TestUTF16toUTF8(
197 { 0xDC00 }, // U+10000 with low surrogate pair only
198 { }, // Unpaired surrogate should be ignored
199 "A single unpaired low surrogate should have a length of 0 chars");
200
201 TestUTF16toUTF8(
202 // U+0030, U+0100, U+10000 with high surrogate pair only, U+2323
203 { 0x0030, 0x0100, 0xDC00, 0x2323 },
204 { '\x30', '\xC4', '\x80', '\xE2', '\x8C', '\xA3' },
205 "Unpaired high surrogate should be skipped in the middle");
206
207 TestUTF16toUTF8(
208 // U+0030, U+0100, U+10000 with high surrogate pair only, U+2323
209 { 0x0030, 0x0100, 0xDC00, 0x2323 },
210 { '\x30', '\xC4', '\x80', '\xE2', '\x8C', '\xA3' },
211 "Unpaired low surrogate should be skipped in the middle");
212 }
213
TEST_F(UnicodeTest,UTF16toUTF8CorrectInvalidSurrogate)214 TEST_F(UnicodeTest, UTF16toUTF8CorrectInvalidSurrogate) {
215 // http://b/29250543
216 // d841d8 is an invalid start for a surrogate pair. Make sure this is handled by ignoring the
217 // first character in the pair and handling the rest correctly.
218 TestUTF16toUTF8(
219 { 0xD841, 0xD841, 0xDC41 }, // U+20441
220 { '\xF0', '\xA0', '\x91', '\x81' },
221 "Invalid start for a surrogate pair should be ignored");
222 }
223
TEST_F(UnicodeTest,UTF16toUTF8Normal)224 TEST_F(UnicodeTest, UTF16toUTF8Normal) {
225 TestUTF16toUTF8({
226 0x0024, // U+0024 ($) --> 0x24, 1 UTF-8 byte
227 0x00A3, // U+00A3 (£) --> 0xC2 0xA3, 2 UTF-8 bytes
228 0x0939, // U+0939 (ह) --> 0xE0 0xA4 0xB9, 3 UTF-8 bytes
229 0x20AC, // U+20AC (€) --> 0xE2 0x82 0xAC, 3 UTF-8 bytes
230 0xD55C, // U+D55C (한)--> 0xED 0x95 0x9C, 3 UTF-8 bytes
231 0xD801, 0xDC37, // U+10437 () --> 0xF0 0x90 0x90 0xB7, 4 UTF-8 bytes
232 }, {
233 '\x24',
234 '\xC2', '\xA3',
235 '\xE0', '\xA4', '\xB9',
236 '\xE2', '\x82', '\xAC',
237 '\xED', '\x95', '\x9C',
238 '\xF0', '\x90', '\x90', '\xB7',
239 });
240 }
241
TEST_F(UnicodeTest,strstr16EmptyTarget)242 TEST_F(UnicodeTest, strstr16EmptyTarget) {
243 EXPECT_EQ(strstr16(kSearchString, u""), kSearchString)
244 << "should return the original pointer";
245 }
246
TEST_F(UnicodeTest,strstr16EmptyTarget_bug)247 TEST_F(UnicodeTest, strstr16EmptyTarget_bug) {
248 // In the original code when target is an empty string strlen16() would
249 // start reading the memory until a "terminating null" (that is, zero)
250 // character is found. This happens because "*target++" in the original
251 // code would increment the pointer beyond the actual string.
252 void* memptr;
253 const size_t alignment = sysconf(_SC_PAGESIZE);
254 const size_t size = 2 * alignment;
255 ASSERT_EQ(posix_memalign(&memptr, alignment, size), 0);
256 // Fill allocated memory.
257 memset(memptr, 'A', size);
258 // Create a pointer to an "empty" string on the first page.
259 char16_t* const emptyString = (char16_t* const)((char*)memptr + alignment - 4);
260 *emptyString = (char16_t)0;
261 // Protect the second page to show that strstr16() violates that.
262 ASSERT_EQ(mprotect((char*)memptr + alignment, alignment, PROT_NONE), 0);
263 // Test strstr16(): when bug is present a segmentation fault is raised.
264 ASSERT_EQ(strstr16((char16_t*)memptr, emptyString), (char16_t*)memptr)
265 << "should not read beyond the first char16_t.";
266 // Reset protection of the second page
267 ASSERT_EQ(mprotect((char*)memptr + alignment, alignment, PROT_READ | PROT_WRITE), 0);
268 // Free allocated memory.
269 free(memptr);
270 }
271
TEST_F(UnicodeTest,strstr16SameString)272 TEST_F(UnicodeTest, strstr16SameString) {
273 const char16_t* result = strstr16(kSearchString, kSearchString);
274 EXPECT_EQ(kSearchString, result)
275 << "should return the original pointer";
276 }
277
TEST_F(UnicodeTest,strstr16TargetStartOfString)278 TEST_F(UnicodeTest, strstr16TargetStartOfString) {
279 const char16_t* result = strstr16(kSearchString, u"I am");
280 EXPECT_EQ(kSearchString, result)
281 << "should return the original pointer";
282 }
283
284
TEST_F(UnicodeTest,strstr16TargetEndOfString)285 TEST_F(UnicodeTest, strstr16TargetEndOfString) {
286 const char16_t* result = strstr16(kSearchString, u"wind.");
287 EXPECT_EQ(kSearchString+19, result);
288 }
289
TEST_F(UnicodeTest,strstr16TargetWithinString)290 TEST_F(UnicodeTest, strstr16TargetWithinString) {
291 const char16_t* result = strstr16(kSearchString, u"leaf");
292 EXPECT_EQ(kSearchString+7, result);
293 }
294
TEST_F(UnicodeTest,strstr16TargetNotPresent)295 TEST_F(UnicodeTest, strstr16TargetNotPresent) {
296 const char16_t* result = strstr16(kSearchString, u"soar");
297 EXPECT_EQ(nullptr, result);
298 }
299
300 // http://b/29267949
301 // Test that overreading in utf8_to_utf16_length is detected
TEST_F(UnicodeTest,InvalidUtf8OverreadDetected)302 TEST_F(UnicodeTest, InvalidUtf8OverreadDetected) {
303 // An utf8 char starting with \xc4 is two bytes long.
304 // Add extra zeros so no extra memory is read in case the code doesn't
305 // work as expected.
306 static char utf8[] = "\xc4\x00\x00\x00";
307 ASSERT_DEATH(utf8_to_utf16_length((uint8_t *) utf8, strlen(utf8),
308 true /* overreadIsFatal */), "" /* regex for ASSERT_DEATH */);
309 }
310
311 }
312