VirtualBox

source: vbox/trunk/src/VBox/Runtime/common/string/utf-8.cpp@ 31229

Last change on this file since 31229 was 31229, checked in by vboxsync, 15 years ago

Runtime/string: remove the worst bit of code duplication in the Latin-1 <-> Utf-8 APIs
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 54.5 KB
Line 
1/* $Id: utf-8.cpp 31229 2010-07-29 21:29:35Z vboxsync $ */
2/** @file
3 * IPRT - UTF-8 Decoding.
4 */
5
6/*
7 * Copyright (C) 2006-2010 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * The contents of this file may alternatively be used under the terms
18 * of the Common Development and Distribution License Version 1.0
19 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20 * VirtualBox OSE distribution, in which case the provisions of the
21 * CDDL are applicable instead of those of the GPL.
22 *
23 * You may elect to license modified versions of this file under the
24 * terms and conditions of either the GPL or the CDDL or both.
25 */
26
27
28/*******************************************************************************
29* Header Files *
30*******************************************************************************/
31#include <iprt/string.h>
32#include "internal/iprt.h"
33
34#include <iprt/uni.h>
35#include <iprt/alloc.h>
36#include <iprt/assert.h>
37#include <iprt/err.h>
38#include "internal/string.h"
39
40
41
42/**
43 * Get get length in code points of a UTF-8 encoded string.
44 * The string is validated while doing this.
45 *
46 * @returns IPRT status code.
47 * @param psz Pointer to the UTF-8 string.
48 * @param cch The max length of the string. (btw cch = cb)
49 * Use RTSTR_MAX if all of the string is to be examined.
50 * @param pcuc Where to store the length in unicode code points.
51 * @param pcchActual Where to store the actual size of the UTF-8 string
52 * on success (cch = cb again). Optional.
53 */
54int rtUtf8Length(const char *psz, size_t cch, size_t *pcuc, size_t *pcchActual)
55{
56 const unsigned char *puch = (const unsigned char *)psz;
57 size_t cCodePoints = 0;
58 while (cch > 0)
59 {
60 const unsigned char uch = *puch;
61 if (!uch)
62 break;
63 if (uch & RT_BIT(7))
64 {
65 /* figure sequence length and validate the first byte */
66 unsigned cb;
67 if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5))) == (RT_BIT(7) | RT_BIT(6)))
68 cb = 2;
69 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5)))
70 cb = 3;
71 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4)))
72 cb = 4;
73 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3)))
74 cb = 5;
75 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2) | RT_BIT(1))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2)))
76 cb = 6;
77 else
78 {
79 RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(cch, 10), puch));
80 return VERR_INVALID_UTF8_ENCODING;
81 }
82
83 /* check length */
84 if (cb > cch)
85 {
86 RTStrAssertMsgFailed(("Invalid UTF-8 length: cb=%d cch=%d (%.*Rhxs)\n", cb, cch, RT_MIN(cch, 10), puch));
87 return VERR_INVALID_UTF8_ENCODING;
88 }
89
90 /* validate the rest */
91 switch (cb)
92 {
93 case 6:
94 RTStrAssertMsgReturn((puch[5] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("6/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
95 case 5:
96 RTStrAssertMsgReturn((puch[4] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("5/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
97 case 4:
98 RTStrAssertMsgReturn((puch[3] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("4/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
99 case 3:
100 RTStrAssertMsgReturn((puch[2] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
101 case 2:
102 RTStrAssertMsgReturn((puch[1] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
103 break;
104 }
105
106 /* validate the code point. */
107 RTUNICP uc;
108 switch (cb)
109 {
110 case 6:
111 uc = (puch[5] & 0x3f)
112 | ((RTUNICP)(puch[4] & 0x3f) << 6)
113 | ((RTUNICP)(puch[3] & 0x3f) << 12)
114 | ((RTUNICP)(puch[2] & 0x3f) << 18)
115 | ((RTUNICP)(puch[1] & 0x3f) << 24)
116 | ((RTUNICP)(uch & 0x01) << 30);
117 RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
118 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
119 break;
120 case 5:
121 uc = (puch[4] & 0x3f)
122 | ((RTUNICP)(puch[3] & 0x3f) << 6)
123 | ((RTUNICP)(puch[2] & 0x3f) << 12)
124 | ((RTUNICP)(puch[1] & 0x3f) << 18)
125 | ((RTUNICP)(uch & 0x03) << 24);
126 RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
127 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
128 break;
129 case 4:
130 uc = (puch[3] & 0x3f)
131 | ((RTUNICP)(puch[2] & 0x3f) << 6)
132 | ((RTUNICP)(puch[1] & 0x3f) << 12)
133 | ((RTUNICP)(uch & 0x07) << 18);
134 RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
135 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
136 break;
137 case 3:
138 uc = (puch[2] & 0x3f)
139 | ((RTUNICP)(puch[1] & 0x3f) << 6)
140 | ((RTUNICP)(uch & 0x0f) << 12);
141 RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
142 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch),
143 uc == 0xffff || uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING);
144 RTStrAssertMsgReturn(uc < 0xd800 || uc > 0xdfff,
145 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_CODE_POINT_SURROGATE);
146 break;
147 case 2:
148 uc = (puch[1] & 0x3f)
149 | ((RTUNICP)(uch & 0x1f) << 6);
150 RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
151 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
152 break;
153 }
154
155 /* advance */
156 cch -= cb;
157 puch += cb;
158 }
159 else
160 {
161 /* one ASCII byte */
162 puch++;
163 cch--;
164 }
165 cCodePoints++;
166 }
167
168 /* done */
169 *pcuc = cCodePoints;
170 if (pcchActual)
171 *pcchActual = puch - (unsigned char const *)psz;
172 return VINF_SUCCESS;
173}
174
175
176/**
177 * Decodes and UTF-8 string into an array of unicode code point.
178 *
179 * Since we know the input is valid, we do *not* perform encoding or length checks.
180 *
181 * @returns iprt status code.
182 * @param psz The UTF-8 string to recode. This is a valid encoding.
183 * @param cch The number of chars (the type char, so bytes if you like) to process of the UTF-8 string.
184 * The recoding will stop when cch or '\\0' is reached. Pass RTSTR_MAX to process up to '\\0'.
185 * @param paCps Where to store the code points array.
186 * @param cCps The number of RTUNICP items the paCps buffer can hold, excluding the terminator ('\\0').
187 */
188static int rtUtf8Decode(const char *psz, size_t cch, PRTUNICP paCps, size_t cCps)
189{
190 int rc = VINF_SUCCESS;
191 const unsigned char *puch = (const unsigned char *)psz;
192 PRTUNICP pCp = paCps;
193 while (cch > 0)
194 {
195 /* read the next char and check for terminator. */
196 const unsigned char uch = *puch;
197 if (!uch)
198 break;
199
200 /* check for output overflow */
201 if (RT_UNLIKELY(cCps < 1))
202 {
203 rc = VERR_BUFFER_OVERFLOW;
204 break;
205 }
206 cCps--;
207
208 /* decode and recode the code point */
209 if (!(uch & RT_BIT(7)))
210 {
211 *pCp++ = uch;
212 puch++;
213 cch--;
214 }
215#ifdef RT_STRICT
216 else if (!(uch & RT_BIT(6)))
217 AssertMsgFailed(("Internal error!\n"));
218#endif
219 else if (!(uch & RT_BIT(5)))
220 {
221 *pCp++ = (puch[1] & 0x3f)
222 | ((uint16_t)(uch & 0x1f) << 6);
223 puch += 2;
224 cch -= 2;
225 }
226 else if (!(uch & RT_BIT(4)))
227 {
228 *pCp++ = (puch[2] & 0x3f)
229 | ((uint16_t)(puch[1] & 0x3f) << 6)
230 | ((uint16_t)(uch & 0x0f) << 12);
231 puch += 3;
232 cch -= 3;
233 }
234 else if (!(uch & RT_BIT(3)))
235 {
236 *pCp++ = (puch[3] & 0x3f)
237 | ((RTUNICP)(puch[2] & 0x3f) << 6)
238 | ((RTUNICP)(puch[1] & 0x3f) << 12)
239 | ((RTUNICP)(uch & 0x07) << 18);
240 puch += 4;
241 cch -= 4;
242 }
243 else if (!(uch & RT_BIT(2)))
244 {
245 *pCp++ = (puch[4] & 0x3f)
246 | ((RTUNICP)(puch[3] & 0x3f) << 6)
247 | ((RTUNICP)(puch[2] & 0x3f) << 12)
248 | ((RTUNICP)(puch[1] & 0x3f) << 18)
249 | ((RTUNICP)(uch & 0x03) << 24);
250 puch += 5;
251 cch -= 6;
252 }
253 else
254 {
255 Assert(!(uch & RT_BIT(1)));
256 *pCp++ = (puch[5] & 0x3f)
257 | ((RTUNICP)(puch[4] & 0x3f) << 6)
258 | ((RTUNICP)(puch[3] & 0x3f) << 12)
259 | ((RTUNICP)(puch[2] & 0x3f) << 18)
260 | ((RTUNICP)(puch[1] & 0x3f) << 24)
261 | ((RTUNICP)(uch & 0x01) << 30);
262 puch += 6;
263 cch -= 6;
264 }
265 }
266
267 /* done */
268 *pCp = 0;
269 return rc;
270}
271
272
273RTDECL(size_t) RTStrUniLen(const char *psz)
274{
275 size_t cCodePoints;
276 int rc = rtUtf8Length(psz, RTSTR_MAX, &cCodePoints, NULL);
277 return RT_SUCCESS(rc) ? cCodePoints : 0;
278}
279RT_EXPORT_SYMBOL(RTStrUniLen);
280
281
282RTDECL(int) RTStrUniLenEx(const char *psz, size_t cch, size_t *pcCps)
283{
284 size_t cCodePoints;
285 int rc = rtUtf8Length(psz, cch, &cCodePoints, NULL);
286 if (pcCps)
287 *pcCps = RT_SUCCESS(rc) ? cCodePoints : 0;
288 return rc;
289}
290RT_EXPORT_SYMBOL(RTStrUniLenEx);
291
292
293RTDECL(int) RTStrValidateEncoding(const char *psz)
294{
295 return RTStrValidateEncodingEx(psz, RTSTR_MAX, 0);
296}
297RT_EXPORT_SYMBOL(RTStrValidateEncoding);
298
299
300RTDECL(int) RTStrValidateEncodingEx(const char *psz, size_t cch, uint32_t fFlags)
301{
302 AssertReturn(!(fFlags & ~(RTSTR_VALIDATE_ENCODING_ZERO_TERMINATED)), VERR_INVALID_PARAMETER);
303 AssertPtr(psz);
304
305 /*
306 * Use rtUtf8Length for the job.
307 */
308 size_t cchActual;
309 size_t cCpsIgnored;
310 int rc = rtUtf8Length(psz, cch, &cCpsIgnored, &cchActual);
311 if (RT_SUCCESS(rc))
312 {
313 if ( (fFlags & RTSTR_VALIDATE_ENCODING_ZERO_TERMINATED)
314 && cchActual >= cch)
315 rc = VERR_BUFFER_OVERFLOW;
316 }
317 return rc;
318}
319RT_EXPORT_SYMBOL(RTStrValidateEncodingEx);
320
321
322RTDECL(bool) RTStrIsValidEncoding(const char *psz)
323{
324 int rc = RTStrValidateEncodingEx(psz, RTSTR_MAX, 0);
325 return RT_SUCCESS(rc);
326}
327RT_EXPORT_SYMBOL(RTStrIsValidEncoding);
328
329
330RTDECL(size_t) RTStrPurgeEncoding(char *psz)
331{
332 size_t cErrors = 0;
333 for (;;)
334 {
335 RTUNICP Cp;
336 int rc = RTStrGetCpEx((const char **)&psz, &Cp);
337 if (RT_SUCCESS(rc))
338 {
339 if (!Cp)
340 break;
341 }
342 else
343 {
344 psz[-1] = '?';
345 cErrors++;
346 }
347 }
348 return cErrors;
349}
350RT_EXPORT_SYMBOL(RTStrPurgeEncoding);
351
352
353RTDECL(int) RTStrToUni(const char *pszString, PRTUNICP *ppaCps)
354{
355 /*
356 * Validate input.
357 */
358 Assert(VALID_PTR(pszString));
359 Assert(VALID_PTR(ppaCps));
360 *ppaCps = NULL;
361
362 /*
363 * Validate the UTF-8 input and count its code points.
364 */
365 size_t cCps;
366 int rc = rtUtf8Length(pszString, RTSTR_MAX, &cCps, NULL);
367 if (RT_SUCCESS(rc))
368 {
369 /*
370 * Allocate buffer.
371 */
372 PRTUNICP paCps = (PRTUNICP)RTMemAlloc((cCps + 1) * sizeof(RTUNICP));
373 if (paCps)
374 {
375 /*
376 * Decode the string.
377 */
378 rc = rtUtf8Decode(pszString, RTSTR_MAX, paCps, cCps);
379 if (RT_SUCCESS(rc))
380 {
381 *ppaCps = paCps;
382 return rc;
383 }
384 RTMemFree(paCps);
385 }
386 else
387 rc = VERR_NO_CODE_POINT_MEMORY;
388 }
389 return rc;
390}
391RT_EXPORT_SYMBOL(RTStrToUni);
392
393
394RTDECL(int) RTStrToUniEx(const char *pszString, size_t cchString, PRTUNICP *ppaCps, size_t cCps, size_t *pcCps)
395{
396 /*
397 * Validate input.
398 */
399 Assert(VALID_PTR(pszString));
400 Assert(VALID_PTR(ppaCps));
401 Assert(!pcCps || VALID_PTR(pcCps));
402
403 /*
404 * Validate the UTF-8 input and count the code points.
405 */
406 size_t cCpsResult;
407 int rc = rtUtf8Length(pszString, cchString, &cCpsResult, NULL);
408 if (RT_SUCCESS(rc))
409 {
410 if (pcCps)
411 *pcCps = cCpsResult;
412
413 /*
414 * Check buffer size / Allocate buffer.
415 */
416 bool fShouldFree;
417 PRTUNICP paCpsResult;
418 if (cCps > 0 && *ppaCps)
419 {
420 fShouldFree = false;
421 if (cCps <= cCpsResult)
422 return VERR_BUFFER_OVERFLOW;
423 paCpsResult = *ppaCps;
424 }
425 else
426 {
427 *ppaCps = NULL;
428 fShouldFree = true;
429 cCps = RT_MAX(cCpsResult + 1, cCps);
430 paCpsResult = (PRTUNICP)RTMemAlloc(cCps * sizeof(RTUNICP));
431 }
432 if (paCpsResult)
433 {
434 /*
435 * Encode the UTF-16 string.
436 */
437 rc = rtUtf8Decode(pszString, cchString, paCpsResult, cCps - 1);
438 if (RT_SUCCESS(rc))
439 {
440 *ppaCps = paCpsResult;
441 return rc;
442 }
443 if (fShouldFree)
444 RTMemFree(paCpsResult);
445 }
446 else
447 rc = VERR_NO_CODE_POINT_MEMORY;
448 }
449 return rc;
450}
451RT_EXPORT_SYMBOL(RTStrToUniEx);
452
453
454/**
455 * Calculates the UTF-16 length of a string, validating the encoding while doing so.
456 *
457 * @returns IPRT status code.
458 * @param psz Pointer to the UTF-8 string.
459 * @param cch The max length of the string. (btw cch = cb)
460 * Use RTSTR_MAX if all of the string is to be examined.
461 * @param pcwc Where to store the length of the UTF-16 string as a number of RTUTF16 characters.
462 */
463static int rtUtf8CalcUtf16Length(const char *psz, size_t cch, size_t *pcwc)
464{
465 const unsigned char *puch = (const unsigned char *)psz;
466 size_t cwc = 0;
467 while (cch > 0)
468 {
469 const unsigned char uch = *puch;
470 if (!uch)
471 break;
472 if (!(uch & RT_BIT(7)))
473 {
474 /* one ASCII byte */
475 cwc++;
476 puch++;
477 cch--;
478 }
479 else
480 {
481 /* figure sequence length and validate the first byte */
482 unsigned cb;
483 if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5))) == (RT_BIT(7) | RT_BIT(6)))
484 cb = 2;
485 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5)))
486 cb = 3;
487 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4)))
488 cb = 4;
489 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3)))
490 cb = 5;
491 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2) | RT_BIT(1))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2)))
492 cb = 6;
493 else
494 {
495 RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(cch, 10), puch));
496 return VERR_INVALID_UTF8_ENCODING;
497 }
498
499 /* check length */
500 if (cb > cch)
501 {
502 RTStrAssertMsgFailed(("Invalid UTF-8 length: cb=%d cch=%d (%.*Rhxs)\n", cb, cch, RT_MIN(cch, 10), puch));
503 return VERR_INVALID_UTF8_ENCODING;
504 }
505
506 /* validate the rest */
507 switch (cb)
508 {
509 case 6:
510 RTStrAssertMsgReturn((puch[5] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("6/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
511 case 5:
512 RTStrAssertMsgReturn((puch[4] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("5/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
513 case 4:
514 RTStrAssertMsgReturn((puch[3] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("4/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
515 case 3:
516 RTStrAssertMsgReturn((puch[2] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
517 case 2:
518 RTStrAssertMsgReturn((puch[1] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
519 break;
520 }
521
522 /* validate the code point. */
523 RTUNICP uc;
524 switch (cb)
525 {
526 case 6:
527 uc = (puch[5] & 0x3f)
528 | ((RTUNICP)(puch[4] & 0x3f) << 6)
529 | ((RTUNICP)(puch[3] & 0x3f) << 12)
530 | ((RTUNICP)(puch[2] & 0x3f) << 18)
531 | ((RTUNICP)(puch[1] & 0x3f) << 24)
532 | ((RTUNICP)(uch & 0x01) << 30);
533 RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
534 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
535 RTStrAssertMsgFailed(("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch));
536 return VERR_CANT_RECODE_AS_UTF16;
537 case 5:
538 uc = (puch[4] & 0x3f)
539 | ((RTUNICP)(puch[3] & 0x3f) << 6)
540 | ((RTUNICP)(puch[2] & 0x3f) << 12)
541 | ((RTUNICP)(puch[1] & 0x3f) << 18)
542 | ((RTUNICP)(uch & 0x03) << 24);
543 RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
544 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
545 RTStrAssertMsgFailed(("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch));
546 return VERR_CANT_RECODE_AS_UTF16;
547 case 4:
548 uc = (puch[3] & 0x3f)
549 | ((RTUNICP)(puch[2] & 0x3f) << 6)
550 | ((RTUNICP)(puch[1] & 0x3f) << 12)
551 | ((RTUNICP)(uch & 0x07) << 18);
552 RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
553 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
554 RTStrAssertMsgReturn(uc <= 0x0010ffff,
555 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_CANT_RECODE_AS_UTF16);
556 cwc++;
557 break;
558 case 3:
559 uc = (puch[2] & 0x3f)
560 | ((RTUNICP)(puch[1] & 0x3f) << 6)
561 | ((RTUNICP)(uch & 0x0f) << 12);
562 RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
563 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch),
564 uc == 0xffff || uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING);
565 RTStrAssertMsgReturn(uc < 0xd800 || uc > 0xdfff,
566 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_CODE_POINT_SURROGATE);
567 break;
568 case 2:
569 uc = (puch[1] & 0x3f)
570 | ((RTUNICP)(uch & 0x1f) << 6);
571 RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
572 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
573 break;
574 }
575
576 /* advance */
577 cch -= cb;
578 puch += cb;
579 cwc++;
580 }
581 }
582
583 /* done */
584 *pcwc = cwc;
585 return VINF_SUCCESS;
586}
587
588
589/**
590 * Recodes a valid UTF-8 string as UTF-16.
591 *
592 * Since we know the input is valid, we do *not* perform encoding or length checks.
593 *
594 * @returns iprt status code.
595 * @param psz The UTF-8 string to recode. This is a valid encoding.
596 * @param cch The number of chars (the type char, so bytes if you like) to process of the UTF-8 string.
597 * The recoding will stop when cch or '\\0' is reached. Pass RTSTR_MAX to process up to '\\0'.
598 * @param pwsz Where to store the UTF-16 string.
599 * @param cwc The number of RTUTF16 items the pwsz buffer can hold, excluding the terminator ('\\0').
600 */
601static int rtUtf8RecodeAsUtf16(const char *psz, size_t cch, PRTUTF16 pwsz, size_t cwc)
602{
603 int rc = VINF_SUCCESS;
604 const unsigned char *puch = (const unsigned char *)psz;
605 PRTUTF16 pwc = pwsz;
606 while (cch > 0)
607 {
608 /* read the next char and check for terminator. */
609 const unsigned char uch = *puch;
610 if (!uch)
611 break;
612
613 /* check for output overflow */
614 if (RT_UNLIKELY(cwc < 1))
615 {
616 rc = VERR_BUFFER_OVERFLOW;
617 break;
618 }
619 cwc--;
620
621 /* decode and recode the code point */
622 if (!(uch & RT_BIT(7)))
623 {
624 *pwc++ = uch;
625 puch++;
626 cch--;
627 }
628 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5))) == (RT_BIT(7) | RT_BIT(6)))
629 {
630 uint16_t uc = (puch[1] & 0x3f)
631 | ((uint16_t)(uch & 0x1f) << 6);
632 *pwc++ = uc;
633 puch += 2;
634 cch -= 2;
635 }
636 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5)))
637 {
638 uint16_t uc = (puch[2] & 0x3f)
639 | ((uint16_t)(puch[1] & 0x3f) << 6)
640 | ((uint16_t)(uch & 0x0f) << 12);
641 *pwc++ = uc;
642 puch += 3;
643 cch -= 3;
644 }
645 else
646 {
647 /* generate surrugate pair */
648 Assert((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4)));
649 RTUNICP uc = (puch[3] & 0x3f)
650 | ((RTUNICP)(puch[2] & 0x3f) << 6)
651 | ((RTUNICP)(puch[1] & 0x3f) << 12)
652 | ((RTUNICP)(uch & 0x07) << 18);
653 if (RT_UNLIKELY(cwc < 1))
654 {
655 rc = VERR_BUFFER_OVERFLOW;
656 break;
657 }
658 cwc--;
659
660 uc -= 0x10000;
661 *pwc++ = 0xd800 | (uc >> 10);
662 *pwc++ = 0xdc00 | (uc & 0x3ff);
663 puch += 4;
664 cch -= 4;
665 }
666 }
667
668 /* done */
669 *pwc = '\0';
670 return rc;
671}
672
673
674RTDECL(int) RTStrToUtf16Tag(const char *pszString, PRTUTF16 *ppwszString, const char *pszTag)
675{
676 /*
677 * Validate input.
678 */
679 Assert(VALID_PTR(ppwszString));
680 Assert(VALID_PTR(pszString));
681 *ppwszString = NULL;
682
683 /*
684 * Validate the UTF-8 input and calculate the length of the UTF-16 string.
685 */
686 size_t cwc;
687 int rc = rtUtf8CalcUtf16Length(pszString, RTSTR_MAX, &cwc);
688 if (RT_SUCCESS(rc))
689 {
690 /*
691 * Allocate buffer.
692 */
693 PRTUTF16 pwsz = (PRTUTF16)RTMemAllocTag((cwc + 1) * sizeof(RTUTF16), pszTag);
694 if (pwsz)
695 {
696 /*
697 * Encode the UTF-16 string.
698 */
699 rc = rtUtf8RecodeAsUtf16(pszString, RTSTR_MAX, pwsz, cwc);
700 if (RT_SUCCESS(rc))
701 {
702 *ppwszString = pwsz;
703 return rc;
704 }
705 RTMemFree(pwsz);
706 }
707 else
708 rc = VERR_NO_UTF16_MEMORY;
709 }
710 return rc;
711}
712RT_EXPORT_SYMBOL(RTStrToUtf16Tag);
713
714
715RTDECL(int) RTStrToUtf16ExTag(const char *pszString, size_t cchString,
716 PRTUTF16 *ppwsz, size_t cwc, size_t *pcwc, const char *pszTag)
717{
718 /*
719 * Validate input.
720 */
721 Assert(VALID_PTR(pszString));
722 Assert(VALID_PTR(ppwsz));
723 Assert(!pcwc || VALID_PTR(pcwc));
724
725 /*
726 * Validate the UTF-8 input and calculate the length of the UTF-16 string.
727 */
728 size_t cwcResult;
729 int rc = rtUtf8CalcUtf16Length(pszString, cchString, &cwcResult);
730 if (RT_SUCCESS(rc))
731 {
732 if (pcwc)
733 *pcwc = cwcResult;
734
735 /*
736 * Check buffer size / Allocate buffer.
737 */
738 bool fShouldFree;
739 PRTUTF16 pwszResult;
740 if (cwc > 0 && *ppwsz)
741 {
742 fShouldFree = false;
743 if (cwc <= cwcResult)
744 return VERR_BUFFER_OVERFLOW;
745 pwszResult = *ppwsz;
746 }
747 else
748 {
749 *ppwsz = NULL;
750 fShouldFree = true;
751 cwc = RT_MAX(cwcResult + 1, cwc);
752 pwszResult = (PRTUTF16)RTMemAllocTag(cwc * sizeof(RTUTF16), pszTag);
753 }
754 if (pwszResult)
755 {
756 /*
757 * Encode the UTF-16 string.
758 */
759 rc = rtUtf8RecodeAsUtf16(pszString, cchString, pwszResult, cwc - 1);
760 if (RT_SUCCESS(rc))
761 {
762 *ppwsz = pwszResult;
763 return rc;
764 }
765 if (fShouldFree)
766 RTMemFree(pwszResult);
767 }
768 else
769 rc = VERR_NO_UTF16_MEMORY;
770 }
771 return rc;
772}
773RT_EXPORT_SYMBOL(RTStrToUtf16ExTag);
774
775
776RTDECL(size_t) RTStrCalcUtf16Len(const char *psz)
777{
778 size_t cwc;
779 int rc = rtUtf8CalcUtf16Length(psz, RTSTR_MAX, &cwc);
780 return RT_SUCCESS(rc) ? cwc : 0;
781}
782RT_EXPORT_SYMBOL(RTStrCalcUtf16Len);
783
784
785RTDECL(int) RTStrCalcUtf16LenEx(const char *psz, size_t cch, size_t *pcwc)
786{
787 size_t cwc;
788 int rc = rtUtf8CalcUtf16Length(psz, cch, &cwc);
789 if (pcwc)
790 *pcwc = RT_SUCCESS(rc) ? cwc : ~(size_t)0;
791 return rc;
792}
793RT_EXPORT_SYMBOL(RTStrCalcUtf16LenEx);
794
795
796/**
797 * Calculates the length of the UTF-8 encoding of a Latin-1 string.
798 *
799 * @returns iprt status code.
800 * @param psz The Latin-1 string.
801 * @param cchIn The max length of the Latin-1 string to consider.
802 * @param pcch Where to store the length (excluding '\\0') of the UTF-8 string. (cch == cb, btw)
803 */
804static int rtLatin1CalcUtf8Length(const char *psz, size_t cchIn, size_t *pcch)
805{
806 size_t cch = 0;
807 while (cchIn > 0)
808 {
809 char ch = *psz++; cchIn--;
810 if (!ch)
811 break;
812 if (!(ch & 0x80))
813 cch++;
814 else
815 cch += 2;
816 }
817
818
819 /* done */
820 *pcch = cch;
821 return VINF_SUCCESS;
822}
823
824
825/**
826 * Recodes a Latin-1 string as UTF-8.
827 *
828 * @returns iprt status code.
829 * @param psz The Latin-1 string.
830 * @param cchIn The number of characters to process from psz. The recoding
831 * will stop when cch or '\\0' is reached.
832 * @param psz Where to store the UTF-8 string.
833 * @param cch The size of the UTF-8 buffer, excluding the terminator.
834 * @param pcch Where to store the number of octets actually encoded.
835 */
836static int rtLatin1RecodeAsUtf8(const char *pszIn, size_t cchIn, char *psz, size_t cch, size_t *pcch)
837{
838 unsigned char *puch = (unsigned char *)psz;
839 int rc = VINF_SUCCESS;
840 while (cchIn > 0)
841 {
842 unsigned char ch = (unsigned char) *pszIn++; cchIn--;
843 if (!ch)
844 break;
845 if (!(ch & 0x80))
846 {
847 if (RT_UNLIKELY(cch < 1))
848 {
849 RTStrAssertMsgFailed(("Buffer overflow! 1\n"));
850 rc = VERR_BUFFER_OVERFLOW;
851 break;
852 }
853 cch--;
854 *puch++ = (unsigned char)ch;
855 }
856 else
857 {
858 if (RT_UNLIKELY(cch < 2))
859 {
860 RTStrAssertMsgFailed(("Buffer overflow! 2\n"));
861 rc = VERR_BUFFER_OVERFLOW;
862 break;
863 }
864 cch -= 2;
865 *puch++ = 0xc0 | (ch >> 6);
866 *puch++ = 0x80 | (ch & 0x3f);
867 }
868 }
869
870 /* done */
871 *puch = '\0';
872 *pcch = (char *)puch - psz;
873 return rc;
874}
875
876
877
878RTDECL(int) RTLatin1ToUtf8Tag(const char *pszString, char **ppszString, const char *pszTag)
879{
880 /*
881 * Validate input.
882 */
883 Assert(VALID_PTR(ppszString));
884 Assert(VALID_PTR(pszString));
885 *ppszString = NULL;
886
887 /*
888 * Calculate the length of the UTF-8 encoding of the Latin-1 string.
889 */
890 size_t cch;
891 int rc = rtLatin1CalcUtf8Length(pszString, RTSTR_MAX, &cch);
892 if (RT_SUCCESS(rc))
893 {
894 /*
895 * Allocate buffer and recode it.
896 */
897 char *pszResult = (char *)RTMemAllocTag(cch + 1, pszTag);
898 if (pszResult)
899 {
900 rc = rtLatin1RecodeAsUtf8(pszString, RTSTR_MAX, pszResult, cch, &cch);
901 if (RT_SUCCESS(rc))
902 {
903 *ppszString = pszResult;
904 return rc;
905 }
906
907 RTMemFree(pszResult);
908 }
909 else
910 rc = VERR_NO_STR_MEMORY;
911 }
912 return rc;
913}
914RT_EXPORT_SYMBOL(RTLatin1ToUtf8Tag);
915
916
917RTDECL(int) RTLatin1ToUtf8ExTag(const char *pszString, size_t cchString, char **ppsz, size_t cch, size_t *pcch, const char *pszTag)
918{
919 /*
920 * Validate input.
921 */
922 Assert(VALID_PTR(pszString));
923 Assert(VALID_PTR(ppsz));
924 Assert(!pcch || VALID_PTR(pcch));
925
926 /*
927 * Calculate the length of the UTF-8 encoding of the Latin-1 string.
928 */
929 size_t cchResult;
930 int rc = rtLatin1CalcUtf8Length(pszString, cchString, &cchResult);
931 if (RT_SUCCESS(rc))
932 {
933 if (pcch)
934 *pcch = cchResult;
935
936 /*
937 * Check buffer size / Allocate buffer and recode it.
938 */
939 bool fShouldFree;
940 char *pszResult;
941 if (cch > 0 && *ppsz)
942 {
943 fShouldFree = false;
944 if (RT_UNLIKELY(cch <= cchResult))
945 return VERR_BUFFER_OVERFLOW;
946 pszResult = *ppsz;
947 }
948 else
949 {
950 *ppsz = NULL;
951 fShouldFree = true;
952 cch = RT_MAX(cch, cchResult + 1);
953 pszResult = (char *)RTStrAllocTag(cch, pszTag);
954 }
955 if (pszResult)
956 {
957 rc = rtLatin1RecodeAsUtf8(pszString, cchString, pszResult, cch - 1, &cch);
958 if (RT_SUCCESS(rc))
959 {
960 *ppsz = pszResult;
961 return rc;
962 }
963
964 if (fShouldFree)
965 RTStrFree(pszResult);
966 }
967 else
968 rc = VERR_NO_STR_MEMORY;
969 }
970 return rc;
971}
972RT_EXPORT_SYMBOL(RTLatin1ToUtf8ExTag);
973
974
975RTDECL(size_t) RTLatin1CalcUtf8Len(const char *psz)
976{
977 size_t cch;
978 int rc = rtLatin1CalcUtf8Length(psz, RTSTR_MAX, &cch);
979 return RT_SUCCESS(rc) ? cch : 0;
980}
981RT_EXPORT_SYMBOL(RTLatin1CalcUtf8Len);
982
983
984RTDECL(int) RTLatin1CalcUtf8LenEx(const char *psz, size_t cchIn, size_t *pcch)
985{
986 size_t cch;
987 int rc = rtLatin1CalcUtf8Length(psz, cchIn, &cch);
988 if (pcch)
989 *pcch = RT_SUCCESS(rc) ? cch : ~(size_t)0;
990 return rc;
991}
992RT_EXPORT_SYMBOL(RTLatin1CalcUtf8LenEx);
993
994
995/**
996 * Calculates the Latin-1 length of a string, validating the encoding while doing so.
997 *
998 * @returns IPRT status code.
999 * @param psz Pointer to the UTF-8 string.
1000 * @param cch The max length of the string. (btw cch = cb)
1001 * Use RTSTR_MAX if all of the string is to be examined.
1002 * @param pcch Where to store the length of the Latin-1 string in bytes.
1003 */
1004static int rtUtf8CalcLatin1Length(const char *psz, size_t cch, size_t *pcch)
1005{
1006 size_t cchOut = 0;
1007 while (true)
1008 {
1009 RTUNICP Cp;
1010 int rc = RTStrGetCpNEx(&psz, &cch, &Cp);
1011 if (Cp == 0 || rc == VERR_END_OF_STRING)
1012 break;
1013 if (RT_FAILURE(rc))
1014 return rc;
1015 if (Cp >= 0x100)
1016 return VERR_NO_TRANSLATION;
1017 cchOut++;
1018 }
1019
1020 /* done */
1021 *pcch = cchOut;
1022 return VINF_SUCCESS;
1023}
1024
1025
1026/**
1027 * Recodes a valid UTF-8 string as Latin-1.
1028 *
1029 * Since we know the input is valid, we do *not* perform encoding or length checks.
1030 *
1031 * @returns iprt status code.
1032 * @param psz The UTF-8 string to recode. This is a valid encoding.
1033 * @param cch The number of chars (the type char, so bytes if you like) to process of the UTF-8 string.
1034 * The recoding will stop when cch or '\\0' is reached. Pass RTSTR_MAX to process up to '\\0'.
1035 * @param pszOut Where to store the Latin-1 string.
1036 * @param cchOut The number of characters the pszOut buffer can hold, excluding the terminator ('\\0').
1037 */
1038static int rtUtf8RecodeAsLatin1(const char *psz, size_t cch, char *pszOut, size_t cchOut)
1039{
1040 int rc = VINF_SUCCESS;
1041 const unsigned char *puch = (const unsigned char *)psz;
1042 unsigned char *puchOut = (unsigned char *)pszOut;
1043 while (cch > 0)
1044 {
1045 /* read the next char and check for terminator. */
1046 const unsigned char uch = *puch;
1047 if (!uch)
1048 break;
1049
1050 /* check for output overflow */
1051 if (RT_UNLIKELY(cchOut < 1))
1052 {
1053 rc = VERR_BUFFER_OVERFLOW;
1054 break;
1055 }
1056 cchOut--;
1057
1058 /* decode and recode the code point */
1059 if (!(uch & RT_BIT(7)))
1060 {
1061 *puchOut++ = uch;
1062 puch++;
1063 cch--;
1064 }
1065 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5))) == (RT_BIT(7) | RT_BIT(6)))
1066 {
1067 uint16_t uc = (puch[1] & 0x3f)
1068 | ((uint16_t)(uch & 0x1f) << 6);
1069 if (uc >= 0x100)
1070 {
1071 rc = VERR_NO_TRANSLATION;
1072 break;
1073 }
1074 *puchOut++ = uc;
1075 puch += 2;
1076 cch -= 2;
1077 }
1078 else
1079 {
1080 rc = VERR_NO_TRANSLATION;
1081 break;
1082 }
1083 }
1084
1085 /* done */
1086 *puchOut = '\0';
1087 return rc;
1088}
1089
1090
1091RTDECL(int) RTStrToLatin1Tag(const char *pszString, char **ppszString, const char *pszTag)
1092{
1093 /*
1094 * Validate input.
1095 */
1096 Assert(VALID_PTR(ppszString));
1097 Assert(VALID_PTR(pszString));
1098 *ppszString = NULL;
1099
1100 /*
1101 * Validate the UTF-8 input and calculate the length of the Latin-1 string.
1102 */
1103 size_t cch;
1104 int rc = rtUtf8CalcLatin1Length(pszString, RTSTR_MAX, &cch);
1105 if (RT_SUCCESS(rc))
1106 {
1107 /*
1108 * Allocate buffer.
1109 */
1110 char *psz = (char *)RTMemAllocTag(cch + 1, pszTag);
1111 if (psz)
1112 {
1113 /*
1114 * Encode the UTF-16 string.
1115 */
1116 rc = rtUtf8RecodeAsLatin1(pszString, RTSTR_MAX, psz, cch);
1117 if (RT_SUCCESS(rc))
1118 {
1119 *ppszString = psz;
1120 return rc;
1121 }
1122 RTMemFree(psz);
1123 }
1124 else
1125 rc = VERR_NO_STR_MEMORY;
1126 }
1127 return rc;
1128}
1129RT_EXPORT_SYMBOL(RTStrToLatin1Tag);
1130
1131
1132RTDECL(int) RTStrToLatin1ExTag(const char *pszString, size_t cchString,
1133 char **ppsz, size_t cch, size_t *pcch, const char *pszTag)
1134{
1135 /*
1136 * Validate input.
1137 */
1138 Assert(VALID_PTR(pszString));
1139 Assert(VALID_PTR(ppsz));
1140 Assert(!pcch || VALID_PTR(pcch));
1141
1142 /*
1143 * Validate the UTF-8 input and calculate the length of the UTF-16 string.
1144 */
1145 size_t cchResult;
1146 int rc = rtUtf8CalcLatin1Length(pszString, cchString, &cchResult);
1147 if (RT_SUCCESS(rc))
1148 {
1149 if (pcch)
1150 *pcch = cchResult;
1151
1152 /*
1153 * Check buffer size / Allocate buffer.
1154 */
1155 bool fShouldFree;
1156 char *pszResult;
1157 if (cch > 0 && *ppsz)
1158 {
1159 fShouldFree = false;
1160 if (cch <= cchResult)
1161 return VERR_BUFFER_OVERFLOW;
1162 pszResult = *ppsz;
1163 }
1164 else
1165 {
1166 *ppsz = NULL;
1167 fShouldFree = true;
1168 cch = RT_MAX(cchResult + 1, cch);
1169 pszResult = (char *)RTMemAllocTag(cch, pszTag);
1170 }
1171 if (pszResult)
1172 {
1173 /*
1174 * Encode the Latin-1 string.
1175 */
1176 rc = rtUtf8RecodeAsLatin1(pszString, cchString, pszResult, cch - 1);
1177 if (RT_SUCCESS(rc))
1178 {
1179 *ppsz = pszResult;
1180 return rc;
1181 }
1182 if (fShouldFree)
1183 RTMemFree(pszResult);
1184 }
1185 else
1186 rc = VERR_NO_STR_MEMORY;
1187 }
1188 return rc;
1189}
1190RT_EXPORT_SYMBOL(RTStrToLatin1Tag);
1191
1192
1193RTDECL(size_t) RTStrCalcLatin1Len(const char *psz)
1194{
1195 size_t cch;
1196 int rc = rtUtf8CalcLatin1Length(psz, RTSTR_MAX, &cch);
1197 return RT_SUCCESS(rc) ? cch : 0;
1198}
1199RT_EXPORT_SYMBOL(RTStrCalcLatin1Len);
1200
1201
1202RTDECL(int) RTStrCalcLatin1LenEx(const char *psz, size_t cchIn, size_t *pcch)
1203{
1204 size_t cch;
1205 int rc = rtUtf8CalcLatin1Length(psz, cchIn, &cch);
1206 if (pcch)
1207 *pcch = RT_SUCCESS(rc) ? cch : ~(size_t)0;
1208 return rc;
1209}
1210RT_EXPORT_SYMBOL(RTStrCalcLatin1LenEx);
1211
1212
1213/**
1214 * Handle invalid encodings passed to RTStrGetCp() and RTStrGetCpEx().
1215 * @returns rc
1216 * @param ppsz The pointer to the string position point.
1217 * @param pCp Where to store RTUNICP_INVALID.
1218 * @param rc The iprt error code.
1219 */
1220static int rtStrGetCpExFailure(const char **ppsz, PRTUNICP pCp, int rc)
1221{
1222 /*
1223 * Try find a valid encoding.
1224 */
1225 (*ppsz)++; /** @todo code this! */
1226 *pCp = RTUNICP_INVALID;
1227 return rc;
1228}
1229
1230
1231RTDECL(RTUNICP) RTStrGetCpInternal(const char *psz)
1232{
1233 RTUNICP Cp;
1234 RTStrGetCpExInternal(&psz, &Cp);
1235 return Cp;
1236}
1237RT_EXPORT_SYMBOL(RTStrGetCpInternal);
1238
1239
1240RTDECL(int) RTStrGetCpExInternal(const char **ppsz, PRTUNICP pCp)
1241{
1242 const unsigned char *puch = (const unsigned char *)*ppsz;
1243 const unsigned char uch = *puch;
1244 RTUNICP uc;
1245
1246 /* ASCII ? */
1247 if (!(uch & RT_BIT(7)))
1248 {
1249 uc = uch;
1250 puch++;
1251 }
1252 else if (uch & RT_BIT(6))
1253 {
1254 /* figure the length and validate the first octet. */
1255 unsigned cb;
1256 if (!(uch & RT_BIT(5)))
1257 cb = 2;
1258 else if (!(uch & RT_BIT(4)))
1259 cb = 3;
1260 else if (!(uch & RT_BIT(3)))
1261 cb = 4;
1262 else if (!(uch & RT_BIT(2)))
1263 cb = 5;
1264 else if (!(uch & RT_BIT(1)))
1265 cb = 6;
1266 else
1267 {
1268 RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(strlen((char *)puch), 10), puch));
1269 return rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING);
1270 }
1271
1272 /* validate the rest */
1273 switch (cb)
1274 {
1275 case 6:
1276 RTStrAssertMsgReturn((puch[5] & 0xc0) == 0x80, ("6/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1277 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1278 case 5:
1279 RTStrAssertMsgReturn((puch[4] & 0xc0) == 0x80, ("5/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1280 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1281 case 4:
1282 RTStrAssertMsgReturn((puch[3] & 0xc0) == 0x80, ("4/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1283 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1284 case 3:
1285 RTStrAssertMsgReturn((puch[2] & 0xc0) == 0x80, ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1286 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1287 case 2:
1288 RTStrAssertMsgReturn((puch[1] & 0xc0) == 0x80, ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1289 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1290 break;
1291 }
1292
1293 /* get and validate the code point. */
1294 switch (cb)
1295 {
1296 case 6:
1297 uc = (puch[5] & 0x3f)
1298 | ((RTUNICP)(puch[4] & 0x3f) << 6)
1299 | ((RTUNICP)(puch[3] & 0x3f) << 12)
1300 | ((RTUNICP)(puch[2] & 0x3f) << 18)
1301 | ((RTUNICP)(puch[1] & 0x3f) << 24)
1302 | ((RTUNICP)(uch & 0x01) << 30);
1303 RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
1304 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1305 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1306 break;
1307 case 5:
1308 uc = (puch[4] & 0x3f)
1309 | ((RTUNICP)(puch[3] & 0x3f) << 6)
1310 | ((RTUNICP)(puch[2] & 0x3f) << 12)
1311 | ((RTUNICP)(puch[1] & 0x3f) << 18)
1312 | ((RTUNICP)(uch & 0x03) << 24);
1313 RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
1314 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1315 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1316 break;
1317 case 4:
1318 uc = (puch[3] & 0x3f)
1319 | ((RTUNICP)(puch[2] & 0x3f) << 6)
1320 | ((RTUNICP)(puch[1] & 0x3f) << 12)
1321 | ((RTUNICP)(uch & 0x07) << 18);
1322 RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
1323 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1324 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1325 break;
1326 case 3:
1327 uc = (puch[2] & 0x3f)
1328 | ((RTUNICP)(puch[1] & 0x3f) << 6)
1329 | ((RTUNICP)(uch & 0x0f) << 12);
1330 RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
1331 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1332 rtStrGetCpExFailure(ppsz, pCp, uc == 0xffff || uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING));
1333 RTStrAssertMsgReturn(uc < 0xd800 || uc > 0xdfff,
1334 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1335 rtStrGetCpExFailure(ppsz, pCp, VERR_CODE_POINT_SURROGATE));
1336 break;
1337 case 2:
1338 uc = (puch[1] & 0x3f)
1339 | ((RTUNICP)(uch & 0x1f) << 6);
1340 RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
1341 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1342 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1343 break;
1344 default: /* impossible, but GCC is bitching. */
1345 uc = RTUNICP_INVALID;
1346 break;
1347 }
1348 puch += cb;
1349 }
1350 else
1351 {
1352 /* 6th bit is always set. */
1353 RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(strlen((char *)puch), 10), puch));
1354 return rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING);
1355 }
1356 *pCp = uc;
1357 *ppsz = (const char *)puch;
1358 return VINF_SUCCESS;
1359}
1360RT_EXPORT_SYMBOL(RTStrGetCpExInternal);
1361
1362
1363/**
1364 * Handle invalid encodings passed to RTStrGetCpNEx().
1365 * @returns rc
1366 * @param ppsz The pointer to the string position point.
1367 * @param pcch Pointer to the string length.
1368 * @param pCp Where to store RTUNICP_INVALID.
1369 * @param rc The iprt error code.
1370 */
1371static int rtStrGetCpNExFailure(const char **ppsz, size_t *pcch, PRTUNICP pCp, int rc)
1372{
1373 /*
1374 * Try find a valid encoding.
1375 */
1376 (*ppsz)++; /** @todo code this! */
1377 (*pcch)--;
1378 *pCp = RTUNICP_INVALID;
1379 return rc;
1380}
1381
1382
1383RTDECL(int) RTStrGetCpNExInternal(const char **ppsz, size_t *pcch, PRTUNICP pCp)
1384{
1385 const unsigned char *puch = (const unsigned char *)*ppsz;
1386 const unsigned char uch = *puch;
1387 size_t cch = *pcch;
1388 RTUNICP uc;
1389
1390 if (cch == 0)
1391 {
1392 *pCp = RTUNICP_INVALID;
1393 return VERR_END_OF_STRING;
1394 }
1395
1396 /* ASCII ? */
1397 if (!(uch & RT_BIT(7)))
1398 {
1399 uc = uch;
1400 puch++;
1401 cch--;
1402 }
1403 else if (uch & RT_BIT(6))
1404 {
1405 /* figure the length and validate the first octet. */
1406 unsigned cb;
1407 if (!(uch & RT_BIT(5)))
1408 cb = 2;
1409 else if (!(uch & RT_BIT(4)))
1410 cb = 3;
1411 else if (!(uch & RT_BIT(3)))
1412 cb = 4;
1413 else if (!(uch & RT_BIT(2)))
1414 cb = 5;
1415 else if (!(uch & RT_BIT(1)))
1416 cb = 6;
1417 else
1418 {
1419 RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(strlen((char *)puch), 10), puch));
1420 return rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING);
1421 }
1422
1423 if (cb > cch)
1424 return rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING);
1425
1426 /* validate the rest */
1427 switch (cb)
1428 {
1429 case 6:
1430 RTStrAssertMsgReturn((puch[5] & 0xc0) == 0x80, ("6/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1431 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1432 case 5:
1433 RTStrAssertMsgReturn((puch[4] & 0xc0) == 0x80, ("5/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1434 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1435 case 4:
1436 RTStrAssertMsgReturn((puch[3] & 0xc0) == 0x80, ("4/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1437 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1438 case 3:
1439 RTStrAssertMsgReturn((puch[2] & 0xc0) == 0x80, ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1440 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1441 case 2:
1442 RTStrAssertMsgReturn((puch[1] & 0xc0) == 0x80, ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1443 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1444 break;
1445 }
1446
1447 /* get and validate the code point. */
1448 switch (cb)
1449 {
1450 case 6:
1451 uc = (puch[5] & 0x3f)
1452 | ((RTUNICP)(puch[4] & 0x3f) << 6)
1453 | ((RTUNICP)(puch[3] & 0x3f) << 12)
1454 | ((RTUNICP)(puch[2] & 0x3f) << 18)
1455 | ((RTUNICP)(puch[1] & 0x3f) << 24)
1456 | ((RTUNICP)(uch & 0x01) << 30);
1457 RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
1458 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1459 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1460 break;
1461 case 5:
1462 uc = (puch[4] & 0x3f)
1463 | ((RTUNICP)(puch[3] & 0x3f) << 6)
1464 | ((RTUNICP)(puch[2] & 0x3f) << 12)
1465 | ((RTUNICP)(puch[1] & 0x3f) << 18)
1466 | ((RTUNICP)(uch & 0x03) << 24);
1467 RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
1468 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1469 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1470 break;
1471 case 4:
1472 uc = (puch[3] & 0x3f)
1473 | ((RTUNICP)(puch[2] & 0x3f) << 6)
1474 | ((RTUNICP)(puch[1] & 0x3f) << 12)
1475 | ((RTUNICP)(uch & 0x07) << 18);
1476 RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
1477 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1478 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1479 break;
1480 case 3:
1481 uc = (puch[2] & 0x3f)
1482 | ((RTUNICP)(puch[1] & 0x3f) << 6)
1483 | ((RTUNICP)(uch & 0x0f) << 12);
1484 RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
1485 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1486 rtStrGetCpNExFailure(ppsz, pcch, pCp, uc == 0xffff || uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING));
1487 RTStrAssertMsgReturn(uc < 0xd800 || uc > 0xdfff,
1488 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1489 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_CODE_POINT_SURROGATE));
1490 break;
1491 case 2:
1492 uc = (puch[1] & 0x3f)
1493 | ((RTUNICP)(uch & 0x1f) << 6);
1494 RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
1495 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1496 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1497 break;
1498 default: /* impossible, but GCC is bitching. */
1499 uc = RTUNICP_INVALID;
1500 break;
1501 }
1502 puch += cb;
1503 cch -= cb;
1504 }
1505 else
1506 {
1507 /* 6th bit is always set. */
1508 RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(strlen((char *)puch), 10), puch));
1509 return rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING);
1510 }
1511 *pCp = uc;
1512 *ppsz = (const char *)puch;
1513 (*pcch) = cch;
1514 return VINF_SUCCESS;
1515}
1516RT_EXPORT_SYMBOL(RTStrGetCpNExInternal);
1517
1518
1519RTDECL(char *) RTStrPutCpInternal(char *psz, RTUNICP uc)
1520{
1521 unsigned char *puch = (unsigned char *)psz;
1522 if (uc < 0x80)
1523 *puch++ = (unsigned char )uc;
1524 else if (uc < 0x00000800)
1525 {
1526 *puch++ = 0xc0 | (uc >> 6);
1527 *puch++ = 0x80 | (uc & 0x3f);
1528 }
1529 else if (uc < 0x00010000)
1530 {
1531 if ( uc < 0x0000d8000
1532 || ( uc > 0x0000dfff
1533 && uc < 0x0000fffe))
1534 {
1535 *puch++ = 0xe0 | (uc >> 12);
1536 *puch++ = 0x80 | ((uc >> 6) & 0x3f);
1537 *puch++ = 0x80 | (uc & 0x3f);
1538 }
1539 else
1540 {
1541 AssertMsgFailed(("Invalid code point U+%05x!\n", uc));
1542 *puch++ = 0x7f;
1543 }
1544 }
1545 else if (uc < 0x00200000)
1546 {
1547 *puch++ = 0xf0 | (uc >> 18);
1548 *puch++ = 0x80 | ((uc >> 12) & 0x3f);
1549 *puch++ = 0x80 | ((uc >> 6) & 0x3f);
1550 *puch++ = 0x80 | (uc & 0x3f);
1551 }
1552 else if (uc < 0x04000000)
1553 {
1554 *puch++ = 0xf8 | (uc >> 24);
1555 *puch++ = 0x80 | ((uc >> 18) & 0x3f);
1556 *puch++ = 0x80 | ((uc >> 12) & 0x3f);
1557 *puch++ = 0x80 | ((uc >> 6) & 0x3f);
1558 *puch++ = 0x80 | (uc & 0x3f);
1559 }
1560 else if (uc <= 0x7fffffff)
1561 {
1562 *puch++ = 0xfc | (uc >> 30);
1563 *puch++ = 0x80 | ((uc >> 24) & 0x3f);
1564 *puch++ = 0x80 | ((uc >> 18) & 0x3f);
1565 *puch++ = 0x80 | ((uc >> 12) & 0x3f);
1566 *puch++ = 0x80 | ((uc >> 6) & 0x3f);
1567 *puch++ = 0x80 | (uc & 0x3f);
1568 }
1569 else
1570 {
1571 AssertMsgFailed(("Invalid code point U+%08x!\n", uc));
1572 *puch++ = 0x7f;
1573 }
1574
1575 return (char *)puch;
1576}
1577RT_EXPORT_SYMBOL(RTStrPutCpInternal);
1578
1579
1580RTDECL(char *) RTStrPrevCp(const char *pszStart, const char *psz)
1581{
1582 if (pszStart < psz)
1583 {
1584 /* simple char? */
1585 const unsigned char *puch = (const unsigned char *)psz;
1586 unsigned uch = *--puch;
1587 if (!(uch & RT_BIT(7)))
1588 return (char *)puch;
1589 RTStrAssertMsgReturn(!(uch & RT_BIT(6)), ("uch=%#x\n", uch), (char *)pszStart);
1590
1591 /* two or more. */
1592 uint32_t uMask = 0xffffffc0;
1593 while ( (const unsigned char *)pszStart < puch
1594 && !(uMask & 1))
1595 {
1596 uch = *--puch;
1597 if ((uch & 0xc0) != 0x80)
1598 {
1599 RTStrAssertMsgReturn((uch & (uMask >> 1)) == (uMask & 0xff),
1600 ("Invalid UTF-8 encoding: %.*Rhxs puch=%p psz=%p\n", psz - (char *)puch, puch, psz),
1601 (char *)pszStart);
1602 return (char *)puch;
1603 }
1604 uMask >>= 1;
1605 }
1606 RTStrAssertMsgFailed(("Invalid UTF-8 encoding: %.*Rhxs puch=%p psz=%p\n", psz - (char *)puch, puch, psz));
1607 }
1608 return (char *)pszStart;
1609}
1610RT_EXPORT_SYMBOL(RTStrPrevCp);
1611
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette