VirtualBox

source: vbox/trunk/src/VBox/Runtime/common/string/utf-8.cpp@ 31246

Last change on this file since 31246 was 31246, checked in by vboxsync, 14 years ago

Runtime/string: clean up the UTF-8 <-> Latin1 APIs a bit more
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 53.7 KB
Line 
1/* $Id: utf-8.cpp 31246 2010-07-30 13:24:53Z vboxsync $ */
2/** @file
3 * IPRT - UTF-8 Decoding.
4 */
5
6/*
7 * Copyright (C) 2006-2010 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * The contents of this file may alternatively be used under the terms
18 * of the Common Development and Distribution License Version 1.0
19 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20 * VirtualBox OSE distribution, in which case the provisions of the
21 * CDDL are applicable instead of those of the GPL.
22 *
23 * You may elect to license modified versions of this file under the
24 * terms and conditions of either the GPL or the CDDL or both.
25 */
26
27
28/*******************************************************************************
29* Header Files *
30*******************************************************************************/
31#include <iprt/string.h>
32#include "internal/iprt.h"
33
34#include <iprt/uni.h>
35#include <iprt/alloc.h>
36#include <iprt/assert.h>
37#include <iprt/err.h>
38#include "internal/string.h"
39
40
41
42/**
43 * Get get length in code points of a UTF-8 encoded string.
44 * The string is validated while doing this.
45 *
46 * @returns IPRT status code.
47 * @param psz Pointer to the UTF-8 string.
48 * @param cch The max length of the string. (btw cch = cb)
49 * Use RTSTR_MAX if all of the string is to be examined.
50 * @param pcuc Where to store the length in unicode code points.
51 * @param pcchActual Where to store the actual size of the UTF-8 string
52 * on success (cch = cb again). Optional.
53 */
54int rtUtf8Length(const char *psz, size_t cch, size_t *pcuc, size_t *pcchActual)
55{
56 const unsigned char *puch = (const unsigned char *)psz;
57 size_t cCodePoints = 0;
58 while (cch > 0)
59 {
60 const unsigned char uch = *puch;
61 if (!uch)
62 break;
63 if (uch & RT_BIT(7))
64 {
65 /* figure sequence length and validate the first byte */
66 unsigned cb;
67 if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5))) == (RT_BIT(7) | RT_BIT(6)))
68 cb = 2;
69 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5)))
70 cb = 3;
71 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4)))
72 cb = 4;
73 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3)))
74 cb = 5;
75 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2) | RT_BIT(1))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2)))
76 cb = 6;
77 else
78 {
79 RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(cch, 10), puch));
80 return VERR_INVALID_UTF8_ENCODING;
81 }
82
83 /* check length */
84 if (cb > cch)
85 {
86 RTStrAssertMsgFailed(("Invalid UTF-8 length: cb=%d cch=%d (%.*Rhxs)\n", cb, cch, RT_MIN(cch, 10), puch));
87 return VERR_INVALID_UTF8_ENCODING;
88 }
89
90 /* validate the rest */
91 switch (cb)
92 {
93 case 6:
94 RTStrAssertMsgReturn((puch[5] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("6/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
95 case 5:
96 RTStrAssertMsgReturn((puch[4] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("5/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
97 case 4:
98 RTStrAssertMsgReturn((puch[3] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("4/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
99 case 3:
100 RTStrAssertMsgReturn((puch[2] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
101 case 2:
102 RTStrAssertMsgReturn((puch[1] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
103 break;
104 }
105
106 /* validate the code point. */
107 RTUNICP uc;
108 switch (cb)
109 {
110 case 6:
111 uc = (puch[5] & 0x3f)
112 | ((RTUNICP)(puch[4] & 0x3f) << 6)
113 | ((RTUNICP)(puch[3] & 0x3f) << 12)
114 | ((RTUNICP)(puch[2] & 0x3f) << 18)
115 | ((RTUNICP)(puch[1] & 0x3f) << 24)
116 | ((RTUNICP)(uch & 0x01) << 30);
117 RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
118 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
119 break;
120 case 5:
121 uc = (puch[4] & 0x3f)
122 | ((RTUNICP)(puch[3] & 0x3f) << 6)
123 | ((RTUNICP)(puch[2] & 0x3f) << 12)
124 | ((RTUNICP)(puch[1] & 0x3f) << 18)
125 | ((RTUNICP)(uch & 0x03) << 24);
126 RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
127 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
128 break;
129 case 4:
130 uc = (puch[3] & 0x3f)
131 | ((RTUNICP)(puch[2] & 0x3f) << 6)
132 | ((RTUNICP)(puch[1] & 0x3f) << 12)
133 | ((RTUNICP)(uch & 0x07) << 18);
134 RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
135 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
136 break;
137 case 3:
138 uc = (puch[2] & 0x3f)
139 | ((RTUNICP)(puch[1] & 0x3f) << 6)
140 | ((RTUNICP)(uch & 0x0f) << 12);
141 RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
142 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch),
143 uc == 0xffff || uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING);
144 RTStrAssertMsgReturn(uc < 0xd800 || uc > 0xdfff,
145 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_CODE_POINT_SURROGATE);
146 break;
147 case 2:
148 uc = (puch[1] & 0x3f)
149 | ((RTUNICP)(uch & 0x1f) << 6);
150 RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
151 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
152 break;
153 }
154
155 /* advance */
156 cch -= cb;
157 puch += cb;
158 }
159 else
160 {
161 /* one ASCII byte */
162 puch++;
163 cch--;
164 }
165 cCodePoints++;
166 }
167
168 /* done */
169 *pcuc = cCodePoints;
170 if (pcchActual)
171 *pcchActual = puch - (unsigned char const *)psz;
172 return VINF_SUCCESS;
173}
174
175
176/**
177 * Decodes and UTF-8 string into an array of unicode code point.
178 *
179 * Since we know the input is valid, we do *not* perform encoding or length checks.
180 *
181 * @returns iprt status code.
182 * @param psz The UTF-8 string to recode. This is a valid encoding.
183 * @param cch The number of chars (the type char, so bytes if you like) to process of the UTF-8 string.
184 * The recoding will stop when cch or '\\0' is reached. Pass RTSTR_MAX to process up to '\\0'.
185 * @param paCps Where to store the code points array.
186 * @param cCps The number of RTUNICP items the paCps buffer can hold, excluding the terminator ('\\0').
187 */
188static int rtUtf8Decode(const char *psz, size_t cch, PRTUNICP paCps, size_t cCps)
189{
190 int rc = VINF_SUCCESS;
191 const unsigned char *puch = (const unsigned char *)psz;
192 PRTUNICP pCp = paCps;
193 while (cch > 0)
194 {
195 /* read the next char and check for terminator. */
196 const unsigned char uch = *puch;
197 if (!uch)
198 break;
199
200 /* check for output overflow */
201 if (RT_UNLIKELY(cCps < 1))
202 {
203 rc = VERR_BUFFER_OVERFLOW;
204 break;
205 }
206 cCps--;
207
208 /* decode and recode the code point */
209 if (!(uch & RT_BIT(7)))
210 {
211 *pCp++ = uch;
212 puch++;
213 cch--;
214 }
215#ifdef RT_STRICT
216 else if (!(uch & RT_BIT(6)))
217 AssertMsgFailed(("Internal error!\n"));
218#endif
219 else if (!(uch & RT_BIT(5)))
220 {
221 *pCp++ = (puch[1] & 0x3f)
222 | ((uint16_t)(uch & 0x1f) << 6);
223 puch += 2;
224 cch -= 2;
225 }
226 else if (!(uch & RT_BIT(4)))
227 {
228 *pCp++ = (puch[2] & 0x3f)
229 | ((uint16_t)(puch[1] & 0x3f) << 6)
230 | ((uint16_t)(uch & 0x0f) << 12);
231 puch += 3;
232 cch -= 3;
233 }
234 else if (!(uch & RT_BIT(3)))
235 {
236 *pCp++ = (puch[3] & 0x3f)
237 | ((RTUNICP)(puch[2] & 0x3f) << 6)
238 | ((RTUNICP)(puch[1] & 0x3f) << 12)
239 | ((RTUNICP)(uch & 0x07) << 18);
240 puch += 4;
241 cch -= 4;
242 }
243 else if (!(uch & RT_BIT(2)))
244 {
245 *pCp++ = (puch[4] & 0x3f)
246 | ((RTUNICP)(puch[3] & 0x3f) << 6)
247 | ((RTUNICP)(puch[2] & 0x3f) << 12)
248 | ((RTUNICP)(puch[1] & 0x3f) << 18)
249 | ((RTUNICP)(uch & 0x03) << 24);
250 puch += 5;
251 cch -= 6;
252 }
253 else
254 {
255 Assert(!(uch & RT_BIT(1)));
256 *pCp++ = (puch[5] & 0x3f)
257 | ((RTUNICP)(puch[4] & 0x3f) << 6)
258 | ((RTUNICP)(puch[3] & 0x3f) << 12)
259 | ((RTUNICP)(puch[2] & 0x3f) << 18)
260 | ((RTUNICP)(puch[1] & 0x3f) << 24)
261 | ((RTUNICP)(uch & 0x01) << 30);
262 puch += 6;
263 cch -= 6;
264 }
265 }
266
267 /* done */
268 *pCp = 0;
269 return rc;
270}
271
272
273RTDECL(size_t) RTStrUniLen(const char *psz)
274{
275 size_t cCodePoints;
276 int rc = rtUtf8Length(psz, RTSTR_MAX, &cCodePoints, NULL);
277 return RT_SUCCESS(rc) ? cCodePoints : 0;
278}
279RT_EXPORT_SYMBOL(RTStrUniLen);
280
281
282RTDECL(int) RTStrUniLenEx(const char *psz, size_t cch, size_t *pcCps)
283{
284 size_t cCodePoints;
285 int rc = rtUtf8Length(psz, cch, &cCodePoints, NULL);
286 if (pcCps)
287 *pcCps = RT_SUCCESS(rc) ? cCodePoints : 0;
288 return rc;
289}
290RT_EXPORT_SYMBOL(RTStrUniLenEx);
291
292
293RTDECL(int) RTStrValidateEncoding(const char *psz)
294{
295 return RTStrValidateEncodingEx(psz, RTSTR_MAX, 0);
296}
297RT_EXPORT_SYMBOL(RTStrValidateEncoding);
298
299
300RTDECL(int) RTStrValidateEncodingEx(const char *psz, size_t cch, uint32_t fFlags)
301{
302 AssertReturn(!(fFlags & ~(RTSTR_VALIDATE_ENCODING_ZERO_TERMINATED)), VERR_INVALID_PARAMETER);
303 AssertPtr(psz);
304
305 /*
306 * Use rtUtf8Length for the job.
307 */
308 size_t cchActual;
309 size_t cCpsIgnored;
310 int rc = rtUtf8Length(psz, cch, &cCpsIgnored, &cchActual);
311 if (RT_SUCCESS(rc))
312 {
313 if ( (fFlags & RTSTR_VALIDATE_ENCODING_ZERO_TERMINATED)
314 && cchActual >= cch)
315 rc = VERR_BUFFER_OVERFLOW;
316 }
317 return rc;
318}
319RT_EXPORT_SYMBOL(RTStrValidateEncodingEx);
320
321
322RTDECL(bool) RTStrIsValidEncoding(const char *psz)
323{
324 int rc = RTStrValidateEncodingEx(psz, RTSTR_MAX, 0);
325 return RT_SUCCESS(rc);
326}
327RT_EXPORT_SYMBOL(RTStrIsValidEncoding);
328
329
330RTDECL(size_t) RTStrPurgeEncoding(char *psz)
331{
332 size_t cErrors = 0;
333 for (;;)
334 {
335 RTUNICP Cp;
336 int rc = RTStrGetCpEx((const char **)&psz, &Cp);
337 if (RT_SUCCESS(rc))
338 {
339 if (!Cp)
340 break;
341 }
342 else
343 {
344 psz[-1] = '?';
345 cErrors++;
346 }
347 }
348 return cErrors;
349}
350RT_EXPORT_SYMBOL(RTStrPurgeEncoding);
351
352
353RTDECL(int) RTStrToUni(const char *pszString, PRTUNICP *ppaCps)
354{
355 /*
356 * Validate input.
357 */
358 Assert(VALID_PTR(pszString));
359 Assert(VALID_PTR(ppaCps));
360 *ppaCps = NULL;
361
362 /*
363 * Validate the UTF-8 input and count its code points.
364 */
365 size_t cCps;
366 int rc = rtUtf8Length(pszString, RTSTR_MAX, &cCps, NULL);
367 if (RT_SUCCESS(rc))
368 {
369 /*
370 * Allocate buffer.
371 */
372 PRTUNICP paCps = (PRTUNICP)RTMemAlloc((cCps + 1) * sizeof(RTUNICP));
373 if (paCps)
374 {
375 /*
376 * Decode the string.
377 */
378 rc = rtUtf8Decode(pszString, RTSTR_MAX, paCps, cCps);
379 if (RT_SUCCESS(rc))
380 {
381 *ppaCps = paCps;
382 return rc;
383 }
384 RTMemFree(paCps);
385 }
386 else
387 rc = VERR_NO_CODE_POINT_MEMORY;
388 }
389 return rc;
390}
391RT_EXPORT_SYMBOL(RTStrToUni);
392
393
394RTDECL(int) RTStrToUniEx(const char *pszString, size_t cchString, PRTUNICP *ppaCps, size_t cCps, size_t *pcCps)
395{
396 /*
397 * Validate input.
398 */
399 Assert(VALID_PTR(pszString));
400 Assert(VALID_PTR(ppaCps));
401 Assert(!pcCps || VALID_PTR(pcCps));
402
403 /*
404 * Validate the UTF-8 input and count the code points.
405 */
406 size_t cCpsResult;
407 int rc = rtUtf8Length(pszString, cchString, &cCpsResult, NULL);
408 if (RT_SUCCESS(rc))
409 {
410 if (pcCps)
411 *pcCps = cCpsResult;
412
413 /*
414 * Check buffer size / Allocate buffer.
415 */
416 bool fShouldFree;
417 PRTUNICP paCpsResult;
418 if (cCps > 0 && *ppaCps)
419 {
420 fShouldFree = false;
421 if (cCps <= cCpsResult)
422 return VERR_BUFFER_OVERFLOW;
423 paCpsResult = *ppaCps;
424 }
425 else
426 {
427 *ppaCps = NULL;
428 fShouldFree = true;
429 cCps = RT_MAX(cCpsResult + 1, cCps);
430 paCpsResult = (PRTUNICP)RTMemAlloc(cCps * sizeof(RTUNICP));
431 }
432 if (paCpsResult)
433 {
434 /*
435 * Encode the UTF-16 string.
436 */
437 rc = rtUtf8Decode(pszString, cchString, paCpsResult, cCps - 1);
438 if (RT_SUCCESS(rc))
439 {
440 *ppaCps = paCpsResult;
441 return rc;
442 }
443 if (fShouldFree)
444 RTMemFree(paCpsResult);
445 }
446 else
447 rc = VERR_NO_CODE_POINT_MEMORY;
448 }
449 return rc;
450}
451RT_EXPORT_SYMBOL(RTStrToUniEx);
452
453
454/**
455 * Calculates the UTF-16 length of a string, validating the encoding while doing so.
456 *
457 * @returns IPRT status code.
458 * @param psz Pointer to the UTF-8 string.
459 * @param cch The max length of the string. (btw cch = cb)
460 * Use RTSTR_MAX if all of the string is to be examined.
461 * @param pcwc Where to store the length of the UTF-16 string as a number of RTUTF16 characters.
462 */
463static int rtUtf8CalcUtf16Length(const char *psz, size_t cch, size_t *pcwc)
464{
465 const unsigned char *puch = (const unsigned char *)psz;
466 size_t cwc = 0;
467 while (cch > 0)
468 {
469 const unsigned char uch = *puch;
470 if (!uch)
471 break;
472 if (!(uch & RT_BIT(7)))
473 {
474 /* one ASCII byte */
475 cwc++;
476 puch++;
477 cch--;
478 }
479 else
480 {
481 /* figure sequence length and validate the first byte */
482 unsigned cb;
483 if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5))) == (RT_BIT(7) | RT_BIT(6)))
484 cb = 2;
485 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5)))
486 cb = 3;
487 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4)))
488 cb = 4;
489 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3)))
490 cb = 5;
491 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2) | RT_BIT(1))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2)))
492 cb = 6;
493 else
494 {
495 RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(cch, 10), puch));
496 return VERR_INVALID_UTF8_ENCODING;
497 }
498
499 /* check length */
500 if (cb > cch)
501 {
502 RTStrAssertMsgFailed(("Invalid UTF-8 length: cb=%d cch=%d (%.*Rhxs)\n", cb, cch, RT_MIN(cch, 10), puch));
503 return VERR_INVALID_UTF8_ENCODING;
504 }
505
506 /* validate the rest */
507 switch (cb)
508 {
509 case 6:
510 RTStrAssertMsgReturn((puch[5] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("6/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
511 case 5:
512 RTStrAssertMsgReturn((puch[4] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("5/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
513 case 4:
514 RTStrAssertMsgReturn((puch[3] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("4/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
515 case 3:
516 RTStrAssertMsgReturn((puch[2] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
517 case 2:
518 RTStrAssertMsgReturn((puch[1] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
519 break;
520 }
521
522 /* validate the code point. */
523 RTUNICP uc;
524 switch (cb)
525 {
526 case 6:
527 uc = (puch[5] & 0x3f)
528 | ((RTUNICP)(puch[4] & 0x3f) << 6)
529 | ((RTUNICP)(puch[3] & 0x3f) << 12)
530 | ((RTUNICP)(puch[2] & 0x3f) << 18)
531 | ((RTUNICP)(puch[1] & 0x3f) << 24)
532 | ((RTUNICP)(uch & 0x01) << 30);
533 RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
534 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
535 RTStrAssertMsgFailed(("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch));
536 return VERR_CANT_RECODE_AS_UTF16;
537 case 5:
538 uc = (puch[4] & 0x3f)
539 | ((RTUNICP)(puch[3] & 0x3f) << 6)
540 | ((RTUNICP)(puch[2] & 0x3f) << 12)
541 | ((RTUNICP)(puch[1] & 0x3f) << 18)
542 | ((RTUNICP)(uch & 0x03) << 24);
543 RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
544 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
545 RTStrAssertMsgFailed(("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch));
546 return VERR_CANT_RECODE_AS_UTF16;
547 case 4:
548 uc = (puch[3] & 0x3f)
549 | ((RTUNICP)(puch[2] & 0x3f) << 6)
550 | ((RTUNICP)(puch[1] & 0x3f) << 12)
551 | ((RTUNICP)(uch & 0x07) << 18);
552 RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
553 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
554 RTStrAssertMsgReturn(uc <= 0x0010ffff,
555 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_CANT_RECODE_AS_UTF16);
556 cwc++;
557 break;
558 case 3:
559 uc = (puch[2] & 0x3f)
560 | ((RTUNICP)(puch[1] & 0x3f) << 6)
561 | ((RTUNICP)(uch & 0x0f) << 12);
562 RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
563 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch),
564 uc == 0xffff || uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING);
565 RTStrAssertMsgReturn(uc < 0xd800 || uc > 0xdfff,
566 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_CODE_POINT_SURROGATE);
567 break;
568 case 2:
569 uc = (puch[1] & 0x3f)
570 | ((RTUNICP)(uch & 0x1f) << 6);
571 RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
572 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
573 break;
574 }
575
576 /* advance */
577 cch -= cb;
578 puch += cb;
579 cwc++;
580 }
581 }
582
583 /* done */
584 *pcwc = cwc;
585 return VINF_SUCCESS;
586}
587
588
589/**
590 * Recodes a valid UTF-8 string as UTF-16.
591 *
592 * Since we know the input is valid, we do *not* perform encoding or length checks.
593 *
594 * @returns iprt status code.
595 * @param psz The UTF-8 string to recode. This is a valid encoding.
596 * @param cch The number of chars (the type char, so bytes if you like) to process of the UTF-8 string.
597 * The recoding will stop when cch or '\\0' is reached. Pass RTSTR_MAX to process up to '\\0'.
598 * @param pwsz Where to store the UTF-16 string.
599 * @param cwc The number of RTUTF16 items the pwsz buffer can hold, excluding the terminator ('\\0').
600 */
601static int rtUtf8RecodeAsUtf16(const char *psz, size_t cch, PRTUTF16 pwsz, size_t cwc)
602{
603 int rc = VINF_SUCCESS;
604 const unsigned char *puch = (const unsigned char *)psz;
605 PRTUTF16 pwc = pwsz;
606 while (cch > 0)
607 {
608 /* read the next char and check for terminator. */
609 const unsigned char uch = *puch;
610 if (!uch)
611 break;
612
613 /* check for output overflow */
614 if (RT_UNLIKELY(cwc < 1))
615 {
616 rc = VERR_BUFFER_OVERFLOW;
617 break;
618 }
619 cwc--;
620
621 /* decode and recode the code point */
622 if (!(uch & RT_BIT(7)))
623 {
624 *pwc++ = uch;
625 puch++;
626 cch--;
627 }
628 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5))) == (RT_BIT(7) | RT_BIT(6)))
629 {
630 uint16_t uc = (puch[1] & 0x3f)
631 | ((uint16_t)(uch & 0x1f) << 6);
632 *pwc++ = uc;
633 puch += 2;
634 cch -= 2;
635 }
636 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5)))
637 {
638 uint16_t uc = (puch[2] & 0x3f)
639 | ((uint16_t)(puch[1] & 0x3f) << 6)
640 | ((uint16_t)(uch & 0x0f) << 12);
641 *pwc++ = uc;
642 puch += 3;
643 cch -= 3;
644 }
645 else
646 {
647 /* generate surrugate pair */
648 Assert((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4)));
649 RTUNICP uc = (puch[3] & 0x3f)
650 | ((RTUNICP)(puch[2] & 0x3f) << 6)
651 | ((RTUNICP)(puch[1] & 0x3f) << 12)
652 | ((RTUNICP)(uch & 0x07) << 18);
653 if (RT_UNLIKELY(cwc < 1))
654 {
655 rc = VERR_BUFFER_OVERFLOW;
656 break;
657 }
658 cwc--;
659
660 uc -= 0x10000;
661 *pwc++ = 0xd800 | (uc >> 10);
662 *pwc++ = 0xdc00 | (uc & 0x3ff);
663 puch += 4;
664 cch -= 4;
665 }
666 }
667
668 /* done */
669 *pwc = '\0';
670 return rc;
671}
672
673
674RTDECL(int) RTStrToUtf16Tag(const char *pszString, PRTUTF16 *ppwszString, const char *pszTag)
675{
676 /*
677 * Validate input.
678 */
679 Assert(VALID_PTR(ppwszString));
680 Assert(VALID_PTR(pszString));
681 *ppwszString = NULL;
682
683 /*
684 * Validate the UTF-8 input and calculate the length of the UTF-16 string.
685 */
686 size_t cwc;
687 int rc = rtUtf8CalcUtf16Length(pszString, RTSTR_MAX, &cwc);
688 if (RT_SUCCESS(rc))
689 {
690 /*
691 * Allocate buffer.
692 */
693 PRTUTF16 pwsz = (PRTUTF16)RTMemAllocTag((cwc + 1) * sizeof(RTUTF16), pszTag);
694 if (pwsz)
695 {
696 /*
697 * Encode the UTF-16 string.
698 */
699 rc = rtUtf8RecodeAsUtf16(pszString, RTSTR_MAX, pwsz, cwc);
700 if (RT_SUCCESS(rc))
701 {
702 *ppwszString = pwsz;
703 return rc;
704 }
705 RTMemFree(pwsz);
706 }
707 else
708 rc = VERR_NO_UTF16_MEMORY;
709 }
710 return rc;
711}
712RT_EXPORT_SYMBOL(RTStrToUtf16Tag);
713
714
715RTDECL(int) RTStrToUtf16ExTag(const char *pszString, size_t cchString,
716 PRTUTF16 *ppwsz, size_t cwc, size_t *pcwc, const char *pszTag)
717{
718 /*
719 * Validate input.
720 */
721 Assert(VALID_PTR(pszString));
722 Assert(VALID_PTR(ppwsz));
723 Assert(!pcwc || VALID_PTR(pcwc));
724
725 /*
726 * Validate the UTF-8 input and calculate the length of the UTF-16 string.
727 */
728 size_t cwcResult;
729 int rc = rtUtf8CalcUtf16Length(pszString, cchString, &cwcResult);
730 if (RT_SUCCESS(rc))
731 {
732 if (pcwc)
733 *pcwc = cwcResult;
734
735 /*
736 * Check buffer size / Allocate buffer.
737 */
738 bool fShouldFree;
739 PRTUTF16 pwszResult;
740 if (cwc > 0 && *ppwsz)
741 {
742 fShouldFree = false;
743 if (cwc <= cwcResult)
744 return VERR_BUFFER_OVERFLOW;
745 pwszResult = *ppwsz;
746 }
747 else
748 {
749 *ppwsz = NULL;
750 fShouldFree = true;
751 cwc = RT_MAX(cwcResult + 1, cwc);
752 pwszResult = (PRTUTF16)RTMemAllocTag(cwc * sizeof(RTUTF16), pszTag);
753 }
754 if (pwszResult)
755 {
756 /*
757 * Encode the UTF-16 string.
758 */
759 rc = rtUtf8RecodeAsUtf16(pszString, cchString, pwszResult, cwc - 1);
760 if (RT_SUCCESS(rc))
761 {
762 *ppwsz = pwszResult;
763 return rc;
764 }
765 if (fShouldFree)
766 RTMemFree(pwszResult);
767 }
768 else
769 rc = VERR_NO_UTF16_MEMORY;
770 }
771 return rc;
772}
773RT_EXPORT_SYMBOL(RTStrToUtf16ExTag);
774
775
776RTDECL(size_t) RTStrCalcUtf16Len(const char *psz)
777{
778 size_t cwc;
779 int rc = rtUtf8CalcUtf16Length(psz, RTSTR_MAX, &cwc);
780 return RT_SUCCESS(rc) ? cwc : 0;
781}
782RT_EXPORT_SYMBOL(RTStrCalcUtf16Len);
783
784
785RTDECL(int) RTStrCalcUtf16LenEx(const char *psz, size_t cch, size_t *pcwc)
786{
787 size_t cwc;
788 int rc = rtUtf8CalcUtf16Length(psz, cch, &cwc);
789 if (pcwc)
790 *pcwc = RT_SUCCESS(rc) ? cwc : ~(size_t)0;
791 return rc;
792}
793RT_EXPORT_SYMBOL(RTStrCalcUtf16LenEx);
794
795
796/**
797 * Calculates the length of the UTF-8 encoding of a Latin-1 string.
798 *
799 * @returns iprt status code.
800 * @param psz The Latin-1 string.
801 * @param cchIn The max length of the Latin-1 string to consider.
802 * @param pcch Where to store the length (excluding '\\0') of the UTF-8 string. (cch == cb, btw)
803 */
804static int rtLatin1CalcUtf8Length(const char *psz, size_t cchIn, size_t *pcch)
805{
806 size_t cch = 0;
807 while (true)
808 {
809 RTUNICP Cp;
810 size_t cchCp;
811 int rc = RTLatin1GetCpNEx(&psz, &cchIn, &Cp);
812 if (Cp == 0 || rc == VERR_END_OF_STRING)
813 break;
814 if (RT_FAILURE(rc))
815 return rc;
816 cchCp = RTStrCpSize(Cp);
817 if (cchCp == 0)
818 return VERR_NO_TRANSLATION;
819 cch += cchCp;
820 }
821
822 /* done */
823 *pcch = cch;
824 return VINF_SUCCESS;
825}
826
827
828/**
829 * Recodes a Latin-1 string as UTF-8.
830 *
831 * @returns iprt status code.
832 * @param psz The Latin-1 string.
833 * @param cchIn The number of characters to process from psz. The recoding
834 * will stop when cch or '\\0' is reached.
835 * @param psz Where to store the UTF-8 string.
836 * @param cch The size of the UTF-8 buffer, excluding the terminator.
837 */
838static int rtLatin1RecodeAsUtf8(const char *pszIn, size_t cchIn, char *psz, size_t cch)
839{
840 int rc = VINF_SUCCESS;
841
842 while (true)
843 {
844 RTUNICP Cp;
845 size_t cchCp;
846 rc = RTLatin1GetCpNEx(&pszIn, &cchIn, &Cp);
847 if (Cp == 0 || RT_FAILURE(rc))
848 break;
849 cchCp = RTStrCpSize(Cp);
850 if (RT_UNLIKELY(cch < cchCp))
851 {
852 RTStrAssertMsgFailed(("Buffer overflow! 1\n"));
853 rc = VERR_BUFFER_OVERFLOW;
854 break;
855 }
856 psz = RTStrPutCp(psz, Cp);
857 cch -= cchCp;
858 }
859
860 /* done */
861 if (rc == VERR_END_OF_STRING)
862 rc = VINF_SUCCESS;
863 *psz = '\0';
864 return rc;
865}
866
867
868
869RTDECL(int) RTLatin1ToUtf8Tag(const char *pszString, char **ppszString, const char *pszTag)
870{
871 /*
872 * Validate input.
873 */
874 Assert(VALID_PTR(ppszString));
875 Assert(VALID_PTR(pszString));
876 *ppszString = NULL;
877
878 /*
879 * Calculate the length of the UTF-8 encoding of the Latin-1 string.
880 */
881 size_t cch;
882 int rc = rtLatin1CalcUtf8Length(pszString, RTSTR_MAX, &cch);
883 if (RT_SUCCESS(rc))
884 {
885 /*
886 * Allocate buffer and recode it.
887 */
888 char *pszResult = (char *)RTMemAllocTag(cch + 1, pszTag);
889 if (pszResult)
890 {
891 rc = rtLatin1RecodeAsUtf8(pszString, RTSTR_MAX, pszResult, cch);
892 if (RT_SUCCESS(rc))
893 {
894 *ppszString = pszResult;
895 return rc;
896 }
897
898 RTMemFree(pszResult);
899 }
900 else
901 rc = VERR_NO_STR_MEMORY;
902 }
903 return rc;
904}
905RT_EXPORT_SYMBOL(RTLatin1ToUtf8Tag);
906
907
908RTDECL(int) RTLatin1ToUtf8ExTag(const char *pszString, size_t cchString, char **ppsz, size_t cch, size_t *pcch, const char *pszTag)
909{
910 /*
911 * Validate input.
912 */
913 Assert(VALID_PTR(pszString));
914 Assert(VALID_PTR(ppsz));
915 Assert(!pcch || VALID_PTR(pcch));
916
917 /*
918 * Calculate the length of the UTF-8 encoding of the Latin-1 string.
919 */
920 size_t cchResult;
921 int rc = rtLatin1CalcUtf8Length(pszString, cchString, &cchResult);
922 if (RT_SUCCESS(rc))
923 {
924 if (pcch)
925 *pcch = cchResult;
926
927 /*
928 * Check buffer size / Allocate buffer and recode it.
929 */
930 bool fShouldFree;
931 char *pszResult;
932 if (cch > 0 && *ppsz)
933 {
934 fShouldFree = false;
935 if (RT_UNLIKELY(cch <= cchResult))
936 return VERR_BUFFER_OVERFLOW;
937 pszResult = *ppsz;
938 }
939 else
940 {
941 *ppsz = NULL;
942 fShouldFree = true;
943 cch = RT_MAX(cch, cchResult + 1);
944 pszResult = (char *)RTStrAllocTag(cch, pszTag);
945 }
946 if (pszResult)
947 {
948 rc = rtLatin1RecodeAsUtf8(pszString, cchString, pszResult, cch - 1);
949 if (RT_SUCCESS(rc))
950 {
951 *ppsz = pszResult;
952 return rc;
953 }
954
955 if (fShouldFree)
956 RTStrFree(pszResult);
957 }
958 else
959 rc = VERR_NO_STR_MEMORY;
960 }
961 return rc;
962}
963RT_EXPORT_SYMBOL(RTLatin1ToUtf8ExTag);
964
965
966RTDECL(size_t) RTLatin1CalcUtf8Len(const char *psz)
967{
968 size_t cch;
969 int rc = rtLatin1CalcUtf8Length(psz, RTSTR_MAX, &cch);
970 return RT_SUCCESS(rc) ? cch : 0;
971}
972RT_EXPORT_SYMBOL(RTLatin1CalcUtf8Len);
973
974
975RTDECL(int) RTLatin1CalcUtf8LenEx(const char *psz, size_t cchIn, size_t *pcch)
976{
977 size_t cch;
978 int rc = rtLatin1CalcUtf8Length(psz, cchIn, &cch);
979 if (pcch)
980 *pcch = RT_SUCCESS(rc) ? cch : ~(size_t)0;
981 return rc;
982}
983RT_EXPORT_SYMBOL(RTLatin1CalcUtf8LenEx);
984
985
986/**
987 * Calculates the Latin-1 length of a string, validating the encoding while doing so.
988 *
989 * @returns IPRT status code.
990 * @param psz Pointer to the UTF-8 string.
991 * @param cchIn The max length of the string. (btw cch = cb)
992 * Use RTSTR_MAX if all of the string is to be examined.
993 * @param pcch Where to store the length of the Latin-1 string in bytes.
994 */
995static int rtUtf8CalcLatin1Length(const char *psz, size_t cchIn, size_t *pcch)
996{
997 size_t cch = 0;
998 while (true)
999 {
1000 RTUNICP Cp;
1001 size_t cchCp;
1002 int rc = RTStrGetCpNEx(&psz, &cchIn, &Cp);
1003 if (Cp == 0 || rc == VERR_END_OF_STRING)
1004 break;
1005 if (RT_FAILURE(rc))
1006 return rc;
1007 cchCp = RTLatin1CpSize(Cp);
1008 if (cchCp == 0)
1009 return VERR_NO_TRANSLATION;
1010 cch += cchCp;
1011 }
1012
1013 /* done */
1014 *pcch = cch;
1015 return VINF_SUCCESS;
1016}
1017
1018
1019/**
1020 * Recodes a valid UTF-8 string as Latin-1.
1021 *
1022 * Since we know the input is valid, we do *not* perform encoding or length checks.
1023 *
1024 * @returns iprt status code.
1025 * @param pszIn The UTF-8 string to recode. This is a valid encoding.
1026 * @param cchIn The number of chars (the type char, so bytes if you like) to process of the UTF-8 string.
1027 * The recoding will stop when cch or '\\0' is reached. Pass RTSTR_MAX to process up to '\\0'.
1028 * @param psz Where to store the Latin-1 string.
1029 * @param cch The number of characters the pszOut buffer can hold, excluding the terminator ('\\0').
1030 */
1031static int rtUtf8RecodeAsLatin1(const char *pszIn, size_t cchIn, char *psz, size_t cch)
1032{
1033 int rc = VINF_SUCCESS;
1034
1035 while (true)
1036 {
1037 RTUNICP Cp;
1038 size_t cchCp;
1039 rc = RTStrGetCpNEx(&pszIn, &cchIn, &Cp);
1040 if (Cp == 0 || RT_FAILURE(rc))
1041 break;
1042 cchCp = RTLatin1CpSize(Cp);
1043 if (RT_UNLIKELY(cch < cchCp))
1044 {
1045 RTStrAssertMsgFailed(("Buffer overflow! 1\n"));
1046 rc = VERR_BUFFER_OVERFLOW;
1047 break;
1048 }
1049 psz = RTLatin1PutCp(psz, Cp);
1050 cch -= cchCp;
1051 }
1052
1053 /* done */
1054 if (rc == VERR_END_OF_STRING)
1055 rc = VINF_SUCCESS;
1056 *psz = '\0';
1057 return rc;
1058}
1059
1060
1061
1062RTDECL(int) RTStrToLatin1Tag(const char *pszString, char **ppszString, const char *pszTag)
1063{
1064 /*
1065 * Validate input.
1066 */
1067 Assert(VALID_PTR(ppszString));
1068 Assert(VALID_PTR(pszString));
1069 *ppszString = NULL;
1070
1071 /*
1072 * Validate the UTF-8 input and calculate the length of the Latin-1 string.
1073 */
1074 size_t cch;
1075 int rc = rtUtf8CalcLatin1Length(pszString, RTSTR_MAX, &cch);
1076 if (RT_SUCCESS(rc))
1077 {
1078 /*
1079 * Allocate buffer.
1080 */
1081 char *psz = (char *)RTMemAllocTag(cch + 1, pszTag);
1082 if (psz)
1083 {
1084 /*
1085 * Encode the UTF-16 string.
1086 */
1087 rc = rtUtf8RecodeAsLatin1(pszString, RTSTR_MAX, psz, cch);
1088 if (RT_SUCCESS(rc))
1089 {
1090 *ppszString = psz;
1091 return rc;
1092 }
1093 RTMemFree(psz);
1094 }
1095 else
1096 rc = VERR_NO_STR_MEMORY;
1097 }
1098 return rc;
1099}
1100RT_EXPORT_SYMBOL(RTStrToLatin1Tag);
1101
1102
1103RTDECL(int) RTStrToLatin1ExTag(const char *pszString, size_t cchString,
1104 char **ppsz, size_t cch, size_t *pcch, const char *pszTag)
1105{
1106 /*
1107 * Validate input.
1108 */
1109 Assert(VALID_PTR(pszString));
1110 Assert(VALID_PTR(ppsz));
1111 Assert(!pcch || VALID_PTR(pcch));
1112
1113 /*
1114 * Validate the UTF-8 input and calculate the length of the UTF-16 string.
1115 */
1116 size_t cchResult;
1117 int rc = rtUtf8CalcLatin1Length(pszString, cchString, &cchResult);
1118 if (RT_SUCCESS(rc))
1119 {
1120 if (pcch)
1121 *pcch = cchResult;
1122
1123 /*
1124 * Check buffer size / Allocate buffer.
1125 */
1126 bool fShouldFree;
1127 char *pszResult;
1128 if (cch > 0 && *ppsz)
1129 {
1130 fShouldFree = false;
1131 if (cch <= cchResult)
1132 return VERR_BUFFER_OVERFLOW;
1133 pszResult = *ppsz;
1134 }
1135 else
1136 {
1137 *ppsz = NULL;
1138 fShouldFree = true;
1139 cch = RT_MAX(cchResult + 1, cch);
1140 pszResult = (char *)RTMemAllocTag(cch, pszTag);
1141 }
1142 if (pszResult)
1143 {
1144 /*
1145 * Encode the Latin-1 string.
1146 */
1147 rc = rtUtf8RecodeAsLatin1(pszString, cchString, pszResult, cch - 1);
1148 if (RT_SUCCESS(rc))
1149 {
1150 *ppsz = pszResult;
1151 return rc;
1152 }
1153 if (fShouldFree)
1154 RTMemFree(pszResult);
1155 }
1156 else
1157 rc = VERR_NO_STR_MEMORY;
1158 }
1159 return rc;
1160}
1161RT_EXPORT_SYMBOL(RTStrToLatin1Tag);
1162
1163
1164RTDECL(size_t) RTStrCalcLatin1Len(const char *psz)
1165{
1166 size_t cch;
1167 int rc = rtUtf8CalcLatin1Length(psz, RTSTR_MAX, &cch);
1168 return RT_SUCCESS(rc) ? cch : 0;
1169}
1170RT_EXPORT_SYMBOL(RTStrCalcLatin1Len);
1171
1172
1173RTDECL(int) RTStrCalcLatin1LenEx(const char *psz, size_t cchIn, size_t *pcch)
1174{
1175 size_t cch;
1176 int rc = rtUtf8CalcLatin1Length(psz, cchIn, &cch);
1177 if (pcch)
1178 *pcch = RT_SUCCESS(rc) ? cch : ~(size_t)0;
1179 return rc;
1180}
1181RT_EXPORT_SYMBOL(RTStrCalcLatin1LenEx);
1182
1183
1184/**
1185 * Handle invalid encodings passed to RTStrGetCp() and RTStrGetCpEx().
1186 * @returns rc
1187 * @param ppsz The pointer to the string position point.
1188 * @param pCp Where to store RTUNICP_INVALID.
1189 * @param rc The iprt error code.
1190 */
1191static int rtStrGetCpExFailure(const char **ppsz, PRTUNICP pCp, int rc)
1192{
1193 /*
1194 * Try find a valid encoding.
1195 */
1196 (*ppsz)++; /** @todo code this! */
1197 *pCp = RTUNICP_INVALID;
1198 return rc;
1199}
1200
1201
1202RTDECL(RTUNICP) RTStrGetCpInternal(const char *psz)
1203{
1204 RTUNICP Cp;
1205 RTStrGetCpExInternal(&psz, &Cp);
1206 return Cp;
1207}
1208RT_EXPORT_SYMBOL(RTStrGetCpInternal);
1209
1210
1211RTDECL(int) RTStrGetCpExInternal(const char **ppsz, PRTUNICP pCp)
1212{
1213 const unsigned char *puch = (const unsigned char *)*ppsz;
1214 const unsigned char uch = *puch;
1215 RTUNICP uc;
1216
1217 /* ASCII ? */
1218 if (!(uch & RT_BIT(7)))
1219 {
1220 uc = uch;
1221 puch++;
1222 }
1223 else if (uch & RT_BIT(6))
1224 {
1225 /* figure the length and validate the first octet. */
1226 unsigned cb;
1227 if (!(uch & RT_BIT(5)))
1228 cb = 2;
1229 else if (!(uch & RT_BIT(4)))
1230 cb = 3;
1231 else if (!(uch & RT_BIT(3)))
1232 cb = 4;
1233 else if (!(uch & RT_BIT(2)))
1234 cb = 5;
1235 else if (!(uch & RT_BIT(1)))
1236 cb = 6;
1237 else
1238 {
1239 RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(strlen((char *)puch), 10), puch));
1240 return rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING);
1241 }
1242
1243 /* validate the rest */
1244 switch (cb)
1245 {
1246 case 6:
1247 RTStrAssertMsgReturn((puch[5] & 0xc0) == 0x80, ("6/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1248 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1249 case 5:
1250 RTStrAssertMsgReturn((puch[4] & 0xc0) == 0x80, ("5/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1251 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1252 case 4:
1253 RTStrAssertMsgReturn((puch[3] & 0xc0) == 0x80, ("4/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1254 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1255 case 3:
1256 RTStrAssertMsgReturn((puch[2] & 0xc0) == 0x80, ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1257 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1258 case 2:
1259 RTStrAssertMsgReturn((puch[1] & 0xc0) == 0x80, ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1260 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1261 break;
1262 }
1263
1264 /* get and validate the code point. */
1265 switch (cb)
1266 {
1267 case 6:
1268 uc = (puch[5] & 0x3f)
1269 | ((RTUNICP)(puch[4] & 0x3f) << 6)
1270 | ((RTUNICP)(puch[3] & 0x3f) << 12)
1271 | ((RTUNICP)(puch[2] & 0x3f) << 18)
1272 | ((RTUNICP)(puch[1] & 0x3f) << 24)
1273 | ((RTUNICP)(uch & 0x01) << 30);
1274 RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
1275 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1276 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1277 break;
1278 case 5:
1279 uc = (puch[4] & 0x3f)
1280 | ((RTUNICP)(puch[3] & 0x3f) << 6)
1281 | ((RTUNICP)(puch[2] & 0x3f) << 12)
1282 | ((RTUNICP)(puch[1] & 0x3f) << 18)
1283 | ((RTUNICP)(uch & 0x03) << 24);
1284 RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
1285 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1286 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1287 break;
1288 case 4:
1289 uc = (puch[3] & 0x3f)
1290 | ((RTUNICP)(puch[2] & 0x3f) << 6)
1291 | ((RTUNICP)(puch[1] & 0x3f) << 12)
1292 | ((RTUNICP)(uch & 0x07) << 18);
1293 RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
1294 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1295 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1296 break;
1297 case 3:
1298 uc = (puch[2] & 0x3f)
1299 | ((RTUNICP)(puch[1] & 0x3f) << 6)
1300 | ((RTUNICP)(uch & 0x0f) << 12);
1301 RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
1302 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1303 rtStrGetCpExFailure(ppsz, pCp, uc == 0xffff || uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING));
1304 RTStrAssertMsgReturn(uc < 0xd800 || uc > 0xdfff,
1305 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1306 rtStrGetCpExFailure(ppsz, pCp, VERR_CODE_POINT_SURROGATE));
1307 break;
1308 case 2:
1309 uc = (puch[1] & 0x3f)
1310 | ((RTUNICP)(uch & 0x1f) << 6);
1311 RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
1312 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1313 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1314 break;
1315 default: /* impossible, but GCC is bitching. */
1316 uc = RTUNICP_INVALID;
1317 break;
1318 }
1319 puch += cb;
1320 }
1321 else
1322 {
1323 /* 6th bit is always set. */
1324 RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(strlen((char *)puch), 10), puch));
1325 return rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING);
1326 }
1327 *pCp = uc;
1328 *ppsz = (const char *)puch;
1329 return VINF_SUCCESS;
1330}
1331RT_EXPORT_SYMBOL(RTStrGetCpExInternal);
1332
1333
1334/**
1335 * Handle invalid encodings passed to RTStrGetCpNEx().
1336 * @returns rc
1337 * @param ppsz The pointer to the string position point.
1338 * @param pcch Pointer to the string length.
1339 * @param pCp Where to store RTUNICP_INVALID.
1340 * @param rc The iprt error code.
1341 */
1342static int rtStrGetCpNExFailure(const char **ppsz, size_t *pcch, PRTUNICP pCp, int rc)
1343{
1344 /*
1345 * Try find a valid encoding.
1346 */
1347 (*ppsz)++; /** @todo code this! */
1348 (*pcch)--;
1349 *pCp = RTUNICP_INVALID;
1350 return rc;
1351}
1352
1353
1354RTDECL(int) RTStrGetCpNExInternal(const char **ppsz, size_t *pcch, PRTUNICP pCp)
1355{
1356 const unsigned char *puch = (const unsigned char *)*ppsz;
1357 const unsigned char uch = *puch;
1358 size_t cch = *pcch;
1359 RTUNICP uc;
1360
1361 if (cch == 0)
1362 {
1363 *pCp = RTUNICP_INVALID;
1364 return VERR_END_OF_STRING;
1365 }
1366
1367 /* ASCII ? */
1368 if (!(uch & RT_BIT(7)))
1369 {
1370 uc = uch;
1371 puch++;
1372 cch--;
1373 }
1374 else if (uch & RT_BIT(6))
1375 {
1376 /* figure the length and validate the first octet. */
1377 unsigned cb;
1378 if (!(uch & RT_BIT(5)))
1379 cb = 2;
1380 else if (!(uch & RT_BIT(4)))
1381 cb = 3;
1382 else if (!(uch & RT_BIT(3)))
1383 cb = 4;
1384 else if (!(uch & RT_BIT(2)))
1385 cb = 5;
1386 else if (!(uch & RT_BIT(1)))
1387 cb = 6;
1388 else
1389 {
1390 RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(strlen((char *)puch), 10), puch));
1391 return rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING);
1392 }
1393
1394 if (cb > cch)
1395 return rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING);
1396
1397 /* validate the rest */
1398 switch (cb)
1399 {
1400 case 6:
1401 RTStrAssertMsgReturn((puch[5] & 0xc0) == 0x80, ("6/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1402 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1403 case 5:
1404 RTStrAssertMsgReturn((puch[4] & 0xc0) == 0x80, ("5/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1405 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1406 case 4:
1407 RTStrAssertMsgReturn((puch[3] & 0xc0) == 0x80, ("4/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1408 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1409 case 3:
1410 RTStrAssertMsgReturn((puch[2] & 0xc0) == 0x80, ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1411 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1412 case 2:
1413 RTStrAssertMsgReturn((puch[1] & 0xc0) == 0x80, ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1414 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1415 break;
1416 }
1417
1418 /* get and validate the code point. */
1419 switch (cb)
1420 {
1421 case 6:
1422 uc = (puch[5] & 0x3f)
1423 | ((RTUNICP)(puch[4] & 0x3f) << 6)
1424 | ((RTUNICP)(puch[3] & 0x3f) << 12)
1425 | ((RTUNICP)(puch[2] & 0x3f) << 18)
1426 | ((RTUNICP)(puch[1] & 0x3f) << 24)
1427 | ((RTUNICP)(uch & 0x01) << 30);
1428 RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
1429 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1430 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1431 break;
1432 case 5:
1433 uc = (puch[4] & 0x3f)
1434 | ((RTUNICP)(puch[3] & 0x3f) << 6)
1435 | ((RTUNICP)(puch[2] & 0x3f) << 12)
1436 | ((RTUNICP)(puch[1] & 0x3f) << 18)
1437 | ((RTUNICP)(uch & 0x03) << 24);
1438 RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
1439 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1440 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1441 break;
1442 case 4:
1443 uc = (puch[3] & 0x3f)
1444 | ((RTUNICP)(puch[2] & 0x3f) << 6)
1445 | ((RTUNICP)(puch[1] & 0x3f) << 12)
1446 | ((RTUNICP)(uch & 0x07) << 18);
1447 RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
1448 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1449 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1450 break;
1451 case 3:
1452 uc = (puch[2] & 0x3f)
1453 | ((RTUNICP)(puch[1] & 0x3f) << 6)
1454 | ((RTUNICP)(uch & 0x0f) << 12);
1455 RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
1456 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1457 rtStrGetCpNExFailure(ppsz, pcch, pCp, uc == 0xffff || uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING));
1458 RTStrAssertMsgReturn(uc < 0xd800 || uc > 0xdfff,
1459 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1460 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_CODE_POINT_SURROGATE));
1461 break;
1462 case 2:
1463 uc = (puch[1] & 0x3f)
1464 | ((RTUNICP)(uch & 0x1f) << 6);
1465 RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
1466 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1467 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1468 break;
1469 default: /* impossible, but GCC is bitching. */
1470 uc = RTUNICP_INVALID;
1471 break;
1472 }
1473 puch += cb;
1474 cch -= cb;
1475 }
1476 else
1477 {
1478 /* 6th bit is always set. */
1479 RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(strlen((char *)puch), 10), puch));
1480 return rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING);
1481 }
1482 *pCp = uc;
1483 *ppsz = (const char *)puch;
1484 (*pcch) = cch;
1485 return VINF_SUCCESS;
1486}
1487RT_EXPORT_SYMBOL(RTStrGetCpNExInternal);
1488
1489
1490RTDECL(char *) RTStrPutCpInternal(char *psz, RTUNICP uc)
1491{
1492 unsigned char *puch = (unsigned char *)psz;
1493 if (uc < 0x80)
1494 *puch++ = (unsigned char )uc;
1495 else if (uc < 0x00000800)
1496 {
1497 *puch++ = 0xc0 | (uc >> 6);
1498 *puch++ = 0x80 | (uc & 0x3f);
1499 }
1500 else if (uc < 0x00010000)
1501 {
1502 if ( uc < 0x0000d8000
1503 || ( uc > 0x0000dfff
1504 && uc < 0x0000fffe))
1505 {
1506 *puch++ = 0xe0 | (uc >> 12);
1507 *puch++ = 0x80 | ((uc >> 6) & 0x3f);
1508 *puch++ = 0x80 | (uc & 0x3f);
1509 }
1510 else
1511 {
1512 AssertMsgFailed(("Invalid code point U+%05x!\n", uc));
1513 *puch++ = 0x7f;
1514 }
1515 }
1516 else if (uc < 0x00200000)
1517 {
1518 *puch++ = 0xf0 | (uc >> 18);
1519 *puch++ = 0x80 | ((uc >> 12) & 0x3f);
1520 *puch++ = 0x80 | ((uc >> 6) & 0x3f);
1521 *puch++ = 0x80 | (uc & 0x3f);
1522 }
1523 else if (uc < 0x04000000)
1524 {
1525 *puch++ = 0xf8 | (uc >> 24);
1526 *puch++ = 0x80 | ((uc >> 18) & 0x3f);
1527 *puch++ = 0x80 | ((uc >> 12) & 0x3f);
1528 *puch++ = 0x80 | ((uc >> 6) & 0x3f);
1529 *puch++ = 0x80 | (uc & 0x3f);
1530 }
1531 else if (uc <= 0x7fffffff)
1532 {
1533 *puch++ = 0xfc | (uc >> 30);
1534 *puch++ = 0x80 | ((uc >> 24) & 0x3f);
1535 *puch++ = 0x80 | ((uc >> 18) & 0x3f);
1536 *puch++ = 0x80 | ((uc >> 12) & 0x3f);
1537 *puch++ = 0x80 | ((uc >> 6) & 0x3f);
1538 *puch++ = 0x80 | (uc & 0x3f);
1539 }
1540 else
1541 {
1542 AssertMsgFailed(("Invalid code point U+%08x!\n", uc));
1543 *puch++ = 0x7f;
1544 }
1545
1546 return (char *)puch;
1547}
1548RT_EXPORT_SYMBOL(RTStrPutCpInternal);
1549
1550
1551RTDECL(char *) RTStrPrevCp(const char *pszStart, const char *psz)
1552{
1553 if (pszStart < psz)
1554 {
1555 /* simple char? */
1556 const unsigned char *puch = (const unsigned char *)psz;
1557 unsigned uch = *--puch;
1558 if (!(uch & RT_BIT(7)))
1559 return (char *)puch;
1560 RTStrAssertMsgReturn(!(uch & RT_BIT(6)), ("uch=%#x\n", uch), (char *)pszStart);
1561
1562 /* two or more. */
1563 uint32_t uMask = 0xffffffc0;
1564 while ( (const unsigned char *)pszStart < puch
1565 && !(uMask & 1))
1566 {
1567 uch = *--puch;
1568 if ((uch & 0xc0) != 0x80)
1569 {
1570 RTStrAssertMsgReturn((uch & (uMask >> 1)) == (uMask & 0xff),
1571 ("Invalid UTF-8 encoding: %.*Rhxs puch=%p psz=%p\n", psz - (char *)puch, puch, psz),
1572 (char *)pszStart);
1573 return (char *)puch;
1574 }
1575 uMask >>= 1;
1576 }
1577 RTStrAssertMsgFailed(("Invalid UTF-8 encoding: %.*Rhxs puch=%p psz=%p\n", psz - (char *)puch, puch, psz));
1578 }
1579 return (char *)pszStart;
1580}
1581RT_EXPORT_SYMBOL(RTStrPrevCp);
1582
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette