utf-8.cpp@ 31157

Last change on this file since 31157 was 31157, checked in by vboxsync, 15 years ago
iprt,++: Tag allocation in all builds with a string, defaulting to FILE.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 43.7 KB

Line
1	/* $Id: utf-8.cpp 31157 2010-07-28 03:15:35Z vboxsync $ */
2	/** @file
3	* IPRT - UTF-8 Decoding.
4	*/
5
6	/*
7	* Copyright (C) 2006-2010 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*******************************************************************************
29	* Header Files *
30	*******************************************************************************/
31	#include <iprt/string.h>
32	#include "internal/iprt.h"
33
34	#include <iprt/uni.h>
35	#include <iprt/alloc.h>
36	#include <iprt/assert.h>
37	#include <iprt/err.h>
38	#include "internal/string.h"
39
40
41
42	/**
43	* Get get length in code points of a UTF-8 encoded string.
44	* The string is validated while doing this.
45	*
46	* @returns IPRT status code.
47	* @param psz Pointer to the UTF-8 string.
48	* @param cch The max length of the string. (btw cch = cb)
49	* Use RTSTR_MAX if all of the string is to be examined.
50	* @param pcuc Where to store the length in unicode code points.
51	* @param pcchActual Where to store the actual size of the UTF-8 string
52	* on success (cch = cb again). Optional.
53	*/
54	int rtUtf8Length(const char psz, size_t cch, size_t pcuc, size_t *pcchActual)
55	{
56	const unsigned char puch = (const unsigned char )psz;
57	size_t cCodePoints = 0;
58	while (cch > 0)
59	{
60	const unsigned char uch = *puch;
61	if (!uch)
62	break;
63	if (uch & RT_BIT(7))
64	{
65	/* figure sequence length and validate the first byte */
66	unsigned cb;
67	if ((uch & (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5))) == (RT_BIT(7) \| RT_BIT(6)))
68	cb = 2;
69	else if ((uch & (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4))) == (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5)))
70	cb = 3;
71	else if ((uch & (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4) \| RT_BIT(3))) == (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4)))
72	cb = 4;
73	else if ((uch & (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4) \| RT_BIT(3) \| RT_BIT(2))) == (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4) \| RT_BIT(3)))
74	cb = 5;
75	else if ((uch & (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4) \| RT_BIT(3) \| RT_BIT(2) \| RT_BIT(1))) == (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4) \| RT_BIT(3) \| RT_BIT(2)))
76	cb = 6;
77	else
78	{
79	RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(cch, 10), puch));
80	return VERR_INVALID_UTF8_ENCODING;
81	}
82
83	/* check length */
84	if (cb > cch)
85	{
86	RTStrAssertMsgFailed(("Invalid UTF-8 length: cb=%d cch=%d (%.*Rhxs)\n", cb, cch, RT_MIN(cch, 10), puch));
87	return VERR_INVALID_UTF8_ENCODING;
88	}
89
90	/* validate the rest */
91	switch (cb)
92	{
93	case 6:
94	RTStrAssertMsgReturn((puch[5] & (RT_BIT(7) \| RT_BIT(6))) == RT_BIT(7), ("6/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
95	case 5:
96	RTStrAssertMsgReturn((puch[4] & (RT_BIT(7) \| RT_BIT(6))) == RT_BIT(7), ("5/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
97	case 4:
98	RTStrAssertMsgReturn((puch[3] & (RT_BIT(7) \| RT_BIT(6))) == RT_BIT(7), ("4/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
99	case 3:
100	RTStrAssertMsgReturn((puch[2] & (RT_BIT(7) \| RT_BIT(6))) == RT_BIT(7), ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
101	case 2:
102	RTStrAssertMsgReturn((puch[1] & (RT_BIT(7) \| RT_BIT(6))) == RT_BIT(7), ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
103	break;
104	}
105
106	/* validate the code point. */
107	RTUNICP uc;
108	switch (cb)
109	{
110	case 6:
111	uc = (puch[5] & 0x3f)
112	\| ((RTUNICP)(puch[4] & 0x3f) << 6)
113	\| ((RTUNICP)(puch[3] & 0x3f) << 12)
114	\| ((RTUNICP)(puch[2] & 0x3f) << 18)
115	\| ((RTUNICP)(puch[1] & 0x3f) << 24)
116	\| ((RTUNICP)(uch & 0x01) << 30);
117	RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
118	("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
119	break;
120	case 5:
121	uc = (puch[4] & 0x3f)
122	\| ((RTUNICP)(puch[3] & 0x3f) << 6)
123	\| ((RTUNICP)(puch[2] & 0x3f) << 12)
124	\| ((RTUNICP)(puch[1] & 0x3f) << 18)
125	\| ((RTUNICP)(uch & 0x03) << 24);
126	RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
127	("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
128	break;
129	case 4:
130	uc = (puch[3] & 0x3f)
131	\| ((RTUNICP)(puch[2] & 0x3f) << 6)
132	\| ((RTUNICP)(puch[1] & 0x3f) << 12)
133	\| ((RTUNICP)(uch & 0x07) << 18);
134	RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
135	("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
136	break;
137	case 3:
138	uc = (puch[2] & 0x3f)
139	\| ((RTUNICP)(puch[1] & 0x3f) << 6)
140	\| ((RTUNICP)(uch & 0x0f) << 12);
141	RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
142	("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch),
143	uc == 0xffff \|\| uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING);
144	RTStrAssertMsgReturn(uc < 0xd800 \|\| uc > 0xdfff,
145	("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_CODE_POINT_SURROGATE);
146	break;
147	case 2:
148	uc = (puch[1] & 0x3f)
149	\| ((RTUNICP)(uch & 0x1f) << 6);
150	RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
151	("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
152	break;
153	}
154
155	/* advance */
156	cch -= cb;
157	puch += cb;
158	}
159	else
160	{
161	/* one ASCII byte */
162	puch++;
163	cch--;
164	}
165	cCodePoints++;
166	}
167
168	/* done */
169	*pcuc = cCodePoints;
170	if (pcchActual)
171	pcchActual = puch - (unsigned char const )psz;
172	return VINF_SUCCESS;
173	}
174
175
176	/**
177	* Decodes and UTF-8 string into an array of unicode code point.
178	*
179	* Since we know the input is valid, we do not perform encoding or length checks.
180	*
181	* @returns iprt status code.
182	* @param psz The UTF-8 string to recode. This is a valid encoding.
183	* @param cch The number of chars (the type char, so bytes if you like) to process of the UTF-8 string.
184	* The recoding will stop when cch or '\\0' is reached. Pass RTSTR_MAX to process up to '\\0'.
185	* @param paCps Where to store the code points array.
186	* @param cCps The number of RTUNICP items the paCps buffer can hold, excluding the terminator ('\\0').
187	*/
188	static int rtUtf8Decode(const char *psz, size_t cch, PRTUNICP paCps, size_t cCps)
189	{
190	int rc = VINF_SUCCESS;
191	const unsigned char puch = (const unsigned char )psz;
192	PRTUNICP pCp = paCps;
193	while (cch > 0)
194	{
195	/* read the next char and check for terminator. */
196	const unsigned char uch = *puch;
197	if (!uch)
198	break;
199
200	/* check for output overflow */
201	if (RT_UNLIKELY(cCps < 1))
202	{
203	rc = VERR_BUFFER_OVERFLOW;
204	break;
205	}
206	cCps--;
207
208	/* decode and recode the code point */
209	if (!(uch & RT_BIT(7)))
210	{
211	*pCp++ = uch;
212	puch++;
213	cch--;
214	}
215	#ifdef RT_STRICT
216	else if (!(uch & RT_BIT(6)))
217	AssertMsgFailed(("Internal error!\n"));
218	#endif
219	else if (!(uch & RT_BIT(5)))
220	{
221	*pCp++ = (puch[1] & 0x3f)
222	\| ((uint16_t)(uch & 0x1f) << 6);
223	puch += 2;
224	cch -= 2;
225	}
226	else if (!(uch & RT_BIT(4)))
227	{
228	*pCp++ = (puch[2] & 0x3f)
229	\| ((uint16_t)(puch[1] & 0x3f) << 6)
230	\| ((uint16_t)(uch & 0x0f) << 12);
231	puch += 3;
232	cch -= 3;
233	}
234	else if (!(uch & RT_BIT(3)))
235	{
236	*pCp++ = (puch[3] & 0x3f)
237	\| ((RTUNICP)(puch[2] & 0x3f) << 6)
238	\| ((RTUNICP)(puch[1] & 0x3f) << 12)
239	\| ((RTUNICP)(uch & 0x07) << 18);
240	puch += 4;
241	cch -= 4;
242	}
243	else if (!(uch & RT_BIT(2)))
244	{
245	*pCp++ = (puch[4] & 0x3f)
246	\| ((RTUNICP)(puch[3] & 0x3f) << 6)
247	\| ((RTUNICP)(puch[2] & 0x3f) << 12)
248	\| ((RTUNICP)(puch[1] & 0x3f) << 18)
249	\| ((RTUNICP)(uch & 0x03) << 24);
250	puch += 5;
251	cch -= 6;
252	}
253	else
254	{
255	Assert(!(uch & RT_BIT(1)));
256	*pCp++ = (puch[5] & 0x3f)
257	\| ((RTUNICP)(puch[4] & 0x3f) << 6)
258	\| ((RTUNICP)(puch[3] & 0x3f) << 12)
259	\| ((RTUNICP)(puch[2] & 0x3f) << 18)
260	\| ((RTUNICP)(puch[1] & 0x3f) << 24)
261	\| ((RTUNICP)(uch & 0x01) << 30);
262	puch += 6;
263	cch -= 6;
264	}
265	}
266
267	/* done */
268	*pCp = 0;
269	return rc;
270	}
271
272
273	RTDECL(size_t) RTStrUniLen(const char *psz)
274	{
275	size_t cCodePoints;
276	int rc = rtUtf8Length(psz, RTSTR_MAX, &cCodePoints, NULL);
277	return RT_SUCCESS(rc) ? cCodePoints : 0;
278	}
279	RT_EXPORT_SYMBOL(RTStrUniLen);
280
281
282	RTDECL(int) RTStrUniLenEx(const char psz, size_t cch, size_t pcCps)
283	{
284	size_t cCodePoints;
285	int rc = rtUtf8Length(psz, cch, &cCodePoints, NULL);
286	if (pcCps)
287	*pcCps = RT_SUCCESS(rc) ? cCodePoints : 0;
288	return rc;
289	}
290	RT_EXPORT_SYMBOL(RTStrUniLenEx);
291
292
293	RTDECL(int) RTStrValidateEncoding(const char *psz)
294	{
295	return RTStrValidateEncodingEx(psz, RTSTR_MAX, 0);
296	}
297	RT_EXPORT_SYMBOL(RTStrValidateEncoding);
298
299
300	RTDECL(int) RTStrValidateEncodingEx(const char *psz, size_t cch, uint32_t fFlags)
301	{
302	AssertReturn(!(fFlags & ~(RTSTR_VALIDATE_ENCODING_ZERO_TERMINATED)), VERR_INVALID_PARAMETER);
303	AssertPtr(psz);
304
305	/*
306	* Use rtUtf8Length for the job.
307	*/
308	size_t cchActual;
309	size_t cCpsIgnored;
310	int rc = rtUtf8Length(psz, cch, &cCpsIgnored, &cchActual);
311	if (RT_SUCCESS(rc))
312	{
313	if ( (fFlags & RTSTR_VALIDATE_ENCODING_ZERO_TERMINATED)
314	&& cchActual >= cch)
315	rc = VERR_BUFFER_OVERFLOW;
316	}
317	return rc;
318	}
319	RT_EXPORT_SYMBOL(RTStrValidateEncodingEx);
320
321
322	RTDECL(bool) RTStrIsValidEncoding(const char *psz)
323	{
324	int rc = RTStrValidateEncodingEx(psz, RTSTR_MAX, 0);
325	return RT_SUCCESS(rc);
326	}
327	RT_EXPORT_SYMBOL(RTStrIsValidEncoding);
328
329
330	RTDECL(size_t) RTStrPurgeEncoding(char *psz)
331	{
332	size_t cErrors = 0;
333	for (;;)
334	{
335	RTUNICP Cp;
336	int rc = RTStrGetCpEx((const char **)&psz, &Cp);
337	if (RT_SUCCESS(rc))
338	{
339	if (!Cp)
340	break;
341	}
342	else
343	{
344	psz[-1] = '?';
345	cErrors++;
346	}
347	}
348	return cErrors;
349	}
350	RT_EXPORT_SYMBOL(RTStrPurgeEncoding);
351
352
353	RTDECL(int) RTStrToUni(const char pszString, PRTUNICP ppaCps)
354	{
355	/*
356	* Validate input.
357	*/
358	Assert(VALID_PTR(pszString));
359	Assert(VALID_PTR(ppaCps));
360	*ppaCps = NULL;
361
362	/*
363	* Validate the UTF-8 input and count its code points.
364	*/
365	size_t cCps;
366	int rc = rtUtf8Length(pszString, RTSTR_MAX, &cCps, NULL);
367	if (RT_SUCCESS(rc))
368	{
369	/*
370	* Allocate buffer.
371	*/
372	PRTUNICP paCps = (PRTUNICP)RTMemAlloc((cCps + 1) * sizeof(RTUNICP));
373	if (paCps)
374	{
375	/*
376	* Decode the string.
377	*/
378	rc = rtUtf8Decode(pszString, RTSTR_MAX, paCps, cCps);
379	if (RT_SUCCESS(rc))
380	{
381	*ppaCps = paCps;
382	return rc;
383	}
384	RTMemFree(paCps);
385	}
386	else
387	rc = VERR_NO_CODE_POINT_MEMORY;
388	}
389	return rc;
390	}
391	RT_EXPORT_SYMBOL(RTStrToUni);
392
393
394	RTDECL(int) RTStrToUniEx(const char pszString, size_t cchString, PRTUNICP ppaCps, size_t cCps, size_t *pcCps)
395	{
396	/*
397	* Validate input.
398	*/
399	Assert(VALID_PTR(pszString));
400	Assert(VALID_PTR(ppaCps));
401	Assert(!pcCps \|\| VALID_PTR(pcCps));
402
403	/*
404	* Validate the UTF-8 input and count the code points.
405	*/
406	size_t cCpsResult;
407	int rc = rtUtf8Length(pszString, cchString, &cCpsResult, NULL);
408	if (RT_SUCCESS(rc))
409	{
410	if (pcCps)
411	*pcCps = cCpsResult;
412
413	/*
414	* Check buffer size / Allocate buffer.
415	*/
416	bool fShouldFree;
417	PRTUNICP paCpsResult;
418	if (cCps > 0 && *ppaCps)
419	{
420	fShouldFree = false;
421	if (cCps <= cCpsResult)
422	return VERR_BUFFER_OVERFLOW;
423	paCpsResult = *ppaCps;
424	}
425	else
426	{
427	*ppaCps = NULL;
428	fShouldFree = true;
429	cCps = RT_MAX(cCpsResult + 1, cCps);
430	paCpsResult = (PRTUNICP)RTMemAlloc(cCps * sizeof(RTUNICP));
431	}
432	if (paCpsResult)
433	{
434	/*
435	* Encode the UTF-16 string.
436	*/
437	rc = rtUtf8Decode(pszString, cchString, paCpsResult, cCps - 1);
438	if (RT_SUCCESS(rc))
439	{
440	*ppaCps = paCpsResult;
441	return rc;
442	}
443	if (fShouldFree)
444	RTMemFree(paCpsResult);
445	}
446	else
447	rc = VERR_NO_CODE_POINT_MEMORY;
448	}
449	return rc;
450	}
451	RT_EXPORT_SYMBOL(RTStrToUniEx);
452
453
454	/**
455	* Calculates the UTF-16 length of a string, validating the encoding while doing so.
456	*
457	* @returns IPRT status code.
458	* @param psz Pointer to the UTF-8 string.
459	* @param cch The max length of the string. (btw cch = cb)
460	* Use RTSTR_MAX if all of the string is to be examined.
461	* @param pcwc Where to store the length of the UTF-16 string as a number of RTUTF16 characters.
462	*/
463	static int rtUtf8CalcUtf16Length(const char psz, size_t cch, size_t pcwc)
464	{
465	const unsigned char puch = (const unsigned char )psz;
466	size_t cwc = 0;
467	while (cch > 0)
468	{
469	const unsigned char uch = *puch;
470	if (!uch)
471	break;
472	if (!(uch & RT_BIT(7)))
473	{
474	/* one ASCII byte */
475	cwc++;
476	puch++;
477	cch--;
478	}
479	else
480	{
481	/* figure sequence length and validate the first byte */
482	unsigned cb;
483	if ((uch & (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5))) == (RT_BIT(7) \| RT_BIT(6)))
484	cb = 2;
485	else if ((uch & (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4))) == (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5)))
486	cb = 3;
487	else if ((uch & (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4) \| RT_BIT(3))) == (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4)))
488	cb = 4;
489	else if ((uch & (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4) \| RT_BIT(3) \| RT_BIT(2))) == (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4) \| RT_BIT(3)))
490	cb = 5;
491	else if ((uch & (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4) \| RT_BIT(3) \| RT_BIT(2) \| RT_BIT(1))) == (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4) \| RT_BIT(3) \| RT_BIT(2)))
492	cb = 6;
493	else
494	{
495	RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(cch, 10), puch));
496	return VERR_INVALID_UTF8_ENCODING;
497	}
498
499	/* check length */
500	if (cb > cch)
501	{
502	RTStrAssertMsgFailed(("Invalid UTF-8 length: cb=%d cch=%d (%.*Rhxs)\n", cb, cch, RT_MIN(cch, 10), puch));
503	return VERR_INVALID_UTF8_ENCODING;
504	}
505
506	/* validate the rest */
507	switch (cb)
508	{
509	case 6:
510	RTStrAssertMsgReturn((puch[5] & (RT_BIT(7) \| RT_BIT(6))) == RT_BIT(7), ("6/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
511	case 5:
512	RTStrAssertMsgReturn((puch[4] & (RT_BIT(7) \| RT_BIT(6))) == RT_BIT(7), ("5/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
513	case 4:
514	RTStrAssertMsgReturn((puch[3] & (RT_BIT(7) \| RT_BIT(6))) == RT_BIT(7), ("4/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
515	case 3:
516	RTStrAssertMsgReturn((puch[2] & (RT_BIT(7) \| RT_BIT(6))) == RT_BIT(7), ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
517	case 2:
518	RTStrAssertMsgReturn((puch[1] & (RT_BIT(7) \| RT_BIT(6))) == RT_BIT(7), ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
519	break;
520	}
521
522	/* validate the code point. */
523	RTUNICP uc;
524	switch (cb)
525	{
526	case 6:
527	uc = (puch[5] & 0x3f)
528	\| ((RTUNICP)(puch[4] & 0x3f) << 6)
529	\| ((RTUNICP)(puch[3] & 0x3f) << 12)
530	\| ((RTUNICP)(puch[2] & 0x3f) << 18)
531	\| ((RTUNICP)(puch[1] & 0x3f) << 24)
532	\| ((RTUNICP)(uch & 0x01) << 30);
533	RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
534	("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
535	RTStrAssertMsgFailed(("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch));
536	return VERR_CANT_RECODE_AS_UTF16;
537	case 5:
538	uc = (puch[4] & 0x3f)
539	\| ((RTUNICP)(puch[3] & 0x3f) << 6)
540	\| ((RTUNICP)(puch[2] & 0x3f) << 12)
541	\| ((RTUNICP)(puch[1] & 0x3f) << 18)
542	\| ((RTUNICP)(uch & 0x03) << 24);
543	RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
544	("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
545	RTStrAssertMsgFailed(("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch));
546	return VERR_CANT_RECODE_AS_UTF16;
547	case 4:
548	uc = (puch[3] & 0x3f)
549	\| ((RTUNICP)(puch[2] & 0x3f) << 6)
550	\| ((RTUNICP)(puch[1] & 0x3f) << 12)
551	\| ((RTUNICP)(uch & 0x07) << 18);
552	RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
553	("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
554	RTStrAssertMsgReturn(uc <= 0x0010ffff,
555	("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_CANT_RECODE_AS_UTF16);
556	cwc++;
557	break;
558	case 3:
559	uc = (puch[2] & 0x3f)
560	\| ((RTUNICP)(puch[1] & 0x3f) << 6)
561	\| ((RTUNICP)(uch & 0x0f) << 12);
562	RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
563	("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch),
564	uc == 0xffff \|\| uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING);
565	RTStrAssertMsgReturn(uc < 0xd800 \|\| uc > 0xdfff,
566	("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_CODE_POINT_SURROGATE);
567	break;
568	case 2:
569	uc = (puch[1] & 0x3f)
570	\| ((RTUNICP)(uch & 0x1f) << 6);
571	RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
572	("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
573	break;
574	}
575
576	/* advance */
577	cch -= cb;
578	puch += cb;
579	cwc++;
580	}
581	}
582
583	/* done */
584	*pcwc = cwc;
585	return VINF_SUCCESS;
586	}
587
588
589	/**
590	* Recodes a valid UTF-8 string as UTF-16.
591	*
592	* Since we know the input is valid, we do not perform encoding or length checks.
593	*
594	* @returns iprt status code.
595	* @param psz The UTF-8 string to recode. This is a valid encoding.
596	* @param cch The number of chars (the type char, so bytes if you like) to process of the UTF-8 string.
597	* The recoding will stop when cch or '\\0' is reached. Pass RTSTR_MAX to process up to '\\0'.
598	* @param pwsz Where to store the UTF-16 string.
599	* @param cwc The number of RTUTF16 items the pwsz buffer can hold, excluding the terminator ('\\0').
600	*/
601	static int rtUtf8RecodeAsUtf16(const char *psz, size_t cch, PRTUTF16 pwsz, size_t cwc)
602	{
603	int rc = VINF_SUCCESS;
604	const unsigned char puch = (const unsigned char )psz;
605	PRTUTF16 pwc = pwsz;
606	while (cch > 0)
607	{
608	/* read the next char and check for terminator. */
609	const unsigned char uch = *puch;
610	if (!uch)
611	break;
612
613	/* check for output overflow */
614	if (RT_UNLIKELY(cwc < 1))
615	{
616	rc = VERR_BUFFER_OVERFLOW;
617	break;
618	}
619	cwc--;
620
621	/* decode and recode the code point */
622	if (!(uch & RT_BIT(7)))
623	{
624	*pwc++ = uch;
625	puch++;
626	cch--;
627	}
628	else if ((uch & (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5))) == (RT_BIT(7) \| RT_BIT(6)))
629	{
630	uint16_t uc = (puch[1] & 0x3f)
631	\| ((uint16_t)(uch & 0x1f) << 6);
632	*pwc++ = uc;
633	puch += 2;
634	cch -= 2;
635	}
636	else if ((uch & (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4))) == (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5)))
637	{
638	uint16_t uc = (puch[2] & 0x3f)
639	\| ((uint16_t)(puch[1] & 0x3f) << 6)
640	\| ((uint16_t)(uch & 0x0f) << 12);
641	*pwc++ = uc;
642	puch += 3;
643	cch -= 3;
644	}
645	else
646	{
647	/* generate surrugate pair */
648	Assert((uch & (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4) \| RT_BIT(3))) == (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4)));
649	RTUNICP uc = (puch[3] & 0x3f)
650	\| ((RTUNICP)(puch[2] & 0x3f) << 6)
651	\| ((RTUNICP)(puch[1] & 0x3f) << 12)
652	\| ((RTUNICP)(uch & 0x07) << 18);
653	if (RT_UNLIKELY(cwc < 1))
654	{
655	rc = VERR_BUFFER_OVERFLOW;
656	break;
657	}
658	cwc--;
659
660	uc -= 0x10000;
661	*pwc++ = 0xd800 \| (uc >> 10);
662	*pwc++ = 0xdc00 \| (uc & 0x3ff);
663	puch += 4;
664	cch -= 4;
665	}
666	}
667
668	/* done */
669	*pwc = '\0';
670	return rc;
671	}
672
673
674	RTDECL(int) RTStrToUtf16Tag(const char pszString, PRTUTF16 ppwszString, const char *pszTag)
675	{
676	/*
677	* Validate input.
678	*/
679	Assert(VALID_PTR(ppwszString));
680	Assert(VALID_PTR(pszString));
681	*ppwszString = NULL;
682
683	/*
684	* Validate the UTF-8 input and calculate the length of the UTF-16 string.
685	*/
686	size_t cwc;
687	int rc = rtUtf8CalcUtf16Length(pszString, RTSTR_MAX, &cwc);
688	if (RT_SUCCESS(rc))
689	{
690	/*
691	* Allocate buffer.
692	*/
693	PRTUTF16 pwsz = (PRTUTF16)RTMemAllocTag((cwc + 1) * sizeof(RTUTF16), pszTag);
694	if (pwsz)
695	{
696	/*
697	* Encode the UTF-16 string.
698	*/
699	rc = rtUtf8RecodeAsUtf16(pszString, RTSTR_MAX, pwsz, cwc);
700	if (RT_SUCCESS(rc))
701	{
702	*ppwszString = pwsz;
703	return rc;
704	}
705	RTMemFree(pwsz);
706	}
707	else
708	rc = VERR_NO_UTF16_MEMORY;
709	}
710	return rc;
711	}
712	RT_EXPORT_SYMBOL(RTStrToUtf16Tag);
713
714
715	RTDECL(int) RTStrToUtf16ExTag(const char *pszString, size_t cchString,
716	PRTUTF16 ppwsz, size_t cwc, size_t pcwc, const char *pszTag)
717	{
718	/*
719	* Validate input.
720	*/
721	Assert(VALID_PTR(pszString));
722	Assert(VALID_PTR(ppwsz));
723	Assert(!pcwc \|\| VALID_PTR(pcwc));
724
725	/*
726	* Validate the UTF-8 input and calculate the length of the UTF-16 string.
727	*/
728	size_t cwcResult;
729	int rc = rtUtf8CalcUtf16Length(pszString, cchString, &cwcResult);
730	if (RT_SUCCESS(rc))
731	{
732	if (pcwc)
733	*pcwc = cwcResult;
734
735	/*
736	* Check buffer size / Allocate buffer.
737	*/
738	bool fShouldFree;
739	PRTUTF16 pwszResult;
740	if (cwc > 0 && *ppwsz)
741	{
742	fShouldFree = false;
743	if (cwc <= cwcResult)
744	return VERR_BUFFER_OVERFLOW;
745	pwszResult = *ppwsz;
746	}
747	else
748	{
749	*ppwsz = NULL;
750	fShouldFree = true;
751	cwc = RT_MAX(cwcResult + 1, cwc);
752	pwszResult = (PRTUTF16)RTMemAllocTag(cwc * sizeof(RTUTF16), pszTag);
753	}
754	if (pwszResult)
755	{
756	/*
757	* Encode the UTF-16 string.
758	*/
759	rc = rtUtf8RecodeAsUtf16(pszString, cchString, pwszResult, cwc - 1);
760	if (RT_SUCCESS(rc))
761	{
762	*ppwsz = pwszResult;
763	return rc;
764	}
765	if (fShouldFree)
766	RTMemFree(pwszResult);
767	}
768	else
769	rc = VERR_NO_UTF16_MEMORY;
770	}
771	return rc;
772	}
773	RT_EXPORT_SYMBOL(RTStrToUtf16ExTag);
774
775
776	RTDECL(size_t) RTStrCalcUtf16Len(const char *psz)
777	{
778	size_t cwc;
779	int rc = rtUtf8CalcUtf16Length(psz, RTSTR_MAX, &cwc);
780	return RT_SUCCESS(rc) ? cwc : 0;
781	}
782	RT_EXPORT_SYMBOL(RTStrCalcUtf16Len);
783
784
785	RTDECL(int) RTStrCalcUtf16LenEx(const char psz, size_t cch, size_t pcwc)
786	{
787	size_t cwc;
788	int rc = rtUtf8CalcUtf16Length(psz, cch, &cwc);
789	if (pcwc)
790	*pcwc = RT_SUCCESS(rc) ? cwc : ~(size_t)0;
791	return rc;
792	}
793	RT_EXPORT_SYMBOL(RTStrCalcUtf16LenEx);
794
795
796	/**
797	* Handle invalid encodings passed to RTStrGetCp() and RTStrGetCpEx().
798	* @returns rc
799	* @param ppsz The pointer to the string position point.
800	* @param pCp Where to store RTUNICP_INVALID.
801	* @param rc The iprt error code.
802	*/
803	static int rtStrGetCpExFailure(const char **ppsz, PRTUNICP pCp, int rc)
804	{
805	/*
806	* Try find a valid encoding.
807	*/
808	(ppsz)++; /* @todo code this! */
809	*pCp = RTUNICP_INVALID;
810	return rc;
811	}
812
813
814	RTDECL(RTUNICP) RTStrGetCpInternal(const char *psz)
815	{
816	RTUNICP Cp;
817	RTStrGetCpExInternal(&psz, &Cp);
818	return Cp;
819	}
820	RT_EXPORT_SYMBOL(RTStrGetCpInternal);
821
822
823	RTDECL(int) RTStrGetCpExInternal(const char **ppsz, PRTUNICP pCp)
824	{
825	const unsigned char puch = (const unsigned char )*ppsz;
826	const unsigned char uch = *puch;
827	RTUNICP uc;
828
829	/* ASCII ? */
830	if (!(uch & RT_BIT(7)))
831	{
832	uc = uch;
833	puch++;
834	}
835	else if (uch & RT_BIT(6))
836	{
837	/* figure the length and validate the first octet. */
838	unsigned cb;
839	if (!(uch & RT_BIT(5)))
840	cb = 2;
841	else if (!(uch & RT_BIT(4)))
842	cb = 3;
843	else if (!(uch & RT_BIT(3)))
844	cb = 4;
845	else if (!(uch & RT_BIT(2)))
846	cb = 5;
847	else if (!(uch & RT_BIT(1)))
848	cb = 6;
849	else
850	{
851	RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.Rhxs\n", RT_MIN(strlen((char )puch), 10), puch));
852	return rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING);
853	}
854
855	/* validate the rest */
856	switch (cb)
857	{
858	case 6:
859	RTStrAssertMsgReturn((puch[5] & 0xc0) == 0x80, ("6/%u: %.Rhxs\n", cb, RT_MIN(cb + 10, strlen((char )puch)), puch),
860	rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
861	case 5:
862	RTStrAssertMsgReturn((puch[4] & 0xc0) == 0x80, ("5/%u: %.Rhxs\n", cb, RT_MIN(cb + 10, strlen((char )puch)), puch),
863	rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
864	case 4:
865	RTStrAssertMsgReturn((puch[3] & 0xc0) == 0x80, ("4/%u: %.Rhxs\n", cb, RT_MIN(cb + 10, strlen((char )puch)), puch),
866	rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
867	case 3:
868	RTStrAssertMsgReturn((puch[2] & 0xc0) == 0x80, ("3/%u: %.Rhxs\n", cb, RT_MIN(cb + 10, strlen((char )puch)), puch),
869	rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
870	case 2:
871	RTStrAssertMsgReturn((puch[1] & 0xc0) == 0x80, ("2/%u: %.Rhxs\n", cb, RT_MIN(cb + 10, strlen((char )puch)), puch),
872	rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
873	break;
874	}
875
876	/* get and validate the code point. */
877	switch (cb)
878	{
879	case 6:
880	uc = (puch[5] & 0x3f)
881	\| ((RTUNICP)(puch[4] & 0x3f) << 6)
882	\| ((RTUNICP)(puch[3] & 0x3f) << 12)
883	\| ((RTUNICP)(puch[2] & 0x3f) << 18)
884	\| ((RTUNICP)(puch[1] & 0x3f) << 24)
885	\| ((RTUNICP)(uch & 0x01) << 30);
886	RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
887	("%u: cp=%#010RX32: %.Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char )puch)), puch),
888	rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
889	break;
890	case 5:
891	uc = (puch[4] & 0x3f)
892	\| ((RTUNICP)(puch[3] & 0x3f) << 6)
893	\| ((RTUNICP)(puch[2] & 0x3f) << 12)
894	\| ((RTUNICP)(puch[1] & 0x3f) << 18)
895	\| ((RTUNICP)(uch & 0x03) << 24);
896	RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
897	("%u: cp=%#010RX32: %.Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char )puch)), puch),
898	rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
899	break;
900	case 4:
901	uc = (puch[3] & 0x3f)
902	\| ((RTUNICP)(puch[2] & 0x3f) << 6)
903	\| ((RTUNICP)(puch[1] & 0x3f) << 12)
904	\| ((RTUNICP)(uch & 0x07) << 18);
905	RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
906	("%u: cp=%#010RX32: %.Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char )puch)), puch),
907	rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
908	break;
909	case 3:
910	uc = (puch[2] & 0x3f)
911	\| ((RTUNICP)(puch[1] & 0x3f) << 6)
912	\| ((RTUNICP)(uch & 0x0f) << 12);
913	RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
914	("%u: cp=%#010RX32: %.Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char )puch)), puch),
915	rtStrGetCpExFailure(ppsz, pCp, uc == 0xffff \|\| uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING));
916	RTStrAssertMsgReturn(uc < 0xd800 \|\| uc > 0xdfff,
917	("%u: cp=%#010RX32: %.Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char )puch)), puch),
918	rtStrGetCpExFailure(ppsz, pCp, VERR_CODE_POINT_SURROGATE));
919	break;
920	case 2:
921	uc = (puch[1] & 0x3f)
922	\| ((RTUNICP)(uch & 0x1f) << 6);
923	RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
924	("%u: cp=%#010RX32: %.Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char )puch)), puch),
925	rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
926	break;
927	default: /* impossible, but GCC is bitching. */
928	uc = RTUNICP_INVALID;
929	break;
930	}
931	puch += cb;
932	}
933	else
934	{
935	/* 6th bit is always set. */
936	RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.Rhxs\n", RT_MIN(strlen((char )puch), 10), puch));
937	return rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING);
938	}
939	*pCp = uc;
940	ppsz = (const char )puch;
941	return VINF_SUCCESS;
942	}
943	RT_EXPORT_SYMBOL(RTStrGetCpExInternal);
944
945
946	/**
947	* Handle invalid encodings passed to RTStrGetCpNEx().
948	* @returns rc
949	* @param ppsz The pointer to the string position point.
950	* @param pcch Pointer to the string length.
951	* @param pCp Where to store RTUNICP_INVALID.
952	* @param rc The iprt error code.
953	*/
954	static int rtStrGetCpNExFailure(const char *ppsz, size_t pcch, PRTUNICP pCp, int rc)
955	{
956	/*
957	* Try find a valid encoding.
958	*/
959	(ppsz)++; /* @todo code this! */
960	(*pcch)--;
961	*pCp = RTUNICP_INVALID;
962	return rc;
963	}
964
965
966	RTDECL(int) RTStrGetCpNExInternal(const char *ppsz, size_t pcch, PRTUNICP pCp)
967	{
968	const unsigned char puch = (const unsigned char )*ppsz;
969	const unsigned char uch = *puch;
970	size_t cch = *pcch;
971	RTUNICP uc;
972
973	if (cch == 0)
974	{
975	*pCp = RTUNICP_INVALID;
976	return VERR_END_OF_STRING;
977	}
978
979	/* ASCII ? */
980	if (!(uch & RT_BIT(7)))
981	{
982	uc = uch;
983	puch++;
984	cch--;
985	}
986	else if (uch & RT_BIT(6))
987	{
988	/* figure the length and validate the first octet. */
989	unsigned cb;
990	if (!(uch & RT_BIT(5)))
991	cb = 2;
992	else if (!(uch & RT_BIT(4)))
993	cb = 3;
994	else if (!(uch & RT_BIT(3)))
995	cb = 4;
996	else if (!(uch & RT_BIT(2)))
997	cb = 5;
998	else if (!(uch & RT_BIT(1)))
999	cb = 6;
1000	else
1001	{
1002	RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.Rhxs\n", RT_MIN(strlen((char )puch), 10), puch));
1003	return rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING);
1004	}
1005
1006	if (cb > cch)
1007	return rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING);
1008
1009	/* validate the rest */
1010	switch (cb)
1011	{
1012	case 6:
1013	RTStrAssertMsgReturn((puch[5] & 0xc0) == 0x80, ("6/%u: %.Rhxs\n", cb, RT_MIN(cb + 10, strlen((char )puch)), puch),
1014	rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1015	case 5:
1016	RTStrAssertMsgReturn((puch[4] & 0xc0) == 0x80, ("5/%u: %.Rhxs\n", cb, RT_MIN(cb + 10, strlen((char )puch)), puch),
1017	rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1018	case 4:
1019	RTStrAssertMsgReturn((puch[3] & 0xc0) == 0x80, ("4/%u: %.Rhxs\n", cb, RT_MIN(cb + 10, strlen((char )puch)), puch),
1020	rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1021	case 3:
1022	RTStrAssertMsgReturn((puch[2] & 0xc0) == 0x80, ("3/%u: %.Rhxs\n", cb, RT_MIN(cb + 10, strlen((char )puch)), puch),
1023	rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1024	case 2:
1025	RTStrAssertMsgReturn((puch[1] & 0xc0) == 0x80, ("2/%u: %.Rhxs\n", cb, RT_MIN(cb + 10, strlen((char )puch)), puch),
1026	rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1027	break;
1028	}
1029
1030	/* get and validate the code point. */
1031	switch (cb)
1032	{
1033	case 6:
1034	uc = (puch[5] & 0x3f)
1035	\| ((RTUNICP)(puch[4] & 0x3f) << 6)
1036	\| ((RTUNICP)(puch[3] & 0x3f) << 12)
1037	\| ((RTUNICP)(puch[2] & 0x3f) << 18)
1038	\| ((RTUNICP)(puch[1] & 0x3f) << 24)
1039	\| ((RTUNICP)(uch & 0x01) << 30);
1040	RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
1041	("%u: cp=%#010RX32: %.Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char )puch)), puch),
1042	rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1043	break;
1044	case 5:
1045	uc = (puch[4] & 0x3f)
1046	\| ((RTUNICP)(puch[3] & 0x3f) << 6)
1047	\| ((RTUNICP)(puch[2] & 0x3f) << 12)
1048	\| ((RTUNICP)(puch[1] & 0x3f) << 18)
1049	\| ((RTUNICP)(uch & 0x03) << 24);
1050	RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
1051	("%u: cp=%#010RX32: %.Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char )puch)), puch),
1052	rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1053	break;
1054	case 4:
1055	uc = (puch[3] & 0x3f)
1056	\| ((RTUNICP)(puch[2] & 0x3f) << 6)
1057	\| ((RTUNICP)(puch[1] & 0x3f) << 12)
1058	\| ((RTUNICP)(uch & 0x07) << 18);
1059	RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
1060	("%u: cp=%#010RX32: %.Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char )puch)), puch),
1061	rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1062	break;
1063	case 3:
1064	uc = (puch[2] & 0x3f)
1065	\| ((RTUNICP)(puch[1] & 0x3f) << 6)
1066	\| ((RTUNICP)(uch & 0x0f) << 12);
1067	RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
1068	("%u: cp=%#010RX32: %.Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char )puch)), puch),
1069	rtStrGetCpNExFailure(ppsz, pcch, pCp, uc == 0xffff \|\| uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING));
1070	RTStrAssertMsgReturn(uc < 0xd800 \|\| uc > 0xdfff,
1071	("%u: cp=%#010RX32: %.Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char )puch)), puch),
1072	rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_CODE_POINT_SURROGATE));
1073	break;
1074	case 2:
1075	uc = (puch[1] & 0x3f)
1076	\| ((RTUNICP)(uch & 0x1f) << 6);
1077	RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
1078	("%u: cp=%#010RX32: %.Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char )puch)), puch),
1079	rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1080	break;
1081	default: /* impossible, but GCC is bitching. */
1082	uc = RTUNICP_INVALID;
1083	break;
1084	}
1085	puch += cb;
1086	cch -= cb;
1087	}
1088	else
1089	{
1090	/* 6th bit is always set. */
1091	RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.Rhxs\n", RT_MIN(strlen((char )puch), 10), puch));
1092	return rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING);
1093	}
1094	*pCp = uc;
1095	ppsz = (const char )puch;
1096	(*pcch) = cch;
1097	return VINF_SUCCESS;
1098	}
1099	RT_EXPORT_SYMBOL(RTStrGetCpNExInternal);
1100
1101
1102	RTDECL(char ) RTStrPutCpInternal(char psz, RTUNICP uc)
1103	{
1104	unsigned char puch = (unsigned char )psz;
1105	if (uc < 0x80)
1106	*puch++ = (unsigned char )uc;
1107	else if (uc < 0x00000800)
1108	{
1109	*puch++ = 0xc0 \| (uc >> 6);
1110	*puch++ = 0x80 \| (uc & 0x3f);
1111	}
1112	else if (uc < 0x00010000)
1113	{
1114	if ( uc < 0x0000d8000
1115	\|\| ( uc > 0x0000dfff
1116	&& uc < 0x0000fffe))
1117	{
1118	*puch++ = 0xe0 \| (uc >> 12);
1119	*puch++ = 0x80 \| ((uc >> 6) & 0x3f);
1120	*puch++ = 0x80 \| (uc & 0x3f);
1121	}
1122	else
1123	{
1124	AssertMsgFailed(("Invalid code point U+%05x!\n", uc));
1125	*puch++ = 0x7f;
1126	}
1127	}
1128	else if (uc < 0x00200000)
1129	{
1130	*puch++ = 0xf0 \| (uc >> 18);
1131	*puch++ = 0x80 \| ((uc >> 12) & 0x3f);
1132	*puch++ = 0x80 \| ((uc >> 6) & 0x3f);
1133	*puch++ = 0x80 \| (uc & 0x3f);
1134	}
1135	else if (uc < 0x04000000)
1136	{
1137	*puch++ = 0xf8 \| (uc >> 24);
1138	*puch++ = 0x80 \| ((uc >> 18) & 0x3f);
1139	*puch++ = 0x80 \| ((uc >> 12) & 0x3f);
1140	*puch++ = 0x80 \| ((uc >> 6) & 0x3f);
1141	*puch++ = 0x80 \| (uc & 0x3f);
1142	}
1143	else if (uc <= 0x7fffffff)
1144	{
1145	*puch++ = 0xfc \| (uc >> 30);
1146	*puch++ = 0x80 \| ((uc >> 24) & 0x3f);
1147	*puch++ = 0x80 \| ((uc >> 18) & 0x3f);
1148	*puch++ = 0x80 \| ((uc >> 12) & 0x3f);
1149	*puch++ = 0x80 \| ((uc >> 6) & 0x3f);
1150	*puch++ = 0x80 \| (uc & 0x3f);
1151	}
1152	else
1153	{
1154	AssertMsgFailed(("Invalid code point U+%08x!\n", uc));
1155	*puch++ = 0x7f;
1156	}
1157
1158	return (char *)puch;
1159	}
1160	RT_EXPORT_SYMBOL(RTStrPutCpInternal);
1161
1162
1163	RTDECL(char ) RTStrPrevCp(const char pszStart, const char *psz)
1164	{
1165	if (pszStart < psz)
1166	{
1167	/* simple char? */
1168	const unsigned char puch = (const unsigned char )psz;
1169	unsigned uch = *--puch;
1170	if (!(uch & RT_BIT(7)))
1171	return (char *)puch;
1172	RTStrAssertMsgReturn(!(uch & RT_BIT(6)), ("uch=%#x\n", uch), (char *)pszStart);
1173
1174	/* two or more. */
1175	uint32_t uMask = 0xffffffc0;
1176	while ( (const unsigned char *)pszStart < puch
1177	&& !(uMask & 1))
1178	{
1179	uch = *--puch;
1180	if ((uch & 0xc0) != 0x80)
1181	{
1182	RTStrAssertMsgReturn((uch & (uMask >> 1)) == (uMask & 0xff),
1183	("Invalid UTF-8 encoding: %.Rhxs puch=%p psz=%p\n", psz - (char )puch, puch, psz),
1184	(char *)pszStart);
1185	return (char *)puch;
1186	}
1187	uMask >>= 1;
1188	}
1189	RTStrAssertMsgFailed(("Invalid UTF-8 encoding: %.Rhxs puch=%p psz=%p\n", psz - (char )puch, puch, psz));
1190	}
1191	return (char *)pszStart;
1192	}
1193	RT_EXPORT_SYMBOL(RTStrPrevCp);
1194

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/common/string/utf-8.cpp@ 31157

Download in other formats: