1 | /* $Id: bldprog-strtab-template.cpp.h 96551 2022-08-30 01:09:00Z vboxsync $ */
|
---|
2 | /** @file
|
---|
3 | * IPRT - Build Program - String Table Generator.
|
---|
4 | */
|
---|
5 |
|
---|
6 | /*
|
---|
7 | * Copyright (C) 2006-2022 Oracle and/or its affiliates.
|
---|
8 | *
|
---|
9 | * This file is part of VirtualBox base platform packages, as
|
---|
10 | * available from https://www.virtualbox.org.
|
---|
11 | *
|
---|
12 | * This program is free software; you can redistribute it and/or
|
---|
13 | * modify it under the terms of the GNU General Public License
|
---|
14 | * as published by the Free Software Foundation, in version 3 of the
|
---|
15 | * License.
|
---|
16 | *
|
---|
17 | * This program is distributed in the hope that it will be useful, but
|
---|
18 | * WITHOUT ANY WARRANTY; without even the implied warranty of
|
---|
19 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
---|
20 | * General Public License for more details.
|
---|
21 | *
|
---|
22 | * You should have received a copy of the GNU General Public License
|
---|
23 | * along with this program; if not, see <https://www.gnu.org/licenses>.
|
---|
24 | *
|
---|
25 | * The contents of this file may alternatively be used under the terms
|
---|
26 | * of the Common Development and Distribution License Version 1.0
|
---|
27 | * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
|
---|
28 | * in the VirtualBox distribution, in which case the provisions of the
|
---|
29 | * CDDL are applicable instead of those of the GPL.
|
---|
30 | *
|
---|
31 | * You may elect to license modified versions of this file under the
|
---|
32 | * terms and conditions of either the GPL or the CDDL or both.
|
---|
33 | *
|
---|
34 | * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
|
---|
35 | */
|
---|
36 |
|
---|
37 |
|
---|
38 | /*
|
---|
39 | * (Avoid include sanity checks as this is ring-3 only, C++ code.)
|
---|
40 | */
|
---|
41 | #if defined(__cplusplus) && defined(IN_RING3)
|
---|
42 |
|
---|
43 |
|
---|
44 | /*********************************************************************************************************************************
|
---|
45 | * Defined Constants And Macros *
|
---|
46 | *********************************************************************************************************************************/
|
---|
47 | /** @def BLDPROG_STRTAB_MAX_STRLEN
|
---|
48 | * The max length of strings in the table. */
|
---|
49 | #if !defined(BLDPROG_STRTAB_MAX_STRLEN) || defined(DOXYGEN_RUNNING)
|
---|
50 | # define BLDPROG_STRTAB_MAX_STRLEN 256
|
---|
51 | #endif
|
---|
52 |
|
---|
53 | /** @def BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
54 | * Enables very simple string compression.
|
---|
55 | */
|
---|
56 | #if defined(DOXYGEN_RUNNING)
|
---|
57 | # define BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
58 | #endif
|
---|
59 |
|
---|
60 | /** @def BLDPROG_STRTAB_WITH_CAMEL_WORDS
|
---|
61 | * Modifies the string compression to look for camel case words.
|
---|
62 | */
|
---|
63 | #if defined(DOXYGEN_RUNNING)
|
---|
64 | # define BLDPROG_STRTAB_WITH_CAMEL_WORDS
|
---|
65 | #endif
|
---|
66 |
|
---|
67 | /** @def BLDPROG_STRTAB_PURE_ASCII
|
---|
68 | * String compression assumes pure 7-bit ASCII and will fail on UTF-8 when this
|
---|
69 | * is defined. Otherwise, the compression code will require IPRT to link.
|
---|
70 | */
|
---|
71 | #if defined(DOXYGEN_RUNNING)
|
---|
72 | # define BLDPROG_STRTAB_PURE_ASCII
|
---|
73 | #endif
|
---|
74 |
|
---|
75 |
|
---|
76 |
|
---|
77 | /*********************************************************************************************************************************
|
---|
78 | * Header Files *
|
---|
79 | *********************************************************************************************************************************/
|
---|
80 | #include <iprt/stdarg.h>
|
---|
81 | #include <iprt/ctype.h>
|
---|
82 | #include <stdlib.h>
|
---|
83 | #include <stdio.h>
|
---|
84 | #include <string.h>
|
---|
85 |
|
---|
86 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
87 | # include <iprt/asm.h>
|
---|
88 | # include <map>
|
---|
89 | # include <iprt/sanitized/string>
|
---|
90 |
|
---|
91 | # define BLDPROG_STRTAB_WITH_WORD_SEP_ALTERNATIVE
|
---|
92 | typedef struct BLDPROGWORDFREQSTATS
|
---|
93 | {
|
---|
94 | uint32_t cWithoutSep; /**< Number of occurances without a separator. */
|
---|
95 | # ifdef BLDPROG_STRTAB_WITH_WORD_SEP_ALTERNATIVE
|
---|
96 | uint32_t cWithSep; /**< Number of occurance with a separator. */
|
---|
97 | char chSep; /**< The separator. First come basis. */
|
---|
98 | # endif
|
---|
99 | } BLDPROGWORDFREQSTATS;
|
---|
100 |
|
---|
101 | typedef std::map<std::string, BLDPROGWORDFREQSTATS> BLDPROGWORDFREQMAP;
|
---|
102 |
|
---|
103 | #endif
|
---|
104 |
|
---|
105 | #include "../../src/VBox/Runtime/include/internal/strhash.h" /** @todo make this one public */
|
---|
106 |
|
---|
107 |
|
---|
108 | /*********************************************************************************************************************************
|
---|
109 | * Structures and Typedefs *
|
---|
110 | *********************************************************************************************************************************/
|
---|
111 | /**
|
---|
112 | * Build table string.
|
---|
113 | */
|
---|
114 | typedef struct BLDPROGSTRING
|
---|
115 | {
|
---|
116 | /** The string.
|
---|
117 | * @note This may be modified or replaced (allocated from heap) when
|
---|
118 | * compressing the string table. */
|
---|
119 | char *pszString;
|
---|
120 | /** The string hash value. */
|
---|
121 | uint32_t uHash;
|
---|
122 | /** The strint table offset. */
|
---|
123 | uint32_t offStrTab;
|
---|
124 | /** The string length. */
|
---|
125 | size_t cchString;
|
---|
126 | /** Pointer to the next string reference (same string table entry). */
|
---|
127 | struct BLDPROGSTRING *pNextRef;
|
---|
128 | /** Pointer to the next string with the same hash value (collision). */
|
---|
129 | struct BLDPROGSTRING *pNextCollision;
|
---|
130 |
|
---|
131 | } BLDPROGSTRING;
|
---|
132 | /** Pointer to a string table string. */
|
---|
133 | typedef BLDPROGSTRING *PBLDPROGSTRING;
|
---|
134 |
|
---|
135 |
|
---|
136 | /** String table data. */
|
---|
137 | typedef struct BLDPROGSTRTAB
|
---|
138 | {
|
---|
139 | /** The size of g_papStrHash. */
|
---|
140 | size_t cStrHash;
|
---|
141 | /** String hash table. */
|
---|
142 | PBLDPROGSTRING *papStrHash;
|
---|
143 | /** Duplicate strings found by AddString. */
|
---|
144 | size_t cDuplicateStrings;
|
---|
145 | /** Total length of the unique strings (no terminators). */
|
---|
146 | size_t cchUniqueStrings;
|
---|
147 | /** Number of unique strings after AddString. */
|
---|
148 | size_t cUniqueStrings;
|
---|
149 | /** Number of collisions. */
|
---|
150 | size_t cCollisions;
|
---|
151 |
|
---|
152 | /** Number of entries in apSortedStrings. */
|
---|
153 | size_t cSortedStrings;
|
---|
154 | /** The sorted string table. */
|
---|
155 | PBLDPROGSTRING *papSortedStrings;
|
---|
156 |
|
---|
157 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
158 | /** The 256 words we've picked to be indexed by reference. */
|
---|
159 | BLDPROGSTRING aCompDict[256];
|
---|
160 | /** The frequency of the 256 dictionary entries. */
|
---|
161 | size_t auCompDictFreq[256];
|
---|
162 | /** Incoming strings pending compression. */
|
---|
163 | PBLDPROGSTRING *papPendingStrings;
|
---|
164 | /** Current number of entries in papStrPending. */
|
---|
165 | size_t cPendingStrings;
|
---|
166 | /** The allocated size of papPendingStrings. */
|
---|
167 | size_t cMaxPendingStrings;
|
---|
168 | /** Work frequency map.
|
---|
169 | * @todo rewrite in plain C. */
|
---|
170 | BLDPROGWORDFREQMAP Frequencies;
|
---|
171 | /** Map of characters used by input strings. */
|
---|
172 | uint64_t bmUsedChars[256/64];
|
---|
173 | #endif
|
---|
174 |
|
---|
175 | /** The string table. */
|
---|
176 | char *pachStrTab;
|
---|
177 | /** The actual string table size. */
|
---|
178 | size_t cchStrTab;
|
---|
179 | } BLDPROGSTRTAB;
|
---|
180 | typedef BLDPROGSTRTAB *PBLDPROGSTRTAB;
|
---|
181 |
|
---|
182 | #if RT_CLANG_PREREQ(4, 0) || RT_GNUC_PREREQ(4, 6)
|
---|
183 | # pragma GCC diagnostic push
|
---|
184 | # pragma GCC diagnostic ignored "-Wunused-function"
|
---|
185 | #endif
|
---|
186 |
|
---|
187 |
|
---|
188 | /**
|
---|
189 | * Initializes the strint table compiler.
|
---|
190 | *
|
---|
191 | * @returns success indicator (out of memory if false).
|
---|
192 | * @param pThis The strint table compiler instance.
|
---|
193 | * @param cMaxStrings The max number of strings we'll be adding.
|
---|
194 | */
|
---|
195 | static bool BldProgStrTab_Init(PBLDPROGSTRTAB pThis, size_t cMaxStrings)
|
---|
196 | {
|
---|
197 | pThis->cStrHash = 0;
|
---|
198 | pThis->papStrHash = NULL;
|
---|
199 | pThis->cDuplicateStrings = 0;
|
---|
200 | pThis->cchUniqueStrings = 0;
|
---|
201 | pThis->cUniqueStrings = 0;
|
---|
202 | pThis->cCollisions = 0;
|
---|
203 | pThis->cSortedStrings = 0;
|
---|
204 | pThis->papSortedStrings = NULL;
|
---|
205 | pThis->pachStrTab = NULL;
|
---|
206 | pThis->cchStrTab = 0;
|
---|
207 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
208 | memset(pThis->aCompDict, 0, sizeof(pThis->aCompDict));
|
---|
209 | pThis->papPendingStrings = NULL;
|
---|
210 | pThis->cPendingStrings = 0;
|
---|
211 | pThis->cMaxPendingStrings = cMaxStrings;
|
---|
212 | memset(pThis->bmUsedChars, 0, sizeof(pThis->bmUsedChars));
|
---|
213 | ASMBitSet(pThis->bmUsedChars, 0); /* Some parts of the code still thinks zero is a terminator, so don't use it for now. */
|
---|
214 | # ifndef BLDPROG_STRTAB_PURE_ASCII
|
---|
215 | ASMBitSet(pThis->bmUsedChars, 0xff); /* Reserve escape byte for codepoints above 127. */
|
---|
216 | # endif
|
---|
217 | #endif
|
---|
218 |
|
---|
219 | /*
|
---|
220 | * Allocate a hash table double the size of all strings (to avoid too
|
---|
221 | * many collisions). Add all strings to it, finding duplicates in the
|
---|
222 | * process.
|
---|
223 | */
|
---|
224 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
225 | cMaxStrings += RT_ELEMENTS(pThis->aCompDict);
|
---|
226 | #endif
|
---|
227 | cMaxStrings *= 2;
|
---|
228 | pThis->papStrHash = (PBLDPROGSTRING *)calloc(sizeof(pThis->papStrHash[0]), cMaxStrings);
|
---|
229 | if (pThis->papStrHash)
|
---|
230 | {
|
---|
231 | pThis->cStrHash = cMaxStrings;
|
---|
232 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
233 | pThis->papPendingStrings = (PBLDPROGSTRING *)calloc(sizeof(pThis->papPendingStrings[0]), pThis->cMaxPendingStrings);
|
---|
234 | if (pThis->papPendingStrings)
|
---|
235 | #endif
|
---|
236 | return true;
|
---|
237 |
|
---|
238 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
239 | free(pThis->papStrHash);
|
---|
240 | pThis->papStrHash = NULL;
|
---|
241 | #endif
|
---|
242 | }
|
---|
243 | return false;
|
---|
244 | }
|
---|
245 |
|
---|
246 |
|
---|
247 | #if 0 /* unused */
|
---|
248 | static void BldProgStrTab_Delete(PBLDPROGSTRTAB pThis)
|
---|
249 | {
|
---|
250 | free(pThis->papStrHash);
|
---|
251 | free(pThis->papSortedStrings);
|
---|
252 | free(pThis->pachStrTab);
|
---|
253 | # ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
254 | free(pThis->papPendingStrings);
|
---|
255 | # endif
|
---|
256 | memset(pThis, 0, sizeof(*pThis));
|
---|
257 | }
|
---|
258 | #endif
|
---|
259 |
|
---|
260 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
261 |
|
---|
262 | DECLINLINE(size_t) bldProgStrTab_compressorFindNextWord(const char *pszSrc, char ch, const char **ppszSrc)
|
---|
263 | {
|
---|
264 | /*
|
---|
265 | * Skip leading word separators.
|
---|
266 | */
|
---|
267 | # ifdef BLDPROG_STRTAB_WITH_CAMEL_WORDS
|
---|
268 | while ( ch == ' '
|
---|
269 | || ch == '-'
|
---|
270 | || ch == '+'
|
---|
271 | || ch == '_')
|
---|
272 | # else
|
---|
273 | while (ch == ' ')
|
---|
274 | # endif
|
---|
275 | ch = *++pszSrc;
|
---|
276 | if (ch)
|
---|
277 | {
|
---|
278 | /*
|
---|
279 | * Find end of word.
|
---|
280 | */
|
---|
281 | size_t cchWord = 1;
|
---|
282 | # ifdef BLDPROG_STRTAB_WITH_CAMEL_WORDS
|
---|
283 | char chPrev = ch;
|
---|
284 | while ( (ch = pszSrc[cchWord]) != ' '
|
---|
285 | && ch != '\0'
|
---|
286 | && ch != '-'
|
---|
287 | && ch != '+'
|
---|
288 | && ch != '_'
|
---|
289 | && ( ch == chPrev
|
---|
290 | || !RT_C_IS_UPPER(ch)
|
---|
291 | || RT_C_IS_UPPER(chPrev)) )
|
---|
292 | {
|
---|
293 | chPrev = ch;
|
---|
294 | cchWord++;
|
---|
295 | }
|
---|
296 | # else
|
---|
297 | while ((ch = pszSrc[cchWord]) != ' ' && ch != '\0')
|
---|
298 | cchWord++;
|
---|
299 | # endif
|
---|
300 | *ppszSrc = pszSrc;
|
---|
301 | return cchWord;
|
---|
302 | }
|
---|
303 |
|
---|
304 | *ppszSrc = pszSrc;
|
---|
305 | return 0;
|
---|
306 | }
|
---|
307 |
|
---|
308 |
|
---|
309 | /**
|
---|
310 | * Analyzes a string.
|
---|
311 | *
|
---|
312 | * @param pThis The strint table compiler instance.
|
---|
313 | * @param pStr The string to analyze.
|
---|
314 | */
|
---|
315 | static void bldProgStrTab_compressorAnalyzeString(PBLDPROGSTRTAB pThis, PBLDPROGSTRING pStr)
|
---|
316 | {
|
---|
317 | /*
|
---|
318 | * Mark all the string characters as used.
|
---|
319 | */
|
---|
320 | const char *psz = pStr->pszString;
|
---|
321 | char ch;
|
---|
322 | while ((ch = *psz++) != '\0')
|
---|
323 | ASMBitSet(pThis->bmUsedChars, (uint8_t)ch);
|
---|
324 |
|
---|
325 | /*
|
---|
326 | * For now we just consider words.
|
---|
327 | */
|
---|
328 | psz = pStr->pszString;
|
---|
329 | while ((ch = *psz) != '\0')
|
---|
330 | {
|
---|
331 | size_t cchWord = bldProgStrTab_compressorFindNextWord(psz, ch, &psz);
|
---|
332 | if (cchWord > 1)
|
---|
333 | {
|
---|
334 | std::string strWord(psz, cchWord);
|
---|
335 | BLDPROGWORDFREQMAP::iterator it = pThis->Frequencies.find(strWord);
|
---|
336 | if (it != pThis->Frequencies.end())
|
---|
337 | {
|
---|
338 | # ifdef BLDPROG_STRTAB_WITH_WORD_SEP_ALTERNATIVE
|
---|
339 | char const chSep = psz[cchWord];
|
---|
340 | if (chSep != '\0' && (it->second.chSep == chSep || it->second.chSep == '\0'))
|
---|
341 | {
|
---|
342 | it->second.chSep = chSep;
|
---|
343 | it->second.cWithSep++;
|
---|
344 | }
|
---|
345 | else
|
---|
346 | # endif
|
---|
347 | it->second.cWithoutSep++;
|
---|
348 | }
|
---|
349 | else
|
---|
350 | {
|
---|
351 | # ifdef BLDPROG_STRTAB_WITH_WORD_SEP_ALTERNATIVE
|
---|
352 | char const chSep = psz[cchWord];
|
---|
353 | if (chSep != '\0')
|
---|
354 | {
|
---|
355 | BLDPROGWORDFREQSTATS NewWord = { 0, 0, chSep };
|
---|
356 | pThis->Frequencies[strWord] = NewWord;
|
---|
357 | }
|
---|
358 | else
|
---|
359 | # endif
|
---|
360 | {
|
---|
361 | static BLDPROGWORDFREQSTATS s_NewWord = { 0 };
|
---|
362 | pThis->Frequencies[strWord] = s_NewWord;
|
---|
363 | }
|
---|
364 | }
|
---|
365 |
|
---|
366 | # if 0 /** @todo need better accounting for overlapping alternatives before this can be enabled. */
|
---|
367 | /* Two words - immediate yields calc may lie when this enabled and we may pick the wrong words. */
|
---|
368 | if (ch == ' ')
|
---|
369 | {
|
---|
370 | ch = psz[++cchWord];
|
---|
371 | if (ch != ' ' && ch != '\0')
|
---|
372 | {
|
---|
373 | size_t const cchSaved = cchWord;
|
---|
374 |
|
---|
375 | do
|
---|
376 | cchWord++;
|
---|
377 | while ((ch = psz[cchWord]) != ' ' && ch != '\0');
|
---|
378 |
|
---|
379 | strWord = std::string(psz, cchWord);
|
---|
380 | BLDPROGWORDFREQMAP::iterator it = pThis->Frequencies.find(strWord);
|
---|
381 | if (it != pThis->Frequencies.end())
|
---|
382 | it->second += cchWord - 1;
|
---|
383 | else
|
---|
384 | pThis->Frequencies[strWord] = 0;
|
---|
385 |
|
---|
386 | cchWord = cchSaved;
|
---|
387 | }
|
---|
388 | }
|
---|
389 | # endif
|
---|
390 | }
|
---|
391 | else if (!cchWord)
|
---|
392 | break;
|
---|
393 |
|
---|
394 | /* Advance. */
|
---|
395 | psz += cchWord;
|
---|
396 | }
|
---|
397 | pStr->cchString = psz - pStr->pszString;
|
---|
398 | if (pStr->cchString > BLDPROG_STRTAB_MAX_STRLEN)
|
---|
399 | {
|
---|
400 | fprintf(stderr, "error: String to long (%u)\n", (unsigned)pStr->cchString);
|
---|
401 | abort();
|
---|
402 | }
|
---|
403 | }
|
---|
404 |
|
---|
405 | #endif /* BLDPROG_STRTAB_WITH_COMPRESSION */
|
---|
406 |
|
---|
407 | /**
|
---|
408 | * Adds a string to the hash table.
|
---|
409 | * @param pThis The strint table compiler instance.
|
---|
410 | * @param pStr The string.
|
---|
411 | */
|
---|
412 | static void bldProgStrTab_AddStringToHashTab(PBLDPROGSTRTAB pThis, PBLDPROGSTRING pStr)
|
---|
413 | {
|
---|
414 | pStr->pNextRef = NULL;
|
---|
415 | pStr->pNextCollision = NULL;
|
---|
416 | pStr->offStrTab = 0;
|
---|
417 | pStr->uHash = sdbm(pStr->pszString, &pStr->cchString);
|
---|
418 | if (pStr->cchString > BLDPROG_STRTAB_MAX_STRLEN)
|
---|
419 | {
|
---|
420 | fprintf(stderr, "error: String to long (%u)\n", (unsigned)pStr->cchString);
|
---|
421 | exit(RTEXITCODE_FAILURE);
|
---|
422 | }
|
---|
423 |
|
---|
424 | size_t idxHash = pStr->uHash % pThis->cStrHash;
|
---|
425 | PBLDPROGSTRING pCur = pThis->papStrHash[idxHash];
|
---|
426 | if (!pCur)
|
---|
427 | pThis->papStrHash[idxHash] = pStr;
|
---|
428 | else
|
---|
429 | {
|
---|
430 | /* Look for matching string. */
|
---|
431 | do
|
---|
432 | {
|
---|
433 | if ( pCur->uHash == pStr->uHash
|
---|
434 | && pCur->cchString == pStr->cchString
|
---|
435 | && memcmp(pCur->pszString, pStr->pszString, pStr->cchString) == 0)
|
---|
436 | {
|
---|
437 | pStr->pNextRef = pCur->pNextRef;
|
---|
438 | pCur->pNextRef = pStr;
|
---|
439 | pThis->cDuplicateStrings++;
|
---|
440 | return;
|
---|
441 | }
|
---|
442 | pCur = pCur->pNextCollision;
|
---|
443 | } while (pCur != NULL);
|
---|
444 |
|
---|
445 | /* No matching string, insert. */
|
---|
446 | pThis->cCollisions++;
|
---|
447 | pStr->pNextCollision = pThis->papStrHash[idxHash];
|
---|
448 | pThis->papStrHash[idxHash] = pStr;
|
---|
449 | }
|
---|
450 |
|
---|
451 | pThis->cUniqueStrings++;
|
---|
452 | pThis->cchUniqueStrings += pStr->cchString;
|
---|
453 | }
|
---|
454 |
|
---|
455 |
|
---|
456 | /**
|
---|
457 | * Adds a string to the string table.
|
---|
458 | *
|
---|
459 | * @param pThis The strint table compiler instance.
|
---|
460 | * @param pStr The string.
|
---|
461 | */
|
---|
462 | static void BldProgStrTab_AddString(PBLDPROGSTRTAB pThis, PBLDPROGSTRING pStr)
|
---|
463 | {
|
---|
464 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
465 | bldProgStrTab_compressorAnalyzeString(pThis, pStr);
|
---|
466 | if (pThis->cPendingStrings < pThis->cMaxPendingStrings)
|
---|
467 | pThis->papPendingStrings[pThis->cPendingStrings++] = pStr;
|
---|
468 | else
|
---|
469 | abort();
|
---|
470 | #else
|
---|
471 | bldProgStrTab_AddStringToHashTab(pThis, pStr);
|
---|
472 | #endif
|
---|
473 | }
|
---|
474 |
|
---|
475 |
|
---|
476 | /**
|
---|
477 | * Adds a string to the string table.
|
---|
478 | *
|
---|
479 | * @param pThis The strint table compiler instance.
|
---|
480 | * @param pStr The string entry (uninitialized).
|
---|
481 | * @param psz The string, will be duplicated if compression is enabled.
|
---|
482 | */
|
---|
483 | DECLINLINE(void) BldProgStrTab_AddStringDup(PBLDPROGSTRTAB pThis, PBLDPROGSTRING pStr, const char *psz)
|
---|
484 | {
|
---|
485 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
486 | pStr->pszString = strdup(psz);
|
---|
487 | if (!pStr->pszString)
|
---|
488 | abort();
|
---|
489 | #else
|
---|
490 | pStr->pszString = (char *)psz;
|
---|
491 | #endif
|
---|
492 | BldProgStrTab_AddString(pThis, pStr);
|
---|
493 | }
|
---|
494 |
|
---|
495 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
496 |
|
---|
497 | /**
|
---|
498 | * Copies @a cchSrc chars from @a pchSrc to @a pszDst, escaping special
|
---|
499 | * sequences.
|
---|
500 | *
|
---|
501 | * @returns New @a pszDst position, NULL if invalid source encoding.
|
---|
502 | * @param pszDst The destination buffer.
|
---|
503 | * @param pszSrc The source buffer.
|
---|
504 | * @param cchSrc How much to copy.
|
---|
505 | */
|
---|
506 | static char *bldProgStrTab_compressorCopyAndEscape(char *pszDst, const char *pszSrc, size_t cchSrc)
|
---|
507 | {
|
---|
508 | while (cchSrc-- > 0)
|
---|
509 | {
|
---|
510 | char ch = *pszSrc;
|
---|
511 | if (!((unsigned char)ch & 0x80))
|
---|
512 | {
|
---|
513 | *pszDst++ = ch;
|
---|
514 | pszSrc++;
|
---|
515 | }
|
---|
516 | else
|
---|
517 | {
|
---|
518 | # ifdef BLDPROG_STRTAB_PURE_ASCII
|
---|
519 | fprintf(stderr, "error: unexpected char value %#x\n", ch);
|
---|
520 | return NULL;
|
---|
521 | # else
|
---|
522 | RTUNICP uc;
|
---|
523 | int rc = RTStrGetCpEx(&pszSrc, &uc);
|
---|
524 | if (RT_SUCCESS(rc))
|
---|
525 | {
|
---|
526 | *pszDst++ = (unsigned char)0xff; /* escape single code point. */
|
---|
527 | pszDst = RTStrPutCp(pszDst, uc);
|
---|
528 | }
|
---|
529 | else
|
---|
530 | {
|
---|
531 | fprintf(stderr, "Error: RTStrGetCpEx failed with rc=%d\n", rc);
|
---|
532 | return NULL;
|
---|
533 | }
|
---|
534 | # endif
|
---|
535 | }
|
---|
536 | }
|
---|
537 | return pszDst;
|
---|
538 | }
|
---|
539 |
|
---|
540 |
|
---|
541 | /**
|
---|
542 | * Replaces the dictionary words and escapes non-ascii chars in a string.
|
---|
543 | *
|
---|
544 | * @param pThis The strint table compiler instance.
|
---|
545 | * @param pString The string to fixup.
|
---|
546 | */
|
---|
547 | static bool bldProgStrTab_compressorFixupString(PBLDPROGSTRTAB pThis, PBLDPROGSTRING pStr)
|
---|
548 | {
|
---|
549 | char szNew[BLDPROG_STRTAB_MAX_STRLEN * 2];
|
---|
550 | char *pszDst = szNew;
|
---|
551 | const char *pszSrc = pStr->pszString;
|
---|
552 | const char *pszSrcEnd = pszSrc + pStr->cchString;
|
---|
553 |
|
---|
554 | char ch;
|
---|
555 | while ((ch = *pszSrc) != '\0')
|
---|
556 | {
|
---|
557 | const char * const pszSrcUncompressed = pszSrc;
|
---|
558 | size_t cchWord = bldProgStrTab_compressorFindNextWord(pszSrc, ch, &pszSrc);
|
---|
559 | size_t cchSrcUncompressed = pszSrc - pszSrcUncompressed;
|
---|
560 | if (cchSrcUncompressed > 0)
|
---|
561 | {
|
---|
562 | pszDst = bldProgStrTab_compressorCopyAndEscape(pszDst, pszSrcUncompressed, cchSrcUncompressed);
|
---|
563 | if (!pszDst)
|
---|
564 | return false;
|
---|
565 | }
|
---|
566 | if (!cchWord)
|
---|
567 | break;
|
---|
568 |
|
---|
569 | /* Check for g_aWord matches. */
|
---|
570 | if (cchWord > 1)
|
---|
571 | {
|
---|
572 | size_t cchMax = pszSrcEnd - pszSrc;
|
---|
573 | for (unsigned i = 0; i < RT_ELEMENTS(pThis->aCompDict); i++)
|
---|
574 | {
|
---|
575 | size_t cchLen = pThis->aCompDict[i].cchString;
|
---|
576 | if ( cchLen >= cchWord
|
---|
577 | && cchLen <= cchMax
|
---|
578 | && memcmp(pThis->aCompDict[i].pszString, pszSrc, cchLen) == 0)
|
---|
579 | {
|
---|
580 | *pszDst++ = (unsigned char)i;
|
---|
581 | pszSrc += cchLen;
|
---|
582 | cchWord = 0;
|
---|
583 | break;
|
---|
584 | }
|
---|
585 | }
|
---|
586 | }
|
---|
587 |
|
---|
588 | if (cchWord > 0)
|
---|
589 | {
|
---|
590 | /* Copy the current word. */
|
---|
591 | pszDst = bldProgStrTab_compressorCopyAndEscape(pszDst, pszSrc, cchWord);
|
---|
592 | if (!pszDst)
|
---|
593 | return false;
|
---|
594 | pszSrc += cchWord;
|
---|
595 | }
|
---|
596 | }
|
---|
597 |
|
---|
598 | /* Just terminate it now. */
|
---|
599 | *pszDst = '\0';
|
---|
600 |
|
---|
601 | /*
|
---|
602 | * Update the string.
|
---|
603 | */
|
---|
604 | size_t cchNew = pszDst - &szNew[0];
|
---|
605 | if (cchNew > pStr->cchString)
|
---|
606 | {
|
---|
607 | pStr->pszString = (char *)malloc(cchNew + 1);
|
---|
608 | if (!pStr->pszString)
|
---|
609 | {
|
---|
610 | fprintf(stderr, "Out of memory!\n");
|
---|
611 | return false;
|
---|
612 | }
|
---|
613 | }
|
---|
614 | memcpy(pStr->pszString, szNew, cchNew + 1);
|
---|
615 | pStr->cchString = cchNew;
|
---|
616 |
|
---|
617 | return true;
|
---|
618 | }
|
---|
619 |
|
---|
620 |
|
---|
621 | /**
|
---|
622 | * Entry in SortedDictionary.
|
---|
623 | *
|
---|
624 | * Uses variable length string member, so not class and allocated via malloc.
|
---|
625 | */
|
---|
626 | struct SortedDictionaryEntry
|
---|
627 | {
|
---|
628 | size_t m_cchGain;
|
---|
629 | size_t m_cchString;
|
---|
630 | RT_FLEXIBLE_ARRAY_EXTENSION
|
---|
631 | char m_szString[RT_FLEXIBLE_ARRAY];
|
---|
632 |
|
---|
633 | /** Allocates and initializes a new entry. */
|
---|
634 | static SortedDictionaryEntry *allocate(const char *a_pch, size_t a_cch, size_t a_cchGain, char a_chSep)
|
---|
635 | {
|
---|
636 | size_t cbString = a_cch + !!a_chSep + 1;
|
---|
637 | SortedDictionaryEntry *pNew = (SortedDictionaryEntry *)malloc(RT_UOFFSETOF(SortedDictionaryEntry, m_szString) + cbString);
|
---|
638 | if (pNew)
|
---|
639 | {
|
---|
640 | pNew->m_cchGain = a_cchGain;
|
---|
641 | memcpy(pNew->m_szString, a_pch, a_cch);
|
---|
642 | if (a_chSep)
|
---|
643 | pNew->m_szString[a_cch++] = a_chSep;
|
---|
644 | pNew->m_szString[a_cch] = '\0';
|
---|
645 | pNew->m_cchString = a_cch;
|
---|
646 | }
|
---|
647 | return pNew;
|
---|
648 | }
|
---|
649 |
|
---|
650 |
|
---|
651 | /** Compares this dictionary entry with an incoming one.
|
---|
652 | * @retval -1 if this entry is of less worth than the new one.
|
---|
653 | * @retval 0 if this entry is of equal worth to the new one.
|
---|
654 | * @retval +1 if this entry is of more worth than the new one.
|
---|
655 | */
|
---|
656 | int compare(size_t a_cchGain, size_t a_cchString)
|
---|
657 | {
|
---|
658 | /* Higher gain is preferred of course: */
|
---|
659 | if (m_cchGain < a_cchGain)
|
---|
660 | return -1;
|
---|
661 | if (m_cchGain > a_cchGain)
|
---|
662 | return 1;
|
---|
663 |
|
---|
664 | /* Gain is the same. Prefer the shorter string, as it will result in a shorter string table: */
|
---|
665 | if (m_cchString > a_cchString)
|
---|
666 | return -1;
|
---|
667 | if (m_cchString < a_cchString)
|
---|
668 | return 1;
|
---|
669 | return 0;
|
---|
670 | }
|
---|
671 | };
|
---|
672 |
|
---|
673 |
|
---|
674 | /**
|
---|
675 | * Insertion sort dictionary that keeps the 256 best words.
|
---|
676 | *
|
---|
677 | * Used by bldProgStrTab_compressorDoStringCompression to pick the dictionary
|
---|
678 | * words.
|
---|
679 | */
|
---|
680 | class SortedDictionary
|
---|
681 | {
|
---|
682 | public:
|
---|
683 | size_t m_cEntries;
|
---|
684 | SortedDictionaryEntry *m_apEntries[256];
|
---|
685 |
|
---|
686 | SortedDictionary()
|
---|
687 | : m_cEntries(0)
|
---|
688 | {
|
---|
689 | for (size_t i = 0; i < RT_ELEMENTS(m_apEntries); i++)
|
---|
690 | m_apEntries[i] = NULL;
|
---|
691 | }
|
---|
692 |
|
---|
693 | ~SortedDictionary()
|
---|
694 | {
|
---|
695 | while (m_cEntries > 0)
|
---|
696 | {
|
---|
697 | free(m_apEntries[--m_cEntries]);
|
---|
698 | m_apEntries[m_cEntries] = NULL;
|
---|
699 | }
|
---|
700 | }
|
---|
701 |
|
---|
702 |
|
---|
703 | /**
|
---|
704 | * Inserts a new entry, if it's worth it.
|
---|
705 | * @returns true on succes, false if out of memory.
|
---|
706 | */
|
---|
707 | bool insert(const char *a_pchString, size_t a_cchStringBase, size_t a_cchGain, char a_chSep = 0)
|
---|
708 | {
|
---|
709 | size_t const cchString = a_cchStringBase + (a_chSep + 1);
|
---|
710 |
|
---|
711 | /*
|
---|
712 | * Drop the insert if the symbol table is full and the insert is less worth the last entry:
|
---|
713 | */
|
---|
714 | if ( m_cEntries >= RT_ELEMENTS(m_apEntries)
|
---|
715 | && m_apEntries[RT_ELEMENTS(m_apEntries) - 1]->compare(a_cchGain, cchString) >= 0)
|
---|
716 | return true;
|
---|
717 |
|
---|
718 | /*
|
---|
719 | * Create a new entry to insert.
|
---|
720 | */
|
---|
721 | SortedDictionaryEntry *pNewEntry = SortedDictionaryEntry::allocate(a_pchString, a_cchStringBase, a_cchGain, a_chSep);
|
---|
722 | if (!pNewEntry)
|
---|
723 | return false;
|
---|
724 |
|
---|
725 | /*
|
---|
726 | * Find the insert point.
|
---|
727 | */
|
---|
728 | if (m_cEntries == 0)
|
---|
729 | {
|
---|
730 | m_apEntries[0] = pNewEntry;
|
---|
731 | m_cEntries = 1;
|
---|
732 | }
|
---|
733 | else
|
---|
734 | {
|
---|
735 | /* If table is full, drop the last entry before we start (already made
|
---|
736 | sure the incoming entry is preferable to the one were dropping): */
|
---|
737 | if (m_cEntries >= RT_ELEMENTS(m_apEntries))
|
---|
738 | {
|
---|
739 | free(m_apEntries[RT_ELEMENTS(m_apEntries) - 1]);
|
---|
740 | m_apEntries[RT_ELEMENTS(m_apEntries) - 1] = NULL;
|
---|
741 | m_cEntries = RT_ELEMENTS(m_apEntries) - 1;
|
---|
742 | }
|
---|
743 |
|
---|
744 | /* Find where to insert the new entry: */
|
---|
745 | /** @todo use binary search. */
|
---|
746 | size_t i = m_cEntries;
|
---|
747 | while (i > 0 && m_apEntries[i - 1]->compare(a_cchGain, cchString) < 0)
|
---|
748 | i--;
|
---|
749 |
|
---|
750 | /* Shift entries to make room and insert the new entry. */
|
---|
751 | if (i < m_cEntries)
|
---|
752 | memmove(&m_apEntries[i + 1], &m_apEntries[i], (m_cEntries - i) * sizeof(m_apEntries[0]));
|
---|
753 | m_apEntries[i] = pNewEntry;
|
---|
754 | m_cEntries++;
|
---|
755 | }
|
---|
756 | return true;
|
---|
757 | }
|
---|
758 | };
|
---|
759 |
|
---|
760 |
|
---|
761 | /**
|
---|
762 | * Compresses the vendor and product strings.
|
---|
763 | *
|
---|
764 | * This is very very simple (a lot less work than the string table for
|
---|
765 | * instance).
|
---|
766 | */
|
---|
767 | static bool bldProgStrTab_compressorDoStringCompression(PBLDPROGSTRTAB pThis, bool fVerbose)
|
---|
768 | {
|
---|
769 | /*
|
---|
770 | * Sort the frequency analyzis result and pick the top entries for any
|
---|
771 | * available dictionary slots.
|
---|
772 | */
|
---|
773 | SortedDictionary SortedDict;
|
---|
774 | for (BLDPROGWORDFREQMAP::iterator it = pThis->Frequencies.begin(); it != pThis->Frequencies.end(); ++it)
|
---|
775 | {
|
---|
776 | bool fInsert;
|
---|
777 | size_t const cchString = it->first.length();
|
---|
778 | # ifndef BLDPROG_STRTAB_WITH_WORD_SEP_ALTERNATIVE
|
---|
779 | size_t const cchGainWithout = it->second.cWithoutSep * cchString;
|
---|
780 | # else
|
---|
781 | size_t const cchGainWithout = (it->second.cWithoutSep + it->second.cWithSep) * cchString;
|
---|
782 | size_t const cchGainWith = it->second.cWithSep * (cchString + 1);
|
---|
783 | if (cchGainWith > cchGainWithout)
|
---|
784 | fInsert = SortedDict.insert(it->first.c_str(), cchString, cchGainWith, it->second.chSep);
|
---|
785 | else
|
---|
786 | # endif
|
---|
787 | fInsert = SortedDict.insert(it->first.c_str(), cchString, cchGainWithout);
|
---|
788 | if (!fInsert)
|
---|
789 | return false;
|
---|
790 | }
|
---|
791 |
|
---|
792 | size_t cb = 0;
|
---|
793 | size_t cWords = 0;
|
---|
794 | size_t iDict = 0;
|
---|
795 | for (size_t i = 0; i < RT_ELEMENTS(pThis->aCompDict); i++)
|
---|
796 | {
|
---|
797 | char szTmp[2] = { (char)i, '\0' };
|
---|
798 | const char *psz = szTmp;
|
---|
799 | if ( ASMBitTest(pThis->bmUsedChars, (int32_t)i)
|
---|
800 | || iDict >= SortedDict.m_cEntries)
|
---|
801 | {
|
---|
802 | /* character entry */
|
---|
803 | pThis->auCompDictFreq[i] = 0;
|
---|
804 | pThis->aCompDict[i].cchString = 1;
|
---|
805 | }
|
---|
806 | else
|
---|
807 | {
|
---|
808 | /* word entry */
|
---|
809 | cb += SortedDict.m_apEntries[iDict]->m_cchGain;
|
---|
810 | pThis->auCompDictFreq[i] = SortedDict.m_apEntries[iDict]->m_cchGain;
|
---|
811 | pThis->aCompDict[i].cchString = SortedDict.m_apEntries[iDict]->m_cchString;
|
---|
812 | psz = SortedDict.m_apEntries[iDict]->m_szString;
|
---|
813 | cWords++;
|
---|
814 | iDict++;
|
---|
815 | }
|
---|
816 | pThis->aCompDict[i].pszString = (char *)malloc(pThis->aCompDict[i].cchString + 1);
|
---|
817 | if (pThis->aCompDict[i].pszString)
|
---|
818 | memcpy(pThis->aCompDict[i].pszString, psz, pThis->aCompDict[i].cchString + 1);
|
---|
819 | else
|
---|
820 | return false;
|
---|
821 | }
|
---|
822 |
|
---|
823 | if (fVerbose)
|
---|
824 | printf("debug: Estimated string compression saving: %u bytes\n"
|
---|
825 | "debug: %u words, %u characters\n"
|
---|
826 | , (unsigned)cb, (unsigned)cWords, (unsigned)(RT_ELEMENTS(pThis->aCompDict) - cWords));
|
---|
827 |
|
---|
828 | /*
|
---|
829 | * Rework the strings.
|
---|
830 | */
|
---|
831 | size_t cchOld = 0;
|
---|
832 | size_t cchOldMax = 0;
|
---|
833 | size_t cchOldMin = BLDPROG_STRTAB_MAX_STRLEN;
|
---|
834 | size_t cchNew = 0;
|
---|
835 | size_t cchNewMax = 0;
|
---|
836 | size_t cchNewMin = BLDPROG_STRTAB_MAX_STRLEN;
|
---|
837 | size_t i = pThis->cPendingStrings;
|
---|
838 | while (i-- > 0)
|
---|
839 | {
|
---|
840 | PBLDPROGSTRING pCurStr = pThis->papPendingStrings[i];
|
---|
841 | cchOld += pCurStr->cchString;
|
---|
842 | if (pCurStr->cchString > cchOldMax)
|
---|
843 | cchOldMax = pCurStr->cchString;
|
---|
844 | if (pCurStr->cchString < cchOldMin)
|
---|
845 | cchOldMin = pCurStr->cchString;
|
---|
846 |
|
---|
847 | if (!bldProgStrTab_compressorFixupString(pThis, pCurStr))
|
---|
848 | return false;
|
---|
849 |
|
---|
850 | cchNew += pCurStr->cchString;
|
---|
851 | if (pCurStr->cchString > cchNewMax)
|
---|
852 | cchNewMax = pCurStr->cchString;
|
---|
853 | if (pCurStr->cchString < cchNewMin)
|
---|
854 | cchNewMin = pCurStr->cchString;
|
---|
855 |
|
---|
856 | bldProgStrTab_AddStringToHashTab(pThis, pCurStr);
|
---|
857 | }
|
---|
858 |
|
---|
859 | /*
|
---|
860 | * Do debug stats.
|
---|
861 | */
|
---|
862 | if (fVerbose)
|
---|
863 | {
|
---|
864 | for (i = 0; i < RT_ELEMENTS(pThis->aCompDict); i++)
|
---|
865 | cchNew += pThis->aCompDict[i].cchString + 1;
|
---|
866 |
|
---|
867 | printf("debug: Strings: original: %u bytes; compressed: %u bytes;", (unsigned)cchOld, (unsigned)cchNew);
|
---|
868 | if (cchNew < cchOld)
|
---|
869 | printf(" saving %u bytes (%u%%)\n", (unsigned)(cchOld - cchNew), (unsigned)((cchOld - cchNew) * 100 / cchOld));
|
---|
870 | else
|
---|
871 | printf(" wasting %u bytes!\n", (unsigned)(cchOld - cchNew));
|
---|
872 | printf("debug: Original string lengths: average %u; min %u; max %u\n",
|
---|
873 | (unsigned)(cchOld / pThis->cPendingStrings), (unsigned)cchOldMin, (unsigned)cchOldMax);
|
---|
874 | printf("debug: Compressed string lengths: average %u; min %u; max %u\n",
|
---|
875 | (unsigned)(cchNew / pThis->cPendingStrings), (unsigned)cchNewMin, (unsigned)cchNewMax);
|
---|
876 | }
|
---|
877 |
|
---|
878 | return true;
|
---|
879 | }
|
---|
880 |
|
---|
881 | #endif /* BLDPROG_STRTAB_WITH_COMPRESSION */
|
---|
882 |
|
---|
883 | /**
|
---|
884 | * Inserts a string into g_apUniqueStrings.
|
---|
885 | * @param pThis The strint table compiler instance.
|
---|
886 | * @param pStr The string.
|
---|
887 | */
|
---|
888 | static void bldProgStrTab_InsertUniqueString(PBLDPROGSTRTAB pThis, PBLDPROGSTRING pStr)
|
---|
889 | {
|
---|
890 | size_t iIdx;
|
---|
891 | size_t iEnd = pThis->cSortedStrings;
|
---|
892 | if (iEnd)
|
---|
893 | {
|
---|
894 | size_t iStart = 0;
|
---|
895 | for (;;)
|
---|
896 | {
|
---|
897 | iIdx = iStart + (iEnd - iStart) / 2;
|
---|
898 | if (pThis->papSortedStrings[iIdx]->cchString < pStr->cchString)
|
---|
899 | {
|
---|
900 | if (iIdx <= iStart)
|
---|
901 | break;
|
---|
902 | iEnd = iIdx;
|
---|
903 | }
|
---|
904 | else if (pThis->papSortedStrings[iIdx]->cchString > pStr->cchString)
|
---|
905 | {
|
---|
906 | if (++iIdx >= iEnd)
|
---|
907 | break;
|
---|
908 | iStart = iIdx;
|
---|
909 | }
|
---|
910 | else
|
---|
911 | break;
|
---|
912 | }
|
---|
913 |
|
---|
914 | if (iIdx != pThis->cSortedStrings)
|
---|
915 | memmove(&pThis->papSortedStrings[iIdx + 1], &pThis->papSortedStrings[iIdx],
|
---|
916 | (pThis->cSortedStrings - iIdx) * sizeof(pThis->papSortedStrings[iIdx]));
|
---|
917 | }
|
---|
918 | else
|
---|
919 | iIdx = 0;
|
---|
920 |
|
---|
921 | pThis->papSortedStrings[iIdx] = pStr;
|
---|
922 | pThis->cSortedStrings++;
|
---|
923 | }
|
---|
924 |
|
---|
925 |
|
---|
926 | /**
|
---|
927 | * Compiles the string table after the string has been added.
|
---|
928 | *
|
---|
929 | * This will save space by dropping string terminators, eliminating duplicates
|
---|
930 | * and try find strings that are sub-strings of others.
|
---|
931 | *
|
---|
932 | * Will initialize the StrRef of all StrTabString instances.
|
---|
933 | *
|
---|
934 | * @returns success indicator (error flagged).
|
---|
935 | * @param pThis The strint table compiler instance.
|
---|
936 | */
|
---|
937 | static bool BldProgStrTab_CompileIt(PBLDPROGSTRTAB pThis, bool fVerbose)
|
---|
938 | {
|
---|
939 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
940 | /*
|
---|
941 | * Do the compression and add all the compressed strings to the table.
|
---|
942 | */
|
---|
943 | if (!bldProgStrTab_compressorDoStringCompression(pThis, fVerbose))
|
---|
944 | return false;
|
---|
945 |
|
---|
946 | /*
|
---|
947 | * Add the dictionary strings.
|
---|
948 | */
|
---|
949 | for (unsigned i = 0; i < RT_ELEMENTS(pThis->aCompDict); i++)
|
---|
950 | if (pThis->aCompDict[i].cchString > 1)
|
---|
951 | bldProgStrTab_AddStringToHashTab(pThis, &pThis->aCompDict[i]);
|
---|
952 | # ifdef RT_STRICT
|
---|
953 | else if (pThis->aCompDict[i].cchString != 1)
|
---|
954 | abort();
|
---|
955 | # endif
|
---|
956 | #endif
|
---|
957 | if (fVerbose)
|
---|
958 | printf("debug: %u unique strings (%u bytes), %u duplicates, %u collisions\n",
|
---|
959 | (unsigned)pThis->cUniqueStrings, (unsigned)pThis->cchUniqueStrings,
|
---|
960 | (unsigned)pThis->cDuplicateStrings, (unsigned)pThis->cCollisions);
|
---|
961 |
|
---|
962 | /*
|
---|
963 | * Create papSortedStrings from the hash table. The table is sorted by
|
---|
964 | * string length, with the longer strings first, so that we increase our
|
---|
965 | * chances of locating duplicate substrings.
|
---|
966 | */
|
---|
967 | pThis->papSortedStrings = (PBLDPROGSTRING *)malloc(sizeof(pThis->papSortedStrings[0]) * pThis->cUniqueStrings);
|
---|
968 | if (!pThis->papSortedStrings)
|
---|
969 | return false;
|
---|
970 | pThis->cSortedStrings = 0;
|
---|
971 | size_t idxHash = pThis->cStrHash;
|
---|
972 | while (idxHash-- > 0)
|
---|
973 | {
|
---|
974 | PBLDPROGSTRING pCur = pThis->papStrHash[idxHash];
|
---|
975 | if (pCur)
|
---|
976 | {
|
---|
977 | do
|
---|
978 | {
|
---|
979 | bldProgStrTab_InsertUniqueString(pThis, pCur);
|
---|
980 | pCur = pCur->pNextCollision;
|
---|
981 | } while (pCur);
|
---|
982 | }
|
---|
983 | }
|
---|
984 |
|
---|
985 | /*
|
---|
986 | * Create the actual string table.
|
---|
987 | */
|
---|
988 | pThis->pachStrTab = (char *)malloc(pThis->cchUniqueStrings + 1);
|
---|
989 | if (!pThis->pachStrTab)
|
---|
990 | return false;
|
---|
991 | pThis->cchStrTab = 0;
|
---|
992 | for (size_t i = 0; i < pThis->cSortedStrings; i++)
|
---|
993 | {
|
---|
994 | PBLDPROGSTRING pCur = pThis->papSortedStrings[i];
|
---|
995 | const char * const pszCur = pCur->pszString;
|
---|
996 | size_t const cchCur = pCur->cchString;
|
---|
997 | size_t offStrTab = pThis->cchStrTab;
|
---|
998 |
|
---|
999 | /*
|
---|
1000 | * See if the string is a substring already found in the string table.
|
---|
1001 | * Excluding the zero terminator increases the chances for this.
|
---|
1002 | */
|
---|
1003 | size_t cchLeft = pThis->cchStrTab >= cchCur ? pThis->cchStrTab - cchCur : 0;
|
---|
1004 | const char *pchLeft = pThis->pachStrTab;
|
---|
1005 | char const chFirst = *pszCur;
|
---|
1006 | while (cchLeft > 0)
|
---|
1007 | {
|
---|
1008 | const char *pchCandidate = (const char *)memchr(pchLeft, chFirst, cchLeft);
|
---|
1009 | if (!pchCandidate)
|
---|
1010 | break;
|
---|
1011 | if (memcmp(pchCandidate, pszCur, cchCur) == 0)
|
---|
1012 | {
|
---|
1013 | offStrTab = pchCandidate - pThis->pachStrTab;
|
---|
1014 | break;
|
---|
1015 | }
|
---|
1016 |
|
---|
1017 | cchLeft -= pchCandidate + 1 - pchLeft;
|
---|
1018 | pchLeft = pchCandidate + 1;
|
---|
1019 | }
|
---|
1020 |
|
---|
1021 | if (offStrTab == pThis->cchStrTab)
|
---|
1022 | {
|
---|
1023 | /*
|
---|
1024 | * See if the start of the string overlaps the end of the string table.
|
---|
1025 | */
|
---|
1026 | if (pThis->cchStrTab && cchCur > 1)
|
---|
1027 | {
|
---|
1028 | cchLeft = RT_MIN(pThis->cchStrTab, cchCur - 1);
|
---|
1029 | pchLeft = &pThis->pachStrTab[pThis->cchStrTab - cchLeft];
|
---|
1030 | while (cchLeft > 0)
|
---|
1031 | {
|
---|
1032 | const char *pchCandidate = (const char *)memchr(pchLeft, chFirst, cchLeft);
|
---|
1033 | if (!pchCandidate)
|
---|
1034 | break;
|
---|
1035 | cchLeft -= pchCandidate - pchLeft;
|
---|
1036 | pchLeft = pchCandidate;
|
---|
1037 | if (memcmp(pchLeft, pszCur, cchLeft) == 0)
|
---|
1038 | {
|
---|
1039 | size_t cchToCopy = cchCur - cchLeft;
|
---|
1040 | memcpy(&pThis->pachStrTab[offStrTab], &pszCur[cchLeft], cchToCopy);
|
---|
1041 | pThis->cchStrTab += cchToCopy;
|
---|
1042 | offStrTab = pchCandidate - pThis->pachStrTab;
|
---|
1043 | break;
|
---|
1044 | }
|
---|
1045 | cchLeft--;
|
---|
1046 | pchLeft++;
|
---|
1047 | }
|
---|
1048 | }
|
---|
1049 |
|
---|
1050 | /*
|
---|
1051 | * If we didn't have any luck above, just append the string.
|
---|
1052 | */
|
---|
1053 | if (offStrTab == pThis->cchStrTab)
|
---|
1054 | {
|
---|
1055 | memcpy(&pThis->pachStrTab[offStrTab], pszCur, cchCur);
|
---|
1056 | pThis->cchStrTab += cchCur;
|
---|
1057 | }
|
---|
1058 | }
|
---|
1059 |
|
---|
1060 | /*
|
---|
1061 | * Set the string table offset for all the references to this string.
|
---|
1062 | */
|
---|
1063 | do
|
---|
1064 | {
|
---|
1065 | pCur->offStrTab = (uint32_t)offStrTab;
|
---|
1066 | pCur = pCur->pNextRef;
|
---|
1067 | } while (pCur != NULL);
|
---|
1068 | }
|
---|
1069 |
|
---|
1070 | if (fVerbose)
|
---|
1071 | printf("debug: String table: %u bytes\n", (unsigned)pThis->cchStrTab);
|
---|
1072 | return true;
|
---|
1073 | }
|
---|
1074 |
|
---|
1075 |
|
---|
1076 | #ifdef RT_STRICT
|
---|
1077 | /**
|
---|
1078 | * Sanity checks a string table string.
|
---|
1079 | * @param pThis The strint table compiler instance.
|
---|
1080 | * @param pStr The string to check.
|
---|
1081 | */
|
---|
1082 | static void BldProgStrTab_CheckStrTabString(PBLDPROGSTRTAB pThis, PBLDPROGSTRING pStr)
|
---|
1083 | {
|
---|
1084 | if (pStr->cchString != strlen(pStr->pszString))
|
---|
1085 | abort();
|
---|
1086 | if (pStr->offStrTab >= pThis->cchStrTab)
|
---|
1087 | abort();
|
---|
1088 | if (pStr->offStrTab + pStr->cchString > pThis->cchStrTab)
|
---|
1089 | abort();
|
---|
1090 | if (memcmp(pStr->pszString, &pThis->pachStrTab[pStr->offStrTab], pStr->cchString) != 0)
|
---|
1091 | abort();
|
---|
1092 | }
|
---|
1093 | #endif
|
---|
1094 |
|
---|
1095 |
|
---|
1096 | /**
|
---|
1097 | * Output the string table string in C string litteral fashion.
|
---|
1098 | *
|
---|
1099 | * @param pThis The strint table instance.
|
---|
1100 | * @param pString The string to print.
|
---|
1101 | * @param pOut The output stream.
|
---|
1102 | */
|
---|
1103 | static void BldProgStrTab_PrintCStringLitteral(PBLDPROGSTRTAB pThis, PBLDPROGSTRING pString, FILE *pOut)
|
---|
1104 | {
|
---|
1105 | const unsigned char *psz = (const unsigned char *)pString->pszString;
|
---|
1106 | unsigned char uch;
|
---|
1107 | while ((uch = *psz++) != '\0')
|
---|
1108 | {
|
---|
1109 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
1110 | if (pThis->aCompDict[uch].cchString == 1)
|
---|
1111 | #else
|
---|
1112 | if (!(uch & 0x80))
|
---|
1113 | #endif
|
---|
1114 | {
|
---|
1115 | if (uch != '\'' && uch != '\\')
|
---|
1116 | fputc((char)uch, pOut);
|
---|
1117 | else
|
---|
1118 | {
|
---|
1119 | fputc('\\', pOut);
|
---|
1120 | fputc((char)uch, pOut);
|
---|
1121 | }
|
---|
1122 | }
|
---|
1123 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
1124 | # ifndef BLDPROG_STRTAB_PURE_ASCII
|
---|
1125 | else if (uch == 0xff)
|
---|
1126 | {
|
---|
1127 | RTUNICP uc = RTStrGetCp((const char *)psz);
|
---|
1128 | psz += RTStrCpSize(uc);
|
---|
1129 | fprintf(pOut, "\\u%04x", uc);
|
---|
1130 | }
|
---|
1131 | # else
|
---|
1132 | else
|
---|
1133 | fputs(pThis->aCompDict[uch].pszString, pOut);
|
---|
1134 | # endif
|
---|
1135 | #else
|
---|
1136 | else
|
---|
1137 | fprintf(pOut, "\\x%02x", (unsigned)uch);
|
---|
1138 | NOREF(pThis);
|
---|
1139 | #endif
|
---|
1140 | }
|
---|
1141 | }
|
---|
1142 |
|
---|
1143 |
|
---|
1144 | /**
|
---|
1145 | * Writes the string table code to the output stream.
|
---|
1146 | *
|
---|
1147 | * @param pThis The strint table compiler instance.
|
---|
1148 | * @param pOut The output stream.
|
---|
1149 | * @param pszScope The scoping ("static " or empty string),
|
---|
1150 | * including trailing space.
|
---|
1151 | * @param pszPrefix The variable prefix. Typically "g_" for C programs,
|
---|
1152 | * whereas C++ classes normally use "class::s_g".
|
---|
1153 | * @param pszBaseName The base name for the variables. The user
|
---|
1154 | * accessible variable will end up as just the
|
---|
1155 | * prefix and basename concatenated.
|
---|
1156 | */
|
---|
1157 | static void BldProgStrTab_WriteStringTable(PBLDPROGSTRTAB pThis, FILE *pOut,
|
---|
1158 | const char *pszScope, const char *pszPrefix, const char *pszBaseName)
|
---|
1159 | {
|
---|
1160 | #ifdef RT_STRICT
|
---|
1161 | /*
|
---|
1162 | * Do some quick sanity checks while we're here.
|
---|
1163 | */
|
---|
1164 | # ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
1165 | for (unsigned i = 0; i < RT_ELEMENTS(pThis->aCompDict); i++)
|
---|
1166 | {
|
---|
1167 | if (ASMBitTest(pThis->bmUsedChars, (int32_t)i)
|
---|
1168 | ? pThis->aCompDict[i].cchString != 1 : pThis->aCompDict[i].cchString < 1)
|
---|
1169 | abort();
|
---|
1170 | if (pThis->aCompDict[i].cchString > 1)
|
---|
1171 | BldProgStrTab_CheckStrTabString(pThis, &pThis->aCompDict[i]);
|
---|
1172 | }
|
---|
1173 | # endif
|
---|
1174 | #endif
|
---|
1175 |
|
---|
1176 | /*
|
---|
1177 | * Create a table for speeding up the character categorization.
|
---|
1178 | */
|
---|
1179 | uint8_t abCharCat[256];
|
---|
1180 | memset(abCharCat, 0, sizeof(abCharCat));
|
---|
1181 | abCharCat[(unsigned char)'\\'] = 1;
|
---|
1182 | abCharCat[(unsigned char)'\''] = 1;
|
---|
1183 | for (unsigned i = 0; i < 0x20; i++)
|
---|
1184 | abCharCat[i] = 2;
|
---|
1185 | for (unsigned i = 0x7f; i < 0x100; i++)
|
---|
1186 | abCharCat[i] = 2;
|
---|
1187 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
1188 | for (unsigned i = 0; i < 0x100; i++)
|
---|
1189 | if (!ASMBitTest(pThis->bmUsedChars, (int32_t)i)) /* Encode table references using '\xYY'. */
|
---|
1190 | abCharCat[i] = 2;
|
---|
1191 | #endif
|
---|
1192 |
|
---|
1193 | /*
|
---|
1194 | * We follow the sorted string table, one string per line.
|
---|
1195 | */
|
---|
1196 | fprintf(pOut,
|
---|
1197 | "#include <iprt/bldprog-strtab.h>\n"
|
---|
1198 | "\n"
|
---|
1199 | "static const char g_achStrTab%s[] =\n"
|
---|
1200 | "{\n",
|
---|
1201 | pszBaseName);
|
---|
1202 |
|
---|
1203 | uint32_t off = 0;
|
---|
1204 | for (uint32_t i = 0; i < pThis->cSortedStrings; i++)
|
---|
1205 | {
|
---|
1206 | PBLDPROGSTRING pCur = pThis->papSortedStrings[i];
|
---|
1207 | uint32_t offEnd = pCur->offStrTab + (uint32_t)pCur->cchString;
|
---|
1208 | if (offEnd > off)
|
---|
1209 | {
|
---|
1210 | /* Comment with an uncompressed and more readable version of the string. */
|
---|
1211 | if (off == pCur->offStrTab)
|
---|
1212 | fprintf(pOut, "/* 0x%05x = \"", off);
|
---|
1213 | else
|
---|
1214 | fprintf(pOut, "/* 0X%05x = \"", off);
|
---|
1215 | BldProgStrTab_PrintCStringLitteral(pThis, pCur, pOut);
|
---|
1216 | fputs("\" */\n", pOut);
|
---|
1217 |
|
---|
1218 | /* Must use char by char here or we may trigger the max string
|
---|
1219 | length limit in the compiler, */
|
---|
1220 | fputs(" ", pOut);
|
---|
1221 | for (; off < offEnd; off++)
|
---|
1222 | {
|
---|
1223 | unsigned char uch = pThis->pachStrTab[off];
|
---|
1224 | fputc('\'', pOut);
|
---|
1225 | if (abCharCat[uch] == 0)
|
---|
1226 | fputc(uch, pOut);
|
---|
1227 | else if (abCharCat[uch] != 1)
|
---|
1228 | fprintf(pOut, "\\x%02x", (unsigned)uch);
|
---|
1229 | else
|
---|
1230 | {
|
---|
1231 | fputc('\\', pOut);
|
---|
1232 | fputc(uch, pOut);
|
---|
1233 | }
|
---|
1234 | fputc('\'', pOut);
|
---|
1235 | fputc(',', pOut);
|
---|
1236 | }
|
---|
1237 | fputc('\n', pOut);
|
---|
1238 | }
|
---|
1239 | }
|
---|
1240 |
|
---|
1241 | fprintf(pOut,
|
---|
1242 | "};\n"
|
---|
1243 | "AssertCompile(sizeof(g_achStrTab%s) == %#x);\n\n",
|
---|
1244 | pszBaseName, (unsigned)pThis->cchStrTab);
|
---|
1245 |
|
---|
1246 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
1247 | /*
|
---|
1248 | * Write the compression dictionary.
|
---|
1249 | */
|
---|
1250 | fprintf(pOut,
|
---|
1251 | "static const RTBLDPROGSTRREF g_aCompDict%s[%u] = \n"
|
---|
1252 | "{\n",
|
---|
1253 | pszBaseName, (unsigned)RT_ELEMENTS(pThis->aCompDict));
|
---|
1254 | for (unsigned i = 0; i < RT_ELEMENTS(pThis->aCompDict); i++)
|
---|
1255 | if (pThis->aCompDict[i].cchString > 1)
|
---|
1256 | fprintf(pOut, " /*[%3u]=*/ { %#08x, %#04x }, // %6lu - %s\n", i,
|
---|
1257 | pThis->aCompDict[i].offStrTab, (unsigned)pThis->aCompDict[i].cchString,
|
---|
1258 | (unsigned long)pThis->auCompDictFreq[i], pThis->aCompDict[i].pszString);
|
---|
1259 | # ifndef BLDPROG_STRTAB_PURE_ASCII
|
---|
1260 | else if (i == 0xff)
|
---|
1261 | fprintf(pOut, " /*[%3u]=*/ { 0x000000, 0x00 }, // UTF-8 escape\n", i);
|
---|
1262 | # endif
|
---|
1263 | else if (i == 0)
|
---|
1264 | fprintf(pOut, " /*[%3u]=*/ { 0x000000, 0x00 }, // unused, because zero terminator\n", i);
|
---|
1265 | else if (i < 0x20)
|
---|
1266 | fprintf(pOut, " /*[%3u]=*/ { 0x000000, 0x00 }, // %02x\n", i, i);
|
---|
1267 | else
|
---|
1268 | fprintf(pOut, " /*[%3u]=*/ { 0x000000, 0x00 }, // '%c'\n", i, (char)i);
|
---|
1269 | fprintf(pOut, "};\n\n");
|
---|
1270 | #endif
|
---|
1271 |
|
---|
1272 |
|
---|
1273 | /*
|
---|
1274 | * Write the string table data structure.
|
---|
1275 | */
|
---|
1276 | fprintf(pOut,
|
---|
1277 | "%sconst RTBLDPROGSTRTAB %s%s = \n"
|
---|
1278 | "{\n"
|
---|
1279 | " /*.pchStrTab = */ &g_achStrTab%s[0],\n"
|
---|
1280 | " /*.cchStrTab = */ sizeof(g_achStrTab%s),\n"
|
---|
1281 | ,
|
---|
1282 | pszScope, pszPrefix, pszBaseName, pszBaseName, pszBaseName);
|
---|
1283 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
1284 | fprintf(pOut,
|
---|
1285 | " /*.cCompDict = */ %u,\n"
|
---|
1286 | " /*.paCompDict = */ &g_aCompDict%s[0]\n"
|
---|
1287 | "};\n"
|
---|
1288 | # ifndef BLDPROG_STRTAB_PURE_ASCII /* 255 or 256 entries is how the decoder knows */
|
---|
1289 | , (unsigned)RT_ELEMENTS(pThis->aCompDict) - 1,
|
---|
1290 | # else
|
---|
1291 | , (unsigned)RT_ELEMENTS(pThis->aCompDict),
|
---|
1292 | # endif
|
---|
1293 | pszBaseName);
|
---|
1294 | #else
|
---|
1295 | fprintf(pOut,
|
---|
1296 | " /*.cCompDict = */ 0,\n"
|
---|
1297 | " /*.paCompDict = */ NULL\n"
|
---|
1298 | "};\n");
|
---|
1299 | #endif
|
---|
1300 | }
|
---|
1301 |
|
---|
1302 | #if RT_CLANG_PREREQ(4, 0) || RT_GNUC_PREREQ(4, 6)
|
---|
1303 | # pragma GCC diagnostic pop
|
---|
1304 | #endif
|
---|
1305 |
|
---|
1306 | #endif /* __cplusplus && IN_RING3 */
|
---|
1307 |
|
---|