Changeset 28876 in vbox for trunk/src/VBox/Runtime

Timestamp:

Apr 28, 2010 7:01:33 PM (15 years ago)

Author:

vboxsync

Message:

uniread.cpp: Updated to cope with version 5.2 of the spec. Preparing for exctracing necessary decomposition and normalization information. Fixed Oracle (C).

File:

• trunk/src/VBox/Runtime/common/string/uniread.cpp (modified) (43 diffs)

trunk/src/VBox/Runtime/common/string/uniread.cpp

@@ -30 +30 @@
 #include <iprt/types.h>
 #include <iprt/stdarg.h>
+#include <iprt/ctype.h>
 
 #include <stdio.h>
@@ -35 +36 @@
 #include <stdlib.h>
 
+
+/*******************************************************************************
+*   Global Variables                                                           *
+*******************************************************************************/
+/** When set, no output is produced.  Very useful when debugging ths code. */
+static bool g_fQuiet = false;
+/** The file we're currently parsing. */
+static const char *g_pszCurFile;
+/** The current line number. */
+static unsigned g_iLine;
+
+
+/**
+ * Exit the program after printing a parse error.
+ *
+ * @param   pszFormat           The message.
+ * @param   ...                 Format arguments.
+ */
+static void ParseError(const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    fprintf(stderr, "parse error: %s:%u: ", g_pszCurFile, g_iLine);
+    vfprintf(stderr, pszFormat, va);
+    va_end(va);
+    exit(1);
+}
 
 /**
@@ -133 +161 @@
 
 /**
+ * Splits a decomposition field.
+ *
+ * This may start with a type that is enclosed in angle brackets.
+ *
+ * @returns Pointer to the mapping values following the type. @a *ppsz if empty.
+ * @param   ppszType    Pointer to the type field pointer.  On input the type
+ *                      field contains the combined type and mapping string.  On
+ *                      output this should only contain the type, no angle
+ *                      brackets.  If no type specified, it is replaced with an
+ *                      empty string (const).
+ */
+static char *SplitDecompField(char **ppszType)
+{
+    /* Empty field? */
+    char *psz = *ppszType;
+    if (!*psz)
+        return psz;
+
+    /* No type? */
+    if (*psz != '<')
+    {
+        *ppszType = (char *)"";
+        return psz;
+    }
+
+    /* Split out the type. */
+    *ppszType = ++psz;
+    psz = strchr(psz, '>');
+    if (!psz)
+    {
+        ParseError("Bad Decomposition Type/Mappings\n");
+        return *ppszType;
+    }
+    *psz++ = '\0';
+
+    psz = StripLine(psz);
+    if (!*psz)
+        ParseError("Missing decomposition mappings\n");
+    return psz;
+}
+
+/**
  * Converts a code point field to a number.
  * @returns Code point.
@@ -142 +212 @@
     unsigned long ul = strtoul(psz, &pszEnd, 16);
     if (pszEnd && *pszEnd)
-        fprintf(stderr, "warning: failed converting '%s' to a number!\n", psz);
+        ParseError("failed converting '%s' to a number!\n", psz);
     return (RTUNICP)ul;
 }
@@ -179 +249 @@
             if (pszEnd && *pszEnd)
             {
-                fprintf(stderr, "warning: failed converting '%s' to a number!\n", psz);
+                ParseError("failed converting '%s' to a number!\n", psz);
                 return ~(RTUNICP)0;
             }
@@ -185 +255 @@
         else
         {
-            fprintf(stderr, "warning: failed converting '%s' to a number!\n", psz);
+            ParseError("failed converting '%s' to a number!\n", psz);
             return ~(RTUNICP)0;
         }
@@ -192 +262 @@
     return (RTUNICP)ulStart;
 
+}
+
+/**
+ * For converting the decompisition mappings field and similar.
+ *
+ * @returns Mapping array or NULL if none.
+ * @param   psz                 The string to convert.  Can be empty.
+ * @param   pcEntries           Where to store the number of entries.
+ * @param   cMax                The max number of entries.
+ */
+static PRTUNICP ToMapping(char *psz, unsigned *pcEntries, unsigned cMax)
+{
+    PRTUNICP paCps  = NULL;
+    unsigned cAlloc = 0;
+    unsigned i      = 0;
+
+    /* Convert the code points. */
+    while (psz)
+    {
+        /* skip leading spaces */
+        while (RT_C_IS_BLANK(*psz))
+            psz++;
+
+        /* the end? */
+        if (!*psz)
+            break;
+
+        /* room left? */
+        if (i >= cMax)
+        {
+            ParseError("Too many mappings.\n");
+            break;
+        }
+        if (i >= cAlloc)
+        {
+            cAlloc += 4;
+            paCps = (PRTUNICP)realloc(paCps, cAlloc * sizeof(paCps[0]));
+            if (!paCps)
+            {
+                fprintf(stderr, "out of memory (%u)\n", (unsigned)(cAlloc * sizeof(paCps[0])));
+                exit(1);
+            }
+        }
+
+        /* Find the end. */
+        char *pszThis = psz;
+        while (RT_C_IS_XDIGIT(*psz))
+            psz++;
+        if (*psz && !RT_C_IS_BLANK(*psz))
+            ParseError("Malformed mappings.\n");
+        if (*psz)
+            *psz++ = '\0';
+
+        /* Convert to number and add it. */
+        paCps[i++] = ToNum(pszThis);
+    }
+
+    *pcEntries = i;
+    return paCps;
 }
 
@@ -223 +352 @@
     RTUNICP     SimpleLowerCaseMapping;
     RTUNICP     SimpleTitleCaseMapping;
+    unsigned    CanonicalCombiningClass;
+    const char *pszDecompositionType;
+    unsigned    cDecompositionMapping;
+    PRTUNICP    paDecompositionMapping;
     const char *pszName;
     /** Set if this is an unused entry */
@@ -230 +363 @@
     unsigned    fASCIIHexDigit : 1;
     unsigned    fBidiControl : 1;
+    unsigned    fCaseIgnorable : 1;
+    unsigned    fCased : 1;
+    unsigned    fChangesWhenCasefolded : 1;
+    unsigned    fChangesWhenCasemapped : 1;
+    unsigned    fChangesWhenLowercased : 1;
+    unsigned    fChangesWhenTitlecased : 1;
+    unsigned    fChangesWhenUppercased : 1;
     unsigned    fDash : 1;
     unsigned    fDefaultIgnorableCodePoint : 1;
@@ -272 +412 @@
     unsigned    fXIDStart : 1;
 
-    /* unprocess stuff, so far. */
+    /** @name DerivedNormalizationProps.txt
+     * @{ */
+    unsigned    fFullCompositionExclusion : 1;
+    unsigned    fInvNFC_QC : 2;     /**< If 1 (NFC_QC == N) then code point 100% sure not part of NFC string. */
+    unsigned    fInvNFD_QC : 2;     /**< If 1 (NFD_QC == N) then code point 100% sure not part of NFD string. */
+    unsigned    fInvNFKC_QC : 2;
+    unsigned    fInvNFKD_QC : 2;
+    unsigned    fExpandsOnNFC : 1;
+    unsigned    fExpandsOnNFD : 1;
+    unsigned    fExpandsOnNFKC : 1;
+    unsigned    fExpandsOnNFKD : 1;
+    /** @}  */
+
+    /* unprocessed stuff, so far. */
     const char *pszGeneralCategory;
-    const char *pszCanonicalCombiningClass;
     const char *pszBidiClass;
-    const char *pszDecompositionType;
-    const char *pszDecompositionMapping;
     const char *pszNumericType;
-    const char *pszNumericValue;
+    const char *pszNumericValueD;
+    const char *pszNumericValueN;
     const char *pszBidiMirrored;
     const char *pszUnicode1Name;
     const char *pszISOComment;
-} g_aCPInfo[0xf0000];
+} g_aCPInfo[0x110000];
 
 
@@ -294 +445 @@
     g_aCPInfo[i].CodePoint = i;
     g_aCPInfo[i].fNullEntry = 1;
-    g_aCPInfo[i].pszName = "";
     g_aCPInfo[i].SimpleUpperCaseMapping = i;
     g_aCPInfo[i].SimpleLowerCaseMapping = i;
     g_aCPInfo[i].SimpleTitleCaseMapping = i;
+    g_aCPInfo[i].pszDecompositionType = "";
+    g_aCPInfo[i].cDecompositionMapping = 0;
+    g_aCPInfo[i].paDecompositionMapping = NULL;
+    g_aCPInfo[i].pszName = "";
     g_aCPInfo[i].pszGeneralCategory = "";
-    g_aCPInfo[i].pszCanonicalCombiningClass = "";
     g_aCPInfo[i].pszBidiClass = "";
-    g_aCPInfo[i].pszDecompositionType = "";
-    g_aCPInfo[i].pszDecompositionMapping = "";
     g_aCPInfo[i].pszNumericType = "";
-    g_aCPInfo[i].pszNumericValue = "";
+    g_aCPInfo[i].pszNumericValueD = "";
+    g_aCPInfo[i].pszNumericValueN = "";
     g_aCPInfo[i].pszBidiMirrored = "";
     g_aCPInfo[i].pszUnicode1Name = "";
@@ -312 +464 @@
 
 /**
+ * Open a file for reading, optionally with a base path prefixed.
+ *
+ * @returns file stream on success, NULL w/ complaint on failure.
+ * @param   pszBasePath         The base path, can be NULL.
+ * @param   pszFilename         The name of the file to open.
+ */
+static FILE *OpenFile(const char *pszBasePath, const char *pszFilename)
+{
+    FILE *pFile;
+    if (   !pszBasePath
+        || *pszFilename == '/'
+#if defined(_MSC_VER) || defined(__OS2__)
+        || *pszFilename == '\\'
+        || (*pszFilename && pszFilename[1] == ':')
+#endif
+       )
+    {
+        pFile = fopen(pszFilename, "r");
+        if (!pFile)
+            fprintf(stderr, "uniread: failed to open '%s' for reading\n", pszFilename);
+    }
+    else
+    {
+        size_t cchBasePath = strlen(pszBasePath);
+        size_t cchFilename = strlen(pszFilename);
+        char  *pszFullName = (char *)malloc(cchBasePath + 1 + cchFilename + 1);
+        if (!pszFullName)
+        {
+            fprintf(stderr, "uniread: failed to allocate %d bytes\n", (int)(cchBasePath + 1 + cchFilename + 1));
+            return NULL;
+        }
+
+        memcpy(pszFullName, pszBasePath, cchBasePath);
+        pszFullName[cchBasePath] = '/';
+        memcpy(&pszFullName[cchBasePath + 1], pszFilename, cchFilename + 1);
+
+        pFile = fopen(pszFullName, "r");
+        if (!pFile)
+            fprintf(stderr, "uniread: failed to open '%s' for reading\n", pszFullName);
+        free(pszFullName);
+    }
+    g_pszCurFile = pszFilename;
+    g_iLine      = 0;
+    return pFile;
+}
+
+
+/**
+ * Wrapper around fgets that keep track of the line number.
+ *
+ * @returns See fgets.
+ * @param   pszBuf              The buffer.  See fgets for output definition.
+ * @param   cbBuf               The buffer size.
+ * @param   pFile               The file to read from.
+ */
+static char *GetLineFromFile(char *pszBuf, int cbBuf, FILE *pFile)
+{
+    g_iLine++;
+    return fgets(pszBuf, cbBuf, pFile);
+}
+
+
+/**
+ * Closes a file opened by OpenFile
+ *
+ * @param   pFile               The file to close.
+ */
+static void CloseFile(FILE *pFile)
+{
+    g_pszCurFile = NULL;
+    g_iLine = 0;
+    fclose(pFile);
+}
+
+
+/**
  * Read the UnicodeData.txt file.
  * @returns 0 on success.
  * @returns !0 on failure.
- * @param   pszFilename     The name of the file.
- */
-static int ReadUnicodeData(const char *pszFilename)
+ * @param   pszBasePath         The base path, can be NULL.
+ * @param   pszFilename         The name of the file.
+ */
+static int ReadUnicodeData(const char *pszBasePath, const char *pszFilename)
 {
     /*
      * Open input.
      */
-    FILE *pFile = fopen(pszFilename, "r");
+    FILE *pFile = OpenFile(pszBasePath, pszFilename);
     if (!pFile)
-    {
-        printf("uniread: failed to open '%s' for reading\n", pszFilename);
         return 1;
-    }
 
     /*
@@ -334 +560 @@
     char szLine[4096];
     RTUNICP i = 0;
-    while (fgets(szLine, sizeof(szLine), pFile) != NULL)
+    while (GetLineFromFile(szLine, sizeof(szLine), pFile) != NULL)
     {
         if (IsCommentOrBlankLine(szLine))
@@ -346 +572 @@
         char *pszBidiClass = NextField(&pszCurField);                     /* 4 */
         char *pszDecompositionType = NextField(&pszCurField);             /* 5 */
-        char *pszDecompositionMapping = NextField(&pszCurField);          /* 6 */
-        char *pszNumericType = NextField(&pszCurField);                   /* 7 */
-        char *pszNumericValue = NextField(&pszCurField);                  /* 8 */
+        char *pszDecompositionMapping = SplitDecompField(&pszDecompositionType);
+        char *pszNumericType = NextField(&pszCurField);                   /* 6 */
+        char *pszNumericValueD = NextField(&pszCurField);                 /* 7 */
+        char *pszNumericValueN = NextField(&pszCurField);                 /* 8 */
         char *pszBidiMirrored = NextField(&pszCurField);                  /* 9 */
         char *pszUnicode1Name = NextField(&pszCurField);                  /* 10 */
@@ -358 +585 @@
         RTUNICP CodePoint = ToNum(pszCodePoint);
         if (CodePoint >= RT_ELEMENTS(g_aCPInfo))
+        {
+            ParseError("U+05X is out of range\n", CodePoint);
             continue;
+        }
 
         /* catchup? */
@@ -365 +595 @@
         if (i != CodePoint)
         {
-            fprintf(stderr, "unitest: error: i=%d CodePoint=%u\n", i, CodePoint);
-            fclose(pFile);
+            ParseError("i=%d CodePoint=%u\n", i, CodePoint);
+            CloseFile(pFile);
             return 1;
         }
@@ -377 +607 @@
         g_aCPInfo[i].SimpleLowerCaseMapping     = ToNumDefault(pszSimpleLowerCaseMapping, CodePoint);
         g_aCPInfo[i].SimpleTitleCaseMapping     = ToNumDefault(pszSimpleTitleCaseMapping, CodePoint);
+        g_aCPInfo[i].CanonicalCombiningClass    = ToNum(pszCanonicalCombiningClass);
+        g_aCPInfo[i].pszDecompositionType       = DupStr(pszDecompositionType);
+        g_aCPInfo[i].paDecompositionMapping     = ToMapping(pszDecompositionMapping, &g_aCPInfo[i].cDecompositionMapping, 20);
         g_aCPInfo[i].pszGeneralCategory         = DupStr(pszGeneralCategory);
-        g_aCPInfo[i].pszCanonicalCombiningClass = DupStr(pszCanonicalCombiningClass);
         g_aCPInfo[i].pszBidiClass               = DupStr(pszBidiClass);
-        g_aCPInfo[i].pszDecompositionType       = DupStr(pszDecompositionType);
-        g_aCPInfo[i].pszDecompositionMapping    = DupStr(pszDecompositionMapping);
         g_aCPInfo[i].pszNumericType             = DupStr(pszNumericType);
-        g_aCPInfo[i].pszNumericValue            = DupStr(pszNumericValue);
+        g_aCPInfo[i].pszNumericValueD           = DupStr(pszNumericValueD);
+        g_aCPInfo[i].pszNumericValueN           = DupStr(pszNumericValueN);
         g_aCPInfo[i].pszBidiMirrored            = DupStr(pszBidiMirrored);
         g_aCPInfo[i].pszUnicode1Name            = DupStr(pszUnicode1Name);
@@ -389 +620 @@
         i++;
     }
+
     /* catchup? */
     while (i < RT_ELEMENTS(g_aCPInfo))
         NullEntry(i++);
-    fclose(pFile);
+    CloseFile(pFile);
 
     return 0;
+}
+
+
+/**
+ * Generates excluded data.
+ *
+ * @returns 0 on success, exit code on failure.
+ */
+static int GenerateExcludedData(void)
+{
+    /*
+     * Hangul Syllables U+AC00 to U+D7A3.
+     */
+    for (RTUNICP i = 0xac00; i <= 0xd7a3; i++)
+    {
+        g_aCPInfo[i].fNullEntry = 0;
+        g_aCPInfo[i].fInvNFD_QC = 1;
+        /** @todo generate the decomposition: http://unicode.org/reports/tr15/#Hangul
+         *         */
+    }
+
+    /** @todo
+     * CJK Ideographs Extension A (U+3400 - U+4DB5)
+     * CJK Ideographs (U+4E00 - U+9FA5)
+     * CJK Ideograph Extension B (U+20000 - U+2A6D6)
+     * CJK Ideograph Extension C (U+2A700 - U+2B734)
+     */
+
+    return 0;
+}
+
+
+
+/**
+ * Worker for ApplyProperty that handles a yes, no, maybe property value.
+ *
+ * @returns 0 (NO), 1 (YES), 2 (MAYBE).
+ * @param   ppszNextField   The field cursor, input and output.
+ */
+static int YesNoMaybePropertyValue(char **ppszNextField)
+{
+    if (!**ppszNextField)
+    {
+        ParseError("Missing Y/N/M field\n");
+        return 0;
+    }
+    char *psz = NextField(ppszNextField);
+    if (!strcmp(psz, "N"))
+        return 0;
+    if (!strcmp(psz, "Y"))
+        return 1;
+    if (!strcmp(psz, "M"))
+        return 2;
+    ParseError("Unexpected Y/N/M value: '%s'\n",  psz);
+    return 0;
+}
+
+
+/**
+ * Inverted version of YesNoMaybePropertyValue
+ *
+ * @returns 1 (NO), 0 (YES), 2 (MAYBE).
+ * @param   ppszNextField   The field cursor, input and output.
+ */
+static int YesNoMaybePropertyValueInv(char **ppszNextField)
+{
+    unsigned rc = YesNoMaybePropertyValue(ppszNextField);
+    switch (rc)
+    {
+        case 0:     return 1;
+        case 1:     return 0;
+        default:    return rc;
+    }
 }
 
@@ -404 +709 @@
  * @param   pszProperty The property name.
  */
-static void ApplyProperty(RTUNICP StartCP, const char *pszProperty)
+static void ApplyProperty(RTUNICP StartCP, const char *pszProperty, char *pszNextField)
 {
     if (StartCP >= RT_ELEMENTS(g_aCPInfo))
+    {
+        ParseError("U+%06X is out of the g_aCPInfo range.\n", StartCP);
         return;
+    }
     struct CPINFO *pCPInfo = &g_aCPInfo[StartCP];
     /* string switch */
-    if (!strcmp(pszProperty, "ASCII_Hex_Digit")) pCPInfo->fASCIIHexDigit = 1;
+         if (!strcmp(pszProperty, "ASCII_Hex_Digit")) pCPInfo->fASCIIHexDigit = 1;
+    else if (!strcmp(pszProperty, "Alphabetic")) pCPInfo->fAlphabetic = 1;
     else if (!strcmp(pszProperty, "Bidi_Control")) pCPInfo->fBidiControl = 1;
+    else if (!strcmp(pszProperty, "Case_Ignorable")) pCPInfo->fCaseIgnorable = 1;
+    else if (!strcmp(pszProperty, "Cased")) pCPInfo->fCased = 1;
+    else if (!strcmp(pszProperty, "Changes_When_Casefolded")) pCPInfo->fChangesWhenCasefolded = 1;
+    else if (!strcmp(pszProperty, "Changes_When_Casemapped")) pCPInfo->fChangesWhenCasemapped = 1;
+    else if (!strcmp(pszProperty, "Changes_When_Lowercased")) pCPInfo->fChangesWhenLowercased = 1;
+    else if (!strcmp(pszProperty, "Changes_When_Titlecased")) pCPInfo->fChangesWhenTitlecased = 1;
+    else if (!strcmp(pszProperty, "Changes_When_Uppercased")) pCPInfo->fChangesWhenUppercased = 1;
     else if (!strcmp(pszProperty, "Dash")) pCPInfo->fDash = 1;
+    else if (!strcmp(pszProperty, "Default_Ignorable_Code_Point")) pCPInfo->fDefaultIgnorableCodePoint = 1;
     else if (!strcmp(pszProperty, "Deprecated")) pCPInfo->fDeprecated = 1;
     else if (!strcmp(pszProperty, "Diacritic")) pCPInfo->fDiacritic = 1;
     else if (!strcmp(pszProperty, "Extender")) pCPInfo->fExtender = 1;
+    else if (!strcmp(pszProperty, "Grapheme_Base")) pCPInfo->fGraphemeBase = 1;
+    else if (!strcmp(pszProperty, "Grapheme_Extend")) pCPInfo->fGraphemeExtend = 1;
     else if (!strcmp(pszProperty, "Grapheme_Link")) pCPInfo->fGraphemeLink = 1;
     else if (!strcmp(pszProperty, "Hex_Digit")) pCPInfo->fHexDigit = 1;
     else if (!strcmp(pszProperty, "Hyphen")) pCPInfo->fHyphen = 1;
+    else if (!strcmp(pszProperty, "ID_Continue")) pCPInfo->fIDContinue = 1;
+    else if (!strcmp(pszProperty, "ID_Start")) pCPInfo->fIDStart = 1;
     else if (!strcmp(pszProperty, "Ideographic")) pCPInfo->fIdeographic = 1;
     else if (!strcmp(pszProperty, "IDS_Binary_Operator")) pCPInfo->fIDSBinaryOperator = 1;
@@ -424 +745 @@
     else if (!strcmp(pszProperty, "Join_Control")) pCPInfo->fJoinControl = 1;
     else if (!strcmp(pszProperty, "Logical_Order_Exception")) pCPInfo->fLogicalOrderException = 1;
+    else if (!strcmp(pszProperty, "Lowercase")) pCPInfo->fLowercase = 1;
+    else if (!strcmp(pszProperty, "Math")) pCPInfo->fMath = 1;
     else if (!strcmp(pszProperty, "Noncharacter_Code_Point")) pCPInfo->fNoncharacterCodePoint = 1;
     else if (!strcmp(pszProperty, "Other_Alphabetic")) pCPInfo->fOtherAlphabetic = 1;
@@ -433 +756 @@
     else if (!strcmp(pszProperty, "Other_Math")) pCPInfo->fOtherMath = 1;
     else if (!strcmp(pszProperty, "Other_Uppercase")) pCPInfo->fOtherUppercase = 1;
-    else if (!strcmp(pszProperty, "Alphabetic")) pCPInfo->fAlphabetic = 1;
-    else if (!strcmp(pszProperty, "Default_Ignorable_Code_Point")) pCPInfo->fDefaultIgnorableCodePoint = 1;
-    else if (!strcmp(pszProperty, "Grapheme_Base")) pCPInfo->fGraphemeBase = 1;
-    else if (!strcmp(pszProperty, "Grapheme_Extend")) pCPInfo->fGraphemeExtend = 1;
-    else if (!strcmp(pszProperty, "ID_Continue")) pCPInfo->fIDContinue = 1;
-    else if (!strcmp(pszProperty, "ID_Start")) pCPInfo->fIDStart = 1;
-    else if (!strcmp(pszProperty, "XID_Continue")) pCPInfo->fXIDContinue = 1;
-    else if (!strcmp(pszProperty, "XID_Start")) pCPInfo->fXIDStart = 1;
-    else if (!strcmp(pszProperty, "Lowercase")) pCPInfo->fLowercase = 1;
-    else if (!strcmp(pszProperty, "Math")) pCPInfo->fMath = 1;
-    else if (!strcmp(pszProperty, "Uppercase")) pCPInfo->fUppercase = 1;
     else if (!strcmp(pszProperty, "Pattern_Syntax")) pCPInfo->fPatternSyntax = 1;
     else if (!strcmp(pszProperty, "Pattern_White_Space")) pCPInfo->fPatternWhiteSpace = 1;
@@ -452 +764 @@
     else if (!strcmp(pszProperty, "Terminal_Punctuation")) pCPInfo->fTerminalPunctuation = 1;
     else if (!strcmp(pszProperty, "Unified_Ideograph")) pCPInfo->fUnifiedIdeograph = 1;
+    else if (!strcmp(pszProperty, "Uppercase")) pCPInfo->fUppercase = 1;
     else if (!strcmp(pszProperty, "Variation_Selector")) pCPInfo->fVariationSelector = 1;
     else if (!strcmp(pszProperty, "White_Space")) pCPInfo->fWhiteSpace = 1;
+    else if (!strcmp(pszProperty, "XID_Continue")) pCPInfo->fXIDContinue = 1;
+    else if (!strcmp(pszProperty, "XID_Start")) pCPInfo->fXIDStart = 1;
+    /* DerivedNormalizationProps: */
+    else if (!strcmp(pszProperty, "FC_NFKC")) return; /* ignored */
+    else if (!strcmp(pszProperty, "Full_Composition_Exclusion")) pCPInfo->fFullCompositionExclusion = 1;
+    else if (!strcmp(pszProperty, "NFC_QC"))  pCPInfo->fInvNFC_QC  = YesNoMaybePropertyValueInv(&pszNextField);
+    else if (!strcmp(pszProperty, "NFD_QC"))  pCPInfo->fInvNFD_QC  = YesNoMaybePropertyValueInv(&pszNextField);
+    else if (!strcmp(pszProperty, "NFKC_QC")) pCPInfo->fInvNFKC_QC = YesNoMaybePropertyValueInv(&pszNextField);
+    else if (!strcmp(pszProperty, "NFKD_QC")) pCPInfo->fInvNFKD_QC = YesNoMaybePropertyValueInv(&pszNextField);
+    else if (!strcmp(pszProperty, "Expands_On_NFC"))  pCPInfo->fExpandsOnNFC  = 1;
+    else if (!strcmp(pszProperty, "Expands_On_NFD"))  pCPInfo->fExpandsOnNFD  = 1;
+    else if (!strcmp(pszProperty, "Expands_On_NFKC")) pCPInfo->fExpandsOnNFKC = 1;
+    else if (!strcmp(pszProperty, "Expands_On_NFKD")) pCPInfo->fExpandsOnNFKD = 1;
+    else if (!strcmp(pszProperty, "NFKC_CF")) return; /*ignore */
+    else if (!strcmp(pszProperty, "Changes_When_NFKC_Casefolded")) return; /*ignore */
     else
-        fprintf(stderr, "uniread: Unknown property '%s'\n", pszProperty);
+    {
+        ParseError("Unknown property '%s'\n", pszProperty);
+        return;
+    }
+
+    if (pszNextField && *pszNextField)
+        ParseError("Unexpected next field: '%s'\n", pszNextField);
 }
 
@@ -467 +801 @@
  * @returns 0 on success.
  * @returns !0 on failure.
+ * @param   pszBasePath         The base path, can be NULL.
  * @param   pszFilename     The name of the file.
  */
-static int ReadProperties(const char *pszFilename)
+static int ReadProperties(const char *pszBasePath, const char *pszFilename)
 {
     /*
      * Open input.
      */
-    FILE *pFile = fopen(pszFilename, "r");
+    FILE *pFile = OpenFile(pszBasePath, pszFilename);
     if (!pFile)
-    {
-        printf("uniread: failed to open '%s' for reading\n", pszFilename);
         return 1;
-    }
 
     /*
@@ -485 +817 @@
      */
     char szLine[4096];
-    while (fgets(szLine, sizeof(szLine), pFile) != NULL)
+    while (GetLineFromFile(szLine, sizeof(szLine), pFile) != NULL)
     {
         if (IsCommentOrBlankLine(szLine))
             continue;
         char *pszCurField;
-        char *pszRange = FirstField(&pszCurField, StripLine(szLine));
+        char *pszRange    = FirstField(&pszCurField, StripLine(szLine));
         char *pszProperty = NextField(&pszCurField);
         if (!*pszProperty)
+        {
+            ParseError("no property field.\n");
             continue;
+        }
 
         RTUNICP LastCP;
@@ -501 +836 @@
 
         while (StartCP <= LastCP)
-            ApplyProperty(StartCP++, pszProperty);
-    }
-
-    fclose(pFile);
+            ApplyProperty(StartCP++, pszProperty, pszCurField);
+    }
+
+    CloseFile(pFile);
 
     return 0;
@@ -547 +882 @@
     if (pInfo->fLowercase || pInfo->fOtherLowercase)
         AppendFlag(pszFlags, "RTUNI_LOWER");
-    //if (pInfo->fNumeric)
-    //    AppendFlag(pszFlags, "RTUNI_NUMERIC");
+    //if (pInfo->???)
+    //    AppendFlag(pszFlags, "RTUNI_BSPACE");
+    if (pInfo->fInvNFD_QC != 0 || pInfo->fInvNFC_QC != 0)
+    {
+        AppendFlag(pszFlags, "RTUNI_QC_NFX");
+        if (!pInfo->paDecompositionMapping && pInfo->fInvNFD_QC)
+            fprintf(stderr, "uniread: U+%05X is QC_NFD but has no mappings.\n", pInfo->CodePoint);
+        else if (*pInfo->pszDecompositionType && pInfo->fInvNFD_QC)
+            fprintf(stderr, "uniread: U+%05X is QC_NFD but has no canonical mappings.\n", pInfo->CodePoint);
+    }
+    else if (pInfo->paDecompositionMapping && !*pInfo->pszDecompositionType)
+        fprintf(stderr, "uniread: U+%05X is not QC_NFX but has canonical mappings.\n", pInfo->CodePoint);
+
     if (!*pszFlags)
     {
@@ -558 +904 @@
 }
 
+
+/**
+ * printf wrapper for the primary output stream.
+ *
+ * @returns See vfprintf.
+ * @param   pszFormat           The vfprintf format string.
+ * @param   ...                 The format arguments.
+ */
+static int Stream1Printf(const char *pszFormat, ...)
+{
+    int     cch;
+    va_list va;
+    va_start(va, pszFormat);
+    if (!g_fQuiet)
+        cch = vfprintf(stdout, pszFormat, va);
+    else
+        cch = strlen(pszFormat);
+    va_end(va);
+    return cch;
+}
+
+
 /** the data store for stream two. */
 static char g_szStream2[10240];
-static unsigned g_offStream2 = 0;
+static unsigned volatile g_offStream2 = 0;
 
 /**
@@ -576 +944 @@
 static int Stream2Flush(void)
 {
-    fwrite(g_szStream2, 1, g_offStream2, stdout);
+    g_szStream2[g_offStream2] = '\0';
+    Stream1Printf("%s", g_szStream2);
+    Stream2Init();
     return 0;
 }
@@ -585 +955 @@
 static int Stream2Printf(const char *pszFormat, ...)
 {
+    unsigned offStream2 = g_offStream2;
     va_list va;
     va_start(va, pszFormat);
-    int cch = vsprintf(&g_szStream2[g_offStream2], pszFormat, va);
+    int cch = vsprintf(&g_szStream2[offStream2], pszFormat, va);
     va_end(va);
-    g_offStream2 += cch;
-    if (g_offStream2 >= sizeof(g_szStream2))
+    offStream2 += cch;
+    if (offStream2 >= sizeof(g_szStream2))
     {
         fprintf(stderr, "error: stream2 overflow!\n");
         exit(1);
     }
+    g_offStream2 = offStream2;
     return cch;
 }
@@ -604 +976 @@
 int PrintHeader(const char *argv0)
 {
-    /*
-     * Print file header.
-     */
-    printf("/** @file\n"
-           " *\n"
-           " * IPRT - Unicode Tables\n"
-           " *\n"
-           " *      Automatically Generated by %s (" __DATE__ " " __TIME__ ")\n"
-           " */\n\n"
-           "/*\n"
-           " * Copyright (C) 2006-2008 Sun Microsystems, Inc.\n"
-           " *\n"
-           " * This file is part of VirtualBox Open Source Edition (OSE), as\n"
-           " * available from http://www.virtualbox.org. This file is free software;\n"
-           " * you can redistribute it and/or modify it under the terms of the GNU\n"
-           " * General Public License as published by the Free Software Foundation,\n"
-           " * in version 2 as it comes in the \"COPYING\" file of the VirtualBox OSE\n"
-           " * distribution. VirtualBox OSE is distributed in the hope that it will\n"
-           " * be useful, but WITHOUT ANY WARRANTY of any kind.\n"
-           " *\n"
-           "\n"
-           "#include <iprt/uni.h>\n"
-           "\n",
-           argv0);
+    Stream1Printf("/** @file\n"
+                  " *\n"
+                  " * IPRT - Unicode Tables.\n"
+                  " *\n"
+                  " * Automatically Generated by %s (" __DATE__ " " __TIME__ ")\n"
+                  " */\n"
+                  "\n"
+                  "/*\n"
+                  " * Copyright (C) 2006-2010 Oracle Corporation\n"
+                  " *\n"
+                  " * This file is part of VirtualBox Open Source Edition (OSE), as\n"
+                  " * available from http://www.virtualbox.org. This file is free software;\n"
+                  " * you can redistribute it and/or modify it under the terms of the GNU\n"
+                  " * General Public License (GPL) as published by the Free Software\n"
+                  " * Foundation, in version 2 as it comes in the \"COPYING\" file of the\n"
+                  " * VirtualBox OSE distribution. VirtualBox OSE is distributed in the\n"
+                  " * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.\n"
+                  " *\n"
+                  " * The contents of this file may alternatively be used under the terms\n"
+                  " * of the Common Development and Distribution License Version 1.0\n"
+                  " * (CDDL) only, as it comes in the \"COPYING.CDDL\" file of the\n"
+                  " * VirtualBox OSE distribution, in which case the provisions of the\n"
+                  " * CDDL are applicable instead of those of the GPL.\n"
+                  " *\n"
+                  " * You may elect to license modified versions of this file under the\n"
+                  " * terms and conditions of either the GPL or the CDDL or both.\n"
+                  " */\n"
+                  "\n"
+                  "#include <iprt/uni.h>\n"
+                  "\n",
+                  argv0);
     return 0;
 }
@@ -650 +1029 @@
         char szFlags[256];
         unsigned iNonNull = i;
-        while (     (g_aCPInfo[iNonNull].fNullEntry || !CalcFlags(&g_aCPInfo[iNonNull], szFlags))
-               &&   iNonNull < RT_ELEMENTS(g_aCPInfo)
-               &&   iNonNull >= 256)
+        while (   iNonNull < RT_ELEMENTS(g_aCPInfo)
+               && iNonNull >= 256
+               && (g_aCPInfo[iNonNull].fNullEntry || !CalcFlags(&g_aCPInfo[iNonNull], szFlags)) )
             iNonNull++;
         if (iNonNull - i > 4096 || iNonNull == RT_ELEMENTS(g_aCPInfo))
@@ -658 +1037 @@
             if (iStart >= 0)
             {
-                printf("};\n\n");
+                Stream1Printf("};\n\n");
                 Stream2Printf("    { 0x%06x, 0x%06x, &g_afRTUniFlags0x%06x[0] },\n", iStart, i, iStart);
                 iStart = -1;
@@ -668 +1047 @@
             if (iStart < 0)
             {
-                printf("static const uint8_t g_afRTUniFlags0x%06x[] = \n"
-                       "{\n", i);
+                Stream1Printf("static const uint8_t g_afRTUniFlags0x%06x[] = \n"
+                              "{\n", i);
                 iStart = i;
             }
             CalcFlags(&g_aCPInfo[i], szFlags);
-            printf("    %50s, /* U+%06x: %s*/\n", szFlags, g_aCPInfo[i].CodePoint, g_aCPInfo[i].pszName);
+            Stream1Printf("    %50s, /* U+%06x: %s*/\n", szFlags, g_aCPInfo[i].CodePoint, g_aCPInfo[i].pszName);
             i++;
         }
@@ -679 +1058 @@
     Stream2Printf("    { ~(RTUNICP)0, ~(RTUNICP)0, NULL }\n"
                   "};\n\n\n");
-    printf("\n");
+    Stream1Printf("\n");
     return Stream2Flush();
 }
@@ -706 +1085 @@
             if (iStart >= 0)
             {
-                printf("};\n\n");
+                Stream1Printf("};\n\n");
                 Stream2Printf("    { 0x%06x, 0x%06x, &g_afRTUniUpper0x%06x[0] },\n", iStart, i, iStart);
                 iStart = -1;
@@ -716 +1095 @@
             if (iStart < 0)
             {
-                printf("static const RTUNICP g_afRTUniUpper0x%06x[] = \n"
-                       "{\n", i);
+                Stream1Printf("static const RTUNICP g_afRTUniUpper0x%06x[] = \n"
+                              "{\n", i);
                 iStart = i;
             }
-            printf("    0x%02x, /* U+%06x: %s*/\n", g_aCPInfo[i].SimpleUpperCaseMapping, g_aCPInfo[i].CodePoint, g_aCPInfo[i].pszName);
+            Stream1Printf("    0x%02x, /* U+%06x: %s*/\n", g_aCPInfo[i].SimpleUpperCaseMapping, g_aCPInfo[i].CodePoint, g_aCPInfo[i].pszName);
             i++;
         }
@@ -726 +1105 @@
     Stream2Printf("    { ~(RTUNICP)0, ~(RTUNICP)0, NULL }\n"
                   "};\n\n\n");
-    printf("\n");
+    Stream1Printf("\n");
     return Stream2Flush();
 }
@@ -753 +1132 @@
             if (iStart >= 0)
             {
-                printf("};\n\n");
+                Stream1Printf("};\n\n");
                 Stream2Printf("    { 0x%06x, 0x%06x, &g_afRTUniLower0x%06x[0] },\n", iStart, i, iStart);
                 iStart = -1;
@@ -763 +1142 @@
             if (iStart < 0)
             {
-                printf("static const RTUNICP g_afRTUniLower0x%06x[] = \n"
-                       "{\n", i);
+                Stream1Printf("static const RTUNICP g_afRTUniLower0x%06x[] = \n"
+                              "{\n", i);
                 iStart = i;
             }
-            printf("    0x%02x, /* U+%06x: %s*/\n", g_aCPInfo[i].SimpleLowerCaseMapping, g_aCPInfo[i].CodePoint, g_aCPInfo[i].pszName);
+            Stream1Printf("    0x%02x, /* U+%06x: %s*/\n",
+                          g_aCPInfo[i].SimpleLowerCaseMapping, g_aCPInfo[i].CodePoint, g_aCPInfo[i].pszName);
             i++;
         }
@@ -773 +1153 @@
     Stream2Printf("    { ~(RTUNICP)0, ~(RTUNICP)0, NULL }\n"
                   "};\n\n\n");
-    printf("\n");
+    Stream1Printf("\n");
     return Stream2Flush();
 }
@@ -785 +1165 @@
     if (argc <= 1)
     {
-        printf("usage: %s [UnicodeData.txt [DerivedCoreProperties.txt [PropList.txt]]]\n", argv[0]);
+        printf("usage: %s [-C|--dir <UCD-dir>] [UnicodeData.txt [DerivedCoreProperties.txt [PropList.txt] [DerivedNormalizationProps.txt]]]\n",
+                argv[0]);
         return 1;
     }
 
-    const char *pszUnicodeData              = "UnicodeData.txt";
-    const char *pszDerivedCoreProperties    = "DerivedCoreProperties.txt";
-    const char *pszPropList                 = "PropList.txt";
+    const char *pszBaseDir                      = NULL;
+    const char *pszUnicodeData                  = "UnicodeData.txt";
+    const char *pszDerivedCoreProperties        = "DerivedCoreProperties.txt";
+    const char *pszPropList                     = "PropList.txt";
+    const char *pszDerivedNormalizationProps    = "DerivedNormalizationProps.txt";
     int iFile = 0;
     for (int argi = 1;  argi < argc; argi++)
@@ -799 +1182 @@
             switch (iFile++)
             {
-                case 0: pszUnicodeData = argv[argi]; break;
-                case 1: pszDerivedCoreProperties = argv[argi]; break;
-                case 2: pszPropList = argv[argi]; break;
+                case 0: pszUnicodeData                  = argv[argi]; break;
+                case 1: pszDerivedCoreProperties        = argv[argi]; break;
+                case 2: pszPropList                     = argv[argi]; break;
+                case 3: pszDerivedNormalizationProps    = argv[argi]; break;
                 default:
-                    printf("uniread: syntax error at '%s': too many filenames\n", argv[argi]);
+                    fprintf(stderr, "uniread: syntax error at '%s': too many filenames\n", argv[argi]);
                     return 1;
             }
         }
+        else if (   !strcmp(argv[argi], "--dir")
+                 || !strcmp(argv[argi], "-C"))
+        {
+            if (argi + 1 >= argc)
+            {
+                fprintf(stderr, "uniread: syntax error: '%s' is missing the directory name.\n", argv[argi]);
+                return 1;
+            }
+            argi++;
+            pszBaseDir = argv[argi];
+        }
+        else if (   !strcmp(argv[argi], "-q")
+                 || !strcmp(argv[argi], "--quiet"))
+            g_fQuiet = true;
         else
         {
-            printf("uniread: syntax error at '%s': Unknown argument\n", argv[argi]);
+            fprintf(stderr, "uniread: syntax error at '%s': Unknown argument\n", argv[argi]);
             return 1;
         }
@@ -817 +1215 @@
      * Read the data.
      */
-    int rc = ReadUnicodeData(pszUnicodeData);
+    int rc = ReadUnicodeData(pszBaseDir, pszUnicodeData);
     if (rc)
         return rc;
-    rc = ReadProperties(pszPropList);
+    rc = GenerateExcludedData();
     if (rc)
         return rc;
-    rc = ReadProperties(pszDerivedCoreProperties);
+    rc = ReadProperties(pszBaseDir, pszPropList);
+    if (rc)
+        return rc;
+    rc = ReadProperties(pszBaseDir, pszDerivedCoreProperties);
+    if (rc)
+        return rc;
+    rc = ReadProperties(pszBaseDir, pszDerivedNormalizationProps);
     if (rc)
         return rc;