Changeset 2900 in kBuild

Timestamp:

Sep 9, 2016 2:42:06 PM (9 years ago)

Author:

bird

Message:

output optimizations

Location:

trunk/src

Files:

• kWorker/kWorker.c (modified) (14 diffs)
• kmk/kmkbuiltin/cat.c (modified) (4 diffs)
• kmk/kmkbuiltin/printf.c (modified) (9 diffs)
• kmk/main.c (modified) (1 diff)
• lib/Makefile.kmk (modified) (1 diff)
• lib/maybe_con_fwrite.c (added)
• lib/maybe_con_write.c (added)
• lib/nt/ntstuff.h (modified) (1 diff)

trunk/src/kWorker/kWorker.c

@@ -69 +69 @@
  * they are included. */
 #define WITH_HASH_MD5_CACHE
+
+/** @def WITH_CONSOLE_OUTPUT_BUFFERING
+ * Enables buffering of all console output as well as removal of annoying
+ * source file echo by cl.exe. */
+#define WITH_CONSOLE_OUTPUT_BUFFERING
 
 
@@ -276 +281 @@
 typedef KFSWCACHEDFILE *PKFSWCACHEDFILE;
 
-
 #ifdef WITH_HASH_MD5_CACHE
+
 /** Pointer to a MD5 hash instance. */
 typedef struct KWHASHMD5 *PKWHASHMD5;
@@ -307 +312 @@
 /** Magic value for KWHASHMD5::uMagic (Les McCann). */
 # define KWHASHMD5_MAGIC    KUPTR_C(0x19350923)
+
 #endif /* WITH_HASH_MD5_CACHE */
-
-
+#ifdef WITH_TEMP_MEMORY_FILES
 
 typedef struct KWFSTEMPFILESEG *PKWFSTEMPFILESEG;
@@ -345 +350 @@
 } KWFSTEMPFILE;
 
+#endif /* WITH_TEMP_MEMORY_FILES */
+#ifdef WITH_CONSOLE_OUTPUT_BUFFERING
+
+/**
+ * Console line buffer.
+ */
+typedef struct KWCONSOLEOUTPUTLINE
+{
+    /** The main output handle. */
+    HANDLE              hOutput;
+    /** Our backup handle. */
+    HANDLE              hBackup;
+    /** Set if this is a console handle. */
+    KBOOL               fIsConsole;
+    /** Amount of pending console output in wchar_t's. */
+    KU32                cwcBuf;
+    /** The allocated buffer size.   */
+    KU32                cwcBufAlloc;
+    /** Pending console output. */
+    wchar_t            *pwcBuf;
+} KWCONSOLEOUTPUTLINE;
+/** Pointer to a console line buffer. */
+typedef KWCONSOLEOUTPUTLINE *PKWCONSOLEOUTPUTLINE;
+
+/**
+ * Combined console buffer of complete lines.
+ */
+typedef struct KWCONSOLEOUTPUT
+{
+    /** The console output handle.
+     * INVALID_HANDLE_VALUE if we haven't got a console and shouldn't be doing any
+     * combined output buffering. */
+    HANDLE              hOutput;
+    /** The current code page for the console. */
+    KU32                uCodepage;
+    /** Amount of pending console output in wchar_t's. */
+    KU32                cwcBuf;
+    /** Number of times we've flushed it in any way (for cl.exe hack). */
+    KU32                cFlushes;
+    /** Pending console output. */
+    wchar_t             wszBuf[8192];
+} KWCONSOLEOUTPUT;
+/** Pointer to a combined console buffer. */
+typedef KWCONSOLEOUTPUT *PKWCONSOLEOUTPUT;
+
+#endif /* WITH_CONSOLE_OUTPUT_BUFFERING */
 
 /** Handle type.   */
@@ -554 +605 @@
     } LastHashRead;
 #endif
+
+#ifdef WITH_CONSOLE_OUTPUT_BUFFERING
+    /** Standard output (and whatever else) line buffer. */
+    KWCONSOLEOUTPUTLINE StdOut;
+    /** Standard error line buffer. */
+    KWCONSOLEOUTPUTLINE StdErr;
+    /** Combined buffer of completed lines. */
+    KWCONSOLEOUTPUT     Combined;
+#endif
 } KWSANDBOX;
 
@@ -640 +700 @@
 static int kwLdrModuleResolveAndLookup(const char *pszName, PKWMODULE pExe, PKWMODULE pImporter, PKWMODULE *ppMod);
 static KBOOL kwSandboxHandleTableEnter(PKWSANDBOX pSandbox, PKWHANDLE pHandle);
+#ifdef WITH_CONSOLE_OUTPUT_BUFFERING
+static void kwSandboxConsoleWriteA(PKWSANDBOX pSandbox, PKWCONSOLEOUTPUTLINE pLineBuf, const char *pchBuffer, KU32 cchToWrite);
+#endif
 
 
@@ -5013 +5076 @@
     }
 
+#ifdef WITH_CONSOLE_OUTPUT_BUFFERING
+    /*
+     * Check for stdout and stderr.
+     */
+    if (   g_Sandbox.StdErr.hOutput == hFile
+        || g_Sandbox.StdOut.hOutput == hFile)
+    {
+        PKWCONSOLEOUTPUTLINE pLineBuf = g_Sandbox.StdErr.hOutput == hFile ? &g_Sandbox.StdErr : &g_Sandbox.StdOut;
+        if (pLineBuf->fIsConsole)
+        {
+            kwSandboxConsoleWriteA(&g_Sandbox, pLineBuf, (const char *)pvBuffer, cbToWrite);
+            KWFS_LOG(("WriteFile(console) -> TRUE\n", hFile));
+            return TRUE;
+        }
+    }
+#endif
+
     KWFS_LOG(("WriteFile(%p)\n", hFile));
     return WriteFile(hFile, pvBuffer, cbToWrite, pcbActuallyWritten, pOverlapped);
@@ -5499 +5579 @@
 }
 #endif /* WITH_TEMP_MEMORY_FILES */
+
+
+
+#ifdef WITH_CONSOLE_OUTPUT_BUFFERING
+
+/*
+ *
+ * Console output buffering.
+ * Console output buffering.
+ * Console output buffering.
+ *
+ */
+
+
+/**
+ * Write a wide char string to the console.
+ *
+ * @param   pSandbox            The sandbox which output buffer to flush.
+ */
+static void kwSandboxConsoleWriteIt(PKWSANDBOX pSandbox, wchar_t const *pwcBuf, KU32 cwcToWrite)
+{
+    if (cwcToWrite > 0)
+    {
+        DWORD cwcWritten = 0;
+        if (WriteConsoleW(pSandbox->Combined.hOutput, pwcBuf, cwcToWrite, &cwcWritten, NULL))
+        {
+            if (cwcWritten == cwcToWrite)
+            { /* likely */ }
+            else
+            {
+                DWORD off = 0;
+                do
+                {
+                    off += cwcWritten;
+                    cwcWritten = 0;
+                }
+                while (   off < cwcToWrite
+                       && WriteConsoleW(pSandbox->Combined.hOutput, &pwcBuf[off], cwcToWrite - off, &cwcWritten, NULL));
+                kHlpAssert(off == cwcWritten);
+            }
+        }
+        else
+            kHlpAssertFailed();
+        pSandbox->Combined.cFlushes++;
+    }
+}
+
+
+/**
+ * Flushes the combined console output buffer.
+ *
+ * @param   pSandbox            The sandbox which output buffer to flush.
+ */
+static void kwSandboxConsoleFlushCombined(PKWSANDBOX pSandbox)
+{
+    if (pSandbox->Combined.cwcBuf > 0)
+    {
+        kwSandboxConsoleWriteIt(pSandbox, pSandbox->Combined.wszBuf, pSandbox->Combined.cwcBuf);
+        pSandbox->Combined.cwcBuf = 0;
+    }
+}
+
+
+/**
+ * For handling combined buffer overflow cases line by line.
+ *
+ * @param   pSandbox            The sandbox.
+ * @param   pwcBuf              What to add to the combined buffer.  Usually a
+ *                              line, unless we're really low on buffer space.
+ * @param   cwcBuf              The length of what to add.
+ * @param   fBrokenLine         Whether this is a broken line.
+ */
+static void kwSandboxConsoleAddToCombined(PKWSANDBOX pSandbox, wchar_t const *pwcBuf, KU32 cwcBuf, KBOOL fBrokenLine)
+{
+    if (fBrokenLine)
+        kwSandboxConsoleFlushCombined(pSandbox);
+    if (pSandbox->Combined.cwcBuf + cwcBuf > K_ELEMENTS(pSandbox->Combined.wszBuf))
+    {
+        kwSandboxConsoleFlushCombined(pSandbox);
+        kwSandboxConsoleWriteIt(pSandbox, pwcBuf, cwcBuf);
+    }
+    else
+    {
+        memcpy(&pSandbox->Combined.wszBuf[pSandbox->Combined.cwcBuf], pwcBuf, cwcBuf * sizeof(wchar_t));
+        pSandbox->Combined.cwcBuf += cwcBuf;
+    }
+}
+
+
+/**
+ * Called to final flush a line buffer via the combined buffer (if applicable).
+ *
+ * @param   pSandbox            The sandbox.
+ * @param   pLineBuf            The line buffer.
+ */
+static void kwSandboxConsoleFinalFlushLineBuf(PKWSANDBOX pSandbox, PKWCONSOLEOUTPUTLINE pLineBuf)
+{
+    if (pLineBuf->cwcBuf > 0)
+    {
+        if (pLineBuf->fIsConsole)
+        {
+            if (pLineBuf->cwcBuf < pLineBuf->cwcBufAlloc)
+            {
+                pLineBuf->pwcBuf[pLineBuf->cwcBuf++] = '\n';
+                kwSandboxConsoleAddToCombined(pSandbox, pLineBuf->pwcBuf, pLineBuf->cwcBuf, K_FALSE /*fBrokenLine*/);
+            }
+            else
+            {
+                kwSandboxConsoleAddToCombined(pSandbox, pLineBuf->pwcBuf, pLineBuf->cwcBuf, K_TRUE /*fBrokenLine*/);
+                kwSandboxConsoleAddToCombined(pSandbox, L"\n", 1, K_TRUE /*fBrokenLine*/);
+            }
+            pLineBuf->cwcBuf = 0;
+        }
+        else
+        {
+            kHlpAssertFailed();
+        }
+    }
+}
+
+
+/**
+ * Called at the end of sandboxed execution to flush both stream buffers.
+ *
+ * @param   pSandbox            The sandbox.
+ */
+static void kwSandboxConsoleFlushAll(PKWSANDBOX pSandbox)
+{
+    /*
+     * First do the cl.exe source file supression trick, if applicable.
+     */
+    if (   pSandbox->Combined.cwcBuf >= 3
+        && pSandbox->StdOut.cwcBuf == 0
+        && pSandbox->StdErr.cwcBuf == 0
+        && pSandbox->Combined.cFlushes == 0
+        && pSandbox->pTool->u.Sandboxed.enmHint == KWTOOLHINT_VISUAL_CPP_CL)
+    {
+        KI32    off = pSandbox->Combined.cwcBuf - 1;
+        if (pSandbox->Combined.wszBuf[off] == '\n')
+        {
+            KBOOL fOk = K_TRUE;
+            while (off-- > 0)
+            {
+                wchar_t const wc = pSandbox->Combined.wszBuf[off];
+                if (iswalnum(wc) || wc == '.' || wc == ' ' || wc == '_' || wc == '-')
+                { /* likely */ }
+                else
+                {
+                    fOk = K_FALSE;
+                    break;
+                }
+            }
+            if (fOk)
+            {
+                pSandbox->Combined.cwcBuf = 0;
+                return;
+            }
+        }
+    }
+
+    /*
+     * Flush the two line buffer, the the combined buffer.
+     */
+    kwSandboxConsoleFinalFlushLineBuf(pSandbox, &pSandbox->StdErr);
+    kwSandboxConsoleFinalFlushLineBuf(pSandbox, &pSandbox->StdOut);
+    kwSandboxConsoleFlushCombined(pSandbox);
+}
+
+
+/**
+ * Writes a string to the given output stream.
+ *
+ * @param   pSandbox            The sandbox.
+ * @param   pLineBuf            The line buffer for the output stream.
+ * @param   pwcBuffer           The buffer to write.
+ * @param   cwcToWrite          The number of wchar_t's in the buffer.
+ */
+static void kwSandboxConsoleWriteW(PKWSANDBOX pSandbox, PKWCONSOLEOUTPUTLINE pLineBuf, wchar_t const *pwcBuffer, KU32 cwcToWrite)
+{
+    if (cwcToWrite > 0)
+    {
+        /*
+         * First, find the start of the last incomplete line so we can figure
+         * out how much line buffering we need to do.
+         */
+        KU32 cchLastIncompleteLine;
+        KU32 offLastIncompleteLine = cwcToWrite;
+        while (   offLastIncompleteLine > 0
+               && pwcBuffer[offLastIncompleteLine - 1] != '\n')
+            offLastIncompleteLine--;
+        cchLastIncompleteLine = cwcToWrite - offLastIncompleteLine;
+
+        /* Was there anything to line buffer? */
+        if (offLastIncompleteLine < cwcToWrite)
+        {
+            /* Need to grow the line buffer? */
+            KU32 cwcNeeded = offLastIncompleteLine != 0 ? offLastIncompleteLine : cchLastIncompleteLine + pLineBuf->cwcBuf;
+            if (cwcNeeded > pLineBuf->cwcBufAlloc)
+            {
+                void *pvNew;
+                KU32  cwcNew = !pLineBuf->cwcBufAlloc ? 1024 : pLineBuf->cwcBufAlloc * 2;
+                while (cwcNew < cwcNeeded)
+                    cwcNew *= 2;
+                pvNew = kHlpRealloc(pLineBuf->pwcBuf, cwcNew * sizeof(wchar_t));
+                if (pvNew)
+                {
+                    pLineBuf->pwcBuf = (wchar_t *)pvNew;
+                    pLineBuf->cwcBufAlloc = cwcNew;
+                }
+                else
+                {
+                    pvNew = kHlpRealloc(pLineBuf->pwcBuf, cwcNeeded * sizeof(wchar_t));
+                    if (pvNew)
+                    {
+                        pLineBuf->pwcBuf = (wchar_t *)pvNew;
+                        pLineBuf->cwcBufAlloc = cwcNeeded;
+                    }
+                    else
+                    {
+                        /* This isn't perfect, but it will have to do for now. */
+                        if (pLineBuf->cwcBuf > 0)
+                        {
+                            kwSandboxConsoleAddToCombined(pSandbox, pLineBuf->pwcBuf, pLineBuf->cwcBuf, K_TRUE /*fBrokenLine*/);
+                            pLineBuf->cwcBuf = 0;
+                        }
+                        kwSandboxConsoleAddToCombined(pSandbox, pwcBuffer, cwcToWrite, K_TRUE /*fBrokenLine*/);
+                        return;
+                    }
+                }
+            }
+
+            /*
+             * Handle the case where we only add to the line buffer.
+             */
+            if (offLastIncompleteLine == 0)
+            {
+                memcpy(&pLineBuf->pwcBuf[pLineBuf->cwcBuf], pwcBuffer, cwcToWrite * sizeof(wchar_t));
+                pLineBuf->cwcBuf += cwcToWrite;
+                return;
+            }
+        }
+
+        /*
+         * If there is sufficient combined buffer to handle this request, this are rather simple.
+         */
+        if (pLineBuf->cwcBuf + cchLastIncompleteLine <= K_ELEMENTS(pSandbox->Combined.wszBuf))
+        {
+            if (pLineBuf->cwcBuf > 0)
+            {
+                memcpy(&pSandbox->Combined.wszBuf[pSandbox->Combined.cwcBuf],
+                       pLineBuf->pwcBuf, pLineBuf->cwcBuf * sizeof(wchar_t));
+                pSandbox->Combined.cwcBuf += pLineBuf->cwcBuf;
+                pLineBuf->cwcBuf = 0;
+            }
+
+            memcpy(&pSandbox->Combined.wszBuf[pSandbox->Combined.cwcBuf],
+                   pwcBuffer, offLastIncompleteLine * sizeof(wchar_t));
+            pSandbox->Combined.cwcBuf += offLastIncompleteLine;
+        }
+        else
+        {
+            /*
+             * Do line-by-line processing of the input, flusing the combined buffer
+             * when it becomes necessary.  We may have to write lines
+             */
+            KU32 off = 0;
+            KU32 offNextLine = 0;
+
+            /* If there is buffered chars, we handle the first line outside the
+               main loop.  We must try our best outputting it as a complete line. */
+            if (pLineBuf->cwcBuf > 0)
+            {
+                while (offNextLine < cwcToWrite && pwcBuffer[offNextLine] != '\n')
+                    offNextLine++;
+                offNextLine++;
+                kHlpAssert(offNextLine <= offLastIncompleteLine);
+
+                if (pLineBuf->cwcBuf + offNextLine + pSandbox->Combined.cwcBuf <= K_ELEMENTS(pSandbox->Combined.wszBuf))
+                {
+                    memcpy(&pSandbox->Combined.wszBuf[pSandbox->Combined.cwcBuf],
+                           pLineBuf->pwcBuf, pLineBuf->cwcBuf * sizeof(wchar_t));
+                    pSandbox->Combined.cwcBuf += pLineBuf->cwcBuf;
+                    pLineBuf->cwcBuf = 0;
+
+                    memcpy(&pSandbox->Combined.wszBuf[pSandbox->Combined.cwcBuf], pwcBuffer, offNextLine * sizeof(wchar_t));
+                    pSandbox->Combined.cwcBuf += offNextLine;
+                }
+                else
+                {
+                    KU32 cwcLeft = pLineBuf->cwcBufAlloc - pLineBuf->cwcBuf;
+                    if (cwcLeft > 0)
+                    {
+                        KU32 cwcCopy = K_MIN(cwcLeft, offNextLine);
+                        memcpy(&pLineBuf->pwcBuf[pLineBuf->cwcBuf], pwcBuffer, cwcCopy * sizeof(wchar_t));
+                        pLineBuf->cwcBuf += cwcCopy;
+                        off += cwcCopy;
+                    }
+                    if (pLineBuf->cwcBuf > 0)
+                    {
+                        kwSandboxConsoleAddToCombined(pSandbox, pLineBuf->pwcBuf, pLineBuf->cwcBuf, K_TRUE /*fBrokenLine*/);
+                        pLineBuf->cwcBuf = 0;
+                    }
+                    if (off < offNextLine)
+                        kwSandboxConsoleAddToCombined(pSandbox, &pwcBuffer[off], offNextLine - off, K_TRUE /*fBrokenLine*/);
+                }
+                off = offNextLine;
+            }
+
+            /* Deal with the remaining lines */
+            while (off < offLastIncompleteLine)
+            {
+                while (offNextLine < offLastIncompleteLine && pwcBuffer[offNextLine] != '\n')
+                    offNextLine++;
+                offNextLine++;
+                kHlpAssert(offNextLine <= offLastIncompleteLine);
+                kwSandboxConsoleAddToCombined(pSandbox, &pwcBuffer[off], offNextLine - off, K_FALSE /*fBrokenLine*/);
+                off = offNextLine;
+            }
+        }
+
+        /*
+         * Buffer any remaining incomplete line chars.
+         */
+        if (offLastIncompleteLine < cwcToWrite)
+        {
+            memcpy(&pLineBuf->pwcBuf, &pwcBuffer[offLastIncompleteLine], cchLastIncompleteLine * sizeof(wchar_t));
+            pLineBuf->cwcBuf = cchLastIncompleteLine;
+        }
+    }
+}
+
+
+/**
+ * Worker for WriteConsoleA and WriteFile.
+ *
+ * @param   pSandbox            The sandbox.
+ * @param   pLineBuf            The line buffer.
+ * @param   pchBuffer           What to write.
+ * @param   cchToWrite          How much to write.
+ */
+static void kwSandboxConsoleWriteA(PKWSANDBOX pSandbox, PKWCONSOLEOUTPUTLINE pLineBuf, const char *pchBuffer, KU32 cchToWrite)
+{
+    /*
+     * Convert it to wide char and use the 'W' to do the work.
+     */
+    int         cwcRet;
+    KU32        cwcBuf = cchToWrite * 2 + 1;
+    wchar_t    *pwcBufFree = NULL;
+    wchar_t    *pwcBuf;
+
+    if (cwcBuf <= 4096)
+        pwcBuf = alloca(cwcBuf * sizeof(wchar_t));
+    else
+        pwcBuf = pwcBufFree = kHlpAlloc(cwcBuf * sizeof(wchar_t));
+
+    cwcRet = MultiByteToWideChar(pSandbox->Combined.uCodepage, 0/*dwFlags*/, pchBuffer, cchToWrite, pwcBuf, cwcBuf);
+    if (cwcRet > 0)
+         kwSandboxConsoleWriteW(pSandbox, pLineBuf, pwcBuf, cwcRet);
+    else
+    {
+        DWORD cchWritten;
+        kHlpAssertFailed();
+
+        /* Flush the line buffer and combined buffer before calling WriteConsoleA. */
+        if (pLineBuf->cwcBuf > 0)
+        {
+            kwSandboxConsoleAddToCombined(pSandbox, pLineBuf->pwcBuf, pLineBuf->cwcBuf, K_TRUE /*fBroken*/);
+            pLineBuf->cwcBuf = 0;
+        }
+        kwSandboxConsoleFlushCombined(pSandbox);
+
+        if (WriteConsoleA(pLineBuf->hBackup, pchBuffer, cchToWrite, &cchWritten, NULL /*pvReserved*/))
+        {
+            if (cchWritten >= cchToWrite)
+            { /* likely */ }
+            else
+            {
+                KU32 off = 0;
+                do
+                {
+                    off += cchWritten;
+                    cchWritten = 0;
+                } while (   off < cchToWrite
+                         && WriteConsoleA(pLineBuf->hBackup, &pchBuffer[off], cchToWrite - off, &cchWritten, NULL));
+            }
+        }
+    }
+
+    if (pwcBufFree)
+        kHlpFree(pwcBufFree);
+}
+
+
+/** Kernel32 - WriteConsoleA  */
+BOOL WINAPI kwSandbox_Kernel32_WriteConsoleA(HANDLE hConOutput, CONST VOID *pvBuffer, DWORD cbToWrite, PDWORD pcbWritten,
+                                             PVOID pvReserved)
+{
+    BOOL                    fRc;
+    PKWCONSOLEOUTPUTLINE    pLineBuf;
+
+    if (hConOutput == g_Sandbox.StdErr.hOutput)
+        pLineBuf = &g_Sandbox.StdErr;
+    else
+        pLineBuf = &g_Sandbox.StdOut;
+    if (pLineBuf->fIsConsole)
+    {
+        kwSandboxConsoleWriteA(&g_Sandbox, pLineBuf, (char const *)pvBuffer, cbToWrite);
+
+        KWFS_LOG(("WriteConsoleA: %p, %p LB %#x (%*.*s), %p, %p -> TRUE [cached]\n",
+                  hConOutput, pvBuffer, cbToWrite, cbToWrite, cbToWrite, pvBuffer, pcbWritten, pvReserved));
+        if (pcbWritten)
+            *pcbWritten = cbToWrite;
+        fRc = TRUE;
+    }
+    else
+    {
+        fRc = WriteConsoleA(hConOutput, pvBuffer, cbToWrite, pcbWritten, pvReserved);
+        KWFS_LOG(("WriteConsoleA: %p, %p LB %#x (%*.*s), %p, %p -> %d !fallback!\n",
+                  hConOutput, pvBuffer, cbToWrite, cbToWrite, cbToWrite, pvBuffer, pcbWritten, pvReserved, fRc));
+    }
+    return fRc;
+}
+
+
+/** Kernel32 - WriteConsoleW  */
+BOOL WINAPI kwSandbox_Kernel32_WriteConsoleW(HANDLE hConOutput, CONST VOID *pvBuffer, DWORD cwcToWrite, PDWORD pcwcWritten,
+                                             PVOID pvReserved)
+{
+    BOOL                    fRc;
+    PKWCONSOLEOUTPUTLINE    pLineBuf;
+
+    if (hConOutput == g_Sandbox.StdErr.hOutput)
+        pLineBuf = &g_Sandbox.StdErr;
+    else
+        pLineBuf = &g_Sandbox.StdOut;
+    if (pLineBuf->fIsConsole)
+    {
+        kwSandboxConsoleWriteW(&g_Sandbox, pLineBuf, (wchar_t const *)pvBuffer, cwcToWrite);
+
+        KWFS_LOG(("WriteConsoleW: %p, %p LB %#x (%*.*ls), %p, %p -> TRUE [cached]\n",
+                  hConOutput, pvBuffer, cwcToWrite, cwcToWrite, cwcToWrite, pvBuffer, pcwcWritten, pvReserved));
+        if (pcwcWritten)
+            *pcwcWritten = cwcToWrite;
+        fRc = TRUE;
+    }
+    else
+    {
+        fRc = WriteConsoleW(hConOutput, pvBuffer, cwcToWrite, pcwcWritten, pvReserved);
+        KWFS_LOG(("WriteConsoleW: %p, %p LB %#x (%*.*ls), %p, %p -> %d !fallback!\n",
+                  hConOutput, pvBuffer, cwcToWrite, cwcToWrite, cwcToWrite, pvBuffer, pcwcWritten, pvReserved, fRc));
+    }
+    return fRc;
+}
+
+#endif /* WITH_CONSOLE_OUTPUT_BUFFERING */
 
 
@@ -6127 +6662 @@
 #endif
 
+    { TUPLE("WriteConsoleA"),               NULL,       (KUPTR)kwSandbox_Kernel32_WriteConsoleA },
+    { TUPLE("WriteConsoleW"),               NULL,       (KUPTR)kwSandbox_Kernel32_WriteConsoleW },
+
     { TUPLE("VirtualAlloc"),                NULL,       (KUPTR)kwSandbox_Kernel32_VirtualAlloc },
     { TUPLE("VirtualFree"),                 NULL,       (KUPTR)kwSandbox_Kernel32_VirtualFree },
@@ -6241 +6779 @@
     { TUPLE("SetConsoleCtrlHandler"),       NULL,       (KUPTR)kwSandbox_Kernel32_SetConsoleCtrlHandler },
 
+    { TUPLE("WriteConsoleA"),               NULL,       (KUPTR)kwSandbox_Kernel32_WriteConsoleA },
+    { TUPLE("WriteConsoleW"),               NULL,       (KUPTR)kwSandbox_Kernel32_WriteConsoleW },
+
 #ifdef WITH_HASH_MD5_CACHE
     { TUPLE("CryptCreateHash"),             NULL,       (KUPTR)kwSandbox_Advapi32_CryptCreateHash },
@@ -6553 +7094 @@
     pSandbox->pgmptr        = (char *)pTool->pszPath;
     pSandbox->wpgmptr       = (wchar_t *)pTool->pwszPath;
+#ifdef WITH_CONSOLE_OUTPUT_BUFFERING
+    pSandbox->StdOut.cwcBuf = 0;
+    pSandbox->StdErr.cwcBuf = 0;
+    pSandbox->Combined.cwcBuf = 0;
+    pSandbox->Combined.cFlushes = 0;
+#endif
     pSandbox->cArgs         = cArgs;
     pSandbox->papszArgs     = (char **)papszArgs;
@@ -6885 +7432 @@
             rcExit = 42 + 4;
 
-        /* Clean up essential bits only, the rest is done after we've replied to kmk. */
+        /*
+         * Flush and clean up the essential bits only, postpone whatever we
+         * can till after we've replied to kmk.
+         */
+#ifdef WITH_CONSOLE_OUTPUT_BUFFERING
+        kwSandboxConsoleFlushAll(&g_Sandbox);
+#endif
         kwSandboxCleanup(&g_Sandbox);
     }
@@ -7376 +7929 @@
     PVOID           pvVecXcptHandler = AddVectoredExceptionHandler(0 /*called last*/, kwSandboxVecXcptEmulateChained);
 #endif
+#ifdef WITH_CONSOLE_OUTPUT_BUFFERING
+    HANDLE                          hCurProc       = GetCurrentProcess();
+    PPEB                            pPeb           = kwSandboxGetProcessEnvironmentBlock();
+    PMY_RTL_USER_PROCESS_PARAMETERS pProcessParams = (PMY_RTL_USER_PROCESS_PARAMETERS)pPeb->ProcessParameters;
+#endif
 
     /*
@@ -7399 +7957 @@
     if (pszTmp && *pszTmp != '\0')
         kFsCacheSetupCustomRevisionForTree(g_pFsCache, kFsCacheLookupA(g_pFsCache, pszTmp, &enmIgnored));
+
+#ifdef WITH_CONSOLE_OUTPUT_BUFFERING
+    /*
+     * Get and duplicate the console handles.
+     */
+    g_Sandbox.StdOut.hOutput = pProcessParams->StandardOutput;
+    if (!DuplicateHandle(hCurProc, pProcessParams->StandardOutput, hCurProc, &g_Sandbox.StdOut.hBackup,
+                         GENERIC_WRITE, FALSE /*fInherit*/, DUPLICATE_SAME_ACCESS))
+        kHlpAssertFailedStmt(g_Sandbox.StdOut.hBackup = pProcessParams->StandardOutput);
+    g_Sandbox.StdOut.fIsConsole = GetFileType(g_Sandbox.StdOut.hOutput) == FILE_TYPE_CHAR;
+
+    g_Sandbox.StdErr.hOutput = pProcessParams->StandardError;
+    if (!DuplicateHandle(hCurProc, pProcessParams->StandardError, hCurProc, &g_Sandbox.StdErr.hBackup,
+                         GENERIC_WRITE, FALSE /*fInherit*/, DUPLICATE_SAME_ACCESS))
+        kHlpAssertFailedStmt(g_Sandbox.StdErr.hBackup = pProcessParams->StandardError);
+    g_Sandbox.StdErr.fIsConsole = GetFileType(g_Sandbox.StdErr.hOutput) == FILE_TYPE_CHAR;
+
+    if (g_Sandbox.StdErr.fIsConsole)
+    {
+        g_Sandbox.Combined.hOutput   = g_Sandbox.StdErr.hBackup;
+        g_Sandbox.Combined.uCodepage = GetConsoleCP();
+    }
+    else if (g_Sandbox.StdOut.fIsConsole)
+    {
+        g_Sandbox.Combined.hOutput   = g_Sandbox.StdOut.hBackup;
+        g_Sandbox.Combined.uCodepage = GetConsoleCP();
+    }
+    else
+    {
+        g_Sandbox.Combined.hOutput   = INVALID_HANDLE_VALUE;
+        g_Sandbox.Combined.uCodepage = CP_ACP;
+    }
+#endif
+
 
     /*

trunk/src/kmk/kmkbuiltin/cat.c

@@ -81 +81 @@
 
 
+#ifdef KBUILD_OS_WINDOWS
+/* This is a trick to seriuosly speed up console output windows. */
+# undef write
+# define write maybe_con_write
+extern ssize_t maybe_con_write(int, void const *, size_t);
+#endif
+
+
 int bflag, eflag, nflag, sflag, tflag, vflag;
 /*int rval;*/
@@ -120 +128 @@
 #ifdef kmk_builtin_cat /* kmk did this already. */
         setlocale(LC_CTYPE, "");
+#else
+        fflush(stdout);
 #endif
 
@@ -288 +298 @@
 raw_cat(int rfd)
 {
-        int off, wfd;
+        int off, wfd = fileno(stdout);
         ssize_t nr, nw;
         static size_t bsize;
@@ -298 +308 @@
                 if (fstat(wfd, &sbuf))
                         return err(1, "%s", filename);
-#ifdef _MSC_VER
-                bsize = 1024;
+#ifdef KBUILD_OS_WINDOWS
+                bsize = 16384;
 #else
                 bsize = MAX(sbuf.st_blksize, 1024);

trunk/src/kmk/kmkbuiltin/printf.c

@@ -76 +76 @@
 #include "../kmkbuiltin.h"
 
+#ifdef KBUILD_OS_WINDOWS
+/* This is a trick to speed up console output on windows. */
+# undef fwrite
+# define fwrite maybe_con_fwrite
+extern size_t maybe_con_fwrite(void const *, size_t, size_t, FILE *);
+#endif
+
 
 #ifdef __GNUC__
@@ -113 +120 @@
 static int       usage(FILE *);
 
+static int      flush_buffer(void);
 static void     b_count(int);
 static void     b_output(int);
@@ -126 +134 @@
 #include "../../bin/sh/bltin/bltin.h"
 #endif /* SHELL */
+
+/* Buffer the output because windows doesn't do line buffering of stdout. */
+static char     g_achBuf[256];
+static size_t   g_cchBuf;
 
 #define PF(f, func) { \
@@ -156 +168 @@
         int ch;
 
-        /* kmk: reset getopt and set progname */
+        /* kmk: reset getopt, set progname and reset buffer. */
         g_progname = argv[0];
         opterr = 1;
@@ -225 +237 @@
         rval = 0;
         gargv = NULL;
+        g_cchBuf = 0;
 
         format = *argv;
@@ -272 +285 @@
                         ch = *fmt;
                         if (!ch) {
+                                flush_buffer();
                                 warnx("missing format character");
                                 return (1);
@@ -354 +368 @@
                         }
                         default:
+                                flush_buffer();
                                 warnx("%s: invalid directive", start);
                                 return 1;
@@ -360 +375 @@
                         *fmt = nextch;
                         /* escape if a \c was encountered */
-                        if (rval & 0x100)
+                        if (rval & 0x100) {
+                                flush_buffer();
                                 return rval & ~0x100;
+                        }
                 }
         } while (gargv != argv && *gargv);
 
+        flush_buffer();
         return rval;
 }
+
 
 /* helper functions for conv_escape_str */
@@ -408 +427 @@
         }
 #endif
-        return putchar(ch);
+        /* Buffered output. */
+        if (g_cchBuf + 1 < sizeof(g_achBuf)) {
+                g_achBuf[g_cchBuf++] = ch;
+        } else {
+                int rc = flush_buffer();
+                g_achBuf[g_cchBuf++] = ch;
+                if (rc)
+                        return -1;
+        }
+        return 0;
 }
 
 static int wrap_printf(const char * fmt, ...)
 {
-        int rc;
+        ssize_t cchRet;
         va_list va;
-
+        char *pszTmp;
+
+        va_start(va, fmt);
+        cchRet = vasprintf(&pszTmp, fmt, va);
+        va_end(va);
+        if (cchRet >= 0) {
 #ifndef kmk_builtin_printf
-        if (g_o) {
-                char *str;
-
-                va_start(va, fmt);
-                rc = vasprintf(&str, fmt, va);
-                va_end(va);
-                if (rc >= 0) {
-                        g_o = variable_buffer_output(g_o, str, rc);
-                        free(str);
-                }
-                return rc;
-        }
-#endif
-
-        va_start(va, fmt);
-        rc = vprintf(fmt, va);
-        va_end(va);
-        return rc;
-}
+                if (g_o) {
+                        g_o = variable_buffer_output(g_o, pszTmp, cchRet);
+                } else
+#endif
+                {
+                        if (cchRet + g_cchBuf <= sizeof(g_achBuf)) {
+                                /* We've got space in the buffer. */
+                                memcpy(&g_achBuf[g_cchBuf], pszTmp, cchRet);
+                                g_cchBuf += cchRet;
+                        } else {
+                                /* Try write out complete lines. */
+                                const char *pszLeft = pszTmp;
+                                ssize_t     cchLeft = cchRet;
+
+                                while (cchLeft > 0) {
+                                        const char *pchNewLine = strchr(pszLeft, '\n');
+                                        ssize_t     cchLine    = pchNewLine ? pchNewLine - pszLeft + 1 : cchLeft;
+                                        if (g_cchBuf + cchLine <= sizeof(g_achBuf)) {
+                                                memcpy(&g_achBuf[g_cchBuf], pszLeft, cchLine);
+                                                g_cchBuf += cchLine;
+                                        } else {
+                                                if (flush_buffer() < 0) {
+                                                        return -1;
+                                                }
+                                                if (fwrite(pszLeft, cchLine, 1, stdout) < 1) {
+                                                        return -1;
+                                                }
+                                        }
+                                        pszLeft += cchLine;
+                                        cchLeft -= cchLine;
+                                }
+                        }
+                }
+                free(pszTmp);
+        }
+        return (int)cchRet;
+}
+
+/**
+ * Flushes the g_abBuf/g_cchBuf.
+ */
+static int flush_buffer(void)
+{
+    if (g_cchBuf > 0) {
+                ssize_t cchToWrite = g_cchBuf;
+                ssize_t cchWritten = fwrite(g_achBuf, 1, g_cchBuf, stdout);
+                g_cchBuf = 0;
+                if (cchWritten >= cchToWrite) {
+                        /* likely */
+                } else {
+                        ssize_t off = cchWritten;
+                        if (cchWritten >= 0) {
+                                off = cchWritten;
+                        } else if (errno == EINTR) {
+                                cchWritten = 0;
+                        } else {
+                                return -1;
+                        }
+
+                        while (off < cchToWrite) {
+                                cchWritten = fwrite(&g_achBuf[off], 1, cchToWrite - off, stdout);
+                                if (cchWritten > 0) {
+                                        off += cchWritten;
+                                } else if (errno == EINTR) {
+                                        /* nothing */
+                                } else {
+                                        return -1;
+                                }
+                        }
+                }
+    }
+    return 0;
+}
+
 
 

trunk/src/kmk/main.c

@@ -1380 +1380 @@
   char *windows32_path = NULL;
 
-#ifndef ELECTRIC_HEAP /* Drop this because it prevents JIT debugging. */
+# ifndef ELECTRIC_HEAP /* Drop this because it prevents JIT debugging. */
   SetUnhandledExceptionFilter(handle_runtime_exceptions);
-#endif /* !ELECTRIC_HEAP */
+# endif /* !ELECTRIC_HEAP */
 
   /* start off assuming we have no shell */

trunk/src/lib/Makefile.kmk

@@ -39 +39 @@
         crc32.c \
         md5.c \
+        maybe_con_write.c \
+        maybe_con_fwrite.c \
         kbuild_version.c
 kUtil_SOURCES.win = \

trunk/src/lib/nt/ntstuff.h

@@ -74 +74 @@
 } MY_STRING;
 typedef MY_STRING MY_ANSI_STRING;
+
+typedef struct MY_CURDIR
+{
+    UNICODE_STRING      DosPath;
+    HANDLE              Handle;
+} MY_CURDIR;
+typedef MY_CURDIR *PMY_CURDIR;
+
+typedef struct MY_RTL_DRIVE_LETTER_CURDIR
+{
+    USHORT              Flags;
+    USHORT              Length;
+    ULONG               TimeStamp;
+    MY_ANSI_STRING      DosPath;
+} MY_RTL_DRIVE_LETTER_CURDIR;
+typedef MY_RTL_DRIVE_LETTER_CURDIR *PRTL_DRIVE_LETTER_CURDIR;
+
+typedef struct MY_RTL_USER_PROCESS_PARAMETERS
+{
+    ULONG               MaximumLength;
+    ULONG               Length;
+    ULONG               Flags;
+    ULONG               DebugFlags;
+    HANDLE              ConsoleHandle;
+    ULONG               ConsoleFlags;
+    HANDLE              StandardInput;
+    HANDLE              StandardOutput;
+    HANDLE              StandardError;
+    MY_CURDIR           CurrentDirectory;
+    MY_UNICODE_STRING   DllPath;
+    MY_UNICODE_STRING   ImagePathName;
+    MY_UNICODE_STRING   CommandLine;
+    PWSTR               Environment;
+    ULONG               StartingX;
+    ULONG               StartingY;
+    ULONG               CountX;
+    ULONG               CountY;
+    ULONG               CountCharsX;
+    ULONG               CountCharsY;
+    ULONG               FillAttribute;
+    ULONG               WindowFlags;
+    ULONG               ShowWindowFlags;
+    MY_UNICODE_STRING   WindowTitle;
+    MY_UNICODE_STRING   DesktopInfo;
+    MY_UNICODE_STRING   ShellInfo;
+    MY_UNICODE_STRING   RuntimeInfo;
+    MY_RTL_DRIVE_LETTER_CURDIR CurrentDirectories[0x20];
+    SIZE_T              EnvironmentSize;        /* >= Vista+ */
+    SIZE_T              EnvironmentVersion;     /* >= Windows 7. */
+    PVOID               PackageDependencyData;  /* >= Windows 8 or Windows 8.1. */
+    ULONG               ProcessGroupId;         /* >= Windows 8 or Windows 8.1. */
+} MY_RTL_USER_PROCESS_PARAMETERS;
+typedef MY_RTL_USER_PROCESS_PARAMETERS *PMY_RTL_USER_PROCESS_PARAMETERS;
 
 typedef struct MY_OBJECT_ATTRIBUTES