Changeset 67860 in vbox

Timestamp:

Jul 7, 2017 4:08:30 PM (8 years ago)

Author:

vboxsync

svn:sync-xref-src-repo-rev:

116816

Message:

isomaker: Some fixes for bugs found when testing different buffer read sizes and read ordering.

Location:

trunk/src/VBox/Runtime/common/fs

Files:

• isomaker.cpp (modified) (4 diffs)
• isomakercmd.cpp (modified) (10 diffs)
• isomakerimport.cpp (modified) (1 diff)

trunk/src/VBox/Runtime/common/fs/isomaker.cpp

@@ -5343 +5343 @@
             pER->cchDescription = sizeof(ISO9660_RRIP_DESC) - 1;
             pER->cchSource      = sizeof(ISO9660_RRIP_SRC)  - 1;
+            pER->bVersion       = ISO9660_RRIP_VER;
             memcpy(&pER->achPayload[0], RT_STR_TUPLE(ISO9660_RRIP_ID));
-            memcpy(&pER->achPayload[sizeof(ISO9660_RRIP_ID)], RT_STR_TUPLE(ISO9660_RRIP_DESC));
-            memcpy(&pER->achPayload[sizeof(ISO9660_RRIP_ID) + sizeof(ISO9660_RRIP_DESC)], RT_STR_TUPLE(ISO9660_RRIP_SRC));
+            memcpy(&pER->achPayload[sizeof(ISO9660_RRIP_ID) - 1], RT_STR_TUPLE(ISO9660_RRIP_DESC));
+            memcpy(&pER->achPayload[sizeof(ISO9660_RRIP_ID) - 1 + sizeof(ISO9660_RRIP_DESC) - 1], RT_STR_TUPLE(ISO9660_RRIP_SRC));
             pbSys += ISO9660_RRIP_ER_LEN;
             cbSys -= ISO9660_RRIP_ER_LEN;
@@ -6332 +6333 @@
     uint8_t abTmpBuf[256];
     Assert(pName->cbDirRec <= sizeof(abTmpBuf));
-    uint32_t const cbOne = rtFsIsoMakerOutFile_GenerateDirRec(pName, fUnicode, pbBuf, pFinalizedDirs);
+    uint32_t const cbOne = rtFsIsoMakerOutFile_GenerateDirRec(pName, fUnicode, abTmpBuf, pFinalizedDirs);
     Assert(cbOne == pName->cbDirRec);
     if (cbOne == pName->cbDirRecTotal)
@@ -6461 +6462 @@
         uint8_t cbSysUse  = pDirRec->cbDirRec - offSysUse;
         if (cbSysUse > 0)
-            memmove(&pDirRec->achFileId[1], &pbBuf[offSysUse], cbSysUse);
+            memmove(&pDirRec->achFileId[1], &abTmpBuf[offSysUse], cbSysUse);
         pDirRec->bFileIdLength = 1;
         cbToCopy = RT_UOFFSETOF(ISO9660DIRREC, achFileId) + 1 + cbSysUse;
@@ -6519 +6520 @@
         pDir = RTListGetFirst(&pFinalizedDirs->FinalizedDirs, RTFSISOMAKERNAMEDIR, FinalizedEntry);
         AssertReturnStmt(pDir, *pbBuf = 0xff, 1);
+        offInDir64 = offUnsigned - pDir->offDir;
     }
     /* Seek backwards: */

trunk/src/VBox/Runtime/common/fs/isomakercmd.cpp

@@ -33 +33 @@
 #include <iprt/fsisomaker.h>
 
+#include <iprt/asm.h>
 #include <iprt/assert.h>
 #include <iprt/buildconfig.h>
@@ -93 +94 @@
     RTFSISOMAKERCMD_OPT_OUTPUT_BUFFER_SIZE,
     RTFSISOMAKERCMD_OPT_RANDOM_OUTPUT_BUFFER_SIZE,
+    RTFSISOMAKERCMD_OPT_RANDOM_ORDER_VERIFICATION,
     RTFSISOMAKERCMD_OPT_NAME_SETUP,
     RTFSISOMAKERCMD_OPT_NO_JOLIET,
@@ -312 +314 @@
      * when this is enabled. */
     bool                fRandomOutputReadBufferSize;
+    /** Do output verification, but do it in random order if non-zero.  The
+     * values gives the block size to use. */
+    uint32_t            cbRandomOrderVerifciationBlock;
 
     /** @name Processing of inputs
@@ -414 +419 @@
     { "--output-buffer-size",           RTFSISOMAKERCMD_OPT_OUTPUT_BUFFER_SIZE,             RTGETOPT_REQ_UINT32  },
     { "--random-output-buffer-size",    RTFSISOMAKERCMD_OPT_RANDOM_OUTPUT_BUFFER_SIZE,      RTGETOPT_REQ_NOTHING },
+    { "--random-order-verficiation",    RTFSISOMAKERCMD_OPT_RANDOM_ORDER_VERIFICATION,      RTGETOPT_REQ_UINT32 },
     { "--name-setup",                   RTFSISOMAKERCMD_OPT_NAME_SETUP,                     RTGETOPT_REQ_STRING  },
     { "--no-joliet",                    RTFSISOMAKERCMD_OPT_NO_JOLIET,                      RTGETOPT_REQ_NOTHING },
@@ -731 +737 @@
 }
 
+static int rtFsIsoMakerCmdVerifyImageInRandomOrder(PRTFSISOMAKERCMDOPTS pOpts, RTVFSFILE hVfsSrcFile, RTVFSFILE hVfsDstFile, uint64_t cbImage)
+{
+    /*
+     * Figure the buffer (block) size and allocate a bitmap for noting down blocks we've covered.
+     */
+    int      rc;
+    size_t   cbBuf     = RT_MAX(pOpts->cbRandomOrderVerifciationBlock, 1);
+    uint64_t cBlocks64 = (cbImage + cbBuf - 1) / cbBuf;
+    if (cBlocks64 > _512M)
+        return rtFsIsoMakerCmdErrorRc(pOpts, VERR_OUT_OF_RANGE,
+                                      "verification block count too high: cBlocks=%#RX64 (cbBuf=%#zx), max 512M", cBlocks64, cbBuf);
+    uint32_t cBlocks   = (uint32_t)cBlocks64;
+    uint32_t cbBitmap  = (cBlocks + 63) / 8;
+    if (cbBitmap > _64M)
+        return rtFsIsoMakerCmdErrorRc(pOpts, VERR_OUT_OF_RANGE,
+                                      "verification bitmap too big: cbBitmap=%#RX32 (cbBuf=%#zx), max 64MB", cbBitmap, cbBuf);
+    void    *pvSrcBuf  = RTMemTmpAlloc(cbBuf);
+    void    *pvDstBuf  = RTMemTmpAlloc(cbBuf);
+    void    *pvBitmap  = RTMemTmpAllocZ(cbBitmap);
+    if (pvSrcBuf && pvDstBuf && pvBitmap)
+    {
+        /* Must set the unused bits in the top qword. */
+        for (uint32_t i = RT_ALIGN_32(cBlocks, 64) - 1; i >= cBlocks; i--)
+            ASMBitSet(pvBitmap, i);
+
+        /*
+         * Do the verification.
+         */
+        rtFsIsoMakerPrintf(pOpts, "Verifying image in random order using %zu (%#zx) byte blocks: %#zx in blocks\n",
+                           cbBuf, cbBuf, cBlocks);
+
+        rc = VINF_SUCCESS;
+        uint64_t cLeft = cBlocks;
+        while (cLeft-- > 0)
+        {
+            /*
+             * Figure out which block to check next.
+             */
+            uint32_t iBlock = RTRandU32Ex(0, cBlocks - 1);
+            if (!ASMBitTestAndSet(pvBitmap, iBlock))
+                Assert(iBlock < (int32_t)cBlocks);
+            else
+            {
+                /* try 32 other random numbers. */
+                bool     fBitSet;
+                unsigned cTries = 0;
+                do
+                {
+                    iBlock = RTRandU32Ex(0, cBlocks - 1);
+                    fBitSet = ASMBitTestAndSet(pvBitmap, iBlock);
+                } while (fBitSet && ++cTries < 32);
+                if (fBitSet)
+                {
+                    /* Look for the next clear bit after it (with wrap around). */
+                    int iHit = ASMBitNextClear(pvBitmap, cBlocks, iBlock);
+                    Assert(iHit < (int32_t)cBlocks);
+                    if (iHit < 0)
+                    {
+                        iHit = ASMBitNextClear(pvBitmap, iBlock, 0);
+                        Assert(iHit < (int32_t)cBlocks);
+                    }
+                    if (iHit >= 0)
+                    {
+                        fBitSet  = ASMBitTestAndSet(pvBitmap, iHit);
+                        if (!fBitSet)
+                            iBlock = iHit;
+                        else
+                        {
+                            rc = rtFsIsoMakerCmdErrorRc(pOpts, VERR_INTERNAL_ERROR_3,
+                                                        "Bitmap weirdness: iHit=%#x iBlock=%#x cBlocks=%#x",
+                                                        iHit, iBlock, cBlocks);
+                            break;
+                        }
+                    }
+                    else
+                    {
+                        rc = rtFsIsoMakerCmdErrorRc(pOpts, VERR_INTERNAL_ERROR_2, "Bitmap weirdness: iBlock=%#x cBlocks=%#x",
+                                                    iBlock, cBlocks);
+                        break;
+                    }
+                }
+            }
+            Assert(ASMBitTest(pvBitmap, iBlock));
+
+            /*
+             * Figure out how much and where to read (last block fun).
+             */
+            uint64_t offBlock = iBlock * (uint64_t)cbBuf;
+            size_t   cbToRead = cbBuf;
+            if (iBlock + 1 < cBlocks)
+            { /* likely */ }
+            else if (cbToRead > cbImage - offBlock)
+                cbToRead = (size_t)(cbImage - offBlock);
+            Assert(offBlock + cbToRead <= cbImage);
+
+            /*
+             * Read the blocks.
+             */
+            //RTPrintf("Reading block #%#x at %#RX64\n", iBlock, offBlock);
+            rc = RTVfsFileReadAt(hVfsDstFile, offBlock, pvDstBuf, cbToRead, NULL);
+            if (RT_SUCCESS(rc))
+            {
+                memset(pvSrcBuf, 0xdd, cbBuf);
+                rc = RTVfsFileReadAt(hVfsSrcFile, offBlock, pvSrcBuf, cbToRead, NULL);
+                if (RT_SUCCESS(rc))
+                {
+                    if (memcmp(pvDstBuf, pvSrcBuf, cbToRead) == 0)
+                        continue;
+                    rc = rtFsIsoMakerCmdErrorRc(pOpts, VERR_MISMATCH,
+                                                "Block #%#x differs! offBlock=%#RX64 cbToRead=%#zu\n"
+                                                "Virtual ISO (source):\n%.*Rhxd\nWritten ISO (destination):\n%.*Rhxd",
+                                                iBlock, offBlock, cbToRead, cbToRead, pvSrcBuf, cbToRead, pvDstBuf);
+                }
+                else
+                    rc = rtFsIsoMakerCmdErrorRc(pOpts, rc,
+                                                "Error reading %#zx bytes source (virtual ISO) block #%#x at %#RX64: %Rrc",
+                                                cbToRead, iBlock, offBlock, rc);
+            }
+            else
+                rc = rtFsIsoMakerCmdErrorRc(pOpts, rc,
+                                            "Error reading %#zx bytes destination (written ISO) block #%#x at %#RX64: %Rrc",
+                                            cbToRead, iBlock, offBlock, rc);
+            break;
+        }
+
+        if (RT_SUCCESS(rc))
+            rtFsIsoMakerPrintf(pOpts, "Written image verified fine!\n");
+    }
+    else if (!pvSrcBuf || !pvDstBuf)
+        rc = rtFsIsoMakerCmdErrorRc(pOpts, VERR_NO_TMP_MEMORY, "RTMemTmpAlloc(%#zx) failed", cbBuf);
+    else
+        rc = rtFsIsoMakerCmdErrorRc(pOpts, VERR_NO_TMP_MEMORY, "RTMemTmpAlloc(%#zx) failed", cbBuf);
+    RTMemTmpFree(pvBitmap);
+    RTMemTmpFree(pvDstBuf);
+    RTMemTmpFree(pvSrcBuf);
+    return rc;
+}
+
+
+/**
+ * Writes the image to file, no checking, no special buffering.
+ *
+ * @returns IPRT status code.
+ * @param   pOpts               The ISO maker command instance.
+ * @param   hVfsSrcFile         The source file from the ISO maker.
+ */
+static int rtFsIsoMakerCmdWriteImageRandomBufferSize(PRTFSISOMAKERCMDOPTS pOpts, RTVFSFILE hVfsSrcFile, RTVFSFILE hVfsDstFile,
+                                                     uint64_t cbImage, void **ppvBuf)
+{
+    /*
+     * Copy the virtual image bits to the destination file.
+     */
+    void    *pvBuf    = *ppvBuf;
+    uint32_t cbMaxBuf = pOpts->cbOutputReadBuffer > 0 ? pOpts->cbOutputReadBuffer : _64K;
+    uint64_t offImage = 0;
+    while (offImage < cbImage)
+    {
+        /* Figure out how much to copy this time. */
+        size_t cbToCopy = RTRandU32Ex(1, cbMaxBuf - 1);
+        if (offImage + cbToCopy < cbImage)
+        { /* likely */ }
+        else
+            cbToCopy = (size_t)(cbImage - offImage);
+        RTMemFree(pvBuf);
+        *ppvBuf = pvBuf = RTMemTmpAlloc(cbToCopy);
+        if (pvBuf)
+        {
+            /* Do the copying. */
+            int rc = RTVfsFileReadAt(hVfsSrcFile, offImage, pvBuf, cbToCopy, NULL);
+            if (RT_SUCCESS(rc))
+            {
+                rc = RTVfsFileWriteAt(hVfsDstFile, offImage, pvBuf, cbToCopy, NULL);
+                if (RT_SUCCESS(rc))
+                    offImage += cbToCopy;
+                else
+                    return rtFsIsoMakerCmdErrorRc(pOpts, rc, "Error %Rrc writing %#zx bytes at offset %#RX64 to '%s'",
+                                                  rc, cbToCopy, offImage, pOpts->pszOutFile);
+            }
+            else
+                return rtFsIsoMakerCmdErrorRc(pOpts, rc, "Error %Rrc read %#zx bytes at offset %#RX64", rc, cbToCopy, offImage);
+        }
+        else
+            return rtFsIsoMakerCmdErrorRc(pOpts, VERR_NO_TMP_MEMORY, "RTMemTmpAlloc(%#zx) failed", cbToCopy);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Writes the image to file, no checking, no special buffering.
+ *
+ * @returns IPRT status code.
+ * @param   pOpts               The ISO maker command instance.
+ * @param   hVfsSrcFile         The source file from the ISO maker.
+ */
+static int rtFsIsoMakerCmdWriteImageSimple(PRTFSISOMAKERCMDOPTS pOpts, RTVFSFILE hVfsSrcFile, RTVFSFILE hVfsDstFile,
+                                           uint64_t cbImage, void *pvBuf, size_t cbBuf)
+{
+    /*
+     * Copy the virtual image bits to the destination file.
+     */
+    uint64_t offImage = 0;
+    while (offImage < cbImage)
+    {
+        /* Figure out how much to copy this time. */
+        size_t cbToCopy = cbBuf;
+        if (offImage + cbToCopy < cbImage)
+        { /* likely */ }
+        else
+            cbToCopy = (size_t)(cbImage - offImage);
+
+        /* Do the copying. */
+        int rc = RTVfsFileReadAt(hVfsSrcFile, offImage, pvBuf, cbToCopy, NULL);
+        if (RT_SUCCESS(rc))
+        {
+            rc = RTVfsFileWriteAt(hVfsDstFile, offImage, pvBuf, cbToCopy, NULL);
+            if (RT_SUCCESS(rc))
+                offImage += cbToCopy;
+            else
+                return rtFsIsoMakerCmdErrorRc(pOpts, rc, "Error %Rrc writing %#zx bytes at offset %#RX64 to '%s'",
+                                              rc, cbToCopy, offImage, pOpts->pszOutFile);
+        }
+        else
+            return rtFsIsoMakerCmdErrorRc(pOpts, rc, "Error %Rrc read %#zx bytes at offset %#RX64", rc, cbToCopy, offImage);
+    }
+    return VINF_SUCCESS;
+}
+
 
 /**
@@ -760 +994 @@
             uint32_t        offError;
             RTERRINFOSTATIC ErrInfo;
-            rc = RTVfsChainOpenFile(pOpts->pszOutFile, RTFILE_O_WRITE | RTFILE_O_CREATE_REPLACE | RTFILE_O_DENY_WRITE,
+            rc = RTVfsChainOpenFile(pOpts->pszOutFile, RTFILE_O_READWRITE | RTFILE_O_CREATE_REPLACE | RTFILE_O_DENY_WRITE,
                                     &hVfsDstFile, &offError, RTErrInfoInitStatic(&ErrInfo));
             if (RT_SUCCESS(rc))
             {
                 /*
-                 * Copy the virtual image bits to the destination file.
+                 * Apply the desired writing method.
                  */
-                uint64_t offImage = 0;
-                while (offImage < cbImage)
-                {
-                    /* Figure out how much to copy this time. */
-                    size_t cbToCopy = cbBuf;
-                    if (pOpts->fRandomOutputReadBufferSize)
-                        cbToCopy = RTRandU32Ex(1, (uint32_t)cbBuf - 1);
-                    if (offImage + cbToCopy < cbImage)
-                    { /* likely */ }
-                    else
-                        cbToCopy = (size_t)(cbImage - offImage);
-
-                    /* Do the copying. */
-                    rc = RTVfsFileReadAt(hVfsSrcFile, offImage, pvBuf, cbToCopy, NULL);
-                    if (RT_SUCCESS(rc))
-                    {
-                        rc = RTVfsFileWriteAt(hVfsDstFile, offImage, pvBuf, cbToCopy, NULL);
-                        if (RT_SUCCESS(rc))
-                            offImage += cbToCopy;
-                        else
-                        {
-                            rc = rtFsIsoMakerCmdErrorRc(pOpts, rc, "Error %Rrc writing %#zx bytes at offset %#RX64 to '%s'",
-                                                        rc, cbToCopy, offImage, pOpts->pszOutFile);
-                            break;
-                        }
-                    }
-                    else
-                    {
-                        rc = rtFsIsoMakerCmdErrorRc(pOpts, rc, "Error %Rrc read %#zx bytes at offset %#RX64", rc, cbToCopy, offImage);
-                        break;
-                    }
-                }
+                if (!pOpts->fRandomOutputReadBufferSize)
+                    rc = rtFsIsoMakerCmdWriteImageRandomBufferSize(pOpts, hVfsSrcFile, hVfsDstFile, cbImage, &pvBuf);
+                else
+                    rc = rtFsIsoMakerCmdWriteImageSimple(pOpts, hVfsSrcFile, hVfsDstFile, cbImage, pvBuf, cbBuf);
+                RTMemTmpFree(pvBuf);
+
+                if (RT_SUCCESS(rc) && pOpts->cbRandomOrderVerifciationBlock > 0)
+                    rc = rtFsIsoMakerCmdVerifyImageInRandomOrder(pOpts, hVfsSrcFile, hVfsDstFile, cbImage);
 
                 /*
@@ -813 +1023 @@
             }
             else
+            {
+                RTMemTmpFree(pvBuf);
                 rc = rtFsIsoMakerCmdChainError(pOpts, "RTVfsChainOpenFile", pOpts->pszOutFile, rc, offError, &ErrInfo.Core);
-
-            RTMemTmpFree(pvBuf);
+            }
         }
         else
-            rc = rtFsIsoMakerCmdErrorRc(pOpts, rc, "RTMemTmpAlloc(%zu) failed: %Rrc", pOpts->cbOutputReadBuffer, rc);
+            rc = rtFsIsoMakerCmdErrorRc(pOpts, VERR_NO_TMP_MEMORY, "RTMemTmpAlloc(%zu) failed", cbBuf);
     }
     else
@@ -1409 +1620 @@
 
     pOpts->cItemsAdded += Results.cAddedFiles;
+    pOpts->cItemsAdded += Results.cAddedSymlinks;
     pOpts->cItemsAdded += Results.cAddedDirs;
     pOpts->cItemsAdded += Results.cBootCatEntries != UINT32_MAX ? Results.cBootCatEntries : 0;
@@ -1418 +1630 @@
     rtFsIsoMakerPrintf(pOpts, "    cbAddedDataBlocks:   %'14RU64 bytes\n", Results.cbAddedDataBlocks);
     rtFsIsoMakerPrintf(pOpts, "    cAddedFiles:         %'14RU32\n", Results.cAddedFiles);
+    rtFsIsoMakerPrintf(pOpts, "    cAddedSymlinks:      %'14RU32\n", Results.cAddedSymlinks);
     if (Results.cBootCatEntries == UINT32_MAX)
         rtFsIsoMakerPrintf(pOpts, "    cBootCatEntries:               none\n");
@@ -2078 +2291 @@
                 break;
 
+            case RTFSISOMAKERCMD_OPT_OUTPUT_BUFFER_SIZE:                /* --output-buffer-size {cb} */
+                pOpts->cbOutputReadBuffer = ValueUnion.u32;
+                break;
+
+            case RTFSISOMAKERCMD_OPT_RANDOM_OUTPUT_BUFFER_SIZE:         /* --random-output-buffer-size */
+                pOpts->fRandomOutputReadBufferSize = true;
+                break;
+
+            case RTFSISOMAKERCMD_OPT_RANDOM_ORDER_VERIFICATION:         /* --random-order-verficiation {cb} */
+                pOpts->cbRandomOrderVerifciationBlock = ValueUnion.u32;
+                break;
+
             case RTFSISOMAKERCMD_OPT_IMPORT_ISO:
                 rc = rtFsIsoMakerCmdOptImportIso(pOpts, ValueUnion.psz);

trunk/src/VBox/Runtime/common/fs/isomakerimport.cpp

@@ -2496 +2496 @@
     pResults->cbAddedDataBlocks = 0;
     pResults->cAddedFiles       = 0;
+    pResults->cAddedSymlinks    = 0;
     pResults->cBootCatEntries   = UINT32_MAX;
     pResults->cbSysArea         = 0;