Changeset 33289 in vbox

Timestamp:

Oct 21, 2010 10:00:15 AM (14 years ago)

Author:

vboxsync

Message:

Runtime;Main-OVF-Import: added online creation of SHA1 sums; preread/calc is done in a second worker thread; reading is cached; directly read out of an ova file; started to make reading fully streaming aware

Location:

trunk

Files:

• include/iprt/manifest.h (modified) (3 diffs)
• include/iprt/tar.h (modified) (2 diffs)
• src/VBox/Main/ApplianceImpl.cpp (modified) (5 diffs)
• src/VBox/Main/ApplianceImplExport.cpp (modified) (7 diffs)
• src/VBox/Main/ApplianceImplIO.cpp (modified) (20 diffs)
• src/VBox/Main/ApplianceImplImport.cpp (modified) (19 diffs)
• src/VBox/Main/include/ApplianceImpl.h (modified) (4 diffs)
• src/VBox/Main/include/ApplianceImplPrivate.h (modified) (4 diffs)
• src/VBox/Runtime/common/checksum/manifest.cpp (modified) (13 diffs)
• src/VBox/Runtime/common/misc/tar.cpp (modified) (18 diffs)

trunk/include/iprt/manifest.h

@@ -44 +44 @@
 {
     /** The filename. */
-    char *pszTestFile;
+    const char *pszTestFile;
     /** The SHA1 digest of the file. */
-    char *pszTestDigest;
+    const char *pszTestDigest;
 } RTMANIFESTTEST;
 /** Pointer to the input structure. */
@@ -106 +106 @@
 
 /**
+ * Verify the given SHA1 digests against the entries in the manifest file in
+ * memory.
+ *
+ * @returns iprt status code.
+ *
+ * @param   pvBuf                Pointer to memory buffer of the manifest file.
+ * @param   cbSize               Size of the memory buffer.
+ * @param   paTests              Array of file names and digests.
+ * @param   cTest                Number of entries in paTests.
+ * @param   piFailed             A index to paTests in the
+ *                               VERR_MANIFEST_DIGEST_MISMATCH error case
+ *                               (optional).
+ */
+RTR3DECL(int) RTManifestVerifyFilesBuf(void *pvBuf, size_t cbSize, PRTMANIFESTTEST paTests, size_t cTests, size_t *piFailed);
+
+/**
  * Creates a manifest file in memory for a set of files. The manifest file
  * contains SHA1 sums of every provided file and could be used to verify the
@@ -114 +130 @@
  * @param   ppvBuf               Pointer to resulting memory buffer.
  * @param   pcbSize              Pointer for the size of the memory buffer.
- * @param   papszFileNames       Array of file names.
- * @param   papszFileDigests     Array of file digests.
- * @param   cFiles               Number of entries in papszFileNames and papszFileDigests.
+ * @param   paFiles              Array of file names and digests.
+ * @param   cFiles               Number of entries in paFiles.
  */
-RTR3DECL(int) RTManifestWriteFilesBuf(void **ppvBuf, size_t *pcbSize, const char * const *papszFileNames, const char * const *papszFileDigests, size_t cFiles);
+RTR3DECL(int) RTManifestWriteFilesBuf(void **ppvBuf, size_t *pcbSize, PRTMANIFESTTEST paFiles, size_t cFiles);
 
 /** @} */

trunk/include/iprt/tar.h

@@ -64 +64 @@
  * @param   fMode          Open flags, i.e a combination of the RTFILE_O_* defines.
  *                         The ACCESS, ACTION and DENY flags are mandatory!
- */
-RTR3DECL(int) RTTarOpen(PRTTAR phTar, const char* pszTarname, uint32_t fMode);
+ * @param   fStream        Open the file in stream mode. Within this mode no
+ *                         seeking is allowed. Use this together with
+ *                         RTTarFileCurrent, RTTarFileOpenCurrent,
+ *                         RTTarFileSeekNextFile and the read method to
+ *                         sequential read a tar file. Currently ignored with
+ *                         RTFILE_O_WRITE.
+ */
+RTR3DECL(int) RTTarOpen(PRTTAR phTar, const char* pszTarname, uint32_t fMode, bool fStream);
 
 /**
@@ -380 +386 @@
 RTR3DECL(int) RTTarCreate(const char *pszTarFile, const char * const *papszFiles, size_t cFiles, PFNRTPROGRESS pfnProgressCallback, void *pvUser);
 
+/******************************************************************************
+ *   Streaming Functions                                                      *
+ ******************************************************************************/
+
+/**
+ * Return the filename where RTTar currently stays at.
+ *
+ * @returns IPRT status code.
+ *
+ * @param   hTar           Handle to the RTTAR interface.
+ * @param   ppszFilename   On success the filename.
+ */
+RTR3DECL(int) RTTarCurrentFile(RTTAR hTar, char **ppszFilename);
+
+/**
+ * Jumps to the next file from the current RTTar position.
+ *
+ * @returns IPRT status code.
+ *
+ * @param   hTar           Handle to the RTTAR interface.
+ */
+RTR3DECL(int) RTTarSeekNextFile(RTTAR hTar);
+
+/**
+ * Opens the file where RTTar currently stays at.
+ *
+ * @returns IPRT status code.
+ *
+ * @param   hTar           Handle to the RTTAR interface.
+ * @param   phFile         Where to store the handle to the opened file.
+ * @param   ppszFilename   On success the filename.
+ * @param   fOpen          Open flags, i.e a combination of the RTFILE_O_* defines.
+ *                         The ACCESS, ACTION flags are mandatory! Currently
+ *                         only RTFILE_O_OPEN | RTFILE_O_READ is supported.
+ */
+RTR3DECL(int) RTTarFileOpenCurrentFile(RTTAR hTar, PRTTARFILE phFile, char **ppszFilename, uint32_t fOpen);
+
+
 /** @} */
 

trunk/src/VBox/Main/ApplianceImpl.cpp

@@ -19 +19 @@
 #include <iprt/path.h>
 #include <iprt/cpp/utils.h>
-#include <iprt/tar.h>
 
 #include <VBox/com/array.h>
@@ -633 +632 @@
 
 /**
+ * Called from Appliance::importImpl() and Appliance::writeImpl() to set up a
+ * progress object with the proper weights and maximum progress values.
+ *
+ * @param pProgress
+ * @param bstrDescription
+ * @param mode
+ * @return
+ */
+HRESULT Appliance::setUpProgress(ComObjPtr<Progress> &pProgress,
+                                 const Bstr &bstrDescription,
+                                 SetUpProgressMode mode)
+{
+    HRESULT rc;
+
+    /* Create the progress object */
+    pProgress.createObject();
+
+    // compute the disks weight (this sets ulTotalDisksMB and cDisks in the instance data)
+    disksWeight();
+
+    m->ulWeightForManifestOperation = 0;
+
+    ULONG cOperations;
+    ULONG ulTotalOperationsWeight;
+
+    cOperations =   1               // one for XML setup
+                  + m->cDisks;      // plus one per disk
+    if (m->ulTotalDisksMB)
+    {
+        m->ulWeightForXmlOperation = (ULONG)((double)m->ulTotalDisksMB * 1 / 100);    // use 1% of the progress for the XML
+        ulTotalOperationsWeight = m->ulTotalDisksMB + m->ulWeightForXmlOperation;
+    }
+    else
+    {
+        // no disks to export:
+        m->ulWeightForXmlOperation = 1;
+        ulTotalOperationsWeight = 1;
+    }
+
+    switch (mode)
+    {
+        case ImportFile:
+        {
+            break;
+        }
+        case WriteFile:
+        {
+            // assume that creating the manifest will take .1% of the time it takes to export the disks
+            if (m->fManifest)
+            {
+                ++cOperations;          // another one for creating the manifest
+
+                m->ulWeightForManifestOperation = (ULONG)((double)m->ulTotalDisksMB * .1 / 100);    // use .5% of the progress for the manifest
+                ulTotalOperationsWeight += m->ulWeightForManifestOperation;
+            }
+            break;
+        }
+        case ImportS3:
+        {
+            cOperations += 1 + 1;     // another one for the manifest file & another one for the import
+            ulTotalOperationsWeight = m->ulTotalDisksMB;
+            if (!m->ulTotalDisksMB)
+                // no disks to export:
+                ulTotalOperationsWeight = 1;
+
+            ULONG ulImportWeight = (ULONG)((double)ulTotalOperationsWeight * 50  / 100);  // use 50% for import
+            ulTotalOperationsWeight += ulImportWeight;
+
+            m->ulWeightForXmlOperation = ulImportWeight; /* save for using later */
+
+            ULONG ulInitWeight = (ULONG)((double)ulTotalOperationsWeight * 0.1  / 100);  // use 0.1% for init
+            ulTotalOperationsWeight += ulInitWeight;
+            break;
+        }
+        case WriteS3:
+        {
+            cOperations += 1 + 1;     // another one for the mf & another one for temporary creation
+
+            if (m->ulTotalDisksMB)
+            {
+                m->ulWeightForXmlOperation = (ULONG)((double)m->ulTotalDisksMB * 1  / 100);    // use 1% of the progress for OVF file upload (we didn't know the size at this point)
+                ulTotalOperationsWeight = m->ulTotalDisksMB + m->ulWeightForXmlOperation;
+            }
+            else
+            {
+                // no disks to export:
+                ulTotalOperationsWeight = 1;
+                m->ulWeightForXmlOperation = 1;
+            }
+            ULONG ulOVFCreationWeight = (ULONG)((double)ulTotalOperationsWeight * 50.0 / 100.0); /* Use 50% for the creation of the OVF & the disks */
+            ulTotalOperationsWeight += ulOVFCreationWeight;
+            break;
+        }
+    }
+
+    Log(("Setting up progress object: ulTotalMB = %d, cDisks = %d, => cOperations = %d, ulTotalOperationsWeight = %d, m->ulWeightForXmlOperation = %d\n",
+         m->ulTotalDisksMB, m->cDisks, cOperations, ulTotalOperationsWeight, m->ulWeightForXmlOperation));
+
+    rc = pProgress->init(mVirtualBox, static_cast<IAppliance*>(this),
+                         bstrDescription.raw(),
+                         TRUE /* aCancelable */,
+                         cOperations, // ULONG cOperations,
+                         ulTotalOperationsWeight, // ULONG ulTotalOperationsWeight,
+                         bstrDescription.raw(), // CBSTR bstrFirstOperationDescription,
+                         m->ulWeightForXmlOperation); // ULONG ulFirstOperationWeight,
+    return rc;
+}
+
+/**
  * Called from the import and export background threads to synchronize the second
  * background disk thread's progress object with the current progress object so
@@ -746 +854 @@
 }
 
-/**
- * Called from Appliance::importImpl() and Appliance::writeImpl() to set up a
- * progress object with the proper weights and maximum progress values.
- *
- * @param pProgress
- * @param bstrDescription
- * @param mode
- * @return
- */
-HRESULT Appliance::setUpProgress(const LocationInfo &locInfo,
-                                 ComObjPtr<Progress> &pProgress,
-                                 const Bstr &bstrDescription,
-                                 SetUpProgressMode mode)
-{
-    HRESULT rc;
-
-    /* Create the progress object */
-    pProgress.createObject();
-
-    // compute the disks weight (this sets ulTotalDisksMB and cDisks in the instance data)
-    disksWeight();
-
-    m->ulWeightForManifestOperation = 0;
-
-    ULONG cOperations;
-    ULONG ulTotalOperationsWeight;
-
-    cOperations =   1               // one for XML setup
-                  + m->cDisks;      // plus one per disk
-    if (m->ulTotalDisksMB)
-    {
-        m->ulWeightForXmlOperation = (ULONG)((double)m->ulTotalDisksMB * 1 / 100);    // use 1% of the progress for the XML
-        ulTotalOperationsWeight = m->ulTotalDisksMB + m->ulWeightForXmlOperation;
-    }
-    else
-    {
-        // no disks to export:
-        m->ulWeightForXmlOperation = 1;
-        ulTotalOperationsWeight = 1;
-    }
-
-    bool fOVA = locInfo.strPath.endsWith(".ova", Utf8Str::CaseInsensitive);
-    switch (mode)
-    {
-        case ImportFileNoManifest:
-        {
-            if (fOVA)
-            {
-                // Another operation for packing
-                ++cOperations;
-
-                // assume that packing the files into the archive has the same weight than creating all files in the ovf exporting step
-                ulTotalOperationsWeight += m->ulTotalDisksMB;
-            }
-            break;
-        }
-        case ImportFileWithManifest:
-        {
-            ++cOperations;          // another one for creating the manifest
-
-            // assume that creating the manifest will take 10% of the time it takes to export the disks
-            m->ulWeightForManifestOperation = m->ulTotalDisksMB / 10;
-            ulTotalOperationsWeight += m->ulWeightForManifestOperation;
-            if (fOVA)
-            {
-                // Another operation for packing
-                ++cOperations;
-
-                // assume that packing the files into the archive has the same weight than creating all files in the ovf exporting step
-                ulTotalOperationsWeight += m->ulTotalDisksMB;
-            }
-            break;
-        }
-        case WriteFile:
-        {
-            // assume that creating the manifest will take .1% of the time it takes to export the disks
-            if (m->fManifest)
-            {
-                ++cOperations;          // another one for creating the manifest
-
-                m->ulWeightForManifestOperation = (ULONG)((double)m->ulTotalDisksMB * .1 / 100);    // use .5% of the progress for the manifest
-                ulTotalOperationsWeight += m->ulWeightForManifestOperation;
-            }
-            break;
-        }
-        case ImportS3:
-        {
-            cOperations += 1 + 1;     // another one for the manifest file & another one for the import
-            ulTotalOperationsWeight = m->ulTotalDisksMB;
-            if (!m->ulTotalDisksMB)
-                // no disks to export:
-                ulTotalOperationsWeight = 1;
-
-            ULONG ulImportWeight = (ULONG)((double)ulTotalOperationsWeight * 50  / 100);  // use 50% for import
-            ulTotalOperationsWeight += ulImportWeight;
-
-            m->ulWeightForXmlOperation = ulImportWeight; /* save for using later */
-
-            ULONG ulInitWeight = (ULONG)((double)ulTotalOperationsWeight * 0.1  / 100);  // use 0.1% for init
-            ulTotalOperationsWeight += ulInitWeight;
-            break;
-        }
-        case WriteS3:
-        {
-            cOperations += 1 + 1;     // another one for the mf & another one for temporary creation
-
-            if (m->ulTotalDisksMB)
-            {
-                m->ulWeightForXmlOperation = (ULONG)((double)m->ulTotalDisksMB * 1  / 100);    // use 1% of the progress for OVF file upload (we didn't know the size at this point)
-                ulTotalOperationsWeight = m->ulTotalDisksMB + m->ulWeightForXmlOperation;
-            }
-            else
-            {
-                // no disks to export:
-                ulTotalOperationsWeight = 1;
-                m->ulWeightForXmlOperation = 1;
-            }
-            ULONG ulOVFCreationWeight = (ULONG)((double)ulTotalOperationsWeight * 50.0 / 100.0); /* Use 50% for the creation of the OVF & the disks */
-            ulTotalOperationsWeight += ulOVFCreationWeight;
-            break;
-        }
-    }
-
-    Log(("Setting up progress object: ulTotalMB = %d, cDisks = %d, => cOperations = %d, ulTotalOperationsWeight = %d, m->ulWeightForXmlOperation = %d\n",
-         m->ulTotalDisksMB, m->cDisks, cOperations, ulTotalOperationsWeight, m->ulWeightForXmlOperation));
-
-    rc = pProgress->init(mVirtualBox, static_cast<IAppliance*>(this),
-                         bstrDescription.raw(),
-                         TRUE /* aCancelable */,
-                         cOperations, // ULONG cOperations,
-                         ulTotalOperationsWeight, // ULONG ulTotalOperationsWeight,
-                         bstrDescription.raw(), // CBSTR bstrFirstOperationDescription,
-                         m->ulWeightForXmlOperation); // ULONG ulFirstOperationWeight,
-    return rc;
-}
-
 void Appliance::parseURI(Utf8Str strUri, LocationInfo &locInfo) const
 {
@@ -947 +919 @@
 }
 
-Utf8Str Appliance::manifestFileName(const Utf8Str& aPath) const
-{
-    Utf8Str strTmpPath = aPath;
-    /* Get the name part */
-    char *pszMfName = RTStrDup(RTPathFilename(strTmpPath.c_str()));
-    /* Strip any extensions */
-    RTPathStripExt(pszMfName);
-    /* Path without the filename */
-    strTmpPath.stripFilename();
-    /* Format the manifest path */
-    Utf8StrFmt strMfFile("%s/%s.mf", strTmpPath.c_str(), pszMfName);
-    RTStrFree(pszMfName);
-    return strMfFile;
-}
-
 /**
  *
@@ -1005 +962 @@
         case TaskOVF::Read:
             if (task->locInfo.storageType == VFSType_File)
-                taskrc = pAppliance->readFS(task->locInfo, task->pProgress);
+                taskrc = pAppliance->readFS(task.get());
             else if (task->locInfo.storageType == VFSType_S3)
                 taskrc = pAppliance->readS3(task.get());

trunk/src/VBox/Main/ApplianceImplExport.cpp

@@ -613 +613 @@
 ////////////////////////////////////////////////////////////////////////////////
 
+/*******************************************************************************
+ * Export stuff
+ ******************************************************************************/
+
 /**
  * Implementation for writing out the OVF to disk. This starts a new thread which will call
@@ -636 +640 @@
     try
     {
-        rc = setUpProgress(aLocInfo,
-                           aProgress,
+        rc = setUpProgress(aProgress,
                            BstrFmt(tr("Export appliance '%s'"), aLocInfo.strPath.c_str()),
                            (aLocInfo.storageType == VFSType_File) ? WriteFile : WriteS3);
@@ -1658 +1661 @@
 
     RTTAR tar;
-    int vrc = RTTarOpen(&tar, pTask->locInfo.strPath.c_str(), RTFILE_O_CREATE | RTFILE_O_WRITE | RTFILE_O_DENY_ALL);
+    int vrc = RTTarOpen(&tar, pTask->locInfo.strPath.c_str(), RTFILE_O_CREATE | RTFILE_O_WRITE | RTFILE_O_DENY_ALL, false);
     if (RT_FAILURE(vrc))
         return setError(VBOX_E_FILE_ERROR,
@@ -1719 +1722 @@
     HRESULT rc = S_OK;
 
-    typedef pair<Utf8Str, Utf8Str> STRPAIR;
     list<STRPAIR> fileList;
     try
@@ -1837 +1839 @@
         {
             // Create & write the manifest file
-            Utf8Str strMfFilePath = manifestFileName(pTask->locInfo.strPath.c_str());
+            Utf8Str strMfFilePath = Utf8Str(pTask->locInfo.strPath).stripExt().append(".mf");
             Utf8Str strMfFileName = Utf8Str(strMfFilePath)
                 .stripPath();
             pTask->pProgress->SetNextOperation(BstrFmt(tr("Creating manifest file '%s'"), strMfFileName.c_str()).raw(),
                                                m->ulWeightForManifestOperation);     // operation's weight, as set up with the IProgress originally);
-            const char** ppManifestFiles = (const char**)RTMemAlloc(sizeof(char*) * fileList.size());
-            const char** ppManifestDigests = (const char**)RTMemAlloc(sizeof(char*) * fileList.size());
+            PRTMANIFESTTEST paManifestFiles = (PRTMANIFESTTEST)RTMemAlloc(sizeof(RTMANIFESTTEST) * fileList.size());
             size_t i = 0;
             list<STRPAIR>::const_iterator it1;
@@ -1850 +1851 @@
                  ++it1, ++i)
             {
-                ppManifestFiles[i] = (*it1).first.c_str();
-                ppManifestDigests[i] = (*it1).second.c_str();
+                paManifestFiles[i].pszTestFile   = (*it1).first.c_str();
+                paManifestFiles[i].pszTestDigest = (*it1).second.c_str();
             }
             void *pvBuf;
             size_t cbSize;
-            vrc = RTManifestWriteFilesBuf(&pvBuf, &cbSize, ppManifestFiles, ppManifestDigests, fileList.size());
-            RTMemFree(ppManifestFiles);
-            RTMemFree(ppManifestDigests);
+            vrc = RTManifestWriteFilesBuf(&pvBuf, &cbSize, paManifestFiles, fileList.size());
+            RTMemFree(paManifestFiles);
             if (RT_FAILURE(vrc))
                 throw setError(VBOX_E_FILE_ERROR,
@@ -1975 +1975 @@
         if (m->fManifest)
         {
-            Utf8Str strMfFile = manifestFileName(strTmpOvf);
+            Utf8Str strMfFile = Utf8Str(strTmpOvf).stripExt().append(".mf");
             filesList.push_back(pair<Utf8Str, ULONG>(strMfFile , m->ulWeightForXmlOperation)); /* Use 1% of the total for the manifest file upload */
         }

trunk/src/VBox/Main/ApplianceImplIO.cpp

@@ -31 +31 @@
 #include <iprt/semaphore.h>
 #include <iprt/stream.h>
+#include <iprt/circbuf.h>
 #include <VBox/VBoxHDD.h>
 
@@ -57 +58 @@
     /** Completion callback. */
     PFNVDCOMPLETED pfnCompleted;
+    /** Our own storage handle. */
+    PSHA1STORAGE pSha1Storage;
     /** Storage handle for the next callback in chain. */
     void *pvStorage;
@@ -83 +86 @@
     /** SHA1 calculation context. */
     RTSHA1CONTEXT ctx;
+    /* Circular buffer in read mode. */
+    PRTCIRCBUF pCircBuf;
+    /* Are we reached end of file. */
+    volatile bool fEOF;
+//    uint64_t calls;
+//    uint64_t waits;
 } SHA1STORAGEINTERNAL, *PSHA1STORAGEINTERNAL;
 
@@ -92 +101 @@
 #define STATUS_CALC UINT32_C(1)
 #define STATUS_END  UINT32_C(3)
+#define STATUS_READ UINT32_C(4)
 
 /* Enable for getting some flow history. */
@@ -227 +237 @@
     AssertPtrReturn(pvStorage, VERR_INVALID_POINTER);
 
-    DEBUG_PRINT_FLOW();
+//    DEBUG_PRINT_FLOW();
 
     PRTFILESTORAGEINTERNAL pInt = (PRTFILESTORAGEINTERNAL)pvStorage;
@@ -257 +267 @@
     AssertPtrReturn(ppInt, VERR_INVALID_POINTER);
     AssertPtrNullReturn(pfnCompleted, VERR_INVALID_PARAMETER);
+//    AssertReturn(!(fOpen & RTFILE_O_READWRITE), VERR_INVALID_PARAMETER);
 
     RTTAR tar = (RTTAR)pvUser;
@@ -268 +279 @@
     pInt->pfnCompleted = pfnCompleted;
 
-    int rc = RTTarFileOpen(tar, &pInt->file, RTPathFilename(pszLocation), fOpen);
+    int rc = VINF_SUCCESS;
+
+    if (   fOpen & RTFILE_O_READ
+        && !(fOpen & RTFILE_O_WRITE))
+    {
+        /* Read only is a little bit more complicated than writing, cause we
+         * need streaming functionality. First try to open the file on the
+         * current file position. If this is the file the caller requested, we
+         * are fine. If not seek to the next file in the stream and check
+         * again. This is repeated until EOF of the OVA. */
+        /*
+         *
+         *
+         *  TODO: recheck this with more VDMKs (or what else) in an test OVA.
+         *
+         *
+         */
+        bool fFound = false;
+        for(;;)
+        {
+            char *pszFilename = 0;
+            rc = RTTarCurrentFile(tar, &pszFilename);
+            if (RT_SUCCESS(rc))
+            {
+                fFound = !strcmp(pszFilename, RTPathFilename(pszLocation));
+                RTStrFree(pszFilename);
+                if (fFound)
+                    break;
+                else
+                {
+                    rc = RTTarSeekNextFile(tar);
+                    if (RT_FAILURE(rc))
+                        break;
+                }
+            }else
+                break;
+        }
+        if (fFound)
+            rc = RTTarFileOpenCurrentFile(tar, &pInt->file, 0, fOpen);
+    }
+    else
+        rc = RTTarFileOpen(tar, &pInt->file, RTPathFilename(pszLocation), fOpen);
 
     if (RT_FAILURE(rc))
@@ -425 +477 @@
         /* What should we do next? */
         uint32_t u32Status = ASMAtomicReadU32(&pInt->u32Status);
+//        RTPrintf("status: %d\n", u32Status);
         switch (u32Status)
         {
@@ -442 +495 @@
                 /* Reset the thread status and signal the main thread that we
                    are finished. */
-                ASMAtomicWriteU32(&pInt->u32Status, STATUS_WAIT);
+                ASMAtomicCmpXchgU32(&pInt->u32Status, STATUS_WAIT, STATUS_CALC);
                 rc = RTSemEventSignal(pInt->calcFinishedEvent);
                 break;
@@ -452 +505 @@
                 break;
             }
+            case STATUS_READ:
+            {
+                PVDINTERFACE pIO = VDInterfaceGet(pInt->pSha1Storage->pVDImageIfaces, VDINTERFACETYPE_IO);
+                AssertPtrReturn(pIO, VERR_INVALID_PARAMETER);
+                PVDINTERFACEIO pCallbacks = VDGetInterfaceIO(pIO);
+                AssertPtrReturn(pCallbacks, VERR_INVALID_PARAMETER);
+
+                size_t cbAvail = RTCircBufFree(pInt->pCircBuf);
+//                RTPrintf("############################################################################### th: avail %ld\n", cbAvail);
+
+                /* ************ CHECK for 0 */
+                /* First loop over all the available memory in the circular
+                 * memory buffer (could be turn around at the end). */
+                size_t cbMemAllWrite = 0;
+                bool fStop = false;
+                bool fEOF = false;
+                for(;;)
+                {
+                    if (   cbMemAllWrite == cbAvail
+                        || fStop == true)
+                        break;
+                    char *pcBuf;
+                    size_t cbMemToWrite = cbAvail - cbMemAllWrite;
+                    size_t cbMemWrite = 0;
+                    /* Try to acquire all the free space of the circular buffer. */
+                    RTCircBufAcquireWriteBlock(pInt->pCircBuf, cbMemToWrite, (void**)&pcBuf, &cbMemWrite);
+                    /* Second, read as long as we filled all the memory. The
+                     * read method could also split the reads up into to
+                     * smaller parts. */
+                    size_t cbAllRead = 0;
+                    for(;;)
+                    {
+                        if (cbAllRead == cbMemWrite)
+                            break;
+                        size_t cbToRead = cbMemWrite - cbAllRead;
+                        size_t cbRead = 0;
+                        rc = pCallbacks->pfnReadSync(pIO->pvUser, pInt->pvStorage, pInt->cbCurFile, &pcBuf[cbAllRead], cbToRead, &cbRead);
+//                        RTPrintf ("%lu %lu %lu %Rrc\n", pInt->cbCurFile, cbToRead, cbRead, rc);
+                        if (RT_FAILURE(rc))
+                        {
+                            fLoop = false;
+                            fStop = true;
+                            break;
+                        }
+                        if (cbRead == 0)
+                        {
+                            fStop = true;
+                            fLoop = false;
+                            fEOF = true;
+//                            RTPrintf("EOF\n");
+                            break;
+                        }
+                        cbAllRead += cbRead;
+                        pInt->cbCurFile += cbRead;
+                    }
+                    /* Update the SHA1 context with the next data block. */
+                    RTSha1Update(&pInt->ctx, pcBuf, cbAllRead);
+                    /* Mark the block as full. */
+                    RTCircBufReleaseWriteBlock(pInt->pCircBuf, cbAllRead);
+                    cbMemAllWrite += cbAllRead;
+                }
+                if (fEOF)
+                    ASMAtomicWriteBool(&pInt->fEOF, true);
+                /* Reset the thread status and signal the main thread that we
+                   are finished. */
+                ASMAtomicCmpXchgU32(&pInt->u32Status, STATUS_WAIT, STATUS_READ);
+                rc = RTSemEventSignal(pInt->calcFinishedEvent);
+                break;
+            }
         }
     }
@@ -470 +592 @@
     {
 //        RTPrintf(" wait\n");
-        if (ASMAtomicReadU32(&pInt->u32Status) != STATUS_CALC)
+        if (!(   ASMAtomicReadU32(&pInt->u32Status) == STATUS_CALC
+              || ASMAtomicReadU32(&pInt->u32Status) == STATUS_READ))
             break;
         rc = RTSemEventWait(pInt->calcFinishedEvent, 100);
@@ -523 +646 @@
     AssertPtrNullReturn(pfnCompleted, VERR_INVALID_PARAMETER);
     AssertPtrReturn(ppInt, VERR_INVALID_POINTER);
+    AssertReturn((fOpen & RTFILE_O_READWRITE) != RTFILE_O_READWRITE, VERR_INVALID_PARAMETER); /* No read/write allowed */
 
     PSHA1STORAGE pSha1Storage = (PSHA1STORAGE)pvUser;
@@ -540 +664 @@
     {
         pInt->pfnCompleted = pfnCompleted;
-        /* For caching reasons and to be able to calculate the sha1 sum of the
-           data we need a memory buffer. */
-        pInt->cbBuf = _1M;
-        pInt->pcBuf = (char*)RTMemAlloc(pInt->cbBuf);
-        if (!pInt->pcBuf)
+        pInt->pSha1Storage = pSha1Storage;
+        pInt->fEOF         = false;
+
+
+        if (fOpen & RTFILE_O_READ)
         {
-            rc = VERR_NO_MEMORY;
-            break;
+            /* Circular buffer in the read case. */
+            rc = RTCircBufCreate(&pInt->pCircBuf, _1M * 2);
+            if (RT_FAILURE(rc))
+                break;
         }
-        /* The zero buffer is used for appending empty parts at the end of the
-         * file (or our buffer) in setSize or when uOffset in writeSync is
-         * increased in steps bigger than a byte. */
-        pInt->cbZeroBuf = _1K;
-        pInt->pvZeroBuf = RTMemAllocZ(pInt->cbZeroBuf);
-        if (!pInt->pvZeroBuf)
+        else if (fOpen & RTFILE_O_WRITE)
         {
-            rc = VERR_NO_MEMORY;
-            break;
+            /* For caching reasons and to be able to calculate the sha1 sum of the
+               data we need a memory buffer. */
+            pInt->cbBuf = _1M;
+            pInt->pcBuf = (char*)RTMemAlloc(pInt->cbBuf);
+            if (!pInt->pcBuf)
+            {
+                rc = VERR_NO_MEMORY;
+                break;
+            }
+            /* The zero buffer is used for appending empty parts at the end of the
+             * file (or our buffer) in setSize or when uOffset in writeSync is
+             * increased in steps bigger than a byte. */
+            pInt->cbZeroBuf = _1K;
+            pInt->pvZeroBuf = RTMemAllocZ(pInt->cbZeroBuf);
+            if (!pInt->pvZeroBuf)
+            {
+                rc = VERR_NO_MEMORY;
+                break;
+            }
         }
 
-        if (pSha1Storage->fCreateDigest)
+        if (   fOpen & RTFILE_O_READ
+            || pSha1Storage->fCreateDigest)
         {
-            /* Create a sha1 context the sha1 worker thread will work with. */
-            RTSha1Init(&pInt->ctx);
             /* Create an event semaphore to indicate a state change for the sha1
                worker thread. */
@@ -579 +716 @@
                 break;
         }
+
+        if (pSha1Storage->fCreateDigest)
+            /* Create a sha1 context the sha1 worker thread will work with. */
+            RTSha1Init(&pInt->ctx);
+
         /* Open the file. */
         rc = pCallbacks->pfnOpen(pIO->pvUser, pszLocation,
@@ -588 +730 @@
     if (RT_FAILURE(rc))
     {
-        if (pSha1Storage->fCreateDigest)
+        if (pInt->pMfThread)
         {
-            if (pInt->pMfThread)
-            {
-                sha1SignalManifestThread(pInt, STATUS_END);
-                RTThreadWait(pInt->pMfThread, RT_INDEFINITE_WAIT, 0);
-            }
-            if (pInt->calcFinishedEvent)
-                RTSemEventDestroy(pInt->calcFinishedEvent);
-            if (pInt->newStatusEvent)
-                RTSemEventDestroy(pInt->newStatusEvent);
+            sha1SignalManifestThread(pInt, STATUS_END);
+            RTThreadWait(pInt->pMfThread, RT_INDEFINITE_WAIT, 0);
         }
+        if (pInt->calcFinishedEvent)
+            RTSemEventDestroy(pInt->calcFinishedEvent);
+        if (pInt->newStatusEvent)
+            RTSemEventDestroy(pInt->newStatusEvent);
+        if (pInt->pCircBuf)
+            RTCircBufDestroy(pInt->pCircBuf);
         if (pInt->pvZeroBuf)
             RTMemFree(pInt->pvZeroBuf);
@@ -634 +775 @@
         rc = sha1FlushCurBuf(pIO, pCallbacks, pInt, pSha1Storage->fCreateDigest);
 
-    if (pSha1Storage->fCreateDigest)
-    {
+    if (pInt->pMfThread)
         /* Signal the worker thread to end himself */
         rc = sha1SignalManifestThread(pInt, STATUS_END);
+
+    if (pSha1Storage->fCreateDigest)
+    {
         /* Finally calculate & format the SHA1 sum */
         unsigned char auchDig[RTSHA1_HASH_SIZE];
@@ -650 +793 @@
             RTStrFree(pszDigest);
         }
+    }
+
+    if (pInt->pMfThread)
         /* Worker thread stopped? */
         rc = RTThreadWait(pInt->pMfThread, RT_INDEFINITE_WAIT, 0);
-    }
+
     /* Close the file */
     rc = pCallbacks->pfnClose(pIO->pvUser, pInt->pvStorage);
 
+//    RTPrintf("%lu %lu\n", pInt->calls, pInt->waits);
+
     /* Cleanup */
-    if (pSha1Storage->fCreateDigest)
-    {
-        if (pInt->calcFinishedEvent)
-            RTSemEventDestroy(pInt->calcFinishedEvent);
-        if (pInt->newStatusEvent)
-            RTSemEventDestroy(pInt->newStatusEvent);
-    }
+    if (pInt->calcFinishedEvent)
+        RTSemEventDestroy(pInt->calcFinishedEvent);
+    if (pInt->newStatusEvent)
+        RTSemEventDestroy(pInt->newStatusEvent);
+    if (pInt->pCircBuf)
+        RTCircBufDestroy(pInt->pCircBuf);
     if (pInt->pvZeroBuf)
         RTMemFree(pInt->pvZeroBuf);
@@ -760 +907 @@
 
     *pcbSize = RT_MAX(pInt->cbCurAll, cbSize);
+
     return VINF_SUCCESS;
 }
@@ -870 +1018 @@
     PSHA1STORAGEINTERNAL pInt = (PSHA1STORAGEINTERNAL)pvStorage;
 
-    return pCallbacks->pfnReadSync(pIO->pvUser, pInt->pvStorage, uOffset, pvBuf, cbRead, pcbRead);
+    int rc = VINF_SUCCESS;
+
+//    pInt->calls++;
+//    RTPrintf("Read uOffset: %7lu cbRead: %7lu = %7lu\n", uOffset, cbRead, uOffset + cbRead);
+
+    /* Check if we jump forward in the file. If so we have to read the
+     * remaining stuff in the gap anyway (SHA1; streaming). */
+    if (pInt->cbCurAll < uOffset)
+    {
+        rc = sha1ReadSyncCallback(pvUser, pvStorage, pInt->cbCurAll, 0, uOffset - pInt->cbCurAll, 0);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    size_t cbAllRead = 0;
+    for(;;)
+    {
+        /* Finished? */
+        if (cbAllRead == cbRead)
+            break;
+        size_t cbAvail = RTCircBufUsed(pInt->pCircBuf);
+        if (   cbAvail == 0
+            && pInt->fEOF)
+            return VERR_EOF;
+        /* If there isn't enough data make sure the worker thread is fetching
+         * more. */
+        if ((cbRead - cbAllRead) > cbAvail)
+        {
+            rc = sha1SignalManifestThread(pInt, STATUS_READ);
+            if(RT_FAILURE(rc))
+                break;
+            /* If there is _no_ data available, we have to wait until it is. */
+            if (cbAvail == 0)
+            {
+                rc = sha1WaitForManifestThreadFinished(pInt);
+                if (RT_FAILURE(rc))
+                    break;
+                cbAvail = RTCircBufUsed(pInt->pCircBuf);
+//                RTPrintf("############## wait %lu %lu %lu \n", cbRead, cbAllRead, cbAvail);
+//                pInt->waits++;
+            }
+        }
+        size_t cbToRead = RT_MIN(cbRead - cbAllRead, cbAvail);
+        char *pcBuf;
+        size_t cbMemRead = 0;
+        /* Acquire a block for reading from our circular buffer. */
+        RTCircBufAcquireReadBlock(pInt->pCircBuf, cbToRead, (void**)&pcBuf, &cbMemRead);
+        if (pvBuf) /* Make it possible to blind read data (for skipping) */
+            memcpy(&((char*)pvBuf)[cbAllRead], pcBuf, cbMemRead);
+        /* Mark the block as empty again. */
+        RTCircBufReleaseReadBlock(pInt->pCircBuf, cbMemRead);
+        cbAllRead += cbMemRead;
+        pInt->cbCurAll += cbMemRead;
+    }
+
+    if (pcbRead)
+        *pcbRead = cbAllRead;
+
+    if (rc == VERR_EOF)
+        rc = VINF_SUCCESS;
+
+    /* Signal the thread to read more data in the mean time. */
+    if (   RT_SUCCESS(rc)
+        && RTCircBufFree(pInt->pCircBuf) >= (RTCircBufSize(pInt->pCircBuf) / 2))
+        rc = sha1SignalManifestThread(pInt, STATUS_READ);
+
+    return rc;
 }
 
@@ -975 +1189 @@
 }
 
+int Sha1ReadBuf(const char *pcszFilename, void **ppvBuf, size_t *pcbSize, PVDINTERFACEIO pCallbacks, void *pvUser)
+{
+    /* Validate input. */
+    AssertPtrReturn(ppvBuf, VERR_INVALID_POINTER);
+    AssertPtrReturn(pcbSize, VERR_INVALID_POINTER);
+    AssertPtrReturn(pCallbacks, VERR_INVALID_POINTER);
+
+    void *pvStorage;
+    int rc = pCallbacks->pfnOpen(pvUser, pcszFilename,
+                                 RTFILE_O_OPEN | RTFILE_O_READ | RTFILE_O_DENY_NONE, 0,
+                                 &pvStorage);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    void *pvBuf = 0;
+    uint64_t cbSize = 0;
+    do
+    {
+        rc = pCallbacks->pfnGetSize(pvUser, pvStorage, &cbSize);
+        if (RT_FAILURE(rc))
+            break;
+
+        pvBuf = RTMemAlloc(cbSize);
+        if (!pvBuf)
+        {
+            rc = VERR_NO_MEMORY;
+            break;
+        }
+
+        size_t cbAllRead = 0;
+        for(;;)
+        {
+            if (cbAllRead == cbSize)
+                break;
+            size_t cbToRead = cbSize - cbAllRead;
+            size_t cbRead = 0;
+            rc = pCallbacks->pfnReadSync(pvUser, pvStorage, cbAllRead, &((char*)pvBuf)[cbAllRead], cbToRead, &cbRead);
+            if (RT_FAILURE(rc))
+                break;
+            cbAllRead += cbRead;
+        }
+    }while(0);
+
+    pCallbacks->pfnClose(pvUser, pvStorage);
+
+    if (RT_SUCCESS(rc))
+    {
+        *ppvBuf = pvBuf;
+        *pcbSize = cbSize;
+    }else
+    {
+        if (pvBuf)
+            RTMemFree(pvBuf);
+    }
+
+    return rc;
+}
+
 int Sha1WriteBuf(const char *pcszFilename, void *pvBuf, size_t cbSize, PVDINTERFACEIO pCallbacks, void *pvUser)
 {
     /* Validate input. */
-    AssertPtrReturn(pCallbacks, VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pvBuf, VERR_INVALID_POINTER);
+    AssertReturn(cbSize, VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pCallbacks, VERR_INVALID_POINTER);
 
     void *pvStorage;

trunk/src/VBox/Main/ApplianceImplImport.cpp

@@ -26 +26 @@
 #include <iprt/stream.h>
 
+#include <VBox/VBoxHDD.h>
 #include <VBox/com/array.h>
 
@@ -33 +34 @@
 #include "ProgressImpl.h"
 #include "MachineImpl.h"
+#include "MediumImpl.h"
+#include "MediumFormatImpl.h"
+#include "SystemPropertiesImpl.h"
 
 #include "AutoCaller.h"
@@ -77 +81 @@
     }
 
-    // see if we can handle this file; for now we insist it has an ".ovf" extension
+    // see if we can handle this file; for now we insist it has an ovf/ova extension
     Utf8Str strPath (path);
     if (!(   strPath.endsWith(".ovf", Utf8Str::CaseInsensitive)
@@ -180 +184 @@
                                nameVBox);
 
+            /* Based on the VM name, create a target machine path. */
+            Bstr bstrMachineFilename;
+            rc = mVirtualBox->ComposeMachineFilename(Bstr(nameVBox).raw(),
+                                                     NULL,
+                                                     bstrMachineFilename.asOutParam());
+            if (FAILED(rc)) throw rc;
+            /* Determine the machine folder from that */
+            Utf8Str strMachineFolder = Utf8Str(bstrMachineFilename).stripFilename();
+
             /* VM Product */
             if (!vsysThis.strProduct.isEmpty())
@@ -236 +249 @@
             /* CPU count */
             ULONG cpuCountVBox = vsysThis.cCPUs;
-            /* Check for the constrains */
+            /* Check for the constraints */
             if (cpuCountVBox > SchemaDefs::MaxCPUCount)
             {
@@ -496 +509 @@
                             strFilename = Utf8StrFmt("%s.vmdk", nameVBox.c_str());
 
+                        Utf8Str strTargetPath = Utf8Str(strMachineFolder)
+                            .append(RTPATH_DELIMITER)
+                            .append(di.strHref);;
+                        searchUniqueDiskImageFilePath(strTargetPath);
+
                         /* find the description for the hard disk controller
                          * that has the same ID as hd.idController */
@@ -512 +530 @@
                                            hd.strDiskId,
                                            di.strHref,
-                                           strFilename,
+                                           strTargetPath,
                                            di.ulSuggestedSizeMB,
                                            strExtraConfig);
@@ -586 +604 @@
 //
 ////////////////////////////////////////////////////////////////////////////////
+
+
+/*******************************************************************************
+ * Read stuff
+ ******************************************************************************/
 
 /**
@@ -655 +678 @@
  * @return
  */
-HRESULT Appliance::readFS(const LocationInfo &locInfo, ComObjPtr<Progress> &pProgress)
-{
-    if (locInfo.strPath.endsWith(".ovf", Utf8Str::CaseInsensitive))
-        return readFSOVF(locInfo, pProgress);
-    else
-        return readFSOVA(locInfo, pProgress);
-}
-
-HRESULT Appliance::readFSOVF(const LocationInfo &locInfo, ComObjPtr<Progress> & /* pProgress */)
+HRESULT Appliance::readFS(TaskOVF *pTask)
 {
     LogFlowFuncEnter();
@@ -675 +690 @@
     HRESULT rc = S_OK;
 
+    if (pTask->locInfo.strPath.endsWith(".ovf", Utf8Str::CaseInsensitive))
+        rc = readFSOVF(pTask);
+    else
+        rc = readFSOVA(pTask);
+
+    LogFlowFunc(("rc=%Rhrc\n", rc));
+    LogFlowFuncLeave();
+
+    return rc;
+}
+
+HRESULT Appliance::readFSOVF(TaskOVF *pTask)
+{
+    LogFlowFuncEnter();
+
+    HRESULT rc = S_OK;
+
+    PVDINTERFACEIO pSha1Callbacks = 0;
+    PVDINTERFACEIO pRTFileCallbacks = 0;
+    do
+    {
+        pSha1Callbacks = Sha1CreateInterface();
+        if (!pSha1Callbacks)
+        {
+            rc = E_OUTOFMEMORY;
+            break;
+        }
+        pRTFileCallbacks = RTFileCreateInterface();
+        if (!pRTFileCallbacks)
+        {
+            rc = E_OUTOFMEMORY;
+            break;
+        }
+        VDINTERFACE VDInterfaceIO;
+        SHA1STORAGE storage;
+        RT_ZERO(storage);
+        int vrc = VDInterfaceAdd(&VDInterfaceIO, "Appliance::IORTFile",
+                                 VDINTERFACETYPE_IO, pRTFileCallbacks,
+                                 0, &storage.pVDImageIfaces);
+        if (RT_FAILURE(vrc))
+        {
+            rc = E_FAIL;
+            break;
+        }
+        rc = readFSImpl(pTask, pSha1Callbacks, &storage);
+    }while(0);
+
+    /* Cleanup */
+    if (pSha1Callbacks)
+        RTMemFree(pSha1Callbacks);
+    if (pRTFileCallbacks)
+        RTMemFree(pRTFileCallbacks);
+
+    LogFlowFunc(("rc=%Rhrc\n", rc));
+    LogFlowFuncLeave();
+
+    return rc;
+}
+
+HRESULT Appliance::readFSOVA(TaskOVF *pTask)
+{
+    LogFlowFuncEnter();
+
+    RTTAR tar;
+    int vrc = RTTarOpen(&tar, pTask->locInfo.strPath.c_str(), RTFILE_O_OPEN | RTFILE_O_READ | RTFILE_O_DENY_NONE, true);
+    if (RT_FAILURE(vrc))
+        return setError(VBOX_E_FILE_ERROR,
+                        tr("Could not open OVA file '%s' (%Rrc)"),
+                        pTask->locInfo.strPath.c_str(), vrc);
+
+    HRESULT rc = S_OK;
+
+    PVDINTERFACEIO pSha1Callbacks = 0;
+    PVDINTERFACEIO pRTTarCallbacks = 0;
+    do
+    {
+        pSha1Callbacks = Sha1CreateInterface();
+        if (!pSha1Callbacks)
+        {
+            rc = E_OUTOFMEMORY;
+            break;
+        }
+        pRTTarCallbacks = RTTarCreateInterface();
+        if (!pRTTarCallbacks)
+        {
+            rc = E_OUTOFMEMORY;
+            break;
+        }
+        VDINTERFACE VDInterfaceIO;
+        SHA1STORAGE storage;
+        RT_ZERO(storage);
+        vrc = VDInterfaceAdd(&VDInterfaceIO, "Appliance::IORTTar",
+                             VDINTERFACETYPE_IO, pRTTarCallbacks,
+                             tar, &storage.pVDImageIfaces);
+        if (RT_FAILURE(vrc))
+        {
+            rc = E_FAIL;
+            break;
+        }
+        rc = readFSImpl(pTask, pSha1Callbacks, &storage);
+    }while(0);
+
+    RTTarClose(tar);
+
+    /* Cleanup */
+    if (pSha1Callbacks)
+        RTMemFree(pSha1Callbacks);
+    if (pRTTarCallbacks)
+        RTMemFree(pRTTarCallbacks);
+
+    LogFlowFunc(("rc=%Rhrc\n", rc));
+    LogFlowFuncLeave();
+
+    return rc;
+}
+
+HRESULT Appliance::readFSImpl(TaskOVF *pTask, PVDINTERFACEIO pCallbacks, PSHA1STORAGE pStorage)
+{
+    LogFlowFuncEnter();
+
+    HRESULT rc = S_OK;
+
+    pStorage->fCreateDigest = true;
+
+    void *pvTmpBuf = 0;
     try
     {
-        /* Read & parse the XML structure of the OVF file */
-        m->pReader = new ovf::OVFReader(locInfo.strPath);
-        /* Create the SHA1 sum of the OVF file for later validation */
-        char *pszDigest;
-        int vrc = RTSha1DigestFromFile(locInfo.strPath.c_str(), &pszDigest, NULL, NULL);
+        Utf8Str strOvfFile = Utf8Str(pTask->locInfo.strPath).stripExt().append(".ovf");
+        /* Read the OVF into a memory buffer */
+        size_t cbSize = 0;
+        int vrc = Sha1ReadBuf(strOvfFile.c_str(), &pvTmpBuf, &cbSize, pCallbacks, pStorage);
         if (RT_FAILURE(vrc))
             throw setError(VBOX_E_FILE_ERROR,
-                           tr("Couldn't calculate SHA1 digest for file '%s' (%Rrc)"),
-                           RTPathFilename(locInfo.strPath.c_str()), vrc);
-        m->strOVFSHA1Digest = pszDigest;
-        RTStrFree(pszDigest);
+                           tr("Could not read OVF file '%s' (%Rrc)"),
+                           RTPathFilename(strOvfFile.c_str()), vrc);
+        /* Copy the SHA1 sum of the OVF file for later validation */
+        m->strOVFSHA1Digest = pStorage->strDigest;
+        /* Read & parse the XML structure of the OVF file */
+        m->pReader = new ovf::OVFReader(pvTmpBuf, cbSize, pTask->locInfo.strPath);
     }
     catch (iprt::Error &x)      // includes all XML exceptions
@@ -699 +840 @@
     }
 
-    LogFlowFunc(("rc=%Rhrc\n", rc));
-    LogFlowFuncLeave();
-
-    return rc;
-}
-
-HRESULT Appliance::readFSOVA(const LocationInfo &locInfo, ComObjPtr<Progress> &pProgress)
-{
-    LogFlowFuncEnter();
-    LogFlowFunc(("Appliance %p\n", this));
-
-    AutoCaller autoCaller(this);
-    if (FAILED(autoCaller.rc())) return autoCaller.rc();
-
-    AutoWriteLock appLock(this COMMA_LOCKVAL_SRC_POS);
-    HRESULT rc = S_OK;
-    void *pvBuf = 0;
-
-    try
-    {
-        Utf8Str tmpPath = locInfo.strPath;
-        /* Remove the ova extension */
-        tmpPath.stripExt();
-        /* add the ovf extension. */
-        tmpPath += ".ovf";
-        char* pcszOVFName = RTPathFilename(tmpPath.c_str());
-
-        /* Read the OVF into a memory buffer */
-        size_t cbSize;
-        int vrc = RTTarExtractFileToBuf(locInfo.strPath.c_str(), &pvBuf, &cbSize, pcszOVFName, 0, 0);
-        if (RT_FAILURE(vrc))
-        {
-            if (vrc == VERR_FILE_NOT_FOUND)
-                throw setError(VBOX_E_IPRT_ERROR,
-                               tr("Can't find ovf file '%s' in archive '%s' (%Rrc)"), pcszOVFName, locInfo.strPath.c_str(), vrc);
-            else
-                throw setError(VBOX_E_IPRT_ERROR,
-                               tr("Can't unpack the archive file '%s' (%Rrc)"), locInfo.strPath.c_str(), vrc);
-        }
-
-        /* Read & parse the XML structure of the OVF file */
-        m->pReader = new ovf::OVFReader(pvBuf, cbSize, locInfo.strPath);
-        /* Create the SHA1 sum of the OVF file for later validation */
-        char *pszDigest;
-        vrc = RTSha1Digest(pvBuf, cbSize, &pszDigest, 0, 0);
-        if (RT_FAILURE(vrc))
-            throw setError(VBOX_E_FILE_ERROR,
-                           tr("Couldn't calculate SHA1 digest for file '%s' (%Rrc)"),
-                           RTPathFilename(locInfo.strPath.c_str()), vrc);
-        m->strOVFSHA1Digest = pszDigest;
-        RTStrFree(pszDigest);
-
-    }
-    catch (iprt::Error &x)      // includes all XML exceptions
-    {
-        rc = setError(VBOX_E_FILE_ERROR,
-                      x.what());
-    }
-    catch (HRESULT aRC)
-    {
-        rc = aRC;
-    }
-
-    /* Cleanup the OVF memory buffer */
-    if (pvBuf)
-        RTMemFree(pvBuf);
+    /* Cleanup */
+    if (pvTmpBuf)
+        RTMemFree(pvTmpBuf);
 
     LogFlowFunc(("rc=%Rhrc\n", rc));
@@ -899 +977 @@
     return rc;
 }
+
+/*******************************************************************************
+ * Import stuff
+ ******************************************************************************/
+
+/**
+ * Implementation for importing OVF data into VirtualBox. This starts a new thread which will call
+ * Appliance::taskThreadImportOrExport().
+ *
+ * This creates one or more new machines according to the VirtualSystemScription instances created by
+ * Appliance::Interpret().
+ *
+ * This is in a separate private method because it is used from two locations:
+ *
+ * 1) from the public Appliance::ImportMachines().
+ * 2) from Appliance::importS3(), which got called from a previous instance of Appliance::taskThreadImportOrExport().
+ *
+ * @param aLocInfo
+ * @param aProgress
+ * @return
+ */
+HRESULT Appliance::importImpl(const LocationInfo &locInfo,
+                              ComObjPtr<Progress> &progress)
+{
+    HRESULT rc = S_OK;
+
+    SetUpProgressMode mode;
+    if (locInfo.storageType == VFSType_File)
+        mode = ImportFile;
+    else
+        mode = ImportS3;
+
+    rc = setUpProgress(progress,
+                       BstrFmt(tr("Importing appliance '%s'"), locInfo.strPath.c_str()),
+                       mode);
+    if (FAILED(rc)) throw rc;
+
+    /* Initialize our worker task */
+    std::auto_ptr<TaskOVF> task(new TaskOVF(this, TaskOVF::Import, locInfo, progress));
+
+    rc = task->startThread();
+    if (FAILED(rc)) throw rc;
+
+    /* Don't destruct on success */
+    task.release();
+
+    return rc;
+}
+
+/**
+ * Actual worker code for importing OVF data into VirtualBox. This is called from Appliance::taskThreadImportOrExport()
+ * and therefore runs on the OVF import worker thread. This creates one or more new machines according to the
+ * VirtualSystemScription instances created by Appliance::Interpret().
+ *
+ * This runs in three contexts:
+ *
+ * 1) in a first worker thread; in that case, Appliance::ImportMachines() called Appliance::importImpl();
+ *
+ * 2) in a second worker thread; in that case, Appliance::ImportMachines() called Appliance::importImpl(), which
+ *    called Appliance::importFSOVA(), which called Appliance::importImpl(), which then called this again.
+ *
+ * 3) in a second worker thread; in that case, Appliance::ImportMachines() called Appliance::importImpl(), which
+ *    called Appliance::importS3(), which called Appliance::importImpl(), which then called this again.
+ *
+ * @param pTask
+ * @return
+ */
+HRESULT Appliance::importFS(TaskOVF *pTask)
+{
+
+    LogFlowFuncEnter();
+    LogFlowFunc(("Appliance %p\n", this));
+
+    AutoCaller autoCaller(this);
+    if (FAILED(autoCaller.rc())) return autoCaller.rc();
+
+    /* Change the appliance state so we can safely leave the lock while doing
+     * time-consuming disk imports; also the below method calls do all kinds of
+     * locking which conflicts with the appliance object lock. */
+    AutoWriteLock writeLock(this COMMA_LOCKVAL_SRC_POS);
+    /* Check if the appliance is currently busy. */
+    if (!isApplianceIdle())
+        return E_ACCESSDENIED;
+    /* Set the internal state to importing. */
+    m->state = Data::ApplianceImporting;
+
+    HRESULT rc = S_OK;
+
+    /* Clear the list of imported machines, if any */
+    m->llGuidsMachinesCreated.clear();
+
+    if (pTask->locInfo.strPath.endsWith(".ovf", Utf8Str::CaseInsensitive))
+        rc = importFSOVF(pTask, writeLock);
+    else
+        rc = importFSOVA(pTask, writeLock);
+
+    if (FAILED(rc))
+    {
+        /* With _whatever_ error we've had, do a complete roll-back of
+         * machines and disks we've created */
+        writeLock.release();
+        for (list<Guid>::iterator itID = m->llGuidsMachinesCreated.begin();
+             itID != m->llGuidsMachinesCreated.end();
+             ++itID)
+        {
+            Guid guid = *itID;
+            Bstr bstrGuid = guid.toUtf16();
+            ComPtr<IMachine> failedMachine;
+            HRESULT rc2 = mVirtualBox->GetMachine(bstrGuid.raw(), failedMachine.asOutParam());
+            if (SUCCEEDED(rc2))
+            {
+                SafeIfaceArray<IMedium> aMedia;
+                rc2 = failedMachine->Unregister(CleanupMode_DetachAllReturnHardDisksOnly, ComSafeArrayAsOutParam(aMedia));
+                ComPtr<IProgress> pProgress2;
+                rc2 = failedMachine->Delete(ComSafeArrayAsInParam(aMedia), pProgress2.asOutParam());
+                pProgress2->WaitForCompletion(-1);
+            }
+        }
+        writeLock.acquire();
+    }
+
+    /* Reset the state so others can call methods again */
+    m->state = Data::ApplianceIdle;
+
+    LogFlowFunc(("rc=%Rhrc\n", rc));
+    LogFlowFuncLeave();
+
+    return rc;
+}
+
+HRESULT Appliance::importFSOVF(TaskOVF *pTask, AutoWriteLockBase& writeLock)
+{
+    LogFlowFuncEnter();
+
+    HRESULT rc = S_OK;
+
+    PVDINTERFACEIO pSha1Callbacks = 0;
+    PVDINTERFACEIO pRTFileCallbacks = 0;
+    void *pvMfBuf = 0;
+    writeLock.release();
+    try
+    {
+        /* Create the necessary file access interfaces. */
+        pSha1Callbacks = Sha1CreateInterface();
+        if (!pSha1Callbacks)
+            throw E_OUTOFMEMORY;
+        pRTFileCallbacks = RTFileCreateInterface();
+        if (!pRTFileCallbacks)
+            throw E_OUTOFMEMORY;
+
+        VDINTERFACE VDInterfaceIO;
+        SHA1STORAGE storage;
+        RT_ZERO(storage);
+        storage.fCreateDigest = true;
+        int vrc = VDInterfaceAdd(&VDInterfaceIO, "Appliance::IORTFile",
+                                 VDINTERFACETYPE_IO, pRTFileCallbacks,
+                                 0, &storage.pVDImageIfaces);
+        if (RT_FAILURE(vrc))
+            throw E_FAIL;
+
+        size_t cbMfSize = 0;
+        Utf8Str strMfFile = Utf8Str(pTask->locInfo.strPath).stripExt().append(".mf");
+        /* Create the import stack for the rollback on errors. */
+        ImportStack stack(pTask->locInfo, m->pReader->m_mapDisks, pTask->pProgress);
+        /* Do we need the digest information? */
+        storage.fCreateDigest = RTFileExists(strMfFile.c_str());
+        /* Now import the appliance. */
+        importMachines(stack, pSha1Callbacks, &storage);
+        /* Read & verify the manifest file, if there is one. */
+        if (storage.fCreateDigest)
+        {
+            /* Add the ovf file to the digest list. */
+            stack.llSrcDisksDigest.push_front(STRPAIR(pTask->locInfo.strPath, m->strOVFSHA1Digest));
+            rc = readManifestFile(strMfFile, &pvMfBuf, &cbMfSize, pSha1Callbacks, &storage);
+            if (FAILED(rc)) throw rc;
+            rc = verifyManifestFile(strMfFile, stack, pvMfBuf, cbMfSize);
+            if (FAILED(rc)) throw rc;
+        }
+    }
+    catch (HRESULT rc2)
+    {
+        rc = rc2;
+    }
+    writeLock.acquire();
+
+    /* Cleanup */
+    if (pvMfBuf)
+        RTMemFree(pvMfBuf);
+    if (pSha1Callbacks)
+        RTMemFree(pSha1Callbacks);
+    if (pRTFileCallbacks)
+        RTMemFree(pRTFileCallbacks);
+
+    LogFlowFunc(("rc=%Rhrc\n", rc));
+    LogFlowFuncLeave();
+
+    return rc;
+}
+
+HRESULT Appliance::importFSOVA(TaskOVF *pTask, AutoWriteLockBase& writeLock)
+{
+    LogFlowFuncEnter();
+
+    RTTAR tar;
+    int vrc = RTTarOpen(&tar, pTask->locInfo.strPath.c_str(), RTFILE_O_OPEN | RTFILE_O_READ | RTFILE_O_DENY_NONE, true);
+    if (RT_FAILURE(vrc))
+        return setError(VBOX_E_FILE_ERROR,
+                        tr("Could not open OVA file '%s' (%Rrc)"),
+                        pTask->locInfo.strPath.c_str(), vrc);
+
+    HRESULT rc = S_OK;
+
+    PVDINTERFACEIO pSha1Callbacks = 0;
+    PVDINTERFACEIO pRTTarCallbacks = 0;
+    void *pvMfBuf = 0;
+    writeLock.release();
+    try
+    {
+        /* Create the necessary file access interfaces. */
+        pSha1Callbacks = Sha1CreateInterface();
+        if (!pSha1Callbacks)
+            throw E_OUTOFMEMORY;
+        pRTTarCallbacks = RTTarCreateInterface();
+        if (!pRTTarCallbacks)
+            throw E_OUTOFMEMORY;
+
+        VDINTERFACE VDInterfaceIO;
+        SHA1STORAGE storage;
+        RT_ZERO(storage);
+        vrc = VDInterfaceAdd(&VDInterfaceIO, "Appliance::IORTTar",
+                             VDINTERFACETYPE_IO, pRTTarCallbacks,
+                             tar, &storage.pVDImageIfaces);
+        if (RT_FAILURE(vrc))
+            throw E_FAIL;
+
+        /* Skip the OVF file, cause this was read in IAppliance::Read already. */
+        vrc = RTTarSeekNextFile(tar);
+        if (RT_FAILURE(vrc))
+            /* Better error .... no unusual error */
+            throw E_FAIL;
+
+        PVDINTERFACEIO pCallbacks = pSha1Callbacks;
+        PSHA1STORAGE pStorage = &storage;
+
+        /* We always need to create the digest, cause we didn't know if there
+         * is a manifest file in the stream. */
+        pStorage->fCreateDigest = true;
+
+        size_t cbMfSize = 0;
+        Utf8Str strMfFile = Utf8Str(pTask->locInfo.strPath).stripExt().append(".mf");
+        /* Create the import stack for the rollback on errors. */
+        ImportStack stack(pTask->locInfo, m->pReader->m_mapDisks, pTask->pProgress);
+        /*
+         * Try to read the manifest file. First try.
+         *
+         * Note: This isn't fatal if the file is not found. The standard
+         * defines 3 cases.
+         * 1. no manifest file
+         * 2. manifest file after the OVF file
+         * 3. manifest file after all disk files
+         * If we want streaming capabilities, we can't check if it is there by
+         * searching for it. We have to try to open it on all possible places.
+         * If it fails here, we will try it again after all disks where read.
+         */
+        rc = readTarManifestFile(tar, strMfFile, &pvMfBuf, &cbMfSize, pCallbacks, pStorage);
+        if (FAILED(rc)) throw rc;
+        /* Now import the appliance. */
+        importMachines(stack, pCallbacks, pStorage);
+        /* Try to read the manifest file. Second try. */
+        if (!pvMfBuf)
+        {
+            rc = readTarManifestFile(tar, strMfFile, &pvMfBuf, &cbMfSize, pCallbacks, pStorage);
+            if (FAILED(rc)) throw rc;
+        }
+        /* If we were able to read a manifest file we can check it now. */
+        if (pvMfBuf)
+        {
+            /* Add the ovf file to the digest list. */
+            stack.llSrcDisksDigest.push_front(STRPAIR(Utf8Str(pTask->locInfo.strPath).stripExt().append(".ovf"), m->strOVFSHA1Digest));
+            rc = verifyManifestFile(strMfFile, stack, pvMfBuf, cbMfSize);
+            if (FAILED(rc)) throw rc;
+        }
+    }
+    catch (HRESULT rc2)
+    {
+        rc = rc2;
+    }
+    writeLock.acquire();
+
+    RTTarClose(tar);
+
+    /* Cleanup */
+    if (pvMfBuf)
+        RTMemFree(pvMfBuf);
+    if (pSha1Callbacks)
+        RTMemFree(pSha1Callbacks);
+    if (pRTTarCallbacks)
+        RTMemFree(pRTTarCallbacks);
+
+    LogFlowFunc(("rc=%Rhrc\n", rc));
+    LogFlowFuncLeave();
+
+    return rc;
+}
+
+/**
+ * Worker code for importing OVF from the cloud. This is called from Appliance::taskThreadImportOrExport()
+ * in S3 mode and therefore runs on the OVF import worker thread. This then starts a second worker
+ * thread to import from temporary files (see Appliance::importFS()).
+ * @param pTask
+ * @return
+ */
+HRESULT Appliance::importS3(TaskOVF *pTask)
+{
+    LogFlowFuncEnter();
+    LogFlowFunc(("Appliance %p\n", this));
+
+    AutoCaller autoCaller(this);
+    if (FAILED(autoCaller.rc())) return autoCaller.rc();
+
+    AutoWriteLock appLock(this COMMA_LOCKVAL_SRC_POS);
+
+    int vrc = VINF_SUCCESS;
+    RTS3 hS3 = NIL_RTS3;
+    char szOSTmpDir[RTPATH_MAX];
+    RTPathTemp(szOSTmpDir, sizeof(szOSTmpDir));
+    /* The template for the temporary directory created below */
+    char *pszTmpDir;
+    RTStrAPrintf(&pszTmpDir, "%s"RTPATH_SLASH_STR"vbox-ovf-XXXXXX", szOSTmpDir);
+    list< pair<Utf8Str, ULONG> > filesList;
+
+    HRESULT rc = S_OK;
+    try
+    {
+        /* Extract the bucket */
+        Utf8Str tmpPath = pTask->locInfo.strPath;
+        Utf8Str bucket;
+        parseBucket(tmpPath, bucket);
+
+        /* We need a temporary directory which we can put the all disk images
+         * in */
+        vrc = RTDirCreateTemp(pszTmpDir);
+        if (RT_FAILURE(vrc))
+            throw setError(VBOX_E_FILE_ERROR,
+                           tr("Cannot create temporary directory '%s' (%Rrc)"), pszTmpDir, vrc);
+
+        /* Add every disks of every virtual system to an internal list */
+        list< ComObjPtr<VirtualSystemDescription> >::const_iterator it;
+        for (it = m->virtualSystemDescriptions.begin();
+             it != m->virtualSystemDescriptions.end();
+             ++it)
+        {
+            ComObjPtr<VirtualSystemDescription> vsdescThis = (*it);
+            std::list<VirtualSystemDescriptionEntry*> avsdeHDs = vsdescThis->findByType(VirtualSystemDescriptionType_HardDiskImage);
+            std::list<VirtualSystemDescriptionEntry*>::const_iterator itH;
+            for (itH = avsdeHDs.begin();
+                 itH != avsdeHDs.end();
+                 ++itH)
+            {
+                const Utf8Str &strTargetFile = (*itH)->strOvf;
+                if (!strTargetFile.isEmpty())
+                {
+                    /* The temporary name of the target disk file */
+                    Utf8StrFmt strTmpDisk("%s/%s", pszTmpDir, RTPathFilename(strTargetFile.c_str()));
+                    filesList.push_back(pair<Utf8Str, ULONG>(strTmpDisk, (*itH)->ulSizeMB));
+                }
+            }
+        }
+
+        /* Next we have to download the disk images */
+        vrc = RTS3Create(&hS3, pTask->locInfo.strUsername.c_str(), pTask->locInfo.strPassword.c_str(), pTask->locInfo.strHostname.c_str(), "virtualbox-agent/"VBOX_VERSION_STRING);
+        if (RT_FAILURE(vrc))
+            throw setError(VBOX_E_IPRT_ERROR,
+                           tr("Cannot create S3 service handler"));
+        RTS3SetProgressCallback(hS3, pTask->updateProgress, &pTask);
+
+        /* Download all files */
+        for (list< pair<Utf8Str, ULONG> >::const_iterator it1 = filesList.begin(); it1 != filesList.end(); ++it1)
+        {
+            const pair<Utf8Str, ULONG> &s = (*it1);
+            const Utf8Str &strSrcFile = s.first;
+            /* Construct the source file name */
+            char *pszFilename = RTPathFilename(strSrcFile.c_str());
+            /* Advance to the next operation */
+            if (!pTask->pProgress.isNull())
+                pTask->pProgress->SetNextOperation(BstrFmt(tr("Downloading file '%s'"), pszFilename).raw(), s.second);
+
+            vrc = RTS3GetKey(hS3, bucket.c_str(), pszFilename, strSrcFile.c_str());
+            if (RT_FAILURE(vrc))
+            {
+                if (vrc == VERR_S3_CANCELED)
+                    throw S_OK; /* todo: !!!!!!!!!!!!! */
+                else if (vrc == VERR_S3_ACCESS_DENIED)
+                    throw setError(E_ACCESSDENIED,
+                                   tr("Cannot download file '%s' from S3 storage server (Access denied). "
+                                      "Make sure that your credentials are right. Also check that your host clock is properly synced"),
+                                   pszFilename);
+                else if (vrc == VERR_S3_NOT_FOUND)
+                    throw setError(VBOX_E_FILE_ERROR,
+                                   tr("Cannot download file '%s' from S3 storage server (File not found)"),
+                                   pszFilename);
+                else
+                    throw setError(VBOX_E_IPRT_ERROR,
+                                   tr("Cannot download file '%s' from S3 storage server (%Rrc)"),
+                                   pszFilename, vrc);
+            }
+        }
+
+        /* Provide a OVF file (haven't to exist) so the import routine can
+         * figure out where the disk images/manifest file are located. */
+        Utf8StrFmt strTmpOvf("%s/%s", pszTmpDir, RTPathFilename(tmpPath.c_str()));
+        /* Now check if there is an manifest file. This is optional. */
+        Utf8Str strManifestFile; //= queryManifestFileName(strTmpOvf);
+//        Utf8Str strManifestFile = queryManifestFileName(strTmpOvf);
+        char *pszFilename = RTPathFilename(strManifestFile.c_str());
+        if (!pTask->pProgress.isNull())
+            pTask->pProgress->SetNextOperation(BstrFmt(tr("Downloading file '%s'"), pszFilename).raw(), 1);
+
+        /* Try to download it. If the error is VERR_S3_NOT_FOUND, it isn't fatal. */
+        vrc = RTS3GetKey(hS3, bucket.c_str(), pszFilename, strManifestFile.c_str());
+        if (RT_SUCCESS(vrc))
+            filesList.push_back(pair<Utf8Str, ULONG>(strManifestFile, 0));
+        else if (RT_FAILURE(vrc))
+        {
+            if (vrc == VERR_S3_CANCELED)
+                throw S_OK; /* todo: !!!!!!!!!!!!! */
+            else if (vrc == VERR_S3_NOT_FOUND)
+                vrc = VINF_SUCCESS; /* Not found is ok */
+            else if (vrc == VERR_S3_ACCESS_DENIED)
+                throw setError(E_ACCESSDENIED,
+                               tr("Cannot download file '%s' from S3 storage server (Access denied)."
+                                  "Make sure that your credentials are right. Also check that your host clock is properly synced"),
+                               pszFilename);
+            else
+                throw setError(VBOX_E_IPRT_ERROR,
+                               tr("Cannot download file '%s' from S3 storage server (%Rrc)"),
+                               pszFilename, vrc);
+        }
+
+        /* Close the connection early */
+        RTS3Destroy(hS3);
+        hS3 = NIL_RTS3;
+
+        pTask->pProgress->SetNextOperation(BstrFmt(tr("Importing appliance")).raw(), m->ulWeightForXmlOperation);
+
+        ComObjPtr<Progress> progress;
+        /* Import the whole temporary OVF & the disk images */
+        LocationInfo li;
+        li.strPath = strTmpOvf;
+        rc = importImpl(li, progress);
+        if (FAILED(rc)) throw rc;
+
+        /* Unlock the appliance for the fs import thread */
+        appLock.release();
+        /* Wait until the import is done, but report the progress back to the
+           caller */
+        ComPtr<IProgress> progressInt(progress);
+        waitForAsyncProgress(pTask->pProgress, progressInt); /* Any errors will be thrown */
+
+        /* Again lock the appliance for the next steps */
+        appLock.acquire();
+    }
+    catch(HRESULT aRC)
+    {
+        rc = aRC;
+    }
+    /* Cleanup */
+    RTS3Destroy(hS3);
+    /* Delete all files which where temporary created */
+    for (list< pair<Utf8Str, ULONG> >::const_iterator it1 = filesList.begin(); it1 != filesList.end(); ++it1)
+    {
+        const char *pszFilePath = (*it1).first.c_str();
+        if (RTPathExists(pszFilePath))
+        {
+            vrc = RTFileDelete(pszFilePath);
+            if (RT_FAILURE(vrc))
+                rc = setError(VBOX_E_FILE_ERROR,
+                              tr("Cannot delete file '%s' (%Rrc)"), pszFilePath, vrc);
+        }
+    }
+    /* Delete the temporary directory */
+    if (RTPathExists(pszTmpDir))
+    {
+        vrc = RTDirRemove(pszTmpDir);
+        if (RT_FAILURE(vrc))
+            rc = setError(VBOX_E_FILE_ERROR,
+                          tr("Cannot delete temporary directory '%s' (%Rrc)"), pszTmpDir, vrc);
+    }
+    if (pszTmpDir)
+        RTStrFree(pszTmpDir);
+
+    LogFlowFunc(("rc=%Rhrc\n", rc));
+    LogFlowFuncLeave();
+
+    return rc;
+}
+
+HRESULT Appliance::readManifestFile(const Utf8Str &strFile, void **ppvBuf, size_t *pcbSize, PVDINTERFACEIO pCallbacks, PSHA1STORAGE pStorage)
+{
+    HRESULT rc = S_OK;
+
+    bool fOldDigest = pStorage->fCreateDigest;
+    pStorage->fCreateDigest = false; /* No digest for the manifest file */
+    int vrc = Sha1ReadBuf(strFile.c_str(), ppvBuf, pcbSize, pCallbacks, pStorage);
+    if (   RT_FAILURE(vrc)
+        && vrc != VERR_FILE_NOT_FOUND)
+        rc = setError(VBOX_E_FILE_ERROR,
+                      tr("Could not read manifest file '%s' (%Rrc)"),
+                      RTPathFilename(strFile.c_str()), vrc);
+    pStorage->fCreateDigest = fOldDigest; /* Restore the old digest creation behavior again. */
+
+    return rc;
+}
+
+HRESULT Appliance::readTarManifestFile(RTTAR tar, const Utf8Str &strFile, void **ppvBuf, size_t *pcbSize, PVDINTERFACEIO pCallbacks, PSHA1STORAGE pStorage)
+{
+    HRESULT rc = S_OK;
+
+    char *pszCurFile;
+    int vrc = RTTarCurrentFile(tar, &pszCurFile);
+    if (RT_SUCCESS(vrc))
+    {
+        if (!strcmp(pszCurFile, RTPathFilename(strFile.c_str())))
+            rc = readManifestFile(strFile, ppvBuf, pcbSize, pCallbacks, pStorage);
+        RTStrFree(pszCurFile);
+    }
+    else if (vrc != VERR_TAR_END_OF_FILE)
+        rc = E_FAIL;
+
+    return rc;
+}
+
+HRESULT Appliance::verifyManifestFile(const Utf8Str &strFile, ImportStack &stack, void *pvBuf, size_t cbSize)
+{
+    HRESULT rc = S_OK;
+
+    PRTMANIFESTTEST paTests = (PRTMANIFESTTEST)RTMemAlloc(sizeof(RTMANIFESTTEST) * stack.llSrcDisksDigest.size());
+    if (!paTests)
+        return E_OUTOFMEMORY;
+
+    size_t i = 0;
+    list<STRPAIR>::const_iterator it1;
+    for (it1 = stack.llSrcDisksDigest.begin();
+         it1 != stack.llSrcDisksDigest.end();
+         ++it1, ++i)
+    {
+        paTests[i].pszTestFile = (*it1).first.c_str();
+        paTests[i].pszTestDigest = (*it1).second.c_str();
+    }
+    size_t iFailed;
+    int vrc = RTManifestVerifyFilesBuf(pvBuf, cbSize, paTests, stack.llSrcDisksDigest.size(), &iFailed);
+    if (RT_UNLIKELY(vrc == VERR_MANIFEST_DIGEST_MISMATCH))
+        rc = setError(VBOX_E_FILE_ERROR,
+                      tr("The SHA1 digest of '%s' does not match the one in '%s' (%Rrc)"),
+                      RTPathFilename(paTests[iFailed].pszTestFile), RTPathFilename(strFile.c_str()), vrc);
+    else if (RT_FAILURE(vrc))
+        rc = setError(VBOX_E_FILE_ERROR,
+                      tr("Could not verify the content of '%s' against the available files (%Rrc)"),
+                      RTPathFilename(strFile.c_str()), vrc);
+
+    RTMemFree(paTests);
+
+    return rc;
+}
+
 
 /**
@@ -996 +1639 @@
 
 /**
- * Implementation for importing OVF data into VirtualBox. This starts a new thread which will call
- * Appliance::taskThreadImportOrExport().
- *
- * This creates one or more new machines according to the VirtualSystemScription instances created by
- * Appliance::Interpret().
- *
- * This is in a separate private method because it is used from two locations:
- *
- * 1) from the public Appliance::ImportMachines().
- * 2) from Appliance::importS3(), which got called from a previous instance of Appliance::taskThreadImportOrExport().
- *
- * @param aLocInfo
- * @param aProgress
- * @return
- */
-HRESULT Appliance::importImpl(const LocationInfo &locInfo,
-                              ComObjPtr<Progress> &progress)
-{
-    HRESULT rc = S_OK;
-
-    SetUpProgressMode mode;
-    if (locInfo.storageType == VFSType_File)
-    {
-        mode = ImportFileNoManifest;
-        Utf8Str strMfFile = queryManifestFileName(locInfo.strPath);
-        if (!strMfFile.isEmpty())
-            mode = ImportFileWithManifest;
-    }
-    else
-         mode = ImportS3;
-
-    rc = setUpProgress(locInfo,
-                       progress,
-                       BstrFmt(tr("Importing appliance '%s'"), locInfo.strPath.c_str()),
-                       mode);
-    if (FAILED(rc)) throw rc;
-
-    /* Initialize our worker task */
-    std::auto_ptr<TaskOVF> task(new TaskOVF(this, TaskOVF::Import, locInfo, progress));
-
-    rc = task->startThread();
-    if (FAILED(rc)) throw rc;
-
-    /* Don't destruct on success */
-    task.release();
-
-    return rc;
-}
-
-Utf8Str Appliance::queryManifestFileName(const Utf8Str& aPath) const
-{
-    Utf8Str strMfFile = manifestFileName(aPath);
-    if (!aPath.endsWith(".ova", Utf8Str::CaseInsensitive))
-    {
-        if (RTPathExists(strMfFile.c_str()))
-            return strMfFile;
-
-    }
-    else
-    {
-        if (RTTarFileExists(aPath.c_str(), RTPathFilename(strMfFile.c_str())) == VINF_SUCCESS)
-            return strMfFile;
-    }
-    return Utf8Str();
-}
-
-/**
- * Checks if a manifest file exists in the given location and, if so, verifies
- * that the relevant files (the OVF XML and the disks referenced by it, as
- * represented by the VirtualSystemDescription instances contained in this appliance)
- * match it. Requires a previous read() and interpret().
- *
- * @param locInfo
- * @param reader
- * @return
- */
-HRESULT Appliance::manifestVerify(const LocationInfo &locInfo,
-                                  const ovf::OVFReader &reader,
-                                  ComObjPtr<Progress> &pProgress)
-{
-    HRESULT rc = S_OK;
-
-    Utf8Str strManifestFile = queryManifestFileName(locInfo.strPath);
-    if (!strManifestFile.isEmpty())
-    {
-        const char *pcszManifestFileOnly = RTPathFilename(strManifestFile.c_str());
-        pProgress->SetNextOperation(BstrFmt(tr("Verifying manifest file '%s'"), pcszManifestFileOnly).raw(),
-                                    m->ulWeightForManifestOperation);     // operation's weight, as set up with the IProgress originally
-
-        list<Utf8Str> filesList;
-        Utf8Str strSrcDir(locInfo.strPath);
-        strSrcDir.stripFilename();
-        // add every disks of every virtual system to an internal list
-        list< ComObjPtr<VirtualSystemDescription> >::const_iterator it;
-        for (it = m->virtualSystemDescriptions.begin();
-             it != m->virtualSystemDescriptions.end();
-             ++it)
-        {
-            ComObjPtr<VirtualSystemDescription> vsdescThis = (*it);
-            std::list<VirtualSystemDescriptionEntry*> avsdeHDs = vsdescThis->findByType(VirtualSystemDescriptionType_HardDiskImage);
-            std::list<VirtualSystemDescriptionEntry*>::const_iterator itH;
-            for (itH = avsdeHDs.begin();
-                 itH != avsdeHDs.end();
-                 ++itH)
-            {
-                VirtualSystemDescriptionEntry *vsdeHD = *itH;
-                // find the disk from the OVF's disk list
-                ovf::DiskImagesMap::const_iterator itDiskImage = reader.m_mapDisks.find(vsdeHD->strRef);
-                const ovf::DiskImage &di = itDiskImage->second;
-                Utf8StrFmt strSrcFilePath("%s%c%s", strSrcDir.c_str(), RTPATH_DELIMITER, di.strHref.c_str());
-                filesList.push_back(strSrcFilePath);
-            }
-        }
-
-        // create the test list
-        PRTMANIFESTTEST pTestList = (PRTMANIFESTTEST)RTMemAllocZ(sizeof(RTMANIFESTTEST) * (filesList.size() + 1));
-        pTestList[0].pszTestFile = (char*)locInfo.strPath.c_str();
-        pTestList[0].pszTestDigest = (char*)m->strOVFSHA1Digest.c_str();
-        int vrc = VINF_SUCCESS;
-        size_t i = 1;
-        list<Utf8Str>::const_iterator it1;
-        for (it1 = filesList.begin();
-             it1 != filesList.end();
-             ++it1, ++i)
-        {
-            char* pszDigest;
-            vrc = RTSha1DigestFromFile((*it1).c_str(), &pszDigest, NULL, NULL);
-            pTestList[i].pszTestFile = (char*)(*it1).c_str();
-            pTestList[i].pszTestDigest = pszDigest;
-        }
-
-        // this call can take a very long time
-        size_t cIndexOnError;
-        vrc = RTManifestVerify(strManifestFile.c_str(),
-                               pTestList,
-                               filesList.size() + 1,
-                               &cIndexOnError);
-
-        if (vrc == VERR_MANIFEST_DIGEST_MISMATCH)
-            rc = setError(VBOX_E_FILE_ERROR,
-                          tr("The SHA1 digest of '%s' does not match the one in '%s'"),
-                          RTPathFilename(pTestList[cIndexOnError].pszTestFile),
-                          pcszManifestFileOnly);
-        else if (RT_FAILURE(vrc))
-            rc = setError(VBOX_E_FILE_ERROR,
-                          tr("Could not verify the content of '%s' against the available files (%Rrc)"),
-                          pcszManifestFileOnly,
-                          vrc);
-
-        // clean up
-        for (size_t j = 1;
-             j < filesList.size();
-             ++j)
-            RTStrFree(pTestList[j].pszTestDigest);
-        RTMemFree(pTestList);
-    }
-
-    return rc;
-}
-
-/**
- * Actual worker code for importing OVF data into VirtualBox. This is called from Appliance::taskThreadImportOrExport()
- * and therefore runs on the OVF import worker thread. This creates one or more new machines according to the
- * VirtualSystemScription instances created by Appliance::Interpret().
- *
- * This runs in three contexts:
- *
- * 1) in a first worker thread; in that case, Appliance::ImportMachines() called Appliance::importImpl();
- *
- * 2) in a second worker thread; in that case, Appliance::ImportMachines() called Appliance::importImpl(), which
- *    called Appliance::importFSOVA(), which called Appliance::importImpl(), which then called this again.
- *
- * 3) in a second worker thread; in that case, Appliance::ImportMachines() called Appliance::importImpl(), which
- *    called Appliance::importS3(), which called Appliance::importImpl(), which then called this again.
- *
- * @param pTask
- * @return
- */
-HRESULT Appliance::importFS(TaskOVF *pTask)
-{
-    if (!Utf8Str(RTPathExt(pTask->locInfo.strPath.c_str())).compare(".ovf", Utf8Str::CaseInsensitive))
-        return importFSOVF(pTask);
-    else
-        return importFSOVA(pTask);
-}
-
-HRESULT Appliance::importFSOVF(TaskOVF *pTask)
-{
-    LogFlowFuncEnter();
-    LogFlowFunc(("Appliance %p\n", this));
-
-    AutoCaller autoCaller(this);
-    if (FAILED(autoCaller.rc())) return autoCaller.rc();
-
-    Assert(!pTask->pProgress.isNull());
-
-    // Change the appliance state so we can safely leave the lock while doing time-consuming
-    // disk imports; also the below method calls do all kinds of locking which conflicts with
-    // the appliance object lock
-    AutoWriteLock appLock(this COMMA_LOCKVAL_SRC_POS);
-    if (!isApplianceIdle())
-        return E_ACCESSDENIED;
-    m->state = Data::ApplianceImporting;
-    appLock.release();
-
-    HRESULT rc = S_OK;
-
-    const ovf::OVFReader &reader = *m->pReader;
-    // this is safe to access because this thread only gets started
-    // if pReader != NULL
-
-    // rollback for errors:
-    ImportStack stack(pTask->locInfo, reader.m_mapDisks, pTask->pProgress);
-
-    // clear the list of imported machines, if any
-    m->llGuidsMachinesCreated.clear();
-
-    try
-    {
-        // if a manifest file exists, verify the content; we then need all files which are referenced by the OVF & the OVF itself
-        rc = manifestVerify(pTask->locInfo, reader, pTask->pProgress);
-        if (FAILED(rc)) throw rc;
-
-        // create a session for the machine + disks we manipulate below
-        rc = stack.pSession.createInprocObject(CLSID_Session);
-        if (FAILED(rc)) throw rc;
-
-        list<ovf::VirtualSystem>::const_iterator it;
-        list< ComObjPtr<VirtualSystemDescription> >::const_iterator it1;
-        /* Iterate through all virtual systems of that appliance */
-        size_t i = 0;
-        for (it = reader.m_llVirtualSystems.begin(),
-                it1 = m->virtualSystemDescriptions.begin();
-             it != reader.m_llVirtualSystems.end();
-             ++it, ++it1, ++i)
-        {
-            const ovf::VirtualSystem &vsysThis = *it;
-            ComObjPtr<VirtualSystemDescription> vsdescThis = (*it1);
-
-            ComPtr<IMachine> pNewMachine;
-
-            // there are two ways in which we can create a vbox machine from OVF:
-            // -- either this OVF was written by vbox 3.2 or later, in which case there is a <vbox:Machine> element
-            //    in the <VirtualSystem>; then the VirtualSystemDescription::Data has a settings::MachineConfigFile
-            //    with all the machine config pretty-parsed;
-            // -- or this is an OVF from an older vbox or an external source, and then we need to translate the
-            //    VirtualSystemDescriptionEntry and do import work
-
-            // Even for the vbox:Machine case, there are a number of configuration items that will be taken from
-            // the OVF because otherwise the "override import parameters" mechanism in the GUI won't work.
-
-            // VM name
-            std::list<VirtualSystemDescriptionEntry*> vsdeName = vsdescThis->findByType(VirtualSystemDescriptionType_Name);
-            if (vsdeName.size() < 1)
-                throw setError(VBOX_E_FILE_ERROR,
-                               tr("Missing VM name"));
-            stack.strNameVBox = vsdeName.front()->strVboxCurrent;
-
-            // have VirtualBox suggest where the filename would be placed so we can
-            // put the disk images in the same directory
-            Bstr bstrMachineFilename;
-            rc = mVirtualBox->ComposeMachineFilename(Bstr(stack.strNameVBox).raw(),
-                                                     NULL,
-                                                     bstrMachineFilename.asOutParam());
-            if (FAILED(rc)) throw rc;
-            // and determine the machine folder from that
-            stack.strMachineFolder = bstrMachineFilename;
-            stack.strMachineFolder.stripFilename();
-
-            // guest OS type
-            std::list<VirtualSystemDescriptionEntry*> vsdeOS;
-            vsdeOS = vsdescThis->findByType(VirtualSystemDescriptionType_OS);
-            if (vsdeOS.size() < 1)
-                throw setError(VBOX_E_FILE_ERROR,
-                               tr("Missing guest OS type"));
-            stack.strOsTypeVBox = vsdeOS.front()->strVboxCurrent;
-
-            // CPU count
-            std::list<VirtualSystemDescriptionEntry*> vsdeCPU = vsdescThis->findByType(VirtualSystemDescriptionType_CPU);
-            if (vsdeCPU.size() != 1)
-                throw setError(VBOX_E_FILE_ERROR, tr("CPU count missing"));
-
-            const Utf8Str &cpuVBox = vsdeCPU.front()->strVboxCurrent;
-            stack.cCPUs = (uint32_t)RTStrToUInt64(cpuVBox.c_str());
-            // We need HWVirt & IO-APIC if more than one CPU is requested
-            if (stack.cCPUs > 1)
-            {
-                stack.fForceHWVirt = true;
-                stack.fForceIOAPIC = true;
-            }
-
-            // RAM
-            std::list<VirtualSystemDescriptionEntry*> vsdeRAM = vsdescThis->findByType(VirtualSystemDescriptionType_Memory);
-            if (vsdeRAM.size() != 1)
-                throw setError(VBOX_E_FILE_ERROR, tr("RAM size missing"));
-            const Utf8Str &memoryVBox = vsdeRAM.front()->strVboxCurrent;
-            stack.ulMemorySizeMB = (uint32_t)RTStrToUInt64(memoryVBox.c_str());
-
-#ifdef VBOX_WITH_USB
-            // USB controller
-            std::list<VirtualSystemDescriptionEntry*> vsdeUSBController = vsdescThis->findByType(VirtualSystemDescriptionType_USBController);
-            // USB support is enabled if there's at least one such entry; to disable USB support,
-            // the type of the USB item would have been changed to "ignore"
-            stack.fUSBEnabled = vsdeUSBController.size() > 0;
-#endif
-            // audio adapter
-            std::list<VirtualSystemDescriptionEntry*> vsdeAudioAdapter = vsdescThis->findByType(VirtualSystemDescriptionType_SoundCard);
-            /* @todo: we support one audio adapter only */
-            if (vsdeAudioAdapter.size() > 0)
-                stack.strAudioAdapter = vsdeAudioAdapter.front()->strVboxCurrent;
-
-            // for the description of the new machine, always use the OVF entry, the user may have changed it in the import config
-            std::list<VirtualSystemDescriptionEntry*> vsdeDescription = vsdescThis->findByType(VirtualSystemDescriptionType_Description);
-            if (vsdeDescription.size())
-                stack.strDescription = vsdeDescription.front()->strVboxCurrent;
-
-            // import vbox:machine or OVF now
-            if (vsdescThis->m->pConfig)
-                // vbox:Machine config
-                importVBoxMachine(vsdescThis, pNewMachine, stack);
-            else
-                // generic OVF config
-                importMachineGeneric(vsysThis, vsdescThis, pNewMachine, stack);
-
-        } // for (it = pAppliance->m->llVirtualSystems.begin() ...
-    }
-    catch (HRESULT rc2)
-    {
-        rc = rc2;
-    }
-
-    if (FAILED(rc))
-    {
-        // with _whatever_ error we've had, do a complete roll-back of
-        // machines and disks we've created
-
-        for (list<Guid>::iterator itID = m->llGuidsMachinesCreated.begin();
-             itID != m->llGuidsMachinesCreated.end();
-             ++itID)
-        {
-            Guid guid = *itID;
-            Bstr bstrGuid = guid.toUtf16();
-            ComPtr<IMachine> failedMachine;
-            HRESULT rc2 = mVirtualBox->GetMachine(bstrGuid.raw(), failedMachine.asOutParam());
-            if (SUCCEEDED(rc2))
-            {
-                SafeIfaceArray<IMedium> aMedia;
-                rc2 = failedMachine->Unregister(CleanupMode_DetachAllReturnHardDisksOnly, ComSafeArrayAsOutParam(aMedia));
-                ComPtr<IProgress> pProgress2;
-                rc2 = failedMachine->Delete(ComSafeArrayAsInParam(aMedia), pProgress2.asOutParam());
-                pProgress2->WaitForCompletion(-1);
-            }
-        }
-    }
-
-    // restore the appliance state
-    appLock.acquire();
-    m->state = Data::ApplianceIdle;
-    appLock.release();
-
-    LogFlowFunc(("rc=%Rhrc\n", rc));
-    LogFlowFuncLeave();
-
-    return rc;
-}
-
-HRESULT Appliance::importFSOVA(TaskOVF *pTask)
-{
-    LogFlowFuncEnter();
-    LogFlowFunc(("Appliance %p\n", this));
-
-    AutoCaller autoCaller(this);
-    if (FAILED(autoCaller.rc())) return autoCaller.rc();
-
-    AutoWriteLock appLock(this COMMA_LOCKVAL_SRC_POS);
-
-    int vrc = VINF_SUCCESS;
-    char szOSTmpDir[RTPATH_MAX];
-    RTPathTemp(szOSTmpDir, sizeof(szOSTmpDir));
-    /* The template for the temporary directory created below */
-    char *pszTmpDir;
-    RTStrAPrintf(&pszTmpDir, "%s"RTPATH_SLASH_STR"vbox-ovf-XXXXXX", szOSTmpDir);
-    list< pair<Utf8Str, ULONG> > filesList;
-    const char** paFiles = 0;
-
-    HRESULT rc = S_OK;
-    try
-    {
-        /* Extract the path */
-        Utf8Str tmpPath = pTask->locInfo.strPath;
-        /* Remove the ova extension */
-        tmpPath.stripExt();
-        tmpPath += ".ovf";
-
-        /* We need a temporary directory which we can put the all disk images
-         * in */
-        vrc = RTDirCreateTemp(pszTmpDir);
-        if (RT_FAILURE(vrc))
-            throw setError(VBOX_E_FILE_ERROR,
-                           tr("Cannot create temporary directory '%s' (%Rrc)"), pszTmpDir, vrc);
-
-        /* Provide a OVF file (haven't to exist) so the import routine can
-         * figure out where the disk images/manifest file are located. */
-        Utf8StrFmt strTmpOvf("%s/%s", pszTmpDir, RTPathFilename(tmpPath.c_str()));
-        /* Add the manifest file to the list of files to extract, but only if
-           one is in the archive. */
-        Utf8Str strManifestFile = queryManifestFileName(strTmpOvf);
-        if (!strManifestFile.isEmpty())
-            filesList.push_back(pair<Utf8Str, ULONG>(strManifestFile.c_str(), 1));
-
-        ULONG ulWeight = m->ulWeightForXmlOperation;
-        /* Add every disks of every virtual system to an internal list */
-        list< ComObjPtr<VirtualSystemDescription> >::const_iterator it;
-        for (it = m->virtualSystemDescriptions.begin();
-             it != m->virtualSystemDescriptions.end();
-             ++it)
-        {
-            ComObjPtr<VirtualSystemDescription> vsdescThis = (*it);
-            std::list<VirtualSystemDescriptionEntry*> avsdeHDs = vsdescThis->findByType(VirtualSystemDescriptionType_HardDiskImage);
-            std::list<VirtualSystemDescriptionEntry*>::const_iterator itH;
-            for (itH = avsdeHDs.begin();
-                 itH != avsdeHDs.end();
-                 ++itH)
-            {
-                const Utf8Str &strTargetFile = (*itH)->strOvf;
-                if (!strTargetFile.isEmpty())
-                {
-                    /* The temporary name of the target disk file */
-                    Utf8StrFmt strTmpDisk("%s/%s", pszTmpDir, RTPathFilename(strTargetFile.c_str()));
-                    filesList.push_back(pair<Utf8Str, ULONG>(strTmpDisk, (*itH)->ulSizeMB));
-                    ulWeight += (*itH)->ulSizeMB;
-                }
-            }
-        }
-
-        /* Download all files */
-        paFiles = (const char**)RTMemAlloc(sizeof(char*) * filesList.size());
-        int i = 0;
-        for (list< pair<Utf8Str, ULONG> >::const_iterator it1 = filesList.begin(); it1 != filesList.end(); ++it1, ++i)
-            paFiles[i] = RTPathFilename((*it1).first.c_str());
-        if (!pTask->pProgress.isNull())
-            pTask->pProgress->SetNextOperation(BstrFmt(tr("Unpacking file '%s'"), RTPathFilename(pTask->locInfo.strPath.c_str())).raw(), ulWeight);
-        vrc = RTTarExtractFiles(pTask->locInfo.strPath.c_str(), pszTmpDir, paFiles, filesList.size(), pTask->updateProgress, &pTask);
-        if (RT_FAILURE(vrc))
-            throw setError(VBOX_E_FILE_ERROR,
-                           tr("Cannot unpack archive file '%s' (%Rrc)"), pTask->locInfo.strPath.c_str(), vrc);
-
-//        if (!pTask->pProgress.isNull())
-//            pTask->pProgress->SetNextOperation(BstrFmt(tr("Importing appliance")), m->ulWeightForXmlOperation);
-
-        ComObjPtr<Progress> progress;
-        /* Import the whole temporary OVF & the disk images */
-        LocationInfo li;
-        li.strPath = strTmpOvf;
-        rc = importImpl(li, progress);
-        if (FAILED(rc)) throw rc;
-
-        /* Unlock the appliance for the fs import thread */
-        appLock.release();
-        /* Wait until the import is done, but report the progress back to the
-           caller */
-        ComPtr<IProgress> progressInt(progress);
-        waitForAsyncProgress(pTask->pProgress, progressInt); /* Any errors will be thrown */
-
-        /* Again lock the appliance for the next steps */
-        appLock.acquire();
-    }
-    catch(HRESULT aRC)
-    {
-        rc = aRC;
-    }
-    /* Delete the temporary files list */
-    if (paFiles)
-        RTMemFree(paFiles);
-    /* Delete all files which where temporary created */
-    for (list< pair<Utf8Str, ULONG> >::const_iterator it1 = filesList.begin(); it1 != filesList.end(); ++it1)
-    {
-        const char *pszFilePath = (*it1).first.c_str();
-        if (RTPathExists(pszFilePath))
-        {
-            vrc = RTFileDelete(pszFilePath);
-            if (RT_FAILURE(vrc))
-                rc = setError(VBOX_E_FILE_ERROR,
-                              tr("Cannot delete file '%s' (%Rrc)"), pszFilePath, vrc);
-        }
-    }
-    /* Delete the temporary directory */
-    if (RTPathExists(pszTmpDir))
-    {
-        vrc = RTDirRemove(pszTmpDir);
-        if (RT_FAILURE(vrc))
-            rc = setError(VBOX_E_FILE_ERROR,
-                          tr("Cannot delete temporary directory '%s' (%Rrc)"), pszTmpDir, vrc);
-    }
-    if (pszTmpDir)
-        RTStrFree(pszTmpDir);
-
-    LogFlowFunc(("rc=%Rhrc\n", rc));
-    LogFlowFuncLeave();
-
-    return rc;
-}
-
-/**
  * Imports one disk image. This is common code shared between
  *  --  importMachineGeneric() for the OVF case; in that case the information comes from
@@ -1523 +1662 @@
 void Appliance::importOneDiskImage(const ovf::DiskImage &di,
                                    const Utf8Str &strTargetPath,
-                                   ComPtr<IMedium> &pTargetHD,
-                                   ImportStack &stack)
+                                   ComObjPtr<Medium> &pTargetHD,
+                                   ImportStack &stack,
+                                   PVDINTERFACEIO pCallbacks,
+                                   PSHA1STORAGE pStorage)
 {
-    ComPtr<IMedium> pSourceHD;
-    bool fSourceHdNeedsClosing = false;
-
-    try
-    {
-        // destination file must not exist
-        if (    strTargetPath.isEmpty()
-             || RTPathExists(strTargetPath.c_str())
+    // destination file must not exist
+    if (    strTargetPath.isEmpty()
+        || RTPathExists(strTargetPath.c_str())
+       )
+        throw setError(VBOX_E_FILE_ERROR,
+                       tr("Destination file '%s' exists"),
+                       strTargetPath.c_str());
+
+    const Utf8Str &strSourceOVF = di.strHref;
+
+    // construct source file path
+    Utf8StrFmt strSrcFilePath("%s%c%s", stack.strSourceDir.c_str(), RTPATH_DELIMITER, strSourceOVF.c_str());
+
+    // subprogress object for hard disk
+    ComPtr<IProgress> pProgress2;
+
+    /**************** Check */
+    ComObjPtr<Progress> pProgress3;
+    pProgress3.createObject();
+    HRESULT rc = pProgress3->init(mVirtualBox, static_cast<IAppliance*>(this), BstrFmt(tr("Creating medium")).raw(), TRUE);
+    //        rc = pProgress3->init(mVirtualBox, static_cast<IAppliance*>(this), BstrFmt(tr("Creating medium '%s'"), strTargetFilePath.c_str()).raw(), TRUE);
+    if (FAILED(rc)) throw rc;
+    /**************** Check */
+
+    pTargetHD.createObject();
+    bool fNeedsGlobalSaveSettings;
+    // create an empty hard disk
+    rc = pTargetHD->init(mVirtualBox,
+                         Utf8Str("VMDK"),
+                         strTargetPath,
+                         Guid::Empty,        // media registry
+                         &fNeedsGlobalSaveSettings);
+    if (FAILED(rc)) throw rc;
+
+    /* If strHref is empty we have to create a new file */
+    if (strSourceOVF.isEmpty())
+    {
+        // which format to use?
+        Bstr srcFormat = L"VDI";
+        if (   di.strFormat.compare("http://www.vmware.com/specifications/vmdk.html#sparse", Utf8Str::CaseInsensitive)
+               || di.strFormat.compare("http://www.vmware.com/interfaces/specifications/vmdk.html#streamOptimized", Utf8Str::CaseInsensitive)
+               || di.strFormat.compare("http://www.vmware.com/specifications/vmdk.html#compressed", Utf8Str::CaseInsensitive)
+               || di.strFormat.compare("http://www.vmware.com/interfaces/specifications/vmdk.html#compressed", Utf8Str::CaseInsensitive)
            )
-            throw setError(VBOX_E_FILE_ERROR,
-                           tr("Destination file '%s' exists"),
-                           strTargetPath.c_str());
-
-        const Utf8Str &strSourceOVF = di.strHref;
-
-        // Make sure target directory exists
-        HRESULT rc = VirtualBox::ensureFilePathExists(strTargetPath.c_str());
+            srcFormat = L"VMDK";
+        rc = mVirtualBox->CreateHardDisk(srcFormat.raw(),
+                                         Bstr(strTargetPath).raw(),
+                                         ComPtr<IMedium>(pTargetHD).asOutParam());
         if (FAILED(rc)) throw rc;
 
-        // subprogress object for hard disk
-        ComPtr<IProgress> pProgress2;
-
-        /* If strHref is empty we have to create a new file */
-        if (strSourceOVF.isEmpty())
-        {
-            // which format to use?
-            Bstr srcFormat = L"VDI";
-            if (   di.strFormat.compare("http://www.vmware.com/specifications/vmdk.html#sparse", Utf8Str::CaseInsensitive)
-                || di.strFormat.compare("http://www.vmware.com/interfaces/specifications/vmdk.html#streamOptimized", Utf8Str::CaseInsensitive)
-                || di.strFormat.compare("http://www.vmware.com/specifications/vmdk.html#compressed", Utf8Str::CaseInsensitive)
-                || di.strFormat.compare("http://www.vmware.com/interfaces/specifications/vmdk.html#compressed", Utf8Str::CaseInsensitive)
-               )
-                srcFormat = L"VMDK";
-            // create an empty hard disk
-            rc = mVirtualBox->CreateHardDisk(srcFormat.raw(),
-                                             Bstr(strTargetPath).raw(),
-                                             pTargetHD.asOutParam());
-            if (FAILED(rc)) throw rc;
-
-            // create a dynamic growing disk image with the given capacity
-            rc = pTargetHD->CreateBaseStorage(di.iCapacity / _1M, MediumVariant_Standard, pProgress2.asOutParam());
-            if (FAILED(rc)) throw rc;
-
-            // advance to the next operation
-            stack.pProgress->SetNextOperation(BstrFmt(tr("Creating disk image '%s'"), strTargetPath.c_str()).raw(),
-                                              di.ulSuggestedSizeMB);     // operation's weight, as set up with the IProgress originally
+        // create a dynamic growing disk image with the given capacity
+        rc = pTargetHD->CreateBaseStorage(di.iCapacity / _1M, MediumVariant_Standard, pProgress2.asOutParam());
+        if (FAILED(rc)) throw rc;
+
+        // advance to the next operation
+        stack.pProgress->SetNextOperation(BstrFmt(tr("Creating disk image '%s'"), strTargetPath.c_str()).raw(),
+                                          di.ulSuggestedSizeMB);     // operation's weight, as set up with the IProgress originally
+    }
+    else
+    {
+        /* Create a new hard disk interface for the destination disk image */
+        rc = mVirtualBox->CreateHardDisk(NULL,
+                                         Bstr(strTargetPath).raw(),
+                                         ComPtr<IMedium>(pTargetHD).asOutParam());
+        if (FAILED(rc)) throw rc;
+
+        // We need a proper format description
+        ComObjPtr<MediumFormat> format;
+        // Scope for the AutoReadLock
+        {
+            SystemProperties *pSysProps = mVirtualBox->getSystemProperties();
+            AutoReadLock propsLock(pSysProps COMMA_LOCKVAL_SRC_POS);
+            // We are always exporting to VMDK stream optimized for now
+            format = pSysProps->mediumFormat("VMDK");
+            if (format.isNull())
+                throw setError(VBOX_E_NOT_SUPPORTED,
+                               tr("Invalid medium storage format"));
         }
-        else
-        {
-            // construct source file path
-            Utf8StrFmt strSrcFilePath("%s%c%s", stack.strSourceDir.c_str(), RTPATH_DELIMITER, strSourceOVF.c_str());
-
-            // Do NOT check here whether the file exists. The clone operation
-            // will figure that out, and filesystem-based tests are simply
-            // wrong in the general case (think of iSCSI).
-
-            // Clone the disk image (this is necessary cause the id has
-            // to be recreated for the case the same hard disk is
-            // attached already from a previous import)
-
-            // First open the existing disk image
-            rc = mVirtualBox->OpenMedium(Bstr(strSrcFilePath).raw(),
-                                         DeviceType_HardDisk,
-                                         AccessMode_ReadOnly,
-                                         pSourceHD.asOutParam());
-            if (FAILED(rc)) throw rc;
-            fSourceHdNeedsClosing = true;
-
-            /* We need the format description of the source disk image */
-            Bstr srcFormat;
-            rc = pSourceHD->COMGETTER(Format)(srcFormat.asOutParam());
-            if (FAILED(rc)) throw rc;
-            /* Create a new hard disk interface for the destination disk image */
-            rc = mVirtualBox->CreateHardDisk(srcFormat.raw(),
-                                             Bstr(strTargetPath).raw(),
-                                             pTargetHD.asOutParam());
-            if (FAILED(rc)) throw rc;
-            /* Clone the source disk image */
-            rc = pSourceHD->CloneTo(pTargetHD, MediumVariant_Standard, NULL, pProgress2.asOutParam());
-            if (FAILED(rc)) throw rc;
-
-            /* Advance to the next operation */
-            stack.pProgress->SetNextOperation(BstrFmt(tr("Importing virtual disk image '%s'"), RTPathFilename(strSrcFilePath.c_str())).raw(),
-                                              di.ulSuggestedSizeMB);     // operation's weight, as set up with the IProgress originally);
-        }
-
-        // now wait for the background disk operation to complete; this throws HRESULTs on error
-        waitForAsyncProgress(stack.pProgress, pProgress2);
-
-        if (fSourceHdNeedsClosing)
-        {
-            rc = pSourceHD->Close();
-            if (FAILED(rc)) throw rc;
-            fSourceHdNeedsClosing = false;
-        }
-
-        stack.llHardDisksCreated.push_back(pTargetHD);
-    }
-    catch (...)
-    {
-        if (fSourceHdNeedsClosing)
-            pSourceHD->Close();
-
-        throw;
-    }
+
+        /* Clone the source disk image */
+        ComObjPtr<Medium> nullParent;
+        rc = pTargetHD->importFile(strSrcFilePath.c_str(),
+                                   format,
+                                   MediumVariant_Standard,
+                                   pCallbacks, pStorage,
+                                   nullParent,
+                                   pProgress3);
+
+        /* Advance to the next operation */
+        stack.pProgress->SetNextOperation(BstrFmt(tr("Importing virtual disk image '%s'"), RTPathFilename(strSrcFilePath.c_str())).raw(),
+                                          di.ulSuggestedSizeMB);     // operation's weight, as set up with the IProgress originally);
+    }
+
+    // now wait for the background disk operation to complete; this throws HRESULTs on error
+    ComPtr<IProgress> pp(pProgress3);
+    waitForAsyncProgress(stack.pProgress, pp);
+
+    stack.llSrcDisksDigest.push_back(STRPAIR(strSrcFilePath, pStorage->strDigest));
 }
 
@@ -1649 +1783 @@
                                      ComObjPtr<VirtualSystemDescription> &vsdescThis,
                                      ComPtr<IMachine> &pNewMachine,
-                                     ImportStack &stack)
+                                     ImportStack &stack,
+                                     PVDINTERFACEIO pCallbacks,
+                                     PSHA1STORAGE pStorage)
 {
     HRESULT rc;
@@ -2112 +2248 @@
                 const ovf::VirtualDisk &ovfVdisk = itVirtualDisk->second;
 
-                ComPtr<IMedium> pTargetHD;
+                ComObjPtr<Medium> pTargetHD;
                 importOneDiskImage(ovfDiskImage,
                                    vsdeHD->strVboxCurrent,
                                    pTargetHD,
-                                   stack);
+                                   stack,
+                                   pCallbacks,
+                                   pStorage);
 
                 // now use the new uuid to attach the disk image to our new machine
@@ -2197 +2335 @@
 void Appliance::importVBoxMachine(ComObjPtr<VirtualSystemDescription> &vsdescThis,
                                   ComPtr<IMachine> &pReturnNewMachine,
-                                  ImportStack &stack)
+                                  ImportStack &stack,
+                                  PVDINTERFACEIO pCallbacks,
+                                  PSHA1STORAGE pStorage)
 {
     Assert(vsdescThis->m->pConfig);
@@ -2304 +2444 @@
                      *
                      */
-                    ComPtr<IMedium> pTargetHD;
+                    ComObjPtr<Medium> pTargetHD;
                     importOneDiskImage(di,
                                        strTargetPath,
                                        pTargetHD,
-                                       stack);
+                                       stack,
+                                       pCallbacks,
+                                       pStorage);
 
                     // ... and replace the old UUID in the machine config with the one of
@@ -2365 +2507 @@
 }
 
-/**
- * Worker code for importing OVF from the cloud. This is called from Appliance::taskThreadImportOrExport()
- * in S3 mode and therefore runs on the OVF import worker thread. This then starts a second worker
- * thread to import from temporary files (see Appliance::importFS()).
- * @param pTask
- * @return
- */
-HRESULT Appliance::importS3(TaskOVF *pTask)
+void Appliance::importMachines(ImportStack &stack,
+                               PVDINTERFACEIO pCallbacks,
+                               PSHA1STORAGE pStorage)
 {
-    LogFlowFuncEnter();
-    LogFlowFunc(("Appliance %p\n", this));
-
-    AutoCaller autoCaller(this);
-    if (FAILED(autoCaller.rc())) return autoCaller.rc();
-
-    AutoWriteLock appLock(this COMMA_LOCKVAL_SRC_POS);
-
-    int vrc = VINF_SUCCESS;
-    RTS3 hS3 = NIL_RTS3;
-    char szOSTmpDir[RTPATH_MAX];
-    RTPathTemp(szOSTmpDir, sizeof(szOSTmpDir));
-    /* The template for the temporary directory created below */
-    char *pszTmpDir;
-    RTStrAPrintf(&pszTmpDir, "%s"RTPATH_SLASH_STR"vbox-ovf-XXXXXX", szOSTmpDir);
-    list< pair<Utf8Str, ULONG> > filesList;
-
     HRESULT rc = S_OK;
-    try
-    {
-        /* Extract the bucket */
-        Utf8Str tmpPath = pTask->locInfo.strPath;
-        Utf8Str bucket;
-        parseBucket(tmpPath, bucket);
-
-        /* We need a temporary directory which we can put the all disk images
-         * in */
-        vrc = RTDirCreateTemp(pszTmpDir);
-        if (RT_FAILURE(vrc))
+
+    // this is safe to access because this thread only gets started
+    // if pReader != NULL
+    const ovf::OVFReader &reader = *m->pReader;
+
+    // create a session for the machine + disks we manipulate below
+    rc = stack.pSession.createInprocObject(CLSID_Session);
+    if (FAILED(rc)) throw rc;
+
+    list<ovf::VirtualSystem>::const_iterator it;
+    list< ComObjPtr<VirtualSystemDescription> >::const_iterator it1;
+    /* Iterate through all virtual systems of that appliance */
+    size_t i = 0;
+    for (it = reader.m_llVirtualSystems.begin(),
+         it1 = m->virtualSystemDescriptions.begin();
+         it != reader.m_llVirtualSystems.end();
+         ++it, ++it1, ++i)
+    {
+        const ovf::VirtualSystem &vsysThis = *it;
+        ComObjPtr<VirtualSystemDescription> vsdescThis = (*it1);
+
+        ComPtr<IMachine> pNewMachine;
+
+        // there are two ways in which we can create a vbox machine from OVF:
+        // -- either this OVF was written by vbox 3.2 or later, in which case there is a <vbox:Machine> element
+        //    in the <VirtualSystem>; then the VirtualSystemDescription::Data has a settings::MachineConfigFile
+        //    with all the machine config pretty-parsed;
+        // -- or this is an OVF from an older vbox or an external source, and then we need to translate the
+        //    VirtualSystemDescriptionEntry and do import work
+
+        // Even for the vbox:Machine case, there are a number of configuration items that will be taken from
+        // the OVF because otherwise the "override import parameters" mechanism in the GUI won't work.
+
+        // VM name
+        std::list<VirtualSystemDescriptionEntry*> vsdeName = vsdescThis->findByType(VirtualSystemDescriptionType_Name);
+        if (vsdeName.size() < 1)
             throw setError(VBOX_E_FILE_ERROR,
-                           tr("Cannot create temporary directory '%s' (%Rrc)"), pszTmpDir, vrc);
-
-        /* Add every disks of every virtual system to an internal list */
-        list< ComObjPtr<VirtualSystemDescription> >::const_iterator it;
-        for (it = m->virtualSystemDescriptions.begin();
-             it != m->virtualSystemDescriptions.end();
-             ++it)
-        {
-            ComObjPtr<VirtualSystemDescription> vsdescThis = (*it);
-            std::list<VirtualSystemDescriptionEntry*> avsdeHDs = vsdescThis->findByType(VirtualSystemDescriptionType_HardDiskImage);
-            std::list<VirtualSystemDescriptionEntry*>::const_iterator itH;
-            for (itH = avsdeHDs.begin();
-                 itH != avsdeHDs.end();
-                 ++itH)
-            {
-                const Utf8Str &strTargetFile = (*itH)->strOvf;
-                if (!strTargetFile.isEmpty())
-                {
-                    /* The temporary name of the target disk file */
-                    Utf8StrFmt strTmpDisk("%s/%s", pszTmpDir, RTPathFilename(strTargetFile.c_str()));
-                    filesList.push_back(pair<Utf8Str, ULONG>(strTmpDisk, (*itH)->ulSizeMB));
-                }
-            }
+                           tr("Missing VM name"));
+        stack.strNameVBox = vsdeName.front()->strVboxCurrent;
+
+        // have VirtualBox suggest where the filename would be placed so we can
+        // put the disk images in the same directory
+        Bstr bstrMachineFilename;
+        rc = mVirtualBox->ComposeMachineFilename(Bstr(stack.strNameVBox).raw(),
+                                                 NULL,
+                                                 bstrMachineFilename.asOutParam());
+        if (FAILED(rc)) throw rc;
+        // and determine the machine folder from that
+        stack.strMachineFolder = bstrMachineFilename;
+        stack.strMachineFolder.stripFilename();
+
+        // guest OS type
+        std::list<VirtualSystemDescriptionEntry*> vsdeOS;
+        vsdeOS = vsdescThis->findByType(VirtualSystemDescriptionType_OS);
+        if (vsdeOS.size() < 1)
+            throw setError(VBOX_E_FILE_ERROR,
+                           tr("Missing guest OS type"));
+        stack.strOsTypeVBox = vsdeOS.front()->strVboxCurrent;
+
+        // CPU count
+        std::list<VirtualSystemDescriptionEntry*> vsdeCPU = vsdescThis->findByType(VirtualSystemDescriptionType_CPU);
+        if (vsdeCPU.size() != 1)
+            throw setError(VBOX_E_FILE_ERROR, tr("CPU count missing"));
+
+        const Utf8Str &cpuVBox = vsdeCPU.front()->strVboxCurrent;
+        stack.cCPUs = (uint32_t)RTStrToUInt64(cpuVBox.c_str());
+        // We need HWVirt & IO-APIC if more than one CPU is requested
+        if (stack.cCPUs > 1)
+        {
+            stack.fForceHWVirt = true;
+            stack.fForceIOAPIC = true;
         }
 
-        /* Next we have to download the disk images */
-        vrc = RTS3Create(&hS3, pTask->locInfo.strUsername.c_str(), pTask->locInfo.strPassword.c_str(), pTask->locInfo.strHostname.c_str(), "virtualbox-agent/"VBOX_VERSION_STRING);
-        if (RT_FAILURE(vrc))
-            throw setError(VBOX_E_IPRT_ERROR,
-                           tr("Cannot create S3 service handler"));
-        RTS3SetProgressCallback(hS3, pTask->updateProgress, &pTask);
-
-        /* Download all files */
-        for (list< pair<Utf8Str, ULONG> >::const_iterator it1 = filesList.begin(); it1 != filesList.end(); ++it1)
-        {
-            const pair<Utf8Str, ULONG> &s = (*it1);
-            const Utf8Str &strSrcFile = s.first;
-            /* Construct the source file name */
-            char *pszFilename = RTPathFilename(strSrcFile.c_str());
-            /* Advance to the next operation */
-            if (!pTask->pProgress.isNull())
-                pTask->pProgress->SetNextOperation(BstrFmt(tr("Downloading file '%s'"), pszFilename).raw(), s.second);
-
-            vrc = RTS3GetKey(hS3, bucket.c_str(), pszFilename, strSrcFile.c_str());
-            if (RT_FAILURE(vrc))
-            {
-                if (vrc == VERR_S3_CANCELED)
-                    throw S_OK; /* todo: !!!!!!!!!!!!! */
-                else if (vrc == VERR_S3_ACCESS_DENIED)
-                    throw setError(E_ACCESSDENIED,
-                                   tr("Cannot download file '%s' from S3 storage server (Access denied). "
-                                      "Make sure that your credentials are right. Also check that your host clock is properly synced"),
-                                   pszFilename);
-                else if (vrc == VERR_S3_NOT_FOUND)
-                    throw setError(VBOX_E_FILE_ERROR,
-                                   tr("Cannot download file '%s' from S3 storage server (File not found)"),
-                                   pszFilename);
-                else
-                    throw setError(VBOX_E_IPRT_ERROR,
-                                   tr("Cannot download file '%s' from S3 storage server (%Rrc)"),
-                                   pszFilename, vrc);
-            }
-        }
-
-        /* Provide a OVF file (haven't to exist) so the import routine can
-         * figure out where the disk images/manifest file are located. */
-        Utf8StrFmt strTmpOvf("%s/%s", pszTmpDir, RTPathFilename(tmpPath.c_str()));
-        /* Now check if there is an manifest file. This is optional. */
-        Utf8Str strManifestFile = queryManifestFileName(strTmpOvf);
-        char *pszFilename = RTPathFilename(strManifestFile.c_str());
-        if (!pTask->pProgress.isNull())
-            pTask->pProgress->SetNextOperation(BstrFmt(tr("Downloading file '%s'"), pszFilename).raw(), 1);
-
-        /* Try to download it. If the error is VERR_S3_NOT_FOUND, it isn't fatal. */
-        vrc = RTS3GetKey(hS3, bucket.c_str(), pszFilename, strManifestFile.c_str());
-        if (RT_SUCCESS(vrc))
-            filesList.push_back(pair<Utf8Str, ULONG>(strManifestFile, 0));
-        else if (RT_FAILURE(vrc))
-        {
-            if (vrc == VERR_S3_CANCELED)
-                throw S_OK; /* todo: !!!!!!!!!!!!! */
-            else if (vrc == VERR_S3_NOT_FOUND)
-                vrc = VINF_SUCCESS; /* Not found is ok */
-            else if (vrc == VERR_S3_ACCESS_DENIED)
-                throw setError(E_ACCESSDENIED,
-                               tr("Cannot download file '%s' from S3 storage server (Access denied)."
-                                  "Make sure that your credentials are right. Also check that your host clock is properly synced"),
-                               pszFilename);
-            else
-                throw setError(VBOX_E_IPRT_ERROR,
-                               tr("Cannot download file '%s' from S3 storage server (%Rrc)"),
-                               pszFilename, vrc);
-        }
-
-        /* Close the connection early */
-        RTS3Destroy(hS3);
-        hS3 = NIL_RTS3;
-
-        pTask->pProgress->SetNextOperation(BstrFmt(tr("Importing appliance")).raw(), m->ulWeightForXmlOperation);
-
-        ComObjPtr<Progress> progress;
-        /* Import the whole temporary OVF & the disk images */
-        LocationInfo li;
-        li.strPath = strTmpOvf;
-        rc = importImpl(li, progress);
-        if (FAILED(rc)) throw rc;
-
-        /* Unlock the appliance for the fs import thread */
-        appLock.release();
-        /* Wait until the import is done, but report the progress back to the
-           caller */
-        ComPtr<IProgress> progressInt(progress);
-        waitForAsyncProgress(pTask->pProgress, progressInt); /* Any errors will be thrown */
-
-        /* Again lock the appliance for the next steps */
-        appLock.acquire();
-    }
-    catch(HRESULT aRC)
-    {
-        rc = aRC;
-    }
-    /* Cleanup */
-    RTS3Destroy(hS3);
-    /* Delete all files which where temporary created */
-    for (list< pair<Utf8Str, ULONG> >::const_iterator it1 = filesList.begin(); it1 != filesList.end(); ++it1)
-    {
-        const char *pszFilePath = (*it1).first.c_str();
-        if (RTPathExists(pszFilePath))
-        {
-            vrc = RTFileDelete(pszFilePath);
-            if (RT_FAILURE(vrc))
-                rc = setError(VBOX_E_FILE_ERROR,
-                              tr("Cannot delete file '%s' (%Rrc)"), pszFilePath, vrc);
-        }
-    }
-    /* Delete the temporary directory */
-    if (RTPathExists(pszTmpDir))
-    {
-        vrc = RTDirRemove(pszTmpDir);
-        if (RT_FAILURE(vrc))
-            rc = setError(VBOX_E_FILE_ERROR,
-                          tr("Cannot delete temporary directory '%s' (%Rrc)"), pszTmpDir, vrc);
-    }
-    if (pszTmpDir)
-        RTStrFree(pszTmpDir);
-
-    LogFlowFunc(("rc=%Rhrc\n", rc));
-    LogFlowFuncLeave();
-
-    return rc;
+        // RAM
+        std::list<VirtualSystemDescriptionEntry*> vsdeRAM = vsdescThis->findByType(VirtualSystemDescriptionType_Memory);
+        if (vsdeRAM.size() != 1)
+            throw setError(VBOX_E_FILE_ERROR, tr("RAM size missing"));
+        const Utf8Str &memoryVBox = vsdeRAM.front()->strVboxCurrent;
+        stack.ulMemorySizeMB = (uint32_t)RTStrToUInt64(memoryVBox.c_str());
+
+#ifdef VBOX_WITH_USB
+        // USB controller
+        std::list<VirtualSystemDescriptionEntry*> vsdeUSBController = vsdescThis->findByType(VirtualSystemDescriptionType_USBController);
+        // USB support is enabled if there's at least one such entry; to disable USB support,
+        // the type of the USB item would have been changed to "ignore"
+        stack.fUSBEnabled = vsdeUSBController.size() > 0;
+#endif
+        // audio adapter
+        std::list<VirtualSystemDescriptionEntry*> vsdeAudioAdapter = vsdescThis->findByType(VirtualSystemDescriptionType_SoundCard);
+        /* @todo: we support one audio adapter only */
+        if (vsdeAudioAdapter.size() > 0)
+            stack.strAudioAdapter = vsdeAudioAdapter.front()->strVboxCurrent;
+
+        // for the description of the new machine, always use the OVF entry, the user may have changed it in the import config
+        std::list<VirtualSystemDescriptionEntry*> vsdeDescription = vsdescThis->findByType(VirtualSystemDescriptionType_Description);
+        if (vsdeDescription.size())
+            stack.strDescription = vsdeDescription.front()->strVboxCurrent;
+
+        // import vbox:machine or OVF now
+        if (vsdescThis->m->pConfig)
+            // vbox:Machine config
+            importVBoxMachine(vsdescThis, pNewMachine, stack, pCallbacks, pStorage);
+        else
+            // generic OVF config
+            importMachineGeneric(vsysThis, vsdescThis, pNewMachine, stack, pCallbacks, pStorage);
+
+    } // for (it = pAppliance->m->llVirtualSystems.begin() ...
 }
 

trunk/src/VBox/Main/include/ApplianceImpl.h

@@ -24 +24 @@
 #include "VirtualBoxBase.h"
 
+/* Todo: This file needs massive cleanup. Split IAppliance in a public and
+ * private classes. */
+#include <iprt/tar.h>
+
 /* VBox forward declarations */
 class Progress;
@@ -114 +118 @@
     VirtualBox* const   mVirtualBox;
 
+    struct ImportStack;
+    struct TaskOVF;
+    struct LocationInfo;
     struct Data;            // opaque, defined in ApplianceImpl.cpp
     Data *m;
 
+    enum SetUpProgressMode { ImportFile, ImportS3, WriteFile, WriteS3 };
+
+    /*******************************************************************************
+     * General stuff
+     ******************************************************************************/
+
     bool isApplianceIdle();
-
     HRESULT searchUniqueVMName(Utf8Str& aName) const;
     HRESULT searchUniqueDiskImageFilePath(Utf8Str& aName) const;
+    HRESULT setUpProgress(ComObjPtr<Progress> &pProgress,
+                          const Bstr &bstrDescription,
+                          SetUpProgressMode mode);
     void waitForAsyncProgress(ComObjPtr<Progress> &pProgressThis, ComPtr<IProgress> &pProgressAsync);
     void addWarning(const char* aWarning, ...);
-
     void disksWeight();
-    struct LocationInfo;
-    enum SetUpProgressMode { ImportFileWithManifest, ImportFileNoManifest, ImportS3, WriteFile, WriteS3 };
-    HRESULT setUpProgress(const LocationInfo &locInfo,
-                          ComObjPtr<Progress> &pProgress,
-                          const Bstr &bstrDescription,
-                          SetUpProgressMode mode);
-
     void parseURI(Utf8Str strUri, LocationInfo &locInfo) const;
     void parseBucket(Utf8Str &aPath, Utf8Str &aBucket);
-    Utf8Str manifestFileName(const Utf8Str& aPath) const;
-    Utf8Str queryManifestFileName(const Utf8Str& aPath) const;
+
+    static DECLCALLBACK(int) taskThreadImportOrExport(RTTHREAD aThread, void *pvUser);
+
+    /*******************************************************************************
+     * Read stuff
+     ******************************************************************************/
 
     HRESULT readImpl(const LocationInfo &aLocInfo, ComObjPtr<Progress> &aProgress);
 
-    struct TaskOVF;
-    static DECLCALLBACK(int) taskThreadImportOrExport(RTTHREAD aThread, void *pvUser);
-
-    HRESULT readFS(const LocationInfo &locInfo, ComObjPtr<Progress> &pProgress);
-    HRESULT readFSOVF(const LocationInfo &locInfo, ComObjPtr<Progress> &pProgress);
-    HRESULT readFSOVA(const LocationInfo &locInfo, ComObjPtr<Progress> &pProgress);
+    HRESULT readFS(TaskOVF *pTask);
+    HRESULT readFSOVF(TaskOVF *pTask);
+    HRESULT readFSOVA(TaskOVF *pTask);
+    HRESULT readFSImpl(TaskOVF *pTask, PVDINTERFACEIO pCallbacks, PSHA1STORAGE pStorage);
     HRESULT readS3(TaskOVF *pTask);
+
+    /*******************************************************************************
+     * Import stuff
+     ******************************************************************************/
+
+    HRESULT importImpl(const LocationInfo &aLocInfo, ComObjPtr<Progress> &aProgress);
+
+    HRESULT importFS(TaskOVF *pTask);
+    HRESULT importFSOVF(TaskOVF *pTask, AutoWriteLockBase& writeLock);
+    HRESULT importFSOVA(TaskOVF *pTask, AutoWriteLockBase& writeLock);
+    HRESULT importS3(TaskOVF *pTask);
+
+    HRESULT readManifestFile(const Utf8Str &strFile, void **ppvBuf, size_t *pcbSize, PVDINTERFACEIO pCallbacks, PSHA1STORAGE pStorage);
+    HRESULT readTarManifestFile(RTTAR tar, const Utf8Str &strFile, void **ppvBuf, size_t *pcbSize, PVDINTERFACEIO pCallbacks, PSHA1STORAGE pStorage);
+    HRESULT verifyManifestFile(const Utf8Str &strFile, ImportStack &stack, void *pvBuf, size_t cbSize);
 
     void convertDiskAttachmentValues(const ovf::HardDiskController &hdc,
@@ -153 +177 @@
                                      int32_t &lDevice);
 
-    HRESULT importImpl(const LocationInfo &aLocInfo, ComObjPtr<Progress> &aProgress);
-    HRESULT manifestVerify(const LocationInfo &locInfo, const ovf::OVFReader &reader, ComObjPtr<Progress> &pProgress);
-
-    HRESULT importFS(TaskOVF *pTask);
-    HRESULT importFSOVF(TaskOVF *pTask);
-    HRESULT importFSOVA(TaskOVF *pTask);
-
-    struct ImportStack;
     void importOneDiskImage(const ovf::DiskImage &di,
                             const Utf8Str &strTargetPath,
-                            ComPtr<IMedium> &pTargetHD,
-                            ImportStack &stack);
+                            ComObjPtr<Medium> &pTargetHD,
+                            ImportStack &stack,
+                            PVDINTERFACEIO pCallbacks,
+                            PSHA1STORAGE pStorage);
     void importMachineGeneric(const ovf::VirtualSystem &vsysThis,
                               ComObjPtr<VirtualSystemDescription> &vsdescThis,
                               ComPtr<IMachine> &pNewMachine,
-                              ImportStack &stack);
+                              ImportStack &stack,
+                              PVDINTERFACEIO pCallbacks,
+                              PSHA1STORAGE pStorage);
     void importVBoxMachine(ComObjPtr<VirtualSystemDescription> &vsdescThis,
                            ComPtr<IMachine> &pNewMachine,
-                           ImportStack &stack);
-
-    HRESULT importS3(TaskOVF *pTask);
+                           ImportStack &stack,
+                           PVDINTERFACEIO pCallbacks,
+                           PSHA1STORAGE pStorage);
+    void importMachines(ImportStack &stack,
+                        PVDINTERFACEIO pCallbacks,
+                        PSHA1STORAGE pStorage);
+
+    /*******************************************************************************
+     * Write stuff
+     ******************************************************************************/
 
     HRESULT writeImpl(OVFFormat aFormat, const LocationInfo &aLocInfo, ComObjPtr<Progress> &aProgress);
+
+    HRESULT writeFS(TaskOVF *pTask);
+    HRESULT writeFSOVF(TaskOVF *pTask, AutoWriteLockBase& writeLock);
+    HRESULT writeFSOVA(TaskOVF *pTask, AutoWriteLockBase& writeLock);
+    HRESULT writeFSImpl(TaskOVF *pTask, AutoWriteLockBase& writeLock, PVDINTERFACEIO pCallbacks, PSHA1STORAGE pStorage);
+    HRESULT writeS3(TaskOVF *pTask);
 
     struct XMLStack;
@@ -186 +219 @@
                                      XMLStack &stack);
 
-    HRESULT writeFS(TaskOVF *pTask);
-    HRESULT writeFSOVF(TaskOVF *pTask, AutoWriteLockBase& writeLock);
-    HRESULT writeFSOVA(TaskOVF *pTask, AutoWriteLockBase& writeLock);
-    HRESULT writeFSImpl(TaskOVF *pTask, AutoWriteLockBase& writeLock, PVDINTERFACEIO pCallbacks, PSHA1STORAGE pStorage);
-    HRESULT writeS3(TaskOVF *pTask);
 
     friend class Machine;

trunk/src/VBox/Main/include/ApplianceImplPrivate.h

@@ -29 +29 @@
 ////////////////////////////////////////////////////////////////////////////////
 
+typedef std::pair<Utf8Str, Utf8Str> STRPAIR;
+
 /* Describe a location for the import/export. The location could be a file on a
  * local hard disk or a remote target based on the supported inet protocols. */
@@ -153 +155 @@
     bool                            fForceHWVirt;       // if true, we force enabling hardware virtualization
     bool                            fForceIOAPIC;       // if true, we force enabling the IOAPIC
-    uint32_t                        ulMemorySizeMB;     // virtual machien RAM in megabytes
+    uint32_t                        ulMemorySizeMB;     // virtual machine RAM in megabytes
 #ifdef VBOX_WITH_USB
     bool                            fUSBEnabled;
@@ -167 +169 @@
     // and will be cleaned up on errors
     std::list<MyHardDiskAttachment> llHardDiskAttachments;      // disks that were attached
-    std::list< ComPtr<IMedium> >    llHardDisksCreated;         // media that were created
+    std::list<STRPAIR>              llSrcDisksDigest;           // Digests of the source disks
 
     ImportStack(const LocationInfo &aLocInfo,
@@ -225 +227 @@
 PVDINTERFACEIO RTFileCreateInterface();
 PVDINTERFACEIO RTTarCreateInterface();
+int Sha1ReadBuf(const char *pcszFilename, void **ppvBuf, size_t *pcbSize, PVDINTERFACEIO pCallbacks, void *pvUser);
 int Sha1WriteBuf(const char *pcszFilename, void *pvBuf, size_t cbSize, PVDINTERFACEIO pCallbacks, void *pvUser);
 

trunk/src/VBox/Runtime/common/checksum/manifest.cpp

@@ -67 +67 @@
 typedef RTMANIFESTCALLBACKDATA* PRTMANIFESTCALLBACKDATA;
 
+/*******************************************************************************
+*   Private functions
+*******************************************************************************/
+
+DECLINLINE(char *) rtManifestPosOfCharInBuf(char const *pv, size_t cb, char c)
+{
+    char *pb = (char *)pv;
+    for (; cb; --cb, ++pb)
+        if (RT_UNLIKELY(*pb == c))
+            return pb;
+    return NULL;
+}
+
+DECLINLINE(size_t) rtManifestIndexOfCharInBuf(char const *pv, size_t cb, char c)
+{
+    char const *pb = (char const *)pv;
+    for (size_t i=0; i < cb; ++i, ++pb)
+        if (RT_UNLIKELY(*pb == c))
+            return i;
+    return cb;
+}
 
 int rtSHAProgressCallback(unsigned uPercent, void *pvUser)
@@ -76 +97 @@
 }
 
+/*******************************************************************************
+*   Public functions
+*******************************************************************************/
+
 RTR3DECL(int) RTManifestVerify(const char *pszManifestFile, PRTMANIFESTTEST paTests, size_t cTests, size_t *piFailed)
 {
     /* Validate input */
     AssertPtrReturn(pszManifestFile, VERR_INVALID_POINTER);
+
+    /* Open the manifest file */
+    RTFILE file;
+    int rc = RTFileOpen(&file, pszManifestFile, RTFILE_O_OPEN | RTFILE_O_READ | RTFILE_O_DENY_NONE);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    void *pvBuf = 0;
+    do
+    {
+        uint64_t cbSize;
+        rc = RTFileGetSize(file, &cbSize);
+        if (RT_FAILURE(rc))
+            break;
+
+        /* Cast down for the case size_t < uint64_t. This isn't really correct,
+           but we consider manifest files bigger than size_t as not supported
+           by now. */
+        size_t cbToRead = (size_t)cbSize;
+        pvBuf = RTMemAlloc(cbToRead);
+        if (!pvBuf)
+        {
+            rc = VERR_NO_MEMORY;
+            break;
+        }
+
+        size_t cbRead = 0;
+        rc = RTFileRead(file, pvBuf, cbToRead, &cbRead);
+        if (RT_FAILURE(rc))
+            break;
+
+        rc = RTManifestVerifyFilesBuf(pvBuf, cbRead, paTests, cTests, piFailed);
+    }while (0);
+
+    /* Cleanup */
+    if (pvBuf)
+        RTMemFree(pvBuf);
+
+    RTFileClose(file);
+
+    return rc;
+}
+
+RTR3DECL(int) RTManifestVerifyFiles(const char *pszManifestFile, const char * const *papszFiles, size_t cFiles, size_t *piFailed,
+                                    PFNRTPROGRESS pfnProgressCallback, void *pvUser)
+{
+    /* Validate input */
+    AssertPtrReturn(pszManifestFile, VERR_INVALID_POINTER);
+    AssertPtrReturn(papszFiles, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pfnProgressCallback, VERR_INVALID_POINTER);
+
+    int rc = VINF_SUCCESS;
+
+    /* Create our compare list */
+    PRTMANIFESTTEST paFiles = (PRTMANIFESTTEST)RTMemTmpAllocZ(sizeof(RTMANIFESTTEST) * cFiles);
+    if (!paFiles)
+        return VERR_NO_MEMORY;
+
+    RTMANIFESTCALLBACKDATA callback = { pfnProgressCallback, pvUser, cFiles, 0 };
+    /* Fill our compare list */
+    for (size_t i = 0; i < cFiles; ++i)
+    {
+        char *pszDigest;
+        if (pfnProgressCallback)
+        {
+            callback.cCurrentFile = i;
+            rc = RTSha1DigestFromFile(papszFiles[i], &pszDigest, rtSHAProgressCallback, &callback);
+        }
+        else
+            rc = RTSha1DigestFromFile(papszFiles[i], &pszDigest, NULL, NULL);
+        if (RT_FAILURE(rc))
+            break;
+        paFiles[i].pszTestFile = (char*)papszFiles[i];
+        paFiles[i].pszTestDigest = pszDigest;
+    }
+
+    /* Do the verification */
+    if (RT_SUCCESS(rc))
+        rc = RTManifestVerify(pszManifestFile, paFiles, cFiles, piFailed);
+
+    /* Cleanup */
+    for (size_t i = 0; i < cFiles; ++i)
+    {
+        if (paFiles[i].pszTestDigest)
+            RTStrFree((char*)paFiles[i].pszTestDigest);
+    }
+    RTMemTmpFree(paFiles);
+
+    return rc;
+}
+
+RTR3DECL(int) RTManifestWriteFiles(const char *pszManifestFile, const char * const *papszFiles, size_t cFiles,
+                                   PFNRTPROGRESS pfnProgressCallback, void *pvUser)
+{
+    /* Validate input */
+    AssertPtrReturn(pszManifestFile, VERR_INVALID_POINTER);
+    AssertPtrReturn(papszFiles, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pfnProgressCallback, VERR_INVALID_POINTER);
+
+    RTFILE file;
+    int rc = RTFileOpen(&file, pszManifestFile, RTFILE_O_CREATE | RTFILE_O_WRITE | RTFILE_O_DENY_ALL);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    PRTMANIFESTTEST paFiles = 0;
+    void *pvBuf = 0;
+    do
+    {
+        paFiles = (PRTMANIFESTTEST)RTMemAllocZ(sizeof(RTMANIFESTTEST) * cFiles);
+        if (!paFiles)
+        {
+            rc = VERR_NO_MEMORY;
+            break;
+        }
+
+        RTMANIFESTCALLBACKDATA callback = { pfnProgressCallback, pvUser, cFiles, 0 };
+        for (size_t i = 0; i < cFiles; ++i)
+        {
+            paFiles[i].pszTestFile = papszFiles[i];
+            /* Calculate the SHA1 digest of every file */
+            if (pfnProgressCallback)
+            {
+                callback.cCurrentFile = i;
+                rc = RTSha1DigestFromFile(paFiles[i].pszTestFile, (char**)&paFiles[i].pszTestDigest, rtSHAProgressCallback, &callback);
+            }
+            else
+                rc = RTSha1DigestFromFile(paFiles[i].pszTestFile, (char**)&paFiles[i].pszTestDigest, NULL, NULL);
+            if (RT_FAILURE(rc))
+                break;
+        }
+
+        if (RT_SUCCESS(rc))
+        {
+            size_t cbSize = 0;
+            rc = RTManifestWriteFilesBuf(&pvBuf, &cbSize, paFiles, cFiles);
+            if (RT_FAILURE(rc))
+                break;
+
+            rc = RTFileWrite(file, pvBuf, cbSize, 0);
+        }
+    }while (0);
+
+    RTFileClose(file);
+
+    /* Cleanup */
+    if (pvBuf)
+        RTMemFree(pvBuf);
+    for (size_t i = 0; i < cFiles; ++i)
+        if (paFiles[i].pszTestDigest)
+            RTStrFree((char*)paFiles[i].pszTestDigest);
+    RTMemFree(paFiles);
+
+    /* Delete the manifest file on failure */
+    if (RT_FAILURE(rc))
+        RTFileDelete(pszManifestFile);
+
+    return rc;
+}
+
+RTR3DECL(int) RTManifestVerifyFilesBuf(void *pvBuf, size_t cbSize, PRTMANIFESTTEST paTests, size_t cTests, size_t *piFailed)
+{
+    /* Validate input */
+    AssertPtrReturn(pvBuf, VERR_INVALID_POINTER);
+    AssertReturn(cbSize > 0, VERR_INVALID_PARAMETER);
     AssertPtrReturn(paTests, VERR_INVALID_POINTER);
     AssertReturn(cTests > 0, VERR_INVALID_PARAMETER);
-
-    /* Open the manifest file */
-    PRTSTREAM pStream;
-    int rc = RTStrmOpen(pszManifestFile, "r", &pStream);
-    if (RT_FAILURE(rc))
-        return rc;
+    AssertPtrNullReturn(piFailed, VERR_INVALID_POINTER);
+
+    int rc = VINF_SUCCESS;
 
     PRTMANIFESTFILEENTRY paFiles = (PRTMANIFESTFILEENTRY)RTMemTmpAllocZ(sizeof(RTMANIFESTFILEENTRY) * cTests);
     if (!paFiles)
-    {
-        RTStrmClose(pStream);
         return VERR_NO_MEMORY;
-    }
 
     /* Fill our compare list */
@@ -100 +283 @@
         paFiles[i].pTestPattern = &paTests[i];
 
+    char *pcBuf = (char*)pvBuf;
+    size_t cbRead = 0;
     /* Parse the manifest file line by line */
-    char szLine[1024];
     for (;;)
     {
-        rc = RTStrmGetLine(pStream, szLine, sizeof(szLine));
-        if (RT_FAILURE(rc))
-            break;
-        size_t cch = strlen(szLine);
-
-        /* Skip empty lines */
-        if (cch == 0)
+        if (cbRead >= cbSize)
+            break;
+
+        size_t cch = rtManifestIndexOfCharInBuf(pcBuf, cbSize - cbRead, '\n') + 1;
+
+        /* Skip empty lines (UNIX/DOS format) */
+        if (   (   cch == 1
+                && pcBuf[0] == '\n')
+            || (   cch == 2
+                && pcBuf[0] == '\r'
+                && pcBuf[1] == '\n'))
+        {
+            pcBuf += cch;
+            cbRead += cch;
             continue;
+        }
 
         /** @todo r=bird:
+         *  -# Better deal with this EOF line platform dependency
          *  -# The SHA1 test should probably include a blank space check.
          *  -# If there is a specific order to the elements in the string, it would be
@@ -122 +315 @@
         /* Check for the digest algorithm */
         if (   cch < 4
-            || !(  szLine[0] == 'S'
-                && szLine[1] == 'H'
-                && szLine[2] == 'A'
-                && szLine[3] == '1'))
+            || !(   pcBuf[0] == 'S'
+                 && pcBuf[1] == 'H'
+                 && pcBuf[2] == 'A'
+                 && pcBuf[3] == '1'))
         {
             /* Digest unsupported */
@@ -133 +326 @@
 
         /* Try to find the filename */
-        char *pszNameStart = strchr(szLine, '(');
+        char *pszNameStart = rtManifestPosOfCharInBuf(pcBuf, cch, '(');
         if (!pszNameStart)
         {
@@ -139 +332 @@
             break;
         }
-        char *pszNameEnd = strchr(szLine, ')');
+        char *pszNameEnd = rtManifestPosOfCharInBuf(pcBuf, cch, ')');
         if (!pszNameEnd)
         {
@@ -158 +351 @@
 
         /* Try to find the digest sum */
-        char *pszDigestStart = strchr(szLine, '=');
+        char *pszDigestStart = rtManifestPosOfCharInBuf(pcBuf, cch, '=') + 1;
         if (!pszDigestStart)
         {
@@ -165 +358 @@
             break;
         }
-        char *pszDigest = ++pszDigestStart;
+        char *pszDigestEnd = rtManifestPosOfCharInBuf(pcBuf, cch, '\r');
+        if (!pszDigestEnd)
+            pszDigestEnd = rtManifestPosOfCharInBuf(pcBuf, cch, '\n');
+        if (!pszDigestEnd)
+        {
+            rc = VERR_MANIFEST_WRONG_FILE_FORMAT;
+            break;
+        }
+        /* Copy the digest part */
+        size_t cchDigest = pszDigestEnd - pszDigestStart - 1;
+        char *pszDigest = (char *)RTMemTmpAlloc(cchDigest + 1);
+        if (!pszDigest)
+        {
+            rc = VERR_NO_MEMORY;
+            break;
+        }
+        memcpy(pszDigest, pszDigestStart + 1, cchDigest);
+        pszDigest[cchDigest] = '\0';
 
         /* Check our file list against the extracted data */
@@ -181 +391 @@
         }
         RTMemTmpFree(pszName);
+        RTMemTmpFree(pszDigest);
         if (!fFound)
         {
@@ -187 +398 @@
             break;
         }
-    }
-    RTStrmClose(pStream);
+
+        pcBuf += cch;
+        cbRead += cch;
+    }
 
     if (   rc == VINF_SUCCESS
@@ -229 +442 @@
 }
 
-
-RTR3DECL(int) RTManifestVerifyFiles(const char *pszManifestFile, const char * const *papszFiles, size_t cFiles, size_t *piFailed,
-                                    PFNRTPROGRESS pfnProgressCallback, void *pvUser)
-{
-    /* Validate input */
-    AssertPtrReturn(pszManifestFile, VERR_INVALID_POINTER);
-    AssertPtrReturn(papszFiles, VERR_INVALID_POINTER);
-    AssertPtrNullReturn(pfnProgressCallback, VERR_INVALID_PARAMETER);
-
-    int rc = VINF_SUCCESS;
-
-    /* Create our compare list */
-    PRTMANIFESTTEST paFiles = (PRTMANIFESTTEST)RTMemTmpAllocZ(sizeof(RTMANIFESTTEST) * cFiles);
-    if (!paFiles)
-        return VERR_NO_MEMORY;
-
-    RTMANIFESTCALLBACKDATA callback = { pfnProgressCallback, pvUser, cFiles, 0 };
-    /* Fill our compare list */
-    for (size_t i = 0; i < cFiles; ++i)
-    {
-        char *pszDigest;
-        if (pfnProgressCallback)
-        {
-            callback.cCurrentFile = i;
-            rc = RTSha1DigestFromFile(papszFiles[i], &pszDigest, rtSHAProgressCallback, &callback);
-        }
-        else
-            rc = RTSha1DigestFromFile(papszFiles[i], &pszDigest, NULL, NULL);
-        if (RT_FAILURE(rc))
-            break;
-        paFiles[i].pszTestFile = (char*)papszFiles[i];
-        paFiles[i].pszTestDigest = pszDigest;
-    }
-
-    /* Do the verification */
-    if (RT_SUCCESS(rc))
-        rc = RTManifestVerify(pszManifestFile, paFiles, cFiles, piFailed);
-
-    /* Cleanup */
-    for (size_t i = 0; i < cFiles; ++i)
-    {
-        if (paFiles[i].pszTestDigest)
-            RTStrFree(paFiles[i].pszTestDigest);
-    }
-    RTMemTmpFree(paFiles);
-
-    return rc;
-}
-
-
-RTR3DECL(int) RTManifestWriteFiles(const char *pszManifestFile, const char * const *papszFiles, size_t cFiles,
-                                   PFNRTPROGRESS pfnProgressCallback, void *pvUser)
-{
-    /* Validate input */
-    AssertPtrReturn(pszManifestFile, VERR_INVALID_POINTER);
-    AssertPtrReturn(papszFiles, VERR_INVALID_POINTER);
-    AssertPtrNullReturn(pfnProgressCallback, VERR_INVALID_PARAMETER);
-
-    /* Open a file to stream in */
-    PRTSTREAM pStream;
-    int rc = RTStrmOpen(pszManifestFile, "w", &pStream);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    RTMANIFESTCALLBACKDATA callback = { pfnProgressCallback, pvUser, cFiles, 0 };
-    for (size_t i = 0; i < cFiles; ++i)
-    {
-        /* Calculate the SHA1 digest of every file */
-        char *pszDigest;
-        if (pfnProgressCallback)
-        {
-            callback.cCurrentFile = i;
-            rc = RTSha1DigestFromFile(papszFiles[i], &pszDigest, rtSHAProgressCallback, &callback);
-        }
-        else
-            rc = RTSha1DigestFromFile(papszFiles[i], &pszDigest, NULL, NULL);
-        if (RT_FAILURE(rc))
-            break;
-
-        /* Add the entry to the manifest file */
-        int cch = RTStrmPrintf(pStream, "SHA1 (%s)= %s\n", RTPathFilename(papszFiles[i]), pszDigest);
-        RTStrFree(pszDigest);
-        if (RT_UNLIKELY(cch < 0))
-        {
-            rc = VERR_INTERNAL_ERROR;
-            break;
-        }
-    }
-    int rc2 = RTStrmClose(pStream);
-    if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
-        rc2 = rc;
-
-    /* Delete the manifest file on failure */
-    if (RT_FAILURE(rc))
-        RTFileDelete(pszManifestFile);
-
-    return rc;
-}
-
-RTR3DECL(int) RTManifestWriteFilesBuf(void **ppvBuf, size_t *pcbSize, const char * const *papszFileNames, const char * const *papszFileDigests, size_t cFiles)
+RTR3DECL(int) RTManifestWriteFilesBuf(void **ppvBuf, size_t *pcbSize, PRTMANIFESTTEST paFiles, size_t cFiles)
 {
     /* Validate input */
     AssertPtrReturn(ppvBuf, VERR_INVALID_POINTER);
     AssertPtrReturn(pcbSize, VERR_INVALID_POINTER);
-    AssertPtrReturn(papszFileNames, VERR_INVALID_POINTER);
-    AssertPtrReturn(papszFileDigests, VERR_INVALID_POINTER);
+    AssertPtrReturn(paFiles, VERR_INVALID_POINTER);
     AssertReturn(cFiles > 0, VERR_INVALID_PARAMETER);
 
@@ -342 +455 @@
     for (size_t i = 0; i < cFiles; ++i)
     {
-        size_t cbTmp = strlen(RTPathFilename(papszFileNames[i])) + strlen(papszFileDigests[i]) + 10;
+        size_t cbTmp = strlen(RTPathFilename(paFiles[i].pszTestFile)) + strlen(paFiles[i].pszTestDigest) + 10;
         cbMaxSize = RT_MAX(cbMaxSize, cbTmp);
         cbSize += cbTmp;
@@ -357 +470 @@
     for (size_t i = 0; i < cFiles; ++i)
     {
-        size_t cch = RTStrPrintf(pszTmp, cbMaxSize + 1, "SHA1 (%s)= %s\n", RTPathFilename(papszFileNames[i]), papszFileDigests[i]);
+        size_t cch = RTStrPrintf(pszTmp, cbMaxSize + 1, "SHA1 (%s)= %s\n", RTPathFilename(paFiles[i].pszTestFile), paFiles[i].pszTestDigest);
         memcpy(&((char*)pvBuf)[cbPos], pszTmp, cch);
         cbPos += cch;

trunk/src/VBox/Runtime/common/misc/tar.cpp

@@ -92 +92 @@
 #endif
 
+typedef struct RTTARFILEINTERNAL* PRTTARFILEINTERNAL;
 typedef struct RTTARINTERNAL
 {
@@ -98 +99 @@
     bool            fFileOpenForWrite;
     uint32_t        fOpenMode;
+    bool            fStreamMode;
+    PRTTARFILEINTERNAL pFileCache;
 } RTTARINTERNAL;
 typedef RTTARINTERNAL* PRTTARINTERNAL;
@@ -275 +278 @@
 
     return rc;
+}
+
+DECLINLINE(PRTTARFILEINTERNAL) rtCreateTarFileInternal(PRTTARINTERNAL pInt, const char *pszFilename, uint32_t fOpen)
+{
+    PRTTARFILEINTERNAL pFileInt = (PRTTARFILEINTERNAL)RTMemAllocZ(sizeof(RTTARFILEINTERNAL));
+    if (!pFileInt)
+        return NULL;
+
+    pFileInt->u32Magic = RTTARFILE_MAGIC;
+    pFileInt->pTar = pInt;
+    pFileInt->pszFilename = RTStrDup(pszFilename);
+    pFileInt->fOpenMode = fOpen;
+
+    return pFileInt;
+}
+
+DECLINLINE(PRTTARFILEINTERNAL) rtCopyTarFileInternal(PRTTARFILEINTERNAL pInt)
+{
+    PRTTARFILEINTERNAL pNewInt = (PRTTARFILEINTERNAL)RTMemAllocZ(sizeof(RTTARFILEINTERNAL));
+    if (!pNewInt)
+        return NULL;
+
+    memcpy(pNewInt, pInt, sizeof(RTTARFILEINTERNAL));
+    pNewInt->pszFilename = RTStrDup(pInt->pszFilename);
+
+    return pNewInt;
+}
+
+DECLINLINE(void) rtDeleteTarFileInternal(PRTTARFILEINTERNAL pInt)
+{
+    if (pInt)
+    {
+        if (pInt->pszFilename)
+            RTStrFree(pInt->pszFilename);
+        pInt->u32Magic = RTTARFILE_MAGIC_DEAD;
+        RTMemFree(pInt);
+    }
 }
 
@@ -563 +603 @@
  ******************************************************************************/
 
-RTR3DECL(int) RTTarOpen(PRTTAR phTar, const char* pszTarname, uint32_t fMode)
+RTR3DECL(int) RTTarOpen(PRTTAR phTar, const char* pszTarname, uint32_t fMode, bool fStream)
 {
     PRTTARINTERNAL pInt = (PRTTARINTERNAL)RTMemAllocZ(sizeof(RTTARINTERNAL));
@@ -571 +611 @@
     pInt->u32Magic = RTTAR_MAGIC;
     pInt->fOpenMode = fMode;
+    pInt->fStreamMode = fStream && (fMode & RTFILE_O_READ);
 
     int rc = VINF_SUCCESS;
@@ -618 +659 @@
         rc = RTFileClose(pInt->hTarFile);
 
+    /* Delete any remaining cached file headers. */
+    if (pInt->pFileCache)
+    {
+        rtDeleteTarFileInternal(pInt->pFileCache);
+        pInt->pFileCache = 0;
+    }
+
     pInt->u32Magic = RTTAR_MAGIC_DEAD;
 
@@ -634 +682 @@
     if (!pInt->hTarFile)
         return VERR_INVALID_HANDLE;
+
+    if (pInt->fStreamMode)
+        return VERR_INVALID_STATE;
 
     if (fOpen & RTFILE_O_WRITE)
@@ -643 +694 @@
     }
 
-    PRTTARFILEINTERNAL pFileInt = (PRTTARFILEINTERNAL)RTMemAllocZ(sizeof(RTTARFILEINTERNAL));
+    PRTTARFILEINTERNAL pFileInt = rtCreateTarFileInternal(pInt, pszFilename, fOpen);
     if (!pFileInt)
         return VERR_NO_MEMORY;
 
-    pFileInt->u32Magic = RTTARFILE_MAGIC;
-
     int rc = VINF_SUCCESS;
     do
     {
-        pFileInt->pTar = pInt;
-        pFileInt->pszFilename = RTStrDup(pszFilename);
-        pFileInt->uStart = 0;
-        pFileInt->cbSize = 0;
-        pFileInt->cbSetSize = 0;
-        pFileInt->fOpenMode = fOpen;
-        pFileInt->uCurrentPos = 0;
-
         if (pFileInt->fOpenMode & RTFILE_O_WRITE)
         {
@@ -718 +759 @@
 
     /* In write mode: */
-    if (pFileInt->fOpenMode & RTFILE_O_WRITE)
+    if (pFileInt->fOpenMode & RTFILE_O_READ)
+    {
+        /* In read mode, we want to make sure to stay at the aligned end of this
+         * file, so the next file could be read immediately. */
+        uint64_t uCurPos = RTFileTell(pFileInt->pTar->hTarFile);
+        /* Check that the file pointer is somewhere within the last open file.
+         * If we are at the beginning (nothing read yet) nothing will be done.
+         * A user could open/close a file more than once, without reading
+         * something. */
+        if (pFileInt->uStart + sizeof(RTTARRECORD) < uCurPos && uCurPos < RT_ALIGN(pFileInt->uStart + sizeof(RTTARRECORD) + pFileInt->cbSize, sizeof(RTTARRECORD)))
+        {
+            /* Seek to the next file header. */
+            uint64_t uNextPos = RT_ALIGN(pFileInt->uStart + sizeof(RTTARRECORD) + pFileInt->cbSize, sizeof(RTTARRECORD));
+            rc = RTFileSeek(pFileInt->pTar->hTarFile, uNextPos - uCurPos, RTFILE_SEEK_CURRENT, NULL);
+        }
+    }
+    else if (pFileInt->fOpenMode & RTFILE_O_WRITE)
     {
         pFileInt->pTar->fFileOpenForWrite = false;
@@ -756 +813 @@
         while(0);
     }
-    /* Nothing special in readmode. */
 
     /* Now cleanup and delete the handle */
-    RTStrFree(pFileInt->pszFilename);
-    pFileInt->u32Magic = RTTARFILE_MAGIC_DEAD;
-    RTMemFree(pFileInt);
+    rtDeleteTarFileInternal(pFileInt);
 
     return rc;
@@ -770 +824 @@
     PRTTARFILEINTERNAL pFileInt = hFile;
     RTTARFILE_VALID_RETURN(pFileInt);
+
+    if (pFileInt->pTar->fStreamMode)
+        return VERR_INVALID_STATE;
 
     switch (uMethod)
@@ -822 +879 @@
     RTTARFILE_VALID_RETURN(pFileInt);
 
+    /* Check that we not read behind the end of file. If so return immediately. */
+    if (uOffset > pFileInt->cbSize)
+    {
+        if (pcbRead)
+            *pcbRead = 0;
+        return VINF_SUCCESS; /* ??? VERR_EOF */
+    }
+
+    size_t cbToCopy = RT_MIN(pFileInt->cbSize - uOffset, cbToRead);
     size_t cbTmpRead = 0;
-    int rc = RTFileReadAt(pFileInt->pTar->hTarFile, pFileInt->uStart + 512 + uOffset, pvBuf, cbToRead, &cbTmpRead);
+    int rc = RTFileReadAt(pFileInt->pTar->hTarFile, pFileInt->uStart + 512 + uOffset, pvBuf, cbToCopy, &cbTmpRead);
     pFileInt->uCurrentPos = uOffset + cbTmpRead;
     if (pcbRead)
@@ -1021 +1087 @@
     /* Open the tar file */
     RTTAR hTar;
-    int rc = RTTarOpen(&hTar, pszTarFile, RTFILE_O_OPEN | RTFILE_O_READ | RTFILE_O_DENY_NONE);
+    int rc = RTTarOpen(&hTar, pszTarFile, RTFILE_O_OPEN | RTFILE_O_READ | RTFILE_O_DENY_NONE, false);
     if (RT_FAILURE(rc))
         return rc;
@@ -1045 +1111 @@
     /* Open the tar file */
     RTTAR hTar;
-    int rc = RTTarOpen(&hTar, pszTarFile, RTFILE_O_OPEN | RTFILE_O_READ | RTFILE_O_DENY_NONE);
+    int rc = RTTarOpen(&hTar, pszTarFile, RTFILE_O_OPEN | RTFILE_O_READ | RTFILE_O_DENY_NONE, false);
     if (RT_FAILURE(rc))
         return rc;
@@ -1152 +1218 @@
     do
     {
-        rc = RTTarOpen(&hTar, pszTarFile, RTFILE_O_OPEN | RTFILE_O_READ | RTFILE_O_DENY_NONE);
+        rc = RTTarOpen(&hTar, pszTarFile, RTFILE_O_OPEN | RTFILE_O_READ | RTFILE_O_DENY_NONE, false);
         if (RT_FAILURE(rc))
             break;
@@ -1215 +1281 @@
     /* Open the tar file */
     RTTAR hTar;
-    int rc = RTTarOpen(&hTar, pszTarFile, RTFILE_O_OPEN | RTFILE_O_READ | RTFILE_O_DENY_NONE);
+    int rc = RTTarOpen(&hTar, pszTarFile, RTFILE_O_OPEN | RTFILE_O_READ | RTFILE_O_DENY_NONE, false);
     if (RT_FAILURE(rc))
         return rc;
@@ -1286 +1352 @@
 
     RTTAR hTar;
-    int rc = RTTarOpen(&hTar, pszTarFile, RTFILE_O_CREATE | RTFILE_O_READWRITE | RTFILE_O_DENY_NONE);
+    int rc = RTTarOpen(&hTar, pszTarFile, RTFILE_O_CREATE | RTFILE_O_READWRITE | RTFILE_O_DENY_NONE, false);
     if (RT_FAILURE(rc))
         return rc;
@@ -1316 +1382 @@
 }
 
+/******************************************************************************
+ *   Streaming Functions                                                      *
+ ******************************************************************************/
+
+RTR3DECL(int) RTTarCurrentFile(RTTAR hTar, char **ppszFilename)
+{
+    /* Validate input. */
+    AssertPtrNullReturn(ppszFilename, VERR_INVALID_POINTER);
+
+    PRTTARINTERNAL pInt = hTar;
+    RTTAR_VALID_RETURN(pInt);
+
+    /* Open and close the file on the current position. This makes sure the
+     * cache is filled in case we never read something before. On success it
+     * will return the current filename. */
+    RTTARFILE hFile;
+    int rc = RTTarFileOpenCurrentFile(hTar, &hFile, ppszFilename, RTFILE_O_OPEN | RTFILE_O_READ | RTFILE_O_DENY_NONE);
+    if (RT_SUCCESS(rc))
+        RTTarFileClose(hFile);
+
+    return rc;
+}
+
+RTR3DECL(int) RTTarSeekNextFile(RTTAR hTar)
+{
+    PRTTARINTERNAL pInt = hTar;
+    RTTAR_VALID_RETURN(pInt);
+
+    int rc = VINF_SUCCESS;
+
+    if (!pInt->fStreamMode)
+        return VERR_INVALID_STATE;
+
+    /* If there is nothing in the cache, it means we never read something. Just
+     * ask for the current filename to fill the cache. */
+    if (!pInt->pFileCache)
+    {
+        rc = RTTarCurrentFile(hTar, 0);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /* Check that the file pointer is somewhere within the last open file.
+     * If not we are somehow busted. */
+    uint64_t uCurPos = RTFileTell(pInt->hTarFile);
+    if (!(pInt->pFileCache->uStart <= uCurPos && uCurPos < pInt->pFileCache->uStart + sizeof(RTTARRECORD) + pInt->pFileCache->cbSize))
+        return VERR_INVALID_STATE;
+
+    /* Seek to the next file header. */
+    uint64_t uNextPos = RT_ALIGN(pInt->pFileCache->uStart + sizeof(RTTARRECORD) + pInt->pFileCache->cbSize, sizeof(RTTARRECORD));
+    rc = RTFileSeek(pInt->hTarFile, uNextPos - uCurPos, RTFILE_SEEK_CURRENT, NULL);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /* Again check the current filename to fill the cache with the new value. */
+    return RTTarCurrentFile(hTar, 0);
+}
+
+RTR3DECL(int) RTTarFileOpenCurrentFile(RTTAR hTar, PRTTARFILE phFile, char **ppszFilename, uint32_t fOpen)
+{
+    /* Validate input. */
+    AssertPtrReturn(phFile, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(ppszFilename, VERR_INVALID_POINTER);
+    AssertReturn((fOpen & RTFILE_O_READ), VERR_INVALID_PARAMETER); /* Only valid in read mode. */
+
+    PRTTARINTERNAL pInt = hTar;
+    RTTAR_VALID_RETURN(pInt);
+
+    if (!pInt->fStreamMode)
+        return VERR_INVALID_STATE;
+
+    int rc = VINF_SUCCESS;
+
+    /* Is there some cached entry? */
+    if (pInt->pFileCache)
+    {
+        /* Are we still direct behind that header? */
+        if (pInt->pFileCache->uStart + sizeof(RTTARRECORD) == RTFileTell(pInt->hTarFile))
+        {
+            /* Yes, so the streaming can start. Just return the cached file
+             * structure to the caller. */
+            *phFile = rtCopyTarFileInternal(pInt->pFileCache);
+            if (ppszFilename)
+                *ppszFilename = RTStrDup(pInt->pFileCache->pszFilename);
+            return VINF_SUCCESS;
+        }else
+        {
+            /* Else delete the last open file cache. Might be recreated below. */
+            rtDeleteTarFileInternal(pInt->pFileCache);
+            pInt->pFileCache = 0;
+        }
+    }
+
+    PRTTARFILEINTERNAL pFileInt = 0;
+    do
+    {
+        /* Try to read a header entry from the current position. If we aren't
+         * on a header record, the header checksum will show and an error will
+         * be returned. */
+        RTTARRECORD record;
+        /* Read & verify a header record */
+        rc = rtTarReadHeaderRecord(pInt->hTarFile, &record);
+        /* Check for error or EOF. */
+        if (RT_FAILURE(rc))
+            break;
+        /* We support normal files only */
+        if (   record.h.linkflag == LF_OLDNORMAL
+            || record.h.linkflag == LF_NORMAL)
+        {
+            pFileInt = rtCreateTarFileInternal(pInt, record.h.name, fOpen);
+            if (!pFileInt)
+            {
+                rc = VERR_NO_MEMORY;
+                break;
+            }
+            /* Get the file size */
+            rc = RTStrToUInt64Full(record.h.size, 8, &pFileInt->cbSize);
+            if (RT_FAILURE(rc))
+                break;
+            /* The start is -512 from here. */
+            pFileInt->uStart = RTFileTell(pInt->hTarFile) - sizeof(RTTARRECORD);
+            /* Copy the new file structure to our cache. */
+            pInt->pFileCache = rtCopyTarFileInternal(pFileInt);
+            if (ppszFilename)
+                *ppszFilename = RTStrDup(pFileInt->pszFilename);
+        }
+    }while (0);
+
+    if (RT_FAILURE(rc))
+    {
+        if (pFileInt)
+            rtDeleteTarFileInternal(pFileInt);
+    }
+    else
+        *phFile = pFileInt;
+
+    return rc;
+}
+