Changeset 22757 in vbox

Timestamp:

Sep 3, 2009 5:22:53 PM (15 years ago)

Author:

vboxsync

Message:

AsyncCompletion: Break the big critical section into smaller parts and let the I/O manager spawn new threads during high I/O load

Location:

trunk/src/VBox/VMM

Files:

• PDMAsyncCompletionFile.cpp (modified) (6 diffs)
• PDMAsyncCompletionFileCache.cpp (modified) (31 diffs)
• PDMAsyncCompletionFileFailsafe.cpp (modified) (1 diff)
• PDMAsyncCompletionFileInternal.h (modified) (6 diffs)
• PDMAsyncCompletionFileNormal.cpp (modified) (12 diffs)

trunk/src/VBox/VMM/PDMAsyncCompletionFile.cpp

@@ -236 +236 @@
 }
 
-static int pdmacFileAioMgrAddEndpoint(PPDMACEPFILEMGR pAioMgr, PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
+int pdmacFileAioMgrAddEndpoint(PPDMACEPFILEMGR pAioMgr, PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
 {
     int rc;
@@ -343 +343 @@
            || (enmTransfer == PDMACTASKFILETRANSFER_WRITE));
 
-    pTaskFile->cbTransferLeft = cbTransfer;
+    ASMAtomicWriteS32(&pTaskFile->cbTransferLeft, cbTransfer);
+    ASMAtomicWriteBool(&pTaskFile->fCompleted, false);
 
     for (unsigned i = 0; i < cSegments; i++)
@@ -366 +367 @@
     AssertMsg(!cbTransfer, ("Incomplete transfer %u bytes left\n", cbTransfer));
 
+    if (ASMAtomicReadS32(&pTaskFile->cbTransferLeft) == 0
+        && !ASMAtomicXchgBool(&pTaskFile->fCompleted, true))
+        pdmR3AsyncCompletionCompleteTask(pTask);
+
     return VINF_SUCCESS;
 }
@@ -376 +381 @@
  * @param   ppAioMgr    Where to store the pointer to the new async I/O manager on success.
  */
-static int pdmacFileAioMgrCreate(PPDMASYNCCOMPLETIONEPCLASSFILE pEpClass, PPPDMACEPFILEMGR ppAioMgr)
+int pdmacFileAioMgrCreate(PPDMASYNCCOMPLETIONEPCLASSFILE pEpClass, PPPDMACEPFILEMGR ppAioMgr)
 {
     int rc = VINF_SUCCESS;
@@ -577 +582 @@
             }
 
+            pEpFile->cbFile = cbSize;
+
             RTFileClose(File);
         }
@@ -737 +744 @@
     PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint;
 
-    return RTFileGetSize(pEpFile->File, pcbSize);
+    *pcbSize = ASMAtomicReadU64(&pEpFile->cbFile);
+
+    return VINF_SUCCESS;
 }
 

trunk/src/VBox/VMM/PDMAsyncCompletionFileCache.cpp

@@ -52 +52 @@
 #include "PDMAsyncCompletionFileInternal.h"
 
+#ifdef VBOX_STRICT
+# define PDMACFILECACHE_IS_CRITSECT_OWNER(Cache) \
+    do \
+    { \
+     AssertMsg(RTCritSectIsOwner(&pCache->CritSect), \
+               ("Thread does not own critical section\n"));\
+    } while(0);
+#else
+# define PDMACFILECACHE_IS_CRITSECT_OWNER(Cache) do { } while(0);
+#endif
+
 /*******************************************************************************
 *   Internal Functions                                                         *
 *******************************************************************************/
 static void pdmacFileCacheTaskCompleted(PPDMACTASKFILE pTask, void *pvUser);
+
+DECLINLINE(void) pdmacFileEpCacheEntryRelease(PPDMACFILECACHEENTRY pEntry)
+{
+    AssertMsg(pEntry->cRefs > 0, ("Trying to release a not referenced entry\n"));
+    ASMAtomicDecU32(&pEntry->cRefs);
+}
+
+DECLINLINE(void) pdmacFileEpCacheEntryRef(PPDMACFILECACHEENTRY pEntry)
+{
+    ASMAtomicIncU32(&pEntry->cRefs);
+}
 
 /**
@@ -217 +239 @@
     size_t cbEvicted = 0;
 
+    PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
+
     AssertMsg(cbData > 0, ("Evicting 0 bytes not possible\n"));
     AssertMsg(   !pGhostListDst
@@ -233 +257 @@
 
         /* We can't evict pages which are currently in progress */
-        if (!(pCurr->fFlags & PDMACFILECACHE_ENTRY_IO_IN_PROGRESS))
+        if (!(pCurr->fFlags & PDMACFILECACHE_ENTRY_IO_IN_PROGRESS)
+            && (ASMAtomicReadU32(&pCurr->cRefs) == 0))
         {
-            LogFlow(("Evicting entry %#p (%u bytes)\n", pCurr, pCurr->cbData));
-            if (pCurr->pbData)
+            /* Ok eviction candidate. Grab the endpoint semaphore and check again
+             * because somebody else might have raced us. */
+            PPDMACFILEENDPOINTCACHE pEndpointCache = &pCurr->pEndpoint->DataCache;
+            RTSemRWRequestWrite(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT);
+
+            if (!(pCurr->fFlags & PDMACFILECACHE_ENTRY_IO_IN_PROGRESS)
+                && (ASMAtomicReadU32(&pCurr->cRefs) == 0))
             {
-                RTMemPageFree(pCurr->pbData);
-                pCurr->pbData = NULL;
+                LogFlow(("Evicting entry %#p (%u bytes)\n", pCurr, pCurr->cbData));
+                if (pCurr->pbData)
+                {
+                    RTMemPageFree(pCurr->pbData);
+                    pCurr->pbData = NULL;
+                }
+
+                cbEvicted += pCurr->cbData;
+
+                if (pGhostListDst)
+                {
+                    pdmacFileCacheEntryAddToList(pGhostListDst, pCurr);
+                }
+                else
+                {
+                    /* Delete the entry from the AVL tree it is assigned to. */
+                    STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
+                    RTAvlrFileOffsetRemove(pCurr->pEndpoint->DataCache.pTree, pCurr->Core.Key);
+                    STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
+
+                    pdmacFileCacheEntryRemoveFromList(pCurr);
+                    pCache->cbCached -= pCurr->cbData;
+                    RTMemFree(pCurr);
+                }
             }
-
-            cbEvicted += pCurr->cbData;
-
-            if (pGhostListDst)
-            {
-                pdmacFileCacheEntryAddToList(pGhostListDst, pCurr);
-            }
-            else
-            {
-                /* Delete the entry from the AVL tree it is assigned to. */
-                STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
-                RTAvlrFileOffsetRemove(pCurr->pEndpoint->DataCache.pTree, pCurr->Core.Key);
-                STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
-
-                pdmacFileCacheEntryRemoveFromList(pCurr);
-                pCache->cbCached -= pCurr->cbData;
-                RTMemFree(pCurr);
-            }
+            RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
         }
         else
@@ -269 +304 @@
 static size_t pdmacFileCacheReplace(PPDMACFILECACHEGLOBAL pCache, size_t cbData, PPDMACFILELRULIST pEntryList)
 {
+    PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
+
     if (   (pCache->LruRecentlyUsed.cbCached)
         && (   (pCache->LruRecentlyUsed.cbCached > pCache->uAdaptVal)
@@ -298 +335 @@
 {
     size_t cbRemoved = ~0;
+
+    PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
 
     if ((pCache->LruRecentlyUsed.cbCached + pCache->LruRecentlyGhost.cbCached) >= pCache->cbMax)
@@ -346 +385 @@
     int32_t uUpdateVal = 0;
 
+    PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
+
     /* Update parameters */
     if (pEntry->pList == &pCache->LruRecentlyGhost)
@@ -436 +477 @@
     PPDMACFILECACHEENTRY pEntry = (PPDMACFILECACHEENTRY)pvUser;
     PPDMACFILECACHEGLOBAL pCache = pEntry->pCache;
-
-    RTCritSectEnter(&pCache->CritSect);
-
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint = pEntry->pEndpoint;
+
+    /* Reference the entry now as we are clearing the I/O in progres flag
+     * which protects the entry till now. */
+    pdmacFileEpCacheEntryRef(pEntry);
+
+    RTSemRWRequestWrite(pEndpoint->DataCache.SemRWEntries, RT_INDEFINITE_WAIT);
     pEntry->fFlags &= ~PDMACFILECACHE_ENTRY_IO_IN_PROGRESS;
 
@@ -503 +548 @@
         pdmacFileCacheWriteToEndpoint(pEntry);
 
-    RTCritSectLeave(&pCache->CritSect);
+    RTSemRWReleaseWrite(pEndpoint->DataCache.SemRWEntries);
+
+    /* Dereference so that it isn't protected anymore except we issued anyother write for it. */
+    pdmacFileEpCacheEntryRelease(pEntry);
 }
 
@@ -648 +696 @@
     PPDMACFILEENDPOINTCACHE pEndpointCache = &pEndpoint->DataCache;
 
-    pEndpointCache->pTree  = (PAVLRFOFFTREE)RTMemAllocZ(sizeof(AVLRFOFFTREE));
     pEndpointCache->pCache = &pClassFile->Cache;
 
-    return VINF_SUCCESS;
+    int rc = RTSemRWCreate(&pEndpointCache->SemRWEntries);
+    if (RT_SUCCESS(rc))
+    {
+        pEndpointCache->pTree  = (PAVLRFOFFTREE)RTMemAllocZ(sizeof(AVLRFOFFTREE));
+        if (!pEndpointCache->pTree)
+        {
+            rc = VERR_NO_MEMORY;
+            RTSemRWDestroy(pEndpointCache->SemRWEntries);
+        }
+    }
+
+    return rc;
 }
 
@@ -690 +748 @@
 
     /* Make sure nobody is accessing the cache while we delete the tree. */
+    RTSemRWRequestWrite(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT);
     RTCritSectEnter(&pCache->CritSect);
     RTAvlrFileOffsetDestroy(pEndpointCache->pTree, pdmacFileEpCacheEntryDestroy, pCache);
     RTCritSectLeave(&pCache->CritSect);
+    RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
+
+    RTSemRWDestroy(pEndpointCache->SemRWEntries);
+}
+
+static PPDMACFILECACHEENTRY pdmacFileEpCacheGetCacheEntryByOffset(PPDMACFILEENDPOINTCACHE pEndpointCache, RTFOFF off)
+{
+    PPDMACFILECACHEGLOBAL pCache = pEndpointCache->pCache;
+    PPDMACFILECACHEENTRY pEntry = NULL;
+
+    STAM_PROFILE_ADV_START(&pCache->StatTreeGet, Cache);
+
+    RTSemRWRequestRead(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT);
+    pEntry = (PPDMACFILECACHEENTRY)RTAvlrFileOffsetRangeGet(pEndpointCache->pTree, off);
+    if (pEntry)
+        pdmacFileEpCacheEntryRef(pEntry);
+    RTSemRWReleaseRead(pEndpointCache->SemRWEntries);
+
+    STAM_PROFILE_ADV_STOP(&pCache->StatTreeGet, Cache);
+
+    return pEntry;
+}
+
+static PPDMACFILECACHEENTRY pdmacFileEpCacheGetCacheBestFitEntryByOffset(PPDMACFILEENDPOINTCACHE pEndpointCache, RTFOFF off)
+{
+    PPDMACFILECACHEGLOBAL pCache = pEndpointCache->pCache;
+    PPDMACFILECACHEENTRY pEntry = NULL;
+
+    STAM_PROFILE_ADV_START(&pCache->StatTreeGet, Cache);
+
+    RTSemRWRequestRead(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT);
+    pEntry = (PPDMACFILECACHEENTRY)RTAvlrFileOffsetGetBestFit(pEndpointCache->pTree, off, true);
+    if (pEntry)
+        pdmacFileEpCacheEntryRef(pEntry);
+    RTSemRWReleaseRead(pEndpointCache->SemRWEntries);
+
+    STAM_PROFILE_ADV_STOP(&pCache->StatTreeGet, Cache);
+
+    return pEntry;
+}
+
+static void pdmacFileEpCacheInsertEntry(PPDMACFILEENDPOINTCACHE pEndpointCache, PPDMACFILECACHEENTRY pEntry)
+{
+    PPDMACFILECACHEGLOBAL pCache = pEndpointCache->pCache;
+
+    STAM_PROFILE_ADV_START(&pCache->StatTreeInsert, Cache);
+    RTSemRWRequestWrite(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT);
+    bool fInserted = RTAvlrFileOffsetInsert(pEndpointCache->pTree, &pEntry->Core);
+    AssertMsg(fInserted, ("Node was not inserted into tree\n"));
+    STAM_PROFILE_ADV_STOP(&pCache->StatTreeInsert, Cache);
+    RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
 }
 
@@ -741 +851 @@
     ASMAtomicWriteBool(&pTask->fCompleted, true);
 
-    RTCritSectEnter(&pCache->CritSect);
-
     int iSegCurr       = 0;
     uint8_t *pbSegBuf  = (uint8_t *)paSegments[iSegCurr].pvSeg;
@@ -751 +859 @@
         size_t cbToRead;
 
-        STAM_PROFILE_ADV_START(&pCache->StatTreeGet, Cache);
-        pEntry = (PPDMACFILECACHEENTRY)RTAvlrFileOffsetRangeGet(pEndpointCache->pTree, off);
-        STAM_PROFILE_ADV_STOP(&pCache->StatTreeGet, Cache);
+        pEntry = pdmacFileEpCacheGetCacheEntryByOffset(pEndpointCache, off);
 
         /*
@@ -793 +899 @@
                     && !(pEntry->fFlags & PDMACFILECACHE_ENTRY_IS_DIRTY))
                 {
-                    /* Entry didn't completed yet. Append to the list */
-                    while (cbToRead)
+                    RTSemRWRequestWrite(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT);
+                    /* Check again. The completion callback might have raced us. */
+
+                    if (   (pEntry->fFlags & PDMACFILECACHE_ENTRY_IO_IN_PROGRESS)
+                        && !(pEntry->fFlags & PDMACFILECACHE_ENTRY_IS_DIRTY))
                     {
-                        PPDMACFILETASKSEG pSeg = (PPDMACFILETASKSEG)RTMemAllocZ(sizeof(PDMACFILETASKSEG));
-
-                        pSeg->pTask      = pTask;
-                        pSeg->uBufOffset = OffDiff;
-                        pSeg->cbTransfer = RT_MIN(cbToRead, cbSegLeft);
-                        pSeg->pvBuf      = pbSegBuf;
-                        pSeg->fWrite     = false;
-
-                        ADVANCE_SEGMENT_BUFFER(pSeg->cbTransfer);
-
-                        pSeg->pNext = pEntry->pHead;
-                        pEntry->pHead = pSeg;
-
-                        off      += pSeg->cbTransfer;
-                        cbToRead -= pSeg->cbTransfer;
-                        OffDiff  += pSeg->cbTransfer;
+                        /* Entry didn't completed yet. Append to the list */
+                        while (cbToRead)
+                        {
+                            PPDMACFILETASKSEG pSeg = (PPDMACFILETASKSEG)RTMemAllocZ(sizeof(PDMACFILETASKSEG));
+
+                            pSeg->pTask      = pTask;
+                            pSeg->uBufOffset = OffDiff;
+                            pSeg->cbTransfer = RT_MIN(cbToRead, cbSegLeft);
+                            pSeg->pvBuf      = pbSegBuf;
+                            pSeg->fWrite     = false;
+
+                            ADVANCE_SEGMENT_BUFFER(pSeg->cbTransfer);
+
+                            pSeg->pNext = pEntry->pHead;
+                            pEntry->pHead = pSeg;
+
+                            off      += pSeg->cbTransfer;
+                            cbToRead -= pSeg->cbTransfer;
+                            OffDiff  += pSeg->cbTransfer;
+                        }
+                        RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
+                    }
+                    else
+                    {
+                        RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
+
+                        /* Read as much as we can from the entry. */
+                        while (cbToRead)
+                        {
+                            size_t cbCopy = RT_MIN(cbSegLeft, cbToRead);
+
+                            memcpy(pbSegBuf, pEntry->pbData + OffDiff, cbCopy);
+
+                            ADVANCE_SEGMENT_BUFFER(cbCopy);
+
+                            cbToRead -= cbCopy;
+                            off      += cbCopy;
+                            OffDiff  += cbCopy;
+                            ASMAtomicSubS32(&pTask->cbTransferLeft, cbCopy);
+                        }
                     }
                 }
@@ -833 +966 @@
 
                 /* Move this entry to the top position */
+                RTCritSectEnter(&pCache->CritSect);
                 pdmacFileCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
+                RTCritSectLeave(&pCache->CritSect);
             }
             else
             {
+                RTCritSectEnter(&pCache->CritSect);
                 pdmacFileCacheUpdate(pCache, pEntry);
                 pdmacFileCacheReplace(pCache, pEntry->cbData, pEntry->pList);
@@ -842 +978 @@
                 /* Move the entry to T2 and fetch it to the cache. */
                 pdmacFileCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
+                RTCritSectLeave(&pCache->CritSect);
 
                 pEntry->pbData  = (uint8_t *)RTMemPageAlloc(pEntry->cbData);
@@ -871 +1008 @@
                 pdmacFileCacheReadFromEndpoint(pEntry);
             }
+            pdmacFileEpCacheEntryRelease(pEntry);
         }
         else
         {
             /* No entry found for this offset. Get best fit entry and fetch the data to the cache. */
-            STAM_PROFILE_ADV_START(&pCache->StatTreeGet, Cache);
-            PPDMACFILECACHEENTRY pEntryBestFit = (PPDMACFILECACHEENTRY)RTAvlrFileOffsetGetBestFit(pEndpointCache->pTree, off, true);
-            STAM_PROFILE_ADV_STOP(&pCache->StatTreeGet, Cache);
+            PPDMACFILECACHEENTRY pEntryBestFit = pdmacFileEpCacheGetCacheBestFitEntryByOffset(pEndpointCache, off);
 
             LogFlow(("%sbest fit entry for off=%RTfoff (BestFit=%RTfoff BestFitEnd=%RTfoff BestFitSize=%u)\n",
@@ -887 +1023 @@
 
             if (pEntryBestFit && ((off + (RTFOFF)cbRead) > pEntryBestFit->Core.Key))
+            {
                 cbToRead = pEntryBestFit->Core.Key - off;
+                pdmacFileEpCacheEntryRelease(pEntryBestFit);
+            }
             else
                 cbToRead = cbRead;
@@ -898 +1037 @@
                 STAM_COUNTER_INC(&pCache->cPartialHits);
 
+            RTCritSectEnter(&pCache->CritSect);
             size_t cbRemoved = pdmacFileCacheEvict(pCache, cbToRead);
+            RTCritSectLeave(&pCache->CritSect);
 
             if (cbRemoved >= cbToRead)
@@ -911 +1052 @@
                 pEntryNew->pCache       = pCache;
                 pEntryNew->fFlags       = 0;
+                pEntryNew->cRefs        = 1; /* We are using it now. */
                 pEntryNew->pList        = NULL;
                 pEntryNew->cbData       = cbToRead;
@@ -916 +1058 @@
                 pEntryNew->pbData       = (uint8_t *)RTMemPageAlloc(cbToRead);
                 AssertPtr(pEntryNew->pbData);
+
+                RTCritSectEnter(&pCache->CritSect);
                 pdmacFileCacheEntryAddToList(&pCache->LruRecentlyUsed, pEntryNew);
-
-                STAM_PROFILE_ADV_START(&pCache->StatTreeInsert, Cache);
-                bool fInserted = RTAvlrFileOffsetInsert(pEndpoint->DataCache.pTree, &pEntryNew->Core);
-                AssertMsg(fInserted, ("Node was not inserted into tree\n"));
-                STAM_PROFILE_ADV_STOP(&pCache->StatTreeInsert, Cache);
-
+                RTCritSectLeave(&pCache->CritSect);
+
+                pdmacFileEpCacheInsertEntry(pEndpointCache, pEntryNew);
                 uint32_t uBufOffset = 0;
 
@@ -947 +1088 @@
 
                 pdmacFileCacheReadFromEndpoint(pEntryNew);
+                pdmacFileEpCacheEntryRelease(pEntryNew); /* it is protected by the I/O in progress flag now. */
             }
             else
@@ -987 +1129 @@
         pdmR3AsyncCompletionCompleteTask(&pTask->Core);
 
-    RTCritSectLeave(&pCache->CritSect);
-
    return rc;
 }
@@ -1019 +1159 @@
     ASMAtomicWriteBool(&pTask->fCompleted, true);
 
-    RTCritSectEnter(&pCache->CritSect);
-
     int iSegCurr       = 0;
     uint8_t *pbSegBuf  = (uint8_t *)paSegments[iSegCurr].pvSeg;
@@ -1029 +1167 @@
         size_t cbToWrite;
 
-        STAM_PROFILE_ADV_START(&pCache->StatTreeGet, Cache);
-        pEntry = (PPDMACFILECACHEENTRY)RTAvlrFileOffsetRangeGet(pEndpointCache->pTree, off);
-        STAM_PROFILE_ADV_STOP(&pCache->StatTreeGet, Cache);
+        pEntry = pdmacFileEpCacheGetCacheEntryByOffset(pEndpointCache, off);
 
         if (pEntry)
@@ -1060 +1196 @@
                 if (pEntry->fFlags & PDMACFILECACHE_ENTRY_IS_DIRTY)
                 {
-                    AssertMsg(pEntry->fFlags & PDMACFILECACHE_ENTRY_IO_IN_PROGRESS,
-                              ("Entry is dirty but not in progress\n"));
-
-                    /* The data isn't written to the file yet */
-                    while (cbToWrite)
+                    RTSemRWRequestWrite(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT);
+                    /* Check again. The completion callback might have raced us. */
+
+                    if (pEntry->fFlags & PDMACFILECACHE_ENTRY_IS_DIRTY)
                     {
-                        PPDMACFILETASKSEG pSeg = (PPDMACFILETASKSEG)RTMemAllocZ(sizeof(PDMACFILETASKSEG));
-
-                        pSeg->pTask      = pTask;
-                        pSeg->uBufOffset = OffDiff;
-                        pSeg->cbTransfer = RT_MIN(cbToWrite, cbSegLeft);
-                        pSeg->pvBuf      = pbSegBuf;
-                        pSeg->fWrite     = true;
-
-                        ADVANCE_SEGMENT_BUFFER(pSeg->cbTransfer);
-
-                        pSeg->pNext = pEntry->pHead;
-                        pEntry->pHead = pSeg;
-
-                        off       += pSeg->cbTransfer;
-                        OffDiff   += pSeg->cbTransfer;
-                        cbToWrite -= pSeg->cbTransfer;
+                        AssertMsg(pEntry->fFlags & PDMACFILECACHE_ENTRY_IO_IN_PROGRESS,
+                                  ("Entry is dirty but not in progress\n"));
+
+                        /* The data isn't written to the file yet */
+                        while (cbToWrite)
+                        {
+                            PPDMACFILETASKSEG pSeg = (PPDMACFILETASKSEG)RTMemAllocZ(sizeof(PDMACFILETASKSEG));
+
+                            pSeg->pTask      = pTask;
+                            pSeg->uBufOffset = OffDiff;
+                            pSeg->cbTransfer = RT_MIN(cbToWrite, cbSegLeft);
+                            pSeg->pvBuf      = pbSegBuf;
+                            pSeg->fWrite     = true;
+
+                            ADVANCE_SEGMENT_BUFFER(pSeg->cbTransfer);
+
+                            pSeg->pNext = pEntry->pHead;
+                            pEntry->pHead = pSeg;
+
+                            off       += pSeg->cbTransfer;
+                            OffDiff   += pSeg->cbTransfer;
+                            cbToWrite -= pSeg->cbTransfer;
+                        }
+                        RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
+                    }
+                    else
+                    {
+                        RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
+
+                        AssertMsg(!(pEntry->fFlags & PDMACFILECACHE_ENTRY_IO_IN_PROGRESS),
+                                  ("Entry is not dirty but in progress\n"));
+
+                        /* Write as much as we can into the entry and update the file. */
+                        while (cbToWrite)
+                        {
+                            size_t cbCopy = RT_MIN(cbSegLeft, cbToWrite);
+
+                            memcpy(pEntry->pbData + OffDiff, pbSegBuf, cbCopy);
+
+                            ADVANCE_SEGMENT_BUFFER(cbCopy);
+
+                            cbToWrite-= cbCopy;
+                            off      += cbCopy;
+                            OffDiff  += cbCopy;
+                            ASMAtomicSubS32(&pTask->cbTransferLeft, cbCopy);
+                        }
+
+                        pEntry->fFlags |= PDMACFILECACHE_ENTRY_IS_DIRTY;
+                        pdmacFileCacheWriteToEndpoint(pEntry);
                     }
                 }
@@ -1109 +1277 @@
 
                 /* Move this entry to the top position */
+                RTCritSectEnter(&pCache->CritSect);
                 pdmacFileCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
+                RTCritSectLeave(&pCache->CritSect);
             }
             else
             {
+                RTCritSectEnter(&pCache->CritSect);
                 pdmacFileCacheUpdate(pCache, pEntry);
                 pdmacFileCacheReplace(pCache, pEntry->cbData, pEntry->pList);
@@ -1118 +1289 @@
                 /* Move the entry to T2 and fetch it to the cache. */
                 pdmacFileCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
+                RTCritSectLeave(&pCache->CritSect);
 
                 pEntry->pbData  = (uint8_t *)RTMemPageAlloc(pEntry->cbData);
@@ -1148 +1320 @@
                 pdmacFileCacheReadFromEndpoint(pEntry);
             }
+
+            /* Release the reference. If it is still needed the I/O in progress flag should protect it now. */
+            pdmacFileEpCacheEntryRelease(pEntry);
         }
         else
@@ -1154 +1329 @@
              * No entry found. Write directly into file.
              */
-            STAM_PROFILE_ADV_START(&pCache->StatTreeGet, Cache);
-            PPDMACFILECACHEENTRY pEntryBestFit = (PPDMACFILECACHEENTRY)RTAvlrFileOffsetGetBestFit(pEndpointCache->pTree, off, true);
-            STAM_PROFILE_ADV_STOP(&pCache->StatTreeGet, Cache);
+            PPDMACFILECACHEENTRY pEntryBestFit = pdmacFileEpCacheGetCacheBestFitEntryByOffset(pEndpointCache, off);
 
             LogFlow(("%sbest fit entry for off=%RTfoff (BestFit=%RTfoff BestFitEnd=%RTfoff BestFitSize=%u)\n",
@@ -1166 +1339 @@
 
             if (pEntryBestFit && ((off + (RTFOFF)cbWrite) > pEntryBestFit->Core.Key))
+            {
                 cbToWrite = pEntryBestFit->Core.Key - off;
+                pdmacFileEpCacheEntryRelease(pEntryBestFit);
+            }
             else
                 cbToWrite = cbWrite;
@@ -1202 +1378 @@
         pdmR3AsyncCompletionCompleteTask(&pTask->Core);
 
-    RTCritSectLeave(&pCache->CritSect);
-
     return VINF_SUCCESS;
 }

trunk/src/VBox/VMM/PDMAsyncCompletionFileFailsafe.cpp

@@ -47 +47 @@
                 if (pCurr->enmTransferType == PDMACTASKFILETRANSFER_READ)
                 {
+                    if (RT_UNLIKELY((pCurr->Off + pCurr->DataSeg.cbSeg) > pEndpoint->cbFile))
+                    {
+                        ASMAtomicWriteU64(&pEndpoint->cbFile, pCurr->Off + pCurr->DataSeg.cbSeg);
+                        RTFileSetSize(pEndpoint->File, pCurr->Off + pCurr->DataSeg.cbSeg);
+                    }
+
                     rc = RTFileReadAt(pEndpoint->File, pCurr->Off,
                                       pCurr->DataSeg.pvSeg,

trunk/src/VBox/VMM/PDMAsyncCompletionFileInternal.h

@@ -134 +134 @@
     /** List of endpoints assigned to this manager. */
     R3PTRTYPE(PPDMASYNCCOMPLETIONENDPOINTFILE) pEndpointsHead;
+    /** Number of endpoints assigned to the manager. */
+    unsigned                               cEndpoints;
     /** Number of requests active currently. */
     unsigned                               cRequestsActive;
@@ -221 +223 @@
     /** Flags for this entry. Combinations of PDMACFILECACHE_* #defines */
     uint32_t                        fFlags;
+    /** Reference counter. Prevents eviction of the entry if > 0. */
+    volatile uint32_t               cRefs;
     /** Size of the entry. */
     size_t                          cbData;
@@ -298 +302 @@
     /** AVL tree managing cache entries. */
     PAVLRFOFFTREE         pTree;
-    /** Critical section protecting the tree. */
-    RTCRITSECT            CritSect;
+    /** R/W semaphore protecting cached entries for this endpoint. */
+    RTSEMRW               SemRWEntries;
     /** Pointer to the gobal cache data */
     PPDMACFILECACHEGLOBAL pCache;
@@ -383 +387 @@
     /** Size of the underlying file.
      * Updated while data is appended. */
-    uint64_t                               cbFile;
+    volatile uint64_t                      cbFile;
     /** Flag whether caching is enabled for this file. */
     bool                                   fCaching;
@@ -444 +448 @@
         /** Current number of processed requests for the current update period. */
         unsigned                                   cReqsProcessed;
+        /** Flag whether the endpoint is about to be moved to another manager. */
+        bool                                       fMoving;
+        /** Destination I/O manager. */
+        PPDMACEPFILEMGR                            pAioMgrDst;
     } AioMgr;
 } PDMASYNCCOMPLETIONENDPOINTFILE;
@@ -514 +522 @@
 void pdmacFileAioMgrNormalDestroy(PPDMACEPFILEMGR pAioMgr);
 
+int pdmacFileAioMgrCreate(PPDMASYNCCOMPLETIONEPCLASSFILE pEpClass, PPPDMACEPFILEMGR ppAioMgr);
+
+int pdmacFileAioMgrAddEndpoint(PPDMACEPFILEMGR pAioMgr, PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint);
+
 PPDMACTASKFILE pdmacFileEpGetNewTasks(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint);
 PPDMACTASKFILE pdmacFileTaskAlloc(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint);

trunk/src/VBox/VMM/PDMAsyncCompletionFileNormal.cpp

@@ -22 +22 @@
 #define LOG_GROUP LOG_GROUP_PDM_ASYNC_COMPLETION
 #include <iprt/types.h>
+#include <iprt/asm.h>
 #include <iprt/file.h>
 #include <iprt/mem.h>
@@ -75 +76 @@
 
     RTMemFree(pAioMgr->pahReqsFree);
+}
+
+/**
+ * Sorts the endpoint list with insertion sort.
+ */
+static void pdmacFileAioMgrNormalEndpointsSortByLoad(PPDMACEPFILEMGR pAioMgr)
+{
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEpPrev, pEpCurr, pEpNextToSort;
+
+    pEpPrev = pAioMgr->pEndpointsHead;
+    pEpCurr = pEpPrev->AioMgr.pEndpointNext;
+
+    while (pEpCurr)
+    {
+        /* Remember the next element to sort because the list might change. */
+        pEpNextToSort = pEpCurr->AioMgr.pEndpointNext;
+
+        /* Unlink the current element from the list. */
+        PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEpCurr->AioMgr.pEndpointPrev;
+        PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEpCurr->AioMgr.pEndpointNext;
+
+        if (pPrev)
+            pPrev->AioMgr.pEndpointNext = pNext;
+        else
+            pAioMgr->pEndpointsHead = pNext;
+
+        if (pNext)
+            pNext->AioMgr.pEndpointPrev = pPrev;
+
+        /* Go back until we reached the place to insert the current endpoint into. */
+        while (pEpPrev && (pEpPrev->AioMgr.cReqsPerSec < pEpCurr->AioMgr.cReqsPerSec))
+            pEpPrev = pEpPrev->AioMgr.pEndpointPrev;
+
+        /* Link the endpoint into the list. */
+        if (pEpPrev)
+            pNext = pEpPrev->AioMgr.pEndpointNext;
+        else
+            pNext = pAioMgr->pEndpointsHead;
+
+        pEpCurr->AioMgr.pEndpointNext = pNext;
+        pEpCurr->AioMgr.pEndpointPrev = pEpPrev;
+        pNext->AioMgr.pEndpointPrev = pEpCurr;
+        if (pEpPrev)
+            pEpPrev->AioMgr.pEndpointNext = pEpCurr;
+        else
+            pAioMgr->pEndpointsHead = pEpCurr;
+
+        pEpCurr = pEpNextToSort;
+    }
+
+#ifdef DEBUG
+    /* Validate sorting alogrithm */
+    unsigned cEndpoints = 0;
+    pEpCurr = pAioMgr->pEndpointsHead;
+
+    AssertMsg(pEpCurr, ("No endpoint in the list?\n"));
+    AssertMsg(!pEpCurr->AioMgr.pEndpointPrev, ("First element in the list points to previous element\n"));
+
+    while (pEpCurr)
+    {
+        cEndpoints++;
+
+        PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEpCurr->AioMgr.pEndpointNext;
+        PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEpCurr->AioMgr.pEndpointPrev;
+
+        Assert(!pNext || pNext->AioMgr.cReqsPerSec <= pEpCurr->AioMgr.cReqsPerSec);
+        Assert(!pPrev || pPrev->AioMgr.cReqsPerSec >= pEpCurr->AioMgr.cReqsPerSec);
+
+        pEpCurr = pNext;
+    }
+
+    AssertMsg(cEndpoints == pAioMgr->cEndpoints, ("Endpoints lost during sort!\n"));
+
+#endif
+}
+
+/**
+ * Removes an endpoint from the currently assigned manager.
+ *
+ * @returns TRUE if there are still requests pending on the current manager for this endpoint.
+ *          FALSE otherwise.
+ * @param   pEndpointRemove    The endpoint to remove.
+ */
+static bool pdmacFileAioMgrNormalRemoveEndpoint(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointRemove)
+{
+    PPDMASYNCCOMPLETIONENDPOINTFILE pPrev   = pEndpointRemove->AioMgr.pEndpointPrev;
+    PPDMASYNCCOMPLETIONENDPOINTFILE pNext   = pEndpointRemove->AioMgr.pEndpointNext;
+    PPDMACEPFILEMGR                 pAioMgr = pEndpointRemove->pAioMgr;
+
+    pAioMgr->cEndpoints--;
+
+    if (pPrev)
+        pPrev->AioMgr.pEndpointNext = pNext;
+    else
+        pAioMgr->pEndpointsHead = pNext;
+
+    if (pNext)
+        pNext->AioMgr.pEndpointPrev = pPrev;
+
+    /* Make sure that there is no request pending on this manager for the endpoint. */
+    if (!pEndpointRemove->AioMgr.cRequestsActive)
+    {
+        Assert(!pEndpointRemove->pFlushReq);
+
+        /* Reopen the file so that the new endpoint can reassociate with the file */
+        RTFileClose(pEndpointRemove->File);
+        int rc = RTFileOpen(&pEndpointRemove->File, pEndpointRemove->Core.pszUri, pEndpointRemove->fFlags);
+        AssertRC(rc);
+        return false;
+    }
+
+    return true;
+}
+
+/**
+ * Creates a new I/O manager and spreads the I/O load of the endpoints
+ * between the given I/O manager and the new one.
+ *
+ * @returns nothing.
+ * @param   pAioMgr    The I/O manager with high I/O load.
+ */
+static void pdmacFileAioMgrNormalBalanceLoad(PPDMACEPFILEMGR pAioMgr)
+{
+    PPDMACEPFILEMGR pAioMgrNew = NULL;
+    int rc = VINF_SUCCESS;
+
+    /* Splitting can't be done with only one open endpoint. */
+    if (pAioMgr->cEndpoints > 1)
+    {
+        rc = pdmacFileAioMgrCreate((PPDMASYNCCOMPLETIONEPCLASSFILE)pAioMgr->pEndpointsHead->Core.pEpClass,
+                                   &pAioMgrNew);
+        if (RT_SUCCESS(rc))
+        {
+            /* We will sort the list by request count per second. */
+            pdmacFileAioMgrNormalEndpointsSortByLoad(pAioMgr);
+
+            /* Now move some endpoints to the new manager. */
+            unsigned cReqsHere  = pAioMgr->pEndpointsHead->AioMgr.cReqsPerSec;
+            unsigned cReqsOther = 0;
+            PPDMASYNCCOMPLETIONENDPOINTFILE pCurr = pAioMgr->pEndpointsHead->AioMgr.pEndpointNext;
+
+            while (pCurr)
+            {
+                if (cReqsHere <= cReqsOther)
+                {
+                    /*
+                     * The other manager has more requests to handle now.
+                     * We will keep the current endpoint.
+                     */
+                    Log(("Keeping endpoint %#p{%s} with %u reqs/s\n", pCurr->Core.pszUri, pCurr->AioMgr.cReqsPerSec));
+                    cReqsHere += pCurr->AioMgr.cReqsPerSec;
+                    pCurr = pCurr->AioMgr.pEndpointNext;
+                }
+                else
+                {
+                    /* Move to other endpoint. */
+                    Log(("Moving endpoint %#p{%s} with %u reqs/s to other manager\n", pCurr->Core.pszUri, pCurr->AioMgr.cReqsPerSec));
+                    cReqsOther += pCurr->AioMgr.cReqsPerSec;
+
+                    PPDMASYNCCOMPLETIONENDPOINTFILE pMove = pCurr;
+
+                    pCurr = pCurr->AioMgr.pEndpointNext;
+
+                    bool fReqsPending = pdmacFileAioMgrNormalRemoveEndpoint(pMove);
+
+                    if (fReqsPending)
+                    {
+                        pMove->enmState          = PDMASYNCCOMPLETIONENDPOINTFILESTATE_REMOVING;
+                        pMove->AioMgr.fMoving    = true;
+                        pMove->AioMgr.pAioMgrDst = pAioMgrNew;
+                    }
+                    else
+                    {
+                        pMove->AioMgr.fMoving    = false;
+                        pMove->AioMgr.pAioMgrDst = NULL;
+                        pdmacFileAioMgrAddEndpoint(pAioMgrNew, pMove);
+                    }
+                }
+            }
+        }
+        else
+        {
+            /* Don't process further but leave a log entry about reduced performance. */
+            LogRel(("AIOMgr: Could not create new I/O manager (rc=%Rrc). Expect reduced performance\n", rc));
+        }
+    }
 }
 
@@ -106 +293 @@
     RTFILEAIOREQ  apReqs[20];
     unsigned      cRequests = 0;
+    unsigned      cMaxRequests = PDMACEPFILEMGR_REQS_MAX - pAioMgr->cRequestsActive;
     int           rc = VINF_SUCCESS;
     PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;
@@ -113 +301 @@
 
     /* Go through the list and queue the requests until we get a flush request */
-    while (pTaskHead && !pEndpoint->pFlushReq)
+    while (pTaskHead && !pEndpoint->pFlushReq && (cMaxRequests > 0))
     {
         PPDMACTASKFILE pCurr = pTaskHead;
@@ -205 +393 @@
 
                 if (pCurr->enmTransferType == PDMACTASKFILETRANSFER_WRITE)
+                {
+                    /* Grow the file if needed. */
+                    if (RT_UNLIKELY((pCurr->Off + pCurr->DataSeg.cbSeg) > pEndpoint->cbFile))
+                    {
+                        ASMAtomicWriteU64(&pEndpoint->cbFile, pCurr->Off + pCurr->DataSeg.cbSeg);
+                        RTFileSetSize(pEndpoint->File, pCurr->Off + pCurr->DataSeg.cbSeg);
+                    }
+
                     rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->File,
                                                   pCurr->Off, pvBuf, pCurr->DataSeg.cbSeg, pCurr);
+                }
                 else
                     rc = RTFileAioReqPrepareRead(hReq, pEndpoint->File,
@@ -214 +411 @@
                 apReqs[cRequests] = hReq;
                 pEndpoint->AioMgr.cReqsProcessed++;
+                cMaxRequests--;
                 cRequests++;
                 if (cRequests == RT_ELEMENTS(apReqs))
@@ -238 +436 @@
         pAioMgr->cRequestsActive += cRequests;
         rc = RTFileAioCtxSubmit(pAioMgr->hAioCtx, apReqs, cRequests);
-        if (RT_FAILURE(rc))
-        {
-            /* Not enough ressources on this context anymore. */
-            /* @todo implement */
-            AssertMsgFailed(("Implement\n"));
-        }
+        AssertMsgReturn(RT_SUCCESS(rc), ("Could not submit %u requests %Rrc\n", cRequests, rc), rc);
+    }
+
+    if (RT_UNLIKELY(!cMaxRequests && pTaskHead && !pEndpoint->pFlushReq))
+    {
+        /*
+         * The I/O manager has no room left for more requests
+         * but there are still requests to process.
+         * Create a new I/O manager and let it handle some endpoints.
+         */
+
+        /* Add the rest of the tasks to the pending list first */
+        if (!pEndpoint->AioMgr.pReqsPendingHead)
+        {
+            Assert(!pEndpoint->AioMgr.pReqsPendingTail);
+            pEndpoint->AioMgr.pReqsPendingHead = pTaskHead;
+        }
+        else
+        {
+            Assert(pEndpoint->AioMgr.pReqsPendingTail);
+            pEndpoint->AioMgr.pReqsPendingTail->pNext = pTaskHead;
+        }
+
+        /* Update the tail. */
+        while (pTaskHead->pNext)
+            pTaskHead = pTaskHead->pNext;
+
+        pEndpoint->AioMgr.pReqsPendingTail = pTaskHead;
+
+        pdmacFileAioMgrNormalBalanceLoad(pAioMgr);
     }
 
@@ -322 +544 @@
             rc = RTFileAioCtxAssociateWithFile(pAioMgr->hAioCtx, pEndpointNew->File);
             fNotifyWaiter = true;
+            pAioMgr->cEndpoints++;
             break;
         }
@@ -329 +552 @@
             AssertMsg(VALID_PTR(pEndpointRemove), ("Removing endpoint event without a endpoint to remove\n"));
 
-            PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEndpointRemove->AioMgr.pEndpointPrev;
-            PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEndpointRemove->AioMgr.pEndpointNext;
-
             pEndpointRemove->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_REMOVING;
-
-            if (pPrev)
-                pPrev->AioMgr.pEndpointNext = pNext;
-            else
-                pAioMgr->pEndpointsHead = pNext;
-
-            if (pNext)
-                pNext->AioMgr.pEndpointPrev = pPrev;
-
-            /* Make sure that there is no request pending on this manager for the endpoint. */
-            if (!pEndpointRemove->AioMgr.cRequestsActive)
-            {
-                Assert(!pEndpointRemove->pFlushReq);
-
-                /* Reopen the file so that the new endpoint can reassociate with the file */
-                RTFileClose(pEndpointRemove->File);
-                rc = RTFileOpen(&pEndpointRemove->File, pEndpointRemove->Core.pszUri, pEndpointRemove->fFlags);
-                AssertRC(rc);
-            }
+            fNotifyWaiter = !pdmacFileAioMgrNormalRemoveEndpoint(pEndpointRemove);
             break;
         }
@@ -364 +566 @@
 
             pEndpointClose->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_CLOSING;
-
-            PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEndpointClose->AioMgr.pEndpointPrev;
-            PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEndpointClose->AioMgr.pEndpointNext;
-
-            if (pPrev)
-                pPrev->AioMgr.pEndpointNext = pNext;
-            else
-                pAioMgr->pEndpointsHead = pNext;
-
-            if (pNext)
-                pNext->AioMgr.pEndpointPrev = pPrev;
-
-            if (!pEndpointClose->AioMgr.cRequestsActive)
-            {
-                Assert(!pEndpointClose->pFlushReq);
-
-                /* Reopen the file to deassociate it from the endpoint. */
-                RTFileClose(pEndpointClose->File);
-                rc = RTFileOpen(&pEndpointClose->File, pEndpointClose->Core.pszUri, pEndpointClose->fFlags);
-                AssertRC(rc);
-                fNotifyWaiter = true;
-            }
+            fNotifyWaiter = !pdmacFileAioMgrNormalRemoveEndpoint(pEndpointClose);
             break;
         }
@@ -483 +664 @@
                 RTFILEAIOREQ  apReqs[20];
                 uint32_t cReqsCompleted = 0;
-
-                rc = RTFileAioCtxWait(pAioMgr->hAioCtx, 1, RT_INDEFINITE_WAIT, apReqs,
+                size_t cReqsWait;
+
+                if (pAioMgr->cRequestsActive > RT_ELEMENTS(apReqs))
+                    cReqsWait = RT_ELEMENTS(apReqs);
+                else
+                    cReqsWait = pAioMgr->cRequestsActive;
+
+                rc = RTFileAioCtxWait(pAioMgr->hAioCtx,
+                                      cReqsWait,
+                                      RT_INDEFINITE_WAIT, apReqs,
                                       RT_ELEMENTS(apReqs), &cReqsCompleted);
-                CHECK_RC(pAioMgr, rc);
+                if (RT_FAILURE(rc) && (rc != VERR_INTERRUPTED))
+                    CHECK_RC(pAioMgr, rc);
 
                 for (uint32_t i = 0; i < cReqsCompleted; i++)
@@ -547 +737 @@
                     else if (!pEndpoint->AioMgr.cRequestsActive)
                     {
-                        Assert(pAioMgr->fBlockingEventPending);
-                        ASMAtomicWriteBool(&pAioMgr->fBlockingEventPending, false);
-
-                        /* Release the waiting thread. */
-                        rc = RTSemEventSignal(pAioMgr->EventSemBlock);
+                        /* Reopen the file so that the new endpoint can reassociate with the file */
+                        RTFileClose(pEndpoint->File);
+                        rc = RTFileOpen(&pEndpoint->File, pEndpoint->Core.pszUri, pEndpoint->fFlags);
                         AssertRC(rc);
+
+                        if (pEndpoint->AioMgr.fMoving)
+                        {
+                            pEndpoint->AioMgr.fMoving = false;
+                            pdmacFileAioMgrAddEndpoint(pEndpoint->AioMgr.pAioMgrDst, pEndpoint);
+                        }
+                        else
+                        {
+                            Assert(pAioMgr->fBlockingEventPending);
+                            ASMAtomicWriteBool(&pAioMgr->fBlockingEventPending, false);
+
+                            /* Release the waiting thread. */
+                            rc = RTSemEventSignal(pAioMgr->EventSemBlock);
+                            AssertRC(rc);
+                        }
                     }
                 }