VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR3/PDMBlkCache.cpp@ 39082

Last change on this file since 39082 was 39078, checked in by vboxsync, 13 years ago

VMM: -Wunused-parameter
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 97.7 KB
Line 
1/* $Id: PDMBlkCache.cpp 39078 2011-10-21 14:18:22Z vboxsync $ */
2/** @file
3 * PDM Block Cache.
4 */
5
6/*
7 * Copyright (C) 2006-2008 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/** @page pg_pdm_block_cache PDM Block Cache - The I/O cache
19 * This component implements an I/O cache based on the 2Q cache algorithm.
20 */
21
22/*******************************************************************************
23* Header Files *
24*******************************************************************************/
25#define LOG_GROUP LOG_GROUP_PDM_BLK_CACHE
26#include "PDMInternal.h"
27#include <iprt/asm.h>
28#include <iprt/mem.h>
29#include <iprt/path.h>
30#include <iprt/string.h>
31#include <VBox/log.h>
32#include <VBox/vmm/stam.h>
33#include <VBox/vmm/uvm.h>
34#include <VBox/vmm/vm.h>
35
36#include "PDMBlkCacheInternal.h"
37
38#ifdef VBOX_STRICT
39# define PDMACFILECACHE_IS_CRITSECT_OWNER(Cache) \
40 do \
41 { \
42 AssertMsg(RTCritSectIsOwner(&Cache->CritSect), \
43 ("Thread does not own critical section\n"));\
44 } while(0)
45
46# define PDMACFILECACHE_EP_IS_SEMRW_WRITE_OWNER(pEpCache) \
47 do \
48 { \
49 AssertMsg(RTSemRWIsWriteOwner(pEpCache->SemRWEntries), \
50 ("Thread is not exclusive owner of the per endpoint RW semaphore\n")); \
51 } while(0)
52
53# define PDMACFILECACHE_EP_IS_SEMRW_READ_OWNER(pEpCache) \
54 do \
55 { \
56 AssertMsg(RTSemRWIsReadOwner(pEpCache->SemRWEntries), \
57 ("Thread is not read owner of the per endpoint RW semaphore\n")); \
58 } while(0)
59
60#else
61# define PDMACFILECACHE_IS_CRITSECT_OWNER(Cache) do { } while(0)
62# define PDMACFILECACHE_EP_IS_SEMRW_WRITE_OWNER(pEpCache) do { } while(0)
63# define PDMACFILECACHE_EP_IS_SEMRW_READ_OWNER(pEpCache) do { } while(0)
64#endif
65
66#define PDM_BLK_CACHE_SAVED_STATE_VERSION 1
67
68/*******************************************************************************
69* Internal Functions *
70*******************************************************************************/
71
72static PPDMBLKCACHEENTRY pdmBlkCacheEntryAlloc(PPDMBLKCACHE pBlkCache,
73 uint64_t off, size_t cbData, uint8_t *pbBuffer);
74static bool pdmBlkCacheAddDirtyEntry(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEENTRY pEntry);
75
76/**
77 * Decrement the reference counter of the given cache entry.
78 *
79 * @returns nothing.
80 * @param pEntry The entry to release.
81 */
82DECLINLINE(void) pdmBlkCacheEntryRelease(PPDMBLKCACHEENTRY pEntry)
83{
84 AssertMsg(pEntry->cRefs > 0, ("Trying to release a not referenced entry\n"));
85 ASMAtomicDecU32(&pEntry->cRefs);
86}
87
88/**
89 * Increment the reference counter of the given cache entry.
90 *
91 * @returns nothing.
92 * @param pEntry The entry to reference.
93 */
94DECLINLINE(void) pdmBlkCacheEntryRef(PPDMBLKCACHEENTRY pEntry)
95{
96 ASMAtomicIncU32(&pEntry->cRefs);
97}
98
99#ifdef VBOX_STRICT
100static void pdmBlkCacheValidate(PPDMBLKCACHEGLOBAL pCache)
101{
102 /* Amount of cached data should never exceed the maximum amount. */
103 AssertMsg(pCache->cbCached <= pCache->cbMax,
104 ("Current amount of cached data exceeds maximum\n"));
105
106 /* The amount of cached data in the LRU and FRU list should match cbCached */
107 AssertMsg(pCache->LruRecentlyUsedIn.cbCached + pCache->LruFrequentlyUsed.cbCached == pCache->cbCached,
108 ("Amount of cached data doesn't match\n"));
109
110 AssertMsg(pCache->LruRecentlyUsedOut.cbCached <= pCache->cbRecentlyUsedOutMax,
111 ("Paged out list exceeds maximum\n"));
112}
113#endif
114
115DECLINLINE(void) pdmBlkCacheLockEnter(PPDMBLKCACHEGLOBAL pCache)
116{
117 RTCritSectEnter(&pCache->CritSect);
118#ifdef VBOX_STRICT
119 pdmBlkCacheValidate(pCache);
120#endif
121}
122
123DECLINLINE(void) pdmBlkCacheLockLeave(PPDMBLKCACHEGLOBAL pCache)
124{
125#ifdef VBOX_STRICT
126 pdmBlkCacheValidate(pCache);
127#endif
128 RTCritSectLeave(&pCache->CritSect);
129}
130
131DECLINLINE(void) pdmBlkCacheSub(PPDMBLKCACHEGLOBAL pCache, uint32_t cbAmount)
132{
133 PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
134 pCache->cbCached -= cbAmount;
135}
136
137DECLINLINE(void) pdmBlkCacheAdd(PPDMBLKCACHEGLOBAL pCache, uint32_t cbAmount)
138{
139 PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
140 pCache->cbCached += cbAmount;
141}
142
143DECLINLINE(void) pdmBlkCacheListAdd(PPDMBLKLRULIST pList, uint32_t cbAmount)
144{
145 pList->cbCached += cbAmount;
146}
147
148DECLINLINE(void) pdmBlkCacheListSub(PPDMBLKLRULIST pList, uint32_t cbAmount)
149{
150 pList->cbCached -= cbAmount;
151}
152
153#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
154/**
155 * Checks consistency of a LRU list.
156 *
157 * @returns nothing
158 * @param pList The LRU list to check.
159 * @param pNotInList Element which is not allowed to occur in the list.
160 */
161static void pdmBlkCacheCheckList(PPDMBLKLRULIST pList, PPDMBLKCACHEENTRY pNotInList)
162{
163 PPDMBLKCACHEENTRY pCurr = pList->pHead;
164
165 /* Check that there are no double entries and no cycles in the list. */
166 while (pCurr)
167 {
168 PPDMBLKCACHEENTRY pNext = pCurr->pNext;
169
170 while (pNext)
171 {
172 AssertMsg(pCurr != pNext,
173 ("Entry %#p is at least two times in list %#p or there is a cycle in the list\n",
174 pCurr, pList));
175 pNext = pNext->pNext;
176 }
177
178 AssertMsg(pCurr != pNotInList, ("Not allowed entry %#p is in list\n", pCurr));
179
180 if (!pCurr->pNext)
181 AssertMsg(pCurr == pList->pTail, ("End of list reached but last element is not list tail\n"));
182
183 pCurr = pCurr->pNext;
184 }
185}
186#endif
187
188/**
189 * Unlinks a cache entry from the LRU list it is assigned to.
190 *
191 * @returns nothing.
192 * @param pEntry The entry to unlink.
193 */
194static void pdmBlkCacheEntryRemoveFromList(PPDMBLKCACHEENTRY pEntry)
195{
196 PPDMBLKLRULIST pList = pEntry->pList;
197 PPDMBLKCACHEENTRY pPrev, pNext;
198
199 LogFlowFunc((": Deleting entry %#p from list %#p\n", pEntry, pList));
200
201 AssertPtr(pList);
202
203#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
204 pdmBlkCacheCheckList(pList, NULL);
205#endif
206
207 pPrev = pEntry->pPrev;
208 pNext = pEntry->pNext;
209
210 AssertMsg(pEntry != pPrev, ("Entry links to itself as previous element\n"));
211 AssertMsg(pEntry != pNext, ("Entry links to itself as next element\n"));
212
213 if (pPrev)
214 pPrev->pNext = pNext;
215 else
216 {
217 pList->pHead = pNext;
218
219 if (pNext)
220 pNext->pPrev = NULL;
221 }
222
223 if (pNext)
224 pNext->pPrev = pPrev;
225 else
226 {
227 pList->pTail = pPrev;
228
229 if (pPrev)
230 pPrev->pNext = NULL;
231 }
232
233 pEntry->pList = NULL;
234 pEntry->pPrev = NULL;
235 pEntry->pNext = NULL;
236 pdmBlkCacheListSub(pList, pEntry->cbData);
237#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
238 pdmBlkCacheCheckList(pList, pEntry);
239#endif
240}
241
242/**
243 * Adds a cache entry to the given LRU list unlinking it from the currently
244 * assigned list if needed.
245 *
246 * @returns nothing.
247 * @param pList List to the add entry to.
248 * @param pEntry Entry to add.
249 */
250static void pdmBlkCacheEntryAddToList(PPDMBLKLRULIST pList, PPDMBLKCACHEENTRY pEntry)
251{
252 LogFlowFunc((": Adding entry %#p to list %#p\n", pEntry, pList));
253#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
254 pdmBlkCacheCheckList(pList, NULL);
255#endif
256
257 /* Remove from old list if needed */
258 if (pEntry->pList)
259 pdmBlkCacheEntryRemoveFromList(pEntry);
260
261 pEntry->pNext = pList->pHead;
262 if (pList->pHead)
263 pList->pHead->pPrev = pEntry;
264 else
265 {
266 Assert(!pList->pTail);
267 pList->pTail = pEntry;
268 }
269
270 pEntry->pPrev = NULL;
271 pList->pHead = pEntry;
272 pdmBlkCacheListAdd(pList, pEntry->cbData);
273 pEntry->pList = pList;
274#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
275 pdmBlkCacheCheckList(pList, NULL);
276#endif
277}
278
279/**
280 * Destroys a LRU list freeing all entries.
281 *
282 * @returns nothing
283 * @param pList Pointer to the LRU list to destroy.
284 *
285 * @note The caller must own the critical section of the cache.
286 */
287static void pdmBlkCacheDestroyList(PPDMBLKLRULIST pList)
288{
289 while (pList->pHead)
290 {
291 PPDMBLKCACHEENTRY pEntry = pList->pHead;
292
293 pList->pHead = pEntry->pNext;
294
295 AssertMsg(!(pEntry->fFlags & (PDMBLKCACHE_ENTRY_IO_IN_PROGRESS | PDMBLKCACHE_ENTRY_IS_DIRTY)),
296 ("Entry is dirty and/or still in progress fFlags=%#x\n", pEntry->fFlags));
297
298 RTMemPageFree(pEntry->pbData, pEntry->cbData);
299 RTMemFree(pEntry);
300 }
301}
302
303/**
304 * Tries to remove the given amount of bytes from a given list in the cache
305 * moving the entries to one of the given ghosts lists
306 *
307 * @returns Amount of data which could be freed.
308 * @param pCache Pointer to the global cache data.
309 * @param cbData The amount of the data to free.
310 * @param pListSrc The source list to evict data from.
311 * @param pGhostListSrc The ghost list removed entries should be moved to
312 * NULL if the entry should be freed.
313 * @param fReuseBuffer Flag whether a buffer should be reused if it has the same size
314 * @param ppbBuf Where to store the address of the buffer if an entry with the
315 * same size was found and fReuseBuffer is true.
316 *
317 * @note This function may return fewer bytes than requested because entries
318 * may be marked as non evictable if they are used for I/O at the
319 * moment.
320 */
321static size_t pdmBlkCacheEvictPagesFrom(PPDMBLKCACHEGLOBAL pCache, size_t cbData,
322 PPDMBLKLRULIST pListSrc, PPDMBLKLRULIST pGhostListDst,
323 bool fReuseBuffer, uint8_t **ppbBuffer)
324{
325 size_t cbEvicted = 0;
326
327 PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
328
329 AssertMsg(cbData > 0, ("Evicting 0 bytes not possible\n"));
330 AssertMsg( !pGhostListDst
331 || (pGhostListDst == &pCache->LruRecentlyUsedOut),
332 ("Destination list must be NULL or the recently used but paged out list\n"));
333
334 if (fReuseBuffer)
335 {
336 AssertPtr(ppbBuffer);
337 *ppbBuffer = NULL;
338 }
339
340 /* Start deleting from the tail. */
341 PPDMBLKCACHEENTRY pEntry = pListSrc->pTail;
342
343 while ((cbEvicted < cbData) && pEntry)
344 {
345 PPDMBLKCACHEENTRY pCurr = pEntry;
346
347 pEntry = pEntry->pPrev;
348
349 /* We can't evict pages which are currently in progress or dirty but not in progress */
350 if ( !(pCurr->fFlags & PDMBLKCACHE_NOT_EVICTABLE)
351 && (ASMAtomicReadU32(&pCurr->cRefs) == 0))
352 {
353 /* Ok eviction candidate. Grab the endpoint semaphore and check again
354 * because somebody else might have raced us. */
355 PPDMBLKCACHE pBlkCache = pCurr->pBlkCache;
356 RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
357
358 if (!(pCurr->fFlags & PDMBLKCACHE_NOT_EVICTABLE)
359 && (ASMAtomicReadU32(&pCurr->cRefs) == 0))
360 {
361 LogFlow(("Evicting entry %#p (%u bytes)\n", pCurr, pCurr->cbData));
362
363 if (fReuseBuffer && pCurr->cbData == cbData)
364 {
365 STAM_COUNTER_INC(&pCache->StatBuffersReused);
366 *ppbBuffer = pCurr->pbData;
367 }
368 else if (pCurr->pbData)
369 RTMemPageFree(pCurr->pbData, pCurr->cbData);
370
371 pCurr->pbData = NULL;
372 cbEvicted += pCurr->cbData;
373
374 pdmBlkCacheEntryRemoveFromList(pCurr);
375 pdmBlkCacheSub(pCache, pCurr->cbData);
376
377 if (pGhostListDst)
378 {
379 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
380
381 PPDMBLKCACHEENTRY pGhostEntFree = pGhostListDst->pTail;
382
383 /* We have to remove the last entries from the paged out list. */
384 while ( pGhostListDst->cbCached + pCurr->cbData > pCache->cbRecentlyUsedOutMax
385 && pGhostEntFree)
386 {
387 PPDMBLKCACHEENTRY pFree = pGhostEntFree;
388 PPDMBLKCACHE pBlkCacheFree = pFree->pBlkCache;
389
390 pGhostEntFree = pGhostEntFree->pPrev;
391
392 RTSemRWRequestWrite(pBlkCacheFree->SemRWEntries, RT_INDEFINITE_WAIT);
393
394 if (ASMAtomicReadU32(&pFree->cRefs) == 0)
395 {
396 pdmBlkCacheEntryRemoveFromList(pFree);
397
398 STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
399 RTAvlrU64Remove(pBlkCacheFree->pTree, pFree->Core.Key);
400 STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
401
402 RTMemFree(pFree);
403 }
404
405 RTSemRWReleaseWrite(pBlkCacheFree->SemRWEntries);
406 }
407
408 if (pGhostListDst->cbCached + pCurr->cbData > pCache->cbRecentlyUsedOutMax)
409 {
410 /* Couldn't remove enough entries. Delete */
411 STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
412 RTAvlrU64Remove(pCurr->pBlkCache->pTree, pCurr->Core.Key);
413 STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
414
415 RTMemFree(pCurr);
416 }
417 else
418 pdmBlkCacheEntryAddToList(pGhostListDst, pCurr);
419 }
420 else
421 {
422 /* Delete the entry from the AVL tree it is assigned to. */
423 STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
424 RTAvlrU64Remove(pCurr->pBlkCache->pTree, pCurr->Core.Key);
425 STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
426
427 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
428 RTMemFree(pCurr);
429 }
430 }
431
432 }
433 else
434 LogFlow(("Entry %#p (%u bytes) is still in progress and can't be evicted\n", pCurr, pCurr->cbData));
435 }
436
437 return cbEvicted;
438}
439
440static bool pdmBlkCacheReclaim(PPDMBLKCACHEGLOBAL pCache, size_t cbData, bool fReuseBuffer, uint8_t **ppbBuffer)
441{
442 size_t cbRemoved = 0;
443
444 if ((pCache->cbCached + cbData) < pCache->cbMax)
445 return true;
446 else if ((pCache->LruRecentlyUsedIn.cbCached + cbData) > pCache->cbRecentlyUsedInMax)
447 {
448 /* Try to evict as many bytes as possible from A1in */
449 cbRemoved = pdmBlkCacheEvictPagesFrom(pCache, cbData, &pCache->LruRecentlyUsedIn,
450 &pCache->LruRecentlyUsedOut, fReuseBuffer, ppbBuffer);
451
452 /*
453 * If it was not possible to remove enough entries
454 * try the frequently accessed cache.
455 */
456 if (cbRemoved < cbData)
457 {
458 Assert(!fReuseBuffer || !*ppbBuffer); /* It is not possible that we got a buffer with the correct size but we didn't freed enough data. */
459
460 /*
461 * If we removed something we can't pass the reuse buffer flag anymore because
462 * we don't need to evict that much data
463 */
464 if (!cbRemoved)
465 cbRemoved += pdmBlkCacheEvictPagesFrom(pCache, cbData, &pCache->LruFrequentlyUsed,
466 NULL, fReuseBuffer, ppbBuffer);
467 else
468 cbRemoved += pdmBlkCacheEvictPagesFrom(pCache, cbData - cbRemoved, &pCache->LruFrequentlyUsed,
469 NULL, false, NULL);
470 }
471 }
472 else
473 {
474 /* We have to remove entries from frequently access list. */
475 cbRemoved = pdmBlkCacheEvictPagesFrom(pCache, cbData, &pCache->LruFrequentlyUsed,
476 NULL, fReuseBuffer, ppbBuffer);
477 }
478
479 LogFlowFunc((": removed %u bytes, requested %u\n", cbRemoved, cbData));
480 return (cbRemoved >= cbData);
481}
482
483DECLINLINE(int) pdmBlkCacheEnqueue(PPDMBLKCACHE pBlkCache, uint64_t off, size_t cbXfer, PPDMBLKCACHEIOXFER pIoXfer)
484{
485 int rc = VINF_SUCCESS;
486
487 LogFlowFunc(("%s: Enqueuing hIoXfer=%#p enmXferDir=%d\n",
488 __FUNCTION__, pIoXfer, pIoXfer->enmXferDir));
489
490 switch (pBlkCache->enmType)
491 {
492 case PDMBLKCACHETYPE_DEV:
493 {
494 rc = pBlkCache->u.Dev.pfnXferEnqueue(pBlkCache->u.Dev.pDevIns,
495 pIoXfer->enmXferDir,
496 off, cbXfer,
497 &pIoXfer->SgBuf, pIoXfer);
498 break;
499 }
500 case PDMBLKCACHETYPE_DRV:
501 {
502 rc = pBlkCache->u.Drv.pfnXferEnqueue(pBlkCache->u.Drv.pDrvIns,
503 pIoXfer->enmXferDir,
504 off, cbXfer,
505 &pIoXfer->SgBuf, pIoXfer);
506 break;
507 }
508 case PDMBLKCACHETYPE_USB:
509 {
510 rc = pBlkCache->u.Usb.pfnXferEnqueue(pBlkCache->u.Usb.pUsbIns,
511 pIoXfer->enmXferDir,
512 off, cbXfer,
513 &pIoXfer->SgBuf, pIoXfer);
514 break;
515 }
516 case PDMBLKCACHETYPE_INTERNAL:
517 {
518 rc = pBlkCache->u.Int.pfnXferEnqueue(pBlkCache->u.Int.pvUser,
519 pIoXfer->enmXferDir,
520 off, cbXfer,
521 &pIoXfer->SgBuf, pIoXfer);
522 break;
523 }
524 default:
525 AssertMsgFailed(("Unknown block cache type!\n"));
526 }
527
528 LogFlowFunc(("%s: returns rc=%Rrc\n", __FUNCTION__, rc));
529 return rc;
530}
531
532/**
533 * Initiates a read I/O task for the given entry.
534 *
535 * @returns VBox status code.
536 * @param pEntry The entry to fetch the data to.
537 */
538static int pdmBlkCacheEntryReadFromMedium(PPDMBLKCACHEENTRY pEntry)
539{
540 PPDMBLKCACHE pBlkCache = pEntry->pBlkCache;
541 LogFlowFunc((": Reading data into cache entry %#p\n", pEntry));
542
543 /* Make sure no one evicts the entry while it is accessed. */
544 pEntry->fFlags |= PDMBLKCACHE_ENTRY_IO_IN_PROGRESS;
545
546 PPDMBLKCACHEIOXFER pIoXfer = (PPDMBLKCACHEIOXFER)RTMemAllocZ(sizeof(PDMBLKCACHEIOXFER));
547 if (RT_UNLIKELY(!pIoXfer))
548 return VERR_NO_MEMORY;
549
550 AssertMsg(pEntry->pbData, ("Entry is in ghost state\n"));
551
552 pIoXfer->fIoCache = true;
553 pIoXfer->pEntry = pEntry;
554 pIoXfer->SgSeg.pvSeg = pEntry->pbData;
555 pIoXfer->SgSeg.cbSeg = pEntry->cbData;
556 pIoXfer->enmXferDir = PDMBLKCACHEXFERDIR_READ;
557 RTSgBufInit(&pIoXfer->SgBuf, &pIoXfer->SgSeg, 1);
558
559 return pdmBlkCacheEnqueue(pBlkCache, pEntry->Core.Key, pEntry->cbData, pIoXfer);
560}
561
562/**
563 * Initiates a write I/O task for the given entry.
564 *
565 * @returns nothing.
566 * @param pEntry The entry to read the data from.
567 */
568static int pdmBlkCacheEntryWriteToMedium(PPDMBLKCACHEENTRY pEntry)
569{
570 PPDMBLKCACHE pBlkCache = pEntry->pBlkCache;
571 LogFlowFunc((": Writing data from cache entry %#p\n", pEntry));
572
573 /* Make sure no one evicts the entry while it is accessed. */
574 pEntry->fFlags |= PDMBLKCACHE_ENTRY_IO_IN_PROGRESS;
575
576 PPDMBLKCACHEIOXFER pIoXfer = (PPDMBLKCACHEIOXFER)RTMemAllocZ(sizeof(PDMBLKCACHEIOXFER));
577 if (RT_UNLIKELY(!pIoXfer))
578 return VERR_NO_MEMORY;
579
580 AssertMsg(pEntry->pbData, ("Entry is in ghost state\n"));
581
582 pIoXfer->fIoCache = true;
583 pIoXfer->pEntry = pEntry;
584 pIoXfer->SgSeg.pvSeg = pEntry->pbData;
585 pIoXfer->SgSeg.cbSeg = pEntry->cbData;
586 pIoXfer->enmXferDir = PDMBLKCACHEXFERDIR_WRITE;
587 RTSgBufInit(&pIoXfer->SgBuf, &pIoXfer->SgSeg, 1);
588
589 return pdmBlkCacheEnqueue(pBlkCache, pEntry->Core.Key, pEntry->cbData, pIoXfer);
590}
591
592/**
593 * Passthrough a part of a request directly to the I/O manager
594 * handling the endpoint.
595 *
596 * @returns VBox status code.
597 * @param pEndpoint The endpoint.
598 * @param pTask The task.
599 * @param pIoMemCtx The I/O memory context to use.
600 * @param offStart Offset to start transfer from.
601 * @param cbData Amount of data to transfer.
602 * @param enmTransferType The transfer type (read/write)
603 */
604static int pdmBlkCacheRequestPassthrough(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEREQ pReq,
605 PRTSGBUF pSgBuf, uint64_t offStart, size_t cbData,
606 PDMBLKCACHEXFERDIR enmXferDir)
607{
608
609 PPDMBLKCACHEIOXFER pIoXfer = (PPDMBLKCACHEIOXFER)RTMemAllocZ(sizeof(PDMBLKCACHEIOXFER));
610 if (RT_UNLIKELY(!pIoXfer))
611 return VERR_NO_MEMORY;
612
613 ASMAtomicIncU32(&pReq->cXfersPending);
614 pIoXfer->fIoCache = false;
615 pIoXfer->pReq = pReq;
616 pIoXfer->enmXferDir = enmXferDir;
617 if (pSgBuf)
618 {
619 RTSgBufClone(&pIoXfer->SgBuf, pSgBuf);
620 RTSgBufAdvance(pSgBuf, cbData);
621 }
622
623 return pdmBlkCacheEnqueue(pBlkCache, offStart, cbData, pIoXfer);
624}
625
626/**
627 * Commit a single dirty entry to the endpoint
628 *
629 * @returns nothing
630 * @param pEntry The entry to commit.
631 */
632static void pdmBlkCacheEntryCommit(PPDMBLKCACHEENTRY pEntry)
633{
634 AssertMsg( (pEntry->fFlags & PDMBLKCACHE_ENTRY_IS_DIRTY)
635 && !(pEntry->fFlags & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS),
636 ("Invalid flags set for entry %#p\n", pEntry));
637
638 pdmBlkCacheEntryWriteToMedium(pEntry);
639}
640
641/**
642 * Commit all dirty entries for a single endpoint.
643 *
644 * @returns nothing.
645 * @param pBlkCache The endpoint cache to commit.
646 */
647static void pdmBlkCacheCommit(PPDMBLKCACHE pBlkCache)
648{
649 uint32_t cbCommitted = 0;
650
651 /* Return if the cache was suspended. */
652 if (pBlkCache->fSuspended)
653 return;
654
655 RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
656
657 /* The list is moved to a new header to reduce locking overhead. */
658 RTLISTNODE ListDirtyNotCommitted;
659 RTSPINLOCKTMP Tmp;
660
661 RTListInit(&ListDirtyNotCommitted);
662 RTSpinlockAcquire(pBlkCache->LockList, &Tmp);
663 RTListMove(&ListDirtyNotCommitted, &pBlkCache->ListDirtyNotCommitted);
664 RTSpinlockRelease(pBlkCache->LockList, &Tmp);
665
666 if (!RTListIsEmpty(&ListDirtyNotCommitted))
667 {
668 PPDMBLKCACHEENTRY pEntry = RTListGetFirst(&ListDirtyNotCommitted, PDMBLKCACHEENTRY, NodeNotCommitted);
669
670 while (!RTListNodeIsLast(&ListDirtyNotCommitted, &pEntry->NodeNotCommitted))
671 {
672 PPDMBLKCACHEENTRY pNext = RTListNodeGetNext(&pEntry->NodeNotCommitted, PDMBLKCACHEENTRY,
673 NodeNotCommitted);
674 pdmBlkCacheEntryCommit(pEntry);
675 cbCommitted += pEntry->cbData;
676 RTListNodeRemove(&pEntry->NodeNotCommitted);
677 pEntry = pNext;
678 }
679
680 /* Commit the last endpoint */
681 Assert(RTListNodeIsLast(&ListDirtyNotCommitted, &pEntry->NodeNotCommitted));
682 pdmBlkCacheEntryCommit(pEntry);
683 RTListNodeRemove(&pEntry->NodeNotCommitted);
684 AssertMsg(RTListIsEmpty(&ListDirtyNotCommitted),
685 ("Committed all entries but list is not empty\n"));
686 }
687
688 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
689 AssertMsg(pBlkCache->pCache->cbDirty >= cbCommitted,
690 ("Number of committed bytes exceeds number of dirty bytes\n"));
691 uint32_t cbDirtyOld = ASMAtomicSubU32(&pBlkCache->pCache->cbDirty, cbCommitted);
692
693 /* Reset the commit timer if we don't have any dirty bits. */
694 if ( !(cbDirtyOld - cbCommitted)
695 && pBlkCache->pCache->u32CommitTimeoutMs != 0)
696 TMTimerStop(pBlkCache->pCache->pTimerCommit);
697}
698
699/**
700 * Commit all dirty entries in the cache.
701 *
702 * @returns nothing.
703 * @param pCache The global cache instance.
704 */
705static void pdmBlkCacheCommitDirtyEntries(PPDMBLKCACHEGLOBAL pCache)
706{
707 bool fCommitInProgress = ASMAtomicXchgBool(&pCache->fCommitInProgress, true);
708
709 if (!fCommitInProgress)
710 {
711 pdmBlkCacheLockEnter(pCache);
712 Assert(!RTListIsEmpty(&pCache->ListUsers));
713
714 PPDMBLKCACHE pBlkCache = RTListGetFirst(&pCache->ListUsers, PDMBLKCACHE, NodeCacheUser);
715 AssertPtr(pBlkCache);
716
717 while (!RTListNodeIsLast(&pCache->ListUsers, &pBlkCache->NodeCacheUser))
718 {
719 pdmBlkCacheCommit(pBlkCache);
720
721 pBlkCache = RTListNodeGetNext(&pBlkCache->NodeCacheUser, PDMBLKCACHE,
722 NodeCacheUser);
723 }
724
725 /* Commit the last endpoint */
726 Assert(RTListNodeIsLast(&pCache->ListUsers, &pBlkCache->NodeCacheUser));
727 pdmBlkCacheCommit(pBlkCache);
728
729 pdmBlkCacheLockLeave(pCache);
730 ASMAtomicWriteBool(&pCache->fCommitInProgress, false);
731 }
732}
733
734/**
735 * Adds the given entry as a dirty to the cache.
736 *
737 * @returns Flag whether the amount of dirty bytes in the cache exceeds the threshold
738 * @param pBlkCache The endpoint cache the entry belongs to.
739 * @param pEntry The entry to add.
740 */
741static bool pdmBlkCacheAddDirtyEntry(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEENTRY pEntry)
742{
743 bool fDirtyBytesExceeded = false;
744 PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
745
746 /* If the commit timer is disabled we commit right away. */
747 if (pCache->u32CommitTimeoutMs == 0)
748 {
749 pEntry->fFlags |= PDMBLKCACHE_ENTRY_IS_DIRTY;
750 pdmBlkCacheEntryCommit(pEntry);
751 }
752 else if (!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IS_DIRTY))
753 {
754 pEntry->fFlags |= PDMBLKCACHE_ENTRY_IS_DIRTY;
755
756 RTSPINLOCKTMP Tmp;
757 RTSpinlockAcquire(pBlkCache->LockList, &Tmp);
758 RTListAppend(&pBlkCache->ListDirtyNotCommitted, &pEntry->NodeNotCommitted);
759 RTSpinlockRelease(pBlkCache->LockList, &Tmp);
760
761 uint32_t cbDirty = ASMAtomicAddU32(&pCache->cbDirty, pEntry->cbData);
762
763 /* Prevent committing if the VM was suspended. */
764 if (RT_LIKELY(!ASMAtomicReadBool(&pCache->fIoErrorVmSuspended)))
765 fDirtyBytesExceeded = (cbDirty + pEntry->cbData >= pCache->cbCommitDirtyThreshold);
766 else if (!cbDirty && pCache->u32CommitTimeoutMs > 0)
767 {
768 /* Arm the commit timer. */
769 TMTimerSetMillies(pCache->pTimerCommit, pCache->u32CommitTimeoutMs);
770 }
771 }
772
773 return fDirtyBytesExceeded;
774}
775
776static PPDMBLKCACHE pdmR3BlkCacheFindById(PPDMBLKCACHEGLOBAL pBlkCacheGlobal, const char *pcszId)
777{
778 bool fFound = false;
779 PPDMBLKCACHE pBlkCache = NULL;
780
781 RTListForEach(&pBlkCacheGlobal->ListUsers, pBlkCache, PDMBLKCACHE, NodeCacheUser)
782 {
783 if (!RTStrCmp(pBlkCache->pszId, pcszId))
784 {
785 fFound = true;
786 break;
787 }
788 }
789
790 return fFound ? pBlkCache : NULL;
791}
792
793/**
794 * Commit timer callback.
795 */
796static DECLCALLBACK(void) pdmBlkCacheCommitTimerCallback(PVM pVM, PTMTIMER pTimer, void *pvUser)
797{
798 PPDMBLKCACHEGLOBAL pCache = (PPDMBLKCACHEGLOBAL)pvUser;
799 NOREF(pVM); NOREF(pTimer);
800
801 LogFlowFunc(("Commit interval expired, commiting dirty entries\n"));
802
803 if ( ASMAtomicReadU32(&pCache->cbDirty) > 0
804 && !ASMAtomicReadBool(&pCache->fIoErrorVmSuspended))
805 pdmBlkCacheCommitDirtyEntries(pCache);
806
807 LogFlowFunc(("Entries committed, going to sleep\n"));
808}
809
810static DECLCALLBACK(int) pdmR3BlkCacheSaveExec(PVM pVM, PSSMHANDLE pSSM)
811{
812 PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
813
814 AssertPtr(pBlkCacheGlobal);
815
816 pdmBlkCacheLockEnter(pBlkCacheGlobal);
817
818 SSMR3PutU32(pSSM, pBlkCacheGlobal->cRefs);
819
820 /* Go through the list and save all dirty entries. */
821 PPDMBLKCACHE pBlkCache;
822 RTListForEach(&pBlkCacheGlobal->ListUsers, pBlkCache, PDMBLKCACHE, NodeCacheUser)
823 {
824 uint32_t cEntries = 0;
825 PPDMBLKCACHEENTRY pEntry;
826
827 RTSemRWRequestRead(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
828 SSMR3PutU32(pSSM, (uint32_t)strlen(pBlkCache->pszId));
829 SSMR3PutStrZ(pSSM, pBlkCache->pszId);
830
831 /* Count the number of entries to safe. */
832 RTListForEach(&pBlkCache->ListDirtyNotCommitted, pEntry, PDMBLKCACHEENTRY, NodeNotCommitted)
833 {
834 cEntries++;
835 }
836
837 SSMR3PutU32(pSSM, cEntries);
838
839 /* Walk the list of all dirty entries and save them. */
840 RTListForEach(&pBlkCache->ListDirtyNotCommitted, pEntry, PDMBLKCACHEENTRY, NodeNotCommitted)
841 {
842 /* A few sanity checks. */
843 AssertMsg(!pEntry->cRefs, ("The entry is still referenced\n"));
844 AssertMsg(pEntry->fFlags & PDMBLKCACHE_ENTRY_IS_DIRTY, ("Entry is not dirty\n"));
845 AssertMsg(!(pEntry->fFlags & ~PDMBLKCACHE_ENTRY_IS_DIRTY), ("Invalid flags set\n"));
846 AssertMsg(!pEntry->pWaitingHead && !pEntry->pWaitingTail, ("There are waiting requests\n"));
847 AssertMsg( pEntry->pList == &pBlkCacheGlobal->LruRecentlyUsedIn
848 || pEntry->pList == &pBlkCacheGlobal->LruFrequentlyUsed,
849 ("Invalid list\n"));
850 AssertMsg(pEntry->cbData == pEntry->Core.KeyLast - pEntry->Core.Key + 1,
851 ("Size and range do not match\n"));
852
853 /* Save */
854 SSMR3PutU64(pSSM, pEntry->Core.Key);
855 SSMR3PutU32(pSSM, pEntry->cbData);
856 SSMR3PutMem(pSSM, pEntry->pbData, pEntry->cbData);
857 }
858
859 RTSemRWReleaseRead(pBlkCache->SemRWEntries);
860 }
861
862 pdmBlkCacheLockLeave(pBlkCacheGlobal);
863
864 /* Terminator */
865 return SSMR3PutU32(pSSM, UINT32_MAX);
866}
867
868static DECLCALLBACK(int) pdmR3BlkCacheLoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
869{
870 PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
871 uint32_t cRefs;
872
873 NOREF(uPass);
874 AssertPtr(pBlkCacheGlobal);
875
876 pdmBlkCacheLockEnter(pBlkCacheGlobal);
877
878 if (uVersion != PDM_BLK_CACHE_SAVED_STATE_VERSION)
879 return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
880
881 SSMR3GetU32(pSSM, &cRefs);
882
883 /*
884 * Fewer users in the saved state than in the current VM are allowed
885 * because that means that there are only new ones which don't have any saved state
886 * which can get lost.
887 * More saved entries that current ones are not allowed because this could result in
888 * lost data.
889 */
890 int rc = VINF_SUCCESS;
891 if (cRefs <= pBlkCacheGlobal->cRefs)
892 {
893 char *pszId = NULL;
894
895 while ( cRefs > 0
896 && RT_SUCCESS(rc))
897 {
898 PPDMBLKCACHE pBlkCache = NULL;
899 uint32_t cbId = 0;
900
901 SSMR3GetU32(pSSM, &cbId);
902 Assert(cbId > 0);
903
904 cbId++; /* Include terminator */
905 pszId = (char *)RTMemAllocZ(cbId * sizeof(char));
906 if (!pszId)
907 {
908 rc = VERR_NO_MEMORY;
909 break;
910 }
911
912 rc = SSMR3GetStrZ(pSSM, pszId, cbId);
913 AssertRC(rc);
914
915 /* Search for the block cache with the provided id. */
916 pBlkCache = pdmR3BlkCacheFindById(pBlkCacheGlobal, pszId);
917 if (!pBlkCache)
918 {
919 rc = SSMR3SetCfgError(pSSM, RT_SRC_POS,
920 N_("The VM is missing a block device. Please make sure the source and target VMs have compatible storage configurations"));
921 break;
922 }
923
924 RTStrFree(pszId);
925 pszId = NULL;
926
927 /* Get the entries */
928 uint32_t cEntries;
929 SSMR3GetU32(pSSM, &cEntries);
930
931 while (cEntries > 0)
932 {
933 PPDMBLKCACHEENTRY pEntry;
934 uint64_t off;
935 uint32_t cbEntry;
936
937 SSMR3GetU64(pSSM, &off);
938 SSMR3GetU32(pSSM, &cbEntry);
939
940 pEntry = pdmBlkCacheEntryAlloc(pBlkCache, off, cbEntry, NULL);
941 if (!pEntry)
942 {
943 rc = VERR_NO_MEMORY;
944 break;
945 }
946
947 rc = SSMR3GetMem(pSSM, pEntry->pbData, cbEntry);
948 if (RT_FAILURE(rc))
949 {
950 RTMemFree(pEntry->pbData);
951 RTMemFree(pEntry);
952 break;
953 }
954
955 /* Insert into the tree. */
956 bool fInserted = RTAvlrU64Insert(pBlkCache->pTree, &pEntry->Core);
957 Assert(fInserted); NOREF(fInserted);
958
959 /* Add to the dirty list. */
960 pdmBlkCacheAddDirtyEntry(pBlkCache, pEntry);
961 pdmBlkCacheEntryAddToList(&pBlkCacheGlobal->LruRecentlyUsedIn, pEntry);
962 pdmBlkCacheAdd(pBlkCacheGlobal, cbEntry);
963 pdmBlkCacheEntryRelease(pEntry);
964 cEntries--;
965 }
966
967 cRefs--;
968 }
969
970 if (pszId)
971 RTStrFree(pszId);
972 }
973 else
974 rc = SSMR3SetCfgError(pSSM, RT_SRC_POS,
975 N_("The VM is missing a block device. Please make sure the source and target VMs have compatible storage configurations"));
976
977 pdmBlkCacheLockLeave(pBlkCacheGlobal);
978
979 if (RT_SUCCESS(rc))
980 {
981 uint32_t u32 = 0;
982 rc = SSMR3GetU32(pSSM, &u32);
983 if (RT_SUCCESS(rc))
984 AssertMsgReturn(u32 == UINT32_MAX, ("%#x\n", u32), VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
985 }
986
987 return rc;
988}
989
990int pdmR3BlkCacheInit(PVM pVM)
991{
992 int rc = VINF_SUCCESS;
993 PUVM pUVM = pVM->pUVM;
994 PPDMBLKCACHEGLOBAL pBlkCacheGlobal;
995
996 LogFlowFunc((": pVM=%p\n", pVM));
997
998 VM_ASSERT_EMT(pVM);
999
1000 PCFGMNODE pCfgRoot = CFGMR3GetRoot(pVM);
1001 PCFGMNODE pCfgBlkCache = CFGMR3GetChild(CFGMR3GetChild(pCfgRoot, "PDM"), "BlkCache");
1002
1003 pBlkCacheGlobal = (PPDMBLKCACHEGLOBAL)RTMemAllocZ(sizeof(PDMBLKCACHEGLOBAL));
1004 if (!pBlkCacheGlobal)
1005 return VERR_NO_MEMORY;
1006
1007 RTListInit(&pBlkCacheGlobal->ListUsers);
1008 pBlkCacheGlobal->pVM = pVM;
1009 pBlkCacheGlobal->cRefs = 0;
1010 pBlkCacheGlobal->cbCached = 0;
1011 pBlkCacheGlobal->fCommitInProgress = false;
1012
1013 /* Initialize members */
1014 pBlkCacheGlobal->LruRecentlyUsedIn.pHead = NULL;
1015 pBlkCacheGlobal->LruRecentlyUsedIn.pTail = NULL;
1016 pBlkCacheGlobal->LruRecentlyUsedIn.cbCached = 0;
1017
1018 pBlkCacheGlobal->LruRecentlyUsedOut.pHead = NULL;
1019 pBlkCacheGlobal->LruRecentlyUsedOut.pTail = NULL;
1020 pBlkCacheGlobal->LruRecentlyUsedOut.cbCached = 0;
1021
1022 pBlkCacheGlobal->LruFrequentlyUsed.pHead = NULL;
1023 pBlkCacheGlobal->LruFrequentlyUsed.pTail = NULL;
1024 pBlkCacheGlobal->LruFrequentlyUsed.cbCached = 0;
1025
1026 do
1027 {
1028 rc = CFGMR3QueryU32Def(pCfgBlkCache, "CacheSize", &pBlkCacheGlobal->cbMax, 5 * _1M);
1029 AssertLogRelRCBreak(rc);
1030 LogFlowFunc(("Maximum number of bytes cached %u\n", pBlkCacheGlobal->cbMax));
1031
1032 pBlkCacheGlobal->cbRecentlyUsedInMax = (pBlkCacheGlobal->cbMax / 100) * 25; /* 25% of the buffer size */
1033 pBlkCacheGlobal->cbRecentlyUsedOutMax = (pBlkCacheGlobal->cbMax / 100) * 50; /* 50% of the buffer size */
1034 LogFlowFunc(("cbRecentlyUsedInMax=%u cbRecentlyUsedOutMax=%u\n",
1035 pBlkCacheGlobal->cbRecentlyUsedInMax, pBlkCacheGlobal->cbRecentlyUsedOutMax));
1036
1037 /** @todo r=aeichner: Experiment to find optimal default values */
1038 rc = CFGMR3QueryU32Def(pCfgBlkCache, "CacheCommitIntervalMs", &pBlkCacheGlobal->u32CommitTimeoutMs, 10000 /* 10sec */);
1039 AssertLogRelRCBreak(rc);
1040 rc = CFGMR3QueryU32Def(pCfgBlkCache, "CacheCommitThreshold", &pBlkCacheGlobal->cbCommitDirtyThreshold, pBlkCacheGlobal->cbMax / 2);
1041 AssertLogRelRCBreak(rc);
1042 } while (0);
1043
1044 if (RT_SUCCESS(rc))
1045 {
1046 STAMR3Register(pVM, &pBlkCacheGlobal->cbMax,
1047 STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
1048 "/PDM/BlkCache/cbMax",
1049 STAMUNIT_BYTES,
1050 "Maximum cache size");
1051 STAMR3Register(pVM, &pBlkCacheGlobal->cbCached,
1052 STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
1053 "/PDM/BlkCache/cbCached",
1054 STAMUNIT_BYTES,
1055 "Currently used cache");
1056 STAMR3Register(pVM, &pBlkCacheGlobal->LruRecentlyUsedIn.cbCached,
1057 STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
1058 "/PDM/BlkCache/cbCachedMruIn",
1059 STAMUNIT_BYTES,
1060 "Number of bytes cached in MRU list");
1061 STAMR3Register(pVM, &pBlkCacheGlobal->LruRecentlyUsedOut.cbCached,
1062 STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
1063 "/PDM/BlkCache/cbCachedMruOut",
1064 STAMUNIT_BYTES,
1065 "Number of bytes cached in FRU list");
1066 STAMR3Register(pVM, &pBlkCacheGlobal->LruFrequentlyUsed.cbCached,
1067 STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
1068 "/PDM/BlkCache/cbCachedFru",
1069 STAMUNIT_BYTES,
1070 "Number of bytes cached in FRU ghost list");
1071
1072#ifdef VBOX_WITH_STATISTICS
1073 STAMR3Register(pVM, &pBlkCacheGlobal->cHits,
1074 STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
1075 "/PDM/BlkCache/CacheHits",
1076 STAMUNIT_COUNT, "Number of hits in the cache");
1077 STAMR3Register(pVM, &pBlkCacheGlobal->cPartialHits,
1078 STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
1079 "/PDM/BlkCache/CachePartialHits",
1080 STAMUNIT_COUNT, "Number of partial hits in the cache");
1081 STAMR3Register(pVM, &pBlkCacheGlobal->cMisses,
1082 STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
1083 "/PDM/BlkCache/CacheMisses",
1084 STAMUNIT_COUNT, "Number of misses when accessing the cache");
1085 STAMR3Register(pVM, &pBlkCacheGlobal->StatRead,
1086 STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
1087 "/PDM/BlkCache/CacheRead",
1088 STAMUNIT_BYTES, "Number of bytes read from the cache");
1089 STAMR3Register(pVM, &pBlkCacheGlobal->StatWritten,
1090 STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
1091 "/PDM/BlkCache/CacheWritten",
1092 STAMUNIT_BYTES, "Number of bytes written to the cache");
1093 STAMR3Register(pVM, &pBlkCacheGlobal->StatTreeGet,
1094 STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
1095 "/PDM/BlkCache/CacheTreeGet",
1096 STAMUNIT_TICKS_PER_CALL, "Time taken to access an entry in the tree");
1097 STAMR3Register(pVM, &pBlkCacheGlobal->StatTreeInsert,
1098 STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
1099 "/PDM/BlkCache/CacheTreeInsert",
1100 STAMUNIT_TICKS_PER_CALL, "Time taken to insert an entry in the tree");
1101 STAMR3Register(pVM, &pBlkCacheGlobal->StatTreeRemove,
1102 STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
1103 "/PDM/BlkCache/CacheTreeRemove",
1104 STAMUNIT_TICKS_PER_CALL, "Time taken to remove an entry an the tree");
1105 STAMR3Register(pVM, &pBlkCacheGlobal->StatBuffersReused,
1106 STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
1107 "/PDM/BlkCache/CacheBuffersReused",
1108 STAMUNIT_COUNT, "Number of times a buffer could be reused");
1109#endif
1110
1111 /* Initialize the critical section */
1112 rc = RTCritSectInit(&pBlkCacheGlobal->CritSect);
1113 }
1114
1115 if (RT_SUCCESS(rc))
1116 {
1117 /* Create the commit timer */
1118 if (pBlkCacheGlobal->u32CommitTimeoutMs > 0)
1119 rc = TMR3TimerCreateInternal(pVM, TMCLOCK_REAL,
1120 pdmBlkCacheCommitTimerCallback,
1121 pBlkCacheGlobal,
1122 "BlkCache-Commit",
1123 &pBlkCacheGlobal->pTimerCommit);
1124
1125 if (RT_SUCCESS(rc))
1126 {
1127 /* Register saved state handler. */
1128 rc = SSMR3RegisterInternal(pVM, "pdmblkcache", 0, PDM_BLK_CACHE_SAVED_STATE_VERSION, pBlkCacheGlobal->cbMax,
1129 NULL, NULL, NULL,
1130 NULL, pdmR3BlkCacheSaveExec, NULL,
1131 NULL, pdmR3BlkCacheLoadExec, NULL);
1132 if (RT_SUCCESS(rc))
1133 {
1134 LogRel(("BlkCache: Cache successfully initialised. Cache size is %u bytes\n", pBlkCacheGlobal->cbMax));
1135 LogRel(("BlkCache: Cache commit interval is %u ms\n", pBlkCacheGlobal->u32CommitTimeoutMs));
1136 LogRel(("BlkCache: Cache commit threshold is %u bytes\n", pBlkCacheGlobal->cbCommitDirtyThreshold));
1137 pUVM->pdm.s.pBlkCacheGlobal = pBlkCacheGlobal;
1138 return VINF_SUCCESS;
1139 }
1140 }
1141
1142 RTCritSectDelete(&pBlkCacheGlobal->CritSect);
1143 }
1144
1145 if (pBlkCacheGlobal)
1146 RTMemFree(pBlkCacheGlobal);
1147
1148 LogFlowFunc((": returns rc=%Rrc\n", pVM, rc));
1149 return rc;
1150}
1151
1152void pdmR3BlkCacheTerm(PVM pVM)
1153{
1154 PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
1155
1156 if (pBlkCacheGlobal)
1157 {
1158 /* Make sure no one else uses the cache now */
1159 pdmBlkCacheLockEnter(pBlkCacheGlobal);
1160
1161 /* Cleanup deleting all cache entries waiting for in progress entries to finish. */
1162 pdmBlkCacheDestroyList(&pBlkCacheGlobal->LruRecentlyUsedIn);
1163 pdmBlkCacheDestroyList(&pBlkCacheGlobal->LruRecentlyUsedOut);
1164 pdmBlkCacheDestroyList(&pBlkCacheGlobal->LruFrequentlyUsed);
1165
1166 pdmBlkCacheLockLeave(pBlkCacheGlobal);
1167
1168 RTCritSectDelete(&pBlkCacheGlobal->CritSect);
1169 RTMemFree(pBlkCacheGlobal);
1170 pVM->pUVM->pdm.s.pBlkCacheGlobal = NULL;
1171 }
1172}
1173
1174int pdmR3BlkCacheResume(PVM pVM)
1175{
1176 PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
1177
1178 LogFlowFunc(("pVM=%#p\n", pVM));
1179
1180 if ( pBlkCacheGlobal
1181 && ASMAtomicXchgBool(&pBlkCacheGlobal->fIoErrorVmSuspended, false))
1182 {
1183 /* The VM was suspended because of an I/O error, commit all dirty entries. */
1184 pdmBlkCacheCommitDirtyEntries(pBlkCacheGlobal);
1185 }
1186
1187 return VINF_SUCCESS;
1188}
1189
1190static int pdmR3BlkCacheRetain(PVM pVM, PPPDMBLKCACHE ppBlkCache, const char *pcszId)
1191{
1192 int rc = VINF_SUCCESS;
1193 PPDMBLKCACHE pBlkCache = NULL;
1194 PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
1195
1196 if (!pBlkCacheGlobal)
1197 return VERR_NOT_SUPPORTED;
1198
1199 /*
1200 * Check that no other user cache has the same id first,
1201 * Unique id's are necessary in case the state is saved.
1202 */
1203 pdmBlkCacheLockEnter(pBlkCacheGlobal);
1204
1205 pBlkCache = pdmR3BlkCacheFindById(pBlkCacheGlobal, pcszId);
1206
1207 if (!pBlkCache)
1208 {
1209 pBlkCache = (PPDMBLKCACHE)RTMemAllocZ(sizeof(PDMBLKCACHE));
1210
1211 if (pBlkCache)
1212 pBlkCache->pszId = RTStrDup(pcszId);
1213
1214 if ( pBlkCache
1215 && pBlkCache->pszId)
1216 {
1217 pBlkCache->fSuspended = false;
1218 pBlkCache->pCache = pBlkCacheGlobal;
1219 RTListInit(&pBlkCache->ListDirtyNotCommitted);
1220
1221 rc = RTSpinlockCreate(&pBlkCache->LockList);
1222 if (RT_SUCCESS(rc))
1223 {
1224 rc = RTSemRWCreate(&pBlkCache->SemRWEntries);
1225 if (RT_SUCCESS(rc))
1226 {
1227 pBlkCache->pTree = (PAVLRU64TREE)RTMemAllocZ(sizeof(AVLRFOFFTREE));
1228 if (pBlkCache->pTree)
1229 {
1230#ifdef VBOX_WITH_STATISTICS
1231 STAMR3RegisterF(pBlkCacheGlobal->pVM, &pBlkCache->StatWriteDeferred,
1232 STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
1233 STAMUNIT_COUNT, "Number of deferred writes",
1234 "/PDM/BlkCache/%s/Cache/DeferredWrites", pBlkCache->pszId);
1235#endif
1236
1237 /* Add to the list of users. */
1238 pBlkCacheGlobal->cRefs++;
1239 RTListAppend(&pBlkCacheGlobal->ListUsers, &pBlkCache->NodeCacheUser);
1240 pdmBlkCacheLockLeave(pBlkCacheGlobal);
1241
1242 *ppBlkCache = pBlkCache;
1243 LogFlowFunc(("returns success\n"));
1244 return VINF_SUCCESS;
1245 }
1246 else
1247 rc = VERR_NO_MEMORY;
1248
1249 RTSemRWDestroy(pBlkCache->SemRWEntries);
1250 }
1251
1252 RTSpinlockDestroy(pBlkCache->LockList);
1253 }
1254
1255 RTStrFree(pBlkCache->pszId);
1256 }
1257 else
1258 rc = VERR_NO_MEMORY;
1259
1260 if (pBlkCache)
1261 RTMemFree(pBlkCache);
1262 }
1263 else
1264 rc = VERR_ALREADY_EXISTS;
1265
1266 pdmBlkCacheLockLeave(pBlkCacheGlobal);
1267
1268 LogFlowFunc(("Leave rc=%Rrc\n", rc));
1269 return rc;
1270}
1271
1272VMMR3DECL(int) PDMR3BlkCacheRetainDriver(PVM pVM, PPDMDRVINS pDrvIns, PPPDMBLKCACHE ppBlkCache,
1273 PFNPDMBLKCACHEXFERCOMPLETEDRV pfnXferComplete,
1274 PFNPDMBLKCACHEXFERENQUEUEDRV pfnXferEnqueue,
1275 PFNPDMBLKCACHEXFERENQUEUEDISCARDDRV pfnXferEnqueueDiscard,
1276 const char *pcszId)
1277{
1278 int rc = VINF_SUCCESS;
1279 PPDMBLKCACHE pBlkCache;
1280
1281 rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
1282 if (RT_SUCCESS(rc))
1283 {
1284 pBlkCache->enmType = PDMBLKCACHETYPE_DRV;
1285 pBlkCache->u.Drv.pfnXferComplete = pfnXferComplete;
1286 pBlkCache->u.Drv.pfnXferEnqueue = pfnXferEnqueue;
1287 pBlkCache->u.Drv.pfnXferEnqueueDiscard = pfnXferEnqueueDiscard;
1288 pBlkCache->u.Drv.pDrvIns = pDrvIns;
1289 *ppBlkCache = pBlkCache;
1290 }
1291
1292 LogFlowFunc(("Leave rc=%Rrc\n", rc));
1293 return rc;
1294}
1295
1296VMMR3DECL(int) PDMR3BlkCacheRetainDevice(PVM pVM, PPDMDEVINS pDevIns, PPPDMBLKCACHE ppBlkCache,
1297 PFNPDMBLKCACHEXFERCOMPLETEDEV pfnXferComplete,
1298 PFNPDMBLKCACHEXFERENQUEUEDEV pfnXferEnqueue,
1299 PFNPDMBLKCACHEXFERENQUEUEDISCARDDEV pfnXferEnqueueDiscard,
1300 const char *pcszId)
1301{
1302 int rc = VINF_SUCCESS;
1303 PPDMBLKCACHE pBlkCache;
1304
1305 rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
1306 if (RT_SUCCESS(rc))
1307 {
1308 pBlkCache->enmType = PDMBLKCACHETYPE_DEV;
1309 pBlkCache->u.Dev.pfnXferComplete = pfnXferComplete;
1310 pBlkCache->u.Dev.pfnXferEnqueue = pfnXferEnqueue;
1311 pBlkCache->u.Dev.pfnXferEnqueueDiscard = pfnXferEnqueueDiscard;
1312 pBlkCache->u.Dev.pDevIns = pDevIns;
1313 *ppBlkCache = pBlkCache;
1314 }
1315
1316 LogFlowFunc(("Leave rc=%Rrc\n", rc));
1317 return rc;
1318
1319}
1320
1321VMMR3DECL(int) PDMR3BlkCacheRetainUsb(PVM pVM, PPDMUSBINS pUsbIns, PPPDMBLKCACHE ppBlkCache,
1322 PFNPDMBLKCACHEXFERCOMPLETEUSB pfnXferComplete,
1323 PFNPDMBLKCACHEXFERENQUEUEUSB pfnXferEnqueue,
1324 PFNPDMBLKCACHEXFERENQUEUEDISCARDUSB pfnXferEnqueueDiscard,
1325 const char *pcszId)
1326{
1327 int rc = VINF_SUCCESS;
1328 PPDMBLKCACHE pBlkCache;
1329
1330 rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
1331 if (RT_SUCCESS(rc))
1332 {
1333 pBlkCache->enmType = PDMBLKCACHETYPE_USB;
1334 pBlkCache->u.Usb.pfnXferComplete = pfnXferComplete;
1335 pBlkCache->u.Usb.pfnXferEnqueue = pfnXferEnqueue;
1336 pBlkCache->u.Usb.pfnXferEnqueueDiscard = pfnXferEnqueueDiscard;
1337 pBlkCache->u.Usb.pUsbIns = pUsbIns;
1338 *ppBlkCache = pBlkCache;
1339 }
1340
1341 LogFlowFunc(("Leave rc=%Rrc\n", rc));
1342 return rc;
1343
1344}
1345
1346VMMR3DECL(int) PDMR3BlkCacheRetainInt(PVM pVM, void *pvUser, PPPDMBLKCACHE ppBlkCache,
1347 PFNPDMBLKCACHEXFERCOMPLETEINT pfnXferComplete,
1348 PFNPDMBLKCACHEXFERENQUEUEINT pfnXferEnqueue,
1349 PFNPDMBLKCACHEXFERENQUEUEDISCARDINT pfnXferEnqueueDiscard,
1350 const char *pcszId)
1351{
1352 int rc = VINF_SUCCESS;
1353 PPDMBLKCACHE pBlkCache;
1354
1355 rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
1356 if (RT_SUCCESS(rc))
1357 {
1358 pBlkCache->enmType = PDMBLKCACHETYPE_INTERNAL;
1359 pBlkCache->u.Int.pfnXferComplete = pfnXferComplete;
1360 pBlkCache->u.Int.pfnXferEnqueue = pfnXferEnqueue;
1361 pBlkCache->u.Int.pfnXferEnqueueDiscard = pfnXferEnqueueDiscard;
1362 pBlkCache->u.Int.pvUser = pvUser;
1363 *ppBlkCache = pBlkCache;
1364 }
1365
1366 LogFlowFunc(("Leave rc=%Rrc\n", rc));
1367 return rc;
1368
1369}
1370
1371/**
1372 * Callback for the AVL destroy routine. Frees a cache entry for this endpoint.
1373 *
1374 * @returns IPRT status code.
1375 * @param pNode The node to destroy.
1376 * @param pvUser Opaque user data.
1377 */
1378static int pdmBlkCacheEntryDestroy(PAVLRU64NODECORE pNode, void *pvUser)
1379{
1380 PPDMBLKCACHEENTRY pEntry = (PPDMBLKCACHEENTRY)pNode;
1381 PPDMBLKCACHEGLOBAL pCache = (PPDMBLKCACHEGLOBAL)pvUser;
1382 PPDMBLKCACHE pBlkCache = pEntry->pBlkCache;
1383
1384 while (ASMAtomicReadU32(&pEntry->fFlags) & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS)
1385 {
1386 /* Leave the locks to let the I/O thread make progress but reference the entry to prevent eviction. */
1387 pdmBlkCacheEntryRef(pEntry);
1388 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
1389 pdmBlkCacheLockLeave(pCache);
1390
1391 RTThreadSleep(250);
1392
1393 /* Re-enter all locks */
1394 pdmBlkCacheLockEnter(pCache);
1395 RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
1396 pdmBlkCacheEntryRelease(pEntry);
1397 }
1398
1399 AssertMsg(!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS),
1400 ("Entry is dirty and/or still in progress fFlags=%#x\n", pEntry->fFlags));
1401
1402 bool fUpdateCache = pEntry->pList == &pCache->LruFrequentlyUsed
1403 || pEntry->pList == &pCache->LruRecentlyUsedIn;
1404
1405 pdmBlkCacheEntryRemoveFromList(pEntry);
1406
1407 if (fUpdateCache)
1408 pdmBlkCacheSub(pCache, pEntry->cbData);
1409
1410 RTMemPageFree(pEntry->pbData, pEntry->cbData);
1411 RTMemFree(pEntry);
1412
1413 return VINF_SUCCESS;
1414}
1415
1416/**
1417 * Destroys all cache resources used by the given endpoint.
1418 *
1419 * @returns nothing.
1420 * @param pEndpoint The endpoint to the destroy.
1421 */
1422VMMR3DECL(void) PDMR3BlkCacheRelease(PPDMBLKCACHE pBlkCache)
1423{
1424 PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
1425
1426 /*
1427 * Commit all dirty entries now (they are waited on for completion during the
1428 * destruction of the AVL tree below).
1429 * The exception is if the VM was paused because of an I/O error before.
1430 */
1431 if (!ASMAtomicReadBool(&pCache->fIoErrorVmSuspended))
1432 pdmBlkCacheCommit(pBlkCache);
1433
1434 /* Make sure nobody is accessing the cache while we delete the tree. */
1435 pdmBlkCacheLockEnter(pCache);
1436 RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
1437 RTAvlrU64Destroy(pBlkCache->pTree, pdmBlkCacheEntryDestroy, pCache);
1438 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
1439
1440 RTSpinlockDestroy(pBlkCache->LockList);
1441
1442 pCache->cRefs--;
1443 RTListNodeRemove(&pBlkCache->NodeCacheUser);
1444
1445 pdmBlkCacheLockLeave(pCache);
1446
1447 RTSemRWDestroy(pBlkCache->SemRWEntries);
1448
1449#ifdef VBOX_WITH_STATISTICS
1450 STAMR3Deregister(pCache->pVM, &pBlkCache->StatWriteDeferred);
1451#endif
1452
1453 RTStrFree(pBlkCache->pszId);
1454 RTMemFree(pBlkCache);
1455}
1456
1457VMMR3DECL(void) PDMR3BlkCacheReleaseDevice(PVM pVM, PPDMDEVINS pDevIns)
1458{
1459 LogFlow(("%s: pDevIns=%p\n", __FUNCTION__, pDevIns));
1460
1461 /*
1462 * Validate input.
1463 */
1464 if (!pDevIns)
1465 return;
1466 VM_ASSERT_EMT(pVM);
1467
1468 PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
1469 PPDMBLKCACHE pBlkCache, pBlkCacheNext;
1470
1471 /* Return silently if not supported. */
1472 if (!pBlkCacheGlobal)
1473 return;
1474
1475 pdmBlkCacheLockEnter(pBlkCacheGlobal);
1476
1477 RTListForEachSafe(&pBlkCacheGlobal->ListUsers, pBlkCache, pBlkCacheNext, PDMBLKCACHE, NodeCacheUser)
1478 {
1479 if ( pBlkCache->enmType == PDMBLKCACHETYPE_DEV
1480 && pBlkCache->u.Dev.pDevIns == pDevIns)
1481 PDMR3BlkCacheRelease(pBlkCache);
1482 }
1483
1484 pdmBlkCacheLockLeave(pBlkCacheGlobal);
1485}
1486
1487VMMR3DECL(void) PDMR3BlkCacheReleaseDriver(PVM pVM, PPDMDRVINS pDrvIns)
1488{
1489 LogFlow(("%s: pDrvIns=%p\n", __FUNCTION__, pDrvIns));
1490
1491 /*
1492 * Validate input.
1493 */
1494 if (!pDrvIns)
1495 return;
1496 VM_ASSERT_EMT(pVM);
1497
1498 PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
1499 PPDMBLKCACHE pBlkCache, pBlkCacheNext;
1500
1501 /* Return silently if not supported. */
1502 if (!pBlkCacheGlobal)
1503 return;
1504
1505 pdmBlkCacheLockEnter(pBlkCacheGlobal);
1506
1507 RTListForEachSafe(&pBlkCacheGlobal->ListUsers, pBlkCache, pBlkCacheNext, PDMBLKCACHE, NodeCacheUser)
1508 {
1509 if ( pBlkCache->enmType == PDMBLKCACHETYPE_DRV
1510 && pBlkCache->u.Drv.pDrvIns == pDrvIns)
1511 PDMR3BlkCacheRelease(pBlkCache);
1512 }
1513
1514 pdmBlkCacheLockLeave(pBlkCacheGlobal);
1515}
1516
1517VMMR3DECL(void) PDMR3BlkCacheReleaseUsb(PVM pVM, PPDMUSBINS pUsbIns)
1518{
1519 LogFlow(("%s: pUsbIns=%p\n", __FUNCTION__, pUsbIns));
1520
1521 /*
1522 * Validate input.
1523 */
1524 if (!pUsbIns)
1525 return;
1526 VM_ASSERT_EMT(pVM);
1527
1528 PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
1529 PPDMBLKCACHE pBlkCache, pBlkCacheNext;
1530
1531 /* Return silently if not supported. */
1532 if (!pBlkCacheGlobal)
1533 return;
1534
1535 pdmBlkCacheLockEnter(pBlkCacheGlobal);
1536
1537 RTListForEachSafe(&pBlkCacheGlobal->ListUsers, pBlkCache, pBlkCacheNext, PDMBLKCACHE, NodeCacheUser)
1538 {
1539 if ( pBlkCache->enmType == PDMBLKCACHETYPE_USB
1540 && pBlkCache->u.Usb.pUsbIns == pUsbIns)
1541 PDMR3BlkCacheRelease(pBlkCache);
1542 }
1543
1544 pdmBlkCacheLockLeave(pBlkCacheGlobal);
1545}
1546
1547static PPDMBLKCACHEENTRY pdmBlkCacheGetCacheEntryByOffset(PPDMBLKCACHE pBlkCache, uint64_t off)
1548{
1549 STAM_PROFILE_ADV_START(&pBlkCache->pCache->StatTreeGet, Cache);
1550
1551 RTSemRWRequestRead(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
1552 PPDMBLKCACHEENTRY pEntry = (PPDMBLKCACHEENTRY)RTAvlrU64RangeGet(pBlkCache->pTree, off);
1553 if (pEntry)
1554 pdmBlkCacheEntryRef(pEntry);
1555 RTSemRWReleaseRead(pBlkCache->SemRWEntries);
1556
1557 STAM_PROFILE_ADV_STOP(&pBlkCache->pCache->StatTreeGet, Cache);
1558
1559 return pEntry;
1560}
1561
1562/**
1563 * Return the best fit cache entries for the given offset.
1564 *
1565 * @returns nothing.
1566 * @param pBlkCache The endpoint cache.
1567 * @param off The offset.
1568 * @param pEntryAbove Where to store the pointer to the best fit entry above the
1569 * the given offset. NULL if not required.
1570 */
1571static void pdmBlkCacheGetCacheBestFitEntryByOffset(PPDMBLKCACHE pBlkCache, uint64_t off,
1572 PPDMBLKCACHEENTRY *ppEntryAbove)
1573{
1574 STAM_PROFILE_ADV_START(&pBlkCache->pCache->StatTreeGet, Cache);
1575
1576 RTSemRWRequestRead(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
1577 if (ppEntryAbove)
1578 {
1579 *ppEntryAbove = (PPDMBLKCACHEENTRY)RTAvlrU64GetBestFit(pBlkCache->pTree, off, true /*fAbove*/);
1580 if (*ppEntryAbove)
1581 pdmBlkCacheEntryRef(*ppEntryAbove);
1582 }
1583
1584 RTSemRWReleaseRead(pBlkCache->SemRWEntries);
1585
1586 STAM_PROFILE_ADV_STOP(&pBlkCache->pCache->StatTreeGet, Cache);
1587}
1588
1589static void pdmBlkCacheInsertEntry(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEENTRY pEntry)
1590{
1591 STAM_PROFILE_ADV_START(&pBlkCache->pCache->StatTreeInsert, Cache);
1592 RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
1593 bool fInserted = RTAvlrU64Insert(pBlkCache->pTree, &pEntry->Core);
1594 AssertMsg(fInserted, ("Node was not inserted into tree\n")); NOREF(fInserted);
1595 STAM_PROFILE_ADV_STOP(&pBlkCache->pCache->StatTreeInsert, Cache);
1596 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
1597}
1598
1599/**
1600 * Allocates and initializes a new entry for the cache.
1601 * The entry has a reference count of 1.
1602 *
1603 * @returns Pointer to the new cache entry or NULL if out of memory.
1604 * @param pBlkCache The cache the entry belongs to.
1605 * @param off Start offset.
1606 * @param cbData Size of the cache entry.
1607 * @param pbBuffer Pointer to the buffer to use.
1608 * NULL if a new buffer should be allocated.
1609 * The buffer needs to have the same size of the entry.
1610 */
1611static PPDMBLKCACHEENTRY pdmBlkCacheEntryAlloc(PPDMBLKCACHE pBlkCache,
1612 uint64_t off, size_t cbData, uint8_t *pbBuffer)
1613{
1614 AssertReturn(cbData <= UINT32_MAX, NULL);
1615 PPDMBLKCACHEENTRY pEntryNew = (PPDMBLKCACHEENTRY)RTMemAllocZ(sizeof(PDMBLKCACHEENTRY));
1616
1617 if (RT_UNLIKELY(!pEntryNew))
1618 return NULL;
1619
1620 pEntryNew->Core.Key = off;
1621 pEntryNew->Core.KeyLast = off + cbData - 1;
1622 pEntryNew->pBlkCache = pBlkCache;
1623 pEntryNew->fFlags = 0;
1624 pEntryNew->cRefs = 1; /* We are using it now. */
1625 pEntryNew->pList = NULL;
1626 pEntryNew->cbData = (uint32_t)cbData;
1627 pEntryNew->pWaitingHead = NULL;
1628 pEntryNew->pWaitingTail = NULL;
1629 if (pbBuffer)
1630 pEntryNew->pbData = pbBuffer;
1631 else
1632 pEntryNew->pbData = (uint8_t *)RTMemPageAlloc(cbData);
1633
1634 if (RT_UNLIKELY(!pEntryNew->pbData))
1635 {
1636 RTMemFree(pEntryNew);
1637 return NULL;
1638 }
1639
1640 return pEntryNew;
1641}
1642
1643/**
1644 * Checks that a set of flags is set/clear acquiring the R/W semaphore
1645 * in exclusive mode.
1646 *
1647 * @returns true if the flag in fSet is set and the one in fClear is clear.
1648 * false otherwise.
1649 * The R/W semaphore is only held if true is returned.
1650 *
1651 * @param pBlkCache The endpoint cache instance data.
1652 * @param pEntry The entry to check the flags for.
1653 * @param fSet The flag which is tested to be set.
1654 * @param fClear The flag which is tested to be clear.
1655 */
1656DECLINLINE(bool) pdmBlkCacheEntryFlagIsSetClearAcquireLock(PPDMBLKCACHE pBlkCache,
1657 PPDMBLKCACHEENTRY pEntry,
1658 uint32_t fSet, uint32_t fClear)
1659{
1660 uint32_t fFlags = ASMAtomicReadU32(&pEntry->fFlags);
1661 bool fPassed = ((fFlags & fSet) && !(fFlags & fClear));
1662
1663 if (fPassed)
1664 {
1665 /* Acquire the lock and check again because the completion callback might have raced us. */
1666 RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
1667
1668 fFlags = ASMAtomicReadU32(&pEntry->fFlags);
1669 fPassed = ((fFlags & fSet) && !(fFlags & fClear));
1670
1671 /* Drop the lock if we didn't passed the test. */
1672 if (!fPassed)
1673 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
1674 }
1675
1676 return fPassed;
1677}
1678
1679/**
1680 * Adds a segment to the waiting list for a cache entry
1681 * which is currently in progress.
1682 *
1683 * @returns nothing.
1684 * @param pEntry The cache entry to add the segment to.
1685 * @param pSeg The segment to add.
1686 */
1687DECLINLINE(void) pdmBlkCacheEntryAddWaiter(PPDMBLKCACHEENTRY pEntry,
1688 PPDMBLKCACHEWAITER pWaiter)
1689{
1690 pWaiter->pNext = NULL;
1691
1692 if (pEntry->pWaitingHead)
1693 {
1694 AssertPtr(pEntry->pWaitingTail);
1695
1696 pEntry->pWaitingTail->pNext = pWaiter;
1697 pEntry->pWaitingTail = pWaiter;
1698 }
1699 else
1700 {
1701 Assert(!pEntry->pWaitingTail);
1702
1703 pEntry->pWaitingHead = pWaiter;
1704 pEntry->pWaitingTail = pWaiter;
1705 }
1706}
1707
1708/**
1709 * Add a buffer described by the I/O memory context
1710 * to the entry waiting for completion.
1711 *
1712 * @returns VBox status code.
1713 * @param pEntry The entry to add the buffer to.
1714 * @param pTask Task associated with the buffer.
1715 * @param pIoMemCtx The memory context to use.
1716 * @param offDiff Offset from the start of the buffer
1717 * in the entry.
1718 * @param cbData Amount of data to wait for onthis entry.
1719 * @param fWrite Flag whether the task waits because it wants to write
1720 * to the cache entry.
1721 */
1722static int pdmBlkCacheEntryWaitersAdd(PPDMBLKCACHEENTRY pEntry,
1723 PPDMBLKCACHEREQ pReq,
1724 PRTSGBUF pSgBuf, uint64_t offDiff,
1725 size_t cbData, bool fWrite)
1726{
1727 PPDMBLKCACHEWAITER pWaiter = (PPDMBLKCACHEWAITER)RTMemAllocZ(sizeof(PDMBLKCACHEWAITER));
1728 if (!pWaiter)
1729 return VERR_NO_MEMORY;
1730
1731 ASMAtomicIncU32(&pReq->cXfersPending);
1732 pWaiter->pReq = pReq;
1733 pWaiter->offCacheEntry = offDiff;
1734 pWaiter->cbTransfer = cbData;
1735 pWaiter->fWrite = fWrite;
1736 RTSgBufClone(&pWaiter->SgBuf, pSgBuf);
1737 RTSgBufAdvance(pSgBuf, cbData);
1738
1739 pdmBlkCacheEntryAddWaiter(pEntry, pWaiter);
1740
1741 return VINF_SUCCESS;
1742}
1743
1744/**
1745 * Calculate aligned offset and size for a new cache entry which do not
1746 * intersect with an already existing entry and the file end.
1747 *
1748 * @returns The number of bytes the entry can hold of the requested amount
1749 * of bytes.
1750 * @param pEndpoint The endpoint.
1751 * @param pBlkCache The endpoint cache.
1752 * @param off The start offset.
1753 * @param cb The number of bytes the entry needs to hold at
1754 * least.
1755 * @param pcbEntry Where to store the number of bytes the entry can hold.
1756 * Can be less than given because of other entries.
1757 */
1758static uint32_t pdmBlkCacheEntryBoundariesCalc(PPDMBLKCACHE pBlkCache,
1759 uint64_t off, uint32_t cb,
1760 uint32_t *pcbEntry)
1761{
1762 /* Get the best fit entries around the offset */
1763 PPDMBLKCACHEENTRY pEntryAbove = NULL;
1764 pdmBlkCacheGetCacheBestFitEntryByOffset(pBlkCache, off, &pEntryAbove);
1765
1766 /* Log the info */
1767 LogFlow(("%sest fit entry above off=%llu (BestFit=%llu BestFitEnd=%llu BestFitSize=%u)\n",
1768 pEntryAbove ? "B" : "No b",
1769 off,
1770 pEntryAbove ? pEntryAbove->Core.Key : 0,
1771 pEntryAbove ? pEntryAbove->Core.KeyLast : 0,
1772 pEntryAbove ? pEntryAbove->cbData : 0));
1773
1774 uint32_t cbNext;
1775 uint32_t cbInEntry;
1776 if ( pEntryAbove
1777 && off + cb > pEntryAbove->Core.Key)
1778 {
1779 cbInEntry = (uint32_t)(pEntryAbove->Core.Key - off);
1780 cbNext = (uint32_t)(pEntryAbove->Core.Key - off);
1781 }
1782 else
1783 {
1784 cbInEntry = cb;
1785 cbNext = cb;
1786 }
1787
1788 /* A few sanity checks */
1789 AssertMsg(!pEntryAbove || off + cbNext <= pEntryAbove->Core.Key,
1790 ("Aligned size intersects with another cache entry\n"));
1791 Assert(cbInEntry <= cbNext);
1792
1793 if (pEntryAbove)
1794 pdmBlkCacheEntryRelease(pEntryAbove);
1795
1796 LogFlow(("off=%llu cbNext=%u\n", off, cbNext));
1797
1798 *pcbEntry = cbNext;
1799
1800 return cbInEntry;
1801}
1802
1803/**
1804 * Create a new cache entry evicting data from the cache if required.
1805 *
1806 * @returns Pointer to the new cache entry or NULL
1807 * if not enough bytes could be evicted from the cache.
1808 * @param pEndpoint The endpoint.
1809 * @param pBlkCache The endpoint cache.
1810 * @param off The offset.
1811 * @param cb Number of bytes the cache entry should have.
1812 * @param pcbData Where to store the number of bytes the new
1813 * entry can hold. May be lower than actually requested
1814 * due to another entry intersecting the access range.
1815 */
1816static PPDMBLKCACHEENTRY pdmBlkCacheEntryCreate(PPDMBLKCACHE pBlkCache,
1817 uint64_t off, size_t cb,
1818 size_t *pcbData)
1819{
1820 uint32_t cbEntry = 0;
1821
1822 *pcbData = pdmBlkCacheEntryBoundariesCalc(pBlkCache, off, (uint32_t)cb, &cbEntry);
1823 AssertReturn(cb <= UINT32_MAX, NULL);
1824
1825 PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
1826 pdmBlkCacheLockEnter(pCache);
1827
1828 PPDMBLKCACHEENTRY pEntryNew = NULL;
1829 uint8_t *pbBuffer = NULL;
1830 bool fEnough = pdmBlkCacheReclaim(pCache, cbEntry, true, &pbBuffer);
1831 if (fEnough)
1832 {
1833 LogFlow(("Evicted enough bytes (%u requested). Creating new cache entry\n", cbEntry));
1834
1835 pEntryNew = pdmBlkCacheEntryAlloc(pBlkCache, off, cbEntry, pbBuffer);
1836 if (RT_LIKELY(pEntryNew))
1837 {
1838 pdmBlkCacheEntryAddToList(&pCache->LruRecentlyUsedIn, pEntryNew);
1839 pdmBlkCacheAdd(pCache, cbEntry);
1840 pdmBlkCacheLockLeave(pCache);
1841
1842 pdmBlkCacheInsertEntry(pBlkCache, pEntryNew);
1843
1844 AssertMsg( (off >= pEntryNew->Core.Key)
1845 && (off + *pcbData <= pEntryNew->Core.KeyLast + 1),
1846 ("Overflow in calculation off=%llu\n", off));
1847 }
1848 else
1849 pdmBlkCacheLockLeave(pCache);
1850 }
1851 else
1852 pdmBlkCacheLockLeave(pCache);
1853
1854 return pEntryNew;
1855}
1856
1857static PPDMBLKCACHEREQ pdmBlkCacheReqAlloc(void *pvUser)
1858{
1859 PPDMBLKCACHEREQ pReq = (PPDMBLKCACHEREQ)RTMemAlloc(sizeof(PDMBLKCACHEREQ));
1860
1861 if (RT_LIKELY(pReq))
1862 {
1863 pReq->pvUser = pvUser;
1864 pReq->rcReq = VINF_SUCCESS;
1865 pReq->cXfersPending = 0;
1866 }
1867
1868 return pReq;
1869}
1870
1871static void pdmBlkCacheReqComplete(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEREQ pReq)
1872{
1873 switch (pBlkCache->enmType)
1874 {
1875 case PDMBLKCACHETYPE_DEV:
1876 {
1877 pBlkCache->u.Dev.pfnXferComplete(pBlkCache->u.Dev.pDevIns,
1878 pReq->pvUser, pReq->rcReq);
1879 break;
1880 }
1881 case PDMBLKCACHETYPE_DRV:
1882 {
1883 pBlkCache->u.Drv.pfnXferComplete(pBlkCache->u.Drv.pDrvIns,
1884 pReq->pvUser, pReq->rcReq);
1885 break;
1886 }
1887 case PDMBLKCACHETYPE_USB:
1888 {
1889 pBlkCache->u.Usb.pfnXferComplete(pBlkCache->u.Usb.pUsbIns,
1890 pReq->pvUser, pReq->rcReq);
1891 break;
1892 }
1893 case PDMBLKCACHETYPE_INTERNAL:
1894 {
1895 pBlkCache->u.Int.pfnXferComplete(pBlkCache->u.Int.pvUser,
1896 pReq->pvUser, pReq->rcReq);
1897 break;
1898 }
1899 default:
1900 AssertMsgFailed(("Unknown block cache type!\n"));
1901 }
1902
1903 RTMemFree(pReq);
1904}
1905
1906static bool pdmBlkCacheReqUpdate(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEREQ pReq,
1907 int rcReq, bool fCallHandler)
1908{
1909 if (RT_FAILURE(rcReq))
1910 ASMAtomicCmpXchgS32(&pReq->rcReq, rcReq, VINF_SUCCESS);
1911
1912 AssertMsg(pReq->cXfersPending > 0, ("No transfers are pending for this request\n"));
1913 uint32_t cXfersPending = ASMAtomicDecU32(&pReq->cXfersPending);
1914
1915 if (!cXfersPending)
1916 {
1917 if (fCallHandler)
1918 pdmBlkCacheReqComplete(pBlkCache, pReq);
1919 else
1920 RTMemFree(pReq);
1921 return true;
1922 }
1923
1924 LogFlowFunc(("pReq=%#p cXfersPending=%u\n", pReq, cXfersPending));
1925 return false;
1926}
1927
1928VMMR3DECL(int) PDMR3BlkCacheRead(PPDMBLKCACHE pBlkCache, uint64_t off,
1929 PCRTSGBUF pcSgBuf, size_t cbRead, void *pvUser)
1930{
1931 int rc = VINF_SUCCESS;
1932 PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
1933 PPDMBLKCACHEENTRY pEntry;
1934 PPDMBLKCACHEREQ pReq;
1935
1936 LogFlowFunc((": pBlkCache=%#p{%s} off=%llu pcSgBuf=%#p cbRead=%u pvUser=%#p\n",
1937 pBlkCache, pBlkCache->pszId, off, pcSgBuf, cbRead, pvUser));
1938
1939 AssertPtrReturn(pBlkCache, VERR_INVALID_POINTER);
1940 AssertReturn(!pBlkCache->fSuspended, VERR_INVALID_STATE);
1941
1942 RTSGBUF SgBuf;
1943 RTSgBufClone(&SgBuf, pcSgBuf);
1944
1945 /* Allocate new request structure. */
1946 pReq = pdmBlkCacheReqAlloc(pvUser);
1947 if (RT_UNLIKELY(!pReq))
1948 return VERR_NO_MEMORY;
1949
1950 /* Increment data transfer counter to keep the request valid while we access it. */
1951 ASMAtomicIncU32(&pReq->cXfersPending);
1952
1953 while (cbRead)
1954 {
1955 size_t cbToRead;
1956
1957 pEntry = pdmBlkCacheGetCacheEntryByOffset(pBlkCache, off);
1958
1959 /*
1960 * If there is no entry we try to create a new one eviciting unused pages
1961 * if the cache is full. If this is not possible we will pass the request through
1962 * and skip the caching (all entries may be still in progress so they can't
1963 * be evicted)
1964 * If we have an entry it can be in one of the LRU lists where the entry
1965 * contains data (recently used or frequently used LRU) so we can just read
1966 * the data we need and put the entry at the head of the frequently used LRU list.
1967 * In case the entry is in one of the ghost lists it doesn't contain any data.
1968 * We have to fetch it again evicting pages from either T1 or T2 to make room.
1969 */
1970 if (pEntry)
1971 {
1972 uint64_t offDiff = off - pEntry->Core.Key;
1973
1974 AssertMsg(off >= pEntry->Core.Key,
1975 ("Overflow in calculation off=%llu OffsetAligned=%llu\n",
1976 off, pEntry->Core.Key));
1977
1978 AssertPtr(pEntry->pList);
1979
1980 cbToRead = RT_MIN(pEntry->cbData - offDiff, cbRead);
1981
1982 AssertMsg(off + cbToRead <= pEntry->Core.Key + pEntry->Core.KeyLast + 1,
1983 ("Buffer of cache entry exceeded off=%llu cbToRead=%d\n",
1984 off, cbToRead));
1985
1986 cbRead -= cbToRead;
1987
1988 if (!cbRead)
1989 STAM_COUNTER_INC(&pCache->cHits);
1990 else
1991 STAM_COUNTER_INC(&pCache->cPartialHits);
1992
1993 STAM_COUNTER_ADD(&pCache->StatRead, cbToRead);
1994
1995 /* Ghost lists contain no data. */
1996 if ( (pEntry->pList == &pCache->LruRecentlyUsedIn)
1997 || (pEntry->pList == &pCache->LruFrequentlyUsed))
1998 {
1999 if (pdmBlkCacheEntryFlagIsSetClearAcquireLock(pBlkCache, pEntry,
2000 PDMBLKCACHE_ENTRY_IO_IN_PROGRESS,
2001 PDMBLKCACHE_ENTRY_IS_DIRTY))
2002 {
2003 /* Entry didn't completed yet. Append to the list */
2004 pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
2005 &SgBuf, offDiff, cbToRead,
2006 false /* fWrite */);
2007 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2008 }
2009 else
2010 {
2011 /* Read as much as we can from the entry. */
2012 RTSgBufCopyFromBuf(&SgBuf, pEntry->pbData + offDiff, cbToRead);
2013 }
2014
2015 /* Move this entry to the top position */
2016 if (pEntry->pList == &pCache->LruFrequentlyUsed)
2017 {
2018 pdmBlkCacheLockEnter(pCache);
2019 pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
2020 pdmBlkCacheLockLeave(pCache);
2021 }
2022 /* Release the entry */
2023 pdmBlkCacheEntryRelease(pEntry);
2024 }
2025 else
2026 {
2027 uint8_t *pbBuffer = NULL;
2028
2029 LogFlow(("Fetching data for ghost entry %#p from file\n", pEntry));
2030
2031 pdmBlkCacheLockEnter(pCache);
2032 pdmBlkCacheEntryRemoveFromList(pEntry); /* Remove it before we remove data, otherwise it may get freed when evicting data. */
2033 bool fEnough = pdmBlkCacheReclaim(pCache, pEntry->cbData, true, &pbBuffer);
2034
2035 /* Move the entry to Am and fetch it to the cache. */
2036 if (fEnough)
2037 {
2038 pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
2039 pdmBlkCacheAdd(pCache, pEntry->cbData);
2040 pdmBlkCacheLockLeave(pCache);
2041
2042 if (pbBuffer)
2043 pEntry->pbData = pbBuffer;
2044 else
2045 pEntry->pbData = (uint8_t *)RTMemPageAlloc(pEntry->cbData);
2046 AssertPtr(pEntry->pbData);
2047
2048 pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
2049 &SgBuf, offDiff, cbToRead,
2050 false /* fWrite */);
2051 pdmBlkCacheEntryReadFromMedium(pEntry);
2052 /* Release the entry */
2053 pdmBlkCacheEntryRelease(pEntry);
2054 }
2055 else
2056 {
2057 RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
2058 STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
2059 RTAvlrU64Remove(pBlkCache->pTree, pEntry->Core.Key);
2060 STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
2061 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2062
2063 pdmBlkCacheLockLeave(pCache);
2064
2065 RTMemFree(pEntry);
2066
2067 pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
2068 &SgBuf, off, cbToRead,
2069 PDMBLKCACHEXFERDIR_READ);
2070 }
2071 }
2072 }
2073 else
2074 {
2075#ifdef VBOX_WITH_IO_READ_CACHE
2076 /* No entry found for this offset. Create a new entry and fetch the data to the cache. */
2077 PPDMBLKCACHEENTRY pEntryNew = pdmBlkCacheEntryCreate(pBlkCache,
2078 off, cbRead,
2079 &cbToRead);
2080
2081 cbRead -= cbToRead;
2082
2083 if (pEntryNew)
2084 {
2085 if (!cbRead)
2086 STAM_COUNTER_INC(&pCache->cMisses);
2087 else
2088 STAM_COUNTER_INC(&pCache->cPartialHits);
2089
2090 pdmBlkCacheEntryWaitersAdd(pEntryNew, pReq,
2091 &SgBuf,
2092 off - pEntryNew->Core.Key,
2093 cbToRead,
2094 false /* fWrite */);
2095 pdmBlkCacheEntryReadFromMedium(pEntryNew);
2096 pdmBlkCacheEntryRelease(pEntryNew); /* it is protected by the I/O in progress flag now. */
2097 }
2098 else
2099 {
2100 /*
2101 * There is not enough free space in the cache.
2102 * Pass the request directly to the I/O manager.
2103 */
2104 LogFlow(("Couldn't evict %u bytes from the cache. Remaining request will be passed through\n", cbToRead));
2105
2106 pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
2107 &SgBuf, off, cbToRead,
2108 PDMBLKCACHEXFERDIR_READ);
2109 }
2110#else
2111 /* Clip read size if necessary. */
2112 PPDMBLKCACHEENTRY pEntryAbove;
2113 pdmBlkCacheGetCacheBestFitEntryByOffset(pBlkCache, off, &pEntryAbove);
2114
2115 if (pEntryAbove)
2116 {
2117 if (off + cbRead > pEntryAbove->Core.Key)
2118 cbToRead = pEntryAbove->Core.Key - off;
2119 else
2120 cbToRead = cbRead;
2121
2122 pdmBlkCacheEntryRelease(pEntryAbove);
2123 }
2124 else
2125 cbToRead = cbRead;
2126
2127 cbRead -= cbToRead;
2128 pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
2129 &SgBuf, off, cbToRead,
2130 PDMBLKCACHEXFERDIR_READ);
2131#endif
2132 }
2133 off += cbToRead;
2134 }
2135
2136 if (!pdmBlkCacheReqUpdate(pBlkCache, pReq, rc, false))
2137 rc = VINF_AIO_TASK_PENDING;
2138
2139 LogFlowFunc((": Leave rc=%Rrc\n", rc));
2140
2141 return rc;
2142}
2143
2144VMMR3DECL(int) PDMR3BlkCacheWrite(PPDMBLKCACHE pBlkCache, uint64_t off,
2145 PCRTSGBUF pcSgBuf, size_t cbWrite, void *pvUser)
2146{
2147 int rc = VINF_SUCCESS;
2148 PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
2149 PPDMBLKCACHEENTRY pEntry;
2150 PPDMBLKCACHEREQ pReq;
2151
2152 LogFlowFunc((": pBlkCache=%#p{%s} off=%llu pcSgBuf=%#p cbWrite=%u pvUser=%#p\n",
2153 pBlkCache, pBlkCache->pszId, off, pcSgBuf, cbWrite, pvUser));
2154
2155 AssertPtrReturn(pBlkCache, VERR_INVALID_POINTER);
2156 AssertReturn(!pBlkCache->fSuspended, VERR_INVALID_STATE);
2157
2158 RTSGBUF SgBuf;
2159 RTSgBufClone(&SgBuf, pcSgBuf);
2160
2161 /* Allocate new request structure. */
2162 pReq = pdmBlkCacheReqAlloc(pvUser);
2163 if (RT_UNLIKELY(!pReq))
2164 return VERR_NO_MEMORY;
2165
2166 /* Increment data transfer counter to keep the request valid while we access it. */
2167 ASMAtomicIncU32(&pReq->cXfersPending);
2168
2169 while (cbWrite)
2170 {
2171 size_t cbToWrite;
2172
2173 pEntry = pdmBlkCacheGetCacheEntryByOffset(pBlkCache, off);
2174 if (pEntry)
2175 {
2176 /* Write the data into the entry and mark it as dirty */
2177 AssertPtr(pEntry->pList);
2178
2179 uint64_t offDiff = off - pEntry->Core.Key;
2180
2181 AssertMsg(off >= pEntry->Core.Key,
2182 ("Overflow in calculation off=%llu OffsetAligned=%llu\n",
2183 off, pEntry->Core.Key));
2184
2185 cbToWrite = RT_MIN(pEntry->cbData - offDiff, cbWrite);
2186 cbWrite -= cbToWrite;
2187
2188 if (!cbWrite)
2189 STAM_COUNTER_INC(&pCache->cHits);
2190 else
2191 STAM_COUNTER_INC(&pCache->cPartialHits);
2192
2193 STAM_COUNTER_ADD(&pCache->StatWritten, cbToWrite);
2194
2195 /* Ghost lists contain no data. */
2196 if ( (pEntry->pList == &pCache->LruRecentlyUsedIn)
2197 || (pEntry->pList == &pCache->LruFrequentlyUsed))
2198 {
2199 /* Check if the entry is dirty. */
2200 if (pdmBlkCacheEntryFlagIsSetClearAcquireLock(pBlkCache, pEntry,
2201 PDMBLKCACHE_ENTRY_IS_DIRTY,
2202 0))
2203 {
2204 /* If it is already dirty but not in progress just update the data. */
2205 if (!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS))
2206 RTSgBufCopyToBuf(&SgBuf, pEntry->pbData + offDiff, cbToWrite);
2207 else
2208 {
2209 /* The data isn't written to the file yet */
2210 pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
2211 &SgBuf, offDiff, cbToWrite,
2212 true /* fWrite */);
2213 STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
2214 }
2215
2216 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2217 }
2218 else /* Dirty bit not set */
2219 {
2220 /*
2221 * Check if a read is in progress for this entry.
2222 * We have to defer processing in that case.
2223 */
2224 if (pdmBlkCacheEntryFlagIsSetClearAcquireLock(pBlkCache, pEntry,
2225 PDMBLKCACHE_ENTRY_IO_IN_PROGRESS,
2226 0))
2227 {
2228 pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
2229 &SgBuf, offDiff, cbToWrite,
2230 true /* fWrite */);
2231 STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
2232 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2233 }
2234 else /* I/O in progress flag not set */
2235 {
2236 /* Write as much as we can into the entry and update the file. */
2237 RTSgBufCopyToBuf(&SgBuf, pEntry->pbData + offDiff, cbToWrite);
2238
2239 bool fCommit = pdmBlkCacheAddDirtyEntry(pBlkCache, pEntry);
2240 if (fCommit)
2241 pdmBlkCacheCommitDirtyEntries(pCache);
2242 }
2243 } /* Dirty bit not set */
2244
2245 /* Move this entry to the top position */
2246 if (pEntry->pList == &pCache->LruFrequentlyUsed)
2247 {
2248 pdmBlkCacheLockEnter(pCache);
2249 pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
2250 pdmBlkCacheLockLeave(pCache);
2251 }
2252
2253 pdmBlkCacheEntryRelease(pEntry);
2254 }
2255 else /* Entry is on the ghost list */
2256 {
2257 uint8_t *pbBuffer = NULL;
2258
2259 pdmBlkCacheLockEnter(pCache);
2260 pdmBlkCacheEntryRemoveFromList(pEntry); /* Remove it before we remove data, otherwise it may get freed when evicting data. */
2261 bool fEnough = pdmBlkCacheReclaim(pCache, pEntry->cbData, true, &pbBuffer);
2262
2263 if (fEnough)
2264 {
2265 /* Move the entry to Am and fetch it to the cache. */
2266 pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
2267 pdmBlkCacheAdd(pCache, pEntry->cbData);
2268 pdmBlkCacheLockLeave(pCache);
2269
2270 if (pbBuffer)
2271 pEntry->pbData = pbBuffer;
2272 else
2273 pEntry->pbData = (uint8_t *)RTMemPageAlloc(pEntry->cbData);
2274 AssertPtr(pEntry->pbData);
2275
2276 pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
2277 &SgBuf, offDiff, cbToWrite,
2278 true /* fWrite */);
2279 STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
2280 pdmBlkCacheEntryReadFromMedium(pEntry);
2281
2282 /* Release the reference. If it is still needed the I/O in progress flag should protect it now. */
2283 pdmBlkCacheEntryRelease(pEntry);
2284 }
2285 else
2286 {
2287 RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
2288 STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
2289 RTAvlrU64Remove(pBlkCache->pTree, pEntry->Core.Key);
2290 STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
2291 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2292
2293 pdmBlkCacheLockLeave(pCache);
2294
2295 RTMemFree(pEntry);
2296 pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
2297 &SgBuf, off, cbToWrite,
2298 PDMBLKCACHEXFERDIR_WRITE);
2299 }
2300 }
2301 }
2302 else /* No entry found */
2303 {
2304 /*
2305 * No entry found. Try to create a new cache entry to store the data in and if that fails
2306 * write directly to the file.
2307 */
2308 PPDMBLKCACHEENTRY pEntryNew = pdmBlkCacheEntryCreate(pBlkCache,
2309 off, cbWrite,
2310 &cbToWrite);
2311
2312 cbWrite -= cbToWrite;
2313
2314 if (pEntryNew)
2315 {
2316 uint64_t offDiff = off - pEntryNew->Core.Key;
2317
2318 STAM_COUNTER_INC(&pCache->cHits);
2319
2320 /*
2321 * Check if it is possible to just write the data without waiting
2322 * for it to get fetched first.
2323 */
2324 if (!offDiff && pEntryNew->cbData == cbToWrite)
2325 {
2326 RTSgBufCopyToBuf(&SgBuf, pEntryNew->pbData, cbToWrite);
2327
2328 bool fCommit = pdmBlkCacheAddDirtyEntry(pBlkCache, pEntryNew);
2329 if (fCommit)
2330 pdmBlkCacheCommitDirtyEntries(pCache);
2331 STAM_COUNTER_ADD(&pCache->StatWritten, cbToWrite);
2332 }
2333 else
2334 {
2335 /* Defer the write and fetch the data from the endpoint. */
2336 pdmBlkCacheEntryWaitersAdd(pEntryNew, pReq,
2337 &SgBuf, offDiff, cbToWrite,
2338 true /* fWrite */);
2339 STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
2340 pdmBlkCacheEntryReadFromMedium(pEntryNew);
2341 }
2342
2343 pdmBlkCacheEntryRelease(pEntryNew);
2344 }
2345 else
2346 {
2347 /*
2348 * There is not enough free space in the cache.
2349 * Pass the request directly to the I/O manager.
2350 */
2351 LogFlow(("Couldn't evict %u bytes from the cache. Remaining request will be passed through\n", cbToWrite));
2352
2353 STAM_COUNTER_INC(&pCache->cMisses);
2354
2355 pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
2356 &SgBuf, off, cbToWrite,
2357 PDMBLKCACHEXFERDIR_WRITE);
2358 }
2359 }
2360
2361 off += cbToWrite;
2362 }
2363
2364 if (!pdmBlkCacheReqUpdate(pBlkCache, pReq, rc, false))
2365 rc = VINF_AIO_TASK_PENDING;
2366
2367 LogFlowFunc((": Leave rc=%Rrc\n", rc));
2368
2369 return rc;
2370}
2371
2372VMMR3DECL(int) PDMR3BlkCacheFlush(PPDMBLKCACHE pBlkCache, void *pvUser)
2373{
2374 int rc = VINF_SUCCESS;
2375 PPDMBLKCACHEREQ pReq;
2376
2377 LogFlowFunc((": pBlkCache=%#p{%s}\n", pBlkCache, pBlkCache->pszId));
2378
2379 AssertPtrReturn(pBlkCache, VERR_INVALID_POINTER);
2380 AssertReturn(!pBlkCache->fSuspended, VERR_INVALID_STATE);
2381
2382 /* Commit dirty entries in the cache. */
2383 pdmBlkCacheCommit(pBlkCache);
2384
2385 /* Allocate new request structure. */
2386 pReq = pdmBlkCacheReqAlloc(pvUser);
2387 if (RT_UNLIKELY(!pReq))
2388 return VERR_NO_MEMORY;
2389
2390 rc = pdmBlkCacheRequestPassthrough(pBlkCache, pReq, NULL, 0, 0,
2391 PDMBLKCACHEXFERDIR_FLUSH);
2392 AssertRC(rc);
2393
2394 LogFlowFunc((": Leave rc=%Rrc\n", rc));
2395 return VINF_AIO_TASK_PENDING;
2396}
2397
2398VMMR3DECL(int) PDMR3BlkCacheDiscard(PPDMBLKCACHE pBlkCache, PCRTRANGE paRanges,
2399 unsigned cRanges, void *pvUser)
2400{
2401 int rc = VINF_SUCCESS;
2402 PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
2403 PPDMBLKCACHEENTRY pEntry;
2404 PPDMBLKCACHEREQ pReq;
2405
2406 LogFlowFunc((": pBlkCache=%#p{%s} paRanges=%#p cRanges=%u pvUser=%#p\n",
2407 pBlkCache, pBlkCache->pszId, paRanges, cRanges, pvUser));
2408
2409 AssertPtrReturn(pBlkCache, VERR_INVALID_POINTER);
2410 AssertReturn(!pBlkCache->fSuspended, VERR_INVALID_STATE);
2411
2412 /* Allocate new request structure. */
2413 pReq = pdmBlkCacheReqAlloc(pvUser);
2414 if (RT_UNLIKELY(!pReq))
2415 return VERR_NO_MEMORY;
2416
2417 /* Increment data transfer counter to keep the request valid while we access it. */
2418 ASMAtomicIncU32(&pReq->cXfersPending);
2419
2420 for (unsigned i = 0; i < cRanges; i++)
2421 {
2422 uint64_t offCur = paRanges[i].offStart;
2423 size_t cbLeft = paRanges[i].cbRange;
2424
2425 while (cbLeft)
2426 {
2427 size_t cbThisDiscard = 0;
2428
2429 pEntry = pdmBlkCacheGetCacheEntryByOffset(pBlkCache, offCur);
2430
2431 if (pEntry)
2432 {
2433 /* Write the data into the entry and mark it as dirty */
2434 AssertPtr(pEntry->pList);
2435
2436 uint64_t offDiff = offCur - pEntry->Core.Key;
2437
2438 AssertMsg(offCur >= pEntry->Core.Key,
2439 ("Overflow in calculation offCur=%llu OffsetAligned=%llu\n",
2440 offCur, pEntry->Core.Key));
2441
2442 cbThisDiscard = RT_MIN(pEntry->cbData - offDiff, cbLeft);
2443
2444 /* Ghost lists contain no data. */
2445 if ( (pEntry->pList == &pCache->LruRecentlyUsedIn)
2446 || (pEntry->pList == &pCache->LruFrequentlyUsed))
2447 {
2448 /* Check if the entry is dirty. */
2449 if (pdmBlkCacheEntryFlagIsSetClearAcquireLock(pBlkCache, pEntry,
2450 PDMBLKCACHE_ENTRY_IS_DIRTY,
2451 0))
2452 {
2453 /* If it is dirty but not yet in progress remove it. */
2454 if (!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS))
2455 {
2456 pdmBlkCacheLockEnter(pCache);
2457 pdmBlkCacheEntryRemoveFromList(pEntry);
2458
2459 STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
2460 RTAvlrU64Remove(pBlkCache->pTree, pEntry->Core.Key);
2461 STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
2462
2463 pdmBlkCacheLockLeave(pCache);
2464
2465 RTMemFree(pEntry);
2466 }
2467 else
2468 {
2469#if 0
2470 /* The data isn't written to the file yet */
2471 pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
2472 &SgBuf, offDiff, cbToWrite,
2473 true /* fWrite */);
2474 STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
2475#endif
2476 }
2477
2478 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2479 pdmBlkCacheEntryRelease(pEntry);
2480 }
2481 else /* Dirty bit not set */
2482 {
2483 /*
2484 * Check if a read is in progress for this entry.
2485 * We have to defer processing in that case.
2486 */
2487 if(pdmBlkCacheEntryFlagIsSetClearAcquireLock(pBlkCache, pEntry,
2488 PDMBLKCACHE_ENTRY_IO_IN_PROGRESS,
2489 0))
2490 {
2491#if 0
2492 pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
2493 &SgBuf, offDiff, cbToWrite,
2494 true /* fWrite */);
2495#endif
2496 STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
2497 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2498 pdmBlkCacheEntryRelease(pEntry);
2499 }
2500 else /* I/O in progress flag not set */
2501 {
2502 pdmBlkCacheLockEnter(pCache);
2503 pdmBlkCacheEntryRemoveFromList(pEntry);
2504
2505 RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
2506 STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
2507 RTAvlrU64Remove(pBlkCache->pTree, pEntry->Core.Key);
2508 STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
2509 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2510
2511 pdmBlkCacheLockLeave(pCache);
2512
2513 RTMemFree(pEntry);
2514 }
2515 } /* Dirty bit not set */
2516 }
2517 else /* Entry is on the ghost list just remove cache entry. */
2518 {
2519 pdmBlkCacheLockEnter(pCache);
2520 pdmBlkCacheEntryRemoveFromList(pEntry);
2521
2522 RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
2523 STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
2524 RTAvlrU64Remove(pBlkCache->pTree, pEntry->Core.Key);
2525 STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
2526 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2527
2528 pdmBlkCacheLockLeave(pCache);
2529
2530 RTMemFree(pEntry);
2531 }
2532 }
2533 /* else: no entry found. */
2534
2535 offCur += cbThisDiscard;
2536 cbLeft -= cbThisDiscard;
2537 }
2538 }
2539
2540 if (!pdmBlkCacheReqUpdate(pBlkCache, pReq, rc, false))
2541 rc = VINF_AIO_TASK_PENDING;
2542
2543 LogFlowFunc((": Leave rc=%Rrc\n", rc));
2544
2545 return rc;
2546}
2547
2548/**
2549 * Completes a task segment freeing all resources and completes the task handle
2550 * if everything was transferred.
2551 *
2552 * @returns Next task segment handle.
2553 * @param pTaskSeg Task segment to complete.
2554 * @param rc Status code to set.
2555 */
2556static PPDMBLKCACHEWAITER pdmBlkCacheWaiterComplete(PPDMBLKCACHE pBlkCache,
2557 PPDMBLKCACHEWAITER pWaiter,
2558 int rc)
2559{
2560 PPDMBLKCACHEWAITER pNext = pWaiter->pNext;
2561 PPDMBLKCACHEREQ pReq = pWaiter->pReq;
2562
2563 pdmBlkCacheReqUpdate(pBlkCache, pReq, rc, true);
2564
2565 RTMemFree(pWaiter);
2566
2567 return pNext;
2568}
2569
2570static void pdmBlkCacheIoXferCompleteEntry(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEIOXFER hIoXfer, int rcIoXfer)
2571{
2572 PPDMBLKCACHEENTRY pEntry = hIoXfer->pEntry;
2573 PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
2574
2575 /* Reference the entry now as we are clearing the I/O in progress flag
2576 * which protected the entry till now. */
2577 pdmBlkCacheEntryRef(pEntry);
2578
2579 RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
2580 pEntry->fFlags &= ~PDMBLKCACHE_ENTRY_IO_IN_PROGRESS;
2581
2582 /* Process waiting segment list. The data in entry might have changed in-between. */
2583 bool fDirty = false;
2584 PPDMBLKCACHEWAITER pComplete = pEntry->pWaitingHead;
2585 PPDMBLKCACHEWAITER pCurr = pComplete;
2586
2587 AssertMsg((pCurr && pEntry->pWaitingTail) || (!pCurr && !pEntry->pWaitingTail),
2588 ("The list tail was not updated correctly\n"));
2589 pEntry->pWaitingTail = NULL;
2590 pEntry->pWaitingHead = NULL;
2591
2592 if (hIoXfer->enmXferDir == PDMBLKCACHEXFERDIR_WRITE)
2593 {
2594 /*
2595 * An error here is difficult to handle as the original request completed already.
2596 * The error is logged for now and the VM is paused.
2597 * If the user continues the entry is written again in the hope
2598 * the user fixed the problem and the next write succeeds.
2599 */
2600 if (RT_FAILURE(rcIoXfer))
2601 {
2602 LogRel(("I/O cache: Error while writing entry at offset %llu (%u bytes) to medium \"%s\" (rc=%Rrc)\n",
2603 pEntry->Core.Key, pEntry->cbData, pBlkCache->pszId, rcIoXfer));
2604
2605 if (!ASMAtomicXchgBool(&pCache->fIoErrorVmSuspended, true))
2606 {
2607 int rc = VMSetRuntimeError(pCache->pVM, VMSETRTERR_FLAGS_SUSPEND | VMSETRTERR_FLAGS_NO_WAIT, "BLKCACHE_IOERR",
2608 N_("The I/O cache encountered an error while updating data in medium \"%s\" (rc=%Rrc). "
2609 "Make sure there is enough free space on the disk and that the disk is working properly. "
2610 "Operation can be resumed afterwards"),
2611 pBlkCache->pszId, rcIoXfer);
2612 AssertRC(rc);
2613 }
2614
2615 /* Mark the entry as dirty again to get it added to the list later on. */
2616 fDirty = true;
2617 }
2618
2619 pEntry->fFlags &= ~PDMBLKCACHE_ENTRY_IS_DIRTY;
2620
2621 while (pCurr)
2622 {
2623 AssertMsg(pCurr->fWrite, ("Completed write entries should never have read tasks attached\n"));
2624
2625 RTSgBufCopyToBuf(&pCurr->SgBuf, pEntry->pbData + pCurr->offCacheEntry, pCurr->cbTransfer);
2626 fDirty = true;
2627 pCurr = pCurr->pNext;
2628 }
2629 }
2630 else
2631 {
2632 AssertMsg(hIoXfer->enmXferDir == PDMBLKCACHEXFERDIR_READ, ("Invalid transfer type\n"));
2633 AssertMsg(!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IS_DIRTY),
2634 ("Invalid flags set\n"));
2635
2636 while (pCurr)
2637 {
2638 if (pCurr->fWrite)
2639 {
2640 RTSgBufCopyToBuf(&pCurr->SgBuf, pEntry->pbData + pCurr->offCacheEntry, pCurr->cbTransfer);
2641 fDirty = true;
2642 }
2643 else
2644 RTSgBufCopyFromBuf(&pCurr->SgBuf, pEntry->pbData + pCurr->offCacheEntry, pCurr->cbTransfer);
2645
2646 pCurr = pCurr->pNext;
2647 }
2648 }
2649
2650 bool fCommit = false;
2651 if (fDirty)
2652 fCommit = pdmBlkCacheAddDirtyEntry(pBlkCache, pEntry);
2653
2654 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2655
2656 /* Dereference so that it isn't protected anymore except we issued anyother write for it. */
2657 pdmBlkCacheEntryRelease(pEntry);
2658
2659 if (fCommit)
2660 pdmBlkCacheCommitDirtyEntries(pCache);
2661
2662 /* Complete waiters now. */
2663 while (pComplete)
2664 pComplete = pdmBlkCacheWaiterComplete(pBlkCache, pComplete, rcIoXfer);
2665}
2666
2667VMMR3DECL(void) PDMR3BlkCacheIoXferComplete(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEIOXFER hIoXfer, int rcIoXfer)
2668{
2669 LogFlowFunc(("pBlkCache=%#p hIoXfer=%#p rcIoXfer=%Rrc\n", pBlkCache, hIoXfer, rcIoXfer));
2670
2671 if (hIoXfer->fIoCache)
2672 pdmBlkCacheIoXferCompleteEntry(pBlkCache, hIoXfer, rcIoXfer);
2673 else
2674 pdmBlkCacheReqUpdate(pBlkCache, hIoXfer->pReq, rcIoXfer, true);
2675 RTMemFree(hIoXfer);
2676}
2677
2678/**
2679 * Callback for the AVL do with all routine. Waits for a cachen entry to finish any pending I/O.
2680 *
2681 * @returns IPRT status code.
2682 * @param pNode The node to destroy.
2683 * @param pvUser Opaque user data.
2684 */
2685static int pdmBlkCacheEntryQuiesce(PAVLRU64NODECORE pNode, void *pvUser)
2686{
2687 PPDMBLKCACHEENTRY pEntry = (PPDMBLKCACHEENTRY)pNode;
2688 PPDMBLKCACHE pBlkCache = pEntry->pBlkCache;
2689 NOREF(pvUser);
2690
2691 while (ASMAtomicReadU32(&pEntry->fFlags) & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS)
2692 {
2693 /* Leave the locks to let the I/O thread make progress but reference the entry to prevent eviction. */
2694 pdmBlkCacheEntryRef(pEntry);
2695 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2696
2697 RTThreadSleep(1);
2698
2699 /* Re-enter all locks and drop the reference. */
2700 RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
2701 pdmBlkCacheEntryRelease(pEntry);
2702 }
2703
2704 AssertMsg(!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS),
2705 ("Entry is dirty and/or still in progress fFlags=%#x\n", pEntry->fFlags));
2706
2707 return VINF_SUCCESS;
2708}
2709
2710VMMR3DECL(int) PDMR3BlkCacheSuspend(PPDMBLKCACHE pBlkCache)
2711{
2712 int rc = VINF_SUCCESS;
2713 LogFlowFunc(("pBlkCache=%#p\n", pBlkCache));
2714
2715 AssertPtrReturn(pBlkCache, VERR_INVALID_POINTER);
2716
2717 if (!ASMAtomicReadBool(&pBlkCache->pCache->fIoErrorVmSuspended))
2718 pdmBlkCacheCommit(pBlkCache); /* Can issue new I/O requests. */
2719 ASMAtomicXchgBool(&pBlkCache->fSuspended, true);
2720
2721 /* Wait for all I/O to complete. */
2722 RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
2723 rc = RTAvlrU64DoWithAll(pBlkCache->pTree, true, pdmBlkCacheEntryQuiesce, NULL);
2724 AssertRC(rc);
2725 RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2726
2727 return rc;
2728}
2729
2730VMMR3DECL(int) PDMR3BlkCacheResume(PPDMBLKCACHE pBlkCache)
2731{
2732 LogFlowFunc(("pBlkCache=%#p\n", pBlkCache));
2733
2734 AssertPtrReturn(pBlkCache, VERR_INVALID_POINTER);
2735
2736 ASMAtomicXchgBool(&pBlkCache->fSuspended, false);
2737
2738 return VINF_SUCCESS;
2739}
2740
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette