PDMBlkCache.cpp@ 34223

Last change on this file since 34223 was 34223, checked in by vboxsync, 14 years ago
BlockCache: Bug fixes
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 84.1 KB

Line
1	/* $Id: PDMBlkCache.cpp 34223 2010-11-21 23:51:36Z vboxsync $ */
2	/** @file
3	* PDM Block Cache.
4	*/
5
6	/*
7	* Copyright (C) 2006-2008 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/** @page pg_pdm_block_cache PDM Block Cache - The I/O cache
19	* This component implements an I/O cache based on the 2Q cache algorithm.
20	*/
21
22	/*******************************************************************************
23	* Header Files *
24	*******************************************************************************/
25	#define LOG_GROUP LOG_GROUP_PDM_BLK_CACHE
26	#include "PDMInternal.h"
27	#include <iprt/asm.h>
28	#include <iprt/mem.h>
29	#include <iprt/path.h>
30	#include <iprt/string.h>
31	#include <VBox/log.h>
32	#include <VBox/stam.h>
33	#include <VBox/uvm.h>
34	#include <VBox/vm.h>
35
36	#include "PDMBlkCacheInternal.h"
37
38	#ifdef VBOX_STRICT
39	# define PDMACFILECACHE_IS_CRITSECT_OWNER(Cache) \
40	do \
41	{ \
42	AssertMsg(RTCritSectIsOwner(&Cache->CritSect), \
43	("Thread does not own critical section\n"));\
44	} while(0)
45
46	# define PDMACFILECACHE_EP_IS_SEMRW_WRITE_OWNER(pEpCache) \
47	do \
48	{ \
49	AssertMsg(RTSemRWIsWriteOwner(pEpCache->SemRWEntries), \
50	("Thread is not exclusive owner of the per endpoint RW semaphore\n")); \
51	} while(0)
52
53	# define PDMACFILECACHE_EP_IS_SEMRW_READ_OWNER(pEpCache) \
54	do \
55	{ \
56	AssertMsg(RTSemRWIsReadOwner(pEpCache->SemRWEntries), \
57	("Thread is not read owner of the per endpoint RW semaphore\n")); \
58	} while(0)
59
60	#else
61	# define PDMACFILECACHE_IS_CRITSECT_OWNER(Cache) do { } while(0)
62	# define PDMACFILECACHE_EP_IS_SEMRW_WRITE_OWNER(pEpCache) do { } while(0)
63	# define PDMACFILECACHE_EP_IS_SEMRW_READ_OWNER(pEpCache) do { } while(0)
64	#endif
65
66	/*******************************************************************************
67	* Internal Functions *
68	*******************************************************************************/
69
70	/**
71	* Decrement the reference counter of the given cache entry.
72	*
73	* @returns nothing.
74	* @param pEntry The entry to release.
75	*/
76	DECLINLINE(void) pdmBlkCacheEntryRelease(PPDMBLKCACHEENTRY pEntry)
77	{
78	AssertMsg(pEntry->cRefs > 0, ("Trying to release a not referenced entry\n"));
79	ASMAtomicDecU32(&pEntry->cRefs);
80	}
81
82	/**
83	* Increment the reference counter of the given cache entry.
84	*
85	* @returns nothing.
86	* @param pEntry The entry to reference.
87	*/
88	DECLINLINE(void) pdmBlkCacheEntryRef(PPDMBLKCACHEENTRY pEntry)
89	{
90	ASMAtomicIncU32(&pEntry->cRefs);
91	}
92
93	#ifdef DEBUG
94	static void pdmBlkCacheValidate(PPDMBLKCACHEGLOBAL pCache)
95	{
96	/* Amount of cached data should never exceed the maximum amount. */
97	AssertMsg(pCache->cbCached <= pCache->cbMax,
98	("Current amount of cached data exceeds maximum\n"));
99
100	/* The amount of cached data in the LRU and FRU list should match cbCached */
101	AssertMsg(pCache->LruRecentlyUsedIn.cbCached + pCache->LruFrequentlyUsed.cbCached == pCache->cbCached,
102	("Amount of cached data doesn't match\n"));
103
104	AssertMsg(pCache->LruRecentlyUsedOut.cbCached <= pCache->cbRecentlyUsedOutMax,
105	("Paged out list exceeds maximum\n"));
106	}
107	#endif
108
109	DECLINLINE(void) pdmBlkCacheLockEnter(PPDMBLKCACHEGLOBAL pCache)
110	{
111	RTCritSectEnter(&pCache->CritSect);
112	#ifdef DEBUG
113	pdmBlkCacheValidate(pCache);
114	#endif
115	}
116
117	DECLINLINE(void) pdmBlkCacheLockLeave(PPDMBLKCACHEGLOBAL pCache)
118	{
119	#ifdef DEBUG
120	pdmBlkCacheValidate(pCache);
121	#endif
122	RTCritSectLeave(&pCache->CritSect);
123	}
124
125	DECLINLINE(void) pdmBlkCacheSub(PPDMBLKCACHEGLOBAL pCache, uint32_t cbAmount)
126	{
127	PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
128	pCache->cbCached -= cbAmount;
129	}
130
131	DECLINLINE(void) pdmBlkCacheAdd(PPDMBLKCACHEGLOBAL pCache, uint32_t cbAmount)
132	{
133	PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
134	pCache->cbCached += cbAmount;
135	}
136
137	DECLINLINE(void) pdmBlkCacheListAdd(PPDMBLKLRULIST pList, uint32_t cbAmount)
138	{
139	pList->cbCached += cbAmount;
140	}
141
142	DECLINLINE(void) pdmBlkCacheListSub(PPDMBLKLRULIST pList, uint32_t cbAmount)
143	{
144	pList->cbCached -= cbAmount;
145	}
146
147	#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
148	/**
149	* Checks consistency of a LRU list.
150	*
151	* @returns nothing
152	* @param pList The LRU list to check.
153	* @param pNotInList Element which is not allowed to occur in the list.
154	*/
155	static void pdmBlkCacheCheckList(PPDMBLKLRULIST pList, PPDMBLKCACHEENTRY pNotInList)
156	{
157	PPDMBLKCACHEENTRY pCurr = pList->pHead;
158
159	/* Check that there are no double entries and no cycles in the list. */
160	while (pCurr)
161	{
162	PPDMBLKCACHEENTRY pNext = pCurr->pNext;
163
164	while (pNext)
165	{
166	AssertMsg(pCurr != pNext,
167	("Entry %#p is at least two times in list %#p or there is a cycle in the list\n",
168	pCurr, pList));
169	pNext = pNext->pNext;
170	}
171
172	AssertMsg(pCurr != pNotInList, ("Not allowed entry %#p is in list\n", pCurr));
173
174	if (!pCurr->pNext)
175	AssertMsg(pCurr == pList->pTail, ("End of list reached but last element is not list tail\n"));
176
177	pCurr = pCurr->pNext;
178	}
179	}
180	#endif
181
182	/**
183	* Unlinks a cache entry from the LRU list it is assigned to.
184	*
185	* @returns nothing.
186	* @param pEntry The entry to unlink.
187	*/
188	static void pdmBlkCacheEntryRemoveFromList(PPDMBLKCACHEENTRY pEntry)
189	{
190	PPDMBLKLRULIST pList = pEntry->pList;
191	PPDMBLKCACHEENTRY pPrev, pNext;
192
193	LogFlowFunc((": Deleting entry %#p from list %#p\n", pEntry, pList));
194
195	AssertPtr(pList);
196
197	#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
198	pdmBlkCacheCheckList(pList, NULL);
199	#endif
200
201	pPrev = pEntry->pPrev;
202	pNext = pEntry->pNext;
203
204	AssertMsg(pEntry != pPrev, ("Entry links to itself as previous element\n"));
205	AssertMsg(pEntry != pNext, ("Entry links to itself as next element\n"));
206
207	if (pPrev)
208	pPrev->pNext = pNext;
209	else
210	{
211	pList->pHead = pNext;
212
213	if (pNext)
214	pNext->pPrev = NULL;
215	}
216
217	if (pNext)
218	pNext->pPrev = pPrev;
219	else
220	{
221	pList->pTail = pPrev;
222
223	if (pPrev)
224	pPrev->pNext = NULL;
225	}
226
227	pEntry->pList = NULL;
228	pEntry->pPrev = NULL;
229	pEntry->pNext = NULL;
230	pdmBlkCacheListSub(pList, pEntry->cbData);
231	#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
232	pdmBlkCacheCheckList(pList, pEntry);
233	#endif
234	}
235
236	/**
237	* Adds a cache entry to the given LRU list unlinking it from the currently
238	* assigned list if needed.
239	*
240	* @returns nothing.
241	* @param pList List to the add entry to.
242	* @param pEntry Entry to add.
243	*/
244	static void pdmBlkCacheEntryAddToList(PPDMBLKLRULIST pList, PPDMBLKCACHEENTRY pEntry)
245	{
246	LogFlowFunc((": Adding entry %#p to list %#p\n", pEntry, pList));
247	#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
248	pdmBlkCacheCheckList(pList, NULL);
249	#endif
250
251	/* Remove from old list if needed */
252	if (pEntry->pList)
253	pdmBlkCacheEntryRemoveFromList(pEntry);
254
255	pEntry->pNext = pList->pHead;
256	if (pList->pHead)
257	pList->pHead->pPrev = pEntry;
258	else
259	{
260	Assert(!pList->pTail);
261	pList->pTail = pEntry;
262	}
263
264	pEntry->pPrev = NULL;
265	pList->pHead = pEntry;
266	pdmBlkCacheListAdd(pList, pEntry->cbData);
267	pEntry->pList = pList;
268	#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
269	pdmBlkCacheCheckList(pList, NULL);
270	#endif
271	}
272
273	/**
274	* Destroys a LRU list freeing all entries.
275	*
276	* @returns nothing
277	* @param pList Pointer to the LRU list to destroy.
278	*
279	* @note The caller must own the critical section of the cache.
280	*/
281	static void pdmBlkCacheDestroyList(PPDMBLKLRULIST pList)
282	{
283	while (pList->pHead)
284	{
285	PPDMBLKCACHEENTRY pEntry = pList->pHead;
286
287	pList->pHead = pEntry->pNext;
288
289	AssertMsg(!(pEntry->fFlags & (PDMBLKCACHE_ENTRY_IO_IN_PROGRESS \| PDMBLKCACHE_ENTRY_IS_DIRTY)),
290	("Entry is dirty and/or still in progress fFlags=%#x\n", pEntry->fFlags));
291
292	RTMemPageFree(pEntry->pbData, pEntry->cbData);
293	RTMemFree(pEntry);
294	}
295	}
296
297	/**
298	* Tries to remove the given amount of bytes from a given list in the cache
299	* moving the entries to one of the given ghosts lists
300	*
301	* @returns Amount of data which could be freed.
302	* @param pCache Pointer to the global cache data.
303	* @param cbData The amount of the data to free.
304	* @param pListSrc The source list to evict data from.
305	* @param pGhostListSrc The ghost list removed entries should be moved to
306	* NULL if the entry should be freed.
307	* @param fReuseBuffer Flag whether a buffer should be reused if it has the same size
308	* @param ppbBuf Where to store the address of the buffer if an entry with the
309	* same size was found and fReuseBuffer is true.
310	*
311	* @note This function may return fewer bytes than requested because entries
312	* may be marked as non evictable if they are used for I/O at the
313	* moment.
314	*/
315	static size_t pdmBlkCacheEvictPagesFrom(PPDMBLKCACHEGLOBAL pCache, size_t cbData,
316	PPDMBLKLRULIST pListSrc, PPDMBLKLRULIST pGhostListDst,
317	bool fReuseBuffer, uint8_t **ppbBuffer)
318	{
319	size_t cbEvicted = 0;
320
321	PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
322
323	AssertMsg(cbData > 0, ("Evicting 0 bytes not possible\n"));
324	AssertMsg( !pGhostListDst
325	\|\| (pGhostListDst == &pCache->LruRecentlyUsedOut),
326	("Destination list must be NULL or the recently used but paged out list\n"));
327
328	if (fReuseBuffer)
329	{
330	AssertPtr(ppbBuffer);
331	*ppbBuffer = NULL;
332	}
333
334	/* Start deleting from the tail. */
335	PPDMBLKCACHEENTRY pEntry = pListSrc->pTail;
336
337	while ((cbEvicted < cbData) && pEntry)
338	{
339	PPDMBLKCACHEENTRY pCurr = pEntry;
340
341	pEntry = pEntry->pPrev;
342
343	/* We can't evict pages which are currently in progress or dirty but not in progress */
344	if ( !(pCurr->fFlags & PDMBLKCACHE_NOT_EVICTABLE)
345	&& (ASMAtomicReadU32(&pCurr->cRefs) == 0))
346	{
347	/* Ok eviction candidate. Grab the endpoint semaphore and check again
348	* because somebody else might have raced us. */
349	PPDMBLKCACHE pBlkCache = pCurr->pBlkCache;
350	RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
351
352	if (!(pCurr->fFlags & PDMBLKCACHE_NOT_EVICTABLE)
353	&& (ASMAtomicReadU32(&pCurr->cRefs) == 0))
354	{
355	LogFlow(("Evicting entry %#p (%u bytes)\n", pCurr, pCurr->cbData));
356
357	if (fReuseBuffer && (pCurr->cbData == cbData))
358	{
359	STAM_COUNTER_INC(&pCache->StatBuffersReused);
360	*ppbBuffer = pCurr->pbData;
361	}
362	else if (pCurr->pbData)
363	RTMemPageFree(pCurr->pbData, pCurr->cbData);
364
365	pCurr->pbData = NULL;
366	cbEvicted += pCurr->cbData;
367
368	pdmBlkCacheEntryRemoveFromList(pCurr);
369	pdmBlkCacheSub(pCache, pCurr->cbData);
370
371	if (pGhostListDst)
372	{
373	RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
374
375	PPDMBLKCACHEENTRY pGhostEntFree = pGhostListDst->pTail;
376
377	/* We have to remove the last entries from the paged out list. */
378	while ( ((pGhostListDst->cbCached + pCurr->cbData) > pCache->cbRecentlyUsedOutMax)
379	&& pGhostEntFree)
380	{
381	PPDMBLKCACHEENTRY pFree = pGhostEntFree;
382	PPDMBLKCACHE pBlkCacheFree = pFree->pBlkCache;
383
384	pGhostEntFree = pGhostEntFree->pPrev;
385
386	RTSemRWRequestWrite(pBlkCacheFree->SemRWEntries, RT_INDEFINITE_WAIT);
387
388	if (ASMAtomicReadU32(&pFree->cRefs) == 0)
389	{
390	pdmBlkCacheEntryRemoveFromList(pFree);
391
392	STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
393	RTAvlrU64Remove(pBlkCacheFree->pTree, pFree->Core.Key);
394	STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
395
396	RTMemFree(pFree);
397	}
398
399	RTSemRWReleaseWrite(pBlkCacheFree->SemRWEntries);
400	}
401
402	if (pGhostListDst->cbCached + pCurr->cbData > pCache->cbRecentlyUsedOutMax)
403	{
404	/* Couldn't remove enough entries. Delete */
405	STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
406	RTAvlrU64Remove(pCurr->pBlkCache->pTree, pCurr->Core.Key);
407	STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
408
409	RTMemFree(pCurr);
410	}
411	else
412	pdmBlkCacheEntryAddToList(pGhostListDst, pCurr);
413	}
414	else
415	{
416	/* Delete the entry from the AVL tree it is assigned to. */
417	STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
418	RTAvlrU64Remove(pCurr->pBlkCache->pTree, pCurr->Core.Key);
419	STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
420
421	RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
422	RTMemFree(pCurr);
423	}
424	}
425
426	}
427	else
428	LogFlow(("Entry %#p (%u bytes) is still in progress and can't be evicted\n", pCurr, pCurr->cbData));
429	}
430
431	return cbEvicted;
432	}
433
434	static bool pdmBlkCacheReclaim(PPDMBLKCACHEGLOBAL pCache, size_t cbData, bool fReuseBuffer, uint8_t **ppbBuffer)
435	{
436	size_t cbRemoved = 0;
437
438	if ((pCache->cbCached + cbData) < pCache->cbMax)
439	return true;
440	else if ((pCache->LruRecentlyUsedIn.cbCached + cbData) > pCache->cbRecentlyUsedInMax)
441	{
442	/* Try to evict as many bytes as possible from A1in */
443	cbRemoved = pdmBlkCacheEvictPagesFrom(pCache, cbData, &pCache->LruRecentlyUsedIn,
444	&pCache->LruRecentlyUsedOut, fReuseBuffer, ppbBuffer);
445
446	/*
447	* If it was not possible to remove enough entries
448	* try the frequently accessed cache.
449	*/
450	if (cbRemoved < cbData)
451	{
452	Assert(!fReuseBuffer \|\| !ppbBuffer); / It is not possible that we got a buffer with the correct size but we didn't freed enough data. */
453
454	/*
455	* If we removed something we can't pass the reuse buffer flag anymore because
456	* we don't need to evict that much data
457	*/
458	if (!cbRemoved)
459	cbRemoved += pdmBlkCacheEvictPagesFrom(pCache, cbData, &pCache->LruFrequentlyUsed,
460	NULL, fReuseBuffer, ppbBuffer);
461	else
462	cbRemoved += pdmBlkCacheEvictPagesFrom(pCache, cbData - cbRemoved, &pCache->LruFrequentlyUsed,
463	NULL, false, NULL);
464	}
465	}
466	else
467	{
468	/* We have to remove entries from frequently access list. */
469	cbRemoved = pdmBlkCacheEvictPagesFrom(pCache, cbData, &pCache->LruFrequentlyUsed,
470	NULL, fReuseBuffer, ppbBuffer);
471	}
472
473	LogFlowFunc((": removed %u bytes, requested %u\n", cbRemoved, cbData));
474	return (cbRemoved >= cbData);
475	}
476
477	DECLINLINE(int) pdmBlkCacheEnqueue(PPDMBLKCACHE pBlkCache, uint64_t off, PPDMBLKCACHEIOXFER pIoXfer)
478	{
479	int rc = VINF_SUCCESS;
480
481	switch (pBlkCache->enmType)
482	{
483	case PDMBLKCACHETYPE_DEV:
484	{
485	rc = pBlkCache->u.Dev.pfnXferEnqueue(pBlkCache->u.Dev.pDevIns,
486	pIoXfer->enmXferDir,
487	off, pIoXfer->cbXfer,
488	&pIoXfer->SgBuf, pIoXfer);
489	break;
490	}
491	case PDMBLKCACHETYPE_DRV:
492	{
493	rc = pBlkCache->u.Drv.pfnXferEnqueue(pBlkCache->u.Drv.pDrvIns,
494	pIoXfer->enmXferDir,
495	off, pIoXfer->cbXfer,
496	&pIoXfer->SgBuf, pIoXfer);
497	break;
498	}
499	case PDMBLKCACHETYPE_USB:
500	{
501	rc = pBlkCache->u.Usb.pfnXferEnqueue(pBlkCache->u.Usb.pUsbIns,
502	pIoXfer->enmXferDir,
503	off, pIoXfer->cbXfer,
504	&pIoXfer->SgBuf, pIoXfer);
505	break;
506	}
507	case PDMBLKCACHETYPE_INTERNAL:
508	{
509	rc = pBlkCache->u.Int.pfnXferEnqueue(pBlkCache->u.Int.pvUser,
510	pIoXfer->enmXferDir,
511	off, pIoXfer->cbXfer,
512	&pIoXfer->SgBuf, pIoXfer);
513	break;
514	}
515	default:
516	AssertMsgFailed(("Unknown block cache type!\n"));
517	}
518
519	return rc;
520	}
521
522	/**
523	* Initiates a read I/O task for the given entry.
524	*
525	* @returns VBox status code.
526	* @param pEntry The entry to fetch the data to.
527	*/
528	static int pdmBlkCacheEntryReadFromMedium(PPDMBLKCACHEENTRY pEntry)
529	{
530	PPDMBLKCACHE pBlkCache = pEntry->pBlkCache;
531	LogFlowFunc((": Reading data into cache entry %#p\n", pEntry));
532
533	/* Make sure no one evicts the entry while it is accessed. */
534	pEntry->fFlags \|= PDMBLKCACHE_ENTRY_IO_IN_PROGRESS;
535
536	PPDMBLKCACHEIOXFER pIoXfer = (PPDMBLKCACHEIOXFER)RTMemAllocZ(sizeof(PDMBLKCACHEIOXFER));
537	if (RT_UNLIKELY(!pIoXfer))
538	return VERR_NO_MEMORY;
539
540	AssertMsg(pEntry->pbData, ("Entry is in ghost state\n"));
541
542	pIoXfer->fIoCache = true;
543	pIoXfer->pEntry = pEntry;
544	pIoXfer->SgSeg.pvSeg = pEntry->pbData;
545	pIoXfer->SgSeg.cbSeg = pEntry->cbData;
546	pIoXfer->cbXfer = pEntry->cbData;
547	pIoXfer->enmXferDir = PDMBLKCACHEXFERDIR_READ;
548	RTSgBufInit(&pIoXfer->SgBuf, &pIoXfer->SgSeg, 1);
549
550	return pdmBlkCacheEnqueue(pBlkCache, pEntry->Core.Key, pIoXfer);
551	}
552
553	/**
554	* Initiates a write I/O task for the given entry.
555	*
556	* @returns nothing.
557	* @param pEntry The entry to read the data from.
558	*/
559	static int pdmBlkCacheEntryWriteToMedium(PPDMBLKCACHEENTRY pEntry)
560	{
561	PPDMBLKCACHE pBlkCache = pEntry->pBlkCache;
562	LogFlowFunc((": Writing data from cache entry %#p\n", pEntry));
563
564	/* Make sure no one evicts the entry while it is accessed. */
565	pEntry->fFlags \|= PDMBLKCACHE_ENTRY_IO_IN_PROGRESS;
566
567	PPDMBLKCACHEIOXFER pIoXfer = (PPDMBLKCACHEIOXFER)RTMemAllocZ(sizeof(PDMBLKCACHEIOXFER));
568	if (RT_UNLIKELY(!pIoXfer))
569	return VERR_NO_MEMORY;
570
571	AssertMsg(pEntry->pbData, ("Entry is in ghost state\n"));
572
573	pIoXfer->fIoCache = true;
574	pIoXfer->pEntry = pEntry;
575	pIoXfer->SgSeg.pvSeg = pEntry->pbData;
576	pIoXfer->SgSeg.cbSeg = pEntry->cbData;
577	pIoXfer->cbXfer = pEntry->cbData;
578	pIoXfer->enmXferDir = PDMBLKCACHEXFERDIR_WRITE;
579	RTSgBufInit(&pIoXfer->SgBuf, &pIoXfer->SgSeg, 1);
580
581	return pdmBlkCacheEnqueue(pBlkCache, pEntry->Core.Key, pIoXfer);
582	}
583
584	/**
585	* Passthrough a part of a request directly to the I/O manager
586	* handling the endpoint.
587	*
588	* @returns VBox status code.
589	* @param pEndpoint The endpoint.
590	* @param pTask The task.
591	* @param pIoMemCtx The I/O memory context to use.
592	* @param offStart Offset to start transfer from.
593	* @param cbData Amount of data to transfer.
594	* @param enmTransferType The transfer type (read/write)
595	*/
596	static int pdmBlkCacheRequestPassthrough(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEREQ pReq,
597	PRTSGBUF pSgBuf, uint64_t offStart, size_t cbData,
598	PDMBLKCACHEXFERDIR enmXferDir)
599	{
600
601	PPDMBLKCACHEIOXFER pIoXfer = (PPDMBLKCACHEIOXFER)RTMemAllocZ(sizeof(PDMBLKCACHEIOXFER));
602	if (RT_UNLIKELY(!pIoXfer))
603	return VERR_NO_MEMORY;
604
605	ASMAtomicIncU32(&pReq->cXfersPending);
606	pIoXfer->fIoCache = false;
607	pIoXfer->pReq = pReq;
608	pIoXfer->cbXfer = cbData;
609	pIoXfer->enmXferDir = enmXferDir;
610	RTSgBufClone(&pIoXfer->SgBuf, pSgBuf);
611
612	return pdmBlkCacheEnqueue(pBlkCache, offStart, pIoXfer);
613	}
614
615	/**
616	* Commit a single dirty entry to the endpoint
617	*
618	* @returns nothing
619	* @param pEntry The entry to commit.
620	*/
621	static void pdmBlkCacheEntryCommit(PPDMBLKCACHEENTRY pEntry)
622	{
623	AssertMsg( (pEntry->fFlags & PDMBLKCACHE_ENTRY_IS_DIRTY)
624	&& !(pEntry->fFlags & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS),
625	("Invalid flags set for entry %#p\n", pEntry));
626
627	pdmBlkCacheEntryWriteToMedium(pEntry);
628	}
629
630	/**
631	* Commit all dirty entries for a single endpoint.
632	*
633	* @returns nothing.
634	* @param pBlkCache The endpoint cache to commit.
635	*/
636	static void pdmBlkCacheCommit(PPDMBLKCACHE pBlkCache)
637	{
638	uint32_t cbCommitted = 0;
639	RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
640
641	/* The list is moved to a new header to reduce locking overhead. */
642	RTLISTNODE ListDirtyNotCommitted;
643	RTSPINLOCKTMP Tmp;
644
645	RTListInit(&ListDirtyNotCommitted);
646	RTSpinlockAcquire(pBlkCache->LockList, &Tmp);
647	RTListMove(&ListDirtyNotCommitted, &pBlkCache->ListDirtyNotCommitted);
648	RTSpinlockRelease(pBlkCache->LockList, &Tmp);
649
650	if (!RTListIsEmpty(&ListDirtyNotCommitted))
651	{
652	PPDMBLKCACHEENTRY pEntry = RTListNodeGetFirst(&ListDirtyNotCommitted,
653	PDMBLKCACHEENTRY,
654	NodeNotCommitted);
655
656	while (!RTListNodeIsLast(&ListDirtyNotCommitted, &pEntry->NodeNotCommitted))
657	{
658	PPDMBLKCACHEENTRY pNext = RTListNodeGetNext(&pEntry->NodeNotCommitted, PDMBLKCACHEENTRY,
659	NodeNotCommitted);
660	pdmBlkCacheEntryCommit(pEntry);
661	cbCommitted += pEntry->cbData;
662	RTListNodeRemove(&pEntry->NodeNotCommitted);
663	pEntry = pNext;
664	}
665
666	/* Commit the last endpoint */
667	Assert(RTListNodeIsLast(&ListDirtyNotCommitted, &pEntry->NodeNotCommitted));
668	pdmBlkCacheEntryCommit(pEntry);
669	RTListNodeRemove(&pEntry->NodeNotCommitted);
670	AssertMsg(RTListIsEmpty(&ListDirtyNotCommitted),
671	("Committed all entries but list is not empty\n"));
672	}
673
674	RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
675	AssertMsg(pBlkCache->pCache->cbDirty >= cbCommitted,
676	("Number of committed bytes exceeds number of dirty bytes\n"));
677	ASMAtomicSubU32(&pBlkCache->pCache->cbDirty, cbCommitted);
678	}
679
680	/**
681	* Commit all dirty entries in the cache.
682	*
683	* @returns nothing.
684	* @param pCache The global cache instance.
685	*/
686	static void pdmBlkCacheCommitDirtyEntries(PPDMBLKCACHEGLOBAL pCache)
687	{
688	bool fCommitInProgress = ASMAtomicXchgBool(&pCache->fCommitInProgress, true);
689
690	if (!fCommitInProgress)
691	{
692	pdmBlkCacheLockEnter(pCache);
693	Assert(!RTListIsEmpty(&pCache->ListUsers));
694
695	PPDMBLKCACHE pBlkCache = RTListNodeGetFirst(&pCache->ListUsers,
696	PDMBLKCACHE,
697	NodeCacheUser);
698	AssertPtr(pBlkCache);
699
700	while (!RTListNodeIsLast(&pCache->ListUsers, &pBlkCache->NodeCacheUser))
701	{
702	pdmBlkCacheCommit(pBlkCache);
703
704	pBlkCache = RTListNodeGetNext(&pBlkCache->NodeCacheUser, PDMBLKCACHE,
705	NodeCacheUser);
706	}
707
708	/* Commit the last endpoint */
709	Assert(RTListNodeIsLast(&pCache->ListUsers, &pBlkCache->NodeCacheUser));
710	pdmBlkCacheCommit(pBlkCache);
711
712	pdmBlkCacheLockLeave(pCache);
713	ASMAtomicWriteBool(&pCache->fCommitInProgress, false);
714	}
715	}
716
717	/**
718	* Adds the given entry as a dirty to the cache.
719	*
720	* @returns Flag whether the amount of dirty bytes in the cache exceeds the threshold
721	* @param pBlkCache The endpoint cache the entry belongs to.
722	* @param pEntry The entry to add.
723	*/
724	static bool pdmBlkCacheAddDirtyEntry(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEENTRY pEntry)
725	{
726	bool fDirtyBytesExceeded = false;
727	PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
728
729	/* If the commit timer is disabled we commit right away. */
730	if (pCache->u32CommitTimeoutMs == 0)
731	{
732	pEntry->fFlags \|= PDMBLKCACHE_ENTRY_IS_DIRTY;
733	pdmBlkCacheEntryCommit(pEntry);
734	}
735	else if (!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IS_DIRTY))
736	{
737	pEntry->fFlags \|= PDMBLKCACHE_ENTRY_IS_DIRTY;
738
739	RTSPINLOCKTMP Tmp;
740	RTSpinlockAcquire(pBlkCache->LockList, &Tmp);
741	RTListAppend(&pBlkCache->ListDirtyNotCommitted, &pEntry->NodeNotCommitted);
742	RTSpinlockRelease(pBlkCache->LockList, &Tmp);
743
744	uint32_t cbDirty = ASMAtomicAddU32(&pCache->cbDirty, pEntry->cbData);
745
746	fDirtyBytesExceeded = (cbDirty >= pCache->cbCommitDirtyThreshold);
747	}
748
749	return fDirtyBytesExceeded;
750	}
751
752	/**
753	* Commit timer callback.
754	*/
755	static void pdmBlkCacheCommitTimerCallback(PVM pVM, PTMTIMER pTimer, void *pvUser)
756	{
757	PPDMBLKCACHEGLOBAL pCache = (PPDMBLKCACHEGLOBAL)pvUser;
758
759	LogFlowFunc(("Commit interval expired, commiting dirty entries\n"));
760
761	if (ASMAtomicReadU32(&pCache->cbDirty) > 0)
762	pdmBlkCacheCommitDirtyEntries(pCache);
763
764	TMTimerSetMillies(pTimer, pCache->u32CommitTimeoutMs);
765	LogFlowFunc(("Entries committed, going to sleep\n"));
766	}
767
768	int pdmR3BlkCacheInit(PVM pVM)
769	{
770	int rc = VINF_SUCCESS;
771	PUVM pUVM = pVM->pUVM;
772	PPDMBLKCACHEGLOBAL pBlkCacheGlobal;
773
774	LogFlowFunc((": pVM=%p\n", pVM));
775
776	VM_ASSERT_EMT(pVM);
777
778	PCFGMNODE pCfgRoot = CFGMR3GetRoot(pVM);
779	PCFGMNODE pCfgBlkCache = CFGMR3GetChild(CFGMR3GetChild(pCfgRoot, "PDM"), "BlkCache");
780
781	pBlkCacheGlobal = (PPDMBLKCACHEGLOBAL)RTMemAllocZ(sizeof(PDMBLKCACHEGLOBAL));
782	if (!pBlkCacheGlobal)
783	return VERR_NO_MEMORY;
784
785	RTListInit(&pBlkCacheGlobal->ListUsers);
786	pBlkCacheGlobal->pVM = pVM;
787	pBlkCacheGlobal->cRefs = 0;
788	pBlkCacheGlobal->cbCached = 0;
789	pBlkCacheGlobal->fCommitInProgress = false;
790
791	/* Initialize members */
792	pBlkCacheGlobal->LruRecentlyUsedIn.pHead = NULL;
793	pBlkCacheGlobal->LruRecentlyUsedIn.pTail = NULL;
794	pBlkCacheGlobal->LruRecentlyUsedIn.cbCached = 0;
795
796	pBlkCacheGlobal->LruRecentlyUsedOut.pHead = NULL;
797	pBlkCacheGlobal->LruRecentlyUsedOut.pTail = NULL;
798	pBlkCacheGlobal->LruRecentlyUsedOut.cbCached = 0;
799
800	pBlkCacheGlobal->LruFrequentlyUsed.pHead = NULL;
801	pBlkCacheGlobal->LruFrequentlyUsed.pTail = NULL;
802	pBlkCacheGlobal->LruFrequentlyUsed.cbCached = 0;
803
804	do
805	{
806	rc = CFGMR3QueryU32Def(pCfgBlkCache, "CacheSize", &pBlkCacheGlobal->cbMax, 5 * _1M);
807	AssertLogRelRCBreak(rc);
808	LogFlowFunc(("Maximum number of bytes cached %u\n", pBlkCacheGlobal->cbMax));
809
810	pBlkCacheGlobal->cbRecentlyUsedInMax = (pBlkCacheGlobal->cbMax / 100) * 25; /* 25% of the buffer size */
811	pBlkCacheGlobal->cbRecentlyUsedOutMax = (pBlkCacheGlobal->cbMax / 100) * 50; /* 50% of the buffer size */
812	LogFlowFunc(("cbRecentlyUsedInMax=%u cbRecentlyUsedOutMax=%u\n",
813	pBlkCacheGlobal->cbRecentlyUsedInMax, pBlkCacheGlobal->cbRecentlyUsedOutMax));
814
815	/** @todo r=aeichner: Experiment to find optimal default values */
816	rc = CFGMR3QueryU32Def(pCfgBlkCache, "CacheCommitIntervalMs", &pBlkCacheGlobal->u32CommitTimeoutMs, 10000 /* 10sec */);
817	AssertLogRelRCBreak(rc);
818	rc = CFGMR3QueryU32Def(pCfgBlkCache, "CacheCommitThreshold", &pBlkCacheGlobal->cbCommitDirtyThreshold, pBlkCacheGlobal->cbMax / 2);
819	AssertLogRelRCBreak(rc);
820	} while (0);
821
822	if (RT_SUCCESS(rc))
823	{
824	STAMR3Register(pVM, &pBlkCacheGlobal->cbMax,
825	STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
826	"/PDM/BlkCache/cbMax",
827	STAMUNIT_BYTES,
828	"Maximum cache size");
829	STAMR3Register(pVM, &pBlkCacheGlobal->cbCached,
830	STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
831	"/PDM/BlkCache/cbCached",
832	STAMUNIT_BYTES,
833	"Currently used cache");
834	STAMR3Register(pVM, &pBlkCacheGlobal->LruRecentlyUsedIn.cbCached,
835	STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
836	"/PDM/BlkCache/cbCachedMruIn",
837	STAMUNIT_BYTES,
838	"Number of bytes cached in MRU list");
839	STAMR3Register(pVM, &pBlkCacheGlobal->LruRecentlyUsedOut.cbCached,
840	STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
841	"/PDM/BlkCache/cbCachedMruOut",
842	STAMUNIT_BYTES,
843	"Number of bytes cached in FRU list");
844	STAMR3Register(pVM, &pBlkCacheGlobal->LruFrequentlyUsed.cbCached,
845	STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
846	"/PDM/BlkCache/cbCachedFru",
847	STAMUNIT_BYTES,
848	"Number of bytes cached in FRU ghost list");
849
850	#ifdef VBOX_WITH_STATISTICS
851	STAMR3Register(pVM, &pBlkCacheGlobal->cHits,
852	STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
853	"/PDM/BlkCache/CacheHits",
854	STAMUNIT_COUNT, "Number of hits in the cache");
855	STAMR3Register(pVM, &pBlkCacheGlobal->cPartialHits,
856	STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
857	"/PDM/BlkCache/CachePartialHits",
858	STAMUNIT_COUNT, "Number of partial hits in the cache");
859	STAMR3Register(pVM, &pBlkCacheGlobal->cMisses,
860	STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
861	"/PDM/BlkCache/CacheMisses",
862	STAMUNIT_COUNT, "Number of misses when accessing the cache");
863	STAMR3Register(pVM, &pBlkCacheGlobal->StatRead,
864	STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
865	"/PDM/BlkCache/CacheRead",
866	STAMUNIT_BYTES, "Number of bytes read from the cache");
867	STAMR3Register(pVM, &pBlkCacheGlobal->StatWritten,
868	STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
869	"/PDM/BlkCache/CacheWritten",
870	STAMUNIT_BYTES, "Number of bytes written to the cache");
871	STAMR3Register(pVM, &pBlkCacheGlobal->StatTreeGet,
872	STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
873	"/PDM/BlkCache/CacheTreeGet",
874	STAMUNIT_TICKS_PER_CALL, "Time taken to access an entry in the tree");
875	STAMR3Register(pVM, &pBlkCacheGlobal->StatTreeInsert,
876	STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
877	"/PDM/BlkCache/CacheTreeInsert",
878	STAMUNIT_TICKS_PER_CALL, "Time taken to insert an entry in the tree");
879	STAMR3Register(pVM, &pBlkCacheGlobal->StatTreeRemove,
880	STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
881	"/PDM/BlkCache/CacheTreeRemove",
882	STAMUNIT_TICKS_PER_CALL, "Time taken to remove an entry an the tree");
883	STAMR3Register(pVM, &pBlkCacheGlobal->StatBuffersReused,
884	STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
885	"/PDM/BlkCache/CacheBuffersReused",
886	STAMUNIT_COUNT, "Number of times a buffer could be reused");
887	#endif
888
889	/* Initialize the critical section */
890	rc = RTCritSectInit(&pBlkCacheGlobal->CritSect);
891	}
892
893	if (RT_SUCCESS(rc))
894	{
895	/* Create the commit timer */
896	if (pBlkCacheGlobal->u32CommitTimeoutMs > 0)
897	rc = TMR3TimerCreateInternal(pVM, TMCLOCK_REAL,
898	pdmBlkCacheCommitTimerCallback,
899	pBlkCacheGlobal,
900	"BlkCache-Commit",
901	&pBlkCacheGlobal->pTimerCommit);
902
903	if (RT_SUCCESS(rc))
904	{
905	LogRel(("BlkCache: Cache successfully initialised. Cache size is %u bytes\n", pBlkCacheGlobal->cbMax));
906	LogRel(("BlkCache: Cache commit interval is %u ms\n", pBlkCacheGlobal->u32CommitTimeoutMs));
907	LogRel(("BlkCache: Cache commit threshold is %u bytes\n", pBlkCacheGlobal->cbCommitDirtyThreshold));
908	pUVM->pdm.s.pBlkCacheGlobal = pBlkCacheGlobal;
909	return VINF_SUCCESS;
910	}
911
912	RTCritSectDelete(&pBlkCacheGlobal->CritSect);
913	}
914
915	if (pBlkCacheGlobal)
916	RTMemFree(pBlkCacheGlobal);
917
918	LogFlowFunc((": returns rc=%Rrc\n", pVM, rc));
919	return rc;
920	}
921
922	void pdmR3BlkCacheTerm(PVM pVM)
923	{
924	PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
925
926	if (pBlkCacheGlobal)
927	{
928	/* Make sure no one else uses the cache now */
929	pdmBlkCacheLockEnter(pBlkCacheGlobal);
930
931	/* Cleanup deleting all cache entries waiting for in progress entries to finish. */
932	pdmBlkCacheDestroyList(&pBlkCacheGlobal->LruRecentlyUsedIn);
933	pdmBlkCacheDestroyList(&pBlkCacheGlobal->LruRecentlyUsedOut);
934	pdmBlkCacheDestroyList(&pBlkCacheGlobal->LruFrequentlyUsed);
935
936	pdmBlkCacheLockLeave(pBlkCacheGlobal);
937
938	RTCritSectDelete(&pBlkCacheGlobal->CritSect);
939	RTMemFree(pBlkCacheGlobal);
940	pVM->pUVM->pdm.s.pBlkCacheGlobal = NULL;
941	}
942	}
943
944	static int pdmR3BlkCacheRetain(PVM pVM, PPPDMBLKCACHE ppBlkCache, const char *pcszId)
945	{
946	int rc = VINF_SUCCESS;
947	PPDMBLKCACHE pBlkCache = NULL;
948	bool fAlreadyExists = false;
949	PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
950
951	if (!pBlkCacheGlobal)
952	return VERR_NOT_SUPPORTED;
953
954	/*
955	* Check that no other user cache has the same id first,
956	* Unique id's are necessary in case the state is saved.
957	*/
958	pdmBlkCacheLockEnter(pBlkCacheGlobal);
959
960	RTListForEach(&pBlkCacheGlobal->ListUsers, pBlkCache, PDMBLKCACHE, NodeCacheUser)
961	{
962	if (!RTStrCmp(pBlkCache->pszId, pcszId))
963	{
964	fAlreadyExists = true;
965	break;
966	}
967	}
968
969	if (!fAlreadyExists)
970	{
971	pBlkCache = (PPDMBLKCACHE)RTMemAllocZ(sizeof(PDMBLKCACHE));
972
973	if (pBlkCache)
974	pBlkCache->pszId = RTStrDup(pcszId);
975
976	if ( pBlkCache
977	&& pBlkCache->pszId)
978	{
979	pBlkCache->pCache = pBlkCacheGlobal;
980	RTListInit(&pBlkCache->ListDirtyNotCommitted);
981
982	rc = RTSpinlockCreate(&pBlkCache->LockList);
983	if (RT_SUCCESS(rc))
984	{
985	rc = RTSemRWCreate(&pBlkCache->SemRWEntries);
986	if (RT_SUCCESS(rc))
987	{
988	pBlkCache->pTree = (PAVLRU64TREE)RTMemAllocZ(sizeof(AVLRFOFFTREE));
989	if (pBlkCache->pTree)
990	{
991	/* Arm the timer if this is the first endpoint. */
992	if ( pBlkCacheGlobal->cRefs == 1
993	&& pBlkCacheGlobal->u32CommitTimeoutMs > 0)
994	rc = TMTimerSetMillies(pBlkCacheGlobal->pTimerCommit, pBlkCacheGlobal->u32CommitTimeoutMs);
995
996	if (RT_SUCCESS(rc))
997	{
998	#ifdef VBOX_WITH_STATISTICS
999	STAMR3RegisterF(pBlkCacheGlobal->pVM, &pBlkCache->StatWriteDeferred,
1000	STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
1001	STAMUNIT_COUNT, "Number of deferred writes",
1002	"/PDM/BlkCache/%s/Cache/DeferredWrites", pBlkCache->pszId);
1003	#endif
1004
1005	/* Add to the list of users. */
1006	pBlkCacheGlobal->cRefs++;
1007	RTListAppend(&pBlkCacheGlobal->ListUsers, &pBlkCache->NodeCacheUser);
1008	pdmBlkCacheLockLeave(pBlkCacheGlobal);
1009
1010	*ppBlkCache = pBlkCache;
1011	LogFlowFunc(("returns success\n"));
1012	return VINF_SUCCESS;
1013	}
1014	}
1015	else
1016	rc = VERR_NO_MEMORY;
1017
1018	RTSemRWDestroy(pBlkCache->SemRWEntries);
1019	}
1020
1021	RTSpinlockDestroy(pBlkCache->LockList);
1022	}
1023
1024	RTStrFree(pBlkCache->pszId);
1025	}
1026	else
1027	rc = VERR_NO_MEMORY;
1028
1029	if (pBlkCache)
1030	RTMemFree(pBlkCache);
1031	}
1032	else
1033	rc = VERR_ALREADY_EXISTS;
1034
1035	pdmBlkCacheLockLeave(pBlkCacheGlobal);
1036
1037	LogFlowFunc(("Leave rc=%Rrc\n", rc));
1038	return rc;
1039	}
1040
1041	VMMR3DECL(int) PDMR3BlkCacheRetainDriver(PVM pVM, PPDMDRVINS pDrvIns, PPPDMBLKCACHE ppBlkCache,
1042	PFNPDMBLKCACHEXFERCOMPLETEDRV pfnXferComplete,
1043	PFNPDMBLKCACHEXFERENQUEUEDRV pfnXferEnqueue,
1044	const char *pcszId)
1045	{
1046	int rc = VINF_SUCCESS;
1047	PPDMBLKCACHE pBlkCache;
1048
1049	rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
1050	if (RT_SUCCESS(rc))
1051	{
1052	pBlkCache->enmType = PDMBLKCACHETYPE_DRV;
1053	pBlkCache->u.Drv.pfnXferComplete = pfnXferComplete;
1054	pBlkCache->u.Drv.pfnXferEnqueue = pfnXferEnqueue;
1055	pBlkCache->u.Drv.pDrvIns = pDrvIns;
1056	*ppBlkCache = pBlkCache;
1057	}
1058
1059	LogFlowFunc(("Leave rc=%Rrc\n", rc));
1060	return rc;
1061	}
1062
1063	VMMR3DECL(int) PDMR3BlkCacheRetainDevice(PVM pVM, PPDMDEVINS pDevIns, PPPDMBLKCACHE ppBlkCache,
1064	PFNPDMBLKCACHEXFERCOMPLETEDEV pfnXferComplete,
1065	PFNPDMBLKCACHEXFERENQUEUEDEV pfnXferEnqueue,
1066	const char *pcszId)
1067	{
1068	int rc = VINF_SUCCESS;
1069	PPDMBLKCACHE pBlkCache;
1070
1071	rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
1072	if (RT_SUCCESS(rc))
1073	{
1074	pBlkCache->enmType = PDMBLKCACHETYPE_DEV;
1075	pBlkCache->u.Dev.pfnXferComplete = pfnXferComplete;
1076	pBlkCache->u.Dev.pfnXferEnqueue = pfnXferEnqueue;
1077	pBlkCache->u.Dev.pDevIns = pDevIns;
1078	*ppBlkCache = pBlkCache;
1079	}
1080
1081	LogFlowFunc(("Leave rc=%Rrc\n", rc));
1082	return rc;
1083
1084	}
1085
1086	VMMR3DECL(int) PDMR3BlkCacheRetainUsb(PVM pVM, PPDMUSBINS pUsbIns, PPPDMBLKCACHE ppBlkCache,
1087	PFNPDMBLKCACHEXFERCOMPLETEUSB pfnXferComplete,
1088	PFNPDMBLKCACHEXFERENQUEUEUSB pfnXferEnqueue,
1089	const char *pcszId)
1090	{
1091	int rc = VINF_SUCCESS;
1092	PPDMBLKCACHE pBlkCache;
1093
1094	rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
1095	if (RT_SUCCESS(rc))
1096	{
1097	pBlkCache->enmType = PDMBLKCACHETYPE_USB;
1098	pBlkCache->u.Usb.pfnXferComplete = pfnXferComplete;
1099	pBlkCache->u.Usb.pfnXferEnqueue = pfnXferEnqueue;
1100	pBlkCache->u.Usb.pUsbIns = pUsbIns;
1101	*ppBlkCache = pBlkCache;
1102	}
1103
1104	LogFlowFunc(("Leave rc=%Rrc\n", rc));
1105	return rc;
1106
1107	}
1108
1109	VMMR3DECL(int) PDMR3BlkCacheRetainInt(PVM pVM, void *pvUser, PPPDMBLKCACHE ppBlkCache,
1110	PFNPDMBLKCACHEXFERCOMPLETEINT pfnXferComplete,
1111	PFNPDMBLKCACHEXFERENQUEUEINT pfnXferEnqueue,
1112	const char *pcszId)
1113	{
1114	int rc = VINF_SUCCESS;
1115	PPDMBLKCACHE pBlkCache;
1116
1117	rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
1118	if (RT_SUCCESS(rc))
1119	{
1120	pBlkCache->enmType = PDMBLKCACHETYPE_INTERNAL;
1121	pBlkCache->u.Int.pfnXferComplete = pfnXferComplete;
1122	pBlkCache->u.Int.pfnXferEnqueue = pfnXferEnqueue;
1123	pBlkCache->u.Int.pvUser = pvUser;
1124	*ppBlkCache = pBlkCache;
1125	}
1126
1127	LogFlowFunc(("Leave rc=%Rrc\n", rc));
1128	return rc;
1129
1130	}
1131
1132	/**
1133	* Callback for the AVL destroy routine. Frees a cache entry for this endpoint.
1134	*
1135	* @returns IPRT status code.
1136	* @param pNode The node to destroy.
1137	* @param pvUser Opaque user data.
1138	*/
1139	static int pdmBlkCacheEntryDestroy(PAVLRU64NODECORE pNode, void *pvUser)
1140	{
1141	PPDMBLKCACHEENTRY pEntry = (PPDMBLKCACHEENTRY)pNode;
1142	PPDMBLKCACHEGLOBAL pCache = (PPDMBLKCACHEGLOBAL)pvUser;
1143	PPDMBLKCACHE pBlkCache = pEntry->pBlkCache;
1144
1145	while (ASMAtomicReadU32(&pEntry->fFlags) & (PDMBLKCACHE_ENTRY_IO_IN_PROGRESS \| PDMBLKCACHE_ENTRY_IS_DIRTY))
1146	{
1147	/* Leave the locks to let the I/O thread make progress but reference the entry to prevent eviction. */
1148	pdmBlkCacheEntryRef(pEntry);
1149	RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
1150	pdmBlkCacheLockLeave(pCache);
1151
1152	RTThreadSleep(250);
1153
1154	/* Re-enter all locks */
1155	pdmBlkCacheLockEnter(pCache);
1156	RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
1157	pdmBlkCacheEntryRelease(pEntry);
1158	}
1159
1160	AssertMsg(!(pEntry->fFlags & (PDMBLKCACHE_ENTRY_IO_IN_PROGRESS \| PDMBLKCACHE_ENTRY_IS_DIRTY)),
1161	("Entry is dirty and/or still in progress fFlags=%#x\n", pEntry->fFlags));
1162
1163	bool fUpdateCache = pEntry->pList == &pCache->LruFrequentlyUsed
1164	\|\| pEntry->pList == &pCache->LruRecentlyUsedIn;
1165
1166	pdmBlkCacheEntryRemoveFromList(pEntry);
1167
1168	if (fUpdateCache)
1169	pdmBlkCacheSub(pCache, pEntry->cbData);
1170
1171	RTMemPageFree(pEntry->pbData, pEntry->cbData);
1172	RTMemFree(pEntry);
1173
1174	return VINF_SUCCESS;
1175	}
1176
1177	/**
1178	* Destroys all cache resources used by the given endpoint.
1179	*
1180	* @returns nothing.
1181	* @param pEndpoint The endpoint to the destroy.
1182	*/
1183	VMMR3DECL(void) PDMR3BlkCacheRelease(PPDMBLKCACHE pBlkCache)
1184	{
1185	PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
1186
1187	/* Make sure nobody is accessing the cache while we delete the tree. */
1188	pdmBlkCacheLockEnter(pCache);
1189	RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
1190	RTAvlrU64Destroy(pBlkCache->pTree, pdmBlkCacheEntryDestroy, pCache);
1191	RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
1192
1193	RTSpinlockDestroy(pBlkCache->LockList);
1194
1195	pCache->cRefs--;
1196	RTListNodeRemove(&pBlkCache->NodeCacheUser);
1197
1198	if ( !pCache->cRefs
1199	&& pCache->u32CommitTimeoutMs > 0)
1200	TMTimerStop(pCache->pTimerCommit);
1201
1202	pdmBlkCacheLockLeave(pCache);
1203
1204	RTSemRWDestroy(pBlkCache->SemRWEntries);
1205
1206	#ifdef VBOX_WITH_STATISTICS
1207	STAMR3Deregister(pCache->pVM, &pBlkCache->StatWriteDeferred);
1208	#endif
1209
1210	RTStrFree(pBlkCache->pszId);
1211	RTMemFree(pBlkCache);
1212	}
1213
1214	VMMR3DECL(void) PDMR3BlkCacheReleaseDevice(PVM pVM, PPDMDEVINS pDevIns)
1215	{
1216	LogFlow(("%s: pDevIns=%p\n", __FUNCTION__, pDevIns));
1217
1218	/*
1219	* Validate input.
1220	*/
1221	if (!pDevIns)
1222	return;
1223	VM_ASSERT_EMT(pVM);
1224
1225	PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
1226	PPDMBLKCACHE pBlkCache, pBlkCacheNext;
1227
1228	/* Return silently if not supported. */
1229	if (!pBlkCacheGlobal)
1230	return;
1231
1232	pdmBlkCacheLockEnter(pBlkCacheGlobal);
1233
1234	RTListForEachSafe(&pBlkCacheGlobal->ListUsers, pBlkCache, pBlkCacheNext, PDMBLKCACHE, NodeCacheUser)
1235	{
1236	if ( pBlkCache->enmType == PDMBLKCACHETYPE_DEV
1237	&& pBlkCache->u.Dev.pDevIns == pDevIns)
1238	PDMR3BlkCacheRelease(pBlkCache);
1239	}
1240
1241	pdmBlkCacheLockLeave(pBlkCacheGlobal);
1242	}
1243
1244	VMMR3DECL(void) PDMR3BlkCacheReleaseDriver(PVM pVM, PPDMDRVINS pDrvIns)
1245	{
1246	LogFlow(("%s: pDrvIns=%p\n", __FUNCTION__, pDrvIns));
1247
1248	/*
1249	* Validate input.
1250	*/
1251	if (!pDrvIns)
1252	return;
1253	VM_ASSERT_EMT(pVM);
1254
1255	PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
1256	PPDMBLKCACHE pBlkCache, pBlkCacheNext;
1257
1258	/* Return silently if not supported. */
1259	if (!pBlkCacheGlobal)
1260	return;
1261
1262	pdmBlkCacheLockEnter(pBlkCacheGlobal);
1263
1264	RTListForEachSafe(&pBlkCacheGlobal->ListUsers, pBlkCache, pBlkCacheNext, PDMBLKCACHE, NodeCacheUser)
1265	{
1266	if ( pBlkCache->enmType == PDMBLKCACHETYPE_DRV
1267	&& pBlkCache->u.Drv.pDrvIns == pDrvIns)
1268	PDMR3BlkCacheRelease(pBlkCache);
1269	}
1270
1271	pdmBlkCacheLockLeave(pBlkCacheGlobal);
1272	}
1273
1274	VMMR3DECL(void) PDMR3BlkCacheReleaseUsb(PVM pVM, PPDMUSBINS pUsbIns)
1275	{
1276	LogFlow(("%s: pUsbIns=%p\n", __FUNCTION__, pUsbIns));
1277
1278	/*
1279	* Validate input.
1280	*/
1281	if (!pUsbIns)
1282	return;
1283	VM_ASSERT_EMT(pVM);
1284
1285	PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
1286	PPDMBLKCACHE pBlkCache, pBlkCacheNext;
1287
1288	/* Return silently if not supported. */
1289	if (!pBlkCacheGlobal)
1290	return;
1291
1292	pdmBlkCacheLockEnter(pBlkCacheGlobal);
1293
1294	RTListForEachSafe(&pBlkCacheGlobal->ListUsers, pBlkCache, pBlkCacheNext, PDMBLKCACHE, NodeCacheUser)
1295	{
1296	if ( pBlkCache->enmType == PDMBLKCACHETYPE_USB
1297	&& pBlkCache->u.Usb.pUsbIns == pUsbIns)
1298	PDMR3BlkCacheRelease(pBlkCache);
1299	}
1300
1301	pdmBlkCacheLockLeave(pBlkCacheGlobal);
1302	}
1303
1304	static PPDMBLKCACHEENTRY pdmBlkCacheGetCacheEntryByOffset(PPDMBLKCACHE pBlkCache, uint64_t off)
1305	{
1306	PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
1307	PPDMBLKCACHEENTRY pEntry = NULL;
1308
1309	STAM_PROFILE_ADV_START(&pCache->StatTreeGet, Cache);
1310
1311	RTSemRWRequestRead(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
1312	pEntry = (PPDMBLKCACHEENTRY)RTAvlrU64RangeGet(pBlkCache->pTree, off);
1313	if (pEntry)
1314	pdmBlkCacheEntryRef(pEntry);
1315	RTSemRWReleaseRead(pBlkCache->SemRWEntries);
1316
1317	STAM_PROFILE_ADV_STOP(&pCache->StatTreeGet, Cache);
1318
1319	return pEntry;
1320	}
1321
1322	/**
1323	* Return the best fit cache entries for the given offset.
1324	*
1325	* @returns nothing.
1326	* @param pBlkCache The endpoint cache.
1327	* @param off The offset.
1328	* @param pEntryAbove Where to store the pointer to the best fit entry above the
1329	* the given offset. NULL if not required.
1330	* @param pEntryBelow Where to store the pointer to the best fit entry below the
1331	* the given offset. NULL if not required.
1332	*/
1333	static void pdmBlkCacheGetCacheBestFitEntryByOffset(PPDMBLKCACHE pBlkCache, uint64_t off,
1334	PPDMBLKCACHEENTRY *ppEntryAbove,
1335	PPDMBLKCACHEENTRY *ppEntryBelow)
1336	{
1337	PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
1338
1339	STAM_PROFILE_ADV_START(&pCache->StatTreeGet, Cache);
1340
1341	RTSemRWRequestRead(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
1342	if (ppEntryAbove)
1343	{
1344	ppEntryAbove = (PPDMBLKCACHEENTRY)RTAvlrU64GetBestFit(pBlkCache->pTree, off, true /fAbove*/);
1345	if (*ppEntryAbove)
1346	pdmBlkCacheEntryRef(*ppEntryAbove);
1347	}
1348
1349	if (ppEntryBelow)
1350	{
1351	ppEntryBelow = (PPDMBLKCACHEENTRY)RTAvlrU64GetBestFit(pBlkCache->pTree, off, false /fAbove*/);
1352	if (*ppEntryBelow)
1353	pdmBlkCacheEntryRef(*ppEntryBelow);
1354	}
1355	RTSemRWReleaseRead(pBlkCache->SemRWEntries);
1356
1357	STAM_PROFILE_ADV_STOP(&pCache->StatTreeGet, Cache);
1358	}
1359
1360	static void pdmBlkCacheInsertEntry(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEENTRY pEntry)
1361	{
1362	PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
1363
1364	STAM_PROFILE_ADV_START(&pCache->StatTreeInsert, Cache);
1365	RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
1366	bool fInserted = RTAvlrU64Insert(pBlkCache->pTree, &pEntry->Core);
1367	AssertMsg(fInserted, ("Node was not inserted into tree\n"));
1368	STAM_PROFILE_ADV_STOP(&pCache->StatTreeInsert, Cache);
1369	RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
1370	}
1371
1372	/**
1373	* Allocates and initializes a new entry for the cache.
1374	* The entry has a reference count of 1.
1375	*
1376	* @returns Pointer to the new cache entry or NULL if out of memory.
1377	* @param pBlkCache The cache the entry belongs to.
1378	* @param off Start offset.
1379	* @param cbData Size of the cache entry.
1380	* @param pbBuffer Pointer to the buffer to use.
1381	* NULL if a new buffer should be allocated.
1382	* The buffer needs to have the same size of the entry.
1383	*/
1384	static PPDMBLKCACHEENTRY pdmBlkCacheEntryAlloc(PPDMBLKCACHE pBlkCache,
1385	uint64_t off, size_t cbData, uint8_t *pbBuffer)
1386	{
1387	PPDMBLKCACHEENTRY pEntryNew = (PPDMBLKCACHEENTRY)RTMemAllocZ(sizeof(PDMBLKCACHEENTRY));
1388
1389	if (RT_UNLIKELY(!pEntryNew))
1390	return NULL;
1391
1392	pEntryNew->Core.Key = off;
1393	pEntryNew->Core.KeyLast = off + cbData - 1;
1394	pEntryNew->pBlkCache = pBlkCache;
1395	pEntryNew->fFlags = 0;
1396	pEntryNew->cRefs = 1; /* We are using it now. */
1397	pEntryNew->pList = NULL;
1398	pEntryNew->cbData = cbData;
1399	pEntryNew->pWaitingHead = NULL;
1400	pEntryNew->pWaitingTail = NULL;
1401	if (pbBuffer)
1402	pEntryNew->pbData = pbBuffer;
1403	else
1404	pEntryNew->pbData = (uint8_t *)RTMemPageAlloc(cbData);
1405
1406	if (RT_UNLIKELY(!pEntryNew->pbData))
1407	{
1408	RTMemFree(pEntryNew);
1409	return NULL;
1410	}
1411
1412	return pEntryNew;
1413	}
1414
1415	/**
1416	* Checks that a set of flags is set/clear acquiring the R/W semaphore
1417	* in exclusive mode.
1418	*
1419	* @returns true if the flag in fSet is set and the one in fClear is clear.
1420	* false otherwise.
1421	* The R/W semaphore is only held if true is returned.
1422	*
1423	* @param pBlkCache The endpoint cache instance data.
1424	* @param pEntry The entry to check the flags for.
1425	* @param fSet The flag which is tested to be set.
1426	* @param fClear The flag which is tested to be clear.
1427	*/
1428	DECLINLINE(bool) pdmBlkCacheEntryFlagIsSetClearAcquireLock(PPDMBLKCACHE pBlkCache,
1429	PPDMBLKCACHEENTRY pEntry,
1430	uint32_t fSet, uint32_t fClear)
1431	{
1432	uint32_t fFlags = ASMAtomicReadU32(&pEntry->fFlags);
1433	bool fPassed = ((fFlags & fSet) && !(fFlags & fClear));
1434
1435	if (fPassed)
1436	{
1437	/* Acquire the lock and check again because the completion callback might have raced us. */
1438	RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
1439
1440	fFlags = ASMAtomicReadU32(&pEntry->fFlags);
1441	fPassed = ((fFlags & fSet) && !(fFlags & fClear));
1442
1443	/* Drop the lock if we didn't passed the test. */
1444	if (!fPassed)
1445	RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
1446	}
1447
1448	return fPassed;
1449	}
1450
1451	/**
1452	* Adds a segment to the waiting list for a cache entry
1453	* which is currently in progress.
1454	*
1455	* @returns nothing.
1456	* @param pEntry The cache entry to add the segment to.
1457	* @param pSeg The segment to add.
1458	*/
1459	DECLINLINE(void) pdmBlkCacheEntryAddWaiter(PPDMBLKCACHEENTRY pEntry,
1460	PPDMBLKCACHEWAITER pWaiter)
1461	{
1462	pWaiter->pNext = NULL;
1463
1464	if (pEntry->pWaitingHead)
1465	{
1466	AssertPtr(pEntry->pWaitingTail);
1467
1468	pEntry->pWaitingTail->pNext = pWaiter;
1469	pEntry->pWaitingTail = pWaiter;
1470	}
1471	else
1472	{
1473	Assert(!pEntry->pWaitingTail);
1474
1475	pEntry->pWaitingHead = pWaiter;
1476	pEntry->pWaitingTail = pWaiter;
1477	}
1478	}
1479
1480	/**
1481	* Add a buffer described by the I/O memory context
1482	* to the entry waiting for completion.
1483	*
1484	* @returns VBox status code.
1485	* @param pEntry The entry to add the buffer to.
1486	* @param pTask Task associated with the buffer.
1487	* @param pIoMemCtx The memory context to use.
1488	* @param offDiff Offset from the start of the buffer
1489	* in the entry.
1490	* @param cbData Amount of data to wait for onthis entry.
1491	* @param fWrite Flag whether the task waits because it wants to write
1492	* to the cache entry.
1493	*/
1494	static int pdmBlkCacheEntryWaitersAdd(PPDMBLKCACHEENTRY pEntry,
1495	PPDMBLKCACHEREQ pReq,
1496	PCRTSGBUF pSgBuf, uint64_t offDiff,
1497	size_t cbData, bool fWrite)
1498	{
1499	PPDMBLKCACHEWAITER pWaiter = (PPDMBLKCACHEWAITER)RTMemAllocZ(sizeof(PDMBLKCACHEWAITER));
1500	if (!pWaiter)
1501	return VERR_NO_MEMORY;
1502
1503	ASMAtomicIncU32(&pReq->cXfersPending);
1504	pWaiter->pReq = pReq;
1505	pWaiter->offCacheEntry = offDiff;
1506	pWaiter->cbTransfer = cbData;
1507	pWaiter->fWrite = fWrite;
1508	RTSgBufClone(&pWaiter->SgBuf, pSgBuf);
1509
1510	pdmBlkCacheEntryAddWaiter(pEntry, pWaiter);
1511
1512	return VINF_SUCCESS;
1513	}
1514
1515	/**
1516	* Calculate aligned offset and size for a new cache entry
1517	* which do not intersect with an already existing entry and the
1518	* file end.
1519	*
1520	* @returns The number of bytes the entry can hold of the requested amount
1521	* of byte.
1522	* @param pEndpoint The endpoint.
1523	* @param pBlkCache The endpoint cache.
1524	* @param off The start offset.
1525	* @param cb The number of bytes the entry needs to hold at least.
1526	* @param uAlignment Alignment of the boundary sizes.
1527	* @param poffAligned Where to store the aligned offset.
1528	* @param pcbAligned Where to store the aligned size of the entry.
1529	*/
1530	static size_t pdmBlkCacheEntryBoundariesCalc(PPDMBLKCACHE pBlkCache,
1531	uint64_t off, size_t cb,
1532	unsigned uAlignment,
1533	uint64_t poffAligned, size_t pcbAligned)
1534	{
1535	size_t cbAligned;
1536	size_t cbInEntry = 0;
1537	uint64_t offAligned;
1538	PPDMBLKCACHEENTRY pEntryAbove = NULL;
1539	PPDMBLKCACHEENTRY pEntryBelow = NULL;
1540
1541	/* Get the best fit entries around the offset */
1542	pdmBlkCacheGetCacheBestFitEntryByOffset(pBlkCache, off, &pEntryAbove, &pEntryBelow);
1543
1544	/* Log the info */
1545	LogFlow(("%sest fit entry below off=%llu (BestFit=%llu BestFitEnd=%llu BestFitSize=%u)\n",
1546	pEntryBelow ? "B" : "No b",
1547	off,
1548	pEntryBelow ? pEntryBelow->Core.Key : 0,
1549	pEntryBelow ? pEntryBelow->Core.KeyLast : 0,
1550	pEntryBelow ? pEntryBelow->cbData : 0));
1551
1552	LogFlow(("%sest fit entry above off=%llu (BestFit=%llu BestFitEnd=%llu BestFitSize=%u)\n",
1553	pEntryAbove ? "B" : "No b",
1554	off,
1555	pEntryAbove ? pEntryAbove->Core.Key : 0,
1556	pEntryAbove ? pEntryAbove->Core.KeyLast : 0,
1557	pEntryAbove ? pEntryAbove->cbData : 0));
1558
1559	/* Align the offset first. */
1560	offAligned = off & ~(uint64_t)(512-1);
1561	if ( pEntryBelow
1562	&& offAligned <= pEntryBelow->Core.KeyLast)
1563	offAligned = pEntryBelow->Core.KeyLast;
1564
1565	if ( pEntryAbove
1566	&& off + cb > pEntryAbove->Core.Key)
1567	{
1568	cbInEntry = pEntryAbove->Core.Key - off;
1569	cbAligned = pEntryAbove->Core.Key - offAligned;
1570	}
1571	else
1572	{
1573	cbAligned = cb;
1574	cbInEntry = cb;
1575	#if 0
1576	/*
1577	* Align the size to a 4KB boundary.
1578	* Memory size is aligned to a page boundary
1579	* and memory is wasted if the size is rather small.
1580	* (For example reads with a size of 512 bytes).
1581	*/
1582	cbInEntry = cb;
1583	cbAligned = RT_ALIGN_Z(cb + (off - offAligned), uAlignment);
1584
1585	/*
1586	* Clip to file size if the original request doesn't
1587	* exceed the file (not an appending write)
1588	*/
1589	uint64_t cbReq = off + cb;
1590	if (cbReq >= pEndpoint->cbFile)
1591	cbAligned = cbReq - offAligned;
1592	else
1593	cbAligned = RT_MIN(pEndpoint->cbFile - offAligned, cbAligned);
1594	if (pEntryAbove)
1595	{
1596	Assert(pEntryAbove->Core.Key >= off);
1597	cbAligned = RT_MIN(cbAligned, pEntryAbove->Core.Key - offAligned);
1598	}
1599	#endif
1600	}
1601
1602	/* A few sanity checks */
1603	AssertMsg(!pEntryBelow \|\| pEntryBelow->Core.KeyLast < offAligned,
1604	("Aligned start offset intersects with another cache entry\n"));
1605	AssertMsg(!pEntryAbove \|\| (offAligned + cbAligned) <= pEntryAbove->Core.Key,
1606	("Aligned size intersects with another cache entry\n"));
1607	Assert(cbInEntry <= cbAligned);
1608
1609	if (pEntryBelow)
1610	pdmBlkCacheEntryRelease(pEntryBelow);
1611	if (pEntryAbove)
1612	pdmBlkCacheEntryRelease(pEntryAbove);
1613
1614	LogFlow(("offAligned=%llu cbAligned=%u\n", offAligned, cbAligned));
1615
1616	*poffAligned = offAligned;
1617	*pcbAligned = cbAligned;
1618
1619	return cbInEntry;
1620	}
1621
1622	/**
1623	* Create a new cache entry evicting data from the cache if required.
1624	*
1625	* @returns Pointer to the new cache entry or NULL
1626	* if not enough bytes could be evicted from the cache.
1627	* @param pEndpoint The endpoint.
1628	* @param pBlkCache The endpoint cache.
1629	* @param off The offset.
1630	* @param cb Number of bytes the cache entry should have.
1631	* @param uAlignment Alignment the size of the entry should have.
1632	* @param pcbData Where to store the number of bytes the new
1633	* entry can hold. May be lower than actually requested
1634	* due to another entry intersecting the access range.
1635	*/
1636	static PPDMBLKCACHEENTRY pdmBlkCacheEntryCreate(PPDMBLKCACHE pBlkCache,
1637	uint64_t off, size_t cb,
1638	unsigned uAlignment,
1639	size_t *pcbData)
1640	{
1641	uint64_t offStart = 0;
1642	size_t cbEntry = 0;
1643	PPDMBLKCACHEENTRY pEntryNew = NULL;
1644	PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
1645	uint8_t *pbBuffer = NULL;
1646
1647	*pcbData = pdmBlkCacheEntryBoundariesCalc(pBlkCache, off, cb, uAlignment,
1648	&offStart, &cbEntry);
1649
1650	pdmBlkCacheLockEnter(pCache);
1651	bool fEnough = pdmBlkCacheReclaim(pCache, cbEntry, true, &pbBuffer);
1652
1653	if (fEnough)
1654	{
1655	LogFlow(("Evicted enough bytes (%u requested). Creating new cache entry\n", cbEntry));
1656
1657	pEntryNew = pdmBlkCacheEntryAlloc(pBlkCache, offStart, cbEntry, pbBuffer);
1658	if (RT_LIKELY(pEntryNew))
1659	{
1660	pdmBlkCacheEntryAddToList(&pCache->LruRecentlyUsedIn, pEntryNew);
1661	pdmBlkCacheAdd(pCache, cbEntry);
1662	pdmBlkCacheLockLeave(pCache);
1663
1664	pdmBlkCacheInsertEntry(pBlkCache, pEntryNew);
1665
1666	AssertMsg( (off >= pEntryNew->Core.Key)
1667	&& (off + *pcbData <= pEntryNew->Core.KeyLast + 1),
1668	("Overflow in calculation off=%llu OffsetAligned=%llu\n",
1669	off, pEntryNew->Core.Key));
1670	}
1671	else
1672	pdmBlkCacheLockLeave(pCache);
1673	}
1674	else
1675	pdmBlkCacheLockLeave(pCache);
1676
1677	return pEntryNew;
1678	}
1679
1680	static PPDMBLKCACHEREQ pdmBlkCacheReqAlloc(size_t cbXfer, void *pvUser)
1681	{
1682	PPDMBLKCACHEREQ pReq = (PPDMBLKCACHEREQ)RTMemAlloc(sizeof(PDMBLKCACHEREQ));
1683
1684	if (RT_LIKELY(pReq))
1685	{
1686	pReq->pvUser = pvUser;
1687	pReq->cbXfer = cbXfer;
1688	pReq->rcReq = VINF_SUCCESS;
1689	pReq->cXfersPending = 0;
1690	}
1691
1692	return pReq;
1693	}
1694
1695	static void pdmBlkCacheReqComplete(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEREQ pReq)
1696	{
1697	switch (pBlkCache->enmType)
1698	{
1699	case PDMBLKCACHETYPE_DEV:
1700	{
1701	pBlkCache->u.Dev.pfnXferComplete(pBlkCache->u.Dev.pDevIns,
1702	pReq->pvUser, pReq->rcReq);
1703	break;
1704	}
1705	case PDMBLKCACHETYPE_DRV:
1706	{
1707	pBlkCache->u.Drv.pfnXferComplete(pBlkCache->u.Drv.pDrvIns,
1708	pReq->pvUser, pReq->rcReq);
1709	break;
1710	}
1711	case PDMBLKCACHETYPE_USB:
1712	{
1713	pBlkCache->u.Usb.pfnXferComplete(pBlkCache->u.Usb.pUsbIns,
1714	pReq->pvUser, pReq->rcReq);
1715	break;
1716	}
1717	case PDMBLKCACHETYPE_INTERNAL:
1718	{
1719	pBlkCache->u.Int.pfnXferComplete(pBlkCache->u.Int.pvUser,
1720	pReq->pvUser, pReq->rcReq);
1721	break;
1722	}
1723	default:
1724	AssertMsgFailed(("Unknown block cache type!\n"));
1725	}
1726
1727	RTMemFree(pReq);
1728	}
1729
1730	static bool pdmBlkCacheReqUpdate(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEREQ pReq,
1731	size_t cbComplete, int rcReq, bool fCallHandler)
1732	{
1733	if (RT_FAILURE(rcReq))
1734	ASMAtomicCmpXchgS32(&pReq->rcReq, rcReq, VINF_SUCCESS);
1735
1736	AssertMsg(pReq->cbXfer >= cbComplete, ("Completed more than left\n"));
1737	AssertMsg(pReq->cXfersPending > 0, ("No transfers are pending for this request\n"));
1738	uint32_t cbOld = ASMAtomicSubU32(&pReq->cbXfer, cbComplete);
1739	uint32_t cXfersPending = ASMAtomicDecU32(&pReq->cXfersPending);
1740
1741	if ( !(cbOld - cbComplete)
1742	&& !cXfersPending)
1743	{
1744	if (fCallHandler)
1745	pdmBlkCacheReqComplete(pBlkCache, pReq);
1746	return true;
1747	}
1748
1749	LogFlowFunc(("pReq=%#p cXfersPending=%u cbXfer=%u\n", pReq, cXfersPending, (cbOld - cbComplete)));
1750	return false;
1751	}
1752
1753	VMMR3DECL(int) PDMR3BlkCacheRead(PPDMBLKCACHE pBlkCache, uint64_t off,
1754	PCRTSGBUF pcSgBuf, size_t cbRead, void *pvUser)
1755	{
1756	int rc = VINF_SUCCESS;
1757	PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
1758	PPDMBLKCACHEENTRY pEntry;
1759	PPDMBLKCACHEREQ pReq;
1760
1761	LogFlowFunc((": pBlkCache=%#p{%s} off=%llu pcSgBuf=%#p cbRead=%u pvUser=%#p\n",
1762	pBlkCache, pBlkCache->pszId, off, pcSgBuf, cbRead, pvUser));
1763
1764	RTSGBUF SgBuf;
1765	RTSgBufClone(&SgBuf, pcSgBuf);
1766
1767	/* Allocate new request structure. */
1768	pReq = pdmBlkCacheReqAlloc(cbRead, pvUser);
1769	if (RT_UNLIKELY(!pReq))
1770	return VERR_NO_MEMORY;
1771
1772	/* Increment data transfer counter to keep the request valid while we access it. */
1773	ASMAtomicIncU32(&pReq->cXfersPending);
1774
1775	while (cbRead)
1776	{
1777	size_t cbToRead;
1778
1779	pEntry = pdmBlkCacheGetCacheEntryByOffset(pBlkCache, off);
1780
1781	/*
1782	* If there is no entry we try to create a new one eviciting unused pages
1783	* if the cache is full. If this is not possible we will pass the request through
1784	* and skip the caching (all entries may be still in progress so they can't
1785	* be evicted)
1786	* If we have an entry it can be in one of the LRU lists where the entry
1787	* contains data (recently used or frequently used LRU) so we can just read
1788	* the data we need and put the entry at the head of the frequently used LRU list.
1789	* In case the entry is in one of the ghost lists it doesn't contain any data.
1790	* We have to fetch it again evicting pages from either T1 or T2 to make room.
1791	*/
1792	if (pEntry)
1793	{
1794	uint64_t offDiff = off - pEntry->Core.Key;
1795
1796	AssertMsg(off >= pEntry->Core.Key,
1797	("Overflow in calculation off=%llu OffsetAligned=%llu\n",
1798	off, pEntry->Core.Key));
1799
1800	AssertPtr(pEntry->pList);
1801
1802	cbToRead = RT_MIN(pEntry->cbData - offDiff, cbRead);
1803
1804	AssertMsg(off + cbToRead <= pEntry->Core.Key + pEntry->Core.KeyLast + 1,
1805	("Buffer of cache entry exceeded off=%llu cbToRead=%d\n",
1806	off, cbToRead));
1807
1808	cbRead -= cbToRead;
1809
1810	if (!cbRead)
1811	STAM_COUNTER_INC(&pCache->cHits);
1812	else
1813	STAM_COUNTER_INC(&pCache->cPartialHits);
1814
1815	STAM_COUNTER_ADD(&pCache->StatRead, cbToRead);
1816
1817	/* Ghost lists contain no data. */
1818	if ( (pEntry->pList == &pCache->LruRecentlyUsedIn)
1819	\|\| (pEntry->pList == &pCache->LruFrequentlyUsed))
1820	{
1821	if (pdmBlkCacheEntryFlagIsSetClearAcquireLock(pBlkCache, pEntry,
1822	PDMBLKCACHE_ENTRY_IO_IN_PROGRESS,
1823	PDMBLKCACHE_ENTRY_IS_DIRTY))
1824	{
1825	/* Entry didn't completed yet. Append to the list */
1826	pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
1827	&SgBuf, offDiff, cbToRead,
1828	false /* fWrite */);
1829	RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
1830	}
1831	else
1832	{
1833	/* Read as much as we can from the entry. */
1834	RTSgBufCopyFromBuf(&SgBuf, pEntry->pbData + offDiff, cbToRead);
1835	ASMAtomicSubU32(&pReq->cbXfer, cbToRead);
1836	}
1837
1838	/* Move this entry to the top position */
1839	if (pEntry->pList == &pCache->LruFrequentlyUsed)
1840	{
1841	pdmBlkCacheLockEnter(pCache);
1842	pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
1843	pdmBlkCacheLockLeave(pCache);
1844	}
1845	/* Release the entry */
1846	pdmBlkCacheEntryRelease(pEntry);
1847	}
1848	else
1849	{
1850	uint8_t *pbBuffer = NULL;
1851
1852	LogFlow(("Fetching data for ghost entry %#p from file\n", pEntry));
1853
1854	pdmBlkCacheLockEnter(pCache);
1855	pdmBlkCacheEntryRemoveFromList(pEntry); /* Remove it before we remove data, otherwise it may get freed when evicting data. */
1856	bool fEnough = pdmBlkCacheReclaim(pCache, pEntry->cbData, true, &pbBuffer);
1857
1858	/* Move the entry to Am and fetch it to the cache. */
1859	if (fEnough)
1860	{
1861	pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
1862	pdmBlkCacheAdd(pCache, pEntry->cbData);
1863	pdmBlkCacheLockLeave(pCache);
1864
1865	if (pbBuffer)
1866	pEntry->pbData = pbBuffer;
1867	else
1868	pEntry->pbData = (uint8_t *)RTMemPageAlloc(pEntry->cbData);
1869	AssertPtr(pEntry->pbData);
1870
1871	pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
1872	&SgBuf, offDiff, cbToRead,
1873	false /* fWrite */);
1874	pdmBlkCacheEntryReadFromMedium(pEntry);
1875	/* Release the entry */
1876	pdmBlkCacheEntryRelease(pEntry);
1877	}
1878	else
1879	{
1880	RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
1881	STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
1882	RTAvlrU64Remove(pBlkCache->pTree, pEntry->Core.Key);
1883	STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
1884	RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
1885
1886	pdmBlkCacheLockLeave(pCache);
1887
1888	RTMemFree(pEntry);
1889
1890	pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
1891	&SgBuf, off, cbToRead,
1892	PDMBLKCACHEXFERDIR_READ);
1893	}
1894	}
1895	}
1896	else
1897	{
1898	#ifdef VBOX_WITH_IO_READ_CACHE
1899	/* No entry found for this offset. Create a new entry and fetch the data to the cache. */
1900	PPDMBLKCACHEENTRY pEntryNew = pdmBlkCacheEntryCreate(pBlkCache,
1901	off, cbRead,
1902	PAGE_SIZE,
1903	&cbToRead);
1904
1905	cbRead -= cbToRead;
1906
1907	if (pEntryNew)
1908	{
1909	if (!cbRead)
1910	STAM_COUNTER_INC(&pCache->cMisses);
1911	else
1912	STAM_COUNTER_INC(&pCache->cPartialHits);
1913
1914	pdmBlkCacheEntryWaitersAdd(pEntryNew, pReq,
1915	&SgBuf,
1916	off - pEntryNew->Core.Key,
1917	cbToRead,
1918	false /* fWrite */);
1919	pdmBlkCacheEntryReadFromMedium(pEntryNew);
1920	pdmBlkCacheEntryRelease(pEntryNew); /* it is protected by the I/O in progress flag now. */
1921	}
1922	else
1923	{
1924	/*
1925	* There is not enough free space in the cache.
1926	* Pass the request directly to the I/O manager.
1927	*/
1928	LogFlow(("Couldn't evict %u bytes from the cache. Remaining request will be passed through\n", cbToRead));
1929
1930	pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
1931	&SgBuf, off, cbToRead,
1932	PDMBLKCACHEXFERDIR_READ);
1933	}
1934	#else
1935	/* Clip read size if necessary. */
1936	PPDMBLKCACHEENTRY pEntryAbove;
1937	pdmBlkCacheGetCacheBestFitEntryByOffset(pBlkCache, off,
1938	&pEntryAbove, NULL);
1939
1940	if (pEntryAbove)
1941	{
1942	if (off + cbRead > pEntryAbove->Core.Key)
1943	cbToRead = pEntryAbove->Core.Key - off;
1944	else
1945	cbToRead = cbRead;
1946
1947	pdmBlkCacheEntryRelease(pEntryAbove);
1948	}
1949	else
1950	cbToRead = cbRead;
1951
1952	cbRead -= cbToRead;
1953	pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
1954	&SgBuf, off, cbToRead,
1955	PDMBLKCACHEXFERDIR_READ);
1956	#endif
1957	}
1958	off += cbToRead;
1959	}
1960
1961	if (!pdmBlkCacheReqUpdate(pBlkCache, pReq, 0, rc, false))
1962	rc = VINF_AIO_TASK_PENDING;
1963
1964	LogFlowFunc((": Leave rc=%Rrc\n", rc));
1965
1966	return rc;
1967	}
1968
1969	VMMR3DECL(int) PDMR3BlkCacheWrite(PPDMBLKCACHE pBlkCache, uint64_t off,
1970	PCRTSGBUF pcSgBuf, size_t cbWrite, void *pvUser)
1971	{
1972	int rc = VINF_SUCCESS;
1973	PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
1974	PPDMBLKCACHEENTRY pEntry;
1975	PPDMBLKCACHEREQ pReq;
1976
1977	LogFlowFunc((": pBlkCache=%#p{%s} off=%llu pcSgBuf=%#p cbWrite=%u pvUser=%#p\n",
1978	pBlkCache, pBlkCache->pszId, off, pcSgBuf, cbWrite, pvUser));
1979
1980	RTSGBUF SgBuf;
1981	RTSgBufClone(&SgBuf, pcSgBuf);
1982
1983	/* Allocate new request structure. */
1984	pReq = pdmBlkCacheReqAlloc(cbWrite, pvUser);
1985	if (RT_UNLIKELY(!pReq))
1986	return VERR_NO_MEMORY;
1987
1988	/* Increment data transfer counter to keep the request valid while we access it. */
1989	ASMAtomicIncU32(&pReq->cXfersPending);
1990
1991	while (cbWrite)
1992	{
1993	size_t cbToWrite;
1994
1995	pEntry = pdmBlkCacheGetCacheEntryByOffset(pBlkCache, off);
1996
1997	if (pEntry)
1998	{
1999	/* Write the data into the entry and mark it as dirty */
2000	AssertPtr(pEntry->pList);
2001
2002	uint64_t offDiff = off - pEntry->Core.Key;
2003
2004	AssertMsg(off >= pEntry->Core.Key,
2005	("Overflow in calculation off=%llu OffsetAligned=%llu\n",
2006	off, pEntry->Core.Key));
2007
2008	cbToWrite = RT_MIN(pEntry->cbData - offDiff, cbWrite);
2009	cbWrite -= cbToWrite;
2010
2011	if (!cbWrite)
2012	STAM_COUNTER_INC(&pCache->cHits);
2013	else
2014	STAM_COUNTER_INC(&pCache->cPartialHits);
2015
2016	STAM_COUNTER_ADD(&pCache->StatWritten, cbToWrite);
2017
2018	/* Ghost lists contain no data. */
2019	if ( (pEntry->pList == &pCache->LruRecentlyUsedIn)
2020	\|\| (pEntry->pList == &pCache->LruFrequentlyUsed))
2021	{
2022	/* Check if the entry is dirty. */
2023	if(pdmBlkCacheEntryFlagIsSetClearAcquireLock(pBlkCache, pEntry,
2024	PDMBLKCACHE_ENTRY_IS_DIRTY,
2025	0))
2026	{
2027	/* If it is already dirty but not in progress just update the data. */
2028	if (!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS))
2029	{
2030	RTSgBufCopyToBuf(&SgBuf, pEntry->pbData + offDiff,
2031	cbToWrite);
2032	ASMAtomicSubU32(&pReq->cbXfer, cbToWrite);
2033	}
2034	else
2035	{
2036	/* The data isn't written to the file yet */
2037	pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
2038	&SgBuf, offDiff, cbToWrite,
2039	true /* fWrite */);
2040	STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
2041	}
2042
2043	RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2044	}
2045	else /* Dirty bit not set */
2046	{
2047	/*
2048	* Check if a read is in progress for this entry.
2049	* We have to defer processing in that case.
2050	*/
2051	if(pdmBlkCacheEntryFlagIsSetClearAcquireLock(pBlkCache, pEntry,
2052	PDMBLKCACHE_ENTRY_IO_IN_PROGRESS,
2053	0))
2054	{
2055	pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
2056	&SgBuf, offDiff, cbToWrite,
2057	true /* fWrite */);
2058	STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
2059	RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2060	}
2061	else /* I/O in progress flag not set */
2062	{
2063	/* Write as much as we can into the entry and update the file. */
2064	RTSgBufCopyToBuf(&SgBuf, pEntry->pbData + offDiff, cbToWrite);
2065	ASMAtomicSubU32(&pReq->cbXfer, cbToWrite);
2066
2067	bool fCommit = pdmBlkCacheAddDirtyEntry(pBlkCache, pEntry);
2068	if (fCommit)
2069	pdmBlkCacheCommitDirtyEntries(pCache);
2070	}
2071	} /* Dirty bit not set */
2072
2073	/* Move this entry to the top position */
2074	if (pEntry->pList == &pCache->LruFrequentlyUsed)
2075	{
2076	pdmBlkCacheLockEnter(pCache);
2077	pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
2078	pdmBlkCacheLockLeave(pCache);
2079	}
2080
2081	pdmBlkCacheEntryRelease(pEntry);
2082	}
2083	else /* Entry is on the ghost list */
2084	{
2085	uint8_t *pbBuffer = NULL;
2086
2087	pdmBlkCacheLockEnter(pCache);
2088	pdmBlkCacheEntryRemoveFromList(pEntry); /* Remove it before we remove data, otherwise it may get freed when evicting data. */
2089	bool fEnough = pdmBlkCacheReclaim(pCache, pEntry->cbData, true, &pbBuffer);
2090
2091	if (fEnough)
2092	{
2093	/* Move the entry to Am and fetch it to the cache. */
2094	pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
2095	pdmBlkCacheAdd(pCache, pEntry->cbData);
2096	pdmBlkCacheLockLeave(pCache);
2097
2098	if (pbBuffer)
2099	pEntry->pbData = pbBuffer;
2100	else
2101	pEntry->pbData = (uint8_t *)RTMemPageAlloc(pEntry->cbData);
2102	AssertPtr(pEntry->pbData);
2103
2104	pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
2105	&SgBuf, offDiff, cbToWrite,
2106	true /* fWrite */);
2107	STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
2108	pdmBlkCacheEntryReadFromMedium(pEntry);
2109
2110	/* Release the reference. If it is still needed the I/O in progress flag should protect it now. */
2111	pdmBlkCacheEntryRelease(pEntry);
2112	}
2113	else
2114	{
2115	RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
2116	STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
2117	RTAvlrU64Remove(pBlkCache->pTree, pEntry->Core.Key);
2118	STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
2119	RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2120
2121	pdmBlkCacheLockLeave(pCache);
2122
2123	RTMemFree(pEntry);
2124	pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
2125	&SgBuf, off, cbToWrite,
2126	PDMBLKCACHEXFERDIR_WRITE);
2127	}
2128	}
2129	}
2130	else /* No entry found */
2131	{
2132	/*
2133	* No entry found. Try to create a new cache entry to store the data in and if that fails
2134	* write directly to the file.
2135	*/
2136	PPDMBLKCACHEENTRY pEntryNew = pdmBlkCacheEntryCreate(pBlkCache,
2137	off, cbWrite,
2138	512, &cbToWrite);
2139
2140	cbWrite -= cbToWrite;
2141
2142	if (pEntryNew)
2143	{
2144	uint64_t offDiff = off - pEntryNew->Core.Key;
2145
2146	STAM_COUNTER_INC(&pCache->cHits);
2147
2148	/*
2149	* Check if it is possible to just write the data without waiting
2150	* for it to get fetched first.
2151	*/
2152	if (!offDiff && pEntryNew->cbData == cbToWrite)
2153	{
2154	RTSgBufCopyToBuf(&SgBuf, pEntryNew->pbData, cbToWrite);
2155	ASMAtomicSubU32(&pReq->cbXfer, cbToWrite);
2156
2157	bool fCommit = pdmBlkCacheAddDirtyEntry(pBlkCache, pEntryNew);
2158	if (fCommit)
2159	pdmBlkCacheCommitDirtyEntries(pCache);
2160	STAM_COUNTER_ADD(&pCache->StatWritten, cbToWrite);
2161	}
2162	else
2163	{
2164	/* Defer the write and fetch the data from the endpoint. */
2165	pdmBlkCacheEntryWaitersAdd(pEntryNew, pReq,
2166	&SgBuf, offDiff, cbToWrite,
2167	true /* fWrite */);
2168	STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
2169	pdmBlkCacheEntryReadFromMedium(pEntryNew);
2170	}
2171
2172	pdmBlkCacheEntryRelease(pEntryNew);
2173	}
2174	else
2175	{
2176	/*
2177	* There is not enough free space in the cache.
2178	* Pass the request directly to the I/O manager.
2179	*/
2180	LogFlow(("Couldn't evict %u bytes from the cache. Remaining request will be passed through\n", cbToWrite));
2181
2182	STAM_COUNTER_INC(&pCache->cMisses);
2183
2184	pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
2185	&SgBuf, off, cbToWrite,
2186	PDMBLKCACHEXFERDIR_WRITE);
2187	}
2188	}
2189
2190	off += cbToWrite;
2191	}
2192
2193	if (!pdmBlkCacheReqUpdate(pBlkCache, pReq, 0, rc, false))
2194	rc = VINF_AIO_TASK_PENDING;
2195
2196	LogFlowFunc((": Leave rc=%Rrc\n", rc));
2197
2198	return rc;
2199	}
2200
2201	VMMR3DECL(int) PDMR3BlkCacheFlush(PPDMBLKCACHE pBlkCache, void *pvUser)
2202	{
2203	int rc = VINF_SUCCESS;
2204
2205	LogFlowFunc((": pBlkCache=%#p{%s}\n", pBlkCache, pBlkCache->pszId));
2206
2207	/* Commit dirty entries in the cache. */
2208	pdmBlkCacheCommit(pBlkCache);
2209
2210	LogFlowFunc((": Leave rc=%Rrc\n", rc));
2211	return rc;
2212	}
2213
2214	/**
2215	* Completes a task segment freeing all resources and completes the task handle
2216	* if everything was transferred.
2217	*
2218	* @returns Next task segment handle.
2219	* @param pTaskSeg Task segment to complete.
2220	* @param rc Status code to set.
2221	*/
2222	static PPDMBLKCACHEWAITER pdmBlkCacheWaiterComplete(PPDMBLKCACHE pBlkCache,
2223	PPDMBLKCACHEWAITER pWaiter,
2224	int rc)
2225	{
2226	PPDMBLKCACHEWAITER pNext = pWaiter->pNext;
2227	PPDMBLKCACHEREQ pReq = pWaiter->pReq;
2228
2229	pdmBlkCacheReqUpdate(pBlkCache, pWaiter->pReq, pWaiter->cbTransfer, rc, true);
2230
2231	RTMemFree(pWaiter);
2232
2233	return pNext;
2234	}
2235
2236	static void pdmBlkCacheIoXferCompleteEntry(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEIOXFER hIoXfer, int rcIoXfer)
2237	{
2238	PPDMBLKCACHEENTRY pEntry = hIoXfer->pEntry;
2239	PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
2240
2241	/* Reference the entry now as we are clearing the I/O in progress flag
2242	* which protected the entry till now. */
2243	pdmBlkCacheEntryRef(pEntry);
2244
2245	RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
2246	pEntry->fFlags &= ~PDMBLKCACHE_ENTRY_IO_IN_PROGRESS;
2247
2248	/* Process waiting segment list. The data in entry might have changed in-between. */
2249	bool fDirty = false;
2250	PPDMBLKCACHEWAITER pComplete = pEntry->pWaitingHead;
2251	PPDMBLKCACHEWAITER pCurr = pComplete;
2252
2253	AssertMsg((pCurr && pEntry->pWaitingTail) \|\| (!pCurr && !pEntry->pWaitingTail),
2254	("The list tail was not updated correctly\n"));
2255	pEntry->pWaitingTail = NULL;
2256	pEntry->pWaitingHead = NULL;
2257
2258	if (hIoXfer->enmXferDir == PDMBLKCACHEXFERDIR_WRITE)
2259	{
2260	/*
2261	* An error here is difficult to handle as the original request completed already.
2262	* The error is logged for now and the VM is paused.
2263	* If the user continues the entry is written again in the hope
2264	* the user fixed the problem and the next write succeeds.
2265	*/
2266	/** @todo r=aeichner: This solution doesn't work
2267	* The user will get the message but the VM will hang afterwards
2268	* VMR3Suspend() returns when the VM is suspended but suspending
2269	* the VM will reopen the images readonly in DrvVD. They are closed first
2270	* which will close the endpoints. This will block EMT while the
2271	* I/O manager processes the close request but the IO manager is stuck
2272	* in the VMR3Suspend call and can't process the request.
2273	* Another problem is that closing the VM means flushing the cache
2274	* but the entry failed and will probably fail again.
2275	* No idea so far how to solve this problem... but the user gets informed
2276	* at least.
2277	*/
2278	if (RT_FAILURE(rcIoXfer))
2279	{
2280	LogRel(("I/O cache: Error while writing entry at offset %llu (%u bytes) to medium \"%s\"\n",
2281	pEntry->Core.Key, pEntry->cbData, pBlkCache->pszId));
2282
2283	int rc = VMSetRuntimeError(pCache->pVM, VMSETRTERR_FLAGS_SUSPEND \| VMSETRTERR_FLAGS_NO_WAIT, "BLKCACHE_IOERR",
2284	N_("The I/O cache encountered an error while updating data in medium \"%s\" (rc=%Rrc)."
2285	"Make sure there is enough free space on the disk and that the disk is working properly."
2286	"Operation can be resumed afterwards."),
2287	pBlkCache->pszId, rcIoXfer);
2288	AssertRC(rc);
2289	}
2290	else
2291	{
2292	pEntry->fFlags &= ~PDMBLKCACHE_ENTRY_IS_DIRTY;
2293
2294	while (pCurr)
2295	{
2296	AssertMsg(pCurr->fWrite, ("Completed write entries should never have read tasks attached\n"));
2297
2298	RTSgBufCopyToBuf(&pCurr->SgBuf, pEntry->pbData + pCurr->offCacheEntry, pCurr->cbTransfer);
2299	fDirty = true;
2300
2301	pCurr = pCurr->pNext;
2302	}
2303	}
2304	}
2305	else
2306	{
2307	AssertMsg(hIoXfer->enmXferDir == PDMBLKCACHEXFERDIR_READ, ("Invalid transfer type\n"));
2308	AssertMsg(!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IS_DIRTY),
2309	("Invalid flags set\n"));
2310
2311	while (pCurr)
2312	{
2313	if (pCurr->fWrite)
2314	{
2315	RTSgBufCopyToBuf(&pCurr->SgBuf, pEntry->pbData + pCurr->offCacheEntry, pCurr->cbTransfer);
2316	fDirty = true;
2317	}
2318	else
2319	RTSgBufCopyFromBuf(&pCurr->SgBuf, pEntry->pbData + pCurr->offCacheEntry, pCurr->cbTransfer);
2320
2321	pCurr = pCurr->pNext;
2322	}
2323	}
2324
2325	bool fCommit = false;
2326	if (fDirty)
2327	fCommit = pdmBlkCacheAddDirtyEntry(pBlkCache, pEntry);
2328
2329	RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2330
2331	/* Dereference so that it isn't protected anymore except we issued anyother write for it. */
2332	pdmBlkCacheEntryRelease(pEntry);
2333
2334	if (fCommit)
2335	pdmBlkCacheCommitDirtyEntries(pCache);
2336
2337	/* Complete waiters now. */
2338	while (pComplete)
2339	pComplete = pdmBlkCacheWaiterComplete(pBlkCache, pComplete, rcIoXfer);
2340	}
2341
2342	VMMR3DECL(void) PDMR3BlkCacheIoXferComplete(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEIOXFER hIoXfer, int rcIoXfer)
2343	{
2344	LogFlowFunc(("pBlkCache=%#p hIoXfer=%#p rcIoXfer=%Rrc\n", pBlkCache, hIoXfer, rcIoXfer));
2345
2346	if (hIoXfer->fIoCache)
2347	pdmBlkCacheIoXferCompleteEntry(pBlkCache, hIoXfer, rcIoXfer);
2348	else
2349	pdmBlkCacheReqUpdate(pBlkCache, hIoXfer->pReq, hIoXfer->cbXfer, rcIoXfer, true);
2350	}
2351

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/PDMBlkCache.cpp@ 34223

Download in other formats: