PDMBlkCache.cpp@ 34347

Last change on this file since 34347 was 34347, checked in by vboxsync, 14 years ago
BlkCache: Suspend the VM if an error occurs during a write
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 84.9 KB

Line
1	/* $Id: PDMBlkCache.cpp 34347 2010-11-24 22:34:21Z vboxsync $ */
2	/** @file
3	* PDM Block Cache.
4	*/
5
6	/*
7	* Copyright (C) 2006-2008 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/** @page pg_pdm_block_cache PDM Block Cache - The I/O cache
19	* This component implements an I/O cache based on the 2Q cache algorithm.
20	*/
21
22	/*******************************************************************************
23	* Header Files *
24	*******************************************************************************/
25	#define LOG_GROUP LOG_GROUP_PDM_BLK_CACHE
26	#include "PDMInternal.h"
27	#include <iprt/asm.h>
28	#include <iprt/mem.h>
29	#include <iprt/path.h>
30	#include <iprt/string.h>
31	#include <VBox/log.h>
32	#include <VBox/stam.h>
33	#include <VBox/uvm.h>
34	#include <VBox/vm.h>
35
36	#include "PDMBlkCacheInternal.h"
37
38	#ifdef VBOX_STRICT
39	# define PDMACFILECACHE_IS_CRITSECT_OWNER(Cache) \
40	do \
41	{ \
42	AssertMsg(RTCritSectIsOwner(&Cache->CritSect), \
43	("Thread does not own critical section\n"));\
44	} while(0)
45
46	# define PDMACFILECACHE_EP_IS_SEMRW_WRITE_OWNER(pEpCache) \
47	do \
48	{ \
49	AssertMsg(RTSemRWIsWriteOwner(pEpCache->SemRWEntries), \
50	("Thread is not exclusive owner of the per endpoint RW semaphore\n")); \
51	} while(0)
52
53	# define PDMACFILECACHE_EP_IS_SEMRW_READ_OWNER(pEpCache) \
54	do \
55	{ \
56	AssertMsg(RTSemRWIsReadOwner(pEpCache->SemRWEntries), \
57	("Thread is not read owner of the per endpoint RW semaphore\n")); \
58	} while(0)
59
60	#else
61	# define PDMACFILECACHE_IS_CRITSECT_OWNER(Cache) do { } while(0)
62	# define PDMACFILECACHE_EP_IS_SEMRW_WRITE_OWNER(pEpCache) do { } while(0)
63	# define PDMACFILECACHE_EP_IS_SEMRW_READ_OWNER(pEpCache) do { } while(0)
64	#endif
65
66	/*******************************************************************************
67	* Internal Functions *
68	*******************************************************************************/
69
70	/**
71	* Decrement the reference counter of the given cache entry.
72	*
73	* @returns nothing.
74	* @param pEntry The entry to release.
75	*/
76	DECLINLINE(void) pdmBlkCacheEntryRelease(PPDMBLKCACHEENTRY pEntry)
77	{
78	AssertMsg(pEntry->cRefs > 0, ("Trying to release a not referenced entry\n"));
79	ASMAtomicDecU32(&pEntry->cRefs);
80	}
81
82	/**
83	* Increment the reference counter of the given cache entry.
84	*
85	* @returns nothing.
86	* @param pEntry The entry to reference.
87	*/
88	DECLINLINE(void) pdmBlkCacheEntryRef(PPDMBLKCACHEENTRY pEntry)
89	{
90	ASMAtomicIncU32(&pEntry->cRefs);
91	}
92
93	#ifdef DEBUG
94	static void pdmBlkCacheValidate(PPDMBLKCACHEGLOBAL pCache)
95	{
96	/* Amount of cached data should never exceed the maximum amount. */
97	AssertMsg(pCache->cbCached <= pCache->cbMax,
98	("Current amount of cached data exceeds maximum\n"));
99
100	/* The amount of cached data in the LRU and FRU list should match cbCached */
101	AssertMsg(pCache->LruRecentlyUsedIn.cbCached + pCache->LruFrequentlyUsed.cbCached == pCache->cbCached,
102	("Amount of cached data doesn't match\n"));
103
104	AssertMsg(pCache->LruRecentlyUsedOut.cbCached <= pCache->cbRecentlyUsedOutMax,
105	("Paged out list exceeds maximum\n"));
106	}
107	#endif
108
109	DECLINLINE(void) pdmBlkCacheLockEnter(PPDMBLKCACHEGLOBAL pCache)
110	{
111	RTCritSectEnter(&pCache->CritSect);
112	#ifdef DEBUG
113	pdmBlkCacheValidate(pCache);
114	#endif
115	}
116
117	DECLINLINE(void) pdmBlkCacheLockLeave(PPDMBLKCACHEGLOBAL pCache)
118	{
119	#ifdef DEBUG
120	pdmBlkCacheValidate(pCache);
121	#endif
122	RTCritSectLeave(&pCache->CritSect);
123	}
124
125	DECLINLINE(void) pdmBlkCacheSub(PPDMBLKCACHEGLOBAL pCache, uint32_t cbAmount)
126	{
127	PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
128	pCache->cbCached -= cbAmount;
129	}
130
131	DECLINLINE(void) pdmBlkCacheAdd(PPDMBLKCACHEGLOBAL pCache, uint32_t cbAmount)
132	{
133	PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
134	pCache->cbCached += cbAmount;
135	}
136
137	DECLINLINE(void) pdmBlkCacheListAdd(PPDMBLKLRULIST pList, uint32_t cbAmount)
138	{
139	pList->cbCached += cbAmount;
140	}
141
142	DECLINLINE(void) pdmBlkCacheListSub(PPDMBLKLRULIST pList, uint32_t cbAmount)
143	{
144	pList->cbCached -= cbAmount;
145	}
146
147	#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
148	/**
149	* Checks consistency of a LRU list.
150	*
151	* @returns nothing
152	* @param pList The LRU list to check.
153	* @param pNotInList Element which is not allowed to occur in the list.
154	*/
155	static void pdmBlkCacheCheckList(PPDMBLKLRULIST pList, PPDMBLKCACHEENTRY pNotInList)
156	{
157	PPDMBLKCACHEENTRY pCurr = pList->pHead;
158
159	/* Check that there are no double entries and no cycles in the list. */
160	while (pCurr)
161	{
162	PPDMBLKCACHEENTRY pNext = pCurr->pNext;
163
164	while (pNext)
165	{
166	AssertMsg(pCurr != pNext,
167	("Entry %#p is at least two times in list %#p or there is a cycle in the list\n",
168	pCurr, pList));
169	pNext = pNext->pNext;
170	}
171
172	AssertMsg(pCurr != pNotInList, ("Not allowed entry %#p is in list\n", pCurr));
173
174	if (!pCurr->pNext)
175	AssertMsg(pCurr == pList->pTail, ("End of list reached but last element is not list tail\n"));
176
177	pCurr = pCurr->pNext;
178	}
179	}
180	#endif
181
182	/**
183	* Unlinks a cache entry from the LRU list it is assigned to.
184	*
185	* @returns nothing.
186	* @param pEntry The entry to unlink.
187	*/
188	static void pdmBlkCacheEntryRemoveFromList(PPDMBLKCACHEENTRY pEntry)
189	{
190	PPDMBLKLRULIST pList = pEntry->pList;
191	PPDMBLKCACHEENTRY pPrev, pNext;
192
193	LogFlowFunc((": Deleting entry %#p from list %#p\n", pEntry, pList));
194
195	AssertPtr(pList);
196
197	#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
198	pdmBlkCacheCheckList(pList, NULL);
199	#endif
200
201	pPrev = pEntry->pPrev;
202	pNext = pEntry->pNext;
203
204	AssertMsg(pEntry != pPrev, ("Entry links to itself as previous element\n"));
205	AssertMsg(pEntry != pNext, ("Entry links to itself as next element\n"));
206
207	if (pPrev)
208	pPrev->pNext = pNext;
209	else
210	{
211	pList->pHead = pNext;
212
213	if (pNext)
214	pNext->pPrev = NULL;
215	}
216
217	if (pNext)
218	pNext->pPrev = pPrev;
219	else
220	{
221	pList->pTail = pPrev;
222
223	if (pPrev)
224	pPrev->pNext = NULL;
225	}
226
227	pEntry->pList = NULL;
228	pEntry->pPrev = NULL;
229	pEntry->pNext = NULL;
230	pdmBlkCacheListSub(pList, pEntry->cbData);
231	#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
232	pdmBlkCacheCheckList(pList, pEntry);
233	#endif
234	}
235
236	/**
237	* Adds a cache entry to the given LRU list unlinking it from the currently
238	* assigned list if needed.
239	*
240	* @returns nothing.
241	* @param pList List to the add entry to.
242	* @param pEntry Entry to add.
243	*/
244	static void pdmBlkCacheEntryAddToList(PPDMBLKLRULIST pList, PPDMBLKCACHEENTRY pEntry)
245	{
246	LogFlowFunc((": Adding entry %#p to list %#p\n", pEntry, pList));
247	#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
248	pdmBlkCacheCheckList(pList, NULL);
249	#endif
250
251	/* Remove from old list if needed */
252	if (pEntry->pList)
253	pdmBlkCacheEntryRemoveFromList(pEntry);
254
255	pEntry->pNext = pList->pHead;
256	if (pList->pHead)
257	pList->pHead->pPrev = pEntry;
258	else
259	{
260	Assert(!pList->pTail);
261	pList->pTail = pEntry;
262	}
263
264	pEntry->pPrev = NULL;
265	pList->pHead = pEntry;
266	pdmBlkCacheListAdd(pList, pEntry->cbData);
267	pEntry->pList = pList;
268	#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
269	pdmBlkCacheCheckList(pList, NULL);
270	#endif
271	}
272
273	/**
274	* Destroys a LRU list freeing all entries.
275	*
276	* @returns nothing
277	* @param pList Pointer to the LRU list to destroy.
278	*
279	* @note The caller must own the critical section of the cache.
280	*/
281	static void pdmBlkCacheDestroyList(PPDMBLKLRULIST pList)
282	{
283	while (pList->pHead)
284	{
285	PPDMBLKCACHEENTRY pEntry = pList->pHead;
286
287	pList->pHead = pEntry->pNext;
288
289	AssertMsg(!(pEntry->fFlags & (PDMBLKCACHE_ENTRY_IO_IN_PROGRESS \| PDMBLKCACHE_ENTRY_IS_DIRTY)),
290	("Entry is dirty and/or still in progress fFlags=%#x\n", pEntry->fFlags));
291
292	RTMemPageFree(pEntry->pbData, pEntry->cbData);
293	RTMemFree(pEntry);
294	}
295	}
296
297	/**
298	* Tries to remove the given amount of bytes from a given list in the cache
299	* moving the entries to one of the given ghosts lists
300	*
301	* @returns Amount of data which could be freed.
302	* @param pCache Pointer to the global cache data.
303	* @param cbData The amount of the data to free.
304	* @param pListSrc The source list to evict data from.
305	* @param pGhostListSrc The ghost list removed entries should be moved to
306	* NULL if the entry should be freed.
307	* @param fReuseBuffer Flag whether a buffer should be reused if it has the same size
308	* @param ppbBuf Where to store the address of the buffer if an entry with the
309	* same size was found and fReuseBuffer is true.
310	*
311	* @note This function may return fewer bytes than requested because entries
312	* may be marked as non evictable if they are used for I/O at the
313	* moment.
314	*/
315	static size_t pdmBlkCacheEvictPagesFrom(PPDMBLKCACHEGLOBAL pCache, size_t cbData,
316	PPDMBLKLRULIST pListSrc, PPDMBLKLRULIST pGhostListDst,
317	bool fReuseBuffer, uint8_t **ppbBuffer)
318	{
319	size_t cbEvicted = 0;
320
321	PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
322
323	AssertMsg(cbData > 0, ("Evicting 0 bytes not possible\n"));
324	AssertMsg( !pGhostListDst
325	\|\| (pGhostListDst == &pCache->LruRecentlyUsedOut),
326	("Destination list must be NULL or the recently used but paged out list\n"));
327
328	if (fReuseBuffer)
329	{
330	AssertPtr(ppbBuffer);
331	*ppbBuffer = NULL;
332	}
333
334	/* Start deleting from the tail. */
335	PPDMBLKCACHEENTRY pEntry = pListSrc->pTail;
336
337	while ((cbEvicted < cbData) && pEntry)
338	{
339	PPDMBLKCACHEENTRY pCurr = pEntry;
340
341	pEntry = pEntry->pPrev;
342
343	/* We can't evict pages which are currently in progress or dirty but not in progress */
344	if ( !(pCurr->fFlags & PDMBLKCACHE_NOT_EVICTABLE)
345	&& (ASMAtomicReadU32(&pCurr->cRefs) == 0))
346	{
347	/* Ok eviction candidate. Grab the endpoint semaphore and check again
348	* because somebody else might have raced us. */
349	PPDMBLKCACHE pBlkCache = pCurr->pBlkCache;
350	RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
351
352	if (!(pCurr->fFlags & PDMBLKCACHE_NOT_EVICTABLE)
353	&& (ASMAtomicReadU32(&pCurr->cRefs) == 0))
354	{
355	LogFlow(("Evicting entry %#p (%u bytes)\n", pCurr, pCurr->cbData));
356
357	if (fReuseBuffer && (pCurr->cbData == cbData))
358	{
359	STAM_COUNTER_INC(&pCache->StatBuffersReused);
360	*ppbBuffer = pCurr->pbData;
361	}
362	else if (pCurr->pbData)
363	RTMemPageFree(pCurr->pbData, pCurr->cbData);
364
365	pCurr->pbData = NULL;
366	cbEvicted += pCurr->cbData;
367
368	pdmBlkCacheEntryRemoveFromList(pCurr);
369	pdmBlkCacheSub(pCache, pCurr->cbData);
370
371	if (pGhostListDst)
372	{
373	RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
374
375	PPDMBLKCACHEENTRY pGhostEntFree = pGhostListDst->pTail;
376
377	/* We have to remove the last entries from the paged out list. */
378	while ( ((pGhostListDst->cbCached + pCurr->cbData) > pCache->cbRecentlyUsedOutMax)
379	&& pGhostEntFree)
380	{
381	PPDMBLKCACHEENTRY pFree = pGhostEntFree;
382	PPDMBLKCACHE pBlkCacheFree = pFree->pBlkCache;
383
384	pGhostEntFree = pGhostEntFree->pPrev;
385
386	RTSemRWRequestWrite(pBlkCacheFree->SemRWEntries, RT_INDEFINITE_WAIT);
387
388	if (ASMAtomicReadU32(&pFree->cRefs) == 0)
389	{
390	pdmBlkCacheEntryRemoveFromList(pFree);
391
392	STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
393	RTAvlrU64Remove(pBlkCacheFree->pTree, pFree->Core.Key);
394	STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
395
396	RTMemFree(pFree);
397	}
398
399	RTSemRWReleaseWrite(pBlkCacheFree->SemRWEntries);
400	}
401
402	if (pGhostListDst->cbCached + pCurr->cbData > pCache->cbRecentlyUsedOutMax)
403	{
404	/* Couldn't remove enough entries. Delete */
405	STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
406	RTAvlrU64Remove(pCurr->pBlkCache->pTree, pCurr->Core.Key);
407	STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
408
409	RTMemFree(pCurr);
410	}
411	else
412	pdmBlkCacheEntryAddToList(pGhostListDst, pCurr);
413	}
414	else
415	{
416	/* Delete the entry from the AVL tree it is assigned to. */
417	STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
418	RTAvlrU64Remove(pCurr->pBlkCache->pTree, pCurr->Core.Key);
419	STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
420
421	RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
422	RTMemFree(pCurr);
423	}
424	}
425
426	}
427	else
428	LogFlow(("Entry %#p (%u bytes) is still in progress and can't be evicted\n", pCurr, pCurr->cbData));
429	}
430
431	return cbEvicted;
432	}
433
434	static bool pdmBlkCacheReclaim(PPDMBLKCACHEGLOBAL pCache, size_t cbData, bool fReuseBuffer, uint8_t **ppbBuffer)
435	{
436	size_t cbRemoved = 0;
437
438	if ((pCache->cbCached + cbData) < pCache->cbMax)
439	return true;
440	else if ((pCache->LruRecentlyUsedIn.cbCached + cbData) > pCache->cbRecentlyUsedInMax)
441	{
442	/* Try to evict as many bytes as possible from A1in */
443	cbRemoved = pdmBlkCacheEvictPagesFrom(pCache, cbData, &pCache->LruRecentlyUsedIn,
444	&pCache->LruRecentlyUsedOut, fReuseBuffer, ppbBuffer);
445
446	/*
447	* If it was not possible to remove enough entries
448	* try the frequently accessed cache.
449	*/
450	if (cbRemoved < cbData)
451	{
452	Assert(!fReuseBuffer \|\| !ppbBuffer); / It is not possible that we got a buffer with the correct size but we didn't freed enough data. */
453
454	/*
455	* If we removed something we can't pass the reuse buffer flag anymore because
456	* we don't need to evict that much data
457	*/
458	if (!cbRemoved)
459	cbRemoved += pdmBlkCacheEvictPagesFrom(pCache, cbData, &pCache->LruFrequentlyUsed,
460	NULL, fReuseBuffer, ppbBuffer);
461	else
462	cbRemoved += pdmBlkCacheEvictPagesFrom(pCache, cbData - cbRemoved, &pCache->LruFrequentlyUsed,
463	NULL, false, NULL);
464	}
465	}
466	else
467	{
468	/* We have to remove entries from frequently access list. */
469	cbRemoved = pdmBlkCacheEvictPagesFrom(pCache, cbData, &pCache->LruFrequentlyUsed,
470	NULL, fReuseBuffer, ppbBuffer);
471	}
472
473	LogFlowFunc((": removed %u bytes, requested %u\n", cbRemoved, cbData));
474	return (cbRemoved >= cbData);
475	}
476
477	DECLINLINE(int) pdmBlkCacheEnqueue(PPDMBLKCACHE pBlkCache, uint64_t off, PPDMBLKCACHEIOXFER pIoXfer)
478	{
479	int rc = VINF_SUCCESS;
480
481	switch (pBlkCache->enmType)
482	{
483	case PDMBLKCACHETYPE_DEV:
484	{
485	rc = pBlkCache->u.Dev.pfnXferEnqueue(pBlkCache->u.Dev.pDevIns,
486	pIoXfer->enmXferDir,
487	off, pIoXfer->cbXfer,
488	&pIoXfer->SgBuf, pIoXfer);
489	break;
490	}
491	case PDMBLKCACHETYPE_DRV:
492	{
493	rc = pBlkCache->u.Drv.pfnXferEnqueue(pBlkCache->u.Drv.pDrvIns,
494	pIoXfer->enmXferDir,
495	off, pIoXfer->cbXfer,
496	&pIoXfer->SgBuf, pIoXfer);
497	break;
498	}
499	case PDMBLKCACHETYPE_USB:
500	{
501	rc = pBlkCache->u.Usb.pfnXferEnqueue(pBlkCache->u.Usb.pUsbIns,
502	pIoXfer->enmXferDir,
503	off, pIoXfer->cbXfer,
504	&pIoXfer->SgBuf, pIoXfer);
505	break;
506	}
507	case PDMBLKCACHETYPE_INTERNAL:
508	{
509	rc = pBlkCache->u.Int.pfnXferEnqueue(pBlkCache->u.Int.pvUser,
510	pIoXfer->enmXferDir,
511	off, pIoXfer->cbXfer,
512	&pIoXfer->SgBuf, pIoXfer);
513	break;
514	}
515	default:
516	AssertMsgFailed(("Unknown block cache type!\n"));
517	}
518
519	return rc;
520	}
521
522	/**
523	* Initiates a read I/O task for the given entry.
524	*
525	* @returns VBox status code.
526	* @param pEntry The entry to fetch the data to.
527	*/
528	static int pdmBlkCacheEntryReadFromMedium(PPDMBLKCACHEENTRY pEntry)
529	{
530	PPDMBLKCACHE pBlkCache = pEntry->pBlkCache;
531	LogFlowFunc((": Reading data into cache entry %#p\n", pEntry));
532
533	/* Make sure no one evicts the entry while it is accessed. */
534	pEntry->fFlags \|= PDMBLKCACHE_ENTRY_IO_IN_PROGRESS;
535
536	PPDMBLKCACHEIOXFER pIoXfer = (PPDMBLKCACHEIOXFER)RTMemAllocZ(sizeof(PDMBLKCACHEIOXFER));
537	if (RT_UNLIKELY(!pIoXfer))
538	return VERR_NO_MEMORY;
539
540	AssertMsg(pEntry->pbData, ("Entry is in ghost state\n"));
541
542	pIoXfer->fIoCache = true;
543	pIoXfer->pEntry = pEntry;
544	pIoXfer->SgSeg.pvSeg = pEntry->pbData;
545	pIoXfer->SgSeg.cbSeg = pEntry->cbData;
546	pIoXfer->cbXfer = pEntry->cbData;
547	pIoXfer->enmXferDir = PDMBLKCACHEXFERDIR_READ;
548	RTSgBufInit(&pIoXfer->SgBuf, &pIoXfer->SgSeg, 1);
549
550	return pdmBlkCacheEnqueue(pBlkCache, pEntry->Core.Key, pIoXfer);
551	}
552
553	/**
554	* Initiates a write I/O task for the given entry.
555	*
556	* @returns nothing.
557	* @param pEntry The entry to read the data from.
558	*/
559	static int pdmBlkCacheEntryWriteToMedium(PPDMBLKCACHEENTRY pEntry)
560	{
561	PPDMBLKCACHE pBlkCache = pEntry->pBlkCache;
562	LogFlowFunc((": Writing data from cache entry %#p\n", pEntry));
563
564	/* Make sure no one evicts the entry while it is accessed. */
565	pEntry->fFlags \|= PDMBLKCACHE_ENTRY_IO_IN_PROGRESS;
566
567	PPDMBLKCACHEIOXFER pIoXfer = (PPDMBLKCACHEIOXFER)RTMemAllocZ(sizeof(PDMBLKCACHEIOXFER));
568	if (RT_UNLIKELY(!pIoXfer))
569	return VERR_NO_MEMORY;
570
571	AssertMsg(pEntry->pbData, ("Entry is in ghost state\n"));
572
573	pIoXfer->fIoCache = true;
574	pIoXfer->pEntry = pEntry;
575	pIoXfer->SgSeg.pvSeg = pEntry->pbData;
576	pIoXfer->SgSeg.cbSeg = pEntry->cbData;
577	pIoXfer->cbXfer = pEntry->cbData;
578	pIoXfer->enmXferDir = PDMBLKCACHEXFERDIR_WRITE;
579	RTSgBufInit(&pIoXfer->SgBuf, &pIoXfer->SgSeg, 1);
580
581	return pdmBlkCacheEnqueue(pBlkCache, pEntry->Core.Key, pIoXfer);
582	}
583
584	/**
585	* Passthrough a part of a request directly to the I/O manager
586	* handling the endpoint.
587	*
588	* @returns VBox status code.
589	* @param pEndpoint The endpoint.
590	* @param pTask The task.
591	* @param pIoMemCtx The I/O memory context to use.
592	* @param offStart Offset to start transfer from.
593	* @param cbData Amount of data to transfer.
594	* @param enmTransferType The transfer type (read/write)
595	*/
596	static int pdmBlkCacheRequestPassthrough(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEREQ pReq,
597	PRTSGBUF pSgBuf, uint64_t offStart, size_t cbData,
598	PDMBLKCACHEXFERDIR enmXferDir)
599	{
600
601	PPDMBLKCACHEIOXFER pIoXfer = (PPDMBLKCACHEIOXFER)RTMemAllocZ(sizeof(PDMBLKCACHEIOXFER));
602	if (RT_UNLIKELY(!pIoXfer))
603	return VERR_NO_MEMORY;
604
605	ASMAtomicIncU32(&pReq->cXfersPending);
606	pIoXfer->fIoCache = false;
607	pIoXfer->pReq = pReq;
608	pIoXfer->cbXfer = cbData;
609	pIoXfer->enmXferDir = enmXferDir;
610	if (pSgBuf)
611	{
612	RTSgBufClone(&pIoXfer->SgBuf, pSgBuf);
613	RTSgBufAdvance(pSgBuf, cbData);
614	}
615
616	return pdmBlkCacheEnqueue(pBlkCache, offStart, pIoXfer);
617	}
618
619	/**
620	* Commit a single dirty entry to the endpoint
621	*
622	* @returns nothing
623	* @param pEntry The entry to commit.
624	*/
625	static void pdmBlkCacheEntryCommit(PPDMBLKCACHEENTRY pEntry)
626	{
627	AssertMsg( (pEntry->fFlags & PDMBLKCACHE_ENTRY_IS_DIRTY)
628	&& !(pEntry->fFlags & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS),
629	("Invalid flags set for entry %#p\n", pEntry));
630
631	pdmBlkCacheEntryWriteToMedium(pEntry);
632	}
633
634	/**
635	* Commit all dirty entries for a single endpoint.
636	*
637	* @returns nothing.
638	* @param pBlkCache The endpoint cache to commit.
639	*/
640	static void pdmBlkCacheCommit(PPDMBLKCACHE pBlkCache)
641	{
642	uint32_t cbCommitted = 0;
643	RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
644
645	/* The list is moved to a new header to reduce locking overhead. */
646	RTLISTNODE ListDirtyNotCommitted;
647	RTSPINLOCKTMP Tmp;
648
649	RTListInit(&ListDirtyNotCommitted);
650	RTSpinlockAcquire(pBlkCache->LockList, &Tmp);
651	RTListMove(&ListDirtyNotCommitted, &pBlkCache->ListDirtyNotCommitted);
652	RTSpinlockRelease(pBlkCache->LockList, &Tmp);
653
654	if (!RTListIsEmpty(&ListDirtyNotCommitted))
655	{
656	PPDMBLKCACHEENTRY pEntry = RTListNodeGetFirst(&ListDirtyNotCommitted,
657	PDMBLKCACHEENTRY,
658	NodeNotCommitted);
659
660	while (!RTListNodeIsLast(&ListDirtyNotCommitted, &pEntry->NodeNotCommitted))
661	{
662	PPDMBLKCACHEENTRY pNext = RTListNodeGetNext(&pEntry->NodeNotCommitted, PDMBLKCACHEENTRY,
663	NodeNotCommitted);
664	pdmBlkCacheEntryCommit(pEntry);
665	cbCommitted += pEntry->cbData;
666	RTListNodeRemove(&pEntry->NodeNotCommitted);
667	pEntry = pNext;
668	}
669
670	/* Commit the last endpoint */
671	Assert(RTListNodeIsLast(&ListDirtyNotCommitted, &pEntry->NodeNotCommitted));
672	pdmBlkCacheEntryCommit(pEntry);
673	RTListNodeRemove(&pEntry->NodeNotCommitted);
674	AssertMsg(RTListIsEmpty(&ListDirtyNotCommitted),
675	("Committed all entries but list is not empty\n"));
676	}
677
678	RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
679	AssertMsg(pBlkCache->pCache->cbDirty >= cbCommitted,
680	("Number of committed bytes exceeds number of dirty bytes\n"));
681	ASMAtomicSubU32(&pBlkCache->pCache->cbDirty, cbCommitted);
682	}
683
684	/**
685	* Commit all dirty entries in the cache.
686	*
687	* @returns nothing.
688	* @param pCache The global cache instance.
689	*/
690	static void pdmBlkCacheCommitDirtyEntries(PPDMBLKCACHEGLOBAL pCache)
691	{
692	bool fCommitInProgress = ASMAtomicXchgBool(&pCache->fCommitInProgress, true);
693
694	if (!fCommitInProgress)
695	{
696	pdmBlkCacheLockEnter(pCache);
697	Assert(!RTListIsEmpty(&pCache->ListUsers));
698
699	PPDMBLKCACHE pBlkCache = RTListNodeGetFirst(&pCache->ListUsers,
700	PDMBLKCACHE,
701	NodeCacheUser);
702	AssertPtr(pBlkCache);
703
704	while (!RTListNodeIsLast(&pCache->ListUsers, &pBlkCache->NodeCacheUser))
705	{
706	pdmBlkCacheCommit(pBlkCache);
707
708	pBlkCache = RTListNodeGetNext(&pBlkCache->NodeCacheUser, PDMBLKCACHE,
709	NodeCacheUser);
710	}
711
712	/* Commit the last endpoint */
713	Assert(RTListNodeIsLast(&pCache->ListUsers, &pBlkCache->NodeCacheUser));
714	pdmBlkCacheCommit(pBlkCache);
715
716	pdmBlkCacheLockLeave(pCache);
717	ASMAtomicWriteBool(&pCache->fCommitInProgress, false);
718	}
719	}
720
721	/**
722	* Adds the given entry as a dirty to the cache.
723	*
724	* @returns Flag whether the amount of dirty bytes in the cache exceeds the threshold
725	* @param pBlkCache The endpoint cache the entry belongs to.
726	* @param pEntry The entry to add.
727	*/
728	static bool pdmBlkCacheAddDirtyEntry(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEENTRY pEntry)
729	{
730	bool fDirtyBytesExceeded = false;
731	PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
732
733	/* If the commit timer is disabled we commit right away. */
734	if (pCache->u32CommitTimeoutMs == 0)
735	{
736	pEntry->fFlags \|= PDMBLKCACHE_ENTRY_IS_DIRTY;
737	pdmBlkCacheEntryCommit(pEntry);
738	}
739	else if (!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IS_DIRTY))
740	{
741	pEntry->fFlags \|= PDMBLKCACHE_ENTRY_IS_DIRTY;
742
743	RTSPINLOCKTMP Tmp;
744	RTSpinlockAcquire(pBlkCache->LockList, &Tmp);
745	RTListAppend(&pBlkCache->ListDirtyNotCommitted, &pEntry->NodeNotCommitted);
746	RTSpinlockRelease(pBlkCache->LockList, &Tmp);
747
748	uint32_t cbDirty = ASMAtomicAddU32(&pCache->cbDirty, pEntry->cbData);
749
750	/* Prevent committing if the VM was suspended. */
751	if (RT_LIKELY(!ASMAtomicReadBool(&pCache->fIoErrorVmSuspended)))
752	fDirtyBytesExceeded = (cbDirty >= pCache->cbCommitDirtyThreshold);
753	}
754
755	return fDirtyBytesExceeded;
756	}
757
758	/**
759	* Commit timer callback.
760	*/
761	static void pdmBlkCacheCommitTimerCallback(PVM pVM, PTMTIMER pTimer, void *pvUser)
762	{
763	PPDMBLKCACHEGLOBAL pCache = (PPDMBLKCACHEGLOBAL)pvUser;
764
765	LogFlowFunc(("Commit interval expired, commiting dirty entries\n"));
766
767	if ( ASMAtomicReadU32(&pCache->cbDirty) > 0
768	&& !ASMAtomicReadBool(&pCache->fIoErrorVmSuspended))
769	pdmBlkCacheCommitDirtyEntries(pCache);
770
771	TMTimerSetMillies(pTimer, pCache->u32CommitTimeoutMs);
772	LogFlowFunc(("Entries committed, going to sleep\n"));
773	}
774
775	int pdmR3BlkCacheInit(PVM pVM)
776	{
777	int rc = VINF_SUCCESS;
778	PUVM pUVM = pVM->pUVM;
779	PPDMBLKCACHEGLOBAL pBlkCacheGlobal;
780
781	LogFlowFunc((": pVM=%p\n", pVM));
782
783	VM_ASSERT_EMT(pVM);
784
785	PCFGMNODE pCfgRoot = CFGMR3GetRoot(pVM);
786	PCFGMNODE pCfgBlkCache = CFGMR3GetChild(CFGMR3GetChild(pCfgRoot, "PDM"), "BlkCache");
787
788	pBlkCacheGlobal = (PPDMBLKCACHEGLOBAL)RTMemAllocZ(sizeof(PDMBLKCACHEGLOBAL));
789	if (!pBlkCacheGlobal)
790	return VERR_NO_MEMORY;
791
792	RTListInit(&pBlkCacheGlobal->ListUsers);
793	pBlkCacheGlobal->pVM = pVM;
794	pBlkCacheGlobal->cRefs = 0;
795	pBlkCacheGlobal->cbCached = 0;
796	pBlkCacheGlobal->fCommitInProgress = false;
797
798	/* Initialize members */
799	pBlkCacheGlobal->LruRecentlyUsedIn.pHead = NULL;
800	pBlkCacheGlobal->LruRecentlyUsedIn.pTail = NULL;
801	pBlkCacheGlobal->LruRecentlyUsedIn.cbCached = 0;
802
803	pBlkCacheGlobal->LruRecentlyUsedOut.pHead = NULL;
804	pBlkCacheGlobal->LruRecentlyUsedOut.pTail = NULL;
805	pBlkCacheGlobal->LruRecentlyUsedOut.cbCached = 0;
806
807	pBlkCacheGlobal->LruFrequentlyUsed.pHead = NULL;
808	pBlkCacheGlobal->LruFrequentlyUsed.pTail = NULL;
809	pBlkCacheGlobal->LruFrequentlyUsed.cbCached = 0;
810
811	do
812	{
813	rc = CFGMR3QueryU32Def(pCfgBlkCache, "CacheSize", &pBlkCacheGlobal->cbMax, 5 * _1M);
814	AssertLogRelRCBreak(rc);
815	LogFlowFunc(("Maximum number of bytes cached %u\n", pBlkCacheGlobal->cbMax));
816
817	pBlkCacheGlobal->cbRecentlyUsedInMax = (pBlkCacheGlobal->cbMax / 100) * 25; /* 25% of the buffer size */
818	pBlkCacheGlobal->cbRecentlyUsedOutMax = (pBlkCacheGlobal->cbMax / 100) * 50; /* 50% of the buffer size */
819	LogFlowFunc(("cbRecentlyUsedInMax=%u cbRecentlyUsedOutMax=%u\n",
820	pBlkCacheGlobal->cbRecentlyUsedInMax, pBlkCacheGlobal->cbRecentlyUsedOutMax));
821
822	/** @todo r=aeichner: Experiment to find optimal default values */
823	rc = CFGMR3QueryU32Def(pCfgBlkCache, "CacheCommitIntervalMs", &pBlkCacheGlobal->u32CommitTimeoutMs, 10000 /* 10sec */);
824	AssertLogRelRCBreak(rc);
825	rc = CFGMR3QueryU32Def(pCfgBlkCache, "CacheCommitThreshold", &pBlkCacheGlobal->cbCommitDirtyThreshold, pBlkCacheGlobal->cbMax / 2);
826	AssertLogRelRCBreak(rc);
827	} while (0);
828
829	if (RT_SUCCESS(rc))
830	{
831	STAMR3Register(pVM, &pBlkCacheGlobal->cbMax,
832	STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
833	"/PDM/BlkCache/cbMax",
834	STAMUNIT_BYTES,
835	"Maximum cache size");
836	STAMR3Register(pVM, &pBlkCacheGlobal->cbCached,
837	STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
838	"/PDM/BlkCache/cbCached",
839	STAMUNIT_BYTES,
840	"Currently used cache");
841	STAMR3Register(pVM, &pBlkCacheGlobal->LruRecentlyUsedIn.cbCached,
842	STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
843	"/PDM/BlkCache/cbCachedMruIn",
844	STAMUNIT_BYTES,
845	"Number of bytes cached in MRU list");
846	STAMR3Register(pVM, &pBlkCacheGlobal->LruRecentlyUsedOut.cbCached,
847	STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
848	"/PDM/BlkCache/cbCachedMruOut",
849	STAMUNIT_BYTES,
850	"Number of bytes cached in FRU list");
851	STAMR3Register(pVM, &pBlkCacheGlobal->LruFrequentlyUsed.cbCached,
852	STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
853	"/PDM/BlkCache/cbCachedFru",
854	STAMUNIT_BYTES,
855	"Number of bytes cached in FRU ghost list");
856
857	#ifdef VBOX_WITH_STATISTICS
858	STAMR3Register(pVM, &pBlkCacheGlobal->cHits,
859	STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
860	"/PDM/BlkCache/CacheHits",
861	STAMUNIT_COUNT, "Number of hits in the cache");
862	STAMR3Register(pVM, &pBlkCacheGlobal->cPartialHits,
863	STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
864	"/PDM/BlkCache/CachePartialHits",
865	STAMUNIT_COUNT, "Number of partial hits in the cache");
866	STAMR3Register(pVM, &pBlkCacheGlobal->cMisses,
867	STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
868	"/PDM/BlkCache/CacheMisses",
869	STAMUNIT_COUNT, "Number of misses when accessing the cache");
870	STAMR3Register(pVM, &pBlkCacheGlobal->StatRead,
871	STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
872	"/PDM/BlkCache/CacheRead",
873	STAMUNIT_BYTES, "Number of bytes read from the cache");
874	STAMR3Register(pVM, &pBlkCacheGlobal->StatWritten,
875	STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
876	"/PDM/BlkCache/CacheWritten",
877	STAMUNIT_BYTES, "Number of bytes written to the cache");
878	STAMR3Register(pVM, &pBlkCacheGlobal->StatTreeGet,
879	STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
880	"/PDM/BlkCache/CacheTreeGet",
881	STAMUNIT_TICKS_PER_CALL, "Time taken to access an entry in the tree");
882	STAMR3Register(pVM, &pBlkCacheGlobal->StatTreeInsert,
883	STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
884	"/PDM/BlkCache/CacheTreeInsert",
885	STAMUNIT_TICKS_PER_CALL, "Time taken to insert an entry in the tree");
886	STAMR3Register(pVM, &pBlkCacheGlobal->StatTreeRemove,
887	STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
888	"/PDM/BlkCache/CacheTreeRemove",
889	STAMUNIT_TICKS_PER_CALL, "Time taken to remove an entry an the tree");
890	STAMR3Register(pVM, &pBlkCacheGlobal->StatBuffersReused,
891	STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
892	"/PDM/BlkCache/CacheBuffersReused",
893	STAMUNIT_COUNT, "Number of times a buffer could be reused");
894	#endif
895
896	/* Initialize the critical section */
897	rc = RTCritSectInit(&pBlkCacheGlobal->CritSect);
898	}
899
900	if (RT_SUCCESS(rc))
901	{
902	/* Create the commit timer */
903	if (pBlkCacheGlobal->u32CommitTimeoutMs > 0)
904	rc = TMR3TimerCreateInternal(pVM, TMCLOCK_REAL,
905	pdmBlkCacheCommitTimerCallback,
906	pBlkCacheGlobal,
907	"BlkCache-Commit",
908	&pBlkCacheGlobal->pTimerCommit);
909
910	if (RT_SUCCESS(rc))
911	{
912	LogRel(("BlkCache: Cache successfully initialised. Cache size is %u bytes\n", pBlkCacheGlobal->cbMax));
913	LogRel(("BlkCache: Cache commit interval is %u ms\n", pBlkCacheGlobal->u32CommitTimeoutMs));
914	LogRel(("BlkCache: Cache commit threshold is %u bytes\n", pBlkCacheGlobal->cbCommitDirtyThreshold));
915	pUVM->pdm.s.pBlkCacheGlobal = pBlkCacheGlobal;
916	return VINF_SUCCESS;
917	}
918
919	RTCritSectDelete(&pBlkCacheGlobal->CritSect);
920	}
921
922	if (pBlkCacheGlobal)
923	RTMemFree(pBlkCacheGlobal);
924
925	LogFlowFunc((": returns rc=%Rrc\n", pVM, rc));
926	return rc;
927	}
928
929	void pdmR3BlkCacheTerm(PVM pVM)
930	{
931	PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
932
933	if (pBlkCacheGlobal)
934	{
935	/* Make sure no one else uses the cache now */
936	pdmBlkCacheLockEnter(pBlkCacheGlobal);
937
938	/* Cleanup deleting all cache entries waiting for in progress entries to finish. */
939	pdmBlkCacheDestroyList(&pBlkCacheGlobal->LruRecentlyUsedIn);
940	pdmBlkCacheDestroyList(&pBlkCacheGlobal->LruRecentlyUsedOut);
941	pdmBlkCacheDestroyList(&pBlkCacheGlobal->LruFrequentlyUsed);
942
943	pdmBlkCacheLockLeave(pBlkCacheGlobal);
944
945	RTCritSectDelete(&pBlkCacheGlobal->CritSect);
946	RTMemFree(pBlkCacheGlobal);
947	pVM->pUVM->pdm.s.pBlkCacheGlobal = NULL;
948	}
949	}
950
951	int pdmR3BlkCacheResume(PVM pVM)
952	{
953	PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
954
955	LogFlowFunc(("pVM=%#p\n", pVM));
956
957	if ( pBlkCacheGlobal
958	&& ASMAtomicXchgBool(&pBlkCacheGlobal->fIoErrorVmSuspended, false))
959	{
960	/* The VM was suspended because of an I/O error, commit all dirty entries. */
961	pdmBlkCacheCommitDirtyEntries(pBlkCacheGlobal);
962	}
963
964	return VINF_SUCCESS;
965	}
966
967	static int pdmR3BlkCacheRetain(PVM pVM, PPPDMBLKCACHE ppBlkCache, const char *pcszId)
968	{
969	int rc = VINF_SUCCESS;
970	PPDMBLKCACHE pBlkCache = NULL;
971	bool fAlreadyExists = false;
972	PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
973
974	if (!pBlkCacheGlobal)
975	return VERR_NOT_SUPPORTED;
976
977	/*
978	* Check that no other user cache has the same id first,
979	* Unique id's are necessary in case the state is saved.
980	*/
981	pdmBlkCacheLockEnter(pBlkCacheGlobal);
982
983	RTListForEach(&pBlkCacheGlobal->ListUsers, pBlkCache, PDMBLKCACHE, NodeCacheUser)
984	{
985	if (!RTStrCmp(pBlkCache->pszId, pcszId))
986	{
987	fAlreadyExists = true;
988	break;
989	}
990	}
991
992	if (!fAlreadyExists)
993	{
994	pBlkCache = (PPDMBLKCACHE)RTMemAllocZ(sizeof(PDMBLKCACHE));
995
996	if (pBlkCache)
997	pBlkCache->pszId = RTStrDup(pcszId);
998
999	if ( pBlkCache
1000	&& pBlkCache->pszId)
1001	{
1002	pBlkCache->pCache = pBlkCacheGlobal;
1003	RTListInit(&pBlkCache->ListDirtyNotCommitted);
1004
1005	rc = RTSpinlockCreate(&pBlkCache->LockList);
1006	if (RT_SUCCESS(rc))
1007	{
1008	rc = RTSemRWCreate(&pBlkCache->SemRWEntries);
1009	if (RT_SUCCESS(rc))
1010	{
1011	pBlkCache->pTree = (PAVLRU64TREE)RTMemAllocZ(sizeof(AVLRFOFFTREE));
1012	if (pBlkCache->pTree)
1013	{
1014	/* Arm the timer if this is the first endpoint. */
1015	if ( pBlkCacheGlobal->cRefs == 1
1016	&& pBlkCacheGlobal->u32CommitTimeoutMs > 0)
1017	rc = TMTimerSetMillies(pBlkCacheGlobal->pTimerCommit, pBlkCacheGlobal->u32CommitTimeoutMs);
1018
1019	if (RT_SUCCESS(rc))
1020	{
1021	#ifdef VBOX_WITH_STATISTICS
1022	STAMR3RegisterF(pBlkCacheGlobal->pVM, &pBlkCache->StatWriteDeferred,
1023	STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
1024	STAMUNIT_COUNT, "Number of deferred writes",
1025	"/PDM/BlkCache/%s/Cache/DeferredWrites", pBlkCache->pszId);
1026	#endif
1027
1028	/* Add to the list of users. */
1029	pBlkCacheGlobal->cRefs++;
1030	RTListAppend(&pBlkCacheGlobal->ListUsers, &pBlkCache->NodeCacheUser);
1031	pdmBlkCacheLockLeave(pBlkCacheGlobal);
1032
1033	*ppBlkCache = pBlkCache;
1034	LogFlowFunc(("returns success\n"));
1035	return VINF_SUCCESS;
1036	}
1037	}
1038	else
1039	rc = VERR_NO_MEMORY;
1040
1041	RTSemRWDestroy(pBlkCache->SemRWEntries);
1042	}
1043
1044	RTSpinlockDestroy(pBlkCache->LockList);
1045	}
1046
1047	RTStrFree(pBlkCache->pszId);
1048	}
1049	else
1050	rc = VERR_NO_MEMORY;
1051
1052	if (pBlkCache)
1053	RTMemFree(pBlkCache);
1054	}
1055	else
1056	rc = VERR_ALREADY_EXISTS;
1057
1058	pdmBlkCacheLockLeave(pBlkCacheGlobal);
1059
1060	LogFlowFunc(("Leave rc=%Rrc\n", rc));
1061	return rc;
1062	}
1063
1064	VMMR3DECL(int) PDMR3BlkCacheRetainDriver(PVM pVM, PPDMDRVINS pDrvIns, PPPDMBLKCACHE ppBlkCache,
1065	PFNPDMBLKCACHEXFERCOMPLETEDRV pfnXferComplete,
1066	PFNPDMBLKCACHEXFERENQUEUEDRV pfnXferEnqueue,
1067	const char *pcszId)
1068	{
1069	int rc = VINF_SUCCESS;
1070	PPDMBLKCACHE pBlkCache;
1071
1072	rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
1073	if (RT_SUCCESS(rc))
1074	{
1075	pBlkCache->enmType = PDMBLKCACHETYPE_DRV;
1076	pBlkCache->u.Drv.pfnXferComplete = pfnXferComplete;
1077	pBlkCache->u.Drv.pfnXferEnqueue = pfnXferEnqueue;
1078	pBlkCache->u.Drv.pDrvIns = pDrvIns;
1079	*ppBlkCache = pBlkCache;
1080	}
1081
1082	LogFlowFunc(("Leave rc=%Rrc\n", rc));
1083	return rc;
1084	}
1085
1086	VMMR3DECL(int) PDMR3BlkCacheRetainDevice(PVM pVM, PPDMDEVINS pDevIns, PPPDMBLKCACHE ppBlkCache,
1087	PFNPDMBLKCACHEXFERCOMPLETEDEV pfnXferComplete,
1088	PFNPDMBLKCACHEXFERENQUEUEDEV pfnXferEnqueue,
1089	const char *pcszId)
1090	{
1091	int rc = VINF_SUCCESS;
1092	PPDMBLKCACHE pBlkCache;
1093
1094	rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
1095	if (RT_SUCCESS(rc))
1096	{
1097	pBlkCache->enmType = PDMBLKCACHETYPE_DEV;
1098	pBlkCache->u.Dev.pfnXferComplete = pfnXferComplete;
1099	pBlkCache->u.Dev.pfnXferEnqueue = pfnXferEnqueue;
1100	pBlkCache->u.Dev.pDevIns = pDevIns;
1101	*ppBlkCache = pBlkCache;
1102	}
1103
1104	LogFlowFunc(("Leave rc=%Rrc\n", rc));
1105	return rc;
1106
1107	}
1108
1109	VMMR3DECL(int) PDMR3BlkCacheRetainUsb(PVM pVM, PPDMUSBINS pUsbIns, PPPDMBLKCACHE ppBlkCache,
1110	PFNPDMBLKCACHEXFERCOMPLETEUSB pfnXferComplete,
1111	PFNPDMBLKCACHEXFERENQUEUEUSB pfnXferEnqueue,
1112	const char *pcszId)
1113	{
1114	int rc = VINF_SUCCESS;
1115	PPDMBLKCACHE pBlkCache;
1116
1117	rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
1118	if (RT_SUCCESS(rc))
1119	{
1120	pBlkCache->enmType = PDMBLKCACHETYPE_USB;
1121	pBlkCache->u.Usb.pfnXferComplete = pfnXferComplete;
1122	pBlkCache->u.Usb.pfnXferEnqueue = pfnXferEnqueue;
1123	pBlkCache->u.Usb.pUsbIns = pUsbIns;
1124	*ppBlkCache = pBlkCache;
1125	}
1126
1127	LogFlowFunc(("Leave rc=%Rrc\n", rc));
1128	return rc;
1129
1130	}
1131
1132	VMMR3DECL(int) PDMR3BlkCacheRetainInt(PVM pVM, void *pvUser, PPPDMBLKCACHE ppBlkCache,
1133	PFNPDMBLKCACHEXFERCOMPLETEINT pfnXferComplete,
1134	PFNPDMBLKCACHEXFERENQUEUEINT pfnXferEnqueue,
1135	const char *pcszId)
1136	{
1137	int rc = VINF_SUCCESS;
1138	PPDMBLKCACHE pBlkCache;
1139
1140	rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
1141	if (RT_SUCCESS(rc))
1142	{
1143	pBlkCache->enmType = PDMBLKCACHETYPE_INTERNAL;
1144	pBlkCache->u.Int.pfnXferComplete = pfnXferComplete;
1145	pBlkCache->u.Int.pfnXferEnqueue = pfnXferEnqueue;
1146	pBlkCache->u.Int.pvUser = pvUser;
1147	*ppBlkCache = pBlkCache;
1148	}
1149
1150	LogFlowFunc(("Leave rc=%Rrc\n", rc));
1151	return rc;
1152
1153	}
1154
1155	/**
1156	* Callback for the AVL destroy routine. Frees a cache entry for this endpoint.
1157	*
1158	* @returns IPRT status code.
1159	* @param pNode The node to destroy.
1160	* @param pvUser Opaque user data.
1161	*/
1162	static int pdmBlkCacheEntryDestroy(PAVLRU64NODECORE pNode, void *pvUser)
1163	{
1164	PPDMBLKCACHEENTRY pEntry = (PPDMBLKCACHEENTRY)pNode;
1165	PPDMBLKCACHEGLOBAL pCache = (PPDMBLKCACHEGLOBAL)pvUser;
1166	PPDMBLKCACHE pBlkCache = pEntry->pBlkCache;
1167
1168	while (ASMAtomicReadU32(&pEntry->fFlags) & (PDMBLKCACHE_ENTRY_IO_IN_PROGRESS \| PDMBLKCACHE_ENTRY_IS_DIRTY))
1169	{
1170	/* Leave the locks to let the I/O thread make progress but reference the entry to prevent eviction. */
1171	pdmBlkCacheEntryRef(pEntry);
1172	RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
1173	pdmBlkCacheLockLeave(pCache);
1174
1175	RTThreadSleep(250);
1176
1177	/* Re-enter all locks */
1178	pdmBlkCacheLockEnter(pCache);
1179	RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
1180	pdmBlkCacheEntryRelease(pEntry);
1181	}
1182
1183	AssertMsg(!(pEntry->fFlags & (PDMBLKCACHE_ENTRY_IO_IN_PROGRESS \| PDMBLKCACHE_ENTRY_IS_DIRTY)),
1184	("Entry is dirty and/or still in progress fFlags=%#x\n", pEntry->fFlags));
1185
1186	bool fUpdateCache = pEntry->pList == &pCache->LruFrequentlyUsed
1187	\|\| pEntry->pList == &pCache->LruRecentlyUsedIn;
1188
1189	pdmBlkCacheEntryRemoveFromList(pEntry);
1190
1191	if (fUpdateCache)
1192	pdmBlkCacheSub(pCache, pEntry->cbData);
1193
1194	RTMemPageFree(pEntry->pbData, pEntry->cbData);
1195	RTMemFree(pEntry);
1196
1197	return VINF_SUCCESS;
1198	}
1199
1200	/**
1201	* Destroys all cache resources used by the given endpoint.
1202	*
1203	* @returns nothing.
1204	* @param pEndpoint The endpoint to the destroy.
1205	*/
1206	VMMR3DECL(void) PDMR3BlkCacheRelease(PPDMBLKCACHE pBlkCache)
1207	{
1208	PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
1209
1210	/* Make sure nobody is accessing the cache while we delete the tree. */
1211	pdmBlkCacheLockEnter(pCache);
1212	RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
1213	RTAvlrU64Destroy(pBlkCache->pTree, pdmBlkCacheEntryDestroy, pCache);
1214	RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
1215
1216	RTSpinlockDestroy(pBlkCache->LockList);
1217
1218	pCache->cRefs--;
1219	RTListNodeRemove(&pBlkCache->NodeCacheUser);
1220
1221	if ( !pCache->cRefs
1222	&& pCache->u32CommitTimeoutMs > 0)
1223	TMTimerStop(pCache->pTimerCommit);
1224
1225	pdmBlkCacheLockLeave(pCache);
1226
1227	RTSemRWDestroy(pBlkCache->SemRWEntries);
1228
1229	#ifdef VBOX_WITH_STATISTICS
1230	STAMR3Deregister(pCache->pVM, &pBlkCache->StatWriteDeferred);
1231	#endif
1232
1233	RTStrFree(pBlkCache->pszId);
1234	RTMemFree(pBlkCache);
1235	}
1236
1237	VMMR3DECL(void) PDMR3BlkCacheReleaseDevice(PVM pVM, PPDMDEVINS pDevIns)
1238	{
1239	LogFlow(("%s: pDevIns=%p\n", __FUNCTION__, pDevIns));
1240
1241	/*
1242	* Validate input.
1243	*/
1244	if (!pDevIns)
1245	return;
1246	VM_ASSERT_EMT(pVM);
1247
1248	PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
1249	PPDMBLKCACHE pBlkCache, pBlkCacheNext;
1250
1251	/* Return silently if not supported. */
1252	if (!pBlkCacheGlobal)
1253	return;
1254
1255	pdmBlkCacheLockEnter(pBlkCacheGlobal);
1256
1257	RTListForEachSafe(&pBlkCacheGlobal->ListUsers, pBlkCache, pBlkCacheNext, PDMBLKCACHE, NodeCacheUser)
1258	{
1259	if ( pBlkCache->enmType == PDMBLKCACHETYPE_DEV
1260	&& pBlkCache->u.Dev.pDevIns == pDevIns)
1261	PDMR3BlkCacheRelease(pBlkCache);
1262	}
1263
1264	pdmBlkCacheLockLeave(pBlkCacheGlobal);
1265	}
1266
1267	VMMR3DECL(void) PDMR3BlkCacheReleaseDriver(PVM pVM, PPDMDRVINS pDrvIns)
1268	{
1269	LogFlow(("%s: pDrvIns=%p\n", __FUNCTION__, pDrvIns));
1270
1271	/*
1272	* Validate input.
1273	*/
1274	if (!pDrvIns)
1275	return;
1276	VM_ASSERT_EMT(pVM);
1277
1278	PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
1279	PPDMBLKCACHE pBlkCache, pBlkCacheNext;
1280
1281	/* Return silently if not supported. */
1282	if (!pBlkCacheGlobal)
1283	return;
1284
1285	pdmBlkCacheLockEnter(pBlkCacheGlobal);
1286
1287	RTListForEachSafe(&pBlkCacheGlobal->ListUsers, pBlkCache, pBlkCacheNext, PDMBLKCACHE, NodeCacheUser)
1288	{
1289	if ( pBlkCache->enmType == PDMBLKCACHETYPE_DRV
1290	&& pBlkCache->u.Drv.pDrvIns == pDrvIns)
1291	PDMR3BlkCacheRelease(pBlkCache);
1292	}
1293
1294	pdmBlkCacheLockLeave(pBlkCacheGlobal);
1295	}
1296
1297	VMMR3DECL(void) PDMR3BlkCacheReleaseUsb(PVM pVM, PPDMUSBINS pUsbIns)
1298	{
1299	LogFlow(("%s: pUsbIns=%p\n", __FUNCTION__, pUsbIns));
1300
1301	/*
1302	* Validate input.
1303	*/
1304	if (!pUsbIns)
1305	return;
1306	VM_ASSERT_EMT(pVM);
1307
1308	PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
1309	PPDMBLKCACHE pBlkCache, pBlkCacheNext;
1310
1311	/* Return silently if not supported. */
1312	if (!pBlkCacheGlobal)
1313	return;
1314
1315	pdmBlkCacheLockEnter(pBlkCacheGlobal);
1316
1317	RTListForEachSafe(&pBlkCacheGlobal->ListUsers, pBlkCache, pBlkCacheNext, PDMBLKCACHE, NodeCacheUser)
1318	{
1319	if ( pBlkCache->enmType == PDMBLKCACHETYPE_USB
1320	&& pBlkCache->u.Usb.pUsbIns == pUsbIns)
1321	PDMR3BlkCacheRelease(pBlkCache);
1322	}
1323
1324	pdmBlkCacheLockLeave(pBlkCacheGlobal);
1325	}
1326
1327	static PPDMBLKCACHEENTRY pdmBlkCacheGetCacheEntryByOffset(PPDMBLKCACHE pBlkCache, uint64_t off)
1328	{
1329	PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
1330	PPDMBLKCACHEENTRY pEntry = NULL;
1331
1332	STAM_PROFILE_ADV_START(&pCache->StatTreeGet, Cache);
1333
1334	RTSemRWRequestRead(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
1335	pEntry = (PPDMBLKCACHEENTRY)RTAvlrU64RangeGet(pBlkCache->pTree, off);
1336	if (pEntry)
1337	pdmBlkCacheEntryRef(pEntry);
1338	RTSemRWReleaseRead(pBlkCache->SemRWEntries);
1339
1340	STAM_PROFILE_ADV_STOP(&pCache->StatTreeGet, Cache);
1341
1342	return pEntry;
1343	}
1344
1345	/**
1346	* Return the best fit cache entries for the given offset.
1347	*
1348	* @returns nothing.
1349	* @param pBlkCache The endpoint cache.
1350	* @param off The offset.
1351	* @param pEntryAbove Where to store the pointer to the best fit entry above the
1352	* the given offset. NULL if not required.
1353	* @param pEntryBelow Where to store the pointer to the best fit entry below the
1354	* the given offset. NULL if not required.
1355	*/
1356	static void pdmBlkCacheGetCacheBestFitEntryByOffset(PPDMBLKCACHE pBlkCache, uint64_t off,
1357	PPDMBLKCACHEENTRY *ppEntryAbove,
1358	PPDMBLKCACHEENTRY *ppEntryBelow)
1359	{
1360	PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
1361
1362	STAM_PROFILE_ADV_START(&pCache->StatTreeGet, Cache);
1363
1364	RTSemRWRequestRead(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
1365	if (ppEntryAbove)
1366	{
1367	ppEntryAbove = (PPDMBLKCACHEENTRY)RTAvlrU64GetBestFit(pBlkCache->pTree, off, true /fAbove*/);
1368	if (*ppEntryAbove)
1369	pdmBlkCacheEntryRef(*ppEntryAbove);
1370	}
1371
1372	if (ppEntryBelow)
1373	{
1374	ppEntryBelow = (PPDMBLKCACHEENTRY)RTAvlrU64GetBestFit(pBlkCache->pTree, off, false /fAbove*/);
1375	if (*ppEntryBelow)
1376	pdmBlkCacheEntryRef(*ppEntryBelow);
1377	}
1378	RTSemRWReleaseRead(pBlkCache->SemRWEntries);
1379
1380	STAM_PROFILE_ADV_STOP(&pCache->StatTreeGet, Cache);
1381	}
1382
1383	static void pdmBlkCacheInsertEntry(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEENTRY pEntry)
1384	{
1385	PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
1386
1387	STAM_PROFILE_ADV_START(&pCache->StatTreeInsert, Cache);
1388	RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
1389	bool fInserted = RTAvlrU64Insert(pBlkCache->pTree, &pEntry->Core);
1390	AssertMsg(fInserted, ("Node was not inserted into tree\n"));
1391	STAM_PROFILE_ADV_STOP(&pCache->StatTreeInsert, Cache);
1392	RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
1393	}
1394
1395	/**
1396	* Allocates and initializes a new entry for the cache.
1397	* The entry has a reference count of 1.
1398	*
1399	* @returns Pointer to the new cache entry or NULL if out of memory.
1400	* @param pBlkCache The cache the entry belongs to.
1401	* @param off Start offset.
1402	* @param cbData Size of the cache entry.
1403	* @param pbBuffer Pointer to the buffer to use.
1404	* NULL if a new buffer should be allocated.
1405	* The buffer needs to have the same size of the entry.
1406	*/
1407	static PPDMBLKCACHEENTRY pdmBlkCacheEntryAlloc(PPDMBLKCACHE pBlkCache,
1408	uint64_t off, size_t cbData, uint8_t *pbBuffer)
1409	{
1410	PPDMBLKCACHEENTRY pEntryNew = (PPDMBLKCACHEENTRY)RTMemAllocZ(sizeof(PDMBLKCACHEENTRY));
1411
1412	if (RT_UNLIKELY(!pEntryNew))
1413	return NULL;
1414
1415	pEntryNew->Core.Key = off;
1416	pEntryNew->Core.KeyLast = off + cbData - 1;
1417	pEntryNew->pBlkCache = pBlkCache;
1418	pEntryNew->fFlags = 0;
1419	pEntryNew->cRefs = 1; /* We are using it now. */
1420	pEntryNew->pList = NULL;
1421	pEntryNew->cbData = cbData;
1422	pEntryNew->pWaitingHead = NULL;
1423	pEntryNew->pWaitingTail = NULL;
1424	if (pbBuffer)
1425	pEntryNew->pbData = pbBuffer;
1426	else
1427	pEntryNew->pbData = (uint8_t *)RTMemPageAlloc(cbData);
1428
1429	if (RT_UNLIKELY(!pEntryNew->pbData))
1430	{
1431	RTMemFree(pEntryNew);
1432	return NULL;
1433	}
1434
1435	return pEntryNew;
1436	}
1437
1438	/**
1439	* Checks that a set of flags is set/clear acquiring the R/W semaphore
1440	* in exclusive mode.
1441	*
1442	* @returns true if the flag in fSet is set and the one in fClear is clear.
1443	* false otherwise.
1444	* The R/W semaphore is only held if true is returned.
1445	*
1446	* @param pBlkCache The endpoint cache instance data.
1447	* @param pEntry The entry to check the flags for.
1448	* @param fSet The flag which is tested to be set.
1449	* @param fClear The flag which is tested to be clear.
1450	*/
1451	DECLINLINE(bool) pdmBlkCacheEntryFlagIsSetClearAcquireLock(PPDMBLKCACHE pBlkCache,
1452	PPDMBLKCACHEENTRY pEntry,
1453	uint32_t fSet, uint32_t fClear)
1454	{
1455	uint32_t fFlags = ASMAtomicReadU32(&pEntry->fFlags);
1456	bool fPassed = ((fFlags & fSet) && !(fFlags & fClear));
1457
1458	if (fPassed)
1459	{
1460	/* Acquire the lock and check again because the completion callback might have raced us. */
1461	RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
1462
1463	fFlags = ASMAtomicReadU32(&pEntry->fFlags);
1464	fPassed = ((fFlags & fSet) && !(fFlags & fClear));
1465
1466	/* Drop the lock if we didn't passed the test. */
1467	if (!fPassed)
1468	RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
1469	}
1470
1471	return fPassed;
1472	}
1473
1474	/**
1475	* Adds a segment to the waiting list for a cache entry
1476	* which is currently in progress.
1477	*
1478	* @returns nothing.
1479	* @param pEntry The cache entry to add the segment to.
1480	* @param pSeg The segment to add.
1481	*/
1482	DECLINLINE(void) pdmBlkCacheEntryAddWaiter(PPDMBLKCACHEENTRY pEntry,
1483	PPDMBLKCACHEWAITER pWaiter)
1484	{
1485	pWaiter->pNext = NULL;
1486
1487	if (pEntry->pWaitingHead)
1488	{
1489	AssertPtr(pEntry->pWaitingTail);
1490
1491	pEntry->pWaitingTail->pNext = pWaiter;
1492	pEntry->pWaitingTail = pWaiter;
1493	}
1494	else
1495	{
1496	Assert(!pEntry->pWaitingTail);
1497
1498	pEntry->pWaitingHead = pWaiter;
1499	pEntry->pWaitingTail = pWaiter;
1500	}
1501	}
1502
1503	/**
1504	* Add a buffer described by the I/O memory context
1505	* to the entry waiting for completion.
1506	*
1507	* @returns VBox status code.
1508	* @param pEntry The entry to add the buffer to.
1509	* @param pTask Task associated with the buffer.
1510	* @param pIoMemCtx The memory context to use.
1511	* @param offDiff Offset from the start of the buffer
1512	* in the entry.
1513	* @param cbData Amount of data to wait for onthis entry.
1514	* @param fWrite Flag whether the task waits because it wants to write
1515	* to the cache entry.
1516	*/
1517	static int pdmBlkCacheEntryWaitersAdd(PPDMBLKCACHEENTRY pEntry,
1518	PPDMBLKCACHEREQ pReq,
1519	PCRTSGBUF pSgBuf, uint64_t offDiff,
1520	size_t cbData, bool fWrite)
1521	{
1522	PPDMBLKCACHEWAITER pWaiter = (PPDMBLKCACHEWAITER)RTMemAllocZ(sizeof(PDMBLKCACHEWAITER));
1523	if (!pWaiter)
1524	return VERR_NO_MEMORY;
1525
1526	ASMAtomicIncU32(&pReq->cXfersPending);
1527	pWaiter->pReq = pReq;
1528	pWaiter->offCacheEntry = offDiff;
1529	pWaiter->cbTransfer = cbData;
1530	pWaiter->fWrite = fWrite;
1531	RTSgBufClone(&pWaiter->SgBuf, pSgBuf);
1532
1533	pdmBlkCacheEntryAddWaiter(pEntry, pWaiter);
1534
1535	return VINF_SUCCESS;
1536	}
1537
1538	/**
1539	* Calculate aligned offset and size for a new cache entry
1540	* which do not intersect with an already existing entry and the
1541	* file end.
1542	*
1543	* @returns The number of bytes the entry can hold of the requested amount
1544	* of byte.
1545	* @param pEndpoint The endpoint.
1546	* @param pBlkCache The endpoint cache.
1547	* @param off The start offset.
1548	* @param cb The number of bytes the entry needs to hold at least.
1549	* @param uAlignment Alignment of the boundary sizes.
1550	* @param poffAligned Where to store the aligned offset.
1551	* @param pcbAligned Where to store the aligned size of the entry.
1552	*/
1553	static size_t pdmBlkCacheEntryBoundariesCalc(PPDMBLKCACHE pBlkCache,
1554	uint64_t off, size_t cb,
1555	unsigned uAlignment,
1556	uint64_t poffAligned, size_t pcbAligned)
1557	{
1558	size_t cbAligned;
1559	size_t cbInEntry = 0;
1560	uint64_t offAligned;
1561	PPDMBLKCACHEENTRY pEntryAbove = NULL;
1562	PPDMBLKCACHEENTRY pEntryBelow = NULL;
1563
1564	/* Get the best fit entries around the offset */
1565	pdmBlkCacheGetCacheBestFitEntryByOffset(pBlkCache, off, &pEntryAbove, &pEntryBelow);
1566
1567	/* Log the info */
1568	LogFlow(("%sest fit entry below off=%llu (BestFit=%llu BestFitEnd=%llu BestFitSize=%u)\n",
1569	pEntryBelow ? "B" : "No b",
1570	off,
1571	pEntryBelow ? pEntryBelow->Core.Key : 0,
1572	pEntryBelow ? pEntryBelow->Core.KeyLast : 0,
1573	pEntryBelow ? pEntryBelow->cbData : 0));
1574
1575	LogFlow(("%sest fit entry above off=%llu (BestFit=%llu BestFitEnd=%llu BestFitSize=%u)\n",
1576	pEntryAbove ? "B" : "No b",
1577	off,
1578	pEntryAbove ? pEntryAbove->Core.Key : 0,
1579	pEntryAbove ? pEntryAbove->Core.KeyLast : 0,
1580	pEntryAbove ? pEntryAbove->cbData : 0));
1581
1582	/* Align the offset first. */
1583	offAligned = off & ~(uint64_t)(512-1);
1584	if ( pEntryBelow
1585	&& offAligned <= pEntryBelow->Core.KeyLast)
1586	offAligned = pEntryBelow->Core.KeyLast;
1587
1588	if ( pEntryAbove
1589	&& off + cb > pEntryAbove->Core.Key)
1590	{
1591	cbInEntry = pEntryAbove->Core.Key - off;
1592	cbAligned = pEntryAbove->Core.Key - offAligned;
1593	}
1594	else
1595	{
1596	cbAligned = cb;
1597	cbInEntry = cb;
1598	}
1599
1600	/* A few sanity checks */
1601	AssertMsg(!pEntryBelow \|\| pEntryBelow->Core.KeyLast < offAligned,
1602	("Aligned start offset intersects with another cache entry\n"));
1603	AssertMsg(!pEntryAbove \|\| (offAligned + cbAligned) <= pEntryAbove->Core.Key,
1604	("Aligned size intersects with another cache entry\n"));
1605	Assert(cbInEntry <= cbAligned);
1606
1607	if (pEntryBelow)
1608	pdmBlkCacheEntryRelease(pEntryBelow);
1609	if (pEntryAbove)
1610	pdmBlkCacheEntryRelease(pEntryAbove);
1611
1612	LogFlow(("offAligned=%llu cbAligned=%u\n", offAligned, cbAligned));
1613
1614	*poffAligned = offAligned;
1615	*pcbAligned = cbAligned;
1616
1617	return cbInEntry;
1618	}
1619
1620	/**
1621	* Create a new cache entry evicting data from the cache if required.
1622	*
1623	* @returns Pointer to the new cache entry or NULL
1624	* if not enough bytes could be evicted from the cache.
1625	* @param pEndpoint The endpoint.
1626	* @param pBlkCache The endpoint cache.
1627	* @param off The offset.
1628	* @param cb Number of bytes the cache entry should have.
1629	* @param uAlignment Alignment the size of the entry should have.
1630	* @param pcbData Where to store the number of bytes the new
1631	* entry can hold. May be lower than actually requested
1632	* due to another entry intersecting the access range.
1633	*/
1634	static PPDMBLKCACHEENTRY pdmBlkCacheEntryCreate(PPDMBLKCACHE pBlkCache,
1635	uint64_t off, size_t cb,
1636	unsigned uAlignment,
1637	size_t *pcbData)
1638	{
1639	uint64_t offStart = 0;
1640	size_t cbEntry = 0;
1641	PPDMBLKCACHEENTRY pEntryNew = NULL;
1642	PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
1643	uint8_t *pbBuffer = NULL;
1644
1645	*pcbData = pdmBlkCacheEntryBoundariesCalc(pBlkCache, off, cb, uAlignment,
1646	&offStart, &cbEntry);
1647
1648	pdmBlkCacheLockEnter(pCache);
1649	bool fEnough = pdmBlkCacheReclaim(pCache, cbEntry, true, &pbBuffer);
1650
1651	if (fEnough)
1652	{
1653	LogFlow(("Evicted enough bytes (%u requested). Creating new cache entry\n", cbEntry));
1654
1655	pEntryNew = pdmBlkCacheEntryAlloc(pBlkCache, offStart, cbEntry, pbBuffer);
1656	if (RT_LIKELY(pEntryNew))
1657	{
1658	pdmBlkCacheEntryAddToList(&pCache->LruRecentlyUsedIn, pEntryNew);
1659	pdmBlkCacheAdd(pCache, cbEntry);
1660	pdmBlkCacheLockLeave(pCache);
1661
1662	pdmBlkCacheInsertEntry(pBlkCache, pEntryNew);
1663
1664	AssertMsg( (off >= pEntryNew->Core.Key)
1665	&& (off + *pcbData <= pEntryNew->Core.KeyLast + 1),
1666	("Overflow in calculation off=%llu OffsetAligned=%llu\n",
1667	off, pEntryNew->Core.Key));
1668	}
1669	else
1670	pdmBlkCacheLockLeave(pCache);
1671	}
1672	else
1673	pdmBlkCacheLockLeave(pCache);
1674
1675	return pEntryNew;
1676	}
1677
1678	static PPDMBLKCACHEREQ pdmBlkCacheReqAlloc(size_t cbXfer, void *pvUser)
1679	{
1680	PPDMBLKCACHEREQ pReq = (PPDMBLKCACHEREQ)RTMemAlloc(sizeof(PDMBLKCACHEREQ));
1681
1682	if (RT_LIKELY(pReq))
1683	{
1684	pReq->pvUser = pvUser;
1685	pReq->cbXfer = cbXfer;
1686	pReq->rcReq = VINF_SUCCESS;
1687	pReq->cXfersPending = 0;
1688	}
1689
1690	return pReq;
1691	}
1692
1693	static void pdmBlkCacheReqComplete(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEREQ pReq)
1694	{
1695	switch (pBlkCache->enmType)
1696	{
1697	case PDMBLKCACHETYPE_DEV:
1698	{
1699	pBlkCache->u.Dev.pfnXferComplete(pBlkCache->u.Dev.pDevIns,
1700	pReq->pvUser, pReq->rcReq);
1701	break;
1702	}
1703	case PDMBLKCACHETYPE_DRV:
1704	{
1705	pBlkCache->u.Drv.pfnXferComplete(pBlkCache->u.Drv.pDrvIns,
1706	pReq->pvUser, pReq->rcReq);
1707	break;
1708	}
1709	case PDMBLKCACHETYPE_USB:
1710	{
1711	pBlkCache->u.Usb.pfnXferComplete(pBlkCache->u.Usb.pUsbIns,
1712	pReq->pvUser, pReq->rcReq);
1713	break;
1714	}
1715	case PDMBLKCACHETYPE_INTERNAL:
1716	{
1717	pBlkCache->u.Int.pfnXferComplete(pBlkCache->u.Int.pvUser,
1718	pReq->pvUser, pReq->rcReq);
1719	break;
1720	}
1721	default:
1722	AssertMsgFailed(("Unknown block cache type!\n"));
1723	}
1724
1725	RTMemFree(pReq);
1726	}
1727
1728	static bool pdmBlkCacheReqUpdate(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEREQ pReq,
1729	size_t cbComplete, int rcReq, bool fCallHandler)
1730	{
1731	if (RT_FAILURE(rcReq))
1732	ASMAtomicCmpXchgS32(&pReq->rcReq, rcReq, VINF_SUCCESS);
1733
1734	AssertMsg(pReq->cbXfer >= cbComplete, ("Completed more than left\n"));
1735	AssertMsg(pReq->cXfersPending > 0, ("No transfers are pending for this request\n"));
1736	uint32_t cbOld = ASMAtomicSubU32(&pReq->cbXfer, cbComplete);
1737	uint32_t cXfersPending = ASMAtomicDecU32(&pReq->cXfersPending);
1738
1739	if ( !(cbOld - cbComplete)
1740	&& !cXfersPending)
1741	{
1742	if (fCallHandler)
1743	pdmBlkCacheReqComplete(pBlkCache, pReq);
1744	return true;
1745	}
1746
1747	LogFlowFunc(("pReq=%#p cXfersPending=%u cbXfer=%u\n", pReq, cXfersPending, (cbOld - cbComplete)));
1748	return false;
1749	}
1750
1751	VMMR3DECL(int) PDMR3BlkCacheRead(PPDMBLKCACHE pBlkCache, uint64_t off,
1752	PCRTSGBUF pcSgBuf, size_t cbRead, void *pvUser)
1753	{
1754	int rc = VINF_SUCCESS;
1755	PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
1756	PPDMBLKCACHEENTRY pEntry;
1757	PPDMBLKCACHEREQ pReq;
1758
1759	LogFlowFunc((": pBlkCache=%#p{%s} off=%llu pcSgBuf=%#p cbRead=%u pvUser=%#p\n",
1760	pBlkCache, pBlkCache->pszId, off, pcSgBuf, cbRead, pvUser));
1761
1762	RTSGBUF SgBuf;
1763	RTSgBufClone(&SgBuf, pcSgBuf);
1764
1765	/* Allocate new request structure. */
1766	pReq = pdmBlkCacheReqAlloc(cbRead, pvUser);
1767	if (RT_UNLIKELY(!pReq))
1768	return VERR_NO_MEMORY;
1769
1770	/* Increment data transfer counter to keep the request valid while we access it. */
1771	ASMAtomicIncU32(&pReq->cXfersPending);
1772
1773	while (cbRead)
1774	{
1775	size_t cbToRead;
1776
1777	pEntry = pdmBlkCacheGetCacheEntryByOffset(pBlkCache, off);
1778
1779	/*
1780	* If there is no entry we try to create a new one eviciting unused pages
1781	* if the cache is full. If this is not possible we will pass the request through
1782	* and skip the caching (all entries may be still in progress so they can't
1783	* be evicted)
1784	* If we have an entry it can be in one of the LRU lists where the entry
1785	* contains data (recently used or frequently used LRU) so we can just read
1786	* the data we need and put the entry at the head of the frequently used LRU list.
1787	* In case the entry is in one of the ghost lists it doesn't contain any data.
1788	* We have to fetch it again evicting pages from either T1 or T2 to make room.
1789	*/
1790	if (pEntry)
1791	{
1792	uint64_t offDiff = off - pEntry->Core.Key;
1793
1794	AssertMsg(off >= pEntry->Core.Key,
1795	("Overflow in calculation off=%llu OffsetAligned=%llu\n",
1796	off, pEntry->Core.Key));
1797
1798	AssertPtr(pEntry->pList);
1799
1800	cbToRead = RT_MIN(pEntry->cbData - offDiff, cbRead);
1801
1802	AssertMsg(off + cbToRead <= pEntry->Core.Key + pEntry->Core.KeyLast + 1,
1803	("Buffer of cache entry exceeded off=%llu cbToRead=%d\n",
1804	off, cbToRead));
1805
1806	cbRead -= cbToRead;
1807
1808	if (!cbRead)
1809	STAM_COUNTER_INC(&pCache->cHits);
1810	else
1811	STAM_COUNTER_INC(&pCache->cPartialHits);
1812
1813	STAM_COUNTER_ADD(&pCache->StatRead, cbToRead);
1814
1815	/* Ghost lists contain no data. */
1816	if ( (pEntry->pList == &pCache->LruRecentlyUsedIn)
1817	\|\| (pEntry->pList == &pCache->LruFrequentlyUsed))
1818	{
1819	if (pdmBlkCacheEntryFlagIsSetClearAcquireLock(pBlkCache, pEntry,
1820	PDMBLKCACHE_ENTRY_IO_IN_PROGRESS,
1821	PDMBLKCACHE_ENTRY_IS_DIRTY))
1822	{
1823	/* Entry didn't completed yet. Append to the list */
1824	pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
1825	&SgBuf, offDiff, cbToRead,
1826	false /* fWrite */);
1827	RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
1828	}
1829	else
1830	{
1831	/* Read as much as we can from the entry. */
1832	RTSgBufCopyFromBuf(&SgBuf, pEntry->pbData + offDiff, cbToRead);
1833	ASMAtomicSubU32(&pReq->cbXfer, cbToRead);
1834	}
1835
1836	/* Move this entry to the top position */
1837	if (pEntry->pList == &pCache->LruFrequentlyUsed)
1838	{
1839	pdmBlkCacheLockEnter(pCache);
1840	pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
1841	pdmBlkCacheLockLeave(pCache);
1842	}
1843	/* Release the entry */
1844	pdmBlkCacheEntryRelease(pEntry);
1845	}
1846	else
1847	{
1848	uint8_t *pbBuffer = NULL;
1849
1850	LogFlow(("Fetching data for ghost entry %#p from file\n", pEntry));
1851
1852	pdmBlkCacheLockEnter(pCache);
1853	pdmBlkCacheEntryRemoveFromList(pEntry); /* Remove it before we remove data, otherwise it may get freed when evicting data. */
1854	bool fEnough = pdmBlkCacheReclaim(pCache, pEntry->cbData, true, &pbBuffer);
1855
1856	/* Move the entry to Am and fetch it to the cache. */
1857	if (fEnough)
1858	{
1859	pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
1860	pdmBlkCacheAdd(pCache, pEntry->cbData);
1861	pdmBlkCacheLockLeave(pCache);
1862
1863	if (pbBuffer)
1864	pEntry->pbData = pbBuffer;
1865	else
1866	pEntry->pbData = (uint8_t *)RTMemPageAlloc(pEntry->cbData);
1867	AssertPtr(pEntry->pbData);
1868
1869	pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
1870	&SgBuf, offDiff, cbToRead,
1871	false /* fWrite */);
1872	pdmBlkCacheEntryReadFromMedium(pEntry);
1873	/* Release the entry */
1874	pdmBlkCacheEntryRelease(pEntry);
1875	}
1876	else
1877	{
1878	RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
1879	STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
1880	RTAvlrU64Remove(pBlkCache->pTree, pEntry->Core.Key);
1881	STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
1882	RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
1883
1884	pdmBlkCacheLockLeave(pCache);
1885
1886	RTMemFree(pEntry);
1887
1888	pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
1889	&SgBuf, off, cbToRead,
1890	PDMBLKCACHEXFERDIR_READ);
1891	}
1892	}
1893	}
1894	else
1895	{
1896	#ifdef VBOX_WITH_IO_READ_CACHE
1897	/* No entry found for this offset. Create a new entry and fetch the data to the cache. */
1898	PPDMBLKCACHEENTRY pEntryNew = pdmBlkCacheEntryCreate(pBlkCache,
1899	off, cbRead,
1900	PAGE_SIZE,
1901	&cbToRead);
1902
1903	cbRead -= cbToRead;
1904
1905	if (pEntryNew)
1906	{
1907	if (!cbRead)
1908	STAM_COUNTER_INC(&pCache->cMisses);
1909	else
1910	STAM_COUNTER_INC(&pCache->cPartialHits);
1911
1912	pdmBlkCacheEntryWaitersAdd(pEntryNew, pReq,
1913	&SgBuf,
1914	off - pEntryNew->Core.Key,
1915	cbToRead,
1916	false /* fWrite */);
1917	pdmBlkCacheEntryReadFromMedium(pEntryNew);
1918	pdmBlkCacheEntryRelease(pEntryNew); /* it is protected by the I/O in progress flag now. */
1919	}
1920	else
1921	{
1922	/*
1923	* There is not enough free space in the cache.
1924	* Pass the request directly to the I/O manager.
1925	*/
1926	LogFlow(("Couldn't evict %u bytes from the cache. Remaining request will be passed through\n", cbToRead));
1927
1928	pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
1929	&SgBuf, off, cbToRead,
1930	PDMBLKCACHEXFERDIR_READ);
1931	}
1932	#else
1933	/* Clip read size if necessary. */
1934	PPDMBLKCACHEENTRY pEntryAbove;
1935	pdmBlkCacheGetCacheBestFitEntryByOffset(pBlkCache, off,
1936	&pEntryAbove, NULL);
1937
1938	if (pEntryAbove)
1939	{
1940	if (off + cbRead > pEntryAbove->Core.Key)
1941	cbToRead = pEntryAbove->Core.Key - off;
1942	else
1943	cbToRead = cbRead;
1944
1945	pdmBlkCacheEntryRelease(pEntryAbove);
1946	}
1947	else
1948	cbToRead = cbRead;
1949
1950	cbRead -= cbToRead;
1951	pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
1952	&SgBuf, off, cbToRead,
1953	PDMBLKCACHEXFERDIR_READ);
1954	#endif
1955	}
1956	off += cbToRead;
1957	}
1958
1959	if (!pdmBlkCacheReqUpdate(pBlkCache, pReq, 0, rc, false))
1960	rc = VINF_AIO_TASK_PENDING;
1961
1962	LogFlowFunc((": Leave rc=%Rrc\n", rc));
1963
1964	return rc;
1965	}
1966
1967	VMMR3DECL(int) PDMR3BlkCacheWrite(PPDMBLKCACHE pBlkCache, uint64_t off,
1968	PCRTSGBUF pcSgBuf, size_t cbWrite, void *pvUser)
1969	{
1970	int rc = VINF_SUCCESS;
1971	PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
1972	PPDMBLKCACHEENTRY pEntry;
1973	PPDMBLKCACHEREQ pReq;
1974
1975	LogFlowFunc((": pBlkCache=%#p{%s} off=%llu pcSgBuf=%#p cbWrite=%u pvUser=%#p\n",
1976	pBlkCache, pBlkCache->pszId, off, pcSgBuf, cbWrite, pvUser));
1977
1978	RTSGBUF SgBuf;
1979	RTSgBufClone(&SgBuf, pcSgBuf);
1980
1981	/* Allocate new request structure. */
1982	pReq = pdmBlkCacheReqAlloc(cbWrite, pvUser);
1983	if (RT_UNLIKELY(!pReq))
1984	return VERR_NO_MEMORY;
1985
1986	/* Increment data transfer counter to keep the request valid while we access it. */
1987	ASMAtomicIncU32(&pReq->cXfersPending);
1988
1989	while (cbWrite)
1990	{
1991	size_t cbToWrite;
1992
1993	pEntry = pdmBlkCacheGetCacheEntryByOffset(pBlkCache, off);
1994
1995	if (pEntry)
1996	{
1997	/* Write the data into the entry and mark it as dirty */
1998	AssertPtr(pEntry->pList);
1999
2000	uint64_t offDiff = off - pEntry->Core.Key;
2001
2002	AssertMsg(off >= pEntry->Core.Key,
2003	("Overflow in calculation off=%llu OffsetAligned=%llu\n",
2004	off, pEntry->Core.Key));
2005
2006	cbToWrite = RT_MIN(pEntry->cbData - offDiff, cbWrite);
2007	cbWrite -= cbToWrite;
2008
2009	if (!cbWrite)
2010	STAM_COUNTER_INC(&pCache->cHits);
2011	else
2012	STAM_COUNTER_INC(&pCache->cPartialHits);
2013
2014	STAM_COUNTER_ADD(&pCache->StatWritten, cbToWrite);
2015
2016	/* Ghost lists contain no data. */
2017	if ( (pEntry->pList == &pCache->LruRecentlyUsedIn)
2018	\|\| (pEntry->pList == &pCache->LruFrequentlyUsed))
2019	{
2020	/* Check if the entry is dirty. */
2021	if(pdmBlkCacheEntryFlagIsSetClearAcquireLock(pBlkCache, pEntry,
2022	PDMBLKCACHE_ENTRY_IS_DIRTY,
2023	0))
2024	{
2025	/* If it is already dirty but not in progress just update the data. */
2026	if (!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS))
2027	{
2028	RTSgBufCopyToBuf(&SgBuf, pEntry->pbData + offDiff,
2029	cbToWrite);
2030	ASMAtomicSubU32(&pReq->cbXfer, cbToWrite);
2031	}
2032	else
2033	{
2034	/* The data isn't written to the file yet */
2035	pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
2036	&SgBuf, offDiff, cbToWrite,
2037	true /* fWrite */);
2038	STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
2039	}
2040
2041	RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2042	}
2043	else /* Dirty bit not set */
2044	{
2045	/*
2046	* Check if a read is in progress for this entry.
2047	* We have to defer processing in that case.
2048	*/
2049	if(pdmBlkCacheEntryFlagIsSetClearAcquireLock(pBlkCache, pEntry,
2050	PDMBLKCACHE_ENTRY_IO_IN_PROGRESS,
2051	0))
2052	{
2053	pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
2054	&SgBuf, offDiff, cbToWrite,
2055	true /* fWrite */);
2056	STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
2057	RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2058	}
2059	else /* I/O in progress flag not set */
2060	{
2061	/* Write as much as we can into the entry and update the file. */
2062	RTSgBufCopyToBuf(&SgBuf, pEntry->pbData + offDiff, cbToWrite);
2063	ASMAtomicSubU32(&pReq->cbXfer, cbToWrite);
2064
2065	bool fCommit = pdmBlkCacheAddDirtyEntry(pBlkCache, pEntry);
2066	if (fCommit)
2067	pdmBlkCacheCommitDirtyEntries(pCache);
2068	}
2069	} /* Dirty bit not set */
2070
2071	/* Move this entry to the top position */
2072	if (pEntry->pList == &pCache->LruFrequentlyUsed)
2073	{
2074	pdmBlkCacheLockEnter(pCache);
2075	pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
2076	pdmBlkCacheLockLeave(pCache);
2077	}
2078
2079	pdmBlkCacheEntryRelease(pEntry);
2080	}
2081	else /* Entry is on the ghost list */
2082	{
2083	uint8_t *pbBuffer = NULL;
2084
2085	pdmBlkCacheLockEnter(pCache);
2086	pdmBlkCacheEntryRemoveFromList(pEntry); /* Remove it before we remove data, otherwise it may get freed when evicting data. */
2087	bool fEnough = pdmBlkCacheReclaim(pCache, pEntry->cbData, true, &pbBuffer);
2088
2089	if (fEnough)
2090	{
2091	/* Move the entry to Am and fetch it to the cache. */
2092	pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
2093	pdmBlkCacheAdd(pCache, pEntry->cbData);
2094	pdmBlkCacheLockLeave(pCache);
2095
2096	if (pbBuffer)
2097	pEntry->pbData = pbBuffer;
2098	else
2099	pEntry->pbData = (uint8_t *)RTMemPageAlloc(pEntry->cbData);
2100	AssertPtr(pEntry->pbData);
2101
2102	pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
2103	&SgBuf, offDiff, cbToWrite,
2104	true /* fWrite */);
2105	STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
2106	pdmBlkCacheEntryReadFromMedium(pEntry);
2107
2108	/* Release the reference. If it is still needed the I/O in progress flag should protect it now. */
2109	pdmBlkCacheEntryRelease(pEntry);
2110	}
2111	else
2112	{
2113	RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
2114	STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
2115	RTAvlrU64Remove(pBlkCache->pTree, pEntry->Core.Key);
2116	STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
2117	RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2118
2119	pdmBlkCacheLockLeave(pCache);
2120
2121	RTMemFree(pEntry);
2122	pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
2123	&SgBuf, off, cbToWrite,
2124	PDMBLKCACHEXFERDIR_WRITE);
2125	}
2126	}
2127	}
2128	else /* No entry found */
2129	{
2130	/*
2131	* No entry found. Try to create a new cache entry to store the data in and if that fails
2132	* write directly to the file.
2133	*/
2134	PPDMBLKCACHEENTRY pEntryNew = pdmBlkCacheEntryCreate(pBlkCache,
2135	off, cbWrite,
2136	512, &cbToWrite);
2137
2138	cbWrite -= cbToWrite;
2139
2140	if (pEntryNew)
2141	{
2142	uint64_t offDiff = off - pEntryNew->Core.Key;
2143
2144	STAM_COUNTER_INC(&pCache->cHits);
2145
2146	/*
2147	* Check if it is possible to just write the data without waiting
2148	* for it to get fetched first.
2149	*/
2150	if (!offDiff && pEntryNew->cbData == cbToWrite)
2151	{
2152	RTSgBufCopyToBuf(&SgBuf, pEntryNew->pbData, cbToWrite);
2153	ASMAtomicSubU32(&pReq->cbXfer, cbToWrite);
2154
2155	bool fCommit = pdmBlkCacheAddDirtyEntry(pBlkCache, pEntryNew);
2156	if (fCommit)
2157	pdmBlkCacheCommitDirtyEntries(pCache);
2158	STAM_COUNTER_ADD(&pCache->StatWritten, cbToWrite);
2159	}
2160	else
2161	{
2162	/* Defer the write and fetch the data from the endpoint. */
2163	pdmBlkCacheEntryWaitersAdd(pEntryNew, pReq,
2164	&SgBuf, offDiff, cbToWrite,
2165	true /* fWrite */);
2166	STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
2167	pdmBlkCacheEntryReadFromMedium(pEntryNew);
2168	}
2169
2170	pdmBlkCacheEntryRelease(pEntryNew);
2171	}
2172	else
2173	{
2174	/*
2175	* There is not enough free space in the cache.
2176	* Pass the request directly to the I/O manager.
2177	*/
2178	LogFlow(("Couldn't evict %u bytes from the cache. Remaining request will be passed through\n", cbToWrite));
2179
2180	STAM_COUNTER_INC(&pCache->cMisses);
2181
2182	pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
2183	&SgBuf, off, cbToWrite,
2184	PDMBLKCACHEXFERDIR_WRITE);
2185	}
2186	}
2187
2188	off += cbToWrite;
2189	}
2190
2191	if (!pdmBlkCacheReqUpdate(pBlkCache, pReq, 0, rc, false))
2192	rc = VINF_AIO_TASK_PENDING;
2193
2194	LogFlowFunc((": Leave rc=%Rrc\n", rc));
2195
2196	return rc;
2197	}
2198
2199	VMMR3DECL(int) PDMR3BlkCacheFlush(PPDMBLKCACHE pBlkCache, void *pvUser)
2200	{
2201	int rc = VINF_SUCCESS;
2202	PPDMBLKCACHEREQ pReq;
2203
2204	LogFlowFunc((": pBlkCache=%#p{%s}\n", pBlkCache, pBlkCache->pszId));
2205
2206	/* Commit dirty entries in the cache. */
2207	pdmBlkCacheCommit(pBlkCache);
2208
2209	/* Allocate new request structure. */
2210	pReq = pdmBlkCacheReqAlloc(0, pvUser);
2211	if (RT_UNLIKELY(!pReq))
2212	return VERR_NO_MEMORY;
2213
2214	rc = pdmBlkCacheRequestPassthrough(pBlkCache, pReq, NULL, 0, 0,
2215	PDMBLKCACHEXFERDIR_FLUSH);
2216	AssertRC(rc);
2217
2218	LogFlowFunc((": Leave rc=%Rrc\n", rc));
2219	return VINF_AIO_TASK_PENDING;
2220	}
2221
2222	/**
2223	* Completes a task segment freeing all resources and completes the task handle
2224	* if everything was transferred.
2225	*
2226	* @returns Next task segment handle.
2227	* @param pTaskSeg Task segment to complete.
2228	* @param rc Status code to set.
2229	*/
2230	static PPDMBLKCACHEWAITER pdmBlkCacheWaiterComplete(PPDMBLKCACHE pBlkCache,
2231	PPDMBLKCACHEWAITER pWaiter,
2232	int rc)
2233	{
2234	PPDMBLKCACHEWAITER pNext = pWaiter->pNext;
2235	PPDMBLKCACHEREQ pReq = pWaiter->pReq;
2236
2237	pdmBlkCacheReqUpdate(pBlkCache, pWaiter->pReq, pWaiter->cbTransfer, rc, true);
2238
2239	RTMemFree(pWaiter);
2240
2241	return pNext;
2242	}
2243
2244	static void pdmBlkCacheIoXferCompleteEntry(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEIOXFER hIoXfer, int rcIoXfer)
2245	{
2246	PPDMBLKCACHEENTRY pEntry = hIoXfer->pEntry;
2247	PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
2248
2249	/* Reference the entry now as we are clearing the I/O in progress flag
2250	* which protected the entry till now. */
2251	pdmBlkCacheEntryRef(pEntry);
2252
2253	RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
2254	pEntry->fFlags &= ~PDMBLKCACHE_ENTRY_IO_IN_PROGRESS;
2255
2256	/* Process waiting segment list. The data in entry might have changed in-between. */
2257	bool fDirty = false;
2258	PPDMBLKCACHEWAITER pComplete = pEntry->pWaitingHead;
2259	PPDMBLKCACHEWAITER pCurr = pComplete;
2260
2261	AssertMsg((pCurr && pEntry->pWaitingTail) \|\| (!pCurr && !pEntry->pWaitingTail),
2262	("The list tail was not updated correctly\n"));
2263	pEntry->pWaitingTail = NULL;
2264	pEntry->pWaitingHead = NULL;
2265
2266	if (hIoXfer->enmXferDir == PDMBLKCACHEXFERDIR_WRITE)
2267	{
2268	/*
2269	* An error here is difficult to handle as the original request completed already.
2270	* The error is logged for now and the VM is paused.
2271	* If the user continues the entry is written again in the hope
2272	* the user fixed the problem and the next write succeeds.
2273	*/
2274	/** @todo r=aeichner: This solution doesn't work
2275	* The user will get the message but the VM will hang afterwards
2276	* VMR3Suspend() returns when the VM is suspended but suspending
2277	* the VM will reopen the images readonly in DrvVD. They are closed first
2278	* which will close the endpoints. This will block EMT while the
2279	* I/O manager processes the close request but the IO manager is stuck
2280	* in the VMR3Suspend call and can't process the request.
2281	* Another problem is that closing the VM means flushing the cache
2282	* but the entry failed and will probably fail again.
2283	* No idea so far how to solve this problem... but the user gets informed
2284	* at least.
2285	*/
2286	if (RT_FAILURE(rcIoXfer))
2287	{
2288	LogRel(("I/O cache: Error while writing entry at offset %llu (%u bytes) to medium \"%s\"\n",
2289	pEntry->Core.Key, pEntry->cbData, pBlkCache->pszId));
2290
2291	if (!ASMAtomicXchgBool(&pCache->fIoErrorVmSuspended, true))
2292	{
2293	int rc = VMSetRuntimeError(pCache->pVM, VMSETRTERR_FLAGS_SUSPEND \| VMSETRTERR_FLAGS_NO_WAIT, "BLKCACHE_IOERR",
2294	N_("The I/O cache encountered an error while updating data in medium \"%s\" (rc=%Rrc). "
2295	"Make sure there is enough free space on the disk and that the disk is working properly. "
2296	"Operation can be resumed afterwards"),
2297	pBlkCache->pszId, rcIoXfer);
2298	AssertRC(rc);
2299	}
2300
2301	/*
2302	* The entry is still marked as dirty which prevents eviction.
2303	* Add the waiters to the list again.
2304	*/
2305	fDirty = true;
2306
2307	if (pComplete)
2308	{
2309	pEntry->pWaitingHead = pComplete;
2310	while (pComplete->pNext)
2311	pComplete = pComplete->pNext;
2312	pEntry->pWaitingTail = pComplete;
2313	pComplete = NULL;
2314	}
2315	}
2316	else
2317	{
2318	pEntry->fFlags &= ~PDMBLKCACHE_ENTRY_IS_DIRTY;
2319
2320	while (pCurr)
2321	{
2322	AssertMsg(pCurr->fWrite, ("Completed write entries should never have read tasks attached\n"));
2323
2324	RTSgBufCopyToBuf(&pCurr->SgBuf, pEntry->pbData + pCurr->offCacheEntry, pCurr->cbTransfer);
2325	fDirty = true;
2326
2327	pCurr = pCurr->pNext;
2328	}
2329	}
2330	}
2331	else
2332	{
2333	AssertMsg(hIoXfer->enmXferDir == PDMBLKCACHEXFERDIR_READ, ("Invalid transfer type\n"));
2334	AssertMsg(!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IS_DIRTY),
2335	("Invalid flags set\n"));
2336
2337	while (pCurr)
2338	{
2339	if (pCurr->fWrite)
2340	{
2341	RTSgBufCopyToBuf(&pCurr->SgBuf, pEntry->pbData + pCurr->offCacheEntry, pCurr->cbTransfer);
2342	fDirty = true;
2343	}
2344	else
2345	RTSgBufCopyFromBuf(&pCurr->SgBuf, pEntry->pbData + pCurr->offCacheEntry, pCurr->cbTransfer);
2346
2347	pCurr = pCurr->pNext;
2348	}
2349	}
2350
2351	bool fCommit = false;
2352	if (fDirty)
2353	fCommit = pdmBlkCacheAddDirtyEntry(pBlkCache, pEntry);
2354
2355	RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
2356
2357	/* Dereference so that it isn't protected anymore except we issued anyother write for it. */
2358	pdmBlkCacheEntryRelease(pEntry);
2359
2360	if (fCommit)
2361	pdmBlkCacheCommitDirtyEntries(pCache);
2362
2363	/* Complete waiters now. */
2364	while (pComplete)
2365	pComplete = pdmBlkCacheWaiterComplete(pBlkCache, pComplete, rcIoXfer);
2366	}
2367
2368	VMMR3DECL(void) PDMR3BlkCacheIoXferComplete(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEIOXFER hIoXfer, int rcIoXfer)
2369	{
2370	LogFlowFunc(("pBlkCache=%#p hIoXfer=%#p rcIoXfer=%Rrc\n", pBlkCache, hIoXfer, rcIoXfer));
2371
2372	if (hIoXfer->fIoCache)
2373	pdmBlkCacheIoXferCompleteEntry(pBlkCache, hIoXfer, rcIoXfer);
2374	else
2375	pdmBlkCacheReqUpdate(pBlkCache, hIoXfer->pReq, hIoXfer->cbXfer, rcIoXfer, true);
2376	}
2377

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/PDMBlkCache.cpp@ 34347

Download in other formats: