PDMAsyncCompletionFileCache.cpp@ 26638

Last change on this file since 26638 was 26526, checked in by vboxsync, 15 years ago
VMM,asm.h,Config.kmk: win.amd64 warnings; fixed the ASMAtomicSubU32 signature.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 70.7 KB

Line
1	/* $Id: PDMAsyncCompletionFileCache.cpp 26526 2010-02-15 03:36:01Z vboxsync $ */
2	/** @file
3	* PDM Async I/O - Transport data asynchronous in R3 using EMT.
4	* File data cache.
5	*/
6
7	/*
8	* Copyright (C) 2006-2008 Sun Microsystems, Inc.
9	*
10	* This file is part of VirtualBox Open Source Edition (OSE), as
11	* available from http://www.virtualbox.org. This file is free software;
12	* you can redistribute it and/or modify it under the terms of the GNU
13	* General Public License (GPL) as published by the Free Software
14	* Foundation, in version 2 as it comes in the "COPYING" file of the
15	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
16	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
17	*
18	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
19	* Clara, CA 95054 USA or visit http://www.sun.com if you need
20	* additional information or have any questions.
21	*/
22
23	/** @page pg_pdm_async_completion_cache PDM Async Completion Cache - The file I/O cache
24	* This component implements an I/O cache for file endpoints based on the 2Q cache algorithm.
25	*/
26
27	/*******************************************************************************
28	* Header Files *
29	*******************************************************************************/
30	#define LOG_GROUP LOG_GROUP_PDM_ASYNC_COMPLETION
31	#include <iprt/types.h>
32	#include <iprt/mem.h>
33	#include <iprt/path.h>
34	#include <VBox/log.h>
35	#include <VBox/stam.h>
36
37	#include "PDMAsyncCompletionFileInternal.h"
38
39	/**
40	* A I/O memory context.
41	*/
42	typedef struct PDMIOMEMCTX
43	{
44	/** Pointer to the scatter/gather list. */
45	PCPDMDATASEG paDataSeg;
46	/** Number of segments. */
47	size_t cSegments;
48	/** Current segment we are in. */
49	unsigned iSegIdx;
50	/** Pointer to the current buffer. */
51	uint8_t *pbBuf;
52	/** Number of bytes left in the current buffer. */
53	size_t cbBufLeft;
54	} PDMIOMEMCTX, *PPDMIOMEMCTX;
55
56	#ifdef VBOX_STRICT
57	# define PDMACFILECACHE_IS_CRITSECT_OWNER(Cache) \
58	do \
59	{ \
60	AssertMsg(RTCritSectIsOwner(&Cache->CritSect), \
61	("Thread does not own critical section\n"));\
62	} while(0);
63	#else
64	# define PDMACFILECACHE_IS_CRITSECT_OWNER(Cache) do { } while(0);
65	#endif
66
67	/*******************************************************************************
68	* Internal Functions *
69	*******************************************************************************/
70	static void pdmacFileCacheTaskCompleted(PPDMACTASKFILE pTask, void *pvUser);
71
72	/**
73	* Decrement the reference counter of the given cache entry.
74	*
75	* @returns nothing.
76	* @param pEntry The entry to release.
77	*/
78	DECLINLINE(void) pdmacFileEpCacheEntryRelease(PPDMACFILECACHEENTRY pEntry)
79	{
80	AssertMsg(pEntry->cRefs > 0, ("Trying to release a not referenced entry\n"));
81	ASMAtomicDecU32(&pEntry->cRefs);
82	}
83
84	/**
85	* Increment the reference counter of the given cache entry.
86	*
87	* @returns nothing.
88	* @param pEntry The entry to reference.
89	*/
90	DECLINLINE(void) pdmacFileEpCacheEntryRef(PPDMACFILECACHEENTRY pEntry)
91	{
92	ASMAtomicIncU32(&pEntry->cRefs);
93	}
94
95	/**
96	* Initialize a I/O memory context.
97	*
98	* @returns nothing
99	* @param pIoMemCtx Pointer to a unitialized I/O memory context.
100	* @param paDataSeg Pointer to the S/G list.
101	* @param cSegments Number of segments in the S/G list.
102	*/
103	DECLINLINE(void) pdmIoMemCtxInit(PPDMIOMEMCTX pIoMemCtx, PCPDMDATASEG paDataSeg, size_t cSegments)
104	{
105	AssertMsg((cSegments > 0) && paDataSeg, ("Trying to initialize a I/O memory context without a S/G list\n"));
106
107	pIoMemCtx->paDataSeg = paDataSeg;
108	pIoMemCtx->cSegments = cSegments;
109	pIoMemCtx->iSegIdx = 0;
110	pIoMemCtx->pbBuf = (uint8_t *)paDataSeg[0].pvSeg;
111	pIoMemCtx->cbBufLeft = paDataSeg[0].cbSeg;
112	}
113
114	/**
115	* Return a buffer from the I/O memory context.
116	*
117	* @returns Pointer to the buffer
118	* @param pIoMemCtx Pointer to the I/O memory context.
119	* @param pcbData Pointer to the amount of byte requested.
120	* If the current buffer doesn't have enough bytes left
121	* the amount is returned in the variable.
122	*/
123	DECLINLINE(uint8_t ) pdmIoMemCtxGetBuffer(PPDMIOMEMCTX pIoMemCtx, size_t pcbData)
124	{
125	size_t cbData = RT_MIN(*pcbData, pIoMemCtx->cbBufLeft);
126	uint8_t *pbBuf = pIoMemCtx->pbBuf;
127
128	pIoMemCtx->cbBufLeft -= cbData;
129
130	/* Advance to the next segment if required. */
131	if (!pIoMemCtx->cbBufLeft)
132	{
133	pIoMemCtx->iSegIdx++;
134
135	if (RT_UNLIKELY(pIoMemCtx->iSegIdx == pIoMemCtx->cSegments))
136	{
137	pIoMemCtx->cbBufLeft = 0;
138	pIoMemCtx->pbBuf = NULL;
139	}
140	else
141	{
142	pIoMemCtx->pbBuf = (uint8_t *)pIoMemCtx->paDataSeg[pIoMemCtx->iSegIdx].pvSeg;
143	pIoMemCtx->cbBufLeft = pIoMemCtx->paDataSeg[pIoMemCtx->iSegIdx].cbSeg;
144	}
145
146	*pcbData = cbData;
147	}
148	else
149	pIoMemCtx->pbBuf += cbData;
150
151	return pbBuf;
152	}
153
154	#ifdef DEBUG
155	static void pdmacFileCacheValidate(PPDMACFILECACHEGLOBAL pCache)
156	{
157	/* Amount of cached data should never exceed the maximum amount. */
158	AssertMsg(pCache->cbCached <= pCache->cbMax,
159	("Current amount of cached data exceeds maximum\n"));
160
161	/* The amount of cached data in the LRU and FRU list should match cbCached */
162	AssertMsg(pCache->LruRecentlyUsedIn.cbCached + pCache->LruFrequentlyUsed.cbCached == pCache->cbCached,
163	("Amount of cached data doesn't match\n"));
164
165	AssertMsg(pCache->LruRecentlyUsedOut.cbCached <= pCache->cbRecentlyUsedOutMax,
166	("Paged out list exceeds maximum\n"));
167	}
168	#endif
169
170	DECLINLINE(void) pdmacFileCacheLockEnter(PPDMACFILECACHEGLOBAL pCache)
171	{
172	RTCritSectEnter(&pCache->CritSect);
173	#ifdef DEBUG
174	pdmacFileCacheValidate(pCache);
175	#endif
176	}
177
178	DECLINLINE(void) pdmacFileCacheLockLeave(PPDMACFILECACHEGLOBAL pCache)
179	{
180	#ifdef DEBUG
181	pdmacFileCacheValidate(pCache);
182	#endif
183	RTCritSectLeave(&pCache->CritSect);
184	}
185
186	DECLINLINE(void) pdmacFileCacheSub(PPDMACFILECACHEGLOBAL pCache, uint32_t cbAmount)
187	{
188	PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
189	pCache->cbCached -= cbAmount;
190	}
191
192	DECLINLINE(void) pdmacFileCacheAdd(PPDMACFILECACHEGLOBAL pCache, uint32_t cbAmount)
193	{
194	PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
195	pCache->cbCached += cbAmount;
196	}
197
198	DECLINLINE(void) pdmacFileCacheListAdd(PPDMACFILELRULIST pList, uint32_t cbAmount)
199	{
200	pList->cbCached += cbAmount;
201	}
202
203	DECLINLINE(void) pdmacFileCacheListSub(PPDMACFILELRULIST pList, uint32_t cbAmount)
204	{
205	pList->cbCached -= cbAmount;
206	}
207
208	#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
209	/**
210	* Checks consistency of a LRU list.
211	*
212	* @returns nothing
213	* @param pList The LRU list to check.
214	* @param pNotInList Element which is not allowed to occur in the list.
215	*/
216	static void pdmacFileCacheCheckList(PPDMACFILELRULIST pList, PPDMACFILECACHEENTRY pNotInList)
217	{
218	PPDMACFILECACHEENTRY pCurr = pList->pHead;
219
220	/* Check that there are no double entries and no cycles in the list. */
221	while (pCurr)
222	{
223	PPDMACFILECACHEENTRY pNext = pCurr->pNext;
224
225	while (pNext)
226	{
227	AssertMsg(pCurr != pNext,
228	("Entry %#p is at least two times in list %#p or there is a cycle in the list\n",
229	pCurr, pList));
230	pNext = pNext->pNext;
231	}
232
233	AssertMsg(pCurr != pNotInList, ("Not allowed entry %#p is in list\n", pCurr));
234
235	if (!pCurr->pNext)
236	AssertMsg(pCurr == pList->pTail, ("End of list reached but last element is not list tail\n"));
237
238	pCurr = pCurr->pNext;
239	}
240	}
241	#endif
242
243	/**
244	* Unlinks a cache entry from the LRU list it is assigned to.
245	*
246	* @returns nothing.
247	* @param pEntry The entry to unlink.
248	*/
249	static void pdmacFileCacheEntryRemoveFromList(PPDMACFILECACHEENTRY pEntry)
250	{
251	PPDMACFILELRULIST pList = pEntry->pList;
252	PPDMACFILECACHEENTRY pPrev, pNext;
253
254	LogFlowFunc((": Deleting entry %#p from list %#p\n", pEntry, pList));
255
256	AssertPtr(pList);
257
258	#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
259	pdmacFileCacheCheckList(pList, NULL);
260	#endif
261
262	pPrev = pEntry->pPrev;
263	pNext = pEntry->pNext;
264
265	AssertMsg(pEntry != pPrev, ("Entry links to itself as previous element\n"));
266	AssertMsg(pEntry != pNext, ("Entry links to itself as next element\n"));
267
268	if (pPrev)
269	pPrev->pNext = pNext;
270	else
271	{
272	pList->pHead = pNext;
273
274	if (pNext)
275	pNext->pPrev = NULL;
276	}
277
278	if (pNext)
279	pNext->pPrev = pPrev;
280	else
281	{
282	pList->pTail = pPrev;
283
284	if (pPrev)
285	pPrev->pNext = NULL;
286	}
287
288	pEntry->pList = NULL;
289	pEntry->pPrev = NULL;
290	pEntry->pNext = NULL;
291	pdmacFileCacheListSub(pList, pEntry->cbData);
292	#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
293	pdmacFileCacheCheckList(pList, pEntry);
294	#endif
295	}
296
297	/**
298	* Adds a cache entry to the given LRU list unlinking it from the currently
299	* assigned list if needed.
300	*
301	* @returns nothing.
302	* @param pList List to the add entry to.
303	* @param pEntry Entry to add.
304	*/
305	static void pdmacFileCacheEntryAddToList(PPDMACFILELRULIST pList, PPDMACFILECACHEENTRY pEntry)
306	{
307	LogFlowFunc((": Adding entry %#p to list %#p\n", pEntry, pList));
308	#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
309	pdmacFileCacheCheckList(pList, NULL);
310	#endif
311
312	/* Remove from old list if needed */
313	if (pEntry->pList)
314	pdmacFileCacheEntryRemoveFromList(pEntry);
315
316	pEntry->pNext = pList->pHead;
317	if (pList->pHead)
318	pList->pHead->pPrev = pEntry;
319	else
320	{
321	Assert(!pList->pTail);
322	pList->pTail = pEntry;
323	}
324
325	pEntry->pPrev = NULL;
326	pList->pHead = pEntry;
327	pdmacFileCacheListAdd(pList, pEntry->cbData);
328	pEntry->pList = pList;
329	#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
330	pdmacFileCacheCheckList(pList, NULL);
331	#endif
332	}
333
334	/**
335	* Destroys a LRU list freeing all entries.
336	*
337	* @returns nothing
338	* @param pList Pointer to the LRU list to destroy.
339	*
340	* @note The caller must own the critical section of the cache.
341	*/
342	static void pdmacFileCacheDestroyList(PPDMACFILELRULIST pList)
343	{
344	while (pList->pHead)
345	{
346	PPDMACFILECACHEENTRY pEntry = pList->pHead;
347
348	pList->pHead = pEntry->pNext;
349
350	AssertMsg(!(pEntry->fFlags & (PDMACFILECACHE_ENTRY_IO_IN_PROGRESS \| PDMACFILECACHE_ENTRY_IS_DIRTY)),
351	("Entry is dirty and/or still in progress fFlags=%#x\n", pEntry->fFlags));
352
353	RTMemPageFree(pEntry->pbData);
354	RTMemFree(pEntry);
355	}
356	}
357
358	/**
359	* Tries to remove the given amount of bytes from a given list in the cache
360	* moving the entries to one of the given ghosts lists
361	*
362	* @returns Amount of data which could be freed.
363	* @param pCache Pointer to the global cache data.
364	* @param cbData The amount of the data to free.
365	* @param pListSrc The source list to evict data from.
366	* @param pGhostListSrc The ghost list removed entries should be moved to
367	* NULL if the entry should be freed.
368	* @param fReuseBuffer Flag whether a buffer should be reused if it has the same size
369	* @param ppbBuf Where to store the address of the buffer if an entry with the
370	* same size was found and fReuseBuffer is true.
371	*
372	* @note This function may return fewer bytes than requested because entries
373	* may be marked as non evictable if they are used for I/O at the
374	* moment.
375	*/
376	static size_t pdmacFileCacheEvictPagesFrom(PPDMACFILECACHEGLOBAL pCache, size_t cbData,
377	PPDMACFILELRULIST pListSrc, PPDMACFILELRULIST pGhostListDst,
378	bool fReuseBuffer, uint8_t **ppbBuffer)
379	{
380	size_t cbEvicted = 0;
381
382	PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
383
384	AssertMsg(cbData > 0, ("Evicting 0 bytes not possible\n"));
385	AssertMsg( !pGhostListDst
386	\|\| (pGhostListDst == &pCache->LruRecentlyUsedOut),
387	("Destination list must be NULL or the recently used but paged out list\n"));
388
389	if (fReuseBuffer)
390	{
391	AssertPtr(ppbBuffer);
392	*ppbBuffer = NULL;
393	}
394
395	/* Start deleting from the tail. */
396	PPDMACFILECACHEENTRY pEntry = pListSrc->pTail;
397
398	while ((cbEvicted < cbData) && pEntry)
399	{
400	PPDMACFILECACHEENTRY pCurr = pEntry;
401
402	pEntry = pEntry->pPrev;
403
404	/* We can't evict pages which are currently in progress */
405	if ( !(pCurr->fFlags & PDMACFILECACHE_ENTRY_IO_IN_PROGRESS)
406	&& (ASMAtomicReadU32(&pCurr->cRefs) == 0))
407	{
408	/* Ok eviction candidate. Grab the endpoint semaphore and check again
409	* because somebody else might have raced us. */
410	PPDMACFILEENDPOINTCACHE pEndpointCache = &pCurr->pEndpoint->DataCache;
411	RTSemRWRequestWrite(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT);
412
413	if (!(pCurr->fFlags & PDMACFILECACHE_ENTRY_IO_IN_PROGRESS)
414	&& (ASMAtomicReadU32(&pCurr->cRefs) == 0))
415	{
416	AssertMsg(!(pCurr->fFlags & PDMACFILECACHE_ENTRY_IS_DEPRECATED),
417	("This entry is deprecated so it should have the I/O in progress flag set\n"));
418	Assert(!pCurr->pbDataReplace);
419
420	LogFlow(("Evicting entry %#p (%u bytes)\n", pCurr, pCurr->cbData));
421
422	if (fReuseBuffer && (pCurr->cbData == cbData))
423	{
424	STAM_COUNTER_INC(&pCache->StatBuffersReused);
425	*ppbBuffer = pCurr->pbData;
426	}
427	else if (pCurr->pbData)
428	RTMemPageFree(pCurr->pbData);
429
430	pCurr->pbData = NULL;
431	cbEvicted += pCurr->cbData;
432
433	pdmacFileCacheEntryRemoveFromList(pCurr);
434	pdmacFileCacheSub(pCache, pCurr->cbData);
435
436	if (pGhostListDst)
437	{
438	RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
439
440	PPDMACFILECACHEENTRY pGhostEntFree = pGhostListDst->pTail;
441
442	/* We have to remove the last entries from the paged out list. */
443	while ( ((pGhostListDst->cbCached + pCurr->cbData) > pCache->cbRecentlyUsedOutMax)
444	&& pGhostEntFree)
445	{
446	PPDMACFILECACHEENTRY pFree = pGhostEntFree;
447	PPDMACFILEENDPOINTCACHE pEndpointCacheFree = &pFree->pEndpoint->DataCache;
448
449	pGhostEntFree = pGhostEntFree->pPrev;
450
451	RTSemRWRequestWrite(pEndpointCacheFree->SemRWEntries, RT_INDEFINITE_WAIT);
452
453	if (ASMAtomicReadU32(&pFree->cRefs) == 0)
454	{
455	pdmacFileCacheEntryRemoveFromList(pFree);
456
457	STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
458	RTAvlrFileOffsetRemove(pEndpointCacheFree->pTree, pFree->Core.Key);
459	STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
460
461	RTMemFree(pFree);
462	}
463
464	RTSemRWReleaseWrite(pEndpointCacheFree->SemRWEntries);
465	}
466
467	if (pGhostListDst->cbCached + pCurr->cbData > pCache->cbRecentlyUsedOutMax)
468	{
469	/* Couldn't remove enough entries. Delete */
470	STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
471	RTAvlrFileOffsetRemove(pCurr->pEndpoint->DataCache.pTree, pCurr->Core.Key);
472	STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
473
474	RTMemFree(pCurr);
475	}
476	else
477	pdmacFileCacheEntryAddToList(pGhostListDst, pCurr);
478	}
479	else
480	{
481	/* Delete the entry from the AVL tree it is assigned to. */
482	STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
483	RTAvlrFileOffsetRemove(pCurr->pEndpoint->DataCache.pTree, pCurr->Core.Key);
484	STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
485
486	RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
487	RTMemFree(pCurr);
488	}
489	}
490
491	}
492	else
493	LogFlow(("Entry %#p (%u bytes) is still in progress and can't be evicted\n", pCurr, pCurr->cbData));
494	}
495
496	return cbEvicted;
497	}
498
499	static bool pdmacFileCacheReclaim(PPDMACFILECACHEGLOBAL pCache, size_t cbData, bool fReuseBuffer, uint8_t **ppbBuffer)
500	{
501	size_t cbRemoved = 0;
502
503	if ((pCache->cbCached + cbData) < pCache->cbMax)
504	return true;
505	else if ((pCache->LruRecentlyUsedIn.cbCached + cbData) > pCache->cbRecentlyUsedInMax)
506	{
507	/* Try to evict as many bytes as possible from A1in */
508	cbRemoved = pdmacFileCacheEvictPagesFrom(pCache, cbData, &pCache->LruRecentlyUsedIn,
509	&pCache->LruRecentlyUsedOut, fReuseBuffer, ppbBuffer);
510
511	/*
512	* If it was not possible to remove enough entries
513	* try the frequently accessed cache.
514	*/
515	if (cbRemoved < cbData)
516	{
517	Assert(!fReuseBuffer \|\| !ppbBuffer); / It is not possible that we got a buffer with the correct size but we didn't freed enough data. */
518
519	cbRemoved += pdmacFileCacheEvictPagesFrom(pCache, cbData - cbRemoved, &pCache->LruFrequentlyUsed,
520	NULL, fReuseBuffer, ppbBuffer);
521	}
522	}
523	else
524	{
525	/* We have to remove entries from frequently access list. */
526	cbRemoved = pdmacFileCacheEvictPagesFrom(pCache, cbData, &pCache->LruFrequentlyUsed,
527	NULL, fReuseBuffer, ppbBuffer);
528	}
529
530	LogFlowFunc((": removed %u bytes, requested %u\n", cbRemoved, cbData));
531	return (cbRemoved >= cbData);
532	}
533
534	/**
535	* Initiates a read I/O task for the given entry.
536	*
537	* @returns nothing.
538	* @param pEntry The entry to fetch the data to.
539	*/
540	static void pdmacFileCacheReadFromEndpoint(PPDMACFILECACHEENTRY pEntry)
541	{
542	LogFlowFunc((": Reading data into cache entry %#p\n", pEntry));
543
544	/* Make sure no one evicts the entry while it is accessed. */
545	pEntry->fFlags \|= PDMACFILECACHE_ENTRY_IO_IN_PROGRESS;
546
547	PPDMACTASKFILE pIoTask = pdmacFileTaskAlloc(pEntry->pEndpoint);
548	AssertPtr(pIoTask);
549
550	AssertMsg(pEntry->pbData, ("Entry is in ghost state\n"));
551
552	pIoTask->pEndpoint = pEntry->pEndpoint;
553	pIoTask->enmTransferType = PDMACTASKFILETRANSFER_READ;
554	pIoTask->Off = pEntry->Core.Key;
555	pIoTask->DataSeg.cbSeg = pEntry->cbData;
556	pIoTask->DataSeg.pvSeg = pEntry->pbData;
557	pIoTask->pvUser = pEntry;
558	pIoTask->pfnCompleted = pdmacFileCacheTaskCompleted;
559
560	/* Send it off to the I/O manager. */
561	pdmacFileEpAddTask(pEntry->pEndpoint, pIoTask);
562	}
563
564	/**
565	* Initiates a write I/O task for the given entry.
566	*
567	* @returns nothing.
568	* @param pEntry The entry to read the data from.
569	*/
570	static void pdmacFileCacheWriteToEndpoint(PPDMACFILECACHEENTRY pEntry)
571	{
572	LogFlowFunc((": Writing data from cache entry %#p\n", pEntry));
573
574	/* Make sure no one evicts the entry while it is accessed. */
575	pEntry->fFlags \|= PDMACFILECACHE_ENTRY_IO_IN_PROGRESS;
576
577	PPDMACTASKFILE pIoTask = pdmacFileTaskAlloc(pEntry->pEndpoint);
578	AssertPtr(pIoTask);
579
580	AssertMsg(pEntry->pbData, ("Entry is in ghost state\n"));
581
582	pIoTask->pEndpoint = pEntry->pEndpoint;
583	pIoTask->enmTransferType = PDMACTASKFILETRANSFER_WRITE;
584	pIoTask->Off = pEntry->Core.Key;
585	pIoTask->DataSeg.cbSeg = pEntry->cbData;
586	pIoTask->DataSeg.pvSeg = pEntry->pbData;
587	pIoTask->pvUser = pEntry;
588	pIoTask->pfnCompleted = pdmacFileCacheTaskCompleted;
589	ASMAtomicIncU32(&pEntry->pEndpoint->DataCache.cWritesOutstanding);
590
591	/* Send it off to the I/O manager. */
592	pdmacFileEpAddTask(pEntry->pEndpoint, pIoTask);
593	}
594
595	/**
596	* Completes a task segment freeing all ressources and completes the task handle
597	* if everything was transfered.
598	*
599	* @returns Next task segment handle.
600	* @param pEndpointCache The endpoint cache.
601	* @param pTaskSeg Task segment to complete.
602	*/
603	static PPDMACFILETASKSEG pdmacFileCacheTaskComplete(PPDMACFILEENDPOINTCACHE pEndpointCache, PPDMACFILETASKSEG pTaskSeg)
604	{
605	PPDMACFILETASKSEG pNext = pTaskSeg->pNext;
606
607	uint32_t uOld = ASMAtomicSubS32(&pTaskSeg->pTask->cbTransferLeft, pTaskSeg->cbTransfer);
608	AssertMsg(uOld >= pTaskSeg->cbTransfer, ("New value would overflow\n"));
609	if (!(uOld - pTaskSeg->cbTransfer)
610	&& !ASMAtomicXchgBool(&pTaskSeg->pTask->fCompleted, true))
611	pdmR3AsyncCompletionCompleteTask(&pTaskSeg->pTask->Core, true);
612
613	RTMemFree(pTaskSeg);
614
615	return pNext;
616	}
617
618	/**
619	* Completion callback for I/O tasks.
620	*
621	* @returns nothing.
622	* @param pTask The completed task.
623	* @param pvUser Opaque user data.
624	*/
625	static void pdmacFileCacheTaskCompleted(PPDMACTASKFILE pTask, void *pvUser)
626	{
627	PPDMACFILECACHEENTRY pEntry = (PPDMACFILECACHEENTRY)pvUser;
628	PPDMACFILECACHEGLOBAL pCache = pEntry->pCache;
629	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint = pEntry->pEndpoint;
630	PPDMACFILEENDPOINTCACHE pEndpointCache = &pEndpoint->DataCache;
631
632	/* Reference the entry now as we are clearing the I/O in progres flag
633	* which protects the entry till now. */
634	pdmacFileEpCacheEntryRef(pEntry);
635
636	RTSemRWRequestWrite(pEndpoint->DataCache.SemRWEntries, RT_INDEFINITE_WAIT);
637	pEntry->fFlags &= ~PDMACFILECACHE_ENTRY_IO_IN_PROGRESS;
638
639	/* Process waiting segment list. The data in entry might have changed inbetween. */
640	PPDMACFILETASKSEG pCurr = pEntry->pWaitingHead;
641
642	AssertMsg((pCurr && pEntry->pWaitingTail) \|\| (!pCurr && !pEntry->pWaitingTail),
643	("The list tail was not updated correctly\n"));
644	pEntry->pWaitingTail = NULL;
645	pEntry->pWaitingHead = NULL;
646
647	if (pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE)
648	{
649	AssertMsg(pEndpointCache->cWritesOutstanding > 0, ("Completed write request but outstanding task count is 0\n"));
650	ASMAtomicDecU32(&pEndpointCache->cWritesOutstanding);
651
652	if (pEntry->fFlags & PDMACFILECACHE_ENTRY_IS_DEPRECATED)
653	{
654	AssertMsg(!pCurr, ("The entry is deprecated but has waiting write segments attached\n"));
655
656	RTMemPageFree(pEntry->pbData);
657	pEntry->pbData = pEntry->pbDataReplace;
658	pEntry->pbDataReplace = NULL;
659	pEntry->fFlags &= ~PDMACFILECACHE_ENTRY_IS_DEPRECATED;
660	}
661	else
662	{
663	pEntry->fFlags &= ~PDMACFILECACHE_ENTRY_IS_DIRTY;
664
665	while (pCurr)
666	{
667	AssertMsg(pCurr->fWrite, ("Completed write entries should never have read tasks attached\n"));
668
669	memcpy(pEntry->pbData + pCurr->uBufOffset, pCurr->pvBuf, pCurr->cbTransfer);
670	pEntry->fFlags \|= PDMACFILECACHE_ENTRY_IS_DIRTY;
671
672	pCurr = pdmacFileCacheTaskComplete(pEndpointCache, pCurr);
673	}
674	}
675	}
676	else
677	{
678	AssertMsg(pTask->enmTransferType == PDMACTASKFILETRANSFER_READ, ("Invalid transfer type\n"));
679	AssertMsg(!(pEntry->fFlags & PDMACFILECACHE_ENTRY_IS_DIRTY),("Invalid flags set\n"));
680
681	while (pCurr)
682	{
683	if (pCurr->fWrite)
684	{
685	memcpy(pEntry->pbData + pCurr->uBufOffset, pCurr->pvBuf, pCurr->cbTransfer);
686	pEntry->fFlags \|= PDMACFILECACHE_ENTRY_IS_DIRTY;
687	}
688	else
689	memcpy(pCurr->pvBuf, pEntry->pbData + pCurr->uBufOffset, pCurr->cbTransfer);
690
691	pCurr = pdmacFileCacheTaskComplete(pEndpointCache, pCurr);
692	}
693	}
694
695	if (pEntry->fFlags & PDMACFILECACHE_ENTRY_IS_DIRTY)
696	pdmacFileCacheWriteToEndpoint(pEntry);
697
698	/* Complete a pending flush if all writes have completed */
699	if (!ASMAtomicReadU32(&pEndpointCache->cWritesOutstanding))
700	{
701	PPDMASYNCCOMPLETIONTASKFILE pTaskFlush = (PPDMASYNCCOMPLETIONTASKFILE)ASMAtomicXchgPtr((void * volatile *)&pEndpointCache->pTaskFlush, NULL);
702	if (pTaskFlush)
703	pdmR3AsyncCompletionCompleteTask(&pTaskFlush->Core, true);
704	}
705
706	RTSemRWReleaseWrite(pEndpoint->DataCache.SemRWEntries);
707
708	/* Dereference so that it isn't protected anymore except we issued anyother write for it. */
709	pdmacFileEpCacheEntryRelease(pEntry);
710	}
711
712	/**
713	* Initializies the I/O cache.
714	*
715	* returns VBox status code.
716	* @param pClassFile The global class data for file endpoints.
717	* @param pCfgNode CFGM node to query configuration data from.
718	*/
719	int pdmacFileCacheInit(PPDMASYNCCOMPLETIONEPCLASSFILE pClassFile, PCFGMNODE pCfgNode)
720	{
721	int rc = VINF_SUCCESS;
722	PPDMACFILECACHEGLOBAL pCache = &pClassFile->Cache;
723
724	rc = CFGMR3QueryU32Def(pCfgNode, "CacheSize", &pCache->cbMax, 5 * _1M);
725	AssertLogRelRCReturn(rc, rc);
726
727	pCache->cbCached = 0;
728	LogFlowFunc((": Maximum number of bytes cached %u\n", pCache->cbMax));
729
730	/* Initialize members */
731	pCache->LruRecentlyUsedIn.pHead = NULL;
732	pCache->LruRecentlyUsedIn.pTail = NULL;
733	pCache->LruRecentlyUsedIn.cbCached = 0;
734
735	pCache->LruRecentlyUsedOut.pHead = NULL;
736	pCache->LruRecentlyUsedOut.pTail = NULL;
737	pCache->LruRecentlyUsedOut.cbCached = 0;
738
739	pCache->LruFrequentlyUsed.pHead = NULL;
740	pCache->LruFrequentlyUsed.pTail = NULL;
741	pCache->LruFrequentlyUsed.cbCached = 0;
742
743	pCache->cbRecentlyUsedInMax = (pCache->cbMax / 100) * 25; /* 25% of the buffer size */
744	pCache->cbRecentlyUsedOutMax = (pCache->cbMax / 100) * 50; /* 50% of the buffer size */
745	LogFlowFunc((": cbRecentlyUsedInMax=%u cbRecentlyUsedOutMax=%u\n", pCache->cbRecentlyUsedInMax, pCache->cbRecentlyUsedOutMax));
746
747	STAMR3Register(pClassFile->Core.pVM, &pCache->cbMax,
748	STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
749	"/PDM/AsyncCompletion/File/cbMax",
750	STAMUNIT_BYTES,
751	"Maximum cache size");
752	STAMR3Register(pClassFile->Core.pVM, &pCache->cbCached,
753	STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
754	"/PDM/AsyncCompletion/File/cbCached",
755	STAMUNIT_BYTES,
756	"Currently used cache");
757	STAMR3Register(pClassFile->Core.pVM, &pCache->LruRecentlyUsedIn.cbCached,
758	STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
759	"/PDM/AsyncCompletion/File/cbCachedMruIn",
760	STAMUNIT_BYTES,
761	"Number of bytes cached in MRU list");
762	STAMR3Register(pClassFile->Core.pVM, &pCache->LruRecentlyUsedOut.cbCached,
763	STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
764	"/PDM/AsyncCompletion/File/cbCachedMruOut",
765	STAMUNIT_BYTES,
766	"Number of bytes cached in FRU list");
767	STAMR3Register(pClassFile->Core.pVM, &pCache->LruFrequentlyUsed.cbCached,
768	STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
769	"/PDM/AsyncCompletion/File/cbCachedFru",
770	STAMUNIT_BYTES,
771	"Number of bytes cached in FRU ghost list");
772
773	#ifdef VBOX_WITH_STATISTICS
774	STAMR3Register(pClassFile->Core.pVM, &pCache->cHits,
775	STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
776	"/PDM/AsyncCompletion/File/CacheHits",
777	STAMUNIT_COUNT, "Number of hits in the cache");
778	STAMR3Register(pClassFile->Core.pVM, &pCache->cPartialHits,
779	STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
780	"/PDM/AsyncCompletion/File/CachePartialHits",
781	STAMUNIT_COUNT, "Number of partial hits in the cache");
782	STAMR3Register(pClassFile->Core.pVM, &pCache->cMisses,
783	STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
784	"/PDM/AsyncCompletion/File/CacheMisses",
785	STAMUNIT_COUNT, "Number of misses when accessing the cache");
786	STAMR3Register(pClassFile->Core.pVM, &pCache->StatRead,
787	STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
788	"/PDM/AsyncCompletion/File/CacheRead",
789	STAMUNIT_BYTES, "Number of bytes read from the cache");
790	STAMR3Register(pClassFile->Core.pVM, &pCache->StatWritten,
791	STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
792	"/PDM/AsyncCompletion/File/CacheWritten",
793	STAMUNIT_BYTES, "Number of bytes written to the cache");
794	STAMR3Register(pClassFile->Core.pVM, &pCache->StatTreeGet,
795	STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
796	"/PDM/AsyncCompletion/File/CacheTreeGet",
797	STAMUNIT_TICKS_PER_CALL, "Time taken to access an entry in the tree");
798	STAMR3Register(pClassFile->Core.pVM, &pCache->StatTreeInsert,
799	STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
800	"/PDM/AsyncCompletion/File/CacheTreeInsert",
801	STAMUNIT_TICKS_PER_CALL, "Time taken to insert an entry in the tree");
802	STAMR3Register(pClassFile->Core.pVM, &pCache->StatTreeRemove,
803	STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
804	"/PDM/AsyncCompletion/File/CacheTreeRemove",
805	STAMUNIT_TICKS_PER_CALL, "Time taken to remove an entry an the tree");
806	STAMR3Register(pClassFile->Core.pVM, &pCache->StatBuffersReused,
807	STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
808	"/PDM/AsyncCompletion/File/CacheBuffersReused",
809	STAMUNIT_COUNT, "Number of times a buffer could be reused");
810	#endif
811
812	/* Initialize the critical section */
813	rc = RTCritSectInit(&pCache->CritSect);
814
815	if (RT_SUCCESS(rc))
816	LogRel(("AIOMgr: Cache successfully initialised. Cache size is %u bytes\n", pCache->cbMax));
817
818	return rc;
819	}
820
821	/**
822	* Destroysthe cache freeing all data.
823	*
824	* returns nothing.
825	* @param pClassFile The global class data for file endpoints.
826	*/
827	void pdmacFileCacheDestroy(PPDMASYNCCOMPLETIONEPCLASSFILE pClassFile)
828	{
829	PPDMACFILECACHEGLOBAL pCache = &pClassFile->Cache;
830
831	/* Make sure no one else uses the cache now */
832	pdmacFileCacheLockEnter(pCache);
833
834	/* Cleanup deleting all cache entries waiting for in progress entries to finish. */
835	pdmacFileCacheDestroyList(&pCache->LruRecentlyUsedIn);
836	pdmacFileCacheDestroyList(&pCache->LruRecentlyUsedOut);
837	pdmacFileCacheDestroyList(&pCache->LruFrequentlyUsed);
838
839	pdmacFileCacheLockLeave(pCache);
840
841	RTCritSectDelete(&pCache->CritSect);
842	}
843
844	/**
845	* Initializes per endpoint cache data
846	* like the AVL tree used to access cached entries.
847	*
848	* @returns VBox status code.
849	* @param pEndpoint The endpoint to init the cache for,
850	* @param pClassFile The global class data for file endpoints.
851	*/
852	int pdmacFileEpCacheInit(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMASYNCCOMPLETIONEPCLASSFILE pClassFile)
853	{
854	PPDMACFILEENDPOINTCACHE pEndpointCache = &pEndpoint->DataCache;
855
856	pEndpointCache->pCache = &pClassFile->Cache;
857
858	int rc = RTSemRWCreate(&pEndpointCache->SemRWEntries);
859	if (RT_SUCCESS(rc))
860	{
861	pEndpointCache->pTree = (PAVLRFOFFTREE)RTMemAllocZ(sizeof(AVLRFOFFTREE));
862	if (!pEndpointCache->pTree)
863	{
864	rc = VERR_NO_MEMORY;
865	RTSemRWDestroy(pEndpointCache->SemRWEntries);
866	}
867	}
868
869	#ifdef VBOX_WITH_STATISTICS
870	if (RT_SUCCESS(rc))
871	{
872	STAMR3RegisterF(pClassFile->Core.pVM, &pEndpointCache->StatWriteDeferred,
873	STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
874	STAMUNIT_COUNT, "Number of deferred writes",
875	"/PDM/AsyncCompletion/File/%s/Cache/DeferredWrites", RTPathFilename(pEndpoint->Core.pszUri));
876	}
877	#endif
878
879	return rc;
880	}
881
882	/**
883	* Callback for the AVL destroy routine. Frees a cache entry for this endpoint.
884	*
885	* @returns IPRT status code.
886	* @param pNode The node to destroy.
887	* @param pvUser Opaque user data.
888	*/
889	static int pdmacFileEpCacheEntryDestroy(PAVLRFOFFNODECORE pNode, void *pvUser)
890	{
891	PPDMACFILECACHEENTRY pEntry = (PPDMACFILECACHEENTRY)pNode;
892	PPDMACFILECACHEGLOBAL pCache = (PPDMACFILECACHEGLOBAL)pvUser;
893	PPDMACFILEENDPOINTCACHE pEndpointCache = &pEntry->pEndpoint->DataCache;
894
895	while (ASMAtomicReadU32(&pEntry->fFlags) & (PDMACFILECACHE_ENTRY_IO_IN_PROGRESS \| PDMACFILECACHE_ENTRY_IS_DIRTY))
896	{
897	RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
898	RTThreadSleep(250);
899	RTSemRWRequestWrite(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT);
900	}
901
902	AssertMsg(!(pEntry->fFlags & (PDMACFILECACHE_ENTRY_IO_IN_PROGRESS \| PDMACFILECACHE_ENTRY_IS_DIRTY)),
903	("Entry is dirty and/or still in progress fFlags=%#x\n", pEntry->fFlags));
904
905	bool fUpdateCache = pEntry->pList == &pCache->LruFrequentlyUsed
906	\|\| pEntry->pList == &pCache->LruRecentlyUsedIn;
907
908	pdmacFileCacheEntryRemoveFromList(pEntry);
909
910	if (fUpdateCache)
911	pdmacFileCacheSub(pCache, pEntry->cbData);
912
913	RTMemPageFree(pEntry->pbData);
914	RTMemFree(pEntry);
915
916	return VINF_SUCCESS;
917	}
918
919	/**
920	* Destroys all cache ressources used by the given endpoint.
921	*
922	* @returns nothing.
923	* @param pEndpoint The endpoint to the destroy.
924	*/
925	void pdmacFileEpCacheDestroy(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
926	{
927	PPDMACFILEENDPOINTCACHE pEndpointCache = &pEndpoint->DataCache;
928	PPDMACFILECACHEGLOBAL pCache = pEndpointCache->pCache;
929
930	/* Make sure nobody is accessing the cache while we delete the tree. */
931	pdmacFileCacheLockEnter(pCache);
932	RTSemRWRequestWrite(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT);
933	RTAvlrFileOffsetDestroy(pEndpointCache->pTree, pdmacFileEpCacheEntryDestroy, pCache);
934	RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
935	pdmacFileCacheLockLeave(pCache);
936
937	RTSemRWDestroy(pEndpointCache->SemRWEntries);
938
939	#ifdef VBOX_WITH_STATISTICS
940	PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;
941
942	STAMR3Deregister(pEpClassFile->Core.pVM, &pEndpointCache->StatWriteDeferred);
943	#endif
944	}
945
946	static PPDMACFILECACHEENTRY pdmacFileEpCacheGetCacheEntryByOffset(PPDMACFILEENDPOINTCACHE pEndpointCache, RTFOFF off)
947	{
948	PPDMACFILECACHEGLOBAL pCache = pEndpointCache->pCache;
949	PPDMACFILECACHEENTRY pEntry = NULL;
950
951	STAM_PROFILE_ADV_START(&pCache->StatTreeGet, Cache);
952
953	RTSemRWRequestRead(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT);
954	pEntry = (PPDMACFILECACHEENTRY)RTAvlrFileOffsetRangeGet(pEndpointCache->pTree, off);
955	if (pEntry)
956	pdmacFileEpCacheEntryRef(pEntry);
957	RTSemRWReleaseRead(pEndpointCache->SemRWEntries);
958
959	STAM_PROFILE_ADV_STOP(&pCache->StatTreeGet, Cache);
960
961	return pEntry;
962	}
963
964	static PPDMACFILECACHEENTRY pdmacFileEpCacheGetCacheBestFitEntryByOffset(PPDMACFILEENDPOINTCACHE pEndpointCache, RTFOFF off)
965	{
966	PPDMACFILECACHEGLOBAL pCache = pEndpointCache->pCache;
967	PPDMACFILECACHEENTRY pEntry = NULL;
968
969	STAM_PROFILE_ADV_START(&pCache->StatTreeGet, Cache);
970
971	RTSemRWRequestRead(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT);
972	pEntry = (PPDMACFILECACHEENTRY)RTAvlrFileOffsetGetBestFit(pEndpointCache->pTree, off, true /fAbove/);
973	if (pEntry)
974	pdmacFileEpCacheEntryRef(pEntry);
975	RTSemRWReleaseRead(pEndpointCache->SemRWEntries);
976
977	STAM_PROFILE_ADV_STOP(&pCache->StatTreeGet, Cache);
978
979	return pEntry;
980	}
981
982	static void pdmacFileEpCacheInsertEntry(PPDMACFILEENDPOINTCACHE pEndpointCache, PPDMACFILECACHEENTRY pEntry)
983	{
984	PPDMACFILECACHEGLOBAL pCache = pEndpointCache->pCache;
985
986	STAM_PROFILE_ADV_START(&pCache->StatTreeInsert, Cache);
987	RTSemRWRequestWrite(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT);
988	bool fInserted = RTAvlrFileOffsetInsert(pEndpointCache->pTree, &pEntry->Core);
989	AssertMsg(fInserted, ("Node was not inserted into tree\n"));
990	STAM_PROFILE_ADV_STOP(&pCache->StatTreeInsert, Cache);
991	RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
992	}
993
994	/**
995	* Allocates and initializes a new entry for the cache.
996	* The entry has a reference count of 1.
997	*
998	* @returns Pointer to the new cache entry or NULL if out of memory.
999	* @param pCache The cache the entry belongs to.
1000	* @param pEndoint The endpoint the entry holds data for.
1001	* @param off Start offset.
1002	* @param cbData Size of the cache entry.
1003	* @param pbBuffer Pointer to the buffer to use.
1004	* NULL if a new buffer should be allocated.
1005	* The buffer needs to have the same size of the entry.
1006	*/
1007	static PPDMACFILECACHEENTRY pdmacFileCacheEntryAlloc(PPDMACFILECACHEGLOBAL pCache,
1008	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
1009	RTFOFF off, size_t cbData, uint8_t *pbBuffer)
1010	{
1011	PPDMACFILECACHEENTRY pEntryNew = (PPDMACFILECACHEENTRY)RTMemAllocZ(sizeof(PDMACFILECACHEENTRY));
1012
1013	if (RT_UNLIKELY(!pEntryNew))
1014	return NULL;
1015
1016	pEntryNew->Core.Key = off;
1017	pEntryNew->Core.KeyLast = off + cbData - 1;
1018	pEntryNew->pEndpoint = pEndpoint;
1019	pEntryNew->pCache = pCache;
1020	pEntryNew->fFlags = 0;
1021	pEntryNew->cRefs = 1; /* We are using it now. */
1022	pEntryNew->pList = NULL;
1023	pEntryNew->cbData = cbData;
1024	pEntryNew->pWaitingHead = NULL;
1025	pEntryNew->pWaitingTail = NULL;
1026	pEntryNew->pbDataReplace = NULL;
1027	if (pbBuffer)
1028	pEntryNew->pbData = pbBuffer;
1029	else
1030	pEntryNew->pbData = (uint8_t *)RTMemPageAlloc(cbData);
1031
1032	if (RT_UNLIKELY(!pEntryNew->pbData))
1033	{
1034	RTMemFree(pEntryNew);
1035	return NULL;
1036	}
1037
1038	return pEntryNew;
1039	}
1040
1041	/**
1042	* Adds a segment to the waiting list for a cache entry
1043	* which is currently in progress.
1044	*
1045	* @returns nothing.
1046	* @param pEntry The cache entry to add the segment to.
1047	* @param pSeg The segment to add.
1048	*/
1049	DECLINLINE(void) pdmacFileEpCacheEntryAddWaitingSegment(PPDMACFILECACHEENTRY pEntry, PPDMACFILETASKSEG pSeg)
1050	{
1051	pSeg->pNext = NULL;
1052
1053	if (pEntry->pWaitingHead)
1054	{
1055	AssertPtr(pEntry->pWaitingTail);
1056
1057	pEntry->pWaitingTail->pNext = pSeg;
1058	pEntry->pWaitingTail = pSeg;
1059	}
1060	else
1061	{
1062	Assert(!pEntry->pWaitingTail);
1063
1064	pEntry->pWaitingHead = pSeg;
1065	pEntry->pWaitingTail = pSeg;
1066	}
1067	}
1068
1069	/**
1070	* Checks that a set of flags is set/clear acquiring the R/W semaphore
1071	* in exclusive mode.
1072	*
1073	* @returns true if the flag in fSet is set and the one in fClear is clear.
1074	* false othwerise.
1075	* The R/W semaphore is only held if true is returned.
1076	*
1077	* @param pEndpointCache The endpoint cache instance data.
1078	* @param pEntry The entry to check the flags for.
1079	* @param fSet The flag which is tested to be set.
1080	* @param fClear The flag which is tested to be clear.
1081	*/
1082	DECLINLINE(bool) pdmacFileEpCacheEntryFlagIsSetClearAcquireLock(PPDMACFILEENDPOINTCACHE pEndpointCache,
1083	PPDMACFILECACHEENTRY pEntry,
1084	uint32_t fSet, uint32_t fClear)
1085	{
1086	uint32_t fFlags = ASMAtomicReadU32(&pEntry->fFlags);
1087	bool fPassed = ((fFlags & fSet) && !(fFlags & fClear));
1088
1089	if (fPassed)
1090	{
1091	/* Acquire the lock and check again becuase the completion callback might have raced us. */
1092	RTSemRWRequestWrite(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT);
1093
1094	fFlags = ASMAtomicReadU32(&pEntry->fFlags);
1095	fPassed = ((fFlags & fSet) && !(fFlags & fClear));
1096
1097	/* Drop the lock if we didn't passed the test. */
1098	if (!fPassed)
1099	RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
1100	}
1101
1102	return fPassed;
1103	}
1104
1105	/**
1106	* Copies data to a buffer described by a I/O memory context.
1107	*
1108	* @returns nothing.
1109	* @param pIoMemCtx The I/O memory context to copy the data into.
1110	* @param pbData Pointer to the data data to copy.
1111	* @param cbData Amount of data to copy.
1112	*/
1113	static void pdmacFileEpCacheCopyToIoMemCtx(PPDMIOMEMCTX pIoMemCtx,
1114	uint8_t *pbData,
1115	size_t cbData)
1116	{
1117	while (cbData)
1118	{
1119	size_t cbCopy = cbData;
1120	uint8_t *pbBuf = pdmIoMemCtxGetBuffer(pIoMemCtx, &cbCopy);
1121
1122	AssertPtr(pbBuf);
1123
1124	memcpy(pbBuf, pbData, cbCopy);
1125
1126	cbData -= cbCopy;
1127	pbData += cbCopy;
1128	}
1129	}
1130
1131	/**
1132	* Copies data from a buffer described by a I/O memory context.
1133	*
1134	* @returns nothing.
1135	* @param pIoMemCtx The I/O memory context to copy the data from.
1136	* @param pbData Pointer to the destination buffer.
1137	* @param cbData Amount of data to copy.
1138	*/
1139	static void pdmacFileEpCacheCopyFromIoMemCtx(PPDMIOMEMCTX pIoMemCtx,
1140	uint8_t *pbData,
1141	size_t cbData)
1142	{
1143	while (cbData)
1144	{
1145	size_t cbCopy = cbData;
1146	uint8_t *pbBuf = pdmIoMemCtxGetBuffer(pIoMemCtx, &cbCopy);
1147
1148	AssertPtr(pbBuf);
1149
1150	memcpy(pbData, pbBuf, cbCopy);
1151
1152	cbData -= cbCopy;
1153	pbData += cbCopy;
1154	}
1155	}
1156
1157	/**
1158	* Add a buffer described by the I/O memory context
1159	* to the entry waiting for completion.
1160	*
1161	* @returns nothing.
1162	* @param pEntry The entry to add the buffer to.
1163	* @param pTask Task associated with the buffer.
1164	* @param pIoMemCtx The memory context to use.
1165	* @param OffDiff Offset from the start of the buffer
1166	* in the entry.
1167	* @param cbData Amount of data to wait for onthis entry.
1168	* @param fWrite Flag whether the task waits because it wants to write
1169	* to the cache entry.
1170	*/
1171	static void pdmacFileEpCacheEntryWaitersAdd(PPDMACFILECACHEENTRY pEntry,
1172	PPDMASYNCCOMPLETIONTASKFILE pTask,
1173	PPDMIOMEMCTX pIoMemCtx,
1174	RTFOFF OffDiff,
1175	size_t cbData,
1176	bool fWrite)
1177	{
1178	while (cbData)
1179	{
1180	PPDMACFILETASKSEG pSeg = (PPDMACFILETASKSEG)RTMemAllocZ(sizeof(PDMACFILETASKSEG));
1181	size_t cbSeg = cbData;
1182	uint8_t *pbBuf = pdmIoMemCtxGetBuffer(pIoMemCtx, &cbSeg);
1183
1184	pSeg->pTask = pTask;
1185	pSeg->uBufOffset = OffDiff;
1186	pSeg->cbTransfer = cbSeg;
1187	pSeg->pvBuf = pbBuf;
1188	pSeg->fWrite = fWrite;
1189
1190	pdmacFileEpCacheEntryAddWaitingSegment(pEntry, pSeg);
1191
1192	cbData -= cbSeg;
1193	OffDiff += cbSeg;
1194	}
1195	}
1196
1197	/**
1198	* Passthrough a part of a request directly to the I/O manager
1199	* handling the endpoint.
1200	*
1201	* @returns nothing.
1202	* @param pEndpoint The endpoint.
1203	* @param pTask The task.
1204	* @param pIoMemCtx The I/O memory context to use.
1205	* @param offStart Offset to start transfer from.
1206	* @param cbData Amount of data to transfer.
1207	* @param enmTransferType The transfer type (read/write)
1208	*/
1209	static void pdmacFileEpCacheRequestPassthrough(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
1210	PPDMASYNCCOMPLETIONTASKFILE pTask,
1211	PPDMIOMEMCTX pIoMemCtx,
1212	RTFOFF offStart, size_t cbData,
1213	PDMACTASKFILETRANSFER enmTransferType)
1214	{
1215	while (cbData)
1216	{
1217	size_t cbSeg = cbData;
1218	uint8_t *pbBuf = pdmIoMemCtxGetBuffer(pIoMemCtx, &cbSeg);
1219	PPDMACTASKFILE pIoTask = pdmacFileTaskAlloc(pEndpoint);
1220	AssertPtr(pIoTask);
1221
1222	pIoTask->pEndpoint = pEndpoint;
1223	pIoTask->enmTransferType = enmTransferType;
1224	pIoTask->Off = offStart;
1225	pIoTask->DataSeg.cbSeg = cbSeg;
1226	pIoTask->DataSeg.pvSeg = pbBuf;
1227	pIoTask->pvUser = pTask;
1228	pIoTask->pfnCompleted = pdmacFileEpTaskCompleted;
1229
1230	offStart += cbSeg;
1231	cbData -= cbSeg;
1232
1233	/* Send it off to the I/O manager. */
1234	pdmacFileEpAddTask(pEndpoint, pIoTask);
1235	}
1236	}
1237
1238	/**
1239	* Reads the specified data from the endpoint using the cache if possible.
1240	*
1241	* @returns VBox status code.
1242	* @param pEndpoint The endpoint to read from.
1243	* @param pTask The task structure used as identifier for this request.
1244	* @param off The offset to start reading from.
1245	* @param paSegments Pointer to the array holding the destination buffers.
1246	* @param cSegments Number of segments in the array.
1247	* @param cbRead Number of bytes to read.
1248	*/
1249	int pdmacFileEpCacheRead(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMASYNCCOMPLETIONTASKFILE pTask,
1250	RTFOFF off, PCPDMDATASEG paSegments, size_t cSegments,
1251	size_t cbRead)
1252	{
1253	int rc = VINF_SUCCESS;
1254	PPDMACFILEENDPOINTCACHE pEndpointCache = &pEndpoint->DataCache;
1255	PPDMACFILECACHEGLOBAL pCache = pEndpointCache->pCache;
1256	PPDMACFILECACHEENTRY pEntry;
1257
1258	LogFlowFunc((": pEndpoint=%#p{%s} pTask=%#p off=%RTfoff paSegments=%#p cSegments=%u cbRead=%u\n",
1259	pEndpoint, pEndpoint->Core.pszUri, pTask, off, paSegments, cSegments, cbRead));
1260
1261	pTask->cbTransferLeft = cbRead;
1262	/* Set to completed to make sure that the task is valid while we access it. */
1263	ASMAtomicWriteBool(&pTask->fCompleted, true);
1264
1265	/* Init the I/O memory context */
1266	PDMIOMEMCTX IoMemCtx;
1267	pdmIoMemCtxInit(&IoMemCtx, paSegments, cSegments);
1268
1269	while (cbRead)
1270	{
1271	size_t cbToRead;
1272
1273	pEntry = pdmacFileEpCacheGetCacheEntryByOffset(pEndpointCache, off);
1274
1275	/*
1276	* If there is no entry we try to create a new one eviciting unused pages
1277	* if the cache is full. If this is not possible we will pass the request through
1278	* and skip the caching (all entries may be still in progress so they can't
1279	* be evicted)
1280	* If we have an entry it can be in one of the LRU lists where the entry
1281	* contains data (recently used or frequently used LRU) so we can just read
1282	* the data we need and put the entry at the head of the frequently used LRU list.
1283	* In case the entry is in one of the ghost lists it doesn't contain any data.
1284	* We have to fetch it again evicting pages from either T1 or T2 to make room.
1285	*/
1286	if (pEntry)
1287	{
1288	RTFOFF OffDiff = off - pEntry->Core.Key;
1289
1290	AssertMsg(off >= pEntry->Core.Key,
1291	("Overflow in calculation off=%RTfoff OffsetAligned=%RTfoff\n",
1292	off, pEntry->Core.Key));
1293
1294	AssertPtr(pEntry->pList);
1295
1296	cbToRead = RT_MIN(pEntry->cbData - OffDiff, cbRead);
1297
1298	AssertMsg(off + (RTFOFF)cbToRead <= pEntry->Core.Key + pEntry->Core.KeyLast + 1,
1299	("Buffer of cache entry exceeded off=%RTfoff cbToRead=%d\n",
1300	off, cbToRead));
1301
1302	cbRead -= cbToRead;
1303
1304	if (!cbRead)
1305	STAM_COUNTER_INC(&pCache->cHits);
1306	else
1307	STAM_COUNTER_INC(&pCache->cPartialHits);
1308
1309	STAM_COUNTER_ADD(&pCache->StatRead, cbToRead);
1310
1311	/* Ghost lists contain no data. */
1312	if ( (pEntry->pList == &pCache->LruRecentlyUsedIn)
1313	\|\| (pEntry->pList == &pCache->LruFrequentlyUsed))
1314	{
1315	if (pdmacFileEpCacheEntryFlagIsSetClearAcquireLock(pEndpointCache, pEntry,
1316	PDMACFILECACHE_ENTRY_IS_DEPRECATED,
1317	0))
1318	{
1319	/* Entry is deprecated. Read data from the new buffer. */
1320	pdmacFileEpCacheCopyToIoMemCtx(&IoMemCtx, pEntry->pbDataReplace + OffDiff, cbToRead);
1321	ASMAtomicSubS32(&pTask->cbTransferLeft, cbToRead);
1322	RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
1323	}
1324	else
1325	{
1326	if (pdmacFileEpCacheEntryFlagIsSetClearAcquireLock(pEndpointCache, pEntry,
1327	PDMACFILECACHE_ENTRY_IO_IN_PROGRESS,
1328	PDMACFILECACHE_ENTRY_IS_DIRTY))
1329	{
1330	/* Entry didn't completed yet. Append to the list */
1331	pdmacFileEpCacheEntryWaitersAdd(pEntry, pTask,
1332	&IoMemCtx,
1333	OffDiff, cbToRead,
1334	false /* fWrite */);
1335	RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
1336	}
1337	else
1338	{
1339	/* Read as much as we can from the entry. */
1340	pdmacFileEpCacheCopyToIoMemCtx(&IoMemCtx, pEntry->pbData + OffDiff, cbToRead);
1341	ASMAtomicSubS32(&pTask->cbTransferLeft, cbToRead);
1342	}
1343	}
1344
1345	/* Move this entry to the top position */
1346	if (pEntry->pList == &pCache->LruFrequentlyUsed)
1347	{
1348	pdmacFileCacheLockEnter(pCache);
1349	pdmacFileCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
1350	pdmacFileCacheLockLeave(pCache);
1351	}
1352	/* Release the entry */
1353	pdmacFileEpCacheEntryRelease(pEntry);
1354	}
1355	else
1356	{
1357	uint8_t *pbBuffer = NULL;
1358
1359	LogFlow(("Fetching data for ghost entry %#p from file\n", pEntry));
1360
1361	pdmacFileCacheLockEnter(pCache);
1362	pdmacFileCacheEntryRemoveFromList(pEntry); /* Remove it before we remove data, otherwise it may get freed when evicting data. */
1363	bool fEnough = pdmacFileCacheReclaim(pCache, pEntry->cbData, true, &pbBuffer);
1364
1365	/* Move the entry to Am and fetch it to the cache. */
1366	if (fEnough)
1367	{
1368	pdmacFileCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
1369	pdmacFileCacheAdd(pCache, pEntry->cbData);
1370	pdmacFileCacheLockLeave(pCache);
1371
1372	if (pbBuffer)
1373	pEntry->pbData = pbBuffer;
1374	else
1375	pEntry->pbData = (uint8_t *)RTMemPageAlloc(pEntry->cbData);
1376	AssertPtr(pEntry->pbData);
1377
1378	pdmacFileEpCacheEntryWaitersAdd(pEntry, pTask,
1379	&IoMemCtx,
1380	OffDiff, cbToRead,
1381	false /* fWrite */);
1382	pdmacFileCacheReadFromEndpoint(pEntry);
1383	/* Release the entry */
1384	pdmacFileEpCacheEntryRelease(pEntry);
1385	}
1386	else
1387	{
1388	RTSemRWRequestWrite(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT);
1389	STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
1390	RTAvlrFileOffsetRemove(pEndpointCache->pTree, pEntry->Core.Key);
1391	STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
1392	RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
1393
1394	pdmacFileCacheLockLeave(pCache);
1395
1396	RTMemFree(pEntry);
1397
1398	pdmacFileEpCacheRequestPassthrough(pEndpoint, pTask,
1399	&IoMemCtx, off, cbToRead,
1400	PDMACTASKFILETRANSFER_READ);
1401	}
1402	}
1403	}
1404	else
1405	{
1406	/* No entry found for this offset. Get best fit entry and fetch the data to the cache. */
1407	size_t cbToReadAligned;
1408	PPDMACFILECACHEENTRY pEntryBestFit = pdmacFileEpCacheGetCacheBestFitEntryByOffset(pEndpointCache, off);
1409
1410	LogFlow(("%sbest fit entry for off=%RTfoff (BestFit=%RTfoff BestFitEnd=%RTfoff BestFitSize=%u)\n",
1411	pEntryBestFit ? "" : "No ",
1412	off,
1413	pEntryBestFit ? pEntryBestFit->Core.Key : 0,
1414	pEntryBestFit ? pEntryBestFit->Core.KeyLast : 0,
1415	pEntryBestFit ? pEntryBestFit->cbData : 0));
1416
1417	if ( pEntryBestFit
1418	&& off + (RTFOFF)cbRead > pEntryBestFit->Core.Key)
1419	{
1420	cbToRead = pEntryBestFit->Core.Key - off;
1421	pdmacFileEpCacheEntryRelease(pEntryBestFit);
1422	cbToReadAligned = cbToRead;
1423	}
1424	else
1425	{
1426	/*
1427	* Align the size to a 4KB boundary.
1428	* Memory size is aligned to a page boundary
1429	* and memory is wasted if the size is rahter small.
1430	* (For example reads with a size of 512 bytes.
1431	*/
1432	cbToRead = cbRead;
1433	cbToReadAligned = RT_ALIGN_Z(cbRead, PAGE_SIZE);
1434
1435	/* Clip read to file size */
1436	cbToReadAligned = RT_MIN(pEndpoint->cbFile - off, cbToReadAligned);
1437	if (pEntryBestFit)
1438	{
1439	Assert(pEntryBestFit->Core.Key >= off);
1440	cbToReadAligned = RT_MIN(cbToReadAligned, (uint64_t)pEntryBestFit->Core.Key - off);
1441	pdmacFileEpCacheEntryRelease(pEntryBestFit);
1442	}
1443	}
1444
1445	cbRead -= cbToRead;
1446
1447	if (!cbRead)
1448	STAM_COUNTER_INC(&pCache->cMisses);
1449	else
1450	STAM_COUNTER_INC(&pCache->cPartialHits);
1451
1452	uint8_t *pbBuffer = NULL;
1453
1454	pdmacFileCacheLockEnter(pCache);
1455	bool fEnough = pdmacFileCacheReclaim(pCache, cbToReadAligned, true, &pbBuffer);
1456
1457	if (fEnough)
1458	{
1459	LogFlow(("Evicted enough bytes (%u requested). Creating new cache entry\n", cbToReadAligned));
1460
1461	PPDMACFILECACHEENTRY pEntryNew = pdmacFileCacheEntryAlloc(pCache, pEndpoint, off, cbToReadAligned, pbBuffer);
1462	AssertPtr(pEntryNew);
1463
1464	pdmacFileCacheEntryAddToList(&pCache->LruRecentlyUsedIn, pEntryNew);
1465	pdmacFileCacheAdd(pCache, cbToReadAligned);
1466	pdmacFileCacheLockLeave(pCache);
1467
1468	pdmacFileEpCacheInsertEntry(pEndpointCache, pEntryNew);
1469
1470	AssertMsg( (off >= pEntryNew->Core.Key)
1471	&& (off + (RTFOFF)cbToRead <= pEntryNew->Core.Key + pEntryNew->Core.KeyLast + 1),
1472	("Overflow in calculation off=%RTfoff OffsetAligned=%RTfoff\n",
1473	off, pEntryNew->Core.Key));
1474
1475	pdmacFileEpCacheEntryWaitersAdd(pEntryNew, pTask,
1476	&IoMemCtx, 0, cbToRead,
1477	false /* fWrite */);
1478	pdmacFileCacheReadFromEndpoint(pEntryNew);
1479	pdmacFileEpCacheEntryRelease(pEntryNew); /* it is protected by the I/O in progress flag now. */
1480	}
1481	else
1482	{
1483	pdmacFileCacheLockLeave(pCache);
1484
1485	/*
1486	* There is not enough free space in the cache.
1487	* Pass the request directly to the I/O manager.
1488	*/
1489	LogFlow(("Couldn't evict %u bytes from the cache. Remaining request will be passed through\n", cbToRead));
1490
1491	pdmacFileEpCacheRequestPassthrough(pEndpoint, pTask,
1492	&IoMemCtx, off, cbToRead,
1493	PDMACTASKFILETRANSFER_READ);
1494	}
1495	}
1496	off += cbToRead;
1497	}
1498
1499	ASMAtomicWriteBool(&pTask->fCompleted, false);
1500
1501	if (ASMAtomicReadS32(&pTask->cbTransferLeft) == 0
1502	&& !ASMAtomicXchgBool(&pTask->fCompleted, true))
1503	pdmR3AsyncCompletionCompleteTask(&pTask->Core, false);
1504	else
1505	rc = VINF_AIO_TASK_PENDING;
1506
1507	LogFlowFunc((": Leave rc=%Rrc\n", rc));
1508
1509	return rc;
1510	}
1511
1512	/**
1513	* Writes the given data to the endpoint using the cache if possible.
1514	*
1515	* @returns VBox status code.
1516	* @param pEndpoint The endpoint to write to.
1517	* @param pTask The task structure used as identifier for this request.
1518	* @param off The offset to start writing to
1519	* @param paSegments Pointer to the array holding the source buffers.
1520	* @param cSegments Number of segments in the array.
1521	* @param cbWrite Number of bytes to write.
1522	*/
1523	int pdmacFileEpCacheWrite(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMASYNCCOMPLETIONTASKFILE pTask,
1524	RTFOFF off, PCPDMDATASEG paSegments, size_t cSegments,
1525	size_t cbWrite)
1526	{
1527	int rc = VINF_SUCCESS;
1528	PPDMACFILEENDPOINTCACHE pEndpointCache = &pEndpoint->DataCache;
1529	PPDMACFILECACHEGLOBAL pCache = pEndpointCache->pCache;
1530	PPDMACFILECACHEENTRY pEntry;
1531
1532	LogFlowFunc((": pEndpoint=%#p{%s} pTask=%#p off=%RTfoff paSegments=%#p cSegments=%u cbWrite=%u\n",
1533	pEndpoint, pEndpoint->Core.pszUri, pTask, off, paSegments, cSegments, cbWrite));
1534
1535	pTask->cbTransferLeft = cbWrite;
1536	/* Set to completed to make sure that the task is valid while we access it. */
1537	ASMAtomicWriteBool(&pTask->fCompleted, true);
1538
1539	/* Init the I/O memory context */
1540	PDMIOMEMCTX IoMemCtx;
1541	pdmIoMemCtxInit(&IoMemCtx, paSegments, cSegments);
1542
1543	while (cbWrite)
1544	{
1545	size_t cbToWrite;
1546
1547	pEntry = pdmacFileEpCacheGetCacheEntryByOffset(pEndpointCache, off);
1548
1549	if (pEntry)
1550	{
1551	/* Write the data into the entry and mark it as dirty */
1552	AssertPtr(pEntry->pList);
1553
1554	RTFOFF OffDiff = off - pEntry->Core.Key;
1555
1556	AssertMsg(off >= pEntry->Core.Key,
1557	("Overflow in calculation off=%RTfoff OffsetAligned=%RTfoff\n",
1558	off, pEntry->Core.Key));
1559
1560	cbToWrite = RT_MIN(pEntry->cbData - OffDiff, cbWrite);
1561	cbWrite -= cbToWrite;
1562
1563	if (!cbWrite)
1564	STAM_COUNTER_INC(&pCache->cHits);
1565	else
1566	STAM_COUNTER_INC(&pCache->cPartialHits);
1567
1568	STAM_COUNTER_ADD(&pCache->StatWritten, cbToWrite);
1569
1570	/* Ghost lists contain no data. */
1571	if ( (pEntry->pList == &pCache->LruRecentlyUsedIn)
1572	\|\| (pEntry->pList == &pCache->LruFrequentlyUsed))
1573	{
1574	/* Check if the buffer is deprecated. */
1575	if(pdmacFileEpCacheEntryFlagIsSetClearAcquireLock(pEndpointCache, pEntry,
1576	PDMACFILECACHE_ENTRY_IS_DEPRECATED,
1577	0))
1578	{
1579	AssertMsg(pEntry->fFlags & PDMACFILECACHE_ENTRY_IO_IN_PROGRESS,
1580	("Entry is deprecated but not in progress\n"));
1581	AssertPtr(pEntry->pbDataReplace);
1582
1583	LogFlow(("Writing to deprecated buffer of entry %#p\n", pEntry));
1584
1585	/* Update the data from the write. */
1586	pdmacFileEpCacheCopyFromIoMemCtx(&IoMemCtx,
1587	pEntry->pbDataReplace + OffDiff,
1588	cbToWrite);
1589	ASMAtomicSubS32(&pTask->cbTransferLeft, cbToWrite);
1590	RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
1591	}
1592	else /* Deprecated flag not set */
1593	{
1594	/* If the entry is dirty it must be also in progress now and we have to defer updating it again. */
1595	if(pdmacFileEpCacheEntryFlagIsSetClearAcquireLock(pEndpointCache, pEntry,
1596	PDMACFILECACHE_ENTRY_IS_DIRTY,
1597	0))
1598	{
1599	AssertMsg(pEntry->fFlags & PDMACFILECACHE_ENTRY_IO_IN_PROGRESS,
1600	("Entry is dirty but not in progress\n"));
1601	Assert(!pEntry->pbDataReplace);
1602
1603	/* Deprecate the current buffer. */
1604	if (!pEntry->pWaitingHead)
1605	pEntry->pbDataReplace = (uint8_t *)RTMemPageAlloc(pEntry->cbData);
1606
1607	/* If we are out of memory or have waiting segments
1608	* defer the write. */
1609	if (!pEntry->pbDataReplace \|\| pEntry->pWaitingHead)
1610	{
1611	/* The data isn't written to the file yet */
1612	pdmacFileEpCacheEntryWaitersAdd(pEntry, pTask,
1613	&IoMemCtx,
1614	OffDiff, cbToWrite,
1615	true /* fWrite */);
1616	STAM_COUNTER_INC(&pEndpointCache->StatWriteDeferred);
1617	}
1618	else /* Deprecate buffer */
1619	{
1620	LogFlow(("Deprecating buffer for entry %#p\n", pEntry));
1621	pEntry->fFlags \|= PDMACFILECACHE_ENTRY_IS_DEPRECATED;
1622
1623	/* Copy the data before the update. */
1624	if (OffDiff)
1625	memcpy(pEntry->pbDataReplace, pEntry->pbData, OffDiff);
1626
1627	/* Copy data behind the update. */
1628	if ((pEntry->cbData - OffDiff - cbToWrite) > 0)
1629	memcpy(pEntry->pbDataReplace + OffDiff + cbToWrite,
1630	pEntry->pbData + OffDiff + cbToWrite,
1631	(pEntry->cbData - OffDiff - cbToWrite));
1632
1633	/* Update the data from the write. */
1634	pdmacFileEpCacheCopyFromIoMemCtx(&IoMemCtx,
1635	pEntry->pbDataReplace + OffDiff,
1636	cbToWrite);
1637	ASMAtomicSubS32(&pTask->cbTransferLeft, cbToWrite);
1638
1639	/* We are done here. A new write is initiated if the current request completes. */
1640	}
1641
1642	RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
1643	}
1644	else /* Dirty bit not set */
1645	{
1646	/*
1647	* Check if a read is in progress for this entry.
1648	* We have to defer processing in that case.
1649	*/
1650	if(pdmacFileEpCacheEntryFlagIsSetClearAcquireLock(pEndpointCache, pEntry,
1651	PDMACFILECACHE_ENTRY_IO_IN_PROGRESS,
1652	0))
1653	{
1654	pdmacFileEpCacheEntryWaitersAdd(pEntry, pTask,
1655	&IoMemCtx,
1656	OffDiff, cbToWrite,
1657	true /* fWrite */);
1658	STAM_COUNTER_INC(&pEndpointCache->StatWriteDeferred);
1659	RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
1660	}
1661	else /* I/O in progres flag not set */
1662	{
1663	/* Write as much as we can into the entry and update the file. */
1664	pdmacFileEpCacheCopyFromIoMemCtx(&IoMemCtx,
1665	pEntry->pbData + OffDiff,
1666	cbToWrite);
1667	ASMAtomicSubS32(&pTask->cbTransferLeft, cbToWrite);
1668
1669	pEntry->fFlags \|= PDMACFILECACHE_ENTRY_IS_DIRTY;
1670	pdmacFileCacheWriteToEndpoint(pEntry);
1671	}
1672	} /* Dirty bit not set */
1673
1674	/* Move this entry to the top position */
1675	if (pEntry->pList == &pCache->LruFrequentlyUsed)
1676	{
1677	pdmacFileCacheLockEnter(pCache);
1678	pdmacFileCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
1679	pdmacFileCacheLockLeave(pCache);
1680	} /* Deprecated flag not set. */
1681	}
1682	pdmacFileEpCacheEntryRelease(pEntry);
1683	}
1684	else /* Entry is on the ghost list */
1685	{
1686	uint8_t *pbBuffer = NULL;
1687
1688	pdmacFileCacheLockEnter(pCache);
1689	pdmacFileCacheEntryRemoveFromList(pEntry); /* Remove it before we remove data, otherwise it may get freed when evicting data. */
1690	bool fEnough = pdmacFileCacheReclaim(pCache, pEntry->cbData, true, &pbBuffer);
1691
1692	if (fEnough)
1693	{
1694	/* Move the entry to Am and fetch it to the cache. */
1695	pdmacFileCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
1696	pdmacFileCacheAdd(pCache, pEntry->cbData);
1697	pdmacFileCacheLockLeave(pCache);
1698
1699	if (pbBuffer)
1700	pEntry->pbData = pbBuffer;
1701	else
1702	pEntry->pbData = (uint8_t *)RTMemPageAlloc(pEntry->cbData);
1703	AssertPtr(pEntry->pbData);
1704
1705	pdmacFileEpCacheEntryWaitersAdd(pEntry, pTask,
1706	&IoMemCtx,
1707	OffDiff, cbToWrite,
1708	true /* fWrite */);
1709	STAM_COUNTER_INC(&pEndpointCache->StatWriteDeferred);
1710	pdmacFileCacheReadFromEndpoint(pEntry);
1711
1712	/* Release the reference. If it is still needed the I/O in progress flag should protect it now. */
1713	pdmacFileEpCacheEntryRelease(pEntry);
1714	}
1715	else
1716	{
1717	RTSemRWRequestWrite(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT);
1718	STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
1719	RTAvlrFileOffsetRemove(pEndpointCache->pTree, pEntry->Core.Key);
1720	STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
1721	RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
1722
1723	pdmacFileCacheLockLeave(pCache);
1724
1725	RTMemFree(pEntry);
1726	pdmacFileEpCacheRequestPassthrough(pEndpoint, pTask,
1727	&IoMemCtx, off, cbToWrite,
1728	PDMACTASKFILETRANSFER_WRITE);
1729	}
1730	}
1731	}
1732	else /* No entry found */
1733	{
1734	/*
1735	* No entry found. Try to create a new cache entry to store the data in and if that fails
1736	* write directly to the file.
1737	*/
1738	PPDMACFILECACHEENTRY pEntryBestFit = pdmacFileEpCacheGetCacheBestFitEntryByOffset(pEndpointCache, off);
1739
1740	LogFlow(("%sest fit entry for off=%RTfoff (BestFit=%RTfoff BestFitEnd=%RTfoff BestFitSize=%u)\n",
1741	pEntryBestFit ? "B" : "No b",
1742	off,
1743	pEntryBestFit ? pEntryBestFit->Core.Key : 0,
1744	pEntryBestFit ? pEntryBestFit->Core.KeyLast : 0,
1745	pEntryBestFit ? pEntryBestFit->cbData : 0));
1746
1747	if (pEntryBestFit && ((off + (RTFOFF)cbWrite) > pEntryBestFit->Core.Key))
1748	{
1749	cbToWrite = pEntryBestFit->Core.Key - off;
1750	pdmacFileEpCacheEntryRelease(pEntryBestFit);
1751	}
1752	else
1753	{
1754	if (pEntryBestFit)
1755	pdmacFileEpCacheEntryRelease(pEntryBestFit);
1756
1757	cbToWrite = cbWrite;
1758	}
1759
1760	cbWrite -= cbToWrite;
1761
1762	STAM_COUNTER_INC(&pCache->cMisses);
1763
1764	uint8_t *pbBuffer = NULL;
1765
1766	pdmacFileCacheLockEnter(pCache);
1767	bool fEnough = pdmacFileCacheReclaim(pCache, cbToWrite, true, &pbBuffer);
1768
1769	if (fEnough)
1770	{
1771	LogFlow(("Evicted enough bytes (%u requested). Creating new cache entry\n", cbToWrite));
1772
1773	PPDMACFILECACHEENTRY pEntryNew;
1774
1775	pEntryNew = pdmacFileCacheEntryAlloc(pCache, pEndpoint, off, cbToWrite, pbBuffer);
1776	AssertPtr(pEntryNew);
1777
1778	pdmacFileCacheEntryAddToList(&pCache->LruRecentlyUsedIn, pEntryNew);
1779	pdmacFileCacheAdd(pCache, cbToWrite);
1780	pdmacFileCacheLockLeave(pCache);
1781
1782	pdmacFileEpCacheInsertEntry(pEndpointCache, pEntryNew);
1783
1784	pdmacFileEpCacheCopyFromIoMemCtx(&IoMemCtx,
1785	pEntryNew->pbData,
1786	cbToWrite);
1787	ASMAtomicSubS32(&pTask->cbTransferLeft, cbToWrite);
1788
1789	pEntryNew->fFlags \|= PDMACFILECACHE_ENTRY_IS_DIRTY;
1790	pdmacFileCacheWriteToEndpoint(pEntryNew);
1791	pdmacFileEpCacheEntryRelease(pEntryNew); /* it is protected by the I/O in progress flag now. */
1792	STAM_COUNTER_ADD(&pCache->StatWritten, cbToWrite);
1793	}
1794	else
1795	{
1796	pdmacFileCacheLockLeave(pCache);
1797
1798	/*
1799	* There is not enough free space in the cache.
1800	* Pass the request directly to the I/O manager.
1801	*/
1802	LogFlow(("Couldn't evict %u bytes from the cache. Remaining request will be passed through\n", cbToWrite));
1803
1804	pdmacFileEpCacheRequestPassthrough(pEndpoint, pTask,
1805	&IoMemCtx, off, cbToWrite,
1806	PDMACTASKFILETRANSFER_WRITE);
1807	}
1808	}
1809
1810	off += cbToWrite;
1811	}
1812
1813	ASMAtomicWriteBool(&pTask->fCompleted, false);
1814
1815	if (ASMAtomicReadS32(&pTask->cbTransferLeft) == 0
1816	&& !ASMAtomicXchgBool(&pTask->fCompleted, true))
1817	pdmR3AsyncCompletionCompleteTask(&pTask->Core, false);
1818	else
1819	rc = VINF_AIO_TASK_PENDING;
1820
1821	LogFlowFunc((": Leave rc=%Rrc\n", rc));
1822
1823	return rc;
1824	}
1825
1826	int pdmacFileEpCacheFlush(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMASYNCCOMPLETIONTASKFILE pTask)
1827	{
1828	int rc = VINF_SUCCESS;
1829
1830	LogFlowFunc((": pEndpoint=%#p{%s} pTask=%#p\n",
1831	pEndpoint, pEndpoint->Core.pszUri, pTask));
1832
1833	if (ASMAtomicReadPtr((void * volatile *)&pEndpoint->DataCache.pTaskFlush))
1834	rc = VERR_RESOURCE_BUSY;
1835	else
1836	{
1837	if (ASMAtomicReadU32(&pEndpoint->DataCache.cWritesOutstanding) > 0)
1838	{
1839	ASMAtomicWritePtr((void * volatile *)&pEndpoint->DataCache.pTaskFlush, pTask);
1840	rc = VINF_AIO_TASK_PENDING;
1841	}
1842	else
1843	pdmR3AsyncCompletionCompleteTask(&pTask->Core, false);
1844	}
1845
1846	LogFlowFunc((": Leave rc=%Rrc\n", rc));
1847	return rc;
1848	}
1849

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/PDMAsyncCompletionFileCache.cpp@ 26638

Download in other formats: