VirtualBox

source: vbox/trunk/src/VBox/VMM/PDMAsyncCompletionFileNormal.cpp@ 29090

Last change on this file since 29090 was 29032, checked in by vboxsync, 15 years ago

wording
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 58.7 KB
Line 
1/* $Id: PDMAsyncCompletionFileNormal.cpp 29032 2010-05-04 14:41:20Z vboxsync $ */
2/** @file
3 * PDM Async I/O - Transport data asynchronous in R3 using EMT.
4 * Async File I/O manager.
5 */
6
7/*
8 * Copyright (C) 2006-2008 Oracle Corporation
9 *
10 * This file is part of VirtualBox Open Source Edition (OSE), as
11 * available from http://www.virtualbox.org. This file is free software;
12 * you can redistribute it and/or modify it under the terms of the GNU
13 * General Public License (GPL) as published by the Free Software
14 * Foundation, in version 2 as it comes in the "COPYING" file of the
15 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
16 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
17 */
18#define LOG_GROUP LOG_GROUP_PDM_ASYNC_COMPLETION
19#include <iprt/types.h>
20#include <iprt/asm.h>
21#include <iprt/file.h>
22#include <iprt/mem.h>
23#include <iprt/string.h>
24#include <iprt/assert.h>
25#include <VBox/log.h>
26
27#include "PDMAsyncCompletionFileInternal.h"
28
29/** The update period for the I/O load statistics in ms. */
30#define PDMACEPFILEMGR_LOAD_UPDATE_PERIOD 1000
31/** Maximum number of requests a manager will handle. */
32#define PDMACEPFILEMGR_REQS_STEP 512
33
34/*******************************************************************************
35* Internal functions *
36*******************************************************************************/
37static int pdmacFileAioMgrNormalProcessTaskList(PPDMACTASKFILE pTaskHead,
38 PPDMACEPFILEMGR pAioMgr,
39 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint);
40
41static PPDMACTASKFILE pdmacFileAioMgrNormalRangeLockFree(PPDMACEPFILEMGR pAioMgr,
42 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
43 PPDMACFILERANGELOCK pRangeLock);
44
45int pdmacFileAioMgrNormalInit(PPDMACEPFILEMGR pAioMgr)
46{
47 int rc = VINF_SUCCESS;
48
49 pAioMgr->cRequestsActiveMax = PDMACEPFILEMGR_REQS_STEP;
50
51 rc = RTFileAioCtxCreate(&pAioMgr->hAioCtx, RTFILEAIO_UNLIMITED_REQS);
52 if (rc == VERR_OUT_OF_RANGE)
53 rc = RTFileAioCtxCreate(&pAioMgr->hAioCtx, pAioMgr->cRequestsActiveMax);
54
55 if (RT_SUCCESS(rc))
56 {
57 /* Initialize request handle array. */
58 pAioMgr->iFreeEntry = 0;
59 pAioMgr->cReqEntries = pAioMgr->cRequestsActiveMax;
60 pAioMgr->pahReqsFree = (RTFILEAIOREQ *)RTMemAllocZ(pAioMgr->cReqEntries * sizeof(RTFILEAIOREQ));
61
62 if (pAioMgr->pahReqsFree)
63 {
64 /* Create the range lock memcache. */
65 rc = RTMemCacheCreate(&pAioMgr->hMemCacheRangeLocks, sizeof(PDMACFILERANGELOCK),
66 0, UINT32_MAX, NULL, NULL, NULL, 0);
67 if (RT_SUCCESS(rc))
68 return VINF_SUCCESS;
69
70 RTMemFree(pAioMgr->pahReqsFree);
71 }
72 else
73 {
74 RTFileAioCtxDestroy(pAioMgr->hAioCtx);
75 rc = VERR_NO_MEMORY;
76 }
77 }
78
79 return rc;
80}
81
82void pdmacFileAioMgrNormalDestroy(PPDMACEPFILEMGR pAioMgr)
83{
84 RTFileAioCtxDestroy(pAioMgr->hAioCtx);
85
86 while (pAioMgr->iFreeEntry > 0)
87 {
88 pAioMgr->iFreeEntry--;
89 Assert(pAioMgr->pahReqsFree[pAioMgr->iFreeEntry] != NIL_RTFILEAIOREQ);
90 RTFileAioReqDestroy(pAioMgr->pahReqsFree[pAioMgr->iFreeEntry]);
91 }
92
93 RTMemFree(pAioMgr->pahReqsFree);
94 RTMemCacheDestroy(pAioMgr->hMemCacheRangeLocks);
95}
96
97/**
98 * Sorts the endpoint list with insertion sort.
99 */
100static void pdmacFileAioMgrNormalEndpointsSortByLoad(PPDMACEPFILEMGR pAioMgr)
101{
102 PPDMASYNCCOMPLETIONENDPOINTFILE pEpPrev, pEpCurr, pEpNextToSort;
103
104 pEpPrev = pAioMgr->pEndpointsHead;
105 pEpCurr = pEpPrev->AioMgr.pEndpointNext;
106
107 while (pEpCurr)
108 {
109 /* Remember the next element to sort because the list might change. */
110 pEpNextToSort = pEpCurr->AioMgr.pEndpointNext;
111
112 /* Unlink the current element from the list. */
113 PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEpCurr->AioMgr.pEndpointPrev;
114 PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEpCurr->AioMgr.pEndpointNext;
115
116 if (pPrev)
117 pPrev->AioMgr.pEndpointNext = pNext;
118 else
119 pAioMgr->pEndpointsHead = pNext;
120
121 if (pNext)
122 pNext->AioMgr.pEndpointPrev = pPrev;
123
124 /* Go back until we reached the place to insert the current endpoint into. */
125 while (pEpPrev && (pEpPrev->AioMgr.cReqsPerSec < pEpCurr->AioMgr.cReqsPerSec))
126 pEpPrev = pEpPrev->AioMgr.pEndpointPrev;
127
128 /* Link the endpoint into the list. */
129 if (pEpPrev)
130 pNext = pEpPrev->AioMgr.pEndpointNext;
131 else
132 pNext = pAioMgr->pEndpointsHead;
133
134 pEpCurr->AioMgr.pEndpointNext = pNext;
135 pEpCurr->AioMgr.pEndpointPrev = pEpPrev;
136
137 if (pNext)
138 pNext->AioMgr.pEndpointPrev = pEpCurr;
139
140 if (pEpPrev)
141 pEpPrev->AioMgr.pEndpointNext = pEpCurr;
142 else
143 pAioMgr->pEndpointsHead = pEpCurr;
144
145 pEpCurr = pEpNextToSort;
146 }
147
148#ifdef DEBUG
149 /* Validate sorting alogrithm */
150 unsigned cEndpoints = 0;
151 pEpCurr = pAioMgr->pEndpointsHead;
152
153 AssertMsg(pEpCurr, ("No endpoint in the list?\n"));
154 AssertMsg(!pEpCurr->AioMgr.pEndpointPrev, ("First element in the list points to previous element\n"));
155
156 while (pEpCurr)
157 {
158 cEndpoints++;
159
160 PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEpCurr->AioMgr.pEndpointNext;
161 PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEpCurr->AioMgr.pEndpointPrev;
162
163 Assert(!pNext || pNext->AioMgr.cReqsPerSec <= pEpCurr->AioMgr.cReqsPerSec);
164 Assert(!pPrev || pPrev->AioMgr.cReqsPerSec >= pEpCurr->AioMgr.cReqsPerSec);
165
166 pEpCurr = pNext;
167 }
168
169 AssertMsg(cEndpoints == pAioMgr->cEndpoints, ("Endpoints lost during sort!\n"));
170
171#endif
172}
173
174/**
175 * Removes an endpoint from the currently assigned manager.
176 *
177 * @returns TRUE if there are still requests pending on the current manager for this endpoint.
178 * FALSE otherwise.
179 * @param pEndpointRemove The endpoint to remove.
180 */
181static bool pdmacFileAioMgrNormalRemoveEndpoint(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointRemove)
182{
183 PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEndpointRemove->AioMgr.pEndpointPrev;
184 PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEndpointRemove->AioMgr.pEndpointNext;
185 PPDMACEPFILEMGR pAioMgr = pEndpointRemove->pAioMgr;
186
187 pAioMgr->cEndpoints--;
188
189 if (pPrev)
190 pPrev->AioMgr.pEndpointNext = pNext;
191 else
192 pAioMgr->pEndpointsHead = pNext;
193
194 if (pNext)
195 pNext->AioMgr.pEndpointPrev = pPrev;
196
197 /* Make sure that there is no request pending on this manager for the endpoint. */
198 if (!pEndpointRemove->AioMgr.cRequestsActive)
199 {
200 Assert(!pEndpointRemove->pFlushReq);
201
202 /* Reopen the file so that the new endpoint can reassociate with the file */
203 RTFileClose(pEndpointRemove->File);
204 int rc = RTFileOpen(&pEndpointRemove->File, pEndpointRemove->Core.pszUri, pEndpointRemove->fFlags);
205 AssertRC(rc);
206 return false;
207 }
208
209 return true;
210}
211
212static bool pdmacFileAioMgrNormalIsBalancePossible(PPDMACEPFILEMGR pAioMgr)
213{
214 /* Balancing doesn't make sense with only one endpoint. */
215 if (pAioMgr->cEndpoints == 1)
216 return false;
217
218 /* Doesn't make sens to move endpoints if only one produces the whole load */
219 unsigned cEndpointsWithLoad = 0;
220
221 PPDMASYNCCOMPLETIONENDPOINTFILE pCurr = pAioMgr->pEndpointsHead;
222
223 while (pCurr)
224 {
225 if (pCurr->AioMgr.cReqsPerSec)
226 cEndpointsWithLoad++;
227
228 pCurr = pCurr->AioMgr.pEndpointNext;
229 }
230
231 return (cEndpointsWithLoad > 1);
232}
233
234/**
235 * Creates a new I/O manager and spreads the I/O load of the endpoints
236 * between the given I/O manager and the new one.
237 *
238 * @returns nothing.
239 * @param pAioMgr The I/O manager with high I/O load.
240 */
241static void pdmacFileAioMgrNormalBalanceLoad(PPDMACEPFILEMGR pAioMgr)
242{
243 PPDMACEPFILEMGR pAioMgrNew = NULL;
244 int rc = VINF_SUCCESS;
245
246 /*
247 * Check if balancing would improve the situation.
248 */
249 if (pdmacFileAioMgrNormalIsBalancePossible(pAioMgr))
250 {
251 PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pAioMgr->pEndpointsHead->Core.pEpClass;
252
253 rc = pdmacFileAioMgrCreate(pEpClassFile, &pAioMgrNew, PDMACEPFILEMGRTYPE_ASYNC);
254 if (RT_SUCCESS(rc))
255 {
256 /* We will sort the list by request count per second. */
257 pdmacFileAioMgrNormalEndpointsSortByLoad(pAioMgr);
258
259 /* Now move some endpoints to the new manager. */
260 unsigned cReqsHere = pAioMgr->pEndpointsHead->AioMgr.cReqsPerSec;
261 unsigned cReqsOther = 0;
262 PPDMASYNCCOMPLETIONENDPOINTFILE pCurr = pAioMgr->pEndpointsHead->AioMgr.pEndpointNext;
263
264 while (pCurr)
265 {
266 if (cReqsHere <= cReqsOther)
267 {
268 /*
269 * The other manager has more requests to handle now.
270 * We will keep the current endpoint.
271 */
272 Log(("Keeping endpoint %#p{%s} with %u reqs/s\n", pCurr->Core.pszUri, pCurr->AioMgr.cReqsPerSec));
273 cReqsHere += pCurr->AioMgr.cReqsPerSec;
274 pCurr = pCurr->AioMgr.pEndpointNext;
275 }
276 else
277 {
278 /* Move to other endpoint. */
279 Log(("Moving endpoint %#p{%s} with %u reqs/s to other manager\n", pCurr, pCurr->Core.pszUri, pCurr->AioMgr.cReqsPerSec));
280 cReqsOther += pCurr->AioMgr.cReqsPerSec;
281
282 PPDMASYNCCOMPLETIONENDPOINTFILE pMove = pCurr;
283
284 pCurr = pCurr->AioMgr.pEndpointNext;
285
286 bool fReqsPending = pdmacFileAioMgrNormalRemoveEndpoint(pMove);
287
288 if (fReqsPending)
289 {
290 pMove->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_REMOVING;
291 pMove->AioMgr.fMoving = true;
292 pMove->AioMgr.pAioMgrDst = pAioMgrNew;
293 }
294 else
295 {
296 pMove->AioMgr.fMoving = false;
297 pMove->AioMgr.pAioMgrDst = NULL;
298 pdmacFileAioMgrAddEndpoint(pAioMgrNew, pMove);
299 }
300 }
301 }
302 }
303 else
304 {
305 /* Don't process further but leave a log entry about reduced performance. */
306 LogRel(("AIOMgr: Could not create new I/O manager (rc=%Rrc). Expect reduced performance\n", rc));
307 }
308 }
309 else
310 Log(("AIOMgr: Load balancing would not improve anything\n"));
311}
312
313/**
314 * Increase the maximum number of active requests for the given I/O manager.
315 *
316 * @returns VBox status code.
317 * @param pAioMgr The I/O manager to grow.
318 */
319static int pdmacFileAioMgrNormalGrow(PPDMACEPFILEMGR pAioMgr)
320{
321 int rc = VINF_SUCCESS;
322 RTFILEAIOCTX hAioCtxNew = NIL_RTFILEAIOCTX;
323
324 LogFlowFunc(("pAioMgr=%#p\n", pAioMgr));
325
326 AssertMsg( pAioMgr->enmState == PDMACEPFILEMGRSTATE_GROWING
327 && !pAioMgr->cRequestsActive,
328 ("Invalid state of the I/O manager\n"));
329
330#ifdef RT_OS_WINDOWS
331 /*
332 * Reopen the files of all assigned endpoints first so we can assign them to the new
333 * I/O context.
334 */
335 PPDMASYNCCOMPLETIONENDPOINTFILE pCurr = pAioMgr->pEndpointsHead;
336
337 while (pCurr)
338 {
339 RTFileClose(pCurr->File);
340 rc = RTFileOpen(&pCurr->File, pCurr->Core.pszUri, pCurr->fFlags);
341 AssertRC(rc);
342
343 pCurr = pCurr->AioMgr.pEndpointNext;
344 }
345#endif
346
347 /* Create the new bigger context. */
348 pAioMgr->cRequestsActiveMax += PDMACEPFILEMGR_REQS_STEP;
349
350 rc = RTFileAioCtxCreate(&hAioCtxNew, RTFILEAIO_UNLIMITED_REQS);
351 if (rc == VERR_OUT_OF_RANGE)
352 rc = RTFileAioCtxCreate(&hAioCtxNew, pAioMgr->cRequestsActiveMax);
353
354 if (RT_SUCCESS(rc))
355 {
356 /* Close the old context. */
357 rc = RTFileAioCtxDestroy(pAioMgr->hAioCtx);
358 AssertRC(rc);
359
360 pAioMgr->hAioCtx = hAioCtxNew;
361
362 /* Create a new I/O task handle array */
363 uint32_t cReqEntriesNew = pAioMgr->cRequestsActiveMax + 1;
364 RTFILEAIOREQ *pahReqNew = (RTFILEAIOREQ *)RTMemAllocZ(cReqEntriesNew * sizeof(RTFILEAIOREQ));
365
366 if (pahReqNew)
367 {
368 /* Copy the cached request handles. */
369 for (uint32_t iReq = 0; iReq < pAioMgr->cReqEntries; iReq++)
370 pahReqNew[iReq] = pAioMgr->pahReqsFree[iReq];
371
372 RTMemFree(pAioMgr->pahReqsFree);
373 pAioMgr->pahReqsFree = pahReqNew;
374 pAioMgr->cReqEntries = cReqEntriesNew;
375 LogFlowFunc(("I/O manager increased to handle a maximum of %u requests\n",
376 pAioMgr->cRequestsActiveMax));
377 }
378 else
379 rc = VERR_NO_MEMORY;
380 }
381
382#ifdef RT_OS_WINDOWS
383 /* Assign the file to the new context. */
384 pCurr = pAioMgr->pEndpointsHead;
385
386 while (pCurr)
387 {
388 rc = RTFileAioCtxAssociateWithFile(pAioMgr->hAioCtx, pCurr->File);
389 AssertRC(rc);
390
391 pCurr = pCurr->AioMgr.pEndpointNext;
392 }
393#endif
394
395 if (RT_FAILURE(rc))
396 {
397 LogFlow(("Increasing size of the I/O manager failed with rc=%Rrc\n", rc));
398 pAioMgr->cRequestsActiveMax -= PDMACEPFILEMGR_REQS_STEP;
399 }
400
401 pAioMgr->enmState = PDMACEPFILEMGRSTATE_RUNNING;
402 LogFlowFunc(("returns rc=%Rrc\n", rc));
403
404 return rc;
405}
406
407/**
408 * Error handler which will create the failsafe managers and destroy the failed I/O manager.
409 *
410 * @returns VBox status code
411 * @param pAioMgr The I/O manager the error ocurred on.
412 * @param rc The error code.
413 */
414static int pdmacFileAioMgrNormalErrorHandler(PPDMACEPFILEMGR pAioMgr, int rc, RT_SRC_POS_DECL)
415{
416 LogRel(("AIOMgr: I/O manager %#p encountered a critical error (rc=%Rrc) during operation. Falling back to failsafe mode. Expect reduced performance\n",
417 pAioMgr, rc));
418 LogRel(("AIOMgr: Error happened in %s:(%u){%s}\n", RT_SRC_POS_ARGS));
419 LogRel(("AIOMgr: Please contact the product vendor\n"));
420
421 PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pAioMgr->pEndpointsHead->Core.pEpClass;
422
423 pAioMgr->enmState = PDMACEPFILEMGRSTATE_FAULT;
424 ASMAtomicWriteU32((volatile uint32_t *)&pEpClassFile->enmMgrTypeOverride, PDMACEPFILEMGRTYPE_SIMPLE);
425
426 AssertMsgFailed(("Implement\n"));
427 return VINF_SUCCESS;
428}
429
430/**
431 * Put a list of tasks in the pending request list of an endpoint.
432 */
433DECLINLINE(void) pdmacFileAioMgrEpAddTaskList(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMACTASKFILE pTaskHead)
434{
435 /* Add the rest of the tasks to the pending list */
436 if (!pEndpoint->AioMgr.pReqsPendingHead)
437 {
438 Assert(!pEndpoint->AioMgr.pReqsPendingTail);
439 pEndpoint->AioMgr.pReqsPendingHead = pTaskHead;
440 }
441 else
442 {
443 Assert(pEndpoint->AioMgr.pReqsPendingTail);
444 pEndpoint->AioMgr.pReqsPendingTail->pNext = pTaskHead;
445 }
446
447 /* Update the tail. */
448 while (pTaskHead->pNext)
449 pTaskHead = pTaskHead->pNext;
450
451 pEndpoint->AioMgr.pReqsPendingTail = pTaskHead;
452 pTaskHead->pNext = NULL;
453}
454
455/**
456 * Put one task in the pending request list of an endpoint.
457 */
458DECLINLINE(void) pdmacFileAioMgrEpAddTask(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMACTASKFILE pTask)
459{
460 /* Add the rest of the tasks to the pending list */
461 if (!pEndpoint->AioMgr.pReqsPendingHead)
462 {
463 Assert(!pEndpoint->AioMgr.pReqsPendingTail);
464 pEndpoint->AioMgr.pReqsPendingHead = pTask;
465 }
466 else
467 {
468 Assert(pEndpoint->AioMgr.pReqsPendingTail);
469 pEndpoint->AioMgr.pReqsPendingTail->pNext = pTask;
470 }
471
472 pEndpoint->AioMgr.pReqsPendingTail = pTask;
473 pTask->pNext = NULL;
474}
475
476/**
477 * Allocates a async I/O request.
478 *
479 * @returns Handle to the request.
480 * @param pAioMgr The I/O manager.
481 */
482static RTFILEAIOREQ pdmacFileAioMgrNormalRequestAlloc(PPDMACEPFILEMGR pAioMgr)
483{
484 RTFILEAIOREQ hReq = NIL_RTFILEAIOREQ;
485
486 /* Get a request handle. */
487 if (pAioMgr->iFreeEntry > 0)
488 {
489 pAioMgr->iFreeEntry--;
490 hReq = pAioMgr->pahReqsFree[pAioMgr->iFreeEntry];
491 pAioMgr->pahReqsFree[pAioMgr->iFreeEntry] = NIL_RTFILEAIOREQ;
492 Assert(hReq != NIL_RTFILEAIOREQ);
493 }
494 else
495 {
496 int rc = RTFileAioReqCreate(&hReq);
497 AssertRC(rc);
498 }
499
500 return hReq;
501}
502
503/**
504 * Frees a async I/O request handle.
505 *
506 * @returns nothing.
507 * @param pAioMgr The I/O manager.
508 * @param hReq The I/O request handle to free.
509 */
510static void pdmacFileAioMgrNormalRequestFree(PPDMACEPFILEMGR pAioMgr, RTFILEAIOREQ hReq)
511{
512 Assert(pAioMgr->iFreeEntry < pAioMgr->cReqEntries);
513 Assert(pAioMgr->pahReqsFree[pAioMgr->iFreeEntry] == NIL_RTFILEAIOREQ);
514
515 pAioMgr->pahReqsFree[pAioMgr->iFreeEntry] = hReq;
516 pAioMgr->iFreeEntry++;
517}
518
519/**
520 * Wrapper around RTFIleAioCtxSubmit() which is also doing error handling.
521 */
522static int pdmacFileAioMgrNormalReqsEnqueue(PPDMACEPFILEMGR pAioMgr,
523 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
524 PRTFILEAIOREQ pahReqs, unsigned cReqs)
525{
526 int rc;
527
528 pAioMgr->cRequestsActive += cReqs;
529 pEndpoint->AioMgr.cRequestsActive += cReqs;
530
531 LogFlow(("Enqueuing %d requests. I/O manager has a total of %d active requests now\n", cReqs, pAioMgr->cRequestsActive));
532 LogFlow(("Endpoint has a total of %d active requests now\n", pEndpoint->AioMgr.cRequestsActive));
533
534 rc = RTFileAioCtxSubmit(pAioMgr->hAioCtx, pahReqs, cReqs);
535 if (RT_FAILURE(rc))
536 {
537 if (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES)
538 {
539 PPDMASYNCCOMPLETIONEPCLASSFILE pEpClass = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;
540
541 /*
542 * We run out of resources.
543 * Need to check which requests got queued
544 * and put the rest on the pending list again.
545 */
546 for (size_t i = 0; i < cReqs; i++)
547 {
548 int rcReq = RTFileAioReqGetRC(pahReqs[i], NULL);
549
550 if (rcReq != VERR_FILE_AIO_IN_PROGRESS)
551 {
552 AssertMsg(rcReq == VERR_FILE_AIO_NOT_SUBMITTED,
553 ("Request returned unexpected return code: rc=%Rrc\n", rcReq));
554
555 PPDMACTASKFILE pTask = (PPDMACTASKFILE)RTFileAioReqGetUser(pahReqs[i]);
556 PPDMACTASKFILE pTasksWaiting;
557
558 pdmacFileAioMgrNormalRequestFree(pAioMgr, pahReqs[i]);
559
560 if (pTask->cbBounceBuffer)
561 RTMemPageFree(pTask->pvBounceBuffer, pTask->cbBounceBuffer);
562
563 pTask->fPrefetch = false;
564 pTask->cbBounceBuffer = 0;
565
566 /* Free the lock and process pending tasks if neccessary */
567 pTasksWaiting = pdmacFileAioMgrNormalRangeLockFree(pAioMgr, pEndpoint, pTask->pRangeLock);
568
569 pdmacFileAioMgrEpAddTask(pEndpoint, pTask);
570 if (pTasksWaiting)
571 pdmacFileAioMgrEpAddTaskList(pEndpoint, pTasksWaiting);
572
573 pAioMgr->cRequestsActive--;
574 pEndpoint->AioMgr.cRequestsActive--;
575 }
576
577 pAioMgr->cRequestsActiveMax = pAioMgr->cRequestsActive;
578 }
579
580 /* Print an entry in the release log */
581 if (RT_UNLIKELY(!pEpClass->fOutOfResourcesWarningPrinted))
582 {
583 pEpClass->fOutOfResourcesWarningPrinted = true;
584 LogRel(("AIOMgr: Host limits number of active IO requests to %u. Expect a performance impact.\n",
585 pAioMgr->cRequestsActive));
586 }
587
588 LogFlow(("Removed requests. I/O manager has a total of %u active requests now\n", pAioMgr->cRequestsActive));
589 LogFlow(("Endpoint has a total of %u active requests now\n", pEndpoint->AioMgr.cRequestsActive));
590 }
591 else
592 AssertMsgFailed(("Unexpected return code rc=%Rrc\n", rc));
593 }
594
595 return rc;
596}
597
598static bool pdmacFileAioMgrNormalIsRangeLocked(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
599 RTFOFF offStart, size_t cbRange,
600 PPDMACTASKFILE pTask)
601{
602 PPDMACFILERANGELOCK pRangeLock = NULL; /** < Range lock */
603
604 AssertMsg( pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE
605 || pTask->enmTransferType == PDMACTASKFILETRANSFER_READ,
606 ("Invalid task type %d\n", pTask->enmTransferType));
607
608 pRangeLock = (PPDMACFILERANGELOCK)RTAvlrFileOffsetRangeGet(pEndpoint->AioMgr.pTreeRangesLocked, offStart);
609 if (!pRangeLock)
610 {
611 pRangeLock = (PPDMACFILERANGELOCK)RTAvlrFileOffsetGetBestFit(pEndpoint->AioMgr.pTreeRangesLocked, offStart, true);
612 /* Check if we intersect with the range. */
613 if ( !pRangeLock
614 || !( (pRangeLock->Core.Key) <= (offStart + (RTFOFF)cbRange - 1)
615 && (pRangeLock->Core.KeyLast) >= offStart))
616 {
617 pRangeLock = NULL; /* False alarm */
618 }
619 }
620
621 /* Check whether we have one of the situations explained below */
622 if ( pRangeLock
623#if 0 /** @todo: later. For now we will just block all requests if they interfere */
624 && ( (pRangeLock->fReadLock && pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE)
625 || (!pRangeLock->fReadLock)
626#endif
627 )
628 {
629 /* Add to the list. */
630 pTask->pNext = NULL;
631
632 if (!pRangeLock->pWaitingTasksHead)
633 {
634 Assert(!pRangeLock->pWaitingTasksTail);
635 pRangeLock->pWaitingTasksHead = pTask;
636 pRangeLock->pWaitingTasksTail = pTask;
637 }
638 else
639 {
640 AssertPtr(pRangeLock->pWaitingTasksTail);
641 pRangeLock->pWaitingTasksTail->pNext = pTask;
642 pRangeLock->pWaitingTasksTail = pTask;
643 }
644 return true;
645 }
646
647 return false;
648}
649
650static int pdmacFileAioMgrNormalRangeLock(PPDMACEPFILEMGR pAioMgr,
651 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
652 RTFOFF offStart, size_t cbRange,
653 PPDMACTASKFILE pTask)
654{
655 AssertMsg(!pdmacFileAioMgrNormalIsRangeLocked(pEndpoint, offStart, cbRange, pTask),
656 ("Range is already locked offStart=%RTfoff cbRange=%u\n",
657 offStart, cbRange));
658
659 PPDMACFILERANGELOCK pRangeLock = (PPDMACFILERANGELOCK)RTMemCacheAlloc(pAioMgr->hMemCacheRangeLocks);
660 if (!pRangeLock)
661 return VERR_NO_MEMORY;
662
663 /* Init the lock. */
664 pRangeLock->Core.Key = offStart;
665 pRangeLock->Core.KeyLast = offStart + cbRange - 1;
666 pRangeLock->cRefs = 1;
667 pRangeLock->fReadLock = pTask->enmTransferType == PDMACTASKFILETRANSFER_READ;
668 pRangeLock->pWaitingTasksHead = NULL;
669 pRangeLock->pWaitingTasksTail = NULL;
670
671 bool fInserted = RTAvlrFileOffsetInsert(pEndpoint->AioMgr.pTreeRangesLocked, &pRangeLock->Core);
672 AssertMsg(fInserted, ("Range lock was not inserted!\n"));
673
674 /* Let the task point to its lock. */
675 pTask->pRangeLock = pRangeLock;
676
677 return VINF_SUCCESS;
678}
679
680static PPDMACTASKFILE pdmacFileAioMgrNormalRangeLockFree(PPDMACEPFILEMGR pAioMgr,
681 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
682 PPDMACFILERANGELOCK pRangeLock)
683{
684 PPDMACTASKFILE pTasksWaitingHead;
685
686 AssertPtr(pRangeLock);
687 Assert(pRangeLock->cRefs == 1);
688
689 RTAvlrFileOffsetRemove(pEndpoint->AioMgr.pTreeRangesLocked, pRangeLock->Core.Key);
690 pTasksWaitingHead = pRangeLock->pWaitingTasksHead;
691 pRangeLock->pWaitingTasksHead = NULL;
692 pRangeLock->pWaitingTasksTail = NULL;
693 RTMemCacheFree(pAioMgr->hMemCacheRangeLocks, pRangeLock);
694
695 return pTasksWaitingHead;
696}
697
698static int pdmacFileAioMgrNormalTaskPrepareBuffered(PPDMACEPFILEMGR pAioMgr,
699 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
700 PPDMACTASKFILE pTask, PRTFILEAIOREQ phReq)
701{
702 int rc = VINF_SUCCESS;
703 RTFILEAIOREQ hReq = NIL_RTFILEAIOREQ;
704 PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;
705 void *pvBuf = pTask->DataSeg.pvSeg;
706
707 AssertMsg( pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE
708 || (uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) <= pEndpoint->cbFile,
709 ("Read exceeds file size offStart=%RTfoff cbToTransfer=%d cbFile=%llu\n",
710 pTask->Off, pTask->DataSeg.cbSeg, pEndpoint->cbFile));
711
712 pTask->fPrefetch = false;
713 pTask->cbBounceBuffer = 0;
714
715 /*
716 * Before we start to setup the request we have to check whether there is a task
717 * already active which range intersects with ours. We have to defer execution
718 * of this task in two cases:
719 * - The pending task is a write and the current is either read or write
720 * - The pending task is a read and the current task is a write task.
721 *
722 * To check whether a range is currently "locked" we use the AVL tree where every pending task
723 * is stored by its file offset range. The current task will be added to the active task
724 * and will be executed when the active one completes. (The method below
725 * which checks whether a range is already used will add the task)
726 *
727 * This is neccessary because of the requirement to align all requests to a 512 boundary
728 * which is enforced by the host OS (Linux and Windows atm). It is possible that
729 * we have to process unaligned tasks and need to align them using bounce buffers.
730 * While the data is fetched from the file another request might arrive writing to
731 * the same range. This will result in data corruption if both are executed concurrently.
732 */
733 bool fLocked = pdmacFileAioMgrNormalIsRangeLocked(pEndpoint, pTask->Off, pTask->DataSeg.cbSeg, pTask);
734
735 if (!fLocked)
736 {
737 /* Get a request handle. */
738 hReq = pdmacFileAioMgrNormalRequestAlloc(pAioMgr);
739 AssertMsg(hReq != NIL_RTFILEAIOREQ, ("Out of request handles\n"));
740
741 if (pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE)
742 {
743 /* Grow the file if needed. */
744 if (RT_UNLIKELY((uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) > pEndpoint->cbFile))
745 {
746 ASMAtomicWriteU64(&pEndpoint->cbFile, pTask->Off + pTask->DataSeg.cbSeg);
747 RTFileSetSize(pEndpoint->File, pTask->Off + pTask->DataSeg.cbSeg);
748 }
749
750 rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->File,
751 pTask->Off, pTask->DataSeg.pvSeg,
752 pTask->DataSeg.cbSeg, pTask);
753 }
754 else
755 rc = RTFileAioReqPrepareRead(hReq, pEndpoint->File,
756 pTask->Off, pTask->DataSeg.pvSeg,
757 pTask->DataSeg.cbSeg, pTask);
758 AssertRC(rc);
759
760 rc = pdmacFileAioMgrNormalRangeLock(pAioMgr, pEndpoint, pTask->Off,
761 pTask->DataSeg.cbSeg,
762 pTask);
763
764 if (RT_SUCCESS(rc))
765 *phReq = hReq;
766 }
767 else
768 LogFlow(("Task %#p was deferred because the access range is locked\n", pTask));
769
770 return rc;
771}
772
773static int pdmacFileAioMgrNormalTaskPrepareNonBuffered(PPDMACEPFILEMGR pAioMgr,
774 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
775 PPDMACTASKFILE pTask, PRTFILEAIOREQ phReq)
776{
777 int rc = VINF_SUCCESS;
778 RTFILEAIOREQ hReq = NIL_RTFILEAIOREQ;
779 PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;
780 void *pvBuf = pTask->DataSeg.pvSeg;
781
782 /*
783 * Check if the alignment requirements are met.
784 * Offset, transfer size and buffer address
785 * need to be on a 512 boundary.
786 */
787 RTFOFF offStart = pTask->Off & ~(RTFOFF)(512-1);
788 size_t cbToTransfer = RT_ALIGN_Z(pTask->DataSeg.cbSeg + (pTask->Off - offStart), 512);
789 PDMACTASKFILETRANSFER enmTransferType = pTask->enmTransferType;
790
791 AssertMsg( pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE
792 || (uint64_t)(offStart + cbToTransfer) <= pEndpoint->cbFile,
793 ("Read exceeds file size offStart=%RTfoff cbToTransfer=%d cbFile=%llu\n",
794 offStart, cbToTransfer, pEndpoint->cbFile));
795
796 pTask->fPrefetch = false;
797
798 /*
799 * Before we start to setup the request we have to check whether there is a task
800 * already active which range intersects with ours. We have to defer execution
801 * of this task in two cases:
802 * - The pending task is a write and the current is either read or write
803 * - The pending task is a read and the current task is a write task.
804 *
805 * To check whether a range is currently "locked" we use the AVL tree where every pending task
806 * is stored by its file offset range. The current task will be added to the active task
807 * and will be executed when the active one completes. (The method below
808 * which checks whether a range is already used will add the task)
809 *
810 * This is neccessary because of the requirement to align all requests to a 512 boundary
811 * which is enforced by the host OS (Linux and Windows atm). It is possible that
812 * we have to process unaligned tasks and need to align them using bounce buffers.
813 * While the data is fetched from the file another request might arrive writing to
814 * the same range. This will result in data corruption if both are executed concurrently.
815 */
816 bool fLocked = pdmacFileAioMgrNormalIsRangeLocked(pEndpoint, offStart, cbToTransfer, pTask);
817
818 if (!fLocked)
819 {
820 /* Get a request handle. */
821 hReq = pdmacFileAioMgrNormalRequestAlloc(pAioMgr);
822 AssertMsg(hReq != NIL_RTFILEAIOREQ, ("Out of request handles\n"));
823
824 if ( RT_UNLIKELY(cbToTransfer != pTask->DataSeg.cbSeg)
825 || RT_UNLIKELY(offStart != pTask->Off)
826 || ((pEpClassFile->uBitmaskAlignment & (RTR3UINTPTR)pvBuf) != (RTR3UINTPTR)pvBuf))
827 {
828 LogFlow(("Using bounce buffer for task %#p cbToTransfer=%zd cbSeg=%zd offStart=%RTfoff off=%RTfoff\n",
829 pTask, cbToTransfer, pTask->DataSeg.cbSeg, offStart, pTask->Off));
830
831 /* Create bounce buffer. */
832 pTask->cbBounceBuffer = cbToTransfer;
833
834 AssertMsg(pTask->Off >= offStart, ("Overflow in calculation Off=%llu offStart=%llu\n",
835 pTask->Off, offStart));
836 pTask->offBounceBuffer = pTask->Off - offStart;
837
838 /** @todo: I think we need something like a RTMemAllocAligned method here.
839 * Current assumption is that the maximum alignment is 4096byte
840 * (GPT disk on Windows)
841 * so we can use RTMemPageAlloc here.
842 */
843 pTask->pvBounceBuffer = RTMemPageAlloc(cbToTransfer);
844 if (RT_LIKELY(pTask->pvBounceBuffer))
845 {
846 pvBuf = pTask->pvBounceBuffer;
847
848 if (pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE)
849 {
850 if ( RT_UNLIKELY(cbToTransfer != pTask->DataSeg.cbSeg)
851 || RT_UNLIKELY(offStart != pTask->Off))
852 {
853 /* We have to fill the buffer first before we can update the data. */
854 LogFlow(("Prefetching data for task %#p\n", pTask));
855 pTask->fPrefetch = true;
856 enmTransferType = PDMACTASKFILETRANSFER_READ;
857 }
858 else
859 memcpy(pvBuf, pTask->DataSeg.pvSeg, pTask->DataSeg.cbSeg);
860 }
861 }
862 else
863 rc = VERR_NO_MEMORY;
864 }
865 else
866 pTask->cbBounceBuffer = 0;
867
868 if (RT_SUCCESS(rc))
869 {
870 AssertMsg((pEpClassFile->uBitmaskAlignment & (RTR3UINTPTR)pvBuf) == (RTR3UINTPTR)pvBuf,
871 ("AIO: Alignment restrictions not met! pvBuf=%p uBitmaskAlignment=%p\n", pvBuf, pEpClassFile->uBitmaskAlignment));
872
873 if (enmTransferType == PDMACTASKFILETRANSFER_WRITE)
874 {
875 /* Grow the file if needed. */
876 if (RT_UNLIKELY((uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) > pEndpoint->cbFile))
877 {
878 ASMAtomicWriteU64(&pEndpoint->cbFile, pTask->Off + pTask->DataSeg.cbSeg);
879 RTFileSetSize(pEndpoint->File, pTask->Off + pTask->DataSeg.cbSeg);
880 }
881
882 rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->File,
883 offStart, pvBuf, cbToTransfer, pTask);
884 }
885 else
886 rc = RTFileAioReqPrepareRead(hReq, pEndpoint->File,
887 offStart, pvBuf, cbToTransfer, pTask);
888 AssertRC(rc);
889
890 rc = pdmacFileAioMgrNormalRangeLock(pAioMgr, pEndpoint, offStart, cbToTransfer, pTask);
891
892 if (RT_SUCCESS(rc))
893 *phReq = hReq;
894 else
895 {
896 /* Cleanup */
897 if (pTask->cbBounceBuffer)
898 RTMemPageFree(pTask->pvBounceBuffer, pTask->cbBounceBuffer);
899 }
900 }
901 }
902 else
903 LogFlow(("Task %#p was deferred because the access range is locked\n", pTask));
904
905 return rc;
906}
907
908static int pdmacFileAioMgrNormalProcessTaskList(PPDMACTASKFILE pTaskHead,
909 PPDMACEPFILEMGR pAioMgr,
910 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
911{
912 RTFILEAIOREQ apReqs[20];
913 unsigned cRequests = 0;
914 unsigned cMaxRequests = pAioMgr->cRequestsActiveMax - pAioMgr->cRequestsActive;
915 int rc = VINF_SUCCESS;
916
917 AssertMsg(pEndpoint->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE,
918 ("Trying to process request lists of a non active endpoint!\n"));
919
920 /* Go through the list and queue the requests until we get a flush request */
921 while ( pTaskHead
922 && !pEndpoint->pFlushReq
923 && (pAioMgr->cRequestsActive + cRequests < pAioMgr->cRequestsActiveMax)
924 && RT_SUCCESS(rc))
925 {
926 PPDMACTASKFILE pCurr = pTaskHead;
927
928 if (!pdmacFileBwMgrIsTransferAllowed(pEndpoint->pBwMgr, (uint32_t)pCurr->DataSeg.cbSeg))
929 {
930 pAioMgr->fBwLimitReached = true;
931 break;
932 }
933
934 pTaskHead = pTaskHead->pNext;
935
936 pCurr->pNext = NULL;
937
938 AssertMsg(VALID_PTR(pCurr->pEndpoint) && (pCurr->pEndpoint == pEndpoint),
939 ("Endpoints do not match\n"));
940
941 switch (pCurr->enmTransferType)
942 {
943 case PDMACTASKFILETRANSFER_FLUSH:
944 {
945 /* If there is no data transfer request this flush request finished immediately. */
946 if (!pEndpoint->AioMgr.cRequestsActive)
947 {
948 pCurr->pfnCompleted(pCurr, pCurr->pvUser, VINF_SUCCESS);
949 pdmacFileTaskFree(pEndpoint, pCurr);
950 }
951 else
952 {
953 Assert(!pEndpoint->pFlushReq);
954 pEndpoint->pFlushReq = pCurr;
955 }
956 break;
957 }
958 case PDMACTASKFILETRANSFER_READ:
959 case PDMACTASKFILETRANSFER_WRITE:
960 {
961 RTFILEAIOREQ hReq = NIL_RTFILEAIOREQ;
962
963 if (pEndpoint->enmBackendType == PDMACFILEEPBACKEND_BUFFERED)
964 rc = pdmacFileAioMgrNormalTaskPrepareBuffered(pAioMgr, pEndpoint, pCurr, &hReq);
965 else if (pEndpoint->enmBackendType == PDMACFILEEPBACKEND_NON_BUFFERED)
966 rc = pdmacFileAioMgrNormalTaskPrepareNonBuffered(pAioMgr, pEndpoint, pCurr, &hReq);
967 else
968 AssertMsgFailed(("Invalid backend type %d\n", pEndpoint->enmBackendType));
969
970 AssertRC(rc);
971
972 if (hReq != NIL_RTFILEAIOREQ)
973 {
974 apReqs[cRequests] = hReq;
975 pEndpoint->AioMgr.cReqsProcessed++;
976 cRequests++;
977 if (cRequests == RT_ELEMENTS(apReqs))
978 {
979 rc = pdmacFileAioMgrNormalReqsEnqueue(pAioMgr, pEndpoint, apReqs, cRequests);
980 cRequests = 0;
981 AssertMsg(RT_SUCCESS(rc) || (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES),
982 ("Unexpected return code\n"));
983 }
984 }
985 break;
986 }
987 default:
988 AssertMsgFailed(("Invalid transfer type %d\n", pCurr->enmTransferType));
989 }
990 }
991
992 if (cRequests)
993 {
994 rc = pdmacFileAioMgrNormalReqsEnqueue(pAioMgr, pEndpoint, apReqs, cRequests);
995 AssertMsg(RT_SUCCESS(rc) || (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES),
996 ("Unexpected return code rc=%Rrc\n", rc));
997 }
998
999 if (pTaskHead)
1000 {
1001 /* Add the rest of the tasks to the pending list */
1002 pdmacFileAioMgrEpAddTaskList(pEndpoint, pTaskHead);
1003
1004 if (RT_UNLIKELY( pAioMgr->cRequestsActiveMax == pAioMgr->cRequestsActive
1005 && !pEndpoint->pFlushReq
1006 && !pAioMgr->fBwLimitReached))
1007 {
1008#if 0
1009 /*
1010 * The I/O manager has no room left for more requests
1011 * but there are still requests to process.
1012 * Create a new I/O manager and let it handle some endpoints.
1013 */
1014 pdmacFileAioMgrNormalBalanceLoad(pAioMgr);
1015#else
1016 /* Grow the I/O manager */
1017 pAioMgr->enmState = PDMACEPFILEMGRSTATE_GROWING;
1018#endif
1019 }
1020 }
1021
1022 /* Insufficient resources are not fatal. */
1023 if (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES)
1024 rc = VINF_SUCCESS;
1025
1026 return rc;
1027}
1028
1029/**
1030 * Adds all pending requests for the given endpoint
1031 * until a flush request is encountered or there is no
1032 * request anymore.
1033 *
1034 * @returns VBox status code.
1035 * @param pAioMgr The async I/O manager for the endpoint
1036 * @param pEndpoint The endpoint to get the requests from.
1037 */
1038static int pdmacFileAioMgrNormalQueueReqs(PPDMACEPFILEMGR pAioMgr,
1039 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
1040{
1041 int rc = VINF_SUCCESS;
1042 PPDMACTASKFILE pTasksHead = NULL;
1043
1044 AssertMsg(pEndpoint->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE,
1045 ("Trying to process request lists of a non active endpoint!\n"));
1046
1047 Assert(!pEndpoint->pFlushReq);
1048
1049 /* Check the pending list first */
1050 if (pEndpoint->AioMgr.pReqsPendingHead)
1051 {
1052 LogFlow(("Queuing pending requests first\n"));
1053
1054 pTasksHead = pEndpoint->AioMgr.pReqsPendingHead;
1055 /*
1056 * Clear the list as the processing routine will insert them into the list
1057 * again if it gets a flush request.
1058 */
1059 pEndpoint->AioMgr.pReqsPendingHead = NULL;
1060 pEndpoint->AioMgr.pReqsPendingTail = NULL;
1061 rc = pdmacFileAioMgrNormalProcessTaskList(pTasksHead, pAioMgr, pEndpoint);
1062 AssertRC(rc);
1063 }
1064
1065 if (!pEndpoint->pFlushReq && !pEndpoint->AioMgr.pReqsPendingHead)
1066 {
1067 /* Now the request queue. */
1068 pTasksHead = pdmacFileEpGetNewTasks(pEndpoint);
1069 if (pTasksHead)
1070 {
1071 rc = pdmacFileAioMgrNormalProcessTaskList(pTasksHead, pAioMgr, pEndpoint);
1072 AssertRC(rc);
1073 }
1074 }
1075
1076 return rc;
1077}
1078
1079static int pdmacFileAioMgrNormalProcessBlockingEvent(PPDMACEPFILEMGR pAioMgr)
1080{
1081 int rc = VINF_SUCCESS;
1082 bool fNotifyWaiter = false;
1083
1084 LogFlowFunc((": Enter\n"));
1085
1086 Assert(pAioMgr->fBlockingEventPending);
1087
1088 switch (pAioMgr->enmBlockingEvent)
1089 {
1090 case PDMACEPFILEAIOMGRBLOCKINGEVENT_ADD_ENDPOINT:
1091 {
1092 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointNew = (PPDMASYNCCOMPLETIONENDPOINTFILE)ASMAtomicReadPtr((void * volatile *)&pAioMgr->BlockingEventData.AddEndpoint.pEndpoint);
1093 AssertMsg(VALID_PTR(pEndpointNew), ("Adding endpoint event without a endpoint to add\n"));
1094
1095 pEndpointNew->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE;
1096
1097 pEndpointNew->AioMgr.pEndpointNext = pAioMgr->pEndpointsHead;
1098 pEndpointNew->AioMgr.pEndpointPrev = NULL;
1099 if (pAioMgr->pEndpointsHead)
1100 pAioMgr->pEndpointsHead->AioMgr.pEndpointPrev = pEndpointNew;
1101 pAioMgr->pEndpointsHead = pEndpointNew;
1102
1103 /* Assign the completion point to this file. */
1104 rc = RTFileAioCtxAssociateWithFile(pAioMgr->hAioCtx, pEndpointNew->File);
1105 fNotifyWaiter = true;
1106 pAioMgr->cEndpoints++;
1107 break;
1108 }
1109 case PDMACEPFILEAIOMGRBLOCKINGEVENT_REMOVE_ENDPOINT:
1110 {
1111 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointRemove = (PPDMASYNCCOMPLETIONENDPOINTFILE)ASMAtomicReadPtr((void * volatile *)&pAioMgr->BlockingEventData.RemoveEndpoint.pEndpoint);
1112 AssertMsg(VALID_PTR(pEndpointRemove), ("Removing endpoint event without a endpoint to remove\n"));
1113
1114 pEndpointRemove->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_REMOVING;
1115 fNotifyWaiter = !pdmacFileAioMgrNormalRemoveEndpoint(pEndpointRemove);
1116 break;
1117 }
1118 case PDMACEPFILEAIOMGRBLOCKINGEVENT_CLOSE_ENDPOINT:
1119 {
1120 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointClose = (PPDMASYNCCOMPLETIONENDPOINTFILE)ASMAtomicReadPtr((void * volatile *)&pAioMgr->BlockingEventData.CloseEndpoint.pEndpoint);
1121 AssertMsg(VALID_PTR(pEndpointClose), ("Close endpoint event without a endpoint to close\n"));
1122
1123 if (pEndpointClose->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE)
1124 {
1125 LogFlowFunc((": Closing endpoint %#p{%s}\n", pEndpointClose, pEndpointClose->Core.pszUri));
1126
1127 /* Make sure all tasks finished. Process the queues a last time first. */
1128 rc = pdmacFileAioMgrNormalQueueReqs(pAioMgr, pEndpointClose);
1129 AssertRC(rc);
1130
1131 pEndpointClose->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_CLOSING;
1132 fNotifyWaiter = !pdmacFileAioMgrNormalRemoveEndpoint(pEndpointClose);
1133 }
1134 else if ( (pEndpointClose->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_CLOSING)
1135 && (!pEndpointClose->AioMgr.cRequestsActive))
1136 fNotifyWaiter = true;
1137 break;
1138 }
1139 case PDMACEPFILEAIOMGRBLOCKINGEVENT_SHUTDOWN:
1140 {
1141 pAioMgr->enmState = PDMACEPFILEMGRSTATE_SHUTDOWN;
1142 if (!pAioMgr->cRequestsActive)
1143 fNotifyWaiter = true;
1144 break;
1145 }
1146 case PDMACEPFILEAIOMGRBLOCKINGEVENT_SUSPEND:
1147 {
1148 pAioMgr->enmState = PDMACEPFILEMGRSTATE_SUSPENDING;
1149 break;
1150 }
1151 case PDMACEPFILEAIOMGRBLOCKINGEVENT_RESUME:
1152 {
1153 pAioMgr->enmState = PDMACEPFILEMGRSTATE_RUNNING;
1154 fNotifyWaiter = true;
1155 break;
1156 }
1157 default:
1158 AssertReleaseMsgFailed(("Invalid event type %d\n", pAioMgr->enmBlockingEvent));
1159 }
1160
1161 if (fNotifyWaiter)
1162 {
1163 ASMAtomicWriteBool(&pAioMgr->fBlockingEventPending, false);
1164 pAioMgr->enmBlockingEvent = PDMACEPFILEAIOMGRBLOCKINGEVENT_INVALID;
1165
1166 /* Release the waiting thread. */
1167 LogFlow(("Signalling waiter\n"));
1168 rc = RTSemEventSignal(pAioMgr->EventSemBlock);
1169 AssertRC(rc);
1170 }
1171
1172 LogFlowFunc((": Leave\n"));
1173 return rc;
1174}
1175
1176/**
1177 * Checks all endpoints for pending events or new requests.
1178 *
1179 * @returns VBox status code.
1180 * @param pAioMgr The I/O manager handle.
1181 */
1182static int pdmacFileAioMgrNormalCheckEndpoints(PPDMACEPFILEMGR pAioMgr)
1183{
1184 /* Check the assigned endpoints for new tasks if there isn't a flush request active at the moment. */
1185 int rc = VINF_SUCCESS;
1186 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint = pAioMgr->pEndpointsHead;
1187
1188 pAioMgr->fBwLimitReached = false;
1189
1190 while (pEndpoint)
1191 {
1192 if (!pEndpoint->pFlushReq
1193 && (pEndpoint->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE)
1194 && !pEndpoint->AioMgr.fMoving)
1195 {
1196 rc = pdmacFileAioMgrNormalQueueReqs(pAioMgr, pEndpoint);
1197 if (RT_FAILURE(rc))
1198 return rc;
1199 }
1200 else if (!pEndpoint->AioMgr.cRequestsActive)
1201 {
1202 /* Reopen the file so that the new endpoint can reassociate with the file */
1203 RTFileClose(pEndpoint->File);
1204 rc = RTFileOpen(&pEndpoint->File, pEndpoint->Core.pszUri, pEndpoint->fFlags);
1205 AssertRC(rc);
1206
1207 if (pEndpoint->AioMgr.fMoving)
1208 {
1209 pEndpoint->AioMgr.fMoving = false;
1210 pdmacFileAioMgrAddEndpoint(pEndpoint->AioMgr.pAioMgrDst, pEndpoint);
1211 }
1212 else
1213 {
1214 Assert(pAioMgr->fBlockingEventPending);
1215 ASMAtomicWriteBool(&pAioMgr->fBlockingEventPending, false);
1216
1217 /* Release the waiting thread. */
1218 LogFlow(("Signalling waiter\n"));
1219 rc = RTSemEventSignal(pAioMgr->EventSemBlock);
1220 AssertRC(rc);
1221 }
1222 }
1223
1224 pEndpoint = pEndpoint->AioMgr.pEndpointNext;
1225 }
1226
1227 return rc;
1228}
1229
1230static void pdmacFileAioMgrNormalReqComplete(PPDMACEPFILEMGR pAioMgr, RTFILEAIOREQ hReq)
1231{
1232 int rc = VINF_SUCCESS;
1233 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint;
1234 size_t cbTransfered = 0;
1235 int rcReq = RTFileAioReqGetRC(hReq, &cbTransfered);
1236 PPDMACTASKFILE pTask = (PPDMACTASKFILE)RTFileAioReqGetUser(hReq);
1237 PPDMACTASKFILE pTasksWaiting;
1238
1239 pEndpoint = pTask->pEndpoint;
1240
1241 /*
1242 * It is possible that the request failed on Linux with kernels < 2.6.23
1243 * if the passed buffer was allocated with remap_pfn_range or if the file
1244 * is on an NFS endpoint which does not support async and direct I/O at the same time.
1245 * The endpoint will be migrated to a failsafe manager in case a request fails.
1246 */
1247 if (RT_FAILURE(rcReq))
1248 {
1249 /* Free bounce buffers and the IPRT request. */
1250 pdmacFileAioMgrNormalRequestFree(pAioMgr, hReq);
1251
1252 /* Free the lock and process pending tasks if neccessary */
1253 pTasksWaiting = pdmacFileAioMgrNormalRangeLockFree(pAioMgr, pEndpoint, pTask->pRangeLock);
1254 rc = pdmacFileAioMgrNormalProcessTaskList(pTasksWaiting, pAioMgr, pEndpoint);
1255 AssertRC(rc);
1256
1257 pAioMgr->cRequestsActive--;
1258 pEndpoint->AioMgr.cRequestsActive--;
1259 pEndpoint->AioMgr.cReqsProcessed++;
1260
1261 if (pTask->cbBounceBuffer)
1262 RTMemPageFree(pTask->pvBounceBuffer, pTask->cbBounceBuffer);
1263
1264 /* Queue the request on the pending list. */
1265 pTask->pNext = pEndpoint->AioMgr.pReqsPendingHead;
1266 pEndpoint->AioMgr.pReqsPendingHead = pTask;
1267
1268 /* Create a new failsafe manager if neccessary. */
1269 if (!pEndpoint->AioMgr.fMoving)
1270 {
1271 PPDMACEPFILEMGR pAioMgrFailsafe;
1272
1273 LogRel(("%s: Request %#p failed with rc=%Rrc, migrating endpoint %s to failsafe manager.\n",
1274 RTThreadGetName(pAioMgr->Thread), pTask, rcReq, pEndpoint->Core.pszUri));
1275
1276 pEndpoint->AioMgr.fMoving = true;
1277
1278 rc = pdmacFileAioMgrCreate((PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass,
1279 &pAioMgrFailsafe, PDMACEPFILEMGRTYPE_SIMPLE);
1280 AssertRC(rc);
1281
1282 pEndpoint->AioMgr.pAioMgrDst = pAioMgrFailsafe;
1283
1284 /* Update the flags to open the file with. Disable async I/O and enable the host cache. */
1285 pEndpoint->fFlags &= ~(RTFILE_O_ASYNC_IO | RTFILE_O_NO_CACHE);
1286 }
1287
1288 /* If this was the last request for the endpoint migrate it to the new manager. */
1289 if (!pEndpoint->AioMgr.cRequestsActive)
1290 {
1291 bool fReqsPending = pdmacFileAioMgrNormalRemoveEndpoint(pEndpoint);
1292 Assert(!fReqsPending);
1293
1294 rc = pdmacFileAioMgrAddEndpoint(pEndpoint->AioMgr.pAioMgrDst, pEndpoint);
1295 AssertRC(rc);
1296 }
1297 }
1298 else
1299 {
1300 /*
1301 * Restart an incomplete transfer.
1302 * This usually means that the request will return an error now
1303 * but to get the cause of the error (disk full, file too big, I/O error, ...)
1304 * the transfer needs to be continued.
1305 */
1306 if (RT_UNLIKELY( cbTransfered < pTask->DataSeg.cbSeg
1307 || ( pTask->cbBounceBuffer
1308 && cbTransfered < pTask->cbBounceBuffer)))
1309 {
1310 RTFOFF offStart;
1311 size_t cbToTransfer;
1312 uint8_t *pbBuf = NULL;
1313
1314 LogFlow(("Restarting incomplete transfer %#p (%zu bytes transfered)\n",
1315 pTask, cbTransfered));
1316 Assert(cbTransfered % 512 == 0);
1317
1318 if (pTask->cbBounceBuffer)
1319 {
1320 AssertPtr(pTask->pvBounceBuffer);
1321 offStart = (pTask->Off & ~((RTFOFF)512-1)) + cbTransfered;
1322 cbToTransfer = pTask->cbBounceBuffer - cbTransfered;
1323 pbBuf = (uint8_t *)pTask->pvBounceBuffer + cbTransfered;
1324 }
1325 else
1326 {
1327 Assert(!pTask->pvBounceBuffer);
1328 offStart = pTask->Off + cbTransfered;
1329 cbToTransfer = pTask->DataSeg.cbSeg - cbTransfered;
1330 pbBuf = (uint8_t *)pTask->DataSeg.pvSeg + cbTransfered;
1331 }
1332
1333 if (pTask->fPrefetch || pTask->enmTransferType == PDMACTASKFILETRANSFER_READ)
1334 {
1335 rc = RTFileAioReqPrepareRead(hReq, pEndpoint->File, offStart,
1336 pbBuf, cbToTransfer, pTask);
1337 }
1338 else
1339 {
1340 AssertMsg(pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE,
1341 ("Invalid transfer type\n"));
1342 rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->File, offStart,
1343 pbBuf, cbToTransfer, pTask);
1344 }
1345
1346 AssertRC(rc);
1347 rc = RTFileAioCtxSubmit(pAioMgr->hAioCtx, &hReq, 1);
1348 AssertRC(rc);
1349 }
1350 else if (pTask->fPrefetch)
1351 {
1352 Assert(pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE);
1353 Assert(pTask->cbBounceBuffer);
1354
1355 memcpy(((uint8_t *)pTask->pvBounceBuffer) + pTask->offBounceBuffer,
1356 pTask->DataSeg.pvSeg,
1357 pTask->DataSeg.cbSeg);
1358
1359 /* Write it now. */
1360 pTask->fPrefetch = false;
1361 size_t cbToTransfer = RT_ALIGN_Z(pTask->DataSeg.cbSeg, 512);
1362 RTFOFF offStart = pTask->Off & ~(RTFOFF)(512-1);
1363
1364 /* Grow the file if needed. */
1365 if (RT_UNLIKELY((uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) > pEndpoint->cbFile))
1366 {
1367 ASMAtomicWriteU64(&pEndpoint->cbFile, pTask->Off + pTask->DataSeg.cbSeg);
1368 RTFileSetSize(pEndpoint->File, pTask->Off + pTask->DataSeg.cbSeg);
1369 }
1370
1371 rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->File,
1372 offStart, pTask->pvBounceBuffer, cbToTransfer, pTask);
1373 AssertRC(rc);
1374 rc = RTFileAioCtxSubmit(pAioMgr->hAioCtx, &hReq, 1);
1375 AssertRC(rc);
1376 }
1377 else
1378 {
1379 if (RT_SUCCESS(rc) && pTask->cbBounceBuffer)
1380 {
1381 if (pTask->enmTransferType == PDMACTASKFILETRANSFER_READ)
1382 memcpy(pTask->DataSeg.pvSeg,
1383 ((uint8_t *)pTask->pvBounceBuffer) + pTask->offBounceBuffer,
1384 pTask->DataSeg.cbSeg);
1385
1386 RTMemPageFree(pTask->pvBounceBuffer, pTask->cbBounceBuffer);
1387 }
1388
1389 pdmacFileAioMgrNormalRequestFree(pAioMgr, hReq);
1390
1391 pAioMgr->cRequestsActive--;
1392 pEndpoint->AioMgr.cRequestsActive--;
1393 pEndpoint->AioMgr.cReqsProcessed++;
1394
1395 /* Free the lock and process pending tasks if neccessary */
1396 pTasksWaiting = pdmacFileAioMgrNormalRangeLockFree(pAioMgr, pEndpoint, pTask->pRangeLock);
1397 rc = pdmacFileAioMgrNormalProcessTaskList(pTasksWaiting, pAioMgr, pEndpoint);
1398 AssertRC(rc);
1399
1400 /* Call completion callback */
1401 LogFlow(("Task=%#p completed with %Rrc\n", pTask, rcReq));
1402 pTask->pfnCompleted(pTask, pTask->pvUser, rcReq);
1403 pdmacFileTaskFree(pEndpoint, pTask);
1404
1405 /*
1406 * If there is no request left on the endpoint but a flush request is set
1407 * it completed now and we notify the owner.
1408 * Furthermore we look for new requests and continue.
1409 */
1410 if (!pEndpoint->AioMgr.cRequestsActive && pEndpoint->pFlushReq)
1411 {
1412 /* Call completion callback */
1413 pTask = pEndpoint->pFlushReq;
1414 pEndpoint->pFlushReq = NULL;
1415
1416 AssertMsg(pTask->pEndpoint == pEndpoint, ("Endpoint of the flush request does not match assigned one\n"));
1417
1418 pTask->pfnCompleted(pTask, pTask->pvUser, VINF_SUCCESS);
1419 pdmacFileTaskFree(pEndpoint, pTask);
1420 }
1421 else if (RT_UNLIKELY(!pEndpoint->AioMgr.cRequestsActive && pEndpoint->AioMgr.fMoving))
1422 {
1423 /* If the endpoint is about to be migrated do it now. */
1424 bool fReqsPending = pdmacFileAioMgrNormalRemoveEndpoint(pEndpoint);
1425 Assert(!fReqsPending);
1426
1427 rc = pdmacFileAioMgrAddEndpoint(pEndpoint->AioMgr.pAioMgrDst, pEndpoint);
1428 AssertRC(rc);
1429 }
1430 }
1431 } /* request completed successfully */
1432}
1433
1434/** Helper macro for checking for error codes. */
1435#define CHECK_RC(pAioMgr, rc) \
1436 if (RT_FAILURE(rc)) \
1437 {\
1438 int rc2 = pdmacFileAioMgrNormalErrorHandler(pAioMgr, rc, RT_SRC_POS);\
1439 return rc2;\
1440 }
1441
1442/**
1443 * The normal I/O manager using the RTFileAio* API
1444 *
1445 * @returns VBox status code.
1446 * @param ThreadSelf Handle of the thread.
1447 * @param pvUser Opaque user data.
1448 */
1449int pdmacFileAioMgrNormal(RTTHREAD ThreadSelf, void *pvUser)
1450{
1451 int rc = VINF_SUCCESS;
1452 PPDMACEPFILEMGR pAioMgr = (PPDMACEPFILEMGR)pvUser;
1453 uint64_t uMillisEnd = RTTimeMilliTS() + PDMACEPFILEMGR_LOAD_UPDATE_PERIOD;
1454
1455 while ( (pAioMgr->enmState == PDMACEPFILEMGRSTATE_RUNNING)
1456 || (pAioMgr->enmState == PDMACEPFILEMGRSTATE_SUSPENDING)
1457 || (pAioMgr->enmState == PDMACEPFILEMGRSTATE_GROWING))
1458 {
1459 if (!pAioMgr->cRequestsActive)
1460 {
1461 ASMAtomicWriteBool(&pAioMgr->fWaitingEventSem, true);
1462 if (!ASMAtomicReadBool(&pAioMgr->fWokenUp))
1463 rc = RTSemEventWait(pAioMgr->EventSem, RT_INDEFINITE_WAIT);
1464 ASMAtomicWriteBool(&pAioMgr->fWaitingEventSem, false);
1465 AssertRC(rc);
1466
1467 LogFlow(("Got woken up\n"));
1468 ASMAtomicWriteBool(&pAioMgr->fWokenUp, false);
1469 }
1470
1471 /* Check for an external blocking event first. */
1472 if (pAioMgr->fBlockingEventPending)
1473 {
1474 rc = pdmacFileAioMgrNormalProcessBlockingEvent(pAioMgr);
1475 CHECK_RC(pAioMgr, rc);
1476 }
1477
1478 if (RT_LIKELY( pAioMgr->enmState == PDMACEPFILEMGRSTATE_RUNNING
1479 || pAioMgr->enmState == PDMACEPFILEMGRSTATE_GROWING))
1480 {
1481 /* We got woken up because an endpoint issued new requests. Queue them. */
1482 rc = pdmacFileAioMgrNormalCheckEndpoints(pAioMgr);
1483 CHECK_RC(pAioMgr, rc);
1484
1485 while ( pAioMgr->cRequestsActive
1486 || pAioMgr->fBwLimitReached)
1487 {
1488 if (pAioMgr->cRequestsActive)
1489 {
1490 RTFILEAIOREQ apReqs[20];
1491 uint32_t cReqsCompleted = 0;
1492 size_t cReqsWait;
1493
1494 if (pAioMgr->cRequestsActive > RT_ELEMENTS(apReqs))
1495 cReqsWait = RT_ELEMENTS(apReqs);
1496 else
1497 cReqsWait = pAioMgr->cRequestsActive;
1498
1499 LogFlow(("Waiting for %d of %d tasks to complete\n", pAioMgr->cRequestsActive, cReqsWait));
1500
1501 rc = RTFileAioCtxWait(pAioMgr->hAioCtx,
1502 cReqsWait,
1503 RT_INDEFINITE_WAIT, apReqs,
1504 RT_ELEMENTS(apReqs), &cReqsCompleted);
1505 if (RT_FAILURE(rc) && (rc != VERR_INTERRUPTED))
1506 CHECK_RC(pAioMgr, rc);
1507
1508 LogFlow(("%d tasks completed\n", cReqsCompleted));
1509
1510 for (uint32_t i = 0; i < cReqsCompleted; i++)
1511 pdmacFileAioMgrNormalReqComplete(pAioMgr, apReqs[i]);
1512
1513 /* Check for an external blocking event before we go to sleep again. */
1514 if (pAioMgr->fBlockingEventPending)
1515 {
1516 rc = pdmacFileAioMgrNormalProcessBlockingEvent(pAioMgr);
1517 CHECK_RC(pAioMgr, rc);
1518 }
1519
1520 /* Update load statistics. */
1521 uint64_t uMillisCurr = RTTimeMilliTS();
1522 if (uMillisCurr > uMillisEnd)
1523 {
1524 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointCurr = pAioMgr->pEndpointsHead;
1525
1526 /* Calculate timespan. */
1527 uMillisCurr -= uMillisEnd;
1528
1529 while (pEndpointCurr)
1530 {
1531 pEndpointCurr->AioMgr.cReqsPerSec = pEndpointCurr->AioMgr.cReqsProcessed / (uMillisCurr + PDMACEPFILEMGR_LOAD_UPDATE_PERIOD);
1532 pEndpointCurr->AioMgr.cReqsProcessed = 0;
1533 pEndpointCurr = pEndpointCurr->AioMgr.pEndpointNext;
1534 }
1535
1536 /* Set new update interval */
1537 uMillisEnd = RTTimeMilliTS() + PDMACEPFILEMGR_LOAD_UPDATE_PERIOD;
1538 }
1539 }
1540 else
1541 {
1542 /*
1543 * Bandwidth limit reached for all endpoints.
1544 * Yield and wait until we have enough resources again.
1545 */
1546 RTThreadYield();
1547 }
1548
1549 /* Check endpoints for new requests. */
1550 if (pAioMgr->enmState != PDMACEPFILEMGRSTATE_GROWING)
1551 {
1552 rc = pdmacFileAioMgrNormalCheckEndpoints(pAioMgr);
1553 CHECK_RC(pAioMgr, rc);
1554 }
1555 } /* while requests are active. */
1556
1557 if (pAioMgr->enmState == PDMACEPFILEMGRSTATE_GROWING)
1558 {
1559 rc = pdmacFileAioMgrNormalGrow(pAioMgr);
1560 AssertRC(rc);
1561 Assert(pAioMgr->enmState == PDMACEPFILEMGRSTATE_RUNNING);
1562
1563 rc = pdmacFileAioMgrNormalCheckEndpoints(pAioMgr);
1564 CHECK_RC(pAioMgr, rc);
1565 }
1566 } /* if still running */
1567 } /* while running */
1568
1569 LogFlowFunc(("rc=%Rrc\n", rc));
1570 return rc;
1571}
1572
1573#undef CHECK_RC
1574
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette