VirtualBox

source: vbox/trunk/src/VBox/VMM/PDMAsyncCompletionFileNormal.cpp@ 29124

Last change on this file since 29124 was 29124, checked in by vboxsync, 15 years ago

AsyncCompletion: Fix assertion
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 63.8 KB
Line 
1/* $Id: PDMAsyncCompletionFileNormal.cpp 29124 2010-05-06 09:36:07Z vboxsync $ */
2/** @file
3 * PDM Async I/O - Transport data asynchronous in R3 using EMT.
4 * Async File I/O manager.
5 */
6
7/*
8 * Copyright (C) 2006-2008 Oracle Corporation
9 *
10 * This file is part of VirtualBox Open Source Edition (OSE), as
11 * available from http://www.virtualbox.org. This file is free software;
12 * you can redistribute it and/or modify it under the terms of the GNU
13 * General Public License (GPL) as published by the Free Software
14 * Foundation, in version 2 as it comes in the "COPYING" file of the
15 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
16 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
17 */
18#define LOG_GROUP LOG_GROUP_PDM_ASYNC_COMPLETION
19#include <iprt/types.h>
20#include <iprt/asm.h>
21#include <iprt/file.h>
22#include <iprt/mem.h>
23#include <iprt/string.h>
24#include <iprt/assert.h>
25#include <VBox/log.h>
26
27#include "PDMAsyncCompletionFileInternal.h"
28
29/** The update period for the I/O load statistics in ms. */
30#define PDMACEPFILEMGR_LOAD_UPDATE_PERIOD 1000
31/** Maximum number of requests a manager will handle. */
32#define PDMACEPFILEMGR_REQS_STEP 512
33
34/*******************************************************************************
35* Internal functions *
36*******************************************************************************/
37static int pdmacFileAioMgrNormalProcessTaskList(PPDMACTASKFILE pTaskHead,
38 PPDMACEPFILEMGR pAioMgr,
39 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint);
40
41static PPDMACTASKFILE pdmacFileAioMgrNormalRangeLockFree(PPDMACEPFILEMGR pAioMgr,
42 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
43 PPDMACFILERANGELOCK pRangeLock);
44
45int pdmacFileAioMgrNormalInit(PPDMACEPFILEMGR pAioMgr)
46{
47 int rc = VINF_SUCCESS;
48
49 pAioMgr->cRequestsActiveMax = PDMACEPFILEMGR_REQS_STEP;
50
51 rc = RTFileAioCtxCreate(&pAioMgr->hAioCtx, RTFILEAIO_UNLIMITED_REQS);
52 if (rc == VERR_OUT_OF_RANGE)
53 rc = RTFileAioCtxCreate(&pAioMgr->hAioCtx, pAioMgr->cRequestsActiveMax);
54
55 if (RT_SUCCESS(rc))
56 {
57 /* Initialize request handle array. */
58 pAioMgr->iFreeEntry = 0;
59 pAioMgr->cReqEntries = pAioMgr->cRequestsActiveMax;
60 pAioMgr->pahReqsFree = (RTFILEAIOREQ *)RTMemAllocZ(pAioMgr->cReqEntries * sizeof(RTFILEAIOREQ));
61
62 if (pAioMgr->pahReqsFree)
63 {
64 /* Create the range lock memcache. */
65 rc = RTMemCacheCreate(&pAioMgr->hMemCacheRangeLocks, sizeof(PDMACFILERANGELOCK),
66 0, UINT32_MAX, NULL, NULL, NULL, 0);
67 if (RT_SUCCESS(rc))
68 return VINF_SUCCESS;
69
70 RTMemFree(pAioMgr->pahReqsFree);
71 }
72 else
73 {
74 RTFileAioCtxDestroy(pAioMgr->hAioCtx);
75 rc = VERR_NO_MEMORY;
76 }
77 }
78
79 return rc;
80}
81
82void pdmacFileAioMgrNormalDestroy(PPDMACEPFILEMGR pAioMgr)
83{
84 RTFileAioCtxDestroy(pAioMgr->hAioCtx);
85
86 while (pAioMgr->iFreeEntry > 0)
87 {
88 pAioMgr->iFreeEntry--;
89 Assert(pAioMgr->pahReqsFree[pAioMgr->iFreeEntry] != NIL_RTFILEAIOREQ);
90 RTFileAioReqDestroy(pAioMgr->pahReqsFree[pAioMgr->iFreeEntry]);
91 }
92
93 RTMemFree(pAioMgr->pahReqsFree);
94 RTMemCacheDestroy(pAioMgr->hMemCacheRangeLocks);
95}
96
97/**
98 * Sorts the endpoint list with insertion sort.
99 */
100static void pdmacFileAioMgrNormalEndpointsSortByLoad(PPDMACEPFILEMGR pAioMgr)
101{
102 PPDMASYNCCOMPLETIONENDPOINTFILE pEpPrev, pEpCurr, pEpNextToSort;
103
104 pEpPrev = pAioMgr->pEndpointsHead;
105 pEpCurr = pEpPrev->AioMgr.pEndpointNext;
106
107 while (pEpCurr)
108 {
109 /* Remember the next element to sort because the list might change. */
110 pEpNextToSort = pEpCurr->AioMgr.pEndpointNext;
111
112 /* Unlink the current element from the list. */
113 PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEpCurr->AioMgr.pEndpointPrev;
114 PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEpCurr->AioMgr.pEndpointNext;
115
116 if (pPrev)
117 pPrev->AioMgr.pEndpointNext = pNext;
118 else
119 pAioMgr->pEndpointsHead = pNext;
120
121 if (pNext)
122 pNext->AioMgr.pEndpointPrev = pPrev;
123
124 /* Go back until we reached the place to insert the current endpoint into. */
125 while (pEpPrev && (pEpPrev->AioMgr.cReqsPerSec < pEpCurr->AioMgr.cReqsPerSec))
126 pEpPrev = pEpPrev->AioMgr.pEndpointPrev;
127
128 /* Link the endpoint into the list. */
129 if (pEpPrev)
130 pNext = pEpPrev->AioMgr.pEndpointNext;
131 else
132 pNext = pAioMgr->pEndpointsHead;
133
134 pEpCurr->AioMgr.pEndpointNext = pNext;
135 pEpCurr->AioMgr.pEndpointPrev = pEpPrev;
136
137 if (pNext)
138 pNext->AioMgr.pEndpointPrev = pEpCurr;
139
140 if (pEpPrev)
141 pEpPrev->AioMgr.pEndpointNext = pEpCurr;
142 else
143 pAioMgr->pEndpointsHead = pEpCurr;
144
145 pEpCurr = pEpNextToSort;
146 }
147
148#ifdef DEBUG
149 /* Validate sorting alogrithm */
150 unsigned cEndpoints = 0;
151 pEpCurr = pAioMgr->pEndpointsHead;
152
153 AssertMsg(pEpCurr, ("No endpoint in the list?\n"));
154 AssertMsg(!pEpCurr->AioMgr.pEndpointPrev, ("First element in the list points to previous element\n"));
155
156 while (pEpCurr)
157 {
158 cEndpoints++;
159
160 PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEpCurr->AioMgr.pEndpointNext;
161 PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEpCurr->AioMgr.pEndpointPrev;
162
163 Assert(!pNext || pNext->AioMgr.cReqsPerSec <= pEpCurr->AioMgr.cReqsPerSec);
164 Assert(!pPrev || pPrev->AioMgr.cReqsPerSec >= pEpCurr->AioMgr.cReqsPerSec);
165
166 pEpCurr = pNext;
167 }
168
169 AssertMsg(cEndpoints == pAioMgr->cEndpoints, ("Endpoints lost during sort!\n"));
170
171#endif
172}
173
174/**
175 * Removes an endpoint from the currently assigned manager.
176 *
177 * @returns TRUE if there are still requests pending on the current manager for this endpoint.
178 * FALSE otherwise.
179 * @param pEndpointRemove The endpoint to remove.
180 */
181static bool pdmacFileAioMgrNormalRemoveEndpoint(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointRemove)
182{
183 PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEndpointRemove->AioMgr.pEndpointPrev;
184 PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEndpointRemove->AioMgr.pEndpointNext;
185 PPDMACEPFILEMGR pAioMgr = pEndpointRemove->pAioMgr;
186
187 pAioMgr->cEndpoints--;
188
189 if (pPrev)
190 pPrev->AioMgr.pEndpointNext = pNext;
191 else
192 pAioMgr->pEndpointsHead = pNext;
193
194 if (pNext)
195 pNext->AioMgr.pEndpointPrev = pPrev;
196
197 /* Make sure that there is no request pending on this manager for the endpoint. */
198 if (!pEndpointRemove->AioMgr.cRequestsActive)
199 {
200 Assert(!pEndpointRemove->pFlushReq);
201
202 /* Reopen the file so that the new endpoint can reassociate with the file */
203 RTFileClose(pEndpointRemove->File);
204 int rc = RTFileOpen(&pEndpointRemove->File, pEndpointRemove->Core.pszUri, pEndpointRemove->fFlags);
205 AssertRC(rc);
206 return false;
207 }
208
209 return true;
210}
211
212static bool pdmacFileAioMgrNormalIsBalancePossible(PPDMACEPFILEMGR pAioMgr)
213{
214 /* Balancing doesn't make sense with only one endpoint. */
215 if (pAioMgr->cEndpoints == 1)
216 return false;
217
218 /* Doesn't make sens to move endpoints if only one produces the whole load */
219 unsigned cEndpointsWithLoad = 0;
220
221 PPDMASYNCCOMPLETIONENDPOINTFILE pCurr = pAioMgr->pEndpointsHead;
222
223 while (pCurr)
224 {
225 if (pCurr->AioMgr.cReqsPerSec)
226 cEndpointsWithLoad++;
227
228 pCurr = pCurr->AioMgr.pEndpointNext;
229 }
230
231 return (cEndpointsWithLoad > 1);
232}
233
234/**
235 * Creates a new I/O manager and spreads the I/O load of the endpoints
236 * between the given I/O manager and the new one.
237 *
238 * @returns nothing.
239 * @param pAioMgr The I/O manager with high I/O load.
240 */
241static void pdmacFileAioMgrNormalBalanceLoad(PPDMACEPFILEMGR pAioMgr)
242{
243 PPDMACEPFILEMGR pAioMgrNew = NULL;
244 int rc = VINF_SUCCESS;
245
246 /*
247 * Check if balancing would improve the situation.
248 */
249 if (pdmacFileAioMgrNormalIsBalancePossible(pAioMgr))
250 {
251 PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pAioMgr->pEndpointsHead->Core.pEpClass;
252
253 rc = pdmacFileAioMgrCreate(pEpClassFile, &pAioMgrNew, PDMACEPFILEMGRTYPE_ASYNC);
254 if (RT_SUCCESS(rc))
255 {
256 /* We will sort the list by request count per second. */
257 pdmacFileAioMgrNormalEndpointsSortByLoad(pAioMgr);
258
259 /* Now move some endpoints to the new manager. */
260 unsigned cReqsHere = pAioMgr->pEndpointsHead->AioMgr.cReqsPerSec;
261 unsigned cReqsOther = 0;
262 PPDMASYNCCOMPLETIONENDPOINTFILE pCurr = pAioMgr->pEndpointsHead->AioMgr.pEndpointNext;
263
264 while (pCurr)
265 {
266 if (cReqsHere <= cReqsOther)
267 {
268 /*
269 * The other manager has more requests to handle now.
270 * We will keep the current endpoint.
271 */
272 Log(("Keeping endpoint %#p{%s} with %u reqs/s\n", pCurr->Core.pszUri, pCurr->AioMgr.cReqsPerSec));
273 cReqsHere += pCurr->AioMgr.cReqsPerSec;
274 pCurr = pCurr->AioMgr.pEndpointNext;
275 }
276 else
277 {
278 /* Move to other endpoint. */
279 Log(("Moving endpoint %#p{%s} with %u reqs/s to other manager\n", pCurr, pCurr->Core.pszUri, pCurr->AioMgr.cReqsPerSec));
280 cReqsOther += pCurr->AioMgr.cReqsPerSec;
281
282 PPDMASYNCCOMPLETIONENDPOINTFILE pMove = pCurr;
283
284 pCurr = pCurr->AioMgr.pEndpointNext;
285
286 bool fReqsPending = pdmacFileAioMgrNormalRemoveEndpoint(pMove);
287
288 if (fReqsPending)
289 {
290 pMove->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_REMOVING;
291 pMove->AioMgr.fMoving = true;
292 pMove->AioMgr.pAioMgrDst = pAioMgrNew;
293 }
294 else
295 {
296 pMove->AioMgr.fMoving = false;
297 pMove->AioMgr.pAioMgrDst = NULL;
298 pdmacFileAioMgrAddEndpoint(pAioMgrNew, pMove);
299 }
300 }
301 }
302 }
303 else
304 {
305 /* Don't process further but leave a log entry about reduced performance. */
306 LogRel(("AIOMgr: Could not create new I/O manager (rc=%Rrc). Expect reduced performance\n", rc));
307 }
308 }
309 else
310 Log(("AIOMgr: Load balancing would not improve anything\n"));
311}
312
313/**
314 * Increase the maximum number of active requests for the given I/O manager.
315 *
316 * @returns VBox status code.
317 * @param pAioMgr The I/O manager to grow.
318 */
319static int pdmacFileAioMgrNormalGrow(PPDMACEPFILEMGR pAioMgr)
320{
321 int rc = VINF_SUCCESS;
322 RTFILEAIOCTX hAioCtxNew = NIL_RTFILEAIOCTX;
323
324 LogFlowFunc(("pAioMgr=%#p\n", pAioMgr));
325
326 AssertMsg( pAioMgr->enmState == PDMACEPFILEMGRSTATE_GROWING
327 && !pAioMgr->cRequestsActive,
328 ("Invalid state of the I/O manager\n"));
329
330#ifdef RT_OS_WINDOWS
331 /*
332 * Reopen the files of all assigned endpoints first so we can assign them to the new
333 * I/O context.
334 */
335 PPDMASYNCCOMPLETIONENDPOINTFILE pCurr = pAioMgr->pEndpointsHead;
336
337 while (pCurr)
338 {
339 RTFileClose(pCurr->File);
340 rc = RTFileOpen(&pCurr->File, pCurr->Core.pszUri, pCurr->fFlags);
341 AssertRC(rc);
342
343 pCurr = pCurr->AioMgr.pEndpointNext;
344 }
345#endif
346
347 /* Create the new bigger context. */
348 pAioMgr->cRequestsActiveMax += PDMACEPFILEMGR_REQS_STEP;
349
350 rc = RTFileAioCtxCreate(&hAioCtxNew, RTFILEAIO_UNLIMITED_REQS);
351 if (rc == VERR_OUT_OF_RANGE)
352 rc = RTFileAioCtxCreate(&hAioCtxNew, pAioMgr->cRequestsActiveMax);
353
354 if (RT_SUCCESS(rc))
355 {
356 /* Close the old context. */
357 rc = RTFileAioCtxDestroy(pAioMgr->hAioCtx);
358 AssertRC(rc);
359
360 pAioMgr->hAioCtx = hAioCtxNew;
361
362 /* Create a new I/O task handle array */
363 uint32_t cReqEntriesNew = pAioMgr->cRequestsActiveMax + 1;
364 RTFILEAIOREQ *pahReqNew = (RTFILEAIOREQ *)RTMemAllocZ(cReqEntriesNew * sizeof(RTFILEAIOREQ));
365
366 if (pahReqNew)
367 {
368 /* Copy the cached request handles. */
369 for (uint32_t iReq = 0; iReq < pAioMgr->cReqEntries; iReq++)
370 pahReqNew[iReq] = pAioMgr->pahReqsFree[iReq];
371
372 RTMemFree(pAioMgr->pahReqsFree);
373 pAioMgr->pahReqsFree = pahReqNew;
374 pAioMgr->cReqEntries = cReqEntriesNew;
375 LogFlowFunc(("I/O manager increased to handle a maximum of %u requests\n",
376 pAioMgr->cRequestsActiveMax));
377 }
378 else
379 rc = VERR_NO_MEMORY;
380 }
381
382#ifdef RT_OS_WINDOWS
383 /* Assign the file to the new context. */
384 pCurr = pAioMgr->pEndpointsHead;
385
386 while (pCurr)
387 {
388 rc = RTFileAioCtxAssociateWithFile(pAioMgr->hAioCtx, pCurr->File);
389 AssertRC(rc);
390
391 pCurr = pCurr->AioMgr.pEndpointNext;
392 }
393#endif
394
395 if (RT_FAILURE(rc))
396 {
397 LogFlow(("Increasing size of the I/O manager failed with rc=%Rrc\n", rc));
398 pAioMgr->cRequestsActiveMax -= PDMACEPFILEMGR_REQS_STEP;
399 }
400
401 pAioMgr->enmState = PDMACEPFILEMGRSTATE_RUNNING;
402 LogFlowFunc(("returns rc=%Rrc\n", rc));
403
404 return rc;
405}
406
407/**
408 * Error handler which will create the failsafe managers and destroy the failed I/O manager.
409 *
410 * @returns VBox status code
411 * @param pAioMgr The I/O manager the error ocurred on.
412 * @param rc The error code.
413 */
414static int pdmacFileAioMgrNormalErrorHandler(PPDMACEPFILEMGR pAioMgr, int rc, RT_SRC_POS_DECL)
415{
416 LogRel(("AIOMgr: I/O manager %#p encountered a critical error (rc=%Rrc) during operation. Falling back to failsafe mode. Expect reduced performance\n",
417 pAioMgr, rc));
418 LogRel(("AIOMgr: Error happened in %s:(%u){%s}\n", RT_SRC_POS_ARGS));
419 LogRel(("AIOMgr: Please contact the product vendor\n"));
420
421 PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pAioMgr->pEndpointsHead->Core.pEpClass;
422
423 pAioMgr->enmState = PDMACEPFILEMGRSTATE_FAULT;
424 ASMAtomicWriteU32((volatile uint32_t *)&pEpClassFile->enmMgrTypeOverride, PDMACEPFILEMGRTYPE_SIMPLE);
425
426 AssertMsgFailed(("Implement\n"));
427 return VINF_SUCCESS;
428}
429
430/**
431 * Put a list of tasks in the pending request list of an endpoint.
432 */
433DECLINLINE(void) pdmacFileAioMgrEpAddTaskList(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMACTASKFILE pTaskHead)
434{
435 /* Add the rest of the tasks to the pending list */
436 if (!pEndpoint->AioMgr.pReqsPendingHead)
437 {
438 Assert(!pEndpoint->AioMgr.pReqsPendingTail);
439 pEndpoint->AioMgr.pReqsPendingHead = pTaskHead;
440 }
441 else
442 {
443 Assert(pEndpoint->AioMgr.pReqsPendingTail);
444 pEndpoint->AioMgr.pReqsPendingTail->pNext = pTaskHead;
445 }
446
447 /* Update the tail. */
448 while (pTaskHead->pNext)
449 pTaskHead = pTaskHead->pNext;
450
451 pEndpoint->AioMgr.pReqsPendingTail = pTaskHead;
452 pTaskHead->pNext = NULL;
453}
454
455/**
456 * Put one task in the pending request list of an endpoint.
457 */
458DECLINLINE(void) pdmacFileAioMgrEpAddTask(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMACTASKFILE pTask)
459{
460 /* Add the rest of the tasks to the pending list */
461 if (!pEndpoint->AioMgr.pReqsPendingHead)
462 {
463 Assert(!pEndpoint->AioMgr.pReqsPendingTail);
464 pEndpoint->AioMgr.pReqsPendingHead = pTask;
465 }
466 else
467 {
468 Assert(pEndpoint->AioMgr.pReqsPendingTail);
469 pEndpoint->AioMgr.pReqsPendingTail->pNext = pTask;
470 }
471
472 pEndpoint->AioMgr.pReqsPendingTail = pTask;
473 pTask->pNext = NULL;
474}
475
476/**
477 * Allocates a async I/O request.
478 *
479 * @returns Handle to the request.
480 * @param pAioMgr The I/O manager.
481 */
482static RTFILEAIOREQ pdmacFileAioMgrNormalRequestAlloc(PPDMACEPFILEMGR pAioMgr)
483{
484 RTFILEAIOREQ hReq = NIL_RTFILEAIOREQ;
485
486 /* Get a request handle. */
487 if (pAioMgr->iFreeEntry > 0)
488 {
489 pAioMgr->iFreeEntry--;
490 hReq = pAioMgr->pahReqsFree[pAioMgr->iFreeEntry];
491 pAioMgr->pahReqsFree[pAioMgr->iFreeEntry] = NIL_RTFILEAIOREQ;
492 Assert(hReq != NIL_RTFILEAIOREQ);
493 }
494 else
495 {
496 int rc = RTFileAioReqCreate(&hReq);
497 AssertRC(rc);
498 }
499
500 return hReq;
501}
502
503/**
504 * Frees a async I/O request handle.
505 *
506 * @returns nothing.
507 * @param pAioMgr The I/O manager.
508 * @param hReq The I/O request handle to free.
509 */
510static void pdmacFileAioMgrNormalRequestFree(PPDMACEPFILEMGR pAioMgr, RTFILEAIOREQ hReq)
511{
512 Assert(pAioMgr->iFreeEntry < pAioMgr->cReqEntries);
513 Assert(pAioMgr->pahReqsFree[pAioMgr->iFreeEntry] == NIL_RTFILEAIOREQ);
514
515 pAioMgr->pahReqsFree[pAioMgr->iFreeEntry] = hReq;
516 pAioMgr->iFreeEntry++;
517}
518
519/**
520 * Wrapper around RTFIleAioCtxSubmit() which is also doing error handling.
521 */
522static int pdmacFileAioMgrNormalReqsEnqueue(PPDMACEPFILEMGR pAioMgr,
523 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
524 PRTFILEAIOREQ pahReqs, unsigned cReqs)
525{
526 int rc;
527
528 pAioMgr->cRequestsActive += cReqs;
529 pEndpoint->AioMgr.cRequestsActive += cReqs;
530
531 LogFlow(("Enqueuing %d requests. I/O manager has a total of %d active requests now\n", cReqs, pAioMgr->cRequestsActive));
532 LogFlow(("Endpoint has a total of %d active requests now\n", pEndpoint->AioMgr.cRequestsActive));
533
534 rc = RTFileAioCtxSubmit(pAioMgr->hAioCtx, pahReqs, cReqs);
535 if (RT_FAILURE(rc))
536 {
537 PPDMASYNCCOMPLETIONEPCLASSFILE pEpClass = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;
538 unsigned cReqsResubmit = 0;
539 RTFILEAIOREQ ahReqsResubmit[20];
540
541 /*
542 * We run out of resources.
543 * Need to check which requests got queued
544 * and put the rest on the pending list again.
545 */
546 for (size_t i = 0; i < cReqs; i++)
547 {
548 int rcReq = RTFileAioReqGetRC(pahReqs[i], NULL);
549
550 if (rcReq != VERR_FILE_AIO_IN_PROGRESS)
551 {
552 PPDMACTASKFILE pTask = (PPDMACTASKFILE)RTFileAioReqGetUser(pahReqs[i]);
553
554 if (pTask->enmTransferType == PDMACTASKFILETRANSFER_FLUSH)
555 {
556 /* Mark as not supported. */
557 if (rcReq != VERR_FILE_AIO_NOT_SUBMITTED)
558 {
559 LogFlow(("Async flushes are not supported for this endpoint, disabling\n"));
560 pEndpoint->fAsyncFlushSupported = false;
561 pdmacFileAioMgrNormalRequestFree(pAioMgr, pahReqs[i]);
562 rc = VINF_SUCCESS;
563 }
564 else
565 {
566 AssertMsg(rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES, ("Flush wasn't submitted but we are not out of ressources\n"));
567 /* Clear the pending flush */
568 pdmacFileAioMgrEpAddTask(pEndpoint, pTask);
569 Assert(pEndpoint->pFlushReq == pTask);
570 pEndpoint->pFlushReq = NULL;
571 }
572 }
573 else
574 {
575 AssertMsg(rcReq == VERR_FILE_AIO_NOT_SUBMITTED,
576 ("Request returned unexpected return code: rc=%Rrc\n", rcReq));
577
578 if (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES)
579 {
580 PPDMACTASKFILE pTasksWaiting;
581
582 pdmacFileAioMgrNormalRequestFree(pAioMgr, pahReqs[i]);
583
584 if (pTask->cbBounceBuffer)
585 RTMemPageFree(pTask->pvBounceBuffer, pTask->cbBounceBuffer);
586
587 pTask->fPrefetch = false;
588 pTask->cbBounceBuffer = 0;
589
590 /* Free the lock and process pending tasks if neccessary */
591 pTasksWaiting = pdmacFileAioMgrNormalRangeLockFree(pAioMgr, pEndpoint, pTask->pRangeLock);
592
593 pdmacFileAioMgrEpAddTask(pEndpoint, pTask);
594 if (pTasksWaiting)
595 pdmacFileAioMgrEpAddTaskList(pEndpoint, pTasksWaiting);
596 }
597 else
598 {
599 ahReqsResubmit[cReqsResubmit] = pahReqs[i];
600 cReqsResubmit++;
601 }
602 }
603
604 pEndpoint->AioMgr.cRequestsActive--;
605 pAioMgr->cRequestsActive--;
606
607 if (cReqsResubmit == RT_ELEMENTS(ahReqsResubmit))
608 {
609 int rc2 = RTFileAioCtxSubmit(pAioMgr->hAioCtx, ahReqsResubmit, cReqsResubmit);
610 AssertRC(rc2);
611 cReqsResubmit = 0;
612 }
613 }
614
615 /* Resubmit tasks. */
616 if (cReqsResubmit)
617 {
618 int rc2 = RTFileAioCtxSubmit(pAioMgr->hAioCtx, ahReqsResubmit, cReqsResubmit);
619 AssertRC(rc2);
620 cReqsResubmit = 0;
621 }
622 else if ( pEndpoint->pFlushReq
623 && !pAioMgr->cRequestsActive
624 && !pEndpoint->fAsyncFlushSupported)
625 {
626 /*
627 * Complete a pending flush if we don't have requests enqueued and the host doesn't support
628 * the async flush API.
629 * Happens only if this we just noticed that this is not supported
630 * and the only active request was a flush.
631 */
632 PPDMACTASKFILE pFlush = pEndpoint->pFlushReq;
633 pEndpoint->pFlushReq = NULL;
634 pFlush->pfnCompleted(pFlush, pFlush->pvUser, VINF_SUCCESS);
635 pdmacFileTaskFree(pEndpoint, pFlush);
636 }
637 }
638
639 if (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES)
640 {
641 pAioMgr->cRequestsActiveMax = pAioMgr->cRequestsActive;
642
643 /* Print an entry in the release log */
644 if (RT_UNLIKELY(!pEpClass->fOutOfResourcesWarningPrinted))
645 {
646 pEpClass->fOutOfResourcesWarningPrinted = true;
647 LogRel(("AIOMgr: Host limits number of active IO requests to %u. Expect a performance impact.\n",
648 pAioMgr->cRequestsActive));
649 }
650 }
651
652 LogFlow(("Removed requests. I/O manager has a total of %u active requests now\n", pAioMgr->cRequestsActive));
653 LogFlow(("Endpoint has a total of %u active requests now\n", pEndpoint->AioMgr.cRequestsActive));
654 }
655
656 return rc;
657}
658
659static bool pdmacFileAioMgrNormalIsRangeLocked(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
660 RTFOFF offStart, size_t cbRange,
661 PPDMACTASKFILE pTask)
662{
663 PPDMACFILERANGELOCK pRangeLock = NULL; /** < Range lock */
664
665 AssertMsg( pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE
666 || pTask->enmTransferType == PDMACTASKFILETRANSFER_READ,
667 ("Invalid task type %d\n", pTask->enmTransferType));
668
669 pRangeLock = (PPDMACFILERANGELOCK)RTAvlrFileOffsetRangeGet(pEndpoint->AioMgr.pTreeRangesLocked, offStart);
670 if (!pRangeLock)
671 {
672 pRangeLock = (PPDMACFILERANGELOCK)RTAvlrFileOffsetGetBestFit(pEndpoint->AioMgr.pTreeRangesLocked, offStart, true);
673 /* Check if we intersect with the range. */
674 if ( !pRangeLock
675 || !( (pRangeLock->Core.Key) <= (offStart + (RTFOFF)cbRange - 1)
676 && (pRangeLock->Core.KeyLast) >= offStart))
677 {
678 pRangeLock = NULL; /* False alarm */
679 }
680 }
681
682 /* Check whether we have one of the situations explained below */
683 if ( pRangeLock
684#if 0 /** @todo: later. For now we will just block all requests if they interfere */
685 && ( (pRangeLock->fReadLock && pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE)
686 || (!pRangeLock->fReadLock)
687#endif
688 )
689 {
690 /* Add to the list. */
691 pTask->pNext = NULL;
692
693 if (!pRangeLock->pWaitingTasksHead)
694 {
695 Assert(!pRangeLock->pWaitingTasksTail);
696 pRangeLock->pWaitingTasksHead = pTask;
697 pRangeLock->pWaitingTasksTail = pTask;
698 }
699 else
700 {
701 AssertPtr(pRangeLock->pWaitingTasksTail);
702 pRangeLock->pWaitingTasksTail->pNext = pTask;
703 pRangeLock->pWaitingTasksTail = pTask;
704 }
705 return true;
706 }
707
708 return false;
709}
710
711static int pdmacFileAioMgrNormalRangeLock(PPDMACEPFILEMGR pAioMgr,
712 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
713 RTFOFF offStart, size_t cbRange,
714 PPDMACTASKFILE pTask)
715{
716 AssertMsg(!pdmacFileAioMgrNormalIsRangeLocked(pEndpoint, offStart, cbRange, pTask),
717 ("Range is already locked offStart=%RTfoff cbRange=%u\n",
718 offStart, cbRange));
719
720 PPDMACFILERANGELOCK pRangeLock = (PPDMACFILERANGELOCK)RTMemCacheAlloc(pAioMgr->hMemCacheRangeLocks);
721 if (!pRangeLock)
722 return VERR_NO_MEMORY;
723
724 /* Init the lock. */
725 pRangeLock->Core.Key = offStart;
726 pRangeLock->Core.KeyLast = offStart + cbRange - 1;
727 pRangeLock->cRefs = 1;
728 pRangeLock->fReadLock = pTask->enmTransferType == PDMACTASKFILETRANSFER_READ;
729 pRangeLock->pWaitingTasksHead = NULL;
730 pRangeLock->pWaitingTasksTail = NULL;
731
732 bool fInserted = RTAvlrFileOffsetInsert(pEndpoint->AioMgr.pTreeRangesLocked, &pRangeLock->Core);
733 AssertMsg(fInserted, ("Range lock was not inserted!\n"));
734
735 /* Let the task point to its lock. */
736 pTask->pRangeLock = pRangeLock;
737
738 return VINF_SUCCESS;
739}
740
741static PPDMACTASKFILE pdmacFileAioMgrNormalRangeLockFree(PPDMACEPFILEMGR pAioMgr,
742 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
743 PPDMACFILERANGELOCK pRangeLock)
744{
745 PPDMACTASKFILE pTasksWaitingHead;
746
747 AssertPtr(pRangeLock);
748 Assert(pRangeLock->cRefs == 1);
749
750 RTAvlrFileOffsetRemove(pEndpoint->AioMgr.pTreeRangesLocked, pRangeLock->Core.Key);
751 pTasksWaitingHead = pRangeLock->pWaitingTasksHead;
752 pRangeLock->pWaitingTasksHead = NULL;
753 pRangeLock->pWaitingTasksTail = NULL;
754 RTMemCacheFree(pAioMgr->hMemCacheRangeLocks, pRangeLock);
755
756 return pTasksWaitingHead;
757}
758
759static int pdmacFileAioMgrNormalTaskPrepareBuffered(PPDMACEPFILEMGR pAioMgr,
760 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
761 PPDMACTASKFILE pTask, PRTFILEAIOREQ phReq)
762{
763 int rc = VINF_SUCCESS;
764 RTFILEAIOREQ hReq = NIL_RTFILEAIOREQ;
765 PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;
766 void *pvBuf = pTask->DataSeg.pvSeg;
767
768 AssertMsg( pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE
769 || (uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) <= pEndpoint->cbFile,
770 ("Read exceeds file size offStart=%RTfoff cbToTransfer=%d cbFile=%llu\n",
771 pTask->Off, pTask->DataSeg.cbSeg, pEndpoint->cbFile));
772
773 pTask->fPrefetch = false;
774 pTask->cbBounceBuffer = 0;
775
776 /*
777 * Before we start to setup the request we have to check whether there is a task
778 * already active which range intersects with ours. We have to defer execution
779 * of this task in two cases:
780 * - The pending task is a write and the current is either read or write
781 * - The pending task is a read and the current task is a write task.
782 *
783 * To check whether a range is currently "locked" we use the AVL tree where every pending task
784 * is stored by its file offset range. The current task will be added to the active task
785 * and will be executed when the active one completes. (The method below
786 * which checks whether a range is already used will add the task)
787 *
788 * This is neccessary because of the requirement to align all requests to a 512 boundary
789 * which is enforced by the host OS (Linux and Windows atm). It is possible that
790 * we have to process unaligned tasks and need to align them using bounce buffers.
791 * While the data is fetched from the file another request might arrive writing to
792 * the same range. This will result in data corruption if both are executed concurrently.
793 */
794 bool fLocked = pdmacFileAioMgrNormalIsRangeLocked(pEndpoint, pTask->Off, pTask->DataSeg.cbSeg, pTask);
795
796 if (!fLocked)
797 {
798 /* Get a request handle. */
799 hReq = pdmacFileAioMgrNormalRequestAlloc(pAioMgr);
800 AssertMsg(hReq != NIL_RTFILEAIOREQ, ("Out of request handles\n"));
801
802 if (pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE)
803 {
804 /* Grow the file if needed. */
805 if (RT_UNLIKELY((uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) > pEndpoint->cbFile))
806 {
807 ASMAtomicWriteU64(&pEndpoint->cbFile, pTask->Off + pTask->DataSeg.cbSeg);
808 RTFileSetSize(pEndpoint->File, pTask->Off + pTask->DataSeg.cbSeg);
809 }
810
811 rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->File,
812 pTask->Off, pTask->DataSeg.pvSeg,
813 pTask->DataSeg.cbSeg, pTask);
814 }
815 else
816 rc = RTFileAioReqPrepareRead(hReq, pEndpoint->File,
817 pTask->Off, pTask->DataSeg.pvSeg,
818 pTask->DataSeg.cbSeg, pTask);
819 AssertRC(rc);
820
821 rc = pdmacFileAioMgrNormalRangeLock(pAioMgr, pEndpoint, pTask->Off,
822 pTask->DataSeg.cbSeg,
823 pTask);
824
825 if (RT_SUCCESS(rc))
826 *phReq = hReq;
827 }
828 else
829 LogFlow(("Task %#p was deferred because the access range is locked\n", pTask));
830
831 return rc;
832}
833
834static int pdmacFileAioMgrNormalTaskPrepareNonBuffered(PPDMACEPFILEMGR pAioMgr,
835 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
836 PPDMACTASKFILE pTask, PRTFILEAIOREQ phReq)
837{
838 int rc = VINF_SUCCESS;
839 RTFILEAIOREQ hReq = NIL_RTFILEAIOREQ;
840 PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;
841 void *pvBuf = pTask->DataSeg.pvSeg;
842
843 /*
844 * Check if the alignment requirements are met.
845 * Offset, transfer size and buffer address
846 * need to be on a 512 boundary.
847 */
848 RTFOFF offStart = pTask->Off & ~(RTFOFF)(512-1);
849 size_t cbToTransfer = RT_ALIGN_Z(pTask->DataSeg.cbSeg + (pTask->Off - offStart), 512);
850 PDMACTASKFILETRANSFER enmTransferType = pTask->enmTransferType;
851
852 AssertMsg( pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE
853 || (uint64_t)(offStart + cbToTransfer) <= pEndpoint->cbFile,
854 ("Read exceeds file size offStart=%RTfoff cbToTransfer=%d cbFile=%llu\n",
855 offStart, cbToTransfer, pEndpoint->cbFile));
856
857 pTask->fPrefetch = false;
858
859 /*
860 * Before we start to setup the request we have to check whether there is a task
861 * already active which range intersects with ours. We have to defer execution
862 * of this task in two cases:
863 * - The pending task is a write and the current is either read or write
864 * - The pending task is a read and the current task is a write task.
865 *
866 * To check whether a range is currently "locked" we use the AVL tree where every pending task
867 * is stored by its file offset range. The current task will be added to the active task
868 * and will be executed when the active one completes. (The method below
869 * which checks whether a range is already used will add the task)
870 *
871 * This is neccessary because of the requirement to align all requests to a 512 boundary
872 * which is enforced by the host OS (Linux and Windows atm). It is possible that
873 * we have to process unaligned tasks and need to align them using bounce buffers.
874 * While the data is fetched from the file another request might arrive writing to
875 * the same range. This will result in data corruption if both are executed concurrently.
876 */
877 bool fLocked = pdmacFileAioMgrNormalIsRangeLocked(pEndpoint, offStart, cbToTransfer, pTask);
878
879 if (!fLocked)
880 {
881 /* Get a request handle. */
882 hReq = pdmacFileAioMgrNormalRequestAlloc(pAioMgr);
883 AssertMsg(hReq != NIL_RTFILEAIOREQ, ("Out of request handles\n"));
884
885 if ( RT_UNLIKELY(cbToTransfer != pTask->DataSeg.cbSeg)
886 || RT_UNLIKELY(offStart != pTask->Off)
887 || ((pEpClassFile->uBitmaskAlignment & (RTR3UINTPTR)pvBuf) != (RTR3UINTPTR)pvBuf))
888 {
889 LogFlow(("Using bounce buffer for task %#p cbToTransfer=%zd cbSeg=%zd offStart=%RTfoff off=%RTfoff\n",
890 pTask, cbToTransfer, pTask->DataSeg.cbSeg, offStart, pTask->Off));
891
892 /* Create bounce buffer. */
893 pTask->cbBounceBuffer = cbToTransfer;
894
895 AssertMsg(pTask->Off >= offStart, ("Overflow in calculation Off=%llu offStart=%llu\n",
896 pTask->Off, offStart));
897 pTask->offBounceBuffer = pTask->Off - offStart;
898
899 /** @todo: I think we need something like a RTMemAllocAligned method here.
900 * Current assumption is that the maximum alignment is 4096byte
901 * (GPT disk on Windows)
902 * so we can use RTMemPageAlloc here.
903 */
904 pTask->pvBounceBuffer = RTMemPageAlloc(cbToTransfer);
905 if (RT_LIKELY(pTask->pvBounceBuffer))
906 {
907 pvBuf = pTask->pvBounceBuffer;
908
909 if (pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE)
910 {
911 if ( RT_UNLIKELY(cbToTransfer != pTask->DataSeg.cbSeg)
912 || RT_UNLIKELY(offStart != pTask->Off))
913 {
914 /* We have to fill the buffer first before we can update the data. */
915 LogFlow(("Prefetching data for task %#p\n", pTask));
916 pTask->fPrefetch = true;
917 enmTransferType = PDMACTASKFILETRANSFER_READ;
918 }
919 else
920 memcpy(pvBuf, pTask->DataSeg.pvSeg, pTask->DataSeg.cbSeg);
921 }
922 }
923 else
924 rc = VERR_NO_MEMORY;
925 }
926 else
927 pTask->cbBounceBuffer = 0;
928
929 if (RT_SUCCESS(rc))
930 {
931 AssertMsg((pEpClassFile->uBitmaskAlignment & (RTR3UINTPTR)pvBuf) == (RTR3UINTPTR)pvBuf,
932 ("AIO: Alignment restrictions not met! pvBuf=%p uBitmaskAlignment=%p\n", pvBuf, pEpClassFile->uBitmaskAlignment));
933
934 if (enmTransferType == PDMACTASKFILETRANSFER_WRITE)
935 {
936 /* Grow the file if needed. */
937 if (RT_UNLIKELY((uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) > pEndpoint->cbFile))
938 {
939 ASMAtomicWriteU64(&pEndpoint->cbFile, pTask->Off + pTask->DataSeg.cbSeg);
940 RTFileSetSize(pEndpoint->File, pTask->Off + pTask->DataSeg.cbSeg);
941 }
942
943 rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->File,
944 offStart, pvBuf, cbToTransfer, pTask);
945 }
946 else
947 rc = RTFileAioReqPrepareRead(hReq, pEndpoint->File,
948 offStart, pvBuf, cbToTransfer, pTask);
949 AssertRC(rc);
950
951 rc = pdmacFileAioMgrNormalRangeLock(pAioMgr, pEndpoint, offStart, cbToTransfer, pTask);
952
953 if (RT_SUCCESS(rc))
954 *phReq = hReq;
955 else
956 {
957 /* Cleanup */
958 if (pTask->cbBounceBuffer)
959 RTMemPageFree(pTask->pvBounceBuffer, pTask->cbBounceBuffer);
960 }
961 }
962 }
963 else
964 LogFlow(("Task %#p was deferred because the access range is locked\n", pTask));
965
966 return rc;
967}
968
969static int pdmacFileAioMgrNormalProcessTaskList(PPDMACTASKFILE pTaskHead,
970 PPDMACEPFILEMGR pAioMgr,
971 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
972{
973 RTFILEAIOREQ apReqs[20];
974 unsigned cRequests = 0;
975 unsigned cMaxRequests = pAioMgr->cRequestsActiveMax - pAioMgr->cRequestsActive;
976 int rc = VINF_SUCCESS;
977
978 AssertMsg(pEndpoint->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE,
979 ("Trying to process request lists of a non active endpoint!\n"));
980
981 /* Go through the list and queue the requests until we get a flush request */
982 while ( pTaskHead
983 && !pEndpoint->pFlushReq
984 && (pAioMgr->cRequestsActive + cRequests < pAioMgr->cRequestsActiveMax)
985 && RT_SUCCESS(rc))
986 {
987 PPDMACTASKFILE pCurr = pTaskHead;
988
989 if (!pdmacFileBwMgrIsTransferAllowed(pEndpoint->pBwMgr, (uint32_t)pCurr->DataSeg.cbSeg))
990 {
991 pAioMgr->fBwLimitReached = true;
992 break;
993 }
994
995 pTaskHead = pTaskHead->pNext;
996
997 pCurr->pNext = NULL;
998
999 AssertMsg(VALID_PTR(pCurr->pEndpoint) && (pCurr->pEndpoint == pEndpoint),
1000 ("Endpoints do not match\n"));
1001
1002 switch (pCurr->enmTransferType)
1003 {
1004 case PDMACTASKFILETRANSFER_FLUSH:
1005 {
1006 /* If there is no data transfer request this flush request finished immediately. */
1007 if (pEndpoint->fAsyncFlushSupported)
1008 {
1009 /* Issue a flush to the host. */
1010 RTFILEAIOREQ hReq = pdmacFileAioMgrNormalRequestAlloc(pAioMgr);
1011 AssertMsg(hReq != NIL_RTFILEAIOREQ, ("Out of request handles\n"));
1012
1013 rc = RTFileAioReqPrepareFlush(hReq, pEndpoint->File, pCurr);
1014 if (RT_FAILURE(rc))
1015 {
1016 pEndpoint->fAsyncFlushSupported = false;
1017 pdmacFileAioMgrNormalRequestFree(pAioMgr, hReq);
1018 }
1019 else
1020 {
1021 apReqs[cRequests] = hReq;
1022 pEndpoint->AioMgr.cReqsProcessed++;
1023 cRequests++;
1024 }
1025 }
1026
1027 if ( !pEndpoint->AioMgr.cRequestsActive
1028 && !pEndpoint->fAsyncFlushSupported)
1029 {
1030 pCurr->pfnCompleted(pCurr, pCurr->pvUser, VINF_SUCCESS);
1031 pdmacFileTaskFree(pEndpoint, pCurr);
1032 }
1033 else
1034 {
1035 Assert(!pEndpoint->pFlushReq);
1036 pEndpoint->pFlushReq = pCurr;
1037 }
1038 break;
1039 }
1040 case PDMACTASKFILETRANSFER_READ:
1041 case PDMACTASKFILETRANSFER_WRITE:
1042 {
1043 RTFILEAIOREQ hReq = NIL_RTFILEAIOREQ;
1044
1045 if (pEndpoint->enmBackendType == PDMACFILEEPBACKEND_BUFFERED)
1046 rc = pdmacFileAioMgrNormalTaskPrepareBuffered(pAioMgr, pEndpoint, pCurr, &hReq);
1047 else if (pEndpoint->enmBackendType == PDMACFILEEPBACKEND_NON_BUFFERED)
1048 rc = pdmacFileAioMgrNormalTaskPrepareNonBuffered(pAioMgr, pEndpoint, pCurr, &hReq);
1049 else
1050 AssertMsgFailed(("Invalid backend type %d\n", pEndpoint->enmBackendType));
1051
1052 AssertRC(rc);
1053
1054 if (hReq != NIL_RTFILEAIOREQ)
1055 {
1056 apReqs[cRequests] = hReq;
1057 pEndpoint->AioMgr.cReqsProcessed++;
1058 cRequests++;
1059 }
1060 break;
1061 }
1062 default:
1063 AssertMsgFailed(("Invalid transfer type %d\n", pCurr->enmTransferType));
1064 } /* switch transfer type */
1065
1066 /* Queue the requests if the array is full. */
1067 if (cRequests == RT_ELEMENTS(apReqs))
1068 {
1069 rc = pdmacFileAioMgrNormalReqsEnqueue(pAioMgr, pEndpoint, apReqs, cRequests);
1070 cRequests = 0;
1071 AssertMsg(RT_SUCCESS(rc) || (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES),
1072 ("Unexpected return code\n"));
1073 }
1074 }
1075
1076 if (cRequests)
1077 {
1078 rc = pdmacFileAioMgrNormalReqsEnqueue(pAioMgr, pEndpoint, apReqs, cRequests);
1079 AssertMsg(RT_SUCCESS(rc) || (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES),
1080 ("Unexpected return code rc=%Rrc\n", rc));
1081 }
1082
1083 if (pTaskHead)
1084 {
1085 /* Add the rest of the tasks to the pending list */
1086 pdmacFileAioMgrEpAddTaskList(pEndpoint, pTaskHead);
1087
1088 if (RT_UNLIKELY( pAioMgr->cRequestsActiveMax == pAioMgr->cRequestsActive
1089 && !pEndpoint->pFlushReq
1090 && !pAioMgr->fBwLimitReached))
1091 {
1092#if 0
1093 /*
1094 * The I/O manager has no room left for more requests
1095 * but there are still requests to process.
1096 * Create a new I/O manager and let it handle some endpoints.
1097 */
1098 pdmacFileAioMgrNormalBalanceLoad(pAioMgr);
1099#else
1100 /* Grow the I/O manager */
1101 pAioMgr->enmState = PDMACEPFILEMGRSTATE_GROWING;
1102#endif
1103 }
1104 }
1105
1106 /* Insufficient resources are not fatal. */
1107 if (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES)
1108 rc = VINF_SUCCESS;
1109
1110 return rc;
1111}
1112
1113/**
1114 * Adds all pending requests for the given endpoint
1115 * until a flush request is encountered or there is no
1116 * request anymore.
1117 *
1118 * @returns VBox status code.
1119 * @param pAioMgr The async I/O manager for the endpoint
1120 * @param pEndpoint The endpoint to get the requests from.
1121 */
1122static int pdmacFileAioMgrNormalQueueReqs(PPDMACEPFILEMGR pAioMgr,
1123 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
1124{
1125 int rc = VINF_SUCCESS;
1126 PPDMACTASKFILE pTasksHead = NULL;
1127
1128 AssertMsg(pEndpoint->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE,
1129 ("Trying to process request lists of a non active endpoint!\n"));
1130
1131 Assert(!pEndpoint->pFlushReq);
1132
1133 /* Check the pending list first */
1134 if (pEndpoint->AioMgr.pReqsPendingHead)
1135 {
1136 LogFlow(("Queuing pending requests first\n"));
1137
1138 pTasksHead = pEndpoint->AioMgr.pReqsPendingHead;
1139 /*
1140 * Clear the list as the processing routine will insert them into the list
1141 * again if it gets a flush request.
1142 */
1143 pEndpoint->AioMgr.pReqsPendingHead = NULL;
1144 pEndpoint->AioMgr.pReqsPendingTail = NULL;
1145 rc = pdmacFileAioMgrNormalProcessTaskList(pTasksHead, pAioMgr, pEndpoint);
1146 AssertRC(rc);
1147 }
1148
1149 if (!pEndpoint->pFlushReq && !pEndpoint->AioMgr.pReqsPendingHead)
1150 {
1151 /* Now the request queue. */
1152 pTasksHead = pdmacFileEpGetNewTasks(pEndpoint);
1153 if (pTasksHead)
1154 {
1155 rc = pdmacFileAioMgrNormalProcessTaskList(pTasksHead, pAioMgr, pEndpoint);
1156 AssertRC(rc);
1157 }
1158 }
1159
1160 return rc;
1161}
1162
1163static int pdmacFileAioMgrNormalProcessBlockingEvent(PPDMACEPFILEMGR pAioMgr)
1164{
1165 int rc = VINF_SUCCESS;
1166 bool fNotifyWaiter = false;
1167
1168 LogFlowFunc((": Enter\n"));
1169
1170 Assert(pAioMgr->fBlockingEventPending);
1171
1172 switch (pAioMgr->enmBlockingEvent)
1173 {
1174 case PDMACEPFILEAIOMGRBLOCKINGEVENT_ADD_ENDPOINT:
1175 {
1176 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointNew = (PPDMASYNCCOMPLETIONENDPOINTFILE)ASMAtomicReadPtr((void * volatile *)&pAioMgr->BlockingEventData.AddEndpoint.pEndpoint);
1177 AssertMsg(VALID_PTR(pEndpointNew), ("Adding endpoint event without a endpoint to add\n"));
1178
1179 pEndpointNew->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE;
1180
1181 pEndpointNew->AioMgr.pEndpointNext = pAioMgr->pEndpointsHead;
1182 pEndpointNew->AioMgr.pEndpointPrev = NULL;
1183 if (pAioMgr->pEndpointsHead)
1184 pAioMgr->pEndpointsHead->AioMgr.pEndpointPrev = pEndpointNew;
1185 pAioMgr->pEndpointsHead = pEndpointNew;
1186
1187 /* Assign the completion point to this file. */
1188 rc = RTFileAioCtxAssociateWithFile(pAioMgr->hAioCtx, pEndpointNew->File);
1189 fNotifyWaiter = true;
1190 pAioMgr->cEndpoints++;
1191 break;
1192 }
1193 case PDMACEPFILEAIOMGRBLOCKINGEVENT_REMOVE_ENDPOINT:
1194 {
1195 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointRemove = (PPDMASYNCCOMPLETIONENDPOINTFILE)ASMAtomicReadPtr((void * volatile *)&pAioMgr->BlockingEventData.RemoveEndpoint.pEndpoint);
1196 AssertMsg(VALID_PTR(pEndpointRemove), ("Removing endpoint event without a endpoint to remove\n"));
1197
1198 pEndpointRemove->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_REMOVING;
1199 fNotifyWaiter = !pdmacFileAioMgrNormalRemoveEndpoint(pEndpointRemove);
1200 break;
1201 }
1202 case PDMACEPFILEAIOMGRBLOCKINGEVENT_CLOSE_ENDPOINT:
1203 {
1204 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointClose = (PPDMASYNCCOMPLETIONENDPOINTFILE)ASMAtomicReadPtr((void * volatile *)&pAioMgr->BlockingEventData.CloseEndpoint.pEndpoint);
1205 AssertMsg(VALID_PTR(pEndpointClose), ("Close endpoint event without a endpoint to close\n"));
1206
1207 if (pEndpointClose->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE)
1208 {
1209 LogFlowFunc((": Closing endpoint %#p{%s}\n", pEndpointClose, pEndpointClose->Core.pszUri));
1210
1211 /* Make sure all tasks finished. Process the queues a last time first. */
1212 rc = pdmacFileAioMgrNormalQueueReqs(pAioMgr, pEndpointClose);
1213 AssertRC(rc);
1214
1215 pEndpointClose->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_CLOSING;
1216 fNotifyWaiter = !pdmacFileAioMgrNormalRemoveEndpoint(pEndpointClose);
1217 }
1218 else if ( (pEndpointClose->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_CLOSING)
1219 && (!pEndpointClose->AioMgr.cRequestsActive))
1220 fNotifyWaiter = true;
1221 break;
1222 }
1223 case PDMACEPFILEAIOMGRBLOCKINGEVENT_SHUTDOWN:
1224 {
1225 pAioMgr->enmState = PDMACEPFILEMGRSTATE_SHUTDOWN;
1226 if (!pAioMgr->cRequestsActive)
1227 fNotifyWaiter = true;
1228 break;
1229 }
1230 case PDMACEPFILEAIOMGRBLOCKINGEVENT_SUSPEND:
1231 {
1232 pAioMgr->enmState = PDMACEPFILEMGRSTATE_SUSPENDING;
1233 break;
1234 }
1235 case PDMACEPFILEAIOMGRBLOCKINGEVENT_RESUME:
1236 {
1237 pAioMgr->enmState = PDMACEPFILEMGRSTATE_RUNNING;
1238 fNotifyWaiter = true;
1239 break;
1240 }
1241 default:
1242 AssertReleaseMsgFailed(("Invalid event type %d\n", pAioMgr->enmBlockingEvent));
1243 }
1244
1245 if (fNotifyWaiter)
1246 {
1247 ASMAtomicWriteBool(&pAioMgr->fBlockingEventPending, false);
1248 pAioMgr->enmBlockingEvent = PDMACEPFILEAIOMGRBLOCKINGEVENT_INVALID;
1249
1250 /* Release the waiting thread. */
1251 LogFlow(("Signalling waiter\n"));
1252 rc = RTSemEventSignal(pAioMgr->EventSemBlock);
1253 AssertRC(rc);
1254 }
1255
1256 LogFlowFunc((": Leave\n"));
1257 return rc;
1258}
1259
1260/**
1261 * Checks all endpoints for pending events or new requests.
1262 *
1263 * @returns VBox status code.
1264 * @param pAioMgr The I/O manager handle.
1265 */
1266static int pdmacFileAioMgrNormalCheckEndpoints(PPDMACEPFILEMGR pAioMgr)
1267{
1268 /* Check the assigned endpoints for new tasks if there isn't a flush request active at the moment. */
1269 int rc = VINF_SUCCESS;
1270 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint = pAioMgr->pEndpointsHead;
1271
1272 pAioMgr->fBwLimitReached = false;
1273
1274 while (pEndpoint)
1275 {
1276 if (!pEndpoint->pFlushReq
1277 && (pEndpoint->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE)
1278 && !pEndpoint->AioMgr.fMoving)
1279 {
1280 rc = pdmacFileAioMgrNormalQueueReqs(pAioMgr, pEndpoint);
1281 if (RT_FAILURE(rc))
1282 return rc;
1283 }
1284 else if ( !pEndpoint->AioMgr.cRequestsActive
1285 && pEndpoint->enmState != PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE)
1286 {
1287 /* Reopen the file so that the new endpoint can reassociate with the file */
1288 RTFileClose(pEndpoint->File);
1289 rc = RTFileOpen(&pEndpoint->File, pEndpoint->Core.pszUri, pEndpoint->fFlags);
1290 AssertRC(rc);
1291
1292 if (pEndpoint->AioMgr.fMoving)
1293 {
1294 pEndpoint->AioMgr.fMoving = false;
1295 pdmacFileAioMgrAddEndpoint(pEndpoint->AioMgr.pAioMgrDst, pEndpoint);
1296 }
1297 else
1298 {
1299 Assert(pAioMgr->fBlockingEventPending);
1300 ASMAtomicWriteBool(&pAioMgr->fBlockingEventPending, false);
1301
1302 /* Release the waiting thread. */
1303 LogFlow(("Signalling waiter\n"));
1304 rc = RTSemEventSignal(pAioMgr->EventSemBlock);
1305 AssertRC(rc);
1306 }
1307 }
1308
1309 pEndpoint = pEndpoint->AioMgr.pEndpointNext;
1310 }
1311
1312 return rc;
1313}
1314
1315static void pdmacFileAioMgrNormalReqComplete(PPDMACEPFILEMGR pAioMgr, RTFILEAIOREQ hReq)
1316{
1317 int rc = VINF_SUCCESS;
1318 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint;
1319 size_t cbTransfered = 0;
1320 int rcReq = RTFileAioReqGetRC(hReq, &cbTransfered);
1321 PPDMACTASKFILE pTask = (PPDMACTASKFILE)RTFileAioReqGetUser(hReq);
1322 PPDMACTASKFILE pTasksWaiting;
1323
1324 pEndpoint = pTask->pEndpoint;
1325
1326 /*
1327 * It is possible that the request failed on Linux with kernels < 2.6.23
1328 * if the passed buffer was allocated with remap_pfn_range or if the file
1329 * is on an NFS endpoint which does not support async and direct I/O at the same time.
1330 * The endpoint will be migrated to a failsafe manager in case a request fails.
1331 */
1332 if (RT_FAILURE(rcReq))
1333 {
1334 /* Free bounce buffers and the IPRT request. */
1335 pdmacFileAioMgrNormalRequestFree(pAioMgr, hReq);
1336
1337 pAioMgr->cRequestsActive--;
1338 pEndpoint->AioMgr.cRequestsActive--;
1339 pEndpoint->AioMgr.cReqsProcessed++;
1340
1341 if (pTask->enmTransferType == PDMACTASKFILETRANSFER_FLUSH)
1342 {
1343 LogFlow(("Async flushes are not supported for this endpoint, disabling\n"));
1344 pEndpoint->fAsyncFlushSupported = false;
1345 AssertMsg(pEndpoint->pFlushReq == pTask, ("Failed flush request doesn't match active one\n"));
1346 /* The other method will take over now. */
1347 }
1348 else
1349 {
1350 /* Free the lock and process pending tasks if neccessary */
1351 pTasksWaiting = pdmacFileAioMgrNormalRangeLockFree(pAioMgr, pEndpoint, pTask->pRangeLock);
1352 rc = pdmacFileAioMgrNormalProcessTaskList(pTasksWaiting, pAioMgr, pEndpoint);
1353 AssertRC(rc);
1354
1355 if (pTask->cbBounceBuffer)
1356 RTMemPageFree(pTask->pvBounceBuffer, pTask->cbBounceBuffer);
1357
1358 /* Queue the request on the pending list. */
1359 pTask->pNext = pEndpoint->AioMgr.pReqsPendingHead;
1360 pEndpoint->AioMgr.pReqsPendingHead = pTask;
1361
1362 /* Create a new failsafe manager if neccessary. */
1363 if (!pEndpoint->AioMgr.fMoving)
1364 {
1365 PPDMACEPFILEMGR pAioMgrFailsafe;
1366
1367 LogRel(("%s: Request %#p failed with rc=%Rrc, migrating endpoint %s to failsafe manager.\n",
1368 RTThreadGetName(pAioMgr->Thread), pTask, rcReq, pEndpoint->Core.pszUri));
1369
1370 pEndpoint->AioMgr.fMoving = true;
1371
1372 rc = pdmacFileAioMgrCreate((PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass,
1373 &pAioMgrFailsafe, PDMACEPFILEMGRTYPE_SIMPLE);
1374 AssertRC(rc);
1375
1376 pEndpoint->AioMgr.pAioMgrDst = pAioMgrFailsafe;
1377
1378 /* Update the flags to open the file with. Disable async I/O and enable the host cache. */
1379 pEndpoint->fFlags &= ~(RTFILE_O_ASYNC_IO | RTFILE_O_NO_CACHE);
1380 }
1381 }
1382
1383 /* If this was the last request for the endpoint migrate it to the new manager. */
1384 if (!pEndpoint->AioMgr.cRequestsActive)
1385 {
1386 bool fReqsPending = pdmacFileAioMgrNormalRemoveEndpoint(pEndpoint);
1387 Assert(!fReqsPending);
1388
1389 rc = pdmacFileAioMgrAddEndpoint(pEndpoint->AioMgr.pAioMgrDst, pEndpoint);
1390 AssertRC(rc);
1391 }
1392 }
1393 else
1394 {
1395 if (pTask->enmTransferType == PDMACTASKFILETRANSFER_FLUSH)
1396 {
1397 /* Clear pending flush */
1398 AssertMsg(pEndpoint->pFlushReq == pTask, ("Completed flush request doesn't match active one\n"));
1399 pEndpoint->pFlushReq = NULL;
1400 pdmacFileAioMgrNormalRequestFree(pAioMgr, hReq);
1401
1402 pAioMgr->cRequestsActive--;
1403 pEndpoint->AioMgr.cRequestsActive--;
1404 pEndpoint->AioMgr.cReqsProcessed++;
1405
1406 /* Call completion callback */
1407 LogFlow(("Flush task=%#p completed with %Rrc\n", pTask, rcReq));
1408 pTask->pfnCompleted(pTask, pTask->pvUser, rcReq);
1409 pdmacFileTaskFree(pEndpoint, pTask);
1410 }
1411 else
1412 {
1413 /*
1414 * Restart an incomplete transfer.
1415 * This usually means that the request will return an error now
1416 * but to get the cause of the error (disk full, file too big, I/O error, ...)
1417 * the transfer needs to be continued.
1418 */
1419 if (RT_UNLIKELY( cbTransfered < pTask->DataSeg.cbSeg
1420 || ( pTask->cbBounceBuffer
1421 && cbTransfered < pTask->cbBounceBuffer)))
1422 {
1423 RTFOFF offStart;
1424 size_t cbToTransfer;
1425 uint8_t *pbBuf = NULL;
1426
1427 LogFlow(("Restarting incomplete transfer %#p (%zu bytes transfered)\n",
1428 pTask, cbTransfered));
1429 Assert(cbTransfered % 512 == 0);
1430
1431 if (pTask->cbBounceBuffer)
1432 {
1433 AssertPtr(pTask->pvBounceBuffer);
1434 offStart = (pTask->Off & ~((RTFOFF)512-1)) + cbTransfered;
1435 cbToTransfer = pTask->cbBounceBuffer - cbTransfered;
1436 pbBuf = (uint8_t *)pTask->pvBounceBuffer + cbTransfered;
1437 }
1438 else
1439 {
1440 Assert(!pTask->pvBounceBuffer);
1441 offStart = pTask->Off + cbTransfered;
1442 cbToTransfer = pTask->DataSeg.cbSeg - cbTransfered;
1443 pbBuf = (uint8_t *)pTask->DataSeg.pvSeg + cbTransfered;
1444 }
1445
1446 if (pTask->fPrefetch || pTask->enmTransferType == PDMACTASKFILETRANSFER_READ)
1447 {
1448 rc = RTFileAioReqPrepareRead(hReq, pEndpoint->File, offStart,
1449 pbBuf, cbToTransfer, pTask);
1450 }
1451 else
1452 {
1453 AssertMsg(pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE,
1454 ("Invalid transfer type\n"));
1455 rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->File, offStart,
1456 pbBuf, cbToTransfer, pTask);
1457 }
1458
1459 AssertRC(rc);
1460 rc = RTFileAioCtxSubmit(pAioMgr->hAioCtx, &hReq, 1);
1461 AssertRC(rc);
1462 }
1463 else if (pTask->fPrefetch)
1464 {
1465 Assert(pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE);
1466 Assert(pTask->cbBounceBuffer);
1467
1468 memcpy(((uint8_t *)pTask->pvBounceBuffer) + pTask->offBounceBuffer,
1469 pTask->DataSeg.pvSeg,
1470 pTask->DataSeg.cbSeg);
1471
1472 /* Write it now. */
1473 pTask->fPrefetch = false;
1474 size_t cbToTransfer = RT_ALIGN_Z(pTask->DataSeg.cbSeg, 512);
1475 RTFOFF offStart = pTask->Off & ~(RTFOFF)(512-1);
1476
1477 /* Grow the file if needed. */
1478 if (RT_UNLIKELY((uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) > pEndpoint->cbFile))
1479 {
1480 ASMAtomicWriteU64(&pEndpoint->cbFile, pTask->Off + pTask->DataSeg.cbSeg);
1481 RTFileSetSize(pEndpoint->File, pTask->Off + pTask->DataSeg.cbSeg);
1482 }
1483
1484 rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->File,
1485 offStart, pTask->pvBounceBuffer, cbToTransfer, pTask);
1486 AssertRC(rc);
1487 rc = RTFileAioCtxSubmit(pAioMgr->hAioCtx, &hReq, 1);
1488 AssertRC(rc);
1489 }
1490 else
1491 {
1492 if (RT_SUCCESS(rc) && pTask->cbBounceBuffer)
1493 {
1494 if (pTask->enmTransferType == PDMACTASKFILETRANSFER_READ)
1495 memcpy(pTask->DataSeg.pvSeg,
1496 ((uint8_t *)pTask->pvBounceBuffer) + pTask->offBounceBuffer,
1497 pTask->DataSeg.cbSeg);
1498
1499 RTMemPageFree(pTask->pvBounceBuffer, pTask->cbBounceBuffer);
1500 }
1501
1502 pdmacFileAioMgrNormalRequestFree(pAioMgr, hReq);
1503
1504 pAioMgr->cRequestsActive--;
1505 pEndpoint->AioMgr.cRequestsActive--;
1506 pEndpoint->AioMgr.cReqsProcessed++;
1507
1508 /* Free the lock and process pending tasks if neccessary */
1509 pTasksWaiting = pdmacFileAioMgrNormalRangeLockFree(pAioMgr, pEndpoint, pTask->pRangeLock);
1510 rc = pdmacFileAioMgrNormalProcessTaskList(pTasksWaiting, pAioMgr, pEndpoint);
1511 AssertRC(rc);
1512
1513 /* Call completion callback */
1514 LogFlow(("Task=%#p completed with %Rrc\n", pTask, rcReq));
1515 pTask->pfnCompleted(pTask, pTask->pvUser, rcReq);
1516 pdmacFileTaskFree(pEndpoint, pTask);
1517
1518 /*
1519 * If there is no request left on the endpoint but a flush request is set
1520 * it completed now and we notify the owner.
1521 * Furthermore we look for new requests and continue.
1522 */
1523 if (!pEndpoint->AioMgr.cRequestsActive && pEndpoint->pFlushReq)
1524 {
1525 /* Call completion callback */
1526 pTask = pEndpoint->pFlushReq;
1527 pEndpoint->pFlushReq = NULL;
1528
1529 AssertMsg(pTask->pEndpoint == pEndpoint, ("Endpoint of the flush request does not match assigned one\n"));
1530
1531 pTask->pfnCompleted(pTask, pTask->pvUser, VINF_SUCCESS);
1532 pdmacFileTaskFree(pEndpoint, pTask);
1533 }
1534 else if (RT_UNLIKELY(!pEndpoint->AioMgr.cRequestsActive && pEndpoint->AioMgr.fMoving))
1535 {
1536 /* If the endpoint is about to be migrated do it now. */
1537 bool fReqsPending = pdmacFileAioMgrNormalRemoveEndpoint(pEndpoint);
1538 Assert(!fReqsPending);
1539
1540 rc = pdmacFileAioMgrAddEndpoint(pEndpoint->AioMgr.pAioMgrDst, pEndpoint);
1541 AssertRC(rc);
1542 }
1543 }
1544 } /* Not a flush request */
1545 } /* request completed successfully */
1546}
1547
1548/** Helper macro for checking for error codes. */
1549#define CHECK_RC(pAioMgr, rc) \
1550 if (RT_FAILURE(rc)) \
1551 {\
1552 int rc2 = pdmacFileAioMgrNormalErrorHandler(pAioMgr, rc, RT_SRC_POS);\
1553 return rc2;\
1554 }
1555
1556/**
1557 * The normal I/O manager using the RTFileAio* API
1558 *
1559 * @returns VBox status code.
1560 * @param ThreadSelf Handle of the thread.
1561 * @param pvUser Opaque user data.
1562 */
1563int pdmacFileAioMgrNormal(RTTHREAD ThreadSelf, void *pvUser)
1564{
1565 int rc = VINF_SUCCESS;
1566 PPDMACEPFILEMGR pAioMgr = (PPDMACEPFILEMGR)pvUser;
1567 uint64_t uMillisEnd = RTTimeMilliTS() + PDMACEPFILEMGR_LOAD_UPDATE_PERIOD;
1568
1569 while ( (pAioMgr->enmState == PDMACEPFILEMGRSTATE_RUNNING)
1570 || (pAioMgr->enmState == PDMACEPFILEMGRSTATE_SUSPENDING)
1571 || (pAioMgr->enmState == PDMACEPFILEMGRSTATE_GROWING))
1572 {
1573 if (!pAioMgr->cRequestsActive)
1574 {
1575 ASMAtomicWriteBool(&pAioMgr->fWaitingEventSem, true);
1576 if (!ASMAtomicReadBool(&pAioMgr->fWokenUp))
1577 rc = RTSemEventWait(pAioMgr->EventSem, RT_INDEFINITE_WAIT);
1578 ASMAtomicWriteBool(&pAioMgr->fWaitingEventSem, false);
1579 AssertRC(rc);
1580
1581 LogFlow(("Got woken up\n"));
1582 ASMAtomicWriteBool(&pAioMgr->fWokenUp, false);
1583 }
1584
1585 /* Check for an external blocking event first. */
1586 if (pAioMgr->fBlockingEventPending)
1587 {
1588 rc = pdmacFileAioMgrNormalProcessBlockingEvent(pAioMgr);
1589 CHECK_RC(pAioMgr, rc);
1590 }
1591
1592 if (RT_LIKELY( pAioMgr->enmState == PDMACEPFILEMGRSTATE_RUNNING
1593 || pAioMgr->enmState == PDMACEPFILEMGRSTATE_GROWING))
1594 {
1595 /* We got woken up because an endpoint issued new requests. Queue them. */
1596 rc = pdmacFileAioMgrNormalCheckEndpoints(pAioMgr);
1597 CHECK_RC(pAioMgr, rc);
1598
1599 while ( pAioMgr->cRequestsActive
1600 || pAioMgr->fBwLimitReached)
1601 {
1602 if (pAioMgr->cRequestsActive)
1603 {
1604 RTFILEAIOREQ apReqs[20];
1605 uint32_t cReqsCompleted = 0;
1606 size_t cReqsWait;
1607
1608 if (pAioMgr->cRequestsActive > RT_ELEMENTS(apReqs))
1609 cReqsWait = RT_ELEMENTS(apReqs);
1610 else
1611 cReqsWait = pAioMgr->cRequestsActive;
1612
1613 LogFlow(("Waiting for %d of %d tasks to complete\n", pAioMgr->cRequestsActive, cReqsWait));
1614
1615 rc = RTFileAioCtxWait(pAioMgr->hAioCtx,
1616 cReqsWait,
1617 RT_INDEFINITE_WAIT, apReqs,
1618 RT_ELEMENTS(apReqs), &cReqsCompleted);
1619 if (RT_FAILURE(rc) && (rc != VERR_INTERRUPTED))
1620 CHECK_RC(pAioMgr, rc);
1621
1622 LogFlow(("%d tasks completed\n", cReqsCompleted));
1623
1624 for (uint32_t i = 0; i < cReqsCompleted; i++)
1625 pdmacFileAioMgrNormalReqComplete(pAioMgr, apReqs[i]);
1626
1627 /* Check for an external blocking event before we go to sleep again. */
1628 if (pAioMgr->fBlockingEventPending)
1629 {
1630 rc = pdmacFileAioMgrNormalProcessBlockingEvent(pAioMgr);
1631 CHECK_RC(pAioMgr, rc);
1632 }
1633
1634 /* Update load statistics. */
1635 uint64_t uMillisCurr = RTTimeMilliTS();
1636 if (uMillisCurr > uMillisEnd)
1637 {
1638 PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointCurr = pAioMgr->pEndpointsHead;
1639
1640 /* Calculate timespan. */
1641 uMillisCurr -= uMillisEnd;
1642
1643 while (pEndpointCurr)
1644 {
1645 pEndpointCurr->AioMgr.cReqsPerSec = pEndpointCurr->AioMgr.cReqsProcessed / (uMillisCurr + PDMACEPFILEMGR_LOAD_UPDATE_PERIOD);
1646 pEndpointCurr->AioMgr.cReqsProcessed = 0;
1647 pEndpointCurr = pEndpointCurr->AioMgr.pEndpointNext;
1648 }
1649
1650 /* Set new update interval */
1651 uMillisEnd = RTTimeMilliTS() + PDMACEPFILEMGR_LOAD_UPDATE_PERIOD;
1652 }
1653 }
1654 else
1655 {
1656 /*
1657 * Bandwidth limit reached for all endpoints.
1658 * Yield and wait until we have enough resources again.
1659 */
1660 RTThreadYield();
1661 }
1662
1663 /* Check endpoints for new requests. */
1664 if (pAioMgr->enmState != PDMACEPFILEMGRSTATE_GROWING)
1665 {
1666 rc = pdmacFileAioMgrNormalCheckEndpoints(pAioMgr);
1667 CHECK_RC(pAioMgr, rc);
1668 }
1669 } /* while requests are active. */
1670
1671 if (pAioMgr->enmState == PDMACEPFILEMGRSTATE_GROWING)
1672 {
1673 rc = pdmacFileAioMgrNormalGrow(pAioMgr);
1674 AssertRC(rc);
1675 Assert(pAioMgr->enmState == PDMACEPFILEMGRSTATE_RUNNING);
1676
1677 rc = pdmacFileAioMgrNormalCheckEndpoints(pAioMgr);
1678 CHECK_RC(pAioMgr, rc);
1679 }
1680 } /* if still running */
1681 } /* while running */
1682
1683 LogFlowFunc(("rc=%Rrc\n", rc));
1684 return rc;
1685}
1686
1687#undef CHECK_RC
1688
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette