| 1 | /* $Id: PDMAllQueue.cpp 93612 2022-02-05 19:17:17Z vboxsync $ */
|
|---|
| 2 | /** @file
|
|---|
| 3 | * PDM Queue - Transport data and tasks to EMT and R3.
|
|---|
| 4 | */
|
|---|
| 5 |
|
|---|
| 6 | /*
|
|---|
| 7 | * Copyright (C) 2006-2022 Oracle Corporation
|
|---|
| 8 | *
|
|---|
| 9 | * This file is part of VirtualBox Open Source Edition (OSE), as
|
|---|
| 10 | * available from http://www.virtualbox.org. This file is free software;
|
|---|
| 11 | * you can redistribute it and/or modify it under the terms of the GNU
|
|---|
| 12 | * General Public License (GPL) as published by the Free Software
|
|---|
| 13 | * Foundation, in version 2 as it comes in the "COPYING" file of the
|
|---|
| 14 | * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
|
|---|
| 15 | * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
|
|---|
| 16 | */
|
|---|
| 17 |
|
|---|
| 18 |
|
|---|
| 19 | /*********************************************************************************************************************************
|
|---|
| 20 | * Header Files *
|
|---|
| 21 | *********************************************************************************************************************************/
|
|---|
| 22 | #define LOG_GROUP LOG_GROUP_PDM_QUEUE
|
|---|
| 23 | #include "PDMInternal.h"
|
|---|
| 24 | #include <VBox/vmm/pdm.h>
|
|---|
| 25 | #ifndef IN_RC
|
|---|
| 26 | # include <VBox/vmm/mm.h>
|
|---|
| 27 | #endif
|
|---|
| 28 | #include <VBox/vmm/vmcc.h>
|
|---|
| 29 | #include <iprt/errcore.h>
|
|---|
| 30 | #include <VBox/log.h>
|
|---|
| 31 | #include <iprt/asm.h>
|
|---|
| 32 | #include <iprt/assert.h>
|
|---|
| 33 | #include <iprt/string.h>
|
|---|
| 34 |
|
|---|
| 35 |
|
|---|
| 36 | /*********************************************************************************************************************************
|
|---|
| 37 | * Defined Constants And Macros *
|
|---|
| 38 | *********************************************************************************************************************************/
|
|---|
| 39 | /*
|
|---|
| 40 | * Macros for thoroughly validating a queue handle and ownership.
|
|---|
| 41 | */
|
|---|
| 42 | #define PDMQUEUE_HANDLE_TO_VARS_RETURN_COMMON(a_cbMax, a_cbTotalMax) \
|
|---|
| 43 | AssertReturn(cbItem >= sizeof(PDMQUEUEITEMCORE), pQueue->rcOkay = VERR_INTERNAL_ERROR_4); \
|
|---|
| 44 | AssertReturn(cbItem <= (a_cbMax), pQueue->rcOkay = VERR_INTERNAL_ERROR_4); \
|
|---|
| 45 | \
|
|---|
| 46 | /* paranoia^3: */ \
|
|---|
| 47 | AssertReturn(cItems > 0, pQueue->rcOkay = VERR_INTERNAL_ERROR_4); \
|
|---|
| 48 | AssertReturn(cItems <= PDMQUEUE_MAX_ITEMS, pQueue->rcOkay = VERR_INTERNAL_ERROR_4); \
|
|---|
| 49 | AssertReturn(cbItem * cItems <= (a_cbTotalMax), pQueue->rcOkay = VERR_INTERNAL_ERROR_4)
|
|---|
| 50 |
|
|---|
| 51 | #ifdef IN_RING0
|
|---|
| 52 | # define PDMQUEUE_HANDLE_TO_VARS_RETURN(a_pVM, a_hQueue, a_pvOwner) \
|
|---|
| 53 | AssertPtrReturn((a_pvOwner), VERR_INVALID_PARAMETER); \
|
|---|
| 54 | \
|
|---|
| 55 | AssertCompile(RT_ELEMENTS((a_pVM)->pdm.s.apRing0Queues) == RT_ELEMENTS((a_pVM)->pdmr0.s.aQueues)); \
|
|---|
| 56 | AssertReturn((a_hQueue) < RT_ELEMENTS((a_pVM)->pdmr0.s.aQueues), VERR_INVALID_HANDLE); \
|
|---|
| 57 | AssertReturn((a_hQueue) < (a_pVM)->pdmr0.s.cQueues, VERR_INVALID_HANDLE); \
|
|---|
| 58 | AssertReturn((a_pVM)->pdmr0.s.aQueues[(a_hQueue)].pvOwner == (a_pvOwner), VERR_INVALID_HANDLE); \
|
|---|
| 59 | PPDMQUEUE pQueue = (a_pVM)->pdmr0.s.aQueues[(a_hQueue)].pQueue; \
|
|---|
| 60 | AssertPtrReturn(pQueue, VERR_INVALID_HANDLE); \
|
|---|
| 61 | AssertReturn(pQueue->u32Magic == PDMQUEUE_MAGIC, VERR_INVALID_HANDLE); \
|
|---|
| 62 | AssertReturn(pQueue->rcOkay == VINF_SUCCESS, pQueue->rcOkay); \
|
|---|
| 63 | \
|
|---|
| 64 | uint32_t const cbItem = (a_pVM)->pdmr0.s.aQueues[(a_hQueue)].cbItem; \
|
|---|
| 65 | uint32_t const cItems = (a_pVM)->pdmr0.s.aQueues[(a_hQueue)].cItems; \
|
|---|
| 66 | uint32_t const offItems = (a_pVM)->pdmr0.s.aQueues[(a_hQueue)].offItems; \
|
|---|
| 67 | \
|
|---|
| 68 | /* paranoia^2: */ \
|
|---|
| 69 | AssertReturn(pQueue->cbItem == cbItem, pQueue->rcOkay = VERR_INTERNAL_ERROR_3); \
|
|---|
| 70 | AssertReturn(pQueue->cItems == cItems, pQueue->rcOkay = VERR_INTERNAL_ERROR_3); \
|
|---|
| 71 | AssertReturn(pQueue->offItems == offItems, pQueue->rcOkay = VERR_INTERNAL_ERROR_3); \
|
|---|
| 72 | \
|
|---|
| 73 | PDMQUEUE_HANDLE_TO_VARS_RETURN_COMMON(PDMQUEUE_MAX_ITEM_SIZE, PDMQUEUE_MAX_TOTAL_SIZE_R0)
|
|---|
| 74 |
|
|---|
| 75 | #else
|
|---|
| 76 | # define PDMQUEUE_HANDLE_TO_VARS_RETURN(a_pVM, a_hQueue, a_pvOwner) \
|
|---|
| 77 | AssertPtrReturn((a_pvOwner), VERR_INVALID_PARAMETER); \
|
|---|
| 78 | \
|
|---|
| 79 | PPDMQUEUE pQueue; \
|
|---|
| 80 | if ((a_hQueue) < RT_ELEMENTS((a_pVM)->pdm.s.apRing0Queues)) \
|
|---|
| 81 | pQueue = (a_pVM)->pdm.s.apRing0Queues[(a_hQueue)]; \
|
|---|
| 82 | else \
|
|---|
| 83 | { \
|
|---|
| 84 | (a_hQueue) -= RT_ELEMENTS((a_pVM)->pdm.s.apRing0Queues); \
|
|---|
| 85 | AssertReturn((a_pVM)->pdm.s.cRing3Queues, VERR_INVALID_HANDLE); \
|
|---|
| 86 | pQueue = (a_pVM)->pdm.s.papRing3Queues[(a_hQueue)]; \
|
|---|
| 87 | } \
|
|---|
| 88 | AssertPtrReturn(pQueue, VERR_INVALID_HANDLE); \
|
|---|
| 89 | AssertReturn(pQueue->u32Magic == PDMQUEUE_MAGIC, VERR_INVALID_HANDLE); \
|
|---|
| 90 | AssertReturn(pQueue->u.Gen.pvOwner == (a_pvOwner), VERR_INVALID_HANDLE); \
|
|---|
| 91 | AssertReturn(pQueue->rcOkay == VINF_SUCCESS, pQueue->rcOkay); \
|
|---|
| 92 | \
|
|---|
| 93 | uint32_t const cbItem = pQueue->cbItem; \
|
|---|
| 94 | uint32_t const cItems = pQueue->cItems; \
|
|---|
| 95 | uint32_t const offItems = pQueue->offItems; \
|
|---|
| 96 | \
|
|---|
| 97 | PDMQUEUE_HANDLE_TO_VARS_RETURN_COMMON(PDMQUEUE_MAX_ITEM_SIZE, PDMQUEUE_MAX_TOTAL_SIZE_R3)
|
|---|
| 98 |
|
|---|
| 99 | #endif
|
|---|
| 100 |
|
|---|
| 101 |
|
|---|
| 102 | /**
|
|---|
| 103 | * Commmon function for initializing the shared queue structure.
|
|---|
| 104 | */
|
|---|
| 105 | void pdmQueueInit(PPDMQUEUE pQueue, uint32_t cbBitmap, uint32_t cbItem, uint32_t cItems,
|
|---|
| 106 | const char *pszName, PDMQUEUETYPE enmType, RTR3PTR pfnCallback, RTR3PTR pvOwner)
|
|---|
| 107 | {
|
|---|
| 108 | Assert(cbBitmap * 8 >= cItems);
|
|---|
| 109 |
|
|---|
| 110 | pQueue->u32Magic = PDMQUEUE_MAGIC;
|
|---|
| 111 | pQueue->cbItem = cbItem;
|
|---|
| 112 | pQueue->cItems = cItems;
|
|---|
| 113 | pQueue->offItems = RT_UOFFSETOF(PDMQUEUE, bmAlloc) + cbBitmap;
|
|---|
| 114 | pQueue->rcOkay = VINF_SUCCESS;
|
|---|
| 115 | pQueue->u32Padding = 0;
|
|---|
| 116 | pQueue->hTimer = NIL_TMTIMERHANDLE;
|
|---|
| 117 | pQueue->cMilliesInterval = 0;
|
|---|
| 118 | pQueue->enmType = enmType;
|
|---|
| 119 | pQueue->u.Gen.pfnCallback = pfnCallback;
|
|---|
| 120 | pQueue->u.Gen.pvOwner = pvOwner;
|
|---|
| 121 | RTStrCopy(pQueue->szName, sizeof(pQueue->szName), pszName);
|
|---|
| 122 | pQueue->iPending = UINT32_MAX;
|
|---|
| 123 | RT_BZERO(pQueue->bmAlloc, cbBitmap);
|
|---|
| 124 | ASMBitSetRange(pQueue->bmAlloc, 0, cItems);
|
|---|
| 125 |
|
|---|
| 126 | uint8_t *pbItem = (uint8_t *)&pQueue->bmAlloc[0] + cbBitmap;
|
|---|
| 127 | while (cItems-- > 0)
|
|---|
| 128 | {
|
|---|
| 129 | ((PPDMQUEUEITEMCORE)pbItem)->u64View = UINT64_C(0xfeedfeedfeedfeed);
|
|---|
| 130 |
|
|---|
| 131 | /* next */
|
|---|
| 132 | pbItem += cbItem;
|
|---|
| 133 | }
|
|---|
| 134 | }
|
|---|
| 135 |
|
|---|
| 136 |
|
|---|
| 137 | /**
|
|---|
| 138 | * Allocate an item from a queue, extended version.
|
|---|
| 139 | *
|
|---|
| 140 | * The allocated item must be handed on to PDMR3QueueInsert() after the
|
|---|
| 141 | * data have been filled in.
|
|---|
| 142 | *
|
|---|
| 143 | * @returns VBox status code.
|
|---|
| 144 | * @param pVM Pointer to the cross context VM structure w/ ring-0.
|
|---|
| 145 | * @param hQueue The queue handle.
|
|---|
| 146 | * @param pvOwner The queue owner.
|
|---|
| 147 | * @param ppNew Where to return the item pointer on success.
|
|---|
| 148 | * @thread Any thread.
|
|---|
| 149 | */
|
|---|
| 150 | VMMDECL(int) PDMQueueAllocEx(PVMCC pVM, PDMQUEUEHANDLE hQueue, void *pvOwner, PPDMQUEUEITEMCORE *ppNew)
|
|---|
| 151 | {
|
|---|
| 152 | /*
|
|---|
| 153 | * Validate and translate input.
|
|---|
| 154 | */
|
|---|
| 155 | *ppNew = NULL;
|
|---|
| 156 | PDMQUEUE_HANDLE_TO_VARS_RETURN(pVM, hQueue, pvOwner);
|
|---|
| 157 |
|
|---|
| 158 | /*
|
|---|
| 159 | * Do the allocation.
|
|---|
| 160 | */
|
|---|
| 161 | uint32_t cEmptyScans = 0;
|
|---|
| 162 | for (;;)
|
|---|
| 163 | {
|
|---|
| 164 | int32_t iBit = ASMBitFirstSet(pQueue->bmAlloc, cItems);
|
|---|
| 165 | if (iBit >= 0)
|
|---|
| 166 | {
|
|---|
| 167 | if (ASMAtomicBitTestAndClear(pQueue->bmAlloc, iBit))
|
|---|
| 168 | {
|
|---|
| 169 | PPDMQUEUEITEMCORE pNew = (PPDMQUEUEITEMCORE)&((uint8_t *)pQueue)[offItems + iBit * cbItem];
|
|---|
| 170 | pNew->u64View = UINT64_C(0xbeefbeefbeefbeef);
|
|---|
| 171 | *ppNew = pNew;
|
|---|
| 172 | return VINF_SUCCESS;
|
|---|
| 173 | }
|
|---|
| 174 | cEmptyScans = 0;
|
|---|
| 175 | }
|
|---|
| 176 | else if (++cEmptyScans < 16)
|
|---|
| 177 | ASMNopPause();
|
|---|
| 178 | else
|
|---|
| 179 | {
|
|---|
| 180 | STAM_REL_COUNTER_INC(&pQueue->StatAllocFailures);
|
|---|
| 181 | return VERR_OUT_OF_RESOURCES;
|
|---|
| 182 | }
|
|---|
| 183 | }
|
|---|
| 184 | }
|
|---|
| 185 |
|
|---|
| 186 |
|
|---|
| 187 | /**
|
|---|
| 188 | * Allocate an item from a queue.
|
|---|
| 189 | *
|
|---|
| 190 | * The allocated item must be handed on to PDMR3QueueInsert() after the
|
|---|
| 191 | * data have been filled in.
|
|---|
| 192 | *
|
|---|
| 193 | * @returns Pointer to the new item on success, NULL on failure.
|
|---|
| 194 | * @param pVM Pointer to the cross context VM structure w/ ring-0.
|
|---|
| 195 | * @param hQueue The queue handle.
|
|---|
| 196 | * @param pvOwner The queue owner.
|
|---|
| 197 | * @thread Any thread.
|
|---|
| 198 | */
|
|---|
| 199 | VMMDECL(PPDMQUEUEITEMCORE) PDMQueueAlloc(PVMCC pVM, PDMQUEUEHANDLE hQueue, void *pvOwner)
|
|---|
| 200 | {
|
|---|
| 201 | PPDMQUEUEITEMCORE pNew = NULL;
|
|---|
| 202 | int rc = PDMQueueAllocEx(pVM, hQueue, pvOwner, &pNew);
|
|---|
| 203 | if (RT_SUCCESS(rc))
|
|---|
| 204 | return pNew;
|
|---|
| 205 | return NULL;
|
|---|
| 206 | }
|
|---|
| 207 |
|
|---|
| 208 |
|
|---|
| 209 | /**
|
|---|
| 210 | * Sets the FFs and fQueueFlushed.
|
|---|
| 211 | *
|
|---|
| 212 | * @param pVM Pointer to the cross context VM structure w/ ring-0.
|
|---|
| 213 | */
|
|---|
| 214 | static void pdmQueueSetFF(PVMCC pVM)
|
|---|
| 215 | {
|
|---|
| 216 | Log2(("PDMQueueInsert: VM_FF_PDM_QUEUES %d -> 1\n", VM_FF_IS_SET(pVM, VM_FF_PDM_QUEUES)));
|
|---|
| 217 | VM_FF_SET(pVM, VM_FF_PDM_QUEUES);
|
|---|
| 218 | ASMAtomicBitSet(&pVM->pdm.s.fQueueFlushing, PDM_QUEUE_FLUSH_FLAG_PENDING_BIT);
|
|---|
| 219 | #ifdef IN_RING3
|
|---|
| 220 | VMR3NotifyGlobalFFU(pVM->pUVM, VMNOTIFYFF_FLAGS_DONE_REM);
|
|---|
| 221 | #endif
|
|---|
| 222 | }
|
|---|
| 223 |
|
|---|
| 224 |
|
|---|
| 225 | /**
|
|---|
| 226 | * Queue an item.
|
|---|
| 227 | *
|
|---|
| 228 | * The item must have been obtained using PDMQueueAlloc(). Once the item
|
|---|
| 229 | * have been passed to this function it must not be touched!
|
|---|
| 230 | *
|
|---|
| 231 | * @returns VBox status code.
|
|---|
| 232 | * @param pVM Pointer to the cross context VM structure w/ ring-0.
|
|---|
| 233 | * @param hQueue The queue handle.
|
|---|
| 234 | * @param pvOwner The queue owner.
|
|---|
| 235 | * @param pInsert The item to insert.
|
|---|
| 236 | * @thread Any thread.
|
|---|
| 237 | */
|
|---|
| 238 | VMMDECL(int) PDMQueueInsert(PVMCC pVM, PDMQUEUEHANDLE hQueue, void *pvOwner, PPDMQUEUEITEMCORE pInsert)
|
|---|
| 239 | {
|
|---|
| 240 | /*
|
|---|
| 241 | * Validate and translate input.
|
|---|
| 242 | */
|
|---|
| 243 | PDMQUEUE_HANDLE_TO_VARS_RETURN(pVM, hQueue, pvOwner);
|
|---|
| 244 |
|
|---|
| 245 | uint8_t * const pbItems = (uint8_t *)pQueue + offItems;
|
|---|
| 246 | uintptr_t const offInsert = (uintptr_t)pInsert - (uintptr_t)pbItems;
|
|---|
| 247 | uintptr_t const iInsert = offInsert / cbItem;
|
|---|
| 248 | AssertReturn(iInsert < cItems, VERR_INVALID_PARAMETER);
|
|---|
| 249 | AssertReturn(iInsert * cbItem == offInsert, VERR_INVALID_PARAMETER);
|
|---|
| 250 |
|
|---|
| 251 | AssertReturn(ASMBitTest(pQueue->bmAlloc, iInsert) == false, VERR_INVALID_PARAMETER);
|
|---|
| 252 |
|
|---|
| 253 | /*
|
|---|
| 254 | * Append the item to the pending list.
|
|---|
| 255 | */
|
|---|
| 256 | for (;;)
|
|---|
| 257 | {
|
|---|
| 258 | uint32_t const iOldPending = ASMAtomicUoReadU32(&pQueue->iPending);
|
|---|
| 259 | pInsert->iNext = iOldPending;
|
|---|
| 260 | if (ASMAtomicCmpXchgU32(&pQueue->iPending, iInsert, iOldPending))
|
|---|
| 261 | break;
|
|---|
| 262 | ASMNopPause();
|
|---|
| 263 | }
|
|---|
| 264 |
|
|---|
| 265 | if (pQueue->hTimer == NIL_TMTIMERHANDLE)
|
|---|
| 266 | pdmQueueSetFF(pVM);
|
|---|
| 267 | STAM_REL_COUNTER_INC(&pQueue->StatInsert);
|
|---|
| 268 | STAM_STATS({ ASMAtomicIncU32(&pQueue->cStatPending); });
|
|---|
| 269 |
|
|---|
| 270 | return VINF_SUCCESS;
|
|---|
| 271 | }
|
|---|
| 272 |
|
|---|
| 273 |
|
|---|
| 274 | /**
|
|---|
| 275 | * Schedule the queue for flushing (processing) if necessary.
|
|---|
| 276 | *
|
|---|
| 277 | * @returns VBox status code.
|
|---|
| 278 | * @retval VINF_SUCCESS if a flush was necessary.
|
|---|
| 279 | * @retval VINF_NO_CHANGE if no flushing needed.
|
|---|
| 280 | *
|
|---|
| 281 | * @param pVM The cross context VM structure.
|
|---|
| 282 | * @param pvOwner The alleged queue owner.
|
|---|
| 283 | * @param hQueue The queueu to maybe flush.
|
|---|
| 284 | */
|
|---|
| 285 | VMMDECL(int) PDMQueueFlushIfNecessary(PVMCC pVM, PDMQUEUEHANDLE hQueue, void *pvOwner)
|
|---|
| 286 | {
|
|---|
| 287 | /*
|
|---|
| 288 | * Validate input.
|
|---|
| 289 | */
|
|---|
| 290 | PDMQUEUE_HANDLE_TO_VARS_RETURN(pVM, hQueue, pvOwner);
|
|---|
| 291 | RT_NOREF(offItems);
|
|---|
| 292 |
|
|---|
| 293 | /*
|
|---|
| 294 | * Check and maybe flush.
|
|---|
| 295 | */
|
|---|
| 296 | if (ASMAtomicUoReadU32(&pQueue->iPending) != UINT32_MAX)
|
|---|
| 297 | {
|
|---|
| 298 | pdmQueueSetFF(pVM);
|
|---|
| 299 | return VINF_SUCCESS;
|
|---|
| 300 | }
|
|---|
| 301 | return VINF_NO_CHANGE;
|
|---|
| 302 | }
|
|---|
| 303 |
|
|---|
