VirtualBox

source: vbox/trunk/src/VBox/VMM/VMReq.cpp@ 20651

Last change on this file since 20651 was 20651, checked in by vboxsync, 16 years ago

VMReq.cpp: paranoid android

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 40.6 KB
Line 
1/* $Id: VMReq.cpp 20651 2009-06-17 08:01:00Z vboxsync $ */
2/** @file
3 * VM - Virtual Machine
4 */
5
6/*
7 * Copyright (C) 2006-2007 Sun Microsystems, Inc.
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18 * Clara, CA 95054 USA or visit http://www.sun.com if you need
19 * additional information or have any questions.
20 */
21
22
23/*******************************************************************************
24* Header Files *
25*******************************************************************************/
26#define LOG_GROUP LOG_GROUP_VM
27#include <VBox/mm.h>
28#include <VBox/vmm.h>
29#include "VMInternal.h"
30#include <VBox/vm.h>
31#include <VBox/uvm.h>
32
33#include <VBox/err.h>
34#include <VBox/param.h>
35#include <VBox/log.h>
36#include <iprt/assert.h>
37#include <iprt/asm.h>
38#include <iprt/string.h>
39#include <iprt/time.h>
40#include <iprt/semaphore.h>
41#include <iprt/thread.h>
42
43
44/*******************************************************************************
45* Internal Functions *
46*******************************************************************************/
47static int vmR3ReqProcessOneU(PUVM pUVM, PVMREQ pReq);
48
49
50/**
51 * Allocate and queue a call request.
52 *
53 * If it's desired to poll on the completion of the request set cMillies
54 * to 0 and use VMR3ReqWait() to check for completation. In the other case
55 * use RT_INDEFINITE_WAIT.
56 * The returned request packet must be freed using VMR3ReqFree().
57 *
58 * @returns VBox status code.
59 * Will not return VERR_INTERRUPTED.
60 * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
61 *
62 * @param pVM The VM handle.
63 * @param idDstCpu The destination CPU(s). Either a specific CPU ID or
64 * one of the following special values:
65 * VMCPUID_ANY, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
66 * @param ppReq Where to store the pointer to the request.
67 * This will be NULL or a valid request pointer not matter what happends.
68 * @param cMillies Number of milliseconds to wait for the request to
69 * be completed. Use RT_INDEFINITE_WAIT to only
70 * wait till it's completed.
71 * @param pfnFunction Pointer to the function to call.
72 * @param cArgs Number of arguments following in the ellipsis.
73 * Not possible to pass 64-bit arguments!
74 * @param ... Function arguments.
75 */
76VMMR3DECL(int) VMR3ReqCall(PVM pVM, VMCPUID idDstCpu, PVMREQ *ppReq, unsigned cMillies, PFNRT pfnFunction, unsigned cArgs, ...)
77{
78 va_list va;
79 va_start(va, cArgs);
80 int rc = VMR3ReqCallVU(pVM->pUVM, idDstCpu, ppReq, cMillies, VMREQFLAGS_VBOX_STATUS, pfnFunction, cArgs, va);
81 va_end(va);
82 return rc;
83}
84
85
86/**
87 * Allocate and queue a call request to a void function.
88 *
89 * If it's desired to poll on the completion of the request set cMillies
90 * to 0 and use VMR3ReqWait() to check for completation. In the other case
91 * use RT_INDEFINITE_WAIT.
92 * The returned request packet must be freed using VMR3ReqFree().
93 *
94 * @returns VBox status code.
95 * Will not return VERR_INTERRUPTED.
96 * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
97 *
98 * @param pUVM Pointer to the user mode VM structure.
99 * @param idDstCpu The destination CPU(s). Either a specific CPU ID or
100 * one of the following special values:
101 * VMCPUID_ANY, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
102 * @param ppReq Where to store the pointer to the request.
103 * This will be NULL or a valid request pointer not matter what happends.
104 * @param cMillies Number of milliseconds to wait for the request to
105 * be completed. Use RT_INDEFINITE_WAIT to only
106 * wait till it's completed.
107 * @param pfnFunction Pointer to the function to call.
108 * @param cArgs Number of arguments following in the ellipsis.
109 * Not possible to pass 64-bit arguments!
110 * @param ... Function arguments.
111 */
112VMMR3DECL(int) VMR3ReqCallVoidU(PUVM pUVM, VMCPUID idDstCpu, PVMREQ *ppReq, unsigned cMillies, PFNRT pfnFunction, unsigned cArgs, ...)
113{
114 va_list va;
115 va_start(va, cArgs);
116 int rc = VMR3ReqCallVU(pUVM, idDstCpu, ppReq, cMillies, VMREQFLAGS_VOID, pfnFunction, cArgs, va);
117 va_end(va);
118 return rc;
119}
120
121
122/**
123 * Allocate and queue a call request to a void function.
124 *
125 * If it's desired to poll on the completion of the request set cMillies
126 * to 0 and use VMR3ReqWait() to check for completation. In the other case
127 * use RT_INDEFINITE_WAIT.
128 * The returned request packet must be freed using VMR3ReqFree().
129 *
130 * @returns VBox status code.
131 * Will not return VERR_INTERRUPTED.
132 * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
133 *
134 * @param pVM The VM handle.
135 * @param idDstCpu The destination CPU(s). Either a specific CPU ID or
136 * one of the following special values:
137 * VMCPUID_ANY, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
138 * @param ppReq Where to store the pointer to the request.
139 * This will be NULL or a valid request pointer not matter what happends.
140 * @param cMillies Number of milliseconds to wait for the request to
141 * be completed. Use RT_INDEFINITE_WAIT to only
142 * wait till it's completed.
143 * @param pfnFunction Pointer to the function to call.
144 * @param cArgs Number of arguments following in the ellipsis.
145 * Not possible to pass 64-bit arguments!
146 * @param ... Function arguments.
147 */
148VMMR3DECL(int) VMR3ReqCallVoid(PVM pVM, VMCPUID idDstCpu, PVMREQ *ppReq, unsigned cMillies, PFNRT pfnFunction, unsigned cArgs, ...)
149{
150 va_list va;
151 va_start(va, cArgs);
152 int rc = VMR3ReqCallVU(pVM->pUVM, idDstCpu, ppReq, cMillies, VMREQFLAGS_VOID, pfnFunction, cArgs, va);
153 va_end(va);
154 return rc;
155}
156
157
158/**
159 * Allocate and queue a call request to a void function.
160 *
161 * If it's desired to poll on the completion of the request set cMillies
162 * to 0 and use VMR3ReqWait() to check for completation. In the other case
163 * use RT_INDEFINITE_WAIT.
164 * The returned request packet must be freed using VMR3ReqFree().
165 *
166 * @returns VBox status code.
167 * Will not return VERR_INTERRUPTED.
168 * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
169 *
170 * @param pVM The VM handle.
171 * @param idDstCpu The destination CPU(s). Either a specific CPU ID or
172 * one of the following special values:
173 * VMCPUID_ANY, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
174 * @param ppReq Where to store the pointer to the request.
175 * This will be NULL or a valid request pointer not matter what happends, unless fFlags
176 * contains VMREQFLAGS_NO_WAIT when it will be optional and always NULL.
177 * @param cMillies Number of milliseconds to wait for the request to
178 * be completed. Use RT_INDEFINITE_WAIT to only
179 * wait till it's completed.
180 * @param fFlags A combination of the VMREQFLAGS values.
181 * @param pfnFunction Pointer to the function to call.
182 * @param cArgs Number of arguments following in the ellipsis.
183 * Not possible to pass 64-bit arguments!
184 * @param ... Function arguments.
185 */
186VMMR3DECL(int) VMR3ReqCallEx(PVM pVM, VMCPUID idDstCpu, PVMREQ *ppReq, unsigned cMillies, unsigned fFlags, PFNRT pfnFunction, unsigned cArgs, ...)
187{
188 va_list va;
189 va_start(va, cArgs);
190 int rc = VMR3ReqCallVU(pVM->pUVM, idDstCpu, ppReq, cMillies, fFlags, pfnFunction, cArgs, va);
191 va_end(va);
192 return rc;
193}
194
195
196/**
197 * Allocate and queue a call request to a void function.
198 *
199 * If it's desired to poll on the completion of the request set cMillies
200 * to 0 and use VMR3ReqWait() to check for completation. In the other case
201 * use RT_INDEFINITE_WAIT.
202 * The returned request packet must be freed using VMR3ReqFree().
203 *
204 * @returns VBox status code.
205 * Will not return VERR_INTERRUPTED.
206 * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
207 *
208 * @param pUVM Pointer to the user mode VM structure.
209 * @param idDstCpu The destination CPU(s). Either a specific CPU ID or
210 * one of the following special values:
211 * VMCPUID_ANY, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
212 * @param ppReq Where to store the pointer to the request.
213 * This will be NULL or a valid request pointer not matter what happends, unless fFlags
214 * contains VMREQFLAGS_NO_WAIT when it will be optional and always NULL.
215 * @param cMillies Number of milliseconds to wait for the request to
216 * be completed. Use RT_INDEFINITE_WAIT to only
217 * wait till it's completed.
218 * @param fFlags A combination of the VMREQFLAGS values.
219 * @param pfnFunction Pointer to the function to call.
220 * @param cArgs Number of arguments following in the ellipsis.
221 * Not possible to pass 64-bit arguments!
222 * @param ... Function arguments.
223 */
224VMMR3DECL(int) VMR3ReqCallU(PUVM pUVM, VMCPUID idDstCpu, PVMREQ *ppReq, unsigned cMillies, unsigned fFlags, PFNRT pfnFunction, unsigned cArgs, ...)
225{
226 va_list va;
227 va_start(va, cArgs);
228 int rc = VMR3ReqCallVU(pUVM, idDstCpu, ppReq, cMillies, fFlags, pfnFunction, cArgs, va);
229 va_end(va);
230 return rc;
231}
232
233
234/**
235 * Allocate and queue a call request.
236 *
237 * If it's desired to poll on the completion of the request set cMillies
238 * to 0 and use VMR3ReqWait() to check for completation. In the other case
239 * use RT_INDEFINITE_WAIT.
240 * The returned request packet must be freed using VMR3ReqFree().
241 *
242 * @returns VBox status code.
243 * Will not return VERR_INTERRUPTED.
244 * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
245 *
246 * @param pUVM Pointer to the user mode VM structure.
247 * @param idDstCpu The destination CPU(s). Either a specific CPU ID or
248 * one of the following special values:
249 * VMCPUID_ANY, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
250 * @param ppReq Where to store the pointer to the request.
251 * This will be NULL or a valid request pointer not matter what happends, unless fFlags
252 * contains VMREQFLAGS_NO_WAIT when it will be optional and always NULL.
253 * @param cMillies Number of milliseconds to wait for the request to
254 * be completed. Use RT_INDEFINITE_WAIT to only
255 * wait till it's completed.
256 * @param pfnFunction Pointer to the function to call.
257 * @param fFlags A combination of the VMREQFLAGS values.
258 * @param cArgs Number of arguments following in the ellipsis.
259 * Stuff which differs in size from uintptr_t is gonna make trouble, so don't try!
260 * @param Args Argument vector.
261 */
262VMMR3DECL(int) VMR3ReqCallVU(PUVM pUVM, VMCPUID idDstCpu, PVMREQ *ppReq, unsigned cMillies, unsigned fFlags, PFNRT pfnFunction, unsigned cArgs, va_list Args)
263{
264 LogFlow(("VMR3ReqCallV: idDstCpu=%u cMillies=%d fFlags=%#x pfnFunction=%p cArgs=%d\n", idDstCpu, cMillies, fFlags, pfnFunction, cArgs));
265
266 /*
267 * Validate input.
268 */
269 AssertPtrReturn(pfnFunction, VERR_INVALID_POINTER);
270 AssertPtrReturn(pUVM, VERR_INVALID_POINTER);
271 AssertReturn(!(fFlags & ~(VMREQFLAGS_RETURN_MASK | VMREQFLAGS_NO_WAIT | VMREQFLAGS_POKE)), VERR_INVALID_PARAMETER);
272 if (!(fFlags & VMREQFLAGS_NO_WAIT) || ppReq)
273 {
274 AssertPtrReturn(ppReq, VERR_INVALID_POINTER);
275 *ppReq = NULL;
276 }
277 PVMREQ pReq = NULL;
278 AssertMsgReturn(cArgs * sizeof(uintptr_t) <= sizeof(pReq->u.Internal.aArgs),
279 ("cArg=%d\n", cArgs),
280 VERR_TOO_MUCH_DATA);
281
282 /*
283 * Allocate request
284 */
285 int rc = VMR3ReqAllocU(pUVM, &pReq, VMREQTYPE_INTERNAL, idDstCpu);
286 if (RT_FAILURE(rc))
287 return rc;
288
289 /*
290 * Initialize the request data.
291 */
292 pReq->fFlags = fFlags;
293 pReq->u.Internal.pfn = pfnFunction;
294 pReq->u.Internal.cArgs = cArgs;
295 for (unsigned iArg = 0; iArg < cArgs; iArg++)
296 pReq->u.Internal.aArgs[iArg] = va_arg(Args, uintptr_t);
297
298 /*
299 * Queue the request and return.
300 */
301 rc = VMR3ReqQueue(pReq, cMillies);
302 if ( RT_FAILURE(rc)
303 && rc != VERR_TIMEOUT)
304 {
305 VMR3ReqFree(pReq);
306 pReq = NULL;
307 }
308 if (!(fFlags & VMREQFLAGS_NO_WAIT))
309 {
310 *ppReq = pReq;
311 LogFlow(("VMR3ReqCallV: returns %Rrc *ppReq=%p\n", rc, pReq));
312 }
313 else
314 LogFlow(("VMR3ReqCallV: returns %Rrc\n", rc));
315 Assert(rc != VERR_INTERRUPTED);
316 return rc;
317}
318
319
320/**
321 * Joins the list pList with whatever is linked up at *pHead.
322 */
323static void vmr3ReqJoinFreeSub(volatile PVMREQ *ppHead, PVMREQ pList)
324{
325 for (unsigned cIterations = 0;; cIterations++)
326 {
327 PVMREQ pHead = (PVMREQ)ASMAtomicXchgPtr((void * volatile *)ppHead, pList);
328 if (!pHead)
329 return;
330 PVMREQ pTail = pHead;
331 while (pTail->pNext)
332 pTail = pTail->pNext;
333 ASMAtomicWritePtr((void * volatile *)&pTail->pNext, pList);
334 ASMCompilerBarrier();
335 if (ASMAtomicCmpXchgPtr((void * volatile *)ppHead, (void *)pHead, pList))
336 return;
337 ASMAtomicWritePtr((void * volatile *)&pTail->pNext, NULL);
338 ASMCompilerBarrier();
339 if (ASMAtomicCmpXchgPtr((void * volatile *)ppHead, (void *)pHead, NULL))
340 return;
341 pList = pHead;
342 Assert(cIterations != 32);
343 Assert(cIterations != 64);
344 }
345}
346
347
348/**
349 * Joins the list pList with whatever is linked up at *pHead.
350 */
351static void vmr3ReqJoinFree(PVMINTUSERPERVM pVMInt, PVMREQ pList)
352{
353 /*
354 * Split the list if it's too long.
355 */
356 unsigned cReqs = 1;
357 PVMREQ pTail = pList;
358 while (pTail->pNext)
359 {
360 if (cReqs++ > 25)
361 {
362 const uint32_t i = pVMInt->iReqFree;
363 vmr3ReqJoinFreeSub(&pVMInt->apReqFree[(i + 2) % RT_ELEMENTS(pVMInt->apReqFree)], pTail->pNext);
364
365 pTail->pNext = NULL;
366 vmr3ReqJoinFreeSub(&pVMInt->apReqFree[(i + 2 + (i == pVMInt->iReqFree)) % RT_ELEMENTS(pVMInt->apReqFree)], pTail->pNext);
367 return;
368 }
369 pTail = pTail->pNext;
370 }
371 vmr3ReqJoinFreeSub(&pVMInt->apReqFree[(pVMInt->iReqFree + 2) % RT_ELEMENTS(pVMInt->apReqFree)], pList);
372}
373
374
375/**
376 * Allocates a request packet.
377 *
378 * The caller allocates a request packet, fills in the request data
379 * union and queues the request.
380 *
381 * @returns VBox status code.
382 *
383 * @param pVM VM handle.
384 * @param ppReq Where to store the pointer to the allocated packet.
385 * @param enmType Package type.
386 * @param idDstCpu The destination CPU(s). Either a specific CPU ID or
387 * one of the following special values:
388 * VMCPUID_ANY, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
389 */
390VMMR3DECL(int) VMR3ReqAlloc(PVM pVM, PVMREQ *ppReq, VMREQTYPE enmType, VMCPUID idDstCpu)
391{
392 return VMR3ReqAllocU(pVM->pUVM, ppReq, enmType, idDstCpu);
393}
394
395
396/**
397 * Allocates a request packet.
398 *
399 * The caller allocates a request packet, fills in the request data
400 * union and queues the request.
401 *
402 * @returns VBox status code.
403 *
404 * @param pUVM Pointer to the user mode VM structure.
405 * @param ppReq Where to store the pointer to the allocated packet.
406 * @param enmType Package type.
407 * @param idDstCpu The destination CPU(s). Either a specific CPU ID or
408 * one of the following special values:
409 * VMCPUID_ANY, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
410 */
411VMMR3DECL(int) VMR3ReqAllocU(PUVM pUVM, PVMREQ *ppReq, VMREQTYPE enmType, VMCPUID idDstCpu)
412{
413 /*
414 * Validate input.
415 */
416 AssertMsgReturn(enmType > VMREQTYPE_INVALID && enmType < VMREQTYPE_MAX,
417 ("Invalid package type %d valid range %d-%d inclusivly.\n",
418 enmType, VMREQTYPE_INVALID + 1, VMREQTYPE_MAX - 1),
419 VERR_VM_REQUEST_INVALID_TYPE);
420 AssertPtrReturn(ppReq, VERR_INVALID_POINTER);
421 AssertMsgReturn( idDstCpu == VMCPUID_ANY
422 || idDstCpu < pUVM->cCpus
423 || idDstCpu == VMCPUID_ALL
424 || idDstCpu == VMCPUID_ALL_REVERSE,
425 ("Invalid destination %u (max=%u)\n", idDstCpu, pUVM->cCpus), VERR_INVALID_PARAMETER);
426
427 /*
428 * Try get a recycled packet.
429 * While this could all be solved with a single list with a lock, it's a sport
430 * of mine to avoid locks.
431 */
432 int cTries = RT_ELEMENTS(pUVM->vm.s.apReqFree) * 2;
433 while (--cTries >= 0)
434 {
435 PVMREQ volatile *ppHead = &pUVM->vm.s.apReqFree[ASMAtomicIncU32(&pUVM->vm.s.iReqFree) % RT_ELEMENTS(pUVM->vm.s.apReqFree)];
436#if 0 /* sad, but this won't work safely because the reading of pReq->pNext. */
437 PVMREQ pNext = NULL;
438 PVMREQ pReq = *ppHead;
439 if ( pReq
440 && !ASMAtomicCmpXchgPtr((void * volatile *)ppHead, (pNext = pReq->pNext), pReq)
441 && (pReq = *ppHead)
442 && !ASMAtomicCmpXchgPtr((void * volatile *)ppHead, (pNext = pReq->pNext), pReq))
443 pReq = NULL;
444 if (pReq)
445 {
446 Assert(pReq->pNext == pNext); NOREF(pReq);
447#else
448 PVMREQ pReq = (PVMREQ)ASMAtomicXchgPtr((void * volatile *)ppHead, NULL);
449 if (pReq)
450 {
451 PVMREQ pNext = pReq->pNext;
452 if ( pNext
453 && !ASMAtomicCmpXchgPtr((void * volatile *)ppHead, pNext, NULL))
454 {
455 STAM_COUNTER_INC(&pUVM->vm.s.StatReqAllocRaces);
456 vmr3ReqJoinFree(&pUVM->vm.s, pReq->pNext);
457 }
458#endif
459 ASMAtomicDecU32(&pUVM->vm.s.cReqFree);
460
461 /*
462 * Make sure the event sem is not signaled.
463 */
464 if (!pReq->fEventSemClear)
465 {
466 int rc = RTSemEventWait(pReq->EventSem, 0);
467 if (rc != VINF_SUCCESS && rc != VERR_TIMEOUT)
468 {
469 /*
470 * This shall not happen, but if it does we'll just destroy
471 * the semaphore and create a new one.
472 */
473 AssertMsgFailed(("rc=%Rrc from RTSemEventWait(%#x).\n", rc, pReq->EventSem));
474 RTSemEventDestroy(pReq->EventSem);
475 rc = RTSemEventCreate(&pReq->EventSem);
476 AssertRC(rc);
477 if (RT_FAILURE(rc))
478 return rc;
479 }
480 pReq->fEventSemClear = true;
481 }
482 else
483 Assert(RTSemEventWait(pReq->EventSem, 0) == VERR_TIMEOUT);
484
485 /*
486 * Initialize the packet and return it.
487 */
488 Assert(pReq->enmType == VMREQTYPE_INVALID);
489 Assert(pReq->enmState == VMREQSTATE_FREE);
490 Assert(pReq->pUVM == pUVM);
491 ASMAtomicXchgSize(&pReq->pNext, NULL);
492 pReq->enmState = VMREQSTATE_ALLOCATED;
493 pReq->iStatus = VERR_VM_REQUEST_STATUS_STILL_PENDING;
494 pReq->fFlags = VMREQFLAGS_VBOX_STATUS;
495 pReq->enmType = enmType;
496 pReq->idDstCpu = idDstCpu;
497
498 *ppReq = pReq;
499 STAM_COUNTER_INC(&pUVM->vm.s.StatReqAllocRecycled);
500 LogFlow(("VMR3ReqAlloc: returns VINF_SUCCESS *ppReq=%p recycled\n", pReq));
501 return VINF_SUCCESS;
502 }
503 }
504
505 /*
506 * Ok allocate one.
507 */
508 PVMREQ pReq = (PVMREQ)MMR3HeapAllocU(pUVM, MM_TAG_VM_REQ, sizeof(*pReq));
509 if (!pReq)
510 return VERR_NO_MEMORY;
511
512 /*
513 * Create the semaphore.
514 */
515 int rc = RTSemEventCreate(&pReq->EventSem);
516 AssertRC(rc);
517 if (RT_FAILURE(rc))
518 {
519 MMR3HeapFree(pReq);
520 return rc;
521 }
522
523 /*
524 * Initialize the packet and return it.
525 */
526 pReq->pNext = NULL;
527 pReq->pUVM = pUVM;
528 pReq->enmState = VMREQSTATE_ALLOCATED;
529 pReq->iStatus = VERR_VM_REQUEST_STATUS_STILL_PENDING;
530 pReq->fEventSemClear = true;
531 pReq->fFlags = VMREQFLAGS_VBOX_STATUS;
532 pReq->enmType = enmType;
533 pReq->idDstCpu = idDstCpu;
534
535 *ppReq = pReq;
536 STAM_COUNTER_INC(&pUVM->vm.s.StatReqAllocNew);
537 LogFlow(("VMR3ReqAlloc: returns VINF_SUCCESS *ppReq=%p new\n", pReq));
538 return VINF_SUCCESS;
539}
540
541
542/**
543 * Free a request packet.
544 *
545 * @returns VBox status code.
546 *
547 * @param pReq Package to free.
548 * @remark The request packet must be in allocated or completed state!
549 */
550VMMR3DECL(int) VMR3ReqFree(PVMREQ pReq)
551{
552 /*
553 * Ignore NULL (all free functions should do this imho).
554 */
555 if (!pReq)
556 return VINF_SUCCESS;
557
558 /*
559 * Check packet state.
560 */
561 switch (pReq->enmState)
562 {
563 case VMREQSTATE_ALLOCATED:
564 case VMREQSTATE_COMPLETED:
565 break;
566 default:
567 AssertMsgFailed(("Invalid state %d!\n", pReq->enmState));
568 return VERR_VM_REQUEST_STATE;
569 }
570
571 /*
572 * Make it a free packet and put it into one of the free packet lists.
573 */
574 pReq->enmState = VMREQSTATE_FREE;
575 pReq->iStatus = VERR_VM_REQUEST_STATUS_FREED;
576 pReq->enmType = VMREQTYPE_INVALID;
577
578 PUVM pUVM = pReq->pUVM;
579 STAM_COUNTER_INC(&pUVM->vm.s.StatReqFree);
580
581 if (pUVM->vm.s.cReqFree < 128)
582 {
583 ASMAtomicIncU32(&pUVM->vm.s.cReqFree);
584 PVMREQ volatile *ppHead = &pUVM->vm.s.apReqFree[ASMAtomicIncU32(&pUVM->vm.s.iReqFree) % RT_ELEMENTS(pUVM->vm.s.apReqFree)];
585 PVMREQ pNext;
586 do
587 {
588 pNext = (PVMREQ)ASMAtomicUoReadPtr((void * volatile *)ppHead);
589 ASMAtomicWritePtr((void * volatile *)&pReq->pNext, pNext);
590 ASMCompilerBarrier();
591 } while (!ASMAtomicCmpXchgPtr((void * volatile *)ppHead, (void *)pReq, (void *)pNext));
592 }
593 else
594 {
595 STAM_COUNTER_INC(&pReq->pUVM->vm.s.StatReqFreeOverflow);
596 RTSemEventDestroy(pReq->EventSem);
597 MMR3HeapFree(pReq);
598 }
599 return VINF_SUCCESS;
600}
601
602
603/**
604 * Queue a request.
605 *
606 * The quest must be allocated using VMR3ReqAlloc() and contain
607 * all the required data.
608 * If it's desired to poll on the completion of the request set cMillies
609 * to 0 and use VMR3ReqWait() to check for completation. In the other case
610 * use RT_INDEFINITE_WAIT.
611 *
612 * @returns VBox status code.
613 * Will not return VERR_INTERRUPTED.
614 * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
615 *
616 * @param pReq The request to queue.
617 * @param cMillies Number of milliseconds to wait for the request to
618 * be completed. Use RT_INDEFINITE_WAIT to only
619 * wait till it's completed.
620 */
621VMMR3DECL(int) VMR3ReqQueue(PVMREQ pReq, unsigned cMillies)
622{
623 LogFlow(("VMR3ReqQueue: pReq=%p cMillies=%d\n", pReq, cMillies));
624 /*
625 * Verify the supplied package.
626 */
627 AssertMsgReturn(pReq->enmState == VMREQSTATE_ALLOCATED, ("%d\n", pReq->enmState), VERR_VM_REQUEST_STATE);
628 AssertMsgReturn( VALID_PTR(pReq->pUVM)
629 && !pReq->pNext
630 && pReq->EventSem != NIL_RTSEMEVENT,
631 ("Invalid request package! Anyone cooking their own packages???\n"),
632 VERR_VM_REQUEST_INVALID_PACKAGE);
633 AssertMsgReturn( pReq->enmType > VMREQTYPE_INVALID
634 && pReq->enmType < VMREQTYPE_MAX,
635 ("Invalid package type %d valid range %d-%d inclusivly. This was verified on alloc too...\n",
636 pReq->enmType, VMREQTYPE_INVALID + 1, VMREQTYPE_MAX - 1),
637 VERR_VM_REQUEST_INVALID_TYPE);
638 Assert(!(pReq->fFlags & ~(VMREQFLAGS_RETURN_MASK | VMREQFLAGS_NO_WAIT | VMREQFLAGS_POKE)));
639
640 /*
641 * Are we the EMT or not?
642 * Also, store pVM (and fFlags) locally since pReq may be invalid after queuing it.
643 */
644 int rc = VINF_SUCCESS;
645 PUVM pUVM = ((VMREQ volatile *)pReq)->pUVM; /* volatile paranoia */
646 PUVMCPU pUVCpu = (PUVMCPU)RTTlsGet(pUVM->vm.s.idxTLS);
647
648 if (pReq->idDstCpu == VMCPUID_ALL)
649 {
650 /* One-by-one. */
651 Assert(!(pReq->fFlags & VMREQFLAGS_NO_WAIT));
652 for (unsigned i = 0; i < pUVM->cCpus; i++)
653 {
654 /* Reinit some members. */
655 pReq->enmState = VMREQSTATE_ALLOCATED;
656 pReq->idDstCpu = i;
657 rc = VMR3ReqQueue(pReq, cMillies);
658 if (RT_FAILURE(rc))
659 break;
660 }
661 }
662 else if (pReq->idDstCpu == VMCPUID_ALL_REVERSE)
663 {
664 /* One-by-one. */
665 Assert(!(pReq->fFlags & VMREQFLAGS_NO_WAIT));
666 for (int i = pUVM->cCpus-1; i >= 0; i--)
667 {
668 /* Reinit some members. */
669 pReq->enmState = VMREQSTATE_ALLOCATED;
670 pReq->idDstCpu = i;
671 rc = VMR3ReqQueue(pReq, cMillies);
672 if (RT_FAILURE(rc))
673 break;
674 }
675 }
676 else if ( pReq->idDstCpu != VMCPUID_ANY /* for a specific VMCPU? */
677 && ( !pUVCpu /* and it's not the current thread. */
678 || pUVCpu->idCpu != pReq->idDstCpu))
679 {
680 VMCPUID idTarget = pReq->idDstCpu; Assert(idTarget < pUVM->cCpus);
681 PVMCPU pVCpu = &pUVM->pVM->aCpus[idTarget];
682 unsigned fFlags = ((VMREQ volatile *)pReq)->fFlags; /* volatile paranoia */
683
684 /* Fetch the right UVMCPU */
685 pUVCpu = &pUVM->aCpus[idTarget];
686
687 /*
688 * Insert it.
689 */
690 pReq->enmState = VMREQSTATE_QUEUED;
691 PVMREQ pNext;
692 do
693 {
694 pNext = (PVMREQ)ASMAtomicUoReadPtr((void * volatile *)&pUVCpu->vm.s.pReqs);
695 ASMAtomicWritePtr((void * volatile *)&pReq->pNext, pNext);
696 ASMCompilerBarrier();
697 } while (!ASMAtomicCmpXchgPtr((void * volatile *)&pUVCpu->vm.s.pReqs, (void *)pReq, (void *)pNext));
698
699 /*
700 * Notify EMT.
701 */
702 if (pUVM->pVM)
703 VMCPU_FF_SET(pVCpu, VMCPU_FF_REQUEST);
704 VMR3NotifyCpuFFU(pUVCpu, fFlags & VMREQFLAGS_POKE ? VMNOTIFYFF_FLAGS_POKE : 0);
705
706 /*
707 * Wait and return.
708 */
709 if (!(fFlags & VMREQFLAGS_NO_WAIT))
710 rc = VMR3ReqWait(pReq, cMillies);
711 LogFlow(("VMR3ReqQueue: returns %Rrc\n", rc));
712 }
713 else if ( pReq->idDstCpu == VMCPUID_ANY
714 && !pUVCpu /* only EMT threads have a valid pointer stored in the TLS slot. */)
715 {
716 unsigned fFlags = ((VMREQ volatile *)pReq)->fFlags; /* volatile paranoia */
717
718 /*
719 * Insert it.
720 */
721 pReq->enmState = VMREQSTATE_QUEUED;
722 PVMREQ pNext;
723 do
724 {
725 pNext = (PVMREQ)ASMAtomicUoReadPtr((void * volatile *)&pUVM->vm.s.pReqs);
726 ASMAtomicWritePtr((void * volatile *)&pReq->pNext, pNext);
727 ASMCompilerBarrier();
728 } while (!ASMAtomicCmpXchgPtr((void * volatile *)&pUVM->vm.s.pReqs, (void *)pReq, (void *)pNext));
729
730 /*
731 * Notify EMT.
732 */
733 if (pUVM->pVM)
734 VM_FF_SET(pUVM->pVM, VM_FF_REQUEST);
735 VMR3NotifyGlobalFFU(pUVM, fFlags & VMREQFLAGS_POKE ? VMNOTIFYFF_FLAGS_POKE : 0);
736
737 /*
738 * Wait and return.
739 */
740 if (!(fFlags & VMREQFLAGS_NO_WAIT))
741 rc = VMR3ReqWait(pReq, cMillies);
742 LogFlow(("VMR3ReqQueue: returns %Rrc\n", rc));
743 }
744 else
745 {
746 Assert(pUVCpu);
747
748 /*
749 * The requester was an EMT, just execute it.
750 */
751 pReq->enmState = VMREQSTATE_QUEUED;
752 rc = vmR3ReqProcessOneU(pUVM, pReq);
753 LogFlow(("VMR3ReqQueue: returns %Rrc (processed)\n", rc));
754 }
755 return rc;
756}
757
758
759/**
760 * Wait for a request to be completed.
761 *
762 * @returns VBox status code.
763 * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
764 *
765 * @param pReq The request to wait for.
766 * @param cMillies Number of milliseconds to wait.
767 * Use RT_INDEFINITE_WAIT to only wait till it's completed.
768 */
769VMMR3DECL(int) VMR3ReqWait(PVMREQ pReq, unsigned cMillies)
770{
771 LogFlow(("VMR3ReqWait: pReq=%p cMillies=%d\n", pReq, cMillies));
772
773 /*
774 * Verify the supplied package.
775 */
776 AssertMsgReturn( pReq->enmState == VMREQSTATE_QUEUED
777 || pReq->enmState == VMREQSTATE_PROCESSING
778 || pReq->enmState == VMREQSTATE_COMPLETED,
779 ("Invalid state %d\n", pReq->enmState),
780 VERR_VM_REQUEST_STATE);
781 AssertMsgReturn( VALID_PTR(pReq->pUVM)
782 && pReq->EventSem != NIL_RTSEMEVENT,
783 ("Invalid request package! Anyone cooking their own packages???\n"),
784 VERR_VM_REQUEST_INVALID_PACKAGE);
785 AssertMsgReturn( pReq->enmType > VMREQTYPE_INVALID
786 && pReq->enmType < VMREQTYPE_MAX,
787 ("Invalid package type %d valid range %d-%d inclusivly. This was verified on alloc too...\n",
788 pReq->enmType, VMREQTYPE_INVALID + 1, VMREQTYPE_MAX - 1),
789 VERR_VM_REQUEST_INVALID_TYPE);
790
791 /*
792 * Check for deadlock condition
793 */
794 PUVM pUVM = pReq->pUVM;
795 NOREF(pUVM);
796
797 /*
798 * Wait on the package.
799 */
800 int rc;
801 if (cMillies != RT_INDEFINITE_WAIT)
802 rc = RTSemEventWait(pReq->EventSem, cMillies);
803 else
804 {
805 do
806 {
807 rc = RTSemEventWait(pReq->EventSem, RT_INDEFINITE_WAIT);
808 Assert(rc != VERR_TIMEOUT);
809 } while ( pReq->enmState != VMREQSTATE_COMPLETED
810 && pReq->enmState != VMREQSTATE_INVALID);
811 }
812 if (RT_SUCCESS(rc))
813 ASMAtomicXchgSize(&pReq->fEventSemClear, true);
814 if (pReq->enmState == VMREQSTATE_COMPLETED)
815 rc = VINF_SUCCESS;
816 LogFlow(("VMR3ReqWait: returns %Rrc\n", rc));
817 Assert(rc != VERR_INTERRUPTED);
818 return rc;
819}
820
821
822/**
823 * Process pending request(s).
824 *
825 * This function is called from a forced action handler in the EMT
826 * or from one of the EMT loops.
827 *
828 * @returns VBox status code.
829 *
830 * @param pUVM Pointer to the user mode VM structure.
831 * @param idDstCpu Pass VMCPUID_ANY to process the common request queue
832 * and the CPU ID for a CPU specific one. In the latter
833 * case the calling thread must be the EMT of that CPU.
834 *
835 * @note SMP safe (multiple EMTs trying to satisfy VM_FF_REQUESTs).
836 */
837VMMR3DECL(int) VMR3ReqProcessU(PUVM pUVM, VMCPUID idDstCpu)
838{
839 LogFlow(("VMR3ReqProcessU: (enmVMState=%d) idDstCpu=%d\n", pUVM->pVM ? pUVM->pVM->enmVMState : VMSTATE_CREATING, idDstCpu));
840
841 /*
842 * Process loop.
843 *
844 * We do not repeat the outer loop if we've got an informational status code
845 * since that code needs processing by our caller.
846 */
847 int rc = VINF_SUCCESS;
848 while (rc <= VINF_SUCCESS)
849 {
850 /*
851 * Get pending requests.
852 */
853 void * volatile *ppReqs;
854 if (idDstCpu == VMCPUID_ANY)
855 {
856 ppReqs = (void * volatile *)&pUVM->vm.s.pReqs;
857 if (RT_LIKELY(pUVM->pVM))
858 VM_FF_CLEAR(pUVM->pVM, VM_FF_REQUEST);
859 }
860 else
861 {
862 Assert(idDstCpu < pUVM->cCpus);
863 Assert(pUVM->aCpus[idDstCpu].vm.s.NativeThreadEMT == RTThreadNativeSelf());
864 ppReqs = (void * volatile *)&pUVM->aCpus[idDstCpu].vm.s.pReqs;
865 if (RT_LIKELY(pUVM->pVM))
866 VMCPU_FF_CLEAR(&pUVM->pVM->aCpus[idDstCpu], VMCPU_FF_REQUEST);
867 }
868 PVMREQ pReqs = (PVMREQ)ASMAtomicXchgPtr(ppReqs, NULL);
869 if (!pReqs)
870 break;
871
872 /*
873 * Reverse the list to process it in FIFO order.
874 */
875 PVMREQ pReq = pReqs;
876 if (pReq->pNext)
877 Log2(("VMR3ReqProcess: 2+ requests: %p %p %p\n", pReq, pReq->pNext, pReq->pNext->pNext));
878 pReqs = NULL;
879 while (pReq)
880 {
881 Assert(pReq->enmState == VMREQSTATE_QUEUED);
882 Assert(pReq->pUVM == pUVM);
883 PVMREQ pCur = pReq;
884 pReq = pReq->pNext;
885 pCur->pNext = pReqs;
886 pReqs = pCur;
887 }
888
889
890 /*
891 * Process the requests.
892 *
893 * Since this is a FF worker certain rules applies to the
894 * status codes. See the EM section in VBox/err.h and EM.cpp for details.
895 */
896 while (pReqs)
897 {
898 /* Unchain the first request and advance the list. */
899 pReq = pReqs;
900 pReqs = pReqs->pNext;
901 pReq->pNext = NULL;
902
903 /* Process the request */
904 int rc2 = vmR3ReqProcessOneU(pUVM, pReq);
905
906 /*
907 * The status code handling extremely important yet very fragile. Should probably
908 * look for a better way of communicating status changes to EM...
909 */
910 if ( rc2 >= VINF_EM_FIRST
911 && rc2 <= VINF_EM_LAST
912 && ( rc == VINF_SUCCESS
913 || rc2 < rc) )
914 rc = rc2;
915 }
916 }
917
918 LogFlow(("VMR3ReqProcess: returns %Rrc (enmVMState=%d)\n", rc, pUVM->pVM ? pUVM->pVM->enmVMState : VMSTATE_CREATING));
919 return rc;
920}
921
922
923/**
924 * Process one request.
925 *
926 * @returns VBox status code.
927 *
928 * @param pVM VM handle.
929 * @param pReq Request packet to process.
930 */
931static int vmR3ReqProcessOneU(PUVM pUVM, PVMREQ pReq)
932{
933 LogFlow(("vmR3ReqProcessOne: pReq=%p type=%d fFlags=%#x\n", pReq, pReq->enmType, pReq->fFlags));
934
935 /*
936 * Process the request.
937 */
938 Assert(pReq->enmState == VMREQSTATE_QUEUED);
939 pReq->enmState = VMREQSTATE_PROCESSING;
940 int rcRet = VINF_SUCCESS; /* the return code of this function. */
941 int rcReq = VERR_NOT_IMPLEMENTED; /* the request status. */
942 switch (pReq->enmType)
943 {
944 /*
945 * A packed down call frame.
946 */
947 case VMREQTYPE_INTERNAL:
948 {
949 uintptr_t *pauArgs = &pReq->u.Internal.aArgs[0];
950 union
951 {
952 PFNRT pfn;
953 DECLCALLBACKMEMBER(int, pfn00)(void);
954 DECLCALLBACKMEMBER(int, pfn01)(uintptr_t);
955 DECLCALLBACKMEMBER(int, pfn02)(uintptr_t, uintptr_t);
956 DECLCALLBACKMEMBER(int, pfn03)(uintptr_t, uintptr_t, uintptr_t);
957 DECLCALLBACKMEMBER(int, pfn04)(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
958 DECLCALLBACKMEMBER(int, pfn05)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
959 DECLCALLBACKMEMBER(int, pfn06)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
960 DECLCALLBACKMEMBER(int, pfn07)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
961 DECLCALLBACKMEMBER(int, pfn08)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
962 DECLCALLBACKMEMBER(int, pfn09)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
963 DECLCALLBACKMEMBER(int, pfn10)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
964 DECLCALLBACKMEMBER(int, pfn11)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
965 DECLCALLBACKMEMBER(int, pfn12)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
966 } u;
967 u.pfn = pReq->u.Internal.pfn;
968#ifdef RT_ARCH_AMD64
969 switch (pReq->u.Internal.cArgs)
970 {
971 case 0: rcRet = u.pfn00(); break;
972 case 1: rcRet = u.pfn01(pauArgs[0]); break;
973 case 2: rcRet = u.pfn02(pauArgs[0], pauArgs[1]); break;
974 case 3: rcRet = u.pfn03(pauArgs[0], pauArgs[1], pauArgs[2]); break;
975 case 4: rcRet = u.pfn04(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3]); break;
976 case 5: rcRet = u.pfn05(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4]); break;
977 case 6: rcRet = u.pfn06(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5]); break;
978 case 7: rcRet = u.pfn07(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6]); break;
979 case 8: rcRet = u.pfn08(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7]); break;
980 case 9: rcRet = u.pfn09(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7], pauArgs[8]); break;
981 case 10: rcRet = u.pfn10(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7], pauArgs[8], pauArgs[9]); break;
982 case 11: rcRet = u.pfn11(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7], pauArgs[8], pauArgs[9], pauArgs[10]); break;
983 case 12: rcRet = u.pfn12(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7], pauArgs[8], pauArgs[9], pauArgs[10], pauArgs[11]); break;
984 default:
985 AssertReleaseMsgFailed(("cArgs=%d\n", pReq->u.Internal.cArgs));
986 rcRet = rcReq = VERR_INTERNAL_ERROR;
987 break;
988 }
989#else /* x86: */
990 size_t cbArgs = pReq->u.Internal.cArgs * sizeof(uintptr_t);
991# ifdef __GNUC__
992 __asm__ __volatile__("movl %%esp, %%edx\n\t"
993 "subl %2, %%esp\n\t"
994 "andl $0xfffffff0, %%esp\n\t"
995 "shrl $2, %2\n\t"
996 "movl %%esp, %%edi\n\t"
997 "rep movsl\n\t"
998 "movl %%edx, %%edi\n\t"
999 "call *%%eax\n\t"
1000 "mov %%edi, %%esp\n\t"
1001 : "=a" (rcRet),
1002 "=S" (pauArgs),
1003 "=c" (cbArgs)
1004 : "0" (u.pfn),
1005 "1" (pauArgs),
1006 "2" (cbArgs)
1007 : "edi", "edx");
1008# else
1009 __asm
1010 {
1011 xor edx, edx /* just mess it up. */
1012 mov eax, u.pfn
1013 mov ecx, cbArgs
1014 shr ecx, 2
1015 mov esi, pauArgs
1016 mov ebx, esp
1017 sub esp, cbArgs
1018 and esp, 0xfffffff0
1019 mov edi, esp
1020 rep movsd
1021 call eax
1022 mov esp, ebx
1023 mov rcRet, eax
1024 }
1025# endif
1026#endif /* x86 */
1027 if ((pReq->fFlags & (VMREQFLAGS_RETURN_MASK)) == VMREQFLAGS_VOID)
1028 rcRet = VINF_SUCCESS;
1029 rcReq = rcRet;
1030 break;
1031 }
1032
1033 default:
1034 AssertMsgFailed(("pReq->enmType=%d\n", pReq->enmType));
1035 rcReq = VERR_NOT_IMPLEMENTED;
1036 break;
1037 }
1038
1039 /*
1040 * Complete the request.
1041 */
1042 pReq->iStatus = rcReq;
1043 pReq->enmState = VMREQSTATE_COMPLETED;
1044 if (pReq->fFlags & VMREQFLAGS_NO_WAIT)
1045 {
1046 /* Free the packet, nobody is waiting. */
1047 LogFlow(("vmR3ReqProcessOne: Completed request %p: rcReq=%Rrc rcRet=%Rrc - freeing it\n",
1048 pReq, rcReq, rcRet));
1049 VMR3ReqFree(pReq);
1050 }
1051 else
1052 {
1053 /* Notify the waiter and him free up the packet. */
1054 LogFlow(("vmR3ReqProcessOne: Completed request %p: rcReq=%Rrc rcRet=%Rrc - notifying waiting thread\n",
1055 pReq, rcReq, rcRet));
1056 ASMAtomicXchgSize(&pReq->fEventSemClear, false);
1057 int rc2 = RTSemEventSignal(pReq->EventSem);
1058 if (RT_FAILURE(rc2))
1059 {
1060 AssertRC(rc2);
1061 rcRet = rc2;
1062 }
1063 }
1064 return rcRet;
1065}
1066
1067
1068
1069
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette