1 | /* $Id: GVMMR0.cpp 13796 2008-11-04 18:37:33Z vboxsync $ */
|
---|
2 | /** @file
|
---|
3 | * GVMM - Global VM Manager.
|
---|
4 | */
|
---|
5 |
|
---|
6 | /*
|
---|
7 | * Copyright (C) 2007 Sun Microsystems, Inc.
|
---|
8 | *
|
---|
9 | * This file is part of VirtualBox Open Source Edition (OSE), as
|
---|
10 | * available from http://www.virtualbox.org. This file is free software;
|
---|
11 | * you can redistribute it and/or modify it under the terms of the GNU
|
---|
12 | * General Public License (GPL) as published by the Free Software
|
---|
13 | * Foundation, in version 2 as it comes in the "COPYING" file of the
|
---|
14 | * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
|
---|
15 | * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
|
---|
16 | *
|
---|
17 | * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
|
---|
18 | * Clara, CA 95054 USA or visit http://www.sun.com if you need
|
---|
19 | * additional information or have any questions.
|
---|
20 | */
|
---|
21 |
|
---|
22 |
|
---|
23 | /** @page pg_gvmm GVMM - The Global VM Manager
|
---|
24 | *
|
---|
25 | * The Global VM Manager lives in ring-0. It's main function at the moment
|
---|
26 | * is to manage a list of all running VMs, keep a ring-0 only structure (GVM)
|
---|
27 | * for each of them, and assign them unique identifiers (so GMM can track
|
---|
28 | * page owners). The idea for the future is to add an idle priority kernel
|
---|
29 | * thread that can take care of tasks like page sharing.
|
---|
30 | *
|
---|
31 | * The GVMM will create a ring-0 object for each VM when it's registered,
|
---|
32 | * this is both for session cleanup purposes and for having a point where
|
---|
33 | * it's possible to implement usage polices later (in SUPR0ObjRegister).
|
---|
34 | */
|
---|
35 |
|
---|
36 |
|
---|
37 | /*******************************************************************************
|
---|
38 | * Header Files *
|
---|
39 | *******************************************************************************/
|
---|
40 | #define LOG_GROUP LOG_GROUP_GVMM
|
---|
41 | #include <VBox/gvmm.h>
|
---|
42 | #include "GVMMR0Internal.h"
|
---|
43 | #include <VBox/gvm.h>
|
---|
44 | #include <VBox/vm.h>
|
---|
45 | #include <VBox/err.h>
|
---|
46 | #include <iprt/alloc.h>
|
---|
47 | #include <iprt/semaphore.h>
|
---|
48 | #include <iprt/time.h>
|
---|
49 | #include <VBox/log.h>
|
---|
50 | #include <iprt/thread.h>
|
---|
51 | #include <iprt/param.h>
|
---|
52 | #include <iprt/string.h>
|
---|
53 | #include <iprt/assert.h>
|
---|
54 | #include <iprt/mem.h>
|
---|
55 | #include <iprt/memobj.h>
|
---|
56 |
|
---|
57 |
|
---|
58 | /*******************************************************************************
|
---|
59 | * Structures and Typedefs *
|
---|
60 | *******************************************************************************/
|
---|
61 |
|
---|
62 | /**
|
---|
63 | * Global VM handle.
|
---|
64 | */
|
---|
65 | typedef struct GVMHANDLE
|
---|
66 | {
|
---|
67 | /** The index of the next handle in the list (free or used). (0 is nil.) */
|
---|
68 | uint16_t volatile iNext;
|
---|
69 | /** Our own index / handle value. */
|
---|
70 | uint16_t iSelf;
|
---|
71 | /** The pointer to the ring-0 only (aka global) VM structure. */
|
---|
72 | PGVM pGVM;
|
---|
73 | /** The ring-0 mapping of the shared VM instance data. */
|
---|
74 | PVM pVM;
|
---|
75 | /** The virtual machine object. */
|
---|
76 | void *pvObj;
|
---|
77 | /** The session this VM is associated with. */
|
---|
78 | PSUPDRVSESSION pSession;
|
---|
79 | /** The ring-0 handle of the EMT thread.
|
---|
80 | * This is used for assertions and similar cases where we need to find the VM handle. */
|
---|
81 | RTNATIVETHREAD hEMT;
|
---|
82 | } GVMHANDLE;
|
---|
83 | /** Pointer to a global VM handle. */
|
---|
84 | typedef GVMHANDLE *PGVMHANDLE;
|
---|
85 |
|
---|
86 | /**
|
---|
87 | * The GVMM instance data.
|
---|
88 | */
|
---|
89 | typedef struct GVMM
|
---|
90 | {
|
---|
91 | /** Eyecatcher / magic. */
|
---|
92 | uint32_t u32Magic;
|
---|
93 | /** The index of the head of the free handle chain. (0 is nil.) */
|
---|
94 | uint16_t volatile iFreeHead;
|
---|
95 | /** The index of the head of the active handle chain. (0 is nil.) */
|
---|
96 | uint16_t volatile iUsedHead;
|
---|
97 | /** The number of VMs. */
|
---|
98 | uint16_t volatile cVMs;
|
---|
99 | // /** The number of halted EMT threads. */
|
---|
100 | // uint16_t volatile cHaltedEMTs;
|
---|
101 | /** The lock used to serialize VM creation, destruction and associated events that
|
---|
102 | * isn't performance critical. Owners may acquire the list lock. */
|
---|
103 | RTSEMFASTMUTEX CreateDestroyLock;
|
---|
104 | /** The lock used to serialize used list updates and accesses.
|
---|
105 | * This indirectly includes scheduling since the scheduler will have to walk the
|
---|
106 | * used list to examin running VMs. Owners may not acquire any other locks. */
|
---|
107 | RTSEMFASTMUTEX UsedLock;
|
---|
108 | /** The handle array.
|
---|
109 | * The size of this array defines the maximum number of currently running VMs.
|
---|
110 | * The first entry is unused as it represents the NIL handle. */
|
---|
111 | GVMHANDLE aHandles[128];
|
---|
112 |
|
---|
113 | /** @gcfgm{/GVMM/cVMsMeansCompany, 32-bit, 0, UINT32_MAX, 1}
|
---|
114 | * The number of VMs that means we no longer consider ourselves alone on a CPU/Core.
|
---|
115 | */
|
---|
116 | uint32_t cVMsMeansCompany;
|
---|
117 | /** @gcfgm{/GVMM/MinSleepAlone,32-bit, 0, 100000000, 750000, ns}
|
---|
118 | * The minimum sleep time for when we're alone, in nano seconds.
|
---|
119 | */
|
---|
120 | uint32_t nsMinSleepAlone;
|
---|
121 | /** @gcfgm{/GVMM/MinSleepCompany,32-bit,0, 100000000, 15000, ns}
|
---|
122 | * The minimum sleep time for when we've got company, in nano seconds.
|
---|
123 | */
|
---|
124 | uint32_t nsMinSleepCompany;
|
---|
125 | /** @gcfgm{/GVMM/EarlyWakeUp1, 32-bit, 0, 100000000, 25000, ns}
|
---|
126 | * The limit for the first round of early wakeups, given in nano seconds.
|
---|
127 | */
|
---|
128 | uint32_t nsEarlyWakeUp1;
|
---|
129 | /** @gcfgm{/GVMM/EarlyWakeUp2, 32-bit, 0, 100000000, 50000, ns}
|
---|
130 | * The limit for the second round of early wakeups, given in nano seconds.
|
---|
131 | */
|
---|
132 | uint32_t nsEarlyWakeUp2;
|
---|
133 | } GVMM;
|
---|
134 | /** Pointer to the GVMM instance data. */
|
---|
135 | typedef GVMM *PGVMM;
|
---|
136 |
|
---|
137 | /** The GVMM::u32Magic value (Charlie Haden). */
|
---|
138 | #define GVMM_MAGIC 0x19370806
|
---|
139 |
|
---|
140 |
|
---|
141 |
|
---|
142 | /*******************************************************************************
|
---|
143 | * Global Variables *
|
---|
144 | *******************************************************************************/
|
---|
145 | /** Pointer to the GVMM instance data.
|
---|
146 | * (Just my general dislike for global variables.) */
|
---|
147 | static PGVMM g_pGVMM = NULL;
|
---|
148 |
|
---|
149 | /** Macro for obtaining and validating the g_pGVMM pointer.
|
---|
150 | * On failure it will return from the invoking function with the specified return value.
|
---|
151 | *
|
---|
152 | * @param pGVMM The name of the pGVMM variable.
|
---|
153 | * @param rc The return value on failure. Use VERR_INTERNAL_ERROR for
|
---|
154 | * VBox status codes.
|
---|
155 | */
|
---|
156 | #define GVMM_GET_VALID_INSTANCE(pGVMM, rc) \
|
---|
157 | do { \
|
---|
158 | (pGVMM) = g_pGVMM;\
|
---|
159 | AssertPtrReturn((pGVMM), (rc)); \
|
---|
160 | AssertMsgReturn((pGVMM)->u32Magic == GVMM_MAGIC, ("%p - %#x\n", (pGVMM), (pGVMM)->u32Magic), (rc)); \
|
---|
161 | } while (0)
|
---|
162 |
|
---|
163 | /** Macro for obtaining and validating the g_pGVMM pointer, void function variant.
|
---|
164 | * On failure it will return from the invoking function.
|
---|
165 | *
|
---|
166 | * @param pGVMM The name of the pGVMM variable.
|
---|
167 | */
|
---|
168 | #define GVMM_GET_VALID_INSTANCE_VOID(pGVMM) \
|
---|
169 | do { \
|
---|
170 | (pGVMM) = g_pGVMM;\
|
---|
171 | AssertPtrReturnVoid((pGVMM)); \
|
---|
172 | AssertMsgReturnVoid((pGVMM)->u32Magic == GVMM_MAGIC, ("%p - %#x\n", (pGVMM), (pGVMM)->u32Magic)); \
|
---|
173 | } while (0)
|
---|
174 |
|
---|
175 |
|
---|
176 | /*******************************************************************************
|
---|
177 | * Internal Functions *
|
---|
178 | *******************************************************************************/
|
---|
179 | static void gvmmR0InitPerVMData(PGVM pGVM);
|
---|
180 | static DECLCALLBACK(void) gvmmR0HandleObjDestructor(void *pvObj, void *pvGVMM, void *pvHandle);
|
---|
181 | static int gvmmR0ByVM(PVM pVM, PGVM *ppGVM, PGVMM *ppGVMM, bool fTakeUsedLock);
|
---|
182 | static int gvmmR0ByVMAndEMT(PVM pVM, PGVM *ppGVM, PGVMM *ppGVMM);
|
---|
183 |
|
---|
184 |
|
---|
185 | /**
|
---|
186 | * Initializes the GVMM.
|
---|
187 | *
|
---|
188 | * This is called while owninng the loader sempahore (see supdrvIOCtl_LdrLoad()).
|
---|
189 | *
|
---|
190 | * @returns VBox status code.
|
---|
191 | */
|
---|
192 | GVMMR0DECL(int) GVMMR0Init(void)
|
---|
193 | {
|
---|
194 | LogFlow(("GVMMR0Init:\n"));
|
---|
195 |
|
---|
196 | /*
|
---|
197 | * Allocate and initialize the instance data.
|
---|
198 | */
|
---|
199 | PGVMM pGVMM = (PGVMM)RTMemAllocZ(sizeof(*pGVMM));
|
---|
200 | if (!pGVMM)
|
---|
201 | return VERR_NO_MEMORY;
|
---|
202 | int rc = RTSemFastMutexCreate(&pGVMM->CreateDestroyLock);
|
---|
203 | if (RT_SUCCESS(rc))
|
---|
204 | {
|
---|
205 | rc = RTSemFastMutexCreate(&pGVMM->UsedLock);
|
---|
206 | if (RT_SUCCESS(rc))
|
---|
207 | {
|
---|
208 | pGVMM->u32Magic = GVMM_MAGIC;
|
---|
209 | pGVMM->iUsedHead = 0;
|
---|
210 | pGVMM->iFreeHead = 1;
|
---|
211 |
|
---|
212 | /* the nil handle */
|
---|
213 | pGVMM->aHandles[0].iSelf = 0;
|
---|
214 | pGVMM->aHandles[0].iNext = 0;
|
---|
215 |
|
---|
216 | /* the tail */
|
---|
217 | unsigned i = RT_ELEMENTS(pGVMM->aHandles) - 1;
|
---|
218 | pGVMM->aHandles[i].iSelf = i;
|
---|
219 | pGVMM->aHandles[i].iNext = 0; /* nil */
|
---|
220 |
|
---|
221 | /* the rest */
|
---|
222 | while (i-- > 1)
|
---|
223 | {
|
---|
224 | pGVMM->aHandles[i].iSelf = i;
|
---|
225 | pGVMM->aHandles[i].iNext = i + 1;
|
---|
226 | }
|
---|
227 |
|
---|
228 | /* The default configuration values. */
|
---|
229 | pGVMM->cVMsMeansCompany = 1; /** @todo should be adjusted to relative to the cpu count or something... */
|
---|
230 | pGVMM->nsMinSleepAlone = 750000 /* ns (0.750 ms) */; /** @todo this should be adjusted to be 75% (or something) of the scheduler granularity... */
|
---|
231 | pGVMM->nsMinSleepCompany = 15000 /* ns (0.015 ms) */;
|
---|
232 | pGVMM->nsEarlyWakeUp1 = 25000 /* ns (0.025 ms) */;
|
---|
233 | pGVMM->nsEarlyWakeUp2 = 50000 /* ns (0.050 ms) */;
|
---|
234 |
|
---|
235 | g_pGVMM = pGVMM;
|
---|
236 | LogFlow(("GVMMR0Init: pGVMM=%p\n", pGVMM));
|
---|
237 | return VINF_SUCCESS;
|
---|
238 | }
|
---|
239 |
|
---|
240 | RTSemFastMutexDestroy(pGVMM->CreateDestroyLock);
|
---|
241 | }
|
---|
242 |
|
---|
243 | RTMemFree(pGVMM);
|
---|
244 | return rc;
|
---|
245 | }
|
---|
246 |
|
---|
247 |
|
---|
248 | /**
|
---|
249 | * Terminates the GVM.
|
---|
250 | *
|
---|
251 | * This is called while owning the loader semaphore (see supdrvLdrFree()).
|
---|
252 | * And unless something is wrong, there should be absolutely no VMs
|
---|
253 | * registered at this point.
|
---|
254 | */
|
---|
255 | GVMMR0DECL(void) GVMMR0Term(void)
|
---|
256 | {
|
---|
257 | LogFlow(("GVMMR0Term:\n"));
|
---|
258 |
|
---|
259 | PGVMM pGVMM = g_pGVMM;
|
---|
260 | g_pGVMM = NULL;
|
---|
261 | if (RT_UNLIKELY(!VALID_PTR(pGVMM)))
|
---|
262 | {
|
---|
263 | SUPR0Printf("GVMMR0Term: pGVMM=%p\n", pGVMM);
|
---|
264 | return;
|
---|
265 | }
|
---|
266 |
|
---|
267 | pGVMM->u32Magic++;
|
---|
268 |
|
---|
269 | RTSemFastMutexDestroy(pGVMM->UsedLock);
|
---|
270 | pGVMM->UsedLock = NIL_RTSEMFASTMUTEX;
|
---|
271 | RTSemFastMutexDestroy(pGVMM->CreateDestroyLock);
|
---|
272 | pGVMM->CreateDestroyLock = NIL_RTSEMFASTMUTEX;
|
---|
273 |
|
---|
274 | pGVMM->iFreeHead = 0;
|
---|
275 | if (pGVMM->iUsedHead)
|
---|
276 | {
|
---|
277 | SUPR0Printf("GVMMR0Term: iUsedHead=%#x! (cVMs=%#x)\n", pGVMM->iUsedHead, pGVMM->cVMs);
|
---|
278 | pGVMM->iUsedHead = 0;
|
---|
279 | }
|
---|
280 |
|
---|
281 | RTMemFree(pGVMM);
|
---|
282 | }
|
---|
283 |
|
---|
284 |
|
---|
285 | /**
|
---|
286 | * A quick hack for setting global config values.
|
---|
287 | *
|
---|
288 | * @returns VBox status code.
|
---|
289 | *
|
---|
290 | * @param pSession The session handle. Used for authentication.
|
---|
291 | * @param pszName The variable name.
|
---|
292 | * @param u64Value The new value.
|
---|
293 | */
|
---|
294 | GVMMR0DECL(int) GVMMR0SetConfig(PSUPDRVSESSION pSession, const char *pszName, uint64_t u64Value)
|
---|
295 | {
|
---|
296 | /*
|
---|
297 | * Validate input.
|
---|
298 | */
|
---|
299 | PGVMM pGVMM;
|
---|
300 | GVMM_GET_VALID_INSTANCE(pGVMM, VERR_INTERNAL_ERROR);
|
---|
301 | AssertPtrReturn(pSession, VERR_INVALID_HANDLE);
|
---|
302 | AssertPtrReturn(pszName, VERR_INVALID_POINTER);
|
---|
303 |
|
---|
304 | /*
|
---|
305 | * String switch time!
|
---|
306 | */
|
---|
307 | if (strncmp(pszName, "/GVMM/", sizeof("/GVMM/") - 1))
|
---|
308 | return VERR_CFGM_VALUE_NOT_FOUND; /* borrow status codes from CFGM... */
|
---|
309 | int rc = VINF_SUCCESS;
|
---|
310 | pszName += sizeof("/GVMM/") - 1;
|
---|
311 | if (!strcmp(pszName, "cVMsMeansCompany"))
|
---|
312 | {
|
---|
313 | if (u64Value <= UINT32_MAX)
|
---|
314 | pGVMM->cVMsMeansCompany = u64Value;
|
---|
315 | else
|
---|
316 | rc = VERR_OUT_OF_RANGE;
|
---|
317 | }
|
---|
318 | else if (!strcmp(pszName, "MinSleepAlone"))
|
---|
319 | {
|
---|
320 | if (u64Value <= 100000000)
|
---|
321 | pGVMM->nsMinSleepAlone = u64Value;
|
---|
322 | else
|
---|
323 | rc = VERR_OUT_OF_RANGE;
|
---|
324 | }
|
---|
325 | else if (!strcmp(pszName, "MinSleepCompany"))
|
---|
326 | {
|
---|
327 | if (u64Value <= 100000000)
|
---|
328 | pGVMM->nsMinSleepCompany = u64Value;
|
---|
329 | else
|
---|
330 | rc = VERR_OUT_OF_RANGE;
|
---|
331 | }
|
---|
332 | else if (!strcmp(pszName, "EarlyWakeUp1"))
|
---|
333 | {
|
---|
334 | if (u64Value <= 100000000)
|
---|
335 | pGVMM->nsEarlyWakeUp1 = u64Value;
|
---|
336 | else
|
---|
337 | rc = VERR_OUT_OF_RANGE;
|
---|
338 | }
|
---|
339 | else if (!strcmp(pszName, "EarlyWakeUp2"))
|
---|
340 | {
|
---|
341 | if (u64Value <= 100000000)
|
---|
342 | pGVMM->nsEarlyWakeUp2 = u64Value;
|
---|
343 | else
|
---|
344 | rc = VERR_OUT_OF_RANGE;
|
---|
345 | }
|
---|
346 | else
|
---|
347 | rc = VERR_CFGM_VALUE_NOT_FOUND;
|
---|
348 | return rc;
|
---|
349 | }
|
---|
350 |
|
---|
351 |
|
---|
352 | /**
|
---|
353 | * A quick hack for getting global config values.
|
---|
354 | *
|
---|
355 | * @returns VBox status code.
|
---|
356 | *
|
---|
357 | * @param pSession The session handle. Used for authentication.
|
---|
358 | * @param pszName The variable name.
|
---|
359 | * @param u64Value The new value.
|
---|
360 | */
|
---|
361 | GVMMR0DECL(int) GVMMR0QueryConfig(PSUPDRVSESSION pSession, const char *pszName, uint64_t *pu64Value)
|
---|
362 | {
|
---|
363 | /*
|
---|
364 | * Validate input.
|
---|
365 | */
|
---|
366 | PGVMM pGVMM;
|
---|
367 | GVMM_GET_VALID_INSTANCE(pGVMM, VERR_INTERNAL_ERROR);
|
---|
368 | AssertPtrReturn(pSession, VERR_INVALID_HANDLE);
|
---|
369 | AssertPtrReturn(pszName, VERR_INVALID_POINTER);
|
---|
370 | AssertPtrReturn(pu64Value, VERR_INVALID_POINTER);
|
---|
371 |
|
---|
372 | /*
|
---|
373 | * String switch time!
|
---|
374 | */
|
---|
375 | if (strncmp(pszName, "/GVMM/", sizeof("/GVMM/") - 1))
|
---|
376 | return VERR_CFGM_VALUE_NOT_FOUND; /* borrow status codes from CFGM... */
|
---|
377 | int rc = VINF_SUCCESS;
|
---|
378 | pszName += sizeof("/GVMM/") - 1;
|
---|
379 | if (!strcmp(pszName, "cVMsMeansCompany"))
|
---|
380 | *pu64Value = pGVMM->cVMsMeansCompany;
|
---|
381 | else if (!strcmp(pszName, "MinSleepAlone"))
|
---|
382 | *pu64Value = pGVMM->nsMinSleepAlone;
|
---|
383 | else if (!strcmp(pszName, "MinSleepCompany"))
|
---|
384 | *pu64Value = pGVMM->nsMinSleepCompany;
|
---|
385 | else if (!strcmp(pszName, "EarlyWakeUp1"))
|
---|
386 | *pu64Value = pGVMM->nsEarlyWakeUp1;
|
---|
387 | else if (!strcmp(pszName, "EarlyWakeUp2"))
|
---|
388 | *pu64Value = pGVMM->nsEarlyWakeUp2;
|
---|
389 | else
|
---|
390 | rc = VERR_CFGM_VALUE_NOT_FOUND;
|
---|
391 | return rc;
|
---|
392 | }
|
---|
393 |
|
---|
394 |
|
---|
395 | /**
|
---|
396 | * Try acquire the 'used' lock.
|
---|
397 | *
|
---|
398 | * @returns IPRT status code, see RTSemFastMutexRequest.
|
---|
399 | * @param pGVMM The GVMM instance data.
|
---|
400 | */
|
---|
401 | DECLINLINE(int) gvmmR0UsedLock(PGVMM pGVMM)
|
---|
402 | {
|
---|
403 | LogFlow(("++gvmmR0UsedLock(%p)\n", pGVMM));
|
---|
404 | int rc = RTSemFastMutexRequest(pGVMM->UsedLock);
|
---|
405 | LogFlow(("gvmmR0UsedLock(%p)->%Rrc\n", pGVMM, rc));
|
---|
406 | return rc;
|
---|
407 | }
|
---|
408 |
|
---|
409 |
|
---|
410 | /**
|
---|
411 | * Release the 'used' lock.
|
---|
412 | *
|
---|
413 | * @returns IPRT status code, see RTSemFastMutexRelease.
|
---|
414 | * @param pGVMM The GVMM instance data.
|
---|
415 | */
|
---|
416 | DECLINLINE(int) gvmmR0UsedUnlock(PGVMM pGVMM)
|
---|
417 | {
|
---|
418 | LogFlow(("--gvmmR0UsedUnlock(%p)\n", pGVMM));
|
---|
419 | int rc = RTSemFastMutexRelease(pGVMM->UsedLock);
|
---|
420 | AssertRC(rc);
|
---|
421 | return rc;
|
---|
422 | }
|
---|
423 |
|
---|
424 |
|
---|
425 | /**
|
---|
426 | * Try acquire the 'create & destroy' lock.
|
---|
427 | *
|
---|
428 | * @returns IPRT status code, see RTSemFastMutexRequest.
|
---|
429 | * @param pGVMM The GVMM instance data.
|
---|
430 | */
|
---|
431 | DECLINLINE(int) gvmmR0CreateDestroyLock(PGVMM pGVMM)
|
---|
432 | {
|
---|
433 | LogFlow(("++gvmmR0CreateDestroyLock(%p)\n", pGVMM));
|
---|
434 | int rc = RTSemFastMutexRequest(pGVMM->CreateDestroyLock);
|
---|
435 | LogFlow(("gvmmR0CreateDestroyLock(%p)->%Rrc\n", pGVMM, rc));
|
---|
436 | return rc;
|
---|
437 | }
|
---|
438 |
|
---|
439 |
|
---|
440 | /**
|
---|
441 | * Release the 'create & destroy' lock.
|
---|
442 | *
|
---|
443 | * @returns IPRT status code, see RTSemFastMutexRequest.
|
---|
444 | * @param pGVMM The GVMM instance data.
|
---|
445 | */
|
---|
446 | DECLINLINE(int) gvmmR0CreateDestroyUnlock(PGVMM pGVMM)
|
---|
447 | {
|
---|
448 | LogFlow(("--gvmmR0CreateDestroyUnlock(%p)\n", pGVMM));
|
---|
449 | int rc = RTSemFastMutexRelease(pGVMM->CreateDestroyLock);
|
---|
450 | AssertRC(rc);
|
---|
451 | return rc;
|
---|
452 | }
|
---|
453 |
|
---|
454 |
|
---|
455 | /**
|
---|
456 | * Request wrapper for the GVMMR0CreateVM API.
|
---|
457 | *
|
---|
458 | * @returns VBox status code.
|
---|
459 | * @param pReq The request buffer.
|
---|
460 | */
|
---|
461 | GVMMR0DECL(int) GVMMR0CreateVMReq(PGVMMCREATEVMREQ pReq)
|
---|
462 | {
|
---|
463 | /*
|
---|
464 | * Validate the request.
|
---|
465 | */
|
---|
466 | if (!VALID_PTR(pReq))
|
---|
467 | return VERR_INVALID_POINTER;
|
---|
468 | if (pReq->Hdr.cbReq != sizeof(*pReq))
|
---|
469 | return VERR_INVALID_PARAMETER;
|
---|
470 | if (!VALID_PTR(pReq->pSession))
|
---|
471 | return VERR_INVALID_POINTER;
|
---|
472 |
|
---|
473 | /*
|
---|
474 | * Execute it.
|
---|
475 | */
|
---|
476 | PVM pVM;
|
---|
477 | pReq->pVMR0 = NULL;
|
---|
478 | pReq->pVMR3 = NIL_RTR3PTR;
|
---|
479 | int rc = GVMMR0CreateVM(pReq->pSession, pReq->cCPUs, &pVM);
|
---|
480 | if (RT_SUCCESS(rc))
|
---|
481 | {
|
---|
482 | pReq->pVMR0 = pVM;
|
---|
483 | pReq->pVMR3 = pVM->pVMR3;
|
---|
484 | }
|
---|
485 | return rc;
|
---|
486 | }
|
---|
487 |
|
---|
488 |
|
---|
489 | /**
|
---|
490 | * Allocates the VM structure and registers it with GVM.
|
---|
491 | *
|
---|
492 | * The caller will become the VM owner and there by the EMT.
|
---|
493 | *
|
---|
494 | * @returns VBox status code.
|
---|
495 | * @param pSession The support driver session.
|
---|
496 | * @param cCPUs Number of virtual CPUs for the new VM.
|
---|
497 | * @param ppVM Where to store the pointer to the VM structure.
|
---|
498 | *
|
---|
499 | * @thread EMT.
|
---|
500 | */
|
---|
501 | GVMMR0DECL(int) GVMMR0CreateVM(PSUPDRVSESSION pSession, uint32_t cCPUs, PVM *ppVM)
|
---|
502 | {
|
---|
503 | LogFlow(("GVMMR0CreateVM: pSession=%p\n", pSession));
|
---|
504 | PGVMM pGVMM;
|
---|
505 | GVMM_GET_VALID_INSTANCE(pGVMM, VERR_INTERNAL_ERROR);
|
---|
506 |
|
---|
507 | AssertPtrReturn(ppVM, VERR_INVALID_POINTER);
|
---|
508 | *ppVM = NULL;
|
---|
509 |
|
---|
510 | if ( cCPUs == 0
|
---|
511 | || cCPUs > VMCPU_MAX_CPU_COUNT)
|
---|
512 | return VERR_INVALID_PARAMETER;
|
---|
513 |
|
---|
514 | RTNATIVETHREAD hEMT = RTThreadNativeSelf();
|
---|
515 | AssertReturn(hEMT != NIL_RTNATIVETHREAD, VERR_INTERNAL_ERROR);
|
---|
516 |
|
---|
517 | /*
|
---|
518 | * The whole allocation process is protected by the lock.
|
---|
519 | */
|
---|
520 | int rc = gvmmR0CreateDestroyLock(pGVMM);
|
---|
521 | AssertRCReturn(rc, rc);
|
---|
522 |
|
---|
523 | /*
|
---|
524 | * Allocate a handle first so we don't waste resources unnecessarily.
|
---|
525 | */
|
---|
526 | uint16_t iHandle = pGVMM->iFreeHead;
|
---|
527 | if (iHandle)
|
---|
528 | {
|
---|
529 | PGVMHANDLE pHandle = &pGVMM->aHandles[iHandle];
|
---|
530 |
|
---|
531 | /* consistency checks, a bit paranoid as always. */
|
---|
532 | if ( !pHandle->pVM
|
---|
533 | && !pHandle->pGVM
|
---|
534 | && !pHandle->pvObj
|
---|
535 | && pHandle->iSelf == iHandle)
|
---|
536 | {
|
---|
537 | pHandle->pvObj = SUPR0ObjRegister(pSession, SUPDRVOBJTYPE_VM, gvmmR0HandleObjDestructor, pGVMM, pHandle);
|
---|
538 | if (pHandle->pvObj)
|
---|
539 | {
|
---|
540 | /*
|
---|
541 | * Move the handle from the free to used list and perform permission checks.
|
---|
542 | */
|
---|
543 | rc = gvmmR0UsedLock(pGVMM);
|
---|
544 | AssertRC(rc);
|
---|
545 |
|
---|
546 | pGVMM->iFreeHead = pHandle->iNext;
|
---|
547 | pHandle->iNext = pGVMM->iUsedHead;
|
---|
548 | pGVMM->iUsedHead = iHandle;
|
---|
549 | pGVMM->cVMs++;
|
---|
550 |
|
---|
551 | pHandle->pVM = NULL;
|
---|
552 | pHandle->pGVM = NULL;
|
---|
553 | pHandle->pSession = pSession;
|
---|
554 | pHandle->hEMT = NIL_RTNATIVETHREAD;
|
---|
555 |
|
---|
556 | gvmmR0UsedUnlock(pGVMM);
|
---|
557 |
|
---|
558 | rc = SUPR0ObjVerifyAccess(pHandle->pvObj, pSession, NULL);
|
---|
559 | if (RT_SUCCESS(rc))
|
---|
560 | {
|
---|
561 | /*
|
---|
562 | * Allocate the global VM structure (GVM) and initialize it.
|
---|
563 | */
|
---|
564 | PGVM pGVM = (PGVM)RTMemAllocZ(sizeof(*pGVM));
|
---|
565 | if (pGVM)
|
---|
566 | {
|
---|
567 | pGVM->u32Magic = GVM_MAGIC;
|
---|
568 | pGVM->hSelf = iHandle;
|
---|
569 | pGVM->hEMT = NIL_RTNATIVETHREAD;
|
---|
570 | pGVM->pVM = NULL;
|
---|
571 |
|
---|
572 | gvmmR0InitPerVMData(pGVM);
|
---|
573 | /* GMMR0InitPerVMData(pGVM); - later */
|
---|
574 |
|
---|
575 | /*
|
---|
576 | * Allocate the shared VM structure and associated page array.
|
---|
577 | */
|
---|
578 | const size_t cbVM = RT_UOFFSETOF(VM, aCpus[cCPUs]);
|
---|
579 | const size_t cPages = RT_ALIGN(cbVM, PAGE_SIZE) >> PAGE_SHIFT;
|
---|
580 | rc = RTR0MemObjAllocLow(&pGVM->gvmm.s.VMMemObj, cPages << PAGE_SHIFT, false /* fExecutable */);
|
---|
581 | if (RT_SUCCESS(rc))
|
---|
582 | {
|
---|
583 | PVM pVM = (PVM)RTR0MemObjAddress(pGVM->gvmm.s.VMMemObj); AssertPtr(pVM);
|
---|
584 | memset(pVM, 0, cPages << PAGE_SHIFT);
|
---|
585 | pVM->enmVMState = VMSTATE_CREATING;
|
---|
586 | pVM->pVMR0 = pVM;
|
---|
587 | pVM->pSession = pSession;
|
---|
588 | pVM->hSelf = iHandle;
|
---|
589 | pVM->cbSelf = cbVM;
|
---|
590 | pVM->cCPUs = cCPUs;
|
---|
591 | pVM->offVMCPU = RT_UOFFSETOF(VM, aCpus);
|
---|
592 |
|
---|
593 | rc = RTR0MemObjAllocPage(&pGVM->gvmm.s.VMPagesMemObj, cPages * sizeof(SUPPAGE), false /* fExecutable */);
|
---|
594 | if (RT_SUCCESS(rc))
|
---|
595 | {
|
---|
596 | PSUPPAGE paPages = (PSUPPAGE)RTR0MemObjAddress(pGVM->gvmm.s.VMPagesMemObj); AssertPtr(paPages);
|
---|
597 | for (size_t iPage = 0; iPage < cPages; iPage++)
|
---|
598 | {
|
---|
599 | paPages[iPage].uReserved = 0;
|
---|
600 | paPages[iPage].Phys = RTR0MemObjGetPagePhysAddr(pGVM->gvmm.s.VMMemObj, iPage);
|
---|
601 | Assert(paPages[iPage].Phys != NIL_RTHCPHYS);
|
---|
602 | }
|
---|
603 |
|
---|
604 | /*
|
---|
605 | * Map them into ring-3.
|
---|
606 | */
|
---|
607 | rc = RTR0MemObjMapUser(&pGVM->gvmm.s.VMMapObj, pGVM->gvmm.s.VMMemObj, (RTR3PTR)-1, 0,
|
---|
608 | RTMEM_PROT_READ | RTMEM_PROT_WRITE, NIL_RTR0PROCESS);
|
---|
609 | if (RT_SUCCESS(rc))
|
---|
610 | {
|
---|
611 | pVM->pVMR3 = RTR0MemObjAddressR3(pGVM->gvmm.s.VMMapObj);
|
---|
612 | AssertPtr((void *)pVM->pVMR3);
|
---|
613 |
|
---|
614 | /* Initialize all the VM pointers. */
|
---|
615 | for (uint32_t i = 0; i < cCPUs; i++)
|
---|
616 | {
|
---|
617 | pVM->aCpus[i].pVMR0 = pVM;
|
---|
618 | pVM->aCpus[i].pVMR3 = pVM->pVMR3;
|
---|
619 | }
|
---|
620 |
|
---|
621 | rc = RTR0MemObjMapUser(&pGVM->gvmm.s.VMPagesMapObj, pGVM->gvmm.s.VMPagesMemObj, (RTR3PTR)-1, 0,
|
---|
622 | RTMEM_PROT_READ | RTMEM_PROT_WRITE, NIL_RTR0PROCESS);
|
---|
623 | if (RT_SUCCESS(rc))
|
---|
624 | {
|
---|
625 | pVM->paVMPagesR3 = RTR0MemObjAddressR3(pGVM->gvmm.s.VMPagesMapObj);
|
---|
626 | AssertPtr((void *)pVM->paVMPagesR3);
|
---|
627 |
|
---|
628 | /* complete the handle - take the UsedLock sem just to be careful. */
|
---|
629 | rc = gvmmR0UsedLock(pGVMM);
|
---|
630 | AssertRC(rc);
|
---|
631 |
|
---|
632 | pHandle->pVM = pVM;
|
---|
633 | pHandle->pGVM = pGVM;
|
---|
634 | pHandle->hEMT = hEMT;
|
---|
635 | pGVM->pVM = pVM;
|
---|
636 | pGVM->hEMT = hEMT;
|
---|
637 |
|
---|
638 | gvmmR0UsedUnlock(pGVMM);
|
---|
639 | gvmmR0CreateDestroyUnlock(pGVMM);
|
---|
640 |
|
---|
641 | *ppVM = pVM;
|
---|
642 | Log(("GVMMR0CreateVM: pVM=%p pVMR3=%p pGVM=%p hGVM=%d\n", pVM, pVM->pVMR3, pGVM, iHandle));
|
---|
643 | return VINF_SUCCESS;
|
---|
644 | }
|
---|
645 |
|
---|
646 | RTR0MemObjFree(pGVM->gvmm.s.VMMapObj, false /* fFreeMappings */);
|
---|
647 | pGVM->gvmm.s.VMMapObj = NIL_RTR0MEMOBJ;
|
---|
648 | }
|
---|
649 | RTR0MemObjFree(pGVM->gvmm.s.VMPagesMemObj, false /* fFreeMappings */);
|
---|
650 | pGVM->gvmm.s.VMPagesMemObj = NIL_RTR0MEMOBJ;
|
---|
651 | }
|
---|
652 | RTR0MemObjFree(pGVM->gvmm.s.VMMemObj, false /* fFreeMappings */);
|
---|
653 | pGVM->gvmm.s.VMMemObj = NIL_RTR0MEMOBJ;
|
---|
654 | }
|
---|
655 | }
|
---|
656 | }
|
---|
657 | /* else: The user wasn't permitted to create this VM. */
|
---|
658 |
|
---|
659 | /*
|
---|
660 | * The handle will be freed by gvmmR0HandleObjDestructor as we release the
|
---|
661 | * object reference here. A little extra mess because of non-recursive lock.
|
---|
662 | */
|
---|
663 | void *pvObj = pHandle->pvObj;
|
---|
664 | pHandle->pvObj = NULL;
|
---|
665 | gvmmR0CreateDestroyUnlock(pGVMM);
|
---|
666 |
|
---|
667 | SUPR0ObjRelease(pvObj, pSession);
|
---|
668 |
|
---|
669 | SUPR0Printf("GVMMR0CreateVM: failed, rc=%d\n", rc);
|
---|
670 | return rc;
|
---|
671 | }
|
---|
672 |
|
---|
673 | rc = VERR_NO_MEMORY;
|
---|
674 | }
|
---|
675 | else
|
---|
676 | rc = VERR_INTERNAL_ERROR;
|
---|
677 | }
|
---|
678 | else
|
---|
679 | rc = VERR_GVM_TOO_MANY_VMS;
|
---|
680 |
|
---|
681 | gvmmR0CreateDestroyUnlock(pGVMM);
|
---|
682 | return rc;
|
---|
683 | }
|
---|
684 |
|
---|
685 |
|
---|
686 | /**
|
---|
687 | * Initializes the per VM data belonging to GVMM.
|
---|
688 | *
|
---|
689 | * @param pGVM Pointer to the global VM structure.
|
---|
690 | */
|
---|
691 | static void gvmmR0InitPerVMData(PGVM pGVM)
|
---|
692 | {
|
---|
693 | AssertCompile(RT_SIZEOFMEMB(GVM,gvmm.s) <= RT_SIZEOFMEMB(GVM,gvmm.padding));
|
---|
694 | Assert(RT_SIZEOFMEMB(GVM,gvmm.s) <= RT_SIZEOFMEMB(GVM,gvmm.padding));
|
---|
695 | pGVM->gvmm.s.VMMemObj = NIL_RTR0MEMOBJ;
|
---|
696 | pGVM->gvmm.s.VMMapObj = NIL_RTR0MEMOBJ;
|
---|
697 | pGVM->gvmm.s.VMPagesMemObj = NIL_RTR0MEMOBJ;
|
---|
698 | pGVM->gvmm.s.VMPagesMapObj = NIL_RTR0MEMOBJ;
|
---|
699 | pGVM->gvmm.s.HaltEventMulti = NIL_RTSEMEVENTMULTI;
|
---|
700 | }
|
---|
701 |
|
---|
702 |
|
---|
703 | /**
|
---|
704 | * Does the VM initialization.
|
---|
705 | *
|
---|
706 | * @returns VBox status code.
|
---|
707 | * @param pVM Pointer to the shared VM structure.
|
---|
708 | */
|
---|
709 | GVMMR0DECL(int) GVMMR0InitVM(PVM pVM)
|
---|
710 | {
|
---|
711 | LogFlow(("GVMMR0InitVM: pVM=%p\n", pVM));
|
---|
712 |
|
---|
713 | /*
|
---|
714 | * Validate the VM structure, state and handle.
|
---|
715 | */
|
---|
716 | PGVM pGVM;
|
---|
717 | PGVMM pGVMM;
|
---|
718 | int rc = gvmmR0ByVMAndEMT(pVM, &pGVM, &pGVMM);
|
---|
719 | if (RT_SUCCESS(rc))
|
---|
720 | {
|
---|
721 | if (pGVM->gvmm.s.HaltEventMulti == NIL_RTSEMEVENTMULTI)
|
---|
722 | {
|
---|
723 | rc = RTSemEventMultiCreate(&pGVM->gvmm.s.HaltEventMulti);
|
---|
724 | if (RT_FAILURE(rc))
|
---|
725 | pGVM->gvmm.s.HaltEventMulti = NIL_RTSEMEVENTMULTI;
|
---|
726 | }
|
---|
727 | else
|
---|
728 | rc = VERR_WRONG_ORDER;
|
---|
729 | }
|
---|
730 |
|
---|
731 | LogFlow(("GVMMR0InitVM: returns %Rrc\n", rc));
|
---|
732 | return rc;
|
---|
733 | }
|
---|
734 |
|
---|
735 |
|
---|
736 | /**
|
---|
737 | * Destroys the VM, freeing all associated resources (the ring-0 ones anyway).
|
---|
738 | *
|
---|
739 | * This is call from the vmR3DestroyFinalBit and from a error path in VMR3Create,
|
---|
740 | * and the caller is not the EMT thread, unfortunately. For security reasons, it
|
---|
741 | * would've been nice if the caller was actually the EMT thread or that we somehow
|
---|
742 | * could've associated the calling thread with the VM up front.
|
---|
743 | *
|
---|
744 | * @returns VBox status code.
|
---|
745 | * @param pVM Where to store the pointer to the VM structure.
|
---|
746 | *
|
---|
747 | * @thread EMT if it's associated with the VM, otherwise any thread.
|
---|
748 | */
|
---|
749 | GVMMR0DECL(int) GVMMR0DestroyVM(PVM pVM)
|
---|
750 | {
|
---|
751 | LogFlow(("GVMMR0DestroyVM: pVM=%p\n", pVM));
|
---|
752 | PGVMM pGVMM;
|
---|
753 | GVMM_GET_VALID_INSTANCE(pGVMM, VERR_INTERNAL_ERROR);
|
---|
754 |
|
---|
755 |
|
---|
756 | /*
|
---|
757 | * Validate the VM structure, state and caller.
|
---|
758 | */
|
---|
759 | AssertPtrReturn(pVM, VERR_INVALID_POINTER);
|
---|
760 | AssertReturn(!((uintptr_t)pVM & PAGE_OFFSET_MASK), VERR_INVALID_POINTER);
|
---|
761 | AssertMsgReturn(pVM->enmVMState >= VMSTATE_CREATING && pVM->enmVMState <= VMSTATE_TERMINATED, ("%d\n", pVM->enmVMState), VERR_WRONG_ORDER);
|
---|
762 |
|
---|
763 | uint32_t hGVM = pVM->hSelf;
|
---|
764 | AssertReturn(hGVM != NIL_GVM_HANDLE, VERR_INVALID_HANDLE);
|
---|
765 | AssertReturn(hGVM < RT_ELEMENTS(pGVMM->aHandles), VERR_INVALID_HANDLE);
|
---|
766 |
|
---|
767 | PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
|
---|
768 | AssertReturn(pHandle->pVM == pVM, VERR_NOT_OWNER);
|
---|
769 |
|
---|
770 | RTNATIVETHREAD hSelf = RTThreadNativeSelf();
|
---|
771 | AssertReturn(pHandle->hEMT == hSelf || pHandle->hEMT == NIL_RTNATIVETHREAD, VERR_NOT_OWNER);
|
---|
772 |
|
---|
773 | /*
|
---|
774 | * Lookup the handle and destroy the object.
|
---|
775 | * Since the lock isn't recursive and we'll have to leave it before dereferencing the
|
---|
776 | * object, we take some precautions against racing callers just in case...
|
---|
777 | */
|
---|
778 | int rc = gvmmR0CreateDestroyLock(pGVMM);
|
---|
779 | AssertRC(rc);
|
---|
780 |
|
---|
781 | /* be careful here because we might theoretically be racing someone else cleaning up. */
|
---|
782 | if ( pHandle->pVM == pVM
|
---|
783 | && ( pHandle->hEMT == hSelf
|
---|
784 | || pHandle->hEMT == NIL_RTNATIVETHREAD)
|
---|
785 | && VALID_PTR(pHandle->pvObj)
|
---|
786 | && VALID_PTR(pHandle->pSession)
|
---|
787 | && VALID_PTR(pHandle->pGVM)
|
---|
788 | && pHandle->pGVM->u32Magic == GVM_MAGIC)
|
---|
789 | {
|
---|
790 | void *pvObj = pHandle->pvObj;
|
---|
791 | pHandle->pvObj = NULL;
|
---|
792 | gvmmR0CreateDestroyUnlock(pGVMM);
|
---|
793 |
|
---|
794 | SUPR0ObjRelease(pvObj, pHandle->pSession);
|
---|
795 | }
|
---|
796 | else
|
---|
797 | {
|
---|
798 | SUPR0Printf("GVMMR0DestroyVM: pHandle=%p:{.pVM=%p, hEMT=%p, .pvObj=%p} pVM=%p hSelf=%p\n",
|
---|
799 | pHandle, pHandle->pVM, pHandle->hEMT, pHandle->pvObj, pVM, hSelf);
|
---|
800 | gvmmR0CreateDestroyUnlock(pGVMM);
|
---|
801 | rc = VERR_INTERNAL_ERROR;
|
---|
802 | }
|
---|
803 |
|
---|
804 | return rc;
|
---|
805 | }
|
---|
806 |
|
---|
807 |
|
---|
808 | /**
|
---|
809 | * Handle destructor.
|
---|
810 | *
|
---|
811 | * @param pvGVMM The GVM instance pointer.
|
---|
812 | * @param pvHandle The handle pointer.
|
---|
813 | */
|
---|
814 | static DECLCALLBACK(void) gvmmR0HandleObjDestructor(void *pvObj, void *pvGVMM, void *pvHandle)
|
---|
815 | {
|
---|
816 | LogFlow(("gvmmR0HandleObjDestructor: %p %p %p\n", pvObj, pvGVMM, pvHandle));
|
---|
817 |
|
---|
818 | /*
|
---|
819 | * Some quick, paranoid, input validation.
|
---|
820 | */
|
---|
821 | PGVMHANDLE pHandle = (PGVMHANDLE)pvHandle;
|
---|
822 | AssertPtr(pHandle);
|
---|
823 | PGVMM pGVMM = (PGVMM)pvGVMM;
|
---|
824 | Assert(pGVMM == g_pGVMM);
|
---|
825 | const uint16_t iHandle = pHandle - &pGVMM->aHandles[0];
|
---|
826 | if ( !iHandle
|
---|
827 | || iHandle >= RT_ELEMENTS(pGVMM->aHandles)
|
---|
828 | || iHandle != pHandle->iSelf)
|
---|
829 | {
|
---|
830 | SUPR0Printf("GVM: handle %d is out of range or corrupt (iSelf=%d)!\n", iHandle, pHandle->iSelf);
|
---|
831 | return;
|
---|
832 | }
|
---|
833 |
|
---|
834 | int rc = gvmmR0CreateDestroyLock(pGVMM);
|
---|
835 | AssertRC(rc);
|
---|
836 | rc = gvmmR0UsedLock(pGVMM);
|
---|
837 | AssertRC(rc);
|
---|
838 |
|
---|
839 | /*
|
---|
840 | * This is a tad slow but a doubly linked list is too much hazzle.
|
---|
841 | */
|
---|
842 | if (RT_UNLIKELY(pHandle->iNext >= RT_ELEMENTS(pGVMM->aHandles)))
|
---|
843 | {
|
---|
844 | SUPR0Printf("GVM: used list index %d is out of range!\n", pHandle->iNext);
|
---|
845 | gvmmR0UsedUnlock(pGVMM);
|
---|
846 | gvmmR0CreateDestroyUnlock(pGVMM);
|
---|
847 | return;
|
---|
848 | }
|
---|
849 |
|
---|
850 | if (pGVMM->iUsedHead == iHandle)
|
---|
851 | pGVMM->iUsedHead = pHandle->iNext;
|
---|
852 | else
|
---|
853 | {
|
---|
854 | uint16_t iPrev = pGVMM->iUsedHead;
|
---|
855 | int c = RT_ELEMENTS(pGVMM->aHandles) + 2;
|
---|
856 | while (iPrev)
|
---|
857 | {
|
---|
858 | if (RT_UNLIKELY(iPrev >= RT_ELEMENTS(pGVMM->aHandles)))
|
---|
859 | {
|
---|
860 | SUPR0Printf("GVM: used list index %d is out of range!\n");
|
---|
861 | gvmmR0UsedUnlock(pGVMM);
|
---|
862 | gvmmR0CreateDestroyUnlock(pGVMM);
|
---|
863 | return;
|
---|
864 | }
|
---|
865 | if (RT_UNLIKELY(c-- <= 0))
|
---|
866 | {
|
---|
867 | iPrev = 0;
|
---|
868 | break;
|
---|
869 | }
|
---|
870 |
|
---|
871 | if (pGVMM->aHandles[iPrev].iNext == iHandle)
|
---|
872 | break;
|
---|
873 | iPrev = pGVMM->aHandles[iPrev].iNext;
|
---|
874 | }
|
---|
875 | if (!iPrev)
|
---|
876 | {
|
---|
877 | SUPR0Printf("GVM: can't find the handle previous previous of %d!\n", pHandle->iSelf);
|
---|
878 | gvmmR0UsedUnlock(pGVMM);
|
---|
879 | gvmmR0CreateDestroyUnlock(pGVMM);
|
---|
880 | return;
|
---|
881 | }
|
---|
882 |
|
---|
883 | Assert(pGVMM->aHandles[iPrev].iNext == iHandle);
|
---|
884 | pGVMM->aHandles[iPrev].iNext = pHandle->iNext;
|
---|
885 | }
|
---|
886 | pHandle->iNext = 0;
|
---|
887 | pGVMM->cVMs--;
|
---|
888 |
|
---|
889 | gvmmR0UsedUnlock(pGVMM);
|
---|
890 |
|
---|
891 | /*
|
---|
892 | * Do the global cleanup round.
|
---|
893 | */
|
---|
894 | PGVM pGVM = pHandle->pGVM;
|
---|
895 | if ( VALID_PTR(pGVM)
|
---|
896 | && pGVM->u32Magic == GVM_MAGIC)
|
---|
897 | {
|
---|
898 | /// @todo GMMR0CleanupVM(pGVM);
|
---|
899 |
|
---|
900 | /*
|
---|
901 | * Do the GVMM cleanup - must be done last.
|
---|
902 | */
|
---|
903 | /* The VM and VM pages mappings/allocations. */
|
---|
904 | if (pGVM->gvmm.s.VMPagesMapObj != NIL_RTR0MEMOBJ)
|
---|
905 | {
|
---|
906 | rc = RTR0MemObjFree(pGVM->gvmm.s.VMPagesMapObj, false /* fFreeMappings */); AssertRC(rc);
|
---|
907 | pGVM->gvmm.s.VMPagesMapObj = NIL_RTR0MEMOBJ;
|
---|
908 | }
|
---|
909 |
|
---|
910 | if (pGVM->gvmm.s.VMMapObj != NIL_RTR0MEMOBJ)
|
---|
911 | {
|
---|
912 | rc = RTR0MemObjFree(pGVM->gvmm.s.VMMapObj, false /* fFreeMappings */); AssertRC(rc);
|
---|
913 | pGVM->gvmm.s.VMMapObj = NIL_RTR0MEMOBJ;
|
---|
914 | }
|
---|
915 |
|
---|
916 | if (pGVM->gvmm.s.VMPagesMemObj != NIL_RTR0MEMOBJ)
|
---|
917 | {
|
---|
918 | rc = RTR0MemObjFree(pGVM->gvmm.s.VMPagesMemObj, false /* fFreeMappings */); AssertRC(rc);
|
---|
919 | pGVM->gvmm.s.VMPagesMemObj = NIL_RTR0MEMOBJ;
|
---|
920 | }
|
---|
921 |
|
---|
922 | if (pGVM->gvmm.s.VMMemObj != NIL_RTR0MEMOBJ)
|
---|
923 | {
|
---|
924 | rc = RTR0MemObjFree(pGVM->gvmm.s.VMMemObj, false /* fFreeMappings */); AssertRC(rc);
|
---|
925 | pGVM->gvmm.s.VMMemObj = NIL_RTR0MEMOBJ;
|
---|
926 | }
|
---|
927 |
|
---|
928 | if (pGVM->gvmm.s.HaltEventMulti != NIL_RTSEMEVENTMULTI)
|
---|
929 | {
|
---|
930 | rc = RTSemEventMultiDestroy(pGVM->gvmm.s.HaltEventMulti); AssertRC(rc);
|
---|
931 | pGVM->gvmm.s.HaltEventMulti = NIL_RTSEMEVENTMULTI;
|
---|
932 | }
|
---|
933 |
|
---|
934 | /* the GVM structure itself. */
|
---|
935 | pGVM->u32Magic++;
|
---|
936 | RTMemFree(pGVM);
|
---|
937 | }
|
---|
938 | /* else: GVMMR0CreateVM cleanup. */
|
---|
939 |
|
---|
940 | /*
|
---|
941 | * Free the handle.
|
---|
942 | * Reacquire the UsedLock here to since we're updating handle fields.
|
---|
943 | */
|
---|
944 | rc = gvmmR0UsedLock(pGVMM);
|
---|
945 | AssertRC(rc);
|
---|
946 |
|
---|
947 | pHandle->iNext = pGVMM->iFreeHead;
|
---|
948 | pGVMM->iFreeHead = iHandle;
|
---|
949 | ASMAtomicXchgPtr((void * volatile *)&pHandle->pGVM, NULL);
|
---|
950 | ASMAtomicXchgPtr((void * volatile *)&pHandle->pVM, NULL);
|
---|
951 | ASMAtomicXchgPtr((void * volatile *)&pHandle->pvObj, NULL);
|
---|
952 | ASMAtomicXchgPtr((void * volatile *)&pHandle->pSession, NULL);
|
---|
953 | ASMAtomicXchgSize(&pHandle->hEMT, NIL_RTNATIVETHREAD);
|
---|
954 |
|
---|
955 | gvmmR0UsedUnlock(pGVMM);
|
---|
956 | gvmmR0CreateDestroyUnlock(pGVMM);
|
---|
957 | LogFlow(("gvmmR0HandleObjDestructor: returns\n"));
|
---|
958 | }
|
---|
959 |
|
---|
960 |
|
---|
961 | /**
|
---|
962 | * Lookup a GVM structure by its handle.
|
---|
963 | *
|
---|
964 | * @returns The GVM pointer on success, NULL on failure.
|
---|
965 | * @param hGVM The global VM handle. Asserts on bad handle.
|
---|
966 | */
|
---|
967 | GVMMR0DECL(PGVM) GVMMR0ByHandle(uint32_t hGVM)
|
---|
968 | {
|
---|
969 | PGVMM pGVMM;
|
---|
970 | GVMM_GET_VALID_INSTANCE(pGVMM, NULL);
|
---|
971 |
|
---|
972 | /*
|
---|
973 | * Validate.
|
---|
974 | */
|
---|
975 | AssertReturn(hGVM != NIL_GVM_HANDLE, NULL);
|
---|
976 | AssertReturn(hGVM < RT_ELEMENTS(pGVMM->aHandles), NULL);
|
---|
977 |
|
---|
978 | /*
|
---|
979 | * Look it up.
|
---|
980 | */
|
---|
981 | PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
|
---|
982 | AssertPtrReturn(pHandle->pVM, NULL);
|
---|
983 | AssertPtrReturn(pHandle->pvObj, NULL);
|
---|
984 | PGVM pGVM = pHandle->pGVM;
|
---|
985 | AssertPtrReturn(pGVM, NULL);
|
---|
986 | AssertReturn(pGVM->pVM == pHandle->pVM, NULL);
|
---|
987 |
|
---|
988 | return pHandle->pGVM;
|
---|
989 | }
|
---|
990 |
|
---|
991 |
|
---|
992 | /**
|
---|
993 | * Lookup a GVM structure by the shared VM structure.
|
---|
994 | *
|
---|
995 | * @returns VBox status code.
|
---|
996 | * @param pVM The shared VM structure (the ring-0 mapping).
|
---|
997 | * @param ppGVM Where to store the GVM pointer.
|
---|
998 | * @param ppGVMM Where to store the pointer to the GVMM instance data.
|
---|
999 | * @param fTakeUsedLock Whether to take the used lock or not.
|
---|
1000 | * Be very careful if not taking the lock as it's possible that
|
---|
1001 | * the VM will disappear then.
|
---|
1002 | *
|
---|
1003 | * @remark This will not assert on an invalid pVM but try return sliently.
|
---|
1004 | */
|
---|
1005 | static int gvmmR0ByVM(PVM pVM, PGVM *ppGVM, PGVMM *ppGVMM, bool fTakeUsedLock)
|
---|
1006 | {
|
---|
1007 | PGVMM pGVMM;
|
---|
1008 | GVMM_GET_VALID_INSTANCE(pGVMM, VERR_INTERNAL_ERROR);
|
---|
1009 |
|
---|
1010 | /*
|
---|
1011 | * Validate.
|
---|
1012 | */
|
---|
1013 | if (RT_UNLIKELY( !VALID_PTR(pVM)
|
---|
1014 | || ((uintptr_t)pVM & PAGE_OFFSET_MASK)))
|
---|
1015 | return VERR_INVALID_POINTER;
|
---|
1016 | if (RT_UNLIKELY( pVM->enmVMState < VMSTATE_CREATING
|
---|
1017 | || pVM->enmVMState >= VMSTATE_TERMINATED))
|
---|
1018 | return VERR_INVALID_POINTER;
|
---|
1019 |
|
---|
1020 | uint16_t hGVM = pVM->hSelf;
|
---|
1021 | if (RT_UNLIKELY( hGVM == NIL_GVM_HANDLE
|
---|
1022 | || hGVM >= RT_ELEMENTS(pGVMM->aHandles)))
|
---|
1023 | return VERR_INVALID_HANDLE;
|
---|
1024 |
|
---|
1025 | /*
|
---|
1026 | * Look it up.
|
---|
1027 | */
|
---|
1028 | PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
|
---|
1029 | PGVM pGVM;
|
---|
1030 | if (fTakeUsedLock)
|
---|
1031 | {
|
---|
1032 | int rc = gvmmR0UsedLock(pGVMM);
|
---|
1033 | AssertRCReturn(rc, rc);
|
---|
1034 |
|
---|
1035 | pGVM = pHandle->pGVM;
|
---|
1036 | if (RT_UNLIKELY( pHandle->pVM != pVM
|
---|
1037 | || !VALID_PTR(pHandle->pvObj)
|
---|
1038 | || !VALID_PTR(pGVM)
|
---|
1039 | || pGVM->pVM != pVM))
|
---|
1040 | {
|
---|
1041 | gvmmR0UsedUnlock(pGVMM);
|
---|
1042 | return VERR_INVALID_HANDLE;
|
---|
1043 | }
|
---|
1044 | }
|
---|
1045 | else
|
---|
1046 | {
|
---|
1047 | if (RT_UNLIKELY(pHandle->pVM != pVM))
|
---|
1048 | return VERR_INVALID_HANDLE;
|
---|
1049 | if (RT_UNLIKELY(!VALID_PTR(pHandle->pvObj)))
|
---|
1050 | return VERR_INVALID_HANDLE;
|
---|
1051 |
|
---|
1052 | pGVM = pHandle->pGVM;
|
---|
1053 | if (RT_UNLIKELY(!VALID_PTR(pGVM)))
|
---|
1054 | return VERR_INVALID_HANDLE;
|
---|
1055 | if (RT_UNLIKELY(pGVM->pVM != pVM))
|
---|
1056 | return VERR_INVALID_HANDLE;
|
---|
1057 | }
|
---|
1058 |
|
---|
1059 | *ppGVM = pGVM;
|
---|
1060 | *ppGVMM = pGVMM;
|
---|
1061 | return VINF_SUCCESS;
|
---|
1062 | }
|
---|
1063 |
|
---|
1064 |
|
---|
1065 | /**
|
---|
1066 | * Lookup a GVM structure by the shared VM structure.
|
---|
1067 | *
|
---|
1068 | * @returns The GVM pointer on success, NULL on failure.
|
---|
1069 | * @param pVM The shared VM structure (the ring-0 mapping).
|
---|
1070 | *
|
---|
1071 | * @remark This will not take the 'used'-lock because it doesn't do
|
---|
1072 | * nesting and this function will be used from under the lock.
|
---|
1073 | */
|
---|
1074 | GVMMR0DECL(PGVM) GVMMR0ByVM(PVM pVM)
|
---|
1075 | {
|
---|
1076 | PGVMM pGVMM;
|
---|
1077 | PGVM pGVM;
|
---|
1078 | int rc = gvmmR0ByVM(pVM, &pGVM, &pGVMM, false /* fTakeUsedLock */);
|
---|
1079 | if (RT_SUCCESS(rc))
|
---|
1080 | return pGVM;
|
---|
1081 | AssertRC(rc);
|
---|
1082 | return NULL;
|
---|
1083 | }
|
---|
1084 |
|
---|
1085 |
|
---|
1086 | /**
|
---|
1087 | * Lookup a GVM structure by the shared VM structure
|
---|
1088 | * and ensuring that the caller is the EMT thread.
|
---|
1089 | *
|
---|
1090 | * @returns VBox status code.
|
---|
1091 | * @param pVM The shared VM structure (the ring-0 mapping).
|
---|
1092 | * @param ppGVM Where to store the GVM pointer.
|
---|
1093 | * @param ppGVMM Where to store the pointer to the GVMM instance data.
|
---|
1094 | * @thread EMT
|
---|
1095 | *
|
---|
1096 | * @remark This will assert in failure paths.
|
---|
1097 | */
|
---|
1098 | static int gvmmR0ByVMAndEMT(PVM pVM, PGVM *ppGVM, PGVMM *ppGVMM)
|
---|
1099 | {
|
---|
1100 | PGVMM pGVMM;
|
---|
1101 | GVMM_GET_VALID_INSTANCE(pGVMM, VERR_INTERNAL_ERROR);
|
---|
1102 |
|
---|
1103 | /*
|
---|
1104 | * Validate.
|
---|
1105 | */
|
---|
1106 | AssertPtrReturn(pVM, VERR_INVALID_POINTER);
|
---|
1107 | AssertReturn(!((uintptr_t)pVM & PAGE_OFFSET_MASK), VERR_INVALID_POINTER);
|
---|
1108 |
|
---|
1109 | uint16_t hGVM = pVM->hSelf;
|
---|
1110 | AssertReturn(hGVM != NIL_GVM_HANDLE, VERR_INVALID_HANDLE);
|
---|
1111 | AssertReturn(hGVM < RT_ELEMENTS(pGVMM->aHandles), VERR_INVALID_HANDLE);
|
---|
1112 |
|
---|
1113 | /*
|
---|
1114 | * Look it up.
|
---|
1115 | */
|
---|
1116 | PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
|
---|
1117 | RTNATIVETHREAD hAllegedEMT = RTThreadNativeSelf();
|
---|
1118 | AssertMsgReturn(pHandle->hEMT == hAllegedEMT, ("hEMT %x hAllegedEMT %x\n", pHandle->hEMT, hAllegedEMT), VERR_NOT_OWNER);
|
---|
1119 | AssertReturn(pHandle->pVM == pVM, VERR_NOT_OWNER);
|
---|
1120 | AssertPtrReturn(pHandle->pvObj, VERR_INTERNAL_ERROR);
|
---|
1121 |
|
---|
1122 | PGVM pGVM = pHandle->pGVM;
|
---|
1123 | AssertPtrReturn(pGVM, VERR_INTERNAL_ERROR);
|
---|
1124 | AssertReturn(pGVM->pVM == pVM, VERR_INTERNAL_ERROR);
|
---|
1125 | AssertReturn(pGVM->hEMT == hAllegedEMT, VERR_INTERNAL_ERROR);
|
---|
1126 |
|
---|
1127 | *ppGVM = pGVM;
|
---|
1128 | *ppGVMM = pGVMM;
|
---|
1129 | return VINF_SUCCESS;
|
---|
1130 | }
|
---|
1131 |
|
---|
1132 |
|
---|
1133 | /**
|
---|
1134 | * Lookup a GVM structure by the shared VM structure
|
---|
1135 | * and ensuring that the caller is the EMT thread.
|
---|
1136 | *
|
---|
1137 | * @returns VBox status code.
|
---|
1138 | * @param pVM The shared VM structure (the ring-0 mapping).
|
---|
1139 | * @param ppGVM Where to store the GVM pointer.
|
---|
1140 | * @thread EMT
|
---|
1141 | */
|
---|
1142 | GVMMR0DECL(int) GVMMR0ByVMAndEMT(PVM pVM, PGVM *ppGVM)
|
---|
1143 | {
|
---|
1144 | AssertPtrReturn(ppGVM, VERR_INVALID_POINTER);
|
---|
1145 | PGVMM pGVMM;
|
---|
1146 | return gvmmR0ByVMAndEMT(pVM, ppGVM, &pGVMM);
|
---|
1147 | }
|
---|
1148 |
|
---|
1149 |
|
---|
1150 | /**
|
---|
1151 | * Lookup a VM by its global handle.
|
---|
1152 | *
|
---|
1153 | * @returns The VM handle on success, NULL on failure.
|
---|
1154 | * @param hGVM The global VM handle. Asserts on bad handle.
|
---|
1155 | */
|
---|
1156 | GVMMR0DECL(PVM) GVMMR0GetVMByHandle(uint32_t hGVM)
|
---|
1157 | {
|
---|
1158 | PGVM pGVM = GVMMR0ByHandle(hGVM);
|
---|
1159 | return pGVM ? pGVM->pVM : NULL;
|
---|
1160 | }
|
---|
1161 |
|
---|
1162 |
|
---|
1163 | /**
|
---|
1164 | * Looks up the VM belonging to the specified EMT thread.
|
---|
1165 | *
|
---|
1166 | * This is used by the assertion machinery in VMMR0.cpp to avoid causing
|
---|
1167 | * unnecessary kernel panics when the EMT thread hits an assertion. The
|
---|
1168 | * call may or not be an EMT thread.
|
---|
1169 | *
|
---|
1170 | * @returns The VM handle on success, NULL on failure.
|
---|
1171 | * @param hEMT The native thread handle of the EMT.
|
---|
1172 | * NIL_RTNATIVETHREAD means the current thread
|
---|
1173 | */
|
---|
1174 | GVMMR0DECL(PVM) GVMMR0GetVMByEMT(RTNATIVETHREAD hEMT)
|
---|
1175 | {
|
---|
1176 | /*
|
---|
1177 | * No Assertions here as we're usually called in a AssertMsgN or
|
---|
1178 | * RTAssert* context.
|
---|
1179 | */
|
---|
1180 | PGVMM pGVMM = g_pGVMM;
|
---|
1181 | if ( !VALID_PTR(pGVMM)
|
---|
1182 | || pGVMM->u32Magic != GVMM_MAGIC)
|
---|
1183 | return NULL;
|
---|
1184 |
|
---|
1185 | if (hEMT == NIL_RTNATIVETHREAD)
|
---|
1186 | hEMT = RTThreadNativeSelf();
|
---|
1187 |
|
---|
1188 | /*
|
---|
1189 | * Search the handles in a linear fashion as we don't dare take the lock (assert).
|
---|
1190 | */
|
---|
1191 | for (unsigned i = 1; i < RT_ELEMENTS(pGVMM->aHandles); i++)
|
---|
1192 | if ( pGVMM->aHandles[i].hEMT == hEMT
|
---|
1193 | && pGVMM->aHandles[i].iSelf == i
|
---|
1194 | && VALID_PTR(pGVMM->aHandles[i].pvObj)
|
---|
1195 | && VALID_PTR(pGVMM->aHandles[i].pVM))
|
---|
1196 | return pGVMM->aHandles[i].pVM;
|
---|
1197 |
|
---|
1198 | return NULL;
|
---|
1199 | }
|
---|
1200 |
|
---|
1201 |
|
---|
1202 | /**
|
---|
1203 | * This is will wake up expired and soon-to-be expired VMs.
|
---|
1204 | *
|
---|
1205 | * @returns Number of VMs that has been woken up.
|
---|
1206 | * @param pGVMM Pointer to the GVMM instance data.
|
---|
1207 | * @param u64Now The current time.
|
---|
1208 | */
|
---|
1209 | static unsigned gvmmR0SchedDoWakeUps(PGVMM pGVMM, uint64_t u64Now)
|
---|
1210 | {
|
---|
1211 | /*
|
---|
1212 | * The first pass will wake up VMs which has actually expired
|
---|
1213 | * and look for VMs that should be woken up in the 2nd and 3rd passes.
|
---|
1214 | */
|
---|
1215 | unsigned cWoken = 0;
|
---|
1216 | unsigned cHalted = 0;
|
---|
1217 | unsigned cTodo2nd = 0;
|
---|
1218 | unsigned cTodo3rd = 0;
|
---|
1219 | for (unsigned i = pGVMM->iUsedHead, cGuard = 0;
|
---|
1220 | i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
|
---|
1221 | i = pGVMM->aHandles[i].iNext)
|
---|
1222 | {
|
---|
1223 | PGVM pCurGVM = pGVMM->aHandles[i].pGVM;
|
---|
1224 | if ( VALID_PTR(pCurGVM)
|
---|
1225 | && pCurGVM->u32Magic == GVM_MAGIC)
|
---|
1226 | {
|
---|
1227 | uint64_t u64 = pCurGVM->gvmm.s.u64HaltExpire;
|
---|
1228 | if (u64)
|
---|
1229 | {
|
---|
1230 | if (u64 <= u64Now)
|
---|
1231 | {
|
---|
1232 | if (ASMAtomicXchgU64(&pCurGVM->gvmm.s.u64HaltExpire, 0))
|
---|
1233 | {
|
---|
1234 | int rc = RTSemEventMultiSignal(pCurGVM->gvmm.s.HaltEventMulti);
|
---|
1235 | AssertRC(rc);
|
---|
1236 | cWoken++;
|
---|
1237 | }
|
---|
1238 | }
|
---|
1239 | else
|
---|
1240 | {
|
---|
1241 | cHalted++;
|
---|
1242 | if (u64 <= u64Now + pGVMM->nsEarlyWakeUp1)
|
---|
1243 | cTodo2nd++;
|
---|
1244 | else if (u64 <= u64Now + pGVMM->nsEarlyWakeUp2)
|
---|
1245 | cTodo3rd++;
|
---|
1246 | }
|
---|
1247 | }
|
---|
1248 | }
|
---|
1249 | AssertLogRelBreak(cGuard++ < RT_ELEMENTS(pGVMM->aHandles));
|
---|
1250 | }
|
---|
1251 |
|
---|
1252 | if (cTodo2nd)
|
---|
1253 | {
|
---|
1254 | for (unsigned i = pGVMM->iUsedHead, cGuard = 0;
|
---|
1255 | i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
|
---|
1256 | i = pGVMM->aHandles[i].iNext)
|
---|
1257 | {
|
---|
1258 | PGVM pCurGVM = pGVMM->aHandles[i].pGVM;
|
---|
1259 | if ( VALID_PTR(pCurGVM)
|
---|
1260 | && pCurGVM->u32Magic == GVM_MAGIC
|
---|
1261 | && pCurGVM->gvmm.s.u64HaltExpire
|
---|
1262 | && pCurGVM->gvmm.s.u64HaltExpire <= u64Now + pGVMM->nsEarlyWakeUp1)
|
---|
1263 | {
|
---|
1264 | if (ASMAtomicXchgU64(&pCurGVM->gvmm.s.u64HaltExpire, 0))
|
---|
1265 | {
|
---|
1266 | int rc = RTSemEventMultiSignal(pCurGVM->gvmm.s.HaltEventMulti);
|
---|
1267 | AssertRC(rc);
|
---|
1268 | cWoken++;
|
---|
1269 | }
|
---|
1270 | }
|
---|
1271 | AssertLogRelBreak(cGuard++ < RT_ELEMENTS(pGVMM->aHandles));
|
---|
1272 | }
|
---|
1273 | }
|
---|
1274 |
|
---|
1275 | if (cTodo3rd)
|
---|
1276 | {
|
---|
1277 | for (unsigned i = pGVMM->iUsedHead, cGuard = 0;
|
---|
1278 | i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
|
---|
1279 | i = pGVMM->aHandles[i].iNext)
|
---|
1280 | {
|
---|
1281 | PGVM pCurGVM = pGVMM->aHandles[i].pGVM;
|
---|
1282 | if ( VALID_PTR(pCurGVM)
|
---|
1283 | && pCurGVM->u32Magic == GVM_MAGIC
|
---|
1284 | && pCurGVM->gvmm.s.u64HaltExpire
|
---|
1285 | && pCurGVM->gvmm.s.u64HaltExpire <= u64Now + pGVMM->nsEarlyWakeUp2)
|
---|
1286 | {
|
---|
1287 | if (ASMAtomicXchgU64(&pCurGVM->gvmm.s.u64HaltExpire, 0))
|
---|
1288 | {
|
---|
1289 | int rc = RTSemEventMultiSignal(pCurGVM->gvmm.s.HaltEventMulti);
|
---|
1290 | AssertRC(rc);
|
---|
1291 | cWoken++;
|
---|
1292 | }
|
---|
1293 | }
|
---|
1294 | AssertLogRelBreak(cGuard++ < RT_ELEMENTS(pGVMM->aHandles));
|
---|
1295 | }
|
---|
1296 | }
|
---|
1297 |
|
---|
1298 | return cWoken;
|
---|
1299 | }
|
---|
1300 |
|
---|
1301 |
|
---|
1302 | /**
|
---|
1303 | * Halt the EMT thread.
|
---|
1304 | *
|
---|
1305 | * @returns VINF_SUCCESS normal wakeup (timeout or kicked by other thread).
|
---|
1306 | * VERR_INTERRUPTED if a signal was scheduled for the thread.
|
---|
1307 | * @param pVM Pointer to the shared VM structure.
|
---|
1308 | * @param u64ExpireGipTime The time for the sleep to expire expressed as GIP time.
|
---|
1309 | * @thread EMT.
|
---|
1310 | */
|
---|
1311 | GVMMR0DECL(int) GVMMR0SchedHalt(PVM pVM, uint64_t u64ExpireGipTime)
|
---|
1312 | {
|
---|
1313 | LogFlow(("GVMMR0SchedHalt: pVM=%p\n", pVM));
|
---|
1314 |
|
---|
1315 | /*
|
---|
1316 | * Validate the VM structure, state and handle.
|
---|
1317 | */
|
---|
1318 | PGVMM pGVMM;
|
---|
1319 | PGVM pGVM;
|
---|
1320 | int rc = gvmmR0ByVMAndEMT(pVM, &pGVM, &pGVMM);
|
---|
1321 | if (RT_FAILURE(rc))
|
---|
1322 | return rc;
|
---|
1323 | pGVM->gvmm.s.StatsSched.cHaltCalls++;
|
---|
1324 |
|
---|
1325 | Assert(!pGVM->gvmm.s.u64HaltExpire);
|
---|
1326 |
|
---|
1327 | /*
|
---|
1328 | * Take the UsedList semaphore, get the current time
|
---|
1329 | * and check if anyone needs waking up.
|
---|
1330 | * Interrupts must NOT be disabled at this point because we ask for GIP time!
|
---|
1331 | */
|
---|
1332 | rc = gvmmR0UsedLock(pGVMM);
|
---|
1333 | AssertRC(rc);
|
---|
1334 |
|
---|
1335 | pGVM->gvmm.s.iCpuEmt = ASMGetApicId();
|
---|
1336 |
|
---|
1337 | Assert(ASMGetFlags() & X86_EFL_IF);
|
---|
1338 | const uint64_t u64Now = RTTimeNanoTS(); /* (GIP time) */
|
---|
1339 | pGVM->gvmm.s.StatsSched.cHaltWakeUps += gvmmR0SchedDoWakeUps(pGVMM, u64Now);
|
---|
1340 |
|
---|
1341 | /*
|
---|
1342 | * Go to sleep if we must...
|
---|
1343 | */
|
---|
1344 | if ( u64Now < u64ExpireGipTime
|
---|
1345 | && u64ExpireGipTime - u64Now > (pGVMM->cVMs > pGVMM->cVMsMeansCompany
|
---|
1346 | ? pGVMM->nsMinSleepCompany
|
---|
1347 | : pGVMM->nsMinSleepAlone))
|
---|
1348 | {
|
---|
1349 | pGVM->gvmm.s.StatsSched.cHaltBlocking++;
|
---|
1350 | ASMAtomicXchgU64(&pGVM->gvmm.s.u64HaltExpire, u64ExpireGipTime);
|
---|
1351 | gvmmR0UsedUnlock(pGVMM);
|
---|
1352 |
|
---|
1353 | uint32_t cMillies = (u64ExpireGipTime - u64Now) / 1000000;
|
---|
1354 | rc = RTSemEventMultiWaitNoResume(pGVM->gvmm.s.HaltEventMulti, cMillies ? cMillies : 1);
|
---|
1355 | ASMAtomicXchgU64(&pGVM->gvmm.s.u64HaltExpire, 0);
|
---|
1356 | if (rc == VERR_TIMEOUT)
|
---|
1357 | {
|
---|
1358 | pGVM->gvmm.s.StatsSched.cHaltTimeouts++;
|
---|
1359 | rc = VINF_SUCCESS;
|
---|
1360 | }
|
---|
1361 | }
|
---|
1362 | else
|
---|
1363 | {
|
---|
1364 | pGVM->gvmm.s.StatsSched.cHaltNotBlocking++;
|
---|
1365 | gvmmR0UsedUnlock(pGVMM);
|
---|
1366 | }
|
---|
1367 |
|
---|
1368 | /* Make sure false wake up calls (gvmmR0SchedDoWakeUps) cause us to spin. */
|
---|
1369 | RTSemEventMultiReset(pGVM->gvmm.s.HaltEventMulti);
|
---|
1370 |
|
---|
1371 | return rc;
|
---|
1372 | }
|
---|
1373 |
|
---|
1374 |
|
---|
1375 | /**
|
---|
1376 | * Wakes up the halted EMT thread so it can service a pending request.
|
---|
1377 | *
|
---|
1378 | * @returns VINF_SUCCESS if not yielded.
|
---|
1379 | * VINF_GVM_NOT_BLOCKED if the EMT thread wasn't blocked.
|
---|
1380 | * @param pVM Pointer to the shared VM structure.
|
---|
1381 | * @thread Any but EMT.
|
---|
1382 | */
|
---|
1383 | GVMMR0DECL(int) GVMMR0SchedWakeUp(PVM pVM)
|
---|
1384 | {
|
---|
1385 | /*
|
---|
1386 | * Validate input and take the UsedLock.
|
---|
1387 | */
|
---|
1388 | PGVM pGVM;
|
---|
1389 | PGVMM pGVMM;
|
---|
1390 | int rc = gvmmR0ByVM(pVM, &pGVM, &pGVMM, true /* fTakeUsedLock */);
|
---|
1391 | if (RT_SUCCESS(rc))
|
---|
1392 | {
|
---|
1393 | pGVM->gvmm.s.StatsSched.cWakeUpCalls++;
|
---|
1394 |
|
---|
1395 | /*
|
---|
1396 | * Signal the semaphore regardless of whether it's current blocked on it.
|
---|
1397 | *
|
---|
1398 | * The reason for this is that there is absolutely no way we can be 100%
|
---|
1399 | * certain that it isn't *about* go to go to sleep on it and just got
|
---|
1400 | * delayed a bit en route. So, we will always signal the semaphore when
|
---|
1401 | * the it is flagged as halted in the VMM.
|
---|
1402 | */
|
---|
1403 | if (pGVM->gvmm.s.u64HaltExpire)
|
---|
1404 | {
|
---|
1405 | rc = VINF_SUCCESS;
|
---|
1406 | ASMAtomicXchgU64(&pGVM->gvmm.s.u64HaltExpire, 0);
|
---|
1407 | }
|
---|
1408 | else
|
---|
1409 | {
|
---|
1410 | rc = VINF_GVM_NOT_BLOCKED;
|
---|
1411 | pGVM->gvmm.s.StatsSched.cWakeUpNotHalted++;
|
---|
1412 | }
|
---|
1413 |
|
---|
1414 | int rc2 = RTSemEventMultiSignal(pGVM->gvmm.s.HaltEventMulti);
|
---|
1415 | AssertRC(rc2);
|
---|
1416 |
|
---|
1417 | /*
|
---|
1418 | * While we're here, do a round of scheduling.
|
---|
1419 | */
|
---|
1420 | Assert(ASMGetFlags() & X86_EFL_IF);
|
---|
1421 | const uint64_t u64Now = RTTimeNanoTS(); /* (GIP time) */
|
---|
1422 | pGVM->gvmm.s.StatsSched.cWakeUpWakeUps += gvmmR0SchedDoWakeUps(pGVMM, u64Now);
|
---|
1423 |
|
---|
1424 |
|
---|
1425 | rc2 = gvmmR0UsedUnlock(pGVMM);
|
---|
1426 | AssertRC(rc2);
|
---|
1427 | }
|
---|
1428 |
|
---|
1429 | LogFlow(("GVMMR0SchedWakeUp: returns %Rrc\n", rc));
|
---|
1430 | return rc;
|
---|
1431 | }
|
---|
1432 |
|
---|
1433 |
|
---|
1434 | /**
|
---|
1435 | * Poll the schedule to see if someone else should get a chance to run.
|
---|
1436 | *
|
---|
1437 | * This is a bit hackish and will not work too well if the machine is
|
---|
1438 | * under heavy load from non-VM processes.
|
---|
1439 | *
|
---|
1440 | * @returns VINF_SUCCESS if not yielded.
|
---|
1441 | * VINF_GVM_YIELDED if an attempt to switch to a different VM task was made.
|
---|
1442 | * @param pVM Pointer to the shared VM structure.
|
---|
1443 | * @param u64ExpireGipTime The time for the sleep to expire expressed as GIP time.
|
---|
1444 | * @param fYield Whether to yield or not.
|
---|
1445 | * This is for when we're spinning in the halt loop.
|
---|
1446 | * @thread EMT.
|
---|
1447 | */
|
---|
1448 | GVMMR0DECL(int) GVMMR0SchedPoll(PVM pVM, bool fYield)
|
---|
1449 | {
|
---|
1450 | /*
|
---|
1451 | * Validate input.
|
---|
1452 | */
|
---|
1453 | PGVM pGVM;
|
---|
1454 | PGVMM pGVMM;
|
---|
1455 | int rc = gvmmR0ByVMAndEMT(pVM, &pGVM, &pGVMM);
|
---|
1456 | if (RT_SUCCESS(rc))
|
---|
1457 | {
|
---|
1458 | rc = gvmmR0UsedLock(pGVMM);
|
---|
1459 | AssertRC(rc);
|
---|
1460 | pGVM->gvmm.s.StatsSched.cPollCalls++;
|
---|
1461 |
|
---|
1462 | Assert(ASMGetFlags() & X86_EFL_IF);
|
---|
1463 | const uint64_t u64Now = RTTimeNanoTS(); /* (GIP time) */
|
---|
1464 |
|
---|
1465 | if (!fYield)
|
---|
1466 | pGVM->gvmm.s.StatsSched.cPollWakeUps += gvmmR0SchedDoWakeUps(pGVMM, u64Now);
|
---|
1467 | else
|
---|
1468 | {
|
---|
1469 | /** @todo implement this... */
|
---|
1470 | rc = VERR_NOT_IMPLEMENTED;
|
---|
1471 | }
|
---|
1472 |
|
---|
1473 | gvmmR0UsedUnlock(pGVMM);
|
---|
1474 | }
|
---|
1475 |
|
---|
1476 | LogFlow(("GVMMR0SchedWakeUp: returns %Rrc\n", rc));
|
---|
1477 | return rc;
|
---|
1478 | }
|
---|
1479 |
|
---|
1480 |
|
---|
1481 |
|
---|
1482 | /**
|
---|
1483 | * Retrieves the GVMM statistics visible to the caller.
|
---|
1484 | *
|
---|
1485 | * @returns VBox status code.
|
---|
1486 | *
|
---|
1487 | * @param pStats Where to put the statistics.
|
---|
1488 | * @param pSession The current session.
|
---|
1489 | * @param pVM The VM to obtain statistics for. Optional.
|
---|
1490 | */
|
---|
1491 | GVMMR0DECL(int) GVMMR0QueryStatistics(PGVMMSTATS pStats, PSUPDRVSESSION pSession, PVM pVM)
|
---|
1492 | {
|
---|
1493 | LogFlow(("GVMMR0QueryStatistics: pStats=%p pSession=%p pVM=%p\n", pStats, pSession, pVM));
|
---|
1494 |
|
---|
1495 | /*
|
---|
1496 | * Validate input.
|
---|
1497 | */
|
---|
1498 | AssertPtrReturn(pSession, VERR_INVALID_POINTER);
|
---|
1499 | AssertPtrReturn(pStats, VERR_INVALID_POINTER);
|
---|
1500 | pStats->cVMs = 0; /* (crash before taking the sem...) */
|
---|
1501 |
|
---|
1502 | /*
|
---|
1503 | * Take the lock and get the VM statistics.
|
---|
1504 | */
|
---|
1505 | PGVMM pGVMM;
|
---|
1506 | if (pVM)
|
---|
1507 | {
|
---|
1508 | PGVM pGVM;
|
---|
1509 | int rc = gvmmR0ByVM(pVM, &pGVM, &pGVMM, true /*fTakeUsedLock*/);
|
---|
1510 | if (RT_FAILURE(rc))
|
---|
1511 | return rc;
|
---|
1512 | pStats->SchedVM = pGVM->gvmm.s.StatsSched;
|
---|
1513 | }
|
---|
1514 | else
|
---|
1515 | {
|
---|
1516 | GVMM_GET_VALID_INSTANCE(pGVMM, VERR_INTERNAL_ERROR);
|
---|
1517 | memset(&pStats->SchedVM, 0, sizeof(pStats->SchedVM));
|
---|
1518 |
|
---|
1519 | int rc = gvmmR0UsedLock(pGVMM);
|
---|
1520 | AssertRCReturn(rc, rc);
|
---|
1521 | }
|
---|
1522 |
|
---|
1523 | /*
|
---|
1524 | * Enumerate the VMs and add the ones visibile to the statistics.
|
---|
1525 | */
|
---|
1526 | pStats->cVMs = 0;
|
---|
1527 | memset(&pStats->SchedSum, 0, sizeof(pStats->SchedSum));
|
---|
1528 |
|
---|
1529 | for (unsigned i = pGVMM->iUsedHead;
|
---|
1530 | i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
|
---|
1531 | i = pGVMM->aHandles[i].iNext)
|
---|
1532 | {
|
---|
1533 | PGVM pGVM = pGVMM->aHandles[i].pGVM;
|
---|
1534 | void *pvObj = pGVMM->aHandles[i].pvObj;
|
---|
1535 | if ( VALID_PTR(pvObj)
|
---|
1536 | && VALID_PTR(pGVM)
|
---|
1537 | && pGVM->u32Magic == GVM_MAGIC
|
---|
1538 | && RT_SUCCESS(SUPR0ObjVerifyAccess(pvObj, pSession, NULL)))
|
---|
1539 | {
|
---|
1540 | pStats->cVMs++;
|
---|
1541 |
|
---|
1542 | pStats->SchedSum.cHaltCalls += pGVM->gvmm.s.StatsSched.cHaltCalls;
|
---|
1543 | pStats->SchedSum.cHaltBlocking += pGVM->gvmm.s.StatsSched.cHaltBlocking;
|
---|
1544 | pStats->SchedSum.cHaltTimeouts += pGVM->gvmm.s.StatsSched.cHaltTimeouts;
|
---|
1545 | pStats->SchedSum.cHaltNotBlocking += pGVM->gvmm.s.StatsSched.cHaltNotBlocking;
|
---|
1546 | pStats->SchedSum.cHaltWakeUps += pGVM->gvmm.s.StatsSched.cHaltWakeUps;
|
---|
1547 |
|
---|
1548 | pStats->SchedSum.cWakeUpCalls += pGVM->gvmm.s.StatsSched.cWakeUpCalls;
|
---|
1549 | pStats->SchedSum.cWakeUpNotHalted += pGVM->gvmm.s.StatsSched.cWakeUpNotHalted;
|
---|
1550 | pStats->SchedSum.cWakeUpWakeUps += pGVM->gvmm.s.StatsSched.cWakeUpWakeUps;
|
---|
1551 |
|
---|
1552 | pStats->SchedSum.cPollCalls += pGVM->gvmm.s.StatsSched.cPollCalls;
|
---|
1553 | pStats->SchedSum.cPollHalts += pGVM->gvmm.s.StatsSched.cPollHalts;
|
---|
1554 | pStats->SchedSum.cPollWakeUps += pGVM->gvmm.s.StatsSched.cPollWakeUps;
|
---|
1555 | }
|
---|
1556 | }
|
---|
1557 |
|
---|
1558 | gvmmR0UsedUnlock(pGVMM);
|
---|
1559 |
|
---|
1560 | return VINF_SUCCESS;
|
---|
1561 | }
|
---|
1562 |
|
---|
1563 |
|
---|
1564 | /**
|
---|
1565 | * VMMR0 request wrapper for GVMMR0QueryStatistics.
|
---|
1566 | *
|
---|
1567 | * @returns see GVMMR0QueryStatistics.
|
---|
1568 | * @param pVM Pointer to the shared VM structure. Optional.
|
---|
1569 | * @param pReq The request packet.
|
---|
1570 | */
|
---|
1571 | GVMMR0DECL(int) GVMMR0QueryStatisticsReq(PVM pVM, PGVMMQUERYSTATISTICSSREQ pReq)
|
---|
1572 | {
|
---|
1573 | /*
|
---|
1574 | * Validate input and pass it on.
|
---|
1575 | */
|
---|
1576 | AssertPtrReturn(pReq, VERR_INVALID_POINTER);
|
---|
1577 | AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER);
|
---|
1578 |
|
---|
1579 | return GVMMR0QueryStatistics(&pReq->Stats, pReq->pSession, pVM);
|
---|
1580 | }
|
---|
1581 |
|
---|
1582 |
|
---|
1583 | /**
|
---|
1584 | * Resets the specified GVMM statistics.
|
---|
1585 | *
|
---|
1586 | * @returns VBox status code.
|
---|
1587 | *
|
---|
1588 | * @param pStats Which statistics to reset, that is, non-zero fields indicates which to reset.
|
---|
1589 | * @param pSession The current session.
|
---|
1590 | * @param pVM The VM to reset statistics for. Optional.
|
---|
1591 | */
|
---|
1592 | GVMMR0DECL(int) GVMMR0ResetStatistics(PCGVMMSTATS pStats, PSUPDRVSESSION pSession, PVM pVM)
|
---|
1593 | {
|
---|
1594 | LogFlow(("GVMMR0ResetStatistics: pStats=%p pSession=%p pVM=%p\n", pStats, pSession, pVM));
|
---|
1595 |
|
---|
1596 | /*
|
---|
1597 | * Validate input.
|
---|
1598 | */
|
---|
1599 | AssertPtrReturn(pSession, VERR_INVALID_POINTER);
|
---|
1600 | AssertPtrReturn(pStats, VERR_INVALID_POINTER);
|
---|
1601 |
|
---|
1602 | /*
|
---|
1603 | * Take the lock and get the VM statistics.
|
---|
1604 | */
|
---|
1605 | PGVMM pGVMM;
|
---|
1606 | if (pVM)
|
---|
1607 | {
|
---|
1608 | PGVM pGVM;
|
---|
1609 | int rc = gvmmR0ByVM(pVM, &pGVM, &pGVMM, true /*fTakeUsedLock*/);
|
---|
1610 | if (RT_FAILURE(rc))
|
---|
1611 | return rc;
|
---|
1612 | # define MAYBE_RESET_FIELD(field) \
|
---|
1613 | do { if (pStats->SchedVM. field ) { pGVM->gvmm.s.StatsSched. field = 0; } } while (0)
|
---|
1614 | MAYBE_RESET_FIELD(cHaltCalls);
|
---|
1615 | MAYBE_RESET_FIELD(cHaltBlocking);
|
---|
1616 | MAYBE_RESET_FIELD(cHaltTimeouts);
|
---|
1617 | MAYBE_RESET_FIELD(cHaltNotBlocking);
|
---|
1618 | MAYBE_RESET_FIELD(cHaltWakeUps);
|
---|
1619 | MAYBE_RESET_FIELD(cWakeUpCalls);
|
---|
1620 | MAYBE_RESET_FIELD(cWakeUpNotHalted);
|
---|
1621 | MAYBE_RESET_FIELD(cWakeUpWakeUps);
|
---|
1622 | MAYBE_RESET_FIELD(cPollCalls);
|
---|
1623 | MAYBE_RESET_FIELD(cPollHalts);
|
---|
1624 | MAYBE_RESET_FIELD(cPollWakeUps);
|
---|
1625 | # undef MAYBE_RESET_FIELD
|
---|
1626 | }
|
---|
1627 | else
|
---|
1628 | {
|
---|
1629 | GVMM_GET_VALID_INSTANCE(pGVMM, VERR_INTERNAL_ERROR);
|
---|
1630 |
|
---|
1631 | int rc = gvmmR0UsedLock(pGVMM);
|
---|
1632 | AssertRCReturn(rc, rc);
|
---|
1633 | }
|
---|
1634 |
|
---|
1635 | /*
|
---|
1636 | * Enumerate the VMs and add the ones visibile to the statistics.
|
---|
1637 | */
|
---|
1638 | if (ASMMemIsAll8(&pStats->SchedSum, sizeof(pStats->SchedSum), 0))
|
---|
1639 | {
|
---|
1640 | for (unsigned i = pGVMM->iUsedHead;
|
---|
1641 | i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
|
---|
1642 | i = pGVMM->aHandles[i].iNext)
|
---|
1643 | {
|
---|
1644 | PGVM pGVM = pGVMM->aHandles[i].pGVM;
|
---|
1645 | void *pvObj = pGVMM->aHandles[i].pvObj;
|
---|
1646 | if ( VALID_PTR(pvObj)
|
---|
1647 | && VALID_PTR(pGVM)
|
---|
1648 | && pGVM->u32Magic == GVM_MAGIC
|
---|
1649 | && RT_SUCCESS(SUPR0ObjVerifyAccess(pvObj, pSession, NULL)))
|
---|
1650 | {
|
---|
1651 | # define MAYBE_RESET_FIELD(field) \
|
---|
1652 | do { if (pStats->SchedSum. field ) { pGVM->gvmm.s.StatsSched. field = 0; } } while (0)
|
---|
1653 | MAYBE_RESET_FIELD(cHaltCalls);
|
---|
1654 | MAYBE_RESET_FIELD(cHaltBlocking);
|
---|
1655 | MAYBE_RESET_FIELD(cHaltTimeouts);
|
---|
1656 | MAYBE_RESET_FIELD(cHaltNotBlocking);
|
---|
1657 | MAYBE_RESET_FIELD(cHaltWakeUps);
|
---|
1658 | MAYBE_RESET_FIELD(cWakeUpCalls);
|
---|
1659 | MAYBE_RESET_FIELD(cWakeUpNotHalted);
|
---|
1660 | MAYBE_RESET_FIELD(cWakeUpWakeUps);
|
---|
1661 | MAYBE_RESET_FIELD(cPollCalls);
|
---|
1662 | MAYBE_RESET_FIELD(cPollHalts);
|
---|
1663 | MAYBE_RESET_FIELD(cPollWakeUps);
|
---|
1664 | # undef MAYBE_RESET_FIELD
|
---|
1665 | }
|
---|
1666 | }
|
---|
1667 | }
|
---|
1668 |
|
---|
1669 | gvmmR0UsedUnlock(pGVMM);
|
---|
1670 |
|
---|
1671 | return VINF_SUCCESS;
|
---|
1672 | }
|
---|
1673 |
|
---|
1674 |
|
---|
1675 | /**
|
---|
1676 | * VMMR0 request wrapper for GVMMR0ResetStatistics.
|
---|
1677 | *
|
---|
1678 | * @returns see GVMMR0ResetStatistics.
|
---|
1679 | * @param pVM Pointer to the shared VM structure. Optional.
|
---|
1680 | * @param pReq The request packet.
|
---|
1681 | */
|
---|
1682 | GVMMR0DECL(int) GVMMR0ResetStatisticsReq(PVM pVM, PGVMMRESETSTATISTICSSREQ pReq)
|
---|
1683 | {
|
---|
1684 | /*
|
---|
1685 | * Validate input and pass it on.
|
---|
1686 | */
|
---|
1687 | AssertPtrReturn(pReq, VERR_INVALID_POINTER);
|
---|
1688 | AssertMsgReturn(pReq->Hdr.cbReq == sizeof(*pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(*pReq)), VERR_INVALID_PARAMETER);
|
---|
1689 |
|
---|
1690 | return GVMMR0ResetStatistics(&pReq->Stats, pReq->pSession, pVM);
|
---|
1691 | }
|
---|
1692 |
|
---|