VMM.cpp@ 73770

Last change on this file since 73770 was 73481, checked in by vboxsync, 6 years ago
VMM,DBGF: Improved unwinding of ring-0 assertion stacks, making the new unwind info stuff deal correctly with ring-0 pointers and such. bugref:3897 [build fix]
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 121.5 KB

Line
1	/* $Id: VMM.cpp 73481 2018-08-03 12:32:03Z vboxsync $ */
2	/** @file
3	* VMM - The Virtual Machine Monitor Core.
4	*/
5
6	/*
7	* Copyright (C) 2006-2017 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	//#define NO_SUPCALLR0VMM
19
20	/** @page pg_vmm VMM - The Virtual Machine Monitor
21	*
22	* The VMM component is two things at the moment, it's a component doing a few
23	* management and routing tasks, and it's the whole virtual machine monitor
24	* thing. For hysterical reasons, it is not doing all the management that one
25	* would expect, this is instead done by @ref pg_vm. We'll address this
26	* misdesign eventually, maybe.
27	*
28	* VMM is made up of these components:
29	* - @subpage pg_cfgm
30	* - @subpage pg_cpum
31	* - @subpage pg_csam
32	* - @subpage pg_dbgf
33	* - @subpage pg_em
34	* - @subpage pg_gim
35	* - @subpage pg_gmm
36	* - @subpage pg_gvmm
37	* - @subpage pg_hm
38	* - @subpage pg_iem
39	* - @subpage pg_iom
40	* - @subpage pg_mm
41	* - @subpage pg_patm
42	* - @subpage pg_pdm
43	* - @subpage pg_pgm
44	* - @subpage pg_rem
45	* - @subpage pg_selm
46	* - @subpage pg_ssm
47	* - @subpage pg_stam
48	* - @subpage pg_tm
49	* - @subpage pg_trpm
50	* - @subpage pg_vm
51	*
52	*
53	* @see @ref grp_vmm @ref grp_vm @subpage pg_vmm_guideline @subpage pg_raw
54	*
55	*
56	* @section sec_vmmstate VMM State
57	*
58	* @image html VM_Statechart_Diagram.gif
59	*
60	* To be written.
61	*
62	*
63	* @subsection subsec_vmm_init VMM Initialization
64	*
65	* To be written.
66	*
67	*
68	* @subsection subsec_vmm_term VMM Termination
69	*
70	* To be written.
71	*
72	*
73	* @section sec_vmm_limits VMM Limits
74	*
75	* There are various resource limits imposed by the VMM and it's
76	* sub-components. We'll list some of them here.
77	*
78	* On 64-bit hosts:
79	* - Max 8191 VMs. Imposed by GVMM's handle allocation (GVMM_MAX_HANDLES),
80	* can be increased up to 64K - 1.
81	* - Max 16TB - 64KB of the host memory can be used for backing VM RAM and
82	* ROM pages. The limit is imposed by the 32-bit page ID used by GMM.
83	* - A VM can be assigned all the memory we can use (16TB), however, the
84	* Main API will restrict this to 2TB (MM_RAM_MAX_IN_MB).
85	* - Max 32 virtual CPUs (VMM_MAX_CPU_COUNT).
86	*
87	* On 32-bit hosts:
88	* - Max 127 VMs. Imposed by GMM's per page structure.
89	* - Max 64GB - 64KB of the host memory can be used for backing VM RAM and
90	* ROM pages. The limit is imposed by the 28-bit page ID used
91	* internally in GMM. It is also limited by PAE.
92	* - A VM can be assigned all the memory GMM can allocate, however, the
93	* Main API will restrict this to 3584MB (MM_RAM_MAX_IN_MB).
94	* - Max 32 virtual CPUs (VMM_MAX_CPU_COUNT).
95	*
96	*/
97
98
99	/*********************************************************************************************************************************
100	* Header Files *
101	*********************************************************************************************************************************/
102	#define LOG_GROUP LOG_GROUP_VMM
103	#include <VBox/vmm/vmm.h>
104	#include <VBox/vmm/vmapi.h>
105	#include <VBox/vmm/pgm.h>
106	#include <VBox/vmm/cfgm.h>
107	#include <VBox/vmm/pdmqueue.h>
108	#include <VBox/vmm/pdmcritsect.h>
109	#include <VBox/vmm/pdmcritsectrw.h>
110	#include <VBox/vmm/pdmapi.h>
111	#include <VBox/vmm/cpum.h>
112	#include <VBox/vmm/gim.h>
113	#include <VBox/vmm/mm.h>
114	#include <VBox/vmm/nem.h>
115	#include <VBox/vmm/iom.h>
116	#include <VBox/vmm/trpm.h>
117	#include <VBox/vmm/selm.h>
118	#include <VBox/vmm/em.h>
119	#include <VBox/sup.h>
120	#include <VBox/vmm/dbgf.h>
121	#include <VBox/vmm/csam.h>
122	#include <VBox/vmm/patm.h>
123	#include <VBox/vmm/apic.h>
124	#ifdef VBOX_WITH_REM
125	# include <VBox/vmm/rem.h>
126	#endif
127	#include <VBox/vmm/ssm.h>
128	#include <VBox/vmm/ftm.h>
129	#include <VBox/vmm/tm.h>
130	#include "VMMInternal.h"
131	#include "VMMSwitcher.h"
132	#include <VBox/vmm/vm.h>
133	#include <VBox/vmm/uvm.h>
134
135	#include <VBox/err.h>
136	#include <VBox/param.h>
137	#include <VBox/version.h>
138	#include <VBox/vmm/hm.h>
139	#include <iprt/assert.h>
140	#include <iprt/alloc.h>
141	#include <iprt/asm.h>
142	#include <iprt/time.h>
143	#include <iprt/semaphore.h>
144	#include <iprt/stream.h>
145	#include <iprt/string.h>
146	#include <iprt/stdarg.h>
147	#include <iprt/ctype.h>
148	#include <iprt/x86.h>
149
150
151	/*********************************************************************************************************************************
152	* Defined Constants And Macros *
153	*********************************************************************************************************************************/
154	/** The saved state version. */
155	#define VMM_SAVED_STATE_VERSION 4
156	/** The saved state version used by v3.0 and earlier. (Teleportation) */
157	#define VMM_SAVED_STATE_VERSION_3_0 3
158
159	/** Macro for flushing the ring-0 logging. */
160	#define VMM_FLUSH_R0_LOG(a_pR0Logger, a_pR3Logger) \
161	do { \
162	PVMMR0LOGGER pVmmLogger = (a_pR0Logger); \
163	if (!pVmmLogger \|\| pVmmLogger->Logger.offScratch == 0) \
164	{ /* likely? */ } \
165	else \
166	RTLogFlushR0(a_pR3Logger, &pVmmLogger->Logger); \
167	} while (0)
168
169
170	/*********************************************************************************************************************************
171	* Internal Functions *
172	*********************************************************************************************************************************/
173	static int vmmR3InitStacks(PVM pVM);
174	static int vmmR3InitLoggers(PVM pVM);
175	static void vmmR3InitRegisterStats(PVM pVM);
176	static DECLCALLBACK(int) vmmR3Save(PVM pVM, PSSMHANDLE pSSM);
177	static DECLCALLBACK(int) vmmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
178	static DECLCALLBACK(void) vmmR3YieldEMT(PVM pVM, PTMTIMER pTimer, void *pvUser);
179	static VBOXSTRICTRC vmmR3EmtRendezvousCommon(PVM pVM, PVMCPU pVCpu, bool fIsCaller,
180	uint32_t fFlags, PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser);
181	static int vmmR3ServiceCallRing3Request(PVM pVM, PVMCPU pVCpu);
182	static DECLCALLBACK(void) vmmR3InfoFF(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
183
184
185	/**
186	* Initializes the VMM.
187	*
188	* @returns VBox status code.
189	* @param pVM The cross context VM structure.
190	*/
191	VMMR3_INT_DECL(int) VMMR3Init(PVM pVM)
192	{
193	LogFlow(("VMMR3Init\n"));
194
195	/*
196	* Assert alignment, sizes and order.
197	*/
198	AssertMsg(pVM->vmm.s.offVM == 0, ("Already initialized!\n"));
199	AssertCompile(sizeof(pVM->vmm.s) <= sizeof(pVM->vmm.padding));
200	AssertCompile(sizeof(pVM->aCpus[0].vmm.s) <= sizeof(pVM->aCpus[0].vmm.padding));
201
202	/*
203	* Init basic VM VMM members.
204	*/
205	pVM->vmm.s.offVM = RT_UOFFSETOF(VM, vmm);
206	pVM->vmm.s.pahEvtRendezvousEnterOrdered = NULL;
207	pVM->vmm.s.hEvtRendezvousEnterOneByOne = NIL_RTSEMEVENT;
208	pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce = NIL_RTSEMEVENTMULTI;
209	pVM->vmm.s.hEvtMulRendezvousDone = NIL_RTSEMEVENTMULTI;
210	pVM->vmm.s.hEvtRendezvousDoneCaller = NIL_RTSEMEVENT;
211	pVM->vmm.s.hEvtMulRendezvousRecursionPush = NIL_RTSEMEVENTMULTI;
212	pVM->vmm.s.hEvtMulRendezvousRecursionPop = NIL_RTSEMEVENTMULTI;
213	pVM->vmm.s.hEvtRendezvousRecursionPushCaller = NIL_RTSEMEVENT;
214	pVM->vmm.s.hEvtRendezvousRecursionPopCaller = NIL_RTSEMEVENT;
215
216	/** @cfgm{/YieldEMTInterval, uint32_t, 1, UINT32_MAX, 23, ms}
217	* The EMT yield interval. The EMT yielding is a hack we employ to play a
218	* bit nicer with the rest of the system (like for instance the GUI).
219	*/
220	int rc = CFGMR3QueryU32Def(CFGMR3GetRoot(pVM), "YieldEMTInterval", &pVM->vmm.s.cYieldEveryMillies,
221	23 /* Value arrived at after experimenting with the grub boot prompt. */);
222	AssertMsgRCReturn(rc, ("Configuration error. Failed to query \"YieldEMTInterval\", rc=%Rrc\n", rc), rc);
223
224
225	/** @cfgm{/VMM/UsePeriodicPreemptionTimers, boolean, true}
226	* Controls whether we employ per-cpu preemption timers to limit the time
227	* spent executing guest code. This option is not available on all
228	* platforms and we will silently ignore this setting then. If we are
229	* running in VT-x mode, we will use the VMX-preemption timer instead of
230	* this one when possible.
231	*/
232	PCFGMNODE pCfgVMM = CFGMR3GetChild(CFGMR3GetRoot(pVM), "VMM");
233	rc = CFGMR3QueryBoolDef(pCfgVMM, "UsePeriodicPreemptionTimers", &pVM->vmm.s.fUsePeriodicPreemptionTimers, true);
234	AssertMsgRCReturn(rc, ("Configuration error. Failed to query \"VMM/UsePeriodicPreemptionTimers\", rc=%Rrc\n", rc), rc);
235
236	/*
237	* Initialize the VMM rendezvous semaphores.
238	*/
239	pVM->vmm.s.pahEvtRendezvousEnterOrdered = (PRTSEMEVENT)MMR3HeapAlloc(pVM, MM_TAG_VMM, sizeof(RTSEMEVENT) * pVM->cCpus);
240	if (!pVM->vmm.s.pahEvtRendezvousEnterOrdered)
241	return VERR_NO_MEMORY;
242	for (VMCPUID i = 0; i < pVM->cCpus; i++)
243	pVM->vmm.s.pahEvtRendezvousEnterOrdered[i] = NIL_RTSEMEVENT;
244	for (VMCPUID i = 0; i < pVM->cCpus; i++)
245	{
246	rc = RTSemEventCreate(&pVM->vmm.s.pahEvtRendezvousEnterOrdered[i]);
247	AssertRCReturn(rc, rc);
248	}
249	rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousEnterOneByOne);
250	AssertRCReturn(rc, rc);
251	rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
252	AssertRCReturn(rc, rc);
253	rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousDone);
254	AssertRCReturn(rc, rc);
255	rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousDoneCaller);
256	AssertRCReturn(rc, rc);
257	rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousRecursionPush);
258	AssertRCReturn(rc, rc);
259	rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousRecursionPop);
260	AssertRCReturn(rc, rc);
261	rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousRecursionPushCaller);
262	AssertRCReturn(rc, rc);
263	rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousRecursionPopCaller);
264	AssertRCReturn(rc, rc);
265
266	/*
267	* Register the saved state data unit.
268	*/
269	rc = SSMR3RegisterInternal(pVM, "vmm", 1, VMM_SAVED_STATE_VERSION, VMM_STACK_SIZE + sizeof(RTGCPTR),
270	NULL, NULL, NULL,
271	NULL, vmmR3Save, NULL,
272	NULL, vmmR3Load, NULL);
273	if (RT_FAILURE(rc))
274	return rc;
275
276	/*
277	* Register the Ring-0 VM handle with the session for fast ioctl calls.
278	*/
279	rc = SUPR3SetVMForFastIOCtl(pVM->pVMR0);
280	if (RT_FAILURE(rc))
281	return rc;
282
283	/*
284	* Init various sub-components.
285	*/
286	rc = vmmR3SwitcherInit(pVM);
287	if (RT_SUCCESS(rc))
288	{
289	rc = vmmR3InitStacks(pVM);
290	if (RT_SUCCESS(rc))
291	{
292	rc = vmmR3InitLoggers(pVM);
293
294	#ifdef VBOX_WITH_NMI
295	/*
296	* Allocate mapping for the host APIC.
297	*/
298	if (RT_SUCCESS(rc))
299	{
300	rc = MMR3HyperReserve(pVM, PAGE_SIZE, "Host APIC", &pVM->vmm.s.GCPtrApicBase);
301	AssertRC(rc);
302	}
303	#endif
304	if (RT_SUCCESS(rc))
305	{
306	/*
307	* Debug info and statistics.
308	*/
309	DBGFR3InfoRegisterInternal(pVM, "fflags", "Displays the current Forced actions Flags.", vmmR3InfoFF);
310	vmmR3InitRegisterStats(pVM);
311	vmmInitFormatTypes();
312
313	return VINF_SUCCESS;
314	}
315	}
316	/** @todo Need failure cleanup. */
317
318	//more todo in here?
319	//if (RT_SUCCESS(rc))
320	//{
321	//}
322	//int rc2 = vmmR3TermCoreCode(pVM);
323	//AssertRC(rc2));
324	}
325
326	return rc;
327	}
328
329
330	/**
331	* Allocate & setup the VMM RC stack(s) (for EMTs).
332	*
333	* The stacks are also used for long jumps in Ring-0.
334	*
335	* @returns VBox status code.
336	* @param pVM The cross context VM structure.
337	*
338	* @remarks The optional guard page gets it protection setup up during R3 init
339	* completion because of init order issues.
340	*/
341	static int vmmR3InitStacks(PVM pVM)
342	{
343	int rc = VINF_SUCCESS;
344	#ifdef VMM_R0_SWITCH_STACK
345	uint32_t fFlags = MMHYPER_AONR_FLAGS_KERNEL_MAPPING;
346	#else
347	uint32_t fFlags = 0;
348	#endif
349
350	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
351	{
352	PVMCPU pVCpu = &pVM->aCpus[idCpu];
353
354	#ifdef VBOX_STRICT_VMM_STACK
355	rc = MMR3HyperAllocOnceNoRelEx(pVM, PAGE_SIZE + VMM_STACK_SIZE + PAGE_SIZE,
356	#else
357	rc = MMR3HyperAllocOnceNoRelEx(pVM, VMM_STACK_SIZE,
358	#endif
359	PAGE_SIZE, MM_TAG_VMM, fFlags, (void **)&pVCpu->vmm.s.pbEMTStackR3);
360	if (RT_SUCCESS(rc))
361	{
362	#ifdef VBOX_STRICT_VMM_STACK
363	pVCpu->vmm.s.pbEMTStackR3 += PAGE_SIZE;
364	#endif
365	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
366	/* MMHyperR3ToR0 returns R3 when not doing hardware assisted virtualization. */
367	if (VM_IS_RAW_MODE_ENABLED(pVM))
368	pVCpu->vmm.s.CallRing3JmpBufR0.pvSavedStack = NIL_RTR0PTR;
369	else
370	#endif
371	pVCpu->vmm.s.CallRing3JmpBufR0.pvSavedStack = MMHyperR3ToR0(pVM, pVCpu->vmm.s.pbEMTStackR3);
372	pVCpu->vmm.s.pbEMTStackRC = MMHyperR3ToRC(pVM, pVCpu->vmm.s.pbEMTStackR3);
373	pVCpu->vmm.s.pbEMTStackBottomRC = pVCpu->vmm.s.pbEMTStackRC + VMM_STACK_SIZE;
374	AssertRelease(pVCpu->vmm.s.pbEMTStackRC);
375
376	CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC);
377	}
378	}
379
380	return rc;
381	}
382
383
384	/**
385	* Initialize the loggers.
386	*
387	* @returns VBox status code.
388	* @param pVM The cross context VM structure.
389	*/
390	static int vmmR3InitLoggers(PVM pVM)
391	{
392	int rc;
393	#define RTLogCalcSizeForR0(cGroups, fFlags) (RT_UOFFSETOF_DYN(VMMR0LOGGER, Logger.afGroups[cGroups]) + PAGE_SIZE)
394
395	/*
396	* Allocate RC & R0 Logger instances (they are finalized in the relocator).
397	*/
398	#ifdef LOG_ENABLED
399	PRTLOGGER pLogger = RTLogDefaultInstance();
400	if (pLogger)
401	{
402	if (VM_IS_RAW_MODE_ENABLED(pVM))
403	{
404	pVM->vmm.s.cbRCLogger = RT_UOFFSETOF_DYN(RTLOGGERRC, afGroups[pLogger->cGroups]);
405	rc = MMR3HyperAllocOnceNoRel(pVM, pVM->vmm.s.cbRCLogger, 0, MM_TAG_VMM, (void **)&pVM->vmm.s.pRCLoggerR3);
406	if (RT_FAILURE(rc))
407	return rc;
408	pVM->vmm.s.pRCLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCLoggerR3);
409	}
410
411	# ifdef VBOX_WITH_R0_LOGGING
412	size_t const cbLogger = RTLogCalcSizeForR0(pLogger->cGroups, 0);
413	for (VMCPUID i = 0; i < pVM->cCpus; i++)
414	{
415	PVMCPU pVCpu = &pVM->aCpus[i];
416	rc = MMR3HyperAllocOnceNoRelEx(pVM, cbLogger, PAGE_SIZE, MM_TAG_VMM, MMHYPER_AONR_FLAGS_KERNEL_MAPPING,
417	(void **)&pVCpu->vmm.s.pR0LoggerR3);
418	if (RT_FAILURE(rc))
419	return rc;
420	pVCpu->vmm.s.pR0LoggerR3->pVM = pVM->pVMR0;
421	//pVCpu->vmm.s.pR0LoggerR3->fCreated = false;
422	pVCpu->vmm.s.pR0LoggerR3->cbLogger = (uint32_t)cbLogger;
423	pVCpu->vmm.s.pR0LoggerR0 = MMHyperR3ToR0(pVM, pVCpu->vmm.s.pR0LoggerR3);
424	}
425	# endif
426	}
427	#endif /* LOG_ENABLED */
428
429	/*
430	* Release logging.
431	*/
432	PRTLOGGER pRelLogger = RTLogRelGetDefaultInstance();
433	if (pRelLogger)
434	{
435	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
436	/*
437	* Allocate RC release logger instances (finalized in the relocator).
438	*/
439	if (VM_IS_RAW_MODE_ENABLED(pVM))
440	{
441	pVM->vmm.s.cbRCRelLogger = RT_UOFFSETOF_DYN(RTLOGGERRC, afGroups[pRelLogger->cGroups]);
442	rc = MMR3HyperAllocOnceNoRel(pVM, pVM->vmm.s.cbRCRelLogger, 0, MM_TAG_VMM, (void **)&pVM->vmm.s.pRCRelLoggerR3);
443	if (RT_FAILURE(rc))
444	return rc;
445	pVM->vmm.s.pRCRelLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCRelLoggerR3);
446	}
447	#endif
448
449	/*
450	* Ring-0 release logger.
451	*/
452	RTR0PTR pfnLoggerWrapper = NIL_RTR0PTR;
453	rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerWrapper", &pfnLoggerWrapper);
454	AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerWrapper not found! rc=%Rra\n", rc), rc);
455
456	RTR0PTR pfnLoggerFlush = NIL_RTR0PTR;
457	rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerFlush", &pfnLoggerFlush);
458	AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerFlush not found! rc=%Rra\n", rc), rc);
459
460	size_t const cbLogger = RTLogCalcSizeForR0(pRelLogger->cGroups, 0);
461
462	for (VMCPUID i = 0; i < pVM->cCpus; i++)
463	{
464	PVMCPU pVCpu = &pVM->aCpus[i];
465	rc = MMR3HyperAllocOnceNoRelEx(pVM, cbLogger, PAGE_SIZE, MM_TAG_VMM, MMHYPER_AONR_FLAGS_KERNEL_MAPPING,
466	(void **)&pVCpu->vmm.s.pR0RelLoggerR3);
467	if (RT_FAILURE(rc))
468	return rc;
469	PVMMR0LOGGER pVmmLogger = pVCpu->vmm.s.pR0RelLoggerR3;
470	RTR0PTR R0PtrVmmLogger = MMHyperR3ToR0(pVM, pVmmLogger);
471	pVCpu->vmm.s.pR0RelLoggerR0 = R0PtrVmmLogger;
472	pVmmLogger->pVM = pVM->pVMR0;
473	pVmmLogger->cbLogger = (uint32_t)cbLogger;
474	pVmmLogger->fCreated = false;
475	pVmmLogger->fFlushingDisabled = false;
476	pVmmLogger->fRegistered = false;
477	pVmmLogger->idCpu = i;
478
479	char szR0ThreadName[16];
480	RTStrPrintf(szR0ThreadName, sizeof(szR0ThreadName), "EMT-%u-R0", i);
481	rc = RTLogCreateForR0(&pVmmLogger->Logger, pVmmLogger->cbLogger, R0PtrVmmLogger + RT_UOFFSETOF(VMMR0LOGGER, Logger),
482	pfnLoggerWrapper, pfnLoggerFlush,
483	RTLOGFLAGS_BUFFERED, RTLOGDEST_DUMMY, szR0ThreadName);
484	AssertReleaseMsgRCReturn(rc, ("RTLogCreateForR0 failed! rc=%Rra\n", rc), rc);
485
486	/* We only update the release log instance here. */
487	rc = RTLogCopyGroupsAndFlagsForR0(&pVmmLogger->Logger, R0PtrVmmLogger + RT_UOFFSETOF(VMMR0LOGGER, Logger),
488	pRelLogger, RTLOGFLAGS_BUFFERED, UINT32_MAX);
489	AssertReleaseMsgRCReturn(rc, ("RTLogCopyGroupsAndFlagsForR0 failed! rc=%Rra\n", rc), rc);
490
491	pVmmLogger->fCreated = true;
492	}
493	}
494
495	return VINF_SUCCESS;
496	}
497
498
499	/**
500	* VMMR3Init worker that register the statistics with STAM.
501	*
502	* @param pVM The cross context VM structure.
503	*/
504	static void vmmR3InitRegisterStats(PVM pVM)
505	{
506	RT_NOREF_PV(pVM);
507
508	/*
509	* Statistics.
510	*/
511	STAM_REG(pVM, &pVM->vmm.s.StatRunRC, STAMTYPE_COUNTER, "/VMM/RunRC", STAMUNIT_OCCURENCES, "Number of context switches.");
512	STAM_REG(pVM, &pVM->vmm.s.StatRZRetNormal, STAMTYPE_COUNTER, "/VMM/RZRet/Normal", STAMUNIT_OCCURENCES, "Number of VINF_SUCCESS returns.");
513	STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterrupt, STAMTYPE_COUNTER, "/VMM/RZRet/Interrupt", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT returns.");
514	STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterruptHyper, STAMTYPE_COUNTER, "/VMM/RZRet/InterruptHyper", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT_HYPER returns.");
515	STAM_REG(pVM, &pVM->vmm.s.StatRZRetGuestTrap, STAMTYPE_COUNTER, "/VMM/RZRet/GuestTrap", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_GUEST_TRAP returns.");
516	STAM_REG(pVM, &pVM->vmm.s.StatRZRetRingSwitch, STAMTYPE_COUNTER, "/VMM/RZRet/RingSwitch", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_RING_SWITCH returns.");
517	STAM_REG(pVM, &pVM->vmm.s.StatRZRetRingSwitchInt, STAMTYPE_COUNTER, "/VMM/RZRet/RingSwitchInt", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_RING_SWITCH_INT returns.");
518	STAM_REG(pVM, &pVM->vmm.s.StatRZRetStaleSelector, STAMTYPE_COUNTER, "/VMM/RZRet/StaleSelector", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_STALE_SELECTOR returns.");
519	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIRETTrap, STAMTYPE_COUNTER, "/VMM/RZRet/IRETTrap", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_IRET_TRAP returns.");
520	STAM_REG(pVM, &pVM->vmm.s.StatRZRetEmulate, STAMTYPE_COUNTER, "/VMM/RZRet/Emulate", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION returns.");
521	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchEmulate, STAMTYPE_COUNTER, "/VMM/RZRet/PatchEmulate", STAMUNIT_OCCURENCES, "Number of VINF_PATCH_EMULATE_INSTR returns.");
522	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIORead, STAMTYPE_COUNTER, "/VMM/RZRet/IORead", STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_IOPORT_READ returns.");
523	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIOWrite, STAMTYPE_COUNTER, "/VMM/RZRet/IOWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_IOPORT_WRITE returns.");
524	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIOCommitWrite, STAMTYPE_COUNTER, "/VMM/RZRet/IOCommitWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_IOPORT_COMMIT_WRITE returns.");
525	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIORead, STAMTYPE_COUNTER, "/VMM/RZRet/MMIORead", STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_MMIO_READ returns.");
526	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOWrite, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_MMIO_WRITE returns.");
527	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOCommitWrite, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOCommitWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_MMIO_COMMIT_WRITE returns.");
528	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOReadWrite, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOReadWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_MMIO_READ_WRITE returns.");
529	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOPatchRead, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOPatchRead", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_PATCH_READ returns.");
530	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOPatchWrite, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOPatchWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_PATCH_WRITE returns.");
531	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMSRRead, STAMTYPE_COUNTER, "/VMM/RZRet/MSRRead", STAMUNIT_OCCURENCES, "Number of VINF_CPUM_R3_MSR_READ returns.");
532	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMSRWrite, STAMTYPE_COUNTER, "/VMM/RZRet/MSRWrite", STAMUNIT_OCCURENCES, "Number of VINF_CPUM_R3_MSR_WRITE returns.");
533	STAM_REG(pVM, &pVM->vmm.s.StatRZRetLDTFault, STAMTYPE_COUNTER, "/VMM/RZRet/LDTFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_GDT_FAULT returns.");
534	STAM_REG(pVM, &pVM->vmm.s.StatRZRetGDTFault, STAMTYPE_COUNTER, "/VMM/RZRet/GDTFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_LDT_FAULT returns.");
535	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIDTFault, STAMTYPE_COUNTER, "/VMM/RZRet/IDTFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_IDT_FAULT returns.");
536	STAM_REG(pVM, &pVM->vmm.s.StatRZRetTSSFault, STAMTYPE_COUNTER, "/VMM/RZRet/TSSFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_TSS_FAULT returns.");
537	STAM_REG(pVM, &pVM->vmm.s.StatRZRetCSAMTask, STAMTYPE_COUNTER, "/VMM/RZRet/CSAMTask", STAMUNIT_OCCURENCES, "Number of VINF_CSAM_PENDING_ACTION returns.");
538	STAM_REG(pVM, &pVM->vmm.s.StatRZRetSyncCR3, STAMTYPE_COUNTER, "/VMM/RZRet/SyncCR", STAMUNIT_OCCURENCES, "Number of VINF_PGM_SYNC_CR3 returns.");
539	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMisc, STAMTYPE_COUNTER, "/VMM/RZRet/Misc", STAMUNIT_OCCURENCES, "Number of misc returns.");
540	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchInt3, STAMTYPE_COUNTER, "/VMM/RZRet/PatchInt3", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_INT3 returns.");
541	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchPF, STAMTYPE_COUNTER, "/VMM/RZRet/PatchPF", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_TRAP_PF returns.");
542	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchGP, STAMTYPE_COUNTER, "/VMM/RZRet/PatchGP", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_TRAP_GP returns.");
543	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchIretIRQ, STAMTYPE_COUNTER, "/VMM/RZRet/PatchIret", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PENDING_IRQ_AFTER_IRET returns.");
544	STAM_REG(pVM, &pVM->vmm.s.StatRZRetRescheduleREM, STAMTYPE_COUNTER, "/VMM/RZRet/ScheduleREM", STAMUNIT_OCCURENCES, "Number of VINF_EM_RESCHEDULE_REM returns.");
545	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Total, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
546	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Unknown, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Unknown", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns without responsible force flag.");
547	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3FF, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/ToR3", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VMCPU_FF_TO_R3.");
548	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3TMVirt, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/TMVirt", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VM_FF_TM_VIRTUAL_SYNC.");
549	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3HandyPages, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Handy", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VM_FF_PGM_NEED_HANDY_PAGES.");
550	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3PDMQueues, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/PDMQueue", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VM_FF_PDM_QUEUES.");
551	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Rendezvous, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Rendezvous", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VM_FF_EMT_RENDEZVOUS.");
552	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Timer, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Timer", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VMCPU_FF_TIMER.");
553	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3DMA, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/DMA", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VM_FF_PDM_DMA.");
554	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3CritSect, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/CritSect", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VMCPU_FF_PDM_CRITSECT.");
555	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Iem, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/IEM", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VMCPU_FF_IEM.");
556	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Iom, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/IOM", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VMCPU_FF_IOM.");
557	STAM_REG(pVM, &pVM->vmm.s.StatRZRetTimerPending, STAMTYPE_COUNTER, "/VMM/RZRet/TimerPending", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TIMER_PENDING returns.");
558	STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterruptPending, STAMTYPE_COUNTER, "/VMM/RZRet/InterruptPending", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT_PENDING returns.");
559	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPATMDuplicateFn, STAMTYPE_COUNTER, "/VMM/RZRet/PATMDuplicateFn", STAMUNIT_OCCURENCES, "Number of VINF_PATM_DUPLICATE_FUNCTION returns.");
560	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPGMChangeMode, STAMTYPE_COUNTER, "/VMM/RZRet/PGMChangeMode", STAMUNIT_OCCURENCES, "Number of VINF_PGM_CHANGE_MODE returns.");
561	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPGMFlushPending, STAMTYPE_COUNTER, "/VMM/RZRet/PGMFlushPending", STAMUNIT_OCCURENCES, "Number of VINF_PGM_POOL_FLUSH_PENDING returns.");
562	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPendingRequest, STAMTYPE_COUNTER, "/VMM/RZRet/PendingRequest", STAMUNIT_OCCURENCES, "Number of VINF_EM_PENDING_REQUEST returns.");
563	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchTPR, STAMTYPE_COUNTER, "/VMM/RZRet/PatchTPR", STAMUNIT_OCCURENCES, "Number of VINF_EM_HM_PATCH_TPR_INSTR returns.");
564	STAM_REG(pVM, &pVM->vmm.s.StatRZRetCallRing3, STAMTYPE_COUNTER, "/VMM/RZCallR3/Misc", STAMUNIT_OCCURENCES, "Number of Other ring-3 calls.");
565	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPDMLock, STAMTYPE_COUNTER, "/VMM/RZCallR3/PDMLock", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PDM_LOCK calls.");
566	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPDMCritSectEnter, STAMTYPE_COUNTER, "/VMM/RZCallR3/PDMCritSectEnter", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PDM_CRITSECT_ENTER calls.");
567	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMLock, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMLock", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_LOCK calls.");
568	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMPoolGrow, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMPoolGrow", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_POOL_GROW calls.");
569	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMMapChunk, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMMapChunk", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_MAP_CHUNK calls.");
570	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMAllocHandy, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMAllocHandy", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES calls.");
571	STAM_REG(pVM, &pVM->vmm.s.StatRZCallRemReplay, STAMTYPE_COUNTER, "/VMM/RZCallR3/REMReplay", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_REM_REPLAY_HANDLER_NOTIFICATIONS calls.");
572	STAM_REG(pVM, &pVM->vmm.s.StatRZCallLogFlush, STAMTYPE_COUNTER, "/VMM/RZCallR3/VMMLogFlush", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VMM_LOGGER_FLUSH calls.");
573	STAM_REG(pVM, &pVM->vmm.s.StatRZCallVMSetError, STAMTYPE_COUNTER, "/VMM/RZCallR3/VMSetError", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VM_SET_ERROR calls.");
574	STAM_REG(pVM, &pVM->vmm.s.StatRZCallVMSetRuntimeError, STAMTYPE_COUNTER, "/VMM/RZCallR3/VMRuntimeError", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VM_SET_RUNTIME_ERROR calls.");
575
576	#ifdef VBOX_WITH_STATISTICS
577	for (VMCPUID i = 0; i < pVM->cCpus; i++)
578	{
579	STAMR3RegisterF(pVM, &pVM->aCpus[i].vmm.s.CallRing3JmpBufR0.cbUsedMax, STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Max amount of stack used.", "/VMM/Stack/CPU%u/Max", i);
580	STAMR3RegisterF(pVM, &pVM->aCpus[i].vmm.s.CallRing3JmpBufR0.cbUsedAvg, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Average stack usage.", "/VMM/Stack/CPU%u/Avg", i);
581	STAMR3RegisterF(pVM, &pVM->aCpus[i].vmm.s.CallRing3JmpBufR0.cUsedTotal, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of stack usages.", "/VMM/Stack/CPU%u/Uses", i);
582	}
583	#endif
584	}
585
586
587	/**
588	* Worker for VMMR3InitR0 that calls ring-0 to do EMT specific initialization.
589	*
590	* @returns VBox status code.
591	* @param pVM The cross context VM structure.
592	* @param pVCpu The cross context per CPU structure.
593	* @thread EMT(pVCpu)
594	*/
595	static DECLCALLBACK(int) vmmR3InitR0Emt(PVM pVM, PVMCPU pVCpu)
596	{
597	return VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_VMMR0_INIT_EMT, 0, NULL);
598	}
599
600
601	/**
602	* Initializes the R0 VMM.
603	*
604	* @returns VBox status code.
605	* @param pVM The cross context VM structure.
606	*/
607	VMMR3_INT_DECL(int) VMMR3InitR0(PVM pVM)
608	{
609	int rc;
610	PVMCPU pVCpu = VMMGetCpu(pVM);
611	Assert(pVCpu && pVCpu->idCpu == 0);
612
613	#ifdef LOG_ENABLED
614	/*
615	* Initialize the ring-0 logger if we haven't done so yet.
616	*/
617	if ( pVCpu->vmm.s.pR0LoggerR3
618	&& !pVCpu->vmm.s.pR0LoggerR3->fCreated)
619	{
620	rc = VMMR3UpdateLoggers(pVM);
621	if (RT_FAILURE(rc))
622	return rc;
623	}
624	#endif
625
626	/*
627	* Call Ring-0 entry with init code.
628	*/
629	for (;;)
630	{
631	#ifdef NO_SUPCALLR0VMM
632	//rc = VERR_GENERAL_FAILURE;
633	rc = VINF_SUCCESS;
634	#else
635	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /idCpu/, VMMR0_DO_VMMR0_INIT, RT_MAKE_U64(VMMGetSvnRev(), vmmGetBuildType()), NULL);
636	#endif
637	/*
638	* Flush the logs.
639	*/
640	#ifdef LOG_ENABLED
641	VMM_FLUSH_R0_LOG(pVCpu->vmm.s.pR0LoggerR3, NULL);
642	#endif
643	VMM_FLUSH_R0_LOG(pVCpu->vmm.s.pR0RelLoggerR3, RTLogRelGetDefaultInstance());
644	if (rc != VINF_VMM_CALL_HOST)
645	break;
646	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
647	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
648	break;
649	/* Resume R0 */
650	}
651
652	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
653	{
654	LogRel(("VMM: R0 init failed, rc=%Rra\n", rc));
655	if (RT_SUCCESS(rc))
656	rc = VERR_IPE_UNEXPECTED_INFO_STATUS;
657	}
658
659	/* Log whether thread-context hooks are used (on Linux this can depend on how the kernel is configured). */
660	if (pVM->aCpus[0].vmm.s.hCtxHook != NIL_RTTHREADCTXHOOK)
661	LogRel(("VMM: Enabled thread-context hooks\n"));
662	else
663	LogRel(("VMM: Thread-context hooks unavailable\n"));
664
665	/* Log RTThreadPreemptIsPendingTrusty() and RTThreadPreemptIsPossible() results. */
666	if (pVM->vmm.s.fIsPreemptPendingApiTrusty)
667	LogRel(("VMM: RTThreadPreemptIsPending() can be trusted\n"));
668	else
669	LogRel(("VMM: Warning! RTThreadPreemptIsPending() cannot be trusted! Need to update kernel info?\n"));
670	if (pVM->vmm.s.fIsPreemptPossible)
671	LogRel(("VMM: Kernel preemption is possible\n"));
672	else
673	LogRel(("VMM: Kernel preemption is not possible it seems\n"));
674
675	/*
676	* Send all EMTs to ring-0 to get their logger initialized.
677	*/
678	for (VMCPUID idCpu = 0; RT_SUCCESS(rc) && idCpu < pVM->cCpus; idCpu++)
679	rc = VMR3ReqCallWait(pVM, idCpu, (PFNRT)vmmR3InitR0Emt, 2, pVM, &pVM->aCpus[idCpu]);
680
681	return rc;
682	}
683
684
685	#ifdef VBOX_WITH_RAW_MODE
686	/**
687	* Initializes the RC VMM.
688	*
689	* @returns VBox status code.
690	* @param pVM The cross context VM structure.
691	*/
692	VMMR3_INT_DECL(int) VMMR3InitRC(PVM pVM)
693	{
694	PVMCPU pVCpu = VMMGetCpu(pVM);
695	Assert(pVCpu && pVCpu->idCpu == 0);
696
697	/* In VMX mode, there's no need to init RC. */
698	if (!VM_IS_RAW_MODE_ENABLED(pVM))
699	return VINF_SUCCESS;
700
701	AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
702
703	/*
704	* Call VMMRCInit():
705	* -# resolve the address.
706	* -# setup stackframe and EIP to use the trampoline.
707	* -# do a generic hypervisor call.
708	*/
709	RTRCPTR RCPtrEP;
710	int rc = PDMR3LdrGetSymbolRC(pVM, VMMRC_MAIN_MODULE_NAME, "VMMRCEntry", &RCPtrEP);
711	if (RT_SUCCESS(rc))
712	{
713	CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC); /* Clear the stack. */
714	uint64_t u64TS = RTTimeProgramStartNanoTS();
715	CPUMPushHyper(pVCpu, RT_HI_U32(u64TS)); /* Param 4: The program startup TS - Hi. */
716	CPUMPushHyper(pVCpu, RT_LO_U32(u64TS)); /* Param 4: The program startup TS - Lo. */
717	CPUMPushHyper(pVCpu, vmmGetBuildType()); /* Param 3: Version argument. */
718	CPUMPushHyper(pVCpu, VMMGetSvnRev()); /* Param 2: Version argument. */
719	CPUMPushHyper(pVCpu, VMMRC_DO_VMMRC_INIT); /* Param 1: Operation. */
720	CPUMPushHyper(pVCpu, pVM->pVMRC); /* Param 0: pVM */
721	CPUMPushHyper(pVCpu, 6 * sizeof(RTRCPTR)); /* trampoline param: stacksize. */
722	CPUMPushHyper(pVCpu, RCPtrEP); /* Call EIP. */
723	CPUMSetHyperEIP(pVCpu, pVM->vmm.s.pfnCallTrampolineRC);
724	Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
725
726	for (;;)
727	{
728	#ifdef NO_SUPCALLR0VMM
729	//rc = VERR_GENERAL_FAILURE;
730	rc = VINF_SUCCESS;
731	#else
732	rc = SUPR3CallVMMR0(pVM->pVMR0, 0 /* VCPU 0 */, VMMR0_DO_CALL_HYPERVISOR, NULL);
733	#endif
734	#ifdef LOG_ENABLED
735	PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
736	if ( pLogger
737	&& pLogger->offScratch > 0)
738	RTLogFlushRC(NULL, pLogger);
739	#endif
740	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
741	PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
742	if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
743	RTLogFlushRC(RTLogRelGetDefaultInstance(), pRelLogger);
744	#endif
745	if (rc != VINF_VMM_CALL_HOST)
746	break;
747	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
748	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
749	break;
750	}
751
752	/* Don't trigger assertions or guru if raw-mode is unavailable. */
753	if (rc != VERR_SUPDRV_NO_RAW_MODE_HYPER_V_ROOT)
754	{
755	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
756	{
757	VMMR3FatalDump(pVM, pVCpu, rc);
758	if (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST)
759	rc = VERR_IPE_UNEXPECTED_INFO_STATUS;
760	}
761	AssertRC(rc);
762	}
763	}
764	return rc;
765	}
766	#endif /* VBOX_WITH_RAW_MODE */
767
768
769	/**
770	* Called when an init phase completes.
771	*
772	* @returns VBox status code.
773	* @param pVM The cross context VM structure.
774	* @param enmWhat Which init phase.
775	*/
776	VMMR3_INT_DECL(int) VMMR3InitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
777	{
778	int rc = VINF_SUCCESS;
779
780	switch (enmWhat)
781	{
782	case VMINITCOMPLETED_RING3:
783	{
784	/*
785	* Set page attributes to r/w for stack pages.
786	*/
787	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
788	{
789	rc = PGMMapSetPage(pVM, pVM->aCpus[idCpu].vmm.s.pbEMTStackRC, VMM_STACK_SIZE,
790	X86_PTE_P \| X86_PTE_A \| X86_PTE_D \| X86_PTE_RW);
791	AssertRCReturn(rc, rc);
792	}
793
794	/*
795	* Create the EMT yield timer.
796	*/
797	rc = TMR3TimerCreateInternal(pVM, TMCLOCK_REAL, vmmR3YieldEMT, NULL, "EMT Yielder", &pVM->vmm.s.pYieldTimer);
798	AssertRCReturn(rc, rc);
799
800	rc = TMTimerSetMillies(pVM->vmm.s.pYieldTimer, pVM->vmm.s.cYieldEveryMillies);
801	AssertRCReturn(rc, rc);
802
803	#ifdef VBOX_WITH_NMI
804	/*
805	* Map the host APIC into GC - This is AMD/Intel + Host OS specific!
806	*/
807	rc = PGMMap(pVM, pVM->vmm.s.GCPtrApicBase, 0xfee00000, PAGE_SIZE,
808	X86_PTE_P \| X86_PTE_RW \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_A \| X86_PTE_D);
809	AssertRCReturn(rc, rc);
810	#endif
811
812	#ifdef VBOX_STRICT_VMM_STACK
813	/*
814	* Setup the stack guard pages: Two inaccessible pages at each sides of the
815	* stack to catch over/under-flows.
816	*/
817	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
818	{
819	uint8_t *pbEMTStackR3 = pVM->aCpus[idCpu].vmm.s.pbEMTStackR3;
820
821	memset(pbEMTStackR3 - PAGE_SIZE, 0xcc, PAGE_SIZE);
822	MMR3HyperSetGuard(pVM, pbEMTStackR3 - PAGE_SIZE, PAGE_SIZE, true /fSet/);
823
824	memset(pbEMTStackR3 + VMM_STACK_SIZE, 0xcc, PAGE_SIZE);
825	MMR3HyperSetGuard(pVM, pbEMTStackR3 + VMM_STACK_SIZE, PAGE_SIZE, true /fSet/);
826	}
827	pVM->vmm.s.fStackGuardsStationed = true;
828	#endif
829	break;
830	}
831
832	case VMINITCOMPLETED_HM:
833	{
834	/*
835	* Disable the periodic preemption timers if we can use the
836	* VMX-preemption timer instead.
837	*/
838	if ( pVM->vmm.s.fUsePeriodicPreemptionTimers
839	&& HMR3IsVmxPreemptionTimerUsed(pVM))
840	pVM->vmm.s.fUsePeriodicPreemptionTimers = false;
841	LogRel(("VMM: fUsePeriodicPreemptionTimers=%RTbool\n", pVM->vmm.s.fUsePeriodicPreemptionTimers));
842
843	/*
844	* Last chance for GIM to update its CPUID leaves if it requires
845	* knowledge/information from HM initialization.
846	*/
847	rc = GIMR3InitCompleted(pVM);
848	AssertRCReturn(rc, rc);
849
850	/*
851	* CPUM's post-initialization (print CPUIDs).
852	*/
853	CPUMR3LogCpuIds(pVM);
854	break;
855	}
856
857	default: /* shuts up gcc */
858	break;
859	}
860
861	return rc;
862	}
863
864
865	/**
866	* Terminate the VMM bits.
867	*
868	* @returns VBox status code.
869	* @param pVM The cross context VM structure.
870	*/
871	VMMR3_INT_DECL(int) VMMR3Term(PVM pVM)
872	{
873	PVMCPU pVCpu = VMMGetCpu(pVM);
874	Assert(pVCpu && pVCpu->idCpu == 0);
875
876	/*
877	* Call Ring-0 entry with termination code.
878	*/
879	int rc;
880	for (;;)
881	{
882	#ifdef NO_SUPCALLR0VMM
883	//rc = VERR_GENERAL_FAILURE;
884	rc = VINF_SUCCESS;
885	#else
886	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /idCpu/, VMMR0_DO_VMMR0_TERM, 0, NULL);
887	#endif
888	/*
889	* Flush the logs.
890	*/
891	#ifdef LOG_ENABLED
892	VMM_FLUSH_R0_LOG(pVCpu->vmm.s.pR0LoggerR3, NULL);
893	#endif
894	VMM_FLUSH_R0_LOG(pVCpu->vmm.s.pR0RelLoggerR3, RTLogRelGetDefaultInstance());
895	if (rc != VINF_VMM_CALL_HOST)
896	break;
897	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
898	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
899	break;
900	/* Resume R0 */
901	}
902	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
903	{
904	LogRel(("VMM: VMMR3Term: R0 term failed, rc=%Rra. (warning)\n", rc));
905	if (RT_SUCCESS(rc))
906	rc = VERR_IPE_UNEXPECTED_INFO_STATUS;
907	}
908
909	for (VMCPUID i = 0; i < pVM->cCpus; i++)
910	{
911	RTSemEventDestroy(pVM->vmm.s.pahEvtRendezvousEnterOrdered[i]);
912	pVM->vmm.s.pahEvtRendezvousEnterOrdered[i] = NIL_RTSEMEVENT;
913	}
914	RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousEnterOneByOne);
915	pVM->vmm.s.hEvtRendezvousEnterOneByOne = NIL_RTSEMEVENT;
916	RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
917	pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce = NIL_RTSEMEVENTMULTI;
918	RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousDone);
919	pVM->vmm.s.hEvtMulRendezvousDone = NIL_RTSEMEVENTMULTI;
920	RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousDoneCaller);
921	pVM->vmm.s.hEvtRendezvousDoneCaller = NIL_RTSEMEVENT;
922	RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousRecursionPush);
923	pVM->vmm.s.hEvtMulRendezvousRecursionPush = NIL_RTSEMEVENTMULTI;
924	RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousRecursionPop);
925	pVM->vmm.s.hEvtMulRendezvousRecursionPop = NIL_RTSEMEVENTMULTI;
926	RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousRecursionPushCaller);
927	pVM->vmm.s.hEvtRendezvousRecursionPushCaller = NIL_RTSEMEVENT;
928	RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousRecursionPopCaller);
929	pVM->vmm.s.hEvtRendezvousRecursionPopCaller = NIL_RTSEMEVENT;
930
931	#ifdef VBOX_STRICT_VMM_STACK
932	/*
933	* Make the two stack guard pages present again.
934	*/
935	if (pVM->vmm.s.fStackGuardsStationed)
936	{
937	for (VMCPUID i = 0; i < pVM->cCpus; i++)
938	{
939	uint8_t *pbEMTStackR3 = pVM->aCpus[i].vmm.s.pbEMTStackR3;
940	MMR3HyperSetGuard(pVM, pbEMTStackR3 - PAGE_SIZE, PAGE_SIZE, false /fSet/);
941	MMR3HyperSetGuard(pVM, pbEMTStackR3 + VMM_STACK_SIZE, PAGE_SIZE, false /fSet/);
942	}
943	pVM->vmm.s.fStackGuardsStationed = false;
944	}
945	#endif
946
947	vmmTermFormatTypes();
948	return rc;
949	}
950
951
952	/**
953	* Applies relocations to data and code managed by this
954	* component. This function will be called at init and
955	* whenever the VMM need to relocate it self inside the GC.
956	*
957	* The VMM will need to apply relocations to the core code.
958	*
959	* @param pVM The cross context VM structure.
960	* @param offDelta The relocation delta.
961	*/
962	VMMR3_INT_DECL(void) VMMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
963	{
964	LogFlow(("VMMR3Relocate: offDelta=%RGv\n", offDelta));
965
966	/*
967	* Recalc the RC address.
968	*/
969	#ifdef VBOX_WITH_RAW_MODE
970	pVM->vmm.s.pvCoreCodeRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pvCoreCodeR3);
971	#endif
972
973	/*
974	* The stack.
975	*/
976	for (VMCPUID i = 0; i < pVM->cCpus; i++)
977	{
978	PVMCPU pVCpu = &pVM->aCpus[i];
979
980	CPUMSetHyperESP(pVCpu, CPUMGetHyperESP(pVCpu) + offDelta);
981
982	pVCpu->vmm.s.pbEMTStackRC = MMHyperR3ToRC(pVM, pVCpu->vmm.s.pbEMTStackR3);
983	pVCpu->vmm.s.pbEMTStackBottomRC = pVCpu->vmm.s.pbEMTStackRC + VMM_STACK_SIZE;
984	}
985
986	/*
987	* All the switchers.
988	*/
989	vmmR3SwitcherRelocate(pVM, offDelta);
990
991	/*
992	* Get other RC entry points.
993	*/
994	if (VM_IS_RAW_MODE_ENABLED(pVM))
995	{
996	int rc = PDMR3LdrGetSymbolRC(pVM, VMMRC_MAIN_MODULE_NAME, "CPUMGCResumeGuest", &pVM->vmm.s.pfnCPUMRCResumeGuest);
997	AssertReleaseMsgRC(rc, ("CPUMGCResumeGuest not found! rc=%Rra\n", rc));
998
999	rc = PDMR3LdrGetSymbolRC(pVM, VMMRC_MAIN_MODULE_NAME, "CPUMGCResumeGuestV86", &pVM->vmm.s.pfnCPUMRCResumeGuestV86);
1000	AssertReleaseMsgRC(rc, ("CPUMGCResumeGuestV86 not found! rc=%Rra\n", rc));
1001	}
1002
1003	/*
1004	* Update the logger.
1005	*/
1006	VMMR3UpdateLoggers(pVM);
1007	}
1008
1009
1010	/**
1011	* Updates the settings for the RC and R0 loggers.
1012	*
1013	* @returns VBox status code.
1014	* @param pVM The cross context VM structure.
1015	*/
1016	VMMR3_INT_DECL(int) VMMR3UpdateLoggers(PVM pVM)
1017	{
1018	/*
1019	* Simply clone the logger instance (for RC).
1020	*/
1021	int rc = VINF_SUCCESS;
1022	RTRCPTR RCPtrLoggerFlush = 0;
1023
1024	if ( pVM->vmm.s.pRCLoggerR3
1025	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
1026	\|\| pVM->vmm.s.pRCRelLoggerR3
1027	#endif
1028	)
1029	{
1030	Assert(VM_IS_RAW_MODE_ENABLED(pVM));
1031	rc = PDMR3LdrGetSymbolRC(pVM, VMMRC_MAIN_MODULE_NAME, "vmmGCLoggerFlush", &RCPtrLoggerFlush);
1032	AssertReleaseMsgRC(rc, ("vmmGCLoggerFlush not found! rc=%Rra\n", rc));
1033	}
1034
1035	if (pVM->vmm.s.pRCLoggerR3)
1036	{
1037	Assert(VM_IS_RAW_MODE_ENABLED(pVM));
1038	RTRCPTR RCPtrLoggerWrapper = 0;
1039	rc = PDMR3LdrGetSymbolRC(pVM, VMMRC_MAIN_MODULE_NAME, "vmmGCLoggerWrapper", &RCPtrLoggerWrapper);
1040	AssertReleaseMsgRC(rc, ("vmmGCLoggerWrapper not found! rc=%Rra\n", rc));
1041
1042	pVM->vmm.s.pRCLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCLoggerR3);
1043	rc = RTLogCloneRC(NULL /* default */, pVM->vmm.s.pRCLoggerR3, pVM->vmm.s.cbRCLogger,
1044	RCPtrLoggerWrapper, RCPtrLoggerFlush, RTLOGFLAGS_BUFFERED);
1045	AssertReleaseMsgRC(rc, ("RTLogCloneRC failed! rc=%Rra\n", rc));
1046	}
1047
1048	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
1049	if (pVM->vmm.s.pRCRelLoggerR3)
1050	{
1051	Assert(VM_IS_RAW_MODE_ENABLED(pVM));
1052	RTRCPTR RCPtrLoggerWrapper = 0;
1053	rc = PDMR3LdrGetSymbolRC(pVM, VMMRC_MAIN_MODULE_NAME, "vmmGCRelLoggerWrapper", &RCPtrLoggerWrapper);
1054	AssertReleaseMsgRC(rc, ("vmmGCRelLoggerWrapper not found! rc=%Rra\n", rc));
1055
1056	pVM->vmm.s.pRCRelLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCRelLoggerR3);
1057	rc = RTLogCloneRC(RTLogRelGetDefaultInstance(), pVM->vmm.s.pRCRelLoggerR3, pVM->vmm.s.cbRCRelLogger,
1058	RCPtrLoggerWrapper, RCPtrLoggerFlush, RTLOGFLAGS_BUFFERED);
1059	AssertReleaseMsgRC(rc, ("RTLogCloneRC failed! rc=%Rra\n", rc));
1060	}
1061	#endif /* VBOX_WITH_RC_RELEASE_LOGGING */
1062
1063	#ifdef LOG_ENABLED
1064	/*
1065	* For the ring-0 EMT logger, we use a per-thread logger instance
1066	* in ring-0. Only initialize it once.
1067	*/
1068	PRTLOGGER const pDefault = RTLogDefaultInstance();
1069	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1070	{
1071	PVMCPU pVCpu = &pVM->aCpus[i];
1072	PVMMR0LOGGER pR0LoggerR3 = pVCpu->vmm.s.pR0LoggerR3;
1073	if (pR0LoggerR3)
1074	{
1075	if (!pR0LoggerR3->fCreated)
1076	{
1077	RTR0PTR pfnLoggerWrapper = NIL_RTR0PTR;
1078	rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerWrapper", &pfnLoggerWrapper);
1079	AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerWrapper not found! rc=%Rra\n", rc), rc);
1080
1081	RTR0PTR pfnLoggerFlush = NIL_RTR0PTR;
1082	rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerFlush", &pfnLoggerFlush);
1083	AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerFlush not found! rc=%Rra\n", rc), rc);
1084
1085	char szR0ThreadName[16];
1086	RTStrPrintf(szR0ThreadName, sizeof(szR0ThreadName), "EMT-%u-R0", i);
1087	rc = RTLogCreateForR0(&pR0LoggerR3->Logger, pR0LoggerR3->cbLogger,
1088	pVCpu->vmm.s.pR0LoggerR0 + RT_UOFFSETOF(VMMR0LOGGER, Logger),
1089	pfnLoggerWrapper, pfnLoggerFlush,
1090	RTLOGFLAGS_BUFFERED, RTLOGDEST_DUMMY, szR0ThreadName);
1091	AssertReleaseMsgRCReturn(rc, ("RTLogCreateForR0 failed! rc=%Rra\n", rc), rc);
1092
1093	pR0LoggerR3->idCpu = i;
1094	pR0LoggerR3->fCreated = true;
1095	pR0LoggerR3->fFlushingDisabled = false;
1096	}
1097
1098	rc = RTLogCopyGroupsAndFlagsForR0(&pR0LoggerR3->Logger, pVCpu->vmm.s.pR0LoggerR0 + RT_UOFFSETOF(VMMR0LOGGER, Logger),
1099	pDefault, RTLOGFLAGS_BUFFERED, UINT32_MAX);
1100	AssertRC(rc);
1101	}
1102	}
1103	#endif
1104	return rc;
1105	}
1106
1107
1108	/**
1109	* Gets the pointer to a buffer containing the R0/RC RTAssertMsg1Weak output.
1110	*
1111	* @returns Pointer to the buffer.
1112	* @param pVM The cross context VM structure.
1113	*/
1114	VMMR3DECL(const char *) VMMR3GetRZAssertMsg1(PVM pVM)
1115	{
1116	if (!VM_IS_RAW_MODE_ENABLED(pVM))
1117	return pVM->vmm.s.szRing0AssertMsg1;
1118
1119	RTRCPTR RCPtr;
1120	int rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_szRTAssertMsg1", &RCPtr);
1121	if (RT_SUCCESS(rc))
1122	return (const char *)MMHyperRCToR3(pVM, RCPtr);
1123
1124	return NULL;
1125	}
1126
1127
1128	/**
1129	* Returns the VMCPU of the specified virtual CPU.
1130	*
1131	* @returns The VMCPU pointer. NULL if @a idCpu or @a pUVM is invalid.
1132	*
1133	* @param pUVM The user mode VM handle.
1134	* @param idCpu The ID of the virtual CPU.
1135	*/
1136	VMMR3DECL(PVMCPU) VMMR3GetCpuByIdU(PUVM pUVM, RTCPUID idCpu)
1137	{
1138	UVM_ASSERT_VALID_EXT_RETURN(pUVM, NULL);
1139	AssertReturn(idCpu < pUVM->cCpus, NULL);
1140	VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, NULL);
1141	return &pUVM->pVM->aCpus[idCpu];
1142	}
1143
1144
1145	/**
1146	* Gets the pointer to a buffer containing the R0/RC RTAssertMsg2Weak output.
1147	*
1148	* @returns Pointer to the buffer.
1149	* @param pVM The cross context VM structure.
1150	*/
1151	VMMR3DECL(const char *) VMMR3GetRZAssertMsg2(PVM pVM)
1152	{
1153	if (!VM_IS_RAW_MODE_ENABLED(pVM))
1154	return pVM->vmm.s.szRing0AssertMsg2;
1155
1156	RTRCPTR RCPtr;
1157	int rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_szRTAssertMsg2", &RCPtr);
1158	if (RT_SUCCESS(rc))
1159	return (const char *)MMHyperRCToR3(pVM, RCPtr);
1160
1161	return NULL;
1162	}
1163
1164
1165	/**
1166	* Execute state save operation.
1167	*
1168	* @returns VBox status code.
1169	* @param pVM The cross context VM structure.
1170	* @param pSSM SSM operation handle.
1171	*/
1172	static DECLCALLBACK(int) vmmR3Save(PVM pVM, PSSMHANDLE pSSM)
1173	{
1174	LogFlow(("vmmR3Save:\n"));
1175
1176	/*
1177	* Save the started/stopped state of all CPUs except 0 as it will always
1178	* be running. This avoids breaking the saved state version. :-)
1179	*/
1180	for (VMCPUID i = 1; i < pVM->cCpus; i++)
1181	SSMR3PutBool(pSSM, VMCPUSTATE_IS_STARTED(VMCPU_GET_STATE(&pVM->aCpus[i])));
1182
1183	return SSMR3PutU32(pSSM, UINT32_MAX); /* terminator */
1184	}
1185
1186
1187	/**
1188	* Execute state load operation.
1189	*
1190	* @returns VBox status code.
1191	* @param pVM The cross context VM structure.
1192	* @param pSSM SSM operation handle.
1193	* @param uVersion Data layout version.
1194	* @param uPass The data pass.
1195	*/
1196	static DECLCALLBACK(int) vmmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
1197	{
1198	LogFlow(("vmmR3Load:\n"));
1199	Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
1200
1201	/*
1202	* Validate version.
1203	*/
1204	if ( uVersion != VMM_SAVED_STATE_VERSION
1205	&& uVersion != VMM_SAVED_STATE_VERSION_3_0)
1206	{
1207	AssertMsgFailed(("vmmR3Load: Invalid version uVersion=%u!\n", uVersion));
1208	return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
1209	}
1210
1211	if (uVersion <= VMM_SAVED_STATE_VERSION_3_0)
1212	{
1213	/* Ignore the stack bottom, stack pointer and stack bits. */
1214	RTRCPTR RCPtrIgnored;
1215	SSMR3GetRCPtr(pSSM, &RCPtrIgnored);
1216	SSMR3GetRCPtr(pSSM, &RCPtrIgnored);
1217	#ifdef RT_OS_DARWIN
1218	if ( SSMR3HandleVersion(pSSM) >= VBOX_FULL_VERSION_MAKE(3,0,0)
1219	&& SSMR3HandleVersion(pSSM) < VBOX_FULL_VERSION_MAKE(3,1,0)
1220	&& SSMR3HandleRevision(pSSM) >= 48858
1221	&& ( !strcmp(SSMR3HandleHostOSAndArch(pSSM), "darwin.x86")
1222	\|\| !strcmp(SSMR3HandleHostOSAndArch(pSSM), "") )
1223	)
1224	SSMR3Skip(pSSM, 16384);
1225	else
1226	SSMR3Skip(pSSM, 8192);
1227	#else
1228	SSMR3Skip(pSSM, 8192);
1229	#endif
1230	}
1231
1232	/*
1233	* Restore the VMCPU states. VCPU 0 is always started.
1234	*/
1235	VMCPU_SET_STATE(&pVM->aCpus[0], VMCPUSTATE_STARTED);
1236	for (VMCPUID i = 1; i < pVM->cCpus; i++)
1237	{
1238	bool fStarted;
1239	int rc = SSMR3GetBool(pSSM, &fStarted);
1240	if (RT_FAILURE(rc))
1241	return rc;
1242	VMCPU_SET_STATE(&pVM->aCpus[i], fStarted ? VMCPUSTATE_STARTED : VMCPUSTATE_STOPPED);
1243	}
1244
1245	/* terminator */
1246	uint32_t u32;
1247	int rc = SSMR3GetU32(pSSM, &u32);
1248	if (RT_FAILURE(rc))
1249	return rc;
1250	if (u32 != UINT32_MAX)
1251	{
1252	AssertMsgFailed(("u32=%#x\n", u32));
1253	return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
1254	}
1255	return VINF_SUCCESS;
1256	}
1257
1258
1259	#ifdef VBOX_WITH_RAW_MODE
1260	/**
1261	* Resolve a builtin RC symbol.
1262	*
1263	* Called by PDM when loading or relocating RC modules.
1264	*
1265	* @returns VBox status
1266	* @param pVM The cross context VM structure.
1267	* @param pszSymbol Symbol to resolve.
1268	* @param pRCPtrValue Where to store the symbol value.
1269	*
1270	* @remark This has to work before VMMR3Relocate() is called.
1271	*/
1272	VMMR3_INT_DECL(int) VMMR3GetImportRC(PVM pVM, const char *pszSymbol, PRTRCPTR pRCPtrValue)
1273	{
1274	if (!strcmp(pszSymbol, "g_Logger"))
1275	{
1276	if (pVM->vmm.s.pRCLoggerR3)
1277	pVM->vmm.s.pRCLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCLoggerR3);
1278	*pRCPtrValue = pVM->vmm.s.pRCLoggerRC;
1279	}
1280	else if (!strcmp(pszSymbol, "g_RelLogger"))
1281	{
1282	# ifdef VBOX_WITH_RC_RELEASE_LOGGING
1283	if (pVM->vmm.s.pRCRelLoggerR3)
1284	pVM->vmm.s.pRCRelLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCRelLoggerR3);
1285	*pRCPtrValue = pVM->vmm.s.pRCRelLoggerRC;
1286	# else
1287	*pRCPtrValue = NIL_RTRCPTR;
1288	# endif
1289	}
1290	else
1291	return VERR_SYMBOL_NOT_FOUND;
1292	return VINF_SUCCESS;
1293	}
1294	#endif /* VBOX_WITH_RAW_MODE */
1295
1296
1297	/**
1298	* Suspends the CPU yielder.
1299	*
1300	* @param pVM The cross context VM structure.
1301	*/
1302	VMMR3_INT_DECL(void) VMMR3YieldSuspend(PVM pVM)
1303	{
1304	VMCPU_ASSERT_EMT(&pVM->aCpus[0]);
1305	if (!pVM->vmm.s.cYieldResumeMillies)
1306	{
1307	uint64_t u64Now = TMTimerGet(pVM->vmm.s.pYieldTimer);
1308	uint64_t u64Expire = TMTimerGetExpire(pVM->vmm.s.pYieldTimer);
1309	if (u64Now >= u64Expire \|\| u64Expire == ~(uint64_t)0)
1310	pVM->vmm.s.cYieldResumeMillies = pVM->vmm.s.cYieldEveryMillies;
1311	else
1312	pVM->vmm.s.cYieldResumeMillies = TMTimerToMilli(pVM->vmm.s.pYieldTimer, u64Expire - u64Now);
1313	TMTimerStop(pVM->vmm.s.pYieldTimer);
1314	}
1315	pVM->vmm.s.u64LastYield = RTTimeNanoTS();
1316	}
1317
1318
1319	/**
1320	* Stops the CPU yielder.
1321	*
1322	* @param pVM The cross context VM structure.
1323	*/
1324	VMMR3_INT_DECL(void) VMMR3YieldStop(PVM pVM)
1325	{
1326	if (!pVM->vmm.s.cYieldResumeMillies)
1327	TMTimerStop(pVM->vmm.s.pYieldTimer);
1328	pVM->vmm.s.cYieldResumeMillies = pVM->vmm.s.cYieldEveryMillies;
1329	pVM->vmm.s.u64LastYield = RTTimeNanoTS();
1330	}
1331
1332
1333	/**
1334	* Resumes the CPU yielder when it has been a suspended or stopped.
1335	*
1336	* @param pVM The cross context VM structure.
1337	*/
1338	VMMR3_INT_DECL(void) VMMR3YieldResume(PVM pVM)
1339	{
1340	if (pVM->vmm.s.cYieldResumeMillies)
1341	{
1342	TMTimerSetMillies(pVM->vmm.s.pYieldTimer, pVM->vmm.s.cYieldResumeMillies);
1343	pVM->vmm.s.cYieldResumeMillies = 0;
1344	}
1345	}
1346
1347
1348	/**
1349	* Internal timer callback function.
1350	*
1351	* @param pVM The cross context VM structure.
1352	* @param pTimer The timer handle.
1353	* @param pvUser User argument specified upon timer creation.
1354	*/
1355	static DECLCALLBACK(void) vmmR3YieldEMT(PVM pVM, PTMTIMER pTimer, void *pvUser)
1356	{
1357	NOREF(pvUser);
1358
1359	/*
1360	* This really needs some careful tuning. While we shouldn't be too greedy since
1361	* that'll cause the rest of the system to stop up, we shouldn't be too nice either
1362	* because that'll cause us to stop up.
1363	*
1364	* The current logic is to use the default interval when there is no lag worth
1365	* mentioning, but when we start accumulating lag we don't bother yielding at all.
1366	*
1367	* (This depends on the TMCLOCK_VIRTUAL_SYNC to be scheduled before TMCLOCK_REAL
1368	* so the lag is up to date.)
1369	*/
1370	const uint64_t u64Lag = TMVirtualSyncGetLag(pVM);
1371	if ( u64Lag < 50000000 /* 50ms */
1372	\|\| ( u64Lag < 1000000000 /* 1s */
1373	&& RTTimeNanoTS() - pVM->vmm.s.u64LastYield < 500000000 /* 500 ms */)
1374	)
1375	{
1376	uint64_t u64Elapsed = RTTimeNanoTS();
1377	pVM->vmm.s.u64LastYield = u64Elapsed;
1378
1379	RTThreadYield();
1380
1381	#ifdef LOG_ENABLED
1382	u64Elapsed = RTTimeNanoTS() - u64Elapsed;
1383	Log(("vmmR3YieldEMT: %RI64 ns\n", u64Elapsed));
1384	#endif
1385	}
1386	TMTimerSetMillies(pTimer, pVM->vmm.s.cYieldEveryMillies);
1387	}
1388
1389
1390	#ifdef VBOX_WITH_RAW_MODE
1391	/**
1392	* Executes guest code in the raw-mode context.
1393	*
1394	* @param pVM The cross context VM structure.
1395	* @param pVCpu The cross context virtual CPU structure.
1396	*/
1397	VMMR3_INT_DECL(int) VMMR3RawRunGC(PVM pVM, PVMCPU pVCpu)
1398	{
1399	Log2(("VMMR3RawRunGC: (cs:eip=%04x:%08x)\n", CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
1400
1401	AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
1402
1403	/*
1404	* Set the hypervisor to resume executing a CPUM resume function
1405	* in CPUMRCA.asm.
1406	*/
1407	CPUMSetHyperState(pVCpu,
1408	CPUMGetGuestEFlags(pVCpu) & X86_EFL_VM
1409	? pVM->vmm.s.pfnCPUMRCResumeGuestV86
1410	: pVM->vmm.s.pfnCPUMRCResumeGuest, /* eip */
1411	pVCpu->vmm.s.pbEMTStackBottomRC, /* esp */
1412	0, /* eax */
1413	VM_RC_ADDR(pVM, &pVCpu->cpum) /* edx */);
1414
1415	/*
1416	* We hide log flushes (outer) and hypervisor interrupts (inner).
1417	*/
1418	for (;;)
1419	{
1420	#ifdef VBOX_STRICT
1421	if (RT_UNLIKELY(!CPUMGetHyperCR3(pVCpu) \|\| CPUMGetHyperCR3(pVCpu) != PGMGetHyperCR3(pVCpu)))
1422	EMR3FatalError(pVCpu, VERR_VMM_HYPER_CR3_MISMATCH);
1423	PGMMapCheck(pVM);
1424	# ifdef VBOX_WITH_SAFE_STR
1425	SELMR3CheckShadowTR(pVM);
1426	# endif
1427	#endif
1428	int rc;
1429	do
1430	{
1431	#ifdef NO_SUPCALLR0VMM
1432	rc = VERR_GENERAL_FAILURE;
1433	#else
1434	rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
1435	if (RT_LIKELY(rc == VINF_SUCCESS))
1436	rc = pVCpu->vmm.s.iLastGZRc;
1437	#endif
1438	} while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
1439
1440	/*
1441	* Flush the logs.
1442	*/
1443	#ifdef LOG_ENABLED
1444	PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
1445	if ( pLogger
1446	&& pLogger->offScratch > 0)
1447	RTLogFlushRC(NULL, pLogger);
1448	#endif
1449	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
1450	PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
1451	if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
1452	RTLogFlushRC(RTLogRelGetDefaultInstance(), pRelLogger);
1453	#endif
1454	if (rc != VINF_VMM_CALL_HOST)
1455	{
1456	Log2(("VMMR3RawRunGC: returns %Rrc (cs:eip=%04x:%08x)\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
1457	return rc;
1458	}
1459	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
1460	if (RT_FAILURE(rc))
1461	return rc;
1462	/* Resume GC */
1463	}
1464	}
1465	#endif /* VBOX_WITH_RAW_MODE */
1466
1467
1468	/**
1469	* Executes guest code (Intel VT-x and AMD-V).
1470	*
1471	* @param pVM The cross context VM structure.
1472	* @param pVCpu The cross context virtual CPU structure.
1473	*/
1474	VMMR3_INT_DECL(int) VMMR3HmRunGC(PVM pVM, PVMCPU pVCpu)
1475	{
1476	Log2(("VMMR3HmRunGC: (cs:rip=%04x:%RX64)\n", CPUMGetGuestCS(pVCpu), CPUMGetGuestRIP(pVCpu)));
1477
1478	for (;;)
1479	{
1480	int rc;
1481	do
1482	{
1483	#ifdef NO_SUPCALLR0VMM
1484	rc = VERR_GENERAL_FAILURE;
1485	#else
1486	rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_HM_RUN, pVCpu->idCpu);
1487	if (RT_LIKELY(rc == VINF_SUCCESS))
1488	rc = pVCpu->vmm.s.iLastGZRc;
1489	#endif
1490	} while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
1491
1492	#if 0 /** @todo triggers too often */
1493	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TO_R3));
1494	#endif
1495
1496	/*
1497	* Flush the logs
1498	*/
1499	#ifdef LOG_ENABLED
1500	VMM_FLUSH_R0_LOG(pVCpu->vmm.s.pR0LoggerR3, NULL);
1501	#endif
1502	VMM_FLUSH_R0_LOG(pVCpu->vmm.s.pR0RelLoggerR3, RTLogRelGetDefaultInstance());
1503	if (rc != VINF_VMM_CALL_HOST)
1504	{
1505	Log2(("VMMR3HmRunGC: returns %Rrc (cs:rip=%04x:%RX64)\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestRIP(pVCpu)));
1506	return rc;
1507	}
1508	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
1509	if (RT_FAILURE(rc))
1510	return rc;
1511	/* Resume R0 */
1512	}
1513	}
1514
1515
1516	/**
1517	* Perform one of the fast I/O control VMMR0 operation.
1518	*
1519	* @returns VBox strict status code.
1520	* @param pVM The cross context VM structure.
1521	* @param pVCpu The cross context virtual CPU structure.
1522	* @param enmOperation The operation to perform.
1523	*/
1524	VMMR3_INT_DECL(VBOXSTRICTRC) VMMR3CallR0EmtFast(PVM pVM, PVMCPU pVCpu, VMMR0OPERATION enmOperation)
1525	{
1526	for (;;)
1527	{
1528	VBOXSTRICTRC rcStrict;
1529	do
1530	{
1531	#ifdef NO_SUPCALLR0VMM
1532	rcStrict = VERR_GENERAL_FAILURE;
1533	#else
1534	rcStrict = SUPR3CallVMMR0Fast(pVM->pVMR0, enmOperation, pVCpu->idCpu);
1535	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
1536	rcStrict = pVCpu->vmm.s.iLastGZRc;
1537	#endif
1538	} while (rcStrict == VINF_EM_RAW_INTERRUPT_HYPER);
1539
1540	/*
1541	* Flush the logs
1542	*/
1543	#ifdef LOG_ENABLED
1544	VMM_FLUSH_R0_LOG(pVCpu->vmm.s.pR0LoggerR3, NULL);
1545	#endif
1546	VMM_FLUSH_R0_LOG(pVCpu->vmm.s.pR0RelLoggerR3, RTLogRelGetDefaultInstance());
1547	if (rcStrict != VINF_VMM_CALL_HOST)
1548	return rcStrict;
1549	int rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
1550	if (RT_FAILURE(rc))
1551	return rc;
1552	/* Resume R0 */
1553	}
1554	}
1555
1556
1557	/**
1558	* VCPU worker for VMMSendStartupIpi.
1559	*
1560	* @param pVM The cross context VM structure.
1561	* @param idCpu Virtual CPU to perform SIPI on.
1562	* @param uVector The SIPI vector.
1563	*/
1564	static DECLCALLBACK(int) vmmR3SendStarupIpi(PVM pVM, VMCPUID idCpu, uint32_t uVector)
1565	{
1566	PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
1567	VMCPU_ASSERT_EMT(pVCpu);
1568
1569	/*
1570	* Active, halt and shutdown states of the processor all block SIPIs.
1571	* So we can safely discard the SIPI. See Intel spec. 26.6.2 "Activity State".
1572	*/
1573	if (EMGetState(pVCpu) != EMSTATE_WAIT_SIPI)
1574	return VERR_ACCESS_DENIED;
1575
1576
1577	PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
1578
1579	pCtx->cs.Sel = uVector << 8;
1580	pCtx->cs.ValidSel = uVector << 8;
1581	pCtx->cs.fFlags = CPUMSELREG_FLAGS_VALID;
1582	pCtx->cs.u64Base = uVector << 12;
1583	pCtx->cs.u32Limit = UINT32_C(0x0000ffff);
1584	pCtx->rip = 0;
1585
1586	Log(("vmmR3SendSipi for VCPU %d with vector %x\n", idCpu, uVector));
1587
1588	# if 1 /* If we keep the EMSTATE_WAIT_SIPI method, then move this to EM.cpp. */
1589	EMSetState(pVCpu, EMSTATE_HALTED);
1590	return VINF_EM_RESCHEDULE;
1591	# else /* And if we go the VMCPU::enmState way it can stay here. */
1592	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STOPPED);
1593	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
1594	return VINF_SUCCESS;
1595	# endif
1596	}
1597
1598
1599	static DECLCALLBACK(int) vmmR3SendInitIpi(PVM pVM, VMCPUID idCpu)
1600	{
1601	PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
1602	VMCPU_ASSERT_EMT(pVCpu);
1603
1604	Log(("vmmR3SendInitIpi for VCPU %d\n", idCpu));
1605
1606	/** @todo Figure out how to handle a nested-guest intercepts here for INIT
1607	* IPI (e.g. SVM_EXIT_INIT). */
1608
1609	PGMR3ResetCpu(pVM, pVCpu);
1610	PDMR3ResetCpu(pVCpu); /* Only clears pending interrupts force flags */
1611	APICR3InitIpi(pVCpu);
1612	TRPMR3ResetCpu(pVCpu);
1613	CPUMR3ResetCpu(pVM, pVCpu);
1614	EMR3ResetCpu(pVCpu);
1615	HMR3ResetCpu(pVCpu);
1616	NEMR3ResetCpu(pVCpu, true /fInitIpi/);
1617
1618	/* This will trickle up on the target EMT. */
1619	return VINF_EM_WAIT_SIPI;
1620	}
1621
1622
1623	/**
1624	* Sends a Startup IPI to the virtual CPU by setting CS:EIP into
1625	* vector-dependent state and unhalting processor.
1626	*
1627	* @param pVM The cross context VM structure.
1628	* @param idCpu Virtual CPU to perform SIPI on.
1629	* @param uVector SIPI vector.
1630	*/
1631	VMMR3_INT_DECL(void) VMMR3SendStartupIpi(PVM pVM, VMCPUID idCpu, uint32_t uVector)
1632	{
1633	AssertReturnVoid(idCpu < pVM->cCpus);
1634
1635	int rc = VMR3ReqCallNoWait(pVM, idCpu, (PFNRT)vmmR3SendStarupIpi, 3, pVM, idCpu, uVector);
1636	AssertRC(rc);
1637	}
1638
1639
1640	/**
1641	* Sends init IPI to the virtual CPU.
1642	*
1643	* @param pVM The cross context VM structure.
1644	* @param idCpu Virtual CPU to perform int IPI on.
1645	*/
1646	VMMR3_INT_DECL(void) VMMR3SendInitIpi(PVM pVM, VMCPUID idCpu)
1647	{
1648	AssertReturnVoid(idCpu < pVM->cCpus);
1649
1650	int rc = VMR3ReqCallNoWait(pVM, idCpu, (PFNRT)vmmR3SendInitIpi, 2, pVM, idCpu);
1651	AssertRC(rc);
1652	}
1653
1654
1655	/**
1656	* Registers the guest memory range that can be used for patching.
1657	*
1658	* @returns VBox status code.
1659	* @param pVM The cross context VM structure.
1660	* @param pPatchMem Patch memory range.
1661	* @param cbPatchMem Size of the memory range.
1662	*/
1663	VMMR3DECL(int) VMMR3RegisterPatchMemory(PVM pVM, RTGCPTR pPatchMem, unsigned cbPatchMem)
1664	{
1665	VM_ASSERT_EMT(pVM);
1666	if (HMIsEnabled(pVM))
1667	return HMR3EnablePatching(pVM, pPatchMem, cbPatchMem);
1668
1669	return VERR_NOT_SUPPORTED;
1670	}
1671
1672
1673	/**
1674	* Deregisters the guest memory range that can be used for patching.
1675	*
1676	* @returns VBox status code.
1677	* @param pVM The cross context VM structure.
1678	* @param pPatchMem Patch memory range.
1679	* @param cbPatchMem Size of the memory range.
1680	*/
1681	VMMR3DECL(int) VMMR3DeregisterPatchMemory(PVM pVM, RTGCPTR pPatchMem, unsigned cbPatchMem)
1682	{
1683	if (HMIsEnabled(pVM))
1684	return HMR3DisablePatching(pVM, pPatchMem, cbPatchMem);
1685
1686	return VINF_SUCCESS;
1687	}
1688
1689
1690	/**
1691	* Common recursion handler for the other EMTs.
1692	*
1693	* @returns Strict VBox status code.
1694	* @param pVM The cross context VM structure.
1695	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1696	* @param rcStrict Current status code to be combined with the one
1697	* from this recursion and returned.
1698	*/
1699	static VBOXSTRICTRC vmmR3EmtRendezvousCommonRecursion(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rcStrict)
1700	{
1701	int rc2;
1702
1703	/*
1704	* We wait here while the initiator of this recursion reconfigures
1705	* everything. The last EMT to get in signals the initiator.
1706	*/
1707	if (ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsRecursingPush) == pVM->cCpus)
1708	{
1709	rc2 = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousRecursionPushCaller);
1710	AssertLogRelRC(rc2);
1711	}
1712
1713	rc2 = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousRecursionPush, RT_INDEFINITE_WAIT);
1714	AssertLogRelRC(rc2);
1715
1716	/*
1717	* Do the normal rendezvous processing.
1718	*/
1719	VBOXSTRICTRC rcStrict2 = vmmR3EmtRendezvousCommon(pVM, pVCpu, false /* fIsCaller */, pVM->vmm.s.fRendezvousFlags,
1720	pVM->vmm.s.pfnRendezvous, pVM->vmm.s.pvRendezvousUser);
1721
1722	/*
1723	* Wait for the initiator to restore everything.
1724	*/
1725	rc2 = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousRecursionPop, RT_INDEFINITE_WAIT);
1726	AssertLogRelRC(rc2);
1727
1728	/*
1729	* Last thread out of here signals the initiator.
1730	*/
1731	if (ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsRecursingPop) == pVM->cCpus)
1732	{
1733	rc2 = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousRecursionPopCaller);
1734	AssertLogRelRC(rc2);
1735	}
1736
1737	/*
1738	* Merge status codes and return.
1739	*/
1740	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
1741	if ( rcStrict2 != VINF_SUCCESS
1742	&& ( rcStrict == VINF_SUCCESS
1743	\|\| rcStrict > rcStrict2))
1744	rcStrict = rcStrict2;
1745	return rcStrict;
1746	}
1747
1748
1749	/**
1750	* Count returns and have the last non-caller EMT wake up the caller.
1751	*
1752	* @returns VBox strict informational status code for EM scheduling. No failures
1753	* will be returned here, those are for the caller only.
1754	*
1755	* @param pVM The cross context VM structure.
1756	* @param rcStrict The current accumulated recursive status code,
1757	* to be merged with i32RendezvousStatus and
1758	* returned.
1759	*/
1760	DECL_FORCE_INLINE(VBOXSTRICTRC) vmmR3EmtRendezvousNonCallerReturn(PVM pVM, VBOXSTRICTRC rcStrict)
1761	{
1762	VBOXSTRICTRC rcStrict2 = ASMAtomicReadS32(&pVM->vmm.s.i32RendezvousStatus);
1763
1764	uint32_t cReturned = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsReturned);
1765	if (cReturned == pVM->cCpus - 1U)
1766	{
1767	int rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousDoneCaller);
1768	AssertLogRelRC(rc);
1769	}
1770
1771	/*
1772	* Merge the status codes, ignoring error statuses in this code path.
1773	*/
1774	AssertLogRelMsgReturn( rcStrict2 <= VINF_SUCCESS
1775	\|\| (rcStrict2 >= VINF_EM_FIRST && rcStrict2 <= VINF_EM_LAST),
1776	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)),
1777	VERR_IPE_UNEXPECTED_INFO_STATUS);
1778
1779	if (RT_SUCCESS(rcStrict2))
1780	{
1781	if ( rcStrict2 != VINF_SUCCESS
1782	&& ( rcStrict == VINF_SUCCESS
1783	\|\| rcStrict > rcStrict2))
1784	rcStrict = rcStrict2;
1785	}
1786	return rcStrict;
1787	}
1788
1789
1790	/**
1791	* Common worker for VMMR3EmtRendezvous and VMMR3EmtRendezvousFF.
1792	*
1793	* @returns VBox strict informational status code for EM scheduling. No failures
1794	* will be returned here, those are for the caller only. When
1795	* fIsCaller is set, VINF_SUCCESS is always returned.
1796	*
1797	* @param pVM The cross context VM structure.
1798	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1799	* @param fIsCaller Whether we're the VMMR3EmtRendezvous caller or
1800	* not.
1801	* @param fFlags The flags.
1802	* @param pfnRendezvous The callback.
1803	* @param pvUser The user argument for the callback.
1804	*/
1805	static VBOXSTRICTRC vmmR3EmtRendezvousCommon(PVM pVM, PVMCPU pVCpu, bool fIsCaller,
1806	uint32_t fFlags, PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser)
1807	{
1808	int rc;
1809	VBOXSTRICTRC rcStrictRecursion = VINF_SUCCESS;
1810
1811	/*
1812	* Enter, the last EMT triggers the next callback phase.
1813	*/
1814	uint32_t cEntered = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsEntered);
1815	if (cEntered != pVM->cCpus)
1816	{
1817	if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE)
1818	{
1819	/* Wait for our turn. */
1820	for (;;)
1821	{
1822	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousEnterOneByOne, RT_INDEFINITE_WAIT);
1823	AssertLogRelRC(rc);
1824	if (!pVM->vmm.s.fRendezvousRecursion)
1825	break;
1826	rcStrictRecursion = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrictRecursion);
1827	}
1828	}
1829	else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE)
1830	{
1831	/* Wait for the last EMT to arrive and wake everyone up. */
1832	rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce, RT_INDEFINITE_WAIT);
1833	AssertLogRelRC(rc);
1834	Assert(!pVM->vmm.s.fRendezvousRecursion);
1835	}
1836	else if ( (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
1837	\|\| (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
1838	{
1839	/* Wait for our turn. */
1840	for (;;)
1841	{
1842	rc = RTSemEventWait(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVCpu->idCpu], RT_INDEFINITE_WAIT);
1843	AssertLogRelRC(rc);
1844	if (!pVM->vmm.s.fRendezvousRecursion)
1845	break;
1846	rcStrictRecursion = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrictRecursion);
1847	}
1848	}
1849	else
1850	{
1851	Assert((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE);
1852
1853	/*
1854	* The execute once is handled specially to optimize the code flow.
1855	*
1856	* The last EMT to arrive will perform the callback and the other
1857	* EMTs will wait on the Done/DoneCaller semaphores (instead of
1858	* the EnterOneByOne/AllAtOnce) in the meanwhile. When the callback
1859	* returns, that EMT will initiate the normal return sequence.
1860	*/
1861	if (!fIsCaller)
1862	{
1863	for (;;)
1864	{
1865	rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousDone, RT_INDEFINITE_WAIT);
1866	AssertLogRelRC(rc);
1867	if (!pVM->vmm.s.fRendezvousRecursion)
1868	break;
1869	rcStrictRecursion = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrictRecursion);
1870	}
1871
1872	return vmmR3EmtRendezvousNonCallerReturn(pVM, rcStrictRecursion);
1873	}
1874	return VINF_SUCCESS;
1875	}
1876	}
1877	else
1878	{
1879	/*
1880	* All EMTs are waiting, clear the FF and take action according to the
1881	* execution method.
1882	*/
1883	VM_FF_CLEAR(pVM, VM_FF_EMT_RENDEZVOUS);
1884
1885	if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE)
1886	{
1887	/* Wake up everyone. */
1888	rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
1889	AssertLogRelRC(rc);
1890	}
1891	else if ( (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
1892	\|\| (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
1893	{
1894	/* Figure out who to wake up and wake it up. If it's ourself, then
1895	it's easy otherwise wait for our turn. */
1896	VMCPUID iFirst = (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
1897	? 0
1898	: pVM->cCpus - 1U;
1899	if (pVCpu->idCpu != iFirst)
1900	{
1901	rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[iFirst]);
1902	AssertLogRelRC(rc);
1903	for (;;)
1904	{
1905	rc = RTSemEventWait(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVCpu->idCpu], RT_INDEFINITE_WAIT);
1906	AssertLogRelRC(rc);
1907	if (!pVM->vmm.s.fRendezvousRecursion)
1908	break;
1909	rcStrictRecursion = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrictRecursion);
1910	}
1911	}
1912	}
1913	/* else: execute the handler on the current EMT and wake up one or more threads afterwards. */
1914	}
1915
1916
1917	/*
1918	* Do the callback and update the status if necessary.
1919	*/
1920	if ( !(fFlags & VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR)
1921	\|\| RT_SUCCESS(ASMAtomicUoReadS32(&pVM->vmm.s.i32RendezvousStatus)) )
1922	{
1923	VBOXSTRICTRC rcStrict2 = pfnRendezvous(pVM, pVCpu, pvUser);
1924	if (rcStrict2 != VINF_SUCCESS)
1925	{
1926	AssertLogRelMsg( rcStrict2 <= VINF_SUCCESS
1927	\|\| (rcStrict2 >= VINF_EM_FIRST && rcStrict2 <= VINF_EM_LAST),
1928	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
1929	int32_t i32RendezvousStatus;
1930	do
1931	{
1932	i32RendezvousStatus = ASMAtomicUoReadS32(&pVM->vmm.s.i32RendezvousStatus);
1933	if ( rcStrict2 == i32RendezvousStatus
1934	\|\| RT_FAILURE(i32RendezvousStatus)
1935	\|\| ( i32RendezvousStatus != VINF_SUCCESS
1936	&& rcStrict2 > i32RendezvousStatus))
1937	break;
1938	} while (!ASMAtomicCmpXchgS32(&pVM->vmm.s.i32RendezvousStatus, VBOXSTRICTRC_VAL(rcStrict2), i32RendezvousStatus));
1939	}
1940	}
1941
1942	/*
1943	* Increment the done counter and take action depending on whether we're
1944	* the last to finish callback execution.
1945	*/
1946	uint32_t cDone = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsDone);
1947	if ( cDone != pVM->cCpus
1948	&& (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE)
1949	{
1950	/* Signal the next EMT? */
1951	if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE)
1952	{
1953	rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousEnterOneByOne);
1954	AssertLogRelRC(rc);
1955	}
1956	else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING)
1957	{
1958	Assert(cDone == pVCpu->idCpu + 1U);
1959	rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVCpu->idCpu + 1U]);
1960	AssertLogRelRC(rc);
1961	}
1962	else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
1963	{
1964	Assert(pVM->cCpus - cDone == pVCpu->idCpu);
1965	rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVM->cCpus - cDone - 1U]);
1966	AssertLogRelRC(rc);
1967	}
1968
1969	/* Wait for the rest to finish (the caller waits on hEvtRendezvousDoneCaller). */
1970	if (!fIsCaller)
1971	{
1972	for (;;)
1973	{
1974	rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousDone, RT_INDEFINITE_WAIT);
1975	AssertLogRelRC(rc);
1976	if (!pVM->vmm.s.fRendezvousRecursion)
1977	break;
1978	rcStrictRecursion = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrictRecursion);
1979	}
1980	}
1981	}
1982	else
1983	{
1984	/* Callback execution is all done, tell the rest to return. */
1985	rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousDone);
1986	AssertLogRelRC(rc);
1987	}
1988
1989	if (!fIsCaller)
1990	return vmmR3EmtRendezvousNonCallerReturn(pVM, rcStrictRecursion);
1991	return rcStrictRecursion;
1992	}
1993
1994
1995	/**
1996	* Called in response to VM_FF_EMT_RENDEZVOUS.
1997	*
1998	* @returns VBox strict status code - EM scheduling. No errors will be returned
1999	* here, nor will any non-EM scheduling status codes be returned.
2000	*
2001	* @param pVM The cross context VM structure.
2002	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2003	*
2004	* @thread EMT
2005	*/
2006	VMMR3_INT_DECL(int) VMMR3EmtRendezvousFF(PVM pVM, PVMCPU pVCpu)
2007	{
2008	Assert(!pVCpu->vmm.s.fInRendezvous);
2009	Log(("VMMR3EmtRendezvousFF: EMT%#u\n", pVCpu->idCpu));
2010	pVCpu->vmm.s.fInRendezvous = true;
2011	VBOXSTRICTRC rcStrict = vmmR3EmtRendezvousCommon(pVM, pVCpu, false /* fIsCaller */, pVM->vmm.s.fRendezvousFlags,
2012	pVM->vmm.s.pfnRendezvous, pVM->vmm.s.pvRendezvousUser);
2013	pVCpu->vmm.s.fInRendezvous = false;
2014	Log(("VMMR3EmtRendezvousFF: EMT%#u returns %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict)));
2015	return VBOXSTRICTRC_TODO(rcStrict);
2016	}
2017
2018
2019	/**
2020	* Helper for resetting an single wakeup event sempahore.
2021	*
2022	* @returns VERR_TIMEOUT on success, RTSemEventWait status otherwise.
2023	* @param hEvt The event semaphore to reset.
2024	*/
2025	static int vmmR3HlpResetEvent(RTSEMEVENT hEvt)
2026	{
2027	for (uint32_t cLoops = 0; ; cLoops++)
2028	{
2029	int rc = RTSemEventWait(hEvt, 0 /cMsTimeout/);
2030	if (rc != VINF_SUCCESS \|\| cLoops > _4K)
2031	return rc;
2032	}
2033	}
2034
2035
2036	/**
2037	* Worker for VMMR3EmtRendezvous that handles recursion.
2038	*
2039	* @returns VBox strict status code. This will be the first error,
2040	* VINF_SUCCESS, or an EM scheduling status code.
2041	*
2042	* @param pVM The cross context VM structure.
2043	* @param pVCpu The cross context virtual CPU structure of the
2044	* calling EMT.
2045	* @param fFlags Flags indicating execution methods. See
2046	* grp_VMMR3EmtRendezvous_fFlags.
2047	* @param pfnRendezvous The callback.
2048	* @param pvUser User argument for the callback.
2049	*
2050	* @thread EMT(pVCpu)
2051	*/
2052	static VBOXSTRICTRC vmmR3EmtRendezvousRecursive(PVM pVM, PVMCPU pVCpu, uint32_t fFlags,
2053	PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser)
2054	{
2055	Log(("vmmR3EmtRendezvousRecursive: %#x EMT#%u depth=%d\n", fFlags, pVCpu->idCpu, pVM->vmm.s.cRendezvousRecursions));
2056	AssertLogRelReturn(pVM->vmm.s.cRendezvousRecursions < 3, VERR_DEADLOCK);
2057	Assert(pVCpu->vmm.s.fInRendezvous);
2058
2059	/*
2060	* Save the current state.
2061	*/
2062	uint32_t const fParentFlags = pVM->vmm.s.fRendezvousFlags;
2063	uint32_t const cParentDone = pVM->vmm.s.cRendezvousEmtsDone;
2064	int32_t const iParentStatus = pVM->vmm.s.i32RendezvousStatus;
2065	PFNVMMEMTRENDEZVOUS const pfnParent = pVM->vmm.s.pfnRendezvous;
2066	void * const pvParentUser = pVM->vmm.s.pvRendezvousUser;
2067
2068	/*
2069	* Check preconditions and save the current state.
2070	*/
2071	AssertReturn( (fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
2072	\|\| (fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING
2073	\|\| (fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE
2074	\|\| (fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE,
2075	VERR_INTERNAL_ERROR);
2076	AssertReturn(pVM->vmm.s.cRendezvousEmtsEntered == pVM->cCpus, VERR_INTERNAL_ERROR_2);
2077	AssertReturn(pVM->vmm.s.cRendezvousEmtsReturned == 0, VERR_INTERNAL_ERROR_3);
2078
2079	/*
2080	* Reset the recursion prep and pop semaphores.
2081	*/
2082	int rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousRecursionPush);
2083	AssertLogRelRCReturn(rc, rc);
2084	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousRecursionPop);
2085	AssertLogRelRCReturn(rc, rc);
2086	rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousRecursionPushCaller);
2087	AssertLogRelMsgReturn(rc == VERR_TIMEOUT, ("%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
2088	rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousRecursionPopCaller);
2089	AssertLogRelMsgReturn(rc == VERR_TIMEOUT, ("%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
2090
2091	/*
2092	* Usher the other thread into the recursion routine.
2093	*/
2094	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsRecursingPush, 0);
2095	ASMAtomicWriteBool(&pVM->vmm.s.fRendezvousRecursion, true);
2096
2097	uint32_t cLeft = pVM->cCpus - (cParentDone + 1U);
2098	if ((fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE)
2099	while (cLeft-- > 0)
2100	{
2101	rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousEnterOneByOne);
2102	AssertLogRelRC(rc);
2103	}
2104	else if ((fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING)
2105	{
2106	Assert(cLeft == pVM->cCpus - (pVCpu->idCpu + 1U));
2107	for (VMCPUID iCpu = pVCpu->idCpu + 1U; iCpu < pVM->cCpus; iCpu++)
2108	{
2109	rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[iCpu]);
2110	AssertLogRelRC(rc);
2111	}
2112	}
2113	else if ((fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
2114	{
2115	Assert(cLeft == pVCpu->idCpu);
2116	for (VMCPUID iCpu = pVCpu->idCpu; iCpu > 0; iCpu--)
2117	{
2118	rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[iCpu - 1U]);
2119	AssertLogRelRC(rc);
2120	}
2121	}
2122	else
2123	AssertLogRelReturn((fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE,
2124	VERR_INTERNAL_ERROR_4);
2125
2126	rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousDone);
2127	AssertLogRelRC(rc);
2128	rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousDoneCaller);
2129	AssertLogRelRC(rc);
2130
2131
2132	/*
2133	* Wait for the EMTs to wake up and get out of the parent rendezvous code.
2134	*/
2135	if (ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsRecursingPush) != pVM->cCpus)
2136	{
2137	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousRecursionPushCaller, RT_INDEFINITE_WAIT);
2138	AssertLogRelRC(rc);
2139	}
2140
2141	ASMAtomicWriteBool(&pVM->vmm.s.fRendezvousRecursion, false);
2142
2143	/*
2144	* Clear the slate and setup the new rendezvous.
2145	*/
2146	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2147	{
2148	rc = vmmR3HlpResetEvent(pVM->vmm.s.pahEvtRendezvousEnterOrdered[i]);
2149	AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
2150	}
2151	rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousEnterOneByOne); AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
2152	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce); AssertLogRelRC(rc);
2153	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousDone); AssertLogRelRC(rc);
2154	rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousDoneCaller); AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
2155
2156	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsEntered, 0);
2157	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsDone, 0);
2158	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsReturned, 0);
2159	ASMAtomicWriteS32(&pVM->vmm.s.i32RendezvousStatus, VINF_SUCCESS);
2160	ASMAtomicWritePtr((void * volatile )&pVM->vmm.s.pfnRendezvous, (void )(uintptr_t)pfnRendezvous);
2161	ASMAtomicWritePtr(&pVM->vmm.s.pvRendezvousUser, pvUser);
2162	ASMAtomicWriteU32(&pVM->vmm.s.fRendezvousFlags, fFlags);
2163	ASMAtomicIncU32(&pVM->vmm.s.cRendezvousRecursions);
2164
2165	/*
2166	* We're ready to go now, do normal rendezvous processing.
2167	*/
2168	rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousRecursionPush);
2169	AssertLogRelRC(rc);
2170
2171	VBOXSTRICTRC rcStrict = vmmR3EmtRendezvousCommon(pVM, pVCpu, true /fIsCaller/, fFlags, pfnRendezvous, pvUser);
2172
2173	/*
2174	* The caller waits for the other EMTs to be done, return and waiting on the
2175	* pop semaphore.
2176	*/
2177	for (;;)
2178	{
2179	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousDoneCaller, RT_INDEFINITE_WAIT);
2180	AssertLogRelRC(rc);
2181	if (!pVM->vmm.s.fRendezvousRecursion)
2182	break;
2183	rcStrict = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrict);
2184	}
2185
2186	/*
2187	* Get the return code and merge it with the above recursion status.
2188	*/
2189	VBOXSTRICTRC rcStrict2 = pVM->vmm.s.i32RendezvousStatus;
2190	if ( rcStrict2 != VINF_SUCCESS
2191	&& ( rcStrict == VINF_SUCCESS
2192	\|\| rcStrict > rcStrict2))
2193	rcStrict = rcStrict2;
2194
2195	/*
2196	* Restore the parent rendezvous state.
2197	*/
2198	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2199	{
2200	rc = vmmR3HlpResetEvent(pVM->vmm.s.pahEvtRendezvousEnterOrdered[i]);
2201	AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
2202	}
2203	rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousEnterOneByOne); AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
2204	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce); AssertLogRelRC(rc);
2205	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousDone); AssertLogRelRC(rc);
2206	rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousDoneCaller); AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
2207
2208	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsEntered, pVM->cCpus);
2209	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsReturned, 0);
2210	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsDone, cParentDone);
2211	ASMAtomicWriteS32(&pVM->vmm.s.i32RendezvousStatus, iParentStatus);
2212	ASMAtomicWriteU32(&pVM->vmm.s.fRendezvousFlags, fParentFlags);
2213	ASMAtomicWritePtr(&pVM->vmm.s.pvRendezvousUser, pvParentUser);
2214	ASMAtomicWritePtr((void * volatile )&pVM->vmm.s.pfnRendezvous, (void )(uintptr_t)pfnParent);
2215
2216	/*
2217	* Usher the other EMTs back to their parent recursion routine, waiting
2218	* for them to all get there before we return (makes sure they've been
2219	* scheduled and are past the pop event sem, see below).
2220	*/
2221	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsRecursingPop, 0);
2222	rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousRecursionPop);
2223	AssertLogRelRC(rc);
2224
2225	if (ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsRecursingPop) != pVM->cCpus)
2226	{
2227	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousRecursionPopCaller, RT_INDEFINITE_WAIT);
2228	AssertLogRelRC(rc);
2229	}
2230
2231	/*
2232	* We must reset the pop semaphore on the way out (doing the pop caller too,
2233	* just in case). The parent may be another recursion.
2234	*/
2235	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousRecursionPop); AssertLogRelRC(rc);
2236	rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousRecursionPopCaller); AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
2237
2238	ASMAtomicDecU32(&pVM->vmm.s.cRendezvousRecursions);
2239
2240	Log(("vmmR3EmtRendezvousRecursive: %#x EMT#%u depth=%d returns %Rrc\n",
2241	fFlags, pVCpu->idCpu, pVM->vmm.s.cRendezvousRecursions, VBOXSTRICTRC_VAL(rcStrict)));
2242	return rcStrict;
2243	}
2244
2245
2246	/**
2247	* EMT rendezvous.
2248	*
2249	* Gathers all the EMTs and execute some code on each of them, either in a one
2250	* by one fashion or all at once.
2251	*
2252	* @returns VBox strict status code. This will be the first error,
2253	* VINF_SUCCESS, or an EM scheduling status code.
2254	*
2255	* @retval VERR_DEADLOCK if recursion is attempted using a rendezvous type that
2256	* doesn't support it or if the recursion is too deep.
2257	*
2258	* @param pVM The cross context VM structure.
2259	* @param fFlags Flags indicating execution methods. See
2260	* grp_VMMR3EmtRendezvous_fFlags. The one-by-one,
2261	* descending and ascending rendezvous types support
2262	* recursion from inside @a pfnRendezvous.
2263	* @param pfnRendezvous The callback.
2264	* @param pvUser User argument for the callback.
2265	*
2266	* @thread Any.
2267	*/
2268	VMMR3DECL(int) VMMR3EmtRendezvous(PVM pVM, uint32_t fFlags, PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser)
2269	{
2270	/*
2271	* Validate input.
2272	*/
2273	AssertReturn(pVM, VERR_INVALID_VM_HANDLE);
2274	AssertMsg( (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_INVALID
2275	&& (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) <= VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING
2276	&& !(fFlags & ~VMMEMTRENDEZVOUS_FLAGS_VALID_MASK), ("%#x\n", fFlags));
2277	AssertMsg( !(fFlags & VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR)
2278	\|\| ( (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE
2279	&& (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE),
2280	("type %u\n", fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK));
2281
2282	VBOXSTRICTRC rcStrict;
2283	PVMCPU pVCpu = VMMGetCpu(pVM);
2284	if (!pVCpu)
2285	{
2286	/*
2287	* Forward the request to an EMT thread.
2288	*/
2289	Log(("VMMR3EmtRendezvous: %#x non-EMT\n", fFlags));
2290	if (!(fFlags & VMMEMTRENDEZVOUS_FLAGS_PRIORITY))
2291	rcStrict = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)VMMR3EmtRendezvous, 4, pVM, fFlags, pfnRendezvous, pvUser);
2292	else
2293	rcStrict = VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)VMMR3EmtRendezvous, 4, pVM, fFlags, pfnRendezvous, pvUser);
2294	Log(("VMMR3EmtRendezvous: %#x non-EMT returns %Rrc\n", fFlags, VBOXSTRICTRC_VAL(rcStrict)));
2295	}
2296	else if ( pVM->cCpus == 1
2297	\|\| ( pVM->enmVMState == VMSTATE_DESTROYING
2298	&& VMR3GetActiveEmts(pVM->pUVM) < pVM->cCpus ) )
2299	{
2300	/*
2301	* Shortcut for the single EMT case.
2302	*
2303	* We also ends up here if EMT(0) (or others) tries to issue a rendezvous
2304	* during vmR3Destroy after other emulation threads have started terminating.
2305	*/
2306	if (!pVCpu->vmm.s.fInRendezvous)
2307	{
2308	Log(("VMMR3EmtRendezvous: %#x EMT (uni)\n", fFlags));
2309	pVCpu->vmm.s.fInRendezvous = true;
2310	pVM->vmm.s.fRendezvousFlags = fFlags;
2311	rcStrict = pfnRendezvous(pVM, pVCpu, pvUser);
2312	pVCpu->vmm.s.fInRendezvous = false;
2313	}
2314	else
2315	{
2316	/* Recursion. Do the same checks as in the SMP case. */
2317	Log(("VMMR3EmtRendezvous: %#x EMT (uni), recursion depth=%d\n", fFlags, pVM->vmm.s.cRendezvousRecursions));
2318	uint32_t fType = pVM->vmm.s.fRendezvousFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK;
2319	AssertLogRelReturn( !pVCpu->vmm.s.fInRendezvous
2320	\|\| fType == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
2321	\|\| fType == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING
2322	\|\| fType == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE
2323	\|\| fType == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE
2324	, VERR_DEADLOCK);
2325
2326	AssertLogRelReturn(pVM->vmm.s.cRendezvousRecursions < 3, VERR_DEADLOCK);
2327	pVM->vmm.s.cRendezvousRecursions++;
2328	uint32_t const fParentFlags = pVM->vmm.s.fRendezvousFlags;
2329	pVM->vmm.s.fRendezvousFlags = fFlags;
2330
2331	rcStrict = pfnRendezvous(pVM, pVCpu, pvUser);
2332
2333	pVM->vmm.s.fRendezvousFlags = fParentFlags;
2334	pVM->vmm.s.cRendezvousRecursions--;
2335	}
2336	Log(("VMMR3EmtRendezvous: %#x EMT (uni) returns %Rrc\n", fFlags, VBOXSTRICTRC_VAL(rcStrict)));
2337	}
2338	else
2339	{
2340	/*
2341	* Spin lock. If busy, check for recursion, if not recursing wait for
2342	* the other EMT to finish while keeping a lookout for the RENDEZVOUS FF.
2343	*/
2344	int rc;
2345	rcStrict = VINF_SUCCESS;
2346	if (RT_UNLIKELY(!ASMAtomicCmpXchgU32(&pVM->vmm.s.u32RendezvousLock, 0x77778888, 0)))
2347	{
2348	/* Allow recursion in some cases. */
2349	if ( pVCpu->vmm.s.fInRendezvous
2350	&& ( (pVM->vmm.s.fRendezvousFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
2351	\|\| (pVM->vmm.s.fRendezvousFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING
2352	\|\| (pVM->vmm.s.fRendezvousFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE
2353	\|\| (pVM->vmm.s.fRendezvousFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE
2354	))
2355	return VBOXSTRICTRC_TODO(vmmR3EmtRendezvousRecursive(pVM, pVCpu, fFlags, pfnRendezvous, pvUser));
2356
2357	AssertLogRelMsgReturn(!pVCpu->vmm.s.fInRendezvous, ("fRendezvousFlags=%#x\n", pVM->vmm.s.fRendezvousFlags),
2358	VERR_DEADLOCK);
2359
2360	Log(("VMMR3EmtRendezvous: %#x EMT#%u, waiting for lock...\n", fFlags, pVCpu->idCpu));
2361	while (!ASMAtomicCmpXchgU32(&pVM->vmm.s.u32RendezvousLock, 0x77778888, 0))
2362	{
2363	if (VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS))
2364	{
2365	rc = VMMR3EmtRendezvousFF(pVM, pVCpu);
2366	if ( rc != VINF_SUCCESS
2367	&& ( rcStrict == VINF_SUCCESS
2368	\|\| rcStrict > rc))
2369	rcStrict = rc;
2370	/** @todo Perhaps deal with termination here? */
2371	}
2372	ASMNopPause();
2373	}
2374	}
2375
2376	Log(("VMMR3EmtRendezvous: %#x EMT#%u\n", fFlags, pVCpu->idCpu));
2377	Assert(!VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS));
2378	Assert(!pVCpu->vmm.s.fInRendezvous);
2379	pVCpu->vmm.s.fInRendezvous = true;
2380
2381	/*
2382	* Clear the slate and setup the rendezvous. This is a semaphore ping-pong orgy. :-)
2383	*/
2384	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2385	{
2386	rc = RTSemEventWait(pVM->vmm.s.pahEvtRendezvousEnterOrdered[i], 0);
2387	AssertLogRelMsg(rc == VERR_TIMEOUT \|\| rc == VINF_SUCCESS, ("%Rrc\n", rc));
2388	}
2389	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousEnterOneByOne, 0); AssertLogRelMsg(rc == VERR_TIMEOUT \|\| rc == VINF_SUCCESS, ("%Rrc\n", rc));
2390	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce); AssertLogRelRC(rc);
2391	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousDone); AssertLogRelRC(rc);
2392	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousDoneCaller, 0); AssertLogRelMsg(rc == VERR_TIMEOUT \|\| rc == VINF_SUCCESS, ("%Rrc\n", rc));
2393	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsEntered, 0);
2394	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsDone, 0);
2395	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsReturned, 0);
2396	ASMAtomicWriteS32(&pVM->vmm.s.i32RendezvousStatus, VINF_SUCCESS);
2397	ASMAtomicWritePtr((void * volatile )&pVM->vmm.s.pfnRendezvous, (void )(uintptr_t)pfnRendezvous);
2398	ASMAtomicWritePtr(&pVM->vmm.s.pvRendezvousUser, pvUser);
2399	ASMAtomicWriteU32(&pVM->vmm.s.fRendezvousFlags, fFlags);
2400
2401	/*
2402	* Set the FF and poke the other EMTs.
2403	*/
2404	VM_FF_SET(pVM, VM_FF_EMT_RENDEZVOUS);
2405	VMR3NotifyGlobalFFU(pVM->pUVM, VMNOTIFYFF_FLAGS_POKE);
2406
2407	/*
2408	* Do the same ourselves.
2409	*/
2410	VBOXSTRICTRC rcStrict2 = vmmR3EmtRendezvousCommon(pVM, pVCpu, true /* fIsCaller */, fFlags, pfnRendezvous, pvUser);
2411
2412	/*
2413	* The caller waits for the other EMTs to be done and return before doing
2414	* the cleanup. This makes away with wakeup / reset races we would otherwise
2415	* risk in the multiple release event semaphore code (hEvtRendezvousDoneCaller).
2416	*/
2417	for (;;)
2418	{
2419	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousDoneCaller, RT_INDEFINITE_WAIT);
2420	AssertLogRelRC(rc);
2421	if (!pVM->vmm.s.fRendezvousRecursion)
2422	break;
2423	rcStrict2 = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrict2);
2424	}
2425
2426	/*
2427	* Get the return code and clean up a little bit.
2428	*/
2429	VBOXSTRICTRC rcStrict3 = pVM->vmm.s.i32RendezvousStatus;
2430	ASMAtomicWriteNullPtr((void * volatile *)&pVM->vmm.s.pfnRendezvous);
2431
2432	ASMAtomicWriteU32(&pVM->vmm.s.u32RendezvousLock, 0);
2433	pVCpu->vmm.s.fInRendezvous = false;
2434
2435	/*
2436	* Merge rcStrict, rcStrict2 and rcStrict3.
2437	*/
2438	AssertRC(VBOXSTRICTRC_VAL(rcStrict));
2439	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
2440	if ( rcStrict2 != VINF_SUCCESS
2441	&& ( rcStrict == VINF_SUCCESS
2442	\|\| rcStrict > rcStrict2))
2443	rcStrict = rcStrict2;
2444	if ( rcStrict3 != VINF_SUCCESS
2445	&& ( rcStrict == VINF_SUCCESS
2446	\|\| rcStrict > rcStrict3))
2447	rcStrict = rcStrict3;
2448	Log(("VMMR3EmtRendezvous: %#x EMT#%u returns %Rrc\n", fFlags, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict)));
2449	}
2450
2451	AssertLogRelMsgReturn( rcStrict <= VINF_SUCCESS
2452	\|\| (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST),
2453	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)),
2454	VERR_IPE_UNEXPECTED_INFO_STATUS);
2455	return VBOXSTRICTRC_VAL(rcStrict);
2456	}
2457
2458
2459	/**
2460	* Read from the ring 0 jump buffer stack.
2461	*
2462	* @returns VBox status code.
2463	*
2464	* @param pVM The cross context VM structure.
2465	* @param idCpu The ID of the source CPU context (for the address).
2466	* @param R0Addr Where to start reading.
2467	* @param pvBuf Where to store the data we've read.
2468	* @param cbRead The number of bytes to read.
2469	*/
2470	VMMR3_INT_DECL(int) VMMR3ReadR0Stack(PVM pVM, VMCPUID idCpu, RTHCUINTPTR R0Addr, void *pvBuf, size_t cbRead)
2471	{
2472	PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
2473	AssertReturn(pVCpu, VERR_INVALID_PARAMETER);
2474	AssertReturn(cbRead < ~(size_t)0 / 2, VERR_INVALID_PARAMETER);
2475
2476	int rc;
2477	#ifdef VMM_R0_SWITCH_STACK
2478	RTHCUINTPTR off = R0Addr - MMHyperCCToR0(pVM, pVCpu->vmm.s.pbEMTStackR3);
2479	#else
2480	RTHCUINTPTR off = pVCpu->vmm.s.CallRing3JmpBufR0.cbSavedStack - (pVCpu->vmm.s.CallRing3JmpBufR0.SpCheck - R0Addr);
2481	#endif
2482	if ( off < VMM_STACK_SIZE
2483	&& off + cbRead <= VMM_STACK_SIZE)
2484	{
2485	memcpy(pvBuf, &pVCpu->vmm.s.pbEMTStackR3[off], cbRead);
2486	rc = VINF_SUCCESS;
2487	}
2488	else
2489	rc = VERR_INVALID_POINTER;
2490
2491	/* Supply the setjmp return RIP/EIP. */
2492	if ( pVCpu->vmm.s.CallRing3JmpBufR0.UnwindRetPcLocation + sizeof(RTR0UINTPTR) > R0Addr
2493	&& pVCpu->vmm.s.CallRing3JmpBufR0.UnwindRetPcLocation < R0Addr + cbRead)
2494	{
2495	uint8_t const pbSrc = (uint8_t const )&pVCpu->vmm.s.CallRing3JmpBufR0.UnwindRetPcValue;
2496	size_t cbSrc = sizeof(pVCpu->vmm.s.CallRing3JmpBufR0.UnwindRetPcValue);
2497	size_t offDst = 0;
2498	if (R0Addr < pVCpu->vmm.s.CallRing3JmpBufR0.UnwindRetPcLocation)
2499	offDst = pVCpu->vmm.s.CallRing3JmpBufR0.UnwindRetPcLocation - R0Addr;
2500	else if (R0Addr > pVCpu->vmm.s.CallRing3JmpBufR0.UnwindRetPcLocation)
2501	{
2502	size_t offSrc = R0Addr - pVCpu->vmm.s.CallRing3JmpBufR0.UnwindRetPcLocation;
2503	Assert(offSrc < cbSrc);
2504	pbSrc -= offSrc;
2505	cbSrc -= offSrc;
2506	}
2507	if (cbSrc > cbRead - offDst)
2508	cbSrc = cbRead - offDst;
2509	memcpy((uint8_t *)pvBuf + offDst, pbSrc, cbSrc);
2510
2511	if (cbSrc == cbRead)
2512	rc = VINF_SUCCESS;
2513	}
2514
2515	return rc;
2516	}
2517
2518
2519	/**
2520	* Used by the DBGF stack unwinder to initialize the register state.
2521	*
2522	* @param pUVM The user mode VM handle.
2523	* @param idCpu The ID of the CPU being unwound.
2524	* @param pState The unwind state to initialize.
2525	*/
2526	VMMR3_INT_DECL(void) VMMR3InitR0StackUnwindState(PUVM pUVM, VMCPUID idCpu, struct RTDBGUNWINDSTATE *pState)
2527	{
2528	PVMCPU pVCpu = VMMR3GetCpuByIdU(pUVM, idCpu);
2529	AssertReturnVoid(pVCpu);
2530
2531	/*
2532	* Locate the resume point on the stack.
2533	*/
2534	#ifdef VMM_R0_SWITCH_STACK
2535	uintptr_t off = pVCpu->vmm.s.CallRing3JmpBufR0.SpResume - MMHyperCCToR0(pVCpu->pVMR3, pVCpu->vmm.s.pbEMTStackR3);
2536	AssertReturnVoid(off < VMM_STACK_SIZE);
2537	#else
2538	uintptr_t off = 0;
2539	#endif
2540
2541	#ifdef RT_ARCH_AMD64
2542	/*
2543	* This code must match the .resume stuff in VMMR0JmpA-amd64.asm exactly.
2544	*/
2545	# ifdef VBOX_STRICT
2546	Assert((uint64_t const )&pVCpu->vmm.s.pbEMTStackR3[off] == UINT32_C(0x7eadf00d));
2547	off += 8; /* RESUME_MAGIC */
2548	# endif
2549	# ifdef RT_OS_WINDOWS
2550	off += 0xa0; /* XMM6 thru XMM15 */
2551	# endif
2552	pState->u.x86.uRFlags = (uint64_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2553	off += 8;
2554	pState->u.x86.auRegs[X86_GREG_xBX] = (uint64_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2555	off += 8;
2556	# ifdef RT_OS_WINDOWS
2557	pState->u.x86.auRegs[X86_GREG_xSI] = (uint64_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2558	off += 8;
2559	pState->u.x86.auRegs[X86_GREG_xDI] = (uint64_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2560	off += 8;
2561	# endif
2562	pState->u.x86.auRegs[X86_GREG_x12] = (uint64_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2563	off += 8;
2564	pState->u.x86.auRegs[X86_GREG_x13] = (uint64_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2565	off += 8;
2566	pState->u.x86.auRegs[X86_GREG_x14] = (uint64_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2567	off += 8;
2568	pState->u.x86.auRegs[X86_GREG_x15] = (uint64_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2569	off += 8;
2570	pState->u.x86.auRegs[X86_GREG_xBP] = (uint64_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2571	off += 8;
2572	pState->uPc = (uint64_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2573	off += 8;
2574
2575	#elif defined(RT_ARCH_X86)
2576	/*
2577	* This code must match the .resume stuff in VMMR0JmpA-x86.asm exactly.
2578	*/
2579	# ifdef VBOX_STRICT
2580	Assert((uint32_t const )&pVCpu->vmm.s.pbEMTStackR3[off] == UINT32_C(0x7eadf00d));
2581	off += 4; /* RESUME_MAGIC */
2582	# endif
2583	pState->u.x86.uRFlags = (uint32_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2584	off += 4;
2585	pState->u.x86.auRegs[X86_GREG_xBX] = (uint32_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2586	off += 4;
2587	pState->u.x86.auRegs[X86_GREG_xSI] = (uint32_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2588	off += 4;
2589	pState->u.x86.auRegs[X86_GREG_xDI] = (uint32_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2590	off += 4;
2591	pState->u.x86.auRegs[X86_GREG_xBP] = (uint32_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2592	off += 4;
2593	pState->uPc = (uint32_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2594	off += 4;
2595	#else
2596	# error "Port me"
2597	#endif
2598
2599	/*
2600	* This is all we really need here, though the above helps if the assembly
2601	* doesn't contain unwind info (currently only on win/64, so that is useful).
2602	*/
2603	pState->u.x86.auRegs[X86_GREG_xBP] = pVCpu->vmm.s.CallRing3JmpBufR0.SavedEbp;
2604	pState->u.x86.auRegs[X86_GREG_xSP] = pVCpu->vmm.s.CallRing3JmpBufR0.SpResume;
2605	}
2606
2607	#ifdef VBOX_WITH_RAW_MODE
2608
2609	/**
2610	* Calls a RC function.
2611	*
2612	* @param pVM The cross context VM structure.
2613	* @param RCPtrEntry The address of the RC function.
2614	* @param cArgs The number of arguments in the ....
2615	* @param ... Arguments to the function.
2616	*/
2617	VMMR3DECL(int) VMMR3CallRC(PVM pVM, RTRCPTR RCPtrEntry, unsigned cArgs, ...)
2618	{
2619	va_list args;
2620	va_start(args, cArgs);
2621	int rc = VMMR3CallRCV(pVM, RCPtrEntry, cArgs, args);
2622	va_end(args);
2623	return rc;
2624	}
2625
2626
2627	/**
2628	* Calls a RC function.
2629	*
2630	* @param pVM The cross context VM structure.
2631	* @param RCPtrEntry The address of the RC function.
2632	* @param cArgs The number of arguments in the ....
2633	* @param args Arguments to the function.
2634	*/
2635	VMMR3DECL(int) VMMR3CallRCV(PVM pVM, RTRCPTR RCPtrEntry, unsigned cArgs, va_list args)
2636	{
2637	/* Raw mode implies 1 VCPU. */
2638	AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
2639	PVMCPU pVCpu = &pVM->aCpus[0];
2640
2641	Log2(("VMMR3CallGCV: RCPtrEntry=%RRv cArgs=%d\n", RCPtrEntry, cArgs));
2642
2643	/*
2644	* Setup the call frame using the trampoline.
2645	*/
2646	CPUMSetHyperState(pVCpu,
2647	pVM->vmm.s.pfnCallTrampolineRC, /* eip */
2648	pVCpu->vmm.s.pbEMTStackBottomRC - cArgs * sizeof(RTGCUINTPTR32), /* esp */
2649	RCPtrEntry, /* eax */
2650	cArgs /* edx */
2651	);
2652
2653	#if 0
2654	memset(pVCpu->vmm.s.pbEMTStackR3, 0xaa, VMM_STACK_SIZE); /* Clear the stack. */
2655	#endif
2656	PRTGCUINTPTR32 pFrame = (PRTGCUINTPTR32)(pVCpu->vmm.s.pbEMTStackR3 + VMM_STACK_SIZE) - cArgs;
2657	int i = cArgs;
2658	while (i-- > 0)
2659	*pFrame++ = va_arg(args, RTGCUINTPTR32);
2660
2661	CPUMPushHyper(pVCpu, cArgs * sizeof(RTGCUINTPTR32)); /* stack frame size */
2662	CPUMPushHyper(pVCpu, RCPtrEntry); /* what to call */
2663
2664	/*
2665	* We hide log flushes (outer) and hypervisor interrupts (inner).
2666	*/
2667	for (;;)
2668	{
2669	int rc;
2670	Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
2671	do
2672	{
2673	#ifdef NO_SUPCALLR0VMM
2674	rc = VERR_GENERAL_FAILURE;
2675	#else
2676	rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
2677	if (RT_LIKELY(rc == VINF_SUCCESS))
2678	rc = pVCpu->vmm.s.iLastGZRc;
2679	#endif
2680	} while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
2681
2682	/*
2683	* Flush the loggers.
2684	*/
2685	#ifdef LOG_ENABLED
2686	PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
2687	if ( pLogger
2688	&& pLogger->offScratch > 0)
2689	RTLogFlushRC(NULL, pLogger);
2690	#endif
2691	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
2692	PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
2693	if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
2694	RTLogFlushRC(RTLogRelGetDefaultInstance(), pRelLogger);
2695	#endif
2696	if (rc == VERR_TRPM_PANIC \|\| rc == VERR_TRPM_DONT_PANIC)
2697	VMMR3FatalDump(pVM, pVCpu, rc);
2698	if (rc != VINF_VMM_CALL_HOST)
2699	{
2700	Log2(("VMMR3CallGCV: returns %Rrc (cs:eip=%04x:%08x)\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
2701	return rc;
2702	}
2703	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
2704	if (RT_FAILURE(rc))
2705	return rc;
2706	}
2707	}
2708
2709	#endif /* VBOX_WITH_RAW_MODE */
2710
2711	/**
2712	* Wrapper for SUPR3CallVMMR0Ex which will deal with VINF_VMM_CALL_HOST returns.
2713	*
2714	* @returns VBox status code.
2715	* @param pVM The cross context VM structure.
2716	* @param uOperation Operation to execute.
2717	* @param u64Arg Constant argument.
2718	* @param pReqHdr Pointer to a request header. See SUPR3CallVMMR0Ex for
2719	* details.
2720	*/
2721	VMMR3DECL(int) VMMR3CallR0(PVM pVM, uint32_t uOperation, uint64_t u64Arg, PSUPVMMR0REQHDR pReqHdr)
2722	{
2723	PVMCPU pVCpu = VMMGetCpu(pVM);
2724	AssertReturn(pVCpu, VERR_VM_THREAD_NOT_EMT);
2725	return VMMR3CallR0Emt(pVM, pVCpu, (VMMR0OPERATION)uOperation, u64Arg, pReqHdr);
2726	}
2727
2728
2729	/**
2730	* Wrapper for SUPR3CallVMMR0Ex which will deal with VINF_VMM_CALL_HOST returns.
2731	*
2732	* @returns VBox status code.
2733	* @param pVM The cross context VM structure.
2734	* @param pVCpu The cross context VM structure.
2735	* @param enmOperation Operation to execute.
2736	* @param u64Arg Constant argument.
2737	* @param pReqHdr Pointer to a request header. See SUPR3CallVMMR0Ex for
2738	* details.
2739	*/
2740	VMMR3_INT_DECL(int) VMMR3CallR0Emt(PVM pVM, PVMCPU pVCpu, VMMR0OPERATION enmOperation, uint64_t u64Arg, PSUPVMMR0REQHDR pReqHdr)
2741	{
2742	int rc;
2743	for (;;)
2744	{
2745	#ifdef NO_SUPCALLR0VMM
2746	rc = VERR_GENERAL_FAILURE;
2747	#else
2748	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, pVCpu->idCpu, enmOperation, u64Arg, pReqHdr);
2749	#endif
2750	/*
2751	* Flush the logs.
2752	*/
2753	#ifdef LOG_ENABLED
2754	VMM_FLUSH_R0_LOG(pVCpu->vmm.s.pR0LoggerR3, NULL);
2755	#endif
2756	VMM_FLUSH_R0_LOG(pVCpu->vmm.s.pR0RelLoggerR3, RTLogRelGetDefaultInstance());
2757	if (rc != VINF_VMM_CALL_HOST)
2758	break;
2759	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
2760	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
2761	break;
2762	/* Resume R0 */
2763	}
2764
2765	AssertLogRelMsgReturn(rc == VINF_SUCCESS \|\| RT_FAILURE(rc),
2766	("enmOperation=%u rc=%Rrc\n", enmOperation, rc),
2767	VERR_IPE_UNEXPECTED_INFO_STATUS);
2768	return rc;
2769	}
2770
2771
2772	#ifdef VBOX_WITH_RAW_MODE
2773	/**
2774	* Resumes executing hypervisor code when interrupted by a queue flush or a
2775	* debug event.
2776	*
2777	* @returns VBox status code.
2778	* @param pVM The cross context VM structure.
2779	* @param pVCpu The cross context virtual CPU structure.
2780	*/
2781	VMMR3DECL(int) VMMR3ResumeHyper(PVM pVM, PVMCPU pVCpu)
2782	{
2783	Log(("VMMR3ResumeHyper: eip=%RRv esp=%RRv\n", CPUMGetHyperEIP(pVCpu), CPUMGetHyperESP(pVCpu)));
2784	AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
2785
2786	/*
2787	* We hide log flushes (outer) and hypervisor interrupts (inner).
2788	*/
2789	for (;;)
2790	{
2791	int rc;
2792	Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
2793	do
2794	{
2795	# ifdef NO_SUPCALLR0VMM
2796	rc = VERR_GENERAL_FAILURE;
2797	# else
2798	rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
2799	if (RT_LIKELY(rc == VINF_SUCCESS))
2800	rc = pVCpu->vmm.s.iLastGZRc;
2801	# endif
2802	} while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
2803
2804	/*
2805	* Flush the loggers.
2806	*/
2807	# ifdef LOG_ENABLED
2808	PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
2809	if ( pLogger
2810	&& pLogger->offScratch > 0)
2811	RTLogFlushRC(NULL, pLogger);
2812	# endif
2813	# ifdef VBOX_WITH_RC_RELEASE_LOGGING
2814	PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
2815	if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
2816	RTLogFlushRC(RTLogRelGetDefaultInstance(), pRelLogger);
2817	# endif
2818	if (rc == VERR_TRPM_PANIC \|\| rc == VERR_TRPM_DONT_PANIC)
2819	VMMR3FatalDump(pVM, pVCpu, rc);
2820	if (rc != VINF_VMM_CALL_HOST)
2821	{
2822	Log(("VMMR3ResumeHyper: returns %Rrc\n", rc));
2823	return rc;
2824	}
2825	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
2826	if (RT_FAILURE(rc))
2827	return rc;
2828	}
2829	}
2830	#endif /* VBOX_WITH_RAW_MODE */
2831
2832
2833	/**
2834	* Service a call to the ring-3 host code.
2835	*
2836	* @returns VBox status code.
2837	* @param pVM The cross context VM structure.
2838	* @param pVCpu The cross context virtual CPU structure.
2839	* @remarks Careful with critsects.
2840	*/
2841	static int vmmR3ServiceCallRing3Request(PVM pVM, PVMCPU pVCpu)
2842	{
2843	/*
2844	* We must also check for pending critsect exits or else we can deadlock
2845	* when entering other critsects here.
2846	*/
2847	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PDM_CRITSECT))
2848	PDMCritSectBothFF(pVCpu);
2849
2850	switch (pVCpu->vmm.s.enmCallRing3Operation)
2851	{
2852	/*
2853	* Acquire a critical section.
2854	*/
2855	case VMMCALLRING3_PDM_CRIT_SECT_ENTER:
2856	{
2857	pVCpu->vmm.s.rcCallRing3 = PDMR3CritSectEnterEx((PPDMCRITSECT)(uintptr_t)pVCpu->vmm.s.u64CallRing3Arg,
2858	true /fCallRing3/);
2859	break;
2860	}
2861
2862	/*
2863	* Enter a r/w critical section exclusively.
2864	*/
2865	case VMMCALLRING3_PDM_CRIT_SECT_RW_ENTER_EXCL:
2866	{
2867	pVCpu->vmm.s.rcCallRing3 = PDMR3CritSectRwEnterExclEx((PPDMCRITSECTRW)(uintptr_t)pVCpu->vmm.s.u64CallRing3Arg,
2868	true /fCallRing3/);
2869	break;
2870	}
2871
2872	/*
2873	* Enter a r/w critical section shared.
2874	*/
2875	case VMMCALLRING3_PDM_CRIT_SECT_RW_ENTER_SHARED:
2876	{
2877	pVCpu->vmm.s.rcCallRing3 = PDMR3CritSectRwEnterSharedEx((PPDMCRITSECTRW)(uintptr_t)pVCpu->vmm.s.u64CallRing3Arg,
2878	true /fCallRing3/);
2879	break;
2880	}
2881
2882	/*
2883	* Acquire the PDM lock.
2884	*/
2885	case VMMCALLRING3_PDM_LOCK:
2886	{
2887	pVCpu->vmm.s.rcCallRing3 = PDMR3LockCall(pVM);
2888	break;
2889	}
2890
2891	/*
2892	* Grow the PGM pool.
2893	*/
2894	case VMMCALLRING3_PGM_POOL_GROW:
2895	{
2896	pVCpu->vmm.s.rcCallRing3 = PGMR3PoolGrow(pVM);
2897	break;
2898	}
2899
2900	/*
2901	* Maps an page allocation chunk into ring-3 so ring-0 can use it.
2902	*/
2903	case VMMCALLRING3_PGM_MAP_CHUNK:
2904	{
2905	pVCpu->vmm.s.rcCallRing3 = PGMR3PhysChunkMap(pVM, pVCpu->vmm.s.u64CallRing3Arg);
2906	break;
2907	}
2908
2909	/*
2910	* Allocates more handy pages.
2911	*/
2912	case VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES:
2913	{
2914	pVCpu->vmm.s.rcCallRing3 = PGMR3PhysAllocateHandyPages(pVM);
2915	break;
2916	}
2917
2918	/*
2919	* Allocates a large page.
2920	*/
2921	case VMMCALLRING3_PGM_ALLOCATE_LARGE_HANDY_PAGE:
2922	{
2923	pVCpu->vmm.s.rcCallRing3 = PGMR3PhysAllocateLargeHandyPage(pVM, pVCpu->vmm.s.u64CallRing3Arg);
2924	break;
2925	}
2926
2927	/*
2928	* Acquire the PGM lock.
2929	*/
2930	case VMMCALLRING3_PGM_LOCK:
2931	{
2932	pVCpu->vmm.s.rcCallRing3 = PGMR3LockCall(pVM);
2933	break;
2934	}
2935
2936	/*
2937	* Acquire the MM hypervisor heap lock.
2938	*/
2939	case VMMCALLRING3_MMHYPER_LOCK:
2940	{
2941	pVCpu->vmm.s.rcCallRing3 = MMR3LockCall(pVM);
2942	break;
2943	}
2944
2945	#ifdef VBOX_WITH_REM
2946	/*
2947	* Flush REM handler notifications.
2948	*/
2949	case VMMCALLRING3_REM_REPLAY_HANDLER_NOTIFICATIONS:
2950	{
2951	REMR3ReplayHandlerNotifications(pVM);
2952	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
2953	break;
2954	}
2955	#endif
2956
2957	/*
2958	* This is a noop. We just take this route to avoid unnecessary
2959	* tests in the loops.
2960	*/
2961	case VMMCALLRING3_VMM_LOGGER_FLUSH:
2962	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
2963	LogAlways(("FLUSH\n"));
2964	break;
2965
2966	/*
2967	* Set the VM error message.
2968	*/
2969	case VMMCALLRING3_VM_SET_ERROR:
2970	VMR3SetErrorWorker(pVM);
2971	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
2972	break;
2973
2974	/*
2975	* Set the VM runtime error message.
2976	*/
2977	case VMMCALLRING3_VM_SET_RUNTIME_ERROR:
2978	pVCpu->vmm.s.rcCallRing3 = VMR3SetRuntimeErrorWorker(pVM);
2979	break;
2980
2981	/*
2982	* Signal a ring 0 hypervisor assertion.
2983	* Cancel the longjmp operation that's in progress.
2984	*/
2985	case VMMCALLRING3_VM_R0_ASSERTION:
2986	pVCpu->vmm.s.enmCallRing3Operation = VMMCALLRING3_INVALID;
2987	pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call = false;
2988	#ifdef RT_ARCH_X86
2989	pVCpu->vmm.s.CallRing3JmpBufR0.eip = 0;
2990	#else
2991	pVCpu->vmm.s.CallRing3JmpBufR0.rip = 0;
2992	#endif
2993	#ifdef VMM_R0_SWITCH_STACK
2994	(uint64_t )pVCpu->vmm.s.pbEMTStackR3 = 0; /* clear marker */
2995	#endif
2996	LogRel(("%s", pVM->vmm.s.szRing0AssertMsg1));
2997	LogRel(("%s", pVM->vmm.s.szRing0AssertMsg2));
2998	return VERR_VMM_RING0_ASSERTION;
2999
3000	/*
3001	* A forced switch to ring 0 for preemption purposes.
3002	*/
3003	case VMMCALLRING3_VM_R0_PREEMPT:
3004	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
3005	break;
3006
3007	case VMMCALLRING3_FTM_SET_CHECKPOINT:
3008	pVCpu->vmm.s.rcCallRing3 = FTMR3SetCheckpoint(pVM, (FTMCHECKPOINTTYPE)pVCpu->vmm.s.u64CallRing3Arg);
3009	break;
3010
3011	default:
3012	AssertMsgFailed(("enmCallRing3Operation=%d\n", pVCpu->vmm.s.enmCallRing3Operation));
3013	return VERR_VMM_UNKNOWN_RING3_CALL;
3014	}
3015
3016	pVCpu->vmm.s.enmCallRing3Operation = VMMCALLRING3_INVALID;
3017	return VINF_SUCCESS;
3018	}
3019
3020
3021	/**
3022	* Displays the Force action Flags.
3023	*
3024	* @param pVM The cross context VM structure.
3025	* @param pHlp The output helpers.
3026	* @param pszArgs The additional arguments (ignored).
3027	*/
3028	static DECLCALLBACK(void) vmmR3InfoFF(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
3029	{
3030	int c;
3031	uint32_t f;
3032	NOREF(pszArgs);
3033
3034	#define PRINT_FLAG(prf,flag) do { \
3035	if (f & (prf##flag)) \
3036	{ \
3037	static const char *s_psz = #flag; \
3038	if (!(c % 6)) \
3039	pHlp->pfnPrintf(pHlp, "%s\n %s", c ? "," : "", s_psz); \
3040	else \
3041	pHlp->pfnPrintf(pHlp, ", %s", s_psz); \
3042	c++; \
3043	f &= ~(prf##flag); \
3044	} \
3045	} while (0)
3046
3047	#define PRINT_GROUP(prf,grp,sfx) do { \
3048	if (f & (prf##grp##sfx)) \
3049	{ \
3050	static const char *s_psz = #grp; \
3051	if (!(c % 5)) \
3052	pHlp->pfnPrintf(pHlp, "%s %s", c ? ",\n" : " Groups:\n", s_psz); \
3053	else \
3054	pHlp->pfnPrintf(pHlp, ", %s", s_psz); \
3055	c++; \
3056	} \
3057	} while (0)
3058
3059	/*
3060	* The global flags.
3061	*/
3062	const uint32_t fGlobalForcedActions = pVM->fGlobalForcedActions;
3063	pHlp->pfnPrintf(pHlp, "Global FFs: %#RX32", fGlobalForcedActions);
3064
3065	/* show the flag mnemonics */
3066	c = 0;
3067	f = fGlobalForcedActions;
3068	PRINT_FLAG(VM_FF_,TM_VIRTUAL_SYNC);
3069	PRINT_FLAG(VM_FF_,PDM_QUEUES);
3070	PRINT_FLAG(VM_FF_,PDM_DMA);
3071	PRINT_FLAG(VM_FF_,DBGF);
3072	PRINT_FLAG(VM_FF_,REQUEST);
3073	PRINT_FLAG(VM_FF_,CHECK_VM_STATE);
3074	PRINT_FLAG(VM_FF_,RESET);
3075	PRINT_FLAG(VM_FF_,EMT_RENDEZVOUS);
3076	PRINT_FLAG(VM_FF_,PGM_NEED_HANDY_PAGES);
3077	PRINT_FLAG(VM_FF_,PGM_NO_MEMORY);
3078	PRINT_FLAG(VM_FF_,PGM_POOL_FLUSH_PENDING);
3079	PRINT_FLAG(VM_FF_,REM_HANDLER_NOTIFY);
3080	PRINT_FLAG(VM_FF_,DEBUG_SUSPEND);
3081	if (f)
3082	pHlp->pfnPrintf(pHlp, "%s\n Unknown bits: %#RX32\n", c ? "," : "", f);
3083	else
3084	pHlp->pfnPrintf(pHlp, "\n");
3085
3086	/* the groups */
3087	c = 0;
3088	f = fGlobalForcedActions;
3089	PRINT_GROUP(VM_FF_,EXTERNAL_SUSPENDED,_MASK);
3090	PRINT_GROUP(VM_FF_,EXTERNAL_HALTED,_MASK);
3091	PRINT_GROUP(VM_FF_,HIGH_PRIORITY_PRE,_MASK);
3092	PRINT_GROUP(VM_FF_,HIGH_PRIORITY_PRE_RAW,_MASK);
3093	PRINT_GROUP(VM_FF_,HIGH_PRIORITY_POST,_MASK);
3094	PRINT_GROUP(VM_FF_,NORMAL_PRIORITY_POST,_MASK);
3095	PRINT_GROUP(VM_FF_,NORMAL_PRIORITY,_MASK);
3096	PRINT_GROUP(VM_FF_,ALL_REM,_MASK);
3097	if (c)
3098	pHlp->pfnPrintf(pHlp, "\n");
3099
3100	/*
3101	* Per CPU flags.
3102	*/
3103	for (VMCPUID i = 0; i < pVM->cCpus; i++)
3104	{
3105	const uint32_t fLocalForcedActions = pVM->aCpus[i].fLocalForcedActions;
3106	pHlp->pfnPrintf(pHlp, "CPU %u FFs: %#RX32", i, fLocalForcedActions);
3107
3108	/* show the flag mnemonics */
3109	c = 0;
3110	f = fLocalForcedActions;
3111	PRINT_FLAG(VMCPU_FF_,INTERRUPT_APIC);
3112	PRINT_FLAG(VMCPU_FF_,INTERRUPT_PIC);
3113	PRINT_FLAG(VMCPU_FF_,TIMER);
3114	PRINT_FLAG(VMCPU_FF_,INTERRUPT_NMI);
3115	PRINT_FLAG(VMCPU_FF_,INTERRUPT_SMI);
3116	PRINT_FLAG(VMCPU_FF_,PDM_CRITSECT);
3117	PRINT_FLAG(VMCPU_FF_,UNHALT);
3118	PRINT_FLAG(VMCPU_FF_,IEM);
3119	PRINT_FLAG(VMCPU_FF_,UPDATE_APIC);
3120	PRINT_FLAG(VMCPU_FF_,DBGF);
3121	PRINT_FLAG(VMCPU_FF_,REQUEST);
3122	PRINT_FLAG(VMCPU_FF_,HM_UPDATE_CR3);
3123	PRINT_FLAG(VMCPU_FF_,HM_UPDATE_PAE_PDPES);
3124	PRINT_FLAG(VMCPU_FF_,PGM_SYNC_CR3);
3125	PRINT_FLAG(VMCPU_FF_,PGM_SYNC_CR3_NON_GLOBAL);
3126	PRINT_FLAG(VMCPU_FF_,TLB_FLUSH);
3127	PRINT_FLAG(VMCPU_FF_,INHIBIT_INTERRUPTS);
3128	PRINT_FLAG(VMCPU_FF_,BLOCK_NMIS);
3129	PRINT_FLAG(VMCPU_FF_,TO_R3);
3130	PRINT_FLAG(VMCPU_FF_,IOM);
3131	#ifdef VBOX_WITH_RAW_MODE
3132	PRINT_FLAG(VMCPU_FF_,TRPM_SYNC_IDT);
3133	PRINT_FLAG(VMCPU_FF_,SELM_SYNC_TSS);
3134	PRINT_FLAG(VMCPU_FF_,SELM_SYNC_GDT);
3135	PRINT_FLAG(VMCPU_FF_,SELM_SYNC_LDT);
3136	PRINT_FLAG(VMCPU_FF_,CSAM_SCAN_PAGE);
3137	PRINT_FLAG(VMCPU_FF_,CSAM_PENDING_ACTION);
3138	PRINT_FLAG(VMCPU_FF_,CPUM);
3139	#endif
3140	if (f)
3141	pHlp->pfnPrintf(pHlp, "%s\n Unknown bits: %#RX32\n", c ? "," : "", f);
3142	else
3143	pHlp->pfnPrintf(pHlp, "\n");
3144
3145	if (fLocalForcedActions & VMCPU_FF_INHIBIT_INTERRUPTS)
3146	pHlp->pfnPrintf(pHlp, " intr inhibit RIP: %RGp\n", EMGetInhibitInterruptsPC(&pVM->aCpus[i]));
3147
3148	/* the groups */
3149	c = 0;
3150	f = fLocalForcedActions;
3151	PRINT_GROUP(VMCPU_FF_,EXTERNAL_SUSPENDED,_MASK);
3152	PRINT_GROUP(VMCPU_FF_,EXTERNAL_HALTED,_MASK);
3153	PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_PRE,_MASK);
3154	PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_PRE_RAW,_MASK);
3155	PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_POST,_MASK);
3156	PRINT_GROUP(VMCPU_FF_,NORMAL_PRIORITY_POST,_MASK);
3157	PRINT_GROUP(VMCPU_FF_,NORMAL_PRIORITY,_MASK);
3158	PRINT_GROUP(VMCPU_FF_,RESUME_GUEST,_MASK);
3159	PRINT_GROUP(VMCPU_FF_,HM_TO_R3,_MASK);
3160	PRINT_GROUP(VMCPU_FF_,ALL_REM,_MASK);
3161	if (c)
3162	pHlp->pfnPrintf(pHlp, "\n");
3163	}
3164
3165	#undef PRINT_FLAG
3166	#undef PRINT_GROUP
3167	}
3168

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR3/VMM.cpp@ 73770

Download in other formats: