VMM.cpp@ 60682

Last change on this file since 60682 was 60377, checked in by vboxsync, 9 years ago
VMM: Fix APIC, CPUM init ordering for the new APIC code while still retaining the old code. Namely, consistent MSR APIC base caching and APIC page dependency on PDM construction, see bugref:8245:46 for more details.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 110.1 KB

Line
1	/* $Id: VMM.cpp 60377 2016-04-07 15:53:36Z vboxsync $ */
2	/** @file
3	* VMM - The Virtual Machine Monitor Core.
4	*/
5
6	/*
7	* Copyright (C) 2006-2015 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	//#define NO_SUPCALLR0VMM
19
20	/** @page pg_vmm VMM - The Virtual Machine Monitor
21	*
22	* The VMM component is two things at the moment, it's a component doing a few
23	* management and routing tasks, and it's the whole virtual machine monitor
24	* thing. For hysterical reasons, it is not doing all the management that one
25	* would expect, this is instead done by @ref pg_vm. We'll address this
26	* misdesign eventually, maybe.
27	*
28	* VMM is made up of these components:
29	* - @subpage pg_cfgm
30	* - @subpage pg_cpum
31	* - @subpage pg_csam
32	* - @subpage pg_dbgf
33	* - @subpage pg_em
34	* - @subpage pg_gim
35	* - @subpage pg_gmm
36	* - @subpage pg_gvmm
37	* - @subpage pg_hm
38	* - @subpage pg_iem
39	* - @subpage pg_iom
40	* - @subpage pg_mm
41	* - @subpage pg_patm
42	* - @subpage pg_pdm
43	* - @subpage pg_pgm
44	* - @subpage pg_rem
45	* - @subpage pg_selm
46	* - @subpage pg_ssm
47	* - @subpage pg_stam
48	* - @subpage pg_tm
49	* - @subpage pg_trpm
50	* - @subpage pg_vm
51	*
52	*
53	* @see @ref grp_vmm @ref grp_vm @subpage pg_vmm_guideline @subpage pg_raw
54	*
55	*
56	* @section sec_vmmstate VMM State
57	*
58	* @image html VM_Statechart_Diagram.gif
59	*
60	* To be written.
61	*
62	*
63	* @subsection subsec_vmm_init VMM Initialization
64	*
65	* To be written.
66	*
67	*
68	* @subsection subsec_vmm_term VMM Termination
69	*
70	* To be written.
71	*
72	*
73	* @section sec_vmm_limits VMM Limits
74	*
75	* There are various resource limits imposed by the VMM and it's
76	* sub-components. We'll list some of them here.
77	*
78	* On 64-bit hosts:
79	* - Max 8191 VMs. Imposed by GVMM's handle allocation (GVMM_MAX_HANDLES),
80	* can be increased up to 64K - 1.
81	* - Max 16TB - 64KB of the host memory can be used for backing VM RAM and
82	* ROM pages. The limit is imposed by the 32-bit page ID used by GMM.
83	* - A VM can be assigned all the memory we can use (16TB), however, the
84	* Main API will restrict this to 2TB (MM_RAM_MAX_IN_MB).
85	* - Max 32 virtual CPUs (VMM_MAX_CPU_COUNT).
86	*
87	* On 32-bit hosts:
88	* - Max 127 VMs. Imposed by GMM's per page structure.
89	* - Max 64GB - 64KB of the host memory can be used for backing VM RAM and
90	* ROM pages. The limit is imposed by the 28-bit page ID used
91	* internally in GMM. It is also limited by PAE.
92	* - A VM can be assigned all the memory GMM can allocate, however, the
93	* Main API will restrict this to 3584MB (MM_RAM_MAX_IN_MB).
94	* - Max 32 virtual CPUs (VMM_MAX_CPU_COUNT).
95	*
96	*/
97
98
99	/*********************************************************************************************************************************
100	* Header Files *
101	*********************************************************************************************************************************/
102	#define LOG_GROUP LOG_GROUP_VMM
103	#include <VBox/vmm/vmm.h>
104	#include <VBox/vmm/vmapi.h>
105	#include <VBox/vmm/pgm.h>
106	#include <VBox/vmm/cfgm.h>
107	#include <VBox/vmm/pdmqueue.h>
108	#include <VBox/vmm/pdmcritsect.h>
109	#include <VBox/vmm/pdmcritsectrw.h>
110	#include <VBox/vmm/pdmapi.h>
111	#include <VBox/vmm/cpum.h>
112	#include <VBox/vmm/gim.h>
113	#include <VBox/vmm/mm.h>
114	#include <VBox/vmm/iom.h>
115	#include <VBox/vmm/trpm.h>
116	#include <VBox/vmm/selm.h>
117	#include <VBox/vmm/em.h>
118	#include <VBox/sup.h>
119	#include <VBox/vmm/dbgf.h>
120	#include <VBox/vmm/csam.h>
121	#include <VBox/vmm/patm.h>
122	#ifdef VBOX_WITH_NEW_APIC
123	# include <VBox/vmm/apic.h>
124	#endif
125	#ifdef VBOX_WITH_REM
126	# include <VBox/vmm/rem.h>
127	#endif
128	#include <VBox/vmm/ssm.h>
129	#include <VBox/vmm/ftm.h>
130	#include <VBox/vmm/tm.h>
131	#include "VMMInternal.h"
132	#include "VMMSwitcher.h"
133	#include <VBox/vmm/vm.h>
134	#include <VBox/vmm/uvm.h>
135
136	#include <VBox/err.h>
137	#include <VBox/param.h>
138	#include <VBox/version.h>
139	#include <VBox/vmm/hm.h>
140	#include <iprt/assert.h>
141	#include <iprt/alloc.h>
142	#include <iprt/asm.h>
143	#include <iprt/time.h>
144	#include <iprt/semaphore.h>
145	#include <iprt/stream.h>
146	#include <iprt/string.h>
147	#include <iprt/stdarg.h>
148	#include <iprt/ctype.h>
149	#include <iprt/x86.h>
150
151
152	/*********************************************************************************************************************************
153	* Defined Constants And Macros *
154	*********************************************************************************************************************************/
155	/** The saved state version. */
156	#define VMM_SAVED_STATE_VERSION 4
157	/** The saved state version used by v3.0 and earlier. (Teleportation) */
158	#define VMM_SAVED_STATE_VERSION_3_0 3
159
160
161	/*********************************************************************************************************************************
162	* Internal Functions *
163	*********************************************************************************************************************************/
164	static int vmmR3InitStacks(PVM pVM);
165	static int vmmR3InitLoggers(PVM pVM);
166	static void vmmR3InitRegisterStats(PVM pVM);
167	static DECLCALLBACK(int) vmmR3Save(PVM pVM, PSSMHANDLE pSSM);
168	static DECLCALLBACK(int) vmmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
169	static DECLCALLBACK(void) vmmR3YieldEMT(PVM pVM, PTMTIMER pTimer, void *pvUser);
170	static VBOXSTRICTRC vmmR3EmtRendezvousCommon(PVM pVM, PVMCPU pVCpu, bool fIsCaller,
171	uint32_t fFlags, PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser);
172	static int vmmR3ServiceCallRing3Request(PVM pVM, PVMCPU pVCpu);
173	static DECLCALLBACK(void) vmmR3InfoFF(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
174
175
176	/**
177	* Initializes the VMM.
178	*
179	* @returns VBox status code.
180	* @param pVM The cross context VM structure.
181	*/
182	VMMR3_INT_DECL(int) VMMR3Init(PVM pVM)
183	{
184	LogFlow(("VMMR3Init\n"));
185
186	/*
187	* Assert alignment, sizes and order.
188	*/
189	AssertMsg(pVM->vmm.s.offVM == 0, ("Already initialized!\n"));
190	AssertCompile(sizeof(pVM->vmm.s) <= sizeof(pVM->vmm.padding));
191	AssertCompile(sizeof(pVM->aCpus[0].vmm.s) <= sizeof(pVM->aCpus[0].vmm.padding));
192
193	/*
194	* Init basic VM VMM members.
195	*/
196	pVM->vmm.s.offVM = RT_OFFSETOF(VM, vmm);
197	pVM->vmm.s.pahEvtRendezvousEnterOrdered = NULL;
198	pVM->vmm.s.hEvtRendezvousEnterOneByOne = NIL_RTSEMEVENT;
199	pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce = NIL_RTSEMEVENTMULTI;
200	pVM->vmm.s.hEvtMulRendezvousDone = NIL_RTSEMEVENTMULTI;
201	pVM->vmm.s.hEvtRendezvousDoneCaller = NIL_RTSEMEVENT;
202	pVM->vmm.s.hEvtMulRendezvousRecursionPush = NIL_RTSEMEVENTMULTI;
203	pVM->vmm.s.hEvtMulRendezvousRecursionPop = NIL_RTSEMEVENTMULTI;
204	pVM->vmm.s.hEvtRendezvousRecursionPushCaller = NIL_RTSEMEVENT;
205	pVM->vmm.s.hEvtRendezvousRecursionPopCaller = NIL_RTSEMEVENT;
206
207	/** @cfgm{/YieldEMTInterval, uint32_t, 1, UINT32_MAX, 23, ms}
208	* The EMT yield interval. The EMT yielding is a hack we employ to play a
209	* bit nicer with the rest of the system (like for instance the GUI).
210	*/
211	int rc = CFGMR3QueryU32Def(CFGMR3GetRoot(pVM), "YieldEMTInterval", &pVM->vmm.s.cYieldEveryMillies,
212	23 /* Value arrived at after experimenting with the grub boot prompt. */);
213	AssertMsgRCReturn(rc, ("Configuration error. Failed to query \"YieldEMTInterval\", rc=%Rrc\n", rc), rc);
214
215
216	/** @cfgm{/VMM/UsePeriodicPreemptionTimers, boolean, true}
217	* Controls whether we employ per-cpu preemption timers to limit the time
218	* spent executing guest code. This option is not available on all
219	* platforms and we will silently ignore this setting then. If we are
220	* running in VT-x mode, we will use the VMX-preemption timer instead of
221	* this one when possible.
222	*/
223	PCFGMNODE pCfgVMM = CFGMR3GetChild(CFGMR3GetRoot(pVM), "VMM");
224	rc = CFGMR3QueryBoolDef(pCfgVMM, "UsePeriodicPreemptionTimers", &pVM->vmm.s.fUsePeriodicPreemptionTimers, true);
225	AssertMsgRCReturn(rc, ("Configuration error. Failed to query \"VMM/UsePeriodicPreemptionTimers\", rc=%Rrc\n", rc), rc);
226
227	/*
228	* Initialize the VMM rendezvous semaphores.
229	*/
230	pVM->vmm.s.pahEvtRendezvousEnterOrdered = (PRTSEMEVENT)MMR3HeapAlloc(pVM, MM_TAG_VMM, sizeof(RTSEMEVENT) * pVM->cCpus);
231	if (!pVM->vmm.s.pahEvtRendezvousEnterOrdered)
232	return VERR_NO_MEMORY;
233	for (VMCPUID i = 0; i < pVM->cCpus; i++)
234	pVM->vmm.s.pahEvtRendezvousEnterOrdered[i] = NIL_RTSEMEVENT;
235	for (VMCPUID i = 0; i < pVM->cCpus; i++)
236	{
237	rc = RTSemEventCreate(&pVM->vmm.s.pahEvtRendezvousEnterOrdered[i]);
238	AssertRCReturn(rc, rc);
239	}
240	rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousEnterOneByOne);
241	AssertRCReturn(rc, rc);
242	rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
243	AssertRCReturn(rc, rc);
244	rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousDone);
245	AssertRCReturn(rc, rc);
246	rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousDoneCaller);
247	AssertRCReturn(rc, rc);
248	rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousRecursionPush);
249	AssertRCReturn(rc, rc);
250	rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousRecursionPop);
251	AssertRCReturn(rc, rc);
252	rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousRecursionPushCaller);
253	AssertRCReturn(rc, rc);
254	rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousRecursionPopCaller);
255	AssertRCReturn(rc, rc);
256
257	/*
258	* Register the saved state data unit.
259	*/
260	rc = SSMR3RegisterInternal(pVM, "vmm", 1, VMM_SAVED_STATE_VERSION, VMM_STACK_SIZE + sizeof(RTGCPTR),
261	NULL, NULL, NULL,
262	NULL, vmmR3Save, NULL,
263	NULL, vmmR3Load, NULL);
264	if (RT_FAILURE(rc))
265	return rc;
266
267	/*
268	* Register the Ring-0 VM handle with the session for fast ioctl calls.
269	*/
270	rc = SUPR3SetVMForFastIOCtl(pVM->pVMR0);
271	if (RT_FAILURE(rc))
272	return rc;
273
274	/*
275	* Init various sub-components.
276	*/
277	rc = vmmR3SwitcherInit(pVM);
278	if (RT_SUCCESS(rc))
279	{
280	rc = vmmR3InitStacks(pVM);
281	if (RT_SUCCESS(rc))
282	{
283	rc = vmmR3InitLoggers(pVM);
284
285	#ifdef VBOX_WITH_NMI
286	/*
287	* Allocate mapping for the host APIC.
288	*/
289	if (RT_SUCCESS(rc))
290	{
291	rc = MMR3HyperReserve(pVM, PAGE_SIZE, "Host APIC", &pVM->vmm.s.GCPtrApicBase);
292	AssertRC(rc);
293	}
294	#endif
295	if (RT_SUCCESS(rc))
296	{
297	/*
298	* Debug info and statistics.
299	*/
300	DBGFR3InfoRegisterInternal(pVM, "fflags", "Displays the current Forced actions Flags.", vmmR3InfoFF);
301	vmmR3InitRegisterStats(pVM);
302	vmmInitFormatTypes();
303
304	return VINF_SUCCESS;
305	}
306	}
307	/** @todo: Need failure cleanup. */
308
309	//more todo in here?
310	//if (RT_SUCCESS(rc))
311	//{
312	//}
313	//int rc2 = vmmR3TermCoreCode(pVM);
314	//AssertRC(rc2));
315	}
316
317	return rc;
318	}
319
320
321	/**
322	* Allocate & setup the VMM RC stack(s) (for EMTs).
323	*
324	* The stacks are also used for long jumps in Ring-0.
325	*
326	* @returns VBox status code.
327	* @param pVM The cross context VM structure.
328	*
329	* @remarks The optional guard page gets it protection setup up during R3 init
330	* completion because of init order issues.
331	*/
332	static int vmmR3InitStacks(PVM pVM)
333	{
334	int rc = VINF_SUCCESS;
335	#ifdef VMM_R0_SWITCH_STACK
336	uint32_t fFlags = MMHYPER_AONR_FLAGS_KERNEL_MAPPING;
337	#else
338	uint32_t fFlags = 0;
339	#endif
340
341	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
342	{
343	PVMCPU pVCpu = &pVM->aCpus[idCpu];
344
345	#ifdef VBOX_STRICT_VMM_STACK
346	rc = MMR3HyperAllocOnceNoRelEx(pVM, PAGE_SIZE + VMM_STACK_SIZE + PAGE_SIZE,
347	#else
348	rc = MMR3HyperAllocOnceNoRelEx(pVM, VMM_STACK_SIZE,
349	#endif
350	PAGE_SIZE, MM_TAG_VMM, fFlags, (void **)&pVCpu->vmm.s.pbEMTStackR3);
351	if (RT_SUCCESS(rc))
352	{
353	#ifdef VBOX_STRICT_VMM_STACK
354	pVCpu->vmm.s.pbEMTStackR3 += PAGE_SIZE;
355	#endif
356	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
357	/* MMHyperR3ToR0 returns R3 when not doing hardware assisted virtualization. */
358	if (!HMIsEnabled(pVM))
359	pVCpu->vmm.s.CallRing3JmpBufR0.pvSavedStack = NIL_RTR0PTR;
360	else
361	#endif
362	pVCpu->vmm.s.CallRing3JmpBufR0.pvSavedStack = MMHyperR3ToR0(pVM, pVCpu->vmm.s.pbEMTStackR3);
363	pVCpu->vmm.s.pbEMTStackRC = MMHyperR3ToRC(pVM, pVCpu->vmm.s.pbEMTStackR3);
364	pVCpu->vmm.s.pbEMTStackBottomRC = pVCpu->vmm.s.pbEMTStackRC + VMM_STACK_SIZE;
365	AssertRelease(pVCpu->vmm.s.pbEMTStackRC);
366
367	CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC);
368	}
369	}
370
371	return rc;
372	}
373
374
375	/**
376	* Initialize the loggers.
377	*
378	* @returns VBox status code.
379	* @param pVM The cross context VM structure.
380	*/
381	static int vmmR3InitLoggers(PVM pVM)
382	{
383	int rc;
384	#define RTLogCalcSizeForR0(cGroups, fFlags) (RT_OFFSETOF(VMMR0LOGGER, Logger.afGroups[cGroups]) + PAGE_SIZE)
385
386	/*
387	* Allocate RC & R0 Logger instances (they are finalized in the relocator).
388	*/
389	#ifdef LOG_ENABLED
390	PRTLOGGER pLogger = RTLogDefaultInstance();
391	if (pLogger)
392	{
393	if (!HMIsEnabled(pVM))
394	{
395	pVM->vmm.s.cbRCLogger = RT_OFFSETOF(RTLOGGERRC, afGroups[pLogger->cGroups]);
396	rc = MMR3HyperAllocOnceNoRel(pVM, pVM->vmm.s.cbRCLogger, 0, MM_TAG_VMM, (void **)&pVM->vmm.s.pRCLoggerR3);
397	if (RT_FAILURE(rc))
398	return rc;
399	pVM->vmm.s.pRCLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCLoggerR3);
400	}
401
402	# ifdef VBOX_WITH_R0_LOGGING
403	size_t const cbLogger = RTLogCalcSizeForR0(pLogger->cGroups, 0);
404	for (VMCPUID i = 0; i < pVM->cCpus; i++)
405	{
406	PVMCPU pVCpu = &pVM->aCpus[i];
407	rc = MMR3HyperAllocOnceNoRelEx(pVM, cbLogger, PAGE_SIZE, MM_TAG_VMM, MMHYPER_AONR_FLAGS_KERNEL_MAPPING,
408	(void **)&pVCpu->vmm.s.pR0LoggerR3);
409	if (RT_FAILURE(rc))
410	return rc;
411	pVCpu->vmm.s.pR0LoggerR3->pVM = pVM->pVMR0;
412	//pVCpu->vmm.s.pR0LoggerR3->fCreated = false;
413	pVCpu->vmm.s.pR0LoggerR3->cbLogger = (uint32_t)cbLogger;
414	pVCpu->vmm.s.pR0LoggerR0 = MMHyperR3ToR0(pVM, pVCpu->vmm.s.pR0LoggerR3);
415	}
416	# endif
417	}
418	#endif /* LOG_ENABLED */
419
420	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
421	/*
422	* Allocate RC release logger instances (finalized in the relocator).
423	*/
424	if (!HMIsEnabled(pVM))
425	{
426	PRTLOGGER pRelLogger = RTLogRelGetDefaultInstance();
427	if (pRelLogger)
428	{
429	pVM->vmm.s.cbRCRelLogger = RT_OFFSETOF(RTLOGGERRC, afGroups[pRelLogger->cGroups]);
430	rc = MMR3HyperAllocOnceNoRel(pVM, pVM->vmm.s.cbRCRelLogger, 0, MM_TAG_VMM, (void **)&pVM->vmm.s.pRCRelLoggerR3);
431	if (RT_FAILURE(rc))
432	return rc;
433	pVM->vmm.s.pRCRelLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCRelLoggerR3);
434	}
435	}
436	#endif /* VBOX_WITH_RC_RELEASE_LOGGING */
437	return VINF_SUCCESS;
438	}
439
440
441	/**
442	* VMMR3Init worker that register the statistics with STAM.
443	*
444	* @param pVM The cross context VM structure.
445	*/
446	static void vmmR3InitRegisterStats(PVM pVM)
447	{
448	/*
449	* Statistics.
450	*/
451	STAM_REG(pVM, &pVM->vmm.s.StatRunRC, STAMTYPE_COUNTER, "/VMM/RunRC", STAMUNIT_OCCURENCES, "Number of context switches.");
452	STAM_REG(pVM, &pVM->vmm.s.StatRZRetNormal, STAMTYPE_COUNTER, "/VMM/RZRet/Normal", STAMUNIT_OCCURENCES, "Number of VINF_SUCCESS returns.");
453	STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterrupt, STAMTYPE_COUNTER, "/VMM/RZRet/Interrupt", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT returns.");
454	STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterruptHyper, STAMTYPE_COUNTER, "/VMM/RZRet/InterruptHyper", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT_HYPER returns.");
455	STAM_REG(pVM, &pVM->vmm.s.StatRZRetGuestTrap, STAMTYPE_COUNTER, "/VMM/RZRet/GuestTrap", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_GUEST_TRAP returns.");
456	STAM_REG(pVM, &pVM->vmm.s.StatRZRetRingSwitch, STAMTYPE_COUNTER, "/VMM/RZRet/RingSwitch", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_RING_SWITCH returns.");
457	STAM_REG(pVM, &pVM->vmm.s.StatRZRetRingSwitchInt, STAMTYPE_COUNTER, "/VMM/RZRet/RingSwitchInt", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_RING_SWITCH_INT returns.");
458	STAM_REG(pVM, &pVM->vmm.s.StatRZRetStaleSelector, STAMTYPE_COUNTER, "/VMM/RZRet/StaleSelector", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_STALE_SELECTOR returns.");
459	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIRETTrap, STAMTYPE_COUNTER, "/VMM/RZRet/IRETTrap", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_IRET_TRAP returns.");
460	STAM_REG(pVM, &pVM->vmm.s.StatRZRetEmulate, STAMTYPE_COUNTER, "/VMM/RZRet/Emulate", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION returns.");
461	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIOBlockEmulate, STAMTYPE_COUNTER, "/VMM/RZRet/EmulateIOBlock", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_EMULATE_IO_BLOCK returns.");
462	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchEmulate, STAMTYPE_COUNTER, "/VMM/RZRet/PatchEmulate", STAMUNIT_OCCURENCES, "Number of VINF_PATCH_EMULATE_INSTR returns.");
463	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIORead, STAMTYPE_COUNTER, "/VMM/RZRet/IORead", STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_IOPORT_READ returns.");
464	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIOWrite, STAMTYPE_COUNTER, "/VMM/RZRet/IOWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_IOPORT_WRITE returns.");
465	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIORead, STAMTYPE_COUNTER, "/VMM/RZRet/MMIORead", STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_MMIO_READ returns.");
466	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOWrite, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_MMIO_WRITE returns.");
467	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOReadWrite, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOReadWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_MMIO_READ_WRITE returns.");
468	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOPatchRead, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOPatchRead", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_PATCH_READ returns.");
469	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOPatchWrite, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOPatchWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_PATCH_WRITE returns.");
470	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMSRRead, STAMTYPE_COUNTER, "/VMM/RZRet/MSRRead", STAMUNIT_OCCURENCES, "Number of VINF_CPUM_R3_MSR_READ returns.");
471	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMSRWrite, STAMTYPE_COUNTER, "/VMM/RZRet/MSRWrite", STAMUNIT_OCCURENCES, "Number of VINF_CPUM_R3_MSR_WRITE returns.");
472	STAM_REG(pVM, &pVM->vmm.s.StatRZRetLDTFault, STAMTYPE_COUNTER, "/VMM/RZRet/LDTFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_GDT_FAULT returns.");
473	STAM_REG(pVM, &pVM->vmm.s.StatRZRetGDTFault, STAMTYPE_COUNTER, "/VMM/RZRet/GDTFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_LDT_FAULT returns.");
474	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIDTFault, STAMTYPE_COUNTER, "/VMM/RZRet/IDTFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_IDT_FAULT returns.");
475	STAM_REG(pVM, &pVM->vmm.s.StatRZRetTSSFault, STAMTYPE_COUNTER, "/VMM/RZRet/TSSFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_TSS_FAULT returns.");
476	STAM_REG(pVM, &pVM->vmm.s.StatRZRetCSAMTask, STAMTYPE_COUNTER, "/VMM/RZRet/CSAMTask", STAMUNIT_OCCURENCES, "Number of VINF_CSAM_PENDING_ACTION returns.");
477	STAM_REG(pVM, &pVM->vmm.s.StatRZRetSyncCR3, STAMTYPE_COUNTER, "/VMM/RZRet/SyncCR", STAMUNIT_OCCURENCES, "Number of VINF_PGM_SYNC_CR3 returns.");
478	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMisc, STAMTYPE_COUNTER, "/VMM/RZRet/Misc", STAMUNIT_OCCURENCES, "Number of misc returns.");
479	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchInt3, STAMTYPE_COUNTER, "/VMM/RZRet/PatchInt3", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_INT3 returns.");
480	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchPF, STAMTYPE_COUNTER, "/VMM/RZRet/PatchPF", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_TRAP_PF returns.");
481	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchGP, STAMTYPE_COUNTER, "/VMM/RZRet/PatchGP", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_TRAP_GP returns.");
482	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchIretIRQ, STAMTYPE_COUNTER, "/VMM/RZRet/PatchIret", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PENDING_IRQ_AFTER_IRET returns.");
483	STAM_REG(pVM, &pVM->vmm.s.StatRZRetRescheduleREM, STAMTYPE_COUNTER, "/VMM/RZRet/ScheduleREM", STAMUNIT_OCCURENCES, "Number of VINF_EM_RESCHEDULE_REM returns.");
484	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
485	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Unknown, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Unknown", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
486	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3TMVirt, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/TMVirt", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
487	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3HandyPages, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Handy", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
488	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3PDMQueues, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/PDMQueue", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
489	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Rendezvous, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Rendezvous", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
490	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Timer, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Timer", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
491	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3DMA, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/DMA", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
492	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3CritSect, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/CritSect", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
493	STAM_REG(pVM, &pVM->vmm.s.StatRZRetTimerPending, STAMTYPE_COUNTER, "/VMM/RZRet/TimerPending", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TIMER_PENDING returns.");
494	STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterruptPending, STAMTYPE_COUNTER, "/VMM/RZRet/InterruptPending", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT_PENDING returns.");
495	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPATMDuplicateFn, STAMTYPE_COUNTER, "/VMM/RZRet/PATMDuplicateFn", STAMUNIT_OCCURENCES, "Number of VINF_PATM_DUPLICATE_FUNCTION returns.");
496	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPGMChangeMode, STAMTYPE_COUNTER, "/VMM/RZRet/PGMChangeMode", STAMUNIT_OCCURENCES, "Number of VINF_PGM_CHANGE_MODE returns.");
497	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPGMFlushPending, STAMTYPE_COUNTER, "/VMM/RZRet/PGMFlushPending", STAMUNIT_OCCURENCES, "Number of VINF_PGM_POOL_FLUSH_PENDING returns.");
498	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPendingRequest, STAMTYPE_COUNTER, "/VMM/RZRet/PendingRequest", STAMUNIT_OCCURENCES, "Number of VINF_EM_PENDING_REQUEST returns.");
499	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchTPR, STAMTYPE_COUNTER, "/VMM/RZRet/PatchTPR", STAMUNIT_OCCURENCES, "Number of VINF_EM_HM_PATCH_TPR_INSTR returns.");
500	STAM_REG(pVM, &pVM->vmm.s.StatRZRetCallRing3, STAMTYPE_COUNTER, "/VMM/RZCallR3/Misc", STAMUNIT_OCCURENCES, "Number of Other ring-3 calls.");
501	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPDMLock, STAMTYPE_COUNTER, "/VMM/RZCallR3/PDMLock", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PDM_LOCK calls.");
502	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPDMCritSectEnter, STAMTYPE_COUNTER, "/VMM/RZCallR3/PDMCritSectEnter", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PDM_CRITSECT_ENTER calls.");
503	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMLock, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMLock", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_LOCK calls.");
504	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMPoolGrow, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMPoolGrow", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_POOL_GROW calls.");
505	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMMapChunk, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMMapChunk", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_MAP_CHUNK calls.");
506	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMAllocHandy, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMAllocHandy", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES calls.");
507	STAM_REG(pVM, &pVM->vmm.s.StatRZCallRemReplay, STAMTYPE_COUNTER, "/VMM/RZCallR3/REMReplay", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_REM_REPLAY_HANDLER_NOTIFICATIONS calls.");
508	STAM_REG(pVM, &pVM->vmm.s.StatRZCallLogFlush, STAMTYPE_COUNTER, "/VMM/RZCallR3/VMMLogFlush", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VMM_LOGGER_FLUSH calls.");
509	STAM_REG(pVM, &pVM->vmm.s.StatRZCallVMSetError, STAMTYPE_COUNTER, "/VMM/RZCallR3/VMSetError", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VM_SET_ERROR calls.");
510	STAM_REG(pVM, &pVM->vmm.s.StatRZCallVMSetRuntimeError, STAMTYPE_COUNTER, "/VMM/RZCallR3/VMRuntimeError", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VM_SET_RUNTIME_ERROR calls.");
511
512	#ifdef VBOX_WITH_STATISTICS
513	for (VMCPUID i = 0; i < pVM->cCpus; i++)
514	{
515	STAMR3RegisterF(pVM, &pVM->aCpus[i].vmm.s.CallRing3JmpBufR0.cbUsedMax, STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Max amount of stack used.", "/VMM/Stack/CPU%u/Max", i);
516	STAMR3RegisterF(pVM, &pVM->aCpus[i].vmm.s.CallRing3JmpBufR0.cbUsedAvg, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Average stack usage.", "/VMM/Stack/CPU%u/Avg", i);
517	STAMR3RegisterF(pVM, &pVM->aCpus[i].vmm.s.CallRing3JmpBufR0.cUsedTotal, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of stack usages.", "/VMM/Stack/CPU%u/Uses", i);
518	}
519	#endif
520	}
521
522
523	/**
524	* Initializes the R0 VMM.
525	*
526	* @returns VBox status code.
527	* @param pVM The cross context VM structure.
528	*/
529	VMMR3_INT_DECL(int) VMMR3InitR0(PVM pVM)
530	{
531	int rc;
532	PVMCPU pVCpu = VMMGetCpu(pVM);
533	Assert(pVCpu && pVCpu->idCpu == 0);
534
535	#ifdef LOG_ENABLED
536	/*
537	* Initialize the ring-0 logger if we haven't done so yet.
538	*/
539	if ( pVCpu->vmm.s.pR0LoggerR3
540	&& !pVCpu->vmm.s.pR0LoggerR3->fCreated)
541	{
542	rc = VMMR3UpdateLoggers(pVM);
543	if (RT_FAILURE(rc))
544	return rc;
545	}
546	#endif
547
548	/*
549	* Call Ring-0 entry with init code.
550	*/
551	for (;;)
552	{
553	#ifdef NO_SUPCALLR0VMM
554	//rc = VERR_GENERAL_FAILURE;
555	rc = VINF_SUCCESS;
556	#else
557	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /idCpu/, VMMR0_DO_VMMR0_INIT,
558	RT_MAKE_U64(VMMGetSvnRev(), vmmGetBuildType()), NULL);
559	#endif
560	/*
561	* Flush the logs.
562	*/
563	#ifdef LOG_ENABLED
564	if ( pVCpu->vmm.s.pR0LoggerR3
565	&& pVCpu->vmm.s.pR0LoggerR3->Logger.offScratch > 0)
566	RTLogFlushR0(NULL, &pVCpu->vmm.s.pR0LoggerR3->Logger);
567	#endif
568	if (rc != VINF_VMM_CALL_HOST)
569	break;
570	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
571	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
572	break;
573	/* Resume R0 */
574	}
575
576	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
577	{
578	LogRel(("VMM: R0 init failed, rc=%Rra\n", rc));
579	if (RT_SUCCESS(rc))
580	rc = VERR_IPE_UNEXPECTED_INFO_STATUS;
581	}
582
583	/* Log whether thread-context hooks are used (on Linux this can depend on how the kernel is configured). */
584	if (pVM->aCpus[0].vmm.s.hCtxHook != NIL_RTTHREADCTXHOOK)
585	LogRel(("VMM: Enabled thread-context hooks\n"));
586	else
587	LogRel(("VMM: Thread-context hooks unavailable\n"));
588
589	return rc;
590	}
591
592
593	#ifdef VBOX_WITH_RAW_MODE
594	/**
595	* Initializes the RC VMM.
596	*
597	* @returns VBox status code.
598	* @param pVM The cross context VM structure.
599	*/
600	VMMR3_INT_DECL(int) VMMR3InitRC(PVM pVM)
601	{
602	PVMCPU pVCpu = VMMGetCpu(pVM);
603	Assert(pVCpu && pVCpu->idCpu == 0);
604
605	/* In VMX mode, there's no need to init RC. */
606	if (HMIsEnabled(pVM))
607	return VINF_SUCCESS;
608
609	AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
610
611	/*
612	* Call VMMRCInit():
613	* -# resolve the address.
614	* -# setup stackframe and EIP to use the trampoline.
615	* -# do a generic hypervisor call.
616	*/
617	RTRCPTR RCPtrEP;
618	int rc = PDMR3LdrGetSymbolRC(pVM, VMMRC_MAIN_MODULE_NAME, "VMMRCEntry", &RCPtrEP);
619	if (RT_SUCCESS(rc))
620	{
621	CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC); /* Clear the stack. */
622	uint64_t u64TS = RTTimeProgramStartNanoTS();
623	CPUMPushHyper(pVCpu, (uint32_t)(u64TS >> 32)); /* Param 4: The program startup TS - Hi. */
624	CPUMPushHyper(pVCpu, (uint32_t)u64TS); /* Param 4: The program startup TS - Lo. */
625	CPUMPushHyper(pVCpu, vmmGetBuildType()); /* Param 3: Version argument. */
626	CPUMPushHyper(pVCpu, VMMGetSvnRev()); /* Param 2: Version argument. */
627	CPUMPushHyper(pVCpu, VMMRC_DO_VMMRC_INIT); /* Param 1: Operation. */
628	CPUMPushHyper(pVCpu, pVM->pVMRC); /* Param 0: pVM */
629	CPUMPushHyper(pVCpu, 6 * sizeof(RTRCPTR)); /* trampoline param: stacksize. */
630	CPUMPushHyper(pVCpu, RCPtrEP); /* Call EIP. */
631	CPUMSetHyperEIP(pVCpu, pVM->vmm.s.pfnCallTrampolineRC);
632	Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
633
634	for (;;)
635	{
636	#ifdef NO_SUPCALLR0VMM
637	//rc = VERR_GENERAL_FAILURE;
638	rc = VINF_SUCCESS;
639	#else
640	rc = SUPR3CallVMMR0(pVM->pVMR0, 0 /* VCPU 0 */, VMMR0_DO_CALL_HYPERVISOR, NULL);
641	#endif
642	#ifdef LOG_ENABLED
643	PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
644	if ( pLogger
645	&& pLogger->offScratch > 0)
646	RTLogFlushRC(NULL, pLogger);
647	#endif
648	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
649	PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
650	if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
651	RTLogFlushRC(RTLogRelGetDefaultInstance(), pRelLogger);
652	#endif
653	if (rc != VINF_VMM_CALL_HOST)
654	break;
655	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
656	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
657	break;
658	}
659
660	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
661	{
662	VMMR3FatalDump(pVM, pVCpu, rc);
663	if (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST)
664	rc = VERR_IPE_UNEXPECTED_INFO_STATUS;
665	}
666	AssertRC(rc);
667	}
668	return rc;
669	}
670	#endif /* VBOX_WITH_RAW_MODE */
671
672
673	/**
674	* Called when an init phase completes.
675	*
676	* @returns VBox status code.
677	* @param pVM The cross context VM structure.
678	* @param enmWhat Which init phase.
679	*/
680	VMMR3_INT_DECL(int) VMMR3InitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
681	{
682	int rc = VINF_SUCCESS;
683
684	switch (enmWhat)
685	{
686	case VMINITCOMPLETED_RING3:
687	{
688	/*
689	* Set page attributes to r/w for stack pages.
690	*/
691	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
692	{
693	rc = PGMMapSetPage(pVM, pVM->aCpus[idCpu].vmm.s.pbEMTStackRC, VMM_STACK_SIZE,
694	X86_PTE_P \| X86_PTE_A \| X86_PTE_D \| X86_PTE_RW);
695	AssertRCReturn(rc, rc);
696	}
697
698	/*
699	* Create the EMT yield timer.
700	*/
701	rc = TMR3TimerCreateInternal(pVM, TMCLOCK_REAL, vmmR3YieldEMT, NULL, "EMT Yielder", &pVM->vmm.s.pYieldTimer);
702	AssertRCReturn(rc, rc);
703
704	rc = TMTimerSetMillies(pVM->vmm.s.pYieldTimer, pVM->vmm.s.cYieldEveryMillies);
705	AssertRCReturn(rc, rc);
706
707	#ifdef VBOX_WITH_NMI
708	/*
709	* Map the host APIC into GC - This is AMD/Intel + Host OS specific!
710	*/
711	rc = PGMMap(pVM, pVM->vmm.s.GCPtrApicBase, 0xfee00000, PAGE_SIZE,
712	X86_PTE_P \| X86_PTE_RW \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_A \| X86_PTE_D);
713	AssertRCReturn(rc, rc);
714	#endif
715
716	#ifdef VBOX_STRICT_VMM_STACK
717	/*
718	* Setup the stack guard pages: Two inaccessible pages at each sides of the
719	* stack to catch over/under-flows.
720	*/
721	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
722	{
723	uint8_t *pbEMTStackR3 = pVM->aCpus[idCpu].vmm.s.pbEMTStackR3;
724
725	memset(pbEMTStackR3 - PAGE_SIZE, 0xcc, PAGE_SIZE);
726	MMR3HyperSetGuard(pVM, pbEMTStackR3 - PAGE_SIZE, PAGE_SIZE, true /fSet/);
727
728	memset(pbEMTStackR3 + VMM_STACK_SIZE, 0xcc, PAGE_SIZE);
729	MMR3HyperSetGuard(pVM, pbEMTStackR3 + VMM_STACK_SIZE, PAGE_SIZE, true /fSet/);
730	}
731	pVM->vmm.s.fStackGuardsStationed = true;
732	#endif
733	break;
734	}
735
736	case VMINITCOMPLETED_HM:
737	{
738	/*
739	* Disable the periodic preemption timers if we can use the
740	* VMX-preemption timer instead.
741	*/
742	if ( pVM->vmm.s.fUsePeriodicPreemptionTimers
743	&& HMR3IsVmxPreemptionTimerUsed(pVM))
744	pVM->vmm.s.fUsePeriodicPreemptionTimers = false;
745	LogRel(("VMM: fUsePeriodicPreemptionTimers=%RTbool\n", pVM->vmm.s.fUsePeriodicPreemptionTimers));
746
747	/*
748	* Last chance for GIM to update its CPUID leaves if it requires
749	* knowledge/information from HM initialization.
750	*/
751	rc = GIMR3InitCompleted(pVM);
752	AssertRCReturn(rc, rc);
753
754	/*
755	* CPUM's post-initialization (print CPUIDs).
756	*/
757	CPUMR3LogCpuIds(pVM);
758	break;
759	}
760
761	default: /* shuts up gcc */
762	break;
763	}
764
765	return rc;
766	}
767
768
769	/**
770	* Terminate the VMM bits.
771	*
772	* @returns VBox status code.
773	* @param pVM The cross context VM structure.
774	*/
775	VMMR3_INT_DECL(int) VMMR3Term(PVM pVM)
776	{
777	PVMCPU pVCpu = VMMGetCpu(pVM);
778	Assert(pVCpu && pVCpu->idCpu == 0);
779
780	/*
781	* Call Ring-0 entry with termination code.
782	*/
783	int rc;
784	for (;;)
785	{
786	#ifdef NO_SUPCALLR0VMM
787	//rc = VERR_GENERAL_FAILURE;
788	rc = VINF_SUCCESS;
789	#else
790	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /idCpu/, VMMR0_DO_VMMR0_TERM, 0, NULL);
791	#endif
792	/*
793	* Flush the logs.
794	*/
795	#ifdef LOG_ENABLED
796	if ( pVCpu->vmm.s.pR0LoggerR3
797	&& pVCpu->vmm.s.pR0LoggerR3->Logger.offScratch > 0)
798	RTLogFlushR0(NULL, &pVCpu->vmm.s.pR0LoggerR3->Logger);
799	#endif
800	if (rc != VINF_VMM_CALL_HOST)
801	break;
802	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
803	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
804	break;
805	/* Resume R0 */
806	}
807	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
808	{
809	LogRel(("VMM: VMMR3Term: R0 term failed, rc=%Rra. (warning)\n", rc));
810	if (RT_SUCCESS(rc))
811	rc = VERR_IPE_UNEXPECTED_INFO_STATUS;
812	}
813
814	for (VMCPUID i = 0; i < pVM->cCpus; i++)
815	{
816	RTSemEventDestroy(pVM->vmm.s.pahEvtRendezvousEnterOrdered[i]);
817	pVM->vmm.s.pahEvtRendezvousEnterOrdered[i] = NIL_RTSEMEVENT;
818	}
819	RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousEnterOneByOne);
820	pVM->vmm.s.hEvtRendezvousEnterOneByOne = NIL_RTSEMEVENT;
821	RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
822	pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce = NIL_RTSEMEVENTMULTI;
823	RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousDone);
824	pVM->vmm.s.hEvtMulRendezvousDone = NIL_RTSEMEVENTMULTI;
825	RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousDoneCaller);
826	pVM->vmm.s.hEvtRendezvousDoneCaller = NIL_RTSEMEVENT;
827	RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousRecursionPush);
828	pVM->vmm.s.hEvtMulRendezvousRecursionPush = NIL_RTSEMEVENTMULTI;
829	RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousRecursionPop);
830	pVM->vmm.s.hEvtMulRendezvousRecursionPop = NIL_RTSEMEVENTMULTI;
831	RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousRecursionPushCaller);
832	pVM->vmm.s.hEvtRendezvousRecursionPushCaller = NIL_RTSEMEVENT;
833	RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousRecursionPopCaller);
834	pVM->vmm.s.hEvtRendezvousRecursionPopCaller = NIL_RTSEMEVENT;
835
836	#ifdef VBOX_STRICT_VMM_STACK
837	/*
838	* Make the two stack guard pages present again.
839	*/
840	if (pVM->vmm.s.fStackGuardsStationed)
841	{
842	for (VMCPUID i = 0; i < pVM->cCpus; i++)
843	{
844	uint8_t *pbEMTStackR3 = pVM->aCpus[i].vmm.s.pbEMTStackR3;
845	MMR3HyperSetGuard(pVM, pbEMTStackR3 - PAGE_SIZE, PAGE_SIZE, false /fSet/);
846	MMR3HyperSetGuard(pVM, pbEMTStackR3 + VMM_STACK_SIZE, PAGE_SIZE, false /fSet/);
847	}
848	pVM->vmm.s.fStackGuardsStationed = false;
849	}
850	#endif
851
852	vmmTermFormatTypes();
853	return rc;
854	}
855
856
857	/**
858	* Applies relocations to data and code managed by this
859	* component. This function will be called at init and
860	* whenever the VMM need to relocate it self inside the GC.
861	*
862	* The VMM will need to apply relocations to the core code.
863	*
864	* @param pVM The cross context VM structure.
865	* @param offDelta The relocation delta.
866	*/
867	VMMR3_INT_DECL(void) VMMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
868	{
869	LogFlow(("VMMR3Relocate: offDelta=%RGv\n", offDelta));
870
871	/*
872	* Recalc the RC address.
873	*/
874	#ifdef VBOX_WITH_RAW_MODE
875	pVM->vmm.s.pvCoreCodeRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pvCoreCodeR3);
876	#endif
877
878	/*
879	* The stack.
880	*/
881	for (VMCPUID i = 0; i < pVM->cCpus; i++)
882	{
883	PVMCPU pVCpu = &pVM->aCpus[i];
884
885	CPUMSetHyperESP(pVCpu, CPUMGetHyperESP(pVCpu) + offDelta);
886
887	pVCpu->vmm.s.pbEMTStackRC = MMHyperR3ToRC(pVM, pVCpu->vmm.s.pbEMTStackR3);
888	pVCpu->vmm.s.pbEMTStackBottomRC = pVCpu->vmm.s.pbEMTStackRC + VMM_STACK_SIZE;
889	}
890
891	/*
892	* All the switchers.
893	*/
894	vmmR3SwitcherRelocate(pVM, offDelta);
895
896	/*
897	* Get other RC entry points.
898	*/
899	if (!HMIsEnabled(pVM))
900	{
901	int rc = PDMR3LdrGetSymbolRC(pVM, VMMRC_MAIN_MODULE_NAME, "CPUMGCResumeGuest", &pVM->vmm.s.pfnCPUMRCResumeGuest);
902	AssertReleaseMsgRC(rc, ("CPUMGCResumeGuest not found! rc=%Rra\n", rc));
903
904	rc = PDMR3LdrGetSymbolRC(pVM, VMMRC_MAIN_MODULE_NAME, "CPUMGCResumeGuestV86", &pVM->vmm.s.pfnCPUMRCResumeGuestV86);
905	AssertReleaseMsgRC(rc, ("CPUMGCResumeGuestV86 not found! rc=%Rra\n", rc));
906	}
907
908	/*
909	* Update the logger.
910	*/
911	VMMR3UpdateLoggers(pVM);
912	}
913
914
915	/**
916	* Updates the settings for the RC and R0 loggers.
917	*
918	* @returns VBox status code.
919	* @param pVM The cross context VM structure.
920	*/
921	VMMR3_INT_DECL(int) VMMR3UpdateLoggers(PVM pVM)
922	{
923	/*
924	* Simply clone the logger instance (for RC).
925	*/
926	int rc = VINF_SUCCESS;
927	RTRCPTR RCPtrLoggerFlush = 0;
928
929	if ( pVM->vmm.s.pRCLoggerR3
930	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
931	\|\| pVM->vmm.s.pRCRelLoggerR3
932	#endif
933	)
934	{
935	Assert(!HMIsEnabled(pVM));
936	rc = PDMR3LdrGetSymbolRC(pVM, VMMRC_MAIN_MODULE_NAME, "vmmGCLoggerFlush", &RCPtrLoggerFlush);
937	AssertReleaseMsgRC(rc, ("vmmGCLoggerFlush not found! rc=%Rra\n", rc));
938	}
939
940	if (pVM->vmm.s.pRCLoggerR3)
941	{
942	Assert(!HMIsEnabled(pVM));
943	RTRCPTR RCPtrLoggerWrapper = 0;
944	rc = PDMR3LdrGetSymbolRC(pVM, VMMRC_MAIN_MODULE_NAME, "vmmGCLoggerWrapper", &RCPtrLoggerWrapper);
945	AssertReleaseMsgRC(rc, ("vmmGCLoggerWrapper not found! rc=%Rra\n", rc));
946
947	pVM->vmm.s.pRCLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCLoggerR3);
948	rc = RTLogCloneRC(NULL /* default */, pVM->vmm.s.pRCLoggerR3, pVM->vmm.s.cbRCLogger,
949	RCPtrLoggerWrapper, RCPtrLoggerFlush, RTLOGFLAGS_BUFFERED);
950	AssertReleaseMsgRC(rc, ("RTLogCloneRC failed! rc=%Rra\n", rc));
951	}
952
953	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
954	if (pVM->vmm.s.pRCRelLoggerR3)
955	{
956	Assert(!HMIsEnabled(pVM));
957	RTRCPTR RCPtrLoggerWrapper = 0;
958	rc = PDMR3LdrGetSymbolRC(pVM, VMMRC_MAIN_MODULE_NAME, "vmmGCRelLoggerWrapper", &RCPtrLoggerWrapper);
959	AssertReleaseMsgRC(rc, ("vmmGCRelLoggerWrapper not found! rc=%Rra\n", rc));
960
961	pVM->vmm.s.pRCRelLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCRelLoggerR3);
962	rc = RTLogCloneRC(RTLogRelGetDefaultInstance(), pVM->vmm.s.pRCRelLoggerR3, pVM->vmm.s.cbRCRelLogger,
963	RCPtrLoggerWrapper, RCPtrLoggerFlush, RTLOGFLAGS_BUFFERED);
964	AssertReleaseMsgRC(rc, ("RTLogCloneRC failed! rc=%Rra\n", rc));
965	}
966	#endif /* VBOX_WITH_RC_RELEASE_LOGGING */
967
968	#ifdef LOG_ENABLED
969	/*
970	* For the ring-0 EMT logger, we use a per-thread logger instance
971	* in ring-0. Only initialize it once.
972	*/
973	PRTLOGGER const pDefault = RTLogDefaultInstance();
974	for (VMCPUID i = 0; i < pVM->cCpus; i++)
975	{
976	PVMCPU pVCpu = &pVM->aCpus[i];
977	PVMMR0LOGGER pR0LoggerR3 = pVCpu->vmm.s.pR0LoggerR3;
978	if (pR0LoggerR3)
979	{
980	if (!pR0LoggerR3->fCreated)
981	{
982	RTR0PTR pfnLoggerWrapper = NIL_RTR0PTR;
983	rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerWrapper", &pfnLoggerWrapper);
984	AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerWrapper not found! rc=%Rra\n", rc), rc);
985
986	RTR0PTR pfnLoggerFlush = NIL_RTR0PTR;
987	rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerFlush", &pfnLoggerFlush);
988	AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerFlush not found! rc=%Rra\n", rc), rc);
989
990	rc = RTLogCreateForR0(&pR0LoggerR3->Logger, pR0LoggerR3->cbLogger,
991	pVCpu->vmm.s.pR0LoggerR0 + RT_OFFSETOF(VMMR0LOGGER, Logger),
992	pfnLoggerWrapper, pfnLoggerFlush,
993	RTLOGFLAGS_BUFFERED, RTLOGDEST_DUMMY);
994	AssertReleaseMsgRCReturn(rc, ("RTLogCreateForR0 failed! rc=%Rra\n", rc), rc);
995
996	RTR0PTR pfnLoggerPrefix = NIL_RTR0PTR;
997	rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerPrefix", &pfnLoggerPrefix);
998	AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerPrefix not found! rc=%Rra\n", rc), rc);
999	rc = RTLogSetCustomPrefixCallbackForR0(&pR0LoggerR3->Logger,
1000	pVCpu->vmm.s.pR0LoggerR0 + RT_OFFSETOF(VMMR0LOGGER, Logger),
1001	pfnLoggerPrefix, NIL_RTR0PTR);
1002	AssertReleaseMsgRCReturn(rc, ("RTLogSetCustomPrefixCallback failed! rc=%Rra\n", rc), rc);
1003
1004	pR0LoggerR3->idCpu = i;
1005	pR0LoggerR3->fCreated = true;
1006	pR0LoggerR3->fFlushingDisabled = false;
1007
1008	}
1009
1010	rc = RTLogCopyGroupsAndFlagsForR0(&pR0LoggerR3->Logger, pVCpu->vmm.s.pR0LoggerR0 + RT_OFFSETOF(VMMR0LOGGER, Logger),
1011	pDefault, RTLOGFLAGS_BUFFERED, UINT32_MAX);
1012	AssertRC(rc);
1013	}
1014	}
1015	#endif
1016	return rc;
1017	}
1018
1019
1020	/**
1021	* Gets the pointer to a buffer containing the R0/RC RTAssertMsg1Weak output.
1022	*
1023	* @returns Pointer to the buffer.
1024	* @param pVM The cross context VM structure.
1025	*/
1026	VMMR3DECL(const char *) VMMR3GetRZAssertMsg1(PVM pVM)
1027	{
1028	if (HMIsEnabled(pVM))
1029	return pVM->vmm.s.szRing0AssertMsg1;
1030
1031	RTRCPTR RCPtr;
1032	int rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_szRTAssertMsg1", &RCPtr);
1033	if (RT_SUCCESS(rc))
1034	return (const char *)MMHyperRCToR3(pVM, RCPtr);
1035
1036	return NULL;
1037	}
1038
1039
1040	/**
1041	* Returns the VMCPU of the specified virtual CPU.
1042	*
1043	* @returns The VMCPU pointer. NULL if @a idCpu or @a pUVM is invalid.
1044	*
1045	* @param pUVM The user mode VM handle.
1046	* @param idCpu The ID of the virtual CPU.
1047	*/
1048	VMMR3DECL(PVMCPU) VMMR3GetCpuByIdU(PUVM pUVM, RTCPUID idCpu)
1049	{
1050	UVM_ASSERT_VALID_EXT_RETURN(pUVM, NULL);
1051	AssertReturn(idCpu < pUVM->cCpus, NULL);
1052	VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, NULL);
1053	return &pUVM->pVM->aCpus[idCpu];
1054	}
1055
1056
1057	/**
1058	* Gets the pointer to a buffer containing the R0/RC RTAssertMsg2Weak output.
1059	*
1060	* @returns Pointer to the buffer.
1061	* @param pVM The cross context VM structure.
1062	*/
1063	VMMR3DECL(const char *) VMMR3GetRZAssertMsg2(PVM pVM)
1064	{
1065	if (HMIsEnabled(pVM))
1066	return pVM->vmm.s.szRing0AssertMsg2;
1067
1068	RTRCPTR RCPtr;
1069	int rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_szRTAssertMsg2", &RCPtr);
1070	if (RT_SUCCESS(rc))
1071	return (const char *)MMHyperRCToR3(pVM, RCPtr);
1072
1073	return NULL;
1074	}
1075
1076
1077	/**
1078	* Execute state save operation.
1079	*
1080	* @returns VBox status code.
1081	* @param pVM The cross context VM structure.
1082	* @param pSSM SSM operation handle.
1083	*/
1084	static DECLCALLBACK(int) vmmR3Save(PVM pVM, PSSMHANDLE pSSM)
1085	{
1086	LogFlow(("vmmR3Save:\n"));
1087
1088	/*
1089	* Save the started/stopped state of all CPUs except 0 as it will always
1090	* be running. This avoids breaking the saved state version. :-)
1091	*/
1092	for (VMCPUID i = 1; i < pVM->cCpus; i++)
1093	SSMR3PutBool(pSSM, VMCPUSTATE_IS_STARTED(VMCPU_GET_STATE(&pVM->aCpus[i])));
1094
1095	return SSMR3PutU32(pSSM, UINT32_MAX); /* terminator */
1096	}
1097
1098
1099	/**
1100	* Execute state load operation.
1101	*
1102	* @returns VBox status code.
1103	* @param pVM The cross context VM structure.
1104	* @param pSSM SSM operation handle.
1105	* @param uVersion Data layout version.
1106	* @param uPass The data pass.
1107	*/
1108	static DECLCALLBACK(int) vmmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
1109	{
1110	LogFlow(("vmmR3Load:\n"));
1111	Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
1112
1113	/*
1114	* Validate version.
1115	*/
1116	if ( uVersion != VMM_SAVED_STATE_VERSION
1117	&& uVersion != VMM_SAVED_STATE_VERSION_3_0)
1118	{
1119	AssertMsgFailed(("vmmR3Load: Invalid version uVersion=%u!\n", uVersion));
1120	return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
1121	}
1122
1123	if (uVersion <= VMM_SAVED_STATE_VERSION_3_0)
1124	{
1125	/* Ignore the stack bottom, stack pointer and stack bits. */
1126	RTRCPTR RCPtrIgnored;
1127	SSMR3GetRCPtr(pSSM, &RCPtrIgnored);
1128	SSMR3GetRCPtr(pSSM, &RCPtrIgnored);
1129	#ifdef RT_OS_DARWIN
1130	if ( SSMR3HandleVersion(pSSM) >= VBOX_FULL_VERSION_MAKE(3,0,0)
1131	&& SSMR3HandleVersion(pSSM) < VBOX_FULL_VERSION_MAKE(3,1,0)
1132	&& SSMR3HandleRevision(pSSM) >= 48858
1133	&& ( !strcmp(SSMR3HandleHostOSAndArch(pSSM), "darwin.x86")
1134	\|\| !strcmp(SSMR3HandleHostOSAndArch(pSSM), "") )
1135	)
1136	SSMR3Skip(pSSM, 16384);
1137	else
1138	SSMR3Skip(pSSM, 8192);
1139	#else
1140	SSMR3Skip(pSSM, 8192);
1141	#endif
1142	}
1143
1144	/*
1145	* Restore the VMCPU states. VCPU 0 is always started.
1146	*/
1147	VMCPU_SET_STATE(&pVM->aCpus[0], VMCPUSTATE_STARTED);
1148	for (VMCPUID i = 1; i < pVM->cCpus; i++)
1149	{
1150	bool fStarted;
1151	int rc = SSMR3GetBool(pSSM, &fStarted);
1152	if (RT_FAILURE(rc))
1153	return rc;
1154	VMCPU_SET_STATE(&pVM->aCpus[i], fStarted ? VMCPUSTATE_STARTED : VMCPUSTATE_STOPPED);
1155	}
1156
1157	/* terminator */
1158	uint32_t u32;
1159	int rc = SSMR3GetU32(pSSM, &u32);
1160	if (RT_FAILURE(rc))
1161	return rc;
1162	if (u32 != UINT32_MAX)
1163	{
1164	AssertMsgFailed(("u32=%#x\n", u32));
1165	return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
1166	}
1167	return VINF_SUCCESS;
1168	}
1169
1170
1171	#ifdef VBOX_WITH_RAW_MODE
1172	/**
1173	* Resolve a builtin RC symbol.
1174	*
1175	* Called by PDM when loading or relocating RC modules.
1176	*
1177	* @returns VBox status
1178	* @param pVM The cross context VM structure.
1179	* @param pszSymbol Symbol to resolve.
1180	* @param pRCPtrValue Where to store the symbol value.
1181	*
1182	* @remark This has to work before VMMR3Relocate() is called.
1183	*/
1184	VMMR3_INT_DECL(int) VMMR3GetImportRC(PVM pVM, const char *pszSymbol, PRTRCPTR pRCPtrValue)
1185	{
1186	if (!strcmp(pszSymbol, "g_Logger"))
1187	{
1188	if (pVM->vmm.s.pRCLoggerR3)
1189	pVM->vmm.s.pRCLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCLoggerR3);
1190	*pRCPtrValue = pVM->vmm.s.pRCLoggerRC;
1191	}
1192	else if (!strcmp(pszSymbol, "g_RelLogger"))
1193	{
1194	# ifdef VBOX_WITH_RC_RELEASE_LOGGING
1195	if (pVM->vmm.s.pRCRelLoggerR3)
1196	pVM->vmm.s.pRCRelLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCRelLoggerR3);
1197	*pRCPtrValue = pVM->vmm.s.pRCRelLoggerRC;
1198	# else
1199	*pRCPtrValue = NIL_RTRCPTR;
1200	# endif
1201	}
1202	else
1203	return VERR_SYMBOL_NOT_FOUND;
1204	return VINF_SUCCESS;
1205	}
1206	#endif /* VBOX_WITH_RAW_MODE */
1207
1208
1209	/**
1210	* Suspends the CPU yielder.
1211	*
1212	* @param pVM The cross context VM structure.
1213	*/
1214	VMMR3_INT_DECL(void) VMMR3YieldSuspend(PVM pVM)
1215	{
1216	VMCPU_ASSERT_EMT(&pVM->aCpus[0]);
1217	if (!pVM->vmm.s.cYieldResumeMillies)
1218	{
1219	uint64_t u64Now = TMTimerGet(pVM->vmm.s.pYieldTimer);
1220	uint64_t u64Expire = TMTimerGetExpire(pVM->vmm.s.pYieldTimer);
1221	if (u64Now >= u64Expire \|\| u64Expire == ~(uint64_t)0)
1222	pVM->vmm.s.cYieldResumeMillies = pVM->vmm.s.cYieldEveryMillies;
1223	else
1224	pVM->vmm.s.cYieldResumeMillies = TMTimerToMilli(pVM->vmm.s.pYieldTimer, u64Expire - u64Now);
1225	TMTimerStop(pVM->vmm.s.pYieldTimer);
1226	}
1227	pVM->vmm.s.u64LastYield = RTTimeNanoTS();
1228	}
1229
1230
1231	/**
1232	* Stops the CPU yielder.
1233	*
1234	* @param pVM The cross context VM structure.
1235	*/
1236	VMMR3_INT_DECL(void) VMMR3YieldStop(PVM pVM)
1237	{
1238	if (!pVM->vmm.s.cYieldResumeMillies)
1239	TMTimerStop(pVM->vmm.s.pYieldTimer);
1240	pVM->vmm.s.cYieldResumeMillies = pVM->vmm.s.cYieldEveryMillies;
1241	pVM->vmm.s.u64LastYield = RTTimeNanoTS();
1242	}
1243
1244
1245	/**
1246	* Resumes the CPU yielder when it has been a suspended or stopped.
1247	*
1248	* @param pVM The cross context VM structure.
1249	*/
1250	VMMR3_INT_DECL(void) VMMR3YieldResume(PVM pVM)
1251	{
1252	if (pVM->vmm.s.cYieldResumeMillies)
1253	{
1254	TMTimerSetMillies(pVM->vmm.s.pYieldTimer, pVM->vmm.s.cYieldResumeMillies);
1255	pVM->vmm.s.cYieldResumeMillies = 0;
1256	}
1257	}
1258
1259
1260	/**
1261	* Internal timer callback function.
1262	*
1263	* @param pVM The cross context VM structure.
1264	* @param pTimer The timer handle.
1265	* @param pvUser User argument specified upon timer creation.
1266	*/
1267	static DECLCALLBACK(void) vmmR3YieldEMT(PVM pVM, PTMTIMER pTimer, void *pvUser)
1268	{
1269	NOREF(pvUser);
1270
1271	/*
1272	* This really needs some careful tuning. While we shouldn't be too greedy since
1273	* that'll cause the rest of the system to stop up, we shouldn't be too nice either
1274	* because that'll cause us to stop up.
1275	*
1276	* The current logic is to use the default interval when there is no lag worth
1277	* mentioning, but when we start accumulating lag we don't bother yielding at all.
1278	*
1279	* (This depends on the TMCLOCK_VIRTUAL_SYNC to be scheduled before TMCLOCK_REAL
1280	* so the lag is up to date.)
1281	*/
1282	const uint64_t u64Lag = TMVirtualSyncGetLag(pVM);
1283	if ( u64Lag < 50000000 /* 50ms */
1284	\|\| ( u64Lag < 1000000000 /* 1s */
1285	&& RTTimeNanoTS() - pVM->vmm.s.u64LastYield < 500000000 /* 500 ms */)
1286	)
1287	{
1288	uint64_t u64Elapsed = RTTimeNanoTS();
1289	pVM->vmm.s.u64LastYield = u64Elapsed;
1290
1291	RTThreadYield();
1292
1293	#ifdef LOG_ENABLED
1294	u64Elapsed = RTTimeNanoTS() - u64Elapsed;
1295	Log(("vmmR3YieldEMT: %RI64 ns\n", u64Elapsed));
1296	#endif
1297	}
1298	TMTimerSetMillies(pTimer, pVM->vmm.s.cYieldEveryMillies);
1299	}
1300
1301
1302	#ifdef VBOX_WITH_RAW_MODE
1303	/**
1304	* Executes guest code in the raw-mode context.
1305	*
1306	* @param pVM The cross context VM structure.
1307	* @param pVCpu The cross context virtual CPU structure.
1308	*/
1309	VMMR3_INT_DECL(int) VMMR3RawRunGC(PVM pVM, PVMCPU pVCpu)
1310	{
1311	Log2(("VMMR3RawRunGC: (cs:eip=%04x:%08x)\n", CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
1312
1313	AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
1314
1315	/*
1316	* Set the hypervisor to resume executing a CPUM resume function
1317	* in CPUMRCA.asm.
1318	*/
1319	CPUMSetHyperState(pVCpu,
1320	CPUMGetGuestEFlags(pVCpu) & X86_EFL_VM
1321	? pVM->vmm.s.pfnCPUMRCResumeGuestV86
1322	: pVM->vmm.s.pfnCPUMRCResumeGuest, /* eip */
1323	pVCpu->vmm.s.pbEMTStackBottomRC, /* esp */
1324	0, /* eax */
1325	VM_RC_ADDR(pVM, &pVCpu->cpum) /* edx */);
1326
1327	/*
1328	* We hide log flushes (outer) and hypervisor interrupts (inner).
1329	*/
1330	for (;;)
1331	{
1332	#ifdef VBOX_STRICT
1333	if (RT_UNLIKELY(!CPUMGetHyperCR3(pVCpu) \|\| CPUMGetHyperCR3(pVCpu) != PGMGetHyperCR3(pVCpu)))
1334	EMR3FatalError(pVCpu, VERR_VMM_HYPER_CR3_MISMATCH);
1335	PGMMapCheck(pVM);
1336	# ifdef VBOX_WITH_SAFE_STR
1337	SELMR3CheckShadowTR(pVM);
1338	# endif
1339	#endif
1340	int rc;
1341	do
1342	{
1343	#ifdef NO_SUPCALLR0VMM
1344	rc = VERR_GENERAL_FAILURE;
1345	#else
1346	rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
1347	if (RT_LIKELY(rc == VINF_SUCCESS))
1348	rc = pVCpu->vmm.s.iLastGZRc;
1349	#endif
1350	} while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
1351
1352	/*
1353	* Flush the logs.
1354	*/
1355	#ifdef LOG_ENABLED
1356	PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
1357	if ( pLogger
1358	&& pLogger->offScratch > 0)
1359	RTLogFlushRC(NULL, pLogger);
1360	#endif
1361	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
1362	PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
1363	if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
1364	RTLogFlushRC(RTLogRelGetDefaultInstance(), pRelLogger);
1365	#endif
1366	if (rc != VINF_VMM_CALL_HOST)
1367	{
1368	Log2(("VMMR3RawRunGC: returns %Rrc (cs:eip=%04x:%08x)\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
1369	return rc;
1370	}
1371	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
1372	if (RT_FAILURE(rc))
1373	return rc;
1374	/* Resume GC */
1375	}
1376	}
1377	#endif /* VBOX_WITH_RAW_MODE */
1378
1379
1380	/**
1381	* Executes guest code (Intel VT-x and AMD-V).
1382	*
1383	* @param pVM The cross context VM structure.
1384	* @param pVCpu The cross context virtual CPU structure.
1385	*/
1386	VMMR3_INT_DECL(int) VMMR3HmRunGC(PVM pVM, PVMCPU pVCpu)
1387	{
1388	Log2(("VMMR3HmRunGC: (cs:rip=%04x:%RX64)\n", CPUMGetGuestCS(pVCpu), CPUMGetGuestRIP(pVCpu)));
1389
1390	for (;;)
1391	{
1392	int rc;
1393	do
1394	{
1395	#ifdef NO_SUPCALLR0VMM
1396	rc = VERR_GENERAL_FAILURE;
1397	#else
1398	rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_HM_RUN, pVCpu->idCpu);
1399	if (RT_LIKELY(rc == VINF_SUCCESS))
1400	rc = pVCpu->vmm.s.iLastGZRc;
1401	#endif
1402	} while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
1403
1404	#if 0 /* todo triggers too often */
1405	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TO_R3));
1406	#endif
1407
1408	#ifdef LOG_ENABLED
1409	/*
1410	* Flush the log
1411	*/
1412	PVMMR0LOGGER pR0LoggerR3 = pVCpu->vmm.s.pR0LoggerR3;
1413	if ( pR0LoggerR3
1414	&& pR0LoggerR3->Logger.offScratch > 0)
1415	RTLogFlushR0(NULL, &pR0LoggerR3->Logger);
1416	#endif /* !LOG_ENABLED */
1417	if (rc != VINF_VMM_CALL_HOST)
1418	{
1419	Log2(("VMMR3HmRunGC: returns %Rrc (cs:rip=%04x:%RX64)\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestRIP(pVCpu)));
1420	return rc;
1421	}
1422	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
1423	if (RT_FAILURE(rc))
1424	return rc;
1425	/* Resume R0 */
1426	}
1427	}
1428
1429
1430	/**
1431	* VCPU worker for VMMSendStartupIpi.
1432	*
1433	* @param pVM The cross context VM structure.
1434	* @param idCpu Virtual CPU to perform SIPI on.
1435	* @param uVector The SIPI vector.
1436	*/
1437	static DECLCALLBACK(int) vmmR3SendStarupIpi(PVM pVM, VMCPUID idCpu, uint32_t uVector)
1438	{
1439	PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
1440	VMCPU_ASSERT_EMT(pVCpu);
1441
1442	/*
1443	* Active, halt and shutdown states of the processor all block SIPIs.
1444	* So we can safely discard the SIPI. See Intel spec. 26.6.2 "Activity State".
1445	*/
1446	if (EMGetState(pVCpu) != EMSTATE_WAIT_SIPI)
1447	return VERR_ACCESS_DENIED;
1448
1449
1450	PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
1451
1452	pCtx->cs.Sel = uVector << 8;
1453	pCtx->cs.ValidSel = uVector << 8;
1454	pCtx->cs.fFlags = CPUMSELREG_FLAGS_VALID;
1455	pCtx->cs.u64Base = uVector << 12;
1456	pCtx->cs.u32Limit = UINT32_C(0x0000ffff);
1457	pCtx->rip = 0;
1458
1459	Log(("vmmR3SendSipi for VCPU %d with vector %x\n", idCpu, uVector));
1460
1461	# if 1 /* If we keep the EMSTATE_WAIT_SIPI method, then move this to EM.cpp. */
1462	EMSetState(pVCpu, EMSTATE_HALTED);
1463	return VINF_EM_RESCHEDULE;
1464	# else /* And if we go the VMCPU::enmState way it can stay here. */
1465	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STOPPED);
1466	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
1467	return VINF_SUCCESS;
1468	# endif
1469	}
1470
1471
1472	static DECLCALLBACK(int) vmmR3SendInitIpi(PVM pVM, VMCPUID idCpu)
1473	{
1474	PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
1475	VMCPU_ASSERT_EMT(pVCpu);
1476
1477	Log(("vmmR3SendInitIpi for VCPU %d\n", idCpu));
1478
1479	PGMR3ResetCpu(pVM, pVCpu);
1480	PDMR3ResetCpu(pVCpu); /* Only clears pending interrupts force flags */
1481	#ifdef VBOX_WITH_NEW_APIC
1482	APICR3InitIpi(pVCpu);
1483	#endif
1484	TRPMR3ResetCpu(pVCpu);
1485	CPUMR3ResetCpu(pVM, pVCpu);
1486	EMR3ResetCpu(pVCpu);
1487	HMR3ResetCpu(pVCpu);
1488
1489	/* This will trickle up on the target EMT. */
1490	return VINF_EM_WAIT_SIPI;
1491	}
1492
1493
1494	/**
1495	* Sends a Startup IPI to the virtual CPU by setting CS:EIP into
1496	* vector-dependent state and unhalting processor.
1497	*
1498	* @param pVM The cross context VM structure.
1499	* @param idCpu Virtual CPU to perform SIPI on.
1500	* @param uVector SIPI vector.
1501	*/
1502	VMMR3_INT_DECL(void) VMMR3SendStartupIpi(PVM pVM, VMCPUID idCpu, uint32_t uVector)
1503	{
1504	AssertReturnVoid(idCpu < pVM->cCpus);
1505
1506	int rc = VMR3ReqCallNoWait(pVM, idCpu, (PFNRT)vmmR3SendStarupIpi, 3, pVM, idCpu, uVector);
1507	AssertRC(rc);
1508	}
1509
1510
1511	/**
1512	* Sends init IPI to the virtual CPU.
1513	*
1514	* @param pVM The cross context VM structure.
1515	* @param idCpu Virtual CPU to perform int IPI on.
1516	*/
1517	VMMR3_INT_DECL(void) VMMR3SendInitIpi(PVM pVM, VMCPUID idCpu)
1518	{
1519	AssertReturnVoid(idCpu < pVM->cCpus);
1520
1521	int rc = VMR3ReqCallNoWait(pVM, idCpu, (PFNRT)vmmR3SendInitIpi, 2, pVM, idCpu);
1522	AssertRC(rc);
1523	}
1524
1525
1526	/**
1527	* Registers the guest memory range that can be used for patching.
1528	*
1529	* @returns VBox status code.
1530	* @param pVM The cross context VM structure.
1531	* @param pPatchMem Patch memory range.
1532	* @param cbPatchMem Size of the memory range.
1533	*/
1534	VMMR3DECL(int) VMMR3RegisterPatchMemory(PVM pVM, RTGCPTR pPatchMem, unsigned cbPatchMem)
1535	{
1536	VM_ASSERT_EMT(pVM);
1537	if (HMIsEnabled(pVM))
1538	return HMR3EnablePatching(pVM, pPatchMem, cbPatchMem);
1539
1540	return VERR_NOT_SUPPORTED;
1541	}
1542
1543
1544	/**
1545	* Deregisters the guest memory range that can be used for patching.
1546	*
1547	* @returns VBox status code.
1548	* @param pVM The cross context VM structure.
1549	* @param pPatchMem Patch memory range.
1550	* @param cbPatchMem Size of the memory range.
1551	*/
1552	VMMR3DECL(int) VMMR3DeregisterPatchMemory(PVM pVM, RTGCPTR pPatchMem, unsigned cbPatchMem)
1553	{
1554	if (HMIsEnabled(pVM))
1555	return HMR3DisablePatching(pVM, pPatchMem, cbPatchMem);
1556
1557	return VINF_SUCCESS;
1558	}
1559
1560
1561	/**
1562	* Common recursion handler for the other EMTs.
1563	*
1564	* @returns Strict VBox status code.
1565	* @param pVM The cross context VM structure.
1566	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1567	* @param rcStrict Current status code to be combined with the one
1568	* from this recursion and returned.
1569	*/
1570	static VBOXSTRICTRC vmmR3EmtRendezvousCommonRecursion(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rcStrict)
1571	{
1572	int rc2;
1573
1574	/*
1575	* We wait here while the initiator of this recursion reconfigures
1576	* everything. The last EMT to get in signals the initiator.
1577	*/
1578	if (ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsRecursingPush) == pVM->cCpus)
1579	{
1580	rc2 = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousRecursionPushCaller);
1581	AssertLogRelRC(rc2);
1582	}
1583
1584	rc2 = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousRecursionPush, RT_INDEFINITE_WAIT);
1585	AssertLogRelRC(rc2);
1586
1587	/*
1588	* Do the normal rendezvous processing.
1589	*/
1590	VBOXSTRICTRC rcStrict2 = vmmR3EmtRendezvousCommon(pVM, pVCpu, false /* fIsCaller */, pVM->vmm.s.fRendezvousFlags,
1591	pVM->vmm.s.pfnRendezvous, pVM->vmm.s.pvRendezvousUser);
1592
1593	/*
1594	* Wait for the initiator to restore everything.
1595	*/
1596	rc2 = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousRecursionPop, RT_INDEFINITE_WAIT);
1597	AssertLogRelRC(rc2);
1598
1599	/*
1600	* Last thread out of here signals the initiator.
1601	*/
1602	if (ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsRecursingPop) == pVM->cCpus)
1603	{
1604	rc2 = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousRecursionPopCaller);
1605	AssertLogRelRC(rc2);
1606	}
1607
1608	/*
1609	* Merge status codes and return.
1610	*/
1611	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
1612	if ( rcStrict2 != VINF_SUCCESS
1613	&& ( rcStrict == VINF_SUCCESS
1614	\|\| rcStrict > rcStrict2))
1615	rcStrict = rcStrict2;
1616	return rcStrict;
1617	}
1618
1619
1620	/**
1621	* Count returns and have the last non-caller EMT wake up the caller.
1622	*
1623	* @returns VBox strict informational status code for EM scheduling. No failures
1624	* will be returned here, those are for the caller only.
1625	*
1626	* @param pVM The cross context VM structure.
1627	* @param rcStrict The current accumulated recursive status code,
1628	* to be merged with i32RendezvousStatus and
1629	* returned.
1630	*/
1631	DECL_FORCE_INLINE(VBOXSTRICTRC) vmmR3EmtRendezvousNonCallerReturn(PVM pVM, VBOXSTRICTRC rcStrict)
1632	{
1633	VBOXSTRICTRC rcStrict2 = ASMAtomicReadS32(&pVM->vmm.s.i32RendezvousStatus);
1634
1635	uint32_t cReturned = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsReturned);
1636	if (cReturned == pVM->cCpus - 1U)
1637	{
1638	int rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousDoneCaller);
1639	AssertLogRelRC(rc);
1640	}
1641
1642	/*
1643	* Merge the status codes, ignoring error statuses in this code path.
1644	*/
1645	AssertLogRelMsgReturn( rcStrict2 <= VINF_SUCCESS
1646	\|\| (rcStrict2 >= VINF_EM_FIRST && rcStrict2 <= VINF_EM_LAST),
1647	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)),
1648	VERR_IPE_UNEXPECTED_INFO_STATUS);
1649
1650	if (RT_SUCCESS(rcStrict2))
1651	{
1652	if ( rcStrict2 != VINF_SUCCESS
1653	&& ( rcStrict == VINF_SUCCESS
1654	\|\| rcStrict > rcStrict2))
1655	rcStrict = rcStrict2;
1656	}
1657	return rcStrict;
1658	}
1659
1660
1661	/**
1662	* Common worker for VMMR3EmtRendezvous and VMMR3EmtRendezvousFF.
1663	*
1664	* @returns VBox strict informational status code for EM scheduling. No failures
1665	* will be returned here, those are for the caller only. When
1666	* fIsCaller is set, VINF_SUCCESS is always returned.
1667	*
1668	* @param pVM The cross context VM structure.
1669	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1670	* @param fIsCaller Whether we're the VMMR3EmtRendezvous caller or
1671	* not.
1672	* @param fFlags The flags.
1673	* @param pfnRendezvous The callback.
1674	* @param pvUser The user argument for the callback.
1675	*/
1676	static VBOXSTRICTRC vmmR3EmtRendezvousCommon(PVM pVM, PVMCPU pVCpu, bool fIsCaller,
1677	uint32_t fFlags, PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser)
1678	{
1679	int rc;
1680	VBOXSTRICTRC rcStrictRecursion = VINF_SUCCESS;
1681
1682	/*
1683	* Enter, the last EMT triggers the next callback phase.
1684	*/
1685	uint32_t cEntered = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsEntered);
1686	if (cEntered != pVM->cCpus)
1687	{
1688	if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE)
1689	{
1690	/* Wait for our turn. */
1691	for (;;)
1692	{
1693	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousEnterOneByOne, RT_INDEFINITE_WAIT);
1694	AssertLogRelRC(rc);
1695	if (!pVM->vmm.s.fRendezvousRecursion)
1696	break;
1697	rcStrictRecursion = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrictRecursion);
1698	}
1699	}
1700	else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE)
1701	{
1702	/* Wait for the last EMT to arrive and wake everyone up. */
1703	rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce, RT_INDEFINITE_WAIT);
1704	AssertLogRelRC(rc);
1705	Assert(!pVM->vmm.s.fRendezvousRecursion);
1706	}
1707	else if ( (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
1708	\|\| (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
1709	{
1710	/* Wait for our turn. */
1711	for (;;)
1712	{
1713	rc = RTSemEventWait(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVCpu->idCpu], RT_INDEFINITE_WAIT);
1714	AssertLogRelRC(rc);
1715	if (!pVM->vmm.s.fRendezvousRecursion)
1716	break;
1717	rcStrictRecursion = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrictRecursion);
1718	}
1719	}
1720	else
1721	{
1722	Assert((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE);
1723
1724	/*
1725	* The execute once is handled specially to optimize the code flow.
1726	*
1727	* The last EMT to arrive will perform the callback and the other
1728	* EMTs will wait on the Done/DoneCaller semaphores (instead of
1729	* the EnterOneByOne/AllAtOnce) in the meanwhile. When the callback
1730	* returns, that EMT will initiate the normal return sequence.
1731	*/
1732	if (!fIsCaller)
1733	{
1734	for (;;)
1735	{
1736	rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousDone, RT_INDEFINITE_WAIT);
1737	AssertLogRelRC(rc);
1738	if (!pVM->vmm.s.fRendezvousRecursion)
1739	break;
1740	rcStrictRecursion = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrictRecursion);
1741	}
1742
1743	return vmmR3EmtRendezvousNonCallerReturn(pVM, rcStrictRecursion);
1744	}
1745	return VINF_SUCCESS;
1746	}
1747	}
1748	else
1749	{
1750	/*
1751	* All EMTs are waiting, clear the FF and take action according to the
1752	* execution method.
1753	*/
1754	VM_FF_CLEAR(pVM, VM_FF_EMT_RENDEZVOUS);
1755
1756	if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE)
1757	{
1758	/* Wake up everyone. */
1759	rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
1760	AssertLogRelRC(rc);
1761	}
1762	else if ( (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
1763	\|\| (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
1764	{
1765	/* Figure out who to wake up and wake it up. If it's ourself, then
1766	it's easy otherwise wait for our turn. */
1767	VMCPUID iFirst = (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
1768	? 0
1769	: pVM->cCpus - 1U;
1770	if (pVCpu->idCpu != iFirst)
1771	{
1772	rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[iFirst]);
1773	AssertLogRelRC(rc);
1774	for (;;)
1775	{
1776	rc = RTSemEventWait(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVCpu->idCpu], RT_INDEFINITE_WAIT);
1777	AssertLogRelRC(rc);
1778	if (!pVM->vmm.s.fRendezvousRecursion)
1779	break;
1780	rcStrictRecursion = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrictRecursion);
1781	}
1782	}
1783	}
1784	/* else: execute the handler on the current EMT and wake up one or more threads afterwards. */
1785	}
1786
1787
1788	/*
1789	* Do the callback and update the status if necessary.
1790	*/
1791	if ( !(fFlags & VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR)
1792	\|\| RT_SUCCESS(ASMAtomicUoReadS32(&pVM->vmm.s.i32RendezvousStatus)) )
1793	{
1794	VBOXSTRICTRC rcStrict2 = pfnRendezvous(pVM, pVCpu, pvUser);
1795	if (rcStrict2 != VINF_SUCCESS)
1796	{
1797	AssertLogRelMsg( rcStrict2 <= VINF_SUCCESS
1798	\|\| (rcStrict2 >= VINF_EM_FIRST && rcStrict2 <= VINF_EM_LAST),
1799	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
1800	int32_t i32RendezvousStatus;
1801	do
1802	{
1803	i32RendezvousStatus = ASMAtomicUoReadS32(&pVM->vmm.s.i32RendezvousStatus);
1804	if ( rcStrict2 == i32RendezvousStatus
1805	\|\| RT_FAILURE(i32RendezvousStatus)
1806	\|\| ( i32RendezvousStatus != VINF_SUCCESS
1807	&& rcStrict2 > i32RendezvousStatus))
1808	break;
1809	} while (!ASMAtomicCmpXchgS32(&pVM->vmm.s.i32RendezvousStatus, VBOXSTRICTRC_VAL(rcStrict2), i32RendezvousStatus));
1810	}
1811	}
1812
1813	/*
1814	* Increment the done counter and take action depending on whether we're
1815	* the last to finish callback execution.
1816	*/
1817	uint32_t cDone = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsDone);
1818	if ( cDone != pVM->cCpus
1819	&& (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE)
1820	{
1821	/* Signal the next EMT? */
1822	if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE)
1823	{
1824	rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousEnterOneByOne);
1825	AssertLogRelRC(rc);
1826	}
1827	else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING)
1828	{
1829	Assert(cDone == pVCpu->idCpu + 1U);
1830	rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVCpu->idCpu + 1U]);
1831	AssertLogRelRC(rc);
1832	}
1833	else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
1834	{
1835	Assert(pVM->cCpus - cDone == pVCpu->idCpu);
1836	rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVM->cCpus - cDone - 1U]);
1837	AssertLogRelRC(rc);
1838	}
1839
1840	/* Wait for the rest to finish (the caller waits on hEvtRendezvousDoneCaller). */
1841	if (!fIsCaller)
1842	{
1843	for (;;)
1844	{
1845	rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousDone, RT_INDEFINITE_WAIT);
1846	AssertLogRelRC(rc);
1847	if (!pVM->vmm.s.fRendezvousRecursion)
1848	break;
1849	rcStrictRecursion = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrictRecursion);
1850	}
1851	}
1852	}
1853	else
1854	{
1855	/* Callback execution is all done, tell the rest to return. */
1856	rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousDone);
1857	AssertLogRelRC(rc);
1858	}
1859
1860	if (!fIsCaller)
1861	return vmmR3EmtRendezvousNonCallerReturn(pVM, rcStrictRecursion);
1862	return rcStrictRecursion;
1863	}
1864
1865
1866	/**
1867	* Called in response to VM_FF_EMT_RENDEZVOUS.
1868	*
1869	* @returns VBox strict status code - EM scheduling. No errors will be returned
1870	* here, nor will any non-EM scheduling status codes be returned.
1871	*
1872	* @param pVM The cross context VM structure.
1873	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1874	*
1875	* @thread EMT
1876	*/
1877	VMMR3_INT_DECL(int) VMMR3EmtRendezvousFF(PVM pVM, PVMCPU pVCpu)
1878	{
1879	Assert(!pVCpu->vmm.s.fInRendezvous);
1880	Log(("VMMR3EmtRendezvousFF: EMT%#u\n", pVCpu->idCpu));
1881	pVCpu->vmm.s.fInRendezvous = true;
1882	VBOXSTRICTRC rcStrict = vmmR3EmtRendezvousCommon(pVM, pVCpu, false /* fIsCaller */, pVM->vmm.s.fRendezvousFlags,
1883	pVM->vmm.s.pfnRendezvous, pVM->vmm.s.pvRendezvousUser);
1884	pVCpu->vmm.s.fInRendezvous = false;
1885	Log(("VMMR3EmtRendezvousFF: EMT%#u returns %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict)));
1886	return VBOXSTRICTRC_TODO(rcStrict);
1887	}
1888
1889
1890	/**
1891	* Helper for resetting an single wakeup event sempahore.
1892	*
1893	* @returns VERR_TIMEOUT on success, RTSemEventWait status otherwise.
1894	* @param hEvt The event semaphore to reset.
1895	*/
1896	static int vmmR3HlpResetEvent(RTSEMEVENT hEvt)
1897	{
1898	for (uint32_t cLoops = 0; ; cLoops++)
1899	{
1900	int rc = RTSemEventWait(hEvt, 0 /cMsTimeout/);
1901	if (rc != VINF_SUCCESS \|\| cLoops > _4K)
1902	return rc;
1903	}
1904	}
1905
1906
1907	/**
1908	* Worker for VMMR3EmtRendezvous that handles recursion.
1909	*
1910	* @returns VBox strict status code. This will be the first error,
1911	* VINF_SUCCESS, or an EM scheduling status code.
1912	*
1913	* @param pVM The cross context VM structure.
1914	* @param pVCpu The cross context virtual CPU structure of the
1915	* calling EMT.
1916	* @param fFlags Flags indicating execution methods. See
1917	* grp_VMMR3EmtRendezvous_fFlags.
1918	* @param pfnRendezvous The callback.
1919	* @param pvUser User argument for the callback.
1920	*
1921	* @thread EMT(pVCpu)
1922	*/
1923	static VBOXSTRICTRC vmmR3EmtRendezvousRecursive(PVM pVM, PVMCPU pVCpu, uint32_t fFlags,
1924	PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser)
1925	{
1926	Log(("vmmR3EmtRendezvousRecursive: %#x EMT#%u depth=%d\n", fFlags, pVCpu->idCpu, pVM->vmm.s.cRendezvousRecursions));
1927	AssertLogRelReturn(pVM->vmm.s.cRendezvousRecursions < 3, VERR_DEADLOCK);
1928	Assert(pVCpu->vmm.s.fInRendezvous);
1929
1930	/*
1931	* Save the current state.
1932	*/
1933	uint32_t const fParentFlags = pVM->vmm.s.fRendezvousFlags;
1934	uint32_t const cParentDone = pVM->vmm.s.cRendezvousEmtsDone;
1935	int32_t const iParentStatus = pVM->vmm.s.i32RendezvousStatus;
1936	PFNVMMEMTRENDEZVOUS const pfnParent = pVM->vmm.s.pfnRendezvous;
1937	void * const pvParentUser = pVM->vmm.s.pvRendezvousUser;
1938
1939	/*
1940	* Check preconditions and save the current state.
1941	*/
1942	AssertReturn( (fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
1943	\|\| (fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING
1944	\|\| (fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE
1945	\|\| (fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE,
1946	VERR_INTERNAL_ERROR);
1947	AssertReturn(pVM->vmm.s.cRendezvousEmtsEntered == pVM->cCpus, VERR_INTERNAL_ERROR_2);
1948	AssertReturn(pVM->vmm.s.cRendezvousEmtsReturned == 0, VERR_INTERNAL_ERROR_3);
1949
1950	/*
1951	* Reset the recursion prep and pop semaphores.
1952	*/
1953	int rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousRecursionPush);
1954	AssertLogRelRCReturn(rc, rc);
1955	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousRecursionPop);
1956	AssertLogRelRCReturn(rc, rc);
1957	rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousRecursionPushCaller);
1958	AssertLogRelMsgReturn(rc == VERR_TIMEOUT, ("%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
1959	rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousRecursionPopCaller);
1960	AssertLogRelMsgReturn(rc == VERR_TIMEOUT, ("%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
1961
1962	/*
1963	* Usher the other thread into the recursion routine.
1964	*/
1965	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsRecursingPush, 0);
1966	ASMAtomicWriteBool(&pVM->vmm.s.fRendezvousRecursion, true);
1967
1968	uint32_t cLeft = pVM->cCpus - (cParentDone + 1U);
1969	if ((fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE)
1970	while (cLeft-- > 0)
1971	{
1972	rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousEnterOneByOne);
1973	AssertLogRelRC(rc);
1974	}
1975	else if ((fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING)
1976	{
1977	Assert(cLeft == pVM->cCpus - (pVCpu->idCpu + 1U));
1978	for (VMCPUID iCpu = pVCpu->idCpu + 1U; iCpu < pVM->cCpus; iCpu++)
1979	{
1980	rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[iCpu]);
1981	AssertLogRelRC(rc);
1982	}
1983	}
1984	else if ((fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
1985	{
1986	Assert(cLeft == pVCpu->idCpu);
1987	for (VMCPUID iCpu = pVCpu->idCpu; iCpu > 0; iCpu--)
1988	{
1989	rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[iCpu - 1U]);
1990	AssertLogRelRC(rc);
1991	}
1992	}
1993	else
1994	AssertLogRelReturn((fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE,
1995	VERR_INTERNAL_ERROR_4);
1996
1997	rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousDone);
1998	AssertLogRelRC(rc);
1999	rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousDoneCaller);
2000	AssertLogRelRC(rc);
2001
2002
2003	/*
2004	* Wait for the EMTs to wake up and get out of the parent rendezvous code.
2005	*/
2006	if (ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsRecursingPush) != pVM->cCpus)
2007	{
2008	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousRecursionPushCaller, RT_INDEFINITE_WAIT);
2009	AssertLogRelRC(rc);
2010	}
2011
2012	ASMAtomicWriteBool(&pVM->vmm.s.fRendezvousRecursion, false);
2013
2014	/*
2015	* Clear the slate and setup the new rendezvous.
2016	*/
2017	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2018	{
2019	rc = vmmR3HlpResetEvent(pVM->vmm.s.pahEvtRendezvousEnterOrdered[i]);
2020	AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
2021	}
2022	rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousEnterOneByOne); AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
2023	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce); AssertLogRelRC(rc);
2024	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousDone); AssertLogRelRC(rc);
2025	rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousDoneCaller); AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
2026
2027	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsEntered, 0);
2028	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsDone, 0);
2029	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsReturned, 0);
2030	ASMAtomicWriteS32(&pVM->vmm.s.i32RendezvousStatus, VINF_SUCCESS);
2031	ASMAtomicWritePtr((void * volatile )&pVM->vmm.s.pfnRendezvous, (void )(uintptr_t)pfnRendezvous);
2032	ASMAtomicWritePtr(&pVM->vmm.s.pvRendezvousUser, pvUser);
2033	ASMAtomicWriteU32(&pVM->vmm.s.fRendezvousFlags, fFlags);
2034	ASMAtomicIncU32(&pVM->vmm.s.cRendezvousRecursions);
2035
2036	/*
2037	* We're ready to go now, do normal rendezvous processing.
2038	*/
2039	rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousRecursionPush);
2040	AssertLogRelRC(rc);
2041
2042	VBOXSTRICTRC rcStrict = vmmR3EmtRendezvousCommon(pVM, pVCpu, true /fIsCaller/, fFlags, pfnRendezvous, pvUser);
2043
2044	/*
2045	* The caller waits for the other EMTs to be done, return and waiting on the
2046	* pop semaphore.
2047	*/
2048	for (;;)
2049	{
2050	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousDoneCaller, RT_INDEFINITE_WAIT);
2051	AssertLogRelRC(rc);
2052	if (!pVM->vmm.s.fRendezvousRecursion)
2053	break;
2054	rcStrict = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrict);
2055	}
2056
2057	/*
2058	* Get the return code and merge it with the above recursion status.
2059	*/
2060	VBOXSTRICTRC rcStrict2 = pVM->vmm.s.i32RendezvousStatus;
2061	if ( rcStrict2 != VINF_SUCCESS
2062	&& ( rcStrict == VINF_SUCCESS
2063	\|\| rcStrict > rcStrict2))
2064	rcStrict = rcStrict2;
2065
2066	/*
2067	* Restore the parent rendezvous state.
2068	*/
2069	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2070	{
2071	rc = vmmR3HlpResetEvent(pVM->vmm.s.pahEvtRendezvousEnterOrdered[i]);
2072	AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
2073	}
2074	rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousEnterOneByOne); AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
2075	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce); AssertLogRelRC(rc);
2076	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousDone); AssertLogRelRC(rc);
2077	rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousDoneCaller); AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
2078
2079	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsEntered, pVM->cCpus);
2080	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsReturned, 0);
2081	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsDone, cParentDone);
2082	ASMAtomicWriteS32(&pVM->vmm.s.i32RendezvousStatus, iParentStatus);
2083	ASMAtomicWriteU32(&pVM->vmm.s.fRendezvousFlags, fParentFlags);
2084	ASMAtomicWritePtr(&pVM->vmm.s.pvRendezvousUser, pvParentUser);
2085	ASMAtomicWritePtr((void * volatile )&pVM->vmm.s.pfnRendezvous, (void )(uintptr_t)pfnParent);
2086
2087	/*
2088	* Usher the other EMTs back to their parent recursion routine, waiting
2089	* for them to all get there before we return (makes sure they've been
2090	* scheduled and are past the pop event sem, see below).
2091	*/
2092	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsRecursingPop, 0);
2093	rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousRecursionPop);
2094	AssertLogRelRC(rc);
2095
2096	if (ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsRecursingPop) != pVM->cCpus)
2097	{
2098	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousRecursionPopCaller, RT_INDEFINITE_WAIT);
2099	AssertLogRelRC(rc);
2100	}
2101
2102	/*
2103	* We must reset the pop semaphore on the way out (doing the pop caller too,
2104	* just in case). The parent may be another recursion.
2105	*/
2106	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousRecursionPop); AssertLogRelRC(rc);
2107	rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousRecursionPopCaller); AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
2108
2109	ASMAtomicDecU32(&pVM->vmm.s.cRendezvousRecursions);
2110
2111	Log(("vmmR3EmtRendezvousRecursive: %#x EMT#%u depth=%d returns %Rrc\n",
2112	fFlags, pVCpu->idCpu, pVM->vmm.s.cRendezvousRecursions, VBOXSTRICTRC_VAL(rcStrict)));
2113	return rcStrict;
2114	}
2115
2116
2117	/**
2118	* EMT rendezvous.
2119	*
2120	* Gathers all the EMTs and execute some code on each of them, either in a one
2121	* by one fashion or all at once.
2122	*
2123	* @returns VBox strict status code. This will be the first error,
2124	* VINF_SUCCESS, or an EM scheduling status code.
2125	*
2126	* @retval VERR_DEADLOCK if recursion is attempted using a rendezvous type that
2127	* doesn't support it or if the recursion is too deep.
2128	*
2129	* @param pVM The cross context VM structure.
2130	* @param fFlags Flags indicating execution methods. See
2131	* grp_VMMR3EmtRendezvous_fFlags. The one-by-one,
2132	* descending and ascending rendezvous types support
2133	* recursion from inside @a pfnRendezvous.
2134	* @param pfnRendezvous The callback.
2135	* @param pvUser User argument for the callback.
2136	*
2137	* @thread Any.
2138	*/
2139	VMMR3DECL(int) VMMR3EmtRendezvous(PVM pVM, uint32_t fFlags, PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser)
2140	{
2141	/*
2142	* Validate input.
2143	*/
2144	AssertReturn(pVM, VERR_INVALID_VM_HANDLE);
2145	AssertMsg( (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_INVALID
2146	&& (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) <= VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING
2147	&& !(fFlags & ~VMMEMTRENDEZVOUS_FLAGS_VALID_MASK), ("%#x\n", fFlags));
2148	AssertMsg( !(fFlags & VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR)
2149	\|\| ( (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE
2150	&& (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE),
2151	("type %u\n", fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK));
2152
2153	VBOXSTRICTRC rcStrict;
2154	PVMCPU pVCpu = VMMGetCpu(pVM);
2155	if (!pVCpu)
2156	/*
2157	* Forward the request to an EMT thread.
2158	*/
2159	{
2160	Log(("VMMR3EmtRendezvous: %#x non-EMT\n", fFlags));
2161	if (!(fFlags & VMMEMTRENDEZVOUS_FLAGS_PRIORITY))
2162	rcStrict = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)VMMR3EmtRendezvous, 4, pVM, fFlags, pfnRendezvous, pvUser);
2163	else
2164	rcStrict = VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)VMMR3EmtRendezvous, 4, pVM, fFlags, pfnRendezvous, pvUser);
2165	Log(("VMMR3EmtRendezvous: %#x non-EMT returns %Rrc\n", fFlags, VBOXSTRICTRC_VAL(rcStrict)));
2166	}
2167	else if (pVM->cCpus == 1)
2168	{
2169	/*
2170	* Shortcut for the single EMT case.
2171	*/
2172	if (!pVCpu->vmm.s.fInRendezvous)
2173	{
2174	Log(("VMMR3EmtRendezvous: %#x EMT (uni)\n", fFlags));
2175	pVCpu->vmm.s.fInRendezvous = true;
2176	pVM->vmm.s.fRendezvousFlags = fFlags;
2177	rcStrict = pfnRendezvous(pVM, pVCpu, pvUser);
2178	pVCpu->vmm.s.fInRendezvous = false;
2179	}
2180	else
2181	{
2182	/* Recursion. Do the same checks as in the SMP case. */
2183	Log(("VMMR3EmtRendezvous: %#x EMT (uni), recursion depth=%d\n", fFlags, pVM->vmm.s.cRendezvousRecursions));
2184	uint32_t fType = pVM->vmm.s.fRendezvousFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK;
2185	AssertLogRelReturn( !pVCpu->vmm.s.fInRendezvous
2186	\|\| fType == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
2187	\|\| fType == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING
2188	\|\| fType == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE
2189	\|\| fType == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE
2190	, VERR_DEADLOCK);
2191
2192	AssertLogRelReturn(pVM->vmm.s.cRendezvousRecursions < 3, VERR_DEADLOCK);
2193	pVM->vmm.s.cRendezvousRecursions++;
2194	uint32_t const fParentFlags = pVM->vmm.s.fRendezvousFlags;
2195	pVM->vmm.s.fRendezvousFlags = fFlags;
2196
2197	rcStrict = pfnRendezvous(pVM, pVCpu, pvUser);
2198
2199	pVM->vmm.s.fRendezvousFlags = fParentFlags;
2200	pVM->vmm.s.cRendezvousRecursions--;
2201	}
2202	Log(("VMMR3EmtRendezvous: %#x EMT (uni) returns %Rrc\n", fFlags, VBOXSTRICTRC_VAL(rcStrict)));
2203	}
2204	else
2205	{
2206	/*
2207	* Spin lock. If busy, check for recursion, if not recursing wait for
2208	* the other EMT to finish while keeping a lookout for the RENDEZVOUS FF.
2209	*/
2210	int rc;
2211	rcStrict = VINF_SUCCESS;
2212	if (RT_UNLIKELY(!ASMAtomicCmpXchgU32(&pVM->vmm.s.u32RendezvousLock, 0x77778888, 0)))
2213	{
2214	/* Allow recursion in some cases. */
2215	if ( pVCpu->vmm.s.fInRendezvous
2216	&& ( (pVM->vmm.s.fRendezvousFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
2217	\|\| (pVM->vmm.s.fRendezvousFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING
2218	\|\| (pVM->vmm.s.fRendezvousFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE
2219	\|\| (pVM->vmm.s.fRendezvousFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE
2220	))
2221	return VBOXSTRICTRC_TODO(vmmR3EmtRendezvousRecursive(pVM, pVCpu, fFlags, pfnRendezvous, pvUser));
2222
2223	AssertLogRelMsgReturn(!pVCpu->vmm.s.fInRendezvous, ("fRendezvousFlags=%#x\n", pVM->vmm.s.fRendezvousFlags),
2224	VERR_DEADLOCK);
2225
2226	Log(("VMMR3EmtRendezvous: %#x EMT#%u, waiting for lock...\n", fFlags, pVCpu->idCpu));
2227	while (!ASMAtomicCmpXchgU32(&pVM->vmm.s.u32RendezvousLock, 0x77778888, 0))
2228	{
2229	if (VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS))
2230	{
2231	rc = VMMR3EmtRendezvousFF(pVM, pVCpu);
2232	if ( rc != VINF_SUCCESS
2233	&& ( rcStrict == VINF_SUCCESS
2234	\|\| rcStrict > rc))
2235	rcStrict = rc;
2236	/** @todo Perhaps deal with termination here? */
2237	}
2238	ASMNopPause();
2239	}
2240	}
2241
2242	Log(("VMMR3EmtRendezvous: %#x EMT#%u\n", fFlags, pVCpu->idCpu));
2243	Assert(!VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS));
2244	Assert(!pVCpu->vmm.s.fInRendezvous);
2245	pVCpu->vmm.s.fInRendezvous = true;
2246
2247	/*
2248	* Clear the slate and setup the rendezvous. This is a semaphore ping-pong orgy. :-)
2249	*/
2250	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2251	{
2252	rc = RTSemEventWait(pVM->vmm.s.pahEvtRendezvousEnterOrdered[i], 0);
2253	AssertLogRelMsg(rc == VERR_TIMEOUT \|\| rc == VINF_SUCCESS, ("%Rrc\n", rc));
2254	}
2255	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousEnterOneByOne, 0); AssertLogRelMsg(rc == VERR_TIMEOUT \|\| rc == VINF_SUCCESS, ("%Rrc\n", rc));
2256	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce); AssertLogRelRC(rc);
2257	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousDone); AssertLogRelRC(rc);
2258	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousDoneCaller, 0); AssertLogRelMsg(rc == VERR_TIMEOUT \|\| rc == VINF_SUCCESS, ("%Rrc\n", rc));
2259	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsEntered, 0);
2260	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsDone, 0);
2261	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsReturned, 0);
2262	ASMAtomicWriteS32(&pVM->vmm.s.i32RendezvousStatus, VINF_SUCCESS);
2263	ASMAtomicWritePtr((void * volatile )&pVM->vmm.s.pfnRendezvous, (void )(uintptr_t)pfnRendezvous);
2264	ASMAtomicWritePtr(&pVM->vmm.s.pvRendezvousUser, pvUser);
2265	ASMAtomicWriteU32(&pVM->vmm.s.fRendezvousFlags, fFlags);
2266
2267	/*
2268	* Set the FF and poke the other EMTs.
2269	*/
2270	VM_FF_SET(pVM, VM_FF_EMT_RENDEZVOUS);
2271	VMR3NotifyGlobalFFU(pVM->pUVM, VMNOTIFYFF_FLAGS_POKE);
2272
2273	/*
2274	* Do the same ourselves.
2275	*/
2276	VBOXSTRICTRC rcStrict2 = vmmR3EmtRendezvousCommon(pVM, pVCpu, true /* fIsCaller */, fFlags, pfnRendezvous, pvUser);
2277
2278	/*
2279	* The caller waits for the other EMTs to be done and return before doing
2280	* the cleanup. This makes away with wakeup / reset races we would otherwise
2281	* risk in the multiple release event semaphore code (hEvtRendezvousDoneCaller).
2282	*/
2283	for (;;)
2284	{
2285	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousDoneCaller, RT_INDEFINITE_WAIT);
2286	AssertLogRelRC(rc);
2287	if (!pVM->vmm.s.fRendezvousRecursion)
2288	break;
2289	rcStrict2 = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrict2);
2290	}
2291
2292	/*
2293	* Get the return code and clean up a little bit.
2294	*/
2295	VBOXSTRICTRC rcStrict3 = pVM->vmm.s.i32RendezvousStatus;
2296	ASMAtomicWriteNullPtr((void * volatile *)&pVM->vmm.s.pfnRendezvous);
2297
2298	ASMAtomicWriteU32(&pVM->vmm.s.u32RendezvousLock, 0);
2299	pVCpu->vmm.s.fInRendezvous = false;
2300
2301	/*
2302	* Merge rcStrict, rcStrict2 and rcStrict3.
2303	*/
2304	AssertRC(VBOXSTRICTRC_VAL(rcStrict));
2305	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
2306	if ( rcStrict2 != VINF_SUCCESS
2307	&& ( rcStrict == VINF_SUCCESS
2308	\|\| rcStrict > rcStrict2))
2309	rcStrict = rcStrict2;
2310	if ( rcStrict3 != VINF_SUCCESS
2311	&& ( rcStrict == VINF_SUCCESS
2312	\|\| rcStrict > rcStrict3))
2313	rcStrict = rcStrict3;
2314	Log(("VMMR3EmtRendezvous: %#x EMT#%u returns %Rrc\n", fFlags, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict)));
2315	}
2316
2317	AssertLogRelMsgReturn( rcStrict <= VINF_SUCCESS
2318	\|\| (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST),
2319	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)),
2320	VERR_IPE_UNEXPECTED_INFO_STATUS);
2321	return VBOXSTRICTRC_VAL(rcStrict);
2322	}
2323
2324
2325	/**
2326	* Read from the ring 0 jump buffer stack
2327	*
2328	* @returns VBox status code.
2329	*
2330	* @param pVM The cross context VM structure.
2331	* @param idCpu The ID of the source CPU context (for the address).
2332	* @param R0Addr Where to start reading.
2333	* @param pvBuf Where to store the data we've read.
2334	* @param cbRead The number of bytes to read.
2335	*/
2336	VMMR3_INT_DECL(int) VMMR3ReadR0Stack(PVM pVM, VMCPUID idCpu, RTHCUINTPTR R0Addr, void *pvBuf, size_t cbRead)
2337	{
2338	PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
2339	AssertReturn(pVCpu, VERR_INVALID_PARAMETER);
2340
2341	#ifdef VMM_R0_SWITCH_STACK
2342	RTHCUINTPTR off = R0Addr - MMHyperCCToR0(pVM, pVCpu->vmm.s.pbEMTStackR3);
2343	#else
2344	RTHCUINTPTR off = pVCpu->vmm.s.CallRing3JmpBufR0.cbSavedStack - (pVCpu->vmm.s.CallRing3JmpBufR0.SpCheck - R0Addr);
2345	#endif
2346	if ( off > VMM_STACK_SIZE
2347	\|\| off + cbRead >= VMM_STACK_SIZE)
2348	return VERR_INVALID_POINTER;
2349
2350	memcpy(pvBuf, &pVCpu->vmm.s.pbEMTStackR3[off], cbRead);
2351	return VINF_SUCCESS;
2352	}
2353
2354	#ifdef VBOX_WITH_RAW_MODE
2355
2356	/**
2357	* Calls a RC function.
2358	*
2359	* @param pVM The cross context VM structure.
2360	* @param RCPtrEntry The address of the RC function.
2361	* @param cArgs The number of arguments in the ....
2362	* @param ... Arguments to the function.
2363	*/
2364	VMMR3DECL(int) VMMR3CallRC(PVM pVM, RTRCPTR RCPtrEntry, unsigned cArgs, ...)
2365	{
2366	va_list args;
2367	va_start(args, cArgs);
2368	int rc = VMMR3CallRCV(pVM, RCPtrEntry, cArgs, args);
2369	va_end(args);
2370	return rc;
2371	}
2372
2373
2374	/**
2375	* Calls a RC function.
2376	*
2377	* @param pVM The cross context VM structure.
2378	* @param RCPtrEntry The address of the RC function.
2379	* @param cArgs The number of arguments in the ....
2380	* @param args Arguments to the function.
2381	*/
2382	VMMR3DECL(int) VMMR3CallRCV(PVM pVM, RTRCPTR RCPtrEntry, unsigned cArgs, va_list args)
2383	{
2384	/* Raw mode implies 1 VCPU. */
2385	AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
2386	PVMCPU pVCpu = &pVM->aCpus[0];
2387
2388	Log2(("VMMR3CallGCV: RCPtrEntry=%RRv cArgs=%d\n", RCPtrEntry, cArgs));
2389
2390	/*
2391	* Setup the call frame using the trampoline.
2392	*/
2393	CPUMSetHyperState(pVCpu,
2394	pVM->vmm.s.pfnCallTrampolineRC, /* eip */
2395	pVCpu->vmm.s.pbEMTStackBottomRC - cArgs * sizeof(RTGCUINTPTR32), /* esp */
2396	RCPtrEntry, /* eax */
2397	cArgs /* edx */
2398	);
2399
2400	#if 0
2401	memset(pVCpu->vmm.s.pbEMTStackR3, 0xaa, VMM_STACK_SIZE); /* Clear the stack. */
2402	#endif
2403	PRTGCUINTPTR32 pFrame = (PRTGCUINTPTR32)(pVCpu->vmm.s.pbEMTStackR3 + VMM_STACK_SIZE) - cArgs;
2404	int i = cArgs;
2405	while (i-- > 0)
2406	*pFrame++ = va_arg(args, RTGCUINTPTR32);
2407
2408	CPUMPushHyper(pVCpu, cArgs * sizeof(RTGCUINTPTR32)); /* stack frame size */
2409	CPUMPushHyper(pVCpu, RCPtrEntry); /* what to call */
2410
2411	/*
2412	* We hide log flushes (outer) and hypervisor interrupts (inner).
2413	*/
2414	for (;;)
2415	{
2416	int rc;
2417	Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
2418	do
2419	{
2420	#ifdef NO_SUPCALLR0VMM
2421	rc = VERR_GENERAL_FAILURE;
2422	#else
2423	rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
2424	if (RT_LIKELY(rc == VINF_SUCCESS))
2425	rc = pVCpu->vmm.s.iLastGZRc;
2426	#endif
2427	} while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
2428
2429	/*
2430	* Flush the loggers.
2431	*/
2432	#ifdef LOG_ENABLED
2433	PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
2434	if ( pLogger
2435	&& pLogger->offScratch > 0)
2436	RTLogFlushRC(NULL, pLogger);
2437	#endif
2438	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
2439	PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
2440	if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
2441	RTLogFlushRC(RTLogRelGetDefaultInstance(), pRelLogger);
2442	#endif
2443	if (rc == VERR_TRPM_PANIC \|\| rc == VERR_TRPM_DONT_PANIC)
2444	VMMR3FatalDump(pVM, pVCpu, rc);
2445	if (rc != VINF_VMM_CALL_HOST)
2446	{
2447	Log2(("VMMR3CallGCV: returns %Rrc (cs:eip=%04x:%08x)\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
2448	return rc;
2449	}
2450	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
2451	if (RT_FAILURE(rc))
2452	return rc;
2453	}
2454	}
2455
2456	#endif /* VBOX_WITH_RAW_MODE */
2457
2458	/**
2459	* Wrapper for SUPR3CallVMMR0Ex which will deal with VINF_VMM_CALL_HOST returns.
2460	*
2461	* @returns VBox status code.
2462	* @param pVM The cross context VM structure.
2463	* @param uOperation Operation to execute.
2464	* @param u64Arg Constant argument.
2465	* @param pReqHdr Pointer to a request header. See SUPR3CallVMMR0Ex for
2466	* details.
2467	*/
2468	VMMR3DECL(int) VMMR3CallR0(PVM pVM, uint32_t uOperation, uint64_t u64Arg, PSUPVMMR0REQHDR pReqHdr)
2469	{
2470	PVMCPU pVCpu = VMMGetCpu(pVM);
2471	AssertReturn(pVCpu, VERR_VM_THREAD_NOT_EMT);
2472
2473	/*
2474	* Call Ring-0 entry with init code.
2475	*/
2476	int rc;
2477	for (;;)
2478	{
2479	#ifdef NO_SUPCALLR0VMM
2480	rc = VERR_GENERAL_FAILURE;
2481	#else
2482	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, pVCpu->idCpu, uOperation, u64Arg, pReqHdr);
2483	#endif
2484	/*
2485	* Flush the logs.
2486	*/
2487	#ifdef LOG_ENABLED
2488	if ( pVCpu->vmm.s.pR0LoggerR3
2489	&& pVCpu->vmm.s.pR0LoggerR3->Logger.offScratch > 0)
2490	RTLogFlushR0(NULL, &pVCpu->vmm.s.pR0LoggerR3->Logger);
2491	#endif
2492	if (rc != VINF_VMM_CALL_HOST)
2493	break;
2494	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
2495	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
2496	break;
2497	/* Resume R0 */
2498	}
2499
2500	AssertLogRelMsgReturn(rc == VINF_SUCCESS \|\| RT_FAILURE(rc),
2501	("uOperation=%u rc=%Rrc\n", uOperation, rc),
2502	VERR_IPE_UNEXPECTED_INFO_STATUS);
2503	return rc;
2504	}
2505
2506
2507	#ifdef VBOX_WITH_RAW_MODE
2508	/**
2509	* Resumes executing hypervisor code when interrupted by a queue flush or a
2510	* debug event.
2511	*
2512	* @returns VBox status code.
2513	* @param pVM The cross context VM structure.
2514	* @param pVCpu The cross context virtual CPU structure.
2515	*/
2516	VMMR3DECL(int) VMMR3ResumeHyper(PVM pVM, PVMCPU pVCpu)
2517	{
2518	Log(("VMMR3ResumeHyper: eip=%RRv esp=%RRv\n", CPUMGetHyperEIP(pVCpu), CPUMGetHyperESP(pVCpu)));
2519	AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
2520
2521	/*
2522	* We hide log flushes (outer) and hypervisor interrupts (inner).
2523	*/
2524	for (;;)
2525	{
2526	int rc;
2527	Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
2528	do
2529	{
2530	# ifdef NO_SUPCALLR0VMM
2531	rc = VERR_GENERAL_FAILURE;
2532	# else
2533	rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
2534	if (RT_LIKELY(rc == VINF_SUCCESS))
2535	rc = pVCpu->vmm.s.iLastGZRc;
2536	# endif
2537	} while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
2538
2539	/*
2540	* Flush the loggers.
2541	*/
2542	# ifdef LOG_ENABLED
2543	PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
2544	if ( pLogger
2545	&& pLogger->offScratch > 0)
2546	RTLogFlushRC(NULL, pLogger);
2547	# endif
2548	# ifdef VBOX_WITH_RC_RELEASE_LOGGING
2549	PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
2550	if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
2551	RTLogFlushRC(RTLogRelGetDefaultInstance(), pRelLogger);
2552	# endif
2553	if (rc == VERR_TRPM_PANIC \|\| rc == VERR_TRPM_DONT_PANIC)
2554	VMMR3FatalDump(pVM, pVCpu, rc);
2555	if (rc != VINF_VMM_CALL_HOST)
2556	{
2557	Log(("VMMR3ResumeHyper: returns %Rrc\n", rc));
2558	return rc;
2559	}
2560	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
2561	if (RT_FAILURE(rc))
2562	return rc;
2563	}
2564	}
2565	#endif /* VBOX_WITH_RAW_MODE */
2566
2567
2568	/**
2569	* Service a call to the ring-3 host code.
2570	*
2571	* @returns VBox status code.
2572	* @param pVM The cross context VM structure.
2573	* @param pVCpu The cross context virtual CPU structure.
2574	* @remarks Careful with critsects.
2575	*/
2576	static int vmmR3ServiceCallRing3Request(PVM pVM, PVMCPU pVCpu)
2577	{
2578	/*
2579	* We must also check for pending critsect exits or else we can deadlock
2580	* when entering other critsects here.
2581	*/
2582	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PDM_CRITSECT))
2583	PDMCritSectBothFF(pVCpu);
2584
2585	switch (pVCpu->vmm.s.enmCallRing3Operation)
2586	{
2587	/*
2588	* Acquire a critical section.
2589	*/
2590	case VMMCALLRING3_PDM_CRIT_SECT_ENTER:
2591	{
2592	pVCpu->vmm.s.rcCallRing3 = PDMR3CritSectEnterEx((PPDMCRITSECT)(uintptr_t)pVCpu->vmm.s.u64CallRing3Arg,
2593	true /fCallRing3/);
2594	break;
2595	}
2596
2597	/*
2598	* Enter a r/w critical section exclusively.
2599	*/
2600	case VMMCALLRING3_PDM_CRIT_SECT_RW_ENTER_EXCL:
2601	{
2602	pVCpu->vmm.s.rcCallRing3 = PDMR3CritSectRwEnterExclEx((PPDMCRITSECTRW)(uintptr_t)pVCpu->vmm.s.u64CallRing3Arg,
2603	true /fCallRing3/);
2604	break;
2605	}
2606
2607	/*
2608	* Enter a r/w critical section shared.
2609	*/
2610	case VMMCALLRING3_PDM_CRIT_SECT_RW_ENTER_SHARED:
2611	{
2612	pVCpu->vmm.s.rcCallRing3 = PDMR3CritSectRwEnterSharedEx((PPDMCRITSECTRW)(uintptr_t)pVCpu->vmm.s.u64CallRing3Arg,
2613	true /fCallRing3/);
2614	break;
2615	}
2616
2617	/*
2618	* Acquire the PDM lock.
2619	*/
2620	case VMMCALLRING3_PDM_LOCK:
2621	{
2622	pVCpu->vmm.s.rcCallRing3 = PDMR3LockCall(pVM);
2623	break;
2624	}
2625
2626	/*
2627	* Grow the PGM pool.
2628	*/
2629	case VMMCALLRING3_PGM_POOL_GROW:
2630	{
2631	pVCpu->vmm.s.rcCallRing3 = PGMR3PoolGrow(pVM);
2632	break;
2633	}
2634
2635	/*
2636	* Maps an page allocation chunk into ring-3 so ring-0 can use it.
2637	*/
2638	case VMMCALLRING3_PGM_MAP_CHUNK:
2639	{
2640	pVCpu->vmm.s.rcCallRing3 = PGMR3PhysChunkMap(pVM, pVCpu->vmm.s.u64CallRing3Arg);
2641	break;
2642	}
2643
2644	/*
2645	* Allocates more handy pages.
2646	*/
2647	case VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES:
2648	{
2649	pVCpu->vmm.s.rcCallRing3 = PGMR3PhysAllocateHandyPages(pVM);
2650	break;
2651	}
2652
2653	/*
2654	* Allocates a large page.
2655	*/
2656	case VMMCALLRING3_PGM_ALLOCATE_LARGE_HANDY_PAGE:
2657	{
2658	pVCpu->vmm.s.rcCallRing3 = PGMR3PhysAllocateLargeHandyPage(pVM, pVCpu->vmm.s.u64CallRing3Arg);
2659	break;
2660	}
2661
2662	/*
2663	* Acquire the PGM lock.
2664	*/
2665	case VMMCALLRING3_PGM_LOCK:
2666	{
2667	pVCpu->vmm.s.rcCallRing3 = PGMR3LockCall(pVM);
2668	break;
2669	}
2670
2671	/*
2672	* Acquire the MM hypervisor heap lock.
2673	*/
2674	case VMMCALLRING3_MMHYPER_LOCK:
2675	{
2676	pVCpu->vmm.s.rcCallRing3 = MMR3LockCall(pVM);
2677	break;
2678	}
2679
2680	#ifdef VBOX_WITH_REM
2681	/*
2682	* Flush REM handler notifications.
2683	*/
2684	case VMMCALLRING3_REM_REPLAY_HANDLER_NOTIFICATIONS:
2685	{
2686	REMR3ReplayHandlerNotifications(pVM);
2687	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
2688	break;
2689	}
2690	#endif
2691
2692	/*
2693	* This is a noop. We just take this route to avoid unnecessary
2694	* tests in the loops.
2695	*/
2696	case VMMCALLRING3_VMM_LOGGER_FLUSH:
2697	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
2698	LogAlways(("FLUSH\n"));
2699	break;
2700
2701	/*
2702	* Set the VM error message.
2703	*/
2704	case VMMCALLRING3_VM_SET_ERROR:
2705	VMR3SetErrorWorker(pVM);
2706	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
2707	break;
2708
2709	/*
2710	* Set the VM runtime error message.
2711	*/
2712	case VMMCALLRING3_VM_SET_RUNTIME_ERROR:
2713	pVCpu->vmm.s.rcCallRing3 = VMR3SetRuntimeErrorWorker(pVM);
2714	break;
2715
2716	/*
2717	* Signal a ring 0 hypervisor assertion.
2718	* Cancel the longjmp operation that's in progress.
2719	*/
2720	case VMMCALLRING3_VM_R0_ASSERTION:
2721	pVCpu->vmm.s.enmCallRing3Operation = VMMCALLRING3_INVALID;
2722	pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call = false;
2723	#ifdef RT_ARCH_X86
2724	pVCpu->vmm.s.CallRing3JmpBufR0.eip = 0;
2725	#else
2726	pVCpu->vmm.s.CallRing3JmpBufR0.rip = 0;
2727	#endif
2728	#ifdef VMM_R0_SWITCH_STACK
2729	(uint64_t )pVCpu->vmm.s.pbEMTStackR3 = 0; /* clear marker */
2730	#endif
2731	LogRel(("%s", pVM->vmm.s.szRing0AssertMsg1));
2732	LogRel(("%s", pVM->vmm.s.szRing0AssertMsg2));
2733	return VERR_VMM_RING0_ASSERTION;
2734
2735	/*
2736	* A forced switch to ring 0 for preemption purposes.
2737	*/
2738	case VMMCALLRING3_VM_R0_PREEMPT:
2739	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
2740	break;
2741
2742	case VMMCALLRING3_FTM_SET_CHECKPOINT:
2743	pVCpu->vmm.s.rcCallRing3 = FTMR3SetCheckpoint(pVM, (FTMCHECKPOINTTYPE)pVCpu->vmm.s.u64CallRing3Arg);
2744	break;
2745
2746	default:
2747	AssertMsgFailed(("enmCallRing3Operation=%d\n", pVCpu->vmm.s.enmCallRing3Operation));
2748	return VERR_VMM_UNKNOWN_RING3_CALL;
2749	}
2750
2751	pVCpu->vmm.s.enmCallRing3Operation = VMMCALLRING3_INVALID;
2752	return VINF_SUCCESS;
2753	}
2754
2755
2756	/**
2757	* Displays the Force action Flags.
2758	*
2759	* @param pVM The cross context VM structure.
2760	* @param pHlp The output helpers.
2761	* @param pszArgs The additional arguments (ignored).
2762	*/
2763	static DECLCALLBACK(void) vmmR3InfoFF(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
2764	{
2765	int c;
2766	uint32_t f;
2767	NOREF(pszArgs);
2768
2769	#define PRINT_FLAG(prf,flag) do { \
2770	if (f & (prf##flag)) \
2771	{ \
2772	static const char *s_psz = #flag; \
2773	if (!(c % 6)) \
2774	pHlp->pfnPrintf(pHlp, "%s\n %s", c ? "," : "", s_psz); \
2775	else \
2776	pHlp->pfnPrintf(pHlp, ", %s", s_psz); \
2777	c++; \
2778	f &= ~(prf##flag); \
2779	} \
2780	} while (0)
2781
2782	#define PRINT_GROUP(prf,grp,sfx) do { \
2783	if (f & (prf##grp##sfx)) \
2784	{ \
2785	static const char *s_psz = #grp; \
2786	if (!(c % 5)) \
2787	pHlp->pfnPrintf(pHlp, "%s %s", c ? ",\n" : " Groups:\n", s_psz); \
2788	else \
2789	pHlp->pfnPrintf(pHlp, ", %s", s_psz); \
2790	c++; \
2791	} \
2792	} while (0)
2793
2794	/*
2795	* The global flags.
2796	*/
2797	const uint32_t fGlobalForcedActions = pVM->fGlobalForcedActions;
2798	pHlp->pfnPrintf(pHlp, "Global FFs: %#RX32", fGlobalForcedActions);
2799
2800	/* show the flag mnemonics */
2801	c = 0;
2802	f = fGlobalForcedActions;
2803	PRINT_FLAG(VM_FF_,TM_VIRTUAL_SYNC);
2804	PRINT_FLAG(VM_FF_,PDM_QUEUES);
2805	PRINT_FLAG(VM_FF_,PDM_DMA);
2806	PRINT_FLAG(VM_FF_,DBGF);
2807	PRINT_FLAG(VM_FF_,REQUEST);
2808	PRINT_FLAG(VM_FF_,CHECK_VM_STATE);
2809	PRINT_FLAG(VM_FF_,RESET);
2810	PRINT_FLAG(VM_FF_,EMT_RENDEZVOUS);
2811	PRINT_FLAG(VM_FF_,PGM_NEED_HANDY_PAGES);
2812	PRINT_FLAG(VM_FF_,PGM_NO_MEMORY);
2813	PRINT_FLAG(VM_FF_,PGM_POOL_FLUSH_PENDING);
2814	PRINT_FLAG(VM_FF_,REM_HANDLER_NOTIFY);
2815	PRINT_FLAG(VM_FF_,DEBUG_SUSPEND);
2816	if (f)
2817	pHlp->pfnPrintf(pHlp, "%s\n Unknown bits: %#RX32\n", c ? "," : "", f);
2818	else
2819	pHlp->pfnPrintf(pHlp, "\n");
2820
2821	/* the groups */
2822	c = 0;
2823	f = fGlobalForcedActions;
2824	PRINT_GROUP(VM_FF_,EXTERNAL_SUSPENDED,_MASK);
2825	PRINT_GROUP(VM_FF_,EXTERNAL_HALTED,_MASK);
2826	PRINT_GROUP(VM_FF_,HIGH_PRIORITY_PRE,_MASK);
2827	PRINT_GROUP(VM_FF_,HIGH_PRIORITY_PRE_RAW,_MASK);
2828	PRINT_GROUP(VM_FF_,HIGH_PRIORITY_POST,_MASK);
2829	PRINT_GROUP(VM_FF_,NORMAL_PRIORITY_POST,_MASK);
2830	PRINT_GROUP(VM_FF_,NORMAL_PRIORITY,_MASK);
2831	PRINT_GROUP(VM_FF_,ALL_REM,_MASK);
2832	if (c)
2833	pHlp->pfnPrintf(pHlp, "\n");
2834
2835	/*
2836	* Per CPU flags.
2837	*/
2838	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2839	{
2840	const uint32_t fLocalForcedActions = pVM->aCpus[i].fLocalForcedActions;
2841	pHlp->pfnPrintf(pHlp, "CPU %u FFs: %#RX32", i, fLocalForcedActions);
2842
2843	/* show the flag mnemonics */
2844	c = 0;
2845	f = fLocalForcedActions;
2846	PRINT_FLAG(VMCPU_FF_,INTERRUPT_APIC);
2847	PRINT_FLAG(VMCPU_FF_,INTERRUPT_PIC);
2848	PRINT_FLAG(VMCPU_FF_,TIMER);
2849	PRINT_FLAG(VMCPU_FF_,INTERRUPT_NMI);
2850	PRINT_FLAG(VMCPU_FF_,INTERRUPT_SMI);
2851	PRINT_FLAG(VMCPU_FF_,PDM_CRITSECT);
2852	PRINT_FLAG(VMCPU_FF_,UNHALT);
2853	PRINT_FLAG(VMCPU_FF_,IEM);
2854	PRINT_FLAG(VMCPU_FF_,REQUEST);
2855	PRINT_FLAG(VMCPU_FF_,HM_UPDATE_CR3);
2856	PRINT_FLAG(VMCPU_FF_,HM_UPDATE_PAE_PDPES);
2857	PRINT_FLAG(VMCPU_FF_,PGM_SYNC_CR3);
2858	PRINT_FLAG(VMCPU_FF_,PGM_SYNC_CR3_NON_GLOBAL);
2859	PRINT_FLAG(VMCPU_FF_,TLB_FLUSH);
2860	PRINT_FLAG(VMCPU_FF_,INHIBIT_INTERRUPTS);
2861	PRINT_FLAG(VMCPU_FF_,BLOCK_NMIS);
2862	PRINT_FLAG(VMCPU_FF_,TO_R3);
2863	#ifdef VBOX_WITH_RAW_MODE
2864	PRINT_FLAG(VMCPU_FF_,TRPM_SYNC_IDT);
2865	PRINT_FLAG(VMCPU_FF_,SELM_SYNC_TSS);
2866	PRINT_FLAG(VMCPU_FF_,SELM_SYNC_GDT);
2867	PRINT_FLAG(VMCPU_FF_,SELM_SYNC_LDT);
2868	PRINT_FLAG(VMCPU_FF_,CSAM_SCAN_PAGE);
2869	PRINT_FLAG(VMCPU_FF_,CSAM_PENDING_ACTION);
2870	#endif
2871	if (f)
2872	pHlp->pfnPrintf(pHlp, "%s\n Unknown bits: %#RX32\n", c ? "," : "", f);
2873	else
2874	pHlp->pfnPrintf(pHlp, "\n");
2875
2876	if (fLocalForcedActions & VMCPU_FF_INHIBIT_INTERRUPTS)
2877	pHlp->pfnPrintf(pHlp, " intr inhibit RIP: %RGp\n", EMGetInhibitInterruptsPC(&pVM->aCpus[i]));
2878
2879	/* the groups */
2880	c = 0;
2881	f = fLocalForcedActions;
2882	PRINT_GROUP(VMCPU_FF_,EXTERNAL_SUSPENDED,_MASK);
2883	PRINT_GROUP(VMCPU_FF_,EXTERNAL_HALTED,_MASK);
2884	PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_PRE,_MASK);
2885	PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_PRE_RAW,_MASK);
2886	PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_POST,_MASK);
2887	PRINT_GROUP(VMCPU_FF_,NORMAL_PRIORITY_POST,_MASK);
2888	PRINT_GROUP(VMCPU_FF_,NORMAL_PRIORITY,_MASK);
2889	PRINT_GROUP(VMCPU_FF_,RESUME_GUEST,_MASK);
2890	PRINT_GROUP(VMCPU_FF_,HM_TO_R3,_MASK);
2891	PRINT_GROUP(VMCPU_FF_,ALL_REM,_MASK);
2892	if (c)
2893	pHlp->pfnPrintf(pHlp, "\n");
2894	}
2895
2896	#undef PRINT_FLAG
2897	#undef PRINT_GROUP
2898	}
2899

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR3/VMM.cpp@ 60682

Download in other formats: