VMM.cpp@ 57009

Last change on this file since 57009 was 56628, checked in by vboxsync, 9 years ago
IEM: Postpone INS memory writes to ring-3 if we hit an access handler. We cannot redo the read, that will only mess things us. This introduces a new per-cpu forced flag, VMCPU_FF_IEM, that must cause immediate return to ring-3 where it will be serviced ASAP. IEM will try return VINF_EM_RAW_TO_R3 as well to help make sure we get back to ring-3.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 90.0 KB

Line
1	/* $Id: VMM.cpp 56628 2015-06-24 19:44:56Z vboxsync $ */
2	/** @file
3	* VMM - The Virtual Machine Monitor Core.
4	*/
5
6	/*
7	* Copyright (C) 2006-2015 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	//#define NO_SUPCALLR0VMM
19
20	/** @page pg_vmm VMM - The Virtual Machine Monitor
21	*
22	* The VMM component is two things at the moment, it's a component doing a few
23	* management and routing tasks, and it's the whole virtual machine monitor
24	* thing. For hysterical reasons, it is not doing all the management that one
25	* would expect, this is instead done by @ref pg_vm. We'll address this
26	* misdesign eventually.
27	*
28	* @see grp_vmm, grp_vm
29	*
30	*
31	* @section sec_vmmstate VMM State
32	*
33	* @image html VM_Statechart_Diagram.gif
34	*
35	* To be written.
36	*
37	*
38	* @subsection subsec_vmm_init VMM Initialization
39	*
40	* To be written.
41	*
42	*
43	* @subsection subsec_vmm_term VMM Termination
44	*
45	* To be written.
46	*
47	*
48	* @sections sec_vmm_limits VMM Limits
49	*
50	* There are various resource limits imposed by the VMM and it's
51	* sub-components. We'll list some of them here.
52	*
53	* On 64-bit hosts:
54	* - Max 8191 VMs. Imposed by GVMM's handle allocation (GVMM_MAX_HANDLES),
55	* can be increased up to 64K - 1.
56	* - Max 16TB - 64KB of the host memory can be used for backing VM RAM and
57	* ROM pages. The limit is imposed by the 32-bit page ID used by GMM.
58	* - A VM can be assigned all the memory we can use (16TB), however, the
59	* Main API will restrict this to 2TB (MM_RAM_MAX_IN_MB).
60	* - Max 32 virtual CPUs (VMM_MAX_CPU_COUNT).
61	*
62	* On 32-bit hosts:
63	* - Max 127 VMs. Imposed by GMM's per page structure.
64	* - Max 64GB - 64KB of the host memory can be used for backing VM RAM and
65	* ROM pages. The limit is imposed by the 28-bit page ID used
66	* internally in GMM. It is also limited by PAE.
67	* - A VM can be assigned all the memory GMM can allocate, however, the
68	* Main API will restrict this to 3584MB (MM_RAM_MAX_IN_MB).
69	* - Max 32 virtual CPUs (VMM_MAX_CPU_COUNT).
70	*
71	*/
72
73	/*******************************************************************************
74	* Header Files *
75	*******************************************************************************/
76	#define LOG_GROUP LOG_GROUP_VMM
77	#include <VBox/vmm/vmm.h>
78	#include <VBox/vmm/vmapi.h>
79	#include <VBox/vmm/pgm.h>
80	#include <VBox/vmm/cfgm.h>
81	#include <VBox/vmm/pdmqueue.h>
82	#include <VBox/vmm/pdmcritsect.h>
83	#include <VBox/vmm/pdmcritsectrw.h>
84	#include <VBox/vmm/pdmapi.h>
85	#include <VBox/vmm/cpum.h>
86	#include <VBox/vmm/gim.h>
87	#include <VBox/vmm/mm.h>
88	#include <VBox/vmm/iom.h>
89	#include <VBox/vmm/trpm.h>
90	#include <VBox/vmm/selm.h>
91	#include <VBox/vmm/em.h>
92	#include <VBox/sup.h>
93	#include <VBox/vmm/dbgf.h>
94	#include <VBox/vmm/csam.h>
95	#include <VBox/vmm/patm.h>
96	#ifdef VBOX_WITH_REM
97	# include <VBox/vmm/rem.h>
98	#endif
99	#include <VBox/vmm/ssm.h>
100	#include <VBox/vmm/ftm.h>
101	#include <VBox/vmm/tm.h>
102	#include "VMMInternal.h"
103	#include "VMMSwitcher.h"
104	#include <VBox/vmm/vm.h>
105	#include <VBox/vmm/uvm.h>
106
107	#include <VBox/err.h>
108	#include <VBox/param.h>
109	#include <VBox/version.h>
110	#include <VBox/vmm/hm.h>
111	#include <iprt/assert.h>
112	#include <iprt/alloc.h>
113	#include <iprt/asm.h>
114	#include <iprt/time.h>
115	#include <iprt/semaphore.h>
116	#include <iprt/stream.h>
117	#include <iprt/string.h>
118	#include <iprt/stdarg.h>
119	#include <iprt/ctype.h>
120	#include <iprt/x86.h>
121
122
123
124	/*******************************************************************************
125	* Defined Constants And Macros *
126	*******************************************************************************/
127	/** The saved state version. */
128	#define VMM_SAVED_STATE_VERSION 4
129	/** The saved state version used by v3.0 and earlier. (Teleportation) */
130	#define VMM_SAVED_STATE_VERSION_3_0 3
131
132
133	/*******************************************************************************
134	* Internal Functions *
135	*******************************************************************************/
136	static int vmmR3InitStacks(PVM pVM);
137	static int vmmR3InitLoggers(PVM pVM);
138	static void vmmR3InitRegisterStats(PVM pVM);
139	static DECLCALLBACK(int) vmmR3Save(PVM pVM, PSSMHANDLE pSSM);
140	static DECLCALLBACK(int) vmmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
141	static DECLCALLBACK(void) vmmR3YieldEMT(PVM pVM, PTMTIMER pTimer, void *pvUser);
142	static int vmmR3ServiceCallRing3Request(PVM pVM, PVMCPU pVCpu);
143	static DECLCALLBACK(void) vmmR3InfoFF(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
144
145
146	/**
147	* Initializes the VMM.
148	*
149	* @returns VBox status code.
150	* @param pVM Pointer to the VM.
151	*/
152	VMMR3_INT_DECL(int) VMMR3Init(PVM pVM)
153	{
154	LogFlow(("VMMR3Init\n"));
155
156	/*
157	* Assert alignment, sizes and order.
158	*/
159	AssertMsg(pVM->vmm.s.offVM == 0, ("Already initialized!\n"));
160	AssertCompile(sizeof(pVM->vmm.s) <= sizeof(pVM->vmm.padding));
161	AssertCompile(sizeof(pVM->aCpus[0].vmm.s) <= sizeof(pVM->aCpus[0].vmm.padding));
162
163	/*
164	* Init basic VM VMM members.
165	*/
166	pVM->vmm.s.offVM = RT_OFFSETOF(VM, vmm);
167	pVM->vmm.s.pahEvtRendezvousEnterOrdered = NULL;
168	pVM->vmm.s.hEvtRendezvousEnterOneByOne = NIL_RTSEMEVENT;
169	pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce = NIL_RTSEMEVENTMULTI;
170	pVM->vmm.s.hEvtMulRendezvousDone = NIL_RTSEMEVENTMULTI;
171	pVM->vmm.s.hEvtRendezvousDoneCaller = NIL_RTSEMEVENT;
172
173	/** @cfgm{/YieldEMTInterval, uint32_t, 1, UINT32_MAX, 23, ms}
174	* The EMT yield interval. The EMT yielding is a hack we employ to play a
175	* bit nicer with the rest of the system (like for instance the GUI).
176	*/
177	int rc = CFGMR3QueryU32Def(CFGMR3GetRoot(pVM), "YieldEMTInterval", &pVM->vmm.s.cYieldEveryMillies,
178	23 /* Value arrived at after experimenting with the grub boot prompt. */);
179	AssertMsgRCReturn(rc, ("Configuration error. Failed to query \"YieldEMTInterval\", rc=%Rrc\n", rc), rc);
180
181
182	/** @cfgm{/VMM/UsePeriodicPreemptionTimers, boolean, true}
183	* Controls whether we employ per-cpu preemption timers to limit the time
184	* spent executing guest code. This option is not available on all
185	* platforms and we will silently ignore this setting then. If we are
186	* running in VT-x mode, we will use the VMX-preemption timer instead of
187	* this one when possible.
188	*/
189	PCFGMNODE pCfgVMM = CFGMR3GetChild(CFGMR3GetRoot(pVM), "VMM");
190	rc = CFGMR3QueryBoolDef(pCfgVMM, "UsePeriodicPreemptionTimers", &pVM->vmm.s.fUsePeriodicPreemptionTimers, true);
191	AssertMsgRCReturn(rc, ("Configuration error. Failed to query \"VMM/UsePeriodicPreemptionTimers\", rc=%Rrc\n", rc), rc);
192
193	/*
194	* Initialize the VMM rendezvous semaphores.
195	*/
196	pVM->vmm.s.pahEvtRendezvousEnterOrdered = (PRTSEMEVENT)MMR3HeapAlloc(pVM, MM_TAG_VMM, sizeof(RTSEMEVENT) * pVM->cCpus);
197	if (!pVM->vmm.s.pahEvtRendezvousEnterOrdered)
198	return VERR_NO_MEMORY;
199	for (VMCPUID i = 0; i < pVM->cCpus; i++)
200	pVM->vmm.s.pahEvtRendezvousEnterOrdered[i] = NIL_RTSEMEVENT;
201	for (VMCPUID i = 0; i < pVM->cCpus; i++)
202	{
203	rc = RTSemEventCreate(&pVM->vmm.s.pahEvtRendezvousEnterOrdered[i]);
204	AssertRCReturn(rc, rc);
205	}
206	rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousEnterOneByOne);
207	AssertRCReturn(rc, rc);
208	rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
209	AssertRCReturn(rc, rc);
210	rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousDone);
211	AssertRCReturn(rc, rc);
212	rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousDoneCaller);
213	AssertRCReturn(rc, rc);
214
215	/*
216	* Register the saved state data unit.
217	*/
218	rc = SSMR3RegisterInternal(pVM, "vmm", 1, VMM_SAVED_STATE_VERSION, VMM_STACK_SIZE + sizeof(RTGCPTR),
219	NULL, NULL, NULL,
220	NULL, vmmR3Save, NULL,
221	NULL, vmmR3Load, NULL);
222	if (RT_FAILURE(rc))
223	return rc;
224
225	/*
226	* Register the Ring-0 VM handle with the session for fast ioctl calls.
227	*/
228	rc = SUPR3SetVMForFastIOCtl(pVM->pVMR0);
229	if (RT_FAILURE(rc))
230	return rc;
231
232	/*
233	* Init various sub-components.
234	*/
235	rc = vmmR3SwitcherInit(pVM);
236	if (RT_SUCCESS(rc))
237	{
238	rc = vmmR3InitStacks(pVM);
239	if (RT_SUCCESS(rc))
240	{
241	rc = vmmR3InitLoggers(pVM);
242
243	#ifdef VBOX_WITH_NMI
244	/*
245	* Allocate mapping for the host APIC.
246	*/
247	if (RT_SUCCESS(rc))
248	{
249	rc = MMR3HyperReserve(pVM, PAGE_SIZE, "Host APIC", &pVM->vmm.s.GCPtrApicBase);
250	AssertRC(rc);
251	}
252	#endif
253	if (RT_SUCCESS(rc))
254	{
255	/*
256	* Debug info and statistics.
257	*/
258	DBGFR3InfoRegisterInternal(pVM, "fflags", "Displays the current Forced actions Flags.", vmmR3InfoFF);
259	vmmR3InitRegisterStats(pVM);
260	vmmInitFormatTypes();
261
262	return VINF_SUCCESS;
263	}
264	}
265	/** @todo: Need failure cleanup. */
266
267	//more todo in here?
268	//if (RT_SUCCESS(rc))
269	//{
270	//}
271	//int rc2 = vmmR3TermCoreCode(pVM);
272	//AssertRC(rc2));
273	}
274
275	return rc;
276	}
277
278
279	/**
280	* Allocate & setup the VMM RC stack(s) (for EMTs).
281	*
282	* The stacks are also used for long jumps in Ring-0.
283	*
284	* @returns VBox status code.
285	* @param pVM Pointer to the VM.
286	*
287	* @remarks The optional guard page gets it protection setup up during R3 init
288	* completion because of init order issues.
289	*/
290	static int vmmR3InitStacks(PVM pVM)
291	{
292	int rc = VINF_SUCCESS;
293	#ifdef VMM_R0_SWITCH_STACK
294	uint32_t fFlags = MMHYPER_AONR_FLAGS_KERNEL_MAPPING;
295	#else
296	uint32_t fFlags = 0;
297	#endif
298
299	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
300	{
301	PVMCPU pVCpu = &pVM->aCpus[idCpu];
302
303	#ifdef VBOX_STRICT_VMM_STACK
304	rc = MMR3HyperAllocOnceNoRelEx(pVM, PAGE_SIZE + VMM_STACK_SIZE + PAGE_SIZE,
305	#else
306	rc = MMR3HyperAllocOnceNoRelEx(pVM, VMM_STACK_SIZE,
307	#endif
308	PAGE_SIZE, MM_TAG_VMM, fFlags, (void **)&pVCpu->vmm.s.pbEMTStackR3);
309	if (RT_SUCCESS(rc))
310	{
311	#ifdef VBOX_STRICT_VMM_STACK
312	pVCpu->vmm.s.pbEMTStackR3 += PAGE_SIZE;
313	#endif
314	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
315	/* MMHyperR3ToR0 returns R3 when not doing hardware assisted virtualization. */
316	if (!HMIsEnabled(pVM))
317	pVCpu->vmm.s.CallRing3JmpBufR0.pvSavedStack = NIL_RTR0PTR;
318	else
319	#endif
320	pVCpu->vmm.s.CallRing3JmpBufR0.pvSavedStack = MMHyperR3ToR0(pVM, pVCpu->vmm.s.pbEMTStackR3);
321	pVCpu->vmm.s.pbEMTStackRC = MMHyperR3ToRC(pVM, pVCpu->vmm.s.pbEMTStackR3);
322	pVCpu->vmm.s.pbEMTStackBottomRC = pVCpu->vmm.s.pbEMTStackRC + VMM_STACK_SIZE;
323	AssertRelease(pVCpu->vmm.s.pbEMTStackRC);
324
325	CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC);
326	}
327	}
328
329	return rc;
330	}
331
332
333	/**
334	* Initialize the loggers.
335	*
336	* @returns VBox status code.
337	* @param pVM Pointer to the VM.
338	*/
339	static int vmmR3InitLoggers(PVM pVM)
340	{
341	int rc;
342	#define RTLogCalcSizeForR0(cGroups, fFlags) (RT_OFFSETOF(VMMR0LOGGER, Logger.afGroups[cGroups]) + PAGE_SIZE)
343
344	/*
345	* Allocate RC & R0 Logger instances (they are finalized in the relocator).
346	*/
347	#ifdef LOG_ENABLED
348	PRTLOGGER pLogger = RTLogDefaultInstance();
349	if (pLogger)
350	{
351	if (!HMIsEnabled(pVM))
352	{
353	pVM->vmm.s.cbRCLogger = RT_OFFSETOF(RTLOGGERRC, afGroups[pLogger->cGroups]);
354	rc = MMR3HyperAllocOnceNoRel(pVM, pVM->vmm.s.cbRCLogger, 0, MM_TAG_VMM, (void **)&pVM->vmm.s.pRCLoggerR3);
355	if (RT_FAILURE(rc))
356	return rc;
357	pVM->vmm.s.pRCLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCLoggerR3);
358	}
359
360	# ifdef VBOX_WITH_R0_LOGGING
361	size_t const cbLogger = RTLogCalcSizeForR0(pLogger->cGroups, 0);
362	for (VMCPUID i = 0; i < pVM->cCpus; i++)
363	{
364	PVMCPU pVCpu = &pVM->aCpus[i];
365	rc = MMR3HyperAllocOnceNoRelEx(pVM, cbLogger, PAGE_SIZE, MM_TAG_VMM, MMHYPER_AONR_FLAGS_KERNEL_MAPPING,
366	(void **)&pVCpu->vmm.s.pR0LoggerR3);
367	if (RT_FAILURE(rc))
368	return rc;
369	pVCpu->vmm.s.pR0LoggerR3->pVM = pVM->pVMR0;
370	//pVCpu->vmm.s.pR0LoggerR3->fCreated = false;
371	pVCpu->vmm.s.pR0LoggerR3->cbLogger = (uint32_t)cbLogger;
372	pVCpu->vmm.s.pR0LoggerR0 = MMHyperR3ToR0(pVM, pVCpu->vmm.s.pR0LoggerR3);
373	}
374	# endif
375	}
376	#endif /* LOG_ENABLED */
377
378	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
379	/*
380	* Allocate RC release logger instances (finalized in the relocator).
381	*/
382	if (!HMIsEnabled(pVM))
383	{
384	PRTLOGGER pRelLogger = RTLogRelGetDefaultInstance();
385	if (pRelLogger)
386	{
387	pVM->vmm.s.cbRCRelLogger = RT_OFFSETOF(RTLOGGERRC, afGroups[pRelLogger->cGroups]);
388	rc = MMR3HyperAllocOnceNoRel(pVM, pVM->vmm.s.cbRCRelLogger, 0, MM_TAG_VMM, (void **)&pVM->vmm.s.pRCRelLoggerR3);
389	if (RT_FAILURE(rc))
390	return rc;
391	pVM->vmm.s.pRCRelLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCRelLoggerR3);
392	}
393	}
394	#endif /* VBOX_WITH_RC_RELEASE_LOGGING */
395	return VINF_SUCCESS;
396	}
397
398
399	/**
400	* VMMR3Init worker that register the statistics with STAM.
401	*
402	* @param pVM The shared VM structure.
403	*/
404	static void vmmR3InitRegisterStats(PVM pVM)
405	{
406	/*
407	* Statistics.
408	*/
409	STAM_REG(pVM, &pVM->vmm.s.StatRunRC, STAMTYPE_COUNTER, "/VMM/RunRC", STAMUNIT_OCCURENCES, "Number of context switches.");
410	STAM_REG(pVM, &pVM->vmm.s.StatRZRetNormal, STAMTYPE_COUNTER, "/VMM/RZRet/Normal", STAMUNIT_OCCURENCES, "Number of VINF_SUCCESS returns.");
411	STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterrupt, STAMTYPE_COUNTER, "/VMM/RZRet/Interrupt", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT returns.");
412	STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterruptHyper, STAMTYPE_COUNTER, "/VMM/RZRet/InterruptHyper", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT_HYPER returns.");
413	STAM_REG(pVM, &pVM->vmm.s.StatRZRetGuestTrap, STAMTYPE_COUNTER, "/VMM/RZRet/GuestTrap", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_GUEST_TRAP returns.");
414	STAM_REG(pVM, &pVM->vmm.s.StatRZRetRingSwitch, STAMTYPE_COUNTER, "/VMM/RZRet/RingSwitch", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_RING_SWITCH returns.");
415	STAM_REG(pVM, &pVM->vmm.s.StatRZRetRingSwitchInt, STAMTYPE_COUNTER, "/VMM/RZRet/RingSwitchInt", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_RING_SWITCH_INT returns.");
416	STAM_REG(pVM, &pVM->vmm.s.StatRZRetStaleSelector, STAMTYPE_COUNTER, "/VMM/RZRet/StaleSelector", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_STALE_SELECTOR returns.");
417	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIRETTrap, STAMTYPE_COUNTER, "/VMM/RZRet/IRETTrap", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_IRET_TRAP returns.");
418	STAM_REG(pVM, &pVM->vmm.s.StatRZRetEmulate, STAMTYPE_COUNTER, "/VMM/RZRet/Emulate", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION returns.");
419	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIOBlockEmulate, STAMTYPE_COUNTER, "/VMM/RZRet/EmulateIOBlock", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_EMULATE_IO_BLOCK returns.");
420	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchEmulate, STAMTYPE_COUNTER, "/VMM/RZRet/PatchEmulate", STAMUNIT_OCCURENCES, "Number of VINF_PATCH_EMULATE_INSTR returns.");
421	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIORead, STAMTYPE_COUNTER, "/VMM/RZRet/IORead", STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_IOPORT_READ returns.");
422	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIOWrite, STAMTYPE_COUNTER, "/VMM/RZRet/IOWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_IOPORT_WRITE returns.");
423	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIORead, STAMTYPE_COUNTER, "/VMM/RZRet/MMIORead", STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_MMIO_READ returns.");
424	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOWrite, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_MMIO_WRITE returns.");
425	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOReadWrite, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOReadWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_MMIO_READ_WRITE returns.");
426	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOPatchRead, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOPatchRead", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_PATCH_READ returns.");
427	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOPatchWrite, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOPatchWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_PATCH_WRITE returns.");
428	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMSRRead, STAMTYPE_COUNTER, "/VMM/RZRet/MSRRead", STAMUNIT_OCCURENCES, "Number of VINF_CPUM_R3_MSR_READ returns.");
429	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMSRWrite, STAMTYPE_COUNTER, "/VMM/RZRet/MSRWrite", STAMUNIT_OCCURENCES, "Number of VINF_CPUM_R3_MSR_WRITE returns.");
430	STAM_REG(pVM, &pVM->vmm.s.StatRZRetLDTFault, STAMTYPE_COUNTER, "/VMM/RZRet/LDTFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_GDT_FAULT returns.");
431	STAM_REG(pVM, &pVM->vmm.s.StatRZRetGDTFault, STAMTYPE_COUNTER, "/VMM/RZRet/GDTFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_LDT_FAULT returns.");
432	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIDTFault, STAMTYPE_COUNTER, "/VMM/RZRet/IDTFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_IDT_FAULT returns.");
433	STAM_REG(pVM, &pVM->vmm.s.StatRZRetTSSFault, STAMTYPE_COUNTER, "/VMM/RZRet/TSSFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_TSS_FAULT returns.");
434	STAM_REG(pVM, &pVM->vmm.s.StatRZRetCSAMTask, STAMTYPE_COUNTER, "/VMM/RZRet/CSAMTask", STAMUNIT_OCCURENCES, "Number of VINF_CSAM_PENDING_ACTION returns.");
435	STAM_REG(pVM, &pVM->vmm.s.StatRZRetSyncCR3, STAMTYPE_COUNTER, "/VMM/RZRet/SyncCR", STAMUNIT_OCCURENCES, "Number of VINF_PGM_SYNC_CR3 returns.");
436	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMisc, STAMTYPE_COUNTER, "/VMM/RZRet/Misc", STAMUNIT_OCCURENCES, "Number of misc returns.");
437	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchInt3, STAMTYPE_COUNTER, "/VMM/RZRet/PatchInt3", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_INT3 returns.");
438	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchPF, STAMTYPE_COUNTER, "/VMM/RZRet/PatchPF", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_TRAP_PF returns.");
439	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchGP, STAMTYPE_COUNTER, "/VMM/RZRet/PatchGP", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_TRAP_GP returns.");
440	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchIretIRQ, STAMTYPE_COUNTER, "/VMM/RZRet/PatchIret", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PENDING_IRQ_AFTER_IRET returns.");
441	STAM_REG(pVM, &pVM->vmm.s.StatRZRetRescheduleREM, STAMTYPE_COUNTER, "/VMM/RZRet/ScheduleREM", STAMUNIT_OCCURENCES, "Number of VINF_EM_RESCHEDULE_REM returns.");
442	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
443	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Unknown, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Unknown", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
444	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3TMVirt, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/TMVirt", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
445	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3HandyPages, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Handy", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
446	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3PDMQueues, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/PDMQueue", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
447	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Rendezvous, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Rendezvous", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
448	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Timer, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Timer", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
449	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3DMA, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/DMA", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
450	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3CritSect, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/CritSect", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
451	STAM_REG(pVM, &pVM->vmm.s.StatRZRetTimerPending, STAMTYPE_COUNTER, "/VMM/RZRet/TimerPending", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TIMER_PENDING returns.");
452	STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterruptPending, STAMTYPE_COUNTER, "/VMM/RZRet/InterruptPending", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT_PENDING returns.");
453	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPATMDuplicateFn, STAMTYPE_COUNTER, "/VMM/RZRet/PATMDuplicateFn", STAMUNIT_OCCURENCES, "Number of VINF_PATM_DUPLICATE_FUNCTION returns.");
454	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPGMChangeMode, STAMTYPE_COUNTER, "/VMM/RZRet/PGMChangeMode", STAMUNIT_OCCURENCES, "Number of VINF_PGM_CHANGE_MODE returns.");
455	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPGMFlushPending, STAMTYPE_COUNTER, "/VMM/RZRet/PGMFlushPending", STAMUNIT_OCCURENCES, "Number of VINF_PGM_POOL_FLUSH_PENDING returns.");
456	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPendingRequest, STAMTYPE_COUNTER, "/VMM/RZRet/PendingRequest", STAMUNIT_OCCURENCES, "Number of VINF_EM_PENDING_REQUEST returns.");
457	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchTPR, STAMTYPE_COUNTER, "/VMM/RZRet/PatchTPR", STAMUNIT_OCCURENCES, "Number of VINF_EM_HM_PATCH_TPR_INSTR returns.");
458	STAM_REG(pVM, &pVM->vmm.s.StatRZRetCallRing3, STAMTYPE_COUNTER, "/VMM/RZCallR3/Misc", STAMUNIT_OCCURENCES, "Number of Other ring-3 calls.");
459	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPDMLock, STAMTYPE_COUNTER, "/VMM/RZCallR3/PDMLock", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PDM_LOCK calls.");
460	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPDMCritSectEnter, STAMTYPE_COUNTER, "/VMM/RZCallR3/PDMCritSectEnter", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PDM_CRITSECT_ENTER calls.");
461	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMLock, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMLock", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_LOCK calls.");
462	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMPoolGrow, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMPoolGrow", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_POOL_GROW calls.");
463	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMMapChunk, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMMapChunk", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_MAP_CHUNK calls.");
464	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMAllocHandy, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMAllocHandy", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES calls.");
465	STAM_REG(pVM, &pVM->vmm.s.StatRZCallRemReplay, STAMTYPE_COUNTER, "/VMM/RZCallR3/REMReplay", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_REM_REPLAY_HANDLER_NOTIFICATIONS calls.");
466	STAM_REG(pVM, &pVM->vmm.s.StatRZCallLogFlush, STAMTYPE_COUNTER, "/VMM/RZCallR3/VMMLogFlush", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VMM_LOGGER_FLUSH calls.");
467	STAM_REG(pVM, &pVM->vmm.s.StatRZCallVMSetError, STAMTYPE_COUNTER, "/VMM/RZCallR3/VMSetError", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VM_SET_ERROR calls.");
468	STAM_REG(pVM, &pVM->vmm.s.StatRZCallVMSetRuntimeError, STAMTYPE_COUNTER, "/VMM/RZCallR3/VMRuntimeError", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VM_SET_RUNTIME_ERROR calls.");
469
470	#ifdef VBOX_WITH_STATISTICS
471	for (VMCPUID i = 0; i < pVM->cCpus; i++)
472	{
473	STAMR3RegisterF(pVM, &pVM->aCpus[i].vmm.s.CallRing3JmpBufR0.cbUsedMax, STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Max amount of stack used.", "/VMM/Stack/CPU%u/Max", i);
474	STAMR3RegisterF(pVM, &pVM->aCpus[i].vmm.s.CallRing3JmpBufR0.cbUsedAvg, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Average stack usage.", "/VMM/Stack/CPU%u/Avg", i);
475	STAMR3RegisterF(pVM, &pVM->aCpus[i].vmm.s.CallRing3JmpBufR0.cUsedTotal, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of stack usages.", "/VMM/Stack/CPU%u/Uses", i);
476	}
477	#endif
478	}
479
480
481	/**
482	* Initializes the R0 VMM.
483	*
484	* @returns VBox status code.
485	* @param pVM Pointer to the VM.
486	*/
487	VMMR3_INT_DECL(int) VMMR3InitR0(PVM pVM)
488	{
489	int rc;
490	PVMCPU pVCpu = VMMGetCpu(pVM);
491	Assert(pVCpu && pVCpu->idCpu == 0);
492
493	#ifdef LOG_ENABLED
494	/*
495	* Initialize the ring-0 logger if we haven't done so yet.
496	*/
497	if ( pVCpu->vmm.s.pR0LoggerR3
498	&& !pVCpu->vmm.s.pR0LoggerR3->fCreated)
499	{
500	rc = VMMR3UpdateLoggers(pVM);
501	if (RT_FAILURE(rc))
502	return rc;
503	}
504	#endif
505
506	/*
507	* Call Ring-0 entry with init code.
508	*/
509	for (;;)
510	{
511	#ifdef NO_SUPCALLR0VMM
512	//rc = VERR_GENERAL_FAILURE;
513	rc = VINF_SUCCESS;
514	#else
515	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /idCpu/, VMMR0_DO_VMMR0_INIT,
516	RT_MAKE_U64(VMMGetSvnRev(), vmmGetBuildType()), NULL);
517	#endif
518	/*
519	* Flush the logs.
520	*/
521	#ifdef LOG_ENABLED
522	if ( pVCpu->vmm.s.pR0LoggerR3
523	&& pVCpu->vmm.s.pR0LoggerR3->Logger.offScratch > 0)
524	RTLogFlushR0(NULL, &pVCpu->vmm.s.pR0LoggerR3->Logger);
525	#endif
526	if (rc != VINF_VMM_CALL_HOST)
527	break;
528	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
529	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
530	break;
531	/* Resume R0 */
532	}
533
534	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
535	{
536	LogRel(("R0 init failed, rc=%Rra\n", rc));
537	if (RT_SUCCESS(rc))
538	rc = VERR_IPE_UNEXPECTED_INFO_STATUS;
539	}
540
541	/* Log whether thread-context hooks are used (on Linux this can depend on how the kernel is configured). */
542	if (pVM->aCpus[0].vmm.s.hCtxHook != NIL_RTTHREADCTXHOOK)
543	LogRel(("VMM: Thread-context hooks enabled!\n"));
544	else
545	LogRel(("VMM: Thread-context hooks unavailable\n"));
546
547	return rc;
548	}
549
550
551	#ifdef VBOX_WITH_RAW_MODE
552	/**
553	* Initializes the RC VMM.
554	*
555	* @returns VBox status code.
556	* @param pVM Pointer to the VM.
557	*/
558	VMMR3_INT_DECL(int) VMMR3InitRC(PVM pVM)
559	{
560	PVMCPU pVCpu = VMMGetCpu(pVM);
561	Assert(pVCpu && pVCpu->idCpu == 0);
562
563	/* In VMX mode, there's no need to init RC. */
564	if (HMIsEnabled(pVM))
565	return VINF_SUCCESS;
566
567	AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
568
569	/*
570	* Call VMMRCInit():
571	* -# resolve the address.
572	* -# setup stackframe and EIP to use the trampoline.
573	* -# do a generic hypervisor call.
574	*/
575	RTRCPTR RCPtrEP;
576	int rc = PDMR3LdrGetSymbolRC(pVM, VMMRC_MAIN_MODULE_NAME, "VMMRCEntry", &RCPtrEP);
577	if (RT_SUCCESS(rc))
578	{
579	CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC); /* Clear the stack. */
580	uint64_t u64TS = RTTimeProgramStartNanoTS();
581	CPUMPushHyper(pVCpu, (uint32_t)(u64TS >> 32)); /* Param 4: The program startup TS - Hi. */
582	CPUMPushHyper(pVCpu, (uint32_t)u64TS); /* Param 4: The program startup TS - Lo. */
583	CPUMPushHyper(pVCpu, vmmGetBuildType()); /* Param 3: Version argument. */
584	CPUMPushHyper(pVCpu, VMMGetSvnRev()); /* Param 2: Version argument. */
585	CPUMPushHyper(pVCpu, VMMRC_DO_VMMRC_INIT); /* Param 1: Operation. */
586	CPUMPushHyper(pVCpu, pVM->pVMRC); /* Param 0: pVM */
587	CPUMPushHyper(pVCpu, 6 * sizeof(RTRCPTR)); /* trampoline param: stacksize. */
588	CPUMPushHyper(pVCpu, RCPtrEP); /* Call EIP. */
589	CPUMSetHyperEIP(pVCpu, pVM->vmm.s.pfnCallTrampolineRC);
590	Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
591
592	for (;;)
593	{
594	#ifdef NO_SUPCALLR0VMM
595	//rc = VERR_GENERAL_FAILURE;
596	rc = VINF_SUCCESS;
597	#else
598	rc = SUPR3CallVMMR0(pVM->pVMR0, 0 /* VCPU 0 */, VMMR0_DO_CALL_HYPERVISOR, NULL);
599	#endif
600	#ifdef LOG_ENABLED
601	PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
602	if ( pLogger
603	&& pLogger->offScratch > 0)
604	RTLogFlushRC(NULL, pLogger);
605	#endif
606	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
607	PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
608	if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
609	RTLogFlushRC(RTLogRelGetDefaultInstance(), pRelLogger);
610	#endif
611	if (rc != VINF_VMM_CALL_HOST)
612	break;
613	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
614	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
615	break;
616	}
617
618	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
619	{
620	VMMR3FatalDump(pVM, pVCpu, rc);
621	if (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST)
622	rc = VERR_IPE_UNEXPECTED_INFO_STATUS;
623	}
624	AssertRC(rc);
625	}
626	return rc;
627	}
628	#endif /* VBOX_WITH_RAW_MODE */
629
630
631	/**
632	* Called when an init phase completes.
633	*
634	* @returns VBox status code.
635	* @param pVM Pointer to the VM.
636	* @param enmWhat Which init phase.
637	*/
638	VMMR3_INT_DECL(int) VMMR3InitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
639	{
640	int rc = VINF_SUCCESS;
641
642	switch (enmWhat)
643	{
644	case VMINITCOMPLETED_RING3:
645	{
646	/*
647	* CPUM's post-initialization (APIC base MSR caching).
648	*/
649	rc = CPUMR3InitCompleted(pVM);
650	AssertRCReturn(rc, rc);
651
652	/*
653	* Set page attributes to r/w for stack pages.
654	*/
655	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
656	{
657	rc = PGMMapSetPage(pVM, pVM->aCpus[idCpu].vmm.s.pbEMTStackRC, VMM_STACK_SIZE,
658	X86_PTE_P \| X86_PTE_A \| X86_PTE_D \| X86_PTE_RW);
659	AssertRCReturn(rc, rc);
660	}
661
662	/*
663	* Create the EMT yield timer.
664	*/
665	rc = TMR3TimerCreateInternal(pVM, TMCLOCK_REAL, vmmR3YieldEMT, NULL, "EMT Yielder", &pVM->vmm.s.pYieldTimer);
666	AssertRCReturn(rc, rc);
667
668	rc = TMTimerSetMillies(pVM->vmm.s.pYieldTimer, pVM->vmm.s.cYieldEveryMillies);
669	AssertRCReturn(rc, rc);
670
671	#ifdef VBOX_WITH_NMI
672	/*
673	* Map the host APIC into GC - This is AMD/Intel + Host OS specific!
674	*/
675	rc = PGMMap(pVM, pVM->vmm.s.GCPtrApicBase, 0xfee00000, PAGE_SIZE,
676	X86_PTE_P \| X86_PTE_RW \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_A \| X86_PTE_D);
677	AssertRCReturn(rc, rc);
678	#endif
679
680	#ifdef VBOX_STRICT_VMM_STACK
681	/*
682	* Setup the stack guard pages: Two inaccessible pages at each sides of the
683	* stack to catch over/under-flows.
684	*/
685	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
686	{
687	uint8_t *pbEMTStackR3 = pVM->aCpus[idCpu].vmm.s.pbEMTStackR3;
688
689	memset(pbEMTStackR3 - PAGE_SIZE, 0xcc, PAGE_SIZE);
690	MMR3HyperSetGuard(pVM, pbEMTStackR3 - PAGE_SIZE, PAGE_SIZE, true /fSet/);
691
692	memset(pbEMTStackR3 + VMM_STACK_SIZE, 0xcc, PAGE_SIZE);
693	MMR3HyperSetGuard(pVM, pbEMTStackR3 + VMM_STACK_SIZE, PAGE_SIZE, true /fSet/);
694	}
695	pVM->vmm.s.fStackGuardsStationed = true;
696	#endif
697	break;
698	}
699
700	case VMINITCOMPLETED_HM:
701	{
702	/*
703	* Disable the periodic preemption timers if we can use the
704	* VMX-preemption timer instead.
705	*/
706	if ( pVM->vmm.s.fUsePeriodicPreemptionTimers
707	&& HMR3IsVmxPreemptionTimerUsed(pVM))
708	pVM->vmm.s.fUsePeriodicPreemptionTimers = false;
709	LogRel(("VMM: fUsePeriodicPreemptionTimers=%RTbool\n", pVM->vmm.s.fUsePeriodicPreemptionTimers));
710
711	/*
712	* Last chance for GIM to update its CPUID leaves if it requires
713	* knowledge/information from HM initialization.
714	*/
715	rc = GIMR3InitCompleted(pVM);
716	AssertRCReturn(rc, rc);
717
718	/*
719	* CPUM's post-initialization (print CPUIDs).
720	*/
721	CPUMR3LogCpuIds(pVM);
722	break;
723	}
724
725	default: /* shuts up gcc */
726	break;
727	}
728
729	return rc;
730	}
731
732
733	/**
734	* Terminate the VMM bits.
735	*
736	* @returns VINF_SUCCESS.
737	* @param pVM Pointer to the VM.
738	*/
739	VMMR3_INT_DECL(int) VMMR3Term(PVM pVM)
740	{
741	PVMCPU pVCpu = VMMGetCpu(pVM);
742	Assert(pVCpu && pVCpu->idCpu == 0);
743
744	/*
745	* Call Ring-0 entry with termination code.
746	*/
747	int rc;
748	for (;;)
749	{
750	#ifdef NO_SUPCALLR0VMM
751	//rc = VERR_GENERAL_FAILURE;
752	rc = VINF_SUCCESS;
753	#else
754	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /idCpu/, VMMR0_DO_VMMR0_TERM, 0, NULL);
755	#endif
756	/*
757	* Flush the logs.
758	*/
759	#ifdef LOG_ENABLED
760	if ( pVCpu->vmm.s.pR0LoggerR3
761	&& pVCpu->vmm.s.pR0LoggerR3->Logger.offScratch > 0)
762	RTLogFlushR0(NULL, &pVCpu->vmm.s.pR0LoggerR3->Logger);
763	#endif
764	if (rc != VINF_VMM_CALL_HOST)
765	break;
766	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
767	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
768	break;
769	/* Resume R0 */
770	}
771	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
772	{
773	LogRel(("VMMR3Term: R0 term failed, rc=%Rra. (warning)\n", rc));
774	if (RT_SUCCESS(rc))
775	rc = VERR_IPE_UNEXPECTED_INFO_STATUS;
776	}
777
778	for (VMCPUID i = 0; i < pVM->cCpus; i++)
779	{
780	RTSemEventDestroy(pVM->vmm.s.pahEvtRendezvousEnterOrdered[i]);
781	pVM->vmm.s.pahEvtRendezvousEnterOrdered[i] = NIL_RTSEMEVENT;
782	}
783	RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousEnterOneByOne);
784	pVM->vmm.s.hEvtRendezvousEnterOneByOne = NIL_RTSEMEVENT;
785	RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
786	pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce = NIL_RTSEMEVENTMULTI;
787	RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousDone);
788	pVM->vmm.s.hEvtMulRendezvousDone = NIL_RTSEMEVENTMULTI;
789	RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousDoneCaller);
790	pVM->vmm.s.hEvtRendezvousDoneCaller = NIL_RTSEMEVENT;
791
792	#ifdef VBOX_STRICT_VMM_STACK
793	/*
794	* Make the two stack guard pages present again.
795	*/
796	if (pVM->vmm.s.fStackGuardsStationed)
797	{
798	for (VMCPUID i = 0; i < pVM->cCpus; i++)
799	{
800	uint8_t *pbEMTStackR3 = pVM->aCpus[i].vmm.s.pbEMTStackR3;
801	MMR3HyperSetGuard(pVM, pbEMTStackR3 - PAGE_SIZE, PAGE_SIZE, false /fSet/);
802	MMR3HyperSetGuard(pVM, pbEMTStackR3 + VMM_STACK_SIZE, PAGE_SIZE, false /fSet/);
803	}
804	pVM->vmm.s.fStackGuardsStationed = false;
805	}
806	#endif
807
808	vmmTermFormatTypes();
809	return rc;
810	}
811
812
813	/**
814	* Applies relocations to data and code managed by this
815	* component. This function will be called at init and
816	* whenever the VMM need to relocate it self inside the GC.
817	*
818	* The VMM will need to apply relocations to the core code.
819	*
820	* @param pVM Pointer to the VM.
821	* @param offDelta The relocation delta.
822	*/
823	VMMR3_INT_DECL(void) VMMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
824	{
825	LogFlow(("VMMR3Relocate: offDelta=%RGv\n", offDelta));
826
827	/*
828	* Recalc the RC address.
829	*/
830	#ifdef VBOX_WITH_RAW_MODE
831	pVM->vmm.s.pvCoreCodeRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pvCoreCodeR3);
832	#endif
833
834	/*
835	* The stack.
836	*/
837	for (VMCPUID i = 0; i < pVM->cCpus; i++)
838	{
839	PVMCPU pVCpu = &pVM->aCpus[i];
840
841	CPUMSetHyperESP(pVCpu, CPUMGetHyperESP(pVCpu) + offDelta);
842
843	pVCpu->vmm.s.pbEMTStackRC = MMHyperR3ToRC(pVM, pVCpu->vmm.s.pbEMTStackR3);
844	pVCpu->vmm.s.pbEMTStackBottomRC = pVCpu->vmm.s.pbEMTStackRC + VMM_STACK_SIZE;
845	}
846
847	/*
848	* All the switchers.
849	*/
850	vmmR3SwitcherRelocate(pVM, offDelta);
851
852	/*
853	* Get other RC entry points.
854	*/
855	if (!HMIsEnabled(pVM))
856	{
857	int rc = PDMR3LdrGetSymbolRC(pVM, VMMRC_MAIN_MODULE_NAME, "CPUMGCResumeGuest", &pVM->vmm.s.pfnCPUMRCResumeGuest);
858	AssertReleaseMsgRC(rc, ("CPUMGCResumeGuest not found! rc=%Rra\n", rc));
859
860	rc = PDMR3LdrGetSymbolRC(pVM, VMMRC_MAIN_MODULE_NAME, "CPUMGCResumeGuestV86", &pVM->vmm.s.pfnCPUMRCResumeGuestV86);
861	AssertReleaseMsgRC(rc, ("CPUMGCResumeGuestV86 not found! rc=%Rra\n", rc));
862	}
863
864	/*
865	* Update the logger.
866	*/
867	VMMR3UpdateLoggers(pVM);
868	}
869
870
871	/**
872	* Updates the settings for the RC and R0 loggers.
873	*
874	* @returns VBox status code.
875	* @param pVM Pointer to the VM.
876	*/
877	VMMR3_INT_DECL(int) VMMR3UpdateLoggers(PVM pVM)
878	{
879	/*
880	* Simply clone the logger instance (for RC).
881	*/
882	int rc = VINF_SUCCESS;
883	RTRCPTR RCPtrLoggerFlush = 0;
884
885	if ( pVM->vmm.s.pRCLoggerR3
886	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
887	\|\| pVM->vmm.s.pRCRelLoggerR3
888	#endif
889	)
890	{
891	Assert(!HMIsEnabled(pVM));
892	rc = PDMR3LdrGetSymbolRC(pVM, VMMRC_MAIN_MODULE_NAME, "vmmGCLoggerFlush", &RCPtrLoggerFlush);
893	AssertReleaseMsgRC(rc, ("vmmGCLoggerFlush not found! rc=%Rra\n", rc));
894	}
895
896	if (pVM->vmm.s.pRCLoggerR3)
897	{
898	Assert(!HMIsEnabled(pVM));
899	RTRCPTR RCPtrLoggerWrapper = 0;
900	rc = PDMR3LdrGetSymbolRC(pVM, VMMRC_MAIN_MODULE_NAME, "vmmGCLoggerWrapper", &RCPtrLoggerWrapper);
901	AssertReleaseMsgRC(rc, ("vmmGCLoggerWrapper not found! rc=%Rra\n", rc));
902
903	pVM->vmm.s.pRCLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCLoggerR3);
904	rc = RTLogCloneRC(NULL /* default */, pVM->vmm.s.pRCLoggerR3, pVM->vmm.s.cbRCLogger,
905	RCPtrLoggerWrapper, RCPtrLoggerFlush, RTLOGFLAGS_BUFFERED);
906	AssertReleaseMsgRC(rc, ("RTLogCloneRC failed! rc=%Rra\n", rc));
907	}
908
909	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
910	if (pVM->vmm.s.pRCRelLoggerR3)
911	{
912	Assert(!HMIsEnabled(pVM));
913	RTRCPTR RCPtrLoggerWrapper = 0;
914	rc = PDMR3LdrGetSymbolRC(pVM, VMMRC_MAIN_MODULE_NAME, "vmmGCRelLoggerWrapper", &RCPtrLoggerWrapper);
915	AssertReleaseMsgRC(rc, ("vmmGCRelLoggerWrapper not found! rc=%Rra\n", rc));
916
917	pVM->vmm.s.pRCRelLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCRelLoggerR3);
918	rc = RTLogCloneRC(RTLogRelGetDefaultInstance(), pVM->vmm.s.pRCRelLoggerR3, pVM->vmm.s.cbRCRelLogger,
919	RCPtrLoggerWrapper, RCPtrLoggerFlush, RTLOGFLAGS_BUFFERED);
920	AssertReleaseMsgRC(rc, ("RTLogCloneRC failed! rc=%Rra\n", rc));
921	}
922	#endif /* VBOX_WITH_RC_RELEASE_LOGGING */
923
924	#ifdef LOG_ENABLED
925	/*
926	* For the ring-0 EMT logger, we use a per-thread logger instance
927	* in ring-0. Only initialize it once.
928	*/
929	PRTLOGGER const pDefault = RTLogDefaultInstance();
930	for (VMCPUID i = 0; i < pVM->cCpus; i++)
931	{
932	PVMCPU pVCpu = &pVM->aCpus[i];
933	PVMMR0LOGGER pR0LoggerR3 = pVCpu->vmm.s.pR0LoggerR3;
934	if (pR0LoggerR3)
935	{
936	if (!pR0LoggerR3->fCreated)
937	{
938	RTR0PTR pfnLoggerWrapper = NIL_RTR0PTR;
939	rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerWrapper", &pfnLoggerWrapper);
940	AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerWrapper not found! rc=%Rra\n", rc), rc);
941
942	RTR0PTR pfnLoggerFlush = NIL_RTR0PTR;
943	rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerFlush", &pfnLoggerFlush);
944	AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerFlush not found! rc=%Rra\n", rc), rc);
945
946	rc = RTLogCreateForR0(&pR0LoggerR3->Logger, pR0LoggerR3->cbLogger,
947	pVCpu->vmm.s.pR0LoggerR0 + RT_OFFSETOF(VMMR0LOGGER, Logger),
948	pfnLoggerWrapper, pfnLoggerFlush,
949	RTLOGFLAGS_BUFFERED, RTLOGDEST_DUMMY);
950	AssertReleaseMsgRCReturn(rc, ("RTLogCreateForR0 failed! rc=%Rra\n", rc), rc);
951
952	RTR0PTR pfnLoggerPrefix = NIL_RTR0PTR;
953	rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerPrefix", &pfnLoggerPrefix);
954	AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerPrefix not found! rc=%Rra\n", rc), rc);
955	rc = RTLogSetCustomPrefixCallbackForR0(&pR0LoggerR3->Logger,
956	pVCpu->vmm.s.pR0LoggerR0 + RT_OFFSETOF(VMMR0LOGGER, Logger),
957	pfnLoggerPrefix, NIL_RTR0PTR);
958	AssertReleaseMsgRCReturn(rc, ("RTLogSetCustomPrefixCallback failed! rc=%Rra\n", rc), rc);
959
960	pR0LoggerR3->idCpu = i;
961	pR0LoggerR3->fCreated = true;
962	pR0LoggerR3->fFlushingDisabled = false;
963
964	}
965
966	rc = RTLogCopyGroupsAndFlagsForR0(&pR0LoggerR3->Logger, pVCpu->vmm.s.pR0LoggerR0 + RT_OFFSETOF(VMMR0LOGGER, Logger),
967	pDefault, RTLOGFLAGS_BUFFERED, UINT32_MAX);
968	AssertRC(rc);
969	}
970	}
971	#endif
972	return rc;
973	}
974
975
976	/**
977	* Gets the pointer to a buffer containing the R0/RC RTAssertMsg1Weak output.
978	*
979	* @returns Pointer to the buffer.
980	* @param pVM Pointer to the VM.
981	*/
982	VMMR3DECL(const char *) VMMR3GetRZAssertMsg1(PVM pVM)
983	{
984	if (HMIsEnabled(pVM))
985	return pVM->vmm.s.szRing0AssertMsg1;
986
987	RTRCPTR RCPtr;
988	int rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_szRTAssertMsg1", &RCPtr);
989	if (RT_SUCCESS(rc))
990	return (const char *)MMHyperRCToR3(pVM, RCPtr);
991
992	return NULL;
993	}
994
995
996	/**
997	* Returns the VMCPU of the specified virtual CPU.
998	*
999	* @returns The VMCPU pointer. NULL if @a idCpu or @a pUVM is invalid.
1000	*
1001	* @param pUVM The user mode VM handle.
1002	* @param idCpu The ID of the virtual CPU.
1003	*/
1004	VMMR3DECL(PVMCPU) VMMR3GetCpuByIdU(PUVM pUVM, RTCPUID idCpu)
1005	{
1006	UVM_ASSERT_VALID_EXT_RETURN(pUVM, NULL);
1007	AssertReturn(idCpu < pUVM->cCpus, NULL);
1008	VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, NULL);
1009	return &pUVM->pVM->aCpus[idCpu];
1010	}
1011
1012
1013	/**
1014	* Gets the pointer to a buffer containing the R0/RC RTAssertMsg2Weak output.
1015	*
1016	* @returns Pointer to the buffer.
1017	* @param pVM Pointer to the VM.
1018	*/
1019	VMMR3DECL(const char *) VMMR3GetRZAssertMsg2(PVM pVM)
1020	{
1021	if (HMIsEnabled(pVM))
1022	return pVM->vmm.s.szRing0AssertMsg2;
1023
1024	RTRCPTR RCPtr;
1025	int rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_szRTAssertMsg2", &RCPtr);
1026	if (RT_SUCCESS(rc))
1027	return (const char *)MMHyperRCToR3(pVM, RCPtr);
1028
1029	return NULL;
1030	}
1031
1032
1033	/**
1034	* Execute state save operation.
1035	*
1036	* @returns VBox status code.
1037	* @param pVM Pointer to the VM.
1038	* @param pSSM SSM operation handle.
1039	*/
1040	static DECLCALLBACK(int) vmmR3Save(PVM pVM, PSSMHANDLE pSSM)
1041	{
1042	LogFlow(("vmmR3Save:\n"));
1043
1044	/*
1045	* Save the started/stopped state of all CPUs except 0 as it will always
1046	* be running. This avoids breaking the saved state version. :-)
1047	*/
1048	for (VMCPUID i = 1; i < pVM->cCpus; i++)
1049	SSMR3PutBool(pSSM, VMCPUSTATE_IS_STARTED(VMCPU_GET_STATE(&pVM->aCpus[i])));
1050
1051	return SSMR3PutU32(pSSM, UINT32_MAX); /* terminator */
1052	}
1053
1054
1055	/**
1056	* Execute state load operation.
1057	*
1058	* @returns VBox status code.
1059	* @param pVM Pointer to the VM.
1060	* @param pSSM SSM operation handle.
1061	* @param uVersion Data layout version.
1062	* @param uPass The data pass.
1063	*/
1064	static DECLCALLBACK(int) vmmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
1065	{
1066	LogFlow(("vmmR3Load:\n"));
1067	Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
1068
1069	/*
1070	* Validate version.
1071	*/
1072	if ( uVersion != VMM_SAVED_STATE_VERSION
1073	&& uVersion != VMM_SAVED_STATE_VERSION_3_0)
1074	{
1075	AssertMsgFailed(("vmmR3Load: Invalid version uVersion=%u!\n", uVersion));
1076	return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
1077	}
1078
1079	if (uVersion <= VMM_SAVED_STATE_VERSION_3_0)
1080	{
1081	/* Ignore the stack bottom, stack pointer and stack bits. */
1082	RTRCPTR RCPtrIgnored;
1083	SSMR3GetRCPtr(pSSM, &RCPtrIgnored);
1084	SSMR3GetRCPtr(pSSM, &RCPtrIgnored);
1085	#ifdef RT_OS_DARWIN
1086	if ( SSMR3HandleVersion(pSSM) >= VBOX_FULL_VERSION_MAKE(3,0,0)
1087	&& SSMR3HandleVersion(pSSM) < VBOX_FULL_VERSION_MAKE(3,1,0)
1088	&& SSMR3HandleRevision(pSSM) >= 48858
1089	&& ( !strcmp(SSMR3HandleHostOSAndArch(pSSM), "darwin.x86")
1090	\|\| !strcmp(SSMR3HandleHostOSAndArch(pSSM), "") )
1091	)
1092	SSMR3Skip(pSSM, 16384);
1093	else
1094	SSMR3Skip(pSSM, 8192);
1095	#else
1096	SSMR3Skip(pSSM, 8192);
1097	#endif
1098	}
1099
1100	/*
1101	* Restore the VMCPU states. VCPU 0 is always started.
1102	*/
1103	VMCPU_SET_STATE(&pVM->aCpus[0], VMCPUSTATE_STARTED);
1104	for (VMCPUID i = 1; i < pVM->cCpus; i++)
1105	{
1106	bool fStarted;
1107	int rc = SSMR3GetBool(pSSM, &fStarted);
1108	if (RT_FAILURE(rc))
1109	return rc;
1110	VMCPU_SET_STATE(&pVM->aCpus[i], fStarted ? VMCPUSTATE_STARTED : VMCPUSTATE_STOPPED);
1111	}
1112
1113	/* terminator */
1114	uint32_t u32;
1115	int rc = SSMR3GetU32(pSSM, &u32);
1116	if (RT_FAILURE(rc))
1117	return rc;
1118	if (u32 != UINT32_MAX)
1119	{
1120	AssertMsgFailed(("u32=%#x\n", u32));
1121	return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
1122	}
1123	return VINF_SUCCESS;
1124	}
1125
1126
1127	#ifdef VBOX_WITH_RAW_MODE
1128	/**
1129	* Resolve a builtin RC symbol.
1130	*
1131	* Called by PDM when loading or relocating RC modules.
1132	*
1133	* @returns VBox status
1134	* @param pVM Pointer to the VM.
1135	* @param pszSymbol Symbol to resolv
1136	* @param pRCPtrValue Where to store the symbol value.
1137	*
1138	* @remark This has to work before VMMR3Relocate() is called.
1139	*/
1140	VMMR3_INT_DECL(int) VMMR3GetImportRC(PVM pVM, const char *pszSymbol, PRTRCPTR pRCPtrValue)
1141	{
1142	if (!strcmp(pszSymbol, "g_Logger"))
1143	{
1144	if (pVM->vmm.s.pRCLoggerR3)
1145	pVM->vmm.s.pRCLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCLoggerR3);
1146	*pRCPtrValue = pVM->vmm.s.pRCLoggerRC;
1147	}
1148	else if (!strcmp(pszSymbol, "g_RelLogger"))
1149	{
1150	# ifdef VBOX_WITH_RC_RELEASE_LOGGING
1151	if (pVM->vmm.s.pRCRelLoggerR3)
1152	pVM->vmm.s.pRCRelLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCRelLoggerR3);
1153	*pRCPtrValue = pVM->vmm.s.pRCRelLoggerRC;
1154	# else
1155	*pRCPtrValue = NIL_RTRCPTR;
1156	# endif
1157	}
1158	else
1159	return VERR_SYMBOL_NOT_FOUND;
1160	return VINF_SUCCESS;
1161	}
1162	#endif /* VBOX_WITH_RAW_MODE */
1163
1164
1165	/**
1166	* Suspends the CPU yielder.
1167	*
1168	* @param pVM Pointer to the VM.
1169	*/
1170	VMMR3_INT_DECL(void) VMMR3YieldSuspend(PVM pVM)
1171	{
1172	VMCPU_ASSERT_EMT(&pVM->aCpus[0]);
1173	if (!pVM->vmm.s.cYieldResumeMillies)
1174	{
1175	uint64_t u64Now = TMTimerGet(pVM->vmm.s.pYieldTimer);
1176	uint64_t u64Expire = TMTimerGetExpire(pVM->vmm.s.pYieldTimer);
1177	if (u64Now >= u64Expire \|\| u64Expire == ~(uint64_t)0)
1178	pVM->vmm.s.cYieldResumeMillies = pVM->vmm.s.cYieldEveryMillies;
1179	else
1180	pVM->vmm.s.cYieldResumeMillies = TMTimerToMilli(pVM->vmm.s.pYieldTimer, u64Expire - u64Now);
1181	TMTimerStop(pVM->vmm.s.pYieldTimer);
1182	}
1183	pVM->vmm.s.u64LastYield = RTTimeNanoTS();
1184	}
1185
1186
1187	/**
1188	* Stops the CPU yielder.
1189	*
1190	* @param pVM Pointer to the VM.
1191	*/
1192	VMMR3_INT_DECL(void) VMMR3YieldStop(PVM pVM)
1193	{
1194	if (!pVM->vmm.s.cYieldResumeMillies)
1195	TMTimerStop(pVM->vmm.s.pYieldTimer);
1196	pVM->vmm.s.cYieldResumeMillies = pVM->vmm.s.cYieldEveryMillies;
1197	pVM->vmm.s.u64LastYield = RTTimeNanoTS();
1198	}
1199
1200
1201	/**
1202	* Resumes the CPU yielder when it has been a suspended or stopped.
1203	*
1204	* @param pVM Pointer to the VM.
1205	*/
1206	VMMR3_INT_DECL(void) VMMR3YieldResume(PVM pVM)
1207	{
1208	if (pVM->vmm.s.cYieldResumeMillies)
1209	{
1210	TMTimerSetMillies(pVM->vmm.s.pYieldTimer, pVM->vmm.s.cYieldResumeMillies);
1211	pVM->vmm.s.cYieldResumeMillies = 0;
1212	}
1213	}
1214
1215
1216	/**
1217	* Internal timer callback function.
1218	*
1219	* @param pVM The VM.
1220	* @param pTimer The timer handle.
1221	* @param pvUser User argument specified upon timer creation.
1222	*/
1223	static DECLCALLBACK(void) vmmR3YieldEMT(PVM pVM, PTMTIMER pTimer, void *pvUser)
1224	{
1225	NOREF(pvUser);
1226
1227	/*
1228	* This really needs some careful tuning. While we shouldn't be too greedy since
1229	* that'll cause the rest of the system to stop up, we shouldn't be too nice either
1230	* because that'll cause us to stop up.
1231	*
1232	* The current logic is to use the default interval when there is no lag worth
1233	* mentioning, but when we start accumulating lag we don't bother yielding at all.
1234	*
1235	* (This depends on the TMCLOCK_VIRTUAL_SYNC to be scheduled before TMCLOCK_REAL
1236	* so the lag is up to date.)
1237	*/
1238	const uint64_t u64Lag = TMVirtualSyncGetLag(pVM);
1239	if ( u64Lag < 50000000 /* 50ms */
1240	\|\| ( u64Lag < 1000000000 /* 1s */
1241	&& RTTimeNanoTS() - pVM->vmm.s.u64LastYield < 500000000 /* 500 ms */)
1242	)
1243	{
1244	uint64_t u64Elapsed = RTTimeNanoTS();
1245	pVM->vmm.s.u64LastYield = u64Elapsed;
1246
1247	RTThreadYield();
1248
1249	#ifdef LOG_ENABLED
1250	u64Elapsed = RTTimeNanoTS() - u64Elapsed;
1251	Log(("vmmR3YieldEMT: %RI64 ns\n", u64Elapsed));
1252	#endif
1253	}
1254	TMTimerSetMillies(pTimer, pVM->vmm.s.cYieldEveryMillies);
1255	}
1256
1257
1258	#ifdef VBOX_WITH_RAW_MODE
1259	/**
1260	* Executes guest code in the raw-mode context.
1261	*
1262	* @param pVM Pointer to the VM.
1263	* @param pVCpu Pointer to the VMCPU.
1264	*/
1265	VMMR3_INT_DECL(int) VMMR3RawRunGC(PVM pVM, PVMCPU pVCpu)
1266	{
1267	Log2(("VMMR3RawRunGC: (cs:eip=%04x:%08x)\n", CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
1268
1269	AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
1270
1271	/*
1272	* Set the hypervisor to resume executing a CPUM resume function
1273	* in CPUMRCA.asm.
1274	*/
1275	CPUMSetHyperState(pVCpu,
1276	CPUMGetGuestEFlags(pVCpu) & X86_EFL_VM
1277	? pVM->vmm.s.pfnCPUMRCResumeGuestV86
1278	: pVM->vmm.s.pfnCPUMRCResumeGuest, /* eip */
1279	pVCpu->vmm.s.pbEMTStackBottomRC, /* esp */
1280	0, /* eax */
1281	VM_RC_ADDR(pVM, &pVCpu->cpum) /* edx */);
1282
1283	/*
1284	* We hide log flushes (outer) and hypervisor interrupts (inner).
1285	*/
1286	for (;;)
1287	{
1288	#ifdef VBOX_STRICT
1289	if (RT_UNLIKELY(!CPUMGetHyperCR3(pVCpu) \|\| CPUMGetHyperCR3(pVCpu) != PGMGetHyperCR3(pVCpu)))
1290	EMR3FatalError(pVCpu, VERR_VMM_HYPER_CR3_MISMATCH);
1291	PGMMapCheck(pVM);
1292	# ifdef VBOX_WITH_SAFE_STR
1293	SELMR3CheckShadowTR(pVM);
1294	# endif
1295	#endif
1296	int rc;
1297	do
1298	{
1299	#ifdef NO_SUPCALLR0VMM
1300	rc = VERR_GENERAL_FAILURE;
1301	#else
1302	rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
1303	if (RT_LIKELY(rc == VINF_SUCCESS))
1304	rc = pVCpu->vmm.s.iLastGZRc;
1305	#endif
1306	} while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
1307
1308	/*
1309	* Flush the logs.
1310	*/
1311	#ifdef LOG_ENABLED
1312	PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
1313	if ( pLogger
1314	&& pLogger->offScratch > 0)
1315	RTLogFlushRC(NULL, pLogger);
1316	#endif
1317	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
1318	PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
1319	if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
1320	RTLogFlushRC(RTLogRelGetDefaultInstance(), pRelLogger);
1321	#endif
1322	if (rc != VINF_VMM_CALL_HOST)
1323	{
1324	Log2(("VMMR3RawRunGC: returns %Rrc (cs:eip=%04x:%08x)\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
1325	return rc;
1326	}
1327	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
1328	if (RT_FAILURE(rc))
1329	return rc;
1330	/* Resume GC */
1331	}
1332	}
1333	#endif /* VBOX_WITH_RAW_MODE */
1334
1335
1336	/**
1337	* Executes guest code (Intel VT-x and AMD-V).
1338	*
1339	* @param pVM Pointer to the VM.
1340	* @param pVCpu Pointer to the VMCPU.
1341	*/
1342	VMMR3_INT_DECL(int) VMMR3HmRunGC(PVM pVM, PVMCPU pVCpu)
1343	{
1344	Log2(("VMMR3HmRunGC: (cs:rip=%04x:%RX64)\n", CPUMGetGuestCS(pVCpu), CPUMGetGuestRIP(pVCpu)));
1345
1346	for (;;)
1347	{
1348	int rc;
1349	do
1350	{
1351	#ifdef NO_SUPCALLR0VMM
1352	rc = VERR_GENERAL_FAILURE;
1353	#else
1354	rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_HM_RUN, pVCpu->idCpu);
1355	if (RT_LIKELY(rc == VINF_SUCCESS))
1356	rc = pVCpu->vmm.s.iLastGZRc;
1357	#endif
1358	} while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
1359
1360	#if 0 /* todo triggers too often */
1361	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TO_R3));
1362	#endif
1363
1364	#ifdef LOG_ENABLED
1365	/*
1366	* Flush the log
1367	*/
1368	PVMMR0LOGGER pR0LoggerR3 = pVCpu->vmm.s.pR0LoggerR3;
1369	if ( pR0LoggerR3
1370	&& pR0LoggerR3->Logger.offScratch > 0)
1371	RTLogFlushR0(NULL, &pR0LoggerR3->Logger);
1372	#endif /* !LOG_ENABLED */
1373	if (rc != VINF_VMM_CALL_HOST)
1374	{
1375	Log2(("VMMR3HmRunGC: returns %Rrc (cs:rip=%04x:%RX64)\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestRIP(pVCpu)));
1376	return rc;
1377	}
1378	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
1379	if (RT_FAILURE(rc))
1380	return rc;
1381	/* Resume R0 */
1382	}
1383	}
1384
1385	/**
1386	* VCPU worker for VMMSendSipi.
1387	*
1388	* @param pVM Pointer to the VM.
1389	* @param idCpu Virtual CPU to perform SIPI on
1390	* @param uVector SIPI vector
1391	*/
1392	DECLCALLBACK(int) vmmR3SendSipi(PVM pVM, VMCPUID idCpu, uint32_t uVector)
1393	{
1394	PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
1395	VMCPU_ASSERT_EMT(pVCpu);
1396
1397	/** @todo what are we supposed to do if the processor is already running? */
1398	if (EMGetState(pVCpu) != EMSTATE_WAIT_SIPI)
1399	return VERR_ACCESS_DENIED;
1400
1401
1402	PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
1403
1404	pCtx->cs.Sel = uVector << 8;
1405	pCtx->cs.ValidSel = uVector << 8;
1406	pCtx->cs.fFlags = CPUMSELREG_FLAGS_VALID;
1407	pCtx->cs.u64Base = uVector << 12;
1408	pCtx->cs.u32Limit = UINT32_C(0x0000ffff);
1409	pCtx->rip = 0;
1410
1411	Log(("vmmR3SendSipi for VCPU %d with vector %x\n", idCpu, uVector));
1412
1413	# if 1 /* If we keep the EMSTATE_WAIT_SIPI method, then move this to EM.cpp. */
1414	EMSetState(pVCpu, EMSTATE_HALTED);
1415	return VINF_EM_RESCHEDULE;
1416	# else /* And if we go the VMCPU::enmState way it can stay here. */
1417	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STOPPED);
1418	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
1419	return VINF_SUCCESS;
1420	# endif
1421	}
1422
1423	DECLCALLBACK(int) vmmR3SendInitIpi(PVM pVM, VMCPUID idCpu)
1424	{
1425	PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
1426	VMCPU_ASSERT_EMT(pVCpu);
1427
1428	Log(("vmmR3SendInitIpi for VCPU %d\n", idCpu));
1429
1430	PGMR3ResetCpu(pVM, pVCpu);
1431	PDMR3ResetCpu(pVCpu); /* Clear any pending interrupts */
1432	TRPMR3ResetCpu(pVCpu);
1433	CPUMR3ResetCpu(pVM, pVCpu);
1434	EMR3ResetCpu(pVCpu);
1435	HMR3ResetCpu(pVCpu);
1436
1437	/* This will trickle up on the target EMT. */
1438	return VINF_EM_WAIT_SIPI;
1439	}
1440
1441	/**
1442	* Sends SIPI to the virtual CPU by setting CS:EIP into vector-dependent state
1443	* and unhalting processor
1444	*
1445	* @param pVM Pointer to the VM.
1446	* @param idCpu Virtual CPU to perform SIPI on
1447	* @param uVector SIPI vector
1448	*/
1449	VMMR3_INT_DECL(void) VMMR3SendSipi(PVM pVM, VMCPUID idCpu, uint32_t uVector)
1450	{
1451	AssertReturnVoid(idCpu < pVM->cCpus);
1452
1453	int rc = VMR3ReqCallNoWait(pVM, idCpu, (PFNRT)vmmR3SendSipi, 3, pVM, idCpu, uVector);
1454	AssertRC(rc);
1455	}
1456
1457	/**
1458	* Sends init IPI to the virtual CPU.
1459	*
1460	* @param pVM Pointer to the VM.
1461	* @param idCpu Virtual CPU to perform int IPI on
1462	*/
1463	VMMR3_INT_DECL(void) VMMR3SendInitIpi(PVM pVM, VMCPUID idCpu)
1464	{
1465	AssertReturnVoid(idCpu < pVM->cCpus);
1466
1467	int rc = VMR3ReqCallNoWait(pVM, idCpu, (PFNRT)vmmR3SendInitIpi, 2, pVM, idCpu);
1468	AssertRC(rc);
1469	}
1470
1471	/**
1472	* Registers the guest memory range that can be used for patching
1473	*
1474	* @returns VBox status code.
1475	* @param pVM Pointer to the VM.
1476	* @param pPatchMem Patch memory range
1477	* @param cbPatchMem Size of the memory range
1478	*/
1479	VMMR3DECL(int) VMMR3RegisterPatchMemory(PVM pVM, RTGCPTR pPatchMem, unsigned cbPatchMem)
1480	{
1481	VM_ASSERT_EMT(pVM);
1482	if (HMIsEnabled(pVM))
1483	return HMR3EnablePatching(pVM, pPatchMem, cbPatchMem);
1484
1485	return VERR_NOT_SUPPORTED;
1486	}
1487
1488	/**
1489	* Deregisters the guest memory range that can be used for patching
1490	*
1491	* @returns VBox status code.
1492	* @param pVM Pointer to the VM.
1493	* @param pPatchMem Patch memory range
1494	* @param cbPatchMem Size of the memory range
1495	*/
1496	VMMR3DECL(int) VMMR3DeregisterPatchMemory(PVM pVM, RTGCPTR pPatchMem, unsigned cbPatchMem)
1497	{
1498	if (HMIsEnabled(pVM))
1499	return HMR3DisablePatching(pVM, pPatchMem, cbPatchMem);
1500
1501	return VINF_SUCCESS;
1502	}
1503
1504
1505	/**
1506	* Count returns and have the last non-caller EMT wake up the caller.
1507	*
1508	* @returns VBox strict informational status code for EM scheduling. No failures
1509	* will be returned here, those are for the caller only.
1510	*
1511	* @param pVM Pointer to the VM.
1512	*/
1513	DECL_FORCE_INLINE(int) vmmR3EmtRendezvousNonCallerReturn(PVM pVM)
1514	{
1515	int rcRet = ASMAtomicReadS32(&pVM->vmm.s.i32RendezvousStatus);
1516	uint32_t cReturned = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsReturned);
1517	if (cReturned == pVM->cCpus - 1U)
1518	{
1519	int rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousDoneCaller);
1520	AssertLogRelRC(rc);
1521	}
1522
1523	AssertLogRelMsgReturn( rcRet <= VINF_SUCCESS
1524	\|\| (rcRet >= VINF_EM_FIRST && rcRet <= VINF_EM_LAST),
1525	("%Rrc\n", rcRet),
1526	VERR_IPE_UNEXPECTED_INFO_STATUS);
1527	return RT_SUCCESS(rcRet) ? rcRet : VINF_SUCCESS;
1528	}
1529
1530
1531	/**
1532	* Common worker for VMMR3EmtRendezvous and VMMR3EmtRendezvousFF.
1533	*
1534	* @returns VBox strict informational status code for EM scheduling. No failures
1535	* will be returned here, those are for the caller only. When
1536	* fIsCaller is set, VINF_SUCCESS is always returned.
1537	*
1538	* @param pVM Pointer to the VM.
1539	* @param pVCpu The VMCPU structure for the calling EMT.
1540	* @param fIsCaller Whether we're the VMMR3EmtRendezvous caller or
1541	* not.
1542	* @param fFlags The flags.
1543	* @param pfnRendezvous The callback.
1544	* @param pvUser The user argument for the callback.
1545	*/
1546	static int vmmR3EmtRendezvousCommon(PVM pVM, PVMCPU pVCpu, bool fIsCaller,
1547	uint32_t fFlags, PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser)
1548	{
1549	int rc;
1550
1551	/*
1552	* Enter, the last EMT triggers the next callback phase.
1553	*/
1554	uint32_t cEntered = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsEntered);
1555	if (cEntered != pVM->cCpus)
1556	{
1557	if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE)
1558	{
1559	/* Wait for our turn. */
1560	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousEnterOneByOne, RT_INDEFINITE_WAIT);
1561	AssertLogRelRC(rc);
1562	}
1563	else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE)
1564	{
1565	/* Wait for the last EMT to arrive and wake everyone up. */
1566	rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce, RT_INDEFINITE_WAIT);
1567	AssertLogRelRC(rc);
1568	}
1569	else if ( (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
1570	\|\| (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
1571	{
1572	/* Wait for our turn. */
1573	rc = RTSemEventWait(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVCpu->idCpu], RT_INDEFINITE_WAIT);
1574	AssertLogRelRC(rc);
1575	}
1576	else
1577	{
1578	Assert((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE);
1579
1580	/*
1581	* The execute once is handled specially to optimize the code flow.
1582	*
1583	* The last EMT to arrive will perform the callback and the other
1584	* EMTs will wait on the Done/DoneCaller semaphores (instead of
1585	* the EnterOneByOne/AllAtOnce) in the meanwhile. When the callback
1586	* returns, that EMT will initiate the normal return sequence.
1587	*/
1588	if (!fIsCaller)
1589	{
1590	rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousDone, RT_INDEFINITE_WAIT);
1591	AssertLogRelRC(rc);
1592
1593	return vmmR3EmtRendezvousNonCallerReturn(pVM);
1594	}
1595	return VINF_SUCCESS;
1596	}
1597	}
1598	else
1599	{
1600	/*
1601	* All EMTs are waiting, clear the FF and take action according to the
1602	* execution method.
1603	*/
1604	VM_FF_CLEAR(pVM, VM_FF_EMT_RENDEZVOUS);
1605
1606	if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE)
1607	{
1608	/* Wake up everyone. */
1609	rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
1610	AssertLogRelRC(rc);
1611	}
1612	else if ( (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
1613	\|\| (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
1614	{
1615	/* Figure out who to wake up and wake it up. If it's ourself, then
1616	it's easy otherwise wait for our turn. */
1617	VMCPUID iFirst = (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
1618	? 0
1619	: pVM->cCpus - 1U;
1620	if (pVCpu->idCpu != iFirst)
1621	{
1622	rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[iFirst]);
1623	AssertLogRelRC(rc);
1624	rc = RTSemEventWait(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVCpu->idCpu], RT_INDEFINITE_WAIT);
1625	AssertLogRelRC(rc);
1626	}
1627	}
1628	/* else: execute the handler on the current EMT and wake up one or more threads afterwards. */
1629	}
1630
1631
1632	/*
1633	* Do the callback and update the status if necessary.
1634	*/
1635	if ( !(fFlags & VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR)
1636	\|\| RT_SUCCESS(ASMAtomicUoReadS32(&pVM->vmm.s.i32RendezvousStatus)) )
1637	{
1638	VBOXSTRICTRC rcStrict = pfnRendezvous(pVM, pVCpu, pvUser);
1639	if (rcStrict != VINF_SUCCESS)
1640	{
1641	AssertLogRelMsg( rcStrict <= VINF_SUCCESS
1642	\|\| (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST),
1643	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
1644	int32_t i32RendezvousStatus;
1645	do
1646	{
1647	i32RendezvousStatus = ASMAtomicUoReadS32(&pVM->vmm.s.i32RendezvousStatus);
1648	if ( rcStrict == i32RendezvousStatus
1649	\|\| RT_FAILURE(i32RendezvousStatus)
1650	\|\| ( i32RendezvousStatus != VINF_SUCCESS
1651	&& rcStrict > i32RendezvousStatus))
1652	break;
1653	} while (!ASMAtomicCmpXchgS32(&pVM->vmm.s.i32RendezvousStatus, VBOXSTRICTRC_VAL(rcStrict), i32RendezvousStatus));
1654	}
1655	}
1656
1657	/*
1658	* Increment the done counter and take action depending on whether we're
1659	* the last to finish callback execution.
1660	*/
1661	uint32_t cDone = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsDone);
1662	if ( cDone != pVM->cCpus
1663	&& (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE)
1664	{
1665	/* Signal the next EMT? */
1666	if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE)
1667	{
1668	rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousEnterOneByOne);
1669	AssertLogRelRC(rc);
1670	}
1671	else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING)
1672	{
1673	Assert(cDone == pVCpu->idCpu + 1U);
1674	rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVCpu->idCpu + 1U]);
1675	AssertLogRelRC(rc);
1676	}
1677	else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
1678	{
1679	Assert(pVM->cCpus - cDone == pVCpu->idCpu);
1680	rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVM->cCpus - cDone - 1U]);
1681	AssertLogRelRC(rc);
1682	}
1683
1684	/* Wait for the rest to finish (the caller waits on hEvtRendezvousDoneCaller). */
1685	if (!fIsCaller)
1686	{
1687	rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousDone, RT_INDEFINITE_WAIT);
1688	AssertLogRelRC(rc);
1689	}
1690	}
1691	else
1692	{
1693	/* Callback execution is all done, tell the rest to return. */
1694	rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousDone);
1695	AssertLogRelRC(rc);
1696	}
1697
1698	if (!fIsCaller)
1699	return vmmR3EmtRendezvousNonCallerReturn(pVM);
1700	return VINF_SUCCESS;
1701	}
1702
1703
1704	/**
1705	* Called in response to VM_FF_EMT_RENDEZVOUS.
1706	*
1707	* @returns VBox strict status code - EM scheduling. No errors will be returned
1708	* here, nor will any non-EM scheduling status codes be returned.
1709	*
1710	* @param pVM Pointer to the VM.
1711	* @param pVCpu The handle of the calling EMT.
1712	*
1713	* @thread EMT
1714	*/
1715	VMMR3_INT_DECL(int) VMMR3EmtRendezvousFF(PVM pVM, PVMCPU pVCpu)
1716	{
1717	Assert(!pVCpu->vmm.s.fInRendezvous);
1718	pVCpu->vmm.s.fInRendezvous = true;
1719	int rc = vmmR3EmtRendezvousCommon(pVM, pVCpu, false /* fIsCaller */, pVM->vmm.s.fRendezvousFlags,
1720	pVM->vmm.s.pfnRendezvous, pVM->vmm.s.pvRendezvousUser);
1721	pVCpu->vmm.s.fInRendezvous = false;
1722	return rc;
1723	}
1724
1725
1726	/**
1727	* EMT rendezvous.
1728	*
1729	* Gathers all the EMTs and execute some code on each of them, either in a one
1730	* by one fashion or all at once.
1731	*
1732	* @returns VBox strict status code. This will be the first error,
1733	* VINF_SUCCESS, or an EM scheduling status code.
1734	*
1735	* @param pVM Pointer to the VM.
1736	* @param fFlags Flags indicating execution methods. See
1737	* grp_VMMR3EmtRendezvous_fFlags.
1738	* @param pfnRendezvous The callback.
1739	* @param pvUser User argument for the callback.
1740	*
1741	* @thread Any.
1742	*/
1743	VMMR3DECL(int) VMMR3EmtRendezvous(PVM pVM, uint32_t fFlags, PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser)
1744	{
1745	/*
1746	* Validate input.
1747	*/
1748	AssertReturn(pVM, VERR_INVALID_VM_HANDLE);
1749	AssertMsg( (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_INVALID
1750	&& (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) <= VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING
1751	&& !(fFlags & ~VMMEMTRENDEZVOUS_FLAGS_VALID_MASK), ("%#x\n", fFlags));
1752	AssertMsg( !(fFlags & VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR)
1753	\|\| ( (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE
1754	&& (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE),
1755	("type %u\n", fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK));
1756
1757	VBOXSTRICTRC rcStrict;
1758	PVMCPU pVCpu = VMMGetCpu(pVM);
1759	if (!pVCpu)
1760	/*
1761	* Forward the request to an EMT thread.
1762	*/
1763	rcStrict = VMR3ReqCallWait(pVM, VMCPUID_ANY,
1764	(PFNRT)VMMR3EmtRendezvous, 4, pVM, fFlags, pfnRendezvous, pvUser);
1765	else if (pVM->cCpus == 1)
1766	{
1767	/*
1768	* Shortcut for the single EMT case.
1769	*/
1770	AssertLogRelReturn(!pVCpu->vmm.s.fInRendezvous, VERR_DEADLOCK);
1771	pVCpu->vmm.s.fInRendezvous = true;
1772	rcStrict = pfnRendezvous(pVM, pVCpu, pvUser);
1773	pVCpu->vmm.s.fInRendezvous = false;
1774	}
1775	else
1776	{
1777	/*
1778	* Spin lock. If busy, wait for the other EMT to finish while keeping a
1779	* lookout of the RENDEZVOUS FF.
1780	*/
1781	int rc;
1782	rcStrict = VINF_SUCCESS;
1783	if (RT_UNLIKELY(!ASMAtomicCmpXchgU32(&pVM->vmm.s.u32RendezvousLock, 0x77778888, 0)))
1784	{
1785	AssertLogRelReturn(!pVCpu->vmm.s.fInRendezvous, VERR_DEADLOCK);
1786
1787	while (!ASMAtomicCmpXchgU32(&pVM->vmm.s.u32RendezvousLock, 0x77778888, 0))
1788	{
1789	if (VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS))
1790	{
1791	rc = VMMR3EmtRendezvousFF(pVM, pVCpu);
1792	if ( rc != VINF_SUCCESS
1793	&& ( rcStrict == VINF_SUCCESS
1794	\|\| rcStrict > rc))
1795	rcStrict = rc;
1796	/** @todo Perhaps deal with termination here? */
1797	}
1798	ASMNopPause();
1799	}
1800	}
1801	Assert(!VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS));
1802	Assert(!pVCpu->vmm.s.fInRendezvous);
1803	pVCpu->vmm.s.fInRendezvous = true;
1804
1805	/*
1806	* Clear the slate. This is a semaphore ping-pong orgy. :-)
1807	*/
1808	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1809	{
1810	rc = RTSemEventWait(pVM->vmm.s.pahEvtRendezvousEnterOrdered[i], 0);
1811	AssertLogRelMsg(rc == VERR_TIMEOUT \|\| rc == VINF_SUCCESS, ("%Rrc\n", rc));
1812	}
1813	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousEnterOneByOne, 0); AssertLogRelMsg(rc == VERR_TIMEOUT \|\| rc == VINF_SUCCESS, ("%Rrc\n", rc));
1814	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce); AssertLogRelRC(rc);
1815	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousDone); AssertLogRelRC(rc);
1816	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousDoneCaller, 0); AssertLogRelMsg(rc == VERR_TIMEOUT \|\| rc == VINF_SUCCESS, ("%Rrc\n", rc));
1817	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsEntered, 0);
1818	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsDone, 0);
1819	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsReturned, 0);
1820	ASMAtomicWriteS32(&pVM->vmm.s.i32RendezvousStatus, VINF_SUCCESS);
1821	ASMAtomicWritePtr((void * volatile )&pVM->vmm.s.pfnRendezvous, (void )(uintptr_t)pfnRendezvous);
1822	ASMAtomicWritePtr(&pVM->vmm.s.pvRendezvousUser, pvUser);
1823	ASMAtomicWriteU32(&pVM->vmm.s.fRendezvousFlags, fFlags);
1824
1825	/*
1826	* Set the FF and poke the other EMTs.
1827	*/
1828	VM_FF_SET(pVM, VM_FF_EMT_RENDEZVOUS);
1829	VMR3NotifyGlobalFFU(pVM->pUVM, VMNOTIFYFF_FLAGS_POKE);
1830
1831	/*
1832	* Do the same ourselves.
1833	*/
1834	vmmR3EmtRendezvousCommon(pVM, pVCpu, true /* fIsCaller */, fFlags, pfnRendezvous, pvUser);
1835
1836	/*
1837	* The caller waits for the other EMTs to be done and return before doing
1838	* the cleanup. This makes away with wakeup / reset races we would otherwise
1839	* risk in the multiple release event semaphore code (hEvtRendezvousDoneCaller).
1840	*/
1841	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousDoneCaller, RT_INDEFINITE_WAIT);
1842	AssertLogRelRC(rc);
1843
1844	/*
1845	* Get the return code and clean up a little bit.
1846	*/
1847	int rcMy = pVM->vmm.s.i32RendezvousStatus;
1848	ASMAtomicWriteNullPtr((void * volatile *)&pVM->vmm.s.pfnRendezvous);
1849
1850	ASMAtomicWriteU32(&pVM->vmm.s.u32RendezvousLock, 0);
1851	pVCpu->vmm.s.fInRendezvous = false;
1852
1853	/*
1854	* Merge rcStrict and rcMy.
1855	*/
1856	AssertRC(VBOXSTRICTRC_VAL(rcStrict));
1857	if ( rcMy != VINF_SUCCESS
1858	&& ( rcStrict == VINF_SUCCESS
1859	\|\| rcStrict > rcMy))
1860	rcStrict = rcMy;
1861	}
1862
1863	AssertLogRelMsgReturn( rcStrict <= VINF_SUCCESS
1864	\|\| (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST),
1865	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)),
1866	VERR_IPE_UNEXPECTED_INFO_STATUS);
1867	return VBOXSTRICTRC_VAL(rcStrict);
1868	}
1869
1870
1871	/**
1872	* Disables/enables EMT rendezvous.
1873	*
1874	* This is used to make sure EMT rendezvous does not take place while
1875	* processing a priority request.
1876	*
1877	* @returns Old rendezvous-disabled state.
1878	* @param pVCpu The handle of the calling EMT.
1879	* @param fDisabled True if disabled, false if enabled.
1880	*/
1881	VMMR3_INT_DECL(bool) VMMR3EmtRendezvousSetDisabled(PVMCPU pVCpu, bool fDisabled)
1882	{
1883	VMCPU_ASSERT_EMT(pVCpu);
1884	bool fOld = pVCpu->vmm.s.fInRendezvous;
1885	pVCpu->vmm.s.fInRendezvous = fDisabled;
1886	return fOld;
1887	}
1888
1889
1890	/**
1891	* Read from the ring 0 jump buffer stack
1892	*
1893	* @returns VBox status code.
1894	*
1895	* @param pVM Pointer to the VM.
1896	* @param idCpu The ID of the source CPU context (for the address).
1897	* @param R0Addr Where to start reading.
1898	* @param pvBuf Where to store the data we've read.
1899	* @param cbRead The number of bytes to read.
1900	*/
1901	VMMR3_INT_DECL(int) VMMR3ReadR0Stack(PVM pVM, VMCPUID idCpu, RTHCUINTPTR R0Addr, void *pvBuf, size_t cbRead)
1902	{
1903	PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
1904	AssertReturn(pVCpu, VERR_INVALID_PARAMETER);
1905
1906	#ifdef VMM_R0_SWITCH_STACK
1907	RTHCUINTPTR off = R0Addr - MMHyperCCToR0(pVM, pVCpu->vmm.s.pbEMTStackR3);
1908	#else
1909	RTHCUINTPTR off = pVCpu->vmm.s.CallRing3JmpBufR0.cbSavedStack - (pVCpu->vmm.s.CallRing3JmpBufR0.SpCheck - R0Addr);
1910	#endif
1911	if ( off > VMM_STACK_SIZE
1912	\|\| off + cbRead >= VMM_STACK_SIZE)
1913	return VERR_INVALID_POINTER;
1914
1915	memcpy(pvBuf, &pVCpu->vmm.s.pbEMTStackR3[off], cbRead);
1916	return VINF_SUCCESS;
1917	}
1918
1919	#ifdef VBOX_WITH_RAW_MODE
1920
1921	/**
1922	* Calls a RC function.
1923	*
1924	* @param pVM Pointer to the VM.
1925	* @param RCPtrEntry The address of the RC function.
1926	* @param cArgs The number of arguments in the ....
1927	* @param ... Arguments to the function.
1928	*/
1929	VMMR3DECL(int) VMMR3CallRC(PVM pVM, RTRCPTR RCPtrEntry, unsigned cArgs, ...)
1930	{
1931	va_list args;
1932	va_start(args, cArgs);
1933	int rc = VMMR3CallRCV(pVM, RCPtrEntry, cArgs, args);
1934	va_end(args);
1935	return rc;
1936	}
1937
1938
1939	/**
1940	* Calls a RC function.
1941	*
1942	* @param pVM Pointer to the VM.
1943	* @param RCPtrEntry The address of the RC function.
1944	* @param cArgs The number of arguments in the ....
1945	* @param args Arguments to the function.
1946	*/
1947	VMMR3DECL(int) VMMR3CallRCV(PVM pVM, RTRCPTR RCPtrEntry, unsigned cArgs, va_list args)
1948	{
1949	/* Raw mode implies 1 VCPU. */
1950	AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
1951	PVMCPU pVCpu = &pVM->aCpus[0];
1952
1953	Log2(("VMMR3CallGCV: RCPtrEntry=%RRv cArgs=%d\n", RCPtrEntry, cArgs));
1954
1955	/*
1956	* Setup the call frame using the trampoline.
1957	*/
1958	CPUMSetHyperState(pVCpu,
1959	pVM->vmm.s.pfnCallTrampolineRC, /* eip */
1960	pVCpu->vmm.s.pbEMTStackBottomRC - cArgs * sizeof(RTGCUINTPTR32), /* esp */
1961	RCPtrEntry, /* eax */
1962	cArgs /* edx */
1963	);
1964
1965	#if 0
1966	memset(pVCpu->vmm.s.pbEMTStackR3, 0xaa, VMM_STACK_SIZE); /* Clear the stack. */
1967	#endif
1968	PRTGCUINTPTR32 pFrame = (PRTGCUINTPTR32)(pVCpu->vmm.s.pbEMTStackR3 + VMM_STACK_SIZE) - cArgs;
1969	int i = cArgs;
1970	while (i-- > 0)
1971	*pFrame++ = va_arg(args, RTGCUINTPTR32);
1972
1973	CPUMPushHyper(pVCpu, cArgs * sizeof(RTGCUINTPTR32)); /* stack frame size */
1974	CPUMPushHyper(pVCpu, RCPtrEntry); /* what to call */
1975
1976	/*
1977	* We hide log flushes (outer) and hypervisor interrupts (inner).
1978	*/
1979	for (;;)
1980	{
1981	int rc;
1982	Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
1983	do
1984	{
1985	#ifdef NO_SUPCALLR0VMM
1986	rc = VERR_GENERAL_FAILURE;
1987	#else
1988	rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
1989	if (RT_LIKELY(rc == VINF_SUCCESS))
1990	rc = pVCpu->vmm.s.iLastGZRc;
1991	#endif
1992	} while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
1993
1994	/*
1995	* Flush the loggers.
1996	*/
1997	#ifdef LOG_ENABLED
1998	PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
1999	if ( pLogger
2000	&& pLogger->offScratch > 0)
2001	RTLogFlushRC(NULL, pLogger);
2002	#endif
2003	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
2004	PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
2005	if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
2006	RTLogFlushRC(RTLogRelGetDefaultInstance(), pRelLogger);
2007	#endif
2008	if (rc == VERR_TRPM_PANIC \|\| rc == VERR_TRPM_DONT_PANIC)
2009	VMMR3FatalDump(pVM, pVCpu, rc);
2010	if (rc != VINF_VMM_CALL_HOST)
2011	{
2012	Log2(("VMMR3CallGCV: returns %Rrc (cs:eip=%04x:%08x)\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
2013	return rc;
2014	}
2015	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
2016	if (RT_FAILURE(rc))
2017	return rc;
2018	}
2019	}
2020
2021	#endif /* VBOX_WITH_RAW_MODE */
2022
2023	/**
2024	* Wrapper for SUPR3CallVMMR0Ex which will deal with VINF_VMM_CALL_HOST returns.
2025	*
2026	* @returns VBox status code.
2027	* @param pVM Pointer to the VM.
2028	* @param uOperation Operation to execute.
2029	* @param u64Arg Constant argument.
2030	* @param pReqHdr Pointer to a request header. See SUPR3CallVMMR0Ex for
2031	* details.
2032	*/
2033	VMMR3DECL(int) VMMR3CallR0(PVM pVM, uint32_t uOperation, uint64_t u64Arg, PSUPVMMR0REQHDR pReqHdr)
2034	{
2035	PVMCPU pVCpu = VMMGetCpu(pVM);
2036	AssertReturn(pVCpu, VERR_VM_THREAD_NOT_EMT);
2037
2038	/*
2039	* Call Ring-0 entry with init code.
2040	*/
2041	int rc;
2042	for (;;)
2043	{
2044	#ifdef NO_SUPCALLR0VMM
2045	rc = VERR_GENERAL_FAILURE;
2046	#else
2047	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, pVCpu->idCpu, uOperation, u64Arg, pReqHdr);
2048	#endif
2049	/*
2050	* Flush the logs.
2051	*/
2052	#ifdef LOG_ENABLED
2053	if ( pVCpu->vmm.s.pR0LoggerR3
2054	&& pVCpu->vmm.s.pR0LoggerR3->Logger.offScratch > 0)
2055	RTLogFlushR0(NULL, &pVCpu->vmm.s.pR0LoggerR3->Logger);
2056	#endif
2057	if (rc != VINF_VMM_CALL_HOST)
2058	break;
2059	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
2060	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
2061	break;
2062	/* Resume R0 */
2063	}
2064
2065	AssertLogRelMsgReturn(rc == VINF_SUCCESS \|\| RT_FAILURE(rc),
2066	("uOperation=%u rc=%Rrc\n", uOperation, rc),
2067	VERR_IPE_UNEXPECTED_INFO_STATUS);
2068	return rc;
2069	}
2070
2071
2072	#ifdef VBOX_WITH_RAW_MODE
2073	/**
2074	* Resumes executing hypervisor code when interrupted by a queue flush or a
2075	* debug event.
2076	*
2077	* @returns VBox status code.
2078	* @param pVM Pointer to the VM.
2079	* @param pVCpu Pointer to the VMCPU.
2080	*/
2081	VMMR3DECL(int) VMMR3ResumeHyper(PVM pVM, PVMCPU pVCpu)
2082	{
2083	Log(("VMMR3ResumeHyper: eip=%RRv esp=%RRv\n", CPUMGetHyperEIP(pVCpu), CPUMGetHyperESP(pVCpu)));
2084	AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
2085
2086	/*
2087	* We hide log flushes (outer) and hypervisor interrupts (inner).
2088	*/
2089	for (;;)
2090	{
2091	int rc;
2092	Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
2093	do
2094	{
2095	# ifdef NO_SUPCALLR0VMM
2096	rc = VERR_GENERAL_FAILURE;
2097	# else
2098	rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
2099	if (RT_LIKELY(rc == VINF_SUCCESS))
2100	rc = pVCpu->vmm.s.iLastGZRc;
2101	# endif
2102	} while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
2103
2104	/*
2105	* Flush the loggers.
2106	*/
2107	# ifdef LOG_ENABLED
2108	PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
2109	if ( pLogger
2110	&& pLogger->offScratch > 0)
2111	RTLogFlushRC(NULL, pLogger);
2112	# endif
2113	# ifdef VBOX_WITH_RC_RELEASE_LOGGING
2114	PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
2115	if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
2116	RTLogFlushRC(RTLogRelGetDefaultInstance(), pRelLogger);
2117	# endif
2118	if (rc == VERR_TRPM_PANIC \|\| rc == VERR_TRPM_DONT_PANIC)
2119	VMMR3FatalDump(pVM, pVCpu, rc);
2120	if (rc != VINF_VMM_CALL_HOST)
2121	{
2122	Log(("VMMR3ResumeHyper: returns %Rrc\n", rc));
2123	return rc;
2124	}
2125	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
2126	if (RT_FAILURE(rc))
2127	return rc;
2128	}
2129	}
2130	#endif /* VBOX_WITH_RAW_MODE */
2131
2132
2133	/**
2134	* Service a call to the ring-3 host code.
2135	*
2136	* @returns VBox status code.
2137	* @param pVM Pointer to the VM.
2138	* @param pVCpu Pointer to the VMCPU.
2139	* @remark Careful with critsects.
2140	*/
2141	static int vmmR3ServiceCallRing3Request(PVM pVM, PVMCPU pVCpu)
2142	{
2143	/*
2144	* We must also check for pending critsect exits or else we can deadlock
2145	* when entering other critsects here.
2146	*/
2147	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PDM_CRITSECT))
2148	PDMCritSectBothFF(pVCpu);
2149
2150	switch (pVCpu->vmm.s.enmCallRing3Operation)
2151	{
2152	/*
2153	* Acquire a critical section.
2154	*/
2155	case VMMCALLRING3_PDM_CRIT_SECT_ENTER:
2156	{
2157	pVCpu->vmm.s.rcCallRing3 = PDMR3CritSectEnterEx((PPDMCRITSECT)(uintptr_t)pVCpu->vmm.s.u64CallRing3Arg,
2158	true /fCallRing3/);
2159	break;
2160	}
2161
2162	/*
2163	* Enter a r/w critical section exclusively.
2164	*/
2165	case VMMCALLRING3_PDM_CRIT_SECT_RW_ENTER_EXCL:
2166	{
2167	pVCpu->vmm.s.rcCallRing3 = PDMR3CritSectRwEnterExclEx((PPDMCRITSECTRW)(uintptr_t)pVCpu->vmm.s.u64CallRing3Arg,
2168	true /fCallRing3/);
2169	break;
2170	}
2171
2172	/*
2173	* Enter a r/w critical section shared.
2174	*/
2175	case VMMCALLRING3_PDM_CRIT_SECT_RW_ENTER_SHARED:
2176	{
2177	pVCpu->vmm.s.rcCallRing3 = PDMR3CritSectRwEnterSharedEx((PPDMCRITSECTRW)(uintptr_t)pVCpu->vmm.s.u64CallRing3Arg,
2178	true /fCallRing3/);
2179	break;
2180	}
2181
2182	/*
2183	* Acquire the PDM lock.
2184	*/
2185	case VMMCALLRING3_PDM_LOCK:
2186	{
2187	pVCpu->vmm.s.rcCallRing3 = PDMR3LockCall(pVM);
2188	break;
2189	}
2190
2191	/*
2192	* Grow the PGM pool.
2193	*/
2194	case VMMCALLRING3_PGM_POOL_GROW:
2195	{
2196	pVCpu->vmm.s.rcCallRing3 = PGMR3PoolGrow(pVM);
2197	break;
2198	}
2199
2200	/*
2201	* Maps an page allocation chunk into ring-3 so ring-0 can use it.
2202	*/
2203	case VMMCALLRING3_PGM_MAP_CHUNK:
2204	{
2205	pVCpu->vmm.s.rcCallRing3 = PGMR3PhysChunkMap(pVM, pVCpu->vmm.s.u64CallRing3Arg);
2206	break;
2207	}
2208
2209	/*
2210	* Allocates more handy pages.
2211	*/
2212	case VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES:
2213	{
2214	pVCpu->vmm.s.rcCallRing3 = PGMR3PhysAllocateHandyPages(pVM);
2215	break;
2216	}
2217
2218	/*
2219	* Allocates a large page.
2220	*/
2221	case VMMCALLRING3_PGM_ALLOCATE_LARGE_HANDY_PAGE:
2222	{
2223	pVCpu->vmm.s.rcCallRing3 = PGMR3PhysAllocateLargeHandyPage(pVM, pVCpu->vmm.s.u64CallRing3Arg);
2224	break;
2225	}
2226
2227	/*
2228	* Acquire the PGM lock.
2229	*/
2230	case VMMCALLRING3_PGM_LOCK:
2231	{
2232	pVCpu->vmm.s.rcCallRing3 = PGMR3LockCall(pVM);
2233	break;
2234	}
2235
2236	/*
2237	* Acquire the MM hypervisor heap lock.
2238	*/
2239	case VMMCALLRING3_MMHYPER_LOCK:
2240	{
2241	pVCpu->vmm.s.rcCallRing3 = MMR3LockCall(pVM);
2242	break;
2243	}
2244
2245	#ifdef VBOX_WITH_REM
2246	/*
2247	* Flush REM handler notifications.
2248	*/
2249	case VMMCALLRING3_REM_REPLAY_HANDLER_NOTIFICATIONS:
2250	{
2251	REMR3ReplayHandlerNotifications(pVM);
2252	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
2253	break;
2254	}
2255	#endif
2256
2257	/*
2258	* This is a noop. We just take this route to avoid unnecessary
2259	* tests in the loops.
2260	*/
2261	case VMMCALLRING3_VMM_LOGGER_FLUSH:
2262	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
2263	LogAlways(("FLUSH\n"));
2264	break;
2265
2266	/*
2267	* Set the VM error message.
2268	*/
2269	case VMMCALLRING3_VM_SET_ERROR:
2270	VMR3SetErrorWorker(pVM);
2271	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
2272	break;
2273
2274	/*
2275	* Set the VM runtime error message.
2276	*/
2277	case VMMCALLRING3_VM_SET_RUNTIME_ERROR:
2278	pVCpu->vmm.s.rcCallRing3 = VMR3SetRuntimeErrorWorker(pVM);
2279	break;
2280
2281	/*
2282	* Signal a ring 0 hypervisor assertion.
2283	* Cancel the longjmp operation that's in progress.
2284	*/
2285	case VMMCALLRING3_VM_R0_ASSERTION:
2286	pVCpu->vmm.s.enmCallRing3Operation = VMMCALLRING3_INVALID;
2287	pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call = false;
2288	#ifdef RT_ARCH_X86
2289	pVCpu->vmm.s.CallRing3JmpBufR0.eip = 0;
2290	#else
2291	pVCpu->vmm.s.CallRing3JmpBufR0.rip = 0;
2292	#endif
2293	#ifdef VMM_R0_SWITCH_STACK
2294	(uint64_t )pVCpu->vmm.s.pbEMTStackR3 = 0; /* clear marker */
2295	#endif
2296	LogRel(("%s", pVM->vmm.s.szRing0AssertMsg1));
2297	LogRel(("%s", pVM->vmm.s.szRing0AssertMsg2));
2298	return VERR_VMM_RING0_ASSERTION;
2299
2300	/*
2301	* A forced switch to ring 0 for preemption purposes.
2302	*/
2303	case VMMCALLRING3_VM_R0_PREEMPT:
2304	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
2305	break;
2306
2307	case VMMCALLRING3_FTM_SET_CHECKPOINT:
2308	pVCpu->vmm.s.rcCallRing3 = FTMR3SetCheckpoint(pVM, (FTMCHECKPOINTTYPE)pVCpu->vmm.s.u64CallRing3Arg);
2309	break;
2310
2311	default:
2312	AssertMsgFailed(("enmCallRing3Operation=%d\n", pVCpu->vmm.s.enmCallRing3Operation));
2313	return VERR_VMM_UNKNOWN_RING3_CALL;
2314	}
2315
2316	pVCpu->vmm.s.enmCallRing3Operation = VMMCALLRING3_INVALID;
2317	return VINF_SUCCESS;
2318	}
2319
2320
2321	/**
2322	* Displays the Force action Flags.
2323	*
2324	* @param pVM Pointer to the VM.
2325	* @param pHlp The output helpers.
2326	* @param pszArgs The additional arguments (ignored).
2327	*/
2328	static DECLCALLBACK(void) vmmR3InfoFF(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
2329	{
2330	int c;
2331	uint32_t f;
2332	NOREF(pszArgs);
2333
2334	#define PRINT_FLAG(prf,flag) do { \
2335	if (f & (prf##flag)) \
2336	{ \
2337	static const char *s_psz = #flag; \
2338	if (!(c % 6)) \
2339	pHlp->pfnPrintf(pHlp, "%s\n %s", c ? "," : "", s_psz); \
2340	else \
2341	pHlp->pfnPrintf(pHlp, ", %s", s_psz); \
2342	c++; \
2343	f &= ~(prf##flag); \
2344	} \
2345	} while (0)
2346
2347	#define PRINT_GROUP(prf,grp,sfx) do { \
2348	if (f & (prf##grp##sfx)) \
2349	{ \
2350	static const char *s_psz = #grp; \
2351	if (!(c % 5)) \
2352	pHlp->pfnPrintf(pHlp, "%s %s", c ? ",\n" : " Groups:\n", s_psz); \
2353	else \
2354	pHlp->pfnPrintf(pHlp, ", %s", s_psz); \
2355	c++; \
2356	} \
2357	} while (0)
2358
2359	/*
2360	* The global flags.
2361	*/
2362	const uint32_t fGlobalForcedActions = pVM->fGlobalForcedActions;
2363	pHlp->pfnPrintf(pHlp, "Global FFs: %#RX32", fGlobalForcedActions);
2364
2365	/* show the flag mnemonics */
2366	c = 0;
2367	f = fGlobalForcedActions;
2368	PRINT_FLAG(VM_FF_,TM_VIRTUAL_SYNC);
2369	PRINT_FLAG(VM_FF_,PDM_QUEUES);
2370	PRINT_FLAG(VM_FF_,PDM_DMA);
2371	PRINT_FLAG(VM_FF_,DBGF);
2372	PRINT_FLAG(VM_FF_,REQUEST);
2373	PRINT_FLAG(VM_FF_,CHECK_VM_STATE);
2374	PRINT_FLAG(VM_FF_,RESET);
2375	PRINT_FLAG(VM_FF_,EMT_RENDEZVOUS);
2376	PRINT_FLAG(VM_FF_,PGM_NEED_HANDY_PAGES);
2377	PRINT_FLAG(VM_FF_,PGM_NO_MEMORY);
2378	PRINT_FLAG(VM_FF_,PGM_POOL_FLUSH_PENDING);
2379	PRINT_FLAG(VM_FF_,REM_HANDLER_NOTIFY);
2380	PRINT_FLAG(VM_FF_,DEBUG_SUSPEND);
2381	if (f)
2382	pHlp->pfnPrintf(pHlp, "%s\n Unknown bits: %#RX32\n", c ? "," : "", f);
2383	else
2384	pHlp->pfnPrintf(pHlp, "\n");
2385
2386	/* the groups */
2387	c = 0;
2388	f = fGlobalForcedActions;
2389	PRINT_GROUP(VM_FF_,EXTERNAL_SUSPENDED,_MASK);
2390	PRINT_GROUP(VM_FF_,EXTERNAL_HALTED,_MASK);
2391	PRINT_GROUP(VM_FF_,HIGH_PRIORITY_PRE,_MASK);
2392	PRINT_GROUP(VM_FF_,HIGH_PRIORITY_PRE_RAW,_MASK);
2393	PRINT_GROUP(VM_FF_,HIGH_PRIORITY_POST,_MASK);
2394	PRINT_GROUP(VM_FF_,NORMAL_PRIORITY_POST,_MASK);
2395	PRINT_GROUP(VM_FF_,NORMAL_PRIORITY,_MASK);
2396	PRINT_GROUP(VM_FF_,ALL_REM,_MASK);
2397	if (c)
2398	pHlp->pfnPrintf(pHlp, "\n");
2399
2400	/*
2401	* Per CPU flags.
2402	*/
2403	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2404	{
2405	const uint32_t fLocalForcedActions = pVM->aCpus[i].fLocalForcedActions;
2406	pHlp->pfnPrintf(pHlp, "CPU %u FFs: %#RX32", i, fLocalForcedActions);
2407
2408	/* show the flag mnemonics */
2409	c = 0;
2410	f = fLocalForcedActions;
2411	PRINT_FLAG(VMCPU_FF_,INTERRUPT_APIC);
2412	PRINT_FLAG(VMCPU_FF_,INTERRUPT_PIC);
2413	PRINT_FLAG(VMCPU_FF_,TIMER);
2414	PRINT_FLAG(VMCPU_FF_,INTERRUPT_NMI);
2415	PRINT_FLAG(VMCPU_FF_,INTERRUPT_SMI);
2416	PRINT_FLAG(VMCPU_FF_,PDM_CRITSECT);
2417	PRINT_FLAG(VMCPU_FF_,UNHALT);
2418	PRINT_FLAG(VMCPU_FF_,IEM);
2419	PRINT_FLAG(VMCPU_FF_,REQUEST);
2420	PRINT_FLAG(VMCPU_FF_,HM_UPDATE_CR3);
2421	PRINT_FLAG(VMCPU_FF_,HM_UPDATE_PAE_PDPES);
2422	PRINT_FLAG(VMCPU_FF_,PGM_SYNC_CR3);
2423	PRINT_FLAG(VMCPU_FF_,PGM_SYNC_CR3_NON_GLOBAL);
2424	PRINT_FLAG(VMCPU_FF_,TLB_SHOOTDOWN);
2425	PRINT_FLAG(VMCPU_FF_,TLB_FLUSH);
2426	PRINT_FLAG(VMCPU_FF_,INHIBIT_INTERRUPTS);
2427	PRINT_FLAG(VMCPU_FF_,BLOCK_NMIS);
2428	PRINT_FLAG(VMCPU_FF_,TO_R3);
2429	#ifdef VBOX_WITH_RAW_MODE
2430	PRINT_FLAG(VMCPU_FF_,TRPM_SYNC_IDT);
2431	PRINT_FLAG(VMCPU_FF_,SELM_SYNC_TSS);
2432	PRINT_FLAG(VMCPU_FF_,SELM_SYNC_GDT);
2433	PRINT_FLAG(VMCPU_FF_,SELM_SYNC_LDT);
2434	PRINT_FLAG(VMCPU_FF_,CSAM_SCAN_PAGE);
2435	PRINT_FLAG(VMCPU_FF_,CSAM_PENDING_ACTION);
2436	#endif
2437	if (f)
2438	pHlp->pfnPrintf(pHlp, "%s\n Unknown bits: %#RX32\n", c ? "," : "", f);
2439	else
2440	pHlp->pfnPrintf(pHlp, "\n");
2441
2442	if (fLocalForcedActions & VMCPU_FF_INHIBIT_INTERRUPTS)
2443	pHlp->pfnPrintf(pHlp, " intr inhibit RIP: %RGp\n", EMGetInhibitInterruptsPC(&pVM->aCpus[i]));
2444
2445	/* the groups */
2446	c = 0;
2447	f = fLocalForcedActions;
2448	PRINT_GROUP(VMCPU_FF_,EXTERNAL_SUSPENDED,_MASK);
2449	PRINT_GROUP(VMCPU_FF_,EXTERNAL_HALTED,_MASK);
2450	PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_PRE,_MASK);
2451	PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_PRE_RAW,_MASK);
2452	PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_POST,_MASK);
2453	PRINT_GROUP(VMCPU_FF_,NORMAL_PRIORITY_POST,_MASK);
2454	PRINT_GROUP(VMCPU_FF_,NORMAL_PRIORITY,_MASK);
2455	PRINT_GROUP(VMCPU_FF_,RESUME_GUEST,_MASK);
2456	PRINT_GROUP(VMCPU_FF_,HM_TO_R3,_MASK);
2457	PRINT_GROUP(VMCPU_FF_,ALL_REM,_MASK);
2458	if (c)
2459	pHlp->pfnPrintf(pHlp, "\n");
2460	}
2461
2462	#undef PRINT_FLAG
2463	#undef PRINT_GROUP
2464	}
2465

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR3/VMM.cpp@ 57009

Download in other formats: