VMM.cpp@ 23495

Last change on this file since 23495 was 23145, checked in by vboxsync, 15 years ago
VMM: Extended VMMR3EmtRendezvous with TYPE_ASCENDING, TYPE_DESCENDING and STOP_ON_ERROR for use with VM state changes. The return type of the callback was changed so that the callback can feed scheduling info to EM.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 80.7 KB

Line
1	/* $Id: VMM.cpp 23145 2009-09-18 20:58:30Z vboxsync $ */
2	/** @file
3	* VMM - The Virtual Machine Monitor Core.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22	//#define NO_SUPCALLR0VMM
23
24	/** @page pg_vmm VMM - The Virtual Machine Monitor
25	*
26	* The VMM component is two things at the moment, it's a component doing a few
27	* management and routing tasks, and it's the whole virtual machine monitor
28	* thing. For hysterical reasons, it is not doing all the management that one
29	* would expect, this is instead done by @ref pg_vm. We'll address this
30	* misdesign eventually.
31	*
32	* @see grp_vmm, grp_vm
33	*
34	*
35	* @section sec_vmmstate VMM State
36	*
37	* @image html VM_Statechart_Diagram.gif
38	*
39	* To be written.
40	*
41	*
42	* @subsection subsec_vmm_init VMM Initialization
43	*
44	* To be written.
45	*
46	*
47	* @subsection subsec_vmm_term VMM Termination
48	*
49	* To be written.
50	*
51	*/
52
53	/*******************************************************************************
54	* Header Files *
55	*******************************************************************************/
56	#define LOG_GROUP LOG_GROUP_VMM
57	#include <VBox/vmm.h>
58	#include <VBox/vmapi.h>
59	#include <VBox/pgm.h>
60	#include <VBox/cfgm.h>
61	#include <VBox/pdmqueue.h>
62	#include <VBox/pdmcritsect.h>
63	#include <VBox/pdmapi.h>
64	#include <VBox/cpum.h>
65	#include <VBox/mm.h>
66	#include <VBox/iom.h>
67	#include <VBox/trpm.h>
68	#include <VBox/selm.h>
69	#include <VBox/em.h>
70	#include <VBox/sup.h>
71	#include <VBox/dbgf.h>
72	#include <VBox/csam.h>
73	#include <VBox/patm.h>
74	#include <VBox/rem.h>
75	#include <VBox/ssm.h>
76	#include <VBox/tm.h>
77	#include "VMMInternal.h"
78	#include "VMMSwitcher/VMMSwitcher.h"
79	#include <VBox/vm.h>
80
81	#include <VBox/err.h>
82	#include <VBox/param.h>
83	#include <VBox/version.h>
84	#include <VBox/x86.h>
85	#include <VBox/hwaccm.h>
86	#include <iprt/assert.h>
87	#include <iprt/alloc.h>
88	#include <iprt/asm.h>
89	#include <iprt/time.h>
90	#include <iprt/semaphore.h>
91	#include <iprt/stream.h>
92	#include <iprt/string.h>
93	#include <iprt/stdarg.h>
94	#include <iprt/ctype.h>
95
96
97
98	/*******************************************************************************
99	* Defined Constants And Macros *
100	*******************************************************************************/
101	/** The saved state version. */
102	#define VMM_SAVED_STATE_VERSION 3
103
104
105	/*******************************************************************************
106	* Internal Functions *
107	*******************************************************************************/
108	static int vmmR3InitStacks(PVM pVM);
109	static int vmmR3InitLoggers(PVM pVM);
110	static void vmmR3InitRegisterStats(PVM pVM);
111	static DECLCALLBACK(int) vmmR3Save(PVM pVM, PSSMHANDLE pSSM);
112	static DECLCALLBACK(int) vmmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
113	static DECLCALLBACK(void) vmmR3YieldEMT(PVM pVM, PTMTIMER pTimer, void *pvUser);
114	static int vmmR3ServiceCallRing3Request(PVM pVM, PVMCPU pVCpu);
115	static DECLCALLBACK(void) vmmR3InfoFF(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
116
117
118	/**
119	* Initializes the VMM.
120	*
121	* @returns VBox status code.
122	* @param pVM The VM to operate on.
123	*/
124	VMMR3DECL(int) VMMR3Init(PVM pVM)
125	{
126	LogFlow(("VMMR3Init\n"));
127
128	/*
129	* Assert alignment, sizes and order.
130	*/
131	AssertMsg(pVM->vmm.s.offVM == 0, ("Already initialized!\n"));
132	AssertCompile(sizeof(pVM->vmm.s) <= sizeof(pVM->vmm.padding));
133	AssertCompile(sizeof(pVM->aCpus[0].vmm.s) <= sizeof(pVM->aCpus[0].vmm.padding));
134
135	/*
136	* Init basic VM VMM members.
137	*/
138	pVM->vmm.s.offVM = RT_OFFSETOF(VM, vmm);
139	pVM->vmm.s.pahEvtRendezvousEnterOrdered = NULL;
140	pVM->vmm.s.hEvtRendezvousEnterOneByOne = NIL_RTSEMEVENT;
141	pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce = NIL_RTSEMEVENTMULTI;
142	pVM->vmm.s.hEvtMulRendezvousDone = NIL_RTSEMEVENTMULTI;
143	pVM->vmm.s.hEvtRendezvousDoneCaller = NIL_RTSEMEVENT;
144	int rc = CFGMR3QueryU32(CFGMR3GetRoot(pVM), "YieldEMTInterval", &pVM->vmm.s.cYieldEveryMillies);
145	if (rc == VERR_CFGM_VALUE_NOT_FOUND)
146	pVM->vmm.s.cYieldEveryMillies = 23; /* Value arrived at after experimenting with the grub boot prompt. */
147	//pVM->vmm.s.cYieldEveryMillies = 8; //debugging
148	else
149	AssertMsgRCReturn(rc, ("Configuration error. Failed to query \"YieldEMTInterval\", rc=%Rrc\n", rc), rc);
150
151	/*
152	* Initialize the VMM sync critical section and semaphores.
153	*/
154	rc = RTCritSectInit(&pVM->vmm.s.CritSectSync);
155	AssertRCReturn(rc, rc);
156	pVM->vmm.s.pahEvtRendezvousEnterOrdered = (PRTSEMEVENT)MMR3HeapAlloc(pVM, MM_TAG_VMM, sizeof(RTSEMEVENT) * pVM->cCpus);
157	if (!pVM->vmm.s.pahEvtRendezvousEnterOrdered)
158	return VERR_NO_MEMORY;
159	for (VMCPUID i = 0; i < pVM->cCpus; i++)
160	pVM->vmm.s.pahEvtRendezvousEnterOrdered[i] = NIL_RTSEMEVENT;
161	for (VMCPUID i = 0; i < pVM->cCpus; i++)
162	{
163	rc = RTSemEventCreate(&pVM->vmm.s.pahEvtRendezvousEnterOrdered[i]);
164	AssertRCReturn(rc, rc);
165	}
166	rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousEnterOneByOne);
167	AssertRCReturn(rc, rc);
168	rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
169	AssertRCReturn(rc, rc);
170	rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousDone);
171	AssertRCReturn(rc, rc);
172	rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousDoneCaller);
173	AssertRCReturn(rc, rc);
174
175	/* GC switchers are enabled by default. Turned off by HWACCM. */
176	pVM->vmm.s.fSwitcherDisabled = false;
177
178	/*
179	* Register the saved state data unit.
180	*/
181	rc = SSMR3RegisterInternal(pVM, "vmm", 1, VMM_SAVED_STATE_VERSION, VMM_STACK_SIZE + sizeof(RTGCPTR),
182	NULL, NULL, NULL,
183	NULL, vmmR3Save, NULL,
184	NULL, vmmR3Load, NULL);
185	if (RT_FAILURE(rc))
186	return rc;
187
188	/*
189	* Register the Ring-0 VM handle with the session for fast ioctl calls.
190	*/
191	rc = SUPR3SetVMForFastIOCtl(pVM->pVMR0);
192	if (RT_FAILURE(rc))
193	return rc;
194
195	/*
196	* Init various sub-components.
197	*/
198	rc = vmmR3SwitcherInit(pVM);
199	if (RT_SUCCESS(rc))
200	{
201	rc = vmmR3InitStacks(pVM);
202	if (RT_SUCCESS(rc))
203	{
204	rc = vmmR3InitLoggers(pVM);
205
206	#ifdef VBOX_WITH_NMI
207	/*
208	* Allocate mapping for the host APIC.
209	*/
210	if (RT_SUCCESS(rc))
211	{
212	rc = MMR3HyperReserve(pVM, PAGE_SIZE, "Host APIC", &pVM->vmm.s.GCPtrApicBase);
213	AssertRC(rc);
214	}
215	#endif
216	if (RT_SUCCESS(rc))
217	{
218	/*
219	* Debug info and statistics.
220	*/
221	DBGFR3InfoRegisterInternal(pVM, "ff", "Displays the current Forced actions Flags.", vmmR3InfoFF);
222	vmmR3InitRegisterStats(pVM);
223
224	return VINF_SUCCESS;
225	}
226	}
227	/** @todo: Need failure cleanup. */
228
229	//more todo in here?
230	//if (RT_SUCCESS(rc))
231	//{
232	//}
233	//int rc2 = vmmR3TermCoreCode(pVM);
234	//AssertRC(rc2));
235	}
236
237	return rc;
238	}
239
240
241	/**
242	* Allocate & setup the VMM RC stack(s) (for EMTs).
243	*
244	* The stacks are also used for long jumps in Ring-0.
245	*
246	* @returns VBox status code.
247	* @param pVM Pointer to the shared VM structure.
248	*
249	* @remarks The optional guard page gets it protection setup up during R3 init
250	* completion because of init order issues.
251	*/
252	static int vmmR3InitStacks(PVM pVM)
253	{
254	int rc = VINF_SUCCESS;
255	#ifdef VMM_R0_SWITCH_STACK
256	uint32_t fFlags = MMHYPER_AONR_FLAGS_KERNEL_MAPPING;
257	#else
258	uint32_t fFlags = 0;
259	#endif
260
261	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
262	{
263	PVMCPU pVCpu = &pVM->aCpus[idCpu];
264
265	#ifdef VBOX_STRICT_VMM_STACK
266	rc = MMR3HyperAllocOnceNoRelEx(pVM, PAGE_SIZE + VMM_STACK_SIZE + PAGE_SIZE,
267	#else
268	rc = MMR3HyperAllocOnceNoRelEx(pVM, VMM_STACK_SIZE,
269	#endif
270	PAGE_SIZE, MM_TAG_VMM, fFlags, (void **)&pVCpu->vmm.s.pbEMTStackR3);
271	if (RT_SUCCESS(rc))
272	{
273	#ifdef VBOX_STRICT_VMM_STACK
274	pVCpu->vmm.s.pbEMTStackR3 += PAGE_SIZE;
275	#endif
276	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
277	/* MMHyperR3ToR0 returns R3 when not doing hardware assisted virtualization. */
278	if (!VMMIsHwVirtExtForced(pVM))
279	pVCpu->vmm.s.CallRing3JmpBufR0.pvSavedStack = NIL_RTR0PTR;
280	else
281	#endif
282	pVCpu->vmm.s.CallRing3JmpBufR0.pvSavedStack = MMHyperR3ToR0(pVM, pVCpu->vmm.s.pbEMTStackR3);
283	pVCpu->vmm.s.pbEMTStackRC = MMHyperR3ToRC(pVM, pVCpu->vmm.s.pbEMTStackR3);
284	pVCpu->vmm.s.pbEMTStackBottomRC = pVCpu->vmm.s.pbEMTStackRC + VMM_STACK_SIZE;
285	AssertRelease(pVCpu->vmm.s.pbEMTStackRC);
286
287	CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC);
288	}
289	}
290
291	return rc;
292	}
293
294
295	/**
296	* Initialize the loggers.
297	*
298	* @returns VBox status code.
299	* @param pVM Pointer to the shared VM structure.
300	*/
301	static int vmmR3InitLoggers(PVM pVM)
302	{
303	int rc;
304
305	/*
306	* Allocate RC & R0 Logger instances (they are finalized in the relocator).
307	*/
308	#ifdef LOG_ENABLED
309	PRTLOGGER pLogger = RTLogDefaultInstance();
310	if (pLogger)
311	{
312	pVM->vmm.s.cbRCLogger = RT_OFFSETOF(RTLOGGERRC, afGroups[pLogger->cGroups]);
313	rc = MMR3HyperAllocOnceNoRel(pVM, pVM->vmm.s.cbRCLogger, 0, MM_TAG_VMM, (void **)&pVM->vmm.s.pRCLoggerR3);
314	if (RT_FAILURE(rc))
315	return rc;
316	pVM->vmm.s.pRCLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCLoggerR3);
317
318	# ifdef VBOX_WITH_R0_LOGGING
319	for (VMCPUID i = 0; i < pVM->cCpus; i++)
320	{
321	PVMCPU pVCpu = &pVM->aCpus[i];
322
323	rc = MMR3HyperAllocOnceNoRelEx(pVM, RT_OFFSETOF(VMMR0LOGGER, Logger.afGroups[pLogger->cGroups]),
324	0, MM_TAG_VMM, MMHYPER_AONR_FLAGS_KERNEL_MAPPING,
325	(void **)&pVCpu->vmm.s.pR0LoggerR3);
326	if (RT_FAILURE(rc))
327	return rc;
328	pVCpu->vmm.s.pR0LoggerR3->pVM = pVM->pVMR0;
329	//pVCpu->vmm.s.pR0LoggerR3->fCreated = false;
330	pVCpu->vmm.s.pR0LoggerR3->cbLogger = RT_OFFSETOF(RTLOGGER, afGroups[pLogger->cGroups]);
331	pVCpu->vmm.s.pR0LoggerR0 = MMHyperR3ToR0(pVM, pVCpu->vmm.s.pR0LoggerR3);
332	}
333	# endif
334	}
335	#endif /* LOG_ENABLED */
336
337	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
338	/*
339	* Allocate RC release logger instances (finalized in the relocator).
340	*/
341	PRTLOGGER pRelLogger = RTLogRelDefaultInstance();
342	if (pRelLogger)
343	{
344	pVM->vmm.s.cbRCRelLogger = RT_OFFSETOF(RTLOGGERRC, afGroups[pRelLogger->cGroups]);
345	rc = MMR3HyperAllocOnceNoRel(pVM, pVM->vmm.s.cbRCRelLogger, 0, MM_TAG_VMM, (void **)&pVM->vmm.s.pRCRelLoggerR3);
346	if (RT_FAILURE(rc))
347	return rc;
348	pVM->vmm.s.pRCRelLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCRelLoggerR3);
349	}
350	#endif /* VBOX_WITH_RC_RELEASE_LOGGING */
351	return VINF_SUCCESS;
352	}
353
354
355	/**
356	* VMMR3Init worker that register the statistics with STAM.
357	*
358	* @param pVM The shared VM structure.
359	*/
360	static void vmmR3InitRegisterStats(PVM pVM)
361	{
362	/*
363	* Statistics.
364	*/
365	STAM_REG(pVM, &pVM->vmm.s.StatRunRC, STAMTYPE_COUNTER, "/VMM/RunRC", STAMUNIT_OCCURENCES, "Number of context switches.");
366	STAM_REG(pVM, &pVM->vmm.s.StatRZRetNormal, STAMTYPE_COUNTER, "/VMM/RZRet/Normal", STAMUNIT_OCCURENCES, "Number of VINF_SUCCESS returns.");
367	STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterrupt, STAMTYPE_COUNTER, "/VMM/RZRet/Interrupt", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT returns.");
368	STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterruptHyper, STAMTYPE_COUNTER, "/VMM/RZRet/InterruptHyper", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT_HYPER returns.");
369	STAM_REG(pVM, &pVM->vmm.s.StatRZRetGuestTrap, STAMTYPE_COUNTER, "/VMM/RZRet/GuestTrap", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_GUEST_TRAP returns.");
370	STAM_REG(pVM, &pVM->vmm.s.StatRZRetRingSwitch, STAMTYPE_COUNTER, "/VMM/RZRet/RingSwitch", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_RING_SWITCH returns.");
371	STAM_REG(pVM, &pVM->vmm.s.StatRZRetRingSwitchInt, STAMTYPE_COUNTER, "/VMM/RZRet/RingSwitchInt", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_RING_SWITCH_INT returns.");
372	STAM_REG(pVM, &pVM->vmm.s.StatRZRetStaleSelector, STAMTYPE_COUNTER, "/VMM/RZRet/StaleSelector", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_STALE_SELECTOR returns.");
373	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIRETTrap, STAMTYPE_COUNTER, "/VMM/RZRet/IRETTrap", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_IRET_TRAP returns.");
374	STAM_REG(pVM, &pVM->vmm.s.StatRZRetEmulate, STAMTYPE_COUNTER, "/VMM/RZRet/Emulate", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION returns.");
375	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIOBlockEmulate, STAMTYPE_COUNTER, "/VMM/RZRet/EmulateIOBlock", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_EMULATE_IO_BLOCK returns.");
376	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchEmulate, STAMTYPE_COUNTER, "/VMM/RZRet/PatchEmulate", STAMUNIT_OCCURENCES, "Number of VINF_PATCH_EMULATE_INSTR returns.");
377	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIORead, STAMTYPE_COUNTER, "/VMM/RZRet/IORead", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_IOPORT_READ returns.");
378	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIOWrite, STAMTYPE_COUNTER, "/VMM/RZRet/IOWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_IOPORT_WRITE returns.");
379	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIORead, STAMTYPE_COUNTER, "/VMM/RZRet/MMIORead", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_READ returns.");
380	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOWrite, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_WRITE returns.");
381	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOReadWrite, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOReadWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_READ_WRITE returns.");
382	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOPatchRead, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOPatchRead", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_PATCH_READ returns.");
383	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOPatchWrite, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOPatchWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_PATCH_WRITE returns.");
384	STAM_REG(pVM, &pVM->vmm.s.StatRZRetLDTFault, STAMTYPE_COUNTER, "/VMM/RZRet/LDTFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_GDT_FAULT returns.");
385	STAM_REG(pVM, &pVM->vmm.s.StatRZRetGDTFault, STAMTYPE_COUNTER, "/VMM/RZRet/GDTFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_LDT_FAULT returns.");
386	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIDTFault, STAMTYPE_COUNTER, "/VMM/RZRet/IDTFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_IDT_FAULT returns.");
387	STAM_REG(pVM, &pVM->vmm.s.StatRZRetTSSFault, STAMTYPE_COUNTER, "/VMM/RZRet/TSSFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_TSS_FAULT returns.");
388	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPDFault, STAMTYPE_COUNTER, "/VMM/RZRet/PDFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_PD_FAULT returns.");
389	STAM_REG(pVM, &pVM->vmm.s.StatRZRetCSAMTask, STAMTYPE_COUNTER, "/VMM/RZRet/CSAMTask", STAMUNIT_OCCURENCES, "Number of VINF_CSAM_PENDING_ACTION returns.");
390	STAM_REG(pVM, &pVM->vmm.s.StatRZRetSyncCR3, STAMTYPE_COUNTER, "/VMM/RZRet/SyncCR", STAMUNIT_OCCURENCES, "Number of VINF_PGM_SYNC_CR3 returns.");
391	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMisc, STAMTYPE_COUNTER, "/VMM/RZRet/Misc", STAMUNIT_OCCURENCES, "Number of misc returns.");
392	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchInt3, STAMTYPE_COUNTER, "/VMM/RZRet/PatchInt3", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_INT3 returns.");
393	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchPF, STAMTYPE_COUNTER, "/VMM/RZRet/PatchPF", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_TRAP_PF returns.");
394	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchGP, STAMTYPE_COUNTER, "/VMM/RZRet/PatchGP", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_TRAP_GP returns.");
395	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchIretIRQ, STAMTYPE_COUNTER, "/VMM/RZRet/PatchIret", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PENDING_IRQ_AFTER_IRET returns.");
396	STAM_REG(pVM, &pVM->vmm.s.StatRZRetRescheduleREM, STAMTYPE_COUNTER, "/VMM/RZRet/ScheduleREM", STAMUNIT_OCCURENCES, "Number of VINF_EM_RESCHEDULE_REM returns.");
397	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
398	STAM_REG(pVM, &pVM->vmm.s.StatRZRetTimerPending, STAMTYPE_COUNTER, "/VMM/RZRet/TimerPending", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TIMER_PENDING returns.");
399	STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterruptPending, STAMTYPE_COUNTER, "/VMM/RZRet/InterruptPending", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT_PENDING returns.");
400	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPATMDuplicateFn, STAMTYPE_COUNTER, "/VMM/RZRet/PATMDuplicateFn", STAMUNIT_OCCURENCES, "Number of VINF_PATM_DUPLICATE_FUNCTION returns.");
401	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPGMChangeMode, STAMTYPE_COUNTER, "/VMM/RZRet/PGMChangeMode", STAMUNIT_OCCURENCES, "Number of VINF_PGM_CHANGE_MODE returns.");
402	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPendingRequest, STAMTYPE_COUNTER, "/VMM/RZRet/PendingRequest", STAMUNIT_OCCURENCES, "Number of VINF_EM_PENDING_REQUEST returns.");
403	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchTPR, STAMTYPE_COUNTER, "/VMM/RZRet/PatchTPR", STAMUNIT_OCCURENCES, "Number of VINF_EM_HWACCM_PATCH_TPR_INSTR returns.");
404	STAM_REG(pVM, &pVM->vmm.s.StatRZRetCallRing3, STAMTYPE_COUNTER, "/VMM/RZCallR3/Misc", STAMUNIT_OCCURENCES, "Number of Other ring-3 calls.");
405	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPDMLock, STAMTYPE_COUNTER, "/VMM/RZCallR3/PDMLock", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PDM_LOCK calls.");
406	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPDMQueueFlush, STAMTYPE_COUNTER, "/VMM/RZCallR3/PDMQueueFlush", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PDM_QUEUE_FLUSH calls.");
407	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMLock, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMLock", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_LOCK calls.");
408	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMPoolGrow, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMPoolGrow", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_POOL_GROW calls.");
409	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMMapChunk, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMMapChunk", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_MAP_CHUNK calls.");
410	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMAllocHandy, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMAllocHandy", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES calls.");
411	STAM_REG(pVM, &pVM->vmm.s.StatRZCallRemReplay, STAMTYPE_COUNTER, "/VMM/RZCallR3/REMReplay", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_REM_REPLAY_HANDLER_NOTIFICATIONS calls.");
412	STAM_REG(pVM, &pVM->vmm.s.StatRZCallLogFlush, STAMTYPE_COUNTER, "/VMM/RZCallR3/VMMLogFlush", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VMM_LOGGER_FLUSH calls.");
413	STAM_REG(pVM, &pVM->vmm.s.StatRZCallVMSetError, STAMTYPE_COUNTER, "/VMM/RZCallR3/VMSetError", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VM_SET_ERROR calls.");
414	STAM_REG(pVM, &pVM->vmm.s.StatRZCallVMSetRuntimeError, STAMTYPE_COUNTER, "/VMM/RZCallR3/VMRuntimeError", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VM_SET_RUNTIME_ERROR calls.");
415
416	#ifdef VBOX_WITH_STATISTICS
417	for (VMCPUID i = 0; i < pVM->cCpus; i++)
418	{
419	STAMR3RegisterF(pVM, &pVM->aCpus[i].vmm.s.CallRing3JmpBufR0.cbUsedMax, STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Max amount of stack used.", "/VMM/Stack/CPU%u/Max", i);
420	STAMR3RegisterF(pVM, &pVM->aCpus[i].vmm.s.CallRing3JmpBufR0.cbUsedAvg, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Average stack usage.", "/VMM/Stack/CPU%u/Avg", i);
421	STAMR3RegisterF(pVM, &pVM->aCpus[i].vmm.s.CallRing3JmpBufR0.cUsedTotal, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of stack usages.", "/VMM/Stack/CPU%u/Uses", i);
422	}
423	#endif
424	}
425
426
427	/**
428	* Initializes the per-VCPU VMM.
429	*
430	* @returns VBox status code.
431	* @param pVM The VM to operate on.
432	*/
433	VMMR3DECL(int) VMMR3InitCPU(PVM pVM)
434	{
435	LogFlow(("VMMR3InitCPU\n"));
436	return VINF_SUCCESS;
437	}
438
439
440	/**
441	* Ring-3 init finalizing.
442	*
443	* @returns VBox status code.
444	* @param pVM The VM handle.
445	*/
446	VMMR3DECL(int) VMMR3InitFinalize(PVM pVM)
447	{
448	int rc = VINF_SUCCESS;
449
450	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
451	{
452	PVMCPU pVCpu = &pVM->aCpus[idCpu];
453
454	#ifdef VBOX_STRICT_VMM_STACK
455	/*
456	* Two inaccessible pages at each sides of the stack to catch over/under-flows.
457	*/
458	memset(pVCpu->vmm.s.pbEMTStackR3 - PAGE_SIZE, 0xcc, PAGE_SIZE);
459	MMR3HyperSetGuard(pVM, pVCpu->vmm.s.pbEMTStackR3 - PAGE_SIZE, PAGE_SIZE, true /fSet/);
460
461	memset(pVCpu->vmm.s.pbEMTStackR3 + VMM_STACK_SIZE, 0xcc, PAGE_SIZE);
462	MMR3HyperSetGuard(pVM, pVCpu->vmm.s.pbEMTStackR3 + VMM_STACK_SIZE, PAGE_SIZE, true /fSet/);
463	#endif
464
465	/*
466	* Set page attributes to r/w for stack pages.
467	*/
468	rc = PGMMapSetPage(pVM, pVCpu->vmm.s.pbEMTStackRC, VMM_STACK_SIZE, X86_PTE_P \| X86_PTE_A \| X86_PTE_D \| X86_PTE_RW);
469	AssertRC(rc);
470	if (RT_FAILURE(rc))
471	break;
472	}
473	if (RT_SUCCESS(rc))
474	{
475	/*
476	* Create the EMT yield timer.
477	*/
478	rc = TMR3TimerCreateInternal(pVM, TMCLOCK_REAL, vmmR3YieldEMT, NULL, "EMT Yielder", &pVM->vmm.s.pYieldTimer);
479	if (RT_SUCCESS(rc))
480	rc = TMTimerSetMillies(pVM->vmm.s.pYieldTimer, pVM->vmm.s.cYieldEveryMillies);
481	}
482
483	#ifdef VBOX_WITH_NMI
484	/*
485	* Map the host APIC into GC - This is AMD/Intel + Host OS specific!
486	*/
487	if (RT_SUCCESS(rc))
488	rc = PGMMap(pVM, pVM->vmm.s.GCPtrApicBase, 0xfee00000, PAGE_SIZE,
489	X86_PTE_P \| X86_PTE_RW \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_A \| X86_PTE_D);
490	#endif
491	return rc;
492	}
493
494
495	/**
496	* Initializes the R0 VMM.
497	*
498	* @returns VBox status code.
499	* @param pVM The VM to operate on.
500	*/
501	VMMR3DECL(int) VMMR3InitR0(PVM pVM)
502	{
503	int rc;
504	PVMCPU pVCpu = VMMGetCpu(pVM);
505	Assert(pVCpu && pVCpu->idCpu == 0);
506
507	#ifdef LOG_ENABLED
508	/*
509	* Initialize the ring-0 logger if we haven't done so yet.
510	*/
511	if ( pVCpu->vmm.s.pR0LoggerR3
512	&& !pVCpu->vmm.s.pR0LoggerR3->fCreated)
513	{
514	rc = VMMR3UpdateLoggers(pVM);
515	if (RT_FAILURE(rc))
516	return rc;
517	}
518	#endif
519
520	/*
521	* Call Ring-0 entry with init code.
522	*/
523	for (;;)
524	{
525	#ifdef NO_SUPCALLR0VMM
526	//rc = VERR_GENERAL_FAILURE;
527	rc = VINF_SUCCESS;
528	#else
529	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /idCpu/, VMMR0_DO_VMMR0_INIT, VMMGetSvnRev(), NULL);
530	#endif
531	/*
532	* Flush the logs.
533	*/
534	#ifdef LOG_ENABLED
535	if ( pVCpu->vmm.s.pR0LoggerR3
536	&& pVCpu->vmm.s.pR0LoggerR3->Logger.offScratch > 0)
537	RTLogFlushToLogger(&pVCpu->vmm.s.pR0LoggerR3->Logger, NULL);
538	#endif
539	if (rc != VINF_VMM_CALL_HOST)
540	break;
541	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
542	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
543	break;
544	/* Resume R0 */
545	}
546
547	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
548	{
549	LogRel(("R0 init failed, rc=%Rra\n", rc));
550	if (RT_SUCCESS(rc))
551	rc = VERR_INTERNAL_ERROR;
552	}
553	return rc;
554	}
555
556
557	/**
558	* Initializes the RC VMM.
559	*
560	* @returns VBox status code.
561	* @param pVM The VM to operate on.
562	*/
563	VMMR3DECL(int) VMMR3InitRC(PVM pVM)
564	{
565	PVMCPU pVCpu = VMMGetCpu(pVM);
566	Assert(pVCpu && pVCpu->idCpu == 0);
567
568	/* In VMX mode, there's no need to init RC. */
569	if (pVM->vmm.s.fSwitcherDisabled)
570	return VINF_SUCCESS;
571
572	AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
573
574	/*
575	* Call VMMGCInit():
576	* -# resolve the address.
577	* -# setup stackframe and EIP to use the trampoline.
578	* -# do a generic hypervisor call.
579	*/
580	RTRCPTR RCPtrEP;
581	int rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "VMMGCEntry", &RCPtrEP);
582	if (RT_SUCCESS(rc))
583	{
584	CPUMHyperSetCtxCore(pVCpu, NULL);
585	CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC); /* Clear the stack. */
586	uint64_t u64TS = RTTimeProgramStartNanoTS();
587	CPUMPushHyper(pVCpu, (uint32_t)(u64TS >> 32)); /* Param 3: The program startup TS - Hi. */
588	CPUMPushHyper(pVCpu, (uint32_t)u64TS); /* Param 3: The program startup TS - Lo. */
589	CPUMPushHyper(pVCpu, VMMGetSvnRev()); /* Param 2: Version argument. */
590	CPUMPushHyper(pVCpu, VMMGC_DO_VMMGC_INIT); /* Param 1: Operation. */
591	CPUMPushHyper(pVCpu, pVM->pVMRC); /* Param 0: pVM */
592	CPUMPushHyper(pVCpu, 5 * sizeof(RTRCPTR)); /* trampoline param: stacksize. */
593	CPUMPushHyper(pVCpu, RCPtrEP); /* Call EIP. */
594	CPUMSetHyperEIP(pVCpu, pVM->vmm.s.pfnCallTrampolineRC);
595	Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
596
597	for (;;)
598	{
599	#ifdef NO_SUPCALLR0VMM
600	//rc = VERR_GENERAL_FAILURE;
601	rc = VINF_SUCCESS;
602	#else
603	rc = SUPR3CallVMMR0(pVM->pVMR0, 0 /* VCPU 0 */, VMMR0_DO_CALL_HYPERVISOR, NULL);
604	#endif
605	#ifdef LOG_ENABLED
606	PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
607	if ( pLogger
608	&& pLogger->offScratch > 0)
609	RTLogFlushRC(NULL, pLogger);
610	#endif
611	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
612	PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
613	if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
614	RTLogFlushRC(RTLogRelDefaultInstance(), pRelLogger);
615	#endif
616	if (rc != VINF_VMM_CALL_HOST)
617	break;
618	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
619	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
620	break;
621	}
622
623	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
624	{
625	VMMR3FatalDump(pVM, pVCpu, rc);
626	if (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST)
627	rc = VERR_INTERNAL_ERROR;
628	}
629	AssertRC(rc);
630	}
631	return rc;
632	}
633
634
635	/**
636	* Terminate the VMM bits.
637	*
638	* @returns VINF_SUCCESS.
639	* @param pVM The VM handle.
640	*/
641	VMMR3DECL(int) VMMR3Term(PVM pVM)
642	{
643	PVMCPU pVCpu = VMMGetCpu(pVM);
644	Assert(pVCpu && pVCpu->idCpu == 0);
645
646	/*
647	* Call Ring-0 entry with termination code.
648	*/
649	int rc;
650	for (;;)
651	{
652	#ifdef NO_SUPCALLR0VMM
653	//rc = VERR_GENERAL_FAILURE;
654	rc = VINF_SUCCESS;
655	#else
656	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /idCpu/, VMMR0_DO_VMMR0_TERM, 0, NULL);
657	#endif
658	/*
659	* Flush the logs.
660	*/
661	#ifdef LOG_ENABLED
662	if ( pVCpu->vmm.s.pR0LoggerR3
663	&& pVCpu->vmm.s.pR0LoggerR3->Logger.offScratch > 0)
664	RTLogFlushToLogger(&pVCpu->vmm.s.pR0LoggerR3->Logger, NULL);
665	#endif
666	if (rc != VINF_VMM_CALL_HOST)
667	break;
668	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
669	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
670	break;
671	/* Resume R0 */
672	}
673	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
674	{
675	LogRel(("VMMR3Term: R0 term failed, rc=%Rra. (warning)\n", rc));
676	if (RT_SUCCESS(rc))
677	rc = VERR_INTERNAL_ERROR;
678	}
679
680	RTCritSectDelete(&pVM->vmm.s.CritSectSync);
681	for (VMCPUID i = 0; i < pVM->cCpus; i++)
682	{
683	RTSemEventDestroy(pVM->vmm.s.pahEvtRendezvousEnterOrdered[i]);
684	pVM->vmm.s.pahEvtRendezvousEnterOrdered[i] = NIL_RTSEMEVENT;
685	}
686	RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousEnterOneByOne);
687	pVM->vmm.s.hEvtRendezvousEnterOneByOne = NIL_RTSEMEVENT;
688	RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
689	pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce = NIL_RTSEMEVENTMULTI;
690	RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousDone);
691	pVM->vmm.s.hEvtMulRendezvousDone = NIL_RTSEMEVENTMULTI;
692	RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousDoneCaller);
693	pVM->vmm.s.hEvtRendezvousDoneCaller = NIL_RTSEMEVENT;
694
695	#ifdef VBOX_STRICT_VMM_STACK
696	/*
697	* Make the two stack guard pages present again.
698	*/
699	for (VMCPUID i = 0; i < pVM->cCpus; i++)
700	{
701	MMR3HyperSetGuard(pVM, pVM->aCpus[i].vmm.s.pbEMTStackR3 - PAGE_SIZE, PAGE_SIZE, false /fSet/);
702	MMR3HyperSetGuard(pVM, pVM->aCpus[i].vmm.s.pbEMTStackR3 + VMM_STACK_SIZE, PAGE_SIZE, false /fSet/);
703	}
704	#endif
705	return rc;
706	}
707
708
709	/**
710	* Terminates the per-VCPU VMM.
711	*
712	* Termination means cleaning up and freeing all resources,
713	* the VM it self is at this point powered off or suspended.
714	*
715	* @returns VBox status code.
716	* @param pVM The VM to operate on.
717	*/
718	VMMR3DECL(int) VMMR3TermCPU(PVM pVM)
719	{
720	return VINF_SUCCESS;
721	}
722
723
724	/**
725	* Applies relocations to data and code managed by this
726	* component. This function will be called at init and
727	* whenever the VMM need to relocate it self inside the GC.
728	*
729	* The VMM will need to apply relocations to the core code.
730	*
731	* @param pVM The VM handle.
732	* @param offDelta The relocation delta.
733	*/
734	VMMR3DECL(void) VMMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
735	{
736	LogFlow(("VMMR3Relocate: offDelta=%RGv\n", offDelta));
737
738	/*
739	* Recalc the RC address.
740	*/
741	pVM->vmm.s.pvCoreCodeRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pvCoreCodeR3);
742
743	/*
744	* The stack.
745	*/
746	for (VMCPUID i = 0; i < pVM->cCpus; i++)
747	{
748	PVMCPU pVCpu = &pVM->aCpus[i];
749
750	CPUMSetHyperESP(pVCpu, CPUMGetHyperESP(pVCpu) + offDelta);
751
752	pVCpu->vmm.s.pbEMTStackRC = MMHyperR3ToRC(pVM, pVCpu->vmm.s.pbEMTStackR3);
753	pVCpu->vmm.s.pbEMTStackBottomRC = pVCpu->vmm.s.pbEMTStackRC + VMM_STACK_SIZE;
754	}
755
756	/*
757	* All the switchers.
758	*/
759	vmmR3SwitcherRelocate(pVM, offDelta);
760
761	/*
762	* Get other RC entry points.
763	*/
764	int rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "CPUMGCResumeGuest", &pVM->vmm.s.pfnCPUMRCResumeGuest);
765	AssertReleaseMsgRC(rc, ("CPUMGCResumeGuest not found! rc=%Rra\n", rc));
766
767	rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "CPUMGCResumeGuestV86", &pVM->vmm.s.pfnCPUMRCResumeGuestV86);
768	AssertReleaseMsgRC(rc, ("CPUMGCResumeGuestV86 not found! rc=%Rra\n", rc));
769
770	/*
771	* Update the logger.
772	*/
773	VMMR3UpdateLoggers(pVM);
774	}
775
776
777	/**
778	* Updates the settings for the RC and R0 loggers.
779	*
780	* @returns VBox status code.
781	* @param pVM The VM handle.
782	*/
783	VMMR3DECL(int) VMMR3UpdateLoggers(PVM pVM)
784	{
785	/*
786	* Simply clone the logger instance (for RC).
787	*/
788	int rc = VINF_SUCCESS;
789	RTRCPTR RCPtrLoggerFlush = 0;
790
791	if (pVM->vmm.s.pRCLoggerR3
792	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
793	\|\| pVM->vmm.s.pRCRelLoggerR3
794	#endif
795	)
796	{
797	rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "vmmGCLoggerFlush", &RCPtrLoggerFlush);
798	AssertReleaseMsgRC(rc, ("vmmGCLoggerFlush not found! rc=%Rra\n", rc));
799	}
800
801	if (pVM->vmm.s.pRCLoggerR3)
802	{
803	RTRCPTR RCPtrLoggerWrapper = 0;
804	rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "vmmGCLoggerWrapper", &RCPtrLoggerWrapper);
805	AssertReleaseMsgRC(rc, ("vmmGCLoggerWrapper not found! rc=%Rra\n", rc));
806
807	pVM->vmm.s.pRCLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCLoggerR3);
808	rc = RTLogCloneRC(NULL /* default */, pVM->vmm.s.pRCLoggerR3, pVM->vmm.s.cbRCLogger,
809	RCPtrLoggerWrapper, RCPtrLoggerFlush, RTLOGFLAGS_BUFFERED);
810	AssertReleaseMsgRC(rc, ("RTLogCloneRC failed! rc=%Rra\n", rc));
811	}
812
813	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
814	if (pVM->vmm.s.pRCRelLoggerR3)
815	{
816	RTRCPTR RCPtrLoggerWrapper = 0;
817	rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "vmmGCRelLoggerWrapper", &RCPtrLoggerWrapper);
818	AssertReleaseMsgRC(rc, ("vmmGCRelLoggerWrapper not found! rc=%Rra\n", rc));
819
820	pVM->vmm.s.pRCRelLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCRelLoggerR3);
821	rc = RTLogCloneRC(RTLogRelDefaultInstance(), pVM->vmm.s.pRCRelLoggerR3, pVM->vmm.s.cbRCRelLogger,
822	RCPtrLoggerWrapper, RCPtrLoggerFlush, RTLOGFLAGS_BUFFERED);
823	AssertReleaseMsgRC(rc, ("RTLogCloneRC failed! rc=%Rra\n", rc));
824	}
825	#endif /* VBOX_WITH_RC_RELEASE_LOGGING */
826
827	#ifdef LOG_ENABLED
828	/*
829	* For the ring-0 EMT logger, we use a per-thread logger instance
830	* in ring-0. Only initialize it once.
831	*/
832	for (VMCPUID i = 0; i < pVM->cCpus; i++)
833	{
834	PVMCPU pVCpu = &pVM->aCpus[i];
835	PVMMR0LOGGER pR0LoggerR3 = pVCpu->vmm.s.pR0LoggerR3;
836	if (pR0LoggerR3)
837	{
838	if (!pR0LoggerR3->fCreated)
839	{
840	RTR0PTR pfnLoggerWrapper = NIL_RTR0PTR;
841	rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerWrapper", &pfnLoggerWrapper);
842	AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerWrapper not found! rc=%Rra\n", rc), rc);
843
844	RTR0PTR pfnLoggerFlush = NIL_RTR0PTR;
845	rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerFlush", &pfnLoggerFlush);
846	AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerFlush not found! rc=%Rra\n", rc), rc);
847
848	rc = RTLogCreateForR0(&pR0LoggerR3->Logger, pR0LoggerR3->cbLogger,
849	(PFNRTLOGGER )&pfnLoggerWrapper, (PFNRTLOGFLUSH )&pfnLoggerFlush,
850	RTLOGFLAGS_BUFFERED, RTLOGDEST_DUMMY);
851	AssertReleaseMsgRCReturn(rc, ("RTLogCreateForR0 failed! rc=%Rra\n", rc), rc);
852
853	RTR0PTR pfnLoggerPrefix = NIL_RTR0PTR;
854	rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerPrefix", &pfnLoggerPrefix);
855	AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerPrefix not found! rc=%Rra\n", rc), rc);
856	rc = RTLogSetCustomPrefixCallback(&pR0LoggerR3->Logger, (PFNRTLOGPREFIX )&pfnLoggerPrefix, NULL);
857	AssertReleaseMsgRCReturn(rc, ("RTLogSetCustomPrefixCallback failed! rc=%Rra\n", rc), rc);
858
859	pR0LoggerR3->idCpu = i;
860	pR0LoggerR3->fCreated = true;
861	pR0LoggerR3->fFlushingDisabled = false;
862
863	}
864
865	rc = RTLogCopyGroupsAndFlags(&pR0LoggerR3->Logger, NULL /* default */, pVM->vmm.s.pRCLoggerR3->fFlags, RTLOGFLAGS_BUFFERED);
866	AssertRC(rc);
867	}
868	}
869	#endif
870	return rc;
871	}
872
873
874	/**
875	* Gets the pointer to a buffer containing the R0/RC AssertMsg1 output.
876	*
877	* @returns Pointer to the buffer.
878	* @param pVM The VM handle.
879	*/
880	VMMR3DECL(const char *) VMMR3GetRZAssertMsg1(PVM pVM)
881	{
882	if (HWACCMIsEnabled(pVM))
883	return pVM->vmm.s.szRing0AssertMsg1;
884
885	RTRCPTR RCPtr;
886	int rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_szRTAssertMsg1", &RCPtr);
887	if (RT_SUCCESS(rc))
888	return (const char *)MMHyperRCToR3(pVM, RCPtr);
889
890	return NULL;
891	}
892
893
894	/**
895	* Gets the pointer to a buffer containing the R0/RC AssertMsg2 output.
896	*
897	* @returns Pointer to the buffer.
898	* @param pVM The VM handle.
899	*/
900	VMMR3DECL(const char *) VMMR3GetRZAssertMsg2(PVM pVM)
901	{
902	if (HWACCMIsEnabled(pVM))
903	return pVM->vmm.s.szRing0AssertMsg2;
904
905	RTRCPTR RCPtr;
906	int rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_szRTAssertMsg2", &RCPtr);
907	if (RT_SUCCESS(rc))
908	return (const char *)MMHyperRCToR3(pVM, RCPtr);
909
910	return NULL;
911	}
912
913
914	/**
915	* Execute state save operation.
916	*
917	* @returns VBox status code.
918	* @param pVM VM Handle.
919	* @param pSSM SSM operation handle.
920	*/
921	static DECLCALLBACK(int) vmmR3Save(PVM pVM, PSSMHANDLE pSSM)
922	{
923	LogFlow(("vmmR3Save:\n"));
924
925	/*
926	* The hypervisor stack.
927	* Note! See note in vmmR3Load (remove this on version change).
928	*/
929	PVMCPU pVCpu0 = &pVM->aCpus[0];
930	SSMR3PutRCPtr(pSSM, pVCpu0->vmm.s.pbEMTStackBottomRC);
931	RTRCPTR RCPtrESP = CPUMGetHyperESP(pVCpu0);
932	AssertMsg(pVCpu0->vmm.s.pbEMTStackBottomRC - RCPtrESP <= VMM_STACK_SIZE, ("Bottom %RRv ESP=%RRv\n", pVCpu0->vmm.s.pbEMTStackBottomRC, RCPtrESP));
933	SSMR3PutRCPtr(pSSM, RCPtrESP);
934	SSMR3PutMem(pSSM, pVCpu0->vmm.s.pbEMTStackR3, VMM_STACK_SIZE);
935
936	/*
937	* Save the started/stopped state of all CPUs except 0 as it will always
938	* be running. This avoids breaking the saved state version. :-)
939	*/
940	for (VMCPUID i = 1; i < pVM->cCpus; i++)
941	SSMR3PutBool(pSSM, VMCPUSTATE_IS_STARTED(VMCPU_GET_STATE(&pVM->aCpus[i])));
942
943	return SSMR3PutU32(pSSM, ~0); /* terminator */
944	}
945
946
947	/**
948	* Execute state load operation.
949	*
950	* @returns VBox status code.
951	* @param pVM VM Handle.
952	* @param pSSM SSM operation handle.
953	* @param uVersion Data layout version.
954	* @param uPass The data pass.
955	*/
956	static DECLCALLBACK(int) vmmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
957	{
958	LogFlow(("vmmR3Load:\n"));
959	Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
960
961	/*
962	* Validate version.
963	*/
964	if (uVersion != VMM_SAVED_STATE_VERSION)
965	{
966	AssertMsgFailed(("vmmR3Load: Invalid version uVersion=%d!\n", uVersion));
967	return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
968	}
969
970	/*
971	* Check that the stack is in the same place, or that it's fearly empty.
972	*
973	* Note! This can be skipped next time we update saved state as we will
974	* never be in a R0/RC -> ring-3 call when saving the state. The
975	* stack and the two associated pointers are not required.
976	*/
977	RTRCPTR RCPtrStackBottom;
978	SSMR3GetRCPtr(pSSM, &RCPtrStackBottom);
979	RTRCPTR RCPtrESP;
980	int rc = SSMR3GetRCPtr(pSSM, &RCPtrESP);
981	if (RT_FAILURE(rc))
982	return rc;
983	SSMR3GetMem(pSSM, pVM->aCpus[0].vmm.s.pbEMTStackR3, VMM_STACK_SIZE);
984
985	/* Restore the VMCPU states. VCPU 0 is always started. */
986	VMCPU_SET_STATE(&pVM->aCpus[0], VMCPUSTATE_STARTED);
987	for (VMCPUID i = 1; i < pVM->cCpus; i++)
988	{
989	bool fStarted;
990	rc = SSMR3GetBool(pSSM, &fStarted);
991	if (RT_FAILURE(rc))
992	return rc;
993	VMCPU_SET_STATE(&pVM->aCpus[i], fStarted ? VMCPUSTATE_STARTED : VMCPUSTATE_STOPPED);
994	}
995
996	/* terminator */
997	uint32_t u32;
998	rc = SSMR3GetU32(pSSM, &u32);
999	if (RT_FAILURE(rc))
1000	return rc;
1001	if (u32 != ~0U)
1002	{
1003	AssertMsgFailed(("u32=%#x\n", u32));
1004	return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
1005	}
1006	return VINF_SUCCESS;
1007	}
1008
1009
1010	/**
1011	* Resolve a builtin RC symbol.
1012	*
1013	* Called by PDM when loading or relocating RC modules.
1014	*
1015	* @returns VBox status
1016	* @param pVM VM Handle.
1017	* @param pszSymbol Symbol to resolv
1018	* @param pRCPtrValue Where to store the symbol value.
1019	*
1020	* @remark This has to work before VMMR3Relocate() is called.
1021	*/
1022	VMMR3DECL(int) VMMR3GetImportRC(PVM pVM, const char *pszSymbol, PRTRCPTR pRCPtrValue)
1023	{
1024	if (!strcmp(pszSymbol, "g_Logger"))
1025	{
1026	if (pVM->vmm.s.pRCLoggerR3)
1027	pVM->vmm.s.pRCLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCLoggerR3);
1028	*pRCPtrValue = pVM->vmm.s.pRCLoggerRC;
1029	}
1030	else if (!strcmp(pszSymbol, "g_RelLogger"))
1031	{
1032	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
1033	if (pVM->vmm.s.pRCRelLoggerR3)
1034	pVM->vmm.s.pRCRelLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCRelLoggerR3);
1035	*pRCPtrValue = pVM->vmm.s.pRCRelLoggerRC;
1036	#else
1037	*pRCPtrValue = NIL_RTRCPTR;
1038	#endif
1039	}
1040	else
1041	return VERR_SYMBOL_NOT_FOUND;
1042	return VINF_SUCCESS;
1043	}
1044
1045
1046	/**
1047	* Suspends the CPU yielder.
1048	*
1049	* @param pVM The VM handle.
1050	*/
1051	VMMR3DECL(void) VMMR3YieldSuspend(PVM pVM)
1052	{
1053	VMCPU_ASSERT_EMT(&pVM->aCpus[0]);
1054	if (!pVM->vmm.s.cYieldResumeMillies)
1055	{
1056	uint64_t u64Now = TMTimerGet(pVM->vmm.s.pYieldTimer);
1057	uint64_t u64Expire = TMTimerGetExpire(pVM->vmm.s.pYieldTimer);
1058	if (u64Now >= u64Expire \|\| u64Expire == ~(uint64_t)0)
1059	pVM->vmm.s.cYieldResumeMillies = pVM->vmm.s.cYieldEveryMillies;
1060	else
1061	pVM->vmm.s.cYieldResumeMillies = TMTimerToMilli(pVM->vmm.s.pYieldTimer, u64Expire - u64Now);
1062	TMTimerStop(pVM->vmm.s.pYieldTimer);
1063	}
1064	pVM->vmm.s.u64LastYield = RTTimeNanoTS();
1065	}
1066
1067
1068	/**
1069	* Stops the CPU yielder.
1070	*
1071	* @param pVM The VM handle.
1072	*/
1073	VMMR3DECL(void) VMMR3YieldStop(PVM pVM)
1074	{
1075	if (!pVM->vmm.s.cYieldResumeMillies)
1076	TMTimerStop(pVM->vmm.s.pYieldTimer);
1077	pVM->vmm.s.cYieldResumeMillies = pVM->vmm.s.cYieldEveryMillies;
1078	pVM->vmm.s.u64LastYield = RTTimeNanoTS();
1079	}
1080
1081
1082	/**
1083	* Resumes the CPU yielder when it has been a suspended or stopped.
1084	*
1085	* @param pVM The VM handle.
1086	*/
1087	VMMR3DECL(void) VMMR3YieldResume(PVM pVM)
1088	{
1089	if (pVM->vmm.s.cYieldResumeMillies)
1090	{
1091	TMTimerSetMillies(pVM->vmm.s.pYieldTimer, pVM->vmm.s.cYieldResumeMillies);
1092	pVM->vmm.s.cYieldResumeMillies = 0;
1093	}
1094	}
1095
1096
1097	/**
1098	* Internal timer callback function.
1099	*
1100	* @param pVM The VM.
1101	* @param pTimer The timer handle.
1102	* @param pvUser User argument specified upon timer creation.
1103	*/
1104	static DECLCALLBACK(void) vmmR3YieldEMT(PVM pVM, PTMTIMER pTimer, void *pvUser)
1105	{
1106	/*
1107	* This really needs some careful tuning. While we shouldn't be too greedy since
1108	* that'll cause the rest of the system to stop up, we shouldn't be too nice either
1109	* because that'll cause us to stop up.
1110	*
1111	* The current logic is to use the default interval when there is no lag worth
1112	* mentioning, but when we start accumulating lag we don't bother yielding at all.
1113	*
1114	* (This depends on the TMCLOCK_VIRTUAL_SYNC to be scheduled before TMCLOCK_REAL
1115	* so the lag is up to date.)
1116	*/
1117	const uint64_t u64Lag = TMVirtualSyncGetLag(pVM);
1118	if ( u64Lag < 50000000 /* 50ms */
1119	\|\| ( u64Lag < 1000000000 /* 1s */
1120	&& RTTimeNanoTS() - pVM->vmm.s.u64LastYield < 500000000 /* 500 ms */)
1121	)
1122	{
1123	uint64_t u64Elapsed = RTTimeNanoTS();
1124	pVM->vmm.s.u64LastYield = u64Elapsed;
1125
1126	RTThreadYield();
1127
1128	#ifdef LOG_ENABLED
1129	u64Elapsed = RTTimeNanoTS() - u64Elapsed;
1130	Log(("vmmR3YieldEMT: %RI64 ns\n", u64Elapsed));
1131	#endif
1132	}
1133	TMTimerSetMillies(pTimer, pVM->vmm.s.cYieldEveryMillies);
1134	}
1135
1136
1137	/**
1138	* Executes guest code in the raw-mode context.
1139	*
1140	* @param pVM VM handle.
1141	* @param pVCpu The VMCPU to operate on.
1142	*/
1143	VMMR3DECL(int) VMMR3RawRunGC(PVM pVM, PVMCPU pVCpu)
1144	{
1145	Log2(("VMMR3RawRunGC: (cs:eip=%04x:%08x)\n", CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
1146
1147	AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
1148
1149	/*
1150	* Set the EIP and ESP.
1151	*/
1152	CPUMSetHyperEIP(pVCpu, CPUMGetGuestEFlags(pVCpu) & X86_EFL_VM
1153	? pVM->vmm.s.pfnCPUMRCResumeGuestV86
1154	: pVM->vmm.s.pfnCPUMRCResumeGuest);
1155	CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC);
1156
1157	/*
1158	* We hide log flushes (outer) and hypervisor interrupts (inner).
1159	*/
1160	for (;;)
1161	{
1162	#ifdef VBOX_STRICT
1163	if (RT_UNLIKELY(!CPUMGetHyperCR3(pVCpu) \|\| CPUMGetHyperCR3(pVCpu) != PGMGetHyperCR3(pVCpu)))
1164	EMR3FatalError(pVCpu, VERR_VMM_HYPER_CR3_MISMATCH);
1165	PGMMapCheck(pVM);
1166	#endif
1167	int rc;
1168	do
1169	{
1170	#ifdef NO_SUPCALLR0VMM
1171	rc = VERR_GENERAL_FAILURE;
1172	#else
1173	rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
1174	if (RT_LIKELY(rc == VINF_SUCCESS))
1175	rc = pVCpu->vmm.s.iLastGZRc;
1176	#endif
1177	} while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
1178
1179	/*
1180	* Flush the logs.
1181	*/
1182	#ifdef LOG_ENABLED
1183	PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
1184	if ( pLogger
1185	&& pLogger->offScratch > 0)
1186	RTLogFlushRC(NULL, pLogger);
1187	#endif
1188	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
1189	PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
1190	if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
1191	RTLogFlushRC(RTLogRelDefaultInstance(), pRelLogger);
1192	#endif
1193	if (rc != VINF_VMM_CALL_HOST)
1194	{
1195	Log2(("VMMR3RawRunGC: returns %Rrc (cs:eip=%04x:%08x)\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
1196	return rc;
1197	}
1198	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
1199	if (RT_FAILURE(rc))
1200	return rc;
1201	/* Resume GC */
1202	}
1203	}
1204
1205
1206	/**
1207	* Executes guest code (Intel VT-x and AMD-V).
1208	*
1209	* @param pVM VM handle.
1210	* @param pVCpu The VMCPU to operate on.
1211	*/
1212	VMMR3DECL(int) VMMR3HwAccRunGC(PVM pVM, PVMCPU pVCpu)
1213	{
1214	Log2(("VMMR3HwAccRunGC: (cs:eip=%04x:%08x)\n", CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
1215
1216	for (;;)
1217	{
1218	int rc;
1219	do
1220	{
1221	#ifdef NO_SUPCALLR0VMM
1222	rc = VERR_GENERAL_FAILURE;
1223	#else
1224	rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_HWACC_RUN, pVCpu->idCpu);
1225	if (RT_LIKELY(rc == VINF_SUCCESS))
1226	rc = pVCpu->vmm.s.iLastGZRc;
1227	#endif
1228	} while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
1229
1230	#ifdef LOG_ENABLED
1231	/*
1232	* Flush the log
1233	*/
1234	PVMMR0LOGGER pR0LoggerR3 = pVCpu->vmm.s.pR0LoggerR3;
1235	if ( pR0LoggerR3
1236	&& pR0LoggerR3->Logger.offScratch > 0)
1237	RTLogFlushToLogger(&pR0LoggerR3->Logger, NULL);
1238	#endif /* !LOG_ENABLED */
1239	if (rc != VINF_VMM_CALL_HOST)
1240	{
1241	Log2(("VMMR3HwAccRunGC: returns %Rrc (cs:eip=%04x:%08x)\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
1242	return rc;
1243	}
1244	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
1245	if (RT_FAILURE(rc))
1246	return rc;
1247	/* Resume R0 */
1248	}
1249	}
1250
1251	/**
1252	* VCPU worker for VMMSendSipi.
1253	*
1254	* @param pVM The VM to operate on.
1255	* @param idCpu Virtual CPU to perform SIPI on
1256	* @param uVector SIPI vector
1257	*/
1258	DECLCALLBACK(int) vmmR3SendSipi(PVM pVM, VMCPUID idCpu, uint32_t uVector)
1259	{
1260	PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
1261	VMCPU_ASSERT_EMT(pVCpu);
1262
1263	/** @todo what are we supposed to do if the processor is already running? */
1264	if (EMGetState(pVCpu) != EMSTATE_WAIT_SIPI)
1265	return VERR_ACCESS_DENIED;
1266
1267
1268	PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
1269
1270	pCtx->cs = uVector << 8;
1271	pCtx->csHid.u64Base = uVector << 12;
1272	pCtx->csHid.u32Limit = 0x0000ffff;
1273	pCtx->rip = 0;
1274
1275	Log(("vmmR3SendSipi for VCPU %d with vector %x\n", uVector));
1276
1277	# if 1 /* If we keep the EMSTATE_WAIT_SIPI method, then move this to EM.cpp. */
1278	EMSetState(pVCpu, EMSTATE_HALTED);
1279	return VINF_EM_RESCHEDULE;
1280	# else /* And if we go the VMCPU::enmState way it can stay here. */
1281	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STOPPED);
1282	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
1283	return VINF_SUCCESS;
1284	# endif
1285	}
1286
1287	DECLCALLBACK(int) vmmR3SendInitIpi(PVM pVM, VMCPUID idCpu)
1288	{
1289	PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
1290	VMCPU_ASSERT_EMT(pVCpu);
1291
1292	Log(("vmmR3SendInitIpi for VCPU %d\n", idCpu));
1293	CPUMR3ResetCpu(pVCpu);
1294	return VINF_EM_WAIT_SIPI;
1295	}
1296
1297	/**
1298	* Sends SIPI to the virtual CPU by setting CS:EIP into vector-dependent state
1299	* and unhalting processor
1300	*
1301	* @param pVM The VM to operate on.
1302	* @param idCpu Virtual CPU to perform SIPI on
1303	* @param uVector SIPI vector
1304	*/
1305	VMMR3DECL(void) VMMR3SendSipi(PVM pVM, VMCPUID idCpu, uint32_t uVector)
1306	{
1307	AssertReturnVoid(idCpu < pVM->cCpus);
1308
1309	int rc = VMR3ReqCallNoWaitU(pVM->pUVM, idCpu, (PFNRT)vmmR3SendSipi, 3, pVM, idCpu, uVector);
1310	AssertRC(rc);
1311	}
1312
1313	/**
1314	* Sends init IPI to the virtual CPU.
1315	*
1316	* @param pVM The VM to operate on.
1317	* @param idCpu Virtual CPU to perform int IPI on
1318	*/
1319	VMMR3DECL(void) VMMR3SendInitIpi(PVM pVM, VMCPUID idCpu)
1320	{
1321	AssertReturnVoid(idCpu < pVM->cCpus);
1322
1323	int rc = VMR3ReqCallNoWaitU(pVM->pUVM, idCpu, (PFNRT)vmmR3SendInitIpi, 2, pVM, idCpu);
1324	AssertRC(rc);
1325	}
1326
1327	/**
1328	* Registers the guest memory range that can be used for patching
1329	*
1330	* @returns VBox status code.
1331	* @param pVM The VM to operate on.
1332	* @param pPatchMem Patch memory range
1333	* @param cbPatchMem Size of the memory range
1334	*/
1335	VMMR3DECL(int) VMMR3RegisterPatchMemory(PVM pVM, RTGCPTR pPatchMem, unsigned cbPatchMem)
1336	{
1337	if (HWACCMIsEnabled(pVM))
1338	return HWACMMR3EnablePatching(pVM, pPatchMem, cbPatchMem);
1339
1340	return VERR_NOT_SUPPORTED;
1341	}
1342
1343	/**
1344	* Deregisters the guest memory range that can be used for patching
1345	*
1346	* @returns VBox status code.
1347	* @param pVM The VM to operate on.
1348	* @param pPatchMem Patch memory range
1349	* @param cbPatchMem Size of the memory range
1350	*/
1351	VMMR3DECL(int) VMMR3DeregisterPatchMemory(PVM pVM, RTGCPTR pPatchMem, unsigned cbPatchMem)
1352	{
1353	if (HWACCMIsEnabled(pVM))
1354	return HWACMMR3DisablePatching(pVM, pPatchMem, cbPatchMem);
1355
1356	return VINF_SUCCESS;
1357	}
1358
1359
1360	/**
1361	* VCPU worker for VMMR3SynchronizeAllVCpus.
1362	*
1363	* @param pVM The VM to operate on.
1364	* @param idCpu Virtual CPU to perform SIPI on
1365	* @param uVector SIPI vector
1366	*/
1367	DECLCALLBACK(int) vmmR3SyncVCpu(PVM pVM)
1368	{
1369	/* Block until the job in the caller has finished. */
1370	RTCritSectEnter(&pVM->vmm.s.CritSectSync);
1371	RTCritSectLeave(&pVM->vmm.s.CritSectSync);
1372	return VINF_SUCCESS;
1373	}
1374
1375
1376	/**
1377	* Atomically execute a callback handler
1378	* Note: This is very expensive; avoid using it frequently!
1379	*
1380	* @param pVM The VM to operate on.
1381	* @param pfnHandler Callback handler
1382	* @param pvUser User specified parameter
1383	*
1384	* @thread EMT
1385	*/
1386	VMMR3DECL(int) VMMR3AtomicExecuteHandler(PVM pVM, PFNATOMICHANDLER pfnHandler, void *pvUser)
1387	{
1388	int rc;
1389	PVMCPU pVCpu = VMMGetCpu(pVM);
1390	AssertReturn(pVCpu, VERR_VM_THREAD_NOT_EMT);
1391
1392	/* Shortcut for the uniprocessor case. */
1393	if (pVM->cCpus == 1)
1394	return pfnHandler(pVM, pvUser);
1395
1396	RTCritSectEnter(&pVM->vmm.s.CritSectSync);
1397	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1398	{
1399	if (idCpu != pVCpu->idCpu)
1400	{
1401	rc = VMR3ReqCallNoWaitU(pVM->pUVM, idCpu, (PFNRT)vmmR3SyncVCpu, 1, pVM);
1402	AssertRC(rc);
1403	}
1404	}
1405	/* Wait until all other VCPUs are waiting for us. */
1406	while (RTCritSectGetWaiters(&pVM->vmm.s.CritSectSync) != (int32_t)(pVM->cCpus - 1))
1407	RTThreadSleep(1);
1408
1409	rc = pfnHandler(pVM, pvUser);
1410	RTCritSectLeave(&pVM->vmm.s.CritSectSync);
1411	return rc;
1412	}
1413
1414
1415	/**
1416	* Count returns and have the last non-caller EMT wake up the caller.
1417	*
1418	* @returns VBox strict informational status code for EM scheduling. No failures
1419	* will be returned here, those are for the caller only.
1420	*
1421	* @param pVM The VM handle.
1422	*/
1423	DECL_FORCE_INLINE(int) vmmR3EmtRendezvousNonCallerReturn(PVM pVM)
1424	{
1425	int rcRet = ASMAtomicReadS32(&pVM->vmm.s.i32RendezvousStatus);
1426	uint32_t cReturned = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsReturned);
1427	if (cReturned == pVM->cCpus - 1U)
1428	{
1429	int rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousDoneCaller);
1430	AssertLogRelRC(rc);
1431	}
1432
1433	AssertLogRelMsgReturn( rcRet <= VINF_SUCCESS
1434	\|\| (rcRet >= VINF_EM_FIRST && rcRet <= VINF_EM_LAST),
1435	("%Rrc\n", rcRet),
1436	VERR_IPE_UNEXPECTED_INFO_STATUS);
1437	return RT_SUCCESS(rcRet) ? rcRet : VINF_SUCCESS;
1438	}
1439
1440
1441	/**
1442	* Common worker for VMMR3EmtRendezvous and VMMR3EmtRendezvousFF.
1443	*
1444	* @returns VBox strict informational status code for EM scheduling. No failures
1445	* will be returned here, those are for the caller only. When
1446	* fIsCaller is set, VINF_SUCESS is always returned.
1447	*
1448	* @param pVM The VM handle.
1449	* @param pVCpu The VMCPU structure for the calling EMT.
1450	* @param fIsCaller Whether we're the VMMR3EmtRendezvous caller or
1451	* not.
1452	* @param fFlags The flags.
1453	* @param pfnRendezvous The callback.
1454	* @param pvUser The user argument for the callback.
1455	*/
1456	static int vmmR3EmtRendezvousCommon(PVM pVM, PVMCPU pVCpu, bool fIsCaller,
1457	uint32_t fFlags, PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser)
1458	{
1459	int rc;
1460
1461	/*
1462	* Enter, the last EMT triggers the next callback phase.
1463	*/
1464	uint32_t cEntered = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsEntered);
1465	if (cEntered != pVM->cCpus)
1466	{
1467	if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE)
1468	{
1469	/* Wait for our turn. */
1470	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousEnterOneByOne, RT_INDEFINITE_WAIT);
1471	AssertLogRelRC(rc);
1472	}
1473	else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE)
1474	{
1475	/* Wait for the last EMT to arrive and wake everyone up. */
1476	rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce, RT_INDEFINITE_WAIT);
1477	AssertLogRelRC(rc);
1478	}
1479	else if ( (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
1480	\|\| (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
1481	{
1482	/* Wait for our turn. */
1483	rc = RTSemEventWait(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVCpu->idCpu], RT_INDEFINITE_WAIT);
1484	AssertLogRelRC(rc);
1485	}
1486	else
1487	{
1488	Assert((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE);
1489
1490	/*
1491	* The execute once is handled specially to optimize the code flow.
1492	*
1493	* The last EMT to arrive will perform the callback and the other
1494	* EMTs will wait on the Done/DoneCaller semaphores (instead of
1495	* the EnterOneByOne/AllAtOnce) in the meanwhile. When the callback
1496	* returns, that EMT will initiate the normal return sequence.
1497	*/
1498	if (!fIsCaller)
1499	{
1500	rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousDone, RT_INDEFINITE_WAIT);
1501	AssertLogRelRC(rc);
1502
1503	return vmmR3EmtRendezvousNonCallerReturn(pVM);
1504	}
1505	return VINF_SUCCESS;
1506	}
1507	}
1508	else
1509	{
1510	/*
1511	* All EMTs are waiting, clear the FF and take action according to the
1512	* execution method.
1513	*/
1514	VM_FF_CLEAR(pVM, VM_FF_EMT_RENDEZVOUS);
1515
1516	if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE)
1517	{
1518	/* Wake up everyone. */
1519	rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
1520	AssertLogRelRC(rc);
1521	}
1522	else if ( (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
1523	\|\| (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
1524	{
1525	/* Figure out who to wake up and wake it up. If it's ourself, then
1526	it's easy otherwise wait for our turn. */
1527	VMCPUID iFirst = (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
1528	? 0
1529	: pVM->cCpus - 1U;
1530	if (pVCpu->idCpu != iFirst)
1531	{
1532	rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[iFirst]);
1533	AssertLogRelRC(rc);
1534	rc = RTSemEventWait(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVCpu->idCpu], RT_INDEFINITE_WAIT);
1535	AssertLogRelRC(rc);
1536	}
1537	}
1538	/* else: execute the handler on the current EMT and wake up one or more threads afterwards. */
1539	}
1540
1541
1542	/*
1543	* Do the callback and update the status if necessary.
1544	*/
1545	if ( !(fFlags & VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR)
1546	\|\| RT_SUCCESS(ASMAtomicUoReadS32(&pVM->vmm.s.i32RendezvousStatus)) )
1547	{
1548	VBOXSTRICTRC rcStrict = pfnRendezvous(pVM, pVCpu, pvUser);
1549	if (rcStrict != VINF_SUCCESS)
1550	{
1551	AssertLogRelMsg( rcStrict <= VINF_SUCCESS
1552	\|\| (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST),
1553	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
1554	int32_t i32RendezvousStatus;
1555	do
1556	{
1557	i32RendezvousStatus = ASMAtomicUoReadS32(&pVM->vmm.s.i32RendezvousStatus);
1558	if ( rcStrict == i32RendezvousStatus
1559	\|\| RT_FAILURE(i32RendezvousStatus)
1560	\|\| ( i32RendezvousStatus != VINF_SUCCESS
1561	&& rcStrict > i32RendezvousStatus))
1562	break;
1563	} while (!ASMAtomicCmpXchgS32(&pVM->vmm.s.i32RendezvousStatus, VBOXSTRICTRC_VAL(rcStrict), i32RendezvousStatus));
1564	}
1565	}
1566
1567	/*
1568	* Increment the done counter and take action depending on whether we're
1569	* the last to finish callback execution.
1570	*/
1571	uint32_t cDone = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsDone);
1572	if ( cDone != pVM->cCpus
1573	&& (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE)
1574	{
1575	/* Signal the next EMT? */
1576	if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE)
1577	{
1578	rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousEnterOneByOne);
1579	AssertLogRelRC(rc);
1580	}
1581	else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING)
1582	{
1583	Assert(cDone == pVCpu->idCpu + 1U);
1584	rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVCpu->idCpu + 1U]);
1585	AssertLogRelRC(rc);
1586	}
1587	else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
1588	{
1589	Assert(pVM->cCpus - cDone == pVCpu->idCpu);
1590	rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVM->cCpus - cDone - 1U]);
1591	AssertLogRelRC(rc);
1592	}
1593
1594	/* Wait for the rest to finish (the caller waits on hEvtRendezvousDoneCaller). */
1595	if (!fIsCaller)
1596	{
1597	rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousDone, RT_INDEFINITE_WAIT);
1598	AssertLogRelRC(rc);
1599	}
1600	}
1601	else
1602	{
1603	/* Callback execution is all done, tell the rest to return. */
1604	rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousDone);
1605	AssertLogRelRC(rc);
1606	}
1607
1608	if (!fIsCaller)
1609	return vmmR3EmtRendezvousNonCallerReturn(pVM);
1610	return VINF_SUCCESS;
1611	}
1612
1613
1614	/**
1615	* Called in response to VM_FF_EMT_RENDEZVOUS.
1616	*
1617	* @returns VBox strict status code - EM scheduling. No errors will be returned
1618	* here, nor will any non-EM scheduling status codes be returned.
1619	*
1620	* @param pVM The VM handle
1621	* @param pVCpu The handle of the calling EMT.
1622	*
1623	* @thread EMT
1624	*/
1625	VMMR3DECL(int) VMMR3EmtRendezvousFF(PVM pVM, PVMCPU pVCpu)
1626	{
1627	return vmmR3EmtRendezvousCommon(pVM, pVCpu, false /* fIsCaller */, pVM->vmm.s.fRendezvousFlags,
1628	pVM->vmm.s.pfnRendezvous, pVM->vmm.s.pvRendezvousUser);
1629	}
1630
1631
1632	/**
1633	* EMT rendezvous.
1634	*
1635	* Gathers all the EMTs and execute some code on each of them, either in a one
1636	* by one fashion or all at once.
1637	*
1638	* @returns VBox strict status code. This will be the the first error,
1639	* VINF_SUCCESS, or an EM scheduling status code.
1640	*
1641	* @param pVM The VM handle.
1642	* @param fFlags Flags indicating execution methods. See
1643	* grp_VMMR3EmtRendezvous_fFlags.
1644	* @param pfnRendezvous The callback.
1645	* @param pvUser User argument for the callback.
1646	*
1647	* @thread Any.
1648	*/
1649	VMMR3DECL(int) VMMR3EmtRendezvous(PVM pVM, uint32_t fFlags, PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser)
1650	{
1651	/*
1652	* Validate input.
1653	*/
1654	AssertMsg( (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_INVALID
1655	&& (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) <= VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING
1656	&& !(fFlags & ~VMMEMTRENDEZVOUS_FLAGS_VALID_MASK), ("%#x\n", fFlags));
1657	AssertMsg( !(fFlags & VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR)
1658	\|\| ( (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE
1659	&& (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE),
1660	("type %u\n", fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK));
1661
1662	VBOXSTRICTRC rcStrict;
1663	PVMCPU pVCpu = VMMGetCpu(pVM);
1664	if (!pVCpu)
1665	/*
1666	* Forward the request to an EMT thread.
1667	*/
1668	rcStrict = VMR3ReqCallWait(pVM, VMCPUID_ANY,
1669	(PFNRT)VMMR3EmtRendezvous, 4, pVM, fFlags, pfnRendezvous, pvUser);
1670	else if (pVM->cCpus == 1)
1671	/*
1672	* Shortcut for the single EMT case.
1673	*/
1674	rcStrict = pfnRendezvous(pVM, pVCpu, pvUser);
1675	else
1676	{
1677	/*
1678	* Spin lock. If busy, wait for the other EMT to finish while keeping a
1679	* lookout of the RENDEZVOUS FF.
1680	*/
1681	int rc;
1682	rcStrict = VINF_SUCCESS;
1683	if (RT_UNLIKELY(!ASMAtomicCmpXchgU32(&pVM->vmm.s.u32RendezvousLock, 0x77778888, 0)))
1684	{
1685	while (!ASMAtomicCmpXchgU32(&pVM->vmm.s.u32RendezvousLock, 0x77778888, 0))
1686	{
1687	if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
1688	{
1689	rc = VMMR3EmtRendezvousFF(pVM, pVCpu);
1690	if ( rc != VINF_SUCCESS
1691	&& ( rcStrict == VINF_SUCCESS
1692	\|\| rcStrict > rc))
1693	rcStrict = rc;
1694	/** @todo Perhaps deal with termination here? */
1695	}
1696	ASMNopPause();
1697	}
1698	}
1699	Assert(!VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS));
1700
1701	/*
1702	* Clear the slate. This is a semaphore ping-pong orgy. :-)
1703	*/
1704	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1705	{
1706	rc = RTSemEventWait(pVM->vmm.s.pahEvtRendezvousEnterOrdered[i], 0);
1707	AssertLogRelMsg(rc == VERR_TIMEOUT \|\| rc == VINF_SUCCESS, ("%Rrc\n", rc));
1708	}
1709	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousEnterOneByOne, 0); AssertLogRelMsg(rc == VERR_TIMEOUT \|\| rc == VINF_SUCCESS, ("%Rrc\n", rc));
1710	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce); AssertLogRelRC(rc);
1711	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousDone); AssertLogRelRC(rc);
1712	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousDoneCaller, 0); AssertLogRelMsg(rc == VERR_TIMEOUT \|\| rc == VINF_SUCCESS, ("%Rrc\n", rc));
1713	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsEntered, 0);
1714	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsDone, 0);
1715	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsReturned, 0);
1716	ASMAtomicWriteS32(&pVM->vmm.s.i32RendezvousStatus, VINF_SUCCESS);
1717	ASMAtomicWritePtr((void * volatile )&pVM->vmm.s.pfnRendezvous, (void )(uintptr_t)pfnRendezvous);
1718	ASMAtomicWritePtr(&pVM->vmm.s.pvRendezvousUser, pvUser);
1719	ASMAtomicWriteU32(&pVM->vmm.s.fRendezvousFlags, fFlags);
1720
1721	/*
1722	* Set the FF and poke the other EMTs.
1723	*/
1724	VM_FF_SET(pVM, VM_FF_EMT_RENDEZVOUS);
1725	VMR3NotifyGlobalFFU(pVM->pUVM, VMNOTIFYFF_FLAGS_POKE);
1726
1727	/*
1728	* Do the same ourselves.
1729	*/
1730	vmmR3EmtRendezvousCommon(pVM, pVCpu, true /* fIsCaller */, fFlags, pfnRendezvous, pvUser);
1731
1732	/*
1733	* The caller waits for the other EMTs to be done and return before doing
1734	* the cleanup. This makes away with wakeup / reset races we would otherwise
1735	* risk in the multiple release event semaphore code (hEvtRendezvousDoneCaller).
1736	*/
1737	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousDoneCaller, RT_INDEFINITE_WAIT);
1738	AssertLogRelRC(rc);
1739
1740	/*
1741	* Get the return code and clean up a little bit.
1742	*/
1743	int rcMy = pVM->vmm.s.i32RendezvousStatus;
1744	ASMAtomicWritePtr((void * volatile *)&pVM->vmm.s.pfnRendezvous, NULL);
1745
1746	ASMAtomicWriteU32(&pVM->vmm.s.u32RendezvousLock, 0);
1747
1748	/*
1749	* Merge rcStrict and rcMy.
1750	*/
1751	AssertRC(VBOXSTRICTRC_VAL(rcStrict));
1752	if ( rcMy != VINF_SUCCESS
1753	&& ( rcStrict == VINF_SUCCESS
1754	\|\| rcStrict > rcMy))
1755	rcStrict = rcMy;
1756	}
1757
1758	AssertLogRelMsgReturn( rcStrict <= VINF_SUCCESS
1759	\|\| (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST),
1760	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)),
1761	VERR_IPE_UNEXPECTED_INFO_STATUS);
1762	return VBOXSTRICTRC_VAL(rcStrict);
1763	}
1764
1765
1766	/**
1767	* Read from the ring 0 jump buffer stack
1768	*
1769	* @returns VBox status code.
1770	*
1771	* @param pVM Pointer to the shared VM structure.
1772	* @param idCpu The ID of the source CPU context (for the address).
1773	* @param pAddress Where to start reading.
1774	* @param pvBuf Where to store the data we've read.
1775	* @param cbRead The number of bytes to read.
1776	*/
1777	VMMR3DECL(int) VMMR3ReadR0Stack(PVM pVM, VMCPUID idCpu, RTHCUINTPTR pAddress, void *pvBuf, size_t cbRead)
1778	{
1779	PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
1780	AssertReturn(pVCpu, VERR_INVALID_PARAMETER);
1781
1782	RTHCUINTPTR offset = pVCpu->vmm.s.CallRing3JmpBufR0.SpCheck - pAddress;
1783	if (offset >= pVCpu->vmm.s.CallRing3JmpBufR0.cbSavedStack)
1784	return VERR_INVALID_POINTER;
1785
1786	memcpy(pvBuf, pVCpu->vmm.s.pbEMTStackR3 + pVCpu->vmm.s.CallRing3JmpBufR0.cbSavedStack - offset, cbRead);
1787	return VINF_SUCCESS;
1788	}
1789
1790
1791	/**
1792	* Calls a RC function.
1793	*
1794	* @param pVM The VM handle.
1795	* @param RCPtrEntry The address of the RC function.
1796	* @param cArgs The number of arguments in the ....
1797	* @param ... Arguments to the function.
1798	*/
1799	VMMR3DECL(int) VMMR3CallRC(PVM pVM, RTRCPTR RCPtrEntry, unsigned cArgs, ...)
1800	{
1801	va_list args;
1802	va_start(args, cArgs);
1803	int rc = VMMR3CallRCV(pVM, RCPtrEntry, cArgs, args);
1804	va_end(args);
1805	return rc;
1806	}
1807
1808
1809	/**
1810	* Calls a RC function.
1811	*
1812	* @param pVM The VM handle.
1813	* @param RCPtrEntry The address of the RC function.
1814	* @param cArgs The number of arguments in the ....
1815	* @param args Arguments to the function.
1816	*/
1817	VMMR3DECL(int) VMMR3CallRCV(PVM pVM, RTRCPTR RCPtrEntry, unsigned cArgs, va_list args)
1818	{
1819	/* Raw mode implies 1 VCPU. */
1820	AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
1821	PVMCPU pVCpu = &pVM->aCpus[0];
1822
1823	Log2(("VMMR3CallGCV: RCPtrEntry=%RRv cArgs=%d\n", RCPtrEntry, cArgs));
1824
1825	/*
1826	* Setup the call frame using the trampoline.
1827	*/
1828	CPUMHyperSetCtxCore(pVCpu, NULL);
1829	memset(pVCpu->vmm.s.pbEMTStackR3, 0xaa, VMM_STACK_SIZE); /* Clear the stack. */
1830	CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC - cArgs * sizeof(RTGCUINTPTR32));
1831	PRTGCUINTPTR32 pFrame = (PRTGCUINTPTR32)(pVCpu->vmm.s.pbEMTStackR3 + VMM_STACK_SIZE) - cArgs;
1832	int i = cArgs;
1833	while (i-- > 0)
1834	*pFrame++ = va_arg(args, RTGCUINTPTR32);
1835
1836	CPUMPushHyper(pVCpu, cArgs * sizeof(RTGCUINTPTR32)); /* stack frame size */
1837	CPUMPushHyper(pVCpu, RCPtrEntry); /* what to call */
1838	CPUMSetHyperEIP(pVCpu, pVM->vmm.s.pfnCallTrampolineRC);
1839
1840	/*
1841	* We hide log flushes (outer) and hypervisor interrupts (inner).
1842	*/
1843	for (;;)
1844	{
1845	int rc;
1846	Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
1847	do
1848	{
1849	#ifdef NO_SUPCALLR0VMM
1850	rc = VERR_GENERAL_FAILURE;
1851	#else
1852	rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
1853	if (RT_LIKELY(rc == VINF_SUCCESS))
1854	rc = pVCpu->vmm.s.iLastGZRc;
1855	#endif
1856	} while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
1857
1858	/*
1859	* Flush the logs.
1860	*/
1861	#ifdef LOG_ENABLED
1862	PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
1863	if ( pLogger
1864	&& pLogger->offScratch > 0)
1865	RTLogFlushRC(NULL, pLogger);
1866	#endif
1867	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
1868	PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
1869	if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
1870	RTLogFlushRC(RTLogRelDefaultInstance(), pRelLogger);
1871	#endif
1872	if (rc == VERR_TRPM_PANIC \|\| rc == VERR_TRPM_DONT_PANIC)
1873	VMMR3FatalDump(pVM, pVCpu, rc);
1874	if (rc != VINF_VMM_CALL_HOST)
1875	{
1876	Log2(("VMMR3CallGCV: returns %Rrc (cs:eip=%04x:%08x)\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
1877	return rc;
1878	}
1879	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
1880	if (RT_FAILURE(rc))
1881	return rc;
1882	}
1883	}
1884
1885
1886	/**
1887	* Wrapper for SUPR3CallVMMR0Ex which will deal with VINF_VMM_CALL_HOST returns.
1888	*
1889	* @returns VBox status code.
1890	* @param pVM The VM to operate on.
1891	* @param uOperation Operation to execute.
1892	* @param u64Arg Constant argument.
1893	* @param pReqHdr Pointer to a request header. See SUPR3CallVMMR0Ex for
1894	* details.
1895	*/
1896	VMMR3DECL(int) VMMR3CallR0(PVM pVM, uint32_t uOperation, uint64_t u64Arg, PSUPVMMR0REQHDR pReqHdr)
1897	{
1898	PVMCPU pVCpu = VMMGetCpu(pVM);
1899	AssertReturn(pVCpu, VERR_VM_THREAD_NOT_EMT);
1900
1901	/*
1902	* Call Ring-0 entry with init code.
1903	*/
1904	int rc;
1905	for (;;)
1906	{
1907	#ifdef NO_SUPCALLR0VMM
1908	rc = VERR_GENERAL_FAILURE;
1909	#else
1910	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, pVCpu->idCpu, uOperation, u64Arg, pReqHdr);
1911	#endif
1912	/*
1913	* Flush the logs.
1914	*/
1915	#ifdef LOG_ENABLED
1916	if ( pVCpu->vmm.s.pR0LoggerR3
1917	&& pVCpu->vmm.s.pR0LoggerR3->Logger.offScratch > 0)
1918	RTLogFlushToLogger(&pVCpu->vmm.s.pR0LoggerR3->Logger, NULL);
1919	#endif
1920	if (rc != VINF_VMM_CALL_HOST)
1921	break;
1922	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
1923	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
1924	break;
1925	/* Resume R0 */
1926	}
1927
1928	AssertLogRelMsgReturn(rc == VINF_SUCCESS \|\| VBOX_FAILURE(rc),
1929	("uOperation=%u rc=%Rrc\n", uOperation, rc),
1930	VERR_INTERNAL_ERROR);
1931	return rc;
1932	}
1933
1934
1935	/**
1936	* Resumes executing hypervisor code when interrupted by a queue flush or a
1937	* debug event.
1938	*
1939	* @returns VBox status code.
1940	* @param pVM VM handle.
1941	* @param pVCpu VMCPU handle.
1942	*/
1943	VMMR3DECL(int) VMMR3ResumeHyper(PVM pVM, PVMCPU pVCpu)
1944	{
1945	Log(("VMMR3ResumeHyper: eip=%RRv esp=%RRv\n", CPUMGetHyperEIP(pVCpu), CPUMGetHyperESP(pVCpu)));
1946	AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
1947
1948	/*
1949	* We hide log flushes (outer) and hypervisor interrupts (inner).
1950	*/
1951	for (;;)
1952	{
1953	int rc;
1954	Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
1955	do
1956	{
1957	#ifdef NO_SUPCALLR0VMM
1958	rc = VERR_GENERAL_FAILURE;
1959	#else
1960	rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
1961	if (RT_LIKELY(rc == VINF_SUCCESS))
1962	rc = pVCpu->vmm.s.iLastGZRc;
1963	#endif
1964	} while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
1965
1966	/*
1967	* Flush the loggers,
1968	*/
1969	#ifdef LOG_ENABLED
1970	PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
1971	if ( pLogger
1972	&& pLogger->offScratch > 0)
1973	RTLogFlushRC(NULL, pLogger);
1974	#endif
1975	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
1976	PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
1977	if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
1978	RTLogFlushRC(RTLogRelDefaultInstance(), pRelLogger);
1979	#endif
1980	if (rc == VERR_TRPM_PANIC \|\| rc == VERR_TRPM_DONT_PANIC)
1981	VMMR3FatalDump(pVM, pVCpu, rc);
1982	if (rc != VINF_VMM_CALL_HOST)
1983	{
1984	Log(("VMMR3ResumeHyper: returns %Rrc\n", rc));
1985	return rc;
1986	}
1987	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
1988	if (RT_FAILURE(rc))
1989	return rc;
1990	}
1991	}
1992
1993
1994	/**
1995	* Service a call to the ring-3 host code.
1996	*
1997	* @returns VBox status code.
1998	* @param pVM VM handle.
1999	* @param pVCpu VMCPU handle
2000	* @remark Careful with critsects.
2001	*/
2002	static int vmmR3ServiceCallRing3Request(PVM pVM, PVMCPU pVCpu)
2003	{
2004	/*
2005	* We must also check for pending critsect exits or else we can deadlock
2006	* when entering other critsects here.
2007	*/
2008	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PDM_CRITSECT))
2009	PDMCritSectFF(pVCpu);
2010
2011	switch (pVCpu->vmm.s.enmCallRing3Operation)
2012	{
2013	/*
2014	* Acquire the PDM lock.
2015	*/
2016	case VMMCALLRING3_PDM_LOCK:
2017	{
2018	pVCpu->vmm.s.rcCallRing3 = PDMR3LockCall(pVM);
2019	break;
2020	}
2021
2022	/*
2023	* Flush a PDM queue.
2024	*/
2025	case VMMCALLRING3_PDM_QUEUE_FLUSH:
2026	{
2027	PDMR3QueueFlushWorker(pVM, NULL);
2028	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
2029	break;
2030	}
2031
2032	/*
2033	* Grow the PGM pool.
2034	*/
2035	case VMMCALLRING3_PGM_POOL_GROW:
2036	{
2037	pVCpu->vmm.s.rcCallRing3 = PGMR3PoolGrow(pVM);
2038	break;
2039	}
2040
2041	/*
2042	* Maps an page allocation chunk into ring-3 so ring-0 can use it.
2043	*/
2044	case VMMCALLRING3_PGM_MAP_CHUNK:
2045	{
2046	pVCpu->vmm.s.rcCallRing3 = PGMR3PhysChunkMap(pVM, pVCpu->vmm.s.u64CallRing3Arg);
2047	break;
2048	}
2049
2050	/*
2051	* Allocates more handy pages.
2052	*/
2053	case VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES:
2054	{
2055	pVCpu->vmm.s.rcCallRing3 = PGMR3PhysAllocateHandyPages(pVM);
2056	break;
2057	}
2058
2059	/*
2060	* Acquire the PGM lock.
2061	*/
2062	case VMMCALLRING3_PGM_LOCK:
2063	{
2064	pVCpu->vmm.s.rcCallRing3 = PGMR3LockCall(pVM);
2065	break;
2066	}
2067
2068	/*
2069	* Acquire the MM hypervisor heap lock.
2070	*/
2071	case VMMCALLRING3_MMHYPER_LOCK:
2072	{
2073	pVCpu->vmm.s.rcCallRing3 = MMR3LockCall(pVM);
2074	break;
2075	}
2076
2077	/*
2078	* Flush REM handler notifications.
2079	*/
2080	case VMMCALLRING3_REM_REPLAY_HANDLER_NOTIFICATIONS:
2081	{
2082	REMR3ReplayHandlerNotifications(pVM);
2083	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
2084	break;
2085	}
2086
2087	/*
2088	* This is a noop. We just take this route to avoid unnecessary
2089	* tests in the loops.
2090	*/
2091	case VMMCALLRING3_VMM_LOGGER_FLUSH:
2092	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
2093	LogAlways(("FLUSH\n"));
2094	break;
2095
2096	/*
2097	* Set the VM error message.
2098	*/
2099	case VMMCALLRING3_VM_SET_ERROR:
2100	VMR3SetErrorWorker(pVM);
2101	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
2102	break;
2103
2104	/*
2105	* Set the VM runtime error message.
2106	*/
2107	case VMMCALLRING3_VM_SET_RUNTIME_ERROR:
2108	pVCpu->vmm.s.rcCallRing3 = VMR3SetRuntimeErrorWorker(pVM);
2109	break;
2110
2111	/*
2112	* Signal a ring 0 hypervisor assertion.
2113	* Cancel the longjmp operation that's in progress.
2114	*/
2115	case VMMCALLRING3_VM_R0_ASSERTION:
2116	pVCpu->vmm.s.enmCallRing3Operation = VMMCALLRING3_INVALID;
2117	pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call = false;
2118	#ifdef RT_ARCH_X86
2119	pVCpu->vmm.s.CallRing3JmpBufR0.eip = 0;
2120	#else
2121	pVCpu->vmm.s.CallRing3JmpBufR0.rip = 0;
2122	#endif
2123	LogRel((pVM->vmm.s.szRing0AssertMsg1));
2124	LogRel((pVM->vmm.s.szRing0AssertMsg2));
2125	return VERR_VMM_RING0_ASSERTION;
2126
2127	/*
2128	* A forced switch to ring 0 for preemption purposes.
2129	*/
2130	case VMMCALLRING3_VM_R0_PREEMPT:
2131	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
2132	break;
2133
2134	default:
2135	AssertMsgFailed(("enmCallRing3Operation=%d\n", pVCpu->vmm.s.enmCallRing3Operation));
2136	return VERR_INTERNAL_ERROR;
2137	}
2138
2139	pVCpu->vmm.s.enmCallRing3Operation = VMMCALLRING3_INVALID;
2140	return VINF_SUCCESS;
2141	}
2142
2143
2144	/**
2145	* Displays the Force action Flags.
2146	*
2147	* @param pVM The VM handle.
2148	* @param pHlp The output helpers.
2149	* @param pszArgs The additional arguments (ignored).
2150	*/
2151	static DECLCALLBACK(void) vmmR3InfoFF(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
2152	{
2153	int c;
2154	uint32_t f;
2155	#define PRINT_FLAG(prf,flag) do { \
2156	if (f & (prf##flag)) \
2157	{ \
2158	static const char *s_psz = #flag; \
2159	if (!(c % 6)) \
2160	pHlp->pfnPrintf(pHlp, "%s\n %s", c ? "," : "", s_psz); \
2161	else \
2162	pHlp->pfnPrintf(pHlp, ", %s", s_psz); \
2163	c++; \
2164	f &= ~(prf##flag); \
2165	} \
2166	} while (0)
2167
2168	#define PRINT_GROUP(prf,grp,sfx) do { \
2169	if (f & (prf##grp##sfx)) \
2170	{ \
2171	static const char *s_psz = #grp; \
2172	if (!(c % 5)) \
2173	pHlp->pfnPrintf(pHlp, "%s %s", c ? ",\n" : " Groups:\n", s_psz); \
2174	else \
2175	pHlp->pfnPrintf(pHlp, ", %s", s_psz); \
2176	c++; \
2177	} \
2178	} while (0)
2179
2180	/*
2181	* The global flags.
2182	*/
2183	const uint32_t fGlobalForcedActions = pVM->fGlobalForcedActions;
2184	pHlp->pfnPrintf(pHlp, "Global FFs: %#RX32", fGlobalForcedActions);
2185
2186	/* show the flag mnemonics */
2187	c = 0;
2188	f = fGlobalForcedActions;
2189	PRINT_FLAG(VM_FF_,TM_VIRTUAL_SYNC);
2190	PRINT_FLAG(VM_FF_,PDM_QUEUES);
2191	PRINT_FLAG(VM_FF_,PDM_DMA);
2192	PRINT_FLAG(VM_FF_,DBGF);
2193	PRINT_FLAG(VM_FF_,REQUEST);
2194	PRINT_FLAG(VM_FF_,TERMINATE);
2195	PRINT_FLAG(VM_FF_,RESET);
2196	PRINT_FLAG(VM_FF_,EMT_RENDEZVOUS);
2197	PRINT_FLAG(VM_FF_,PGM_NEED_HANDY_PAGES);
2198	PRINT_FLAG(VM_FF_,PGM_NO_MEMORY);
2199	PRINT_FLAG(VM_FF_,REM_HANDLER_NOTIFY);
2200	PRINT_FLAG(VM_FF_,DEBUG_SUSPEND);
2201	if (f)
2202	pHlp->pfnPrintf(pHlp, "%s\n Unknown bits: %#RX32\n", c ? "," : "", f);
2203	else
2204	pHlp->pfnPrintf(pHlp, "\n");
2205
2206	/* the groups */
2207	c = 0;
2208	f = fGlobalForcedActions;
2209	PRINT_GROUP(VM_FF_,EXTERNAL_SUSPENDED,_MASK);
2210	PRINT_GROUP(VM_FF_,EXTERNAL_HALTED,_MASK);
2211	PRINT_GROUP(VM_FF_,HIGH_PRIORITY_PRE,_MASK);
2212	PRINT_GROUP(VM_FF_,HIGH_PRIORITY_PRE_RAW,_MASK);
2213	PRINT_GROUP(VM_FF_,HIGH_PRIORITY_POST,_MASK);
2214	PRINT_GROUP(VM_FF_,NORMAL_PRIORITY_POST,_MASK);
2215	PRINT_GROUP(VM_FF_,NORMAL_PRIORITY,_MASK);
2216	PRINT_GROUP(VM_FF_,ALL_BUT_RAW,_MASK);
2217	if (c)
2218	pHlp->pfnPrintf(pHlp, "\n");
2219
2220	/*
2221	* Per CPU flags.
2222	*/
2223	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2224	{
2225	const uint32_t fLocalForcedActions = pVM->aCpus[i].fLocalForcedActions;
2226	pHlp->pfnPrintf(pHlp, "CPU %u FFs: %#RX32", i, fLocalForcedActions);
2227
2228	/* show the flag mnemonics */
2229	c = 0;
2230	f = fLocalForcedActions;
2231	PRINT_FLAG(VMCPU_FF_,INTERRUPT_APIC);
2232	PRINT_FLAG(VMCPU_FF_,INTERRUPT_PIC);
2233	PRINT_FLAG(VMCPU_FF_,TIMER);
2234	PRINT_FLAG(VMCPU_FF_,PDM_CRITSECT);
2235	PRINT_FLAG(VMCPU_FF_,PGM_SYNC_CR3);
2236	PRINT_FLAG(VMCPU_FF_,PGM_SYNC_CR3_NON_GLOBAL);
2237	PRINT_FLAG(VMCPU_FF_,TRPM_SYNC_IDT);
2238	PRINT_FLAG(VMCPU_FF_,SELM_SYNC_TSS);
2239	PRINT_FLAG(VMCPU_FF_,SELM_SYNC_GDT);
2240	PRINT_FLAG(VMCPU_FF_,SELM_SYNC_LDT);
2241	PRINT_FLAG(VMCPU_FF_,INHIBIT_INTERRUPTS);
2242	PRINT_FLAG(VMCPU_FF_,CSAM_SCAN_PAGE);
2243	PRINT_FLAG(VMCPU_FF_,CSAM_PENDING_ACTION);
2244	PRINT_FLAG(VMCPU_FF_,TO_R3);
2245	if (f)
2246	pHlp->pfnPrintf(pHlp, "%s\n Unknown bits: %#RX32\n", c ? "," : "", f);
2247	else
2248	pHlp->pfnPrintf(pHlp, "\n");
2249
2250	/* the groups */
2251	c = 0;
2252	f = fLocalForcedActions;
2253	PRINT_GROUP(VMCPU_FF_,EXTERNAL_SUSPENDED,_MASK);
2254	PRINT_GROUP(VMCPU_FF_,EXTERNAL_HALTED,_MASK);
2255	PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_PRE,_MASK);
2256	PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_PRE_RAW,_MASK);
2257	PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_POST,_MASK);
2258	PRINT_GROUP(VMCPU_FF_,NORMAL_PRIORITY_POST,_MASK);
2259	PRINT_GROUP(VMCPU_FF_,NORMAL_PRIORITY,_MASK);
2260	PRINT_GROUP(VMCPU_FF_,RESUME_GUEST,_MASK);
2261	PRINT_GROUP(VMCPU_FF_,HWACCM_TO_R3,_MASK);
2262	PRINT_GROUP(VMCPU_FF_,ALL_BUT_RAW,_MASK);
2263	if (c)
2264	pHlp->pfnPrintf(pHlp, "\n");
2265	}
2266
2267	#undef PRINT_FLAG
2268	#undef PRINT_GROUP
2269	}
2270

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMM.cpp@ 23495

Download in other formats: