VMM.cpp@ 29576

Last change on this file since 29576 was 28800, checked in by vboxsync, 15 years ago
Automated rebranding to Oracle copyright/license strings via filemuncher
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 81.1 KB

Line
1	/* $Id: VMM.cpp 28800 2010-04-27 08:22:32Z vboxsync $ */
2	/** @file
3	* VMM - The Virtual Machine Monitor Core.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	//#define NO_SUPCALLR0VMM
19
20	/** @page pg_vmm VMM - The Virtual Machine Monitor
21	*
22	* The VMM component is two things at the moment, it's a component doing a few
23	* management and routing tasks, and it's the whole virtual machine monitor
24	* thing. For hysterical reasons, it is not doing all the management that one
25	* would expect, this is instead done by @ref pg_vm. We'll address this
26	* misdesign eventually.
27	*
28	* @see grp_vmm, grp_vm
29	*
30	*
31	* @section sec_vmmstate VMM State
32	*
33	* @image html VM_Statechart_Diagram.gif
34	*
35	* To be written.
36	*
37	*
38	* @subsection subsec_vmm_init VMM Initialization
39	*
40	* To be written.
41	*
42	*
43	* @subsection subsec_vmm_term VMM Termination
44	*
45	* To be written.
46	*
47	*/
48
49	/*******************************************************************************
50	* Header Files *
51	*******************************************************************************/
52	#define LOG_GROUP LOG_GROUP_VMM
53	#include <VBox/vmm.h>
54	#include <VBox/vmapi.h>
55	#include <VBox/pgm.h>
56	#include <VBox/cfgm.h>
57	#include <VBox/pdmqueue.h>
58	#include <VBox/pdmcritsect.h>
59	#include <VBox/pdmapi.h>
60	#include <VBox/cpum.h>
61	#include <VBox/mm.h>
62	#include <VBox/iom.h>
63	#include <VBox/trpm.h>
64	#include <VBox/selm.h>
65	#include <VBox/em.h>
66	#include <VBox/sup.h>
67	#include <VBox/dbgf.h>
68	#include <VBox/csam.h>
69	#include <VBox/patm.h>
70	#include <VBox/rem.h>
71	#include <VBox/ssm.h>
72	#include <VBox/tm.h>
73	#include "VMMInternal.h"
74	#include "VMMSwitcher/VMMSwitcher.h"
75	#include <VBox/vm.h>
76
77	#include <VBox/err.h>
78	#include <VBox/param.h>
79	#include <VBox/version.h>
80	#include <VBox/x86.h>
81	#include <VBox/hwaccm.h>
82	#include <iprt/assert.h>
83	#include <iprt/alloc.h>
84	#include <iprt/asm.h>
85	#include <iprt/time.h>
86	#include <iprt/semaphore.h>
87	#include <iprt/stream.h>
88	#include <iprt/string.h>
89	#include <iprt/stdarg.h>
90	#include <iprt/ctype.h>
91
92
93
94	/*******************************************************************************
95	* Defined Constants And Macros *
96	*******************************************************************************/
97	/** The saved state version. */
98	#define VMM_SAVED_STATE_VERSION 4
99	/** The saved state version used by v3.0 and earlier. (Teleportation) */
100	#define VMM_SAVED_STATE_VERSION_3_0 3
101
102
103	/*******************************************************************************
104	* Internal Functions *
105	*******************************************************************************/
106	static int vmmR3InitStacks(PVM pVM);
107	static int vmmR3InitLoggers(PVM pVM);
108	static void vmmR3InitRegisterStats(PVM pVM);
109	static DECLCALLBACK(int) vmmR3Save(PVM pVM, PSSMHANDLE pSSM);
110	static DECLCALLBACK(int) vmmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
111	static DECLCALLBACK(void) vmmR3YieldEMT(PVM pVM, PTMTIMER pTimer, void *pvUser);
112	static int vmmR3ServiceCallRing3Request(PVM pVM, PVMCPU pVCpu);
113	static DECLCALLBACK(void) vmmR3InfoFF(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
114
115
116	/**
117	* Initializes the VMM.
118	*
119	* @returns VBox status code.
120	* @param pVM The VM to operate on.
121	*/
122	VMMR3DECL(int) VMMR3Init(PVM pVM)
123	{
124	LogFlow(("VMMR3Init\n"));
125
126	/*
127	* Assert alignment, sizes and order.
128	*/
129	AssertMsg(pVM->vmm.s.offVM == 0, ("Already initialized!\n"));
130	AssertCompile(sizeof(pVM->vmm.s) <= sizeof(pVM->vmm.padding));
131	AssertCompile(sizeof(pVM->aCpus[0].vmm.s) <= sizeof(pVM->aCpus[0].vmm.padding));
132
133	/*
134	* Init basic VM VMM members.
135	*/
136	pVM->vmm.s.offVM = RT_OFFSETOF(VM, vmm);
137	pVM->vmm.s.pahEvtRendezvousEnterOrdered = NULL;
138	pVM->vmm.s.hEvtRendezvousEnterOneByOne = NIL_RTSEMEVENT;
139	pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce = NIL_RTSEMEVENTMULTI;
140	pVM->vmm.s.hEvtMulRendezvousDone = NIL_RTSEMEVENTMULTI;
141	pVM->vmm.s.hEvtRendezvousDoneCaller = NIL_RTSEMEVENT;
142	int rc = CFGMR3QueryU32(CFGMR3GetRoot(pVM), "YieldEMTInterval", &pVM->vmm.s.cYieldEveryMillies);
143	if (rc == VERR_CFGM_VALUE_NOT_FOUND)
144	pVM->vmm.s.cYieldEveryMillies = 23; /* Value arrived at after experimenting with the grub boot prompt. */
145	//pVM->vmm.s.cYieldEveryMillies = 8; //debugging
146	else
147	AssertMsgRCReturn(rc, ("Configuration error. Failed to query \"YieldEMTInterval\", rc=%Rrc\n", rc), rc);
148
149	/*
150	* Initialize the VMM sync critical section and semaphores.
151	*/
152	rc = RTCritSectInit(&pVM->vmm.s.CritSectSync);
153	AssertRCReturn(rc, rc);
154	pVM->vmm.s.pahEvtRendezvousEnterOrdered = (PRTSEMEVENT)MMR3HeapAlloc(pVM, MM_TAG_VMM, sizeof(RTSEMEVENT) * pVM->cCpus);
155	if (!pVM->vmm.s.pahEvtRendezvousEnterOrdered)
156	return VERR_NO_MEMORY;
157	for (VMCPUID i = 0; i < pVM->cCpus; i++)
158	pVM->vmm.s.pahEvtRendezvousEnterOrdered[i] = NIL_RTSEMEVENT;
159	for (VMCPUID i = 0; i < pVM->cCpus; i++)
160	{
161	rc = RTSemEventCreate(&pVM->vmm.s.pahEvtRendezvousEnterOrdered[i]);
162	AssertRCReturn(rc, rc);
163	}
164	rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousEnterOneByOne);
165	AssertRCReturn(rc, rc);
166	rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
167	AssertRCReturn(rc, rc);
168	rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousDone);
169	AssertRCReturn(rc, rc);
170	rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousDoneCaller);
171	AssertRCReturn(rc, rc);
172
173	/* GC switchers are enabled by default. Turned off by HWACCM. */
174	pVM->vmm.s.fSwitcherDisabled = false;
175
176	/*
177	* Register the saved state data unit.
178	*/
179	rc = SSMR3RegisterInternal(pVM, "vmm", 1, VMM_SAVED_STATE_VERSION, VMM_STACK_SIZE + sizeof(RTGCPTR),
180	NULL, NULL, NULL,
181	NULL, vmmR3Save, NULL,
182	NULL, vmmR3Load, NULL);
183	if (RT_FAILURE(rc))
184	return rc;
185
186	/*
187	* Register the Ring-0 VM handle with the session for fast ioctl calls.
188	*/
189	rc = SUPR3SetVMForFastIOCtl(pVM->pVMR0);
190	if (RT_FAILURE(rc))
191	return rc;
192
193	/*
194	* Init various sub-components.
195	*/
196	rc = vmmR3SwitcherInit(pVM);
197	if (RT_SUCCESS(rc))
198	{
199	rc = vmmR3InitStacks(pVM);
200	if (RT_SUCCESS(rc))
201	{
202	rc = vmmR3InitLoggers(pVM);
203
204	#ifdef VBOX_WITH_NMI
205	/*
206	* Allocate mapping for the host APIC.
207	*/
208	if (RT_SUCCESS(rc))
209	{
210	rc = MMR3HyperReserve(pVM, PAGE_SIZE, "Host APIC", &pVM->vmm.s.GCPtrApicBase);
211	AssertRC(rc);
212	}
213	#endif
214	if (RT_SUCCESS(rc))
215	{
216	/*
217	* Debug info and statistics.
218	*/
219	DBGFR3InfoRegisterInternal(pVM, "ff", "Displays the current Forced actions Flags.", vmmR3InfoFF);
220	vmmR3InitRegisterStats(pVM);
221
222	return VINF_SUCCESS;
223	}
224	}
225	/** @todo: Need failure cleanup. */
226
227	//more todo in here?
228	//if (RT_SUCCESS(rc))
229	//{
230	//}
231	//int rc2 = vmmR3TermCoreCode(pVM);
232	//AssertRC(rc2));
233	}
234
235	return rc;
236	}
237
238
239	/**
240	* Allocate & setup the VMM RC stack(s) (for EMTs).
241	*
242	* The stacks are also used for long jumps in Ring-0.
243	*
244	* @returns VBox status code.
245	* @param pVM Pointer to the shared VM structure.
246	*
247	* @remarks The optional guard page gets it protection setup up during R3 init
248	* completion because of init order issues.
249	*/
250	static int vmmR3InitStacks(PVM pVM)
251	{
252	int rc = VINF_SUCCESS;
253	#ifdef VMM_R0_SWITCH_STACK
254	uint32_t fFlags = MMHYPER_AONR_FLAGS_KERNEL_MAPPING;
255	#else
256	uint32_t fFlags = 0;
257	#endif
258
259	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
260	{
261	PVMCPU pVCpu = &pVM->aCpus[idCpu];
262
263	#ifdef VBOX_STRICT_VMM_STACK
264	rc = MMR3HyperAllocOnceNoRelEx(pVM, PAGE_SIZE + VMM_STACK_SIZE + PAGE_SIZE,
265	#else
266	rc = MMR3HyperAllocOnceNoRelEx(pVM, VMM_STACK_SIZE,
267	#endif
268	PAGE_SIZE, MM_TAG_VMM, fFlags, (void **)&pVCpu->vmm.s.pbEMTStackR3);
269	if (RT_SUCCESS(rc))
270	{
271	#ifdef VBOX_STRICT_VMM_STACK
272	pVCpu->vmm.s.pbEMTStackR3 += PAGE_SIZE;
273	#endif
274	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
275	/* MMHyperR3ToR0 returns R3 when not doing hardware assisted virtualization. */
276	if (!VMMIsHwVirtExtForced(pVM))
277	pVCpu->vmm.s.CallRing3JmpBufR0.pvSavedStack = NIL_RTR0PTR;
278	else
279	#endif
280	pVCpu->vmm.s.CallRing3JmpBufR0.pvSavedStack = MMHyperR3ToR0(pVM, pVCpu->vmm.s.pbEMTStackR3);
281	pVCpu->vmm.s.pbEMTStackRC = MMHyperR3ToRC(pVM, pVCpu->vmm.s.pbEMTStackR3);
282	pVCpu->vmm.s.pbEMTStackBottomRC = pVCpu->vmm.s.pbEMTStackRC + VMM_STACK_SIZE;
283	AssertRelease(pVCpu->vmm.s.pbEMTStackRC);
284
285	CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC);
286	}
287	}
288
289	return rc;
290	}
291
292
293	/**
294	* Initialize the loggers.
295	*
296	* @returns VBox status code.
297	* @param pVM Pointer to the shared VM structure.
298	*/
299	static int vmmR3InitLoggers(PVM pVM)
300	{
301	int rc;
302
303	/*
304	* Allocate RC & R0 Logger instances (they are finalized in the relocator).
305	*/
306	#ifdef LOG_ENABLED
307	PRTLOGGER pLogger = RTLogDefaultInstance();
308	if (pLogger)
309	{
310	pVM->vmm.s.cbRCLogger = RT_OFFSETOF(RTLOGGERRC, afGroups[pLogger->cGroups]);
311	rc = MMR3HyperAllocOnceNoRel(pVM, pVM->vmm.s.cbRCLogger, 0, MM_TAG_VMM, (void **)&pVM->vmm.s.pRCLoggerR3);
312	if (RT_FAILURE(rc))
313	return rc;
314	pVM->vmm.s.pRCLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCLoggerR3);
315
316	# ifdef VBOX_WITH_R0_LOGGING
317	for (VMCPUID i = 0; i < pVM->cCpus; i++)
318	{
319	PVMCPU pVCpu = &pVM->aCpus[i];
320
321	rc = MMR3HyperAllocOnceNoRelEx(pVM, RT_OFFSETOF(VMMR0LOGGER, Logger.afGroups[pLogger->cGroups]),
322	0, MM_TAG_VMM, MMHYPER_AONR_FLAGS_KERNEL_MAPPING,
323	(void **)&pVCpu->vmm.s.pR0LoggerR3);
324	if (RT_FAILURE(rc))
325	return rc;
326	pVCpu->vmm.s.pR0LoggerR3->pVM = pVM->pVMR0;
327	//pVCpu->vmm.s.pR0LoggerR3->fCreated = false;
328	pVCpu->vmm.s.pR0LoggerR3->cbLogger = RT_OFFSETOF(RTLOGGER, afGroups[pLogger->cGroups]);
329	pVCpu->vmm.s.pR0LoggerR0 = MMHyperR3ToR0(pVM, pVCpu->vmm.s.pR0LoggerR3);
330	}
331	# endif
332	}
333	#endif /* LOG_ENABLED */
334
335	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
336	/*
337	* Allocate RC release logger instances (finalized in the relocator).
338	*/
339	PRTLOGGER pRelLogger = RTLogRelDefaultInstance();
340	if (pRelLogger)
341	{
342	pVM->vmm.s.cbRCRelLogger = RT_OFFSETOF(RTLOGGERRC, afGroups[pRelLogger->cGroups]);
343	rc = MMR3HyperAllocOnceNoRel(pVM, pVM->vmm.s.cbRCRelLogger, 0, MM_TAG_VMM, (void **)&pVM->vmm.s.pRCRelLoggerR3);
344	if (RT_FAILURE(rc))
345	return rc;
346	pVM->vmm.s.pRCRelLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCRelLoggerR3);
347	}
348	#endif /* VBOX_WITH_RC_RELEASE_LOGGING */
349	return VINF_SUCCESS;
350	}
351
352
353	/**
354	* VMMR3Init worker that register the statistics with STAM.
355	*
356	* @param pVM The shared VM structure.
357	*/
358	static void vmmR3InitRegisterStats(PVM pVM)
359	{
360	/*
361	* Statistics.
362	*/
363	STAM_REG(pVM, &pVM->vmm.s.StatRunRC, STAMTYPE_COUNTER, "/VMM/RunRC", STAMUNIT_OCCURENCES, "Number of context switches.");
364	STAM_REG(pVM, &pVM->vmm.s.StatRZRetNormal, STAMTYPE_COUNTER, "/VMM/RZRet/Normal", STAMUNIT_OCCURENCES, "Number of VINF_SUCCESS returns.");
365	STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterrupt, STAMTYPE_COUNTER, "/VMM/RZRet/Interrupt", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT returns.");
366	STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterruptHyper, STAMTYPE_COUNTER, "/VMM/RZRet/InterruptHyper", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT_HYPER returns.");
367	STAM_REG(pVM, &pVM->vmm.s.StatRZRetGuestTrap, STAMTYPE_COUNTER, "/VMM/RZRet/GuestTrap", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_GUEST_TRAP returns.");
368	STAM_REG(pVM, &pVM->vmm.s.StatRZRetRingSwitch, STAMTYPE_COUNTER, "/VMM/RZRet/RingSwitch", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_RING_SWITCH returns.");
369	STAM_REG(pVM, &pVM->vmm.s.StatRZRetRingSwitchInt, STAMTYPE_COUNTER, "/VMM/RZRet/RingSwitchInt", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_RING_SWITCH_INT returns.");
370	STAM_REG(pVM, &pVM->vmm.s.StatRZRetStaleSelector, STAMTYPE_COUNTER, "/VMM/RZRet/StaleSelector", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_STALE_SELECTOR returns.");
371	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIRETTrap, STAMTYPE_COUNTER, "/VMM/RZRet/IRETTrap", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_IRET_TRAP returns.");
372	STAM_REG(pVM, &pVM->vmm.s.StatRZRetEmulate, STAMTYPE_COUNTER, "/VMM/RZRet/Emulate", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION returns.");
373	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIOBlockEmulate, STAMTYPE_COUNTER, "/VMM/RZRet/EmulateIOBlock", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_EMULATE_IO_BLOCK returns.");
374	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchEmulate, STAMTYPE_COUNTER, "/VMM/RZRet/PatchEmulate", STAMUNIT_OCCURENCES, "Number of VINF_PATCH_EMULATE_INSTR returns.");
375	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIORead, STAMTYPE_COUNTER, "/VMM/RZRet/IORead", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_IOPORT_READ returns.");
376	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIOWrite, STAMTYPE_COUNTER, "/VMM/RZRet/IOWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_IOPORT_WRITE returns.");
377	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIORead, STAMTYPE_COUNTER, "/VMM/RZRet/MMIORead", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_READ returns.");
378	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOWrite, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_WRITE returns.");
379	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOReadWrite, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOReadWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_READ_WRITE returns.");
380	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOPatchRead, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOPatchRead", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_PATCH_READ returns.");
381	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOPatchWrite, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOPatchWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_PATCH_WRITE returns.");
382	STAM_REG(pVM, &pVM->vmm.s.StatRZRetLDTFault, STAMTYPE_COUNTER, "/VMM/RZRet/LDTFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_GDT_FAULT returns.");
383	STAM_REG(pVM, &pVM->vmm.s.StatRZRetGDTFault, STAMTYPE_COUNTER, "/VMM/RZRet/GDTFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_LDT_FAULT returns.");
384	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIDTFault, STAMTYPE_COUNTER, "/VMM/RZRet/IDTFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_IDT_FAULT returns.");
385	STAM_REG(pVM, &pVM->vmm.s.StatRZRetTSSFault, STAMTYPE_COUNTER, "/VMM/RZRet/TSSFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_TSS_FAULT returns.");
386	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPDFault, STAMTYPE_COUNTER, "/VMM/RZRet/PDFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_PD_FAULT returns.");
387	STAM_REG(pVM, &pVM->vmm.s.StatRZRetCSAMTask, STAMTYPE_COUNTER, "/VMM/RZRet/CSAMTask", STAMUNIT_OCCURENCES, "Number of VINF_CSAM_PENDING_ACTION returns.");
388	STAM_REG(pVM, &pVM->vmm.s.StatRZRetSyncCR3, STAMTYPE_COUNTER, "/VMM/RZRet/SyncCR", STAMUNIT_OCCURENCES, "Number of VINF_PGM_SYNC_CR3 returns.");
389	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMisc, STAMTYPE_COUNTER, "/VMM/RZRet/Misc", STAMUNIT_OCCURENCES, "Number of misc returns.");
390	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchInt3, STAMTYPE_COUNTER, "/VMM/RZRet/PatchInt3", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_INT3 returns.");
391	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchPF, STAMTYPE_COUNTER, "/VMM/RZRet/PatchPF", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_TRAP_PF returns.");
392	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchGP, STAMTYPE_COUNTER, "/VMM/RZRet/PatchGP", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_TRAP_GP returns.");
393	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchIretIRQ, STAMTYPE_COUNTER, "/VMM/RZRet/PatchIret", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PENDING_IRQ_AFTER_IRET returns.");
394	STAM_REG(pVM, &pVM->vmm.s.StatRZRetRescheduleREM, STAMTYPE_COUNTER, "/VMM/RZRet/ScheduleREM", STAMUNIT_OCCURENCES, "Number of VINF_EM_RESCHEDULE_REM returns.");
395	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
396	STAM_REG(pVM, &pVM->vmm.s.StatRZRetTimerPending, STAMTYPE_COUNTER, "/VMM/RZRet/TimerPending", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TIMER_PENDING returns.");
397	STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterruptPending, STAMTYPE_COUNTER, "/VMM/RZRet/InterruptPending", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT_PENDING returns.");
398	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPATMDuplicateFn, STAMTYPE_COUNTER, "/VMM/RZRet/PATMDuplicateFn", STAMUNIT_OCCURENCES, "Number of VINF_PATM_DUPLICATE_FUNCTION returns.");
399	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPGMChangeMode, STAMTYPE_COUNTER, "/VMM/RZRet/PGMChangeMode", STAMUNIT_OCCURENCES, "Number of VINF_PGM_CHANGE_MODE returns.");
400	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPGMFlushPending, STAMTYPE_COUNTER, "/VMM/RZRet/PGMFlushPending", STAMUNIT_OCCURENCES, "Number of VINF_PGM_POOL_FLUSH_PENDING returns.");
401	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPendingRequest, STAMTYPE_COUNTER, "/VMM/RZRet/PendingRequest", STAMUNIT_OCCURENCES, "Number of VINF_EM_PENDING_REQUEST returns.");
402	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchTPR, STAMTYPE_COUNTER, "/VMM/RZRet/PatchTPR", STAMUNIT_OCCURENCES, "Number of VINF_EM_HWACCM_PATCH_TPR_INSTR returns.");
403	STAM_REG(pVM, &pVM->vmm.s.StatRZRetCallRing3, STAMTYPE_COUNTER, "/VMM/RZCallR3/Misc", STAMUNIT_OCCURENCES, "Number of Other ring-3 calls.");
404	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPDMLock, STAMTYPE_COUNTER, "/VMM/RZCallR3/PDMLock", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PDM_LOCK calls.");
405	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPDMQueueFlush, STAMTYPE_COUNTER, "/VMM/RZCallR3/PDMQueueFlush", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PDM_QUEUE_FLUSH calls.");
406	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMLock, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMLock", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_LOCK calls.");
407	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMPoolGrow, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMPoolGrow", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_POOL_GROW calls.");
408	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMMapChunk, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMMapChunk", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_MAP_CHUNK calls.");
409	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMAllocHandy, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMAllocHandy", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES calls.");
410	STAM_REG(pVM, &pVM->vmm.s.StatRZCallRemReplay, STAMTYPE_COUNTER, "/VMM/RZCallR3/REMReplay", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_REM_REPLAY_HANDLER_NOTIFICATIONS calls.");
411	STAM_REG(pVM, &pVM->vmm.s.StatRZCallLogFlush, STAMTYPE_COUNTER, "/VMM/RZCallR3/VMMLogFlush", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VMM_LOGGER_FLUSH calls.");
412	STAM_REG(pVM, &pVM->vmm.s.StatRZCallVMSetError, STAMTYPE_COUNTER, "/VMM/RZCallR3/VMSetError", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VM_SET_ERROR calls.");
413	STAM_REG(pVM, &pVM->vmm.s.StatRZCallVMSetRuntimeError, STAMTYPE_COUNTER, "/VMM/RZCallR3/VMRuntimeError", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VM_SET_RUNTIME_ERROR calls.");
414
415	#ifdef VBOX_WITH_STATISTICS
416	for (VMCPUID i = 0; i < pVM->cCpus; i++)
417	{
418	STAMR3RegisterF(pVM, &pVM->aCpus[i].vmm.s.CallRing3JmpBufR0.cbUsedMax, STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Max amount of stack used.", "/VMM/Stack/CPU%u/Max", i);
419	STAMR3RegisterF(pVM, &pVM->aCpus[i].vmm.s.CallRing3JmpBufR0.cbUsedAvg, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Average stack usage.", "/VMM/Stack/CPU%u/Avg", i);
420	STAMR3RegisterF(pVM, &pVM->aCpus[i].vmm.s.CallRing3JmpBufR0.cUsedTotal, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of stack usages.", "/VMM/Stack/CPU%u/Uses", i);
421	}
422	#endif
423	}
424
425
426	/**
427	* Initializes the per-VCPU VMM.
428	*
429	* @returns VBox status code.
430	* @param pVM The VM to operate on.
431	*/
432	VMMR3DECL(int) VMMR3InitCPU(PVM pVM)
433	{
434	LogFlow(("VMMR3InitCPU\n"));
435	return VINF_SUCCESS;
436	}
437
438
439	/**
440	* Ring-3 init finalizing.
441	*
442	* @returns VBox status code.
443	* @param pVM The VM handle.
444	*/
445	VMMR3DECL(int) VMMR3InitFinalize(PVM pVM)
446	{
447	int rc;
448
449	/*
450	* Set page attributes to r/w for stack pages.
451	*/
452	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
453	{
454	rc = PGMMapSetPage(pVM, pVM->aCpus[idCpu].vmm.s.pbEMTStackRC, VMM_STACK_SIZE,
455	X86_PTE_P \| X86_PTE_A \| X86_PTE_D \| X86_PTE_RW);
456	AssertRCReturn(rc, rc);
457	}
458
459	/*
460	* Create the EMT yield timer.
461	*/
462	rc = TMR3TimerCreateInternal(pVM, TMCLOCK_REAL, vmmR3YieldEMT, NULL, "EMT Yielder", &pVM->vmm.s.pYieldTimer);
463	AssertRCReturn(rc, rc);
464
465	rc = TMTimerSetMillies(pVM->vmm.s.pYieldTimer, pVM->vmm.s.cYieldEveryMillies);
466	AssertRCReturn(rc, rc);
467
468	#ifdef VBOX_WITH_NMI
469	/*
470	* Map the host APIC into GC - This is AMD/Intel + Host OS specific!
471	*/
472	rc = PGMMap(pVM, pVM->vmm.s.GCPtrApicBase, 0xfee00000, PAGE_SIZE,
473	X86_PTE_P \| X86_PTE_RW \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_A \| X86_PTE_D);
474	AssertRCReturn(rc, rc);
475	#endif
476
477	#ifdef VBOX_STRICT_VMM_STACK
478	/*
479	* Setup the stack guard pages: Two inaccessible pages at each sides of the
480	* stack to catch over/under-flows.
481	*/
482	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
483	{
484	uint8_t *pbEMTStackR3 = pVM->aCpus[idCpu].vmm.s.pbEMTStackR3;
485
486	memset(pbEMTStackR3 - PAGE_SIZE, 0xcc, PAGE_SIZE);
487	MMR3HyperSetGuard(pVM, pbEMTStackR3 - PAGE_SIZE, PAGE_SIZE, true /fSet/);
488
489	memset(pbEMTStackR3 + VMM_STACK_SIZE, 0xcc, PAGE_SIZE);
490	MMR3HyperSetGuard(pVM, pbEMTStackR3 + VMM_STACK_SIZE, PAGE_SIZE, true /fSet/);
491	}
492	pVM->vmm.s.fStackGuardsStationed = true;
493	#endif
494
495	return VINF_SUCCESS;
496	}
497
498
499	/**
500	* Initializes the R0 VMM.
501	*
502	* @returns VBox status code.
503	* @param pVM The VM to operate on.
504	*/
505	VMMR3DECL(int) VMMR3InitR0(PVM pVM)
506	{
507	int rc;
508	PVMCPU pVCpu = VMMGetCpu(pVM);
509	Assert(pVCpu && pVCpu->idCpu == 0);
510
511	#ifdef LOG_ENABLED
512	/*
513	* Initialize the ring-0 logger if we haven't done so yet.
514	*/
515	if ( pVCpu->vmm.s.pR0LoggerR3
516	&& !pVCpu->vmm.s.pR0LoggerR3->fCreated)
517	{
518	rc = VMMR3UpdateLoggers(pVM);
519	if (RT_FAILURE(rc))
520	return rc;
521	}
522	#endif
523
524	/*
525	* Call Ring-0 entry with init code.
526	*/
527	for (;;)
528	{
529	#ifdef NO_SUPCALLR0VMM
530	//rc = VERR_GENERAL_FAILURE;
531	rc = VINF_SUCCESS;
532	#else
533	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /idCpu/, VMMR0_DO_VMMR0_INIT, VMMGetSvnRev(), NULL);
534	#endif
535	/*
536	* Flush the logs.
537	*/
538	#ifdef LOG_ENABLED
539	if ( pVCpu->vmm.s.pR0LoggerR3
540	&& pVCpu->vmm.s.pR0LoggerR3->Logger.offScratch > 0)
541	RTLogFlushToLogger(&pVCpu->vmm.s.pR0LoggerR3->Logger, NULL);
542	#endif
543	if (rc != VINF_VMM_CALL_HOST)
544	break;
545	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
546	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
547	break;
548	/* Resume R0 */
549	}
550
551	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
552	{
553	LogRel(("R0 init failed, rc=%Rra\n", rc));
554	if (RT_SUCCESS(rc))
555	rc = VERR_INTERNAL_ERROR;
556	}
557	return rc;
558	}
559
560
561	/**
562	* Initializes the RC VMM.
563	*
564	* @returns VBox status code.
565	* @param pVM The VM to operate on.
566	*/
567	VMMR3DECL(int) VMMR3InitRC(PVM pVM)
568	{
569	PVMCPU pVCpu = VMMGetCpu(pVM);
570	Assert(pVCpu && pVCpu->idCpu == 0);
571
572	/* In VMX mode, there's no need to init RC. */
573	if (pVM->vmm.s.fSwitcherDisabled)
574	return VINF_SUCCESS;
575
576	AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
577
578	/*
579	* Call VMMGCInit():
580	* -# resolve the address.
581	* -# setup stackframe and EIP to use the trampoline.
582	* -# do a generic hypervisor call.
583	*/
584	RTRCPTR RCPtrEP;
585	int rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "VMMGCEntry", &RCPtrEP);
586	if (RT_SUCCESS(rc))
587	{
588	CPUMHyperSetCtxCore(pVCpu, NULL);
589	CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC); /* Clear the stack. */
590	uint64_t u64TS = RTTimeProgramStartNanoTS();
591	CPUMPushHyper(pVCpu, (uint32_t)(u64TS >> 32)); /* Param 3: The program startup TS - Hi. */
592	CPUMPushHyper(pVCpu, (uint32_t)u64TS); /* Param 3: The program startup TS - Lo. */
593	CPUMPushHyper(pVCpu, VMMGetSvnRev()); /* Param 2: Version argument. */
594	CPUMPushHyper(pVCpu, VMMGC_DO_VMMGC_INIT); /* Param 1: Operation. */
595	CPUMPushHyper(pVCpu, pVM->pVMRC); /* Param 0: pVM */
596	CPUMPushHyper(pVCpu, 5 * sizeof(RTRCPTR)); /* trampoline param: stacksize. */
597	CPUMPushHyper(pVCpu, RCPtrEP); /* Call EIP. */
598	CPUMSetHyperEIP(pVCpu, pVM->vmm.s.pfnCallTrampolineRC);
599	Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
600
601	for (;;)
602	{
603	#ifdef NO_SUPCALLR0VMM
604	//rc = VERR_GENERAL_FAILURE;
605	rc = VINF_SUCCESS;
606	#else
607	rc = SUPR3CallVMMR0(pVM->pVMR0, 0 /* VCPU 0 */, VMMR0_DO_CALL_HYPERVISOR, NULL);
608	#endif
609	#ifdef LOG_ENABLED
610	PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
611	if ( pLogger
612	&& pLogger->offScratch > 0)
613	RTLogFlushRC(NULL, pLogger);
614	#endif
615	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
616	PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
617	if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
618	RTLogFlushRC(RTLogRelDefaultInstance(), pRelLogger);
619	#endif
620	if (rc != VINF_VMM_CALL_HOST)
621	break;
622	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
623	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
624	break;
625	}
626
627	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
628	{
629	VMMR3FatalDump(pVM, pVCpu, rc);
630	if (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST)
631	rc = VERR_INTERNAL_ERROR;
632	}
633	AssertRC(rc);
634	}
635	return rc;
636	}
637
638
639	/**
640	* Terminate the VMM bits.
641	*
642	* @returns VINF_SUCCESS.
643	* @param pVM The VM handle.
644	*/
645	VMMR3DECL(int) VMMR3Term(PVM pVM)
646	{
647	PVMCPU pVCpu = VMMGetCpu(pVM);
648	Assert(pVCpu && pVCpu->idCpu == 0);
649
650	/*
651	* Call Ring-0 entry with termination code.
652	*/
653	int rc;
654	for (;;)
655	{
656	#ifdef NO_SUPCALLR0VMM
657	//rc = VERR_GENERAL_FAILURE;
658	rc = VINF_SUCCESS;
659	#else
660	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /idCpu/, VMMR0_DO_VMMR0_TERM, 0, NULL);
661	#endif
662	/*
663	* Flush the logs.
664	*/
665	#ifdef LOG_ENABLED
666	if ( pVCpu->vmm.s.pR0LoggerR3
667	&& pVCpu->vmm.s.pR0LoggerR3->Logger.offScratch > 0)
668	RTLogFlushToLogger(&pVCpu->vmm.s.pR0LoggerR3->Logger, NULL);
669	#endif
670	if (rc != VINF_VMM_CALL_HOST)
671	break;
672	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
673	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
674	break;
675	/* Resume R0 */
676	}
677	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
678	{
679	LogRel(("VMMR3Term: R0 term failed, rc=%Rra. (warning)\n", rc));
680	if (RT_SUCCESS(rc))
681	rc = VERR_INTERNAL_ERROR;
682	}
683
684	RTCritSectDelete(&pVM->vmm.s.CritSectSync);
685	for (VMCPUID i = 0; i < pVM->cCpus; i++)
686	{
687	RTSemEventDestroy(pVM->vmm.s.pahEvtRendezvousEnterOrdered[i]);
688	pVM->vmm.s.pahEvtRendezvousEnterOrdered[i] = NIL_RTSEMEVENT;
689	}
690	RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousEnterOneByOne);
691	pVM->vmm.s.hEvtRendezvousEnterOneByOne = NIL_RTSEMEVENT;
692	RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
693	pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce = NIL_RTSEMEVENTMULTI;
694	RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousDone);
695	pVM->vmm.s.hEvtMulRendezvousDone = NIL_RTSEMEVENTMULTI;
696	RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousDoneCaller);
697	pVM->vmm.s.hEvtRendezvousDoneCaller = NIL_RTSEMEVENT;
698
699	#ifdef VBOX_STRICT_VMM_STACK
700	/*
701	* Make the two stack guard pages present again.
702	*/
703	if (pVM->vmm.s.fStackGuardsStationed)
704	{
705	for (VMCPUID i = 0; i < pVM->cCpus; i++)
706	{
707	uint8_t *pbEMTStackR3 = pVM->aCpus[i].vmm.s.pbEMTStackR3;
708	MMR3HyperSetGuard(pVM, pbEMTStackR3 - PAGE_SIZE, PAGE_SIZE, false /fSet/);
709	MMR3HyperSetGuard(pVM, pbEMTStackR3 + VMM_STACK_SIZE, PAGE_SIZE, false /fSet/);
710	}
711	pVM->vmm.s.fStackGuardsStationed = false;
712	}
713	#endif
714	return rc;
715	}
716
717
718	/**
719	* Terminates the per-VCPU VMM.
720	*
721	* Termination means cleaning up and freeing all resources,
722	* the VM it self is at this point powered off or suspended.
723	*
724	* @returns VBox status code.
725	* @param pVM The VM to operate on.
726	*/
727	VMMR3DECL(int) VMMR3TermCPU(PVM pVM)
728	{
729	return VINF_SUCCESS;
730	}
731
732
733	/**
734	* Applies relocations to data and code managed by this
735	* component. This function will be called at init and
736	* whenever the VMM need to relocate it self inside the GC.
737	*
738	* The VMM will need to apply relocations to the core code.
739	*
740	* @param pVM The VM handle.
741	* @param offDelta The relocation delta.
742	*/
743	VMMR3DECL(void) VMMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
744	{
745	LogFlow(("VMMR3Relocate: offDelta=%RGv\n", offDelta));
746
747	/*
748	* Recalc the RC address.
749	*/
750	#ifdef VBOX_WITH_RAW_MODE
751	pVM->vmm.s.pvCoreCodeRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pvCoreCodeR3);
752	#endif
753
754	/*
755	* The stack.
756	*/
757	for (VMCPUID i = 0; i < pVM->cCpus; i++)
758	{
759	PVMCPU pVCpu = &pVM->aCpus[i];
760
761	CPUMSetHyperESP(pVCpu, CPUMGetHyperESP(pVCpu) + offDelta);
762
763	pVCpu->vmm.s.pbEMTStackRC = MMHyperR3ToRC(pVM, pVCpu->vmm.s.pbEMTStackR3);
764	pVCpu->vmm.s.pbEMTStackBottomRC = pVCpu->vmm.s.pbEMTStackRC + VMM_STACK_SIZE;
765	}
766
767	/*
768	* All the switchers.
769	*/
770	vmmR3SwitcherRelocate(pVM, offDelta);
771
772	/*
773	* Get other RC entry points.
774	*/
775	int rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "CPUMGCResumeGuest", &pVM->vmm.s.pfnCPUMRCResumeGuest);
776	AssertReleaseMsgRC(rc, ("CPUMGCResumeGuest not found! rc=%Rra\n", rc));
777
778	rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "CPUMGCResumeGuestV86", &pVM->vmm.s.pfnCPUMRCResumeGuestV86);
779	AssertReleaseMsgRC(rc, ("CPUMGCResumeGuestV86 not found! rc=%Rra\n", rc));
780
781	/*
782	* Update the logger.
783	*/
784	VMMR3UpdateLoggers(pVM);
785	}
786
787
788	/**
789	* Updates the settings for the RC and R0 loggers.
790	*
791	* @returns VBox status code.
792	* @param pVM The VM handle.
793	*/
794	VMMR3DECL(int) VMMR3UpdateLoggers(PVM pVM)
795	{
796	/*
797	* Simply clone the logger instance (for RC).
798	*/
799	int rc = VINF_SUCCESS;
800	RTRCPTR RCPtrLoggerFlush = 0;
801
802	if (pVM->vmm.s.pRCLoggerR3
803	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
804	\|\| pVM->vmm.s.pRCRelLoggerR3
805	#endif
806	)
807	{
808	rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "vmmGCLoggerFlush", &RCPtrLoggerFlush);
809	AssertReleaseMsgRC(rc, ("vmmGCLoggerFlush not found! rc=%Rra\n", rc));
810	}
811
812	if (pVM->vmm.s.pRCLoggerR3)
813	{
814	RTRCPTR RCPtrLoggerWrapper = 0;
815	rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "vmmGCLoggerWrapper", &RCPtrLoggerWrapper);
816	AssertReleaseMsgRC(rc, ("vmmGCLoggerWrapper not found! rc=%Rra\n", rc));
817
818	pVM->vmm.s.pRCLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCLoggerR3);
819	rc = RTLogCloneRC(NULL /* default */, pVM->vmm.s.pRCLoggerR3, pVM->vmm.s.cbRCLogger,
820	RCPtrLoggerWrapper, RCPtrLoggerFlush, RTLOGFLAGS_BUFFERED);
821	AssertReleaseMsgRC(rc, ("RTLogCloneRC failed! rc=%Rra\n", rc));
822	}
823
824	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
825	if (pVM->vmm.s.pRCRelLoggerR3)
826	{
827	RTRCPTR RCPtrLoggerWrapper = 0;
828	rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "vmmGCRelLoggerWrapper", &RCPtrLoggerWrapper);
829	AssertReleaseMsgRC(rc, ("vmmGCRelLoggerWrapper not found! rc=%Rra\n", rc));
830
831	pVM->vmm.s.pRCRelLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCRelLoggerR3);
832	rc = RTLogCloneRC(RTLogRelDefaultInstance(), pVM->vmm.s.pRCRelLoggerR3, pVM->vmm.s.cbRCRelLogger,
833	RCPtrLoggerWrapper, RCPtrLoggerFlush, RTLOGFLAGS_BUFFERED);
834	AssertReleaseMsgRC(rc, ("RTLogCloneRC failed! rc=%Rra\n", rc));
835	}
836	#endif /* VBOX_WITH_RC_RELEASE_LOGGING */
837
838	#ifdef LOG_ENABLED
839	/*
840	* For the ring-0 EMT logger, we use a per-thread logger instance
841	* in ring-0. Only initialize it once.
842	*/
843	for (VMCPUID i = 0; i < pVM->cCpus; i++)
844	{
845	PVMCPU pVCpu = &pVM->aCpus[i];
846	PVMMR0LOGGER pR0LoggerR3 = pVCpu->vmm.s.pR0LoggerR3;
847	if (pR0LoggerR3)
848	{
849	if (!pR0LoggerR3->fCreated)
850	{
851	RTR0PTR pfnLoggerWrapper = NIL_RTR0PTR;
852	rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerWrapper", &pfnLoggerWrapper);
853	AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerWrapper not found! rc=%Rra\n", rc), rc);
854
855	RTR0PTR pfnLoggerFlush = NIL_RTR0PTR;
856	rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerFlush", &pfnLoggerFlush);
857	AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerFlush not found! rc=%Rra\n", rc), rc);
858
859	rc = RTLogCreateForR0(&pR0LoggerR3->Logger, pR0LoggerR3->cbLogger,
860	(PFNRTLOGGER )&pfnLoggerWrapper, (PFNRTLOGFLUSH )&pfnLoggerFlush,
861	RTLOGFLAGS_BUFFERED, RTLOGDEST_DUMMY);
862	AssertReleaseMsgRCReturn(rc, ("RTLogCreateForR0 failed! rc=%Rra\n", rc), rc);
863
864	RTR0PTR pfnLoggerPrefix = NIL_RTR0PTR;
865	rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerPrefix", &pfnLoggerPrefix);
866	AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerPrefix not found! rc=%Rra\n", rc), rc);
867	rc = RTLogSetCustomPrefixCallback(&pR0LoggerR3->Logger, (PFNRTLOGPREFIX )&pfnLoggerPrefix, NULL);
868	AssertReleaseMsgRCReturn(rc, ("RTLogSetCustomPrefixCallback failed! rc=%Rra\n", rc), rc);
869
870	pR0LoggerR3->idCpu = i;
871	pR0LoggerR3->fCreated = true;
872	pR0LoggerR3->fFlushingDisabled = false;
873
874	}
875
876	rc = RTLogCopyGroupsAndFlags(&pR0LoggerR3->Logger, NULL /* default */, pVM->vmm.s.pRCLoggerR3->fFlags, RTLOGFLAGS_BUFFERED);
877	AssertRC(rc);
878	}
879	}
880	#endif
881	return rc;
882	}
883
884
885	/**
886	* Gets the pointer to a buffer containing the R0/RC RTAssertMsg1Weak output.
887	*
888	* @returns Pointer to the buffer.
889	* @param pVM The VM handle.
890	*/
891	VMMR3DECL(const char *) VMMR3GetRZAssertMsg1(PVM pVM)
892	{
893	if (HWACCMIsEnabled(pVM))
894	return pVM->vmm.s.szRing0AssertMsg1;
895
896	RTRCPTR RCPtr;
897	int rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_szRTAssertMsg1", &RCPtr);
898	if (RT_SUCCESS(rc))
899	return (const char *)MMHyperRCToR3(pVM, RCPtr);
900
901	return NULL;
902	}
903
904
905	/**
906	* Gets the pointer to a buffer containing the R0/RC RTAssertMsg2Weak output.
907	*
908	* @returns Pointer to the buffer.
909	* @param pVM The VM handle.
910	*/
911	VMMR3DECL(const char *) VMMR3GetRZAssertMsg2(PVM pVM)
912	{
913	if (HWACCMIsEnabled(pVM))
914	return pVM->vmm.s.szRing0AssertMsg2;
915
916	RTRCPTR RCPtr;
917	int rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_szRTAssertMsg2", &RCPtr);
918	if (RT_SUCCESS(rc))
919	return (const char *)MMHyperRCToR3(pVM, RCPtr);
920
921	return NULL;
922	}
923
924
925	/**
926	* Execute state save operation.
927	*
928	* @returns VBox status code.
929	* @param pVM VM Handle.
930	* @param pSSM SSM operation handle.
931	*/
932	static DECLCALLBACK(int) vmmR3Save(PVM pVM, PSSMHANDLE pSSM)
933	{
934	LogFlow(("vmmR3Save:\n"));
935
936	/*
937	* Save the started/stopped state of all CPUs except 0 as it will always
938	* be running. This avoids breaking the saved state version. :-)
939	*/
940	for (VMCPUID i = 1; i < pVM->cCpus; i++)
941	SSMR3PutBool(pSSM, VMCPUSTATE_IS_STARTED(VMCPU_GET_STATE(&pVM->aCpus[i])));
942
943	return SSMR3PutU32(pSSM, UINT32_MAX); /* terminator */
944	}
945
946
947	/**
948	* Execute state load operation.
949	*
950	* @returns VBox status code.
951	* @param pVM VM Handle.
952	* @param pSSM SSM operation handle.
953	* @param uVersion Data layout version.
954	* @param uPass The data pass.
955	*/
956	static DECLCALLBACK(int) vmmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
957	{
958	LogFlow(("vmmR3Load:\n"));
959	Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
960
961	/*
962	* Validate version.
963	*/
964	if ( uVersion != VMM_SAVED_STATE_VERSION
965	&& uVersion != VMM_SAVED_STATE_VERSION_3_0)
966	{
967	AssertMsgFailed(("vmmR3Load: Invalid version uVersion=%u!\n", uVersion));
968	return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
969	}
970
971	if (uVersion <= VMM_SAVED_STATE_VERSION_3_0)
972	{
973	/* Ignore the stack bottom, stack pointer and stack bits. */
974	RTRCPTR RCPtrIgnored;
975	SSMR3GetRCPtr(pSSM, &RCPtrIgnored);
976	SSMR3GetRCPtr(pSSM, &RCPtrIgnored);
977	#ifdef RT_OS_DARWIN
978	if ( SSMR3HandleVersion(pSSM) >= VBOX_FULL_VERSION_MAKE(3,0,0)
979	&& SSMR3HandleVersion(pSSM) < VBOX_FULL_VERSION_MAKE(3,1,0)
980	&& SSMR3HandleRevision(pSSM) >= 48858
981	&& ( !strcmp(SSMR3HandleHostOSAndArch(pSSM), "darwin.x86")
982	\|\| !strcmp(SSMR3HandleHostOSAndArch(pSSM), "") )
983	)
984	SSMR3Skip(pSSM, 16384);
985	else
986	SSMR3Skip(pSSM, 8192);
987	#else
988	SSMR3Skip(pSSM, 8192);
989	#endif
990	}
991
992	/*
993	* Restore the VMCPU states. VCPU 0 is always started.
994	*/
995	VMCPU_SET_STATE(&pVM->aCpus[0], VMCPUSTATE_STARTED);
996	for (VMCPUID i = 1; i < pVM->cCpus; i++)
997	{
998	bool fStarted;
999	int rc = SSMR3GetBool(pSSM, &fStarted);
1000	if (RT_FAILURE(rc))
1001	return rc;
1002	VMCPU_SET_STATE(&pVM->aCpus[i], fStarted ? VMCPUSTATE_STARTED : VMCPUSTATE_STOPPED);
1003	}
1004
1005	/* terminator */
1006	uint32_t u32;
1007	int rc = SSMR3GetU32(pSSM, &u32);
1008	if (RT_FAILURE(rc))
1009	return rc;
1010	if (u32 != UINT32_MAX)
1011	{
1012	AssertMsgFailed(("u32=%#x\n", u32));
1013	return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
1014	}
1015	return VINF_SUCCESS;
1016	}
1017
1018
1019	/**
1020	* Resolve a builtin RC symbol.
1021	*
1022	* Called by PDM when loading or relocating RC modules.
1023	*
1024	* @returns VBox status
1025	* @param pVM VM Handle.
1026	* @param pszSymbol Symbol to resolv
1027	* @param pRCPtrValue Where to store the symbol value.
1028	*
1029	* @remark This has to work before VMMR3Relocate() is called.
1030	*/
1031	VMMR3DECL(int) VMMR3GetImportRC(PVM pVM, const char *pszSymbol, PRTRCPTR pRCPtrValue)
1032	{
1033	if (!strcmp(pszSymbol, "g_Logger"))
1034	{
1035	if (pVM->vmm.s.pRCLoggerR3)
1036	pVM->vmm.s.pRCLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCLoggerR3);
1037	*pRCPtrValue = pVM->vmm.s.pRCLoggerRC;
1038	}
1039	else if (!strcmp(pszSymbol, "g_RelLogger"))
1040	{
1041	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
1042	if (pVM->vmm.s.pRCRelLoggerR3)
1043	pVM->vmm.s.pRCRelLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCRelLoggerR3);
1044	*pRCPtrValue = pVM->vmm.s.pRCRelLoggerRC;
1045	#else
1046	*pRCPtrValue = NIL_RTRCPTR;
1047	#endif
1048	}
1049	else
1050	return VERR_SYMBOL_NOT_FOUND;
1051	return VINF_SUCCESS;
1052	}
1053
1054
1055	/**
1056	* Suspends the CPU yielder.
1057	*
1058	* @param pVM The VM handle.
1059	*/
1060	VMMR3DECL(void) VMMR3YieldSuspend(PVM pVM)
1061	{
1062	VMCPU_ASSERT_EMT(&pVM->aCpus[0]);
1063	if (!pVM->vmm.s.cYieldResumeMillies)
1064	{
1065	uint64_t u64Now = TMTimerGet(pVM->vmm.s.pYieldTimer);
1066	uint64_t u64Expire = TMTimerGetExpire(pVM->vmm.s.pYieldTimer);
1067	if (u64Now >= u64Expire \|\| u64Expire == ~(uint64_t)0)
1068	pVM->vmm.s.cYieldResumeMillies = pVM->vmm.s.cYieldEveryMillies;
1069	else
1070	pVM->vmm.s.cYieldResumeMillies = TMTimerToMilli(pVM->vmm.s.pYieldTimer, u64Expire - u64Now);
1071	TMTimerStop(pVM->vmm.s.pYieldTimer);
1072	}
1073	pVM->vmm.s.u64LastYield = RTTimeNanoTS();
1074	}
1075
1076
1077	/**
1078	* Stops the CPU yielder.
1079	*
1080	* @param pVM The VM handle.
1081	*/
1082	VMMR3DECL(void) VMMR3YieldStop(PVM pVM)
1083	{
1084	if (!pVM->vmm.s.cYieldResumeMillies)
1085	TMTimerStop(pVM->vmm.s.pYieldTimer);
1086	pVM->vmm.s.cYieldResumeMillies = pVM->vmm.s.cYieldEveryMillies;
1087	pVM->vmm.s.u64LastYield = RTTimeNanoTS();
1088	}
1089
1090
1091	/**
1092	* Resumes the CPU yielder when it has been a suspended or stopped.
1093	*
1094	* @param pVM The VM handle.
1095	*/
1096	VMMR3DECL(void) VMMR3YieldResume(PVM pVM)
1097	{
1098	if (pVM->vmm.s.cYieldResumeMillies)
1099	{
1100	TMTimerSetMillies(pVM->vmm.s.pYieldTimer, pVM->vmm.s.cYieldResumeMillies);
1101	pVM->vmm.s.cYieldResumeMillies = 0;
1102	}
1103	}
1104
1105
1106	/**
1107	* Internal timer callback function.
1108	*
1109	* @param pVM The VM.
1110	* @param pTimer The timer handle.
1111	* @param pvUser User argument specified upon timer creation.
1112	*/
1113	static DECLCALLBACK(void) vmmR3YieldEMT(PVM pVM, PTMTIMER pTimer, void *pvUser)
1114	{
1115	/*
1116	* This really needs some careful tuning. While we shouldn't be too greedy since
1117	* that'll cause the rest of the system to stop up, we shouldn't be too nice either
1118	* because that'll cause us to stop up.
1119	*
1120	* The current logic is to use the default interval when there is no lag worth
1121	* mentioning, but when we start accumulating lag we don't bother yielding at all.
1122	*
1123	* (This depends on the TMCLOCK_VIRTUAL_SYNC to be scheduled before TMCLOCK_REAL
1124	* so the lag is up to date.)
1125	*/
1126	const uint64_t u64Lag = TMVirtualSyncGetLag(pVM);
1127	if ( u64Lag < 50000000 /* 50ms */
1128	\|\| ( u64Lag < 1000000000 /* 1s */
1129	&& RTTimeNanoTS() - pVM->vmm.s.u64LastYield < 500000000 /* 500 ms */)
1130	)
1131	{
1132	uint64_t u64Elapsed = RTTimeNanoTS();
1133	pVM->vmm.s.u64LastYield = u64Elapsed;
1134
1135	RTThreadYield();
1136
1137	#ifdef LOG_ENABLED
1138	u64Elapsed = RTTimeNanoTS() - u64Elapsed;
1139	Log(("vmmR3YieldEMT: %RI64 ns\n", u64Elapsed));
1140	#endif
1141	}
1142	TMTimerSetMillies(pTimer, pVM->vmm.s.cYieldEveryMillies);
1143	}
1144
1145
1146	/**
1147	* Executes guest code in the raw-mode context.
1148	*
1149	* @param pVM VM handle.
1150	* @param pVCpu The VMCPU to operate on.
1151	*/
1152	VMMR3DECL(int) VMMR3RawRunGC(PVM pVM, PVMCPU pVCpu)
1153	{
1154	Log2(("VMMR3RawRunGC: (cs:eip=%04x:%08x)\n", CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
1155
1156	AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
1157
1158	/*
1159	* Set the EIP and ESP.
1160	*/
1161	CPUMSetHyperEIP(pVCpu, CPUMGetGuestEFlags(pVCpu) & X86_EFL_VM
1162	? pVM->vmm.s.pfnCPUMRCResumeGuestV86
1163	: pVM->vmm.s.pfnCPUMRCResumeGuest);
1164	CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC);
1165
1166	/*
1167	* We hide log flushes (outer) and hypervisor interrupts (inner).
1168	*/
1169	for (;;)
1170	{
1171	#ifdef VBOX_STRICT
1172	if (RT_UNLIKELY(!CPUMGetHyperCR3(pVCpu) \|\| CPUMGetHyperCR3(pVCpu) != PGMGetHyperCR3(pVCpu)))
1173	EMR3FatalError(pVCpu, VERR_VMM_HYPER_CR3_MISMATCH);
1174	PGMMapCheck(pVM);
1175	#endif
1176	int rc;
1177	do
1178	{
1179	#ifdef NO_SUPCALLR0VMM
1180	rc = VERR_GENERAL_FAILURE;
1181	#else
1182	rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
1183	if (RT_LIKELY(rc == VINF_SUCCESS))
1184	rc = pVCpu->vmm.s.iLastGZRc;
1185	#endif
1186	} while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
1187
1188	/*
1189	* Flush the logs.
1190	*/
1191	#ifdef LOG_ENABLED
1192	PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
1193	if ( pLogger
1194	&& pLogger->offScratch > 0)
1195	RTLogFlushRC(NULL, pLogger);
1196	#endif
1197	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
1198	PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
1199	if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
1200	RTLogFlushRC(RTLogRelDefaultInstance(), pRelLogger);
1201	#endif
1202	if (rc != VINF_VMM_CALL_HOST)
1203	{
1204	Log2(("VMMR3RawRunGC: returns %Rrc (cs:eip=%04x:%08x)\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
1205	return rc;
1206	}
1207	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
1208	if (RT_FAILURE(rc))
1209	return rc;
1210	/* Resume GC */
1211	}
1212	}
1213
1214
1215	/**
1216	* Executes guest code (Intel VT-x and AMD-V).
1217	*
1218	* @param pVM VM handle.
1219	* @param pVCpu The VMCPU to operate on.
1220	*/
1221	VMMR3DECL(int) VMMR3HwAccRunGC(PVM pVM, PVMCPU pVCpu)
1222	{
1223	Log2(("VMMR3HwAccRunGC: (cs:eip=%04x:%08x)\n", CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
1224
1225	for (;;)
1226	{
1227	int rc;
1228	do
1229	{
1230	#ifdef NO_SUPCALLR0VMM
1231	rc = VERR_GENERAL_FAILURE;
1232	#else
1233	rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_HWACC_RUN, pVCpu->idCpu);
1234	if (RT_LIKELY(rc == VINF_SUCCESS))
1235	rc = pVCpu->vmm.s.iLastGZRc;
1236	#endif
1237	} while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
1238
1239	#ifdef LOG_ENABLED
1240	/*
1241	* Flush the log
1242	*/
1243	PVMMR0LOGGER pR0LoggerR3 = pVCpu->vmm.s.pR0LoggerR3;
1244	if ( pR0LoggerR3
1245	&& pR0LoggerR3->Logger.offScratch > 0)
1246	RTLogFlushToLogger(&pR0LoggerR3->Logger, NULL);
1247	#endif /* !LOG_ENABLED */
1248	if (rc != VINF_VMM_CALL_HOST)
1249	{
1250	Log2(("VMMR3HwAccRunGC: returns %Rrc (cs:eip=%04x:%08x)\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
1251	return rc;
1252	}
1253	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
1254	if (RT_FAILURE(rc))
1255	return rc;
1256	/* Resume R0 */
1257	}
1258	}
1259
1260	/**
1261	* VCPU worker for VMMSendSipi.
1262	*
1263	* @param pVM The VM to operate on.
1264	* @param idCpu Virtual CPU to perform SIPI on
1265	* @param uVector SIPI vector
1266	*/
1267	DECLCALLBACK(int) vmmR3SendSipi(PVM pVM, VMCPUID idCpu, uint32_t uVector)
1268	{
1269	PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
1270	VMCPU_ASSERT_EMT(pVCpu);
1271
1272	/** @todo what are we supposed to do if the processor is already running? */
1273	if (EMGetState(pVCpu) != EMSTATE_WAIT_SIPI)
1274	return VERR_ACCESS_DENIED;
1275
1276
1277	PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
1278
1279	pCtx->cs = uVector << 8;
1280	pCtx->csHid.u64Base = uVector << 12;
1281	pCtx->csHid.u32Limit = 0x0000ffff;
1282	pCtx->rip = 0;
1283
1284	Log(("vmmR3SendSipi for VCPU %d with vector %x\n", uVector));
1285
1286	# if 1 /* If we keep the EMSTATE_WAIT_SIPI method, then move this to EM.cpp. */
1287	EMSetState(pVCpu, EMSTATE_HALTED);
1288	return VINF_EM_RESCHEDULE;
1289	# else /* And if we go the VMCPU::enmState way it can stay here. */
1290	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STOPPED);
1291	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
1292	return VINF_SUCCESS;
1293	# endif
1294	}
1295
1296	DECLCALLBACK(int) vmmR3SendInitIpi(PVM pVM, VMCPUID idCpu)
1297	{
1298	PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
1299	VMCPU_ASSERT_EMT(pVCpu);
1300
1301	Log(("vmmR3SendInitIpi for VCPU %d\n", idCpu));
1302	CPUMR3ResetCpu(pVCpu);
1303	return VINF_EM_WAIT_SIPI;
1304	}
1305
1306	/**
1307	* Sends SIPI to the virtual CPU by setting CS:EIP into vector-dependent state
1308	* and unhalting processor
1309	*
1310	* @param pVM The VM to operate on.
1311	* @param idCpu Virtual CPU to perform SIPI on
1312	* @param uVector SIPI vector
1313	*/
1314	VMMR3DECL(void) VMMR3SendSipi(PVM pVM, VMCPUID idCpu, uint32_t uVector)
1315	{
1316	AssertReturnVoid(idCpu < pVM->cCpus);
1317
1318	int rc = VMR3ReqCallNoWaitU(pVM->pUVM, idCpu, (PFNRT)vmmR3SendSipi, 3, pVM, idCpu, uVector);
1319	AssertRC(rc);
1320	}
1321
1322	/**
1323	* Sends init IPI to the virtual CPU.
1324	*
1325	* @param pVM The VM to operate on.
1326	* @param idCpu Virtual CPU to perform int IPI on
1327	*/
1328	VMMR3DECL(void) VMMR3SendInitIpi(PVM pVM, VMCPUID idCpu)
1329	{
1330	AssertReturnVoid(idCpu < pVM->cCpus);
1331
1332	int rc = VMR3ReqCallNoWaitU(pVM->pUVM, idCpu, (PFNRT)vmmR3SendInitIpi, 2, pVM, idCpu);
1333	AssertRC(rc);
1334	}
1335
1336	/**
1337	* Registers the guest memory range that can be used for patching
1338	*
1339	* @returns VBox status code.
1340	* @param pVM The VM to operate on.
1341	* @param pPatchMem Patch memory range
1342	* @param cbPatchMem Size of the memory range
1343	*/
1344	VMMR3DECL(int) VMMR3RegisterPatchMemory(PVM pVM, RTGCPTR pPatchMem, unsigned cbPatchMem)
1345	{
1346	if (HWACCMIsEnabled(pVM))
1347	return HWACMMR3EnablePatching(pVM, pPatchMem, cbPatchMem);
1348
1349	return VERR_NOT_SUPPORTED;
1350	}
1351
1352	/**
1353	* Deregisters the guest memory range that can be used for patching
1354	*
1355	* @returns VBox status code.
1356	* @param pVM The VM to operate on.
1357	* @param pPatchMem Patch memory range
1358	* @param cbPatchMem Size of the memory range
1359	*/
1360	VMMR3DECL(int) VMMR3DeregisterPatchMemory(PVM pVM, RTGCPTR pPatchMem, unsigned cbPatchMem)
1361	{
1362	if (HWACCMIsEnabled(pVM))
1363	return HWACMMR3DisablePatching(pVM, pPatchMem, cbPatchMem);
1364
1365	return VINF_SUCCESS;
1366	}
1367
1368
1369	/**
1370	* VCPU worker for VMMR3SynchronizeAllVCpus.
1371	*
1372	* @param pVM The VM to operate on.
1373	* @param idCpu Virtual CPU to perform SIPI on
1374	* @param uVector SIPI vector
1375	*/
1376	DECLCALLBACK(int) vmmR3SyncVCpu(PVM pVM)
1377	{
1378	/* Block until the job in the caller has finished. */
1379	RTCritSectEnter(&pVM->vmm.s.CritSectSync);
1380	RTCritSectLeave(&pVM->vmm.s.CritSectSync);
1381	return VINF_SUCCESS;
1382	}
1383
1384
1385	/**
1386	* Atomically execute a callback handler
1387	* Note: This is very expensive; avoid using it frequently!
1388	*
1389	* @param pVM The VM to operate on.
1390	* @param pfnHandler Callback handler
1391	* @param pvUser User specified parameter
1392	*
1393	* @thread EMT
1394	*/
1395	VMMR3DECL(int) VMMR3AtomicExecuteHandler(PVM pVM, PFNATOMICHANDLER pfnHandler, void *pvUser)
1396	{
1397	int rc;
1398	PVMCPU pVCpu = VMMGetCpu(pVM);
1399	AssertReturn(pVCpu, VERR_VM_THREAD_NOT_EMT);
1400
1401	/* Shortcut for the uniprocessor case. */
1402	if (pVM->cCpus == 1)
1403	return pfnHandler(pVM, pvUser);
1404
1405	RTCritSectEnter(&pVM->vmm.s.CritSectSync);
1406	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1407	{
1408	if (idCpu != pVCpu->idCpu)
1409	{
1410	rc = VMR3ReqCallNoWaitU(pVM->pUVM, idCpu, (PFNRT)vmmR3SyncVCpu, 1, pVM);
1411	AssertRC(rc);
1412	}
1413	}
1414	/* Wait until all other VCPUs are waiting for us. */
1415	while (RTCritSectGetWaiters(&pVM->vmm.s.CritSectSync) != (int32_t)(pVM->cCpus - 1))
1416	RTThreadSleep(1);
1417
1418	rc = pfnHandler(pVM, pvUser);
1419	RTCritSectLeave(&pVM->vmm.s.CritSectSync);
1420	return rc;
1421	}
1422
1423
1424	/**
1425	* Count returns and have the last non-caller EMT wake up the caller.
1426	*
1427	* @returns VBox strict informational status code for EM scheduling. No failures
1428	* will be returned here, those are for the caller only.
1429	*
1430	* @param pVM The VM handle.
1431	*/
1432	DECL_FORCE_INLINE(int) vmmR3EmtRendezvousNonCallerReturn(PVM pVM)
1433	{
1434	int rcRet = ASMAtomicReadS32(&pVM->vmm.s.i32RendezvousStatus);
1435	uint32_t cReturned = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsReturned);
1436	if (cReturned == pVM->cCpus - 1U)
1437	{
1438	int rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousDoneCaller);
1439	AssertLogRelRC(rc);
1440	}
1441
1442	AssertLogRelMsgReturn( rcRet <= VINF_SUCCESS
1443	\|\| (rcRet >= VINF_EM_FIRST && rcRet <= VINF_EM_LAST),
1444	("%Rrc\n", rcRet),
1445	VERR_IPE_UNEXPECTED_INFO_STATUS);
1446	return RT_SUCCESS(rcRet) ? rcRet : VINF_SUCCESS;
1447	}
1448
1449
1450	/**
1451	* Common worker for VMMR3EmtRendezvous and VMMR3EmtRendezvousFF.
1452	*
1453	* @returns VBox strict informational status code for EM scheduling. No failures
1454	* will be returned here, those are for the caller only. When
1455	* fIsCaller is set, VINF_SUCESS is always returned.
1456	*
1457	* @param pVM The VM handle.
1458	* @param pVCpu The VMCPU structure for the calling EMT.
1459	* @param fIsCaller Whether we're the VMMR3EmtRendezvous caller or
1460	* not.
1461	* @param fFlags The flags.
1462	* @param pfnRendezvous The callback.
1463	* @param pvUser The user argument for the callback.
1464	*/
1465	static int vmmR3EmtRendezvousCommon(PVM pVM, PVMCPU pVCpu, bool fIsCaller,
1466	uint32_t fFlags, PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser)
1467	{
1468	int rc;
1469
1470	/*
1471	* Enter, the last EMT triggers the next callback phase.
1472	*/
1473	uint32_t cEntered = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsEntered);
1474	if (cEntered != pVM->cCpus)
1475	{
1476	if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE)
1477	{
1478	/* Wait for our turn. */
1479	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousEnterOneByOne, RT_INDEFINITE_WAIT);
1480	AssertLogRelRC(rc);
1481	}
1482	else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE)
1483	{
1484	/* Wait for the last EMT to arrive and wake everyone up. */
1485	rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce, RT_INDEFINITE_WAIT);
1486	AssertLogRelRC(rc);
1487	}
1488	else if ( (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
1489	\|\| (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
1490	{
1491	/* Wait for our turn. */
1492	rc = RTSemEventWait(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVCpu->idCpu], RT_INDEFINITE_WAIT);
1493	AssertLogRelRC(rc);
1494	}
1495	else
1496	{
1497	Assert((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE);
1498
1499	/*
1500	* The execute once is handled specially to optimize the code flow.
1501	*
1502	* The last EMT to arrive will perform the callback and the other
1503	* EMTs will wait on the Done/DoneCaller semaphores (instead of
1504	* the EnterOneByOne/AllAtOnce) in the meanwhile. When the callback
1505	* returns, that EMT will initiate the normal return sequence.
1506	*/
1507	if (!fIsCaller)
1508	{
1509	rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousDone, RT_INDEFINITE_WAIT);
1510	AssertLogRelRC(rc);
1511
1512	return vmmR3EmtRendezvousNonCallerReturn(pVM);
1513	}
1514	return VINF_SUCCESS;
1515	}
1516	}
1517	else
1518	{
1519	/*
1520	* All EMTs are waiting, clear the FF and take action according to the
1521	* execution method.
1522	*/
1523	VM_FF_CLEAR(pVM, VM_FF_EMT_RENDEZVOUS);
1524
1525	if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE)
1526	{
1527	/* Wake up everyone. */
1528	rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
1529	AssertLogRelRC(rc);
1530	}
1531	else if ( (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
1532	\|\| (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
1533	{
1534	/* Figure out who to wake up and wake it up. If it's ourself, then
1535	it's easy otherwise wait for our turn. */
1536	VMCPUID iFirst = (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
1537	? 0
1538	: pVM->cCpus - 1U;
1539	if (pVCpu->idCpu != iFirst)
1540	{
1541	rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[iFirst]);
1542	AssertLogRelRC(rc);
1543	rc = RTSemEventWait(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVCpu->idCpu], RT_INDEFINITE_WAIT);
1544	AssertLogRelRC(rc);
1545	}
1546	}
1547	/* else: execute the handler on the current EMT and wake up one or more threads afterwards. */
1548	}
1549
1550
1551	/*
1552	* Do the callback and update the status if necessary.
1553	*/
1554	if ( !(fFlags & VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR)
1555	\|\| RT_SUCCESS(ASMAtomicUoReadS32(&pVM->vmm.s.i32RendezvousStatus)) )
1556	{
1557	VBOXSTRICTRC rcStrict = pfnRendezvous(pVM, pVCpu, pvUser);
1558	if (rcStrict != VINF_SUCCESS)
1559	{
1560	AssertLogRelMsg( rcStrict <= VINF_SUCCESS
1561	\|\| (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST),
1562	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
1563	int32_t i32RendezvousStatus;
1564	do
1565	{
1566	i32RendezvousStatus = ASMAtomicUoReadS32(&pVM->vmm.s.i32RendezvousStatus);
1567	if ( rcStrict == i32RendezvousStatus
1568	\|\| RT_FAILURE(i32RendezvousStatus)
1569	\|\| ( i32RendezvousStatus != VINF_SUCCESS
1570	&& rcStrict > i32RendezvousStatus))
1571	break;
1572	} while (!ASMAtomicCmpXchgS32(&pVM->vmm.s.i32RendezvousStatus, VBOXSTRICTRC_VAL(rcStrict), i32RendezvousStatus));
1573	}
1574	}
1575
1576	/*
1577	* Increment the done counter and take action depending on whether we're
1578	* the last to finish callback execution.
1579	*/
1580	uint32_t cDone = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsDone);
1581	if ( cDone != pVM->cCpus
1582	&& (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE)
1583	{
1584	/* Signal the next EMT? */
1585	if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE)
1586	{
1587	rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousEnterOneByOne);
1588	AssertLogRelRC(rc);
1589	}
1590	else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING)
1591	{
1592	Assert(cDone == pVCpu->idCpu + 1U);
1593	rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVCpu->idCpu + 1U]);
1594	AssertLogRelRC(rc);
1595	}
1596	else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
1597	{
1598	Assert(pVM->cCpus - cDone == pVCpu->idCpu);
1599	rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVM->cCpus - cDone - 1U]);
1600	AssertLogRelRC(rc);
1601	}
1602
1603	/* Wait for the rest to finish (the caller waits on hEvtRendezvousDoneCaller). */
1604	if (!fIsCaller)
1605	{
1606	rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousDone, RT_INDEFINITE_WAIT);
1607	AssertLogRelRC(rc);
1608	}
1609	}
1610	else
1611	{
1612	/* Callback execution is all done, tell the rest to return. */
1613	rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousDone);
1614	AssertLogRelRC(rc);
1615	}
1616
1617	if (!fIsCaller)
1618	return vmmR3EmtRendezvousNonCallerReturn(pVM);
1619	return VINF_SUCCESS;
1620	}
1621
1622
1623	/**
1624	* Called in response to VM_FF_EMT_RENDEZVOUS.
1625	*
1626	* @returns VBox strict status code - EM scheduling. No errors will be returned
1627	* here, nor will any non-EM scheduling status codes be returned.
1628	*
1629	* @param pVM The VM handle
1630	* @param pVCpu The handle of the calling EMT.
1631	*
1632	* @thread EMT
1633	*/
1634	VMMR3DECL(int) VMMR3EmtRendezvousFF(PVM pVM, PVMCPU pVCpu)
1635	{
1636	return vmmR3EmtRendezvousCommon(pVM, pVCpu, false /* fIsCaller */, pVM->vmm.s.fRendezvousFlags,
1637	pVM->vmm.s.pfnRendezvous, pVM->vmm.s.pvRendezvousUser);
1638	}
1639
1640
1641	/**
1642	* EMT rendezvous.
1643	*
1644	* Gathers all the EMTs and execute some code on each of them, either in a one
1645	* by one fashion or all at once.
1646	*
1647	* @returns VBox strict status code. This will be the the first error,
1648	* VINF_SUCCESS, or an EM scheduling status code.
1649	*
1650	* @param pVM The VM handle.
1651	* @param fFlags Flags indicating execution methods. See
1652	* grp_VMMR3EmtRendezvous_fFlags.
1653	* @param pfnRendezvous The callback.
1654	* @param pvUser User argument for the callback.
1655	*
1656	* @thread Any.
1657	*/
1658	VMMR3DECL(int) VMMR3EmtRendezvous(PVM pVM, uint32_t fFlags, PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser)
1659	{
1660	/*
1661	* Validate input.
1662	*/
1663	AssertMsg( (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_INVALID
1664	&& (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) <= VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING
1665	&& !(fFlags & ~VMMEMTRENDEZVOUS_FLAGS_VALID_MASK), ("%#x\n", fFlags));
1666	AssertMsg( !(fFlags & VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR)
1667	\|\| ( (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE
1668	&& (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE),
1669	("type %u\n", fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK));
1670
1671	VBOXSTRICTRC rcStrict;
1672	PVMCPU pVCpu = VMMGetCpu(pVM);
1673	if (!pVCpu)
1674	/*
1675	* Forward the request to an EMT thread.
1676	*/
1677	rcStrict = VMR3ReqCallWait(pVM, VMCPUID_ANY,
1678	(PFNRT)VMMR3EmtRendezvous, 4, pVM, fFlags, pfnRendezvous, pvUser);
1679	else if (pVM->cCpus == 1)
1680	/*
1681	* Shortcut for the single EMT case.
1682	*/
1683	rcStrict = pfnRendezvous(pVM, pVCpu, pvUser);
1684	else
1685	{
1686	/*
1687	* Spin lock. If busy, wait for the other EMT to finish while keeping a
1688	* lookout of the RENDEZVOUS FF.
1689	*/
1690	int rc;
1691	rcStrict = VINF_SUCCESS;
1692	if (RT_UNLIKELY(!ASMAtomicCmpXchgU32(&pVM->vmm.s.u32RendezvousLock, 0x77778888, 0)))
1693	{
1694	while (!ASMAtomicCmpXchgU32(&pVM->vmm.s.u32RendezvousLock, 0x77778888, 0))
1695	{
1696	if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
1697	{
1698	rc = VMMR3EmtRendezvousFF(pVM, pVCpu);
1699	if ( rc != VINF_SUCCESS
1700	&& ( rcStrict == VINF_SUCCESS
1701	\|\| rcStrict > rc))
1702	rcStrict = rc;
1703	/** @todo Perhaps deal with termination here? */
1704	}
1705	ASMNopPause();
1706	}
1707	}
1708	Assert(!VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS));
1709
1710	/*
1711	* Clear the slate. This is a semaphore ping-pong orgy. :-)
1712	*/
1713	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1714	{
1715	rc = RTSemEventWait(pVM->vmm.s.pahEvtRendezvousEnterOrdered[i], 0);
1716	AssertLogRelMsg(rc == VERR_TIMEOUT \|\| rc == VINF_SUCCESS, ("%Rrc\n", rc));
1717	}
1718	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousEnterOneByOne, 0); AssertLogRelMsg(rc == VERR_TIMEOUT \|\| rc == VINF_SUCCESS, ("%Rrc\n", rc));
1719	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce); AssertLogRelRC(rc);
1720	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousDone); AssertLogRelRC(rc);
1721	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousDoneCaller, 0); AssertLogRelMsg(rc == VERR_TIMEOUT \|\| rc == VINF_SUCCESS, ("%Rrc\n", rc));
1722	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsEntered, 0);
1723	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsDone, 0);
1724	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsReturned, 0);
1725	ASMAtomicWriteS32(&pVM->vmm.s.i32RendezvousStatus, VINF_SUCCESS);
1726	ASMAtomicWritePtr((void * volatile )&pVM->vmm.s.pfnRendezvous, (void )(uintptr_t)pfnRendezvous);
1727	ASMAtomicWritePtr(&pVM->vmm.s.pvRendezvousUser, pvUser);
1728	ASMAtomicWriteU32(&pVM->vmm.s.fRendezvousFlags, fFlags);
1729
1730	/*
1731	* Set the FF and poke the other EMTs.
1732	*/
1733	VM_FF_SET(pVM, VM_FF_EMT_RENDEZVOUS);
1734	VMR3NotifyGlobalFFU(pVM->pUVM, VMNOTIFYFF_FLAGS_POKE);
1735
1736	/*
1737	* Do the same ourselves.
1738	*/
1739	vmmR3EmtRendezvousCommon(pVM, pVCpu, true /* fIsCaller */, fFlags, pfnRendezvous, pvUser);
1740
1741	/*
1742	* The caller waits for the other EMTs to be done and return before doing
1743	* the cleanup. This makes away with wakeup / reset races we would otherwise
1744	* risk in the multiple release event semaphore code (hEvtRendezvousDoneCaller).
1745	*/
1746	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousDoneCaller, RT_INDEFINITE_WAIT);
1747	AssertLogRelRC(rc);
1748
1749	/*
1750	* Get the return code and clean up a little bit.
1751	*/
1752	int rcMy = pVM->vmm.s.i32RendezvousStatus;
1753	ASMAtomicWritePtr((void * volatile *)&pVM->vmm.s.pfnRendezvous, NULL);
1754
1755	ASMAtomicWriteU32(&pVM->vmm.s.u32RendezvousLock, 0);
1756
1757	/*
1758	* Merge rcStrict and rcMy.
1759	*/
1760	AssertRC(VBOXSTRICTRC_VAL(rcStrict));
1761	if ( rcMy != VINF_SUCCESS
1762	&& ( rcStrict == VINF_SUCCESS
1763	\|\| rcStrict > rcMy))
1764	rcStrict = rcMy;
1765	}
1766
1767	AssertLogRelMsgReturn( rcStrict <= VINF_SUCCESS
1768	\|\| (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST),
1769	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)),
1770	VERR_IPE_UNEXPECTED_INFO_STATUS);
1771	return VBOXSTRICTRC_VAL(rcStrict);
1772	}
1773
1774
1775	/**
1776	* Read from the ring 0 jump buffer stack
1777	*
1778	* @returns VBox status code.
1779	*
1780	* @param pVM Pointer to the shared VM structure.
1781	* @param idCpu The ID of the source CPU context (for the address).
1782	* @param pAddress Where to start reading.
1783	* @param pvBuf Where to store the data we've read.
1784	* @param cbRead The number of bytes to read.
1785	*/
1786	VMMR3DECL(int) VMMR3ReadR0Stack(PVM pVM, VMCPUID idCpu, RTHCUINTPTR pAddress, void *pvBuf, size_t cbRead)
1787	{
1788	PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
1789	AssertReturn(pVCpu, VERR_INVALID_PARAMETER);
1790
1791	RTHCUINTPTR offset = pVCpu->vmm.s.CallRing3JmpBufR0.SpCheck - pAddress;
1792	if (offset >= pVCpu->vmm.s.CallRing3JmpBufR0.cbSavedStack)
1793	return VERR_INVALID_POINTER;
1794
1795	memcpy(pvBuf, pVCpu->vmm.s.pbEMTStackR3 + pVCpu->vmm.s.CallRing3JmpBufR0.cbSavedStack - offset, cbRead);
1796	return VINF_SUCCESS;
1797	}
1798
1799
1800	/**
1801	* Calls a RC function.
1802	*
1803	* @param pVM The VM handle.
1804	* @param RCPtrEntry The address of the RC function.
1805	* @param cArgs The number of arguments in the ....
1806	* @param ... Arguments to the function.
1807	*/
1808	VMMR3DECL(int) VMMR3CallRC(PVM pVM, RTRCPTR RCPtrEntry, unsigned cArgs, ...)
1809	{
1810	va_list args;
1811	va_start(args, cArgs);
1812	int rc = VMMR3CallRCV(pVM, RCPtrEntry, cArgs, args);
1813	va_end(args);
1814	return rc;
1815	}
1816
1817
1818	/**
1819	* Calls a RC function.
1820	*
1821	* @param pVM The VM handle.
1822	* @param RCPtrEntry The address of the RC function.
1823	* @param cArgs The number of arguments in the ....
1824	* @param args Arguments to the function.
1825	*/
1826	VMMR3DECL(int) VMMR3CallRCV(PVM pVM, RTRCPTR RCPtrEntry, unsigned cArgs, va_list args)
1827	{
1828	/* Raw mode implies 1 VCPU. */
1829	AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
1830	PVMCPU pVCpu = &pVM->aCpus[0];
1831
1832	Log2(("VMMR3CallGCV: RCPtrEntry=%RRv cArgs=%d\n", RCPtrEntry, cArgs));
1833
1834	/*
1835	* Setup the call frame using the trampoline.
1836	*/
1837	CPUMHyperSetCtxCore(pVCpu, NULL);
1838	memset(pVCpu->vmm.s.pbEMTStackR3, 0xaa, VMM_STACK_SIZE); /* Clear the stack. */
1839	CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC - cArgs * sizeof(RTGCUINTPTR32));
1840	PRTGCUINTPTR32 pFrame = (PRTGCUINTPTR32)(pVCpu->vmm.s.pbEMTStackR3 + VMM_STACK_SIZE) - cArgs;
1841	int i = cArgs;
1842	while (i-- > 0)
1843	*pFrame++ = va_arg(args, RTGCUINTPTR32);
1844
1845	CPUMPushHyper(pVCpu, cArgs * sizeof(RTGCUINTPTR32)); /* stack frame size */
1846	CPUMPushHyper(pVCpu, RCPtrEntry); /* what to call */
1847	CPUMSetHyperEIP(pVCpu, pVM->vmm.s.pfnCallTrampolineRC);
1848
1849	/*
1850	* We hide log flushes (outer) and hypervisor interrupts (inner).
1851	*/
1852	for (;;)
1853	{
1854	int rc;
1855	Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
1856	do
1857	{
1858	#ifdef NO_SUPCALLR0VMM
1859	rc = VERR_GENERAL_FAILURE;
1860	#else
1861	rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
1862	if (RT_LIKELY(rc == VINF_SUCCESS))
1863	rc = pVCpu->vmm.s.iLastGZRc;
1864	#endif
1865	} while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
1866
1867	/*
1868	* Flush the logs.
1869	*/
1870	#ifdef LOG_ENABLED
1871	PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
1872	if ( pLogger
1873	&& pLogger->offScratch > 0)
1874	RTLogFlushRC(NULL, pLogger);
1875	#endif
1876	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
1877	PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
1878	if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
1879	RTLogFlushRC(RTLogRelDefaultInstance(), pRelLogger);
1880	#endif
1881	if (rc == VERR_TRPM_PANIC \|\| rc == VERR_TRPM_DONT_PANIC)
1882	VMMR3FatalDump(pVM, pVCpu, rc);
1883	if (rc != VINF_VMM_CALL_HOST)
1884	{
1885	Log2(("VMMR3CallGCV: returns %Rrc (cs:eip=%04x:%08x)\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
1886	return rc;
1887	}
1888	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
1889	if (RT_FAILURE(rc))
1890	return rc;
1891	}
1892	}
1893
1894
1895	/**
1896	* Wrapper for SUPR3CallVMMR0Ex which will deal with VINF_VMM_CALL_HOST returns.
1897	*
1898	* @returns VBox status code.
1899	* @param pVM The VM to operate on.
1900	* @param uOperation Operation to execute.
1901	* @param u64Arg Constant argument.
1902	* @param pReqHdr Pointer to a request header. See SUPR3CallVMMR0Ex for
1903	* details.
1904	*/
1905	VMMR3DECL(int) VMMR3CallR0(PVM pVM, uint32_t uOperation, uint64_t u64Arg, PSUPVMMR0REQHDR pReqHdr)
1906	{
1907	PVMCPU pVCpu = VMMGetCpu(pVM);
1908	AssertReturn(pVCpu, VERR_VM_THREAD_NOT_EMT);
1909
1910	/*
1911	* Call Ring-0 entry with init code.
1912	*/
1913	int rc;
1914	for (;;)
1915	{
1916	#ifdef NO_SUPCALLR0VMM
1917	rc = VERR_GENERAL_FAILURE;
1918	#else
1919	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, pVCpu->idCpu, uOperation, u64Arg, pReqHdr);
1920	#endif
1921	/*
1922	* Flush the logs.
1923	*/
1924	#ifdef LOG_ENABLED
1925	if ( pVCpu->vmm.s.pR0LoggerR3
1926	&& pVCpu->vmm.s.pR0LoggerR3->Logger.offScratch > 0)
1927	RTLogFlushToLogger(&pVCpu->vmm.s.pR0LoggerR3->Logger, NULL);
1928	#endif
1929	if (rc != VINF_VMM_CALL_HOST)
1930	break;
1931	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
1932	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
1933	break;
1934	/* Resume R0 */
1935	}
1936
1937	AssertLogRelMsgReturn(rc == VINF_SUCCESS \|\| RT_FAILURE(rc),
1938	("uOperation=%u rc=%Rrc\n", uOperation, rc),
1939	VERR_INTERNAL_ERROR);
1940	return rc;
1941	}
1942
1943
1944	/**
1945	* Resumes executing hypervisor code when interrupted by a queue flush or a
1946	* debug event.
1947	*
1948	* @returns VBox status code.
1949	* @param pVM VM handle.
1950	* @param pVCpu VMCPU handle.
1951	*/
1952	VMMR3DECL(int) VMMR3ResumeHyper(PVM pVM, PVMCPU pVCpu)
1953	{
1954	Log(("VMMR3ResumeHyper: eip=%RRv esp=%RRv\n", CPUMGetHyperEIP(pVCpu), CPUMGetHyperESP(pVCpu)));
1955	AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
1956
1957	/*
1958	* We hide log flushes (outer) and hypervisor interrupts (inner).
1959	*/
1960	for (;;)
1961	{
1962	int rc;
1963	Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
1964	do
1965	{
1966	#ifdef NO_SUPCALLR0VMM
1967	rc = VERR_GENERAL_FAILURE;
1968	#else
1969	rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
1970	if (RT_LIKELY(rc == VINF_SUCCESS))
1971	rc = pVCpu->vmm.s.iLastGZRc;
1972	#endif
1973	} while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
1974
1975	/*
1976	* Flush the loggers,
1977	*/
1978	#ifdef LOG_ENABLED
1979	PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
1980	if ( pLogger
1981	&& pLogger->offScratch > 0)
1982	RTLogFlushRC(NULL, pLogger);
1983	#endif
1984	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
1985	PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
1986	if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
1987	RTLogFlushRC(RTLogRelDefaultInstance(), pRelLogger);
1988	#endif
1989	if (rc == VERR_TRPM_PANIC \|\| rc == VERR_TRPM_DONT_PANIC)
1990	VMMR3FatalDump(pVM, pVCpu, rc);
1991	if (rc != VINF_VMM_CALL_HOST)
1992	{
1993	Log(("VMMR3ResumeHyper: returns %Rrc\n", rc));
1994	return rc;
1995	}
1996	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
1997	if (RT_FAILURE(rc))
1998	return rc;
1999	}
2000	}
2001
2002
2003	/**
2004	* Service a call to the ring-3 host code.
2005	*
2006	* @returns VBox status code.
2007	* @param pVM VM handle.
2008	* @param pVCpu VMCPU handle
2009	* @remark Careful with critsects.
2010	*/
2011	static int vmmR3ServiceCallRing3Request(PVM pVM, PVMCPU pVCpu)
2012	{
2013	/*
2014	* We must also check for pending critsect exits or else we can deadlock
2015	* when entering other critsects here.
2016	*/
2017	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PDM_CRITSECT))
2018	PDMCritSectFF(pVCpu);
2019
2020	switch (pVCpu->vmm.s.enmCallRing3Operation)
2021	{
2022	/*
2023	* Acquire the PDM lock.
2024	*/
2025	case VMMCALLRING3_PDM_LOCK:
2026	{
2027	pVCpu->vmm.s.rcCallRing3 = PDMR3LockCall(pVM);
2028	break;
2029	}
2030
2031	/*
2032	* Flush a PDM queue.
2033	*/
2034	case VMMCALLRING3_PDM_QUEUE_FLUSH:
2035	{
2036	PDMR3QueueFlushWorker(pVM, NULL);
2037	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
2038	break;
2039	}
2040
2041	/*
2042	* Grow the PGM pool.
2043	*/
2044	case VMMCALLRING3_PGM_POOL_GROW:
2045	{
2046	pVCpu->vmm.s.rcCallRing3 = PGMR3PoolGrow(pVM);
2047	break;
2048	}
2049
2050	/*
2051	* Maps an page allocation chunk into ring-3 so ring-0 can use it.
2052	*/
2053	case VMMCALLRING3_PGM_MAP_CHUNK:
2054	{
2055	pVCpu->vmm.s.rcCallRing3 = PGMR3PhysChunkMap(pVM, pVCpu->vmm.s.u64CallRing3Arg);
2056	break;
2057	}
2058
2059	/*
2060	* Allocates more handy pages.
2061	*/
2062	case VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES:
2063	{
2064	pVCpu->vmm.s.rcCallRing3 = PGMR3PhysAllocateHandyPages(pVM);
2065	break;
2066	}
2067
2068	/*
2069	* Allocates a large page.
2070	*/
2071	case VMMCALLRING3_PGM_ALLOCATE_LARGE_HANDY_PAGE:
2072	{
2073	pVCpu->vmm.s.rcCallRing3 = PGMR3PhysAllocateLargeHandyPage(pVM, pVCpu->vmm.s.u64CallRing3Arg);
2074	break;
2075	}
2076
2077	/*
2078	* Acquire the PGM lock.
2079	*/
2080	case VMMCALLRING3_PGM_LOCK:
2081	{
2082	pVCpu->vmm.s.rcCallRing3 = PGMR3LockCall(pVM);
2083	break;
2084	}
2085
2086	/*
2087	* Acquire the MM hypervisor heap lock.
2088	*/
2089	case VMMCALLRING3_MMHYPER_LOCK:
2090	{
2091	pVCpu->vmm.s.rcCallRing3 = MMR3LockCall(pVM);
2092	break;
2093	}
2094
2095	/*
2096	* Flush REM handler notifications.
2097	*/
2098	case VMMCALLRING3_REM_REPLAY_HANDLER_NOTIFICATIONS:
2099	{
2100	REMR3ReplayHandlerNotifications(pVM);
2101	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
2102	break;
2103	}
2104
2105	/*
2106	* This is a noop. We just take this route to avoid unnecessary
2107	* tests in the loops.
2108	*/
2109	case VMMCALLRING3_VMM_LOGGER_FLUSH:
2110	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
2111	LogAlways(("FLUSH\n"));
2112	break;
2113
2114	/*
2115	* Set the VM error message.
2116	*/
2117	case VMMCALLRING3_VM_SET_ERROR:
2118	VMR3SetErrorWorker(pVM);
2119	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
2120	break;
2121
2122	/*
2123	* Set the VM runtime error message.
2124	*/
2125	case VMMCALLRING3_VM_SET_RUNTIME_ERROR:
2126	pVCpu->vmm.s.rcCallRing3 = VMR3SetRuntimeErrorWorker(pVM);
2127	break;
2128
2129	/*
2130	* Signal a ring 0 hypervisor assertion.
2131	* Cancel the longjmp operation that's in progress.
2132	*/
2133	case VMMCALLRING3_VM_R0_ASSERTION:
2134	pVCpu->vmm.s.enmCallRing3Operation = VMMCALLRING3_INVALID;
2135	pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call = false;
2136	#ifdef RT_ARCH_X86
2137	pVCpu->vmm.s.CallRing3JmpBufR0.eip = 0;
2138	#else
2139	pVCpu->vmm.s.CallRing3JmpBufR0.rip = 0;
2140	#endif
2141	LogRel((pVM->vmm.s.szRing0AssertMsg1));
2142	LogRel((pVM->vmm.s.szRing0AssertMsg2));
2143	return VERR_VMM_RING0_ASSERTION;
2144
2145	/*
2146	* A forced switch to ring 0 for preemption purposes.
2147	*/
2148	case VMMCALLRING3_VM_R0_PREEMPT:
2149	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
2150	break;
2151
2152	default:
2153	AssertMsgFailed(("enmCallRing3Operation=%d\n", pVCpu->vmm.s.enmCallRing3Operation));
2154	return VERR_INTERNAL_ERROR;
2155	}
2156
2157	pVCpu->vmm.s.enmCallRing3Operation = VMMCALLRING3_INVALID;
2158	return VINF_SUCCESS;
2159	}
2160
2161
2162	/**
2163	* Displays the Force action Flags.
2164	*
2165	* @param pVM The VM handle.
2166	* @param pHlp The output helpers.
2167	* @param pszArgs The additional arguments (ignored).
2168	*/
2169	static DECLCALLBACK(void) vmmR3InfoFF(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
2170	{
2171	int c;
2172	uint32_t f;
2173	#define PRINT_FLAG(prf,flag) do { \
2174	if (f & (prf##flag)) \
2175	{ \
2176	static const char *s_psz = #flag; \
2177	if (!(c % 6)) \
2178	pHlp->pfnPrintf(pHlp, "%s\n %s", c ? "," : "", s_psz); \
2179	else \
2180	pHlp->pfnPrintf(pHlp, ", %s", s_psz); \
2181	c++; \
2182	f &= ~(prf##flag); \
2183	} \
2184	} while (0)
2185
2186	#define PRINT_GROUP(prf,grp,sfx) do { \
2187	if (f & (prf##grp##sfx)) \
2188	{ \
2189	static const char *s_psz = #grp; \
2190	if (!(c % 5)) \
2191	pHlp->pfnPrintf(pHlp, "%s %s", c ? ",\n" : " Groups:\n", s_psz); \
2192	else \
2193	pHlp->pfnPrintf(pHlp, ", %s", s_psz); \
2194	c++; \
2195	} \
2196	} while (0)
2197
2198	/*
2199	* The global flags.
2200	*/
2201	const uint32_t fGlobalForcedActions = pVM->fGlobalForcedActions;
2202	pHlp->pfnPrintf(pHlp, "Global FFs: %#RX32", fGlobalForcedActions);
2203
2204	/* show the flag mnemonics */
2205	c = 0;
2206	f = fGlobalForcedActions;
2207	PRINT_FLAG(VM_FF_,TM_VIRTUAL_SYNC);
2208	PRINT_FLAG(VM_FF_,PDM_QUEUES);
2209	PRINT_FLAG(VM_FF_,PDM_DMA);
2210	PRINT_FLAG(VM_FF_,DBGF);
2211	PRINT_FLAG(VM_FF_,REQUEST);
2212	PRINT_FLAG(VM_FF_,TERMINATE);
2213	PRINT_FLAG(VM_FF_,RESET);
2214	PRINT_FLAG(VM_FF_,EMT_RENDEZVOUS);
2215	PRINT_FLAG(VM_FF_,PGM_NEED_HANDY_PAGES);
2216	PRINT_FLAG(VM_FF_,PGM_NO_MEMORY);
2217	PRINT_FLAG(VM_FF_,REM_HANDLER_NOTIFY);
2218	PRINT_FLAG(VM_FF_,DEBUG_SUSPEND);
2219	if (f)
2220	pHlp->pfnPrintf(pHlp, "%s\n Unknown bits: %#RX32\n", c ? "," : "", f);
2221	else
2222	pHlp->pfnPrintf(pHlp, "\n");
2223
2224	/* the groups */
2225	c = 0;
2226	f = fGlobalForcedActions;
2227	PRINT_GROUP(VM_FF_,EXTERNAL_SUSPENDED,_MASK);
2228	PRINT_GROUP(VM_FF_,EXTERNAL_HALTED,_MASK);
2229	PRINT_GROUP(VM_FF_,HIGH_PRIORITY_PRE,_MASK);
2230	PRINT_GROUP(VM_FF_,HIGH_PRIORITY_PRE_RAW,_MASK);
2231	PRINT_GROUP(VM_FF_,HIGH_PRIORITY_POST,_MASK);
2232	PRINT_GROUP(VM_FF_,NORMAL_PRIORITY_POST,_MASK);
2233	PRINT_GROUP(VM_FF_,NORMAL_PRIORITY,_MASK);
2234	PRINT_GROUP(VM_FF_,ALL_BUT_RAW,_MASK);
2235	if (c)
2236	pHlp->pfnPrintf(pHlp, "\n");
2237
2238	/*
2239	* Per CPU flags.
2240	*/
2241	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2242	{
2243	const uint32_t fLocalForcedActions = pVM->aCpus[i].fLocalForcedActions;
2244	pHlp->pfnPrintf(pHlp, "CPU %u FFs: %#RX32", i, fLocalForcedActions);
2245
2246	/* show the flag mnemonics */
2247	c = 0;
2248	f = fLocalForcedActions;
2249	PRINT_FLAG(VMCPU_FF_,INTERRUPT_APIC);
2250	PRINT_FLAG(VMCPU_FF_,INTERRUPT_PIC);
2251	PRINT_FLAG(VMCPU_FF_,TIMER);
2252	PRINT_FLAG(VMCPU_FF_,PDM_CRITSECT);
2253	PRINT_FLAG(VMCPU_FF_,PGM_SYNC_CR3);
2254	PRINT_FLAG(VMCPU_FF_,PGM_SYNC_CR3_NON_GLOBAL);
2255	PRINT_FLAG(VMCPU_FF_,TRPM_SYNC_IDT);
2256	PRINT_FLAG(VMCPU_FF_,SELM_SYNC_TSS);
2257	PRINT_FLAG(VMCPU_FF_,SELM_SYNC_GDT);
2258	PRINT_FLAG(VMCPU_FF_,SELM_SYNC_LDT);
2259	PRINT_FLAG(VMCPU_FF_,INHIBIT_INTERRUPTS);
2260	PRINT_FLAG(VMCPU_FF_,CSAM_SCAN_PAGE);
2261	PRINT_FLAG(VMCPU_FF_,CSAM_PENDING_ACTION);
2262	PRINT_FLAG(VMCPU_FF_,TO_R3);
2263	if (f)
2264	pHlp->pfnPrintf(pHlp, "%s\n Unknown bits: %#RX32\n", c ? "," : "", f);
2265	else
2266	pHlp->pfnPrintf(pHlp, "\n");
2267
2268	/* the groups */
2269	c = 0;
2270	f = fLocalForcedActions;
2271	PRINT_GROUP(VMCPU_FF_,EXTERNAL_SUSPENDED,_MASK);
2272	PRINT_GROUP(VMCPU_FF_,EXTERNAL_HALTED,_MASK);
2273	PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_PRE,_MASK);
2274	PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_PRE_RAW,_MASK);
2275	PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_POST,_MASK);
2276	PRINT_GROUP(VMCPU_FF_,NORMAL_PRIORITY_POST,_MASK);
2277	PRINT_GROUP(VMCPU_FF_,NORMAL_PRIORITY,_MASK);
2278	PRINT_GROUP(VMCPU_FF_,RESUME_GUEST,_MASK);
2279	PRINT_GROUP(VMCPU_FF_,HWACCM_TO_R3,_MASK);
2280	PRINT_GROUP(VMCPU_FF_,ALL_BUT_RAW,_MASK);
2281	if (c)
2282	pHlp->pfnPrintf(pHlp, "\n");
2283	}
2284
2285	#undef PRINT_FLAG
2286	#undef PRINT_GROUP
2287	}
2288

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMM.cpp@ 29576

Download in other formats: