VMMR0.cpp@ 75646

Last change on this file since 75646 was 75646, checked in by vboxsync, 6 years ago
VMM: HLT/MWAIT optimizations for busy guests: don't go back to ring-3 just to call GVMMR0SchedHalt(), do the first call in ring-0. This saves a reduces interrupt latency for some workloads. bugref:9172
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 103.8 KB

Line
1	/* $Id: VMMR0.cpp 75646 2018-11-21 15:38:10Z vboxsync $ */
2	/** @file
3	* VMM - Host Context Ring 0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2017 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_VMM
23	#include <VBox/vmm/vmm.h>
24	#include <VBox/sup.h>
25	#include <VBox/vmm/trpm.h>
26	#include <VBox/vmm/cpum.h>
27	#include <VBox/vmm/pdmapi.h>
28	#include <VBox/vmm/pgm.h>
29	#ifdef VBOX_WITH_NEM_R0
30	# include <VBox/vmm/nem.h>
31	#endif
32	#include <VBox/vmm/em.h>
33	#include <VBox/vmm/stam.h>
34	#include <VBox/vmm/tm.h>
35	#include "VMMInternal.h"
36	#include <VBox/vmm/vm.h>
37	#include <VBox/vmm/gvm.h>
38	#ifdef VBOX_WITH_PCI_PASSTHROUGH
39	# include <VBox/vmm/pdmpci.h>
40	#endif
41	#include <VBox/vmm/apic.h>
42
43	#include <VBox/vmm/gvmm.h>
44	#include <VBox/vmm/gmm.h>
45	#include <VBox/vmm/gim.h>
46	#include <VBox/intnet.h>
47	#include <VBox/vmm/hm.h>
48	#include <VBox/param.h>
49	#include <VBox/err.h>
50	#include <VBox/version.h>
51	#include <VBox/log.h>
52
53	#include <iprt/asm-amd64-x86.h>
54	#include <iprt/assert.h>
55	#include <iprt/crc.h>
56	#include <iprt/mp.h>
57	#include <iprt/once.h>
58	#include <iprt/stdarg.h>
59	#include <iprt/string.h>
60	#include <iprt/thread.h>
61	#include <iprt/timer.h>
62	#include <iprt/time.h>
63
64	#include "dtrace/VBoxVMM.h"
65
66
67	#if defined(_MSC_VER) && defined(RT_ARCH_AMD64) /** @todo check this with with VC7! */
68	# pragma intrinsic(_AddressOfReturnAddress)
69	#endif
70
71	#if defined(RT_OS_DARWIN) && ARCH_BITS == 32
72	# error "32-bit darwin is no longer supported. Go back to 4.3 or earlier!"
73	#endif
74
75
76
77	/*********************************************************************************************************************************
78	* Defined Constants And Macros *
79	*********************************************************************************************************************************/
80	/** @def VMM_CHECK_SMAP_SETUP
81	* SMAP check setup. */
82	/** @def VMM_CHECK_SMAP_CHECK
83	* Checks that the AC flag is set if SMAP is enabled. If AC is not set,
84	* it will be logged and @a a_BadExpr is executed. */
85	/** @def VMM_CHECK_SMAP_CHECK2
86	* Checks that the AC flag is set if SMAP is enabled. If AC is not set, it will
87	* be logged, written to the VMs assertion text buffer, and @a a_BadExpr is
88	* executed. */
89	#if defined(VBOX_STRICT) \|\| 1
90	# define VMM_CHECK_SMAP_SETUP() uint32_t const fKernelFeatures = SUPR0GetKernelFeatures()
91	# define VMM_CHECK_SMAP_CHECK(a_BadExpr) \
92	do { \
93	if (fKernelFeatures & SUPKERNELFEATURES_SMAP) \
94	{ \
95	RTCCUINTREG fEflCheck = ASMGetFlags(); \
96	if (RT_LIKELY(fEflCheck & X86_EFL_AC)) \
97	{ /* likely */ } \
98	else \
99	{ \
100	SUPR0Printf("%s, line %d: EFLAGS.AC is clear! (%#x)\n", __FUNCTION__, __LINE__, (uint32_t)fEflCheck); \
101	a_BadExpr; \
102	} \
103	} \
104	} while (0)
105	# define VMM_CHECK_SMAP_CHECK2(a_pVM, a_BadExpr) \
106	do { \
107	if (fKernelFeatures & SUPKERNELFEATURES_SMAP) \
108	{ \
109	RTCCUINTREG fEflCheck = ASMGetFlags(); \
110	if (RT_LIKELY(fEflCheck & X86_EFL_AC)) \
111	{ /* likely */ } \
112	else \
113	{ \
114	SUPR0BadContext((a_pVM) ? (a_pVM)->pSession : NULL, __FILE__, __LINE__, "EFLAGS.AC is zero!"); \
115	RTStrPrintf(pVM->vmm.s.szRing0AssertMsg1, sizeof(pVM->vmm.s.szRing0AssertMsg1), \
116	"%s, line %d: EFLAGS.AC is clear! (%#x)\n", __FUNCTION__, __LINE__, (uint32_t)fEflCheck); \
117	a_BadExpr; \
118	} \
119	} \
120	} while (0)
121	#else
122	# define VMM_CHECK_SMAP_SETUP() uint32_t const fKernelFeatures = 0
123	# define VMM_CHECK_SMAP_CHECK(a_BadExpr) NOREF(fKernelFeatures)
124	# define VMM_CHECK_SMAP_CHECK2(a_pVM, a_BadExpr) NOREF(fKernelFeatures)
125	#endif
126
127
128	/*********************************************************************************************************************************
129	* Internal Functions *
130	*********************************************************************************************************************************/
131	RT_C_DECLS_BEGIN
132	#if defined(RT_ARCH_X86) && (defined(RT_OS_SOLARIS) \|\| defined(RT_OS_FREEBSD))
133	extern uint64_t __udivdi3(uint64_t, uint64_t);
134	extern uint64_t __umoddi3(uint64_t, uint64_t);
135	#endif
136	RT_C_DECLS_END
137
138
139	/*********************************************************************************************************************************
140	* Global Variables *
141	*********************************************************************************************************************************/
142	/** Drag in necessary library bits.
143	* The runtime lives here (in VMMR0.r0) and VBoxDDR0.r0 links against us. /
144	PFNRT g_VMMR0Deps[] =
145	{
146	(PFNRT)RTCrc32,
147	(PFNRT)RTOnce,
148	#if defined(RT_ARCH_X86) && (defined(RT_OS_SOLARIS) \|\| defined(RT_OS_FREEBSD))
149	(PFNRT)__udivdi3,
150	(PFNRT)__umoddi3,
151	#endif
152	NULL
153	};
154
155	#ifdef RT_OS_SOLARIS
156	/* Dependency information for the native solaris loader. */
157	extern "C" { char _depends_on[] = "vboxdrv"; }
158	#endif
159
160	/** The result of SUPR0GetRawModeUsability(), set by ModuleInit(). */
161	int g_rcRawModeUsability = VINF_SUCCESS;
162
163
164	/**
165	* Initialize the module.
166	* This is called when we're first loaded.
167	*
168	* @returns 0 on success.
169	* @returns VBox status on failure.
170	* @param hMod Image handle for use in APIs.
171	*/
172	DECLEXPORT(int) ModuleInit(void *hMod)
173	{
174	VMM_CHECK_SMAP_SETUP();
175	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
176
177	#ifdef VBOX_WITH_DTRACE_R0
178	/*
179	* The first thing to do is register the static tracepoints.
180	* (Deregistration is automatic.)
181	*/
182	int rc2 = SUPR0TracerRegisterModule(hMod, &g_VTGObjHeader);
183	if (RT_FAILURE(rc2))
184	return rc2;
185	#endif
186	LogFlow(("ModuleInit:\n"));
187
188	#ifdef VBOX_WITH_64ON32_CMOS_DEBUG
189	/*
190	* Display the CMOS debug code.
191	*/
192	ASMOutU8(0x72, 0x03);
193	uint8_t bDebugCode = ASMInU8(0x73);
194	LogRel(("CMOS Debug Code: %#x (%d)\n", bDebugCode, bDebugCode));
195	RTLogComPrintf("CMOS Debug Code: %#x (%d)\n", bDebugCode, bDebugCode);
196	#endif
197
198	/*
199	* Initialize the VMM, GVMM, GMM, HM, PGM (Darwin) and INTNET.
200	*/
201	int rc = vmmInitFormatTypes();
202	if (RT_SUCCESS(rc))
203	{
204	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
205	rc = GVMMR0Init();
206	if (RT_SUCCESS(rc))
207	{
208	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
209	rc = GMMR0Init();
210	if (RT_SUCCESS(rc))
211	{
212	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
213	rc = HMR0Init();
214	if (RT_SUCCESS(rc))
215	{
216	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
217	rc = PGMRegisterStringFormatTypes();
218	if (RT_SUCCESS(rc))
219	{
220	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
221	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
222	rc = PGMR0DynMapInit();
223	#endif
224	if (RT_SUCCESS(rc))
225	{
226	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
227	rc = IntNetR0Init();
228	if (RT_SUCCESS(rc))
229	{
230	#ifdef VBOX_WITH_PCI_PASSTHROUGH
231	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
232	rc = PciRawR0Init();
233	#endif
234	if (RT_SUCCESS(rc))
235	{
236	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
237	rc = CPUMR0ModuleInit();
238	if (RT_SUCCESS(rc))
239	{
240	#ifdef VBOX_WITH_TRIPLE_FAULT_HACK
241	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
242	rc = vmmR0TripleFaultHackInit();
243	if (RT_SUCCESS(rc))
244	#endif
245	{
246	VMM_CHECK_SMAP_CHECK(rc = VERR_VMM_SMAP_BUT_AC_CLEAR);
247	if (RT_SUCCESS(rc))
248	{
249	g_rcRawModeUsability = SUPR0GetRawModeUsability();
250	if (g_rcRawModeUsability != VINF_SUCCESS)
251	SUPR0Printf("VMMR0!ModuleInit: SUPR0GetRawModeUsability -> %Rrc\n",
252	g_rcRawModeUsability);
253	LogFlow(("ModuleInit: returns success\n"));
254	return VINF_SUCCESS;
255	}
256	}
257
258	/*
259	* Bail out.
260	*/
261	#ifdef VBOX_WITH_TRIPLE_FAULT_HACK
262	vmmR0TripleFaultHackTerm();
263	#endif
264	}
265	else
266	LogRel(("ModuleInit: CPUMR0ModuleInit -> %Rrc\n", rc));
267	#ifdef VBOX_WITH_PCI_PASSTHROUGH
268	PciRawR0Term();
269	#endif
270	}
271	else
272	LogRel(("ModuleInit: PciRawR0Init -> %Rrc\n", rc));
273	IntNetR0Term();
274	}
275	else
276	LogRel(("ModuleInit: IntNetR0Init -> %Rrc\n", rc));
277	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
278	PGMR0DynMapTerm();
279	#endif
280	}
281	else
282	LogRel(("ModuleInit: PGMR0DynMapInit -> %Rrc\n", rc));
283	PGMDeregisterStringFormatTypes();
284	}
285	else
286	LogRel(("ModuleInit: PGMRegisterStringFormatTypes -> %Rrc\n", rc));
287	HMR0Term();
288	}
289	else
290	LogRel(("ModuleInit: HMR0Init -> %Rrc\n", rc));
291	GMMR0Term();
292	}
293	else
294	LogRel(("ModuleInit: GMMR0Init -> %Rrc\n", rc));
295	GVMMR0Term();
296	}
297	else
298	LogRel(("ModuleInit: GVMMR0Init -> %Rrc\n", rc));
299	vmmTermFormatTypes();
300	}
301	else
302	LogRel(("ModuleInit: vmmInitFormatTypes -> %Rrc\n", rc));
303
304	LogFlow(("ModuleInit: failed %Rrc\n", rc));
305	return rc;
306	}
307
308
309	/**
310	* Terminate the module.
311	* This is called when we're finally unloaded.
312	*
313	* @param hMod Image handle for use in APIs.
314	*/
315	DECLEXPORT(void) ModuleTerm(void *hMod)
316	{
317	NOREF(hMod);
318	LogFlow(("ModuleTerm:\n"));
319
320	/*
321	* Terminate the CPUM module (Local APIC cleanup).
322	*/
323	CPUMR0ModuleTerm();
324
325	/*
326	* Terminate the internal network service.
327	*/
328	IntNetR0Term();
329
330	/*
331	* PGM (Darwin), HM and PciRaw global cleanup.
332	*/
333	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
334	PGMR0DynMapTerm();
335	#endif
336	#ifdef VBOX_WITH_PCI_PASSTHROUGH
337	PciRawR0Term();
338	#endif
339	PGMDeregisterStringFormatTypes();
340	HMR0Term();
341	#ifdef VBOX_WITH_TRIPLE_FAULT_HACK
342	vmmR0TripleFaultHackTerm();
343	#endif
344
345	/*
346	* Destroy the GMM and GVMM instances.
347	*/
348	GMMR0Term();
349	GVMMR0Term();
350
351	vmmTermFormatTypes();
352
353	LogFlow(("ModuleTerm: returns\n"));
354	}
355
356
357	/**
358	* Initiates the R0 driver for a particular VM instance.
359	*
360	* @returns VBox status code.
361	*
362	* @param pGVM The global (ring-0) VM structure.
363	* @param pVM The cross context VM structure.
364	* @param uSvnRev The SVN revision of the ring-3 part.
365	* @param uBuildType Build type indicator.
366	* @thread EMT(0)
367	*/
368	static int vmmR0InitVM(PGVM pGVM, PVM pVM, uint32_t uSvnRev, uint32_t uBuildType)
369	{
370	VMM_CHECK_SMAP_SETUP();
371	VMM_CHECK_SMAP_CHECK(return VERR_VMM_SMAP_BUT_AC_CLEAR);
372
373	/*
374	* Match the SVN revisions and build type.
375	*/
376	if (uSvnRev != VMMGetSvnRev())
377	{
378	LogRel(("VMMR0InitVM: Revision mismatch, r3=%d r0=%d\n", uSvnRev, VMMGetSvnRev()));
379	SUPR0Printf("VMMR0InitVM: Revision mismatch, r3=%d r0=%d\n", uSvnRev, VMMGetSvnRev());
380	return VERR_VMM_R0_VERSION_MISMATCH;
381	}
382	if (uBuildType != vmmGetBuildType())
383	{
384	LogRel(("VMMR0InitVM: Build type mismatch, r3=%#x r0=%#x\n", uBuildType, vmmGetBuildType()));
385	SUPR0Printf("VMMR0InitVM: Build type mismatch, r3=%#x r0=%#x\n", uBuildType, vmmGetBuildType());
386	return VERR_VMM_R0_VERSION_MISMATCH;
387	}
388
389	int rc = GVMMR0ValidateGVMandVMandEMT(pGVM, pVM, 0 /idCpu/);
390	if (RT_FAILURE(rc))
391	return rc;
392
393	#ifdef LOG_ENABLED
394	/*
395	* Register the EMT R0 logger instance for VCPU 0.
396	*/
397	PVMCPU pVCpu = &pVM->aCpus[0];
398
399	PVMMR0LOGGER pR0Logger = pVCpu->vmm.s.pR0LoggerR0;
400	if (pR0Logger)
401	{
402	# if 0 /* testing of the logger. */
403	LogCom(("vmmR0InitVM: before %p\n", RTLogDefaultInstance()));
404	LogCom(("vmmR0InitVM: pfnFlush=%p actual=%p\n", pR0Logger->Logger.pfnFlush, vmmR0LoggerFlush));
405	LogCom(("vmmR0InitVM: pfnLogger=%p actual=%p\n", pR0Logger->Logger.pfnLogger, vmmR0LoggerWrapper));
406	LogCom(("vmmR0InitVM: offScratch=%d fFlags=%#x fDestFlags=%#x\n", pR0Logger->Logger.offScratch, pR0Logger->Logger.fFlags, pR0Logger->Logger.fDestFlags));
407
408	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pVM->pSession);
409	LogCom(("vmmR0InitVM: after %p reg\n", RTLogDefaultInstance()));
410	RTLogSetDefaultInstanceThread(NULL, pVM->pSession);
411	LogCom(("vmmR0InitVM: after %p dereg\n", RTLogDefaultInstance()));
412
413	pR0Logger->Logger.pfnLogger("hello ring-0 logger\n");
414	LogCom(("vmmR0InitVM: returned successfully from direct logger call.\n"));
415	pR0Logger->Logger.pfnFlush(&pR0Logger->Logger);
416	LogCom(("vmmR0InitVM: returned successfully from direct flush call.\n"));
417
418	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pVM->pSession);
419	LogCom(("vmmR0InitVM: after %p reg2\n", RTLogDefaultInstance()));
420	pR0Logger->Logger.pfnLogger("hello ring-0 logger\n");
421	LogCom(("vmmR0InitVM: returned successfully from direct logger call (2). offScratch=%d\n", pR0Logger->Logger.offScratch));
422	RTLogSetDefaultInstanceThread(NULL, pVM->pSession);
423	LogCom(("vmmR0InitVM: after %p dereg2\n", RTLogDefaultInstance()));
424
425	RTLogLoggerEx(&pR0Logger->Logger, 0, ~0U, "hello ring-0 logger (RTLogLoggerEx)\n");
426	LogCom(("vmmR0InitVM: RTLogLoggerEx returned fine offScratch=%d\n", pR0Logger->Logger.offScratch));
427
428	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pVM->pSession);
429	RTLogPrintf("hello ring-0 logger (RTLogPrintf)\n");
430	LogCom(("vmmR0InitVM: RTLogPrintf returned fine offScratch=%d\n", pR0Logger->Logger.offScratch));
431	# endif
432	Log(("Switching to per-thread logging instance %p (key=%p)\n", &pR0Logger->Logger, pVM->pSession));
433	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pVM->pSession);
434	pR0Logger->fRegistered = true;
435	}
436	#endif /* LOG_ENABLED */
437
438	/*
439	* Check if the host supports high resolution timers or not.
440	*/
441	if ( pVM->vmm.s.fUsePeriodicPreemptionTimers
442	&& !RTTimerCanDoHighResolution())
443	pVM->vmm.s.fUsePeriodicPreemptionTimers = false;
444
445	/*
446	* Initialize the per VM data for GVMM and GMM.
447	*/
448	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
449	rc = GVMMR0InitVM(pGVM);
450	// if (RT_SUCCESS(rc))
451	// rc = GMMR0InitPerVMData(pVM);
452	if (RT_SUCCESS(rc))
453	{
454	/*
455	* Init HM, CPUM and PGM (Darwin only).
456	*/
457	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
458	rc = HMR0InitVM(pVM);
459	if (RT_SUCCESS(rc))
460	VMM_CHECK_SMAP_CHECK2(pVM, rc = VERR_VMM_RING0_ASSERTION); /* CPUR0InitVM will otherwise panic the host */
461	if (RT_SUCCESS(rc))
462	{
463	rc = CPUMR0InitVM(pVM);
464	if (RT_SUCCESS(rc))
465	{
466	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
467	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
468	rc = PGMR0DynMapInitVM(pVM);
469	#endif
470	if (RT_SUCCESS(rc))
471	{
472	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
473	rc = EMR0InitVM(pGVM, pVM);
474	if (RT_SUCCESS(rc))
475	{
476	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
477	#ifdef VBOX_WITH_PCI_PASSTHROUGH
478	rc = PciRawR0InitVM(pGVM, pVM);
479	#endif
480	if (RT_SUCCESS(rc))
481	{
482	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
483	rc = GIMR0InitVM(pVM);
484	if (RT_SUCCESS(rc))
485	{
486	VMM_CHECK_SMAP_CHECK2(pVM, rc = VERR_VMM_RING0_ASSERTION);
487	if (RT_SUCCESS(rc))
488	{
489	GVMMR0DoneInitVM(pGVM);
490
491	/*
492	* Collect a bit of info for the VM release log.
493	*/
494	pVM->vmm.s.fIsPreemptPendingApiTrusty = RTThreadPreemptIsPendingTrusty();
495	pVM->vmm.s.fIsPreemptPossible = RTThreadPreemptIsPossible();;
496
497	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
498	return rc;
499	}
500
501	/* bail out*/
502	GIMR0TermVM(pVM);
503	}
504	#ifdef VBOX_WITH_PCI_PASSTHROUGH
505	PciRawR0TermVM(pGVM, pVM);
506	#endif
507	}
508	}
509	}
510	}
511	HMR0TermVM(pVM);
512	}
513	}
514
515	RTLogSetDefaultInstanceThread(NULL, (uintptr_t)pVM->pSession);
516	return rc;
517	}
518
519
520	/**
521	* Does EMT specific VM initialization.
522	*
523	* @returns VBox status code.
524	* @param pGVM The ring-0 VM structure.
525	* @param pVM The cross context VM structure.
526	* @param idCpu The EMT that's calling.
527	*/
528	static int vmmR0InitVMEmt(PGVM pGVM, PVM pVM, VMCPUID idCpu)
529	{
530	/* Paranoia (caller checked these already). */
531	AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID);
532	AssertReturn(pGVM->aCpus[idCpu].hEMT == RTThreadNativeSelf(), VERR_INVALID_CPU_ID);
533
534	#ifdef LOG_ENABLED
535	/*
536	* Registration of ring 0 loggers.
537	*/
538	PVMCPU pVCpu = &pVM->aCpus[idCpu];
539	PVMMR0LOGGER pR0Logger = pVCpu->vmm.s.pR0LoggerR0;
540	if ( pR0Logger
541	&& !pR0Logger->fRegistered)
542	{
543	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pVM->pSession);
544	pR0Logger->fRegistered = true;
545	}
546	#endif
547	RT_NOREF(pVM);
548
549	return VINF_SUCCESS;
550	}
551
552
553
554	/**
555	* Terminates the R0 bits for a particular VM instance.
556	*
557	* This is normally called by ring-3 as part of the VM termination process, but
558	* may alternatively be called during the support driver session cleanup when
559	* the VM object is destroyed (see GVMM).
560	*
561	* @returns VBox status code.
562	*
563	* @param pGVM The global (ring-0) VM structure.
564	* @param pVM The cross context VM structure.
565	* @param idCpu Set to 0 if EMT(0) or NIL_VMCPUID if session cleanup
566	* thread.
567	* @thread EMT(0) or session clean up thread.
568	*/
569	VMMR0_INT_DECL(int) VMMR0TermVM(PGVM pGVM, PVM pVM, VMCPUID idCpu)
570	{
571	/*
572	* Check EMT(0) claim if we're called from userland.
573	*/
574	if (idCpu != NIL_VMCPUID)
575	{
576	AssertReturn(idCpu == 0, VERR_INVALID_CPU_ID);
577	int rc = GVMMR0ValidateGVMandVMandEMT(pGVM, pVM, idCpu);
578	if (RT_FAILURE(rc))
579	return rc;
580	}
581
582	#ifdef VBOX_WITH_PCI_PASSTHROUGH
583	PciRawR0TermVM(pGVM, pVM);
584	#endif
585
586	/*
587	* Tell GVMM what we're up to and check that we only do this once.
588	*/
589	if (GVMMR0DoingTermVM(pGVM))
590	{
591	GIMR0TermVM(pVM);
592
593	/** @todo I wish to call PGMR0PhysFlushHandyPages(pVM, &pVM->aCpus[idCpu])
594	* here to make sure we don't leak any shared pages if we crash... */
595	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
596	PGMR0DynMapTermVM(pVM);
597	#endif
598	HMR0TermVM(pVM);
599	}
600
601	/*
602	* Deregister the logger.
603	*/
604	RTLogSetDefaultInstanceThread(NULL, (uintptr_t)pVM->pSession);
605	return VINF_SUCCESS;
606	}
607
608
609	/**
610	* An interrupt or unhalt force flag is set, deal with it.
611	*
612	* @returns VINF_SUCCESS (or VINF_EM_HALT).
613	* @param pVCpu The cross context virtual CPU structure.
614	* @param uMWait Result from EMMonitorWaitIsActive().
615	* @param enmInterruptibility Guest CPU interruptbility level.
616	*/
617	static int vmmR0DoHaltInterrupt(PVMCPU pVCpu, unsigned uMWait, CPUMINTERRUPTIBILITY enmInterruptibility)
618	{
619	Assert(!TRPMHasTrap(pVCpu));
620
621	/*
622	* Pending interrupts w/o any SMIs or NMIs? That the usual case.
623	*/
624	if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
625	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI \| VMCPU_FF_INTERRUPT_NMI))
626	{
627	if (enmInterruptibility <= CPUMINTERRUPTIBILITY_INT_INHIBITED)
628	{
629	uint8_t u8Interrupt = 0;
630	int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
631	Log(("vmmR0DoHaltInterrupt: CPU%d u8Interrupt=%d (%#x) rc=%Rrc\n", pVCpu->idCpu, u8Interrupt, u8Interrupt, rc));
632	if (RT_SUCCESS(rc))
633	{
634	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_UNHALT);
635
636	rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
637	AssertRCSuccess(rc);
638	STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltExec);
639	return rc;
640	}
641	}
642	}
643	/*
644	* SMI is not implemented yet, at least not here.
645	*/
646	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI))
647	{
648	return VINF_EM_HALT;
649	}
650	/*
651	* NMI.
652	*/
653	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
654	{
655	if (enmInterruptibility < CPUMINTERRUPTIBILITY_NMI_INHIBIT)
656	{
657	/** @todo later. */
658	return VINF_EM_HALT;
659	}
660	}
661
662	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UNHALT))
663	{
664	STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltExec);
665	return VINF_SUCCESS;
666	}
667	if (uMWait > 1)
668	{
669	STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltExec);
670	return VINF_SUCCESS;
671	}
672
673	return VINF_EM_HALT;
674	}
675
676
677	/**
678	* This does one round of vmR3HaltGlobal1Halt().
679	*
680	* The rational here is that we'll reduce latency in interrupt situations if we
681	* don't go to ring-3 immediately on a VINF_EM_HALT (guest executed HLT or
682	* MWAIT), but do one round of blocking here instead and hope the interrupt is
683	* raised in the meanwhile.
684	*
685	* If we go to ring-3 we'll quit the inner HM/NEM loop in EM and end up in the
686	* outer loop, which will then call VMR3WaitHalted() and that in turn will do a
687	* ring-0 call (unless we're too close to a timer event). When the interrupt
688	* wakes us up, we'll return from ring-0 and EM will by instinct do a
689	* rescheduling (because of raw-mode) before it resumes the HM/NEM loop and gets
690	* back to VMMR0EntryFast().
691	*
692	* @returns VINF_SUCCESS or VINF_EM_HALT.
693	* @param pGVM The ring-0 VM structure.
694	* @param pVM The cross context VM structure.
695	* @param pGVCpu The ring-0 virtual CPU structure.
696	* @param pVCpu The cross context virtual CPU structure.
697	*
698	* @todo r=bird: All the blocking/waiting and EMT managment should move out of
699	* the VM module, probably to VMM. Then this would be more weird wrt
700	* parameters and statistics.
701	*/
702	static int vmmR0DoHalt(PGVM pGVM, PVM pVM, PGVMCPU pGVCpu, PVMCPU pVCpu)
703	{
704	Assert(pVCpu == pGVCpu->pVCpu);
705
706	/*
707	* Do spin stat historization.
708	*/
709	if (++pVCpu->vmm.s.cR0Halts & 0xff)
710	{ /* likely */ }
711	else if (pVCpu->vmm.s.cR0HaltsSucceeded > pVCpu->vmm.s.cR0HaltsToRing3)
712	{
713	pVCpu->vmm.s.cR0HaltsSucceeded = 2;
714	pVCpu->vmm.s.cR0HaltsToRing3 = 0;
715	}
716	else
717	{
718	pVCpu->vmm.s.cR0HaltsSucceeded = 0;
719	pVCpu->vmm.s.cR0HaltsToRing3 = 2;
720	}
721
722	/*
723	* Flags that makes us go to ring-3.
724	*/
725	uint32_t const fVmFFs = VM_FF_TM_VIRTUAL_SYNC \| VM_FF_PDM_QUEUES \| VM_FF_PDM_DMA
726	\| VM_FF_DBGF \| VM_FF_REQUEST \| VM_FF_CHECK_VM_STATE
727	\| VM_FF_RESET \| VM_FF_EMT_RENDEZVOUS \| VM_FF_PGM_NEED_HANDY_PAGES
728	\| VM_FF_PGM_NO_MEMORY \| VM_FF_REM_HANDLER_NOTIFY \| VM_FF_DEBUG_SUSPEND;
729	uint64_t const fCpuFFs = VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT \| VMCPU_FF_IEM
730	\| VMCPU_FF_REQUEST \| VMCPU_FF_DBGF \| VMCPU_FF_HM_UPDATE_CR3
731	\| VMCPU_FF_HM_UPDATE_PAE_PDPES \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
732	\| VMCPU_FF_TO_R3 \| VMCPU_FF_IOM \| VMCPU_FF_INTERRUPT_NESTED_GUEST /?/
733	\| VMCPU_FF_VMX_PREEMPT_TIMER /?/ \| VMCPU_FF_VMX_APIC_WRITE /?/ \| VMCPU_FF_VMX_MTF /?/
734	#ifdef VBOX_WITH_RAW_MODE
735	\| VMCPU_FF_TRPM_SYNC_IDT \| VMCPU_FF_SELM_SYNC_TSS \| VMCPU_FF_SELM_SYNC_GDT
736	\| VMCPU_FF_SELM_SYNC_LDT \| VMCPU_FF_CSAM_SCAN_PAGE \| VMCPU_FF_CSAM_PENDING_ACTION
737	\| VMCPU_FF_CPUM
738	#endif
739	;
740
741	/*
742	* Check preconditions.
743	*/
744	unsigned const uMWait = EMMonitorWaitIsActive(pVCpu);
745	CPUMINTERRUPTIBILITY const enmInterruptibility = CPUMGetGuestInterruptibility(pVCpu);
746	if ( pVCpu->vmm.s.fMayHaltInRing0
747	&& !TRPMHasTrap(pVCpu)
748	&& ( enmInterruptibility <= CPUMINTERRUPTIBILITY_INT_INHIBITED
749	\|\| uMWait > 1))
750	{
751	if ( !VM_FF_IS_ANY_SET(pVM, fVmFFs)
752	&& !VMCPU_FF_IS_ANY_SET(pVCpu, fCpuFFs))
753	{
754	/*
755	* Interrupts pending already?
756	*/
757	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
758	APICUpdatePendingInterrupts(pVCpu);
759
760	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC
761	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI \| VMCPU_FF_UNHALT))
762	return vmmR0DoHaltInterrupt(pVCpu, uMWait, enmInterruptibility);
763	ASMNopPause();
764
765	/*
766	* Check out how long till the next timer event.
767	*/
768	uint64_t u64Delta;
769	uint64_t u64GipTime = TMTimerPollGIP(pVM, pVCpu, &u64Delta);
770
771	if ( !VM_FF_IS_ANY_SET(pVM, fVmFFs)
772	&& !VMCPU_FF_IS_ANY_SET(pVCpu, fCpuFFs))
773	{
774	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
775	APICUpdatePendingInterrupts(pVCpu);
776
777	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC
778	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI \| VMCPU_FF_UNHALT))
779	return vmmR0DoHaltInterrupt(pVCpu, uMWait, enmInterruptibility);
780
781	/*
782	* Wait if there is enough time to the next timer event.
783	*/
784	if (u64Delta >= pVCpu->vmm.s.cNsSpinBlockThreshold)
785	{
786	/* If there are few other CPU cores around, we will procrastinate a
787	little before going to sleep, hoping for some device raising an
788	interrupt or similar. Though, the best thing here would be to
789	dynamically adjust the spin count according to its usfulness or
790	something... */
791	if ( pVCpu->vmm.s.cR0HaltsSucceeded > pVCpu->vmm.s.cR0HaltsToRing3
792	&& RTMpGetOnlineCount() >= 4)
793	{
794	/** @todo Figure out how we can skip this if it haven't help recently... */
795	uint32_t cSpinLoops = 42;
796	while (cSpinLoops-- > 0)
797	{
798	ASMNopPause();
799	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
800	APICUpdatePendingInterrupts(pVCpu);
801	ASMNopPause();
802	if (VM_FF_IS_ANY_SET(pVM, fVmFFs))
803	{
804	STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltToR3FromSpin);
805	return VINF_EM_HALT;
806	}
807	ASMNopPause();
808	if (VMCPU_FF_IS_ANY_SET(pVCpu, fCpuFFs))
809	{
810	STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltToR3FromSpin);
811	return VINF_EM_HALT;
812	}
813	ASMNopPause();
814	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC
815	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI \| VMCPU_FF_UNHALT))
816	{
817	STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltExecFromSpin);
818	return vmmR0DoHaltInterrupt(pVCpu, uMWait, enmInterruptibility);
819	}
820	ASMNopPause();
821	}
822	}
823
824	/* Block. We have to set the state to VMCPUSTATE_STARTED_HALTED here so ring-3
825	knows when to notify us (cannot access VMINTUSERPERVMCPU::fWait from here). */
826	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HALTED, VMCPUSTATE_STARTED);
827	uint64_t const u64StartSchedHalt = RTTimeNanoTS();
828	int rc = GVMMR0SchedHalt(pGVM, pVM, pGVCpu, u64GipTime);
829	uint64_t const u64EndSchedHalt = RTTimeNanoTS();
830	uint64_t const cNsElapsedSchedHalt = u64EndSchedHalt - u64StartSchedHalt;
831	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_HALTED);
832	STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->vmm.s.StatR0HaltBlock, cNsElapsedSchedHalt);
833	if ( rc == VINF_SUCCESS
834	\|\| rc == VERR_INTERRUPTED)
835
836	{
837	/* Keep some stats like ring-3 does. */
838	int64_t const cNsOverslept = u64EndSchedHalt - u64GipTime;
839	if (cNsOverslept > 50000)
840	STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->vmm.s.StatR0HaltBlockOverslept, cNsOverslept);
841	else if (cNsOverslept < -50000)
842	STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->vmm.s.StatR0HaltBlockInsomnia, cNsElapsedSchedHalt);
843	else
844	STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->vmm.s.StatR0HaltBlockOnTime, cNsElapsedSchedHalt);
845
846	/*
847	* Recheck whether we can resume execution or have to go to ring-3.
848	*/
849	if ( !VM_FF_IS_ANY_SET(pVM, fVmFFs)
850	&& !VMCPU_FF_IS_ANY_SET(pVCpu, fCpuFFs))
851	{
852	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
853	APICUpdatePendingInterrupts(pVCpu);
854	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC
855	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI \| VMCPU_FF_UNHALT))
856	{
857	STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltExecFromBlock);
858	return vmmR0DoHaltInterrupt(pVCpu, uMWait, enmInterruptibility);
859	}
860	}
861	}
862	}
863	}
864	}
865	}
866	return VINF_EM_HALT;
867	}
868
869
870	/**
871	* VMM ring-0 thread-context callback.
872	*
873	* This does common HM state updating and calls the HM-specific thread-context
874	* callback.
875	*
876	* @param enmEvent The thread-context event.
877	* @param pvUser Opaque pointer to the VMCPU.
878	*
879	* @thread EMT(pvUser)
880	*/
881	static DECLCALLBACK(void) vmmR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, void *pvUser)
882	{
883	PVMCPU pVCpu = (PVMCPU)pvUser;
884
885	switch (enmEvent)
886	{
887	case RTTHREADCTXEVENT_IN:
888	{
889	/*
890	* Linux may call us with preemption enabled (really!) but technically we
891	* cannot get preempted here, otherwise we end up in an infinite recursion
892	* scenario (i.e. preempted in resume hook -> preempt hook -> resume hook...
893	* ad infinitum). Let's just disable preemption for now...
894	*/
895	/** @todo r=bird: I don't believe the above. The linux code is clearly enabling
896	* preemption after doing the callout (one or two functions up the
897	* call chain). */
898	/** @todo r=ramshankar: See @bugref{5313#c30}. */
899	RTTHREADPREEMPTSTATE ParanoidPreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
900	RTThreadPreemptDisable(&ParanoidPreemptState);
901
902	/* We need to update the VCPU <-> host CPU mapping. */
903	RTCPUID idHostCpu;
904	uint32_t iHostCpuSet = RTMpCurSetIndexAndId(&idHostCpu);
905	pVCpu->iHostCpuSet = iHostCpuSet;
906	ASMAtomicWriteU32(&pVCpu->idHostCpu, idHostCpu);
907
908	/* In the very unlikely event that the GIP delta for the CPU we're
909	rescheduled needs calculating, try force a return to ring-3.
910	We unfortunately cannot do the measurements right here. */
911	if (RT_UNLIKELY(SUPIsTscDeltaAvailableForCpuSetIndex(iHostCpuSet)))
912	VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
913
914	/* Invoke the HM-specific thread-context callback. */
915	HMR0ThreadCtxCallback(enmEvent, pvUser);
916
917	/* Restore preemption. */
918	RTThreadPreemptRestore(&ParanoidPreemptState);
919	break;
920	}
921
922	case RTTHREADCTXEVENT_OUT:
923	{
924	/* Invoke the HM-specific thread-context callback. */
925	HMR0ThreadCtxCallback(enmEvent, pvUser);
926
927	/*
928	* Sigh. See VMMGetCpu() used by VMCPU_ASSERT_EMT(). We cannot let several VCPUs
929	* have the same host CPU associated with it.
930	*/
931	pVCpu->iHostCpuSet = UINT32_MAX;
932	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
933	break;
934	}
935
936	default:
937	/* Invoke the HM-specific thread-context callback. */
938	HMR0ThreadCtxCallback(enmEvent, pvUser);
939	break;
940	}
941	}
942
943
944	/**
945	* Creates thread switching hook for the current EMT thread.
946	*
947	* This is called by GVMMR0CreateVM and GVMMR0RegisterVCpu. If the host
948	* platform does not implement switcher hooks, no hooks will be create and the
949	* member set to NIL_RTTHREADCTXHOOK.
950	*
951	* @returns VBox status code.
952	* @param pVCpu The cross context virtual CPU structure.
953	* @thread EMT(pVCpu)
954	*/
955	VMMR0_INT_DECL(int) VMMR0ThreadCtxHookCreateForEmt(PVMCPU pVCpu)
956	{
957	VMCPU_ASSERT_EMT(pVCpu);
958	Assert(pVCpu->vmm.s.hCtxHook == NIL_RTTHREADCTXHOOK);
959
960	#if 1 /* To disable this stuff change to zero. */
961	int rc = RTThreadCtxHookCreate(&pVCpu->vmm.s.hCtxHook, 0, vmmR0ThreadCtxCallback, pVCpu);
962	if (RT_SUCCESS(rc))
963	return rc;
964	#else
965	RT_NOREF(vmmR0ThreadCtxCallback);
966	int rc = VERR_NOT_SUPPORTED;
967	#endif
968
969	pVCpu->vmm.s.hCtxHook = NIL_RTTHREADCTXHOOK;
970	if (rc == VERR_NOT_SUPPORTED)
971	return VINF_SUCCESS;
972
973	LogRelMax(32, ("RTThreadCtxHookCreate failed! rc=%Rrc pVCpu=%p idCpu=%RU32\n", rc, pVCpu, pVCpu->idCpu));
974	return VINF_SUCCESS; /* Just ignore it, we can live without context hooks. */
975	}
976
977
978	/**
979	* Destroys the thread switching hook for the specified VCPU.
980	*
981	* @param pVCpu The cross context virtual CPU structure.
982	* @remarks Can be called from any thread.
983	*/
984	VMMR0_INT_DECL(void) VMMR0ThreadCtxHookDestroyForEmt(PVMCPU pVCpu)
985	{
986	int rc = RTThreadCtxHookDestroy(pVCpu->vmm.s.hCtxHook);
987	AssertRC(rc);
988	pVCpu->vmm.s.hCtxHook = NIL_RTTHREADCTXHOOK;
989	}
990
991
992	/**
993	* Disables the thread switching hook for this VCPU (if we got one).
994	*
995	* @param pVCpu The cross context virtual CPU structure.
996	* @thread EMT(pVCpu)
997	*
998	* @remarks This also clears VMCPU::idHostCpu, so the mapping is invalid after
999	* this call. This means you have to be careful with what you do!
1000	*/
1001	VMMR0_INT_DECL(void) VMMR0ThreadCtxHookDisable(PVMCPU pVCpu)
1002	{
1003	/*
1004	* Clear the VCPU <-> host CPU mapping as we've left HM context.
1005	* @bugref{7726#c19} explains the need for this trick:
1006	*
1007	* hmR0VmxCallRing3Callback/hmR0SvmCallRing3Callback &
1008	* hmR0VmxLeaveSession/hmR0SvmLeaveSession disables context hooks during
1009	* longjmp & normal return to ring-3, which opens a window where we may be
1010	* rescheduled without changing VMCPUID::idHostCpu and cause confusion if
1011	* the CPU starts executing a different EMT. Both functions first disables
1012	* preemption and then calls HMR0LeaveCpu which invalids idHostCpu, leaving
1013	* an opening for getting preempted.
1014	*/
1015	/** @todo Make HM not need this API! Then we could leave the hooks enabled
1016	* all the time. */
1017	/** @todo move this into the context hook disabling if(). */
1018	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
1019
1020	/*
1021	* Disable the context hook, if we got one.
1022	*/
1023	if (pVCpu->vmm.s.hCtxHook != NIL_RTTHREADCTXHOOK)
1024	{
1025	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1026	int rc = RTThreadCtxHookDisable(pVCpu->vmm.s.hCtxHook);
1027	AssertRC(rc);
1028	}
1029	}
1030
1031
1032	/**
1033	* Internal version of VMMR0ThreadCtxHooksAreRegistered.
1034	*
1035	* @returns true if registered, false otherwise.
1036	* @param pVCpu The cross context virtual CPU structure.
1037	*/
1038	DECLINLINE(bool) vmmR0ThreadCtxHookIsEnabled(PVMCPU pVCpu)
1039	{
1040	return RTThreadCtxHookIsEnabled(pVCpu->vmm.s.hCtxHook);
1041	}
1042
1043
1044	/**
1045	* Whether thread-context hooks are registered for this VCPU.
1046	*
1047	* @returns true if registered, false otherwise.
1048	* @param pVCpu The cross context virtual CPU structure.
1049	*/
1050	VMMR0_INT_DECL(bool) VMMR0ThreadCtxHookIsEnabled(PVMCPU pVCpu)
1051	{
1052	return vmmR0ThreadCtxHookIsEnabled(pVCpu);
1053	}
1054
1055
1056	#ifdef VBOX_WITH_STATISTICS
1057	/**
1058	* Record return code statistics
1059	* @param pVM The cross context VM structure.
1060	* @param pVCpu The cross context virtual CPU structure.
1061	* @param rc The status code.
1062	*/
1063	static void vmmR0RecordRC(PVM pVM, PVMCPU pVCpu, int rc)
1064	{
1065	/*
1066	* Collect statistics.
1067	*/
1068	switch (rc)
1069	{
1070	case VINF_SUCCESS:
1071	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetNormal);
1072	break;
1073	case VINF_EM_RAW_INTERRUPT:
1074	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetInterrupt);
1075	break;
1076	case VINF_EM_RAW_INTERRUPT_HYPER:
1077	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetInterruptHyper);
1078	break;
1079	case VINF_EM_RAW_GUEST_TRAP:
1080	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetGuestTrap);
1081	break;
1082	case VINF_EM_RAW_RING_SWITCH:
1083	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetRingSwitch);
1084	break;
1085	case VINF_EM_RAW_RING_SWITCH_INT:
1086	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetRingSwitchInt);
1087	break;
1088	case VINF_EM_RAW_STALE_SELECTOR:
1089	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetStaleSelector);
1090	break;
1091	case VINF_EM_RAW_IRET_TRAP:
1092	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIRETTrap);
1093	break;
1094	case VINF_IOM_R3_IOPORT_READ:
1095	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIORead);
1096	break;
1097	case VINF_IOM_R3_IOPORT_WRITE:
1098	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIOWrite);
1099	break;
1100	case VINF_IOM_R3_IOPORT_COMMIT_WRITE:
1101	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIOCommitWrite);
1102	break;
1103	case VINF_IOM_R3_MMIO_READ:
1104	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIORead);
1105	break;
1106	case VINF_IOM_R3_MMIO_WRITE:
1107	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOWrite);
1108	break;
1109	case VINF_IOM_R3_MMIO_COMMIT_WRITE:
1110	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOCommitWrite);
1111	break;
1112	case VINF_IOM_R3_MMIO_READ_WRITE:
1113	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOReadWrite);
1114	break;
1115	case VINF_PATM_HC_MMIO_PATCH_READ:
1116	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOPatchRead);
1117	break;
1118	case VINF_PATM_HC_MMIO_PATCH_WRITE:
1119	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOPatchWrite);
1120	break;
1121	case VINF_CPUM_R3_MSR_READ:
1122	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMSRRead);
1123	break;
1124	case VINF_CPUM_R3_MSR_WRITE:
1125	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMSRWrite);
1126	break;
1127	case VINF_EM_RAW_EMULATE_INSTR:
1128	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetEmulate);
1129	break;
1130	case VINF_PATCH_EMULATE_INSTR:
1131	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchEmulate);
1132	break;
1133	case VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT:
1134	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetLDTFault);
1135	break;
1136	case VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT:
1137	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetGDTFault);
1138	break;
1139	case VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT:
1140	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIDTFault);
1141	break;
1142	case VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT:
1143	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetTSSFault);
1144	break;
1145	case VINF_CSAM_PENDING_ACTION:
1146	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetCSAMTask);
1147	break;
1148	case VINF_PGM_SYNC_CR3:
1149	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetSyncCR3);
1150	break;
1151	case VINF_PATM_PATCH_INT3:
1152	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchInt3);
1153	break;
1154	case VINF_PATM_PATCH_TRAP_PF:
1155	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchPF);
1156	break;
1157	case VINF_PATM_PATCH_TRAP_GP:
1158	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchGP);
1159	break;
1160	case VINF_PATM_PENDING_IRQ_AFTER_IRET:
1161	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchIretIRQ);
1162	break;
1163	case VINF_EM_RESCHEDULE_REM:
1164	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetRescheduleREM);
1165	break;
1166	case VINF_EM_RAW_TO_R3:
1167	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Total);
1168	if (VM_FF_IS_SET(pVM, VM_FF_TM_VIRTUAL_SYNC))
1169	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3TMVirt);
1170	else if (VM_FF_IS_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES))
1171	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3HandyPages);
1172	else if (VM_FF_IS_SET(pVM, VM_FF_PDM_QUEUES))
1173	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3PDMQueues);
1174	else if (VM_FF_IS_SET(pVM, VM_FF_EMT_RENDEZVOUS))
1175	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Rendezvous);
1176	else if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
1177	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3DMA);
1178	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TIMER))
1179	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Timer);
1180	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PDM_CRITSECT))
1181	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3CritSect);
1182	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TO_R3))
1183	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3FF);
1184	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM))
1185	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Iem);
1186	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IOM))
1187	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Iom);
1188	else
1189	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Unknown);
1190	break;
1191
1192	case VINF_EM_RAW_TIMER_PENDING:
1193	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetTimerPending);
1194	break;
1195	case VINF_EM_RAW_INTERRUPT_PENDING:
1196	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetInterruptPending);
1197	break;
1198	case VINF_VMM_CALL_HOST:
1199	switch (pVCpu->vmm.s.enmCallRing3Operation)
1200	{
1201	case VMMCALLRING3_PDM_CRIT_SECT_ENTER:
1202	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPDMCritSectEnter);
1203	break;
1204	case VMMCALLRING3_PDM_LOCK:
1205	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPDMLock);
1206	break;
1207	case VMMCALLRING3_PGM_POOL_GROW:
1208	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPGMPoolGrow);
1209	break;
1210	case VMMCALLRING3_PGM_LOCK:
1211	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPGMLock);
1212	break;
1213	case VMMCALLRING3_PGM_MAP_CHUNK:
1214	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPGMMapChunk);
1215	break;
1216	case VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES:
1217	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPGMAllocHandy);
1218	break;
1219	case VMMCALLRING3_REM_REPLAY_HANDLER_NOTIFICATIONS:
1220	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallRemReplay);
1221	break;
1222	case VMMCALLRING3_VMM_LOGGER_FLUSH:
1223	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallLogFlush);
1224	break;
1225	case VMMCALLRING3_VM_SET_ERROR:
1226	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallVMSetError);
1227	break;
1228	case VMMCALLRING3_VM_SET_RUNTIME_ERROR:
1229	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallVMSetRuntimeError);
1230	break;
1231	case VMMCALLRING3_VM_R0_ASSERTION:
1232	default:
1233	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetCallRing3);
1234	break;
1235	}
1236	break;
1237	case VINF_PATM_DUPLICATE_FUNCTION:
1238	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPATMDuplicateFn);
1239	break;
1240	case VINF_PGM_CHANGE_MODE:
1241	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPGMChangeMode);
1242	break;
1243	case VINF_PGM_POOL_FLUSH_PENDING:
1244	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPGMFlushPending);
1245	break;
1246	case VINF_EM_PENDING_REQUEST:
1247	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPendingRequest);
1248	break;
1249	case VINF_EM_HM_PATCH_TPR_INSTR:
1250	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchTPR);
1251	break;
1252	default:
1253	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMisc);
1254	break;
1255	}
1256	}
1257	#endif /* VBOX_WITH_STATISTICS */
1258
1259
1260	/**
1261	* The Ring 0 entry point, called by the fast-ioctl path.
1262	*
1263	* @param pGVM The global (ring-0) VM structure.
1264	* @param pVM The cross context VM structure.
1265	* The return code is stored in pVM->vmm.s.iLastGZRc.
1266	* @param idCpu The Virtual CPU ID of the calling EMT.
1267	* @param enmOperation Which operation to execute.
1268	* @remarks Assume called with interrupts _enabled_.
1269	*/
1270	VMMR0DECL(void) VMMR0EntryFast(PGVM pGVM, PVM pVM, VMCPUID idCpu, VMMR0OPERATION enmOperation)
1271	{
1272	/*
1273	* Validation.
1274	*/
1275	if ( idCpu < pGVM->cCpus
1276	&& pGVM->cCpus == pVM->cCpus)
1277	{ /likely/ }
1278	else
1279	{
1280	SUPR0Printf("VMMR0EntryFast: Bad idCpu=%#x cCpus=%#x/%#x\n", idCpu, pGVM->cCpus, pVM->cCpus);
1281	return;
1282	}
1283
1284	PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
1285	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1286	RTNATIVETHREAD const hNativeThread = RTThreadNativeSelf();
1287	if (RT_LIKELY( pGVCpu->hEMT == hNativeThread
1288	&& pVCpu->hNativeThreadR0 == hNativeThread))
1289	{ /* likely */ }
1290	else
1291	{
1292	SUPR0Printf("VMMR0EntryFast: Bad thread idCpu=%#x hNativeSelf=%p pGVCpu->hEmt=%p pVCpu->hNativeThreadR0=%p\n",
1293	idCpu, hNativeThread, pGVCpu->hEMT, pVCpu->hNativeThreadR0);
1294	return;
1295	}
1296
1297	/*
1298	* SMAP fun.
1299	*/
1300	VMM_CHECK_SMAP_SETUP();
1301	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1302
1303	/*
1304	* Perform requested operation.
1305	*/
1306	switch (enmOperation)
1307	{
1308	/*
1309	* Switch to GC and run guest raw mode code.
1310	* Disable interrupts before doing the world switch.
1311	*/
1312	case VMMR0_DO_RAW_RUN:
1313	{
1314	#ifdef VBOX_WITH_RAW_MODE
1315	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
1316	/* Some safety precautions first. */
1317	if (RT_UNLIKELY(!PGMGetHyperCR3(pVCpu)))
1318	{
1319	pVCpu->vmm.s.iLastGZRc = VERR_PGM_NO_CR3_SHADOW_ROOT;
1320	break;
1321	}
1322	# endif
1323	if (RT_SUCCESS(g_rcRawModeUsability))
1324	{ /* likely */ }
1325	else
1326	{
1327	pVCpu->vmm.s.iLastGZRc = g_rcRawModeUsability;
1328	break;
1329	}
1330
1331	/*
1332	* Disable preemption.
1333	*/
1334	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
1335	RTThreadPreemptDisable(&PreemptState);
1336
1337	/*
1338	* Get the host CPU identifiers, make sure they are valid and that
1339	* we've got a TSC delta for the CPU.
1340	*/
1341	RTCPUID idHostCpu;
1342	uint32_t iHostCpuSet = RTMpCurSetIndexAndId(&idHostCpu);
1343	if (RT_LIKELY( iHostCpuSet < RTCPUSET_MAX_CPUS
1344	&& SUPIsTscDeltaAvailableForCpuSetIndex(iHostCpuSet)))
1345	{
1346	/*
1347	* Commit the CPU identifiers and update the periodict preemption timer if it's active.
1348	*/
1349	# ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
1350	CPUMR0SetLApic(pVCpu, iHostCpuSet);
1351	# endif
1352	pVCpu->iHostCpuSet = iHostCpuSet;
1353	ASMAtomicWriteU32(&pVCpu->idHostCpu, idHostCpu);
1354
1355	if (pVM->vmm.s.fUsePeriodicPreemptionTimers)
1356	GVMMR0SchedUpdatePeriodicPreemptionTimer(pVM, pVCpu->idHostCpu, TMCalcHostTimerFrequency(pVM, pVCpu));
1357
1358	/*
1359	* We might need to disable VT-x if the active switcher turns off paging.
1360	*/
1361	bool fVTxDisabled;
1362	int rc = HMR0EnterSwitcher(pVM, pVM->vmm.s.enmSwitcher, &fVTxDisabled);
1363	if (RT_SUCCESS(rc))
1364	{
1365	/*
1366	* Disable interrupts and run raw-mode code. The loop is for efficiently
1367	* dispatching tracepoints that fired in raw-mode context.
1368	*/
1369	RTCCUINTREG uFlags = ASMIntDisableFlags();
1370
1371	for (;;)
1372	{
1373	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
1374	TMNotifyStartOfExecution(pVCpu);
1375
1376	rc = pVM->vmm.s.pfnR0ToRawMode(pVM);
1377	pVCpu->vmm.s.iLastGZRc = rc;
1378
1379	TMNotifyEndOfExecution(pVCpu);
1380	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
1381
1382	if (rc != VINF_VMM_CALL_TRACER)
1383	break;
1384	SUPR0TracerUmodProbeFire(pVM->pSession, &pVCpu->vmm.s.TracerCtx);
1385	}
1386
1387	/*
1388	* Re-enable VT-x before we dispatch any pending host interrupts and
1389	* re-enables interrupts.
1390	*/
1391	HMR0LeaveSwitcher(pVM, fVTxDisabled);
1392
1393	if ( rc == VINF_EM_RAW_INTERRUPT
1394	\|\| rc == VINF_EM_RAW_INTERRUPT_HYPER)
1395	TRPMR0DispatchHostInterrupt(pVM);
1396
1397	ASMSetFlags(uFlags);
1398
1399	/* Fire dtrace probe and collect statistics. */
1400	VBOXVMM_R0_VMM_RETURN_TO_RING3_RC(pVCpu, CPUMQueryGuestCtxPtr(pVCpu), rc);
1401	# ifdef VBOX_WITH_STATISTICS
1402	STAM_COUNTER_INC(&pVM->vmm.s.StatRunRC);
1403	vmmR0RecordRC(pVM, pVCpu, rc);
1404	# endif
1405	}
1406	else
1407	pVCpu->vmm.s.iLastGZRc = rc;
1408
1409	/*
1410	* Invalidate the host CPU identifiers as we restore preemption.
1411	*/
1412	pVCpu->iHostCpuSet = UINT32_MAX;
1413	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
1414
1415	RTThreadPreemptRestore(&PreemptState);
1416	}
1417	/*
1418	* Invalid CPU set index or TSC delta in need of measuring.
1419	*/
1420	else
1421	{
1422	RTThreadPreemptRestore(&PreemptState);
1423	if (iHostCpuSet < RTCPUSET_MAX_CPUS)
1424	{
1425	int rc = SUPR0TscDeltaMeasureBySetIndex(pVM->pSession, iHostCpuSet, 0 /fFlags/,
1426	2 /cMsWaitRetry/, 5RT_MS_1SEC /cMsWaitThread*/,
1427	0 /default cTries/);
1428	if (RT_SUCCESS(rc) \|\| rc == VERR_CPU_OFFLINE)
1429	pVCpu->vmm.s.iLastGZRc = VINF_EM_RAW_TO_R3;
1430	else
1431	pVCpu->vmm.s.iLastGZRc = rc;
1432	}
1433	else
1434	pVCpu->vmm.s.iLastGZRc = VERR_INVALID_CPU_INDEX;
1435	}
1436
1437	#else /* !VBOX_WITH_RAW_MODE */
1438	pVCpu->vmm.s.iLastGZRc = VERR_RAW_MODE_NOT_SUPPORTED;
1439	#endif
1440	break;
1441	}
1442
1443	/*
1444	* Run guest code using the available hardware acceleration technology.
1445	*/
1446	case VMMR0_DO_HM_RUN:
1447	{
1448	for (;;) /* hlt loop */
1449	{
1450	/*
1451	* Disable preemption.
1452	*/
1453	Assert(!vmmR0ThreadCtxHookIsEnabled(pVCpu));
1454	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
1455	RTThreadPreemptDisable(&PreemptState);
1456
1457	/*
1458	* Get the host CPU identifiers, make sure they are valid and that
1459	* we've got a TSC delta for the CPU.
1460	*/
1461	RTCPUID idHostCpu;
1462	uint32_t iHostCpuSet = RTMpCurSetIndexAndId(&idHostCpu);
1463	if (RT_LIKELY( iHostCpuSet < RTCPUSET_MAX_CPUS
1464	&& SUPIsTscDeltaAvailableForCpuSetIndex(iHostCpuSet)))
1465	{
1466	pVCpu->iHostCpuSet = iHostCpuSet;
1467	ASMAtomicWriteU32(&pVCpu->idHostCpu, idHostCpu);
1468
1469	/*
1470	* Update the periodic preemption timer if it's active.
1471	*/
1472	if (pVM->vmm.s.fUsePeriodicPreemptionTimers)
1473	GVMMR0SchedUpdatePeriodicPreemptionTimer(pVM, pVCpu->idHostCpu, TMCalcHostTimerFrequency(pVM, pVCpu));
1474	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1475
1476	#ifdef VMM_R0_TOUCH_FPU
1477	/*
1478	* Make sure we've got the FPU state loaded so and we don't need to clear
1479	* CR0.TS and get out of sync with the host kernel when loading the guest
1480	* FPU state. @ref sec_cpum_fpu (CPUM.cpp) and @bugref{4053}.
1481	*/
1482	CPUMR0TouchHostFpu();
1483	#endif
1484	int rc;
1485	bool fPreemptRestored = false;
1486	if (!HMR0SuspendPending())
1487	{
1488	/*
1489	* Enable the context switching hook.
1490	*/
1491	if (pVCpu->vmm.s.hCtxHook != NIL_RTTHREADCTXHOOK)
1492	{
1493	Assert(!RTThreadCtxHookIsEnabled(pVCpu->vmm.s.hCtxHook));
1494	int rc2 = RTThreadCtxHookEnable(pVCpu->vmm.s.hCtxHook); AssertRC(rc2);
1495	}
1496
1497	/*
1498	* Enter HM context.
1499	*/
1500	rc = HMR0Enter(pVCpu);
1501	if (RT_SUCCESS(rc))
1502	{
1503	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
1504
1505	/*
1506	* When preemption hooks are in place, enable preemption now that
1507	* we're in HM context.
1508	*/
1509	if (vmmR0ThreadCtxHookIsEnabled(pVCpu))
1510	{
1511	fPreemptRestored = true;
1512	RTThreadPreemptRestore(&PreemptState);
1513	}
1514
1515	/*
1516	* Setup the longjmp machinery and execute guest code (calls HMR0RunGuestCode).
1517	*/
1518	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1519	rc = vmmR0CallRing3SetJmp(&pVCpu->vmm.s.CallRing3JmpBufR0, HMR0RunGuestCode, pVM, pVCpu);
1520	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1521
1522	/*
1523	* Assert sanity on the way out. Using manual assertions code here as normal
1524	* assertions are going to panic the host since we're outside the setjmp/longjmp zone.
1525	*/
1526	if (RT_UNLIKELY( VMCPU_GET_STATE(pVCpu) != VMCPUSTATE_STARTED_HM
1527	&& RT_SUCCESS_NP(rc) && rc != VINF_VMM_CALL_HOST ))
1528	{
1529	pVM->vmm.s.szRing0AssertMsg1[0] = '\0';
1530	RTStrPrintf(pVM->vmm.s.szRing0AssertMsg2, sizeof(pVM->vmm.s.szRing0AssertMsg2),
1531	"Got VMCPU state %d expected %d.\n", VMCPU_GET_STATE(pVCpu), VMCPUSTATE_STARTED_HM);
1532	rc = VERR_VMM_WRONG_HM_VMCPU_STATE;
1533	}
1534	/** @todo Get rid of this. HM shouldn't disable the context hook. */
1535	else if (RT_UNLIKELY(vmmR0ThreadCtxHookIsEnabled(pVCpu)))
1536	{
1537	pVM->vmm.s.szRing0AssertMsg1[0] = '\0';
1538	RTStrPrintf(pVM->vmm.s.szRing0AssertMsg2, sizeof(pVM->vmm.s.szRing0AssertMsg2),
1539	"Thread-context hooks still enabled! VCPU=%p Id=%u rc=%d.\n", pVCpu, pVCpu->idCpu, rc);
1540	rc = VERR_INVALID_STATE;
1541	}
1542
1543	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
1544	}
1545	STAM_COUNTER_INC(&pVM->vmm.s.StatRunRC);
1546
1547	/*
1548	* Invalidate the host CPU identifiers before we disable the context
1549	* hook / restore preemption.
1550	*/
1551	pVCpu->iHostCpuSet = UINT32_MAX;
1552	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
1553
1554	/*
1555	* Disable context hooks. Due to unresolved cleanup issues, we
1556	* cannot leave the hooks enabled when we return to ring-3.
1557	*
1558	* Note! At the moment HM may also have disabled the hook
1559	* when we get here, but the IPRT API handles that.
1560	*/
1561	if (pVCpu->vmm.s.hCtxHook != NIL_RTTHREADCTXHOOK)
1562	{
1563	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
1564	RTThreadCtxHookDisable(pVCpu->vmm.s.hCtxHook);
1565	}
1566	}
1567	/*
1568	* The system is about to go into suspend mode; go back to ring 3.
1569	*/
1570	else
1571	{
1572	rc = VINF_EM_RAW_INTERRUPT;
1573	pVCpu->iHostCpuSet = UINT32_MAX;
1574	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
1575	}
1576
1577	/** @todo When HM stops messing with the context hook state, we'll disable
1578	* preemption again before the RTThreadCtxHookDisable call. */
1579	if (!fPreemptRestored)
1580	RTThreadPreemptRestore(&PreemptState);
1581
1582	pVCpu->vmm.s.iLastGZRc = rc;
1583
1584	/* Fire dtrace probe and collect statistics. */
1585	VBOXVMM_R0_VMM_RETURN_TO_RING3_HM(pVCpu, CPUMQueryGuestCtxPtr(pVCpu), rc);
1586	#ifdef VBOX_WITH_STATISTICS
1587	vmmR0RecordRC(pVM, pVCpu, rc);
1588	#endif
1589
1590	/*
1591	* If this is a halt.
1592	*/
1593	if (rc != VINF_EM_HALT)
1594	{ /* we're not in a hurry for a HLT, so prefer this path */ }
1595	else //if (VMCPU_FF_IS_ANY_SET())
1596	{
1597	pVCpu->vmm.s.iLastGZRc = rc = vmmR0DoHalt(pGVM, pVM, pGVCpu, pVCpu);
1598	if (rc == VINF_SUCCESS)
1599	{
1600	pVCpu->vmm.s.cR0HaltsSucceeded++;
1601	continue;
1602	}
1603	pVCpu->vmm.s.cR0HaltsToRing3++;
1604	}
1605	}
1606	/*
1607	* Invalid CPU set index or TSC delta in need of measuring.
1608	*/
1609	else
1610	{
1611	pVCpu->iHostCpuSet = UINT32_MAX;
1612	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
1613	RTThreadPreemptRestore(&PreemptState);
1614	if (iHostCpuSet < RTCPUSET_MAX_CPUS)
1615	{
1616	int rc = SUPR0TscDeltaMeasureBySetIndex(pVM->pSession, iHostCpuSet, 0 /fFlags/,
1617	2 /cMsWaitRetry/, 5RT_MS_1SEC /cMsWaitThread*/,
1618	0 /default cTries/);
1619	if (RT_SUCCESS(rc) \|\| rc == VERR_CPU_OFFLINE)
1620	pVCpu->vmm.s.iLastGZRc = VINF_EM_RAW_TO_R3;
1621	else
1622	pVCpu->vmm.s.iLastGZRc = rc;
1623	}
1624	else
1625	pVCpu->vmm.s.iLastGZRc = VERR_INVALID_CPU_INDEX;
1626	}
1627	break;
1628
1629	} /* halt loop. */
1630	break;
1631	}
1632
1633	#ifdef VBOX_WITH_NEM_R0
1634	# if defined(RT_ARCH_AMD64) && defined(RT_OS_WINDOWS)
1635	case VMMR0_DO_NEM_RUN:
1636	{
1637	/*
1638	* Setup the longjmp machinery and execute guest code (calls NEMR0RunGuestCode).
1639	*/
1640	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1641	int rc = vmmR0CallRing3SetJmp2(&pVCpu->vmm.s.CallRing3JmpBufR0, NEMR0RunGuestCode, pGVM, idCpu);
1642	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1643	STAM_COUNTER_INC(&pVM->vmm.s.StatRunRC);
1644
1645	pVCpu->vmm.s.iLastGZRc = rc;
1646
1647	/*
1648	* Fire dtrace probe and collect statistics.
1649	*/
1650	VBOXVMM_R0_VMM_RETURN_TO_RING3_NEM(pVCpu, CPUMQueryGuestCtxPtr(pVCpu), rc);
1651	# ifdef VBOX_WITH_STATISTICS
1652	vmmR0RecordRC(pVM, pVCpu, rc);
1653	# endif
1654	break;
1655	}
1656	# endif
1657	#endif
1658
1659
1660	/*
1661	* For profiling.
1662	*/
1663	case VMMR0_DO_NOP:
1664	pVCpu->vmm.s.iLastGZRc = VINF_SUCCESS;
1665	break;
1666
1667	/*
1668	* Shouldn't happen.
1669	*/
1670	default:
1671	AssertMsgFailed(("%#x\n", enmOperation));
1672	pVCpu->vmm.s.iLastGZRc = VERR_NOT_SUPPORTED;
1673	break;
1674	}
1675	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1676	}
1677
1678
1679	/**
1680	* Validates a session or VM session argument.
1681	*
1682	* @returns true / false accordingly.
1683	* @param pVM The cross context VM structure.
1684	* @param pClaimedSession The session claim to validate.
1685	* @param pSession The session argument.
1686	*/
1687	DECLINLINE(bool) vmmR0IsValidSession(PVM pVM, PSUPDRVSESSION pClaimedSession, PSUPDRVSESSION pSession)
1688	{
1689	/* This must be set! */
1690	if (!pSession)
1691	return false;
1692
1693	/* Only one out of the two. */
1694	if (pVM && pClaimedSession)
1695	return false;
1696	if (pVM)
1697	pClaimedSession = pVM->pSession;
1698	return pClaimedSession == pSession;
1699	}
1700
1701
1702	/**
1703	* VMMR0EntryEx worker function, either called directly or when ever possible
1704	* called thru a longjmp so we can exit safely on failure.
1705	*
1706	* @returns VBox status code.
1707	* @param pGVM The global (ring-0) VM structure.
1708	* @param pVM The cross context VM structure.
1709	* @param idCpu Virtual CPU ID argument. Must be NIL_VMCPUID if pVM
1710	* is NIL_RTR0PTR, and may be NIL_VMCPUID if it isn't
1711	* @param enmOperation Which operation to execute.
1712	* @param pReqHdr This points to a SUPVMMR0REQHDR packet. Optional.
1713	* The support driver validates this if it's present.
1714	* @param u64Arg Some simple constant argument.
1715	* @param pSession The session of the caller.
1716	*
1717	* @remarks Assume called with interrupts _enabled_.
1718	*/
1719	static int vmmR0EntryExWorker(PGVM pGVM, PVM pVM, VMCPUID idCpu, VMMR0OPERATION enmOperation,
1720	PSUPVMMR0REQHDR pReqHdr, uint64_t u64Arg, PSUPDRVSESSION pSession)
1721	{
1722	/*
1723	* Validate pGVM, pVM and idCpu for consistency and validity.
1724	*/
1725	if ( pGVM != NULL
1726	\|\| pVM != NULL)
1727	{
1728	if (RT_LIKELY( RT_VALID_PTR(pGVM)
1729	&& RT_VALID_PTR(pVM)
1730	&& ((uintptr_t)pVM & PAGE_OFFSET_MASK) == 0))
1731	{ /* likely */ }
1732	else
1733	{
1734	SUPR0Printf("vmmR0EntryExWorker: Invalid pGVM=%p and/or pVM=%p! (op=%d)\n", pGVM, pVM, enmOperation);
1735	return VERR_INVALID_POINTER;
1736	}
1737
1738	if (RT_LIKELY(pGVM->pVM == pVM))
1739	{ /* likely */ }
1740	else
1741	{
1742	SUPR0Printf("vmmR0EntryExWorker: pVM mismatch: got %p, pGVM->pVM=%p\n", pVM, pGVM->pVM);
1743	return VERR_INVALID_PARAMETER;
1744	}
1745
1746	if (RT_LIKELY(idCpu == NIL_VMCPUID \|\| idCpu < pGVM->cCpus))
1747	{ /* likely */ }
1748	else
1749	{
1750	SUPR0Printf("vmmR0EntryExWorker: Invalid idCpu %#x (cCpus=%#x)\n", idCpu, pGVM->cCpus);
1751	return VERR_INVALID_PARAMETER;
1752	}
1753
1754	if (RT_LIKELY( pVM->enmVMState >= VMSTATE_CREATING
1755	&& pVM->enmVMState <= VMSTATE_TERMINATED
1756	&& pVM->cCpus == pGVM->cCpus
1757	&& pVM->pSession == pSession
1758	&& pVM->pVMR0 == pVM))
1759	{ /* likely */ }
1760	else
1761	{
1762	SUPR0Printf("vmmR0EntryExWorker: Invalid pVM=%p:{.enmVMState=%d, .cCpus=%#x(==%#x), .pSession=%p(==%p), .pVMR0=%p(==%p)}! (op=%d)\n",
1763	pVM, pVM->enmVMState, pVM->cCpus, pGVM->cCpus, pVM->pSession, pSession, pVM->pVMR0, pVM, enmOperation);
1764	return VERR_INVALID_POINTER;
1765	}
1766	}
1767	else if (RT_LIKELY(idCpu == NIL_VMCPUID))
1768	{ /* likely */ }
1769	else
1770	{
1771	SUPR0Printf("vmmR0EntryExWorker: Invalid idCpu=%u\n", idCpu);
1772	return VERR_INVALID_PARAMETER;
1773	}
1774
1775	/*
1776	* SMAP fun.
1777	*/
1778	VMM_CHECK_SMAP_SETUP();
1779	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
1780
1781	/*
1782	* Process the request.
1783	*/
1784	int rc;
1785	switch (enmOperation)
1786	{
1787	/*
1788	* GVM requests
1789	*/
1790	case VMMR0_DO_GVMM_CREATE_VM:
1791	if (pGVM == NULL && pVM == NULL && u64Arg == 0 && idCpu == NIL_VMCPUID)
1792	rc = GVMMR0CreateVMReq((PGVMMCREATEVMREQ)pReqHdr, pSession);
1793	else
1794	rc = VERR_INVALID_PARAMETER;
1795	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
1796	break;
1797
1798	case VMMR0_DO_GVMM_DESTROY_VM:
1799	if (pReqHdr == NULL && u64Arg == 0)
1800	rc = GVMMR0DestroyVM(pGVM, pVM);
1801	else
1802	rc = VERR_INVALID_PARAMETER;
1803	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
1804	break;
1805
1806	case VMMR0_DO_GVMM_REGISTER_VMCPU:
1807	if (pGVM != NULL && pVM != NULL)
1808	rc = GVMMR0RegisterVCpu(pGVM, pVM, idCpu);
1809	else
1810	rc = VERR_INVALID_PARAMETER;
1811	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1812	break;
1813
1814	case VMMR0_DO_GVMM_DEREGISTER_VMCPU:
1815	if (pGVM != NULL && pVM != NULL)
1816	rc = GVMMR0DeregisterVCpu(pGVM, pVM, idCpu);
1817	else
1818	rc = VERR_INVALID_PARAMETER;
1819	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1820	break;
1821
1822	case VMMR0_DO_GVMM_SCHED_HALT:
1823	if (pReqHdr)
1824	return VERR_INVALID_PARAMETER;
1825	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1826	rc = GVMMR0SchedHaltReq(pGVM, pVM, idCpu, u64Arg);
1827	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1828	break;
1829
1830	case VMMR0_DO_GVMM_SCHED_WAKE_UP:
1831	if (pReqHdr \|\| u64Arg)
1832	return VERR_INVALID_PARAMETER;
1833	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1834	rc = GVMMR0SchedWakeUp(pGVM, pVM, idCpu);
1835	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1836	break;
1837
1838	case VMMR0_DO_GVMM_SCHED_POKE:
1839	if (pReqHdr \|\| u64Arg)
1840	return VERR_INVALID_PARAMETER;
1841	rc = GVMMR0SchedPoke(pGVM, pVM, idCpu);
1842	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1843	break;
1844
1845	case VMMR0_DO_GVMM_SCHED_WAKE_UP_AND_POKE_CPUS:
1846	if (u64Arg)
1847	return VERR_INVALID_PARAMETER;
1848	rc = GVMMR0SchedWakeUpAndPokeCpusReq(pGVM, pVM, (PGVMMSCHEDWAKEUPANDPOKECPUSREQ)pReqHdr);
1849	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1850	break;
1851
1852	case VMMR0_DO_GVMM_SCHED_POLL:
1853	if (pReqHdr \|\| u64Arg > 1)
1854	return VERR_INVALID_PARAMETER;
1855	rc = GVMMR0SchedPoll(pGVM, pVM, idCpu, !!u64Arg);
1856	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1857	break;
1858
1859	case VMMR0_DO_GVMM_QUERY_STATISTICS:
1860	if (u64Arg)
1861	return VERR_INVALID_PARAMETER;
1862	rc = GVMMR0QueryStatisticsReq(pGVM, pVM, (PGVMMQUERYSTATISTICSSREQ)pReqHdr, pSession);
1863	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1864	break;
1865
1866	case VMMR0_DO_GVMM_RESET_STATISTICS:
1867	if (u64Arg)
1868	return VERR_INVALID_PARAMETER;
1869	rc = GVMMR0ResetStatisticsReq(pGVM, pVM, (PGVMMRESETSTATISTICSSREQ)pReqHdr, pSession);
1870	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1871	break;
1872
1873	/*
1874	* Initialize the R0 part of a VM instance.
1875	*/
1876	case VMMR0_DO_VMMR0_INIT:
1877	rc = vmmR0InitVM(pGVM, pVM, RT_LODWORD(u64Arg), RT_HIDWORD(u64Arg));
1878	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1879	break;
1880
1881	/*
1882	* Does EMT specific ring-0 init.
1883	*/
1884	case VMMR0_DO_VMMR0_INIT_EMT:
1885	rc = vmmR0InitVMEmt(pGVM, pVM, idCpu);
1886	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1887	break;
1888
1889	/*
1890	* Terminate the R0 part of a VM instance.
1891	*/
1892	case VMMR0_DO_VMMR0_TERM:
1893	rc = VMMR0TermVM(pGVM, pVM, 0 /idCpu/);
1894	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1895	break;
1896
1897	/*
1898	* Attempt to enable hm mode and check the current setting.
1899	*/
1900	case VMMR0_DO_HM_ENABLE:
1901	rc = HMR0EnableAllCpus(pVM);
1902	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1903	break;
1904
1905	/*
1906	* Setup the hardware accelerated session.
1907	*/
1908	case VMMR0_DO_HM_SETUP_VM:
1909	rc = HMR0SetupVM(pVM);
1910	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1911	break;
1912
1913	/*
1914	* Switch to RC to execute Hypervisor function.
1915	*/
1916	case VMMR0_DO_CALL_HYPERVISOR:
1917	{
1918	#ifdef VBOX_WITH_RAW_MODE
1919	/*
1920	* Validate input / context.
1921	*/
1922	if (RT_UNLIKELY(idCpu != 0))
1923	return VERR_INVALID_CPU_ID;
1924	if (RT_UNLIKELY(pVM->cCpus != 1))
1925	return VERR_INVALID_PARAMETER;
1926	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1927	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
1928	if (RT_UNLIKELY(!PGMGetHyperCR3(pVCpu)))
1929	return VERR_PGM_NO_CR3_SHADOW_ROOT;
1930	# endif
1931	if (RT_FAILURE(g_rcRawModeUsability))
1932	return g_rcRawModeUsability;
1933
1934	/*
1935	* Disable interrupts.
1936	*/
1937	RTCCUINTREG fFlags = ASMIntDisableFlags();
1938
1939	/*
1940	* Get the host CPU identifiers, make sure they are valid and that
1941	* we've got a TSC delta for the CPU.
1942	*/
1943	RTCPUID idHostCpu;
1944	uint32_t iHostCpuSet = RTMpCurSetIndexAndId(&idHostCpu);
1945	if (RT_UNLIKELY(iHostCpuSet >= RTCPUSET_MAX_CPUS))
1946	{
1947	ASMSetFlags(fFlags);
1948	return VERR_INVALID_CPU_INDEX;
1949	}
1950	if (RT_UNLIKELY(!SUPIsTscDeltaAvailableForCpuSetIndex(iHostCpuSet)))
1951	{
1952	ASMSetFlags(fFlags);
1953	rc = SUPR0TscDeltaMeasureBySetIndex(pVM->pSession, iHostCpuSet, 0 /fFlags/,
1954	2 /cMsWaitRetry/, 5RT_MS_1SEC /cMsWaitThread*/,
1955	0 /default cTries/);
1956	if (RT_FAILURE(rc) && rc != VERR_CPU_OFFLINE)
1957	{
1958	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1959	return rc;
1960	}
1961	}
1962
1963	/*
1964	* Commit the CPU identifiers.
1965	*/
1966	# ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
1967	CPUMR0SetLApic(pVCpu, iHostCpuSet);
1968	# endif
1969	pVCpu->iHostCpuSet = iHostCpuSet;
1970	ASMAtomicWriteU32(&pVCpu->idHostCpu, idHostCpu);
1971
1972	/*
1973	* We might need to disable VT-x if the active switcher turns off paging.
1974	*/
1975	bool fVTxDisabled;
1976	rc = HMR0EnterSwitcher(pVM, pVM->vmm.s.enmSwitcher, &fVTxDisabled);
1977	if (RT_SUCCESS(rc))
1978	{
1979	/*
1980	* Go through the wormhole...
1981	*/
1982	rc = pVM->vmm.s.pfnR0ToRawMode(pVM);
1983
1984	/*
1985	* Re-enable VT-x before we dispatch any pending host interrupts.
1986	*/
1987	HMR0LeaveSwitcher(pVM, fVTxDisabled);
1988
1989	if ( rc == VINF_EM_RAW_INTERRUPT
1990	\|\| rc == VINF_EM_RAW_INTERRUPT_HYPER)
1991	TRPMR0DispatchHostInterrupt(pVM);
1992	}
1993
1994	/*
1995	* Invalidate the host CPU identifiers as we restore interrupts.
1996	*/
1997	pVCpu->iHostCpuSet = UINT32_MAX;
1998	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
1999	ASMSetFlags(fFlags);
2000
2001	#else /* !VBOX_WITH_RAW_MODE */
2002	rc = VERR_RAW_MODE_NOT_SUPPORTED;
2003	#endif
2004	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2005	break;
2006	}
2007
2008	/*
2009	* PGM wrappers.
2010	*/
2011	case VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES:
2012	if (idCpu == NIL_VMCPUID)
2013	return VERR_INVALID_CPU_ID;
2014	rc = PGMR0PhysAllocateHandyPages(pGVM, pVM, idCpu);
2015	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2016	break;
2017
2018	case VMMR0_DO_PGM_FLUSH_HANDY_PAGES:
2019	if (idCpu == NIL_VMCPUID)
2020	return VERR_INVALID_CPU_ID;
2021	rc = PGMR0PhysFlushHandyPages(pGVM, pVM, idCpu);
2022	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2023	break;
2024
2025	case VMMR0_DO_PGM_ALLOCATE_LARGE_HANDY_PAGE:
2026	if (idCpu == NIL_VMCPUID)
2027	return VERR_INVALID_CPU_ID;
2028	rc = PGMR0PhysAllocateLargeHandyPage(pGVM, pVM, idCpu);
2029	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2030	break;
2031
2032	case VMMR0_DO_PGM_PHYS_SETUP_IOMMU:
2033	if (idCpu != 0)
2034	return VERR_INVALID_CPU_ID;
2035	rc = PGMR0PhysSetupIoMmu(pGVM, pVM);
2036	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2037	break;
2038
2039	/*
2040	* GMM wrappers.
2041	*/
2042	case VMMR0_DO_GMM_INITIAL_RESERVATION:
2043	if (u64Arg)
2044	return VERR_INVALID_PARAMETER;
2045	rc = GMMR0InitialReservationReq(pGVM, pVM, idCpu, (PGMMINITIALRESERVATIONREQ)pReqHdr);
2046	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2047	break;
2048
2049	case VMMR0_DO_GMM_UPDATE_RESERVATION:
2050	if (u64Arg)
2051	return VERR_INVALID_PARAMETER;
2052	rc = GMMR0UpdateReservationReq(pGVM, pVM, idCpu, (PGMMUPDATERESERVATIONREQ)pReqHdr);
2053	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2054	break;
2055
2056	case VMMR0_DO_GMM_ALLOCATE_PAGES:
2057	if (u64Arg)
2058	return VERR_INVALID_PARAMETER;
2059	rc = GMMR0AllocatePagesReq(pGVM, pVM, idCpu, (PGMMALLOCATEPAGESREQ)pReqHdr);
2060	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2061	break;
2062
2063	case VMMR0_DO_GMM_FREE_PAGES:
2064	if (u64Arg)
2065	return VERR_INVALID_PARAMETER;
2066	rc = GMMR0FreePagesReq(pGVM, pVM, idCpu, (PGMMFREEPAGESREQ)pReqHdr);
2067	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2068	break;
2069
2070	case VMMR0_DO_GMM_FREE_LARGE_PAGE:
2071	if (u64Arg)
2072	return VERR_INVALID_PARAMETER;
2073	rc = GMMR0FreeLargePageReq(pGVM, pVM, idCpu, (PGMMFREELARGEPAGEREQ)pReqHdr);
2074	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2075	break;
2076
2077	case VMMR0_DO_GMM_QUERY_HYPERVISOR_MEM_STATS:
2078	if (u64Arg)
2079	return VERR_INVALID_PARAMETER;
2080	rc = GMMR0QueryHypervisorMemoryStatsReq((PGMMMEMSTATSREQ)pReqHdr);
2081	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2082	break;
2083
2084	case VMMR0_DO_GMM_QUERY_MEM_STATS:
2085	if (idCpu == NIL_VMCPUID)
2086	return VERR_INVALID_CPU_ID;
2087	if (u64Arg)
2088	return VERR_INVALID_PARAMETER;
2089	rc = GMMR0QueryMemoryStatsReq(pGVM, pVM, idCpu, (PGMMMEMSTATSREQ)pReqHdr);
2090	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2091	break;
2092
2093	case VMMR0_DO_GMM_BALLOONED_PAGES:
2094	if (u64Arg)
2095	return VERR_INVALID_PARAMETER;
2096	rc = GMMR0BalloonedPagesReq(pGVM, pVM, idCpu, (PGMMBALLOONEDPAGESREQ)pReqHdr);
2097	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2098	break;
2099
2100	case VMMR0_DO_GMM_MAP_UNMAP_CHUNK:
2101	if (u64Arg)
2102	return VERR_INVALID_PARAMETER;
2103	rc = GMMR0MapUnmapChunkReq(pGVM, pVM, (PGMMMAPUNMAPCHUNKREQ)pReqHdr);
2104	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2105	break;
2106
2107	case VMMR0_DO_GMM_SEED_CHUNK:
2108	if (pReqHdr)
2109	return VERR_INVALID_PARAMETER;
2110	rc = GMMR0SeedChunk(pGVM, pVM, idCpu, (RTR3PTR)u64Arg);
2111	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2112	break;
2113
2114	case VMMR0_DO_GMM_REGISTER_SHARED_MODULE:
2115	if (idCpu == NIL_VMCPUID)
2116	return VERR_INVALID_CPU_ID;
2117	if (u64Arg)
2118	return VERR_INVALID_PARAMETER;
2119	rc = GMMR0RegisterSharedModuleReq(pGVM, pVM, idCpu, (PGMMREGISTERSHAREDMODULEREQ)pReqHdr);
2120	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2121	break;
2122
2123	case VMMR0_DO_GMM_UNREGISTER_SHARED_MODULE:
2124	if (idCpu == NIL_VMCPUID)
2125	return VERR_INVALID_CPU_ID;
2126	if (u64Arg)
2127	return VERR_INVALID_PARAMETER;
2128	rc = GMMR0UnregisterSharedModuleReq(pGVM, pVM, idCpu, (PGMMUNREGISTERSHAREDMODULEREQ)pReqHdr);
2129	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2130	break;
2131
2132	case VMMR0_DO_GMM_RESET_SHARED_MODULES:
2133	if (idCpu == NIL_VMCPUID)
2134	return VERR_INVALID_CPU_ID;
2135	if ( u64Arg
2136	\|\| pReqHdr)
2137	return VERR_INVALID_PARAMETER;
2138	rc = GMMR0ResetSharedModules(pGVM, pVM, idCpu);
2139	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2140	break;
2141
2142	#ifdef VBOX_WITH_PAGE_SHARING
2143	case VMMR0_DO_GMM_CHECK_SHARED_MODULES:
2144	{
2145	if (idCpu == NIL_VMCPUID)
2146	return VERR_INVALID_CPU_ID;
2147	if ( u64Arg
2148	\|\| pReqHdr)
2149	return VERR_INVALID_PARAMETER;
2150	rc = GMMR0CheckSharedModules(pGVM, pVM, idCpu);
2151	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2152	break;
2153	}
2154	#endif
2155
2156	#if defined(VBOX_STRICT) && HC_ARCH_BITS == 64
2157	case VMMR0_DO_GMM_FIND_DUPLICATE_PAGE:
2158	if (u64Arg)
2159	return VERR_INVALID_PARAMETER;
2160	rc = GMMR0FindDuplicatePageReq(pGVM, pVM, (PGMMFINDDUPLICATEPAGEREQ)pReqHdr);
2161	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2162	break;
2163	#endif
2164
2165	case VMMR0_DO_GMM_QUERY_STATISTICS:
2166	if (u64Arg)
2167	return VERR_INVALID_PARAMETER;
2168	rc = GMMR0QueryStatisticsReq(pGVM, pVM, (PGMMQUERYSTATISTICSSREQ)pReqHdr);
2169	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2170	break;
2171
2172	case VMMR0_DO_GMM_RESET_STATISTICS:
2173	if (u64Arg)
2174	return VERR_INVALID_PARAMETER;
2175	rc = GMMR0ResetStatisticsReq(pGVM, pVM, (PGMMRESETSTATISTICSSREQ)pReqHdr);
2176	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2177	break;
2178
2179	/*
2180	* A quick GCFGM mock-up.
2181	*/
2182	/** @todo GCFGM with proper access control, ring-3 management interface and all that. */
2183	case VMMR0_DO_GCFGM_SET_VALUE:
2184	case VMMR0_DO_GCFGM_QUERY_VALUE:
2185	{
2186	if (pGVM \|\| pVM \|\| !pReqHdr \|\| u64Arg \|\| idCpu != NIL_VMCPUID)
2187	return VERR_INVALID_PARAMETER;
2188	PGCFGMVALUEREQ pReq = (PGCFGMVALUEREQ)pReqHdr;
2189	if (pReq->Hdr.cbReq != sizeof(*pReq))
2190	return VERR_INVALID_PARAMETER;
2191	if (enmOperation == VMMR0_DO_GCFGM_SET_VALUE)
2192	{
2193	rc = GVMMR0SetConfig(pReq->pSession, &pReq->szName[0], pReq->u64Value);
2194	//if (rc == VERR_CFGM_VALUE_NOT_FOUND)
2195	// rc = GMMR0SetConfig(pReq->pSession, &pReq->szName[0], pReq->u64Value);
2196	}
2197	else
2198	{
2199	rc = GVMMR0QueryConfig(pReq->pSession, &pReq->szName[0], &pReq->u64Value);
2200	//if (rc == VERR_CFGM_VALUE_NOT_FOUND)
2201	// rc = GMMR0QueryConfig(pReq->pSession, &pReq->szName[0], &pReq->u64Value);
2202	}
2203	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2204	break;
2205	}
2206
2207	/*
2208	* PDM Wrappers.
2209	*/
2210	case VMMR0_DO_PDM_DRIVER_CALL_REQ_HANDLER:
2211	{
2212	if (!pReqHdr \|\| u64Arg \|\| idCpu != NIL_VMCPUID)
2213	return VERR_INVALID_PARAMETER;
2214	rc = PDMR0DriverCallReqHandler(pGVM, pVM, (PPDMDRIVERCALLREQHANDLERREQ)pReqHdr);
2215	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2216	break;
2217	}
2218
2219	case VMMR0_DO_PDM_DEVICE_CALL_REQ_HANDLER:
2220	{
2221	if (!pReqHdr \|\| u64Arg \|\| idCpu != NIL_VMCPUID)
2222	return VERR_INVALID_PARAMETER;
2223	rc = PDMR0DeviceCallReqHandler(pGVM, pVM, (PPDMDEVICECALLREQHANDLERREQ)pReqHdr);
2224	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2225	break;
2226	}
2227
2228	/*
2229	* Requests to the internal networking service.
2230	*/
2231	case VMMR0_DO_INTNET_OPEN:
2232	{
2233	PINTNETOPENREQ pReq = (PINTNETOPENREQ)pReqHdr;
2234	if (u64Arg \|\| !pReq \|\| !vmmR0IsValidSession(pVM, pReq->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2235	return VERR_INVALID_PARAMETER;
2236	rc = IntNetR0OpenReq(pSession, pReq);
2237	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2238	break;
2239	}
2240
2241	case VMMR0_DO_INTNET_IF_CLOSE:
2242	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFCLOSEREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2243	return VERR_INVALID_PARAMETER;
2244	rc = IntNetR0IfCloseReq(pSession, (PINTNETIFCLOSEREQ)pReqHdr);
2245	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2246	break;
2247
2248
2249	case VMMR0_DO_INTNET_IF_GET_BUFFER_PTRS:
2250	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFGETBUFFERPTRSREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2251	return VERR_INVALID_PARAMETER;
2252	rc = IntNetR0IfGetBufferPtrsReq(pSession, (PINTNETIFGETBUFFERPTRSREQ)pReqHdr);
2253	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2254	break;
2255
2256	case VMMR0_DO_INTNET_IF_SET_PROMISCUOUS_MODE:
2257	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFSETPROMISCUOUSMODEREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2258	return VERR_INVALID_PARAMETER;
2259	rc = IntNetR0IfSetPromiscuousModeReq(pSession, (PINTNETIFSETPROMISCUOUSMODEREQ)pReqHdr);
2260	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2261	break;
2262
2263	case VMMR0_DO_INTNET_IF_SET_MAC_ADDRESS:
2264	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFSETMACADDRESSREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2265	return VERR_INVALID_PARAMETER;
2266	rc = IntNetR0IfSetMacAddressReq(pSession, (PINTNETIFSETMACADDRESSREQ)pReqHdr);
2267	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2268	break;
2269
2270	case VMMR0_DO_INTNET_IF_SET_ACTIVE:
2271	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFSETACTIVEREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2272	return VERR_INVALID_PARAMETER;
2273	rc = IntNetR0IfSetActiveReq(pSession, (PINTNETIFSETACTIVEREQ)pReqHdr);
2274	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2275	break;
2276
2277	case VMMR0_DO_INTNET_IF_SEND:
2278	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFSENDREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2279	return VERR_INVALID_PARAMETER;
2280	rc = IntNetR0IfSendReq(pSession, (PINTNETIFSENDREQ)pReqHdr);
2281	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2282	break;
2283
2284	case VMMR0_DO_INTNET_IF_WAIT:
2285	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFWAITREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2286	return VERR_INVALID_PARAMETER;
2287	rc = IntNetR0IfWaitReq(pSession, (PINTNETIFWAITREQ)pReqHdr);
2288	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2289	break;
2290
2291	case VMMR0_DO_INTNET_IF_ABORT_WAIT:
2292	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFWAITREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2293	return VERR_INVALID_PARAMETER;
2294	rc = IntNetR0IfAbortWaitReq(pSession, (PINTNETIFABORTWAITREQ)pReqHdr);
2295	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2296	break;
2297
2298	#ifdef VBOX_WITH_PCI_PASSTHROUGH
2299	/*
2300	* Requests to host PCI driver service.
2301	*/
2302	case VMMR0_DO_PCIRAW_REQ:
2303	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PPCIRAWSENDREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2304	return VERR_INVALID_PARAMETER;
2305	rc = PciRawR0ProcessReq(pGVM, pVM, pSession, (PPCIRAWSENDREQ)pReqHdr);
2306	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2307	break;
2308	#endif
2309
2310	/*
2311	* NEM requests.
2312	*/
2313	#ifdef VBOX_WITH_NEM_R0
2314	# if defined(RT_ARCH_AMD64) && defined(RT_OS_WINDOWS)
2315	case VMMR0_DO_NEM_INIT_VM:
2316	if (u64Arg \|\| pReqHdr \|\| idCpu != 0)
2317	return VERR_INVALID_PARAMETER;
2318	rc = NEMR0InitVM(pGVM, pVM);
2319	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2320	break;
2321
2322	case VMMR0_DO_NEM_INIT_VM_PART_2:
2323	if (u64Arg \|\| pReqHdr \|\| idCpu != 0)
2324	return VERR_INVALID_PARAMETER;
2325	rc = NEMR0InitVMPart2(pGVM, pVM);
2326	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2327	break;
2328
2329	case VMMR0_DO_NEM_MAP_PAGES:
2330	if (u64Arg \|\| pReqHdr \|\| idCpu == NIL_VMCPUID)
2331	return VERR_INVALID_PARAMETER;
2332	rc = NEMR0MapPages(pGVM, pVM, idCpu);
2333	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2334	break;
2335
2336	case VMMR0_DO_NEM_UNMAP_PAGES:
2337	if (u64Arg \|\| pReqHdr \|\| idCpu == NIL_VMCPUID)
2338	return VERR_INVALID_PARAMETER;
2339	rc = NEMR0UnmapPages(pGVM, pVM, idCpu);
2340	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2341	break;
2342
2343	case VMMR0_DO_NEM_EXPORT_STATE:
2344	if (u64Arg \|\| pReqHdr \|\| idCpu == NIL_VMCPUID)
2345	return VERR_INVALID_PARAMETER;
2346	rc = NEMR0ExportState(pGVM, pVM, idCpu);
2347	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2348	break;
2349
2350	case VMMR0_DO_NEM_IMPORT_STATE:
2351	if (pReqHdr \|\| idCpu == NIL_VMCPUID)
2352	return VERR_INVALID_PARAMETER;
2353	rc = NEMR0ImportState(pGVM, pVM, idCpu, u64Arg);
2354	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2355	break;
2356
2357	case VMMR0_DO_NEM_QUERY_CPU_TICK:
2358	if (u64Arg \|\| pReqHdr \|\| idCpu == NIL_VMCPUID)
2359	return VERR_INVALID_PARAMETER;
2360	rc = NEMR0QueryCpuTick(pGVM, pVM, idCpu);
2361	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2362	break;
2363
2364	case VMMR0_DO_NEM_RESUME_CPU_TICK_ON_ALL:
2365	if (pReqHdr \|\| idCpu == NIL_VMCPUID)
2366	return VERR_INVALID_PARAMETER;
2367	rc = NEMR0ResumeCpuTickOnAll(pGVM, pVM, idCpu, u64Arg);
2368	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2369	break;
2370
2371	case VMMR0_DO_NEM_UPDATE_STATISTICS:
2372	if (u64Arg \|\| pReqHdr)
2373	return VERR_INVALID_PARAMETER;
2374	rc = NEMR0UpdateStatistics(pGVM, pVM, idCpu);
2375	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2376	break;
2377
2378	# if 1 && defined(DEBUG_bird)
2379	case VMMR0_DO_NEM_EXPERIMENT:
2380	if (pReqHdr)
2381	return VERR_INVALID_PARAMETER;
2382	rc = NEMR0DoExperiment(pGVM, pVM, idCpu, u64Arg);
2383	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2384	break;
2385	# endif
2386	# endif
2387	#endif
2388
2389	/*
2390	* For profiling.
2391	*/
2392	case VMMR0_DO_NOP:
2393	case VMMR0_DO_SLOW_NOP:
2394	return VINF_SUCCESS;
2395
2396	/*
2397	* For testing Ring-0 APIs invoked in this environment.
2398	*/
2399	case VMMR0_DO_TESTS:
2400	/** @todo make new test */
2401	return VINF_SUCCESS;
2402
2403
2404	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
2405	case VMMR0_DO_TEST_SWITCHER3264:
2406	if (idCpu == NIL_VMCPUID)
2407	return VERR_INVALID_CPU_ID;
2408	rc = HMR0TestSwitcher3264(pVM);
2409	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2410	break;
2411	#endif
2412	default:
2413	/*
2414	* We're returning VERR_NOT_SUPPORT here so we've got something else
2415	* than -1 which the interrupt gate glue code might return.
2416	*/
2417	Log(("operation %#x is not supported\n", enmOperation));
2418	return VERR_NOT_SUPPORTED;
2419	}
2420	return rc;
2421	}
2422
2423
2424	/**
2425	* Argument for vmmR0EntryExWrapper containing the arguments for VMMR0EntryEx.
2426	*/
2427	typedef struct VMMR0ENTRYEXARGS
2428	{
2429	PGVM pGVM;
2430	PVM pVM;
2431	VMCPUID idCpu;
2432	VMMR0OPERATION enmOperation;
2433	PSUPVMMR0REQHDR pReq;
2434	uint64_t u64Arg;
2435	PSUPDRVSESSION pSession;
2436	} VMMR0ENTRYEXARGS;
2437	/** Pointer to a vmmR0EntryExWrapper argument package. */
2438	typedef VMMR0ENTRYEXARGS *PVMMR0ENTRYEXARGS;
2439
2440	/**
2441	* This is just a longjmp wrapper function for VMMR0EntryEx calls.
2442	*
2443	* @returns VBox status code.
2444	* @param pvArgs The argument package
2445	*/
2446	static DECLCALLBACK(int) vmmR0EntryExWrapper(void *pvArgs)
2447	{
2448	return vmmR0EntryExWorker(((PVMMR0ENTRYEXARGS)pvArgs)->pGVM,
2449	((PVMMR0ENTRYEXARGS)pvArgs)->pVM,
2450	((PVMMR0ENTRYEXARGS)pvArgs)->idCpu,
2451	((PVMMR0ENTRYEXARGS)pvArgs)->enmOperation,
2452	((PVMMR0ENTRYEXARGS)pvArgs)->pReq,
2453	((PVMMR0ENTRYEXARGS)pvArgs)->u64Arg,
2454	((PVMMR0ENTRYEXARGS)pvArgs)->pSession);
2455	}
2456
2457
2458	/**
2459	* The Ring 0 entry point, called by the support library (SUP).
2460	*
2461	* @returns VBox status code.
2462	* @param pGVM The global (ring-0) VM structure.
2463	* @param pVM The cross context VM structure.
2464	* @param idCpu Virtual CPU ID argument. Must be NIL_VMCPUID if pVM
2465	* is NIL_RTR0PTR, and may be NIL_VMCPUID if it isn't
2466	* @param enmOperation Which operation to execute.
2467	* @param pReq Pointer to the SUPVMMR0REQHDR packet. Optional.
2468	* @param u64Arg Some simple constant argument.
2469	* @param pSession The session of the caller.
2470	* @remarks Assume called with interrupts _enabled_.
2471	*/
2472	VMMR0DECL(int) VMMR0EntryEx(PGVM pGVM, PVM pVM, VMCPUID idCpu, VMMR0OPERATION enmOperation,
2473	PSUPVMMR0REQHDR pReq, uint64_t u64Arg, PSUPDRVSESSION pSession)
2474	{
2475	/*
2476	* Requests that should only happen on the EMT thread will be
2477	* wrapped in a setjmp so we can assert without causing trouble.
2478	*/
2479	if ( pVM != NULL
2480	&& pGVM != NULL
2481	&& idCpu < pGVM->cCpus
2482	&& pVM->pVMR0 != NULL)
2483	{
2484	switch (enmOperation)
2485	{
2486	/* These might/will be called before VMMR3Init. */
2487	case VMMR0_DO_GMM_INITIAL_RESERVATION:
2488	case VMMR0_DO_GMM_UPDATE_RESERVATION:
2489	case VMMR0_DO_GMM_ALLOCATE_PAGES:
2490	case VMMR0_DO_GMM_FREE_PAGES:
2491	case VMMR0_DO_GMM_BALLOONED_PAGES:
2492	/* On the mac we might not have a valid jmp buf, so check these as well. */
2493	case VMMR0_DO_VMMR0_INIT:
2494	case VMMR0_DO_VMMR0_TERM:
2495	{
2496	PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
2497	PVMCPU pVCpu = &pVM->aCpus[idCpu];
2498	RTNATIVETHREAD hNativeThread = RTThreadNativeSelf();
2499	if (RT_LIKELY( pGVCpu->hEMT == hNativeThread
2500	&& pVCpu->hNativeThreadR0 == hNativeThread))
2501	{
2502	if (!pVCpu->vmm.s.CallRing3JmpBufR0.pvSavedStack)
2503	break;
2504
2505	/** @todo validate this EMT claim... GVM knows. */
2506	VMMR0ENTRYEXARGS Args;
2507	Args.pGVM = pGVM;
2508	Args.pVM = pVM;
2509	Args.idCpu = idCpu;
2510	Args.enmOperation = enmOperation;
2511	Args.pReq = pReq;
2512	Args.u64Arg = u64Arg;
2513	Args.pSession = pSession;
2514	return vmmR0CallRing3SetJmpEx(&pVCpu->vmm.s.CallRing3JmpBufR0, vmmR0EntryExWrapper, &Args);
2515	}
2516	return VERR_VM_THREAD_NOT_EMT;
2517	}
2518
2519	default:
2520	break;
2521	}
2522	}
2523	return vmmR0EntryExWorker(pGVM, pVM, idCpu, enmOperation, pReq, u64Arg, pSession);
2524	}
2525
2526
2527	/**
2528	* Checks whether we've armed the ring-0 long jump machinery.
2529	*
2530	* @returns @c true / @c false
2531	* @param pVCpu The cross context virtual CPU structure.
2532	* @thread EMT
2533	* @sa VMMIsLongJumpArmed
2534	*/
2535	VMMR0_INT_DECL(bool) VMMR0IsLongJumpArmed(PVMCPU pVCpu)
2536	{
2537	#ifdef RT_ARCH_X86
2538	return pVCpu->vmm.s.CallRing3JmpBufR0.eip
2539	&& !pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call;
2540	#else
2541	return pVCpu->vmm.s.CallRing3JmpBufR0.rip
2542	&& !pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call;
2543	#endif
2544	}
2545
2546
2547	/**
2548	* Checks whether we've done a ring-3 long jump.
2549	*
2550	* @returns @c true / @c false
2551	* @param pVCpu The cross context virtual CPU structure.
2552	* @thread EMT
2553	*/
2554	VMMR0_INT_DECL(bool) VMMR0IsInRing3LongJump(PVMCPU pVCpu)
2555	{
2556	return pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call;
2557	}
2558
2559
2560	/**
2561	* Internal R0 logger worker: Flush logger.
2562	*
2563	* @param pLogger The logger instance to flush.
2564	* @remark This function must be exported!
2565	*/
2566	VMMR0DECL(void) vmmR0LoggerFlush(PRTLOGGER pLogger)
2567	{
2568	#ifdef LOG_ENABLED
2569	/*
2570	* Convert the pLogger into a VM handle and 'call' back to Ring-3.
2571	* (This is a bit paranoid code.)
2572	*/
2573	PVMMR0LOGGER pR0Logger = (PVMMR0LOGGER)((uintptr_t)pLogger - RT_UOFFSETOF(VMMR0LOGGER, Logger));
2574	if ( !VALID_PTR(pR0Logger)
2575	\|\| !VALID_PTR(pR0Logger + 1)
2576	\|\| pLogger->u32Magic != RTLOGGER_MAGIC)
2577	{
2578	# ifdef DEBUG
2579	SUPR0Printf("vmmR0LoggerFlush: pLogger=%p!\n", pLogger);
2580	# endif
2581	return;
2582	}
2583	if (pR0Logger->fFlushingDisabled)
2584	return; /* quietly */
2585
2586	PVM pVM = pR0Logger->pVM;
2587	if ( !VALID_PTR(pVM)
2588	\|\| pVM->pVMR0 != pVM)
2589	{
2590	# ifdef DEBUG
2591	SUPR0Printf("vmmR0LoggerFlush: pVM=%p! pVMR0=%p! pLogger=%p\n", pVM, pVM->pVMR0, pLogger);
2592	# endif
2593	return;
2594	}
2595
2596	PVMCPU pVCpu = VMMGetCpu(pVM);
2597	if (pVCpu)
2598	{
2599	/*
2600	* Check that the jump buffer is armed.
2601	*/
2602	# ifdef RT_ARCH_X86
2603	if ( !pVCpu->vmm.s.CallRing3JmpBufR0.eip
2604	\|\| pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call)
2605	# else
2606	if ( !pVCpu->vmm.s.CallRing3JmpBufR0.rip
2607	\|\| pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call)
2608	# endif
2609	{
2610	# ifdef DEBUG
2611	SUPR0Printf("vmmR0LoggerFlush: Jump buffer isn't armed!\n");
2612	# endif
2613	return;
2614	}
2615	VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_VMM_LOGGER_FLUSH, 0);
2616	}
2617	# ifdef DEBUG
2618	else
2619	SUPR0Printf("vmmR0LoggerFlush: invalid VCPU context!\n");
2620	# endif
2621	#else
2622	NOREF(pLogger);
2623	#endif /* LOG_ENABLED */
2624	}
2625
2626	#ifdef LOG_ENABLED
2627
2628	/**
2629	* Disables flushing of the ring-0 debug log.
2630	*
2631	* @param pVCpu The cross context virtual CPU structure.
2632	*/
2633	VMMR0_INT_DECL(void) VMMR0LogFlushDisable(PVMCPU pVCpu)
2634	{
2635	if (pVCpu->vmm.s.pR0LoggerR0)
2636	pVCpu->vmm.s.pR0LoggerR0->fFlushingDisabled = true;
2637	if (pVCpu->vmm.s.pR0RelLoggerR0)
2638	pVCpu->vmm.s.pR0RelLoggerR0->fFlushingDisabled = true;
2639	}
2640
2641
2642	/**
2643	* Enables flushing of the ring-0 debug log.
2644	*
2645	* @param pVCpu The cross context virtual CPU structure.
2646	*/
2647	VMMR0_INT_DECL(void) VMMR0LogFlushEnable(PVMCPU pVCpu)
2648	{
2649	if (pVCpu->vmm.s.pR0LoggerR0)
2650	pVCpu->vmm.s.pR0LoggerR0->fFlushingDisabled = false;
2651	if (pVCpu->vmm.s.pR0RelLoggerR0)
2652	pVCpu->vmm.s.pR0RelLoggerR0->fFlushingDisabled = false;
2653	}
2654
2655
2656	/**
2657	* Checks if log flushing is disabled or not.
2658	*
2659	* @param pVCpu The cross context virtual CPU structure.
2660	*/
2661	VMMR0_INT_DECL(bool) VMMR0IsLogFlushDisabled(PVMCPU pVCpu)
2662	{
2663	if (pVCpu->vmm.s.pR0LoggerR0)
2664	return pVCpu->vmm.s.pR0LoggerR0->fFlushingDisabled;
2665	if (pVCpu->vmm.s.pR0RelLoggerR0)
2666	return pVCpu->vmm.s.pR0RelLoggerR0->fFlushingDisabled;
2667	return true;
2668	}
2669
2670	#endif /* LOG_ENABLED */
2671
2672	/**
2673	* Override RTLogRelGetDefaultInstanceEx so we can do LogRel to VBox.log from EMTs in ring-0.
2674	*/
2675	DECLEXPORT(PRTLOGGER) RTLogRelGetDefaultInstanceEx(uint32_t fFlagsAndGroup)
2676	{
2677	PGVMCPU pGVCpu = GVMMR0GetGVCpuByEMT(NIL_RTNATIVETHREAD);
2678	if (pGVCpu)
2679	{
2680	PVMCPU pVCpu = pGVCpu->pVCpu;
2681	if (RT_VALID_PTR(pVCpu))
2682	{
2683	PVMMR0LOGGER pVmmLogger = pVCpu->vmm.s.pR0RelLoggerR0;
2684	if (RT_VALID_PTR(pVmmLogger))
2685	{
2686	if ( pVmmLogger->fCreated
2687	&& pVmmLogger->pVM == pGVCpu->pVM)
2688	{
2689	if (pVmmLogger->Logger.fFlags & RTLOGFLAGS_DISABLED)
2690	return NULL;
2691	uint16_t const fFlags = RT_LO_U16(fFlagsAndGroup);
2692	uint16_t const iGroup = RT_HI_U16(fFlagsAndGroup);
2693	if ( iGroup != UINT16_MAX
2694	&& ( ( pVmmLogger->Logger.afGroups[iGroup < pVmmLogger->Logger.cGroups ? iGroup : 0]
2695	& (fFlags \| (uint32_t)RTLOGGRPFLAGS_ENABLED))
2696	!= (fFlags \| (uint32_t)RTLOGGRPFLAGS_ENABLED)))
2697	return NULL;
2698	return &pVmmLogger->Logger;
2699	}
2700	}
2701	}
2702	}
2703	return SUPR0GetDefaultLogRelInstanceEx(fFlagsAndGroup);
2704	}
2705
2706
2707	/**
2708	* Jump back to ring-3 if we're the EMT and the longjmp is armed.
2709	*
2710	* @returns true if the breakpoint should be hit, false if it should be ignored.
2711	*/
2712	DECLEXPORT(bool) RTCALL RTAssertShouldPanic(void)
2713	{
2714	#if 0
2715	return true;
2716	#else
2717	PVM pVM = GVMMR0GetVMByEMT(NIL_RTNATIVETHREAD);
2718	if (pVM)
2719	{
2720	PVMCPU pVCpu = VMMGetCpu(pVM);
2721
2722	if (pVCpu)
2723	{
2724	#ifdef RT_ARCH_X86
2725	if ( pVCpu->vmm.s.CallRing3JmpBufR0.eip
2726	&& !pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call)
2727	#else
2728	if ( pVCpu->vmm.s.CallRing3JmpBufR0.rip
2729	&& !pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call)
2730	#endif
2731	{
2732	int rc = VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_VM_R0_ASSERTION, 0);
2733	return RT_FAILURE_NP(rc);
2734	}
2735	}
2736	}
2737	#ifdef RT_OS_LINUX
2738	return true;
2739	#else
2740	return false;
2741	#endif
2742	#endif
2743	}
2744
2745
2746	/**
2747	* Override this so we can push it up to ring-3.
2748	*
2749	* @param pszExpr Expression. Can be NULL.
2750	* @param uLine Location line number.
2751	* @param pszFile Location file name.
2752	* @param pszFunction Location function name.
2753	*/
2754	DECLEXPORT(void) RTCALL RTAssertMsg1Weak(const char pszExpr, unsigned uLine, const char pszFile, const char *pszFunction)
2755	{
2756	/*
2757	* To the log.
2758	*/
2759	LogAlways(("\n!!R0-Assertion Failed!!\n"
2760	"Expression: %s\n"
2761	"Location : %s(%d) %s\n",
2762	pszExpr, pszFile, uLine, pszFunction));
2763
2764	/*
2765	* To the global VMM buffer.
2766	*/
2767	PVM pVM = GVMMR0GetVMByEMT(NIL_RTNATIVETHREAD);
2768	if (pVM)
2769	RTStrPrintf(pVM->vmm.s.szRing0AssertMsg1, sizeof(pVM->vmm.s.szRing0AssertMsg1),
2770	"\n!!R0-Assertion Failed!!\n"
2771	"Expression: %.*s\n"
2772	"Location : %s(%d) %s\n",
2773	sizeof(pVM->vmm.s.szRing0AssertMsg1) / 4 * 3, pszExpr,
2774	pszFile, uLine, pszFunction);
2775
2776	/*
2777	* Continue the normal way.
2778	*/
2779	RTAssertMsg1(pszExpr, uLine, pszFile, pszFunction);
2780	}
2781
2782
2783	/**
2784	* Callback for RTLogFormatV which writes to the ring-3 log port.
2785	* See PFNLOGOUTPUT() for details.
2786	*/
2787	static DECLCALLBACK(size_t) rtLogOutput(void pv, const char pachChars, size_t cbChars)
2788	{
2789	for (size_t i = 0; i < cbChars; i++)
2790	{
2791	LogAlways(("%c", pachChars[i])); NOREF(pachChars);
2792	}
2793
2794	NOREF(pv);
2795	return cbChars;
2796	}
2797
2798
2799	/**
2800	* Override this so we can push it up to ring-3.
2801	*
2802	* @param pszFormat The format string.
2803	* @param va Arguments.
2804	*/
2805	DECLEXPORT(void) RTCALL RTAssertMsg2WeakV(const char *pszFormat, va_list va)
2806	{
2807	va_list vaCopy;
2808
2809	/*
2810	* Push the message to the loggers.
2811	*/
2812	PRTLOGGER pLog = RTLogGetDefaultInstance(); /* Don't initialize it here... */
2813	if (pLog)
2814	{
2815	va_copy(vaCopy, va);
2816	RTLogFormatV(rtLogOutput, pLog, pszFormat, vaCopy);
2817	va_end(vaCopy);
2818	}
2819	pLog = RTLogRelGetDefaultInstance();
2820	if (pLog)
2821	{
2822	va_copy(vaCopy, va);
2823	RTLogFormatV(rtLogOutput, pLog, pszFormat, vaCopy);
2824	va_end(vaCopy);
2825	}
2826
2827	/*
2828	* Push it to the global VMM buffer.
2829	*/
2830	PVM pVM = GVMMR0GetVMByEMT(NIL_RTNATIVETHREAD);
2831	if (pVM)
2832	{
2833	va_copy(vaCopy, va);
2834	RTStrPrintfV(pVM->vmm.s.szRing0AssertMsg2, sizeof(pVM->vmm.s.szRing0AssertMsg2), pszFormat, vaCopy);
2835	va_end(vaCopy);
2836	}
2837
2838	/*
2839	* Continue the normal way.
2840	*/
2841	RTAssertMsg2V(pszFormat, va);
2842	}
2843

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/VMMR0.cpp@ 75646

Download in other formats: