VMMR0.cpp@ 72747

Last change on this file since 72747 was 72642, checked in by vboxsync, 6 years ago
EM,IEM,VMX: Working on configuring exit history optimziations. Currently enabled in ring-0 for NEM but disabled for HM. bugref:9198
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 90.1 KB

Line
1	/* $Id: VMMR0.cpp 72642 2018-06-21 15:41:14Z vboxsync $ */
2	/** @file
3	* VMM - Host Context Ring 0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2017 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_VMM
23	#include <VBox/vmm/vmm.h>
24	#include <VBox/sup.h>
25	#include <VBox/vmm/trpm.h>
26	#include <VBox/vmm/cpum.h>
27	#include <VBox/vmm/pdmapi.h>
28	#include <VBox/vmm/pgm.h>
29	#ifdef VBOX_WITH_NEM_R0
30	# include <VBox/vmm/nem.h>
31	#endif
32	#include <VBox/vmm/em.h>
33	#include <VBox/vmm/stam.h>
34	#include <VBox/vmm/tm.h>
35	#include "VMMInternal.h"
36	#include <VBox/vmm/vm.h>
37	#include <VBox/vmm/gvm.h>
38	#ifdef VBOX_WITH_PCI_PASSTHROUGH
39	# include <VBox/vmm/pdmpci.h>
40	#endif
41	#include <VBox/vmm/apic.h>
42
43	#include <VBox/vmm/gvmm.h>
44	#include <VBox/vmm/gmm.h>
45	#include <VBox/vmm/gim.h>
46	#include <VBox/intnet.h>
47	#include <VBox/vmm/hm.h>
48	#include <VBox/param.h>
49	#include <VBox/err.h>
50	#include <VBox/version.h>
51	#include <VBox/log.h>
52
53	#include <iprt/asm-amd64-x86.h>
54	#include <iprt/assert.h>
55	#include <iprt/crc.h>
56	#include <iprt/mp.h>
57	#include <iprt/once.h>
58	#include <iprt/stdarg.h>
59	#include <iprt/string.h>
60	#include <iprt/thread.h>
61	#include <iprt/timer.h>
62
63	#include "dtrace/VBoxVMM.h"
64
65
66	#if defined(_MSC_VER) && defined(RT_ARCH_AMD64) /** @todo check this with with VC7! */
67	# pragma intrinsic(_AddressOfReturnAddress)
68	#endif
69
70	#if defined(RT_OS_DARWIN) && ARCH_BITS == 32
71	# error "32-bit darwin is no longer supported. Go back to 4.3 or earlier!"
72	#endif
73
74
75
76	/*********************************************************************************************************************************
77	* Defined Constants And Macros *
78	*********************************************************************************************************************************/
79	/** @def VMM_CHECK_SMAP_SETUP
80	* SMAP check setup. */
81	/** @def VMM_CHECK_SMAP_CHECK
82	* Checks that the AC flag is set if SMAP is enabled. If AC is not set,
83	* it will be logged and @a a_BadExpr is executed. */
84	/** @def VMM_CHECK_SMAP_CHECK2
85	* Checks that the AC flag is set if SMAP is enabled. If AC is not set, it will
86	* be logged, written to the VMs assertion text buffer, and @a a_BadExpr is
87	* executed. */
88	#if defined(VBOX_STRICT) \|\| 1
89	# define VMM_CHECK_SMAP_SETUP() uint32_t const fKernelFeatures = SUPR0GetKernelFeatures()
90	# define VMM_CHECK_SMAP_CHECK(a_BadExpr) \
91	do { \
92	if (fKernelFeatures & SUPKERNELFEATURES_SMAP) \
93	{ \
94	RTCCUINTREG fEflCheck = ASMGetFlags(); \
95	if (RT_LIKELY(fEflCheck & X86_EFL_AC)) \
96	{ /* likely */ } \
97	else \
98	{ \
99	SUPR0Printf("%s, line %d: EFLAGS.AC is clear! (%#x)\n", __FUNCTION__, __LINE__, (uint32_t)fEflCheck); \
100	a_BadExpr; \
101	} \
102	} \
103	} while (0)
104	# define VMM_CHECK_SMAP_CHECK2(a_pVM, a_BadExpr) \
105	do { \
106	if (fKernelFeatures & SUPKERNELFEATURES_SMAP) \
107	{ \
108	RTCCUINTREG fEflCheck = ASMGetFlags(); \
109	if (RT_LIKELY(fEflCheck & X86_EFL_AC)) \
110	{ /* likely */ } \
111	else \
112	{ \
113	SUPR0BadContext((a_pVM) ? (a_pVM)->pSession : NULL, __FILE__, __LINE__, "EFLAGS.AC is zero!"); \
114	RTStrPrintf(pVM->vmm.s.szRing0AssertMsg1, sizeof(pVM->vmm.s.szRing0AssertMsg1), \
115	"%s, line %d: EFLAGS.AC is clear! (%#x)\n", __FUNCTION__, __LINE__, (uint32_t)fEflCheck); \
116	a_BadExpr; \
117	} \
118	} \
119	} while (0)
120	#else
121	# define VMM_CHECK_SMAP_SETUP() uint32_t const fKernelFeatures = 0
122	# define VMM_CHECK_SMAP_CHECK(a_BadExpr) NOREF(fKernelFeatures)
123	# define VMM_CHECK_SMAP_CHECK2(a_pVM, a_BadExpr) NOREF(fKernelFeatures)
124	#endif
125
126
127	/*********************************************************************************************************************************
128	* Internal Functions *
129	*********************************************************************************************************************************/
130	RT_C_DECLS_BEGIN
131	#if defined(RT_ARCH_X86) && (defined(RT_OS_SOLARIS) \|\| defined(RT_OS_FREEBSD))
132	extern uint64_t __udivdi3(uint64_t, uint64_t);
133	extern uint64_t __umoddi3(uint64_t, uint64_t);
134	#endif
135	RT_C_DECLS_END
136
137
138	/*********************************************************************************************************************************
139	* Global Variables *
140	*********************************************************************************************************************************/
141	/** Drag in necessary library bits.
142	* The runtime lives here (in VMMR0.r0) and VBoxDDR0.r0 links against us. /
143	PFNRT g_VMMR0Deps[] =
144	{
145	(PFNRT)RTCrc32,
146	(PFNRT)RTOnce,
147	#if defined(RT_ARCH_X86) && (defined(RT_OS_SOLARIS) \|\| defined(RT_OS_FREEBSD))
148	(PFNRT)__udivdi3,
149	(PFNRT)__umoddi3,
150	#endif
151	NULL
152	};
153
154	#ifdef RT_OS_SOLARIS
155	/* Dependency information for the native solaris loader. */
156	extern "C" { char _depends_on[] = "vboxdrv"; }
157	#endif
158
159	/** The result of SUPR0GetRawModeUsability(), set by ModuleInit(). */
160	int g_rcRawModeUsability = VINF_SUCCESS;
161
162
163	/**
164	* Initialize the module.
165	* This is called when we're first loaded.
166	*
167	* @returns 0 on success.
168	* @returns VBox status on failure.
169	* @param hMod Image handle for use in APIs.
170	*/
171	DECLEXPORT(int) ModuleInit(void *hMod)
172	{
173	VMM_CHECK_SMAP_SETUP();
174	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
175
176	#ifdef VBOX_WITH_DTRACE_R0
177	/*
178	* The first thing to do is register the static tracepoints.
179	* (Deregistration is automatic.)
180	*/
181	int rc2 = SUPR0TracerRegisterModule(hMod, &g_VTGObjHeader);
182	if (RT_FAILURE(rc2))
183	return rc2;
184	#endif
185	LogFlow(("ModuleInit:\n"));
186
187	#ifdef VBOX_WITH_64ON32_CMOS_DEBUG
188	/*
189	* Display the CMOS debug code.
190	*/
191	ASMOutU8(0x72, 0x03);
192	uint8_t bDebugCode = ASMInU8(0x73);
193	LogRel(("CMOS Debug Code: %#x (%d)\n", bDebugCode, bDebugCode));
194	RTLogComPrintf("CMOS Debug Code: %#x (%d)\n", bDebugCode, bDebugCode);
195	#endif
196
197	/*
198	* Initialize the VMM, GVMM, GMM, HM, PGM (Darwin) and INTNET.
199	*/
200	int rc = vmmInitFormatTypes();
201	if (RT_SUCCESS(rc))
202	{
203	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
204	rc = GVMMR0Init();
205	if (RT_SUCCESS(rc))
206	{
207	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
208	rc = GMMR0Init();
209	if (RT_SUCCESS(rc))
210	{
211	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
212	rc = HMR0Init();
213	if (RT_SUCCESS(rc))
214	{
215	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
216	rc = PGMRegisterStringFormatTypes();
217	if (RT_SUCCESS(rc))
218	{
219	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
220	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
221	rc = PGMR0DynMapInit();
222	#endif
223	if (RT_SUCCESS(rc))
224	{
225	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
226	rc = IntNetR0Init();
227	if (RT_SUCCESS(rc))
228	{
229	#ifdef VBOX_WITH_PCI_PASSTHROUGH
230	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
231	rc = PciRawR0Init();
232	#endif
233	if (RT_SUCCESS(rc))
234	{
235	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
236	rc = CPUMR0ModuleInit();
237	if (RT_SUCCESS(rc))
238	{
239	#ifdef VBOX_WITH_TRIPLE_FAULT_HACK
240	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
241	rc = vmmR0TripleFaultHackInit();
242	if (RT_SUCCESS(rc))
243	#endif
244	{
245	VMM_CHECK_SMAP_CHECK(rc = VERR_VMM_SMAP_BUT_AC_CLEAR);
246	if (RT_SUCCESS(rc))
247	{
248	g_rcRawModeUsability = SUPR0GetRawModeUsability();
249	if (g_rcRawModeUsability != VINF_SUCCESS)
250	SUPR0Printf("VMMR0!ModuleInit: SUPR0GetRawModeUsability -> %Rrc\n",
251	g_rcRawModeUsability);
252	LogFlow(("ModuleInit: returns success\n"));
253	return VINF_SUCCESS;
254	}
255	}
256
257	/*
258	* Bail out.
259	*/
260	#ifdef VBOX_WITH_TRIPLE_FAULT_HACK
261	vmmR0TripleFaultHackTerm();
262	#endif
263	}
264	else
265	LogRel(("ModuleInit: CPUMR0ModuleInit -> %Rrc\n", rc));
266	#ifdef VBOX_WITH_PCI_PASSTHROUGH
267	PciRawR0Term();
268	#endif
269	}
270	else
271	LogRel(("ModuleInit: PciRawR0Init -> %Rrc\n", rc));
272	IntNetR0Term();
273	}
274	else
275	LogRel(("ModuleInit: IntNetR0Init -> %Rrc\n", rc));
276	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
277	PGMR0DynMapTerm();
278	#endif
279	}
280	else
281	LogRel(("ModuleInit: PGMR0DynMapInit -> %Rrc\n", rc));
282	PGMDeregisterStringFormatTypes();
283	}
284	else
285	LogRel(("ModuleInit: PGMRegisterStringFormatTypes -> %Rrc\n", rc));
286	HMR0Term();
287	}
288	else
289	LogRel(("ModuleInit: HMR0Init -> %Rrc\n", rc));
290	GMMR0Term();
291	}
292	else
293	LogRel(("ModuleInit: GMMR0Init -> %Rrc\n", rc));
294	GVMMR0Term();
295	}
296	else
297	LogRel(("ModuleInit: GVMMR0Init -> %Rrc\n", rc));
298	vmmTermFormatTypes();
299	}
300	else
301	LogRel(("ModuleInit: vmmInitFormatTypes -> %Rrc\n", rc));
302
303	LogFlow(("ModuleInit: failed %Rrc\n", rc));
304	return rc;
305	}
306
307
308	/**
309	* Terminate the module.
310	* This is called when we're finally unloaded.
311	*
312	* @param hMod Image handle for use in APIs.
313	*/
314	DECLEXPORT(void) ModuleTerm(void *hMod)
315	{
316	NOREF(hMod);
317	LogFlow(("ModuleTerm:\n"));
318
319	/*
320	* Terminate the CPUM module (Local APIC cleanup).
321	*/
322	CPUMR0ModuleTerm();
323
324	/*
325	* Terminate the internal network service.
326	*/
327	IntNetR0Term();
328
329	/*
330	* PGM (Darwin), HM and PciRaw global cleanup.
331	*/
332	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
333	PGMR0DynMapTerm();
334	#endif
335	#ifdef VBOX_WITH_PCI_PASSTHROUGH
336	PciRawR0Term();
337	#endif
338	PGMDeregisterStringFormatTypes();
339	HMR0Term();
340	#ifdef VBOX_WITH_TRIPLE_FAULT_HACK
341	vmmR0TripleFaultHackTerm();
342	#endif
343
344	/*
345	* Destroy the GMM and GVMM instances.
346	*/
347	GMMR0Term();
348	GVMMR0Term();
349
350	vmmTermFormatTypes();
351
352	LogFlow(("ModuleTerm: returns\n"));
353	}
354
355
356	/**
357	* Initiates the R0 driver for a particular VM instance.
358	*
359	* @returns VBox status code.
360	*
361	* @param pGVM The global (ring-0) VM structure.
362	* @param pVM The cross context VM structure.
363	* @param uSvnRev The SVN revision of the ring-3 part.
364	* @param uBuildType Build type indicator.
365	* @thread EMT(0)
366	*/
367	static int vmmR0InitVM(PGVM pGVM, PVM pVM, uint32_t uSvnRev, uint32_t uBuildType)
368	{
369	VMM_CHECK_SMAP_SETUP();
370	VMM_CHECK_SMAP_CHECK(return VERR_VMM_SMAP_BUT_AC_CLEAR);
371
372	/*
373	* Match the SVN revisions and build type.
374	*/
375	if (uSvnRev != VMMGetSvnRev())
376	{
377	LogRel(("VMMR0InitVM: Revision mismatch, r3=%d r0=%d\n", uSvnRev, VMMGetSvnRev()));
378	SUPR0Printf("VMMR0InitVM: Revision mismatch, r3=%d r0=%d\n", uSvnRev, VMMGetSvnRev());
379	return VERR_VMM_R0_VERSION_MISMATCH;
380	}
381	if (uBuildType != vmmGetBuildType())
382	{
383	LogRel(("VMMR0InitVM: Build type mismatch, r3=%#x r0=%#x\n", uBuildType, vmmGetBuildType()));
384	SUPR0Printf("VMMR0InitVM: Build type mismatch, r3=%#x r0=%#x\n", uBuildType, vmmGetBuildType());
385	return VERR_VMM_R0_VERSION_MISMATCH;
386	}
387
388	int rc = GVMMR0ValidateGVMandVMandEMT(pGVM, pVM, 0 /idCpu/);
389	if (RT_FAILURE(rc))
390	return rc;
391
392
393	#ifdef LOG_ENABLED
394	/*
395	* Register the EMT R0 logger instance for VCPU 0.
396	*/
397	PVMCPU pVCpu = &pVM->aCpus[0];
398
399	PVMMR0LOGGER pR0Logger = pVCpu->vmm.s.pR0LoggerR0;
400	if (pR0Logger)
401	{
402	# if 0 /* testing of the logger. */
403	LogCom(("vmmR0InitVM: before %p\n", RTLogDefaultInstance()));
404	LogCom(("vmmR0InitVM: pfnFlush=%p actual=%p\n", pR0Logger->Logger.pfnFlush, vmmR0LoggerFlush));
405	LogCom(("vmmR0InitVM: pfnLogger=%p actual=%p\n", pR0Logger->Logger.pfnLogger, vmmR0LoggerWrapper));
406	LogCom(("vmmR0InitVM: offScratch=%d fFlags=%#x fDestFlags=%#x\n", pR0Logger->Logger.offScratch, pR0Logger->Logger.fFlags, pR0Logger->Logger.fDestFlags));
407
408	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pVM->pSession);
409	LogCom(("vmmR0InitVM: after %p reg\n", RTLogDefaultInstance()));
410	RTLogSetDefaultInstanceThread(NULL, pVM->pSession);
411	LogCom(("vmmR0InitVM: after %p dereg\n", RTLogDefaultInstance()));
412
413	pR0Logger->Logger.pfnLogger("hello ring-0 logger\n");
414	LogCom(("vmmR0InitVM: returned successfully from direct logger call.\n"));
415	pR0Logger->Logger.pfnFlush(&pR0Logger->Logger);
416	LogCom(("vmmR0InitVM: returned successfully from direct flush call.\n"));
417
418	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pVM->pSession);
419	LogCom(("vmmR0InitVM: after %p reg2\n", RTLogDefaultInstance()));
420	pR0Logger->Logger.pfnLogger("hello ring-0 logger\n");
421	LogCom(("vmmR0InitVM: returned successfully from direct logger call (2). offScratch=%d\n", pR0Logger->Logger.offScratch));
422	RTLogSetDefaultInstanceThread(NULL, pVM->pSession);
423	LogCom(("vmmR0InitVM: after %p dereg2\n", RTLogDefaultInstance()));
424
425	RTLogLoggerEx(&pR0Logger->Logger, 0, ~0U, "hello ring-0 logger (RTLogLoggerEx)\n");
426	LogCom(("vmmR0InitVM: RTLogLoggerEx returned fine offScratch=%d\n", pR0Logger->Logger.offScratch));
427
428	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pVM->pSession);
429	RTLogPrintf("hello ring-0 logger (RTLogPrintf)\n");
430	LogCom(("vmmR0InitVM: RTLogPrintf returned fine offScratch=%d\n", pR0Logger->Logger.offScratch));
431	# endif
432	Log(("Switching to per-thread logging instance %p (key=%p)\n", &pR0Logger->Logger, pVM->pSession));
433	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pVM->pSession);
434	pR0Logger->fRegistered = true;
435	}
436	#endif /* LOG_ENABLED */
437
438	/*
439	* Check if the host supports high resolution timers or not.
440	*/
441	if ( pVM->vmm.s.fUsePeriodicPreemptionTimers
442	&& !RTTimerCanDoHighResolution())
443	pVM->vmm.s.fUsePeriodicPreemptionTimers = false;
444
445	/*
446	* Initialize the per VM data for GVMM and GMM.
447	*/
448	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
449	rc = GVMMR0InitVM(pGVM);
450	// if (RT_SUCCESS(rc))
451	// rc = GMMR0InitPerVMData(pVM);
452	if (RT_SUCCESS(rc))
453	{
454	/*
455	* Init HM, CPUM and PGM (Darwin only).
456	*/
457	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
458	rc = HMR0InitVM(pVM);
459	if (RT_SUCCESS(rc))
460	VMM_CHECK_SMAP_CHECK2(pVM, rc = VERR_VMM_RING0_ASSERTION); /* CPUR0InitVM will otherwise panic the host */
461	if (RT_SUCCESS(rc))
462	{
463	rc = CPUMR0InitVM(pVM);
464	if (RT_SUCCESS(rc))
465	{
466	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
467	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
468	rc = PGMR0DynMapInitVM(pVM);
469	#endif
470	if (RT_SUCCESS(rc))
471	{
472	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
473	rc = EMR0InitVM(pGVM, pVM);
474	if (RT_SUCCESS(rc))
475	{
476	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
477	#ifdef VBOX_WITH_PCI_PASSTHROUGH
478	rc = PciRawR0InitVM(pGVM, pVM);
479	#endif
480	if (RT_SUCCESS(rc))
481	{
482	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
483	rc = GIMR0InitVM(pVM);
484	if (RT_SUCCESS(rc))
485	{
486	VMM_CHECK_SMAP_CHECK2(pVM, rc = VERR_VMM_RING0_ASSERTION);
487	if (RT_SUCCESS(rc))
488	{
489	GVMMR0DoneInitVM(pGVM);
490
491	/*
492	* Collect a bit of info for the VM release log.
493	*/
494	pVM->vmm.s.fIsPreemptPendingApiTrusty = RTThreadPreemptIsPendingTrusty();
495	pVM->vmm.s.fIsPreemptPossible = RTThreadPreemptIsPossible();;
496
497	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
498	return rc;
499	}
500
501	/* bail out*/
502	GIMR0TermVM(pVM);
503	}
504	#ifdef VBOX_WITH_PCI_PASSTHROUGH
505	PciRawR0TermVM(pGVM, pVM);
506	#endif
507	}
508	}
509	}
510	}
511	HMR0TermVM(pVM);
512	}
513	}
514
515	RTLogSetDefaultInstanceThread(NULL, (uintptr_t)pVM->pSession);
516	return rc;
517	}
518
519
520	/**
521	* Does EMT specific VM initialization.
522	*
523	* @returns VBox status code.
524	* @param pGVM The ring-0 VM structure.
525	* @param pVM The cross context VM structure.
526	* @param idCpu The EMT that's calling.
527	*/
528	static int vmmR0InitVMEmt(PGVM pGVM, PVM pVM, VMCPUID idCpu)
529	{
530	/* Paranoia (caller checked these already). */
531	AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID);
532	AssertReturn(pGVM->aCpus[idCpu].hEMT == RTThreadNativeSelf(), VERR_INVALID_CPU_ID);
533
534	#ifdef LOG_ENABLED
535	/*
536	* Registration of ring 0 loggers.
537	*/
538	PVMCPU pVCpu = &pVM->aCpus[idCpu];
539	PVMMR0LOGGER pR0Logger = pVCpu->vmm.s.pR0LoggerR0;
540	if ( pR0Logger
541	&& !pR0Logger->fRegistered)
542	{
543	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pVM->pSession);
544	pR0Logger->fRegistered = true;
545	}
546	#endif
547	RT_NOREF(pVM);
548
549	return VINF_SUCCESS;
550	}
551
552
553
554	/**
555	* Terminates the R0 bits for a particular VM instance.
556	*
557	* This is normally called by ring-3 as part of the VM termination process, but
558	* may alternatively be called during the support driver session cleanup when
559	* the VM object is destroyed (see GVMM).
560	*
561	* @returns VBox status code.
562	*
563	* @param pGVM The global (ring-0) VM structure.
564	* @param pVM The cross context VM structure.
565	* @param idCpu Set to 0 if EMT(0) or NIL_VMCPUID if session cleanup
566	* thread.
567	* @thread EMT(0) or session clean up thread.
568	*/
569	VMMR0_INT_DECL(int) VMMR0TermVM(PGVM pGVM, PVM pVM, VMCPUID idCpu)
570	{
571	/*
572	* Check EMT(0) claim if we're called from userland.
573	*/
574	if (idCpu != NIL_VMCPUID)
575	{
576	AssertReturn(idCpu == 0, VERR_INVALID_CPU_ID);
577	int rc = GVMMR0ValidateGVMandVMandEMT(pGVM, pVM, idCpu);
578	if (RT_FAILURE(rc))
579	return rc;
580	}
581
582	#ifdef VBOX_WITH_PCI_PASSTHROUGH
583	PciRawR0TermVM(pGVM, pVM);
584	#endif
585
586	/*
587	* Tell GVMM what we're up to and check that we only do this once.
588	*/
589	if (GVMMR0DoingTermVM(pGVM))
590	{
591	GIMR0TermVM(pVM);
592
593	/** @todo I wish to call PGMR0PhysFlushHandyPages(pVM, &pVM->aCpus[idCpu])
594	* here to make sure we don't leak any shared pages if we crash... */
595	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
596	PGMR0DynMapTermVM(pVM);
597	#endif
598	HMR0TermVM(pVM);
599	}
600
601	/*
602	* Deregister the logger.
603	*/
604	RTLogSetDefaultInstanceThread(NULL, (uintptr_t)pVM->pSession);
605	return VINF_SUCCESS;
606	}
607
608
609	/**
610	* VMM ring-0 thread-context callback.
611	*
612	* This does common HM state updating and calls the HM-specific thread-context
613	* callback.
614	*
615	* @param enmEvent The thread-context event.
616	* @param pvUser Opaque pointer to the VMCPU.
617	*
618	* @thread EMT(pvUser)
619	*/
620	static DECLCALLBACK(void) vmmR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, void *pvUser)
621	{
622	PVMCPU pVCpu = (PVMCPU)pvUser;
623
624	switch (enmEvent)
625	{
626	case RTTHREADCTXEVENT_IN:
627	{
628	/*
629	* Linux may call us with preemption enabled (really!) but technically we
630	* cannot get preempted here, otherwise we end up in an infinite recursion
631	* scenario (i.e. preempted in resume hook -> preempt hook -> resume hook...
632	* ad infinitum). Let's just disable preemption for now...
633	*/
634	/** @todo r=bird: I don't believe the above. The linux code is clearly enabling
635	* preemption after doing the callout (one or two functions up the
636	* call chain). */
637	/** @todo r=ramshankar: See @bugref{5313#c30}. */
638	RTTHREADPREEMPTSTATE ParanoidPreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
639	RTThreadPreemptDisable(&ParanoidPreemptState);
640
641	/* We need to update the VCPU <-> host CPU mapping. */
642	RTCPUID idHostCpu;
643	uint32_t iHostCpuSet = RTMpCurSetIndexAndId(&idHostCpu);
644	pVCpu->iHostCpuSet = iHostCpuSet;
645	ASMAtomicWriteU32(&pVCpu->idHostCpu, idHostCpu);
646
647	/* In the very unlikely event that the GIP delta for the CPU we're
648	rescheduled needs calculating, try force a return to ring-3.
649	We unfortunately cannot do the measurements right here. */
650	if (RT_UNLIKELY(SUPIsTscDeltaAvailableForCpuSetIndex(iHostCpuSet)))
651	VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
652
653	/* Invoke the HM-specific thread-context callback. */
654	HMR0ThreadCtxCallback(enmEvent, pvUser);
655
656	/* Restore preemption. */
657	RTThreadPreemptRestore(&ParanoidPreemptState);
658	break;
659	}
660
661	case RTTHREADCTXEVENT_OUT:
662	{
663	/* Invoke the HM-specific thread-context callback. */
664	HMR0ThreadCtxCallback(enmEvent, pvUser);
665
666	/*
667	* Sigh. See VMMGetCpu() used by VMCPU_ASSERT_EMT(). We cannot let several VCPUs
668	* have the same host CPU associated with it.
669	*/
670	pVCpu->iHostCpuSet = UINT32_MAX;
671	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
672	break;
673	}
674
675	default:
676	/* Invoke the HM-specific thread-context callback. */
677	HMR0ThreadCtxCallback(enmEvent, pvUser);
678	break;
679	}
680	}
681
682
683	/**
684	* Creates thread switching hook for the current EMT thread.
685	*
686	* This is called by GVMMR0CreateVM and GVMMR0RegisterVCpu. If the host
687	* platform does not implement switcher hooks, no hooks will be create and the
688	* member set to NIL_RTTHREADCTXHOOK.
689	*
690	* @returns VBox status code.
691	* @param pVCpu The cross context virtual CPU structure.
692	* @thread EMT(pVCpu)
693	*/
694	VMMR0_INT_DECL(int) VMMR0ThreadCtxHookCreateForEmt(PVMCPU pVCpu)
695	{
696	VMCPU_ASSERT_EMT(pVCpu);
697	Assert(pVCpu->vmm.s.hCtxHook == NIL_RTTHREADCTXHOOK);
698
699	#if 1 /* To disable this stuff change to zero. */
700	int rc = RTThreadCtxHookCreate(&pVCpu->vmm.s.hCtxHook, 0, vmmR0ThreadCtxCallback, pVCpu);
701	if (RT_SUCCESS(rc))
702	return rc;
703	#else
704	RT_NOREF(vmmR0ThreadCtxCallback);
705	int rc = VERR_NOT_SUPPORTED;
706	#endif
707
708	pVCpu->vmm.s.hCtxHook = NIL_RTTHREADCTXHOOK;
709	if (rc == VERR_NOT_SUPPORTED)
710	return VINF_SUCCESS;
711
712	LogRelMax(32, ("RTThreadCtxHookCreate failed! rc=%Rrc pVCpu=%p idCpu=%RU32\n", rc, pVCpu, pVCpu->idCpu));
713	return VINF_SUCCESS; /* Just ignore it, we can live without context hooks. */
714	}
715
716
717	/**
718	* Destroys the thread switching hook for the specified VCPU.
719	*
720	* @param pVCpu The cross context virtual CPU structure.
721	* @remarks Can be called from any thread.
722	*/
723	VMMR0_INT_DECL(void) VMMR0ThreadCtxHookDestroyForEmt(PVMCPU pVCpu)
724	{
725	int rc = RTThreadCtxHookDestroy(pVCpu->vmm.s.hCtxHook);
726	AssertRC(rc);
727	pVCpu->vmm.s.hCtxHook = NIL_RTTHREADCTXHOOK;
728	}
729
730
731	/**
732	* Disables the thread switching hook for this VCPU (if we got one).
733	*
734	* @param pVCpu The cross context virtual CPU structure.
735	* @thread EMT(pVCpu)
736	*
737	* @remarks This also clears VMCPU::idHostCpu, so the mapping is invalid after
738	* this call. This means you have to be careful with what you do!
739	*/
740	VMMR0_INT_DECL(void) VMMR0ThreadCtxHookDisable(PVMCPU pVCpu)
741	{
742	/*
743	* Clear the VCPU <-> host CPU mapping as we've left HM context.
744	* @bugref{7726#c19} explains the need for this trick:
745	*
746	* hmR0VmxCallRing3Callback/hmR0SvmCallRing3Callback &
747	* hmR0VmxLeaveSession/hmR0SvmLeaveSession disables context hooks during
748	* longjmp & normal return to ring-3, which opens a window where we may be
749	* rescheduled without changing VMCPUID::idHostCpu and cause confusion if
750	* the CPU starts executing a different EMT. Both functions first disables
751	* preemption and then calls HMR0LeaveCpu which invalids idHostCpu, leaving
752	* an opening for getting preempted.
753	*/
754	/** @todo Make HM not need this API! Then we could leave the hooks enabled
755	* all the time. */
756	/** @todo move this into the context hook disabling if(). */
757	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
758
759	/*
760	* Disable the context hook, if we got one.
761	*/
762	if (pVCpu->vmm.s.hCtxHook != NIL_RTTHREADCTXHOOK)
763	{
764	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
765	int rc = RTThreadCtxHookDisable(pVCpu->vmm.s.hCtxHook);
766	AssertRC(rc);
767	}
768	}
769
770
771	/**
772	* Internal version of VMMR0ThreadCtxHooksAreRegistered.
773	*
774	* @returns true if registered, false otherwise.
775	* @param pVCpu The cross context virtual CPU structure.
776	*/
777	DECLINLINE(bool) vmmR0ThreadCtxHookIsEnabled(PVMCPU pVCpu)
778	{
779	return RTThreadCtxHookIsEnabled(pVCpu->vmm.s.hCtxHook);
780	}
781
782
783	/**
784	* Whether thread-context hooks are registered for this VCPU.
785	*
786	* @returns true if registered, false otherwise.
787	* @param pVCpu The cross context virtual CPU structure.
788	*/
789	VMMR0_INT_DECL(bool) VMMR0ThreadCtxHookIsEnabled(PVMCPU pVCpu)
790	{
791	return vmmR0ThreadCtxHookIsEnabled(pVCpu);
792	}
793
794
795	#ifdef VBOX_WITH_STATISTICS
796	/**
797	* Record return code statistics
798	* @param pVM The cross context VM structure.
799	* @param pVCpu The cross context virtual CPU structure.
800	* @param rc The status code.
801	*/
802	static void vmmR0RecordRC(PVM pVM, PVMCPU pVCpu, int rc)
803	{
804	/*
805	* Collect statistics.
806	*/
807	switch (rc)
808	{
809	case VINF_SUCCESS:
810	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetNormal);
811	break;
812	case VINF_EM_RAW_INTERRUPT:
813	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetInterrupt);
814	break;
815	case VINF_EM_RAW_INTERRUPT_HYPER:
816	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetInterruptHyper);
817	break;
818	case VINF_EM_RAW_GUEST_TRAP:
819	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetGuestTrap);
820	break;
821	case VINF_EM_RAW_RING_SWITCH:
822	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetRingSwitch);
823	break;
824	case VINF_EM_RAW_RING_SWITCH_INT:
825	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetRingSwitchInt);
826	break;
827	case VINF_EM_RAW_STALE_SELECTOR:
828	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetStaleSelector);
829	break;
830	case VINF_EM_RAW_IRET_TRAP:
831	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIRETTrap);
832	break;
833	case VINF_IOM_R3_IOPORT_READ:
834	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIORead);
835	break;
836	case VINF_IOM_R3_IOPORT_WRITE:
837	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIOWrite);
838	break;
839	case VINF_IOM_R3_IOPORT_COMMIT_WRITE:
840	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIOCommitWrite);
841	break;
842	case VINF_IOM_R3_MMIO_READ:
843	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIORead);
844	break;
845	case VINF_IOM_R3_MMIO_WRITE:
846	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOWrite);
847	break;
848	case VINF_IOM_R3_MMIO_COMMIT_WRITE:
849	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOCommitWrite);
850	break;
851	case VINF_IOM_R3_MMIO_READ_WRITE:
852	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOReadWrite);
853	break;
854	case VINF_PATM_HC_MMIO_PATCH_READ:
855	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOPatchRead);
856	break;
857	case VINF_PATM_HC_MMIO_PATCH_WRITE:
858	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOPatchWrite);
859	break;
860	case VINF_CPUM_R3_MSR_READ:
861	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMSRRead);
862	break;
863	case VINF_CPUM_R3_MSR_WRITE:
864	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMSRWrite);
865	break;
866	case VINF_EM_RAW_EMULATE_INSTR:
867	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetEmulate);
868	break;
869	case VINF_EM_RAW_EMULATE_IO_BLOCK:
870	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIOBlockEmulate);
871	break;
872	case VINF_PATCH_EMULATE_INSTR:
873	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchEmulate);
874	break;
875	case VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT:
876	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetLDTFault);
877	break;
878	case VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT:
879	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetGDTFault);
880	break;
881	case VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT:
882	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIDTFault);
883	break;
884	case VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT:
885	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetTSSFault);
886	break;
887	case VINF_CSAM_PENDING_ACTION:
888	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetCSAMTask);
889	break;
890	case VINF_PGM_SYNC_CR3:
891	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetSyncCR3);
892	break;
893	case VINF_PATM_PATCH_INT3:
894	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchInt3);
895	break;
896	case VINF_PATM_PATCH_TRAP_PF:
897	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchPF);
898	break;
899	case VINF_PATM_PATCH_TRAP_GP:
900	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchGP);
901	break;
902	case VINF_PATM_PENDING_IRQ_AFTER_IRET:
903	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchIretIRQ);
904	break;
905	case VINF_EM_RESCHEDULE_REM:
906	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetRescheduleREM);
907	break;
908	case VINF_EM_RAW_TO_R3:
909	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Total);
910	if (VM_FF_IS_PENDING(pVM, VM_FF_TM_VIRTUAL_SYNC))
911	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3TMVirt);
912	else if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_NEED_HANDY_PAGES))
913	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3HandyPages);
914	else if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_QUEUES))
915	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3PDMQueues);
916	else if (VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS))
917	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Rendezvous);
918	else if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
919	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3DMA);
920	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TIMER))
921	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Timer);
922	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PDM_CRITSECT))
923	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3CritSect);
924	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TO_R3))
925	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3FF);
926	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_IEM))
927	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Iem);
928	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_IOM))
929	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Iom);
930	else
931	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Unknown);
932	break;
933
934	case VINF_EM_RAW_TIMER_PENDING:
935	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetTimerPending);
936	break;
937	case VINF_EM_RAW_INTERRUPT_PENDING:
938	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetInterruptPending);
939	break;
940	case VINF_VMM_CALL_HOST:
941	switch (pVCpu->vmm.s.enmCallRing3Operation)
942	{
943	case VMMCALLRING3_PDM_CRIT_SECT_ENTER:
944	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPDMCritSectEnter);
945	break;
946	case VMMCALLRING3_PDM_LOCK:
947	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPDMLock);
948	break;
949	case VMMCALLRING3_PGM_POOL_GROW:
950	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPGMPoolGrow);
951	break;
952	case VMMCALLRING3_PGM_LOCK:
953	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPGMLock);
954	break;
955	case VMMCALLRING3_PGM_MAP_CHUNK:
956	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPGMMapChunk);
957	break;
958	case VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES:
959	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPGMAllocHandy);
960	break;
961	case VMMCALLRING3_REM_REPLAY_HANDLER_NOTIFICATIONS:
962	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallRemReplay);
963	break;
964	case VMMCALLRING3_VMM_LOGGER_FLUSH:
965	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallLogFlush);
966	break;
967	case VMMCALLRING3_VM_SET_ERROR:
968	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallVMSetError);
969	break;
970	case VMMCALLRING3_VM_SET_RUNTIME_ERROR:
971	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallVMSetRuntimeError);
972	break;
973	case VMMCALLRING3_VM_R0_ASSERTION:
974	default:
975	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetCallRing3);
976	break;
977	}
978	break;
979	case VINF_PATM_DUPLICATE_FUNCTION:
980	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPATMDuplicateFn);
981	break;
982	case VINF_PGM_CHANGE_MODE:
983	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPGMChangeMode);
984	break;
985	case VINF_PGM_POOL_FLUSH_PENDING:
986	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPGMFlushPending);
987	break;
988	case VINF_EM_PENDING_REQUEST:
989	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPendingRequest);
990	break;
991	case VINF_EM_HM_PATCH_TPR_INSTR:
992	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchTPR);
993	break;
994	default:
995	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMisc);
996	break;
997	}
998	}
999	#endif /* VBOX_WITH_STATISTICS */
1000
1001
1002	/**
1003	* The Ring 0 entry point, called by the fast-ioctl path.
1004	*
1005	* @param pGVM The global (ring-0) VM structure.
1006	* @param pVM The cross context VM structure.
1007	* The return code is stored in pVM->vmm.s.iLastGZRc.
1008	* @param idCpu The Virtual CPU ID of the calling EMT.
1009	* @param enmOperation Which operation to execute.
1010	* @remarks Assume called with interrupts _enabled_.
1011	*/
1012	VMMR0DECL(void) VMMR0EntryFast(PGVM pGVM, PVM pVM, VMCPUID idCpu, VMMR0OPERATION enmOperation)
1013	{
1014	/*
1015	* Validation.
1016	*/
1017	if ( idCpu < pGVM->cCpus
1018	&& pGVM->cCpus == pVM->cCpus)
1019	{ /likely/ }
1020	else
1021	{
1022	SUPR0Printf("VMMR0EntryFast: Bad idCpu=%#x cCpus=%#x/%#x\n", idCpu, pGVM->cCpus, pVM->cCpus);
1023	return;
1024	}
1025
1026	PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
1027	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1028	RTNATIVETHREAD const hNativeThread = RTThreadNativeSelf();
1029	if (RT_LIKELY( pGVCpu->hEMT == hNativeThread
1030	&& pVCpu->hNativeThreadR0 == hNativeThread))
1031	{ /* likely */ }
1032	else
1033	{
1034	SUPR0Printf("VMMR0EntryFast: Bad thread idCpu=%#x hNativeSelf=%p pGVCpu->hEmt=%p pVCpu->hNativeThreadR0=%p\n",
1035	idCpu, hNativeThread, pGVCpu->hEMT, pVCpu->hNativeThreadR0);
1036	return;
1037	}
1038
1039	/*
1040	* SMAP fun.
1041	*/
1042	VMM_CHECK_SMAP_SETUP();
1043	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1044
1045	/*
1046	* Perform requested operation.
1047	*/
1048	switch (enmOperation)
1049	{
1050	/*
1051	* Switch to GC and run guest raw mode code.
1052	* Disable interrupts before doing the world switch.
1053	*/
1054	case VMMR0_DO_RAW_RUN:
1055	{
1056	#ifdef VBOX_WITH_RAW_MODE
1057	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
1058	/* Some safety precautions first. */
1059	if (RT_UNLIKELY(!PGMGetHyperCR3(pVCpu)))
1060	{
1061	pVCpu->vmm.s.iLastGZRc = VERR_PGM_NO_CR3_SHADOW_ROOT;
1062	break;
1063	}
1064	# endif
1065	if (RT_SUCCESS(g_rcRawModeUsability))
1066	{ /* likely */ }
1067	else
1068	{
1069	pVCpu->vmm.s.iLastGZRc = g_rcRawModeUsability;
1070	break;
1071	}
1072
1073	/*
1074	* Disable preemption.
1075	*/
1076	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
1077	RTThreadPreemptDisable(&PreemptState);
1078
1079	/*
1080	* Get the host CPU identifiers, make sure they are valid and that
1081	* we've got a TSC delta for the CPU.
1082	*/
1083	RTCPUID idHostCpu;
1084	uint32_t iHostCpuSet = RTMpCurSetIndexAndId(&idHostCpu);
1085	if (RT_LIKELY( iHostCpuSet < RTCPUSET_MAX_CPUS
1086	&& SUPIsTscDeltaAvailableForCpuSetIndex(iHostCpuSet)))
1087	{
1088	/*
1089	* Commit the CPU identifiers and update the periodict preemption timer if it's active.
1090	*/
1091	# ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
1092	CPUMR0SetLApic(pVCpu, iHostCpuSet);
1093	# endif
1094	pVCpu->iHostCpuSet = iHostCpuSet;
1095	ASMAtomicWriteU32(&pVCpu->idHostCpu, idHostCpu);
1096
1097	if (pVM->vmm.s.fUsePeriodicPreemptionTimers)
1098	GVMMR0SchedUpdatePeriodicPreemptionTimer(pVM, pVCpu->idHostCpu, TMCalcHostTimerFrequency(pVM, pVCpu));
1099
1100	/*
1101	* We might need to disable VT-x if the active switcher turns off paging.
1102	*/
1103	bool fVTxDisabled;
1104	int rc = HMR0EnterSwitcher(pVM, pVM->vmm.s.enmSwitcher, &fVTxDisabled);
1105	if (RT_SUCCESS(rc))
1106	{
1107	/*
1108	* Disable interrupts and run raw-mode code. The loop is for efficiently
1109	* dispatching tracepoints that fired in raw-mode context.
1110	*/
1111	RTCCUINTREG uFlags = ASMIntDisableFlags();
1112
1113	for (;;)
1114	{
1115	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
1116	TMNotifyStartOfExecution(pVCpu);
1117
1118	rc = pVM->vmm.s.pfnR0ToRawMode(pVM);
1119	pVCpu->vmm.s.iLastGZRc = rc;
1120
1121	TMNotifyEndOfExecution(pVCpu);
1122	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
1123
1124	if (rc != VINF_VMM_CALL_TRACER)
1125	break;
1126	SUPR0TracerUmodProbeFire(pVM->pSession, &pVCpu->vmm.s.TracerCtx);
1127	}
1128
1129	/*
1130	* Re-enable VT-x before we dispatch any pending host interrupts and
1131	* re-enables interrupts.
1132	*/
1133	HMR0LeaveSwitcher(pVM, fVTxDisabled);
1134
1135	if ( rc == VINF_EM_RAW_INTERRUPT
1136	\|\| rc == VINF_EM_RAW_INTERRUPT_HYPER)
1137	TRPMR0DispatchHostInterrupt(pVM);
1138
1139	ASMSetFlags(uFlags);
1140
1141	/* Fire dtrace probe and collect statistics. */
1142	VBOXVMM_R0_VMM_RETURN_TO_RING3_RC(pVCpu, CPUMQueryGuestCtxPtr(pVCpu), rc);
1143	# ifdef VBOX_WITH_STATISTICS
1144	STAM_COUNTER_INC(&pVM->vmm.s.StatRunRC);
1145	vmmR0RecordRC(pVM, pVCpu, rc);
1146	# endif
1147	}
1148	else
1149	pVCpu->vmm.s.iLastGZRc = rc;
1150
1151	/*
1152	* Invalidate the host CPU identifiers as we restore preemption.
1153	*/
1154	pVCpu->iHostCpuSet = UINT32_MAX;
1155	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
1156
1157	RTThreadPreemptRestore(&PreemptState);
1158	}
1159	/*
1160	* Invalid CPU set index or TSC delta in need of measuring.
1161	*/
1162	else
1163	{
1164	RTThreadPreemptRestore(&PreemptState);
1165	if (iHostCpuSet < RTCPUSET_MAX_CPUS)
1166	{
1167	int rc = SUPR0TscDeltaMeasureBySetIndex(pVM->pSession, iHostCpuSet, 0 /fFlags/,
1168	2 /cMsWaitRetry/, 5RT_MS_1SEC /cMsWaitThread*/,
1169	0 /default cTries/);
1170	if (RT_SUCCESS(rc) \|\| rc == VERR_CPU_OFFLINE)
1171	pVCpu->vmm.s.iLastGZRc = VINF_EM_RAW_TO_R3;
1172	else
1173	pVCpu->vmm.s.iLastGZRc = rc;
1174	}
1175	else
1176	pVCpu->vmm.s.iLastGZRc = VERR_INVALID_CPU_INDEX;
1177	}
1178
1179	#else /* !VBOX_WITH_RAW_MODE */
1180	pVCpu->vmm.s.iLastGZRc = VERR_RAW_MODE_NOT_SUPPORTED;
1181	#endif
1182	break;
1183	}
1184
1185	/*
1186	* Run guest code using the available hardware acceleration technology.
1187	*/
1188	case VMMR0_DO_HM_RUN:
1189	{
1190	/*
1191	* Disable preemption.
1192	*/
1193	Assert(!vmmR0ThreadCtxHookIsEnabled(pVCpu));
1194	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
1195	RTThreadPreemptDisable(&PreemptState);
1196
1197	/*
1198	* Get the host CPU identifiers, make sure they are valid and that
1199	* we've got a TSC delta for the CPU.
1200	*/
1201	RTCPUID idHostCpu;
1202	uint32_t iHostCpuSet = RTMpCurSetIndexAndId(&idHostCpu);
1203	if (RT_LIKELY( iHostCpuSet < RTCPUSET_MAX_CPUS
1204	&& SUPIsTscDeltaAvailableForCpuSetIndex(iHostCpuSet)))
1205	{
1206	pVCpu->iHostCpuSet = iHostCpuSet;
1207	ASMAtomicWriteU32(&pVCpu->idHostCpu, idHostCpu);
1208
1209	/*
1210	* Update the periodic preemption timer if it's active.
1211	*/
1212	if (pVM->vmm.s.fUsePeriodicPreemptionTimers)
1213	GVMMR0SchedUpdatePeriodicPreemptionTimer(pVM, pVCpu->idHostCpu, TMCalcHostTimerFrequency(pVM, pVCpu));
1214	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1215
1216	#ifdef VMM_R0_TOUCH_FPU
1217	/*
1218	* Make sure we've got the FPU state loaded so and we don't need to clear
1219	* CR0.TS and get out of sync with the host kernel when loading the guest
1220	* FPU state. @ref sec_cpum_fpu (CPUM.cpp) and @bugref{4053}.
1221	*/
1222	CPUMR0TouchHostFpu();
1223	#endif
1224	int rc;
1225	bool fPreemptRestored = false;
1226	if (!HMR0SuspendPending())
1227	{
1228	/*
1229	* Enable the context switching hook.
1230	*/
1231	if (pVCpu->vmm.s.hCtxHook != NIL_RTTHREADCTXHOOK)
1232	{
1233	Assert(!RTThreadCtxHookIsEnabled(pVCpu->vmm.s.hCtxHook));
1234	int rc2 = RTThreadCtxHookEnable(pVCpu->vmm.s.hCtxHook); AssertRC(rc2);
1235	}
1236
1237	/*
1238	* Enter HM context.
1239	*/
1240	rc = HMR0Enter(pVM, pVCpu);
1241	if (RT_SUCCESS(rc))
1242	{
1243	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
1244
1245	/*
1246	* When preemption hooks are in place, enable preemption now that
1247	* we're in HM context.
1248	*/
1249	if (vmmR0ThreadCtxHookIsEnabled(pVCpu))
1250	{
1251	fPreemptRestored = true;
1252	RTThreadPreemptRestore(&PreemptState);
1253	}
1254
1255	/*
1256	* Setup the longjmp machinery and execute guest code (calls HMR0RunGuestCode).
1257	*/
1258	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1259	rc = vmmR0CallRing3SetJmp(&pVCpu->vmm.s.CallRing3JmpBufR0, HMR0RunGuestCode, pVM, pVCpu);
1260	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1261
1262	/*
1263	* Assert sanity on the way out. Using manual assertions code here as normal
1264	* assertions are going to panic the host since we're outside the setjmp/longjmp zone.
1265	*/
1266	if (RT_UNLIKELY( VMCPU_GET_STATE(pVCpu) != VMCPUSTATE_STARTED_HM
1267	&& RT_SUCCESS_NP(rc) && rc != VINF_VMM_CALL_HOST ))
1268	{
1269	pVM->vmm.s.szRing0AssertMsg1[0] = '\0';
1270	RTStrPrintf(pVM->vmm.s.szRing0AssertMsg2, sizeof(pVM->vmm.s.szRing0AssertMsg2),
1271	"Got VMCPU state %d expected %d.\n", VMCPU_GET_STATE(pVCpu), VMCPUSTATE_STARTED_HM);
1272	rc = VERR_VMM_WRONG_HM_VMCPU_STATE;
1273	}
1274	/** @todo Get rid of this. HM shouldn't disable the context hook. */
1275	else if (RT_UNLIKELY(vmmR0ThreadCtxHookIsEnabled(pVCpu)))
1276	{
1277	pVM->vmm.s.szRing0AssertMsg1[0] = '\0';
1278	RTStrPrintf(pVM->vmm.s.szRing0AssertMsg2, sizeof(pVM->vmm.s.szRing0AssertMsg2),
1279	"Thread-context hooks still enabled! VCPU=%p Id=%u rc=%d.\n", pVCpu, pVCpu->idCpu, rc);
1280	rc = VERR_INVALID_STATE;
1281	}
1282
1283	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
1284	}
1285	STAM_COUNTER_INC(&pVM->vmm.s.StatRunRC);
1286
1287	/*
1288	* Invalidate the host CPU identifiers before we disable the context
1289	* hook / restore preemption.
1290	*/
1291	pVCpu->iHostCpuSet = UINT32_MAX;
1292	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
1293
1294	/*
1295	* Disable context hooks. Due to unresolved cleanup issues, we
1296	* cannot leave the hooks enabled when we return to ring-3.
1297	*
1298	* Note! At the moment HM may also have disabled the hook
1299	* when we get here, but the IPRT API handles that.
1300	*/
1301	if (pVCpu->vmm.s.hCtxHook != NIL_RTTHREADCTXHOOK)
1302	{
1303	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
1304	RTThreadCtxHookDisable(pVCpu->vmm.s.hCtxHook);
1305	}
1306	}
1307	/*
1308	* The system is about to go into suspend mode; go back to ring 3.
1309	*/
1310	else
1311	{
1312	rc = VINF_EM_RAW_INTERRUPT;
1313	pVCpu->iHostCpuSet = UINT32_MAX;
1314	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
1315	}
1316
1317	/** @todo When HM stops messing with the context hook state, we'll disable
1318	* preemption again before the RTThreadCtxHookDisable call. */
1319	if (!fPreemptRestored)
1320	RTThreadPreemptRestore(&PreemptState);
1321
1322	pVCpu->vmm.s.iLastGZRc = rc;
1323
1324	/* Fire dtrace probe and collect statistics. */
1325	VBOXVMM_R0_VMM_RETURN_TO_RING3_HM(pVCpu, CPUMQueryGuestCtxPtr(pVCpu), rc);
1326	#ifdef VBOX_WITH_STATISTICS
1327	vmmR0RecordRC(pVM, pVCpu, rc);
1328	#endif
1329	}
1330	/*
1331	* Invalid CPU set index or TSC delta in need of measuring.
1332	*/
1333	else
1334	{
1335	pVCpu->iHostCpuSet = UINT32_MAX;
1336	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
1337	RTThreadPreemptRestore(&PreemptState);
1338	if (iHostCpuSet < RTCPUSET_MAX_CPUS)
1339	{
1340	int rc = SUPR0TscDeltaMeasureBySetIndex(pVM->pSession, iHostCpuSet, 0 /fFlags/,
1341	2 /cMsWaitRetry/, 5RT_MS_1SEC /cMsWaitThread*/,
1342	0 /default cTries/);
1343	if (RT_SUCCESS(rc) \|\| rc == VERR_CPU_OFFLINE)
1344	pVCpu->vmm.s.iLastGZRc = VINF_EM_RAW_TO_R3;
1345	else
1346	pVCpu->vmm.s.iLastGZRc = rc;
1347	}
1348	else
1349	pVCpu->vmm.s.iLastGZRc = VERR_INVALID_CPU_INDEX;
1350	}
1351	break;
1352	}
1353
1354	#ifdef VBOX_WITH_NEM_R0
1355	# if defined(RT_ARCH_AMD64) && defined(RT_OS_WINDOWS)
1356	case VMMR0_DO_NEM_RUN:
1357	{
1358	/*
1359	* Setup the longjmp machinery and execute guest code (calls NEMR0RunGuestCode).
1360	*/
1361	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1362	int rc = vmmR0CallRing3SetJmp2(&pVCpu->vmm.s.CallRing3JmpBufR0, NEMR0RunGuestCode, pGVM, idCpu);
1363	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1364	STAM_COUNTER_INC(&pVM->vmm.s.StatRunRC);
1365
1366	pVCpu->vmm.s.iLastGZRc = rc;
1367
1368	/*
1369	* Fire dtrace probe and collect statistics.
1370	*/
1371	VBOXVMM_R0_VMM_RETURN_TO_RING3_NEM(pVCpu, CPUMQueryGuestCtxPtr(pVCpu), rc);
1372	# ifdef VBOX_WITH_STATISTICS
1373	vmmR0RecordRC(pVM, pVCpu, rc);
1374	# endif
1375	break;
1376	}
1377	# endif
1378	#endif
1379
1380
1381	/*
1382	* For profiling.
1383	*/
1384	case VMMR0_DO_NOP:
1385	pVCpu->vmm.s.iLastGZRc = VINF_SUCCESS;
1386	break;
1387
1388	/*
1389	* Shouldn't happen.
1390	*/
1391	default:
1392	AssertMsgFailed(("%#x\n", enmOperation));
1393	pVCpu->vmm.s.iLastGZRc = VERR_NOT_SUPPORTED;
1394	break;
1395	}
1396	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1397	}
1398
1399
1400	/**
1401	* Validates a session or VM session argument.
1402	*
1403	* @returns true / false accordingly.
1404	* @param pVM The cross context VM structure.
1405	* @param pClaimedSession The session claim to validate.
1406	* @param pSession The session argument.
1407	*/
1408	DECLINLINE(bool) vmmR0IsValidSession(PVM pVM, PSUPDRVSESSION pClaimedSession, PSUPDRVSESSION pSession)
1409	{
1410	/* This must be set! */
1411	if (!pSession)
1412	return false;
1413
1414	/* Only one out of the two. */
1415	if (pVM && pClaimedSession)
1416	return false;
1417	if (pVM)
1418	pClaimedSession = pVM->pSession;
1419	return pClaimedSession == pSession;
1420	}
1421
1422
1423	/**
1424	* VMMR0EntryEx worker function, either called directly or when ever possible
1425	* called thru a longjmp so we can exit safely on failure.
1426	*
1427	* @returns VBox status code.
1428	* @param pGVM The global (ring-0) VM structure.
1429	* @param pVM The cross context VM structure.
1430	* @param idCpu Virtual CPU ID argument. Must be NIL_VMCPUID if pVM
1431	* is NIL_RTR0PTR, and may be NIL_VMCPUID if it isn't
1432	* @param enmOperation Which operation to execute.
1433	* @param pReqHdr This points to a SUPVMMR0REQHDR packet. Optional.
1434	* The support driver validates this if it's present.
1435	* @param u64Arg Some simple constant argument.
1436	* @param pSession The session of the caller.
1437	*
1438	* @remarks Assume called with interrupts _enabled_.
1439	*/
1440	static int vmmR0EntryExWorker(PGVM pGVM, PVM pVM, VMCPUID idCpu, VMMR0OPERATION enmOperation,
1441	PSUPVMMR0REQHDR pReqHdr, uint64_t u64Arg, PSUPDRVSESSION pSession)
1442	{
1443	/*
1444	* Validate pGVM, pVM and idCpu for consistency and validity.
1445	*/
1446	if ( pGVM != NULL
1447	\|\| pVM != NULL)
1448	{
1449	if (RT_LIKELY( RT_VALID_PTR(pGVM)
1450	&& RT_VALID_PTR(pVM)
1451	&& ((uintptr_t)pVM & PAGE_OFFSET_MASK) == 0))
1452	{ /* likely */ }
1453	else
1454	{
1455	SUPR0Printf("vmmR0EntryExWorker: Invalid pGVM=%p and/or pVM=%p! (op=%d)\n", pGVM, pVM, enmOperation);
1456	return VERR_INVALID_POINTER;
1457	}
1458
1459	if (RT_LIKELY(pGVM->pVM == pVM))
1460	{ /* likely */ }
1461	else
1462	{
1463	SUPR0Printf("vmmR0EntryExWorker: pVM mismatch: got %p, pGVM->pVM=%p\n", pVM, pGVM->pVM);
1464	return VERR_INVALID_PARAMETER;
1465	}
1466
1467	if (RT_LIKELY(idCpu == NIL_VMCPUID \|\| idCpu < pGVM->cCpus))
1468	{ /* likely */ }
1469	else
1470	{
1471	SUPR0Printf("vmmR0EntryExWorker: Invalid idCpu %#x (cCpus=%#x)\n", idCpu, pGVM->cCpus);
1472	return VERR_INVALID_PARAMETER;
1473	}
1474
1475	if (RT_LIKELY( pVM->enmVMState >= VMSTATE_CREATING
1476	&& pVM->enmVMState <= VMSTATE_TERMINATED
1477	&& pVM->cCpus == pGVM->cCpus
1478	&& pVM->pSession == pSession
1479	&& pVM->pVMR0 == pVM))
1480	{ /* likely */ }
1481	else
1482	{
1483	SUPR0Printf("vmmR0EntryExWorker: Invalid pVM=%p:{.enmVMState=%d, .cCpus=%#x(==%#x), .pSession=%p(==%p), .pVMR0=%p(==%p)}! (op=%d)\n",
1484	pVM, pVM->enmVMState, pVM->cCpus, pGVM->cCpus, pVM->pSession, pSession, pVM->pVMR0, pVM, enmOperation);
1485	return VERR_INVALID_POINTER;
1486	}
1487	}
1488	else if (RT_LIKELY(idCpu == NIL_VMCPUID))
1489	{ /* likely */ }
1490	else
1491	{
1492	SUPR0Printf("vmmR0EntryExWorker: Invalid idCpu=%u\n", idCpu);
1493	return VERR_INVALID_PARAMETER;
1494	}
1495
1496	/*
1497	* SMAP fun.
1498	*/
1499	VMM_CHECK_SMAP_SETUP();
1500	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
1501
1502	/*
1503	* Process the request.
1504	*/
1505	int rc;
1506	switch (enmOperation)
1507	{
1508	/*
1509	* GVM requests
1510	*/
1511	case VMMR0_DO_GVMM_CREATE_VM:
1512	if (pGVM == NULL && pVM == NULL && u64Arg == 0 && idCpu == NIL_VMCPUID)
1513	rc = GVMMR0CreateVMReq((PGVMMCREATEVMREQ)pReqHdr, pSession);
1514	else
1515	rc = VERR_INVALID_PARAMETER;
1516	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
1517	break;
1518
1519	case VMMR0_DO_GVMM_DESTROY_VM:
1520	if (pReqHdr == NULL && u64Arg == 0)
1521	rc = GVMMR0DestroyVM(pGVM, pVM);
1522	else
1523	rc = VERR_INVALID_PARAMETER;
1524	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
1525	break;
1526
1527	case VMMR0_DO_GVMM_REGISTER_VMCPU:
1528	if (pGVM != NULL && pVM != NULL)
1529	rc = GVMMR0RegisterVCpu(pGVM, pVM, idCpu);
1530	else
1531	rc = VERR_INVALID_PARAMETER;
1532	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1533	break;
1534
1535	case VMMR0_DO_GVMM_DEREGISTER_VMCPU:
1536	if (pGVM != NULL && pVM != NULL)
1537	rc = GVMMR0DeregisterVCpu(pGVM, pVM, idCpu);
1538	else
1539	rc = VERR_INVALID_PARAMETER;
1540	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1541	break;
1542
1543	case VMMR0_DO_GVMM_SCHED_HALT:
1544	if (pReqHdr)
1545	return VERR_INVALID_PARAMETER;
1546	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1547	rc = GVMMR0SchedHalt(pGVM, pVM, idCpu, u64Arg);
1548	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1549	break;
1550
1551	case VMMR0_DO_GVMM_SCHED_WAKE_UP:
1552	if (pReqHdr \|\| u64Arg)
1553	return VERR_INVALID_PARAMETER;
1554	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1555	rc = GVMMR0SchedWakeUp(pGVM, pVM, idCpu);
1556	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1557	break;
1558
1559	case VMMR0_DO_GVMM_SCHED_POKE:
1560	if (pReqHdr \|\| u64Arg)
1561	return VERR_INVALID_PARAMETER;
1562	rc = GVMMR0SchedPoke(pGVM, pVM, idCpu);
1563	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1564	break;
1565
1566	case VMMR0_DO_GVMM_SCHED_WAKE_UP_AND_POKE_CPUS:
1567	if (u64Arg)
1568	return VERR_INVALID_PARAMETER;
1569	rc = GVMMR0SchedWakeUpAndPokeCpusReq(pGVM, pVM, (PGVMMSCHEDWAKEUPANDPOKECPUSREQ)pReqHdr);
1570	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1571	break;
1572
1573	case VMMR0_DO_GVMM_SCHED_POLL:
1574	if (pReqHdr \|\| u64Arg > 1)
1575	return VERR_INVALID_PARAMETER;
1576	rc = GVMMR0SchedPoll(pGVM, pVM, idCpu, !!u64Arg);
1577	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1578	break;
1579
1580	case VMMR0_DO_GVMM_QUERY_STATISTICS:
1581	if (u64Arg)
1582	return VERR_INVALID_PARAMETER;
1583	rc = GVMMR0QueryStatisticsReq(pGVM, pVM, (PGVMMQUERYSTATISTICSSREQ)pReqHdr, pSession);
1584	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1585	break;
1586
1587	case VMMR0_DO_GVMM_RESET_STATISTICS:
1588	if (u64Arg)
1589	return VERR_INVALID_PARAMETER;
1590	rc = GVMMR0ResetStatisticsReq(pGVM, pVM, (PGVMMRESETSTATISTICSSREQ)pReqHdr, pSession);
1591	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1592	break;
1593
1594	/*
1595	* Initialize the R0 part of a VM instance.
1596	*/
1597	case VMMR0_DO_VMMR0_INIT:
1598	rc = vmmR0InitVM(pGVM, pVM, RT_LODWORD(u64Arg), RT_HIDWORD(u64Arg));
1599	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1600	break;
1601
1602	/*
1603	* Does EMT specific ring-0 init.
1604	*/
1605	case VMMR0_DO_VMMR0_INIT_EMT:
1606	rc = vmmR0InitVMEmt(pGVM, pVM, idCpu);
1607	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1608	break;
1609
1610	/*
1611	* Terminate the R0 part of a VM instance.
1612	*/
1613	case VMMR0_DO_VMMR0_TERM:
1614	rc = VMMR0TermVM(pGVM, pVM, 0 /idCpu/);
1615	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1616	break;
1617
1618	/*
1619	* Attempt to enable hm mode and check the current setting.
1620	*/
1621	case VMMR0_DO_HM_ENABLE:
1622	rc = HMR0EnableAllCpus(pVM);
1623	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1624	break;
1625
1626	/*
1627	* Setup the hardware accelerated session.
1628	*/
1629	case VMMR0_DO_HM_SETUP_VM:
1630	rc = HMR0SetupVM(pVM);
1631	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1632	break;
1633
1634	/*
1635	* Switch to RC to execute Hypervisor function.
1636	*/
1637	case VMMR0_DO_CALL_HYPERVISOR:
1638	{
1639	#ifdef VBOX_WITH_RAW_MODE
1640	/*
1641	* Validate input / context.
1642	*/
1643	if (RT_UNLIKELY(idCpu != 0))
1644	return VERR_INVALID_CPU_ID;
1645	if (RT_UNLIKELY(pVM->cCpus != 1))
1646	return VERR_INVALID_PARAMETER;
1647	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1648	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
1649	if (RT_UNLIKELY(!PGMGetHyperCR3(pVCpu)))
1650	return VERR_PGM_NO_CR3_SHADOW_ROOT;
1651	# endif
1652	if (RT_FAILURE(g_rcRawModeUsability))
1653	return g_rcRawModeUsability;
1654
1655	/*
1656	* Disable interrupts.
1657	*/
1658	RTCCUINTREG fFlags = ASMIntDisableFlags();
1659
1660	/*
1661	* Get the host CPU identifiers, make sure they are valid and that
1662	* we've got a TSC delta for the CPU.
1663	*/
1664	RTCPUID idHostCpu;
1665	uint32_t iHostCpuSet = RTMpCurSetIndexAndId(&idHostCpu);
1666	if (RT_UNLIKELY(iHostCpuSet >= RTCPUSET_MAX_CPUS))
1667	{
1668	ASMSetFlags(fFlags);
1669	return VERR_INVALID_CPU_INDEX;
1670	}
1671	if (RT_UNLIKELY(!SUPIsTscDeltaAvailableForCpuSetIndex(iHostCpuSet)))
1672	{
1673	ASMSetFlags(fFlags);
1674	rc = SUPR0TscDeltaMeasureBySetIndex(pVM->pSession, iHostCpuSet, 0 /fFlags/,
1675	2 /cMsWaitRetry/, 5RT_MS_1SEC /cMsWaitThread*/,
1676	0 /default cTries/);
1677	if (RT_FAILURE(rc) && rc != VERR_CPU_OFFLINE)
1678	{
1679	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1680	return rc;
1681	}
1682	}
1683
1684	/*
1685	* Commit the CPU identifiers.
1686	*/
1687	# ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
1688	CPUMR0SetLApic(pVCpu, iHostCpuSet);
1689	# endif
1690	pVCpu->iHostCpuSet = iHostCpuSet;
1691	ASMAtomicWriteU32(&pVCpu->idHostCpu, idHostCpu);
1692
1693	/*
1694	* We might need to disable VT-x if the active switcher turns off paging.
1695	*/
1696	bool fVTxDisabled;
1697	rc = HMR0EnterSwitcher(pVM, pVM->vmm.s.enmSwitcher, &fVTxDisabled);
1698	if (RT_SUCCESS(rc))
1699	{
1700	/*
1701	* Go through the wormhole...
1702	*/
1703	rc = pVM->vmm.s.pfnR0ToRawMode(pVM);
1704
1705	/*
1706	* Re-enable VT-x before we dispatch any pending host interrupts.
1707	*/
1708	HMR0LeaveSwitcher(pVM, fVTxDisabled);
1709
1710	if ( rc == VINF_EM_RAW_INTERRUPT
1711	\|\| rc == VINF_EM_RAW_INTERRUPT_HYPER)
1712	TRPMR0DispatchHostInterrupt(pVM);
1713	}
1714
1715	/*
1716	* Invalidate the host CPU identifiers as we restore interrupts.
1717	*/
1718	pVCpu->iHostCpuSet = UINT32_MAX;
1719	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
1720	ASMSetFlags(fFlags);
1721
1722	#else /* !VBOX_WITH_RAW_MODE */
1723	rc = VERR_RAW_MODE_NOT_SUPPORTED;
1724	#endif
1725	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1726	break;
1727	}
1728
1729	/*
1730	* PGM wrappers.
1731	*/
1732	case VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES:
1733	if (idCpu == NIL_VMCPUID)
1734	return VERR_INVALID_CPU_ID;
1735	rc = PGMR0PhysAllocateHandyPages(pGVM, pVM, idCpu);
1736	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1737	break;
1738
1739	case VMMR0_DO_PGM_FLUSH_HANDY_PAGES:
1740	if (idCpu == NIL_VMCPUID)
1741	return VERR_INVALID_CPU_ID;
1742	rc = PGMR0PhysFlushHandyPages(pGVM, pVM, idCpu);
1743	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1744	break;
1745
1746	case VMMR0_DO_PGM_ALLOCATE_LARGE_HANDY_PAGE:
1747	if (idCpu == NIL_VMCPUID)
1748	return VERR_INVALID_CPU_ID;
1749	rc = PGMR0PhysAllocateLargeHandyPage(pGVM, pVM, idCpu);
1750	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1751	break;
1752
1753	case VMMR0_DO_PGM_PHYS_SETUP_IOMMU:
1754	if (idCpu != 0)
1755	return VERR_INVALID_CPU_ID;
1756	rc = PGMR0PhysSetupIoMmu(pGVM, pVM);
1757	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1758	break;
1759
1760	/*
1761	* GMM wrappers.
1762	*/
1763	case VMMR0_DO_GMM_INITIAL_RESERVATION:
1764	if (u64Arg)
1765	return VERR_INVALID_PARAMETER;
1766	rc = GMMR0InitialReservationReq(pGVM, pVM, idCpu, (PGMMINITIALRESERVATIONREQ)pReqHdr);
1767	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1768	break;
1769
1770	case VMMR0_DO_GMM_UPDATE_RESERVATION:
1771	if (u64Arg)
1772	return VERR_INVALID_PARAMETER;
1773	rc = GMMR0UpdateReservationReq(pGVM, pVM, idCpu, (PGMMUPDATERESERVATIONREQ)pReqHdr);
1774	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1775	break;
1776
1777	case VMMR0_DO_GMM_ALLOCATE_PAGES:
1778	if (u64Arg)
1779	return VERR_INVALID_PARAMETER;
1780	rc = GMMR0AllocatePagesReq(pGVM, pVM, idCpu, (PGMMALLOCATEPAGESREQ)pReqHdr);
1781	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1782	break;
1783
1784	case VMMR0_DO_GMM_FREE_PAGES:
1785	if (u64Arg)
1786	return VERR_INVALID_PARAMETER;
1787	rc = GMMR0FreePagesReq(pGVM, pVM, idCpu, (PGMMFREEPAGESREQ)pReqHdr);
1788	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1789	break;
1790
1791	case VMMR0_DO_GMM_FREE_LARGE_PAGE:
1792	if (u64Arg)
1793	return VERR_INVALID_PARAMETER;
1794	rc = GMMR0FreeLargePageReq(pGVM, pVM, idCpu, (PGMMFREELARGEPAGEREQ)pReqHdr);
1795	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1796	break;
1797
1798	case VMMR0_DO_GMM_QUERY_HYPERVISOR_MEM_STATS:
1799	if (u64Arg)
1800	return VERR_INVALID_PARAMETER;
1801	rc = GMMR0QueryHypervisorMemoryStatsReq((PGMMMEMSTATSREQ)pReqHdr);
1802	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1803	break;
1804
1805	case VMMR0_DO_GMM_QUERY_MEM_STATS:
1806	if (idCpu == NIL_VMCPUID)
1807	return VERR_INVALID_CPU_ID;
1808	if (u64Arg)
1809	return VERR_INVALID_PARAMETER;
1810	rc = GMMR0QueryMemoryStatsReq(pGVM, pVM, idCpu, (PGMMMEMSTATSREQ)pReqHdr);
1811	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1812	break;
1813
1814	case VMMR0_DO_GMM_BALLOONED_PAGES:
1815	if (u64Arg)
1816	return VERR_INVALID_PARAMETER;
1817	rc = GMMR0BalloonedPagesReq(pGVM, pVM, idCpu, (PGMMBALLOONEDPAGESREQ)pReqHdr);
1818	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1819	break;
1820
1821	case VMMR0_DO_GMM_MAP_UNMAP_CHUNK:
1822	if (u64Arg)
1823	return VERR_INVALID_PARAMETER;
1824	rc = GMMR0MapUnmapChunkReq(pGVM, pVM, (PGMMMAPUNMAPCHUNKREQ)pReqHdr);
1825	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1826	break;
1827
1828	case VMMR0_DO_GMM_SEED_CHUNK:
1829	if (pReqHdr)
1830	return VERR_INVALID_PARAMETER;
1831	rc = GMMR0SeedChunk(pGVM, pVM, idCpu, (RTR3PTR)u64Arg);
1832	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1833	break;
1834
1835	case VMMR0_DO_GMM_REGISTER_SHARED_MODULE:
1836	if (idCpu == NIL_VMCPUID)
1837	return VERR_INVALID_CPU_ID;
1838	if (u64Arg)
1839	return VERR_INVALID_PARAMETER;
1840	rc = GMMR0RegisterSharedModuleReq(pGVM, pVM, idCpu, (PGMMREGISTERSHAREDMODULEREQ)pReqHdr);
1841	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1842	break;
1843
1844	case VMMR0_DO_GMM_UNREGISTER_SHARED_MODULE:
1845	if (idCpu == NIL_VMCPUID)
1846	return VERR_INVALID_CPU_ID;
1847	if (u64Arg)
1848	return VERR_INVALID_PARAMETER;
1849	rc = GMMR0UnregisterSharedModuleReq(pGVM, pVM, idCpu, (PGMMUNREGISTERSHAREDMODULEREQ)pReqHdr);
1850	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1851	break;
1852
1853	case VMMR0_DO_GMM_RESET_SHARED_MODULES:
1854	if (idCpu == NIL_VMCPUID)
1855	return VERR_INVALID_CPU_ID;
1856	if ( u64Arg
1857	\|\| pReqHdr)
1858	return VERR_INVALID_PARAMETER;
1859	rc = GMMR0ResetSharedModules(pGVM, pVM, idCpu);
1860	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1861	break;
1862
1863	#ifdef VBOX_WITH_PAGE_SHARING
1864	case VMMR0_DO_GMM_CHECK_SHARED_MODULES:
1865	{
1866	if (idCpu == NIL_VMCPUID)
1867	return VERR_INVALID_CPU_ID;
1868	if ( u64Arg
1869	\|\| pReqHdr)
1870	return VERR_INVALID_PARAMETER;
1871	rc = GMMR0CheckSharedModules(pGVM, pVM, idCpu);
1872	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1873	break;
1874	}
1875	#endif
1876
1877	#if defined(VBOX_STRICT) && HC_ARCH_BITS == 64
1878	case VMMR0_DO_GMM_FIND_DUPLICATE_PAGE:
1879	if (u64Arg)
1880	return VERR_INVALID_PARAMETER;
1881	rc = GMMR0FindDuplicatePageReq(pGVM, pVM, (PGMMFINDDUPLICATEPAGEREQ)pReqHdr);
1882	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1883	break;
1884	#endif
1885
1886	case VMMR0_DO_GMM_QUERY_STATISTICS:
1887	if (u64Arg)
1888	return VERR_INVALID_PARAMETER;
1889	rc = GMMR0QueryStatisticsReq(pGVM, pVM, (PGMMQUERYSTATISTICSSREQ)pReqHdr);
1890	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1891	break;
1892
1893	case VMMR0_DO_GMM_RESET_STATISTICS:
1894	if (u64Arg)
1895	return VERR_INVALID_PARAMETER;
1896	rc = GMMR0ResetStatisticsReq(pGVM, pVM, (PGMMRESETSTATISTICSSREQ)pReqHdr);
1897	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1898	break;
1899
1900	/*
1901	* A quick GCFGM mock-up.
1902	*/
1903	/** @todo GCFGM with proper access control, ring-3 management interface and all that. */
1904	case VMMR0_DO_GCFGM_SET_VALUE:
1905	case VMMR0_DO_GCFGM_QUERY_VALUE:
1906	{
1907	if (pGVM \|\| pVM \|\| !pReqHdr \|\| u64Arg \|\| idCpu != NIL_VMCPUID)
1908	return VERR_INVALID_PARAMETER;
1909	PGCFGMVALUEREQ pReq = (PGCFGMVALUEREQ)pReqHdr;
1910	if (pReq->Hdr.cbReq != sizeof(*pReq))
1911	return VERR_INVALID_PARAMETER;
1912	if (enmOperation == VMMR0_DO_GCFGM_SET_VALUE)
1913	{
1914	rc = GVMMR0SetConfig(pReq->pSession, &pReq->szName[0], pReq->u64Value);
1915	//if (rc == VERR_CFGM_VALUE_NOT_FOUND)
1916	// rc = GMMR0SetConfig(pReq->pSession, &pReq->szName[0], pReq->u64Value);
1917	}
1918	else
1919	{
1920	rc = GVMMR0QueryConfig(pReq->pSession, &pReq->szName[0], &pReq->u64Value);
1921	//if (rc == VERR_CFGM_VALUE_NOT_FOUND)
1922	// rc = GMMR0QueryConfig(pReq->pSession, &pReq->szName[0], &pReq->u64Value);
1923	}
1924	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1925	break;
1926	}
1927
1928	/*
1929	* PDM Wrappers.
1930	*/
1931	case VMMR0_DO_PDM_DRIVER_CALL_REQ_HANDLER:
1932	{
1933	if (!pReqHdr \|\| u64Arg \|\| idCpu != NIL_VMCPUID)
1934	return VERR_INVALID_PARAMETER;
1935	rc = PDMR0DriverCallReqHandler(pGVM, pVM, (PPDMDRIVERCALLREQHANDLERREQ)pReqHdr);
1936	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1937	break;
1938	}
1939
1940	case VMMR0_DO_PDM_DEVICE_CALL_REQ_HANDLER:
1941	{
1942	if (!pReqHdr \|\| u64Arg \|\| idCpu != NIL_VMCPUID)
1943	return VERR_INVALID_PARAMETER;
1944	rc = PDMR0DeviceCallReqHandler(pGVM, pVM, (PPDMDEVICECALLREQHANDLERREQ)pReqHdr);
1945	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1946	break;
1947	}
1948
1949	/*
1950	* Requests to the internal networking service.
1951	*/
1952	case VMMR0_DO_INTNET_OPEN:
1953	{
1954	PINTNETOPENREQ pReq = (PINTNETOPENREQ)pReqHdr;
1955	if (u64Arg \|\| !pReq \|\| !vmmR0IsValidSession(pVM, pReq->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
1956	return VERR_INVALID_PARAMETER;
1957	rc = IntNetR0OpenReq(pSession, pReq);
1958	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1959	break;
1960	}
1961
1962	case VMMR0_DO_INTNET_IF_CLOSE:
1963	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFCLOSEREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
1964	return VERR_INVALID_PARAMETER;
1965	rc = IntNetR0IfCloseReq(pSession, (PINTNETIFCLOSEREQ)pReqHdr);
1966	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1967	break;
1968
1969
1970	case VMMR0_DO_INTNET_IF_GET_BUFFER_PTRS:
1971	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFGETBUFFERPTRSREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
1972	return VERR_INVALID_PARAMETER;
1973	rc = IntNetR0IfGetBufferPtrsReq(pSession, (PINTNETIFGETBUFFERPTRSREQ)pReqHdr);
1974	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1975	break;
1976
1977	case VMMR0_DO_INTNET_IF_SET_PROMISCUOUS_MODE:
1978	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFSETPROMISCUOUSMODEREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
1979	return VERR_INVALID_PARAMETER;
1980	rc = IntNetR0IfSetPromiscuousModeReq(pSession, (PINTNETIFSETPROMISCUOUSMODEREQ)pReqHdr);
1981	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1982	break;
1983
1984	case VMMR0_DO_INTNET_IF_SET_MAC_ADDRESS:
1985	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFSETMACADDRESSREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
1986	return VERR_INVALID_PARAMETER;
1987	rc = IntNetR0IfSetMacAddressReq(pSession, (PINTNETIFSETMACADDRESSREQ)pReqHdr);
1988	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1989	break;
1990
1991	case VMMR0_DO_INTNET_IF_SET_ACTIVE:
1992	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFSETACTIVEREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
1993	return VERR_INVALID_PARAMETER;
1994	rc = IntNetR0IfSetActiveReq(pSession, (PINTNETIFSETACTIVEREQ)pReqHdr);
1995	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1996	break;
1997
1998	case VMMR0_DO_INTNET_IF_SEND:
1999	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFSENDREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2000	return VERR_INVALID_PARAMETER;
2001	rc = IntNetR0IfSendReq(pSession, (PINTNETIFSENDREQ)pReqHdr);
2002	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2003	break;
2004
2005	case VMMR0_DO_INTNET_IF_WAIT:
2006	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFWAITREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2007	return VERR_INVALID_PARAMETER;
2008	rc = IntNetR0IfWaitReq(pSession, (PINTNETIFWAITREQ)pReqHdr);
2009	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2010	break;
2011
2012	case VMMR0_DO_INTNET_IF_ABORT_WAIT:
2013	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFWAITREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2014	return VERR_INVALID_PARAMETER;
2015	rc = IntNetR0IfAbortWaitReq(pSession, (PINTNETIFABORTWAITREQ)pReqHdr);
2016	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2017	break;
2018
2019	#ifdef VBOX_WITH_PCI_PASSTHROUGH
2020	/*
2021	* Requests to host PCI driver service.
2022	*/
2023	case VMMR0_DO_PCIRAW_REQ:
2024	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PPCIRAWSENDREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2025	return VERR_INVALID_PARAMETER;
2026	rc = PciRawR0ProcessReq(pGVM, pVM, pSession, (PPCIRAWSENDREQ)pReqHdr);
2027	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2028	break;
2029	#endif
2030
2031	/*
2032	* NEM requests.
2033	*/
2034	#ifdef VBOX_WITH_NEM_R0
2035	# if defined(RT_ARCH_AMD64) && defined(RT_OS_WINDOWS)
2036	case VMMR0_DO_NEM_INIT_VM:
2037	if (u64Arg \|\| pReqHdr \|\| idCpu != 0)
2038	return VERR_INVALID_PARAMETER;
2039	rc = NEMR0InitVM(pGVM, pVM);
2040	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2041	break;
2042
2043	case VMMR0_DO_NEM_INIT_VM_PART_2:
2044	if (u64Arg \|\| pReqHdr \|\| idCpu != 0)
2045	return VERR_INVALID_PARAMETER;
2046	rc = NEMR0InitVMPart2(pGVM, pVM);
2047	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2048	break;
2049
2050	case VMMR0_DO_NEM_MAP_PAGES:
2051	if (u64Arg \|\| pReqHdr \|\| idCpu == NIL_VMCPUID)
2052	return VERR_INVALID_PARAMETER;
2053	rc = NEMR0MapPages(pGVM, pVM, idCpu);
2054	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2055	break;
2056
2057	case VMMR0_DO_NEM_UNMAP_PAGES:
2058	if (u64Arg \|\| pReqHdr \|\| idCpu == NIL_VMCPUID)
2059	return VERR_INVALID_PARAMETER;
2060	rc = NEMR0UnmapPages(pGVM, pVM, idCpu);
2061	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2062	break;
2063
2064	case VMMR0_DO_NEM_EXPORT_STATE:
2065	if (u64Arg \|\| pReqHdr \|\| idCpu == NIL_VMCPUID)
2066	return VERR_INVALID_PARAMETER;
2067	rc = NEMR0ExportState(pGVM, pVM, idCpu);
2068	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2069	break;
2070
2071	case VMMR0_DO_NEM_IMPORT_STATE:
2072	if (pReqHdr \|\| idCpu == NIL_VMCPUID)
2073	return VERR_INVALID_PARAMETER;
2074	rc = NEMR0ImportState(pGVM, pVM, idCpu, u64Arg);
2075	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2076	break;
2077
2078	case VMMR0_DO_NEM_QUERY_CPU_TICK:
2079	if (u64Arg \|\| pReqHdr \|\| idCpu == NIL_VMCPUID)
2080	return VERR_INVALID_PARAMETER;
2081	rc = NEMR0QueryCpuTick(pGVM, pVM, idCpu);
2082	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2083	break;
2084
2085	case VMMR0_DO_NEM_RESUME_CPU_TICK_ON_ALL:
2086	if (pReqHdr \|\| idCpu == NIL_VMCPUID)
2087	return VERR_INVALID_PARAMETER;
2088	rc = NEMR0ResumeCpuTickOnAll(pGVM, pVM, idCpu, u64Arg);
2089	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2090	break;
2091
2092	case VMMR0_DO_NEM_UPDATE_STATISTICS:
2093	if (u64Arg \|\| pReqHdr)
2094	return VERR_INVALID_PARAMETER;
2095	rc = NEMR0UpdateStatistics(pGVM, pVM, idCpu);
2096	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2097	break;
2098
2099	# if 1 && defined(DEBUG_bird)
2100	case VMMR0_DO_NEM_EXPERIMENT:
2101	if (pReqHdr)
2102	return VERR_INVALID_PARAMETER;
2103	rc = NEMR0DoExperiment(pGVM, pVM, idCpu, u64Arg);
2104	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2105	break;
2106	# endif
2107	# endif
2108	#endif
2109
2110	/*
2111	* For profiling.
2112	*/
2113	case VMMR0_DO_NOP:
2114	case VMMR0_DO_SLOW_NOP:
2115	return VINF_SUCCESS;
2116
2117	/*
2118	* For testing Ring-0 APIs invoked in this environment.
2119	*/
2120	case VMMR0_DO_TESTS:
2121	/** @todo make new test */
2122	return VINF_SUCCESS;
2123
2124
2125	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
2126	case VMMR0_DO_TEST_SWITCHER3264:
2127	if (idCpu == NIL_VMCPUID)
2128	return VERR_INVALID_CPU_ID;
2129	rc = HMR0TestSwitcher3264(pVM);
2130	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2131	break;
2132	#endif
2133	default:
2134	/*
2135	* We're returning VERR_NOT_SUPPORT here so we've got something else
2136	* than -1 which the interrupt gate glue code might return.
2137	*/
2138	Log(("operation %#x is not supported\n", enmOperation));
2139	return VERR_NOT_SUPPORTED;
2140	}
2141	return rc;
2142	}
2143
2144
2145	/**
2146	* Argument for vmmR0EntryExWrapper containing the arguments for VMMR0EntryEx.
2147	*/
2148	typedef struct VMMR0ENTRYEXARGS
2149	{
2150	PGVM pGVM;
2151	PVM pVM;
2152	VMCPUID idCpu;
2153	VMMR0OPERATION enmOperation;
2154	PSUPVMMR0REQHDR pReq;
2155	uint64_t u64Arg;
2156	PSUPDRVSESSION pSession;
2157	} VMMR0ENTRYEXARGS;
2158	/** Pointer to a vmmR0EntryExWrapper argument package. */
2159	typedef VMMR0ENTRYEXARGS *PVMMR0ENTRYEXARGS;
2160
2161	/**
2162	* This is just a longjmp wrapper function for VMMR0EntryEx calls.
2163	*
2164	* @returns VBox status code.
2165	* @param pvArgs The argument package
2166	*/
2167	static DECLCALLBACK(int) vmmR0EntryExWrapper(void *pvArgs)
2168	{
2169	return vmmR0EntryExWorker(((PVMMR0ENTRYEXARGS)pvArgs)->pGVM,
2170	((PVMMR0ENTRYEXARGS)pvArgs)->pVM,
2171	((PVMMR0ENTRYEXARGS)pvArgs)->idCpu,
2172	((PVMMR0ENTRYEXARGS)pvArgs)->enmOperation,
2173	((PVMMR0ENTRYEXARGS)pvArgs)->pReq,
2174	((PVMMR0ENTRYEXARGS)pvArgs)->u64Arg,
2175	((PVMMR0ENTRYEXARGS)pvArgs)->pSession);
2176	}
2177
2178
2179	/**
2180	* The Ring 0 entry point, called by the support library (SUP).
2181	*
2182	* @returns VBox status code.
2183	* @param pGVM The global (ring-0) VM structure.
2184	* @param pVM The cross context VM structure.
2185	* @param idCpu Virtual CPU ID argument. Must be NIL_VMCPUID if pVM
2186	* is NIL_RTR0PTR, and may be NIL_VMCPUID if it isn't
2187	* @param enmOperation Which operation to execute.
2188	* @param pReq Pointer to the SUPVMMR0REQHDR packet. Optional.
2189	* @param u64Arg Some simple constant argument.
2190	* @param pSession The session of the caller.
2191	* @remarks Assume called with interrupts _enabled_.
2192	*/
2193	VMMR0DECL(int) VMMR0EntryEx(PGVM pGVM, PVM pVM, VMCPUID idCpu, VMMR0OPERATION enmOperation,
2194	PSUPVMMR0REQHDR pReq, uint64_t u64Arg, PSUPDRVSESSION pSession)
2195	{
2196	/*
2197	* Requests that should only happen on the EMT thread will be
2198	* wrapped in a setjmp so we can assert without causing trouble.
2199	*/
2200	if ( pVM != NULL
2201	&& pGVM != NULL
2202	&& idCpu < pGVM->cCpus
2203	&& pVM->pVMR0 != NULL)
2204	{
2205	switch (enmOperation)
2206	{
2207	/* These might/will be called before VMMR3Init. */
2208	case VMMR0_DO_GMM_INITIAL_RESERVATION:
2209	case VMMR0_DO_GMM_UPDATE_RESERVATION:
2210	case VMMR0_DO_GMM_ALLOCATE_PAGES:
2211	case VMMR0_DO_GMM_FREE_PAGES:
2212	case VMMR0_DO_GMM_BALLOONED_PAGES:
2213	/* On the mac we might not have a valid jmp buf, so check these as well. */
2214	case VMMR0_DO_VMMR0_INIT:
2215	case VMMR0_DO_VMMR0_TERM:
2216	{
2217	PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
2218	PVMCPU pVCpu = &pVM->aCpus[idCpu];
2219	RTNATIVETHREAD hNativeThread = RTThreadNativeSelf();
2220	if (RT_LIKELY( pGVCpu->hEMT == hNativeThread
2221	&& pVCpu->hNativeThreadR0 == hNativeThread))
2222	{
2223	if (!pVCpu->vmm.s.CallRing3JmpBufR0.pvSavedStack)
2224	break;
2225
2226	/** @todo validate this EMT claim... GVM knows. */
2227	VMMR0ENTRYEXARGS Args;
2228	Args.pGVM = pGVM;
2229	Args.pVM = pVM;
2230	Args.idCpu = idCpu;
2231	Args.enmOperation = enmOperation;
2232	Args.pReq = pReq;
2233	Args.u64Arg = u64Arg;
2234	Args.pSession = pSession;
2235	return vmmR0CallRing3SetJmpEx(&pVCpu->vmm.s.CallRing3JmpBufR0, vmmR0EntryExWrapper, &Args);
2236	}
2237	return VERR_VM_THREAD_NOT_EMT;
2238	}
2239
2240	default:
2241	break;
2242	}
2243	}
2244	return vmmR0EntryExWorker(pGVM, pVM, idCpu, enmOperation, pReq, u64Arg, pSession);
2245	}
2246
2247
2248	/**
2249	* Checks whether we've armed the ring-0 long jump machinery.
2250	*
2251	* @returns @c true / @c false
2252	* @param pVCpu The cross context virtual CPU structure.
2253	* @thread EMT
2254	* @sa VMMIsLongJumpArmed
2255	*/
2256	VMMR0_INT_DECL(bool) VMMR0IsLongJumpArmed(PVMCPU pVCpu)
2257	{
2258	#ifdef RT_ARCH_X86
2259	return pVCpu->vmm.s.CallRing3JmpBufR0.eip
2260	&& !pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call;
2261	#else
2262	return pVCpu->vmm.s.CallRing3JmpBufR0.rip
2263	&& !pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call;
2264	#endif
2265	}
2266
2267
2268	/**
2269	* Checks whether we've done a ring-3 long jump.
2270	*
2271	* @returns @c true / @c false
2272	* @param pVCpu The cross context virtual CPU structure.
2273	* @thread EMT
2274	*/
2275	VMMR0_INT_DECL(bool) VMMR0IsInRing3LongJump(PVMCPU pVCpu)
2276	{
2277	return pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call;
2278	}
2279
2280
2281	/**
2282	* Internal R0 logger worker: Flush logger.
2283	*
2284	* @param pLogger The logger instance to flush.
2285	* @remark This function must be exported!
2286	*/
2287	VMMR0DECL(void) vmmR0LoggerFlush(PRTLOGGER pLogger)
2288	{
2289	#ifdef LOG_ENABLED
2290	/*
2291	* Convert the pLogger into a VM handle and 'call' back to Ring-3.
2292	* (This is a bit paranoid code.)
2293	*/
2294	PVMMR0LOGGER pR0Logger = (PVMMR0LOGGER)((uintptr_t)pLogger - RT_OFFSETOF(VMMR0LOGGER, Logger));
2295	if ( !VALID_PTR(pR0Logger)
2296	\|\| !VALID_PTR(pR0Logger + 1)
2297	\|\| pLogger->u32Magic != RTLOGGER_MAGIC)
2298	{
2299	# ifdef DEBUG
2300	SUPR0Printf("vmmR0LoggerFlush: pLogger=%p!\n", pLogger);
2301	# endif
2302	return;
2303	}
2304	if (pR0Logger->fFlushingDisabled)
2305	return; /* quietly */
2306
2307	PVM pVM = pR0Logger->pVM;
2308	if ( !VALID_PTR(pVM)
2309	\|\| pVM->pVMR0 != pVM)
2310	{
2311	# ifdef DEBUG
2312	SUPR0Printf("vmmR0LoggerFlush: pVM=%p! pVMR0=%p! pLogger=%p\n", pVM, pVM->pVMR0, pLogger);
2313	# endif
2314	return;
2315	}
2316
2317	PVMCPU pVCpu = VMMGetCpu(pVM);
2318	if (pVCpu)
2319	{
2320	/*
2321	* Check that the jump buffer is armed.
2322	*/
2323	# ifdef RT_ARCH_X86
2324	if ( !pVCpu->vmm.s.CallRing3JmpBufR0.eip
2325	\|\| pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call)
2326	# else
2327	if ( !pVCpu->vmm.s.CallRing3JmpBufR0.rip
2328	\|\| pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call)
2329	# endif
2330	{
2331	# ifdef DEBUG
2332	SUPR0Printf("vmmR0LoggerFlush: Jump buffer isn't armed!\n");
2333	# endif
2334	return;
2335	}
2336	VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_VMM_LOGGER_FLUSH, 0);
2337	}
2338	# ifdef DEBUG
2339	else
2340	SUPR0Printf("vmmR0LoggerFlush: invalid VCPU context!\n");
2341	# endif
2342	#else
2343	NOREF(pLogger);
2344	#endif /* LOG_ENABLED */
2345	}
2346
2347	/**
2348	* Internal R0 logger worker: Custom prefix.
2349	*
2350	* @returns Number of chars written.
2351	*
2352	* @param pLogger The logger instance.
2353	* @param pchBuf The output buffer.
2354	* @param cchBuf The size of the buffer.
2355	* @param pvUser User argument (ignored).
2356	*/
2357	VMMR0DECL(size_t) vmmR0LoggerPrefix(PRTLOGGER pLogger, char pchBuf, size_t cchBuf, void pvUser)
2358	{
2359	NOREF(pvUser);
2360	#ifdef LOG_ENABLED
2361	PVMMR0LOGGER pR0Logger = (PVMMR0LOGGER)((uintptr_t)pLogger - RT_OFFSETOF(VMMR0LOGGER, Logger));
2362	if ( !VALID_PTR(pR0Logger)
2363	\|\| !VALID_PTR(pR0Logger + 1)
2364	\|\| pLogger->u32Magic != RTLOGGER_MAGIC
2365	\|\| cchBuf < 2)
2366	return 0;
2367
2368	static const char s_szHex[17] = "0123456789abcdef";
2369	VMCPUID const idCpu = pR0Logger->idCpu;
2370	pchBuf[1] = s_szHex[ idCpu & 15];
2371	pchBuf[0] = s_szHex[(idCpu >> 4) & 15];
2372
2373	return 2;
2374	#else
2375	NOREF(pLogger); NOREF(pchBuf); NOREF(cchBuf);
2376	return 0;
2377	#endif
2378	}
2379
2380	#ifdef LOG_ENABLED
2381
2382	/**
2383	* Disables flushing of the ring-0 debug log.
2384	*
2385	* @param pVCpu The cross context virtual CPU structure.
2386	*/
2387	VMMR0_INT_DECL(void) VMMR0LogFlushDisable(PVMCPU pVCpu)
2388	{
2389	if (pVCpu->vmm.s.pR0LoggerR0)
2390	pVCpu->vmm.s.pR0LoggerR0->fFlushingDisabled = true;
2391	}
2392
2393
2394	/**
2395	* Enables flushing of the ring-0 debug log.
2396	*
2397	* @param pVCpu The cross context virtual CPU structure.
2398	*/
2399	VMMR0_INT_DECL(void) VMMR0LogFlushEnable(PVMCPU pVCpu)
2400	{
2401	if (pVCpu->vmm.s.pR0LoggerR0)
2402	pVCpu->vmm.s.pR0LoggerR0->fFlushingDisabled = false;
2403	}
2404
2405
2406	/**
2407	* Checks if log flushing is disabled or not.
2408	*
2409	* @param pVCpu The cross context virtual CPU structure.
2410	*/
2411	VMMR0_INT_DECL(bool) VMMR0IsLogFlushDisabled(PVMCPU pVCpu)
2412	{
2413	if (pVCpu->vmm.s.pR0LoggerR0)
2414	return pVCpu->vmm.s.pR0LoggerR0->fFlushingDisabled;
2415	return true;
2416	}
2417	#endif /* LOG_ENABLED */
2418
2419	/**
2420	* Jump back to ring-3 if we're the EMT and the longjmp is armed.
2421	*
2422	* @returns true if the breakpoint should be hit, false if it should be ignored.
2423	*/
2424	DECLEXPORT(bool) RTCALL RTAssertShouldPanic(void)
2425	{
2426	#if 0
2427	return true;
2428	#else
2429	PVM pVM = GVMMR0GetVMByEMT(NIL_RTNATIVETHREAD);
2430	if (pVM)
2431	{
2432	PVMCPU pVCpu = VMMGetCpu(pVM);
2433
2434	if (pVCpu)
2435	{
2436	#ifdef RT_ARCH_X86
2437	if ( pVCpu->vmm.s.CallRing3JmpBufR0.eip
2438	&& !pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call)
2439	#else
2440	if ( pVCpu->vmm.s.CallRing3JmpBufR0.rip
2441	&& !pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call)
2442	#endif
2443	{
2444	int rc = VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_VM_R0_ASSERTION, 0);
2445	return RT_FAILURE_NP(rc);
2446	}
2447	}
2448	}
2449	#ifdef RT_OS_LINUX
2450	return true;
2451	#else
2452	return false;
2453	#endif
2454	#endif
2455	}
2456
2457
2458	/**
2459	* Override this so we can push it up to ring-3.
2460	*
2461	* @param pszExpr Expression. Can be NULL.
2462	* @param uLine Location line number.
2463	* @param pszFile Location file name.
2464	* @param pszFunction Location function name.
2465	*/
2466	DECLEXPORT(void) RTCALL RTAssertMsg1Weak(const char pszExpr, unsigned uLine, const char pszFile, const char *pszFunction)
2467	{
2468	/*
2469	* To the log.
2470	*/
2471	LogAlways(("\n!!R0-Assertion Failed!!\n"
2472	"Expression: %s\n"
2473	"Location : %s(%d) %s\n",
2474	pszExpr, pszFile, uLine, pszFunction));
2475
2476	/*
2477	* To the global VMM buffer.
2478	*/
2479	PVM pVM = GVMMR0GetVMByEMT(NIL_RTNATIVETHREAD);
2480	if (pVM)
2481	RTStrPrintf(pVM->vmm.s.szRing0AssertMsg1, sizeof(pVM->vmm.s.szRing0AssertMsg1),
2482	"\n!!R0-Assertion Failed!!\n"
2483	"Expression: %.*s\n"
2484	"Location : %s(%d) %s\n",
2485	sizeof(pVM->vmm.s.szRing0AssertMsg1) / 4 * 3, pszExpr,
2486	pszFile, uLine, pszFunction);
2487
2488	/*
2489	* Continue the normal way.
2490	*/
2491	RTAssertMsg1(pszExpr, uLine, pszFile, pszFunction);
2492	}
2493
2494
2495	/**
2496	* Callback for RTLogFormatV which writes to the ring-3 log port.
2497	* See PFNLOGOUTPUT() for details.
2498	*/
2499	static DECLCALLBACK(size_t) rtLogOutput(void pv, const char pachChars, size_t cbChars)
2500	{
2501	for (size_t i = 0; i < cbChars; i++)
2502	{
2503	LogAlways(("%c", pachChars[i])); NOREF(pachChars);
2504	}
2505
2506	NOREF(pv);
2507	return cbChars;
2508	}
2509
2510
2511	/**
2512	* Override this so we can push it up to ring-3.
2513	*
2514	* @param pszFormat The format string.
2515	* @param va Arguments.
2516	*/
2517	DECLEXPORT(void) RTCALL RTAssertMsg2WeakV(const char *pszFormat, va_list va)
2518	{
2519	va_list vaCopy;
2520
2521	/*
2522	* Push the message to the loggers.
2523	*/
2524	PRTLOGGER pLog = RTLogGetDefaultInstance(); /* Don't initialize it here... */
2525	if (pLog)
2526	{
2527	va_copy(vaCopy, va);
2528	RTLogFormatV(rtLogOutput, pLog, pszFormat, vaCopy);
2529	va_end(vaCopy);
2530	}
2531	pLog = RTLogRelGetDefaultInstance();
2532	if (pLog)
2533	{
2534	va_copy(vaCopy, va);
2535	RTLogFormatV(rtLogOutput, pLog, pszFormat, vaCopy);
2536	va_end(vaCopy);
2537	}
2538
2539	/*
2540	* Push it to the global VMM buffer.
2541	*/
2542	PVM pVM = GVMMR0GetVMByEMT(NIL_RTNATIVETHREAD);
2543	if (pVM)
2544	{
2545	va_copy(vaCopy, va);
2546	RTStrPrintfV(pVM->vmm.s.szRing0AssertMsg2, sizeof(pVM->vmm.s.szRing0AssertMsg2), pszFormat, vaCopy);
2547	va_end(vaCopy);
2548	}
2549
2550	/*
2551	* Continue the normal way.
2552	*/
2553	RTAssertMsg2V(pszFormat, va);
2554	}
2555

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/VMMR0.cpp@ 72747

Download in other formats: