VMMR0.cpp@ 78647

Last change on this file since 78647 was 78431, checked in by vboxsync, 6 years ago
VMM: Started refactoring GVM & VM structures for bugref:9217
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 104.8 KB

Line
1	/* $Id: VMMR0.cpp 78431 2019-05-07 14:01:45Z vboxsync $ */
2	/** @file
3	* VMM - Host Context Ring 0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_VMM
23	#include <VBox/vmm/vmm.h>
24	#include <VBox/sup.h>
25	#include <VBox/vmm/trpm.h>
26	#include <VBox/vmm/cpum.h>
27	#include <VBox/vmm/pdmapi.h>
28	#include <VBox/vmm/pgm.h>
29	#ifdef VBOX_WITH_NEM_R0
30	# include <VBox/vmm/nem.h>
31	#endif
32	#include <VBox/vmm/em.h>
33	#include <VBox/vmm/stam.h>
34	#include <VBox/vmm/tm.h>
35	#include "VMMInternal.h"
36	#include <VBox/vmm/vm.h>
37	#include <VBox/vmm/gvm.h>
38	#ifdef VBOX_WITH_PCI_PASSTHROUGH
39	# include <VBox/vmm/pdmpci.h>
40	#endif
41	#include <VBox/vmm/apic.h>
42
43	#include <VBox/vmm/gvmm.h>
44	#include <VBox/vmm/gmm.h>
45	#include <VBox/vmm/gim.h>
46	#include <VBox/intnet.h>
47	#include <VBox/vmm/hm.h>
48	#include <VBox/param.h>
49	#include <VBox/err.h>
50	#include <VBox/version.h>
51	#include <VBox/log.h>
52
53	#include <iprt/asm-amd64-x86.h>
54	#include <iprt/assert.h>
55	#include <iprt/crc.h>
56	#include <iprt/mp.h>
57	#include <iprt/once.h>
58	#include <iprt/stdarg.h>
59	#include <iprt/string.h>
60	#include <iprt/thread.h>
61	#include <iprt/timer.h>
62	#include <iprt/time.h>
63
64	#include "dtrace/VBoxVMM.h"
65
66
67	#if defined(_MSC_VER) && defined(RT_ARCH_AMD64) /** @todo check this with with VC7! */
68	# pragma intrinsic(_AddressOfReturnAddress)
69	#endif
70
71	#if defined(RT_OS_DARWIN) && ARCH_BITS == 32
72	# error "32-bit darwin is no longer supported. Go back to 4.3 or earlier!"
73	#endif
74
75
76
77	/*********************************************************************************************************************************
78	* Defined Constants And Macros *
79	*********************************************************************************************************************************/
80	/** @def VMM_CHECK_SMAP_SETUP
81	* SMAP check setup. */
82	/** @def VMM_CHECK_SMAP_CHECK
83	* Checks that the AC flag is set if SMAP is enabled. If AC is not set,
84	* it will be logged and @a a_BadExpr is executed. */
85	/** @def VMM_CHECK_SMAP_CHECK2
86	* Checks that the AC flag is set if SMAP is enabled. If AC is not set, it will
87	* be logged, written to the VMs assertion text buffer, and @a a_BadExpr is
88	* executed. */
89	#if defined(VBOX_STRICT) \|\| 1
90	# define VMM_CHECK_SMAP_SETUP() uint32_t const fKernelFeatures = SUPR0GetKernelFeatures()
91	# define VMM_CHECK_SMAP_CHECK(a_BadExpr) \
92	do { \
93	if (fKernelFeatures & SUPKERNELFEATURES_SMAP) \
94	{ \
95	RTCCUINTREG fEflCheck = ASMGetFlags(); \
96	if (RT_LIKELY(fEflCheck & X86_EFL_AC)) \
97	{ /* likely */ } \
98	else \
99	{ \
100	SUPR0Printf("%s, line %d: EFLAGS.AC is clear! (%#x)\n", __FUNCTION__, __LINE__, (uint32_t)fEflCheck); \
101	a_BadExpr; \
102	} \
103	} \
104	} while (0)
105	# define VMM_CHECK_SMAP_CHECK2(a_pVM, a_BadExpr) \
106	do { \
107	if (fKernelFeatures & SUPKERNELFEATURES_SMAP) \
108	{ \
109	RTCCUINTREG fEflCheck = ASMGetFlags(); \
110	if (RT_LIKELY(fEflCheck & X86_EFL_AC)) \
111	{ /* likely */ } \
112	else \
113	{ \
114	SUPR0BadContext((a_pVM) ? (a_pVM)->pSession : NULL, __FILE__, __LINE__, "EFLAGS.AC is zero!"); \
115	RTStrPrintf(pVM->vmm.s.szRing0AssertMsg1, sizeof(pVM->vmm.s.szRing0AssertMsg1), \
116	"%s, line %d: EFLAGS.AC is clear! (%#x)\n", __FUNCTION__, __LINE__, (uint32_t)fEflCheck); \
117	a_BadExpr; \
118	} \
119	} \
120	} while (0)
121	#else
122	# define VMM_CHECK_SMAP_SETUP() uint32_t const fKernelFeatures = 0
123	# define VMM_CHECK_SMAP_CHECK(a_BadExpr) NOREF(fKernelFeatures)
124	# define VMM_CHECK_SMAP_CHECK2(a_pVM, a_BadExpr) NOREF(fKernelFeatures)
125	#endif
126
127
128	/*********************************************************************************************************************************
129	* Internal Functions *
130	*********************************************************************************************************************************/
131	RT_C_DECLS_BEGIN
132	#if defined(RT_ARCH_X86) && (defined(RT_OS_SOLARIS) \|\| defined(RT_OS_FREEBSD))
133	extern uint64_t __udivdi3(uint64_t, uint64_t);
134	extern uint64_t __umoddi3(uint64_t, uint64_t);
135	#endif
136	RT_C_DECLS_END
137
138
139	/*********************************************************************************************************************************
140	* Global Variables *
141	*********************************************************************************************************************************/
142	/** Drag in necessary library bits.
143	* The runtime lives here (in VMMR0.r0) and VBoxDDR0.r0 links against us. /
144	PFNRT g_VMMR0Deps[] =
145	{
146	(PFNRT)RTCrc32,
147	(PFNRT)RTOnce,
148	#if defined(RT_ARCH_X86) && (defined(RT_OS_SOLARIS) \|\| defined(RT_OS_FREEBSD))
149	(PFNRT)__udivdi3,
150	(PFNRT)__umoddi3,
151	#endif
152	NULL
153	};
154
155	#ifdef RT_OS_SOLARIS
156	/* Dependency information for the native solaris loader. */
157	extern "C" { char _depends_on[] = "vboxdrv"; }
158	#endif
159
160	/** The result of SUPR0GetRawModeUsability(), set by ModuleInit(). */
161	int g_rcRawModeUsability = VINF_SUCCESS;
162
163
164	/**
165	* Initialize the module.
166	* This is called when we're first loaded.
167	*
168	* @returns 0 on success.
169	* @returns VBox status on failure.
170	* @param hMod Image handle for use in APIs.
171	*/
172	DECLEXPORT(int) ModuleInit(void *hMod)
173	{
174	VMM_CHECK_SMAP_SETUP();
175	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
176
177	#ifdef VBOX_WITH_DTRACE_R0
178	/*
179	* The first thing to do is register the static tracepoints.
180	* (Deregistration is automatic.)
181	*/
182	int rc2 = SUPR0TracerRegisterModule(hMod, &g_VTGObjHeader);
183	if (RT_FAILURE(rc2))
184	return rc2;
185	#endif
186	LogFlow(("ModuleInit:\n"));
187
188	#ifdef VBOX_WITH_64ON32_CMOS_DEBUG
189	/*
190	* Display the CMOS debug code.
191	*/
192	ASMOutU8(0x72, 0x03);
193	uint8_t bDebugCode = ASMInU8(0x73);
194	LogRel(("CMOS Debug Code: %#x (%d)\n", bDebugCode, bDebugCode));
195	RTLogComPrintf("CMOS Debug Code: %#x (%d)\n", bDebugCode, bDebugCode);
196	#endif
197
198	/*
199	* Initialize the VMM, GVMM, GMM, HM, PGM (Darwin) and INTNET.
200	*/
201	int rc = vmmInitFormatTypes();
202	if (RT_SUCCESS(rc))
203	{
204	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
205	rc = GVMMR0Init();
206	if (RT_SUCCESS(rc))
207	{
208	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
209	rc = GMMR0Init();
210	if (RT_SUCCESS(rc))
211	{
212	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
213	rc = HMR0Init();
214	if (RT_SUCCESS(rc))
215	{
216	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
217	rc = PGMRegisterStringFormatTypes();
218	if (RT_SUCCESS(rc))
219	{
220	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
221	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
222	rc = PGMR0DynMapInit();
223	#endif
224	if (RT_SUCCESS(rc))
225	{
226	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
227	rc = IntNetR0Init();
228	if (RT_SUCCESS(rc))
229	{
230	#ifdef VBOX_WITH_PCI_PASSTHROUGH
231	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
232	rc = PciRawR0Init();
233	#endif
234	if (RT_SUCCESS(rc))
235	{
236	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
237	rc = CPUMR0ModuleInit();
238	if (RT_SUCCESS(rc))
239	{
240	#ifdef VBOX_WITH_TRIPLE_FAULT_HACK
241	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
242	rc = vmmR0TripleFaultHackInit();
243	if (RT_SUCCESS(rc))
244	#endif
245	{
246	VMM_CHECK_SMAP_CHECK(rc = VERR_VMM_SMAP_BUT_AC_CLEAR);
247	if (RT_SUCCESS(rc))
248	{
249	g_rcRawModeUsability = SUPR0GetRawModeUsability();
250	if (g_rcRawModeUsability != VINF_SUCCESS)
251	SUPR0Printf("VMMR0!ModuleInit: SUPR0GetRawModeUsability -> %Rrc\n",
252	g_rcRawModeUsability);
253	LogFlow(("ModuleInit: returns success\n"));
254	return VINF_SUCCESS;
255	}
256	}
257
258	/*
259	* Bail out.
260	*/
261	#ifdef VBOX_WITH_TRIPLE_FAULT_HACK
262	vmmR0TripleFaultHackTerm();
263	#endif
264	}
265	else
266	LogRel(("ModuleInit: CPUMR0ModuleInit -> %Rrc\n", rc));
267	#ifdef VBOX_WITH_PCI_PASSTHROUGH
268	PciRawR0Term();
269	#endif
270	}
271	else
272	LogRel(("ModuleInit: PciRawR0Init -> %Rrc\n", rc));
273	IntNetR0Term();
274	}
275	else
276	LogRel(("ModuleInit: IntNetR0Init -> %Rrc\n", rc));
277	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
278	PGMR0DynMapTerm();
279	#endif
280	}
281	else
282	LogRel(("ModuleInit: PGMR0DynMapInit -> %Rrc\n", rc));
283	PGMDeregisterStringFormatTypes();
284	}
285	else
286	LogRel(("ModuleInit: PGMRegisterStringFormatTypes -> %Rrc\n", rc));
287	HMR0Term();
288	}
289	else
290	LogRel(("ModuleInit: HMR0Init -> %Rrc\n", rc));
291	GMMR0Term();
292	}
293	else
294	LogRel(("ModuleInit: GMMR0Init -> %Rrc\n", rc));
295	GVMMR0Term();
296	}
297	else
298	LogRel(("ModuleInit: GVMMR0Init -> %Rrc\n", rc));
299	vmmTermFormatTypes();
300	}
301	else
302	LogRel(("ModuleInit: vmmInitFormatTypes -> %Rrc\n", rc));
303
304	LogFlow(("ModuleInit: failed %Rrc\n", rc));
305	return rc;
306	}
307
308
309	/**
310	* Terminate the module.
311	* This is called when we're finally unloaded.
312	*
313	* @param hMod Image handle for use in APIs.
314	*/
315	DECLEXPORT(void) ModuleTerm(void *hMod)
316	{
317	NOREF(hMod);
318	LogFlow(("ModuleTerm:\n"));
319
320	/*
321	* Terminate the CPUM module (Local APIC cleanup).
322	*/
323	CPUMR0ModuleTerm();
324
325	/*
326	* Terminate the internal network service.
327	*/
328	IntNetR0Term();
329
330	/*
331	* PGM (Darwin), HM and PciRaw global cleanup.
332	*/
333	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
334	PGMR0DynMapTerm();
335	#endif
336	#ifdef VBOX_WITH_PCI_PASSTHROUGH
337	PciRawR0Term();
338	#endif
339	PGMDeregisterStringFormatTypes();
340	HMR0Term();
341	#ifdef VBOX_WITH_TRIPLE_FAULT_HACK
342	vmmR0TripleFaultHackTerm();
343	#endif
344
345	/*
346	* Destroy the GMM and GVMM instances.
347	*/
348	GMMR0Term();
349	GVMMR0Term();
350
351	vmmTermFormatTypes();
352
353	LogFlow(("ModuleTerm: returns\n"));
354	}
355
356
357	/**
358	* Initiates the R0 driver for a particular VM instance.
359	*
360	* @returns VBox status code.
361	*
362	* @param pGVM The global (ring-0) VM structure.
363	* @param pVM The cross context VM structure.
364	* @param uSvnRev The SVN revision of the ring-3 part.
365	* @param uBuildType Build type indicator.
366	* @thread EMT(0)
367	*/
368	static int vmmR0InitVM(PGVM pGVM, PVM pVM, uint32_t uSvnRev, uint32_t uBuildType)
369	{
370	VMM_CHECK_SMAP_SETUP();
371	VMM_CHECK_SMAP_CHECK(return VERR_VMM_SMAP_BUT_AC_CLEAR);
372
373	/*
374	* Match the SVN revisions and build type.
375	*/
376	if (uSvnRev != VMMGetSvnRev())
377	{
378	LogRel(("VMMR0InitVM: Revision mismatch, r3=%d r0=%d\n", uSvnRev, VMMGetSvnRev()));
379	SUPR0Printf("VMMR0InitVM: Revision mismatch, r3=%d r0=%d\n", uSvnRev, VMMGetSvnRev());
380	return VERR_VMM_R0_VERSION_MISMATCH;
381	}
382	if (uBuildType != vmmGetBuildType())
383	{
384	LogRel(("VMMR0InitVM: Build type mismatch, r3=%#x r0=%#x\n", uBuildType, vmmGetBuildType()));
385	SUPR0Printf("VMMR0InitVM: Build type mismatch, r3=%#x r0=%#x\n", uBuildType, vmmGetBuildType());
386	return VERR_VMM_R0_VERSION_MISMATCH;
387	}
388
389	int rc = GVMMR0ValidateGVMandVMandEMT(pGVM, pVM, 0 /idCpu/);
390	if (RT_FAILURE(rc))
391	return rc;
392
393	#ifdef LOG_ENABLED
394	/*
395	* Register the EMT R0 logger instance for VCPU 0.
396	*/
397	#ifdef VBOX_BUGREF_9217
398	PVMCPU pVCpu = &pGVM->aCpus[0];
399	#else
400	PVMCPU pVCpu = &pVM->aCpus[0];
401	#endif
402
403	PVMMR0LOGGER pR0Logger = pVCpu->vmm.s.pR0LoggerR0;
404	if (pR0Logger)
405	{
406	# if 0 /* testing of the logger. */
407	LogCom(("vmmR0InitVM: before %p\n", RTLogDefaultInstance()));
408	LogCom(("vmmR0InitVM: pfnFlush=%p actual=%p\n", pR0Logger->Logger.pfnFlush, vmmR0LoggerFlush));
409	LogCom(("vmmR0InitVM: pfnLogger=%p actual=%p\n", pR0Logger->Logger.pfnLogger, vmmR0LoggerWrapper));
410	LogCom(("vmmR0InitVM: offScratch=%d fFlags=%#x fDestFlags=%#x\n", pR0Logger->Logger.offScratch, pR0Logger->Logger.fFlags, pR0Logger->Logger.fDestFlags));
411
412	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pVM->pSession);
413	LogCom(("vmmR0InitVM: after %p reg\n", RTLogDefaultInstance()));
414	RTLogSetDefaultInstanceThread(NULL, pVM->pSession);
415	LogCom(("vmmR0InitVM: after %p dereg\n", RTLogDefaultInstance()));
416
417	pR0Logger->Logger.pfnLogger("hello ring-0 logger\n");
418	LogCom(("vmmR0InitVM: returned successfully from direct logger call.\n"));
419	pR0Logger->Logger.pfnFlush(&pR0Logger->Logger);
420	LogCom(("vmmR0InitVM: returned successfully from direct flush call.\n"));
421
422	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pVM->pSession);
423	LogCom(("vmmR0InitVM: after %p reg2\n", RTLogDefaultInstance()));
424	pR0Logger->Logger.pfnLogger("hello ring-0 logger\n");
425	LogCom(("vmmR0InitVM: returned successfully from direct logger call (2). offScratch=%d\n", pR0Logger->Logger.offScratch));
426	RTLogSetDefaultInstanceThread(NULL, pVM->pSession);
427	LogCom(("vmmR0InitVM: after %p dereg2\n", RTLogDefaultInstance()));
428
429	RTLogLoggerEx(&pR0Logger->Logger, 0, ~0U, "hello ring-0 logger (RTLogLoggerEx)\n");
430	LogCom(("vmmR0InitVM: RTLogLoggerEx returned fine offScratch=%d\n", pR0Logger->Logger.offScratch));
431
432	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pVM->pSession);
433	RTLogPrintf("hello ring-0 logger (RTLogPrintf)\n");
434	LogCom(("vmmR0InitVM: RTLogPrintf returned fine offScratch=%d\n", pR0Logger->Logger.offScratch));
435	# endif
436	Log(("Switching to per-thread logging instance %p (key=%p)\n", &pR0Logger->Logger, pVM->pSession));
437	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pVM->pSession);
438	pR0Logger->fRegistered = true;
439	}
440	#endif /* LOG_ENABLED */
441
442	/*
443	* Check if the host supports high resolution timers or not.
444	*/
445	if ( pVM->vmm.s.fUsePeriodicPreemptionTimers
446	&& !RTTimerCanDoHighResolution())
447	pVM->vmm.s.fUsePeriodicPreemptionTimers = false;
448
449	/*
450	* Initialize the per VM data for GVMM and GMM.
451	*/
452	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
453	rc = GVMMR0InitVM(pGVM);
454	// if (RT_SUCCESS(rc))
455	// rc = GMMR0InitPerVMData(pVM);
456	if (RT_SUCCESS(rc))
457	{
458	/*
459	* Init HM, CPUM and PGM (Darwin only).
460	*/
461	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
462	rc = HMR0InitVM(pVM);
463	if (RT_SUCCESS(rc))
464	VMM_CHECK_SMAP_CHECK2(pVM, rc = VERR_VMM_RING0_ASSERTION); /* CPUR0InitVM will otherwise panic the host */
465	if (RT_SUCCESS(rc))
466	{
467	rc = CPUMR0InitVM(pVM);
468	if (RT_SUCCESS(rc))
469	{
470	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
471	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
472	rc = PGMR0DynMapInitVM(pVM);
473	#endif
474	if (RT_SUCCESS(rc))
475	{
476	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
477	#ifdef VBOX_BUGREF_9217
478	rc = EMR0InitVM(pGVM);
479	#else
480	rc = EMR0InitVM(pGVM, pVM);
481	#endif
482	if (RT_SUCCESS(rc))
483	{
484	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
485	#ifdef VBOX_WITH_PCI_PASSTHROUGH
486	rc = PciRawR0InitVM(pGVM, pVM);
487	#endif
488	if (RT_SUCCESS(rc))
489	{
490	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
491	rc = GIMR0InitVM(pVM);
492	if (RT_SUCCESS(rc))
493	{
494	VMM_CHECK_SMAP_CHECK2(pVM, rc = VERR_VMM_RING0_ASSERTION);
495	if (RT_SUCCESS(rc))
496	{
497	GVMMR0DoneInitVM(pGVM);
498
499	/*
500	* Collect a bit of info for the VM release log.
501	*/
502	pVM->vmm.s.fIsPreemptPendingApiTrusty = RTThreadPreemptIsPendingTrusty();
503	pVM->vmm.s.fIsPreemptPossible = RTThreadPreemptIsPossible();;
504
505	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
506	return rc;
507	}
508
509	/* bail out*/
510	GIMR0TermVM(pVM);
511	}
512	#ifdef VBOX_WITH_PCI_PASSTHROUGH
513	PciRawR0TermVM(pGVM, pVM);
514	#endif
515	}
516	}
517	}
518	}
519	HMR0TermVM(pVM);
520	}
521	}
522
523	RTLogSetDefaultInstanceThread(NULL, (uintptr_t)pVM->pSession);
524	return rc;
525	}
526
527
528	/**
529	* Does EMT specific VM initialization.
530	*
531	* @returns VBox status code.
532	* @param pGVM The ring-0 VM structure.
533	* @param pVM The cross context VM structure.
534	* @param idCpu The EMT that's calling.
535	*/
536	static int vmmR0InitVMEmt(PGVM pGVM, PVM pVM, VMCPUID idCpu)
537	{
538	/* Paranoia (caller checked these already). */
539	AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID);
540	AssertReturn(pGVM->aCpus[idCpu].hEMT == RTThreadNativeSelf(), VERR_INVALID_CPU_ID);
541
542	#ifdef LOG_ENABLED
543	/*
544	* Registration of ring 0 loggers.
545	*/
546	#ifdef VBOX_BUGREF_9217
547	PVMCPU pVCpu = &pGVM->aCpus[idCpu];
548	#else
549	PVMCPU pVCpu = &pVM->aCpus[idCpu];
550	#endif
551	PVMMR0LOGGER pR0Logger = pVCpu->vmm.s.pR0LoggerR0;
552	if ( pR0Logger
553	&& !pR0Logger->fRegistered)
554	{
555	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pVM->pSession);
556	pR0Logger->fRegistered = true;
557	}
558	#endif
559	RT_NOREF(pVM);
560
561	return VINF_SUCCESS;
562	}
563
564
565
566	/**
567	* Terminates the R0 bits for a particular VM instance.
568	*
569	* This is normally called by ring-3 as part of the VM termination process, but
570	* may alternatively be called during the support driver session cleanup when
571	* the VM object is destroyed (see GVMM).
572	*
573	* @returns VBox status code.
574	*
575	* @param pGVM The global (ring-0) VM structure.
576	* @param pVM The cross context VM structure.
577	* @param idCpu Set to 0 if EMT(0) or NIL_VMCPUID if session cleanup
578	* thread.
579	* @thread EMT(0) or session clean up thread.
580	*/
581	VMMR0_INT_DECL(int) VMMR0TermVM(PGVM pGVM, PVM pVM, VMCPUID idCpu)
582	{
583	/*
584	* Check EMT(0) claim if we're called from userland.
585	*/
586	if (idCpu != NIL_VMCPUID)
587	{
588	AssertReturn(idCpu == 0, VERR_INVALID_CPU_ID);
589	int rc = GVMMR0ValidateGVMandVMandEMT(pGVM, pVM, idCpu);
590	if (RT_FAILURE(rc))
591	return rc;
592	}
593
594	#ifdef VBOX_WITH_PCI_PASSTHROUGH
595	PciRawR0TermVM(pGVM, pVM);
596	#endif
597
598	/*
599	* Tell GVMM what we're up to and check that we only do this once.
600	*/
601	if (GVMMR0DoingTermVM(pGVM))
602	{
603	GIMR0TermVM(pVM);
604
605	/** @todo I wish to call PGMR0PhysFlushHandyPages(pVM, &pVM->aCpus[idCpu])
606	* here to make sure we don't leak any shared pages if we crash... */
607	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
608	PGMR0DynMapTermVM(pVM);
609	#endif
610	HMR0TermVM(pVM);
611	}
612
613	/*
614	* Deregister the logger.
615	*/
616	RTLogSetDefaultInstanceThread(NULL, (uintptr_t)pVM->pSession);
617	return VINF_SUCCESS;
618	}
619
620
621	/**
622	* An interrupt or unhalt force flag is set, deal with it.
623	*
624	* @returns VINF_SUCCESS (or VINF_EM_HALT).
625	* @param pVCpu The cross context virtual CPU structure.
626	* @param uMWait Result from EMMonitorWaitIsActive().
627	* @param enmInterruptibility Guest CPU interruptbility level.
628	*/
629	static int vmmR0DoHaltInterrupt(PVMCPU pVCpu, unsigned uMWait, CPUMINTERRUPTIBILITY enmInterruptibility)
630	{
631	Assert(!TRPMHasTrap(pVCpu));
632	Assert( enmInterruptibility > CPUMINTERRUPTIBILITY_INVALID
633	&& enmInterruptibility < CPUMINTERRUPTIBILITY_END);
634
635	/*
636	* Pending interrupts w/o any SMIs or NMIs? That the usual case.
637	*/
638	if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
639	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI \| VMCPU_FF_INTERRUPT_NMI))
640	{
641	if (enmInterruptibility <= CPUMINTERRUPTIBILITY_UNRESTRAINED)
642	{
643	uint8_t u8Interrupt = 0;
644	int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
645	Log(("vmmR0DoHaltInterrupt: CPU%d u8Interrupt=%d (%#x) rc=%Rrc\n", pVCpu->idCpu, u8Interrupt, u8Interrupt, rc));
646	if (RT_SUCCESS(rc))
647	{
648	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_UNHALT);
649
650	rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
651	AssertRCSuccess(rc);
652	STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltExec);
653	return rc;
654	}
655	}
656	}
657	/*
658	* SMI is not implemented yet, at least not here.
659	*/
660	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI))
661	{
662	return VINF_EM_HALT;
663	}
664	/*
665	* NMI.
666	*/
667	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
668	{
669	if (enmInterruptibility < CPUMINTERRUPTIBILITY_NMI_INHIBIT)
670	{
671	/** @todo later. */
672	return VINF_EM_HALT;
673	}
674	}
675	/*
676	* Nested-guest virtual interrupt.
677	*/
678	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))
679	{
680	if (enmInterruptibility < CPUMINTERRUPTIBILITY_VIRT_INT_DISABLED)
681	{
682	/** @todo NSTVMX: NSTSVM: Remember, we might have to check and perform VM-exits
683	* here before injecting the virtual interrupt. See emR3ForcedActions
684	* for details. */
685	return VINF_EM_HALT;
686	}
687	}
688
689	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UNHALT))
690	{
691	STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltExec);
692	return VINF_SUCCESS;
693	}
694	if (uMWait > 1)
695	{
696	STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltExec);
697	return VINF_SUCCESS;
698	}
699
700	return VINF_EM_HALT;
701	}
702
703
704	/**
705	* This does one round of vmR3HaltGlobal1Halt().
706	*
707	* The rational here is that we'll reduce latency in interrupt situations if we
708	* don't go to ring-3 immediately on a VINF_EM_HALT (guest executed HLT or
709	* MWAIT), but do one round of blocking here instead and hope the interrupt is
710	* raised in the meanwhile.
711	*
712	* If we go to ring-3 we'll quit the inner HM/NEM loop in EM and end up in the
713	* outer loop, which will then call VMR3WaitHalted() and that in turn will do a
714	* ring-0 call (unless we're too close to a timer event). When the interrupt
715	* wakes us up, we'll return from ring-0 and EM will by instinct do a
716	* rescheduling (because of raw-mode) before it resumes the HM/NEM loop and gets
717	* back to VMMR0EntryFast().
718	*
719	* @returns VINF_SUCCESS or VINF_EM_HALT.
720	* @param pGVM The ring-0 VM structure.
721	* @param pVM The cross context VM structure.
722	* @param pGVCpu The ring-0 virtual CPU structure.
723	* @param pVCpu The cross context virtual CPU structure.
724	*
725	* @todo r=bird: All the blocking/waiting and EMT managment should move out of
726	* the VM module, probably to VMM. Then this would be more weird wrt
727	* parameters and statistics.
728	*/
729	static int vmmR0DoHalt(PGVM pGVM, PVM pVM, PGVMCPU pGVCpu, PVMCPU pVCpu)
730	{
731	#ifdef VBOX_BUGREF_9217
732	Assert(pVCpu == pGVCpu);
733	#else
734	Assert(pVCpu == pGVCpu->pVCpu);
735	#endif
736
737	/*
738	* Do spin stat historization.
739	*/
740	if (++pVCpu->vmm.s.cR0Halts & 0xff)
741	{ /* likely */ }
742	else if (pVCpu->vmm.s.cR0HaltsSucceeded > pVCpu->vmm.s.cR0HaltsToRing3)
743	{
744	pVCpu->vmm.s.cR0HaltsSucceeded = 2;
745	pVCpu->vmm.s.cR0HaltsToRing3 = 0;
746	}
747	else
748	{
749	pVCpu->vmm.s.cR0HaltsSucceeded = 0;
750	pVCpu->vmm.s.cR0HaltsToRing3 = 2;
751	}
752
753	/*
754	* Flags that makes us go to ring-3.
755	*/
756	uint32_t const fVmFFs = VM_FF_TM_VIRTUAL_SYNC \| VM_FF_PDM_QUEUES \| VM_FF_PDM_DMA
757	\| VM_FF_DBGF \| VM_FF_REQUEST \| VM_FF_CHECK_VM_STATE
758	\| VM_FF_RESET \| VM_FF_EMT_RENDEZVOUS \| VM_FF_PGM_NEED_HANDY_PAGES
759	\| VM_FF_PGM_NO_MEMORY \| VM_FF_REM_HANDLER_NOTIFY \| VM_FF_DEBUG_SUSPEND;
760	uint64_t const fCpuFFs = VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT \| VMCPU_FF_IEM
761	\| VMCPU_FF_REQUEST \| VMCPU_FF_DBGF \| VMCPU_FF_HM_UPDATE_CR3
762	\| VMCPU_FF_HM_UPDATE_PAE_PDPES \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
763	\| VMCPU_FF_TO_R3 \| VMCPU_FF_IOM
764	#ifdef VBOX_WITH_RAW_MODE
765	\| VMCPU_FF_TRPM_SYNC_IDT \| VMCPU_FF_SELM_SYNC_TSS \| VMCPU_FF_SELM_SYNC_GDT
766	\| VMCPU_FF_SELM_SYNC_LDT \| VMCPU_FF_CSAM_SCAN_PAGE \| VMCPU_FF_CSAM_PENDING_ACTION
767	\| VMCPU_FF_CPUM
768	#endif
769	;
770
771	/*
772	* Check preconditions.
773	*/
774	unsigned const uMWait = EMMonitorWaitIsActive(pVCpu);
775	CPUMINTERRUPTIBILITY const enmInterruptibility = CPUMGetGuestInterruptibility(pVCpu);
776	if ( pVCpu->vmm.s.fMayHaltInRing0
777	&& !TRPMHasTrap(pVCpu)
778	&& ( enmInterruptibility == CPUMINTERRUPTIBILITY_UNRESTRAINED
779	\|\| uMWait > 1))
780	{
781	if ( !VM_FF_IS_ANY_SET(pVM, fVmFFs)
782	&& !VMCPU_FF_IS_ANY_SET(pVCpu, fCpuFFs))
783	{
784	/*
785	* Interrupts pending already?
786	*/
787	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
788	APICUpdatePendingInterrupts(pVCpu);
789
790	/*
791	* Flags that wake up from the halted state.
792	*/
793	uint64_t const fIntMask = VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC \| VMCPU_FF_INTERRUPT_NESTED_GUEST
794	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI \| VMCPU_FF_UNHALT;
795
796	if (VMCPU_FF_IS_ANY_SET(pVCpu, fIntMask))
797	return vmmR0DoHaltInterrupt(pVCpu, uMWait, enmInterruptibility);
798	ASMNopPause();
799
800	/*
801	* Check out how long till the next timer event.
802	*/
803	uint64_t u64Delta;
804	uint64_t u64GipTime = TMTimerPollGIP(pVM, pVCpu, &u64Delta);
805
806	if ( !VM_FF_IS_ANY_SET(pVM, fVmFFs)
807	&& !VMCPU_FF_IS_ANY_SET(pVCpu, fCpuFFs))
808	{
809	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
810	APICUpdatePendingInterrupts(pVCpu);
811
812	if (VMCPU_FF_IS_ANY_SET(pVCpu, fIntMask))
813	return vmmR0DoHaltInterrupt(pVCpu, uMWait, enmInterruptibility);
814
815	/*
816	* Wait if there is enough time to the next timer event.
817	*/
818	if (u64Delta >= pVCpu->vmm.s.cNsSpinBlockThreshold)
819	{
820	/* If there are few other CPU cores around, we will procrastinate a
821	little before going to sleep, hoping for some device raising an
822	interrupt or similar. Though, the best thing here would be to
823	dynamically adjust the spin count according to its usfulness or
824	something... */
825	if ( pVCpu->vmm.s.cR0HaltsSucceeded > pVCpu->vmm.s.cR0HaltsToRing3
826	&& RTMpGetOnlineCount() >= 4)
827	{
828	/** @todo Figure out how we can skip this if it hasn't help recently...
829	* @bugref{9172#c12} */
830	uint32_t cSpinLoops = 42;
831	while (cSpinLoops-- > 0)
832	{
833	ASMNopPause();
834	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
835	APICUpdatePendingInterrupts(pVCpu);
836	ASMNopPause();
837	if (VM_FF_IS_ANY_SET(pVM, fVmFFs))
838	{
839	STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltToR3FromSpin);
840	return VINF_EM_HALT;
841	}
842	ASMNopPause();
843	if (VMCPU_FF_IS_ANY_SET(pVCpu, fCpuFFs))
844	{
845	STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltToR3FromSpin);
846	return VINF_EM_HALT;
847	}
848	ASMNopPause();
849	if (VMCPU_FF_IS_ANY_SET(pVCpu, fIntMask))
850	{
851	STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltExecFromSpin);
852	return vmmR0DoHaltInterrupt(pVCpu, uMWait, enmInterruptibility);
853	}
854	ASMNopPause();
855	}
856	}
857
858	/* Block. We have to set the state to VMCPUSTATE_STARTED_HALTED here so ring-3
859	knows when to notify us (cannot access VMINTUSERPERVMCPU::fWait from here). */
860	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HALTED, VMCPUSTATE_STARTED);
861	uint64_t const u64StartSchedHalt = RTTimeNanoTS();
862	int rc = GVMMR0SchedHalt(pGVM, pVM, pGVCpu, u64GipTime);
863	uint64_t const u64EndSchedHalt = RTTimeNanoTS();
864	uint64_t const cNsElapsedSchedHalt = u64EndSchedHalt - u64StartSchedHalt;
865	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_HALTED);
866	STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->vmm.s.StatR0HaltBlock, cNsElapsedSchedHalt);
867	if ( rc == VINF_SUCCESS
868	\|\| rc == VERR_INTERRUPTED)
869
870	{
871	/* Keep some stats like ring-3 does. */
872	int64_t const cNsOverslept = u64EndSchedHalt - u64GipTime;
873	if (cNsOverslept > 50000)
874	STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->vmm.s.StatR0HaltBlockOverslept, cNsOverslept);
875	else if (cNsOverslept < -50000)
876	STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->vmm.s.StatR0HaltBlockInsomnia, cNsElapsedSchedHalt);
877	else
878	STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->vmm.s.StatR0HaltBlockOnTime, cNsElapsedSchedHalt);
879
880	/*
881	* Recheck whether we can resume execution or have to go to ring-3.
882	*/
883	if ( !VM_FF_IS_ANY_SET(pVM, fVmFFs)
884	&& !VMCPU_FF_IS_ANY_SET(pVCpu, fCpuFFs))
885	{
886	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
887	APICUpdatePendingInterrupts(pVCpu);
888	if (VMCPU_FF_IS_ANY_SET(pVCpu, fIntMask))
889	{
890	STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltExecFromBlock);
891	return vmmR0DoHaltInterrupt(pVCpu, uMWait, enmInterruptibility);
892	}
893	}
894	}
895	}
896	}
897	}
898	}
899	return VINF_EM_HALT;
900	}
901
902
903	/**
904	* VMM ring-0 thread-context callback.
905	*
906	* This does common HM state updating and calls the HM-specific thread-context
907	* callback.
908	*
909	* @param enmEvent The thread-context event.
910	* @param pvUser Opaque pointer to the VMCPU.
911	*
912	* @thread EMT(pvUser)
913	*/
914	static DECLCALLBACK(void) vmmR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, void *pvUser)
915	{
916	PVMCPU pVCpu = (PVMCPU)pvUser;
917
918	switch (enmEvent)
919	{
920	case RTTHREADCTXEVENT_IN:
921	{
922	/*
923	* Linux may call us with preemption enabled (really!) but technically we
924	* cannot get preempted here, otherwise we end up in an infinite recursion
925	* scenario (i.e. preempted in resume hook -> preempt hook -> resume hook...
926	* ad infinitum). Let's just disable preemption for now...
927	*/
928	/** @todo r=bird: I don't believe the above. The linux code is clearly enabling
929	* preemption after doing the callout (one or two functions up the
930	* call chain). */
931	/** @todo r=ramshankar: See @bugref{5313#c30}. */
932	RTTHREADPREEMPTSTATE ParanoidPreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
933	RTThreadPreemptDisable(&ParanoidPreemptState);
934
935	/* We need to update the VCPU <-> host CPU mapping. */
936	RTCPUID idHostCpu;
937	uint32_t iHostCpuSet = RTMpCurSetIndexAndId(&idHostCpu);
938	pVCpu->iHostCpuSet = iHostCpuSet;
939	ASMAtomicWriteU32(&pVCpu->idHostCpu, idHostCpu);
940
941	/* In the very unlikely event that the GIP delta for the CPU we're
942	rescheduled needs calculating, try force a return to ring-3.
943	We unfortunately cannot do the measurements right here. */
944	if (RT_UNLIKELY(SUPIsTscDeltaAvailableForCpuSetIndex(iHostCpuSet)))
945	VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
946
947	/* Invoke the HM-specific thread-context callback. */
948	HMR0ThreadCtxCallback(enmEvent, pvUser);
949
950	/* Restore preemption. */
951	RTThreadPreemptRestore(&ParanoidPreemptState);
952	break;
953	}
954
955	case RTTHREADCTXEVENT_OUT:
956	{
957	/* Invoke the HM-specific thread-context callback. */
958	HMR0ThreadCtxCallback(enmEvent, pvUser);
959
960	/*
961	* Sigh. See VMMGetCpu() used by VMCPU_ASSERT_EMT(). We cannot let several VCPUs
962	* have the same host CPU associated with it.
963	*/
964	pVCpu->iHostCpuSet = UINT32_MAX;
965	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
966	break;
967	}
968
969	default:
970	/* Invoke the HM-specific thread-context callback. */
971	HMR0ThreadCtxCallback(enmEvent, pvUser);
972	break;
973	}
974	}
975
976
977	/**
978	* Creates thread switching hook for the current EMT thread.
979	*
980	* This is called by GVMMR0CreateVM and GVMMR0RegisterVCpu. If the host
981	* platform does not implement switcher hooks, no hooks will be create and the
982	* member set to NIL_RTTHREADCTXHOOK.
983	*
984	* @returns VBox status code.
985	* @param pVCpu The cross context virtual CPU structure.
986	* @thread EMT(pVCpu)
987	*/
988	VMMR0_INT_DECL(int) VMMR0ThreadCtxHookCreateForEmt(PVMCPU pVCpu)
989	{
990	VMCPU_ASSERT_EMT(pVCpu);
991	Assert(pVCpu->vmm.s.hCtxHook == NIL_RTTHREADCTXHOOK);
992
993	#if 1 /* To disable this stuff change to zero. */
994	int rc = RTThreadCtxHookCreate(&pVCpu->vmm.s.hCtxHook, 0, vmmR0ThreadCtxCallback, pVCpu);
995	if (RT_SUCCESS(rc))
996	return rc;
997	#else
998	RT_NOREF(vmmR0ThreadCtxCallback);
999	int rc = VERR_NOT_SUPPORTED;
1000	#endif
1001
1002	pVCpu->vmm.s.hCtxHook = NIL_RTTHREADCTXHOOK;
1003	if (rc == VERR_NOT_SUPPORTED)
1004	return VINF_SUCCESS;
1005
1006	LogRelMax(32, ("RTThreadCtxHookCreate failed! rc=%Rrc pVCpu=%p idCpu=%RU32\n", rc, pVCpu, pVCpu->idCpu));
1007	return VINF_SUCCESS; /* Just ignore it, we can live without context hooks. */
1008	}
1009
1010
1011	/**
1012	* Destroys the thread switching hook for the specified VCPU.
1013	*
1014	* @param pVCpu The cross context virtual CPU structure.
1015	* @remarks Can be called from any thread.
1016	*/
1017	VMMR0_INT_DECL(void) VMMR0ThreadCtxHookDestroyForEmt(PVMCPU pVCpu)
1018	{
1019	int rc = RTThreadCtxHookDestroy(pVCpu->vmm.s.hCtxHook);
1020	AssertRC(rc);
1021	pVCpu->vmm.s.hCtxHook = NIL_RTTHREADCTXHOOK;
1022	}
1023
1024
1025	/**
1026	* Disables the thread switching hook for this VCPU (if we got one).
1027	*
1028	* @param pVCpu The cross context virtual CPU structure.
1029	* @thread EMT(pVCpu)
1030	*
1031	* @remarks This also clears VMCPU::idHostCpu, so the mapping is invalid after
1032	* this call. This means you have to be careful with what you do!
1033	*/
1034	VMMR0_INT_DECL(void) VMMR0ThreadCtxHookDisable(PVMCPU pVCpu)
1035	{
1036	/*
1037	* Clear the VCPU <-> host CPU mapping as we've left HM context.
1038	* @bugref{7726#c19} explains the need for this trick:
1039	*
1040	* hmR0VmxCallRing3Callback/hmR0SvmCallRing3Callback &
1041	* hmR0VmxLeaveSession/hmR0SvmLeaveSession disables context hooks during
1042	* longjmp & normal return to ring-3, which opens a window where we may be
1043	* rescheduled without changing VMCPUID::idHostCpu and cause confusion if
1044	* the CPU starts executing a different EMT. Both functions first disables
1045	* preemption and then calls HMR0LeaveCpu which invalids idHostCpu, leaving
1046	* an opening for getting preempted.
1047	*/
1048	/** @todo Make HM not need this API! Then we could leave the hooks enabled
1049	* all the time. */
1050	/** @todo move this into the context hook disabling if(). */
1051	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
1052
1053	/*
1054	* Disable the context hook, if we got one.
1055	*/
1056	if (pVCpu->vmm.s.hCtxHook != NIL_RTTHREADCTXHOOK)
1057	{
1058	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1059	int rc = RTThreadCtxHookDisable(pVCpu->vmm.s.hCtxHook);
1060	AssertRC(rc);
1061	}
1062	}
1063
1064
1065	/**
1066	* Internal version of VMMR0ThreadCtxHooksAreRegistered.
1067	*
1068	* @returns true if registered, false otherwise.
1069	* @param pVCpu The cross context virtual CPU structure.
1070	*/
1071	DECLINLINE(bool) vmmR0ThreadCtxHookIsEnabled(PVMCPU pVCpu)
1072	{
1073	return RTThreadCtxHookIsEnabled(pVCpu->vmm.s.hCtxHook);
1074	}
1075
1076
1077	/**
1078	* Whether thread-context hooks are registered for this VCPU.
1079	*
1080	* @returns true if registered, false otherwise.
1081	* @param pVCpu The cross context virtual CPU structure.
1082	*/
1083	VMMR0_INT_DECL(bool) VMMR0ThreadCtxHookIsEnabled(PVMCPU pVCpu)
1084	{
1085	return vmmR0ThreadCtxHookIsEnabled(pVCpu);
1086	}
1087
1088
1089	#ifdef VBOX_WITH_STATISTICS
1090	/**
1091	* Record return code statistics
1092	* @param pVM The cross context VM structure.
1093	* @param pVCpu The cross context virtual CPU structure.
1094	* @param rc The status code.
1095	*/
1096	static void vmmR0RecordRC(PVM pVM, PVMCPU pVCpu, int rc)
1097	{
1098	/*
1099	* Collect statistics.
1100	*/
1101	switch (rc)
1102	{
1103	case VINF_SUCCESS:
1104	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetNormal);
1105	break;
1106	case VINF_EM_RAW_INTERRUPT:
1107	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetInterrupt);
1108	break;
1109	case VINF_EM_RAW_INTERRUPT_HYPER:
1110	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetInterruptHyper);
1111	break;
1112	case VINF_EM_RAW_GUEST_TRAP:
1113	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetGuestTrap);
1114	break;
1115	case VINF_EM_RAW_RING_SWITCH:
1116	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetRingSwitch);
1117	break;
1118	case VINF_EM_RAW_RING_SWITCH_INT:
1119	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetRingSwitchInt);
1120	break;
1121	case VINF_EM_RAW_STALE_SELECTOR:
1122	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetStaleSelector);
1123	break;
1124	case VINF_EM_RAW_IRET_TRAP:
1125	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIRETTrap);
1126	break;
1127	case VINF_IOM_R3_IOPORT_READ:
1128	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIORead);
1129	break;
1130	case VINF_IOM_R3_IOPORT_WRITE:
1131	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIOWrite);
1132	break;
1133	case VINF_IOM_R3_IOPORT_COMMIT_WRITE:
1134	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIOCommitWrite);
1135	break;
1136	case VINF_IOM_R3_MMIO_READ:
1137	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIORead);
1138	break;
1139	case VINF_IOM_R3_MMIO_WRITE:
1140	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOWrite);
1141	break;
1142	case VINF_IOM_R3_MMIO_COMMIT_WRITE:
1143	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOCommitWrite);
1144	break;
1145	case VINF_IOM_R3_MMIO_READ_WRITE:
1146	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOReadWrite);
1147	break;
1148	case VINF_PATM_HC_MMIO_PATCH_READ:
1149	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOPatchRead);
1150	break;
1151	case VINF_PATM_HC_MMIO_PATCH_WRITE:
1152	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOPatchWrite);
1153	break;
1154	case VINF_CPUM_R3_MSR_READ:
1155	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMSRRead);
1156	break;
1157	case VINF_CPUM_R3_MSR_WRITE:
1158	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMSRWrite);
1159	break;
1160	case VINF_EM_RAW_EMULATE_INSTR:
1161	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetEmulate);
1162	break;
1163	case VINF_PATCH_EMULATE_INSTR:
1164	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchEmulate);
1165	break;
1166	case VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT:
1167	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetLDTFault);
1168	break;
1169	case VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT:
1170	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetGDTFault);
1171	break;
1172	case VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT:
1173	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIDTFault);
1174	break;
1175	case VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT:
1176	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetTSSFault);
1177	break;
1178	case VINF_CSAM_PENDING_ACTION:
1179	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetCSAMTask);
1180	break;
1181	case VINF_PGM_SYNC_CR3:
1182	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetSyncCR3);
1183	break;
1184	case VINF_PATM_PATCH_INT3:
1185	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchInt3);
1186	break;
1187	case VINF_PATM_PATCH_TRAP_PF:
1188	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchPF);
1189	break;
1190	case VINF_PATM_PATCH_TRAP_GP:
1191	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchGP);
1192	break;
1193	case VINF_PATM_PENDING_IRQ_AFTER_IRET:
1194	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchIretIRQ);
1195	break;
1196	case VINF_EM_RESCHEDULE_REM:
1197	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetRescheduleREM);
1198	break;
1199	case VINF_EM_RAW_TO_R3:
1200	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Total);
1201	if (VM_FF_IS_SET(pVM, VM_FF_TM_VIRTUAL_SYNC))
1202	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3TMVirt);
1203	else if (VM_FF_IS_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES))
1204	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3HandyPages);
1205	else if (VM_FF_IS_SET(pVM, VM_FF_PDM_QUEUES))
1206	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3PDMQueues);
1207	else if (VM_FF_IS_SET(pVM, VM_FF_EMT_RENDEZVOUS))
1208	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Rendezvous);
1209	else if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
1210	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3DMA);
1211	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TIMER))
1212	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Timer);
1213	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PDM_CRITSECT))
1214	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3CritSect);
1215	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TO_R3))
1216	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3FF);
1217	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM))
1218	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Iem);
1219	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IOM))
1220	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Iom);
1221	else
1222	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Unknown);
1223	break;
1224
1225	case VINF_EM_RAW_TIMER_PENDING:
1226	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetTimerPending);
1227	break;
1228	case VINF_EM_RAW_INTERRUPT_PENDING:
1229	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetInterruptPending);
1230	break;
1231	case VINF_VMM_CALL_HOST:
1232	switch (pVCpu->vmm.s.enmCallRing3Operation)
1233	{
1234	case VMMCALLRING3_PDM_CRIT_SECT_ENTER:
1235	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPDMCritSectEnter);
1236	break;
1237	case VMMCALLRING3_PDM_LOCK:
1238	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPDMLock);
1239	break;
1240	case VMMCALLRING3_PGM_POOL_GROW:
1241	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPGMPoolGrow);
1242	break;
1243	case VMMCALLRING3_PGM_LOCK:
1244	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPGMLock);
1245	break;
1246	case VMMCALLRING3_PGM_MAP_CHUNK:
1247	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPGMMapChunk);
1248	break;
1249	case VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES:
1250	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPGMAllocHandy);
1251	break;
1252	case VMMCALLRING3_REM_REPLAY_HANDLER_NOTIFICATIONS:
1253	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallRemReplay);
1254	break;
1255	case VMMCALLRING3_VMM_LOGGER_FLUSH:
1256	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallLogFlush);
1257	break;
1258	case VMMCALLRING3_VM_SET_ERROR:
1259	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallVMSetError);
1260	break;
1261	case VMMCALLRING3_VM_SET_RUNTIME_ERROR:
1262	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallVMSetRuntimeError);
1263	break;
1264	case VMMCALLRING3_VM_R0_ASSERTION:
1265	default:
1266	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetCallRing3);
1267	break;
1268	}
1269	break;
1270	case VINF_PATM_DUPLICATE_FUNCTION:
1271	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPATMDuplicateFn);
1272	break;
1273	case VINF_PGM_CHANGE_MODE:
1274	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPGMChangeMode);
1275	break;
1276	case VINF_PGM_POOL_FLUSH_PENDING:
1277	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPGMFlushPending);
1278	break;
1279	case VINF_EM_PENDING_REQUEST:
1280	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPendingRequest);
1281	break;
1282	case VINF_EM_HM_PATCH_TPR_INSTR:
1283	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchTPR);
1284	break;
1285	default:
1286	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMisc);
1287	break;
1288	}
1289	}
1290	#endif /* VBOX_WITH_STATISTICS */
1291
1292
1293	/**
1294	* The Ring 0 entry point, called by the fast-ioctl path.
1295	*
1296	* @param pGVM The global (ring-0) VM structure.
1297	* @param pVM The cross context VM structure.
1298	* The return code is stored in pVM->vmm.s.iLastGZRc.
1299	* @param idCpu The Virtual CPU ID of the calling EMT.
1300	* @param enmOperation Which operation to execute.
1301	* @remarks Assume called with interrupts _enabled_.
1302	*/
1303	VMMR0DECL(void) VMMR0EntryFast(PGVM pGVM, PVM pVM, VMCPUID idCpu, VMMR0OPERATION enmOperation)
1304	{
1305	/*
1306	* Validation.
1307	*/
1308	if ( idCpu < pGVM->cCpus
1309	&& pGVM->cCpus == pVM->cCpus)
1310	{ /likely/ }
1311	else
1312	{
1313	SUPR0Printf("VMMR0EntryFast: Bad idCpu=%#x cCpus=%#x/%#x\n", idCpu, pGVM->cCpus, pVM->cCpus);
1314	return;
1315	}
1316
1317	PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
1318	#ifdef VBOX_BUGREF_9217
1319	PVMCPU pVCpu = pGVCpu;
1320	#else
1321	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1322	#endif
1323	RTNATIVETHREAD const hNativeThread = RTThreadNativeSelf();
1324	if (RT_LIKELY( pGVCpu->hEMT == hNativeThread
1325	&& pVCpu->hNativeThreadR0 == hNativeThread))
1326	{ /* likely */ }
1327	else
1328	{
1329	SUPR0Printf("VMMR0EntryFast: Bad thread idCpu=%#x hNativeSelf=%p pGVCpu->hEmt=%p pVCpu->hNativeThreadR0=%p\n",
1330	idCpu, hNativeThread, pGVCpu->hEMT, pVCpu->hNativeThreadR0);
1331	return;
1332	}
1333
1334	/*
1335	* SMAP fun.
1336	*/
1337	VMM_CHECK_SMAP_SETUP();
1338	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1339
1340	/*
1341	* Perform requested operation.
1342	*/
1343	switch (enmOperation)
1344	{
1345	/*
1346	* Switch to GC and run guest raw mode code.
1347	* Disable interrupts before doing the world switch.
1348	*/
1349	case VMMR0_DO_RAW_RUN:
1350	{
1351	#ifdef VBOX_WITH_RAW_MODE
1352	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
1353	/* Some safety precautions first. */
1354	if (RT_UNLIKELY(!PGMGetHyperCR3(pVCpu)))
1355	{
1356	pVCpu->vmm.s.iLastGZRc = VERR_PGM_NO_CR3_SHADOW_ROOT;
1357	break;
1358	}
1359	# endif
1360	if (RT_SUCCESS(g_rcRawModeUsability))
1361	{ /* likely */ }
1362	else
1363	{
1364	pVCpu->vmm.s.iLastGZRc = g_rcRawModeUsability;
1365	break;
1366	}
1367
1368	/*
1369	* Disable preemption.
1370	*/
1371	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
1372	RTThreadPreemptDisable(&PreemptState);
1373
1374	/*
1375	* Get the host CPU identifiers, make sure they are valid and that
1376	* we've got a TSC delta for the CPU.
1377	*/
1378	RTCPUID idHostCpu;
1379	uint32_t iHostCpuSet = RTMpCurSetIndexAndId(&idHostCpu);
1380	if (RT_LIKELY( iHostCpuSet < RTCPUSET_MAX_CPUS
1381	&& SUPIsTscDeltaAvailableForCpuSetIndex(iHostCpuSet)))
1382	{
1383	/*
1384	* Commit the CPU identifiers and update the periodict preemption timer if it's active.
1385	*/
1386	# ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
1387	CPUMR0SetLApic(pVCpu, iHostCpuSet);
1388	# endif
1389	pVCpu->iHostCpuSet = iHostCpuSet;
1390	ASMAtomicWriteU32(&pVCpu->idHostCpu, idHostCpu);
1391
1392	if (pVM->vmm.s.fUsePeriodicPreemptionTimers)
1393	GVMMR0SchedUpdatePeriodicPreemptionTimer(pVM, pVCpu->idHostCpu, TMCalcHostTimerFrequency(pVM, pVCpu));
1394
1395	/*
1396	* We might need to disable VT-x if the active switcher turns off paging.
1397	*/
1398	bool fVTxDisabled;
1399	int rc = HMR0EnterSwitcher(pVM, pVM->vmm.s.enmSwitcher, &fVTxDisabled);
1400	if (RT_SUCCESS(rc))
1401	{
1402	/*
1403	* Disable interrupts and run raw-mode code. The loop is for efficiently
1404	* dispatching tracepoints that fired in raw-mode context.
1405	*/
1406	RTCCUINTREG uFlags = ASMIntDisableFlags();
1407
1408	for (;;)
1409	{
1410	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
1411	TMNotifyStartOfExecution(pVCpu);
1412
1413	rc = pVM->vmm.s.pfnR0ToRawMode(pVM);
1414	pVCpu->vmm.s.iLastGZRc = rc;
1415
1416	TMNotifyEndOfExecution(pVCpu);
1417	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
1418
1419	if (rc != VINF_VMM_CALL_TRACER)
1420	break;
1421	SUPR0TracerUmodProbeFire(pVM->pSession, &pVCpu->vmm.s.TracerCtx);
1422	}
1423
1424	/*
1425	* Re-enable VT-x before we dispatch any pending host interrupts and
1426	* re-enables interrupts.
1427	*/
1428	HMR0LeaveSwitcher(pVM, fVTxDisabled);
1429
1430	if ( rc == VINF_EM_RAW_INTERRUPT
1431	\|\| rc == VINF_EM_RAW_INTERRUPT_HYPER)
1432	TRPMR0DispatchHostInterrupt(pVM);
1433
1434	ASMSetFlags(uFlags);
1435
1436	/* Fire dtrace probe and collect statistics. */
1437	VBOXVMM_R0_VMM_RETURN_TO_RING3_RC(pVCpu, CPUMQueryGuestCtxPtr(pVCpu), rc);
1438	# ifdef VBOX_WITH_STATISTICS
1439	STAM_COUNTER_INC(&pVM->vmm.s.StatRunRC);
1440	vmmR0RecordRC(pVM, pVCpu, rc);
1441	# endif
1442	}
1443	else
1444	pVCpu->vmm.s.iLastGZRc = rc;
1445
1446	/*
1447	* Invalidate the host CPU identifiers as we restore preemption.
1448	*/
1449	pVCpu->iHostCpuSet = UINT32_MAX;
1450	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
1451
1452	RTThreadPreemptRestore(&PreemptState);
1453	}
1454	/*
1455	* Invalid CPU set index or TSC delta in need of measuring.
1456	*/
1457	else
1458	{
1459	RTThreadPreemptRestore(&PreemptState);
1460	if (iHostCpuSet < RTCPUSET_MAX_CPUS)
1461	{
1462	int rc = SUPR0TscDeltaMeasureBySetIndex(pVM->pSession, iHostCpuSet, 0 /fFlags/,
1463	2 /cMsWaitRetry/, 5RT_MS_1SEC /cMsWaitThread*/,
1464	0 /default cTries/);
1465	if (RT_SUCCESS(rc) \|\| rc == VERR_CPU_OFFLINE)
1466	pVCpu->vmm.s.iLastGZRc = VINF_EM_RAW_TO_R3;
1467	else
1468	pVCpu->vmm.s.iLastGZRc = rc;
1469	}
1470	else
1471	pVCpu->vmm.s.iLastGZRc = VERR_INVALID_CPU_INDEX;
1472	}
1473
1474	#else /* !VBOX_WITH_RAW_MODE */
1475	pVCpu->vmm.s.iLastGZRc = VERR_RAW_MODE_NOT_SUPPORTED;
1476	#endif
1477	break;
1478	}
1479
1480	/*
1481	* Run guest code using the available hardware acceleration technology.
1482	*/
1483	case VMMR0_DO_HM_RUN:
1484	{
1485	for (;;) /* hlt loop */
1486	{
1487	/*
1488	* Disable preemption.
1489	*/
1490	Assert(!vmmR0ThreadCtxHookIsEnabled(pVCpu));
1491	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
1492	RTThreadPreemptDisable(&PreemptState);
1493
1494	/*
1495	* Get the host CPU identifiers, make sure they are valid and that
1496	* we've got a TSC delta for the CPU.
1497	*/
1498	RTCPUID idHostCpu;
1499	uint32_t iHostCpuSet = RTMpCurSetIndexAndId(&idHostCpu);
1500	if (RT_LIKELY( iHostCpuSet < RTCPUSET_MAX_CPUS
1501	&& SUPIsTscDeltaAvailableForCpuSetIndex(iHostCpuSet)))
1502	{
1503	pVCpu->iHostCpuSet = iHostCpuSet;
1504	ASMAtomicWriteU32(&pVCpu->idHostCpu, idHostCpu);
1505
1506	/*
1507	* Update the periodic preemption timer if it's active.
1508	*/
1509	if (pVM->vmm.s.fUsePeriodicPreemptionTimers)
1510	GVMMR0SchedUpdatePeriodicPreemptionTimer(pVM, pVCpu->idHostCpu, TMCalcHostTimerFrequency(pVM, pVCpu));
1511	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1512
1513	#ifdef VMM_R0_TOUCH_FPU
1514	/*
1515	* Make sure we've got the FPU state loaded so and we don't need to clear
1516	* CR0.TS and get out of sync with the host kernel when loading the guest
1517	* FPU state. @ref sec_cpum_fpu (CPUM.cpp) and @bugref{4053}.
1518	*/
1519	CPUMR0TouchHostFpu();
1520	#endif
1521	int rc;
1522	bool fPreemptRestored = false;
1523	if (!HMR0SuspendPending())
1524	{
1525	/*
1526	* Enable the context switching hook.
1527	*/
1528	if (pVCpu->vmm.s.hCtxHook != NIL_RTTHREADCTXHOOK)
1529	{
1530	Assert(!RTThreadCtxHookIsEnabled(pVCpu->vmm.s.hCtxHook));
1531	int rc2 = RTThreadCtxHookEnable(pVCpu->vmm.s.hCtxHook); AssertRC(rc2);
1532	}
1533
1534	/*
1535	* Enter HM context.
1536	*/
1537	rc = HMR0Enter(pVCpu);
1538	if (RT_SUCCESS(rc))
1539	{
1540	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
1541
1542	/*
1543	* When preemption hooks are in place, enable preemption now that
1544	* we're in HM context.
1545	*/
1546	if (vmmR0ThreadCtxHookIsEnabled(pVCpu))
1547	{
1548	fPreemptRestored = true;
1549	RTThreadPreemptRestore(&PreemptState);
1550	}
1551
1552	/*
1553	* Setup the longjmp machinery and execute guest code (calls HMR0RunGuestCode).
1554	*/
1555	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1556	rc = vmmR0CallRing3SetJmp(&pVCpu->vmm.s.CallRing3JmpBufR0, HMR0RunGuestCode, pVM, pVCpu);
1557	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1558
1559	/*
1560	* Assert sanity on the way out. Using manual assertions code here as normal
1561	* assertions are going to panic the host since we're outside the setjmp/longjmp zone.
1562	*/
1563	if (RT_UNLIKELY( VMCPU_GET_STATE(pVCpu) != VMCPUSTATE_STARTED_HM
1564	&& RT_SUCCESS_NP(rc) && rc != VINF_VMM_CALL_HOST ))
1565	{
1566	pVM->vmm.s.szRing0AssertMsg1[0] = '\0';
1567	RTStrPrintf(pVM->vmm.s.szRing0AssertMsg2, sizeof(pVM->vmm.s.szRing0AssertMsg2),
1568	"Got VMCPU state %d expected %d.\n", VMCPU_GET_STATE(pVCpu), VMCPUSTATE_STARTED_HM);
1569	rc = VERR_VMM_WRONG_HM_VMCPU_STATE;
1570	}
1571	/** @todo Get rid of this. HM shouldn't disable the context hook. */
1572	else if (RT_UNLIKELY(vmmR0ThreadCtxHookIsEnabled(pVCpu)))
1573	{
1574	pVM->vmm.s.szRing0AssertMsg1[0] = '\0';
1575	RTStrPrintf(pVM->vmm.s.szRing0AssertMsg2, sizeof(pVM->vmm.s.szRing0AssertMsg2),
1576	"Thread-context hooks still enabled! VCPU=%p Id=%u rc=%d.\n", pVCpu, pVCpu->idCpu, rc);
1577	rc = VERR_INVALID_STATE;
1578	}
1579
1580	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
1581	}
1582	STAM_COUNTER_INC(&pVM->vmm.s.StatRunRC);
1583
1584	/*
1585	* Invalidate the host CPU identifiers before we disable the context
1586	* hook / restore preemption.
1587	*/
1588	pVCpu->iHostCpuSet = UINT32_MAX;
1589	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
1590
1591	/*
1592	* Disable context hooks. Due to unresolved cleanup issues, we
1593	* cannot leave the hooks enabled when we return to ring-3.
1594	*
1595	* Note! At the moment HM may also have disabled the hook
1596	* when we get here, but the IPRT API handles that.
1597	*/
1598	if (pVCpu->vmm.s.hCtxHook != NIL_RTTHREADCTXHOOK)
1599	{
1600	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
1601	RTThreadCtxHookDisable(pVCpu->vmm.s.hCtxHook);
1602	}
1603	}
1604	/*
1605	* The system is about to go into suspend mode; go back to ring 3.
1606	*/
1607	else
1608	{
1609	rc = VINF_EM_RAW_INTERRUPT;
1610	pVCpu->iHostCpuSet = UINT32_MAX;
1611	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
1612	}
1613
1614	/** @todo When HM stops messing with the context hook state, we'll disable
1615	* preemption again before the RTThreadCtxHookDisable call. */
1616	if (!fPreemptRestored)
1617	RTThreadPreemptRestore(&PreemptState);
1618
1619	pVCpu->vmm.s.iLastGZRc = rc;
1620
1621	/* Fire dtrace probe and collect statistics. */
1622	VBOXVMM_R0_VMM_RETURN_TO_RING3_HM(pVCpu, CPUMQueryGuestCtxPtr(pVCpu), rc);
1623	#ifdef VBOX_WITH_STATISTICS
1624	vmmR0RecordRC(pVM, pVCpu, rc);
1625	#endif
1626	#if 1
1627	/*
1628	* If this is a halt.
1629	*/
1630	if (rc != VINF_EM_HALT)
1631	{ /* we're not in a hurry for a HLT, so prefer this path */ }
1632	else
1633	{
1634	pVCpu->vmm.s.iLastGZRc = rc = vmmR0DoHalt(pGVM, pVM, pGVCpu, pVCpu);
1635	if (rc == VINF_SUCCESS)
1636	{
1637	pVCpu->vmm.s.cR0HaltsSucceeded++;
1638	continue;
1639	}
1640	pVCpu->vmm.s.cR0HaltsToRing3++;
1641	}
1642	#endif
1643	}
1644	/*
1645	* Invalid CPU set index or TSC delta in need of measuring.
1646	*/
1647	else
1648	{
1649	pVCpu->iHostCpuSet = UINT32_MAX;
1650	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
1651	RTThreadPreemptRestore(&PreemptState);
1652	if (iHostCpuSet < RTCPUSET_MAX_CPUS)
1653	{
1654	int rc = SUPR0TscDeltaMeasureBySetIndex(pVM->pSession, iHostCpuSet, 0 /fFlags/,
1655	2 /cMsWaitRetry/, 5RT_MS_1SEC /cMsWaitThread*/,
1656	0 /default cTries/);
1657	if (RT_SUCCESS(rc) \|\| rc == VERR_CPU_OFFLINE)
1658	pVCpu->vmm.s.iLastGZRc = VINF_EM_RAW_TO_R3;
1659	else
1660	pVCpu->vmm.s.iLastGZRc = rc;
1661	}
1662	else
1663	pVCpu->vmm.s.iLastGZRc = VERR_INVALID_CPU_INDEX;
1664	}
1665	break;
1666
1667	} /* halt loop. */
1668	break;
1669	}
1670
1671	#ifdef VBOX_WITH_NEM_R0
1672	# if defined(RT_ARCH_AMD64) && defined(RT_OS_WINDOWS)
1673	case VMMR0_DO_NEM_RUN:
1674	{
1675	/*
1676	* Setup the longjmp machinery and execute guest code (calls NEMR0RunGuestCode).
1677	*/
1678	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1679	int rc = vmmR0CallRing3SetJmp2(&pVCpu->vmm.s.CallRing3JmpBufR0, NEMR0RunGuestCode, pGVM, idCpu);
1680	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1681	STAM_COUNTER_INC(&pVM->vmm.s.StatRunRC);
1682
1683	pVCpu->vmm.s.iLastGZRc = rc;
1684
1685	/*
1686	* Fire dtrace probe and collect statistics.
1687	*/
1688	VBOXVMM_R0_VMM_RETURN_TO_RING3_NEM(pVCpu, CPUMQueryGuestCtxPtr(pVCpu), rc);
1689	# ifdef VBOX_WITH_STATISTICS
1690	vmmR0RecordRC(pVM, pVCpu, rc);
1691	# endif
1692	break;
1693	}
1694	# endif
1695	#endif
1696
1697
1698	/*
1699	* For profiling.
1700	*/
1701	case VMMR0_DO_NOP:
1702	pVCpu->vmm.s.iLastGZRc = VINF_SUCCESS;
1703	break;
1704
1705	/*
1706	* Shouldn't happen.
1707	*/
1708	default:
1709	AssertMsgFailed(("%#x\n", enmOperation));
1710	pVCpu->vmm.s.iLastGZRc = VERR_NOT_SUPPORTED;
1711	break;
1712	}
1713	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1714	}
1715
1716
1717	/**
1718	* Validates a session or VM session argument.
1719	*
1720	* @returns true / false accordingly.
1721	* @param pVM The cross context VM structure.
1722	* @param pClaimedSession The session claim to validate.
1723	* @param pSession The session argument.
1724	*/
1725	DECLINLINE(bool) vmmR0IsValidSession(PVM pVM, PSUPDRVSESSION pClaimedSession, PSUPDRVSESSION pSession)
1726	{
1727	/* This must be set! */
1728	if (!pSession)
1729	return false;
1730
1731	/* Only one out of the two. */
1732	if (pVM && pClaimedSession)
1733	return false;
1734	if (pVM)
1735	pClaimedSession = pVM->pSession;
1736	return pClaimedSession == pSession;
1737	}
1738
1739
1740	/**
1741	* VMMR0EntryEx worker function, either called directly or when ever possible
1742	* called thru a longjmp so we can exit safely on failure.
1743	*
1744	* @returns VBox status code.
1745	* @param pGVM The global (ring-0) VM structure.
1746	* @param pVM The cross context VM structure.
1747	* @param idCpu Virtual CPU ID argument. Must be NIL_VMCPUID if pVM
1748	* is NIL_RTR0PTR, and may be NIL_VMCPUID if it isn't
1749	* @param enmOperation Which operation to execute.
1750	* @param pReqHdr This points to a SUPVMMR0REQHDR packet. Optional.
1751	* The support driver validates this if it's present.
1752	* @param u64Arg Some simple constant argument.
1753	* @param pSession The session of the caller.
1754	*
1755	* @remarks Assume called with interrupts _enabled_.
1756	*/
1757	static int vmmR0EntryExWorker(PGVM pGVM, PVM pVM, VMCPUID idCpu, VMMR0OPERATION enmOperation,
1758	PSUPVMMR0REQHDR pReqHdr, uint64_t u64Arg, PSUPDRVSESSION pSession)
1759	{
1760	/*
1761	* Validate pGVM, pVM and idCpu for consistency and validity.
1762	*/
1763	if ( pGVM != NULL
1764	\|\| pVM != NULL)
1765	{
1766	if (RT_LIKELY( RT_VALID_PTR(pGVM)
1767	&& RT_VALID_PTR(pVM)
1768	&& ((uintptr_t)pVM & PAGE_OFFSET_MASK) == 0))
1769	{ /* likely */ }
1770	else
1771	{
1772	SUPR0Printf("vmmR0EntryExWorker: Invalid pGVM=%p and/or pVM=%p! (op=%d)\n", pGVM, pVM, enmOperation);
1773	return VERR_INVALID_POINTER;
1774	}
1775
1776	#ifdef VBOX_BUGREF_9217
1777	if (RT_LIKELY(pGVM == pVM))
1778	#else
1779	if (RT_LIKELY(pGVM->pVM == pVM))
1780	#endif
1781	{ /* likely */ }
1782	else
1783	{
1784	#ifdef VBOX_BUGREF_9217
1785	SUPR0Printf("vmmR0EntryExWorker: pVM mismatch: got %p, pGVM/pVM=%p\n", pVM, pGVM);
1786	#else
1787	SUPR0Printf("vmmR0EntryExWorker: pVM mismatch: got %p, pGVM->pVM=%p\n", pVM, pGVM->pVM);
1788	#endif
1789	return VERR_INVALID_PARAMETER;
1790	}
1791
1792	if (RT_LIKELY(idCpu == NIL_VMCPUID \|\| idCpu < pGVM->cCpus))
1793	{ /* likely */ }
1794	else
1795	{
1796	SUPR0Printf("vmmR0EntryExWorker: Invalid idCpu %#x (cCpus=%#x)\n", idCpu, pGVM->cCpus);
1797	return VERR_INVALID_PARAMETER;
1798	}
1799
1800	if (RT_LIKELY( pVM->enmVMState >= VMSTATE_CREATING
1801	&& pVM->enmVMState <= VMSTATE_TERMINATED
1802	&& pVM->cCpus == pGVM->cCpus
1803	&& pVM->pSession == pSession
1804	&& pVM->pVMR0 == pVM))
1805	{ /* likely */ }
1806	else
1807	{
1808	SUPR0Printf("vmmR0EntryExWorker: Invalid pVM=%p:{.enmVMState=%d, .cCpus=%#x(==%#x), .pSession=%p(==%p), .pVMR0=%p(==%p)}! (op=%d)\n",
1809	pVM, pVM->enmVMState, pVM->cCpus, pGVM->cCpus, pVM->pSession, pSession, pVM->pVMR0, pVM, enmOperation);
1810	return VERR_INVALID_POINTER;
1811	}
1812	}
1813	else if (RT_LIKELY(idCpu == NIL_VMCPUID))
1814	{ /* likely */ }
1815	else
1816	{
1817	SUPR0Printf("vmmR0EntryExWorker: Invalid idCpu=%u\n", idCpu);
1818	return VERR_INVALID_PARAMETER;
1819	}
1820
1821	/*
1822	* SMAP fun.
1823	*/
1824	VMM_CHECK_SMAP_SETUP();
1825	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
1826
1827	/*
1828	* Process the request.
1829	*/
1830	int rc;
1831	switch (enmOperation)
1832	{
1833	/*
1834	* GVM requests
1835	*/
1836	case VMMR0_DO_GVMM_CREATE_VM:
1837	if (pGVM == NULL && pVM == NULL && u64Arg == 0 && idCpu == NIL_VMCPUID)
1838	rc = GVMMR0CreateVMReq((PGVMMCREATEVMREQ)pReqHdr, pSession);
1839	else
1840	rc = VERR_INVALID_PARAMETER;
1841	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
1842	break;
1843
1844	case VMMR0_DO_GVMM_DESTROY_VM:
1845	if (pReqHdr == NULL && u64Arg == 0)
1846	rc = GVMMR0DestroyVM(pGVM, pVM);
1847	else
1848	rc = VERR_INVALID_PARAMETER;
1849	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
1850	break;
1851
1852	case VMMR0_DO_GVMM_REGISTER_VMCPU:
1853	if (pGVM != NULL && pVM != NULL)
1854	rc = GVMMR0RegisterVCpu(pGVM, pVM, idCpu);
1855	else
1856	rc = VERR_INVALID_PARAMETER;
1857	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1858	break;
1859
1860	case VMMR0_DO_GVMM_DEREGISTER_VMCPU:
1861	if (pGVM != NULL && pVM != NULL)
1862	rc = GVMMR0DeregisterVCpu(pGVM, pVM, idCpu);
1863	else
1864	rc = VERR_INVALID_PARAMETER;
1865	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1866	break;
1867
1868	case VMMR0_DO_GVMM_SCHED_HALT:
1869	if (pReqHdr)
1870	return VERR_INVALID_PARAMETER;
1871	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1872	rc = GVMMR0SchedHaltReq(pGVM, pVM, idCpu, u64Arg);
1873	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1874	break;
1875
1876	case VMMR0_DO_GVMM_SCHED_WAKE_UP:
1877	if (pReqHdr \|\| u64Arg)
1878	return VERR_INVALID_PARAMETER;
1879	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1880	rc = GVMMR0SchedWakeUp(pGVM, pVM, idCpu);
1881	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1882	break;
1883
1884	case VMMR0_DO_GVMM_SCHED_POKE:
1885	if (pReqHdr \|\| u64Arg)
1886	return VERR_INVALID_PARAMETER;
1887	rc = GVMMR0SchedPoke(pGVM, pVM, idCpu);
1888	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1889	break;
1890
1891	case VMMR0_DO_GVMM_SCHED_WAKE_UP_AND_POKE_CPUS:
1892	if (u64Arg)
1893	return VERR_INVALID_PARAMETER;
1894	rc = GVMMR0SchedWakeUpAndPokeCpusReq(pGVM, pVM, (PGVMMSCHEDWAKEUPANDPOKECPUSREQ)pReqHdr);
1895	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1896	break;
1897
1898	case VMMR0_DO_GVMM_SCHED_POLL:
1899	if (pReqHdr \|\| u64Arg > 1)
1900	return VERR_INVALID_PARAMETER;
1901	rc = GVMMR0SchedPoll(pGVM, pVM, idCpu, !!u64Arg);
1902	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1903	break;
1904
1905	case VMMR0_DO_GVMM_QUERY_STATISTICS:
1906	if (u64Arg)
1907	return VERR_INVALID_PARAMETER;
1908	rc = GVMMR0QueryStatisticsReq(pGVM, pVM, (PGVMMQUERYSTATISTICSSREQ)pReqHdr, pSession);
1909	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1910	break;
1911
1912	case VMMR0_DO_GVMM_RESET_STATISTICS:
1913	if (u64Arg)
1914	return VERR_INVALID_PARAMETER;
1915	rc = GVMMR0ResetStatisticsReq(pGVM, pVM, (PGVMMRESETSTATISTICSSREQ)pReqHdr, pSession);
1916	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1917	break;
1918
1919	/*
1920	* Initialize the R0 part of a VM instance.
1921	*/
1922	case VMMR0_DO_VMMR0_INIT:
1923	rc = vmmR0InitVM(pGVM, pVM, RT_LODWORD(u64Arg), RT_HIDWORD(u64Arg));
1924	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1925	break;
1926
1927	/*
1928	* Does EMT specific ring-0 init.
1929	*/
1930	case VMMR0_DO_VMMR0_INIT_EMT:
1931	rc = vmmR0InitVMEmt(pGVM, pVM, idCpu);
1932	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1933	break;
1934
1935	/*
1936	* Terminate the R0 part of a VM instance.
1937	*/
1938	case VMMR0_DO_VMMR0_TERM:
1939	rc = VMMR0TermVM(pGVM, pVM, 0 /idCpu/);
1940	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1941	break;
1942
1943	/*
1944	* Attempt to enable hm mode and check the current setting.
1945	*/
1946	case VMMR0_DO_HM_ENABLE:
1947	rc = HMR0EnableAllCpus(pVM);
1948	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1949	break;
1950
1951	/*
1952	* Setup the hardware accelerated session.
1953	*/
1954	case VMMR0_DO_HM_SETUP_VM:
1955	rc = HMR0SetupVM(pVM);
1956	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1957	break;
1958
1959	/*
1960	* Switch to RC to execute Hypervisor function.
1961	*/
1962	case VMMR0_DO_CALL_HYPERVISOR:
1963	{
1964	#ifdef VBOX_WITH_RAW_MODE
1965	/*
1966	* Validate input / context.
1967	*/
1968	if (RT_UNLIKELY(idCpu != 0))
1969	return VERR_INVALID_CPU_ID;
1970	if (RT_UNLIKELY(pVM->cCpus != 1))
1971	return VERR_INVALID_PARAMETER;
1972	# ifdef VBOX_BUGREF_9217
1973	PVMCPU pVCpu = &pGVM->aCpus[idCpu];
1974	# else
1975	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1976	# endif
1977	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
1978	if (RT_UNLIKELY(!PGMGetHyperCR3(pVCpu)))
1979	return VERR_PGM_NO_CR3_SHADOW_ROOT;
1980	# endif
1981	if (RT_FAILURE(g_rcRawModeUsability))
1982	return g_rcRawModeUsability;
1983
1984	/*
1985	* Disable interrupts.
1986	*/
1987	RTCCUINTREG fFlags = ASMIntDisableFlags();
1988
1989	/*
1990	* Get the host CPU identifiers, make sure they are valid and that
1991	* we've got a TSC delta for the CPU.
1992	*/
1993	RTCPUID idHostCpu;
1994	uint32_t iHostCpuSet = RTMpCurSetIndexAndId(&idHostCpu);
1995	if (RT_UNLIKELY(iHostCpuSet >= RTCPUSET_MAX_CPUS))
1996	{
1997	ASMSetFlags(fFlags);
1998	return VERR_INVALID_CPU_INDEX;
1999	}
2000	if (RT_UNLIKELY(!SUPIsTscDeltaAvailableForCpuSetIndex(iHostCpuSet)))
2001	{
2002	ASMSetFlags(fFlags);
2003	rc = SUPR0TscDeltaMeasureBySetIndex(pVM->pSession, iHostCpuSet, 0 /fFlags/,
2004	2 /cMsWaitRetry/, 5RT_MS_1SEC /cMsWaitThread*/,
2005	0 /default cTries/);
2006	if (RT_FAILURE(rc) && rc != VERR_CPU_OFFLINE)
2007	{
2008	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2009	return rc;
2010	}
2011	}
2012
2013	/*
2014	* Commit the CPU identifiers.
2015	*/
2016	# ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
2017	CPUMR0SetLApic(pVCpu, iHostCpuSet);
2018	# endif
2019	pVCpu->iHostCpuSet = iHostCpuSet;
2020	ASMAtomicWriteU32(&pVCpu->idHostCpu, idHostCpu);
2021
2022	/*
2023	* We might need to disable VT-x if the active switcher turns off paging.
2024	*/
2025	bool fVTxDisabled;
2026	rc = HMR0EnterSwitcher(pVM, pVM->vmm.s.enmSwitcher, &fVTxDisabled);
2027	if (RT_SUCCESS(rc))
2028	{
2029	/*
2030	* Go through the wormhole...
2031	*/
2032	rc = pVM->vmm.s.pfnR0ToRawMode(pVM);
2033
2034	/*
2035	* Re-enable VT-x before we dispatch any pending host interrupts.
2036	*/
2037	HMR0LeaveSwitcher(pVM, fVTxDisabled);
2038
2039	if ( rc == VINF_EM_RAW_INTERRUPT
2040	\|\| rc == VINF_EM_RAW_INTERRUPT_HYPER)
2041	TRPMR0DispatchHostInterrupt(pVM);
2042	}
2043
2044	/*
2045	* Invalidate the host CPU identifiers as we restore interrupts.
2046	*/
2047	pVCpu->iHostCpuSet = UINT32_MAX;
2048	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
2049	ASMSetFlags(fFlags);
2050
2051	#else /* !VBOX_WITH_RAW_MODE */
2052	rc = VERR_RAW_MODE_NOT_SUPPORTED;
2053	#endif
2054	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2055	break;
2056	}
2057
2058	/*
2059	* PGM wrappers.
2060	*/
2061	case VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES:
2062	if (idCpu == NIL_VMCPUID)
2063	return VERR_INVALID_CPU_ID;
2064	rc = PGMR0PhysAllocateHandyPages(pGVM, pVM, idCpu);
2065	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2066	break;
2067
2068	case VMMR0_DO_PGM_FLUSH_HANDY_PAGES:
2069	if (idCpu == NIL_VMCPUID)
2070	return VERR_INVALID_CPU_ID;
2071	rc = PGMR0PhysFlushHandyPages(pGVM, pVM, idCpu);
2072	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2073	break;
2074
2075	case VMMR0_DO_PGM_ALLOCATE_LARGE_HANDY_PAGE:
2076	if (idCpu == NIL_VMCPUID)
2077	return VERR_INVALID_CPU_ID;
2078	rc = PGMR0PhysAllocateLargeHandyPage(pGVM, pVM, idCpu);
2079	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2080	break;
2081
2082	case VMMR0_DO_PGM_PHYS_SETUP_IOMMU:
2083	if (idCpu != 0)
2084	return VERR_INVALID_CPU_ID;
2085	rc = PGMR0PhysSetupIoMmu(pGVM, pVM);
2086	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2087	break;
2088
2089	/*
2090	* GMM wrappers.
2091	*/
2092	case VMMR0_DO_GMM_INITIAL_RESERVATION:
2093	if (u64Arg)
2094	return VERR_INVALID_PARAMETER;
2095	rc = GMMR0InitialReservationReq(pGVM, pVM, idCpu, (PGMMINITIALRESERVATIONREQ)pReqHdr);
2096	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2097	break;
2098
2099	case VMMR0_DO_GMM_UPDATE_RESERVATION:
2100	if (u64Arg)
2101	return VERR_INVALID_PARAMETER;
2102	rc = GMMR0UpdateReservationReq(pGVM, pVM, idCpu, (PGMMUPDATERESERVATIONREQ)pReqHdr);
2103	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2104	break;
2105
2106	case VMMR0_DO_GMM_ALLOCATE_PAGES:
2107	if (u64Arg)
2108	return VERR_INVALID_PARAMETER;
2109	rc = GMMR0AllocatePagesReq(pGVM, pVM, idCpu, (PGMMALLOCATEPAGESREQ)pReqHdr);
2110	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2111	break;
2112
2113	case VMMR0_DO_GMM_FREE_PAGES:
2114	if (u64Arg)
2115	return VERR_INVALID_PARAMETER;
2116	rc = GMMR0FreePagesReq(pGVM, pVM, idCpu, (PGMMFREEPAGESREQ)pReqHdr);
2117	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2118	break;
2119
2120	case VMMR0_DO_GMM_FREE_LARGE_PAGE:
2121	if (u64Arg)
2122	return VERR_INVALID_PARAMETER;
2123	rc = GMMR0FreeLargePageReq(pGVM, pVM, idCpu, (PGMMFREELARGEPAGEREQ)pReqHdr);
2124	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2125	break;
2126
2127	case VMMR0_DO_GMM_QUERY_HYPERVISOR_MEM_STATS:
2128	if (u64Arg)
2129	return VERR_INVALID_PARAMETER;
2130	rc = GMMR0QueryHypervisorMemoryStatsReq((PGMMMEMSTATSREQ)pReqHdr);
2131	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2132	break;
2133
2134	case VMMR0_DO_GMM_QUERY_MEM_STATS:
2135	if (idCpu == NIL_VMCPUID)
2136	return VERR_INVALID_CPU_ID;
2137	if (u64Arg)
2138	return VERR_INVALID_PARAMETER;
2139	rc = GMMR0QueryMemoryStatsReq(pGVM, pVM, idCpu, (PGMMMEMSTATSREQ)pReqHdr);
2140	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2141	break;
2142
2143	case VMMR0_DO_GMM_BALLOONED_PAGES:
2144	if (u64Arg)
2145	return VERR_INVALID_PARAMETER;
2146	rc = GMMR0BalloonedPagesReq(pGVM, pVM, idCpu, (PGMMBALLOONEDPAGESREQ)pReqHdr);
2147	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2148	break;
2149
2150	case VMMR0_DO_GMM_MAP_UNMAP_CHUNK:
2151	if (u64Arg)
2152	return VERR_INVALID_PARAMETER;
2153	rc = GMMR0MapUnmapChunkReq(pGVM, pVM, (PGMMMAPUNMAPCHUNKREQ)pReqHdr);
2154	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2155	break;
2156
2157	case VMMR0_DO_GMM_SEED_CHUNK:
2158	if (pReqHdr)
2159	return VERR_INVALID_PARAMETER;
2160	rc = GMMR0SeedChunk(pGVM, pVM, idCpu, (RTR3PTR)u64Arg);
2161	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2162	break;
2163
2164	case VMMR0_DO_GMM_REGISTER_SHARED_MODULE:
2165	if (idCpu == NIL_VMCPUID)
2166	return VERR_INVALID_CPU_ID;
2167	if (u64Arg)
2168	return VERR_INVALID_PARAMETER;
2169	rc = GMMR0RegisterSharedModuleReq(pGVM, pVM, idCpu, (PGMMREGISTERSHAREDMODULEREQ)pReqHdr);
2170	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2171	break;
2172
2173	case VMMR0_DO_GMM_UNREGISTER_SHARED_MODULE:
2174	if (idCpu == NIL_VMCPUID)
2175	return VERR_INVALID_CPU_ID;
2176	if (u64Arg)
2177	return VERR_INVALID_PARAMETER;
2178	rc = GMMR0UnregisterSharedModuleReq(pGVM, pVM, idCpu, (PGMMUNREGISTERSHAREDMODULEREQ)pReqHdr);
2179	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2180	break;
2181
2182	case VMMR0_DO_GMM_RESET_SHARED_MODULES:
2183	if (idCpu == NIL_VMCPUID)
2184	return VERR_INVALID_CPU_ID;
2185	if ( u64Arg
2186	\|\| pReqHdr)
2187	return VERR_INVALID_PARAMETER;
2188	rc = GMMR0ResetSharedModules(pGVM, pVM, idCpu);
2189	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2190	break;
2191
2192	#ifdef VBOX_WITH_PAGE_SHARING
2193	case VMMR0_DO_GMM_CHECK_SHARED_MODULES:
2194	{
2195	if (idCpu == NIL_VMCPUID)
2196	return VERR_INVALID_CPU_ID;
2197	if ( u64Arg
2198	\|\| pReqHdr)
2199	return VERR_INVALID_PARAMETER;
2200	rc = GMMR0CheckSharedModules(pGVM, pVM, idCpu);
2201	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2202	break;
2203	}
2204	#endif
2205
2206	#if defined(VBOX_STRICT) && HC_ARCH_BITS == 64
2207	case VMMR0_DO_GMM_FIND_DUPLICATE_PAGE:
2208	if (u64Arg)
2209	return VERR_INVALID_PARAMETER;
2210	rc = GMMR0FindDuplicatePageReq(pGVM, pVM, (PGMMFINDDUPLICATEPAGEREQ)pReqHdr);
2211	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2212	break;
2213	#endif
2214
2215	case VMMR0_DO_GMM_QUERY_STATISTICS:
2216	if (u64Arg)
2217	return VERR_INVALID_PARAMETER;
2218	rc = GMMR0QueryStatisticsReq(pGVM, pVM, (PGMMQUERYSTATISTICSSREQ)pReqHdr);
2219	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2220	break;
2221
2222	case VMMR0_DO_GMM_RESET_STATISTICS:
2223	if (u64Arg)
2224	return VERR_INVALID_PARAMETER;
2225	rc = GMMR0ResetStatisticsReq(pGVM, pVM, (PGMMRESETSTATISTICSSREQ)pReqHdr);
2226	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2227	break;
2228
2229	/*
2230	* A quick GCFGM mock-up.
2231	*/
2232	/** @todo GCFGM with proper access control, ring-3 management interface and all that. */
2233	case VMMR0_DO_GCFGM_SET_VALUE:
2234	case VMMR0_DO_GCFGM_QUERY_VALUE:
2235	{
2236	if (pGVM \|\| pVM \|\| !pReqHdr \|\| u64Arg \|\| idCpu != NIL_VMCPUID)
2237	return VERR_INVALID_PARAMETER;
2238	PGCFGMVALUEREQ pReq = (PGCFGMVALUEREQ)pReqHdr;
2239	if (pReq->Hdr.cbReq != sizeof(*pReq))
2240	return VERR_INVALID_PARAMETER;
2241	if (enmOperation == VMMR0_DO_GCFGM_SET_VALUE)
2242	{
2243	rc = GVMMR0SetConfig(pReq->pSession, &pReq->szName[0], pReq->u64Value);
2244	//if (rc == VERR_CFGM_VALUE_NOT_FOUND)
2245	// rc = GMMR0SetConfig(pReq->pSession, &pReq->szName[0], pReq->u64Value);
2246	}
2247	else
2248	{
2249	rc = GVMMR0QueryConfig(pReq->pSession, &pReq->szName[0], &pReq->u64Value);
2250	//if (rc == VERR_CFGM_VALUE_NOT_FOUND)
2251	// rc = GMMR0QueryConfig(pReq->pSession, &pReq->szName[0], &pReq->u64Value);
2252	}
2253	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2254	break;
2255	}
2256
2257	/*
2258	* PDM Wrappers.
2259	*/
2260	case VMMR0_DO_PDM_DRIVER_CALL_REQ_HANDLER:
2261	{
2262	if (!pReqHdr \|\| u64Arg \|\| idCpu != NIL_VMCPUID)
2263	return VERR_INVALID_PARAMETER;
2264	rc = PDMR0DriverCallReqHandler(pGVM, pVM, (PPDMDRIVERCALLREQHANDLERREQ)pReqHdr);
2265	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2266	break;
2267	}
2268
2269	case VMMR0_DO_PDM_DEVICE_CALL_REQ_HANDLER:
2270	{
2271	if (!pReqHdr \|\| u64Arg \|\| idCpu != NIL_VMCPUID)
2272	return VERR_INVALID_PARAMETER;
2273	rc = PDMR0DeviceCallReqHandler(pGVM, pVM, (PPDMDEVICECALLREQHANDLERREQ)pReqHdr);
2274	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2275	break;
2276	}
2277
2278	/*
2279	* Requests to the internal networking service.
2280	*/
2281	case VMMR0_DO_INTNET_OPEN:
2282	{
2283	PINTNETOPENREQ pReq = (PINTNETOPENREQ)pReqHdr;
2284	if (u64Arg \|\| !pReq \|\| !vmmR0IsValidSession(pVM, pReq->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2285	return VERR_INVALID_PARAMETER;
2286	rc = IntNetR0OpenReq(pSession, pReq);
2287	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2288	break;
2289	}
2290
2291	case VMMR0_DO_INTNET_IF_CLOSE:
2292	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFCLOSEREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2293	return VERR_INVALID_PARAMETER;
2294	rc = IntNetR0IfCloseReq(pSession, (PINTNETIFCLOSEREQ)pReqHdr);
2295	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2296	break;
2297
2298
2299	case VMMR0_DO_INTNET_IF_GET_BUFFER_PTRS:
2300	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFGETBUFFERPTRSREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2301	return VERR_INVALID_PARAMETER;
2302	rc = IntNetR0IfGetBufferPtrsReq(pSession, (PINTNETIFGETBUFFERPTRSREQ)pReqHdr);
2303	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2304	break;
2305
2306	case VMMR0_DO_INTNET_IF_SET_PROMISCUOUS_MODE:
2307	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFSETPROMISCUOUSMODEREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2308	return VERR_INVALID_PARAMETER;
2309	rc = IntNetR0IfSetPromiscuousModeReq(pSession, (PINTNETIFSETPROMISCUOUSMODEREQ)pReqHdr);
2310	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2311	break;
2312
2313	case VMMR0_DO_INTNET_IF_SET_MAC_ADDRESS:
2314	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFSETMACADDRESSREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2315	return VERR_INVALID_PARAMETER;
2316	rc = IntNetR0IfSetMacAddressReq(pSession, (PINTNETIFSETMACADDRESSREQ)pReqHdr);
2317	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2318	break;
2319
2320	case VMMR0_DO_INTNET_IF_SET_ACTIVE:
2321	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFSETACTIVEREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2322	return VERR_INVALID_PARAMETER;
2323	rc = IntNetR0IfSetActiveReq(pSession, (PINTNETIFSETACTIVEREQ)pReqHdr);
2324	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2325	break;
2326
2327	case VMMR0_DO_INTNET_IF_SEND:
2328	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFSENDREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2329	return VERR_INVALID_PARAMETER;
2330	rc = IntNetR0IfSendReq(pSession, (PINTNETIFSENDREQ)pReqHdr);
2331	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2332	break;
2333
2334	case VMMR0_DO_INTNET_IF_WAIT:
2335	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFWAITREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2336	return VERR_INVALID_PARAMETER;
2337	rc = IntNetR0IfWaitReq(pSession, (PINTNETIFWAITREQ)pReqHdr);
2338	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2339	break;
2340
2341	case VMMR0_DO_INTNET_IF_ABORT_WAIT:
2342	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFWAITREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2343	return VERR_INVALID_PARAMETER;
2344	rc = IntNetR0IfAbortWaitReq(pSession, (PINTNETIFABORTWAITREQ)pReqHdr);
2345	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2346	break;
2347
2348	#ifdef VBOX_WITH_PCI_PASSTHROUGH
2349	/*
2350	* Requests to host PCI driver service.
2351	*/
2352	case VMMR0_DO_PCIRAW_REQ:
2353	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PPCIRAWSENDREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2354	return VERR_INVALID_PARAMETER;
2355	rc = PciRawR0ProcessReq(pGVM, pVM, pSession, (PPCIRAWSENDREQ)pReqHdr);
2356	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2357	break;
2358	#endif
2359
2360	/*
2361	* NEM requests.
2362	*/
2363	#ifdef VBOX_WITH_NEM_R0
2364	# if defined(RT_ARCH_AMD64) && defined(RT_OS_WINDOWS)
2365	case VMMR0_DO_NEM_INIT_VM:
2366	if (u64Arg \|\| pReqHdr \|\| idCpu != 0)
2367	return VERR_INVALID_PARAMETER;
2368	rc = NEMR0InitVM(pGVM, pVM);
2369	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2370	break;
2371
2372	case VMMR0_DO_NEM_INIT_VM_PART_2:
2373	if (u64Arg \|\| pReqHdr \|\| idCpu != 0)
2374	return VERR_INVALID_PARAMETER;
2375	rc = NEMR0InitVMPart2(pGVM, pVM);
2376	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2377	break;
2378
2379	case VMMR0_DO_NEM_MAP_PAGES:
2380	if (u64Arg \|\| pReqHdr \|\| idCpu == NIL_VMCPUID)
2381	return VERR_INVALID_PARAMETER;
2382	rc = NEMR0MapPages(pGVM, pVM, idCpu);
2383	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2384	break;
2385
2386	case VMMR0_DO_NEM_UNMAP_PAGES:
2387	if (u64Arg \|\| pReqHdr \|\| idCpu == NIL_VMCPUID)
2388	return VERR_INVALID_PARAMETER;
2389	rc = NEMR0UnmapPages(pGVM, pVM, idCpu);
2390	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2391	break;
2392
2393	case VMMR0_DO_NEM_EXPORT_STATE:
2394	if (u64Arg \|\| pReqHdr \|\| idCpu == NIL_VMCPUID)
2395	return VERR_INVALID_PARAMETER;
2396	rc = NEMR0ExportState(pGVM, pVM, idCpu);
2397	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2398	break;
2399
2400	case VMMR0_DO_NEM_IMPORT_STATE:
2401	if (pReqHdr \|\| idCpu == NIL_VMCPUID)
2402	return VERR_INVALID_PARAMETER;
2403	rc = NEMR0ImportState(pGVM, pVM, idCpu, u64Arg);
2404	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2405	break;
2406
2407	case VMMR0_DO_NEM_QUERY_CPU_TICK:
2408	if (u64Arg \|\| pReqHdr \|\| idCpu == NIL_VMCPUID)
2409	return VERR_INVALID_PARAMETER;
2410	rc = NEMR0QueryCpuTick(pGVM, pVM, idCpu);
2411	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2412	break;
2413
2414	case VMMR0_DO_NEM_RESUME_CPU_TICK_ON_ALL:
2415	if (pReqHdr \|\| idCpu == NIL_VMCPUID)
2416	return VERR_INVALID_PARAMETER;
2417	rc = NEMR0ResumeCpuTickOnAll(pGVM, pVM, idCpu, u64Arg);
2418	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2419	break;
2420
2421	case VMMR0_DO_NEM_UPDATE_STATISTICS:
2422	if (u64Arg \|\| pReqHdr)
2423	return VERR_INVALID_PARAMETER;
2424	rc = NEMR0UpdateStatistics(pGVM, pVM, idCpu);
2425	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2426	break;
2427
2428	# if 1 && defined(DEBUG_bird)
2429	case VMMR0_DO_NEM_EXPERIMENT:
2430	if (pReqHdr)
2431	return VERR_INVALID_PARAMETER;
2432	rc = NEMR0DoExperiment(pGVM, pVM, idCpu, u64Arg);
2433	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2434	break;
2435	# endif
2436	# endif
2437	#endif
2438
2439	/*
2440	* For profiling.
2441	*/
2442	case VMMR0_DO_NOP:
2443	case VMMR0_DO_SLOW_NOP:
2444	return VINF_SUCCESS;
2445
2446	/*
2447	* For testing Ring-0 APIs invoked in this environment.
2448	*/
2449	case VMMR0_DO_TESTS:
2450	/** @todo make new test */
2451	return VINF_SUCCESS;
2452
2453
2454	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
2455	case VMMR0_DO_TEST_SWITCHER3264:
2456	if (idCpu == NIL_VMCPUID)
2457	return VERR_INVALID_CPU_ID;
2458	rc = HMR0TestSwitcher3264(pVM);
2459	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2460	break;
2461	#endif
2462	default:
2463	/*
2464	* We're returning VERR_NOT_SUPPORT here so we've got something else
2465	* than -1 which the interrupt gate glue code might return.
2466	*/
2467	Log(("operation %#x is not supported\n", enmOperation));
2468	return VERR_NOT_SUPPORTED;
2469	}
2470	return rc;
2471	}
2472
2473
2474	/**
2475	* Argument for vmmR0EntryExWrapper containing the arguments for VMMR0EntryEx.
2476	*/
2477	typedef struct VMMR0ENTRYEXARGS
2478	{
2479	PGVM pGVM;
2480	PVM pVM;
2481	VMCPUID idCpu;
2482	VMMR0OPERATION enmOperation;
2483	PSUPVMMR0REQHDR pReq;
2484	uint64_t u64Arg;
2485	PSUPDRVSESSION pSession;
2486	} VMMR0ENTRYEXARGS;
2487	/** Pointer to a vmmR0EntryExWrapper argument package. */
2488	typedef VMMR0ENTRYEXARGS *PVMMR0ENTRYEXARGS;
2489
2490	/**
2491	* This is just a longjmp wrapper function for VMMR0EntryEx calls.
2492	*
2493	* @returns VBox status code.
2494	* @param pvArgs The argument package
2495	*/
2496	static DECLCALLBACK(int) vmmR0EntryExWrapper(void *pvArgs)
2497	{
2498	return vmmR0EntryExWorker(((PVMMR0ENTRYEXARGS)pvArgs)->pGVM,
2499	((PVMMR0ENTRYEXARGS)pvArgs)->pVM,
2500	((PVMMR0ENTRYEXARGS)pvArgs)->idCpu,
2501	((PVMMR0ENTRYEXARGS)pvArgs)->enmOperation,
2502	((PVMMR0ENTRYEXARGS)pvArgs)->pReq,
2503	((PVMMR0ENTRYEXARGS)pvArgs)->u64Arg,
2504	((PVMMR0ENTRYEXARGS)pvArgs)->pSession);
2505	}
2506
2507
2508	/**
2509	* The Ring 0 entry point, called by the support library (SUP).
2510	*
2511	* @returns VBox status code.
2512	* @param pGVM The global (ring-0) VM structure.
2513	* @param pVM The cross context VM structure.
2514	* @param idCpu Virtual CPU ID argument. Must be NIL_VMCPUID if pVM
2515	* is NIL_RTR0PTR, and may be NIL_VMCPUID if it isn't
2516	* @param enmOperation Which operation to execute.
2517	* @param pReq Pointer to the SUPVMMR0REQHDR packet. Optional.
2518	* @param u64Arg Some simple constant argument.
2519	* @param pSession The session of the caller.
2520	* @remarks Assume called with interrupts _enabled_.
2521	*/
2522	VMMR0DECL(int) VMMR0EntryEx(PGVM pGVM, PVM pVM, VMCPUID idCpu, VMMR0OPERATION enmOperation,
2523	PSUPVMMR0REQHDR pReq, uint64_t u64Arg, PSUPDRVSESSION pSession)
2524	{
2525	/*
2526	* Requests that should only happen on the EMT thread will be
2527	* wrapped in a setjmp so we can assert without causing trouble.
2528	*/
2529	if ( pVM != NULL
2530	&& pGVM != NULL
2531	&& idCpu < pGVM->cCpus
2532	&& pVM->pVMR0 != NULL)
2533	{
2534	switch (enmOperation)
2535	{
2536	/* These might/will be called before VMMR3Init. */
2537	case VMMR0_DO_GMM_INITIAL_RESERVATION:
2538	case VMMR0_DO_GMM_UPDATE_RESERVATION:
2539	case VMMR0_DO_GMM_ALLOCATE_PAGES:
2540	case VMMR0_DO_GMM_FREE_PAGES:
2541	case VMMR0_DO_GMM_BALLOONED_PAGES:
2542	/* On the mac we might not have a valid jmp buf, so check these as well. */
2543	case VMMR0_DO_VMMR0_INIT:
2544	case VMMR0_DO_VMMR0_TERM:
2545	{
2546	PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
2547	#ifdef VBOX_BUGREF_9217
2548	PVMCPU pVCpu = pGVCpu;
2549	#else
2550	PVMCPU pVCpu = &pVM->aCpus[idCpu];
2551	#endif
2552	RTNATIVETHREAD hNativeThread = RTThreadNativeSelf();
2553	if (RT_LIKELY( pGVCpu->hEMT == hNativeThread
2554	&& pVCpu->hNativeThreadR0 == hNativeThread))
2555	{
2556	if (!pVCpu->vmm.s.CallRing3JmpBufR0.pvSavedStack)
2557	break;
2558
2559	/** @todo validate this EMT claim... GVM knows. */
2560	VMMR0ENTRYEXARGS Args;
2561	Args.pGVM = pGVM;
2562	Args.pVM = pVM;
2563	Args.idCpu = idCpu;
2564	Args.enmOperation = enmOperation;
2565	Args.pReq = pReq;
2566	Args.u64Arg = u64Arg;
2567	Args.pSession = pSession;
2568	return vmmR0CallRing3SetJmpEx(&pVCpu->vmm.s.CallRing3JmpBufR0, vmmR0EntryExWrapper, &Args);
2569	}
2570	return VERR_VM_THREAD_NOT_EMT;
2571	}
2572
2573	default:
2574	break;
2575	}
2576	}
2577	return vmmR0EntryExWorker(pGVM, pVM, idCpu, enmOperation, pReq, u64Arg, pSession);
2578	}
2579
2580
2581	/**
2582	* Checks whether we've armed the ring-0 long jump machinery.
2583	*
2584	* @returns @c true / @c false
2585	* @param pVCpu The cross context virtual CPU structure.
2586	* @thread EMT
2587	* @sa VMMIsLongJumpArmed
2588	*/
2589	VMMR0_INT_DECL(bool) VMMR0IsLongJumpArmed(PVMCPU pVCpu)
2590	{
2591	#ifdef RT_ARCH_X86
2592	return pVCpu->vmm.s.CallRing3JmpBufR0.eip
2593	&& !pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call;
2594	#else
2595	return pVCpu->vmm.s.CallRing3JmpBufR0.rip
2596	&& !pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call;
2597	#endif
2598	}
2599
2600
2601	/**
2602	* Checks whether we've done a ring-3 long jump.
2603	*
2604	* @returns @c true / @c false
2605	* @param pVCpu The cross context virtual CPU structure.
2606	* @thread EMT
2607	*/
2608	VMMR0_INT_DECL(bool) VMMR0IsInRing3LongJump(PVMCPU pVCpu)
2609	{
2610	return pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call;
2611	}
2612
2613
2614	/**
2615	* Internal R0 logger worker: Flush logger.
2616	*
2617	* @param pLogger The logger instance to flush.
2618	* @remark This function must be exported!
2619	*/
2620	VMMR0DECL(void) vmmR0LoggerFlush(PRTLOGGER pLogger)
2621	{
2622	#ifdef LOG_ENABLED
2623	/*
2624	* Convert the pLogger into a VM handle and 'call' back to Ring-3.
2625	* (This is a bit paranoid code.)
2626	*/
2627	PVMMR0LOGGER pR0Logger = (PVMMR0LOGGER)((uintptr_t)pLogger - RT_UOFFSETOF(VMMR0LOGGER, Logger));
2628	if ( !VALID_PTR(pR0Logger)
2629	\|\| !VALID_PTR(pR0Logger + 1)
2630	\|\| pLogger->u32Magic != RTLOGGER_MAGIC)
2631	{
2632	# ifdef DEBUG
2633	SUPR0Printf("vmmR0LoggerFlush: pLogger=%p!\n", pLogger);
2634	# endif
2635	return;
2636	}
2637	if (pR0Logger->fFlushingDisabled)
2638	return; /* quietly */
2639
2640	PVM pVM = pR0Logger->pVM;
2641	if ( !VALID_PTR(pVM)
2642	\|\| pVM->pVMR0 != pVM)
2643	{
2644	# ifdef DEBUG
2645	SUPR0Printf("vmmR0LoggerFlush: pVM=%p! pVMR0=%p! pLogger=%p\n", pVM, pVM->pVMR0, pLogger);
2646	# endif
2647	return;
2648	}
2649
2650	PVMCPU pVCpu = VMMGetCpu(pVM);
2651	if (pVCpu)
2652	{
2653	/*
2654	* Check that the jump buffer is armed.
2655	*/
2656	# ifdef RT_ARCH_X86
2657	if ( !pVCpu->vmm.s.CallRing3JmpBufR0.eip
2658	\|\| pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call)
2659	# else
2660	if ( !pVCpu->vmm.s.CallRing3JmpBufR0.rip
2661	\|\| pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call)
2662	# endif
2663	{
2664	# ifdef DEBUG
2665	SUPR0Printf("vmmR0LoggerFlush: Jump buffer isn't armed!\n");
2666	# endif
2667	return;
2668	}
2669	VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_VMM_LOGGER_FLUSH, 0);
2670	}
2671	# ifdef DEBUG
2672	else
2673	SUPR0Printf("vmmR0LoggerFlush: invalid VCPU context!\n");
2674	# endif
2675	#else
2676	NOREF(pLogger);
2677	#endif /* LOG_ENABLED */
2678	}
2679
2680	#ifdef LOG_ENABLED
2681
2682	/**
2683	* Disables flushing of the ring-0 debug log.
2684	*
2685	* @param pVCpu The cross context virtual CPU structure.
2686	*/
2687	VMMR0_INT_DECL(void) VMMR0LogFlushDisable(PVMCPU pVCpu)
2688	{
2689	if (pVCpu->vmm.s.pR0LoggerR0)
2690	pVCpu->vmm.s.pR0LoggerR0->fFlushingDisabled = true;
2691	if (pVCpu->vmm.s.pR0RelLoggerR0)
2692	pVCpu->vmm.s.pR0RelLoggerR0->fFlushingDisabled = true;
2693	}
2694
2695
2696	/**
2697	* Enables flushing of the ring-0 debug log.
2698	*
2699	* @param pVCpu The cross context virtual CPU structure.
2700	*/
2701	VMMR0_INT_DECL(void) VMMR0LogFlushEnable(PVMCPU pVCpu)
2702	{
2703	if (pVCpu->vmm.s.pR0LoggerR0)
2704	pVCpu->vmm.s.pR0LoggerR0->fFlushingDisabled = false;
2705	if (pVCpu->vmm.s.pR0RelLoggerR0)
2706	pVCpu->vmm.s.pR0RelLoggerR0->fFlushingDisabled = false;
2707	}
2708
2709
2710	/**
2711	* Checks if log flushing is disabled or not.
2712	*
2713	* @param pVCpu The cross context virtual CPU structure.
2714	*/
2715	VMMR0_INT_DECL(bool) VMMR0IsLogFlushDisabled(PVMCPU pVCpu)
2716	{
2717	if (pVCpu->vmm.s.pR0LoggerR0)
2718	return pVCpu->vmm.s.pR0LoggerR0->fFlushingDisabled;
2719	if (pVCpu->vmm.s.pR0RelLoggerR0)
2720	return pVCpu->vmm.s.pR0RelLoggerR0->fFlushingDisabled;
2721	return true;
2722	}
2723
2724	#endif /* LOG_ENABLED */
2725
2726	/**
2727	* Override RTLogRelGetDefaultInstanceEx so we can do LogRel to VBox.log from EMTs in ring-0.
2728	*/
2729	DECLEXPORT(PRTLOGGER) RTLogRelGetDefaultInstanceEx(uint32_t fFlagsAndGroup)
2730	{
2731	PGVMCPU pGVCpu = GVMMR0GetGVCpuByEMT(NIL_RTNATIVETHREAD);
2732	if (pGVCpu)
2733	{
2734	#ifdef VBOX_BUGREF_9217
2735	PVMCPU pVCpu = pGVCpu;
2736	#else
2737	PVMCPU pVCpu = pGVCpu->pVCpu;
2738	#endif
2739	if (RT_VALID_PTR(pVCpu))
2740	{
2741	PVMMR0LOGGER pVmmLogger = pVCpu->vmm.s.pR0RelLoggerR0;
2742	if (RT_VALID_PTR(pVmmLogger))
2743	{
2744	if ( pVmmLogger->fCreated
2745	#ifdef VBOX_BUGREF_9217
2746	&& pVmmLogger->pVM == pGVCpu->pGVM
2747	#else
2748	&& pVmmLogger->pVM == pGVCpu->pVM
2749	#endif
2750	)
2751	{
2752	if (pVmmLogger->Logger.fFlags & RTLOGFLAGS_DISABLED)
2753	return NULL;
2754	uint16_t const fFlags = RT_LO_U16(fFlagsAndGroup);
2755	uint16_t const iGroup = RT_HI_U16(fFlagsAndGroup);
2756	if ( iGroup != UINT16_MAX
2757	&& ( ( pVmmLogger->Logger.afGroups[iGroup < pVmmLogger->Logger.cGroups ? iGroup : 0]
2758	& (fFlags \| (uint32_t)RTLOGGRPFLAGS_ENABLED))
2759	!= (fFlags \| (uint32_t)RTLOGGRPFLAGS_ENABLED)))
2760	return NULL;
2761	return &pVmmLogger->Logger;
2762	}
2763	}
2764	}
2765	}
2766	return SUPR0GetDefaultLogRelInstanceEx(fFlagsAndGroup);
2767	}
2768
2769
2770	/**
2771	* Jump back to ring-3 if we're the EMT and the longjmp is armed.
2772	*
2773	* @returns true if the breakpoint should be hit, false if it should be ignored.
2774	*/
2775	DECLEXPORT(bool) RTCALL RTAssertShouldPanic(void)
2776	{
2777	#if 0
2778	return true;
2779	#else
2780	PVM pVM = GVMMR0GetVMByEMT(NIL_RTNATIVETHREAD);
2781	if (pVM)
2782	{
2783	PVMCPU pVCpu = VMMGetCpu(pVM);
2784
2785	if (pVCpu)
2786	{
2787	#ifdef RT_ARCH_X86
2788	if ( pVCpu->vmm.s.CallRing3JmpBufR0.eip
2789	&& !pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call)
2790	#else
2791	if ( pVCpu->vmm.s.CallRing3JmpBufR0.rip
2792	&& !pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call)
2793	#endif
2794	{
2795	int rc = VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_VM_R0_ASSERTION, 0);
2796	return RT_FAILURE_NP(rc);
2797	}
2798	}
2799	}
2800	#ifdef RT_OS_LINUX
2801	return true;
2802	#else
2803	return false;
2804	#endif
2805	#endif
2806	}
2807
2808
2809	/**
2810	* Override this so we can push it up to ring-3.
2811	*
2812	* @param pszExpr Expression. Can be NULL.
2813	* @param uLine Location line number.
2814	* @param pszFile Location file name.
2815	* @param pszFunction Location function name.
2816	*/
2817	DECLEXPORT(void) RTCALL RTAssertMsg1Weak(const char pszExpr, unsigned uLine, const char pszFile, const char *pszFunction)
2818	{
2819	/*
2820	* To the log.
2821	*/
2822	LogAlways(("\n!!R0-Assertion Failed!!\n"
2823	"Expression: %s\n"
2824	"Location : %s(%d) %s\n",
2825	pszExpr, pszFile, uLine, pszFunction));
2826
2827	/*
2828	* To the global VMM buffer.
2829	*/
2830	PVM pVM = GVMMR0GetVMByEMT(NIL_RTNATIVETHREAD);
2831	if (pVM)
2832	RTStrPrintf(pVM->vmm.s.szRing0AssertMsg1, sizeof(pVM->vmm.s.szRing0AssertMsg1),
2833	"\n!!R0-Assertion Failed!!\n"
2834	"Expression: %.*s\n"
2835	"Location : %s(%d) %s\n",
2836	sizeof(pVM->vmm.s.szRing0AssertMsg1) / 4 * 3, pszExpr,
2837	pszFile, uLine, pszFunction);
2838
2839	/*
2840	* Continue the normal way.
2841	*/
2842	RTAssertMsg1(pszExpr, uLine, pszFile, pszFunction);
2843	}
2844
2845
2846	/**
2847	* Callback for RTLogFormatV which writes to the ring-3 log port.
2848	* See PFNLOGOUTPUT() for details.
2849	*/
2850	static DECLCALLBACK(size_t) rtLogOutput(void pv, const char pachChars, size_t cbChars)
2851	{
2852	for (size_t i = 0; i < cbChars; i++)
2853	{
2854	LogAlways(("%c", pachChars[i])); NOREF(pachChars);
2855	}
2856
2857	NOREF(pv);
2858	return cbChars;
2859	}
2860
2861
2862	/**
2863	* Override this so we can push it up to ring-3.
2864	*
2865	* @param pszFormat The format string.
2866	* @param va Arguments.
2867	*/
2868	DECLEXPORT(void) RTCALL RTAssertMsg2WeakV(const char *pszFormat, va_list va)
2869	{
2870	va_list vaCopy;
2871
2872	/*
2873	* Push the message to the loggers.
2874	*/
2875	PRTLOGGER pLog = RTLogGetDefaultInstance(); /* Don't initialize it here... */
2876	if (pLog)
2877	{
2878	va_copy(vaCopy, va);
2879	RTLogFormatV(rtLogOutput, pLog, pszFormat, vaCopy);
2880	va_end(vaCopy);
2881	}
2882	pLog = RTLogRelGetDefaultInstance();
2883	if (pLog)
2884	{
2885	va_copy(vaCopy, va);
2886	RTLogFormatV(rtLogOutput, pLog, pszFormat, vaCopy);
2887	va_end(vaCopy);
2888	}
2889
2890	/*
2891	* Push it to the global VMM buffer.
2892	*/
2893	PVM pVM = GVMMR0GetVMByEMT(NIL_RTNATIVETHREAD);
2894	if (pVM)
2895	{
2896	va_copy(vaCopy, va);
2897	RTStrPrintfV(pVM->vmm.s.szRing0AssertMsg2, sizeof(pVM->vmm.s.szRing0AssertMsg2), pszFormat, vaCopy);
2898	va_end(vaCopy);
2899	}
2900
2901	/*
2902	* Continue the normal way.
2903	*/
2904	RTAssertMsg2V(pszFormat, va);
2905	}
2906

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/VMMR0.cpp@ 78647

Download in other formats: