VMMR0.cpp@ 61605

Last change on this file since 61605 was 61348, checked in by vboxsync, 9 years ago
CPUM,VMM: Touch the FPU state before doing HM on all platforms which allows us do (VMM_R0_TOUCH_FPU, see Makefile.kmk). No special treatment of win.amd64 (could save a CR0 read, maybe). Cleaned up the fix from this morning.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 81.8 KB

Line
1	/* $Id: VMMR0.cpp 61348 2016-05-31 17:59:34Z vboxsync $ */
2	/** @file
3	* VMM - Host Context Ring 0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2015 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_VMM
23	#include <VBox/vmm/vmm.h>
24	#include <VBox/sup.h>
25	#include <VBox/vmm/trpm.h>
26	#include <VBox/vmm/cpum.h>
27	#include <VBox/vmm/pdmapi.h>
28	#include <VBox/vmm/pgm.h>
29	#include <VBox/vmm/stam.h>
30	#include <VBox/vmm/tm.h>
31	#include "VMMInternal.h"
32	#include <VBox/vmm/vm.h>
33	#ifdef VBOX_WITH_PCI_PASSTHROUGH
34	# include <VBox/vmm/pdmpci.h>
35	#endif
36	#ifdef VBOX_WITH_NEW_APIC
37	# include <VBox/vmm/apic.h>
38	#endif
39
40	#include <VBox/vmm/gvmm.h>
41	#include <VBox/vmm/gmm.h>
42	#include <VBox/vmm/gim.h>
43	#include <VBox/intnet.h>
44	#include <VBox/vmm/hm.h>
45	#include <VBox/param.h>
46	#include <VBox/err.h>
47	#include <VBox/version.h>
48	#include <VBox/log.h>
49
50	#include <iprt/asm-amd64-x86.h>
51	#include <iprt/assert.h>
52	#include <iprt/crc.h>
53	#include <iprt/mp.h>
54	#include <iprt/once.h>
55	#include <iprt/stdarg.h>
56	#include <iprt/string.h>
57	#include <iprt/thread.h>
58	#include <iprt/timer.h>
59
60	#include "dtrace/VBoxVMM.h"
61
62
63	#if defined(_MSC_VER) && defined(RT_ARCH_AMD64) /** @todo check this with with VC7! */
64	# pragma intrinsic(_AddressOfReturnAddress)
65	#endif
66
67	#if defined(RT_OS_DARWIN) && ARCH_BITS == 32
68	# error "32-bit darwin is no longer supported. Go back to 4.3 or earlier!"
69	#endif
70
71
72
73	/*********************************************************************************************************************************
74	* Defined Constants And Macros *
75	*********************************************************************************************************************************/
76	/** @def VMM_CHECK_SMAP_SETUP
77	* SMAP check setup. */
78	/** @def VMM_CHECK_SMAP_CHECK
79	* Checks that the AC flag is set if SMAP is enabled. If AC is not set,
80	* it will be logged and @a a_BadExpr is executed. */
81	/** @def VMM_CHECK_SMAP_CHECK2
82	* Checks that the AC flag is set if SMAP is enabled. If AC is not set, it will
83	* be logged, written to the VMs assertion text buffer, and @a a_BadExpr is
84	* executed. */
85	#if defined(VBOX_STRICT) \|\| 1
86	# define VMM_CHECK_SMAP_SETUP() uint32_t const fKernelFeatures = SUPR0GetKernelFeatures()
87	# define VMM_CHECK_SMAP_CHECK(a_BadExpr) \
88	do { \
89	if (fKernelFeatures & SUPKERNELFEATURES_SMAP) \
90	{ \
91	RTCCUINTREG fEflCheck = ASMGetFlags(); \
92	if (RT_LIKELY(fEflCheck & X86_EFL_AC)) \
93	{ /* likely */ } \
94	else \
95	{ \
96	SUPR0Printf("%s, line %d: EFLAGS.AC is clear! (%#x)\n", __FUNCTION__, __LINE__, (uint32_t)fEflCheck); \
97	a_BadExpr; \
98	} \
99	} \
100	} while (0)
101	# define VMM_CHECK_SMAP_CHECK2(a_pVM, a_BadExpr) \
102	do { \
103	if (fKernelFeatures & SUPKERNELFEATURES_SMAP) \
104	{ \
105	RTCCUINTREG fEflCheck = ASMGetFlags(); \
106	if (RT_LIKELY(fEflCheck & X86_EFL_AC)) \
107	{ /* likely */ } \
108	else \
109	{ \
110	SUPR0BadContext((a_pVM) ? (a_pVM)->pSession : NULL, __FILE__, __LINE__, "EFLAGS.AC is zero!"); \
111	RTStrPrintf(pVM->vmm.s.szRing0AssertMsg1, sizeof(pVM->vmm.s.szRing0AssertMsg1), \
112	"%s, line %d: EFLAGS.AC is clear! (%#x)\n", __FUNCTION__, __LINE__, (uint32_t)fEflCheck); \
113	a_BadExpr; \
114	} \
115	} \
116	} while (0)
117	#else
118	# define VMM_CHECK_SMAP_SETUP() uint32_t const fKernelFeatures = 0
119	# define VMM_CHECK_SMAP_CHECK(a_BadExpr) NOREF(fKernelFeatures)
120	# define VMM_CHECK_SMAP_CHECK2(a_pVM, a_BadExpr) NOREF(fKernelFeatures)
121	#endif
122
123
124	/*********************************************************************************************************************************
125	* Internal Functions *
126	*********************************************************************************************************************************/
127	RT_C_DECLS_BEGIN
128	#if defined(RT_ARCH_X86) && (defined(RT_OS_SOLARIS) \|\| defined(RT_OS_FREEBSD))
129	extern uint64_t __udivdi3(uint64_t, uint64_t);
130	extern uint64_t __umoddi3(uint64_t, uint64_t);
131	#endif
132	RT_C_DECLS_END
133
134
135	/*********************************************************************************************************************************
136	* Global Variables *
137	*********************************************************************************************************************************/
138	/** Drag in necessary library bits.
139	* The runtime lives here (in VMMR0.r0) and VBoxDDR0.r0 links against us. /
140	PFNRT g_VMMR0Deps[] =
141	{
142	(PFNRT)RTCrc32,
143	(PFNRT)RTOnce,
144	#if defined(RT_ARCH_X86) && (defined(RT_OS_SOLARIS) \|\| defined(RT_OS_FREEBSD))
145	(PFNRT)__udivdi3,
146	(PFNRT)__umoddi3,
147	#endif
148	NULL
149	};
150
151	#ifdef RT_OS_SOLARIS
152	/* Dependency information for the native solaris loader. */
153	extern "C" { char _depends_on[] = "vboxdrv"; }
154	#endif
155
156
157
158	/**
159	* Initialize the module.
160	* This is called when we're first loaded.
161	*
162	* @returns 0 on success.
163	* @returns VBox status on failure.
164	* @param hMod Image handle for use in APIs.
165	*/
166	DECLEXPORT(int) ModuleInit(void *hMod)
167	{
168	VMM_CHECK_SMAP_SETUP();
169	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
170
171	#ifdef VBOX_WITH_DTRACE_R0
172	/*
173	* The first thing to do is register the static tracepoints.
174	* (Deregistration is automatic.)
175	*/
176	int rc2 = SUPR0TracerRegisterModule(hMod, &g_VTGObjHeader);
177	if (RT_FAILURE(rc2))
178	return rc2;
179	#endif
180	LogFlow(("ModuleInit:\n"));
181
182	#ifdef VBOX_WITH_64ON32_CMOS_DEBUG
183	/*
184	* Display the CMOS debug code.
185	*/
186	ASMOutU8(0x72, 0x03);
187	uint8_t bDebugCode = ASMInU8(0x73);
188	LogRel(("CMOS Debug Code: %#x (%d)\n", bDebugCode, bDebugCode));
189	RTLogComPrintf("CMOS Debug Code: %#x (%d)\n", bDebugCode, bDebugCode);
190	#endif
191
192	/*
193	* Initialize the VMM, GVMM, GMM, HM, PGM (Darwin) and INTNET.
194	*/
195	int rc = vmmInitFormatTypes();
196	if (RT_SUCCESS(rc))
197	{
198	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
199	rc = GVMMR0Init();
200	if (RT_SUCCESS(rc))
201	{
202	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
203	rc = GMMR0Init();
204	if (RT_SUCCESS(rc))
205	{
206	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
207	rc = HMR0Init();
208	if (RT_SUCCESS(rc))
209	{
210	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
211	rc = PGMRegisterStringFormatTypes();
212	if (RT_SUCCESS(rc))
213	{
214	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
215	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
216	rc = PGMR0DynMapInit();
217	#endif
218	if (RT_SUCCESS(rc))
219	{
220	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
221	rc = IntNetR0Init();
222	if (RT_SUCCESS(rc))
223	{
224	#ifdef VBOX_WITH_PCI_PASSTHROUGH
225	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
226	rc = PciRawR0Init();
227	#endif
228	if (RT_SUCCESS(rc))
229	{
230	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
231	rc = CPUMR0ModuleInit();
232	if (RT_SUCCESS(rc))
233	{
234	#ifdef VBOX_WITH_TRIPLE_FAULT_HACK
235	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
236	rc = vmmR0TripleFaultHackInit();
237	if (RT_SUCCESS(rc))
238	#endif
239	{
240	VMM_CHECK_SMAP_CHECK(rc = VERR_VMM_SMAP_BUT_AC_CLEAR);
241	if (RT_SUCCESS(rc))
242	{
243	LogFlow(("ModuleInit: returns success.\n"));
244	return VINF_SUCCESS;
245	}
246	}
247
248	/*
249	* Bail out.
250	*/
251	#ifdef VBOX_WITH_TRIPLE_FAULT_HACK
252	vmmR0TripleFaultHackTerm();
253	#endif
254	}
255	else
256	LogRel(("ModuleInit: CPUMR0ModuleInit -> %Rrc\n", rc));
257	#ifdef VBOX_WITH_PCI_PASSTHROUGH
258	PciRawR0Term();
259	#endif
260	}
261	else
262	LogRel(("ModuleInit: PciRawR0Init -> %Rrc\n", rc));
263	IntNetR0Term();
264	}
265	else
266	LogRel(("ModuleInit: IntNetR0Init -> %Rrc\n", rc));
267	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
268	PGMR0DynMapTerm();
269	#endif
270	}
271	else
272	LogRel(("ModuleInit: PGMR0DynMapInit -> %Rrc\n", rc));
273	PGMDeregisterStringFormatTypes();
274	}
275	else
276	LogRel(("ModuleInit: PGMRegisterStringFormatTypes -> %Rrc\n", rc));
277	HMR0Term();
278	}
279	else
280	LogRel(("ModuleInit: HMR0Init -> %Rrc\n", rc));
281	GMMR0Term();
282	}
283	else
284	LogRel(("ModuleInit: GMMR0Init -> %Rrc\n", rc));
285	GVMMR0Term();
286	}
287	else
288	LogRel(("ModuleInit: GVMMR0Init -> %Rrc\n", rc));
289	vmmTermFormatTypes();
290	}
291	else
292	LogRel(("ModuleInit: vmmInitFormatTypes -> %Rrc\n", rc));
293
294	LogFlow(("ModuleInit: failed %Rrc\n", rc));
295	return rc;
296	}
297
298
299	/**
300	* Terminate the module.
301	* This is called when we're finally unloaded.
302	*
303	* @param hMod Image handle for use in APIs.
304	*/
305	DECLEXPORT(void) ModuleTerm(void *hMod)
306	{
307	NOREF(hMod);
308	LogFlow(("ModuleTerm:\n"));
309
310	/*
311	* Terminate the CPUM module (Local APIC cleanup).
312	*/
313	CPUMR0ModuleTerm();
314
315	/*
316	* Terminate the internal network service.
317	*/
318	IntNetR0Term();
319
320	/*
321	* PGM (Darwin), HM and PciRaw global cleanup.
322	*/
323	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
324	PGMR0DynMapTerm();
325	#endif
326	#ifdef VBOX_WITH_PCI_PASSTHROUGH
327	PciRawR0Term();
328	#endif
329	PGMDeregisterStringFormatTypes();
330	HMR0Term();
331	#ifdef VBOX_WITH_TRIPLE_FAULT_HACK
332	vmmR0TripleFaultHackTerm();
333	#endif
334
335	/*
336	* Destroy the GMM and GVMM instances.
337	*/
338	GMMR0Term();
339	GVMMR0Term();
340
341	vmmTermFormatTypes();
342
343	LogFlow(("ModuleTerm: returns\n"));
344	}
345
346
347	/**
348	* Initiates the R0 driver for a particular VM instance.
349	*
350	* @returns VBox status code.
351	*
352	* @param pVM The cross context VM structure.
353	* @param uSvnRev The SVN revision of the ring-3 part.
354	* @param uBuildType Build type indicator.
355	* @thread EMT.
356	*/
357	static int vmmR0InitVM(PVM pVM, uint32_t uSvnRev, uint32_t uBuildType)
358	{
359	VMM_CHECK_SMAP_SETUP();
360	VMM_CHECK_SMAP_CHECK(return VERR_VMM_SMAP_BUT_AC_CLEAR);
361
362	/*
363	* Match the SVN revisions and build type.
364	*/
365	if (uSvnRev != VMMGetSvnRev())
366	{
367	LogRel(("VMMR0InitVM: Revision mismatch, r3=%d r0=%d\n", uSvnRev, VMMGetSvnRev()));
368	SUPR0Printf("VMMR0InitVM: Revision mismatch, r3=%d r0=%d\n", uSvnRev, VMMGetSvnRev());
369	return VERR_VMM_R0_VERSION_MISMATCH;
370	}
371	if (uBuildType != vmmGetBuildType())
372	{
373	LogRel(("VMMR0InitVM: Build type mismatch, r3=%#x r0=%#x\n", uBuildType, vmmGetBuildType()));
374	SUPR0Printf("VMMR0InitVM: Build type mismatch, r3=%#x r0=%#x\n", uBuildType, vmmGetBuildType());
375	return VERR_VMM_R0_VERSION_MISMATCH;
376	}
377	if ( !VALID_PTR(pVM)
378	\|\| pVM->pVMR0 != pVM)
379	return VERR_INVALID_PARAMETER;
380
381
382	#ifdef LOG_ENABLED
383	/*
384	* Register the EMT R0 logger instance for VCPU 0.
385	*/
386	PVMCPU pVCpu = &pVM->aCpus[0];
387
388	PVMMR0LOGGER pR0Logger = pVCpu->vmm.s.pR0LoggerR0;
389	if (pR0Logger)
390	{
391	# if 0 /* testing of the logger. */
392	LogCom(("vmmR0InitVM: before %p\n", RTLogDefaultInstance()));
393	LogCom(("vmmR0InitVM: pfnFlush=%p actual=%p\n", pR0Logger->Logger.pfnFlush, vmmR0LoggerFlush));
394	LogCom(("vmmR0InitVM: pfnLogger=%p actual=%p\n", pR0Logger->Logger.pfnLogger, vmmR0LoggerWrapper));
395	LogCom(("vmmR0InitVM: offScratch=%d fFlags=%#x fDestFlags=%#x\n", pR0Logger->Logger.offScratch, pR0Logger->Logger.fFlags, pR0Logger->Logger.fDestFlags));
396
397	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pVM->pSession);
398	LogCom(("vmmR0InitVM: after %p reg\n", RTLogDefaultInstance()));
399	RTLogSetDefaultInstanceThread(NULL, pVM->pSession);
400	LogCom(("vmmR0InitVM: after %p dereg\n", RTLogDefaultInstance()));
401
402	pR0Logger->Logger.pfnLogger("hello ring-0 logger\n");
403	LogCom(("vmmR0InitVM: returned successfully from direct logger call.\n"));
404	pR0Logger->Logger.pfnFlush(&pR0Logger->Logger);
405	LogCom(("vmmR0InitVM: returned successfully from direct flush call.\n"));
406
407	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pVM->pSession);
408	LogCom(("vmmR0InitVM: after %p reg2\n", RTLogDefaultInstance()));
409	pR0Logger->Logger.pfnLogger("hello ring-0 logger\n");
410	LogCom(("vmmR0InitVM: returned successfully from direct logger call (2). offScratch=%d\n", pR0Logger->Logger.offScratch));
411	RTLogSetDefaultInstanceThread(NULL, pVM->pSession);
412	LogCom(("vmmR0InitVM: after %p dereg2\n", RTLogDefaultInstance()));
413
414	RTLogLoggerEx(&pR0Logger->Logger, 0, ~0U, "hello ring-0 logger (RTLogLoggerEx)\n");
415	LogCom(("vmmR0InitVM: RTLogLoggerEx returned fine offScratch=%d\n", pR0Logger->Logger.offScratch));
416
417	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pVM->pSession);
418	RTLogPrintf("hello ring-0 logger (RTLogPrintf)\n");
419	LogCom(("vmmR0InitVM: RTLogPrintf returned fine offScratch=%d\n", pR0Logger->Logger.offScratch));
420	# endif
421	Log(("Switching to per-thread logging instance %p (key=%p)\n", &pR0Logger->Logger, pVM->pSession));
422	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pVM->pSession);
423	pR0Logger->fRegistered = true;
424	}
425	#endif /* LOG_ENABLED */
426
427	/*
428	* Check if the host supports high resolution timers or not.
429	*/
430	if ( pVM->vmm.s.fUsePeriodicPreemptionTimers
431	&& !RTTimerCanDoHighResolution())
432	pVM->vmm.s.fUsePeriodicPreemptionTimers = false;
433
434	/*
435	* Initialize the per VM data for GVMM and GMM.
436	*/
437	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
438	int rc = GVMMR0InitVM(pVM);
439	// if (RT_SUCCESS(rc))
440	// rc = GMMR0InitPerVMData(pVM);
441	if (RT_SUCCESS(rc))
442	{
443	/*
444	* Init HM, CPUM and PGM (Darwin only).
445	*/
446	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
447	rc = HMR0InitVM(pVM);
448	if (RT_SUCCESS(rc))
449	VMM_CHECK_SMAP_CHECK2(pVM, rc = VERR_VMM_RING0_ASSERTION); /* CPUR0InitVM will otherwise panic the host */
450	if (RT_SUCCESS(rc))
451	{
452	rc = CPUMR0InitVM(pVM);
453	if (RT_SUCCESS(rc))
454	{
455	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
456	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
457	rc = PGMR0DynMapInitVM(pVM);
458	#endif
459	if (RT_SUCCESS(rc))
460	{
461	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
462	#ifdef VBOX_WITH_PCI_PASSTHROUGH
463	rc = PciRawR0InitVM(pVM);
464	#endif
465	if (RT_SUCCESS(rc))
466	{
467	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
468	rc = GIMR0InitVM(pVM);
469	if (RT_SUCCESS(rc))
470	{
471	VMM_CHECK_SMAP_CHECK2(pVM, rc = VERR_VMM_RING0_ASSERTION);
472	if (RT_SUCCESS(rc))
473	{
474	GVMMR0DoneInitVM(pVM);
475	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
476	return rc;
477	}
478
479	/* bail out*/
480	GIMR0TermVM(pVM);
481	}
482	#ifdef VBOX_WITH_PCI_PASSTHROUGH
483	PciRawR0TermVM(pVM);
484	#endif
485	}
486	}
487	}
488	HMR0TermVM(pVM);
489	}
490	}
491
492	RTLogSetDefaultInstanceThread(NULL, (uintptr_t)pVM->pSession);
493	return rc;
494	}
495
496
497	/**
498	* Terminates the R0 bits for a particular VM instance.
499	*
500	* This is normally called by ring-3 as part of the VM termination process, but
501	* may alternatively be called during the support driver session cleanup when
502	* the VM object is destroyed (see GVMM).
503	*
504	* @returns VBox status code.
505	*
506	* @param pVM The cross context VM structure.
507	* @param pGVM Pointer to the global VM structure. Optional.
508	* @thread EMT or session clean up thread.
509	*/
510	VMMR0_INT_DECL(int) VMMR0TermVM(PVM pVM, PGVM pGVM)
511	{
512	#ifdef VBOX_WITH_PCI_PASSTHROUGH
513	PciRawR0TermVM(pVM);
514	#endif
515
516	/*
517	* Tell GVMM what we're up to and check that we only do this once.
518	*/
519	if (GVMMR0DoingTermVM(pVM, pGVM))
520	{
521	GIMR0TermVM(pVM);
522
523	/** @todo I wish to call PGMR0PhysFlushHandyPages(pVM, &pVM->aCpus[idCpu])
524	* here to make sure we don't leak any shared pages if we crash... */
525	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
526	PGMR0DynMapTermVM(pVM);
527	#endif
528	HMR0TermVM(pVM);
529	}
530
531	/*
532	* Deregister the logger.
533	*/
534	RTLogSetDefaultInstanceThread(NULL, (uintptr_t)pVM->pSession);
535	return VINF_SUCCESS;
536	}
537
538
539	/**
540	* VMM ring-0 thread-context callback.
541	*
542	* This does common HM state updating and calls the HM-specific thread-context
543	* callback.
544	*
545	* @param enmEvent The thread-context event.
546	* @param pvUser Opaque pointer to the VMCPU.
547	*
548	* @thread EMT(pvUser)
549	*/
550	static DECLCALLBACK(void) vmmR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, void *pvUser)
551	{
552	PVMCPU pVCpu = (PVMCPU)pvUser;
553
554	switch (enmEvent)
555	{
556	case RTTHREADCTXEVENT_IN:
557	{
558	/*
559	* Linux may call us with preemption enabled (really!) but technically we
560	* cannot get preempted here, otherwise we end up in an infinite recursion
561	* scenario (i.e. preempted in resume hook -> preempt hook -> resume hook...
562	* ad infinitum). Let's just disable preemption for now...
563	*/
564	/** @todo r=bird: I don't believe the above. The linux code is clearly enabling
565	* preemption after doing the callout (one or two functions up the
566	* call chain). */
567	/** @todo r=ramshankar: See @bugref{5313#c30}. */
568	RTTHREADPREEMPTSTATE ParanoidPreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
569	RTThreadPreemptDisable(&ParanoidPreemptState);
570
571	/* We need to update the VCPU <-> host CPU mapping. */
572	RTCPUID idHostCpu;
573	uint32_t iHostCpuSet = RTMpCurSetIndexAndId(&idHostCpu);
574	pVCpu->iHostCpuSet = iHostCpuSet;
575	ASMAtomicWriteU32(&pVCpu->idHostCpu, idHostCpu);
576
577	/* In the very unlikely event that the GIP delta for the CPU we're
578	rescheduled needs calculating, try force a return to ring-3.
579	We unfortunately cannot do the measurements right here. */
580	if (RT_UNLIKELY(SUPIsTscDeltaAvailableForCpuSetIndex(iHostCpuSet)))
581	VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
582
583	/* Invoke the HM-specific thread-context callback. */
584	HMR0ThreadCtxCallback(enmEvent, pvUser);
585
586	/* Restore preemption. */
587	RTThreadPreemptRestore(&ParanoidPreemptState);
588	break;
589	}
590
591	case RTTHREADCTXEVENT_OUT:
592	{
593	/* Invoke the HM-specific thread-context callback. */
594	HMR0ThreadCtxCallback(enmEvent, pvUser);
595
596	/*
597	* Sigh. See VMMGetCpu() used by VMCPU_ASSERT_EMT(). We cannot let several VCPUs
598	* have the same host CPU associated with it.
599	*/
600	pVCpu->iHostCpuSet = UINT32_MAX;
601	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
602	break;
603	}
604
605	default:
606	/* Invoke the HM-specific thread-context callback. */
607	HMR0ThreadCtxCallback(enmEvent, pvUser);
608	break;
609	}
610	}
611
612
613	/**
614	* Creates thread switching hook for the current EMT thread.
615	*
616	* This is called by GVMMR0CreateVM and GVMMR0RegisterVCpu. If the host
617	* platform does not implement switcher hooks, no hooks will be create and the
618	* member set to NIL_RTTHREADCTXHOOK.
619	*
620	* @returns VBox status code.
621	* @param pVCpu The cross context virtual CPU structure.
622	* @thread EMT(pVCpu)
623	*/
624	VMMR0_INT_DECL(int) VMMR0ThreadCtxHookCreateForEmt(PVMCPU pVCpu)
625	{
626	VMCPU_ASSERT_EMT(pVCpu);
627	Assert(pVCpu->vmm.s.hCtxHook == NIL_RTTHREADCTXHOOK);
628
629	int rc = RTThreadCtxHookCreate(&pVCpu->vmm.s.hCtxHook, 0, vmmR0ThreadCtxCallback, pVCpu);
630	if (RT_SUCCESS(rc))
631	return rc;
632
633	pVCpu->vmm.s.hCtxHook = NIL_RTTHREADCTXHOOK;
634	if (rc == VERR_NOT_SUPPORTED)
635	return VINF_SUCCESS;
636
637	LogRelMax(32, ("RTThreadCtxHookCreate failed! rc=%Rrc pVCpu=%p idCpu=%RU32\n", rc, pVCpu, pVCpu->idCpu));
638	return VINF_SUCCESS; /* Just ignore it, we can live without context hooks. */
639	}
640
641
642	/**
643	* Destroys the thread switching hook for the specified VCPU.
644	*
645	* @param pVCpu The cross context virtual CPU structure.
646	* @remarks Can be called from any thread.
647	*/
648	VMMR0_INT_DECL(void) VMMR0ThreadCtxHookDestroyForEmt(PVMCPU pVCpu)
649	{
650	int rc = RTThreadCtxHookDestroy(pVCpu->vmm.s.hCtxHook);
651	AssertRC(rc);
652	}
653
654
655	/**
656	* Disables the thread switching hook for this VCPU (if we got one).
657	*
658	* @param pVCpu The cross context virtual CPU structure.
659	* @thread EMT(pVCpu)
660	*
661	* @remarks This also clears VMCPU::idHostCpu, so the mapping is invalid after
662	* this call. This means you have to be careful with what you do!
663	*/
664	VMMR0_INT_DECL(void) VMMR0ThreadCtxHookDisable(PVMCPU pVCpu)
665	{
666	/*
667	* Clear the VCPU <-> host CPU mapping as we've left HM context.
668	* @bugref{7726#c19} explains the need for this trick:
669	*
670	* hmR0VmxCallRing3Callback/hmR0SvmCallRing3Callback &
671	* hmR0VmxLeaveSession/hmR0SvmLeaveSession disables context hooks during
672	* longjmp & normal return to ring-3, which opens a window where we may be
673	* rescheduled without changing VMCPUID::idHostCpu and cause confusion if
674	* the CPU starts executing a different EMT. Both functions first disables
675	* preemption and then calls HMR0LeaveCpu which invalids idHostCpu, leaving
676	* an opening for getting preempted.
677	*/
678	/** @todo Make HM not need this API! Then we could leave the hooks enabled
679	* all the time. */
680	/** @todo move this into the context hook disabling if(). */
681	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
682
683	/*
684	* Disable the context hook, if we got one.
685	*/
686	if (pVCpu->vmm.s.hCtxHook != NIL_RTTHREADCTXHOOK)
687	{
688	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
689	int rc = RTThreadCtxHookDisable(pVCpu->vmm.s.hCtxHook);
690	AssertRC(rc);
691	}
692	}
693
694
695	/**
696	* Internal version of VMMR0ThreadCtxHooksAreRegistered.
697	*
698	* @returns true if registered, false otherwise.
699	* @param pVCpu The cross context virtual CPU structure.
700	*/
701	DECLINLINE(bool) vmmR0ThreadCtxHookIsEnabled(PVMCPU pVCpu)
702	{
703	return RTThreadCtxHookIsEnabled(pVCpu->vmm.s.hCtxHook);
704	}
705
706
707	/**
708	* Whether thread-context hooks are registered for this VCPU.
709	*
710	* @returns true if registered, false otherwise.
711	* @param pVCpu The cross context virtual CPU structure.
712	*/
713	VMMR0_INT_DECL(bool) VMMR0ThreadCtxHookIsEnabled(PVMCPU pVCpu)
714	{
715	return vmmR0ThreadCtxHookIsEnabled(pVCpu);
716	}
717
718
719	#ifdef VBOX_WITH_STATISTICS
720	/**
721	* Record return code statistics
722	* @param pVM The cross context VM structure.
723	* @param pVCpu The cross context virtual CPU structure.
724	* @param rc The status code.
725	*/
726	static void vmmR0RecordRC(PVM pVM, PVMCPU pVCpu, int rc)
727	{
728	/*
729	* Collect statistics.
730	*/
731	switch (rc)
732	{
733	case VINF_SUCCESS:
734	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetNormal);
735	break;
736	case VINF_EM_RAW_INTERRUPT:
737	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetInterrupt);
738	break;
739	case VINF_EM_RAW_INTERRUPT_HYPER:
740	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetInterruptHyper);
741	break;
742	case VINF_EM_RAW_GUEST_TRAP:
743	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetGuestTrap);
744	break;
745	case VINF_EM_RAW_RING_SWITCH:
746	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetRingSwitch);
747	break;
748	case VINF_EM_RAW_RING_SWITCH_INT:
749	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetRingSwitchInt);
750	break;
751	case VINF_EM_RAW_STALE_SELECTOR:
752	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetStaleSelector);
753	break;
754	case VINF_EM_RAW_IRET_TRAP:
755	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIRETTrap);
756	break;
757	case VINF_IOM_R3_IOPORT_READ:
758	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIORead);
759	break;
760	case VINF_IOM_R3_IOPORT_WRITE:
761	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIOWrite);
762	break;
763	case VINF_IOM_R3_IOPORT_COMMIT_WRITE:
764	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIOCommitWrite);
765	break;
766	case VINF_IOM_R3_MMIO_READ:
767	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIORead);
768	break;
769	case VINF_IOM_R3_MMIO_WRITE:
770	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOWrite);
771	break;
772	case VINF_IOM_R3_MMIO_COMMIT_WRITE:
773	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOCommitWrite);
774	break;
775	case VINF_IOM_R3_MMIO_READ_WRITE:
776	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOReadWrite);
777	break;
778	case VINF_PATM_HC_MMIO_PATCH_READ:
779	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOPatchRead);
780	break;
781	case VINF_PATM_HC_MMIO_PATCH_WRITE:
782	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOPatchWrite);
783	break;
784	case VINF_CPUM_R3_MSR_READ:
785	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMSRRead);
786	break;
787	case VINF_CPUM_R3_MSR_WRITE:
788	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMSRWrite);
789	break;
790	case VINF_EM_RAW_EMULATE_INSTR:
791	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetEmulate);
792	break;
793	case VINF_EM_RAW_EMULATE_IO_BLOCK:
794	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIOBlockEmulate);
795	break;
796	case VINF_PATCH_EMULATE_INSTR:
797	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchEmulate);
798	break;
799	case VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT:
800	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetLDTFault);
801	break;
802	case VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT:
803	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetGDTFault);
804	break;
805	case VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT:
806	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIDTFault);
807	break;
808	case VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT:
809	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetTSSFault);
810	break;
811	case VINF_CSAM_PENDING_ACTION:
812	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetCSAMTask);
813	break;
814	case VINF_PGM_SYNC_CR3:
815	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetSyncCR3);
816	break;
817	case VINF_PATM_PATCH_INT3:
818	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchInt3);
819	break;
820	case VINF_PATM_PATCH_TRAP_PF:
821	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchPF);
822	break;
823	case VINF_PATM_PATCH_TRAP_GP:
824	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchGP);
825	break;
826	case VINF_PATM_PENDING_IRQ_AFTER_IRET:
827	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchIretIRQ);
828	break;
829	case VINF_EM_RESCHEDULE_REM:
830	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetRescheduleREM);
831	break;
832	case VINF_EM_RAW_TO_R3:
833	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Total);
834	if (VM_FF_IS_PENDING(pVM, VM_FF_TM_VIRTUAL_SYNC))
835	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3TMVirt);
836	else if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_NEED_HANDY_PAGES))
837	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3HandyPages);
838	else if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_QUEUES))
839	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3PDMQueues);
840	else if (VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS))
841	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Rendezvous);
842	else if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
843	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3DMA);
844	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TIMER))
845	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Timer);
846	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PDM_CRITSECT))
847	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3CritSect);
848	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TO_R3))
849	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3FF);
850	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_IEM))
851	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Iem);
852	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_IOM))
853	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Iom);
854	else
855	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Unknown);
856	break;
857
858	case VINF_EM_RAW_TIMER_PENDING:
859	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetTimerPending);
860	break;
861	case VINF_EM_RAW_INTERRUPT_PENDING:
862	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetInterruptPending);
863	break;
864	case VINF_VMM_CALL_HOST:
865	switch (pVCpu->vmm.s.enmCallRing3Operation)
866	{
867	case VMMCALLRING3_PDM_CRIT_SECT_ENTER:
868	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPDMCritSectEnter);
869	break;
870	case VMMCALLRING3_PDM_LOCK:
871	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPDMLock);
872	break;
873	case VMMCALLRING3_PGM_POOL_GROW:
874	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPGMPoolGrow);
875	break;
876	case VMMCALLRING3_PGM_LOCK:
877	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPGMLock);
878	break;
879	case VMMCALLRING3_PGM_MAP_CHUNK:
880	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPGMMapChunk);
881	break;
882	case VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES:
883	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPGMAllocHandy);
884	break;
885	case VMMCALLRING3_REM_REPLAY_HANDLER_NOTIFICATIONS:
886	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallRemReplay);
887	break;
888	case VMMCALLRING3_VMM_LOGGER_FLUSH:
889	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallLogFlush);
890	break;
891	case VMMCALLRING3_VM_SET_ERROR:
892	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallVMSetError);
893	break;
894	case VMMCALLRING3_VM_SET_RUNTIME_ERROR:
895	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallVMSetRuntimeError);
896	break;
897	case VMMCALLRING3_VM_R0_ASSERTION:
898	default:
899	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetCallRing3);
900	break;
901	}
902	break;
903	case VINF_PATM_DUPLICATE_FUNCTION:
904	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPATMDuplicateFn);
905	break;
906	case VINF_PGM_CHANGE_MODE:
907	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPGMChangeMode);
908	break;
909	case VINF_PGM_POOL_FLUSH_PENDING:
910	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPGMFlushPending);
911	break;
912	case VINF_EM_PENDING_REQUEST:
913	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPendingRequest);
914	break;
915	case VINF_EM_HM_PATCH_TPR_INSTR:
916	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchTPR);
917	break;
918	default:
919	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMisc);
920	break;
921	}
922	}
923	#endif /* VBOX_WITH_STATISTICS */
924
925
926	/**
927	* The Ring 0 entry point, called by the fast-ioctl path.
928	*
929	* @param pVM The cross context VM structure.
930	* The return code is stored in pVM->vmm.s.iLastGZRc.
931	* @param idCpu The Virtual CPU ID of the calling EMT.
932	* @param enmOperation Which operation to execute.
933	* @remarks Assume called with interrupts _enabled_.
934	*/
935	VMMR0DECL(void) VMMR0EntryFast(PVM pVM, VMCPUID idCpu, VMMR0OPERATION enmOperation)
936	{
937	/*
938	* Validation.
939	*/
940	if (RT_UNLIKELY(idCpu >= pVM->cCpus))
941	return;
942	PVMCPU pVCpu = &pVM->aCpus[idCpu];
943	if (RT_UNLIKELY(pVCpu->hNativeThreadR0 != RTThreadNativeSelf()))
944	return;
945	VMM_CHECK_SMAP_SETUP();
946	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
947
948	/*
949	* Perform requested operation.
950	*/
951	switch (enmOperation)
952	{
953	/*
954	* Switch to GC and run guest raw mode code.
955	* Disable interrupts before doing the world switch.
956	*/
957	case VMMR0_DO_RAW_RUN:
958	{
959	#ifdef VBOX_WITH_RAW_MODE
960	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
961	/* Some safety precautions first. */
962	if (RT_UNLIKELY(!PGMGetHyperCR3(pVCpu)))
963	{
964	pVCpu->vmm.s.iLastGZRc = VERR_PGM_NO_CR3_SHADOW_ROOT;
965	break;
966	}
967	# endif
968
969	/*
970	* Disable preemption.
971	*/
972	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
973	RTThreadPreemptDisable(&PreemptState);
974
975	/*
976	* Get the host CPU identifiers, make sure they are valid and that
977	* we've got a TSC delta for the CPU.
978	*/
979	RTCPUID idHostCpu;
980	uint32_t iHostCpuSet = RTMpCurSetIndexAndId(&idHostCpu);
981	if (RT_LIKELY( iHostCpuSet < RTCPUSET_MAX_CPUS
982	&& SUPIsTscDeltaAvailableForCpuSetIndex(iHostCpuSet)))
983	{
984	/*
985	* Commit the CPU identifiers and update the periodict preemption timer if it's active.
986	*/
987	# ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
988	CPUMR0SetLApic(pVCpu, iHostCpuSet);
989	# endif
990	pVCpu->iHostCpuSet = iHostCpuSet;
991	ASMAtomicWriteU32(&pVCpu->idHostCpu, idHostCpu);
992
993	if (pVM->vmm.s.fUsePeriodicPreemptionTimers)
994	GVMMR0SchedUpdatePeriodicPreemptionTimer(pVM, pVCpu->idHostCpu, TMCalcHostTimerFrequency(pVM, pVCpu));
995
996	/*
997	* We might need to disable VT-x if the active switcher turns off paging.
998	*/
999	bool fVTxDisabled;
1000	int rc = HMR0EnterSwitcher(pVM, pVM->vmm.s.enmSwitcher, &fVTxDisabled);
1001	if (RT_SUCCESS(rc))
1002	{
1003	/*
1004	* Disable interrupts and run raw-mode code. The loop is for efficiently
1005	* dispatching tracepoints that fired in raw-mode context.
1006	*/
1007	RTCCUINTREG uFlags = ASMIntDisableFlags();
1008
1009	for (;;)
1010	{
1011	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
1012	TMNotifyStartOfExecution(pVCpu);
1013
1014	rc = pVM->vmm.s.pfnR0ToRawMode(pVM);
1015	pVCpu->vmm.s.iLastGZRc = rc;
1016
1017	TMNotifyEndOfExecution(pVCpu);
1018	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
1019
1020	if (rc != VINF_VMM_CALL_TRACER)
1021	break;
1022	SUPR0TracerUmodProbeFire(pVM->pSession, &pVCpu->vmm.s.TracerCtx);
1023	}
1024
1025	/*
1026	* Re-enable VT-x before we dispatch any pending host interrupts and
1027	* re-enables interrupts.
1028	*/
1029	HMR0LeaveSwitcher(pVM, fVTxDisabled);
1030
1031	if ( rc == VINF_EM_RAW_INTERRUPT
1032	\|\| rc == VINF_EM_RAW_INTERRUPT_HYPER)
1033	TRPMR0DispatchHostInterrupt(pVM);
1034
1035	ASMSetFlags(uFlags);
1036
1037	/* Fire dtrace probe and collect statistics. */
1038	VBOXVMM_R0_VMM_RETURN_TO_RING3_RC(pVCpu, CPUMQueryGuestCtxPtr(pVCpu), rc);
1039	# ifdef VBOX_WITH_STATISTICS
1040	STAM_COUNTER_INC(&pVM->vmm.s.StatRunRC);
1041	vmmR0RecordRC(pVM, pVCpu, rc);
1042	# endif
1043	}
1044	else
1045	pVCpu->vmm.s.iLastGZRc = rc;
1046
1047	/*
1048	* Invalidate the host CPU identifiers as we restore preemption.
1049	*/
1050	pVCpu->iHostCpuSet = UINT32_MAX;
1051	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
1052
1053	RTThreadPreemptRestore(&PreemptState);
1054	}
1055	/*
1056	* Invalid CPU set index or TSC delta in need of measuring.
1057	*/
1058	else
1059	{
1060	RTThreadPreemptRestore(&PreemptState);
1061	if (iHostCpuSet < RTCPUSET_MAX_CPUS)
1062	{
1063	int rc = SUPR0TscDeltaMeasureBySetIndex(pVM->pSession, iHostCpuSet, 0 /fFlags/,
1064	2 /cMsWaitRetry/, 5RT_MS_1SEC /cMsWaitThread*/,
1065	0 /default cTries/);
1066	if (RT_SUCCESS(rc) \|\| rc == VERR_CPU_OFFLINE)
1067	pVCpu->vmm.s.iLastGZRc = VINF_EM_RAW_TO_R3;
1068	else
1069	pVCpu->vmm.s.iLastGZRc = rc;
1070	}
1071	else
1072	pVCpu->vmm.s.iLastGZRc = VERR_INVALID_CPU_INDEX;
1073	}
1074
1075	#else /* !VBOX_WITH_RAW_MODE */
1076	pVCpu->vmm.s.iLastGZRc = VERR_RAW_MODE_NOT_SUPPORTED;
1077	#endif
1078	break;
1079	}
1080
1081	/*
1082	* Run guest code using the available hardware acceleration technology.
1083	*/
1084	case VMMR0_DO_HM_RUN:
1085	{
1086	/*
1087	* Disable preemption.
1088	*/
1089	Assert(!vmmR0ThreadCtxHookIsEnabled(pVCpu));
1090	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
1091	RTThreadPreemptDisable(&PreemptState);
1092
1093	/*
1094	* Get the host CPU identifiers, make sure they are valid and that
1095	* we've got a TSC delta for the CPU.
1096	*/
1097	RTCPUID idHostCpu;
1098	uint32_t iHostCpuSet = RTMpCurSetIndexAndId(&idHostCpu);
1099	if (RT_LIKELY( iHostCpuSet < RTCPUSET_MAX_CPUS
1100	&& SUPIsTscDeltaAvailableForCpuSetIndex(iHostCpuSet)))
1101	{
1102	pVCpu->iHostCpuSet = iHostCpuSet;
1103	ASMAtomicWriteU32(&pVCpu->idHostCpu, idHostCpu);
1104
1105	/*
1106	* Update the periodic preemption timer if it's active.
1107	*/
1108	if (pVM->vmm.s.fUsePeriodicPreemptionTimers)
1109	GVMMR0SchedUpdatePeriodicPreemptionTimer(pVM, pVCpu->idHostCpu, TMCalcHostTimerFrequency(pVM, pVCpu));
1110	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1111
1112	#ifdef LOG_ENABLED
1113	/*
1114	* Ugly: Lazy registration of ring 0 loggers.
1115	*/
1116	if (pVCpu->idCpu > 0)
1117	{
1118	PVMMR0LOGGER pR0Logger = pVCpu->vmm.s.pR0LoggerR0;
1119	if ( pR0Logger
1120	&& RT_UNLIKELY(!pR0Logger->fRegistered))
1121	{
1122	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pVM->pSession);
1123	pR0Logger->fRegistered = true;
1124	}
1125	}
1126	#endif
1127
1128	#ifdef VMM_R0_TOUCH_FPU
1129	/*
1130	* Make sure we've got the FPU state loaded so and we don't need to clear
1131	* CR0.TS and get out of sync with the host kernel when loading the guest
1132	* FPU state. @ref sec_cpum_fpu (CPUM.cpp) and @bugref{4053}.
1133	*/
1134	CPUMR0TouchHostFpu();
1135	#endif
1136	int rc;
1137	bool fPreemptRestored = false;
1138	if (!HMR0SuspendPending())
1139	{
1140	/*
1141	* Enable the context switching hook.
1142	*/
1143	if (pVCpu->vmm.s.hCtxHook != NIL_RTTHREADCTXHOOK)
1144	{
1145	Assert(!RTThreadCtxHookIsEnabled(pVCpu->vmm.s.hCtxHook));
1146	int rc2 = RTThreadCtxHookEnable(pVCpu->vmm.s.hCtxHook); AssertRC(rc2);
1147	}
1148
1149	/*
1150	* Enter HM context.
1151	*/
1152	rc = HMR0Enter(pVM, pVCpu);
1153	if (RT_SUCCESS(rc))
1154	{
1155	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
1156
1157	/*
1158	* When preemption hooks are in place, enable preemption now that
1159	* we're in HM context.
1160	*/
1161	if (vmmR0ThreadCtxHookIsEnabled(pVCpu))
1162	{
1163	fPreemptRestored = true;
1164	RTThreadPreemptRestore(&PreemptState);
1165	}
1166
1167	/*
1168	* Setup the longjmp machinery and execute guest code (calls HMR0RunGuestCode).
1169	*/
1170	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1171	rc = vmmR0CallRing3SetJmp(&pVCpu->vmm.s.CallRing3JmpBufR0, HMR0RunGuestCode, pVM, pVCpu);
1172	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1173
1174	/*
1175	* Assert sanity on the way out. Using manual assertions code here as normal
1176	* assertions are going to panic the host since we're outside the setjmp/longjmp zone.
1177	*/
1178	if (RT_UNLIKELY( VMCPU_GET_STATE(pVCpu) != VMCPUSTATE_STARTED_HM
1179	&& RT_SUCCESS_NP(rc) && rc != VINF_VMM_CALL_HOST ))
1180	{
1181	pVM->vmm.s.szRing0AssertMsg1[0] = '\0';
1182	RTStrPrintf(pVM->vmm.s.szRing0AssertMsg2, sizeof(pVM->vmm.s.szRing0AssertMsg2),
1183	"Got VMCPU state %d expected %d.\n", VMCPU_GET_STATE(pVCpu), VMCPUSTATE_STARTED_HM);
1184	rc = VERR_VMM_WRONG_HM_VMCPU_STATE;
1185	}
1186	/** @todo Get rid of this. HM shouldn't disable the context hook. */
1187	else if (RT_UNLIKELY(vmmR0ThreadCtxHookIsEnabled(pVCpu)))
1188	{
1189	pVM->vmm.s.szRing0AssertMsg1[0] = '\0';
1190	RTStrPrintf(pVM->vmm.s.szRing0AssertMsg2, sizeof(pVM->vmm.s.szRing0AssertMsg2),
1191	"Thread-context hooks still enabled! VCPU=%p Id=%u rc=%d.\n", pVCpu, pVCpu->idCpu, rc);
1192	rc = VERR_INVALID_STATE;
1193	}
1194
1195	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
1196	}
1197	STAM_COUNTER_INC(&pVM->vmm.s.StatRunRC);
1198
1199	/*
1200	* Invalidate the host CPU identifiers before we disable the context
1201	* hook / restore preemption.
1202	*/
1203	pVCpu->iHostCpuSet = UINT32_MAX;
1204	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
1205
1206	/*
1207	* Disable context hooks. Due to unresolved cleanup issues, we
1208	* cannot leave the hooks enabled when we return to ring-3.
1209	*
1210	* Note! At the moment HM may also have disabled the hook
1211	* when we get here, but the IPRT API handles that.
1212	*/
1213	if (pVCpu->vmm.s.hCtxHook != NIL_RTTHREADCTXHOOK)
1214	{
1215	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
1216	RTThreadCtxHookDisable(pVCpu->vmm.s.hCtxHook);
1217	}
1218	}
1219	/*
1220	* The system is about to go into suspend mode; go back to ring 3.
1221	*/
1222	else
1223	{
1224	rc = VINF_EM_RAW_INTERRUPT;
1225	pVCpu->iHostCpuSet = UINT32_MAX;
1226	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
1227	}
1228
1229	/** @todo When HM stops messing with the context hook state, we'll disable
1230	* preemption again before the RTThreadCtxHookDisable call. */
1231	if (!fPreemptRestored)
1232	RTThreadPreemptRestore(&PreemptState);
1233
1234	pVCpu->vmm.s.iLastGZRc = rc;
1235
1236	/* Fire dtrace probe and collect statistics. */
1237	VBOXVMM_R0_VMM_RETURN_TO_RING3_HM(pVCpu, CPUMQueryGuestCtxPtr(pVCpu), rc);
1238	#ifdef VBOX_WITH_STATISTICS
1239	vmmR0RecordRC(pVM, pVCpu, rc);
1240	#endif
1241	}
1242	/*
1243	* Invalid CPU set index or TSC delta in need of measuring.
1244	*/
1245	else
1246	{
1247	pVCpu->iHostCpuSet = UINT32_MAX;
1248	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
1249	RTThreadPreemptRestore(&PreemptState);
1250	if (iHostCpuSet < RTCPUSET_MAX_CPUS)
1251	{
1252	int rc = SUPR0TscDeltaMeasureBySetIndex(pVM->pSession, iHostCpuSet, 0 /fFlags/,
1253	2 /cMsWaitRetry/, 5RT_MS_1SEC /cMsWaitThread*/,
1254	0 /default cTries/);
1255	if (RT_SUCCESS(rc) \|\| rc == VERR_CPU_OFFLINE)
1256	pVCpu->vmm.s.iLastGZRc = VINF_EM_RAW_TO_R3;
1257	else
1258	pVCpu->vmm.s.iLastGZRc = rc;
1259	}
1260	else
1261	pVCpu->vmm.s.iLastGZRc = VERR_INVALID_CPU_INDEX;
1262	}
1263	break;
1264	}
1265
1266	/*
1267	* For profiling.
1268	*/
1269	case VMMR0_DO_NOP:
1270	pVCpu->vmm.s.iLastGZRc = VINF_SUCCESS;
1271	break;
1272
1273	/*
1274	* Impossible.
1275	*/
1276	default:
1277	AssertMsgFailed(("%#x\n", enmOperation));
1278	pVCpu->vmm.s.iLastGZRc = VERR_NOT_SUPPORTED;
1279	break;
1280	}
1281	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1282	}
1283
1284
1285	/**
1286	* Validates a session or VM session argument.
1287	*
1288	* @returns true / false accordingly.
1289	* @param pVM The cross context VM structure.
1290	* @param pClaimedSession The session claim to validate.
1291	* @param pSession The session argument.
1292	*/
1293	DECLINLINE(bool) vmmR0IsValidSession(PVM pVM, PSUPDRVSESSION pClaimedSession, PSUPDRVSESSION pSession)
1294	{
1295	/* This must be set! */
1296	if (!pSession)
1297	return false;
1298
1299	/* Only one out of the two. */
1300	if (pVM && pClaimedSession)
1301	return false;
1302	if (pVM)
1303	pClaimedSession = pVM->pSession;
1304	return pClaimedSession == pSession;
1305	}
1306
1307
1308	/**
1309	* VMMR0EntryEx worker function, either called directly or when ever possible
1310	* called thru a longjmp so we can exit safely on failure.
1311	*
1312	* @returns VBox status code.
1313	* @param pVM The cross context VM structure.
1314	* @param idCpu Virtual CPU ID argument. Must be NIL_VMCPUID if pVM
1315	* is NIL_RTR0PTR, and may be NIL_VMCPUID if it isn't
1316	* @param enmOperation Which operation to execute.
1317	* @param pReqHdr This points to a SUPVMMR0REQHDR packet. Optional.
1318	* The support driver validates this if it's present.
1319	* @param u64Arg Some simple constant argument.
1320	* @param pSession The session of the caller.
1321	* @remarks Assume called with interrupts _enabled_.
1322	*/
1323	static int vmmR0EntryExWorker(PVM pVM, VMCPUID idCpu, VMMR0OPERATION enmOperation, PSUPVMMR0REQHDR pReqHdr, uint64_t u64Arg, PSUPDRVSESSION pSession)
1324	{
1325	/*
1326	* Common VM pointer validation.
1327	*/
1328	if (pVM)
1329	{
1330	if (RT_UNLIKELY( !VALID_PTR(pVM)
1331	\|\| ((uintptr_t)pVM & PAGE_OFFSET_MASK)))
1332	{
1333	SUPR0Printf("vmmR0EntryExWorker: Invalid pVM=%p! (op=%d)\n", pVM, enmOperation);
1334	return VERR_INVALID_POINTER;
1335	}
1336	if (RT_UNLIKELY( pVM->enmVMState < VMSTATE_CREATING
1337	\|\| pVM->enmVMState > VMSTATE_TERMINATED
1338	\|\| pVM->pVMR0 != pVM))
1339	{
1340	SUPR0Printf("vmmR0EntryExWorker: Invalid pVM=%p:{enmVMState=%d, .pVMR0=%p}! (op=%d)\n",
1341	pVM, pVM->enmVMState, pVM->pVMR0, enmOperation);
1342	return VERR_INVALID_POINTER;
1343	}
1344
1345	if (RT_UNLIKELY(idCpu >= pVM->cCpus && idCpu != NIL_VMCPUID))
1346	{
1347	SUPR0Printf("vmmR0EntryExWorker: Invalid idCpu (%u vs cCpus=%u)\n", idCpu, pVM->cCpus);
1348	return VERR_INVALID_PARAMETER;
1349	}
1350	}
1351	else if (RT_UNLIKELY(idCpu != NIL_VMCPUID))
1352	{
1353	SUPR0Printf("vmmR0EntryExWorker: Invalid idCpu=%u\n", idCpu);
1354	return VERR_INVALID_PARAMETER;
1355	}
1356	VMM_CHECK_SMAP_SETUP();
1357	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
1358	int rc;
1359
1360	switch (enmOperation)
1361	{
1362	/*
1363	* GVM requests
1364	*/
1365	case VMMR0_DO_GVMM_CREATE_VM:
1366	if (pVM \|\| u64Arg \|\| idCpu != NIL_VMCPUID)
1367	return VERR_INVALID_PARAMETER;
1368	rc = GVMMR0CreateVMReq((PGVMMCREATEVMREQ)pReqHdr);
1369	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
1370	break;
1371
1372	case VMMR0_DO_GVMM_DESTROY_VM:
1373	if (pReqHdr \|\| u64Arg)
1374	return VERR_INVALID_PARAMETER;
1375	rc = GVMMR0DestroyVM(pVM);
1376	VMM_CHECK_SMAP_CHECK(RT_NOTHING);
1377	break;
1378
1379	case VMMR0_DO_GVMM_REGISTER_VMCPU:
1380	{
1381	if (!pVM)
1382	return VERR_INVALID_PARAMETER;
1383	rc = GVMMR0RegisterVCpu(pVM, idCpu);
1384	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1385	break;
1386	}
1387
1388	case VMMR0_DO_GVMM_SCHED_HALT:
1389	if (pReqHdr)
1390	return VERR_INVALID_PARAMETER;
1391	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1392	rc = GVMMR0SchedHalt(pVM, idCpu, u64Arg);
1393	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1394	break;
1395
1396	case VMMR0_DO_GVMM_SCHED_WAKE_UP:
1397	if (pReqHdr \|\| u64Arg)
1398	return VERR_INVALID_PARAMETER;
1399	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1400	rc = GVMMR0SchedWakeUp(pVM, idCpu);
1401	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1402	break;
1403
1404	case VMMR0_DO_GVMM_SCHED_POKE:
1405	if (pReqHdr \|\| u64Arg)
1406	return VERR_INVALID_PARAMETER;
1407	rc = GVMMR0SchedPoke(pVM, idCpu);
1408	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1409	break;
1410
1411	case VMMR0_DO_GVMM_SCHED_WAKE_UP_AND_POKE_CPUS:
1412	if (u64Arg)
1413	return VERR_INVALID_PARAMETER;
1414	rc = GVMMR0SchedWakeUpAndPokeCpusReq(pVM, (PGVMMSCHEDWAKEUPANDPOKECPUSREQ)pReqHdr);
1415	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1416	break;
1417
1418	case VMMR0_DO_GVMM_SCHED_POLL:
1419	if (pReqHdr \|\| u64Arg > 1)
1420	return VERR_INVALID_PARAMETER;
1421	rc = GVMMR0SchedPoll(pVM, idCpu, !!u64Arg);
1422	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1423	break;
1424
1425	case VMMR0_DO_GVMM_QUERY_STATISTICS:
1426	if (u64Arg)
1427	return VERR_INVALID_PARAMETER;
1428	rc = GVMMR0QueryStatisticsReq(pVM, (PGVMMQUERYSTATISTICSSREQ)pReqHdr);
1429	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1430	break;
1431
1432	case VMMR0_DO_GVMM_RESET_STATISTICS:
1433	if (u64Arg)
1434	return VERR_INVALID_PARAMETER;
1435	rc = GVMMR0ResetStatisticsReq(pVM, (PGVMMRESETSTATISTICSSREQ)pReqHdr);
1436	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1437	break;
1438
1439	/*
1440	* Initialize the R0 part of a VM instance.
1441	*/
1442	case VMMR0_DO_VMMR0_INIT:
1443	rc = vmmR0InitVM(pVM, RT_LODWORD(u64Arg), RT_HIDWORD(u64Arg));
1444	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1445	break;
1446
1447	/*
1448	* Terminate the R0 part of a VM instance.
1449	*/
1450	case VMMR0_DO_VMMR0_TERM:
1451	rc = VMMR0TermVM(pVM, NULL);
1452	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1453	break;
1454
1455	/*
1456	* Attempt to enable hm mode and check the current setting.
1457	*/
1458	case VMMR0_DO_HM_ENABLE:
1459	rc = HMR0EnableAllCpus(pVM);
1460	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1461	break;
1462
1463	/*
1464	* Setup the hardware accelerated session.
1465	*/
1466	case VMMR0_DO_HM_SETUP_VM:
1467	rc = HMR0SetupVM(pVM);
1468	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1469	break;
1470
1471	/*
1472	* Switch to RC to execute Hypervisor function.
1473	*/
1474	case VMMR0_DO_CALL_HYPERVISOR:
1475	{
1476	#ifdef VBOX_WITH_RAW_MODE
1477	/*
1478	* Validate input / context.
1479	*/
1480	if (RT_UNLIKELY(idCpu != 0))
1481	return VERR_INVALID_CPU_ID;
1482	if (RT_UNLIKELY(pVM->cCpus != 1))
1483	return VERR_INVALID_PARAMETER;
1484	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1485	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
1486	if (RT_UNLIKELY(!PGMGetHyperCR3(pVCpu)))
1487	return VERR_PGM_NO_CR3_SHADOW_ROOT;
1488	# endif
1489
1490	/*
1491	* Disable interrupts.
1492	*/
1493	RTCCUINTREG fFlags = ASMIntDisableFlags();
1494
1495	/*
1496	* Get the host CPU identifiers, make sure they are valid and that
1497	* we've got a TSC delta for the CPU.
1498	*/
1499	RTCPUID idHostCpu;
1500	uint32_t iHostCpuSet = RTMpCurSetIndexAndId(&idHostCpu);
1501	if (RT_UNLIKELY(iHostCpuSet >= RTCPUSET_MAX_CPUS))
1502	{
1503	ASMSetFlags(fFlags);
1504	return VERR_INVALID_CPU_INDEX;
1505	}
1506	if (RT_UNLIKELY(!SUPIsTscDeltaAvailableForCpuSetIndex(iHostCpuSet)))
1507	{
1508	ASMSetFlags(fFlags);
1509	rc = SUPR0TscDeltaMeasureBySetIndex(pVM->pSession, iHostCpuSet, 0 /fFlags/,
1510	2 /cMsWaitRetry/, 5RT_MS_1SEC /cMsWaitThread*/,
1511	0 /default cTries/);
1512	if (RT_FAILURE(rc) && rc != VERR_CPU_OFFLINE)
1513	{
1514	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1515	return rc;
1516	}
1517	}
1518
1519	/*
1520	* Commit the CPU identifiers.
1521	*/
1522	# ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
1523	CPUMR0SetLApic(pVCpu, iHostCpuSet);
1524	# endif
1525	pVCpu->iHostCpuSet = iHostCpuSet;
1526	ASMAtomicWriteU32(&pVCpu->idHostCpu, idHostCpu);
1527
1528	/*
1529	* We might need to disable VT-x if the active switcher turns off paging.
1530	*/
1531	bool fVTxDisabled;
1532	rc = HMR0EnterSwitcher(pVM, pVM->vmm.s.enmSwitcher, &fVTxDisabled);
1533	if (RT_SUCCESS(rc))
1534	{
1535	/*
1536	* Go through the wormhole...
1537	*/
1538	rc = pVM->vmm.s.pfnR0ToRawMode(pVM);
1539
1540	/*
1541	* Re-enable VT-x before we dispatch any pending host interrupts.
1542	*/
1543	HMR0LeaveSwitcher(pVM, fVTxDisabled);
1544
1545	if ( rc == VINF_EM_RAW_INTERRUPT
1546	\|\| rc == VINF_EM_RAW_INTERRUPT_HYPER)
1547	TRPMR0DispatchHostInterrupt(pVM);
1548	}
1549
1550	/*
1551	* Invalidate the host CPU identifiers as we restore interrupts.
1552	*/
1553	pVCpu->iHostCpuSet = UINT32_MAX;
1554	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
1555	ASMSetFlags(fFlags);
1556
1557	#else /* !VBOX_WITH_RAW_MODE */
1558	rc = VERR_RAW_MODE_NOT_SUPPORTED;
1559	#endif
1560	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1561	break;
1562	}
1563
1564	/*
1565	* PGM wrappers.
1566	*/
1567	case VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES:
1568	if (idCpu == NIL_VMCPUID)
1569	return VERR_INVALID_CPU_ID;
1570	rc = PGMR0PhysAllocateHandyPages(pVM, &pVM->aCpus[idCpu]);
1571	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1572	break;
1573
1574	case VMMR0_DO_PGM_FLUSH_HANDY_PAGES:
1575	if (idCpu == NIL_VMCPUID)
1576	return VERR_INVALID_CPU_ID;
1577	rc = PGMR0PhysFlushHandyPages(pVM, &pVM->aCpus[idCpu]);
1578	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1579	break;
1580
1581	case VMMR0_DO_PGM_ALLOCATE_LARGE_HANDY_PAGE:
1582	if (idCpu == NIL_VMCPUID)
1583	return VERR_INVALID_CPU_ID;
1584	rc = PGMR0PhysAllocateLargeHandyPage(pVM, &pVM->aCpus[idCpu]);
1585	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1586	break;
1587
1588	case VMMR0_DO_PGM_PHYS_SETUP_IOMMU:
1589	if (idCpu != 0)
1590	return VERR_INVALID_CPU_ID;
1591	rc = PGMR0PhysSetupIommu(pVM);
1592	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1593	break;
1594
1595	/*
1596	* GMM wrappers.
1597	*/
1598	case VMMR0_DO_GMM_INITIAL_RESERVATION:
1599	if (u64Arg)
1600	return VERR_INVALID_PARAMETER;
1601	rc = GMMR0InitialReservationReq(pVM, idCpu, (PGMMINITIALRESERVATIONREQ)pReqHdr);
1602	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1603	break;
1604
1605	case VMMR0_DO_GMM_UPDATE_RESERVATION:
1606	if (u64Arg)
1607	return VERR_INVALID_PARAMETER;
1608	rc = GMMR0UpdateReservationReq(pVM, idCpu, (PGMMUPDATERESERVATIONREQ)pReqHdr);
1609	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1610	break;
1611
1612	case VMMR0_DO_GMM_ALLOCATE_PAGES:
1613	if (u64Arg)
1614	return VERR_INVALID_PARAMETER;
1615	rc = GMMR0AllocatePagesReq(pVM, idCpu, (PGMMALLOCATEPAGESREQ)pReqHdr);
1616	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1617	break;
1618
1619	case VMMR0_DO_GMM_FREE_PAGES:
1620	if (u64Arg)
1621	return VERR_INVALID_PARAMETER;
1622	rc = GMMR0FreePagesReq(pVM, idCpu, (PGMMFREEPAGESREQ)pReqHdr);
1623	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1624	break;
1625
1626	case VMMR0_DO_GMM_FREE_LARGE_PAGE:
1627	if (u64Arg)
1628	return VERR_INVALID_PARAMETER;
1629	rc = GMMR0FreeLargePageReq(pVM, idCpu, (PGMMFREELARGEPAGEREQ)pReqHdr);
1630	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1631	break;
1632
1633	case VMMR0_DO_GMM_QUERY_HYPERVISOR_MEM_STATS:
1634	if (u64Arg)
1635	return VERR_INVALID_PARAMETER;
1636	rc = GMMR0QueryHypervisorMemoryStatsReq(pVM, (PGMMMEMSTATSREQ)pReqHdr);
1637	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1638	break;
1639
1640	case VMMR0_DO_GMM_QUERY_MEM_STATS:
1641	if (idCpu == NIL_VMCPUID)
1642	return VERR_INVALID_CPU_ID;
1643	if (u64Arg)
1644	return VERR_INVALID_PARAMETER;
1645	rc = GMMR0QueryMemoryStatsReq(pVM, idCpu, (PGMMMEMSTATSREQ)pReqHdr);
1646	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1647	break;
1648
1649	case VMMR0_DO_GMM_BALLOONED_PAGES:
1650	if (u64Arg)
1651	return VERR_INVALID_PARAMETER;
1652	rc = GMMR0BalloonedPagesReq(pVM, idCpu, (PGMMBALLOONEDPAGESREQ)pReqHdr);
1653	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1654	break;
1655
1656	case VMMR0_DO_GMM_MAP_UNMAP_CHUNK:
1657	if (u64Arg)
1658	return VERR_INVALID_PARAMETER;
1659	rc = GMMR0MapUnmapChunkReq(pVM, (PGMMMAPUNMAPCHUNKREQ)pReqHdr);
1660	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1661	break;
1662
1663	case VMMR0_DO_GMM_SEED_CHUNK:
1664	if (pReqHdr)
1665	return VERR_INVALID_PARAMETER;
1666	rc = GMMR0SeedChunk(pVM, idCpu, (RTR3PTR)u64Arg);
1667	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1668	break;
1669
1670	case VMMR0_DO_GMM_REGISTER_SHARED_MODULE:
1671	if (idCpu == NIL_VMCPUID)
1672	return VERR_INVALID_CPU_ID;
1673	if (u64Arg)
1674	return VERR_INVALID_PARAMETER;
1675	rc = GMMR0RegisterSharedModuleReq(pVM, idCpu, (PGMMREGISTERSHAREDMODULEREQ)pReqHdr);
1676	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1677	break;
1678
1679	case VMMR0_DO_GMM_UNREGISTER_SHARED_MODULE:
1680	if (idCpu == NIL_VMCPUID)
1681	return VERR_INVALID_CPU_ID;
1682	if (u64Arg)
1683	return VERR_INVALID_PARAMETER;
1684	rc = GMMR0UnregisterSharedModuleReq(pVM, idCpu, (PGMMUNREGISTERSHAREDMODULEREQ)pReqHdr);
1685	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1686	break;
1687
1688	case VMMR0_DO_GMM_RESET_SHARED_MODULES:
1689	if (idCpu == NIL_VMCPUID)
1690	return VERR_INVALID_CPU_ID;
1691	if ( u64Arg
1692	\|\| pReqHdr)
1693	return VERR_INVALID_PARAMETER;
1694	rc = GMMR0ResetSharedModules(pVM, idCpu);
1695	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1696	break;
1697
1698	#ifdef VBOX_WITH_PAGE_SHARING
1699	case VMMR0_DO_GMM_CHECK_SHARED_MODULES:
1700	{
1701	if (idCpu == NIL_VMCPUID)
1702	return VERR_INVALID_CPU_ID;
1703	if ( u64Arg
1704	\|\| pReqHdr)
1705	return VERR_INVALID_PARAMETER;
1706
1707	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1708	Assert(pVCpu->hNativeThreadR0 == RTThreadNativeSelf());
1709
1710	# ifdef DEBUG_sandervl
1711	/* Make sure that log flushes can jump back to ring-3; annoying to get an incomplete log (this is risky though as the code doesn't take this into account). */
1712	/* Todo: this can have bad side effects for unexpected jumps back to r3. */
1713	rc = GMMR0CheckSharedModulesStart(pVM);
1714	if (rc == VINF_SUCCESS)
1715	{
1716	rc = vmmR0CallRing3SetJmp(&pVCpu->vmm.s.CallRing3JmpBufR0, GMMR0CheckSharedModules, pVM, pVCpu); /* this may resume code. */
1717	Assert( rc == VINF_SUCCESS
1718	\|\| (rc == VINF_VMM_CALL_HOST && pVCpu->vmm.s.enmCallRing3Operation == VMMCALLRING3_VMM_LOGGER_FLUSH));
1719	GMMR0CheckSharedModulesEnd(pVM);
1720	}
1721	# else
1722	rc = GMMR0CheckSharedModules(pVM, pVCpu);
1723	# endif
1724	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1725	break;
1726	}
1727	#endif
1728
1729	#if defined(VBOX_STRICT) && HC_ARCH_BITS == 64
1730	case VMMR0_DO_GMM_FIND_DUPLICATE_PAGE:
1731	if (u64Arg)
1732	return VERR_INVALID_PARAMETER;
1733	rc = GMMR0FindDuplicatePageReq(pVM, (PGMMFINDDUPLICATEPAGEREQ)pReqHdr);
1734	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1735	break;
1736	#endif
1737
1738	case VMMR0_DO_GMM_QUERY_STATISTICS:
1739	if (u64Arg)
1740	return VERR_INVALID_PARAMETER;
1741	rc = GMMR0QueryStatisticsReq(pVM, (PGMMQUERYSTATISTICSSREQ)pReqHdr);
1742	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1743	break;
1744
1745	case VMMR0_DO_GMM_RESET_STATISTICS:
1746	if (u64Arg)
1747	return VERR_INVALID_PARAMETER;
1748	rc = GMMR0ResetStatisticsReq(pVM, (PGMMRESETSTATISTICSSREQ)pReqHdr);
1749	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1750	break;
1751
1752	/*
1753	* A quick GCFGM mock-up.
1754	*/
1755	/** @todo GCFGM with proper access control, ring-3 management interface and all that. */
1756	case VMMR0_DO_GCFGM_SET_VALUE:
1757	case VMMR0_DO_GCFGM_QUERY_VALUE:
1758	{
1759	if (pVM \|\| !pReqHdr \|\| u64Arg \|\| idCpu != NIL_VMCPUID)
1760	return VERR_INVALID_PARAMETER;
1761	PGCFGMVALUEREQ pReq = (PGCFGMVALUEREQ)pReqHdr;
1762	if (pReq->Hdr.cbReq != sizeof(*pReq))
1763	return VERR_INVALID_PARAMETER;
1764	if (enmOperation == VMMR0_DO_GCFGM_SET_VALUE)
1765	{
1766	rc = GVMMR0SetConfig(pReq->pSession, &pReq->szName[0], pReq->u64Value);
1767	//if (rc == VERR_CFGM_VALUE_NOT_FOUND)
1768	// rc = GMMR0SetConfig(pReq->pSession, &pReq->szName[0], pReq->u64Value);
1769	}
1770	else
1771	{
1772	rc = GVMMR0QueryConfig(pReq->pSession, &pReq->szName[0], &pReq->u64Value);
1773	//if (rc == VERR_CFGM_VALUE_NOT_FOUND)
1774	// rc = GMMR0QueryConfig(pReq->pSession, &pReq->szName[0], &pReq->u64Value);
1775	}
1776	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1777	break;
1778	}
1779
1780	/*
1781	* PDM Wrappers.
1782	*/
1783	case VMMR0_DO_PDM_DRIVER_CALL_REQ_HANDLER:
1784	{
1785	if (!pVM \|\| !pReqHdr \|\| u64Arg \|\| idCpu != NIL_VMCPUID)
1786	return VERR_INVALID_PARAMETER;
1787	rc = PDMR0DriverCallReqHandler(pVM, (PPDMDRIVERCALLREQHANDLERREQ)pReqHdr);
1788	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1789	break;
1790	}
1791
1792	case VMMR0_DO_PDM_DEVICE_CALL_REQ_HANDLER:
1793	{
1794	if (!pVM \|\| !pReqHdr \|\| u64Arg \|\| idCpu != NIL_VMCPUID)
1795	return VERR_INVALID_PARAMETER;
1796	rc = PDMR0DeviceCallReqHandler(pVM, (PPDMDEVICECALLREQHANDLERREQ)pReqHdr);
1797	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1798	break;
1799	}
1800
1801	/*
1802	* Requests to the internal networking service.
1803	*/
1804	case VMMR0_DO_INTNET_OPEN:
1805	{
1806	PINTNETOPENREQ pReq = (PINTNETOPENREQ)pReqHdr;
1807	if (u64Arg \|\| !pReq \|\| !vmmR0IsValidSession(pVM, pReq->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
1808	return VERR_INVALID_PARAMETER;
1809	rc = IntNetR0OpenReq(pSession, pReq);
1810	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1811	break;
1812	}
1813
1814	case VMMR0_DO_INTNET_IF_CLOSE:
1815	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFCLOSEREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
1816	return VERR_INVALID_PARAMETER;
1817	rc = IntNetR0IfCloseReq(pSession, (PINTNETIFCLOSEREQ)pReqHdr);
1818	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1819	break;
1820
1821
1822	case VMMR0_DO_INTNET_IF_GET_BUFFER_PTRS:
1823	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFGETBUFFERPTRSREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
1824	return VERR_INVALID_PARAMETER;
1825	rc = IntNetR0IfGetBufferPtrsReq(pSession, (PINTNETIFGETBUFFERPTRSREQ)pReqHdr);
1826	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1827	break;
1828
1829	case VMMR0_DO_INTNET_IF_SET_PROMISCUOUS_MODE:
1830	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFSETPROMISCUOUSMODEREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
1831	return VERR_INVALID_PARAMETER;
1832	rc = IntNetR0IfSetPromiscuousModeReq(pSession, (PINTNETIFSETPROMISCUOUSMODEREQ)pReqHdr);
1833	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1834	break;
1835
1836	case VMMR0_DO_INTNET_IF_SET_MAC_ADDRESS:
1837	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFSETMACADDRESSREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
1838	return VERR_INVALID_PARAMETER;
1839	rc = IntNetR0IfSetMacAddressReq(pSession, (PINTNETIFSETMACADDRESSREQ)pReqHdr);
1840	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1841	break;
1842
1843	case VMMR0_DO_INTNET_IF_SET_ACTIVE:
1844	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFSETACTIVEREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
1845	return VERR_INVALID_PARAMETER;
1846	rc = IntNetR0IfSetActiveReq(pSession, (PINTNETIFSETACTIVEREQ)pReqHdr);
1847	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1848	break;
1849
1850	case VMMR0_DO_INTNET_IF_SEND:
1851	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFSENDREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
1852	return VERR_INVALID_PARAMETER;
1853	rc = IntNetR0IfSendReq(pSession, (PINTNETIFSENDREQ)pReqHdr);
1854	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1855	break;
1856
1857	case VMMR0_DO_INTNET_IF_WAIT:
1858	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFWAITREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
1859	return VERR_INVALID_PARAMETER;
1860	rc = IntNetR0IfWaitReq(pSession, (PINTNETIFWAITREQ)pReqHdr);
1861	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1862	break;
1863
1864	case VMMR0_DO_INTNET_IF_ABORT_WAIT:
1865	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFWAITREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
1866	return VERR_INVALID_PARAMETER;
1867	rc = IntNetR0IfAbortWaitReq(pSession, (PINTNETIFABORTWAITREQ)pReqHdr);
1868	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1869	break;
1870
1871	#ifdef VBOX_WITH_PCI_PASSTHROUGH
1872	/*
1873	* Requests to host PCI driver service.
1874	*/
1875	case VMMR0_DO_PCIRAW_REQ:
1876	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PPCIRAWSENDREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
1877	return VERR_INVALID_PARAMETER;
1878	rc = PciRawR0ProcessReq(pSession, pVM, (PPCIRAWSENDREQ)pReqHdr);
1879	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1880	break;
1881	#endif
1882	/*
1883	* For profiling.
1884	*/
1885	case VMMR0_DO_NOP:
1886	case VMMR0_DO_SLOW_NOP:
1887	return VINF_SUCCESS;
1888
1889	/*
1890	* For testing Ring-0 APIs invoked in this environment.
1891	*/
1892	case VMMR0_DO_TESTS:
1893	/** @todo make new test */
1894	return VINF_SUCCESS;
1895
1896
1897	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
1898	case VMMR0_DO_TEST_SWITCHER3264:
1899	if (idCpu == NIL_VMCPUID)
1900	return VERR_INVALID_CPU_ID;
1901	rc = HMR0TestSwitcher3264(pVM);
1902	VMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
1903	break;
1904	#endif
1905	default:
1906	/*
1907	* We're returning VERR_NOT_SUPPORT here so we've got something else
1908	* than -1 which the interrupt gate glue code might return.
1909	*/
1910	Log(("operation %#x is not supported\n", enmOperation));
1911	return VERR_NOT_SUPPORTED;
1912	}
1913	return rc;
1914	}
1915
1916
1917	/**
1918	* Argument for vmmR0EntryExWrapper containing the arguments for VMMR0EntryEx.
1919	*/
1920	typedef struct VMMR0ENTRYEXARGS
1921	{
1922	PVM pVM;
1923	VMCPUID idCpu;
1924	VMMR0OPERATION enmOperation;
1925	PSUPVMMR0REQHDR pReq;
1926	uint64_t u64Arg;
1927	PSUPDRVSESSION pSession;
1928	} VMMR0ENTRYEXARGS;
1929	/** Pointer to a vmmR0EntryExWrapper argument package. */
1930	typedef VMMR0ENTRYEXARGS *PVMMR0ENTRYEXARGS;
1931
1932	/**
1933	* This is just a longjmp wrapper function for VMMR0EntryEx calls.
1934	*
1935	* @returns VBox status code.
1936	* @param pvArgs The argument package
1937	*/
1938	static DECLCALLBACK(int) vmmR0EntryExWrapper(void *pvArgs)
1939	{
1940	return vmmR0EntryExWorker(((PVMMR0ENTRYEXARGS)pvArgs)->pVM,
1941	((PVMMR0ENTRYEXARGS)pvArgs)->idCpu,
1942	((PVMMR0ENTRYEXARGS)pvArgs)->enmOperation,
1943	((PVMMR0ENTRYEXARGS)pvArgs)->pReq,
1944	((PVMMR0ENTRYEXARGS)pvArgs)->u64Arg,
1945	((PVMMR0ENTRYEXARGS)pvArgs)->pSession);
1946	}
1947
1948
1949	/**
1950	* The Ring 0 entry point, called by the support library (SUP).
1951	*
1952	* @returns VBox status code.
1953	* @param pVM The cross context VM structure.
1954	* @param idCpu Virtual CPU ID argument. Must be NIL_VMCPUID if pVM
1955	* is NIL_RTR0PTR, and may be NIL_VMCPUID if it isn't
1956	* @param enmOperation Which operation to execute.
1957	* @param pReq Pointer to the SUPVMMR0REQHDR packet. Optional.
1958	* @param u64Arg Some simple constant argument.
1959	* @param pSession The session of the caller.
1960	* @remarks Assume called with interrupts _enabled_.
1961	*/
1962	VMMR0DECL(int) VMMR0EntryEx(PVM pVM, VMCPUID idCpu, VMMR0OPERATION enmOperation, PSUPVMMR0REQHDR pReq, uint64_t u64Arg, PSUPDRVSESSION pSession)
1963	{
1964	/*
1965	* Requests that should only happen on the EMT thread will be
1966	* wrapped in a setjmp so we can assert without causing trouble.
1967	*/
1968	if ( VALID_PTR(pVM)
1969	&& pVM->pVMR0
1970	&& idCpu < pVM->cCpus)
1971	{
1972	switch (enmOperation)
1973	{
1974	/* These might/will be called before VMMR3Init. */
1975	case VMMR0_DO_GMM_INITIAL_RESERVATION:
1976	case VMMR0_DO_GMM_UPDATE_RESERVATION:
1977	case VMMR0_DO_GMM_ALLOCATE_PAGES:
1978	case VMMR0_DO_GMM_FREE_PAGES:
1979	case VMMR0_DO_GMM_BALLOONED_PAGES:
1980	/* On the mac we might not have a valid jmp buf, so check these as well. */
1981	case VMMR0_DO_VMMR0_INIT:
1982	case VMMR0_DO_VMMR0_TERM:
1983	{
1984	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1985
1986	if (!pVCpu->vmm.s.CallRing3JmpBufR0.pvSavedStack)
1987	break;
1988
1989	/** @todo validate this EMT claim... GVM knows. */
1990	VMMR0ENTRYEXARGS Args;
1991	Args.pVM = pVM;
1992	Args.idCpu = idCpu;
1993	Args.enmOperation = enmOperation;
1994	Args.pReq = pReq;
1995	Args.u64Arg = u64Arg;
1996	Args.pSession = pSession;
1997	return vmmR0CallRing3SetJmpEx(&pVCpu->vmm.s.CallRing3JmpBufR0, vmmR0EntryExWrapper, &Args);
1998	}
1999
2000	default:
2001	break;
2002	}
2003	}
2004	return vmmR0EntryExWorker(pVM, idCpu, enmOperation, pReq, u64Arg, pSession);
2005	}
2006
2007
2008	/**
2009	* Checks whether we've armed the ring-0 long jump machinery.
2010	*
2011	* @returns @c true / @c false
2012	* @param pVCpu The cross context virtual CPU structure.
2013	* @thread EMT
2014	* @sa VMMIsLongJumpArmed
2015	*/
2016	VMMR0_INT_DECL(bool) VMMR0IsLongJumpArmed(PVMCPU pVCpu)
2017	{
2018	#ifdef RT_ARCH_X86
2019	return pVCpu->vmm.s.CallRing3JmpBufR0.eip
2020	&& !pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call;
2021	#else
2022	return pVCpu->vmm.s.CallRing3JmpBufR0.rip
2023	&& !pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call;
2024	#endif
2025	}
2026
2027
2028	/**
2029	* Checks whether we've done a ring-3 long jump.
2030	*
2031	* @returns @c true / @c false
2032	* @param pVCpu The cross context virtual CPU structure.
2033	* @thread EMT
2034	*/
2035	VMMR0_INT_DECL(bool) VMMR0IsInRing3LongJump(PVMCPU pVCpu)
2036	{
2037	return pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call;
2038	}
2039
2040
2041	/**
2042	* Internal R0 logger worker: Flush logger.
2043	*
2044	* @param pLogger The logger instance to flush.
2045	* @remark This function must be exported!
2046	*/
2047	VMMR0DECL(void) vmmR0LoggerFlush(PRTLOGGER pLogger)
2048	{
2049	#ifdef LOG_ENABLED
2050	/*
2051	* Convert the pLogger into a VM handle and 'call' back to Ring-3.
2052	* (This is a bit paranoid code.)
2053	*/
2054	PVMMR0LOGGER pR0Logger = (PVMMR0LOGGER)((uintptr_t)pLogger - RT_OFFSETOF(VMMR0LOGGER, Logger));
2055	if ( !VALID_PTR(pR0Logger)
2056	\|\| !VALID_PTR(pR0Logger + 1)
2057	\|\| pLogger->u32Magic != RTLOGGER_MAGIC)
2058	{
2059	# ifdef DEBUG
2060	SUPR0Printf("vmmR0LoggerFlush: pLogger=%p!\n", pLogger);
2061	# endif
2062	return;
2063	}
2064	if (pR0Logger->fFlushingDisabled)
2065	return; /* quietly */
2066
2067	PVM pVM = pR0Logger->pVM;
2068	if ( !VALID_PTR(pVM)
2069	\|\| pVM->pVMR0 != pVM)
2070	{
2071	# ifdef DEBUG
2072	SUPR0Printf("vmmR0LoggerFlush: pVM=%p! pVMR0=%p! pLogger=%p\n", pVM, pVM->pVMR0, pLogger);
2073	# endif
2074	return;
2075	}
2076
2077	PVMCPU pVCpu = VMMGetCpu(pVM);
2078	if (pVCpu)
2079	{
2080	/*
2081	* Check that the jump buffer is armed.
2082	*/
2083	# ifdef RT_ARCH_X86
2084	if ( !pVCpu->vmm.s.CallRing3JmpBufR0.eip
2085	\|\| pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call)
2086	# else
2087	if ( !pVCpu->vmm.s.CallRing3JmpBufR0.rip
2088	\|\| pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call)
2089	# endif
2090	{
2091	# ifdef DEBUG
2092	SUPR0Printf("vmmR0LoggerFlush: Jump buffer isn't armed!\n");
2093	# endif
2094	return;
2095	}
2096	VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_VMM_LOGGER_FLUSH, 0);
2097	}
2098	# ifdef DEBUG
2099	else
2100	SUPR0Printf("vmmR0LoggerFlush: invalid VCPU context!\n");
2101	# endif
2102	#else
2103	NOREF(pLogger);
2104	#endif /* LOG_ENABLED */
2105	}
2106
2107	/**
2108	* Internal R0 logger worker: Custom prefix.
2109	*
2110	* @returns Number of chars written.
2111	*
2112	* @param pLogger The logger instance.
2113	* @param pchBuf The output buffer.
2114	* @param cchBuf The size of the buffer.
2115	* @param pvUser User argument (ignored).
2116	*/
2117	VMMR0DECL(size_t) vmmR0LoggerPrefix(PRTLOGGER pLogger, char pchBuf, size_t cchBuf, void pvUser)
2118	{
2119	NOREF(pvUser);
2120	#ifdef LOG_ENABLED
2121	PVMMR0LOGGER pR0Logger = (PVMMR0LOGGER)((uintptr_t)pLogger - RT_OFFSETOF(VMMR0LOGGER, Logger));
2122	if ( !VALID_PTR(pR0Logger)
2123	\|\| !VALID_PTR(pR0Logger + 1)
2124	\|\| pLogger->u32Magic != RTLOGGER_MAGIC
2125	\|\| cchBuf < 2)
2126	return 0;
2127
2128	static const char s_szHex[17] = "0123456789abcdef";
2129	VMCPUID const idCpu = pR0Logger->idCpu;
2130	pchBuf[1] = s_szHex[ idCpu & 15];
2131	pchBuf[0] = s_szHex[(idCpu >> 4) & 15];
2132
2133	return 2;
2134	#else
2135	NOREF(pLogger); NOREF(pchBuf); NOREF(cchBuf);
2136	return 0;
2137	#endif
2138	}
2139
2140	#ifdef LOG_ENABLED
2141
2142	/**
2143	* Disables flushing of the ring-0 debug log.
2144	*
2145	* @param pVCpu The cross context virtual CPU structure.
2146	*/
2147	VMMR0_INT_DECL(void) VMMR0LogFlushDisable(PVMCPU pVCpu)
2148	{
2149	if (pVCpu->vmm.s.pR0LoggerR0)
2150	pVCpu->vmm.s.pR0LoggerR0->fFlushingDisabled = true;
2151	}
2152
2153
2154	/**
2155	* Enables flushing of the ring-0 debug log.
2156	*
2157	* @param pVCpu The cross context virtual CPU structure.
2158	*/
2159	VMMR0_INT_DECL(void) VMMR0LogFlushEnable(PVMCPU pVCpu)
2160	{
2161	if (pVCpu->vmm.s.pR0LoggerR0)
2162	pVCpu->vmm.s.pR0LoggerR0->fFlushingDisabled = false;
2163	}
2164
2165
2166	/**
2167	* Checks if log flushing is disabled or not.
2168	*
2169	* @param pVCpu The cross context virtual CPU structure.
2170	*/
2171	VMMR0_INT_DECL(bool) VMMR0IsLogFlushDisabled(PVMCPU pVCpu)
2172	{
2173	if (pVCpu->vmm.s.pR0LoggerR0)
2174	return pVCpu->vmm.s.pR0LoggerR0->fFlushingDisabled;
2175	return true;
2176	}
2177	#endif /* LOG_ENABLED */
2178
2179	/**
2180	* Jump back to ring-3 if we're the EMT and the longjmp is armed.
2181	*
2182	* @returns true if the breakpoint should be hit, false if it should be ignored.
2183	*/
2184	DECLEXPORT(bool) RTCALL RTAssertShouldPanic(void)
2185	{
2186	#if 0
2187	return true;
2188	#else
2189	PVM pVM = GVMMR0GetVMByEMT(NIL_RTNATIVETHREAD);
2190	if (pVM)
2191	{
2192	PVMCPU pVCpu = VMMGetCpu(pVM);
2193
2194	if (pVCpu)
2195	{
2196	#ifdef RT_ARCH_X86
2197	if ( pVCpu->vmm.s.CallRing3JmpBufR0.eip
2198	&& !pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call)
2199	#else
2200	if ( pVCpu->vmm.s.CallRing3JmpBufR0.rip
2201	&& !pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call)
2202	#endif
2203	{
2204	int rc = VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_VM_R0_ASSERTION, 0);
2205	return RT_FAILURE_NP(rc);
2206	}
2207	}
2208	}
2209	#ifdef RT_OS_LINUX
2210	return true;
2211	#else
2212	return false;
2213	#endif
2214	#endif
2215	}
2216
2217
2218	/**
2219	* Override this so we can push it up to ring-3.
2220	*
2221	* @param pszExpr Expression. Can be NULL.
2222	* @param uLine Location line number.
2223	* @param pszFile Location file name.
2224	* @param pszFunction Location function name.
2225	*/
2226	DECLEXPORT(void) RTCALL RTAssertMsg1Weak(const char pszExpr, unsigned uLine, const char pszFile, const char *pszFunction)
2227	{
2228	/*
2229	* To the log.
2230	*/
2231	LogAlways(("\n!!R0-Assertion Failed!!\n"
2232	"Expression: %s\n"
2233	"Location : %s(%d) %s\n",
2234	pszExpr, pszFile, uLine, pszFunction));
2235
2236	/*
2237	* To the global VMM buffer.
2238	*/
2239	PVM pVM = GVMMR0GetVMByEMT(NIL_RTNATIVETHREAD);
2240	if (pVM)
2241	RTStrPrintf(pVM->vmm.s.szRing0AssertMsg1, sizeof(pVM->vmm.s.szRing0AssertMsg1),
2242	"\n!!R0-Assertion Failed!!\n"
2243	"Expression: %s\n"
2244	"Location : %s(%d) %s\n",
2245	pszExpr, pszFile, uLine, pszFunction);
2246
2247	/*
2248	* Continue the normal way.
2249	*/
2250	RTAssertMsg1(pszExpr, uLine, pszFile, pszFunction);
2251	}
2252
2253
2254	/**
2255	* Callback for RTLogFormatV which writes to the ring-3 log port.
2256	* See PFNLOGOUTPUT() for details.
2257	*/
2258	static DECLCALLBACK(size_t) rtLogOutput(void pv, const char pachChars, size_t cbChars)
2259	{
2260	for (size_t i = 0; i < cbChars; i++)
2261	{
2262	LogAlways(("%c", pachChars[i])); NOREF(pachChars);
2263	}
2264
2265	NOREF(pv);
2266	return cbChars;
2267	}
2268
2269
2270	/**
2271	* Override this so we can push it up to ring-3.
2272	*
2273	* @param pszFormat The format string.
2274	* @param va Arguments.
2275	*/
2276	DECLEXPORT(void) RTCALL RTAssertMsg2WeakV(const char *pszFormat, va_list va)
2277	{
2278	va_list vaCopy;
2279
2280	/*
2281	* Push the message to the loggers.
2282	*/
2283	PRTLOGGER pLog = RTLogGetDefaultInstance(); /* Don't initialize it here... */
2284	if (pLog)
2285	{
2286	va_copy(vaCopy, va);
2287	RTLogFormatV(rtLogOutput, pLog, pszFormat, vaCopy);
2288	va_end(vaCopy);
2289	}
2290	pLog = RTLogRelGetDefaultInstance();
2291	if (pLog)
2292	{
2293	va_copy(vaCopy, va);
2294	RTLogFormatV(rtLogOutput, pLog, pszFormat, vaCopy);
2295	va_end(vaCopy);
2296	}
2297
2298	/*
2299	* Push it to the global VMM buffer.
2300	*/
2301	PVM pVM = GVMMR0GetVMByEMT(NIL_RTNATIVETHREAD);
2302	if (pVM)
2303	{
2304	va_copy(vaCopy, va);
2305	RTStrPrintfV(pVM->vmm.s.szRing0AssertMsg2, sizeof(pVM->vmm.s.szRing0AssertMsg2), pszFormat, vaCopy);
2306	va_end(vaCopy);
2307	}
2308
2309	/*
2310	* Continue the normal way.
2311	*/
2312	RTAssertMsg2V(pszFormat, va);
2313	}
2314

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/VMMR0.cpp@ 61605

Download in other formats: