VMMR0.cpp@ 36329

Last change on this file since 36329 was 36329, checked in by vboxsync, 14 years ago
PCI: per-VM initialization (for IOMMU domains)
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 52.9 KB

Line
1	/* $Id: VMMR0.cpp 36329 2011-03-21 16:47:48Z vboxsync $ */
2	/** @file
3	* VMM - Host Context Ring 0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2010 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_VMM
22	#include <VBox/vmm/vmm.h>
23	#include <VBox/sup.h>
24	#include <VBox/vmm/trpm.h>
25	#include <VBox/vmm/cpum.h>
26	#include <VBox/vmm/pdmapi.h>
27	#include <VBox/vmm/pgm.h>
28	#include <VBox/vmm/stam.h>
29	#include <VBox/vmm/tm.h>
30	#include "VMMInternal.h"
31	#include <VBox/vmm/vm.h>
32	#ifdef VBOX_WITH_PCI_PASSTHROUGH
33	# include <VBox/vmm/pdmpci.h>
34	#endif
35
36	#include <VBox/vmm/gvmm.h>
37	#include <VBox/vmm/gmm.h>
38	#include <VBox/intnet.h>
39	#include <VBox/vmm/hwaccm.h>
40	#include <VBox/param.h>
41	#include <VBox/err.h>
42	#include <VBox/version.h>
43	#include <VBox/log.h>
44
45	#include <iprt/asm-amd64-x86.h>
46	#include <iprt/assert.h>
47	#include <iprt/crc.h>
48	#include <iprt/mp.h>
49	#include <iprt/once.h>
50	#include <iprt/stdarg.h>
51	#include <iprt/string.h>
52	#include <iprt/thread.h>
53	#include <iprt/timer.h>
54
55	#if defined(_MSC_VER) && defined(RT_ARCH_AMD64) /** @todo check this with with VC7! */
56	# pragma intrinsic(_AddressOfReturnAddress)
57	#endif
58
59
60	/*******************************************************************************
61	* Internal Functions *
62	*******************************************************************************/
63	RT_C_DECLS_BEGIN
64	VMMR0DECL(int) ModuleInit(void);
65	VMMR0DECL(void) ModuleTerm(void);
66
67	#if defined(RT_ARCH_X86) && (defined(RT_OS_SOLARIS) \|\| defined(RT_OS_FREEBSD))
68	extern uint64_t __udivdi3(uint64_t, uint64_t);
69	extern uint64_t __umoddi3(uint64_t, uint64_t);
70	#endif // RT_ARCH_X86 && (RT_OS_SOLARIS \|\| RT_OS_FREEBSD)
71	RT_C_DECLS_END
72
73
74	/*******************************************************************************
75	* Global Variables *
76	*******************************************************************************/
77	/** Drag in necessary library bits.
78	* The runtime lives here (in VMMR0.r0) and VBoxDDR0.r0 links against us. /
79	PFNRT g_VMMGCDeps[] =
80	{
81	(PFNRT)RTCrc32,
82	(PFNRT)RTOnce,
83	#if defined(RT_ARCH_X86) && (defined(RT_OS_SOLARIS) \|\| defined(RT_OS_FREEBSD))
84	(PFNRT)__udivdi3,
85	(PFNRT)__umoddi3,
86	#endif // RT_ARCH_X86 && (RT_OS_SOLARIS \|\| RT_OS_FREEBSD)
87	NULL
88	};
89
90
91	#if defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
92	/* Increase the size of the image to work around the refusal of Win64 to
93	* load images in the 0x80000 range.
94	*/
95	static uint64_t u64BloatImage[8192] = {0};
96	#endif
97
98	/**
99	* Initialize the module.
100	* This is called when we're first loaded.
101	*
102	* @returns 0 on success.
103	* @returns VBox status on failure.
104	*/
105	VMMR0DECL(int) ModuleInit(void)
106	{
107	LogFlow(("ModuleInit:\n"));
108
109	/*
110	* Initialize the GVMM, GMM, HWACCM, PGM (Darwin) and INTNET.
111	*/
112	int rc = GVMMR0Init();
113	if (RT_SUCCESS(rc))
114	{
115	rc = GMMR0Init();
116	if (RT_SUCCESS(rc))
117	{
118	rc = HWACCMR0Init();
119	if (RT_SUCCESS(rc))
120	{
121	rc = PGMRegisterStringFormatTypes();
122	if (RT_SUCCESS(rc))
123	{
124	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
125	rc = PGMR0DynMapInit();
126	#endif
127	if (RT_SUCCESS(rc))
128	{
129	rc = IntNetR0Init();
130	if (RT_SUCCESS(rc))
131	{
132	#ifdef VBOX_WITH_PCI_PASSTHROUGH
133	rc = PciRawR0Init();
134	#endif
135	if (RT_SUCCESS(rc))
136	{
137	rc = CPUMR0ModuleInit();
138	if (RT_SUCCESS(rc))
139	{
140	LogFlow(("ModuleInit: returns success.\n"));
141	return VINF_SUCCESS;
142	}
143
144	/*
145	* Bail out.
146	*/
147	#ifdef VBOX_WITH_PCI_PASSTHROUGH
148	PciRawR0Term();
149	#endif
150	}
151	IntNetR0Term();
152	}
153	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
154	PGMR0DynMapTerm();
155	#endif
156	}
157	PGMDeregisterStringFormatTypes();
158	}
159	HWACCMR0Term();
160	}
161	GMMR0Term();
162	}
163	GVMMR0Term();
164	}
165
166	LogFlow(("ModuleInit: failed %Rrc\n", rc));
167	return rc;
168	}
169
170
171	/**
172	* Terminate the module.
173	* This is called when we're finally unloaded.
174	*/
175	VMMR0DECL(void) ModuleTerm(void)
176	{
177	LogFlow(("ModuleTerm:\n"));
178
179	/*
180	* Terminate the CPUM module (Local APIC cleanup).
181	*/
182	CPUMR0ModuleTerm();
183
184	/*
185	* Terminate the internal network service.
186	*/
187	IntNetR0Term();
188
189	/*
190	* PGM (Darwin), HWACCM and PciRaw global cleanup.
191	*/
192	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
193	PGMR0DynMapTerm();
194	#endif
195	#ifdef VBOX_WITH_PCI_PASSTHROUGH
196	PciRawR0Term();
197	#endif
198	PGMDeregisterStringFormatTypes();
199	HWACCMR0Term();
200
201	/*
202	* Destroy the GMM and GVMM instances.
203	*/
204	GMMR0Term();
205	GVMMR0Term();
206
207	LogFlow(("ModuleTerm: returns\n"));
208	}
209
210
211	/**
212	* Initiates the R0 driver for a particular VM instance.
213	*
214	* @returns VBox status code.
215	*
216	* @param pVM The VM instance in question.
217	* @param uSvnRev The SVN revision of the ring-3 part.
218	* @thread EMT.
219	*/
220	static int vmmR0InitVM(PVM pVM, uint32_t uSvnRev)
221	{
222	/*
223	* Match the SVN revisions.
224	*/
225	if (uSvnRev != VMMGetSvnRev())
226	{
227	LogRel(("VMMR0InitVM: Revision mismatch, r3=%d r0=%d\n", uSvnRev, VMMGetSvnRev()));
228	SUPR0Printf("VMMR0InitVM: Revision mismatch, r3=%d r0=%d\n", uSvnRev, VMMGetSvnRev());
229	return VERR_VMM_R0_VERSION_MISMATCH;
230	}
231	if ( !VALID_PTR(pVM)
232	\|\| pVM->pVMR0 != pVM)
233	return VERR_INVALID_PARAMETER;
234
235	#ifdef LOG_ENABLED
236	/*
237	* Register the EMT R0 logger instance for VCPU 0.
238	*/
239	PVMCPU pVCpu = &pVM->aCpus[0];
240
241	PVMMR0LOGGER pR0Logger = pVCpu->vmm.s.pR0LoggerR0;
242	if (pR0Logger)
243	{
244	# if 0 /* testing of the logger. */
245	LogCom(("vmmR0InitVM: before %p\n", RTLogDefaultInstance()));
246	LogCom(("vmmR0InitVM: pfnFlush=%p actual=%p\n", pR0Logger->Logger.pfnFlush, vmmR0LoggerFlush));
247	LogCom(("vmmR0InitVM: pfnLogger=%p actual=%p\n", pR0Logger->Logger.pfnLogger, vmmR0LoggerWrapper));
248	LogCom(("vmmR0InitVM: offScratch=%d fFlags=%#x fDestFlags=%#x\n", pR0Logger->Logger.offScratch, pR0Logger->Logger.fFlags, pR0Logger->Logger.fDestFlags));
249
250	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pVM->pSession);
251	LogCom(("vmmR0InitVM: after %p reg\n", RTLogDefaultInstance()));
252	RTLogSetDefaultInstanceThread(NULL, pVM->pSession);
253	LogCom(("vmmR0InitVM: after %p dereg\n", RTLogDefaultInstance()));
254
255	pR0Logger->Logger.pfnLogger("hello ring-0 logger\n");
256	LogCom(("vmmR0InitVM: returned successfully from direct logger call.\n"));
257	pR0Logger->Logger.pfnFlush(&pR0Logger->Logger);
258	LogCom(("vmmR0InitVM: returned successfully from direct flush call.\n"));
259
260	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pVM->pSession);
261	LogCom(("vmmR0InitVM: after %p reg2\n", RTLogDefaultInstance()));
262	pR0Logger->Logger.pfnLogger("hello ring-0 logger\n");
263	LogCom(("vmmR0InitVM: returned successfully from direct logger call (2). offScratch=%d\n", pR0Logger->Logger.offScratch));
264	RTLogSetDefaultInstanceThread(NULL, pVM->pSession);
265	LogCom(("vmmR0InitVM: after %p dereg2\n", RTLogDefaultInstance()));
266
267	RTLogLoggerEx(&pR0Logger->Logger, 0, ~0U, "hello ring-0 logger (RTLogLoggerEx)\n");
268	LogCom(("vmmR0InitVM: RTLogLoggerEx returned fine offScratch=%d\n", pR0Logger->Logger.offScratch));
269
270	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pVM->pSession);
271	RTLogPrintf("hello ring-0 logger (RTLogPrintf)\n");
272	LogCom(("vmmR0InitVM: RTLogPrintf returned fine offScratch=%d\n", pR0Logger->Logger.offScratch));
273	# endif
274	Log(("Switching to per-thread logging instance %p (key=%p)\n", &pR0Logger->Logger, pVM->pSession));
275	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pVM->pSession);
276	pR0Logger->fRegistered = true;
277	}
278	#endif /* LOG_ENABLED */
279
280	/*
281	* Check if the host supports high resolution timers or not.
282	*/
283	if ( pVM->vmm.s.fUsePeriodicPreemptionTimers
284	&& !RTTimerCanDoHighResolution())
285	pVM->vmm.s.fUsePeriodicPreemptionTimers = false;
286
287	/*
288	* Initialize the per VM data for GVMM and GMM.
289	*/
290	int rc = GVMMR0InitVM(pVM);
291	// if (RT_SUCCESS(rc))
292	// rc = GMMR0InitPerVMData(pVM);
293	if (RT_SUCCESS(rc))
294	{
295	/*
296	* Init HWACCM, CPUM and PGM (Darwin only).
297	*/
298	rc = HWACCMR0InitVM(pVM);
299	if (RT_SUCCESS(rc))
300	{
301	rc = CPUMR0Init(pVM); /** @todo rename to CPUMR0InitVM */
302	if (RT_SUCCESS(rc))
303	{
304	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
305	rc = PGMR0DynMapInitVM(pVM);
306	#endif
307	if (RT_SUCCESS(rc))
308	{
309	#ifdef VBOX_WITH_PCI_PASSTHROUGH
310	rc = PciRawR0InitVM(pVM);
311	#endif
312	if (RT_SUCCESS(rc))
313	{
314	GVMMR0DoneInitVM(pVM);
315	return rc;
316	}
317	}
318
319	/* bail out */
320	}
321	#ifdef VBOX_WITH_PCI_PASSTHROUGH
322	PciRawR0TermVM(pVM);
323	#endif
324	HWACCMR0TermVM(pVM);
325	}
326	}
327
328
329	RTLogSetDefaultInstanceThread(NULL, (uintptr_t)pVM->pSession);
330	return rc;
331	}
332
333
334	/**
335	* Terminates the R0 driver for a particular VM instance.
336	*
337	* This is normally called by ring-3 as part of the VM termination process, but
338	* may alternatively be called during the support driver session cleanup when
339	* the VM object is destroyed (see GVMM).
340	*
341	* @returns VBox status code.
342	*
343	* @param pVM The VM instance in question.
344	* @param pGVM Pointer to the global VM structure. Optional.
345	* @thread EMT or session clean up thread.
346	*/
347	VMMR0DECL(int) VMMR0TermVM(PVM pVM, PGVM pGVM)
348	{
349	#ifdef VBOX_WITH_PCI_PASSTHROUGH
350	PciRawR0TermVM(pVM);
351	#endif
352
353	/*
354	* Tell GVMM what we're up to and check that we only do this once.
355	*/
356	if (GVMMR0DoingTermVM(pVM, pGVM))
357	{
358	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
359	PGMR0DynMapTermVM(pVM);
360	#endif
361	HWACCMR0TermVM(pVM);
362	}
363
364	/*
365	* Deregister the logger.
366	*/
367	RTLogSetDefaultInstanceThread(NULL, (uintptr_t)pVM->pSession);
368	return VINF_SUCCESS;
369	}
370
371
372	#ifdef VBOX_WITH_STATISTICS
373	/**
374	* Record return code statistics
375	* @param pVM The VM handle.
376	* @param pVCpu The VMCPU handle.
377	* @param rc The status code.
378	*/
379	static void vmmR0RecordRC(PVM pVM, PVMCPU pVCpu, int rc)
380	{
381	/*
382	* Collect statistics.
383	*/
384	switch (rc)
385	{
386	case VINF_SUCCESS:
387	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetNormal);
388	break;
389	case VINF_EM_RAW_INTERRUPT:
390	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetInterrupt);
391	break;
392	case VINF_EM_RAW_INTERRUPT_HYPER:
393	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetInterruptHyper);
394	break;
395	case VINF_EM_RAW_GUEST_TRAP:
396	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetGuestTrap);
397	break;
398	case VINF_EM_RAW_RING_SWITCH:
399	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetRingSwitch);
400	break;
401	case VINF_EM_RAW_RING_SWITCH_INT:
402	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetRingSwitchInt);
403	break;
404	case VINF_EM_RAW_STALE_SELECTOR:
405	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetStaleSelector);
406	break;
407	case VINF_EM_RAW_IRET_TRAP:
408	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIRETTrap);
409	break;
410	case VINF_IOM_HC_IOPORT_READ:
411	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIORead);
412	break;
413	case VINF_IOM_HC_IOPORT_WRITE:
414	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIOWrite);
415	break;
416	case VINF_IOM_HC_MMIO_READ:
417	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIORead);
418	break;
419	case VINF_IOM_HC_MMIO_WRITE:
420	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOWrite);
421	break;
422	case VINF_IOM_HC_MMIO_READ_WRITE:
423	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOReadWrite);
424	break;
425	case VINF_PATM_HC_MMIO_PATCH_READ:
426	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOPatchRead);
427	break;
428	case VINF_PATM_HC_MMIO_PATCH_WRITE:
429	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOPatchWrite);
430	break;
431	case VINF_EM_RAW_EMULATE_INSTR:
432	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetEmulate);
433	break;
434	case VINF_EM_RAW_EMULATE_IO_BLOCK:
435	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIOBlockEmulate);
436	break;
437	case VINF_PATCH_EMULATE_INSTR:
438	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchEmulate);
439	break;
440	case VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT:
441	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetLDTFault);
442	break;
443	case VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT:
444	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetGDTFault);
445	break;
446	case VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT:
447	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIDTFault);
448	break;
449	case VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT:
450	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetTSSFault);
451	break;
452	case VINF_EM_RAW_EMULATE_INSTR_PD_FAULT:
453	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPDFault);
454	break;
455	case VINF_CSAM_PENDING_ACTION:
456	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetCSAMTask);
457	break;
458	case VINF_PGM_SYNC_CR3:
459	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetSyncCR3);
460	break;
461	case VINF_PATM_PATCH_INT3:
462	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchInt3);
463	break;
464	case VINF_PATM_PATCH_TRAP_PF:
465	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchPF);
466	break;
467	case VINF_PATM_PATCH_TRAP_GP:
468	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchGP);
469	break;
470	case VINF_PATM_PENDING_IRQ_AFTER_IRET:
471	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchIretIRQ);
472	break;
473	case VINF_EM_RESCHEDULE_REM:
474	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetRescheduleREM);
475	break;
476	case VINF_EM_RAW_TO_R3:
477	if (VM_FF_ISPENDING(pVM, VM_FF_TM_VIRTUAL_SYNC))
478	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3TMVirt);
479	else
480	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NEED_HANDY_PAGES))
481	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3HandyPages);
482	else
483	if (VM_FF_ISPENDING(pVM, VM_FF_PDM_QUEUES))
484	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3PDMQueues);
485	else
486	if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
487	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Rendezvous);
488	else
489	if (VM_FF_ISPENDING(pVM, VM_FF_PDM_DMA))
490	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3DMA);
491	else
492	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TIMER))
493	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Timer);
494	else
495	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PDM_CRITSECT))
496	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3CritSect);
497	else
498	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TO_R3))
499	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3);
500	else
501	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Unknown);
502	break;
503
504	case VINF_EM_RAW_TIMER_PENDING:
505	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetTimerPending);
506	break;
507	case VINF_EM_RAW_INTERRUPT_PENDING:
508	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetInterruptPending);
509	break;
510	case VINF_VMM_CALL_HOST:
511	switch (pVCpu->vmm.s.enmCallRing3Operation)
512	{
513	case VMMCALLRING3_PDM_LOCK:
514	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPDMLock);
515	break;
516	case VMMCALLRING3_PGM_POOL_GROW:
517	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPGMPoolGrow);
518	break;
519	case VMMCALLRING3_PGM_LOCK:
520	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPGMLock);
521	break;
522	case VMMCALLRING3_PGM_MAP_CHUNK:
523	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPGMMapChunk);
524	break;
525	case VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES:
526	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallPGMAllocHandy);
527	break;
528	case VMMCALLRING3_REM_REPLAY_HANDLER_NOTIFICATIONS:
529	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallRemReplay);
530	break;
531	case VMMCALLRING3_VMM_LOGGER_FLUSH:
532	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallLogFlush);
533	break;
534	case VMMCALLRING3_VM_SET_ERROR:
535	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallVMSetError);
536	break;
537	case VMMCALLRING3_VM_SET_RUNTIME_ERROR:
538	STAM_COUNTER_INC(&pVM->vmm.s.StatRZCallVMSetRuntimeError);
539	break;
540	case VMMCALLRING3_VM_R0_ASSERTION:
541	default:
542	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetCallRing3);
543	break;
544	}
545	break;
546	case VINF_PATM_DUPLICATE_FUNCTION:
547	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPATMDuplicateFn);
548	break;
549	case VINF_PGM_CHANGE_MODE:
550	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPGMChangeMode);
551	break;
552	case VINF_PGM_POOL_FLUSH_PENDING:
553	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPGMFlushPending);
554	break;
555	case VINF_EM_PENDING_REQUEST:
556	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPendingRequest);
557	break;
558	case VINF_EM_HWACCM_PATCH_TPR_INSTR:
559	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchTPR);
560	break;
561	default:
562	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMisc);
563	break;
564	}
565	}
566	#endif /* VBOX_WITH_STATISTICS */
567
568
569	/**
570	* Unused ring-0 entry point that used to be called from the interrupt gate.
571	*
572	* Will be removed one of the next times we do a major SUPDrv version bump.
573	*
574	* @returns VBox status code.
575	* @param pVM The VM to operate on.
576	* @param enmOperation Which operation to execute.
577	* @param pvArg Argument to the operation.
578	* @remarks Assume called with interrupts disabled.
579	*/
580	VMMR0DECL(int) VMMR0EntryInt(PVM pVM, VMMR0OPERATION enmOperation, void *pvArg)
581	{
582	/*
583	* We're returning VERR_NOT_SUPPORT here so we've got something else
584	* than -1 which the interrupt gate glue code might return.
585	*/
586	Log(("operation %#x is not supported\n", enmOperation));
587	return VERR_NOT_SUPPORTED;
588	}
589
590
591	/**
592	* The Ring 0 entry point, called by the fast-ioctl path.
593	*
594	* @param pVM The VM to operate on.
595	* The return code is stored in pVM->vmm.s.iLastGZRc.
596	* @param idCpu The Virtual CPU ID of the calling EMT.
597	* @param enmOperation Which operation to execute.
598	* @remarks Assume called with interrupts _enabled_.
599	*/
600	VMMR0DECL(void) VMMR0EntryFast(PVM pVM, VMCPUID idCpu, VMMR0OPERATION enmOperation)
601	{
602	if (RT_UNLIKELY(idCpu >= pVM->cCpus))
603	return;
604	PVMCPU pVCpu = &pVM->aCpus[idCpu];
605
606	switch (enmOperation)
607	{
608	/*
609	* Switch to GC and run guest raw mode code.
610	* Disable interrupts before doing the world switch.
611	*/
612	case VMMR0_DO_RAW_RUN:
613	{
614	/* Some safety precautions first. */
615	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
616	if (RT_LIKELY( !pVM->vmm.s.fSwitcherDisabled /* hwaccm */
617	&& pVM->cCpus == 1 /* !smp */
618	&& PGMGetHyperCR3(pVCpu)))
619	#else
620	if (RT_LIKELY( !pVM->vmm.s.fSwitcherDisabled
621	&& pVM->cCpus == 1))
622	#endif
623	{
624	/* Disable preemption and update the periodic preemption timer. */
625	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
626	RTThreadPreemptDisable(&PreemptState);
627	RTCPUID idHostCpu = RTMpCpuId();
628	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
629	CPUMR0SetLApic(pVM, idHostCpu);
630	#endif
631	ASMAtomicWriteU32(&pVCpu->idHostCpu, idHostCpu);
632	if (pVM->vmm.s.fUsePeriodicPreemptionTimers)
633	GVMMR0SchedUpdatePeriodicPreemptionTimer(pVM, pVCpu->idHostCpu, TMCalcHostTimerFrequency(pVM, pVCpu));
634
635	/* We might need to disable VT-x if the active switcher turns off paging. */
636	bool fVTxDisabled;
637	int rc = HWACCMR0EnterSwitcher(pVM, &fVTxDisabled);
638	if (RT_SUCCESS(rc))
639	{
640	RTCCUINTREG uFlags = ASMIntDisableFlags();
641	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
642
643	TMNotifyStartOfExecution(pVCpu);
644	rc = pVM->vmm.s.pfnHostToGuestR0(pVM);
645	pVCpu->vmm.s.iLastGZRc = rc;
646	TMNotifyEndOfExecution(pVCpu);
647
648	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
649
650	/* Re-enable VT-x if previously turned off. */
651	HWACCMR0LeaveSwitcher(pVM, fVTxDisabled);
652
653	if ( rc == VINF_EM_RAW_INTERRUPT
654	\|\| rc == VINF_EM_RAW_INTERRUPT_HYPER)
655	TRPMR0DispatchHostInterrupt(pVM);
656
657	ASMSetFlags(uFlags);
658
659	#ifdef VBOX_WITH_STATISTICS
660	STAM_COUNTER_INC(&pVM->vmm.s.StatRunRC);
661	vmmR0RecordRC(pVM, pVCpu, rc);
662	#endif
663	}
664	else
665	pVCpu->vmm.s.iLastGZRc = rc;
666	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
667	RTThreadPreemptRestore(&PreemptState);
668	}
669	else
670	{
671	Assert(!pVM->vmm.s.fSwitcherDisabled);
672	pVCpu->vmm.s.iLastGZRc = VERR_NOT_SUPPORTED;
673	if (pVM->cCpus != 1)
674	pVCpu->vmm.s.iLastGZRc = VERR_RAW_MODE_INVALID_SMP;
675	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
676	if (!PGMGetHyperCR3(pVCpu))
677	pVCpu->vmm.s.iLastGZRc = VERR_PGM_NO_CR3_SHADOW_ROOT;
678	#endif
679	}
680	break;
681	}
682
683	/*
684	* Run guest code using the available hardware acceleration technology.
685	*
686	* Disable interrupts before we do anything interesting. On Windows we avoid
687	* this by having the support driver raise the IRQL before calling us, this way
688	* we hope to get away with page faults and later calling into the kernel.
689	*/
690	case VMMR0_DO_HWACC_RUN:
691	{
692	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
693	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
694	RTThreadPreemptDisable(&PreemptState);
695	#elif !defined(RT_OS_WINDOWS)
696	RTCCUINTREG uFlags = ASMIntDisableFlags();
697	#endif
698	ASMAtomicWriteU32(&pVCpu->idHostCpu, RTMpCpuId());
699	if (pVM->vmm.s.fUsePeriodicPreemptionTimers)
700	GVMMR0SchedUpdatePeriodicPreemptionTimer(pVM, pVCpu->idHostCpu, TMCalcHostTimerFrequency(pVM, pVCpu));
701
702	#ifdef LOG_ENABLED
703	if (pVCpu->idCpu > 0)
704	{
705	/* Lazy registration of ring 0 loggers. */
706	PVMMR0LOGGER pR0Logger = pVCpu->vmm.s.pR0LoggerR0;
707	if ( pR0Logger
708	&& !pR0Logger->fRegistered)
709	{
710	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pVM->pSession);
711	pR0Logger->fRegistered = true;
712	}
713	}
714	#endif
715	int rc;
716	if (!HWACCMR0SuspendPending())
717	{
718	rc = HWACCMR0Enter(pVM, pVCpu);
719	if (RT_SUCCESS(rc))
720	{
721	rc = vmmR0CallRing3SetJmp(&pVCpu->vmm.s.CallRing3JmpBufR0, HWACCMR0RunGuestCode, pVM, pVCpu); /* this may resume code. */
722	int rc2 = HWACCMR0Leave(pVM, pVCpu);
723	AssertRC(rc2);
724	}
725	STAM_COUNTER_INC(&pVM->vmm.s.StatRunRC);
726	}
727	else
728	{
729	/* System is about to go into suspend mode; go back to ring 3. */
730	rc = VINF_EM_RAW_INTERRUPT;
731	}
732	pVCpu->vmm.s.iLastGZRc = rc;
733
734	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
735	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
736	RTThreadPreemptRestore(&PreemptState);
737	#elif !defined(RT_OS_WINDOWS)
738	ASMSetFlags(uFlags);
739	#endif
740
741	#ifdef VBOX_WITH_STATISTICS
742	vmmR0RecordRC(pVM, pVCpu, rc);
743	#endif
744	/* No special action required for external interrupts, just return. */
745	break;
746	}
747
748	/*
749	* For profiling.
750	*/
751	case VMMR0_DO_NOP:
752	pVCpu->vmm.s.iLastGZRc = VINF_SUCCESS;
753	break;
754
755	/*
756	* Impossible.
757	*/
758	default:
759	AssertMsgFailed(("%#x\n", enmOperation));
760	pVCpu->vmm.s.iLastGZRc = VERR_NOT_SUPPORTED;
761	break;
762	}
763	}
764
765
766	/**
767	* Validates a session or VM session argument.
768	*
769	* @returns true / false accordingly.
770	* @param pVM The VM argument.
771	* @param pSession The session argument.
772	*/
773	DECLINLINE(bool) vmmR0IsValidSession(PVM pVM, PSUPDRVSESSION pClaimedSession, PSUPDRVSESSION pSession)
774	{
775	/* This must be set! */
776	if (!pSession)
777	return false;
778
779	/* Only one out of the two. */
780	if (pVM && pClaimedSession)
781	return false;
782	if (pVM)
783	pClaimedSession = pVM->pSession;
784	return pClaimedSession == pSession;
785	}
786
787
788	/**
789	* VMMR0EntryEx worker function, either called directly or when ever possible
790	* called thru a longjmp so we can exit safely on failure.
791	*
792	* @returns VBox status code.
793	* @param pVM The VM to operate on.
794	* @param idCpu Virtual CPU ID argument. Must be NIL_VMCPUID if pVM
795	* is NIL_RTR0PTR, and may be NIL_VMCPUID if it isn't
796	* @param enmOperation Which operation to execute.
797	* @param pReqHdr This points to a SUPVMMR0REQHDR packet. Optional.
798	* The support driver validates this if it's present.
799	* @param u64Arg Some simple constant argument.
800	* @param pSession The session of the caller.
801	* @remarks Assume called with interrupts _enabled_.
802	*/
803	static int vmmR0EntryExWorker(PVM pVM, VMCPUID idCpu, VMMR0OPERATION enmOperation, PSUPVMMR0REQHDR pReqHdr, uint64_t u64Arg, PSUPDRVSESSION pSession)
804	{
805	/*
806	* Common VM pointer validation.
807	*/
808	if (pVM)
809	{
810	if (RT_UNLIKELY( !VALID_PTR(pVM)
811	\|\| ((uintptr_t)pVM & PAGE_OFFSET_MASK)))
812	{
813	SUPR0Printf("vmmR0EntryExWorker: Invalid pVM=%p! (op=%d)\n", pVM, enmOperation);
814	return VERR_INVALID_POINTER;
815	}
816	if (RT_UNLIKELY( pVM->enmVMState < VMSTATE_CREATING
817	\|\| pVM->enmVMState > VMSTATE_TERMINATED
818	\|\| pVM->pVMR0 != pVM))
819	{
820	SUPR0Printf("vmmR0EntryExWorker: Invalid pVM=%p:{enmVMState=%d, .pVMR0=%p}! (op=%d)\n",
821	pVM, pVM->enmVMState, pVM->pVMR0, enmOperation);
822	return VERR_INVALID_POINTER;
823	}
824
825	if (RT_UNLIKELY(idCpu >= pVM->cCpus && idCpu != NIL_VMCPUID))
826	{
827	SUPR0Printf("vmmR0EntryExWorker: Invalid idCpu (%u vs cCpus=%u)\n", idCpu, pVM->cCpus);
828	return VERR_INVALID_PARAMETER;
829	}
830	}
831	else if (RT_UNLIKELY(idCpu != NIL_VMCPUID))
832	{
833	SUPR0Printf("vmmR0EntryExWorker: Invalid idCpu=%u\n", idCpu);
834	return VERR_INVALID_PARAMETER;
835	}
836
837
838	switch (enmOperation)
839	{
840	/*
841	* GVM requests
842	*/
843	case VMMR0_DO_GVMM_CREATE_VM:
844	if (pVM \|\| u64Arg \|\| idCpu != NIL_VMCPUID)
845	return VERR_INVALID_PARAMETER;
846	return GVMMR0CreateVMReq((PGVMMCREATEVMREQ)pReqHdr);
847
848	case VMMR0_DO_GVMM_DESTROY_VM:
849	if (pReqHdr \|\| u64Arg)
850	return VERR_INVALID_PARAMETER;
851	return GVMMR0DestroyVM(pVM);
852
853	case VMMR0_DO_GVMM_REGISTER_VMCPU:
854	{
855	if (!pVM)
856	return VERR_INVALID_PARAMETER;
857	return GVMMR0RegisterVCpu(pVM, idCpu);
858	}
859
860	case VMMR0_DO_GVMM_SCHED_HALT:
861	if (pReqHdr)
862	return VERR_INVALID_PARAMETER;
863	return GVMMR0SchedHalt(pVM, idCpu, u64Arg);
864
865	case VMMR0_DO_GVMM_SCHED_WAKE_UP:
866	if (pReqHdr \|\| u64Arg)
867	return VERR_INVALID_PARAMETER;
868	return GVMMR0SchedWakeUp(pVM, idCpu);
869
870	case VMMR0_DO_GVMM_SCHED_POKE:
871	if (pReqHdr \|\| u64Arg)
872	return VERR_INVALID_PARAMETER;
873	return GVMMR0SchedPoke(pVM, idCpu);
874
875	case VMMR0_DO_GVMM_SCHED_WAKE_UP_AND_POKE_CPUS:
876	if (u64Arg)
877	return VERR_INVALID_PARAMETER;
878	return GVMMR0SchedWakeUpAndPokeCpusReq(pVM, (PGVMMSCHEDWAKEUPANDPOKECPUSREQ)pReqHdr);
879
880	case VMMR0_DO_GVMM_SCHED_POLL:
881	if (pReqHdr \|\| u64Arg > 1)
882	return VERR_INVALID_PARAMETER;
883	return GVMMR0SchedPoll(pVM, idCpu, !!u64Arg);
884
885	case VMMR0_DO_GVMM_QUERY_STATISTICS:
886	if (u64Arg)
887	return VERR_INVALID_PARAMETER;
888	return GVMMR0QueryStatisticsReq(pVM, (PGVMMQUERYSTATISTICSSREQ)pReqHdr);
889
890	case VMMR0_DO_GVMM_RESET_STATISTICS:
891	if (u64Arg)
892	return VERR_INVALID_PARAMETER;
893	return GVMMR0ResetStatisticsReq(pVM, (PGVMMRESETSTATISTICSSREQ)pReqHdr);
894
895	/*
896	* Initialize the R0 part of a VM instance.
897	*/
898	case VMMR0_DO_VMMR0_INIT:
899	return vmmR0InitVM(pVM, (uint32_t)u64Arg);
900
901	/*
902	* Terminate the R0 part of a VM instance.
903	*/
904	case VMMR0_DO_VMMR0_TERM:
905	return VMMR0TermVM(pVM, NULL);
906
907	/*
908	* Attempt to enable hwacc mode and check the current setting.
909	*/
910	case VMMR0_DO_HWACC_ENABLE:
911	return HWACCMR0EnableAllCpus(pVM);
912
913	/*
914	* Setup the hardware accelerated session.
915	*/
916	case VMMR0_DO_HWACC_SETUP_VM:
917	{
918	RTCCUINTREG fFlags = ASMIntDisableFlags();
919	int rc = HWACCMR0SetupVM(pVM);
920	ASMSetFlags(fFlags);
921	return rc;
922	}
923
924	/*
925	* Switch to RC to execute Hypervisor function.
926	*/
927	case VMMR0_DO_CALL_HYPERVISOR:
928	{
929	int rc;
930	bool fVTxDisabled;
931
932	/* Safety precaution as HWACCM can disable the switcher. */
933	Assert(!pVM->vmm.s.fSwitcherDisabled);
934	if (RT_UNLIKELY(pVM->vmm.s.fSwitcherDisabled))
935	return VERR_NOT_SUPPORTED;
936
937	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
938	if (RT_UNLIKELY(!PGMGetHyperCR3(VMMGetCpu0(pVM))))
939	return VERR_PGM_NO_CR3_SHADOW_ROOT;
940	#endif
941
942	RTCCUINTREG fFlags = ASMIntDisableFlags();
943
944	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
945	RTCPUID idHostCpu = RTMpCpuId();
946	CPUMR0SetLApic(pVM, idHostCpu);
947	#endif
948
949	/* We might need to disable VT-x if the active switcher turns off paging. */
950	rc = HWACCMR0EnterSwitcher(pVM, &fVTxDisabled);
951	if (RT_FAILURE(rc))
952	return rc;
953
954	rc = pVM->vmm.s.pfnHostToGuestR0(pVM);
955
956	/* Re-enable VT-x if previously turned off. */
957	HWACCMR0LeaveSwitcher(pVM, fVTxDisabled);
958
959	/** @todo dispatch interrupts? */
960	ASMSetFlags(fFlags);
961	return rc;
962	}
963
964	/*
965	* PGM wrappers.
966	*/
967	case VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES:
968	if (idCpu == NIL_VMCPUID)
969	return VERR_INVALID_CPU_ID;
970	return PGMR0PhysAllocateHandyPages(pVM, &pVM->aCpus[idCpu]);
971
972	case VMMR0_DO_PGM_ALLOCATE_LARGE_HANDY_PAGE:
973	if (idCpu == NIL_VMCPUID)
974	return VERR_INVALID_CPU_ID;
975	return PGMR0PhysAllocateLargeHandyPage(pVM, &pVM->aCpus[idCpu]);
976
977	/*
978	* GMM wrappers.
979	*/
980	case VMMR0_DO_GMM_INITIAL_RESERVATION:
981	if (u64Arg)
982	return VERR_INVALID_PARAMETER;
983	return GMMR0InitialReservationReq(pVM, idCpu, (PGMMINITIALRESERVATIONREQ)pReqHdr);
984
985	case VMMR0_DO_GMM_UPDATE_RESERVATION:
986	if (u64Arg)
987	return VERR_INVALID_PARAMETER;
988	return GMMR0UpdateReservationReq(pVM, idCpu, (PGMMUPDATERESERVATIONREQ)pReqHdr);
989
990	case VMMR0_DO_GMM_ALLOCATE_PAGES:
991	if (u64Arg)
992	return VERR_INVALID_PARAMETER;
993	return GMMR0AllocatePagesReq(pVM, idCpu, (PGMMALLOCATEPAGESREQ)pReqHdr);
994
995	case VMMR0_DO_GMM_FREE_PAGES:
996	if (u64Arg)
997	return VERR_INVALID_PARAMETER;
998	return GMMR0FreePagesReq(pVM, idCpu, (PGMMFREEPAGESREQ)pReqHdr);
999
1000	case VMMR0_DO_GMM_FREE_LARGE_PAGE:
1001	if (u64Arg)
1002	return VERR_INVALID_PARAMETER;
1003	return GMMR0FreeLargePageReq(pVM, idCpu, (PGMMFREELARGEPAGEREQ)pReqHdr);
1004
1005	case VMMR0_DO_GMM_QUERY_HYPERVISOR_MEM_STATS:
1006	if (u64Arg)
1007	return VERR_INVALID_PARAMETER;
1008	return GMMR0QueryHypervisorMemoryStatsReq(pVM, (PGMMMEMSTATSREQ)pReqHdr);
1009
1010	case VMMR0_DO_GMM_QUERY_MEM_STATS:
1011	if (idCpu == NIL_VMCPUID)
1012	return VERR_INVALID_CPU_ID;
1013	if (u64Arg)
1014	return VERR_INVALID_PARAMETER;
1015	return GMMR0QueryMemoryStatsReq(pVM, idCpu, (PGMMMEMSTATSREQ)pReqHdr);
1016
1017	case VMMR0_DO_GMM_BALLOONED_PAGES:
1018	if (u64Arg)
1019	return VERR_INVALID_PARAMETER;
1020	return GMMR0BalloonedPagesReq(pVM, idCpu, (PGMMBALLOONEDPAGESREQ)pReqHdr);
1021
1022	case VMMR0_DO_GMM_MAP_UNMAP_CHUNK:
1023	if (u64Arg)
1024	return VERR_INVALID_PARAMETER;
1025	return GMMR0MapUnmapChunkReq(pVM, (PGMMMAPUNMAPCHUNKREQ)pReqHdr);
1026
1027	case VMMR0_DO_GMM_SEED_CHUNK:
1028	if (pReqHdr)
1029	return VERR_INVALID_PARAMETER;
1030	return GMMR0SeedChunk(pVM, idCpu, (RTR3PTR)u64Arg);
1031
1032	case VMMR0_DO_GMM_REGISTER_SHARED_MODULE:
1033	if (idCpu == NIL_VMCPUID)
1034	return VERR_INVALID_CPU_ID;
1035	if (u64Arg)
1036	return VERR_INVALID_PARAMETER;
1037	return GMMR0RegisterSharedModuleReq(pVM, idCpu, (PGMMREGISTERSHAREDMODULEREQ)pReqHdr);
1038
1039	case VMMR0_DO_GMM_UNREGISTER_SHARED_MODULE:
1040	if (idCpu == NIL_VMCPUID)
1041	return VERR_INVALID_CPU_ID;
1042	if (u64Arg)
1043	return VERR_INVALID_PARAMETER;
1044	return GMMR0UnregisterSharedModuleReq(pVM, idCpu, (PGMMUNREGISTERSHAREDMODULEREQ)pReqHdr);
1045
1046	case VMMR0_DO_GMM_RESET_SHARED_MODULES:
1047	if (idCpu == NIL_VMCPUID)
1048	return VERR_INVALID_CPU_ID;
1049	if ( u64Arg
1050	\|\| pReqHdr)
1051	return VERR_INVALID_PARAMETER;
1052	return GMMR0ResetSharedModules(pVM, idCpu);
1053
1054	#ifdef VBOX_WITH_PAGE_SHARING
1055	case VMMR0_DO_GMM_CHECK_SHARED_MODULES:
1056	{
1057	if (idCpu == NIL_VMCPUID)
1058	return VERR_INVALID_CPU_ID;
1059	if ( u64Arg
1060	\|\| pReqHdr)
1061	return VERR_INVALID_PARAMETER;
1062
1063	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1064	Assert(pVCpu->hNativeThreadR0 == RTThreadNativeSelf());
1065
1066	# ifdef DEBUG_sandervl
1067	/* Make sure that log flushes can jump back to ring-3; annoying to get an incomplete log (this is risky though as the code doesn't take this into account). */
1068	/* Todo: this can have bad side effects for unexpected jumps back to r3. */
1069	int rc = GMMR0CheckSharedModulesStart(pVM);
1070	if (rc == VINF_SUCCESS)
1071	{
1072	rc = vmmR0CallRing3SetJmp(&pVCpu->vmm.s.CallRing3JmpBufR0, GMMR0CheckSharedModules, pVM, pVCpu); /* this may resume code. */
1073	Assert( rc == VINF_SUCCESS
1074	\|\| (rc == VINF_VMM_CALL_HOST && pVCpu->vmm.s.enmCallRing3Operation == VMMCALLRING3_VMM_LOGGER_FLUSH));
1075	GMMR0CheckSharedModulesEnd(pVM);
1076	}
1077	# else
1078	int rc = GMMR0CheckSharedModules(pVM, pVCpu);
1079	# endif
1080	return rc;
1081	}
1082	#endif
1083
1084	#if defined(VBOX_STRICT) && HC_ARCH_BITS == 64
1085	case VMMR0_DO_GMM_FIND_DUPLICATE_PAGE:
1086	{
1087	if (u64Arg)
1088	return VERR_INVALID_PARAMETER;
1089	return GMMR0FindDuplicatePageReq(pVM, (PGMMFINDDUPLICATEPAGEREQ)pReqHdr);
1090	}
1091	#endif
1092
1093	/*
1094	* A quick GCFGM mock-up.
1095	*/
1096	/** @todo GCFGM with proper access control, ring-3 management interface and all that. */
1097	case VMMR0_DO_GCFGM_SET_VALUE:
1098	case VMMR0_DO_GCFGM_QUERY_VALUE:
1099	{
1100	if (pVM \|\| !pReqHdr \|\| u64Arg \|\| idCpu != NIL_VMCPUID)
1101	return VERR_INVALID_PARAMETER;
1102	PGCFGMVALUEREQ pReq = (PGCFGMVALUEREQ)pReqHdr;
1103	if (pReq->Hdr.cbReq != sizeof(*pReq))
1104	return VERR_INVALID_PARAMETER;
1105	int rc;
1106	if (enmOperation == VMMR0_DO_GCFGM_SET_VALUE)
1107	{
1108	rc = GVMMR0SetConfig(pReq->pSession, &pReq->szName[0], pReq->u64Value);
1109	//if (rc == VERR_CFGM_VALUE_NOT_FOUND)
1110	// rc = GMMR0SetConfig(pReq->pSession, &pReq->szName[0], pReq->u64Value);
1111	}
1112	else
1113	{
1114	rc = GVMMR0QueryConfig(pReq->pSession, &pReq->szName[0], &pReq->u64Value);
1115	//if (rc == VERR_CFGM_VALUE_NOT_FOUND)
1116	// rc = GMMR0QueryConfig(pReq->pSession, &pReq->szName[0], &pReq->u64Value);
1117	}
1118	return rc;
1119	}
1120
1121	/*
1122	* PDM Wrappers.
1123	*/
1124	case VMMR0_DO_PDM_DRIVER_CALL_REQ_HANDLER:
1125	{
1126	if (!pVM \|\| !pReqHdr \|\| u64Arg \|\| idCpu != NIL_VMCPUID)
1127	return VERR_INVALID_PARAMETER;
1128	return PDMR0DriverCallReqHandler(pVM, (PPDMDRIVERCALLREQHANDLERREQ)pReqHdr);
1129	}
1130
1131	case VMMR0_DO_PDM_DEVICE_CALL_REQ_HANDLER:
1132	{
1133	if (!pVM \|\| !pReqHdr \|\| u64Arg \|\| idCpu != NIL_VMCPUID)
1134	return VERR_INVALID_PARAMETER;
1135	return PDMR0DeviceCallReqHandler(pVM, (PPDMDEVICECALLREQHANDLERREQ)pReqHdr);
1136	}
1137
1138	/*
1139	* Requests to the internal networking service.
1140	*/
1141	case VMMR0_DO_INTNET_OPEN:
1142	{
1143	PINTNETOPENREQ pReq = (PINTNETOPENREQ)pReqHdr;
1144	if (u64Arg \|\| !pReq \|\| !vmmR0IsValidSession(pVM, pReq->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
1145	return VERR_INVALID_PARAMETER;
1146	return IntNetR0OpenReq(pSession, pReq);
1147	}
1148
1149	case VMMR0_DO_INTNET_IF_CLOSE:
1150	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFCLOSEREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
1151	return VERR_INVALID_PARAMETER;
1152	return IntNetR0IfCloseReq(pSession, (PINTNETIFCLOSEREQ)pReqHdr);
1153
1154	case VMMR0_DO_INTNET_IF_GET_BUFFER_PTRS:
1155	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFGETBUFFERPTRSREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
1156	return VERR_INVALID_PARAMETER;
1157	return IntNetR0IfGetBufferPtrsReq(pSession, (PINTNETIFGETBUFFERPTRSREQ)pReqHdr);
1158
1159	case VMMR0_DO_INTNET_IF_SET_PROMISCUOUS_MODE:
1160	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFSETPROMISCUOUSMODEREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
1161	return VERR_INVALID_PARAMETER;
1162	return IntNetR0IfSetPromiscuousModeReq(pSession, (PINTNETIFSETPROMISCUOUSMODEREQ)pReqHdr);
1163
1164	case VMMR0_DO_INTNET_IF_SET_MAC_ADDRESS:
1165	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFSETMACADDRESSREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
1166	return VERR_INVALID_PARAMETER;
1167	return IntNetR0IfSetMacAddressReq(pSession, (PINTNETIFSETMACADDRESSREQ)pReqHdr);
1168
1169	case VMMR0_DO_INTNET_IF_SET_ACTIVE:
1170	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFSETACTIVEREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
1171	return VERR_INVALID_PARAMETER;
1172	return IntNetR0IfSetActiveReq(pSession, (PINTNETIFSETACTIVEREQ)pReqHdr);
1173
1174	case VMMR0_DO_INTNET_IF_SEND:
1175	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFSENDREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
1176	return VERR_INVALID_PARAMETER;
1177	return IntNetR0IfSendReq(pSession, (PINTNETIFSENDREQ)pReqHdr);
1178
1179	case VMMR0_DO_INTNET_IF_WAIT:
1180	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFWAITREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
1181	return VERR_INVALID_PARAMETER;
1182	return IntNetR0IfWaitReq(pSession, (PINTNETIFWAITREQ)pReqHdr);
1183
1184	case VMMR0_DO_INTNET_IF_ABORT_WAIT:
1185	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PINTNETIFWAITREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
1186	return VERR_INVALID_PARAMETER;
1187	return IntNetR0IfAbortWaitReq(pSession, (PINTNETIFABORTWAITREQ)pReqHdr);
1188
1189	#ifdef VBOX_WITH_PCI_PASSTHROUGH
1190	/*
1191	* Requests to host PCI driver service.
1192	*/
1193	case VMMR0_DO_PCIRAW_REQ:
1194	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pVM, ((PPCIRAWSENDREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
1195	return VERR_INVALID_PARAMETER;
1196	return PciRawR0ProcessReq(pSession, pVM, (PPCIRAWSENDREQ)pReqHdr);
1197	#endif
1198	/*
1199	* For profiling.
1200	*/
1201	case VMMR0_DO_NOP:
1202	case VMMR0_DO_SLOW_NOP:
1203	return VINF_SUCCESS;
1204
1205	/*
1206	* For testing Ring-0 APIs invoked in this environment.
1207	*/
1208	case VMMR0_DO_TESTS:
1209	/** @todo make new test */
1210	return VINF_SUCCESS;
1211
1212
1213	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1214	case VMMR0_DO_TEST_SWITCHER3264:
1215	if (idCpu == NIL_VMCPUID)
1216	return VERR_INVALID_CPU_ID;
1217	return HWACCMR0TestSwitcher3264(pVM);
1218	#endif
1219	default:
1220	/*
1221	* We're returning VERR_NOT_SUPPORT here so we've got something else
1222	* than -1 which the interrupt gate glue code might return.
1223	*/
1224	Log(("operation %#x is not supported\n", enmOperation));
1225	return VERR_NOT_SUPPORTED;
1226	}
1227	}
1228
1229
1230	/**
1231	* Argument for vmmR0EntryExWrapper containing the arguments for VMMR0EntryEx.
1232	*/
1233	typedef struct VMMR0ENTRYEXARGS
1234	{
1235	PVM pVM;
1236	VMCPUID idCpu;
1237	VMMR0OPERATION enmOperation;
1238	PSUPVMMR0REQHDR pReq;
1239	uint64_t u64Arg;
1240	PSUPDRVSESSION pSession;
1241	} VMMR0ENTRYEXARGS;
1242	/** Pointer to a vmmR0EntryExWrapper argument package. */
1243	typedef VMMR0ENTRYEXARGS *PVMMR0ENTRYEXARGS;
1244
1245	/**
1246	* This is just a longjmp wrapper function for VMMR0EntryEx calls.
1247	*
1248	* @returns VBox status code.
1249	* @param pvArgs The argument package
1250	*/
1251	static int vmmR0EntryExWrapper(void *pvArgs)
1252	{
1253	return vmmR0EntryExWorker(((PVMMR0ENTRYEXARGS)pvArgs)->pVM,
1254	((PVMMR0ENTRYEXARGS)pvArgs)->idCpu,
1255	((PVMMR0ENTRYEXARGS)pvArgs)->enmOperation,
1256	((PVMMR0ENTRYEXARGS)pvArgs)->pReq,
1257	((PVMMR0ENTRYEXARGS)pvArgs)->u64Arg,
1258	((PVMMR0ENTRYEXARGS)pvArgs)->pSession);
1259	}
1260
1261
1262	/**
1263	* The Ring 0 entry point, called by the support library (SUP).
1264	*
1265	* @returns VBox status code.
1266	* @param pVM The VM to operate on.
1267	* @param idCpu Virtual CPU ID argument. Must be NIL_VMCPUID if pVM
1268	* is NIL_RTR0PTR, and may be NIL_VMCPUID if it isn't
1269	* @param enmOperation Which operation to execute.
1270	* @param pReq This points to a SUPVMMR0REQHDR packet. Optional.
1271	* @param u64Arg Some simple constant argument.
1272	* @param pSession The session of the caller.
1273	* @remarks Assume called with interrupts _enabled_.
1274	*/
1275	VMMR0DECL(int) VMMR0EntryEx(PVM pVM, VMCPUID idCpu, VMMR0OPERATION enmOperation, PSUPVMMR0REQHDR pReq, uint64_t u64Arg, PSUPDRVSESSION pSession)
1276	{
1277	/*
1278	* Requests that should only happen on the EMT thread will be
1279	* wrapped in a setjmp so we can assert without causing trouble.
1280	*/
1281	if ( VALID_PTR(pVM)
1282	&& pVM->pVMR0
1283	&& idCpu < pVM->cCpus)
1284	{
1285	switch (enmOperation)
1286	{
1287	/* These might/will be called before VMMR3Init. */
1288	case VMMR0_DO_GMM_INITIAL_RESERVATION:
1289	case VMMR0_DO_GMM_UPDATE_RESERVATION:
1290	case VMMR0_DO_GMM_ALLOCATE_PAGES:
1291	case VMMR0_DO_GMM_FREE_PAGES:
1292	case VMMR0_DO_GMM_BALLOONED_PAGES:
1293	/* On the mac we might not have a valid jmp buf, so check these as well. */
1294	case VMMR0_DO_VMMR0_INIT:
1295	case VMMR0_DO_VMMR0_TERM:
1296	{
1297	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1298
1299	if (!pVCpu->vmm.s.CallRing3JmpBufR0.pvSavedStack)
1300	break;
1301
1302	/** @todo validate this EMT claim... GVM knows. */
1303	VMMR0ENTRYEXARGS Args;
1304	Args.pVM = pVM;
1305	Args.idCpu = idCpu;
1306	Args.enmOperation = enmOperation;
1307	Args.pReq = pReq;
1308	Args.u64Arg = u64Arg;
1309	Args.pSession = pSession;
1310	return vmmR0CallRing3SetJmpEx(&pVCpu->vmm.s.CallRing3JmpBufR0, vmmR0EntryExWrapper, &Args);
1311	}
1312
1313	default:
1314	break;
1315	}
1316	}
1317	return vmmR0EntryExWorker(pVM, idCpu, enmOperation, pReq, u64Arg, pSession);
1318	}
1319
1320	/**
1321	* Internal R0 logger worker: Flush logger.
1322	*
1323	* @param pLogger The logger instance to flush.
1324	* @remark This function must be exported!
1325	*/
1326	VMMR0DECL(void) vmmR0LoggerFlush(PRTLOGGER pLogger)
1327	{
1328	#ifdef LOG_ENABLED
1329	/*
1330	* Convert the pLogger into a VM handle and 'call' back to Ring-3.
1331	* (This is a bit paranoid code.)
1332	*/
1333	PVMMR0LOGGER pR0Logger = (PVMMR0LOGGER)((uintptr_t)pLogger - RT_OFFSETOF(VMMR0LOGGER, Logger));
1334	if ( !VALID_PTR(pR0Logger)
1335	\|\| !VALID_PTR(pR0Logger + 1)
1336	\|\| pLogger->u32Magic != RTLOGGER_MAGIC)
1337	{
1338	# ifdef DEBUG
1339	SUPR0Printf("vmmR0LoggerFlush: pLogger=%p!\n", pLogger);
1340	# endif
1341	return;
1342	}
1343	if (pR0Logger->fFlushingDisabled)
1344	return; /* quietly */
1345
1346	PVM pVM = pR0Logger->pVM;
1347	if ( !VALID_PTR(pVM)
1348	\|\| pVM->pVMR0 != pVM)
1349	{
1350	# ifdef DEBUG
1351	SUPR0Printf("vmmR0LoggerFlush: pVM=%p! pVMR0=%p! pLogger=%p\n", pVM, pVM->pVMR0, pLogger);
1352	# endif
1353	return;
1354	}
1355
1356	PVMCPU pVCpu = VMMGetCpu(pVM);
1357	if (pVCpu)
1358	{
1359	/*
1360	* Check that the jump buffer is armed.
1361	*/
1362	# ifdef RT_ARCH_X86
1363	if ( !pVCpu->vmm.s.CallRing3JmpBufR0.eip
1364	\|\| pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call)
1365	# else
1366	if ( !pVCpu->vmm.s.CallRing3JmpBufR0.rip
1367	\|\| pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call)
1368	# endif
1369	{
1370	# ifdef DEBUG
1371	SUPR0Printf("vmmR0LoggerFlush: Jump buffer isn't armed!\n");
1372	# endif
1373	return;
1374	}
1375	VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_VMM_LOGGER_FLUSH, 0);
1376	}
1377	# ifdef DEBUG
1378	else
1379	SUPR0Printf("vmmR0LoggerFlush: invalid VCPU context!\n");
1380	# endif
1381	#endif
1382	}
1383
1384	/**
1385	* Internal R0 logger worker: Custom prefix.
1386	*
1387	* @returns Number of chars written.
1388	*
1389	* @param pLogger The logger instance.
1390	* @param pchBuf The output buffer.
1391	* @param cchBuf The size of the buffer.
1392	* @param pvUser User argument (ignored).
1393	*/
1394	VMMR0DECL(size_t) vmmR0LoggerPrefix(PRTLOGGER pLogger, char pchBuf, size_t cchBuf, void pvUser)
1395	{
1396	NOREF(pvUser);
1397	#ifdef LOG_ENABLED
1398	PVMMR0LOGGER pR0Logger = (PVMMR0LOGGER)((uintptr_t)pLogger - RT_OFFSETOF(VMMR0LOGGER, Logger));
1399	if ( !VALID_PTR(pR0Logger)
1400	\|\| !VALID_PTR(pR0Logger + 1)
1401	\|\| pLogger->u32Magic != RTLOGGER_MAGIC
1402	\|\| cchBuf < 2)
1403	return 0;
1404
1405	static const char s_szHex[17] = "0123456789abcdef";
1406	VMCPUID const idCpu = pR0Logger->idCpu;
1407	pchBuf[1] = s_szHex[ idCpu & 15];
1408	pchBuf[0] = s_szHex[(idCpu >> 4) & 15];
1409
1410	return 2;
1411	#else
1412	return 0;
1413	#endif
1414	}
1415
1416	#ifdef LOG_ENABLED
1417
1418	/**
1419	* Disables flushing of the ring-0 debug log.
1420	*
1421	* @param pVCpu The shared virtual cpu structure.
1422	*/
1423	VMMR0DECL(void) VMMR0LogFlushDisable(PVMCPU pVCpu)
1424	{
1425	PVM pVM = pVCpu->pVMR0;
1426	if (pVCpu->vmm.s.pR0LoggerR0)
1427	pVCpu->vmm.s.pR0LoggerR0->fFlushingDisabled = true;
1428	}
1429
1430
1431	/**
1432	* Enables flushing of the ring-0 debug log.
1433	*
1434	* @param pVCpu The shared virtual cpu structure.
1435	*/
1436	VMMR0DECL(void) VMMR0LogFlushEnable(PVMCPU pVCpu)
1437	{
1438	PVM pVM = pVCpu->pVMR0;
1439	if (pVCpu->vmm.s.pR0LoggerR0)
1440	pVCpu->vmm.s.pR0LoggerR0->fFlushingDisabled = false;
1441	}
1442
1443	#endif /* LOG_ENABLED */
1444
1445	/**
1446	* Jump back to ring-3 if we're the EMT and the longjmp is armed.
1447	*
1448	* @returns true if the breakpoint should be hit, false if it should be ignored.
1449	*/
1450	DECLEXPORT(bool) RTCALL RTAssertShouldPanic(void)
1451	{
1452	#if 0
1453	return true;
1454	#else
1455	PVM pVM = GVMMR0GetVMByEMT(NIL_RTNATIVETHREAD);
1456	if (pVM)
1457	{
1458	PVMCPU pVCpu = VMMGetCpu(pVM);
1459
1460	if (pVCpu)
1461	{
1462	#ifdef RT_ARCH_X86
1463	if ( pVCpu->vmm.s.CallRing3JmpBufR0.eip
1464	&& !pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call)
1465	#else
1466	if ( pVCpu->vmm.s.CallRing3JmpBufR0.rip
1467	&& !pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call)
1468	#endif
1469	{
1470	int rc = VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_VM_R0_ASSERTION, 0);
1471	return RT_FAILURE_NP(rc);
1472	}
1473	}
1474	}
1475	#ifdef RT_OS_LINUX
1476	return true;
1477	#else
1478	return false;
1479	#endif
1480	#endif
1481	}
1482
1483
1484	/**
1485	* Override this so we can push it up to ring-3.
1486	*
1487	* @param pszExpr Expression. Can be NULL.
1488	* @param uLine Location line number.
1489	* @param pszFile Location file name.
1490	* @param pszFunction Location function name.
1491	*/
1492	DECLEXPORT(void) RTCALL RTAssertMsg1Weak(const char pszExpr, unsigned uLine, const char pszFile, const char *pszFunction)
1493	{
1494	/*
1495	* To the log.
1496	*/
1497	LogAlways(("\n!!R0-Assertion Failed!!\n"
1498	"Expression: %s\n"
1499	"Location : %s(%d) %s\n",
1500	pszExpr, pszFile, uLine, pszFunction));
1501
1502	/*
1503	* To the global VMM buffer.
1504	*/
1505	PVM pVM = GVMMR0GetVMByEMT(NIL_RTNATIVETHREAD);
1506	if (pVM)
1507	RTStrPrintf(pVM->vmm.s.szRing0AssertMsg1, sizeof(pVM->vmm.s.szRing0AssertMsg1),
1508	"\n!!R0-Assertion Failed!!\n"
1509	"Expression: %s\n"
1510	"Location : %s(%d) %s\n",
1511	pszExpr, pszFile, uLine, pszFunction);
1512
1513	/*
1514	* Continue the normal way.
1515	*/
1516	RTAssertMsg1(pszExpr, uLine, pszFile, pszFunction);
1517	}
1518
1519
1520	/**
1521	* Callback for RTLogFormatV which writes to the ring-3 log port.
1522	* See PFNLOGOUTPUT() for details.
1523	*/
1524	static DECLCALLBACK(size_t) rtLogOutput(void pv, const char pachChars, size_t cbChars)
1525	{
1526	for (size_t i = 0; i < cbChars; i++)
1527	LogAlways(("%c", pachChars[i]));
1528
1529	return cbChars;
1530	}
1531
1532
1533	/**
1534	* Override this so we can push it up to ring-3.
1535	*
1536	* @param pszFormat The format string.
1537	* @param va Arguments.
1538	*/
1539	DECLEXPORT(void) RTCALL RTAssertMsg2WeakV(const char *pszFormat, va_list va)
1540	{
1541	va_list vaCopy;
1542
1543	/*
1544	* Push the message to the logger.
1545	*/
1546	PRTLOGGER pLog = RTLogDefaultInstance(); /** @todo we want this for release as well! */
1547	if (pLog)
1548	{
1549	va_copy(vaCopy, va);
1550	RTLogFormatV(rtLogOutput, pLog, pszFormat, vaCopy);
1551	va_end(vaCopy);
1552	}
1553
1554	/*
1555	* Push it to the global VMM buffer.
1556	*/
1557	PVM pVM = GVMMR0GetVMByEMT(NIL_RTNATIVETHREAD);
1558	if (pVM)
1559	{
1560	va_copy(vaCopy, va);
1561	RTStrPrintfV(pVM->vmm.s.szRing0AssertMsg2, sizeof(pVM->vmm.s.szRing0AssertMsg2), pszFormat, vaCopy);
1562	va_end(vaCopy);
1563	}
1564
1565	/*
1566	* Continue the normal way.
1567	*/
1568	RTAssertMsg2V(pszFormat, va);
1569	}

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/VMMR0.cpp@ 36329

Download in other formats: