PGMR0.cpp@ 99725

Last change on this file since 99725 was 98103, checked in by vboxsync, 2 years ago
Copyright year updates by scm.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 54.2 KB

Line
1	/* $Id: PGMR0.cpp 98103 2023-01-17 14:15:46Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2007-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_PGM
33	#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */
34	#include <VBox/rawpci.h>
35	#include <VBox/vmm/pgm.h>
36	#include <VBox/vmm/iem.h>
37	#include <VBox/vmm/gmm.h>
38	#include "PGMInternal.h"
39	#include <VBox/vmm/pdmdev.h>
40	#include <VBox/vmm/vmcc.h>
41	#include <VBox/vmm/gvm.h>
42	#include "PGMInline.h"
43	#include <VBox/log.h>
44	#include <VBox/err.h>
45	#include <iprt/assert.h>
46	#include <iprt/mem.h>
47	#include <iprt/memobj.h>
48	#include <iprt/process.h>
49	#include <iprt/rand.h>
50	#include <iprt/string.h>
51	#include <iprt/time.h>
52
53
54	/*
55	* Instantiate the ring-0 header/code templates.
56	*/
57	/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
58	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_PROT(name)
59	#include "PGMR0Bth.h"
60	#undef PGM_BTH_NAME
61
62	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PROT(name)
63	#include "PGMR0Bth.h"
64	#undef PGM_BTH_NAME
65
66	#define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_PROT(name)
67	#include "PGMR0Bth.h"
68	#undef PGM_BTH_NAME
69
70	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PROT(name)
71	#include "PGMR0Bth.h"
72	#undef PGM_BTH_NAME
73
74
75	/**
76	* Initializes the per-VM data for the PGM.
77	*
78	* This is called from under the GVMM lock, so it should only initialize the
79	* data so PGMR0CleanupVM and others will work smoothly.
80	*
81	* @returns VBox status code.
82	* @param pGVM Pointer to the global VM structure.
83	* @param hMemObj Handle to the memory object backing pGVM.
84	*/
85	VMMR0_INT_DECL(int) PGMR0InitPerVMData(PGVM pGVM, RTR0MEMOBJ hMemObj)
86	{
87	AssertCompile(sizeof(pGVM->pgm.s) <= sizeof(pGVM->pgm.padding));
88	AssertCompile(sizeof(pGVM->pgmr0.s) <= sizeof(pGVM->pgmr0.padding));
89
90	AssertCompile(RT_ELEMENTS(pGVM->pgmr0.s.ahPoolMemObjs) == RT_ELEMENTS(pGVM->pgmr0.s.ahPoolMapObjs));
91	for (uint32_t i = 0; i < RT_ELEMENTS(pGVM->pgmr0.s.ahPoolMemObjs); i++)
92	{
93	pGVM->pgmr0.s.ahPoolMemObjs[i] = NIL_RTR0MEMOBJ;
94	pGVM->pgmr0.s.ahPoolMapObjs[i] = NIL_RTR0MEMOBJ;
95	}
96	pGVM->pgmr0.s.hPhysHandlerMemObj = NIL_RTR0MEMOBJ;
97	pGVM->pgmr0.s.hPhysHandlerMapObj = NIL_RTR0MEMOBJ;
98
99	/*
100	* Initialize the handler type table with return to ring-3 callbacks so we
101	* don't have to do anything special for ring-3 only registrations.
102	*
103	* Note! The random bits of the hType value is mainly for prevent trouble
104	* with zero initialized handles w/o needing to sacrifice handle zero.
105	*/
106	for (size_t i = 0; i < RT_ELEMENTS(pGVM->pgm.s.aPhysHandlerTypes); i++)
107	{
108	pGVM->pgmr0.s.aPhysHandlerTypes[i].hType = i \| (RTRandU64() & ~(uint64_t)PGMPHYSHANDLERTYPE_IDX_MASK);
109	pGVM->pgmr0.s.aPhysHandlerTypes[i].enmKind = PGMPHYSHANDLERKIND_INVALID;
110	pGVM->pgmr0.s.aPhysHandlerTypes[i].pfnHandler = pgmR0HandlerPhysicalHandlerToRing3;
111	pGVM->pgmr0.s.aPhysHandlerTypes[i].pfnPfHandler = pgmR0HandlerPhysicalPfHandlerToRing3;
112
113	pGVM->pgm.s.aPhysHandlerTypes[i].hType = pGVM->pgmr0.s.aPhysHandlerTypes[i].hType;
114	pGVM->pgm.s.aPhysHandlerTypes[i].enmKind = PGMPHYSHANDLERKIND_INVALID;
115	}
116
117	/*
118	* Get the physical address of the ZERO and MMIO-dummy pages.
119	*/
120	AssertReturn(((uintptr_t)&pGVM->pgm.s.abZeroPg[0] & HOST_PAGE_OFFSET_MASK) == 0, VERR_INTERNAL_ERROR_2);
121	pGVM->pgm.s.HCPhysZeroPg = RTR0MemObjGetPagePhysAddr(hMemObj, RT_UOFFSETOF_DYN(GVM, pgm.s.abZeroPg) >> HOST_PAGE_SHIFT);
122	AssertReturn(pGVM->pgm.s.HCPhysZeroPg != NIL_RTHCPHYS, VERR_INTERNAL_ERROR_3);
123
124	AssertReturn(((uintptr_t)&pGVM->pgm.s.abMmioPg[0] & HOST_PAGE_OFFSET_MASK) == 0, VERR_INTERNAL_ERROR_2);
125	pGVM->pgm.s.HCPhysMmioPg = RTR0MemObjGetPagePhysAddr(hMemObj, RT_UOFFSETOF_DYN(GVM, pgm.s.abMmioPg) >> HOST_PAGE_SHIFT);
126	AssertReturn(pGVM->pgm.s.HCPhysMmioPg != NIL_RTHCPHYS, VERR_INTERNAL_ERROR_3);
127
128	pGVM->pgm.s.HCPhysInvMmioPg = pGVM->pgm.s.HCPhysMmioPg;
129
130	return RTCritSectInit(&pGVM->pgmr0.s.PoolGrowCritSect);
131	}
132
133
134	/**
135	* Initalize the per-VM PGM for ring-0.
136	*
137	* @returns VBox status code.
138	* @param pGVM Pointer to the global VM structure.
139	*/
140	VMMR0_INT_DECL(int) PGMR0InitVM(PGVM pGVM)
141	{
142	/*
143	* Set up the ring-0 context for our access handlers.
144	*/
145	int rc = PGMR0HandlerPhysicalTypeSetUpContext(pGVM, PGMPHYSHANDLERKIND_WRITE, 0 /fFlags/,
146	pgmPhysRomWriteHandler, pgmPhysRomWritePfHandler,
147	"ROM write protection", pGVM->pgm.s.hRomPhysHandlerType);
148	AssertLogRelRCReturn(rc, rc);
149
150	/*
151	* Register the physical access handler doing dirty MMIO2 tracing.
152	*/
153	rc = PGMR0HandlerPhysicalTypeSetUpContext(pGVM, PGMPHYSHANDLERKIND_WRITE, PGMPHYSHANDLER_F_KEEP_PGM_LOCK,
154	pgmPhysMmio2WriteHandler, pgmPhysMmio2WritePfHandler,
155	"MMIO2 dirty page tracing", pGVM->pgm.s.hMmio2DirtyPhysHandlerType);
156	AssertLogRelRCReturn(rc, rc);
157
158	/*
159	* The page pool.
160	*/
161	return pgmR0PoolInitVM(pGVM);
162	}
163
164
165	/**
166	* Called at the end of the ring-0 initialization to seal access handler types.
167	*
168	* @returns VBox status code.
169	* @param pGVM Pointer to the global VM structure.
170	*/
171	VMMR0_INT_DECL(void) PGMR0DoneInitVM(PGVM pGVM)
172	{
173	/*
174	* Seal all the access handler types. Does both ring-3 and ring-0.
175	*
176	* Note! Since this is a void function and we don't have any ring-0 state
177	* machinery for marking the VM as bogus, this code will just
178	* override corrupted values as best as it can.
179	*/
180	AssertCompile(RT_ELEMENTS(pGVM->pgmr0.s.aPhysHandlerTypes) == RT_ELEMENTS(pGVM->pgm.s.aPhysHandlerTypes));
181	for (size_t i = 0; i < RT_ELEMENTS(pGVM->pgmr0.s.aPhysHandlerTypes); i++)
182	{
183	PPGMPHYSHANDLERTYPEINTR0 const pTypeR0 = &pGVM->pgmr0.s.aPhysHandlerTypes[i];
184	PPGMPHYSHANDLERTYPEINTR3 const pTypeR3 = &pGVM->pgm.s.aPhysHandlerTypes[i];
185	PGMPHYSHANDLERKIND const enmKindR3 = pTypeR3->enmKind;
186	PGMPHYSHANDLERKIND const enmKindR0 = pTypeR0->enmKind;
187	AssertLogRelMsgStmt(pTypeR0->hType == pTypeR3->hType,
188	("i=%u %#RX64 vs %#RX64 %s\n", i, pTypeR0->hType, pTypeR3->hType, pTypeR0->pszDesc),
189	pTypeR3->hType = pTypeR0->hType);
190	switch (enmKindR3)
191	{
192	case PGMPHYSHANDLERKIND_ALL:
193	case PGMPHYSHANDLERKIND_MMIO:
194	if ( enmKindR0 == enmKindR3
195	\|\| enmKindR0 == PGMPHYSHANDLERKIND_INVALID)
196	{
197	pTypeR3->fRing0Enabled = enmKindR0 == enmKindR3;
198	pTypeR0->uState = PGM_PAGE_HNDL_PHYS_STATE_ALL;
199	pTypeR3->uState = PGM_PAGE_HNDL_PHYS_STATE_ALL;
200	continue;
201	}
202	break;
203
204	case PGMPHYSHANDLERKIND_WRITE:
205	if ( enmKindR0 == enmKindR3
206	\|\| enmKindR0 == PGMPHYSHANDLERKIND_INVALID)
207	{
208	pTypeR3->fRing0Enabled = enmKindR0 == enmKindR3;
209	pTypeR0->uState = PGM_PAGE_HNDL_PHYS_STATE_WRITE;
210	pTypeR3->uState = PGM_PAGE_HNDL_PHYS_STATE_WRITE;
211	continue;
212	}
213	break;
214
215	default:
216	AssertLogRelMsgFailed(("i=%u enmKindR3=%d\n", i, enmKindR3));
217	RT_FALL_THROUGH();
218	case PGMPHYSHANDLERKIND_INVALID:
219	AssertLogRelMsg(enmKindR0 == PGMPHYSHANDLERKIND_INVALID,
220	("i=%u enmKind=%d %s\n", i, enmKindR0, pTypeR0->pszDesc));
221	AssertLogRelMsg(pTypeR0->pfnHandler == pgmR0HandlerPhysicalHandlerToRing3,
222	("i=%u pfnHandler=%p %s\n", i, pTypeR0->pfnHandler, pTypeR0->pszDesc));
223	AssertLogRelMsg(pTypeR0->pfnPfHandler == pgmR0HandlerPhysicalPfHandlerToRing3,
224	("i=%u pfnPfHandler=%p %s\n", i, pTypeR0->pfnPfHandler, pTypeR0->pszDesc));
225
226	/* Unused of bad ring-3 entry, make it and the ring-0 one harmless. */
227	pTypeR3->enmKind = PGMPHYSHANDLERKIND_END;
228	pTypeR3->fRing0DevInsIdx = false;
229	pTypeR3->fKeepPgmLock = false;
230	pTypeR3->uState = 0;
231	break;
232	}
233	pTypeR3->fRing0Enabled = false;
234
235	/* Make sure the entry is harmless and goes to ring-3. */
236	pTypeR0->enmKind = PGMPHYSHANDLERKIND_END;
237	pTypeR0->pfnHandler = pgmR0HandlerPhysicalHandlerToRing3;
238	pTypeR0->pfnPfHandler = pgmR0HandlerPhysicalPfHandlerToRing3;
239	pTypeR0->fRing0DevInsIdx = false;
240	pTypeR0->fKeepPgmLock = false;
241	pTypeR0->uState = 0;
242	pTypeR0->pszDesc = "invalid";
243	}
244	}
245
246
247	/**
248	* Cleans up any loose ends before the GVM structure is destroyed.
249	*/
250	VMMR0_INT_DECL(void) PGMR0CleanupVM(PGVM pGVM)
251	{
252	for (uint32_t i = 0; i < RT_ELEMENTS(pGVM->pgmr0.s.ahPoolMemObjs); i++)
253	{
254	if (pGVM->pgmr0.s.ahPoolMapObjs[i] != NIL_RTR0MEMOBJ)
255	{
256	int rc = RTR0MemObjFree(pGVM->pgmr0.s.ahPoolMapObjs[i], true /fFreeMappings/);
257	AssertRC(rc);
258	pGVM->pgmr0.s.ahPoolMapObjs[i] = NIL_RTR0MEMOBJ;
259	}
260
261	if (pGVM->pgmr0.s.ahPoolMemObjs[i] != NIL_RTR0MEMOBJ)
262	{
263	int rc = RTR0MemObjFree(pGVM->pgmr0.s.ahPoolMemObjs[i], true /fFreeMappings/);
264	AssertRC(rc);
265	pGVM->pgmr0.s.ahPoolMemObjs[i] = NIL_RTR0MEMOBJ;
266	}
267	}
268
269	if (pGVM->pgmr0.s.hPhysHandlerMapObj != NIL_RTR0MEMOBJ)
270	{
271	int rc = RTR0MemObjFree(pGVM->pgmr0.s.hPhysHandlerMapObj, true /fFreeMappings/);
272	AssertRC(rc);
273	pGVM->pgmr0.s.hPhysHandlerMapObj = NIL_RTR0MEMOBJ;
274	}
275
276	if (pGVM->pgmr0.s.hPhysHandlerMemObj != NIL_RTR0MEMOBJ)
277	{
278	int rc = RTR0MemObjFree(pGVM->pgmr0.s.hPhysHandlerMemObj, true /fFreeMappings/);
279	AssertRC(rc);
280	pGVM->pgmr0.s.hPhysHandlerMemObj = NIL_RTR0MEMOBJ;
281	}
282
283	if (RTCritSectIsInitialized(&pGVM->pgmr0.s.PoolGrowCritSect))
284	RTCritSectDelete(&pGVM->pgmr0.s.PoolGrowCritSect);
285	}
286
287
288	/**
289	* Worker function for PGMR3PhysAllocateHandyPages and pgmPhysEnsureHandyPage.
290	*
291	* @returns The following VBox status codes.
292	* @retval VINF_SUCCESS on success. FF cleared.
293	* @retval VINF_EM_NO_MEMORY if we're out of memory. The FF is set in this case.
294	*
295	* @param pGVM The global (ring-0) VM structure.
296	* @param idCpu The ID of the calling EMT.
297	* @param fRing3 Set if the caller is ring-3. Determins whether to
298	* return VINF_EM_NO_MEMORY or not.
299	*
300	* @thread EMT(idCpu)
301	*
302	* @remarks Must be called from within the PGM critical section. The caller
303	* must clear the new pages.
304	*/
305	int pgmR0PhysAllocateHandyPages(PGVM pGVM, VMCPUID idCpu, bool fRing3)
306	{
307	/*
308	* Validate inputs.
309	*/
310	AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID); /* caller already checked this, but just to be sure. */
311	Assert(pGVM->aCpus[idCpu].hEMT == RTThreadNativeSelf());
312	PGM_LOCK_ASSERT_OWNER_EX(pGVM, &pGVM->aCpus[idCpu]);
313
314	/*
315	* Check for error injection.
316	*/
317	if (RT_LIKELY(!pGVM->pgm.s.fErrInjHandyPages))
318	{ /* likely */ }
319	else
320	return VERR_NO_MEMORY;
321
322	/*
323	* Try allocate a full set of handy pages.
324	*/
325	uint32_t const iFirst = pGVM->pgm.s.cHandyPages;
326	AssertMsgReturn(iFirst <= RT_ELEMENTS(pGVM->pgm.s.aHandyPages), ("%#x\n", iFirst), VERR_PGM_HANDY_PAGE_IPE);
327
328	uint32_t const cPages = RT_ELEMENTS(pGVM->pgm.s.aHandyPages) - iFirst;
329	if (!cPages)
330	return VINF_SUCCESS;
331
332	int rc = GMMR0AllocateHandyPages(pGVM, idCpu, cPages, cPages, &pGVM->pgm.s.aHandyPages[iFirst]);
333	if (RT_SUCCESS(rc))
334	{
335	uint32_t const cHandyPages = RT_ELEMENTS(pGVM->pgm.s.aHandyPages); /** @todo allow allocating less... */
336	pGVM->pgm.s.cHandyPages = cHandyPages;
337	VM_FF_CLEAR(pGVM, VM_FF_PGM_NEED_HANDY_PAGES);
338	VM_FF_CLEAR(pGVM, VM_FF_PGM_NO_MEMORY);
339
340	#ifdef VBOX_STRICT
341	for (uint32_t i = 0; i < cHandyPages; i++)
342	{
343	Assert(pGVM->pgm.s.aHandyPages[i].idPage != NIL_GMM_PAGEID);
344	Assert(pGVM->pgm.s.aHandyPages[i].idPage <= GMM_PAGEID_LAST);
345	Assert(pGVM->pgm.s.aHandyPages[i].idSharedPage == NIL_GMM_PAGEID);
346	Assert(pGVM->pgm.s.aHandyPages[i].HCPhysGCPhys != NIL_GMMPAGEDESC_PHYS);
347	Assert(!(pGVM->pgm.s.aHandyPages[i].HCPhysGCPhys & ~X86_PTE_PAE_PG_MASK));
348	}
349	#endif
350
351	/*
352	* Clear the pages.
353	*/
354	for (uint32_t iPage = iFirst; iPage < cHandyPages; iPage++)
355	{
356	PGMMPAGEDESC pPage = &pGVM->pgm.s.aHandyPages[iPage];
357	if (!pPage->fZeroed)
358	{
359	void *pv = NULL;
360	#ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM
361	rc = SUPR0HCPhysToVirt(pPage->HCPhysGCPhys, &pv);
362	#else
363	rc = GMMR0PageIdToVirt(pGVM, pPage->idPage, &pv);
364	#endif
365	AssertMsgRCReturn(rc, ("idPage=%#x HCPhys=%RHp rc=%Rrc\n", pPage->idPage, pPage->HCPhysGCPhys, rc), rc);
366
367	RT_BZERO(pv, GUEST_PAGE_SIZE);
368	pPage->fZeroed = true;
369	}
370	#ifdef VBOX_STRICT
371	else
372	{
373	void *pv = NULL;
374	# ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM
375	rc = SUPR0HCPhysToVirt(pPage->HCPhysGCPhys, &pv);
376	# else
377	rc = GMMR0PageIdToVirt(pGVM, pPage->idPage, &pv);
378	# endif
379	AssertMsgRCReturn(rc, ("idPage=%#x HCPhys=%RHp rc=%Rrc\n", pPage->idPage, pPage->HCPhysGCPhys, rc), rc);
380	AssertReturn(ASMMemIsZero(pv, GUEST_PAGE_SIZE), VERR_PGM_HANDY_PAGE_IPE);
381	}
382	#endif
383	Log3(("PGMR0PhysAllocateHandyPages: idPage=%#x HCPhys=%RGp\n", pPage->idPage, pPage->HCPhysGCPhys));
384	}
385	}
386	else
387	{
388	/*
389	* We should never get here unless there is a genuine shortage of
390	* memory (or some internal error). Flag the error so the VM can be
391	* suspended ASAP and the user informed. If we're totally out of
392	* handy pages we will return failure.
393	*/
394	/* Report the failure. */
395	LogRel(("PGM: Failed to procure handy pages; rc=%Rrc cHandyPages=%#x\n"
396	" cAllPages=%#x cPrivatePages=%#x cSharedPages=%#x cZeroPages=%#x\n",
397	rc, pGVM->pgm.s.cHandyPages,
398	pGVM->pgm.s.cAllPages, pGVM->pgm.s.cPrivatePages, pGVM->pgm.s.cSharedPages, pGVM->pgm.s.cZeroPages));
399
400	GMMMEMSTATSREQ Stats = { { SUPVMMR0REQHDR_MAGIC, sizeof(Stats) }, 0, 0, 0, 0, 0 };
401	if (RT_SUCCESS(GMMR0QueryMemoryStatsReq(pGVM, idCpu, &Stats)))
402	LogRel(("GMM: Statistics:\n"
403	" Allocated pages: %RX64\n"
404	" Free pages: %RX64\n"
405	" Shared pages: %RX64\n"
406	" Maximum pages: %RX64\n"
407	" Ballooned pages: %RX64\n",
408	Stats.cAllocPages, Stats.cFreePages, Stats.cSharedPages, Stats.cMaxPages, Stats.cBalloonedPages));
409
410	if ( rc != VERR_NO_MEMORY
411	&& rc != VERR_NO_PHYS_MEMORY
412	&& rc != VERR_LOCK_FAILED)
413	for (uint32_t iPage = 0; iPage < RT_ELEMENTS(pGVM->pgm.s.aHandyPages); iPage++)
414	LogRel(("PGM: aHandyPages[#%#04x] = {.HCPhysGCPhys=%RHp, .idPage=%#08x, .idSharedPage=%#08x}\n",
415	iPage, pGVM->pgm.s.aHandyPages[iPage].HCPhysGCPhys, pGVM->pgm.s.aHandyPages[iPage].idPage,
416	pGVM->pgm.s.aHandyPages[iPage].idSharedPage));
417
418	/* Set the FFs and adjust rc. */
419	VM_FF_SET(pGVM, VM_FF_PGM_NEED_HANDY_PAGES);
420	VM_FF_SET(pGVM, VM_FF_PGM_NO_MEMORY);
421	if (!fRing3)
422	if ( rc == VERR_NO_MEMORY
423	\|\| rc == VERR_NO_PHYS_MEMORY
424	\|\| rc == VERR_LOCK_FAILED
425	\|\| rc == VERR_MAP_FAILED)
426	rc = VINF_EM_NO_MEMORY;
427	}
428
429	LogFlow(("PGMR0PhysAllocateHandyPages: cPages=%d rc=%Rrc\n", cPages, rc));
430	return rc;
431	}
432
433
434	/**
435	* Worker function for PGMR3PhysAllocateHandyPages / VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES.
436	*
437	* @returns The following VBox status codes.
438	* @retval VINF_SUCCESS on success. FF cleared.
439	* @retval VINF_EM_NO_MEMORY if we're out of memory. The FF is set in this case.
440	*
441	* @param pGVM The global (ring-0) VM structure.
442	* @param idCpu The ID of the calling EMT.
443	*
444	* @thread EMT(idCpu)
445	*
446	* @remarks Must be called from within the PGM critical section. The caller
447	* must clear the new pages.
448	*/
449	VMMR0_INT_DECL(int) PGMR0PhysAllocateHandyPages(PGVM pGVM, VMCPUID idCpu)
450	{
451	/*
452	* Validate inputs.
453	*/
454	AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID); /* caller already checked this, but just to be sure. */
455	AssertReturn(pGVM->aCpus[idCpu].hEMT == RTThreadNativeSelf(), VERR_NOT_OWNER);
456
457	/*
458	* Enter the PGM lock and call the worker.
459	*/
460	int rc = PGM_LOCK(pGVM);
461	if (RT_SUCCESS(rc))
462	{
463	rc = pgmR0PhysAllocateHandyPages(pGVM, idCpu, true /fRing3/);
464	PGM_UNLOCK(pGVM);
465	}
466	return rc;
467	}
468
469
470	/**
471	* Flushes any changes pending in the handy page array.
472	*
473	* It is very important that this gets done when page sharing is enabled.
474	*
475	* @returns The following VBox status codes.
476	* @retval VINF_SUCCESS on success. FF cleared.
477	*
478	* @param pGVM The global (ring-0) VM structure.
479	* @param idCpu The ID of the calling EMT.
480	*
481	* @thread EMT(idCpu)
482	*
483	* @remarks Must be called from within the PGM critical section.
484	*/
485	VMMR0_INT_DECL(int) PGMR0PhysFlushHandyPages(PGVM pGVM, VMCPUID idCpu)
486	{
487	/*
488	* Validate inputs.
489	*/
490	AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID); /* caller already checked this, but just to be sure. */
491	AssertReturn(pGVM->aCpus[idCpu].hEMT == RTThreadNativeSelf(), VERR_NOT_OWNER);
492	PGM_LOCK_ASSERT_OWNER_EX(pGVM, &pGVM->aCpus[idCpu]);
493
494	/*
495	* Try allocate a full set of handy pages.
496	*/
497	uint32_t iFirst = pGVM->pgm.s.cHandyPages;
498	AssertReturn(iFirst <= RT_ELEMENTS(pGVM->pgm.s.aHandyPages), VERR_PGM_HANDY_PAGE_IPE);
499	uint32_t cPages = RT_ELEMENTS(pGVM->pgm.s.aHandyPages) - iFirst;
500	if (!cPages)
501	return VINF_SUCCESS;
502	int rc = GMMR0AllocateHandyPages(pGVM, idCpu, cPages, 0, &pGVM->pgm.s.aHandyPages[iFirst]);
503
504	LogFlow(("PGMR0PhysFlushHandyPages: cPages=%d rc=%Rrc\n", cPages, rc));
505	return rc;
506	}
507
508
509	/**
510	* Allocate a large page at @a GCPhys.
511	*
512	* @returns The following VBox status codes.
513	* @retval VINF_SUCCESS on success.
514	* @retval VINF_EM_NO_MEMORY if we're out of memory.
515	*
516	* @param pGVM The global (ring-0) VM structure.
517	* @param idCpu The ID of the calling EMT.
518	* @param GCPhys The guest physical address of the page.
519	*
520	* @thread EMT(idCpu)
521	*
522	* @remarks Must be called from within the PGM critical section. The caller
523	* must clear the new pages.
524	*/
525	int pgmR0PhysAllocateLargePage(PGVM pGVM, VMCPUID idCpu, RTGCPHYS GCPhys)
526	{
527	STAM_PROFILE_START(&pGVM->pgm.s.Stats.StatLargePageAlloc2, a);
528	PGM_LOCK_ASSERT_OWNER_EX(pGVM, &pGVM->aCpus[idCpu]);
529
530	/*
531	* Allocate a large page.
532	*/
533	RTHCPHYS HCPhys = NIL_GMMPAGEDESC_PHYS;
534	uint32_t idPage = NIL_GMM_PAGEID;
535
536	if (true) /** @todo pre-allocate 2-3 pages on the allocation thread. */
537	{
538	uint64_t const nsAllocStart = RTTimeNanoTS();
539	if (nsAllocStart < pGVM->pgm.s.nsLargePageRetry)
540	{
541	LogFlowFunc(("returns VERR_TRY_AGAIN - %RU64 ns left of hold off period\n", pGVM->pgm.s.nsLargePageRetry - nsAllocStart));
542	return VERR_TRY_AGAIN;
543	}
544
545	int const rc = GMMR0AllocateLargePage(pGVM, idCpu, _2M, &idPage, &HCPhys);
546
547	uint64_t const nsAllocEnd = RTTimeNanoTS();
548	uint64_t const cNsElapsed = nsAllocEnd - nsAllocStart;
549	STAM_REL_PROFILE_ADD_PERIOD(&pGVM->pgm.s.StatLargePageAlloc, cNsElapsed);
550	if (cNsElapsed < RT_NS_100MS)
551	pGVM->pgm.s.cLargePageLongAllocRepeats = 0;
552	else
553	{
554	/* If a large page allocation takes more than 100ms back off for a
555	while so the host OS can reshuffle memory and make some more large
556	pages available. However if it took over a second, just disable it. */
557	STAM_REL_COUNTER_INC(&pGVM->pgm.s.StatLargePageOverflow);
558	pGVM->pgm.s.cLargePageLongAllocRepeats++;
559	if (cNsElapsed > RT_NS_1SEC)
560	{
561	LogRel(("PGMR0PhysAllocateLargePage: Disabling large pages after %'RU64 ns allocation time.\n", cNsElapsed));
562	PGMSetLargePageUsage(pGVM, false);
563	}
564	else
565	{
566	Log(("PGMR0PhysAllocateLargePage: Suspending large page allocations for %u sec after %'RU64 ns allocation time.\n",
567	30 * pGVM->pgm.s.cLargePageLongAllocRepeats, cNsElapsed));
568	pGVM->pgm.s.nsLargePageRetry = nsAllocEnd + RT_NS_30SEC * pGVM->pgm.s.cLargePageLongAllocRepeats;
569	}
570	}
571
572	if (RT_FAILURE(rc))
573	{
574	Log(("PGMR0PhysAllocateLargePage: Failed: %Rrc\n", rc));
575	STAM_REL_COUNTER_INC(&pGVM->pgm.s.StatLargePageAllocFailed);
576	if (rc == VERR_NOT_SUPPORTED)
577	{
578	LogRel(("PGM: Disabling large pages because of VERR_NOT_SUPPORTED status.\n"));
579	PGMSetLargePageUsage(pGVM, false);
580	}
581	return rc;
582	}
583	}
584
585	STAM_PROFILE_STOP_START(&pGVM->pgm.s.Stats.StatLargePageAlloc2, &pGVM->pgm.s.Stats.StatLargePageSetup, a);
586
587	/*
588	* Enter the pages into PGM.
589	*/
590	bool fFlushTLBs = false;
591	VBOXSTRICTRC rc = VINF_SUCCESS;
592	unsigned cLeft = _2M / GUEST_PAGE_SIZE;
593	while (cLeft-- > 0)
594	{
595	PPGMPAGE const pPage = pgmPhysGetPage(pGVM, GCPhys);
596	AssertReturn(pPage && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM && PGM_PAGE_IS_ZERO(pPage), VERR_PGM_UNEXPECTED_PAGE_STATE);
597
598	/* Make sure there are no zero mappings. */
599	uint16_t const u16Tracking = PGM_PAGE_GET_TRACKING(pPage);
600	if (u16Tracking == 0)
601	Assert(PGM_PAGE_GET_PTE_INDEX(pPage) == 0);
602	else
603	{
604	STAM_REL_COUNTER_INC(&pGVM->pgm.s.StatLargePageZeroEvict);
605	VBOXSTRICTRC rc3 = pgmPoolTrackUpdateGCPhys(pGVM, GCPhys, pPage, true /fFlushPTEs/, &fFlushTLBs);
606	Log(("PGMR0PhysAllocateLargePage: GCPhys=%RGp: tracking=%#x rc3=%Rrc\n", GCPhys, u16Tracking, VBOXSTRICTRC_VAL(rc3)));
607	if (rc3 != VINF_SUCCESS && rc == VINF_SUCCESS)
608	rc = rc3; /** @todo not perfect... */
609	PGM_PAGE_SET_PTE_INDEX(pGVM, pPage, 0);
610	PGM_PAGE_SET_TRACKING(pGVM, pPage, 0);
611	}
612
613	/* Setup the new page. */
614	PGM_PAGE_SET_HCPHYS(pGVM, pPage, HCPhys);
615	PGM_PAGE_SET_STATE(pGVM, pPage, PGM_PAGE_STATE_ALLOCATED);
616	PGM_PAGE_SET_PDE_TYPE(pGVM, pPage, PGM_PAGE_PDE_TYPE_PDE);
617	PGM_PAGE_SET_PAGEID(pGVM, pPage, idPage);
618	Log3(("PGMR0PhysAllocateLargePage: GCPhys=%RGp: idPage=%#x HCPhys=%RGp (old tracking=%#x)\n",
619	GCPhys, idPage, HCPhys, u16Tracking));
620
621	/* advance */
622	idPage++;
623	HCPhys += GUEST_PAGE_SIZE;
624	GCPhys += GUEST_PAGE_SIZE;
625	}
626
627	STAM_COUNTER_ADD(&pGVM->pgm.s.Stats.StatRZPageReplaceZero, _2M / GUEST_PAGE_SIZE);
628	pGVM->pgm.s.cZeroPages -= _2M / GUEST_PAGE_SIZE;
629	pGVM->pgm.s.cPrivatePages += _2M / GUEST_PAGE_SIZE;
630
631	/*
632	* Flush all TLBs.
633	*/
634	if (!fFlushTLBs)
635	{ /* likely as we shouldn't normally map zero pages */ }
636	else
637	{
638	STAM_REL_COUNTER_INC(&pGVM->pgm.s.StatLargePageTlbFlush);
639	PGM_INVL_ALL_VCPU_TLBS(pGVM);
640	}
641	/** @todo this is a little expensive (~3000 ticks) since we'll have to
642	* invalidate everything. Add a version to the TLB? */
643	pgmPhysInvalidatePageMapTLB(pGVM);
644	IEMTlbInvalidateAllPhysicalAllCpus(pGVM, idCpu);
645
646	STAM_PROFILE_STOP(&pGVM->pgm.s.Stats.StatLargePageSetup, a);
647	#if 0 /** @todo returning info statuses here might not be a great idea... */
648	LogFlow(("PGMR0PhysAllocateLargePage: returns %Rrc\n", VBOXSTRICTRC_VAL(rc) ));
649	return VBOXSTRICTRC_TODO(rc);
650	#else
651	LogFlow(("PGMR0PhysAllocateLargePage: returns VINF_SUCCESS (rc=%Rrc)\n", VBOXSTRICTRC_VAL(rc) ));
652	return VINF_SUCCESS;
653	#endif
654	}
655
656
657	/**
658	* Allocate a large page at @a GCPhys.
659	*
660	* @returns The following VBox status codes.
661	* @retval VINF_SUCCESS on success.
662	* @retval VINF_EM_NO_MEMORY if we're out of memory.
663	*
664	* @param pGVM The global (ring-0) VM structure.
665	* @param idCpu The ID of the calling EMT.
666	* @param GCPhys The guest physical address of the page.
667	*
668	* @thread EMT(idCpu)
669	*
670	* @remarks Must be called from within the PGM critical section. The caller
671	* must clear the new pages.
672	*/
673	VMMR0_INT_DECL(int) PGMR0PhysAllocateLargePage(PGVM pGVM, VMCPUID idCpu, RTGCPHYS GCPhys)
674	{
675	/*
676	* Validate inputs.
677	*/
678	AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID);
679	AssertReturn(pGVM->aCpus[idCpu].hEMT == RTThreadNativeSelf(), VERR_NOT_OWNER);
680
681	int rc = PGM_LOCK(pGVM);
682	AssertRCReturn(rc, rc);
683
684	/* The caller might have done this already, but since we're ring-3 callable we
685	need to make sure everything is fine before starting the allocation here. */
686	for (unsigned i = 0; i < _2M / GUEST_PAGE_SIZE; i++)
687	{
688	PPGMPAGE pPage;
689	rc = pgmPhysGetPageEx(pGVM, GCPhys + i * GUEST_PAGE_SIZE, &pPage);
690	AssertRCReturnStmt(rc, PGM_UNLOCK(pGVM), rc);
691	AssertReturnStmt(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM, PGM_UNLOCK(pGVM), VERR_PGM_PHYS_NOT_RAM);
692	AssertReturnStmt(PGM_PAGE_IS_ZERO(pPage), PGM_UNLOCK(pGVM), VERR_PGM_UNEXPECTED_PAGE_STATE);
693	}
694
695	/*
696	* Call common code.
697	*/
698	rc = pgmR0PhysAllocateLargePage(pGVM, idCpu, GCPhys);
699
700	PGM_UNLOCK(pGVM);
701	return rc;
702	}
703
704
705	/**
706	* Locate a MMIO2 range.
707	*
708	* @returns Pointer to the MMIO2 range.
709	* @param pGVM The global (ring-0) VM structure.
710	* @param pDevIns The device instance owning the region.
711	* @param hMmio2 Handle to look up.
712	*/
713	DECLINLINE(PPGMREGMMIO2RANGE) pgmR0PhysMmio2Find(PGVM pGVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2)
714	{
715	/*
716	* We use the lookup table here as list walking is tedious in ring-0 when using
717	* ring-3 pointers and this probably will require some kind of refactoring anyway.
718	*/
719	if (hMmio2 <= RT_ELEMENTS(pGVM->pgm.s.apMmio2RangesR0) && hMmio2 != 0)
720	{
721	PPGMREGMMIO2RANGE pCur = pGVM->pgm.s.apMmio2RangesR0[hMmio2 - 1];
722	if (pCur && pCur->pDevInsR3 == pDevIns->pDevInsForR3)
723	{
724	Assert(pCur->idMmio2 == hMmio2);
725	return pCur;
726	}
727	Assert(!pCur);
728	}
729	return NULL;
730	}
731
732
733	/**
734	* Worker for PDMDEVHLPR0::pfnMmio2SetUpContext.
735	*
736	* @returns VBox status code.
737	* @param pGVM The global (ring-0) VM structure.
738	* @param pDevIns The device instance.
739	* @param hMmio2 The MMIO2 region to map into ring-0 address space.
740	* @param offSub The offset into the region.
741	* @param cbSub The size of the mapping, zero meaning all the rest.
742	* @param ppvMapping Where to return the ring-0 mapping address.
743	*/
744	VMMR0_INT_DECL(int) PGMR0PhysMMIO2MapKernel(PGVM pGVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2,
745	size_t offSub, size_t cbSub, void **ppvMapping)
746	{
747	AssertReturn(!(offSub & HOST_PAGE_OFFSET_MASK), VERR_UNSUPPORTED_ALIGNMENT);
748	AssertReturn(!(cbSub & HOST_PAGE_OFFSET_MASK), VERR_UNSUPPORTED_ALIGNMENT);
749
750	/*
751	* Translate hRegion into a range pointer.
752	*/
753	PPGMREGMMIO2RANGE pFirstRegMmio = pgmR0PhysMmio2Find(pGVM, pDevIns, hMmio2);
754	AssertReturn(pFirstRegMmio, VERR_NOT_FOUND);
755	#ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
756	uint8_t * const pvR0 = (uint8_t *)pFirstRegMmio->pvR0;
757	#else
758	RTR3PTR const pvR3 = pFirstRegMmio->pvR3;
759	#endif
760	RTGCPHYS const cbReal = pFirstRegMmio->cbReal;
761	pFirstRegMmio = NULL;
762	ASMCompilerBarrier();
763
764	AssertReturn(offSub < cbReal, VERR_OUT_OF_RANGE);
765	if (cbSub == 0)
766	cbSub = cbReal - offSub;
767	else
768	AssertReturn(cbSub < cbReal && cbSub + offSub <= cbReal, VERR_OUT_OF_RANGE);
769
770	/*
771	* Do the mapping.
772	*/
773	#ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
774	AssertPtr(pvR0);
775	*ppvMapping = pvR0 + offSub;
776	return VINF_SUCCESS;
777	#else
778	return SUPR0PageMapKernel(pGVM->pSession, pvR3, (uint32_t)offSub, (uint32_t)cbSub, 0 /fFlags/, ppvMapping);
779	#endif
780	}
781
782
783	/**
784	* This is called during PGMR3Init to init the physical access handler allocator
785	* and tree.
786	*
787	* @returns VBox status code.
788	* @param pGVM Pointer to the global VM structure.
789	* @param cEntries Desired number of physical access handlers to reserve
790	* space for (will be adjusted).
791	* @thread EMT(0)
792	*/
793	VMMR0_INT_DECL(int) PGMR0PhysHandlerInitReqHandler(PGVM pGVM, uint32_t cEntries)
794	{
795	/*
796	* Validate the input and state.
797	*/
798	int rc = GVMMR0ValidateGVMandEMT(pGVM, 0);
799	AssertRCReturn(rc, rc);
800	VM_ASSERT_STATE_RETURN(pGVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE); /** @todo ring-0 safe state check. */
801
802	AssertReturn(pGVM->pgmr0.s.PhysHandlerAllocator.m_paNodes == NULL, VERR_WRONG_ORDER);
803	AssertReturn(pGVM->pgm.s.PhysHandlerAllocator.m_paNodes == NULL, VERR_WRONG_ORDER);
804
805	AssertLogRelMsgReturn(cEntries <= _64K, ("%#x\n", cEntries), VERR_OUT_OF_RANGE);
806
807	/*
808	* Calculate the table size and allocate it.
809	*/
810	uint32_t cbTreeAndBitmap = 0;
811	uint32_t const cbTotalAligned = pgmHandlerPhysicalCalcTableSizes(&cEntries, &cbTreeAndBitmap);
812	RTR0MEMOBJ hMemObj = NIL_RTR0MEMOBJ;
813	rc = RTR0MemObjAllocPage(&hMemObj, cbTotalAligned, false);
814	if (RT_SUCCESS(rc))
815	{
816	RTR0MEMOBJ hMapObj = NIL_RTR0MEMOBJ;
817	rc = RTR0MemObjMapUser(&hMapObj, hMemObj, (RTR3PTR)-1, 0, RTMEM_PROT_READ \| RTMEM_PROT_WRITE, RTR0ProcHandleSelf());
818	if (RT_SUCCESS(rc))
819	{
820	uint8_t pb = (uint8_t )RTR0MemObjAddress(hMemObj);
821	if (!RTR0MemObjWasZeroInitialized(hMemObj))
822	RT_BZERO(pb, cbTotalAligned);
823
824	pGVM->pgmr0.s.PhysHandlerAllocator.initSlabAllocator(cEntries, (PPGMPHYSHANDLER)&pb[cbTreeAndBitmap],
825	(uint64_t *)&pb[sizeof(PGMPHYSHANDLERTREE)]);
826	pGVM->pgmr0.s.pPhysHandlerTree = (PPGMPHYSHANDLERTREE)pb;
827	pGVM->pgmr0.s.pPhysHandlerTree->initWithAllocator(&pGVM->pgmr0.s.PhysHandlerAllocator);
828	pGVM->pgmr0.s.hPhysHandlerMemObj = hMemObj;
829	pGVM->pgmr0.s.hPhysHandlerMapObj = hMapObj;
830
831	AssertCompile(sizeof(pGVM->pgm.s.PhysHandlerAllocator) == sizeof(pGVM->pgmr0.s.PhysHandlerAllocator));
832	RTR3PTR R3Ptr = RTR0MemObjAddressR3(hMapObj);
833	pGVM->pgm.s.pPhysHandlerTree = R3Ptr;
834	pGVM->pgm.s.PhysHandlerAllocator.m_paNodes = R3Ptr + cbTreeAndBitmap;
835	pGVM->pgm.s.PhysHandlerAllocator.m_pbmAlloc = R3Ptr + sizeof(PGMPHYSHANDLERTREE);
836	pGVM->pgm.s.PhysHandlerAllocator.m_cNodes = cEntries;
837	pGVM->pgm.s.PhysHandlerAllocator.m_cErrors = 0;
838	pGVM->pgm.s.PhysHandlerAllocator.m_idxAllocHint = 0;
839	pGVM->pgm.s.PhysHandlerAllocator.m_uPadding = 0;
840	return VINF_SUCCESS;
841	}
842
843	RTR0MemObjFree(hMemObj, true /fFreeMappings/);
844	}
845	return rc;
846	}
847
848
849	/**
850	* Updates a physical access handler type with ring-0 callback functions.
851	*
852	* The handler type must first have been registered in ring-3.
853	*
854	* @returns VBox status code.
855	* @param pGVM The global (ring-0) VM structure.
856	* @param enmKind The kind of access handler.
857	* @param fFlags PGMPHYSHANDLER_F_XXX
858	* @param pfnHandler Pointer to the ring-0 handler callback.
859	* @param pfnPfHandler Pointer to the ring-0 \#PF handler callback.
860	* callback. Can be NULL (not recommended though).
861	* @param pszDesc The type description.
862	* @param hType The handle to do ring-0 callback registrations for.
863	* @thread EMT(0)
864	*/
865	VMMR0_INT_DECL(int) PGMR0HandlerPhysicalTypeSetUpContext(PGVM pGVM, PGMPHYSHANDLERKIND enmKind, uint32_t fFlags,
866	PFNPGMPHYSHANDLER pfnHandler, PFNPGMRZPHYSPFHANDLER pfnPfHandler,
867	const char *pszDesc, PGMPHYSHANDLERTYPE hType)
868	{
869	/*
870	* Validate input.
871	*/
872	AssertPtrReturn(pfnHandler, VERR_INVALID_POINTER);
873	AssertPtrNullReturn(pfnPfHandler, VERR_INVALID_POINTER);
874
875	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
876	AssertReturn( enmKind == PGMPHYSHANDLERKIND_WRITE
877	\|\| enmKind == PGMPHYSHANDLERKIND_ALL
878	\|\| enmKind == PGMPHYSHANDLERKIND_MMIO,
879	VERR_INVALID_PARAMETER);
880	AssertMsgReturn(!(fFlags & ~PGMPHYSHANDLER_F_VALID_MASK), ("%#x\n", fFlags), VERR_INVALID_FLAGS);
881
882	PPGMPHYSHANDLERTYPEINTR0 const pTypeR0 = &pGVM->pgmr0.s.aPhysHandlerTypes[hType & PGMPHYSHANDLERTYPE_IDX_MASK];
883	AssertMsgReturn(hType == pTypeR0->hType, ("%#RX64, expected=%#RX64\n", hType, pTypeR0->hType), VERR_INVALID_HANDLE);
884	AssertCompile(RT_ELEMENTS(pGVM->pgmr0.s.aPhysHandlerTypes) == RT_ELEMENTS(pGVM->pgm.s.aPhysHandlerTypes));
885	AssertCompile(RT_ELEMENTS(pGVM->pgmr0.s.aPhysHandlerTypes) == PGMPHYSHANDLERTYPE_IDX_MASK + 1);
886	AssertReturn(pTypeR0->enmKind == PGMPHYSHANDLERKIND_INVALID, VERR_ALREADY_INITIALIZED);
887
888	int rc = GVMMR0ValidateGVMandEMT(pGVM, 0);
889	AssertRCReturn(rc, rc);
890	VM_ASSERT_STATE_RETURN(pGVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE); /** @todo ring-0 safe state check. */
891
892	PPGMPHYSHANDLERTYPEINTR3 const pTypeR3 = &pGVM->pgm.s.aPhysHandlerTypes[hType & PGMPHYSHANDLERTYPE_IDX_MASK];
893	AssertMsgReturn(pTypeR3->enmKind == enmKind,
894	("%#x: %d, expected %d\n", hType, pTypeR3->enmKind, enmKind),
895	VERR_INVALID_HANDLE);
896	AssertMsgReturn(pTypeR3->fKeepPgmLock == RT_BOOL(fFlags & PGMPHYSHANDLER_F_KEEP_PGM_LOCK),
897	("%#x: %d, fFlags=%#x\n", hType, pTypeR3->fKeepPgmLock, fFlags),
898	VERR_INVALID_HANDLE);
899	AssertMsgReturn(pTypeR3->fRing0DevInsIdx == RT_BOOL(fFlags & PGMPHYSHANDLER_F_R0_DEVINS_IDX),
900	("%#x: %d, fFlags=%#x\n", hType, pTypeR3->fRing0DevInsIdx, fFlags),
901	VERR_INVALID_HANDLE);
902	AssertMsgReturn(pTypeR3->fNotInHm == RT_BOOL(fFlags & PGMPHYSHANDLER_F_NOT_IN_HM),
903	("%#x: %d, fFlags=%#x\n", hType, pTypeR3->fNotInHm, fFlags),
904	VERR_INVALID_HANDLE);
905
906	/*
907	* Update the entry.
908	*/
909	pTypeR0->enmKind = enmKind;
910	pTypeR0->uState = enmKind == PGMPHYSHANDLERKIND_WRITE
911	? PGM_PAGE_HNDL_PHYS_STATE_WRITE : PGM_PAGE_HNDL_PHYS_STATE_ALL;
912	pTypeR0->fKeepPgmLock = RT_BOOL(fFlags & PGMPHYSHANDLER_F_KEEP_PGM_LOCK);
913	pTypeR0->fRing0DevInsIdx = RT_BOOL(fFlags & PGMPHYSHANDLER_F_R0_DEVINS_IDX);
914	pTypeR0->fNotInHm = RT_BOOL(fFlags & PGMPHYSHANDLER_F_NOT_IN_HM);
915	pTypeR0->pfnHandler = pfnHandler;
916	pTypeR0->pfnPfHandler = pfnPfHandler;
917	pTypeR0->pszDesc = pszDesc;
918
919	pTypeR3->fRing0Enabled = true;
920
921	LogFlow(("PGMR0HandlerPhysicalTypeRegister: hType=%#x: enmKind=%d fFlags=%#x pfnHandler=%p pfnPfHandler=%p pszDesc=%s\n",
922	hType, enmKind, fFlags, pfnHandler, pfnPfHandler, pszDesc));
923	return VINF_SUCCESS;
924	}
925
926
927	#ifdef VBOX_WITH_PCI_PASSTHROUGH
928	/* Interface sketch. The interface belongs to a global PCI pass-through
929	manager. It shall use the global VM handle, not the user VM handle to
930	store the per-VM info (domain) since that is all ring-0 stuff, thus
931	passing pGVM here. I've tentitively prefixed the functions 'GPciRawR0',
932	we can discuss the PciRaw code re-organtization when I'm back from
933	vacation.
934
935	I've implemented the initial IOMMU set up below. For things to work
936	reliably, we will probably need add a whole bunch of checks and
937	GPciRawR0GuestPageUpdate call to the PGM code. For the present,
938	assuming nested paging (enforced) and prealloc (enforced), no
939	ballooning (check missing), page sharing (check missing) or live
940	migration (check missing), it might work fine. At least if some
941	VM power-off hook is present and can tear down the IOMMU page tables. */
942
943	/**
944	* Tells the global PCI pass-through manager that we are about to set up the
945	* guest page to host page mappings for the specfied VM.
946	*
947	* @returns VBox status code.
948	*
949	* @param pGVM The ring-0 VM structure.
950	*/
951	VMMR0_INT_DECL(int) GPciRawR0GuestPageBeginAssignments(PGVM pGVM)
952	{
953	NOREF(pGVM);
954	return VINF_SUCCESS;
955	}
956
957
958	/**
959	* Assigns a host page mapping for a guest page.
960	*
961	* This is only used when setting up the mappings, i.e. between
962	* GPciRawR0GuestPageBeginAssignments and GPciRawR0GuestPageEndAssignments.
963	*
964	* @returns VBox status code.
965	* @param pGVM The ring-0 VM structure.
966	* @param GCPhys The address of the guest page (page aligned).
967	* @param HCPhys The address of the host page (page aligned).
968	*/
969	VMMR0_INT_DECL(int) GPciRawR0GuestPageAssign(PGVM pGVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys)
970	{
971	AssertReturn(!(GCPhys & HOST_PAGE_OFFSET_MASK), VERR_INTERNAL_ERROR_3);
972	AssertReturn(!(HCPhys & HOST_PAGE_OFFSET_MASK), VERR_INTERNAL_ERROR_3);
973
974	if (pGVM->rawpci.s.pfnContigMemInfo)
975	/** @todo what do we do on failure? */
976	pGVM->rawpci.s.pfnContigMemInfo(&pGVM->rawpci.s, HCPhys, GCPhys, HOST_PAGE_SIZE, PCIRAW_MEMINFO_MAP);
977
978	return VINF_SUCCESS;
979	}
980
981
982	/**
983	* Indicates that the specified guest page doesn't exists but doesn't have host
984	* page mapping we trust PCI pass-through with.
985	*
986	* This is only used when setting up the mappings, i.e. between
987	* GPciRawR0GuestPageBeginAssignments and GPciRawR0GuestPageEndAssignments.
988	*
989	* @returns VBox status code.
990	* @param pGVM The ring-0 VM structure.
991	* @param GCPhys The address of the guest page (page aligned).
992	* @param HCPhys The address of the host page (page aligned).
993	*/
994	VMMR0_INT_DECL(int) GPciRawR0GuestPageUnassign(PGVM pGVM, RTGCPHYS GCPhys)
995	{
996	AssertReturn(!(GCPhys & HOST_PAGE_OFFSET_MASK), VERR_INTERNAL_ERROR_3);
997
998	if (pGVM->rawpci.s.pfnContigMemInfo)
999	/** @todo what do we do on failure? */
1000	pGVM->rawpci.s.pfnContigMemInfo(&pGVM->rawpci.s, 0, GCPhys, HOST_PAGE_SIZE, PCIRAW_MEMINFO_UNMAP);
1001
1002	return VINF_SUCCESS;
1003	}
1004
1005
1006	/**
1007	* Tells the global PCI pass-through manager that we have completed setting up
1008	* the guest page to host page mappings for the specfied VM.
1009	*
1010	* This complements GPciRawR0GuestPageBeginAssignments and will be called even
1011	* if some page assignment failed.
1012	*
1013	* @returns VBox status code.
1014	*
1015	* @param pGVM The ring-0 VM structure.
1016	*/
1017	VMMR0_INT_DECL(int) GPciRawR0GuestPageEndAssignments(PGVM pGVM)
1018	{
1019	NOREF(pGVM);
1020	return VINF_SUCCESS;
1021	}
1022
1023
1024	/**
1025	* Tells the global PCI pass-through manager that a guest page mapping has
1026	* changed after the initial setup.
1027	*
1028	* @returns VBox status code.
1029	* @param pGVM The ring-0 VM structure.
1030	* @param GCPhys The address of the guest page (page aligned).
1031	* @param HCPhys The new host page address or NIL_RTHCPHYS if
1032	* now unassigned.
1033	*/
1034	VMMR0_INT_DECL(int) GPciRawR0GuestPageUpdate(PGVM pGVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys)
1035	{
1036	AssertReturn(!(GCPhys & HOST_PAGE_OFFSET_MASK), VERR_INTERNAL_ERROR_4);
1037	AssertReturn(!(HCPhys & HOST_PAGE_OFFSET_MASK) \|\| HCPhys == NIL_RTHCPHYS, VERR_INTERNAL_ERROR_4);
1038	NOREF(pGVM);
1039	return VINF_SUCCESS;
1040	}
1041
1042	#endif /* VBOX_WITH_PCI_PASSTHROUGH */
1043
1044
1045	/**
1046	* Sets up the IOMMU when raw PCI device is enabled.
1047	*
1048	* @note This is a hack that will probably be remodelled and refined later!
1049	*
1050	* @returns VBox status code.
1051	*
1052	* @param pGVM The global (ring-0) VM structure.
1053	*/
1054	VMMR0_INT_DECL(int) PGMR0PhysSetupIoMmu(PGVM pGVM)
1055	{
1056	int rc = GVMMR0ValidateGVM(pGVM);
1057	if (RT_FAILURE(rc))
1058	return rc;
1059
1060	#ifdef VBOX_WITH_PCI_PASSTHROUGH
1061	if (pGVM->pgm.s.fPciPassthrough)
1062	{
1063	/*
1064	* The Simplistic Approach - Enumerate all the pages and call tell the
1065	* IOMMU about each of them.
1066	*/
1067	PGM_LOCK_VOID(pGVM);
1068	rc = GPciRawR0GuestPageBeginAssignments(pGVM);
1069	if (RT_SUCCESS(rc))
1070	{
1071	for (PPGMRAMRANGE pRam = pGVM->pgm.s.pRamRangesXR0; RT_SUCCESS(rc) && pRam; pRam = pRam->pNextR0)
1072	{
1073	PPGMPAGE pPage = &pRam->aPages[0];
1074	RTGCPHYS GCPhys = pRam->GCPhys;
1075	uint32_t cLeft = pRam->cb >> GUEST_PAGE_SHIFT;
1076	while (cLeft-- > 0)
1077	{
1078	/* Only expose pages that are 100% safe for now. */
1079	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1080	&& PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED
1081	&& !PGM_PAGE_HAS_ANY_HANDLERS(pPage))
1082	rc = GPciRawR0GuestPageAssign(pGVM, GCPhys, PGM_PAGE_GET_HCPHYS(pPage));
1083	else
1084	rc = GPciRawR0GuestPageUnassign(pGVM, GCPhys);
1085
1086	/* next */
1087	pPage++;
1088	GCPhys += HOST_PAGE_SIZE;
1089	}
1090	}
1091
1092	int rc2 = GPciRawR0GuestPageEndAssignments(pGVM);
1093	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
1094	rc = rc2;
1095	}
1096	PGM_UNLOCK(pGVM);
1097	}
1098	else
1099	#endif
1100	rc = VERR_NOT_SUPPORTED;
1101	return rc;
1102	}
1103
1104
1105	/**
1106	* \#PF Handler for nested paging.
1107	*
1108	* @returns VBox status code (appropriate for trap handling and GC return).
1109	* @param pGVM The global (ring-0) VM structure.
1110	* @param pGVCpu The global (ring-0) CPU structure of the calling
1111	* EMT.
1112	* @param enmShwPagingMode Paging mode for the nested page tables.
1113	* @param uErr The trap error code.
1114	* @param pCtx Pointer to the register context for the CPU.
1115	* @param GCPhysFault The fault address.
1116	*/
1117	VMMR0DECL(int) PGMR0Trap0eHandlerNestedPaging(PGVM pGVM, PGVMCPU pGVCpu, PGMMODE enmShwPagingMode, RTGCUINT uErr,
1118	PCPUMCTX pCtx, RTGCPHYS GCPhysFault)
1119	{
1120	int rc;
1121
1122	LogFlow(("PGMTrap0eHandler: uErr=%RGx GCPhysFault=%RGp eip=%RGv\n", uErr, GCPhysFault, (RTGCPTR)pCtx->rip));
1123	STAM_PROFILE_START(&pGVCpu->pgm.s.StatRZTrap0e, a);
1124	STAM_STATS({ pGVCpu->pgmr0.s.pStatTrap0eAttributionR0 = NULL; } );
1125
1126	/* AMD uses the host's paging mode; Intel has a single mode (EPT). */
1127	AssertMsg( enmShwPagingMode == PGMMODE_32_BIT \|\| enmShwPagingMode == PGMMODE_PAE \|\| enmShwPagingMode == PGMMODE_PAE_NX
1128	\|\| enmShwPagingMode == PGMMODE_AMD64 \|\| enmShwPagingMode == PGMMODE_AMD64_NX \|\| enmShwPagingMode == PGMMODE_EPT,
1129	("enmShwPagingMode=%d\n", enmShwPagingMode));
1130
1131	/* Reserved shouldn't end up here. */
1132	Assert(!(uErr & X86_TRAP_PF_RSVD));
1133
1134	#ifdef VBOX_WITH_STATISTICS
1135	/*
1136	* Error code stats.
1137	*/
1138	if (uErr & X86_TRAP_PF_US)
1139	{
1140	if (!(uErr & X86_TRAP_PF_P))
1141	{
1142	if (uErr & X86_TRAP_PF_RW)
1143	STAM_COUNTER_INC(&pGVCpu->pgm.s.Stats.StatRZTrap0eUSNotPresentWrite);
1144	else
1145	STAM_COUNTER_INC(&pGVCpu->pgm.s.Stats.StatRZTrap0eUSNotPresentRead);
1146	}
1147	else if (uErr & X86_TRAP_PF_RW)
1148	STAM_COUNTER_INC(&pGVCpu->pgm.s.Stats.StatRZTrap0eUSWrite);
1149	else if (uErr & X86_TRAP_PF_RSVD)
1150	STAM_COUNTER_INC(&pGVCpu->pgm.s.Stats.StatRZTrap0eUSReserved);
1151	else if (uErr & X86_TRAP_PF_ID)
1152	STAM_COUNTER_INC(&pGVCpu->pgm.s.Stats.StatRZTrap0eUSNXE);
1153	else
1154	STAM_COUNTER_INC(&pGVCpu->pgm.s.Stats.StatRZTrap0eUSRead);
1155	}
1156	else
1157	{ /* Supervisor */
1158	if (!(uErr & X86_TRAP_PF_P))
1159	{
1160	if (uErr & X86_TRAP_PF_RW)
1161	STAM_COUNTER_INC(&pGVCpu->pgm.s.Stats.StatRZTrap0eSVNotPresentWrite);
1162	else
1163	STAM_COUNTER_INC(&pGVCpu->pgm.s.Stats.StatRZTrap0eSVNotPresentRead);
1164	}
1165	else if (uErr & X86_TRAP_PF_RW)
1166	STAM_COUNTER_INC(&pGVCpu->pgm.s.Stats.StatRZTrap0eSVWrite);
1167	else if (uErr & X86_TRAP_PF_ID)
1168	STAM_COUNTER_INC(&pGVCpu->pgm.s.Stats.StatRZTrap0eSNXE);
1169	else if (uErr & X86_TRAP_PF_RSVD)
1170	STAM_COUNTER_INC(&pGVCpu->pgm.s.Stats.StatRZTrap0eSVReserved);
1171	}
1172	#endif
1173
1174	/*
1175	* Call the worker.
1176	*
1177	* Note! We pretend the guest is in protected mode without paging, so we
1178	* can use existing code to build the nested page tables.
1179	*/
1180	/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
1181	bool fLockTaken = false;
1182	switch (enmShwPagingMode)
1183	{
1184	case PGMMODE_32_BIT:
1185	rc = PGM_BTH_NAME_32BIT_PROT(Trap0eHandler)(pGVCpu, uErr, pCtx, GCPhysFault, &fLockTaken);
1186	break;
1187	case PGMMODE_PAE:
1188	case PGMMODE_PAE_NX:
1189	rc = PGM_BTH_NAME_PAE_PROT(Trap0eHandler)(pGVCpu, uErr, pCtx, GCPhysFault, &fLockTaken);
1190	break;
1191	case PGMMODE_AMD64:
1192	case PGMMODE_AMD64_NX:
1193	rc = PGM_BTH_NAME_AMD64_PROT(Trap0eHandler)(pGVCpu, uErr, pCtx, GCPhysFault, &fLockTaken);
1194	break;
1195	case PGMMODE_EPT:
1196	rc = PGM_BTH_NAME_EPT_PROT(Trap0eHandler)(pGVCpu, uErr, pCtx, GCPhysFault, &fLockTaken);
1197	break;
1198	default:
1199	AssertFailed();
1200	rc = VERR_INVALID_PARAMETER;
1201	break;
1202	}
1203	if (fLockTaken)
1204	{
1205	PGM_LOCK_ASSERT_OWNER(pGVM);
1206	PGM_UNLOCK(pGVM);
1207	}
1208
1209	if (rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
1210	rc = VINF_SUCCESS;
1211	/*
1212	* Handle the case where we cannot interpret the instruction because we cannot get the guest physical address
1213	* via its page tables, see @bugref{6043}.
1214	*/
1215	else if ( rc == VERR_PAGE_NOT_PRESENT /* SMP only ; disassembly might fail. */
1216	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT /* seen with UNI & SMP */
1217	\|\| rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT /* seen with SMP */
1218	\|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT) /* precaution */
1219	{
1220	Log(("WARNING: Unexpected VERR_PAGE_TABLE_NOT_PRESENT (%d) for page fault at %RGp error code %x (rip=%RGv)\n", rc, GCPhysFault, uErr, pCtx->rip));
1221	/* Some kind of inconsistency in the SMP case; it's safe to just execute the instruction again; not sure about
1222	single VCPU VMs though. */
1223	rc = VINF_SUCCESS;
1224	}
1225
1226	STAM_STATS({ if (!pGVCpu->pgmr0.s.pStatTrap0eAttributionR0)
1227	pGVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pGVCpu->pgm.s.Stats.StatRZTrap0eTime2Misc; });
1228	STAM_PROFILE_STOP_EX(&pGVCpu->pgm.s.Stats.StatRZTrap0e, pGVCpu->pgmr0.s.pStatTrap0eAttributionR0, a);
1229	return rc;
1230	}
1231
1232
1233	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1234	/**
1235	* Nested \#PF Handler for nested-guest execution using nested paging.
1236	*
1237	* @returns Strict VBox status code (appropriate for trap handling and GC return).
1238	* @param pGVM The global (ring-0) VM structure.
1239	* @param pGVCpu The global (ring-0) CPU structure of the calling
1240	* EMT.
1241	* @param uErr The trap error code.
1242	* @param pCtx Pointer to the register context for the CPU.
1243	* @param GCPhysNestedFault The nested-guest physical address causing the fault.
1244	* @param fIsLinearAddrValid Whether translation of a nested-guest linear address
1245	* caused this fault. If @c false, GCPtrNestedFault
1246	* must be 0.
1247	* @param GCPtrNestedFault The nested-guest linear address that caused this
1248	* fault.
1249	* @param pWalk Where to store the SLAT walk result.
1250	*/
1251	VMMR0DECL(VBOXSTRICTRC) PGMR0NestedTrap0eHandlerNestedPaging(PGVMCPU pGVCpu, PGMMODE enmShwPagingMode, RTGCUINT uErr,
1252	PCPUMCTX pCtx, RTGCPHYS GCPhysNestedFault,
1253	bool fIsLinearAddrValid, RTGCPTR GCPtrNestedFault, PPGMPTWALK pWalk)
1254	{
1255	Assert(enmShwPagingMode == PGMMODE_EPT);
1256	NOREF(enmShwPagingMode);
1257
1258	bool fLockTaken;
1259	VBOXSTRICTRC rcStrict = PGM_BTH_NAME_EPT_PROT(NestedTrap0eHandler)(pGVCpu, uErr, pCtx, GCPhysNestedFault,
1260	fIsLinearAddrValid, GCPtrNestedFault, pWalk, &fLockTaken);
1261	if (fLockTaken)
1262	{
1263	PGM_LOCK_ASSERT_OWNER(pGVCpu->CTX_SUFF(pVM));
1264	PGM_UNLOCK(pGVCpu->CTX_SUFF(pVM));
1265	}
1266	Assert(rcStrict != VINF_PGM_SYNCPAGE_MODIFIED_PDE); /* This rc isn't used with Nested Paging and nested-EPT. */
1267	return rcStrict;
1268	}
1269	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
1270
1271
1272	/**
1273	* \#PF Handler for deliberate nested paging misconfiguration (/reserved bit)
1274	* employed for MMIO pages.
1275	*
1276	* @returns VBox status code (appropriate for trap handling and GC return).
1277	* @param pGVM The global (ring-0) VM structure.
1278	* @param pGVCpu The global (ring-0) CPU structure of the calling
1279	* EMT.
1280	* @param enmShwPagingMode Paging mode for the nested page tables.
1281	* @param pCtx Pointer to the register context for the CPU.
1282	* @param GCPhysFault The fault address.
1283	* @param uErr The error code, UINT32_MAX if not available
1284	* (VT-x).
1285	*/
1286	VMMR0DECL(VBOXSTRICTRC) PGMR0Trap0eHandlerNPMisconfig(PGVM pGVM, PGVMCPU pGVCpu, PGMMODE enmShwPagingMode,
1287	PCPUMCTX pCtx, RTGCPHYS GCPhysFault, uint32_t uErr)
1288	{
1289	#ifdef PGM_WITH_MMIO_OPTIMIZATIONS
1290	STAM_PROFILE_START(&pGVCpu->CTX_SUFF(pStats)->StatR0NpMiscfg, a);
1291	VBOXSTRICTRC rc;
1292
1293	/*
1294	* Try lookup the all access physical handler for the address.
1295	*/
1296	PGM_LOCK_VOID(pGVM);
1297	PPGMPHYSHANDLER pHandler;
1298	rc = pgmHandlerPhysicalLookup(pGVM, GCPhysFault, &pHandler);
1299	if (RT_SUCCESS(rc))
1300	{
1301	PCPGMPHYSHANDLERTYPEINT pHandlerType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pGVM, pHandler);
1302	if (RT_LIKELY( pHandlerType->enmKind != PGMPHYSHANDLERKIND_WRITE
1303	&& !pHandlerType->fNotInHm /paranoia/ ))
1304	{
1305	/*
1306	* If the handle has aliases page or pages that have been temporarily
1307	* disabled, we'll have to take a detour to make sure we resync them
1308	* to avoid lots of unnecessary exits.
1309	*/
1310	PPGMPAGE pPage;
1311	if ( ( pHandler->cAliasedPages
1312	\|\| pHandler->cTmpOffPages)
1313	&& ( (pPage = pgmPhysGetPage(pGVM, GCPhysFault)) == NULL
1314	\|\| PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) == PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
1315	)
1316	{
1317	Log(("PGMR0Trap0eHandlerNPMisconfig: Resyncing aliases / tmp-off page at %RGp (uErr=%#x) %R[pgmpage]\n", GCPhysFault, uErr, pPage));
1318	STAM_COUNTER_INC(&pGVCpu->pgm.s.Stats.StatR0NpMiscfgSyncPage);
1319	rc = pgmShwSyncNestedPageLocked(pGVCpu, GCPhysFault, 1 /cPages/, enmShwPagingMode);
1320	PGM_UNLOCK(pGVM);
1321	}
1322	else
1323	{
1324	if (pHandlerType->pfnPfHandler)
1325	{
1326	uint64_t const uUser = !pHandlerType->fRing0DevInsIdx ? pHandler->uUser
1327	: (uintptr_t)PDMDeviceRing0IdxToInstance(pGVM, pHandler->uUser);
1328	STAM_PROFILE_START(&pHandler->Stat, h);
1329	PGM_UNLOCK(pGVM);
1330
1331	Log6(("PGMR0Trap0eHandlerNPMisconfig: calling %p(,%#x,,%RGp,%p)\n", pHandlerType->pfnPfHandler, uErr, GCPhysFault, uUser));
1332	rc = pHandlerType->pfnPfHandler(pGVM, pGVCpu, uErr == UINT32_MAX ? RTGCPTR_MAX : uErr, pCtx,
1333	GCPhysFault, GCPhysFault, uUser);
1334
1335	STAM_PROFILE_STOP(&pHandler->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
1336	}
1337	else
1338	{
1339	PGM_UNLOCK(pGVM);
1340	Log(("PGMR0Trap0eHandlerNPMisconfig: %RGp (uErr=%#x) -> R3\n", GCPhysFault, uErr));
1341	rc = VINF_EM_RAW_EMULATE_INSTR;
1342	}
1343	}
1344	STAM_PROFILE_STOP(&pGVCpu->pgm.s.Stats.StatR0NpMiscfg, a);
1345	return rc;
1346	}
1347	}
1348	else
1349	AssertMsgReturn(rc == VERR_NOT_FOUND, ("%Rrc GCPhysFault=%RGp\n", VBOXSTRICTRC_VAL(rc), GCPhysFault), rc);
1350
1351	/*
1352	* Must be out of sync, so do a SyncPage and restart the instruction.
1353	*
1354	* ASSUMES that ALL handlers are page aligned and covers whole pages
1355	* (assumption asserted in PGMHandlerPhysicalRegisterEx).
1356	*/
1357	Log(("PGMR0Trap0eHandlerNPMisconfig: Out of sync page at %RGp (uErr=%#x)\n", GCPhysFault, uErr));
1358	STAM_COUNTER_INC(&pGVCpu->pgm.s.Stats.StatR0NpMiscfgSyncPage);
1359	rc = pgmShwSyncNestedPageLocked(pGVCpu, GCPhysFault, 1 /cPages/, enmShwPagingMode);
1360	PGM_UNLOCK(pGVM);
1361
1362	STAM_PROFILE_STOP(&pGVCpu->pgm.s.Stats.StatR0NpMiscfg, a);
1363	return rc;
1364
1365	#else
1366	AssertLogRelFailed();
1367	return VERR_PGM_NOT_USED_IN_MODE;
1368	#endif
1369	}
1370

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/PGMR0.cpp@ 99725

Download in other formats: