PGMAllBth.h@ 31167

Last change on this file since 31167 was 31167, checked in by vboxsync, 14 years ago
PGM: Pass pVCpu instead of pPGM (PPGMCPU) and eliminate PGMPOOL_PAGE_2_PTR_BY_PGM and PGMPOOL_PAGE_2_PTR_BY_PGMCPU.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 197.7 KB

Line
1	/* $Id: PGMAllBth.h 31167 2010-07-28 14:38:43Z vboxsync $ */
2	/** @file
3	* VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4	*
5	* @remarks The nested page tables on AMD makes use of PGM_SHW_TYPE in
6	* {PGM_TYPE_AMD64, PGM_TYPE_PAE and PGM_TYPE_32BIT} and PGM_GST_TYPE
7	* set to PGM_TYPE_PROT. Half of the code in this file is not
8	* exercised with PGM_SHW_TYPE set to PGM_TYPE_NESTED.
9	*
10	* @remarks Extended page tables (intel) are built with PGM_GST_TYPE set to
11	* PGM_TYPE_PROT (and PGM_SHW_TYPE set to PGM_TYPE_EPT).
12	*
13	* @remarks This file is one big \#ifdef-orgy!
14	*
15	*/
16
17	/*
18	* Copyright (C) 2006-2010 Oracle Corporation
19	*
20	* This file is part of VirtualBox Open Source Edition (OSE), as
21	* available from http://www.virtualbox.org. This file is free software;
22	* you can redistribute it and/or modify it under the terms of the GNU
23	* General Public License (GPL) as published by the Free Software
24	* Foundation, in version 2 as it comes in the "COPYING" file of the
25	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
26	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
27	*/
28
29
30	/*******************************************************************************
31	* Internal Functions *
32	*******************************************************************************/
33	RT_C_DECLS_BEGIN
34	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken);
35	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
36	static int PGM_BTH_NAME(SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr);
37	static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, RTGCPTR GCPtrPage);
38	static int PGM_BTH_NAME(SyncPT)(PVMCPU pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage);
39	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uErr);
40	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
41	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
42	#ifdef VBOX_STRICT
43	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
44	#endif
45	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3);
46	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu);
47	RT_C_DECLS_END
48
49
50	/*
51	* Filter out some illegal combinations of guest and shadow paging, so we can
52	* remove redundant checks inside functions.
53	*/
54	#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
55	# error "Invalid combination; PAE guest implies PAE shadow"
56	#endif
57
58	#if (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
59	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT)
60	# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
61	#endif
62
63	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) \
64	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT)
65	# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
66	#endif
67
68	#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT) \
69	\|\| (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
70	# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
71	#endif
72
73	#ifndef IN_RING3
74
75	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
76	/**
77	* Deal with a guest page fault.
78	*
79	* @returns Strict VBox status code.
80	* @retval VINF_EM_RAW_GUEST_TRAP
81	* @retval VINF_EM_RAW_EMULATE_INSTR
82	*
83	* @param pVCpu The current CPU.
84	* @param pGstWalk The guest page table walk result.
85	* @param uErr The error code.
86	*/
87	PGM_BTH_DECL(VBOXSTRICTRC, Trap0eHandlerGuestFault)(PVMCPU pVCpu, PGSTPTWALK pGstWalk, RTGCUINT uErr)
88	{
89	# if !defined(PGM_WITHOUT_MAPPINGS) && (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE)
90	/*
91	* Check for write conflicts with our hypervisor mapping.
92	*
93	* If the guest happens to access a non-present page, where our hypervisor
94	* is currently mapped, then we'll create a #PF storm in the guest.
95	*/
96	if ( (uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RW)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RW)
97	&& MMHyperIsInsideArea(pVCpu->CTX_SUFF(pVM), pGstWalk->Core.GCPtr))
98	{
99	/* Force a CR3 sync to check for conflicts and emulate the instruction. */
100	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
101	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
102	return VINF_EM_RAW_EMULATE_INSTR;
103	}
104	# endif
105
106	/*
107	* Calc the error code for the guest trap.
108	*/
109	uint32_t uNewErr = GST_IS_NX_ACTIVE(pVCpu)
110	? uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID)
111	: uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US);
112	if (pGstWalk->Core.fBadPhysAddr)
113	{
114	uNewErr \|= X86_TRAP_PF_RSVD \| X86_TRAP_PF_P;
115	Assert(!pGstWalk->Core.fNotPresent);
116	}
117	else if (!pGstWalk->Core.fNotPresent)
118	uNewErr \|= X86_TRAP_PF_P;
119	TRPMSetErrorCode(pVCpu, uNewErr);
120
121	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
122	return VINF_EM_RAW_GUEST_TRAP;
123	}
124	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
125
126
127	/**
128	* Deal with a guest page fault.
129	*
130	* @returns Strict VBox status code.
131	*
132	* @param pVCpu The current CPU.
133	* @param uErr The error code.
134	* @param pRegFrame The register frame.
135	* @param pvFault The fault address.
136	* @param pPage The guest page at @a pvFault.
137	* @param pGstWalk The guest page table walk result.
138	*/
139	static VBOXSTRICTRC PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame,
140	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
141	RTGCPTR pvFault, PPGMPAGE pPage, PGSTPTWALK pGstWalk)
142	# else
143	RTGCPTR pvFault, PPGMPAGE pPage)
144	# endif
145	{
146	# if !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
147	GSTPDE const PdeSrcDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A};
148	#endif
149	PVM pVM = pVCpu->CTX_SUFF(pVM);
150	int rc;
151
152	if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
153	{
154	/*
155	* Physical page access handler.
156	*/
157	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
158	const RTGCPHYS GCPhysFault = pGstWalk->Core.GCPhys;
159	# else
160	const RTGCPHYS GCPhysFault = (RTGCPHYS)pvFault;
161	# endif
162	PPGMPHYSHANDLER pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
163	if (pCur)
164	{
165	# ifdef PGM_SYNC_N_PAGES
166	/*
167	* If the region is write protected and we got a page not present fault, then sync
168	* the pages. If the fault was caused by a read, then restart the instruction.
169	* In case of write access continue to the GC write handler.
170	*
171	* ASSUMES that there is only one handler per page or that they have similar write properties.
172	*/
173	if ( pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
174	&& !(uErr & X86_TRAP_PF_P))
175	{
176	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
177	rc = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
178	# else
179	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
180	# endif
181	if ( RT_FAILURE(rc)
182	\|\| !(uErr & X86_TRAP_PF_RW)
183	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
184	{
185	AssertRC(rc);
186	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
187	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
188	return rc;
189	}
190	}
191	# endif
192
193	AssertMsg( pCur->enmType != PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
194	\|\| (pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE && (uErr & X86_TRAP_PF_RW)),
195	("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enum=%d\n",
196	pvFault, GCPhysFault, pPage, uErr, pCur->enmType));
197
198	# if defined(IN_RC) \|\| defined(IN_RING0) /** @todo remove this */
199	if (pCur->CTX_SUFF(pfnHandler))
200	{
201	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
202	# ifdef IN_RING0
203	PFNPGMR0PHYSHANDLER pfnHandler = pCur->CTX_SUFF(pfnHandler);
204	# else
205	PFNPGMRCPHYSHANDLER pfnHandler = pCur->CTX_SUFF(pfnHandler);
206	# endif
207	bool fLeaveLock = (pfnHandler != pPool->CTX_SUFF(pfnAccessHandler));
208	void *pvUser = pCur->CTX_SUFF(pvUser);
209
210	STAM_PROFILE_START(&pCur->Stat, h);
211	if (fLeaveLock)
212	pgmUnlock(pVM); /** @todo: Not entirely safe. */
213
214	rc = pfnHandler(pVM, uErr, pRegFrame, pvFault, GCPhysFault, pvUser);
215	if (fLeaveLock)
216	pgmLock(pVM);
217	# ifdef VBOX_WITH_STATISTICS
218	pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
219	if (pCur)
220	STAM_PROFILE_STOP(&pCur->Stat, h);
221	# else
222	pCur = NULL; /* might be invalid by now. */
223	# endif
224
225	}
226	else
227	# endif /* IN_RC \|\| IN_RING0 */
228	rc = VINF_EM_RAW_EMULATE_INSTR;
229
230	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysical);
231	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndPhys; });
232	return rc;
233	}
234	}
235	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
236	else
237	{
238	# ifdef PGM_SYNC_N_PAGES
239	/*
240	* If the region is write protected and we got a page not present fault, then sync
241	* the pages. If the fault was caused by a read, then restart the instruction.
242	* In case of write access continue to the GC write handler.
243	*/
244	if ( PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) < PGM_PAGE_HNDL_PHYS_STATE_ALL
245	&& !(uErr & X86_TRAP_PF_P))
246	{
247	rc = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
248	if ( RT_FAILURE(rc)
249	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
250	\|\| !(uErr & X86_TRAP_PF_RW))
251	{
252	AssertRC(rc);
253	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
254	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndVirt; });
255	return rc;
256	}
257	}
258	# endif
259	/*
260	* Ok, it's an virtual page access handler.
261	*
262	* Since it's faster to search by address, we'll do that first
263	* and then retry by GCPhys if that fails.
264	*/
265	/** @todo r=bird: perhaps we should consider looking up by physical address directly now?
266	* r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be
267	* out of sync, because the page was changed without us noticing it (not-present -> present
268	* without invlpg or mov cr3, xxx).
269	*/
270	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
271	if (pCur)
272	{
273	AssertMsg(!(pvFault - pCur->Core.Key < pCur->cb)
274	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
275	\|\| !(uErr & X86_TRAP_PF_P)
276	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
277	("Unexpected trap for virtual handler: %RGv (phys=%RGp) pPage=%R[pgmpage] uErr=%X, enum=%d\n",
278	pvFault, pGstWalk->Core.GCPhys, pPage, uErr, pCur->enmType));
279
280	if ( pvFault - pCur->Core.Key < pCur->cb
281	&& ( uErr & X86_TRAP_PF_RW
282	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
283	{
284	# ifdef IN_RC
285	STAM_PROFILE_START(&pCur->Stat, h);
286	pgmUnlock(pVM);
287	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
288	pgmLock(pVM);
289	STAM_PROFILE_STOP(&pCur->Stat, h);
290	# else
291	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
292	# endif
293	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersVirtual);
294	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
295	return rc;
296	}
297	/* Unhandled part of a monitored page */
298	}
299	else
300	{
301	/* Check by physical address. */
302	unsigned iPage;
303	rc = pgmHandlerVirtualFindByPhysAddr(pVM, pGstWalk->Core.GCPhys, &pCur, &iPage);
304	Assert(RT_SUCCESS(rc) \|\| !pCur);
305	if ( pCur
306	&& ( uErr & X86_TRAP_PF_RW
307	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
308	{
309	Assert((pCur->aPhysToVirt[iPage].Core.Key & X86_PTE_PAE_PG_MASK) == (pGstWalk->Core.GCPhys & X86_PTE_PAE_PG_MASK));
310	# ifdef IN_RC
311	RTGCPTR off = (iPage << PAGE_SHIFT) + (pvFault & PAGE_OFFSET_MASK) - (pCur->Core.Key & PAGE_OFFSET_MASK);
312	Assert(off < pCur->cb);
313	STAM_PROFILE_START(&pCur->Stat, h);
314	pgmUnlock(pVM);
315	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, off);
316	pgmLock(pVM);
317	STAM_PROFILE_STOP(&pCur->Stat, h);
318	# else
319	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
320	# endif
321	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersVirtualByPhys);
322	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
323	return rc;
324	}
325	}
326	}
327	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
328
329	/*
330	* There is a handled area of the page, but this fault doesn't belong to it.
331	* We must emulate the instruction.
332	*
333	* To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
334	* we first check if this was a page-not-present fault for a page with only
335	* write access handlers. Restart the instruction if it wasn't a write access.
336	*/
337	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersUnhandled);
338
339	if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
340	&& !(uErr & X86_TRAP_PF_P))
341	{
342	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
343	rc = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
344	# else
345	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
346	# endif
347	if ( RT_FAILURE(rc)
348	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
349	\|\| !(uErr & X86_TRAP_PF_RW))
350	{
351	AssertRC(rc);
352	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
353	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
354	return rc;
355	}
356	}
357
358	/** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
359	* It's writing to an unhandled part of the LDT page several million times.
360	*/
361	rc = PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault);
362	LogFlow(("PGM: PGMInterpretInstruction -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
363	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndUnhandled; });
364	return rc;
365	} /* if any kind of handler */
366
367
368	/**
369	* #PF Handler for raw-mode guest execution.
370	*
371	* @returns VBox status code (appropriate for trap handling and GC return).
372	*
373	* @param pVCpu VMCPU Handle.
374	* @param uErr The trap error code.
375	* @param pRegFrame Trap register frame.
376	* @param pvFault The fault address.
377	* @param pfLockTaken PGM lock taken here or not (out)
378	*/
379	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken)
380	{
381	PVM pVM = pVCpu->CTX_SUFF(pVM);
382
383	*pfLockTaken = false;
384
385	# if defined(IN_RC) && defined(VBOX_STRICT)
386	PGMDynCheckLocks(pVM);
387	# endif
388
389	# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \
390	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
391	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
392	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT)
393	int rc;
394
395	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
396	/*
397	* Walk the guest page translation tables and check if it's a guest fault.
398	*/
399	GSTPTWALK GstWalk;
400	rc = PGM_GST_NAME(Walk)(pVCpu, pvFault, &GstWalk);
401	if (RT_FAILURE_NP(rc))
402	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &GstWalk, uErr));
403
404	/* assert some GstWalk sanity. */
405	# if PGM_GST_TYPE == PGM_TYPE_AMD64
406	AssertMsg(GstWalk.Pml4e.u == GstWalk.pPml4e->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pml4e.u, (uint64_t)GstWalk.pPml4e->u));
407	# endif
408	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
409	AssertMsg(GstWalk.Pdpe.u == GstWalk.pPdpe->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pdpe.u, (uint64_t)GstWalk.pPdpe->u));
410	# endif
411	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u));
412	AssertMsg(GstWalk.Core.fBigPage \|\| GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u));
413	Assert(GstWalk.Core.fSucceeded);
414
415	if (uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID))
416	{
417	if ( ( (uErr & X86_TRAP_PF_RW)
418	&& !GstWalk.Core.fEffectiveRW
419	&& ( (uErr & X86_TRAP_PF_US)
420	\|\| CPUMIsGuestR0WriteProtEnabled(pVCpu)) )
421	\|\| ((uErr & X86_TRAP_PF_US) && !GstWalk.Core.fEffectiveUS)
422	\|\| ((uErr & X86_TRAP_PF_ID) && GstWalk.Core.fEffectiveNX)
423	)
424	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &GstWalk, uErr));
425	}
426	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
427
428	# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
429	/*
430	* If it is a reserved bit fault we know that it is an MMIO or access
431	* handler related fault and can skip the dirty page stuff below.
432	*/
433	if (uErr & X86_TRAP_PF_RSVD)
434	{
435	Assert(uErr & X86_TRAP_PF_P);
436	PPGMPAGE pPage;
437	/** @todo Only all physical access handlers here, so optimize further. */
438	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
439	rc = pgmPhysGetPageEx(&pVM->pgm.s, GstWalk.Core.GCPhys, &pPage);
440	if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
441	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
442	&GstWalk));
443	# else
444	rc = pgmPhysGetPageEx(&pVM->pgm.s, (RTGCPHYS)pvFault, &pPage);
445	if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
446	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage));
447	# endif
448	}
449	# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
450
451	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
452	/*
453	* Set the accessed and dirty flags.
454	*/
455	# if PGM_GST_TYPE == PGM_TYPE_AMD64
456	GstWalk.Pml4e.u \|= X86_PML4E_A;
457	GstWalk.pPml4e->u \|= X86_PML4E_A;
458	GstWalk.Pdpe.u \|= X86_PDPE_A;
459	GstWalk.pPdpe->u \|= X86_PDPE_A;
460	# endif
461	if (GstWalk.Core.fBigPage)
462	{
463	Assert(GstWalk.Pde.b.u1Size);
464	if (uErr & X86_TRAP_PF_RW)
465	{
466	GstWalk.Pde.u \|= X86_PDE4M_A \| X86_PDE4M_D;
467	GstWalk.pPde->u \|= X86_PDE4M_A \| X86_PDE4M_D;
468	}
469	else
470	{
471	GstWalk.Pde.u \|= X86_PDE4M_A;
472	GstWalk.pPde->u \|= X86_PDE4M_A;
473	}
474	}
475	else
476	{
477	Assert(!GstWalk.Pde.b.u1Size);
478	GstWalk.Pde.u \|= X86_PDE_A;
479	GstWalk.pPde->u \|= X86_PDE_A;
480	if (uErr & X86_TRAP_PF_RW)
481	{
482	# ifdef VBOX_WITH_STATISTICS
483	if (!GstWalk.Pte.n.u1Dirty)
484	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtiedPage));
485	else
486	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageAlreadyDirty));
487	# endif
488	GstWalk.Pte.u \|= X86_PTE_A \| X86_PTE_D;
489	GstWalk.pPte->u \|= X86_PTE_A \| X86_PTE_D;
490	}
491	else
492	{
493	GstWalk.Pte.u \|= X86_PTE_A;
494	GstWalk.pPte->u \|= X86_PTE_A;
495	}
496	Assert(GstWalk.Pte.u == GstWalk.pPte->u);
497	}
498	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u \|\| GstWalk.pPte->u == GstWalk.pPde->u,
499	("%RX64 %RX64 pPte=%p pPde=%p Pte=%RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u, GstWalk.pPte, GstWalk.pPde, (uint64_t)GstWalk.pPte->u));
500	# else /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
501	GSTPDE const PdeSrcDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A}; /** @todo eliminate this */
502	# endif /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
503
504	/* Take the big lock now. */
505	*pfLockTaken = true;
506	pgmLock(pVM);
507
508	/*
509	* Fetch the guest PDE, PDPE and PML4E.
510	*/
511	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
512	const unsigned iPDDst = pvFault >> SHW_PD_SHIFT;
513	PX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
514
515	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
516	const unsigned iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK; /* pPDDst index, not used with the pool. */
517	PX86PDPAE pPDDst;
518	# if PGM_GST_TYPE == PGM_TYPE_PAE
519	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, GstWalk.Pdpe.u, &pPDDst);
520	# else
521	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, X86_PDPE_P, &pPDDst); /* RW, US and A are reserved in PAE mode. */
522	# endif
523	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_INTERNAL_ERROR_4);
524
525	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
526	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
527	PX86PDPAE pPDDst;
528	# if PGM_GST_TYPE == PGM_TYPE_PROT /* (AMD-V nested paging) */
529	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A,
530	X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A, &pPDDst);
531	# else
532	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, GstWalk.Pml4e.u, GstWalk.Pdpe.u, &pPDDst);
533	# endif
534	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_INTERNAL_ERROR_4);
535
536	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
537	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
538	PEPTPD pPDDst;
539	rc = pgmShwGetEPTPDPtr(pVCpu, pvFault, NULL, &pPDDst);
540	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_INTERNAL_ERROR_4);
541	# endif
542	Assert(pPDDst);
543
544	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
545	/*
546	* Dirty page handling.
547	*
548	* If we successfully correct the write protection fault due to dirty bit
549	* tracking, then return immediately.
550	*/
551	if (uErr & X86_TRAP_PF_RW) /* write fault? */
552	{
553	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyBitTracking), a);
554	rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, &pPDDst->a[iPDDst], GstWalk.pPde, pvFault);
555	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyBitTracking), a);
556	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
557	{
558	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution)
559	= rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
560	? &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2DirtyAndAccessed
561	: &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
562	LogBird(("Trap0eHandler: returns VINF_SUCCESS\n"));
563	return VINF_SUCCESS;
564	}
565	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u \|\| GstWalk.pPte->u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u));
566	AssertMsg(GstWalk.Core.fBigPage \|\| GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u));
567	}
568
569	# if 0 /* rarely useful; leave for debugging. */
570	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0ePD[iPDSrc]);
571	# endif
572	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
573
574	/*
575	* A common case is the not-present error caused by lazy page table syncing.
576	*
577	* It is IMPORTANT that we weed out any access to non-present shadow PDEs
578	* here so we can safely assume that the shadow PT is present when calling
579	* SyncPage later.
580	*
581	* On failure, we ASSUME that SyncPT is out of memory or detected some kind
582	* of mapping conflict and defer to SyncCR3 in R3.
583	* (Again, we do NOT support access handlers for non-present guest pages.)
584	*
585	*/
586	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
587	Assert(GstWalk.Pde.n.u1Present);
588	# endif
589	if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
590	&& !pPDDst->a[iPDDst].n.u1Present)
591	{
592	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2SyncPT; });
593	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
594	LogFlow(("=>SyncPT %04x = %08RX64\n", (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, (uint64_t)GstWalk.Pde.u));
595	rc = PGM_BTH_NAME(SyncPT)(pVCpu, (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, GstWalk.pPd, pvFault);
596	# else
597	LogFlow(("=>SyncPT pvFault=%RGv\n", pvFault));
598	rc = PGM_BTH_NAME(SyncPT)(pVCpu, 0, NULL, pvFault);
599	# endif
600	if (RT_SUCCESS(rc))
601	return rc;
602	Log(("SyncPT: %RGv failed!! rc=%Rrc\n", pvFault, rc));
603	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
604	return VINF_PGM_SYNC_CR3;
605	}
606
607	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(PGM_WITHOUT_MAPPINGS)
608	/*
609	* Check if this address is within any of our mappings.
610	*
611	* This is very fast and it's gonna save us a bit of effort below and prevent
612	* us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
613	* (BTW, it's impossible to have physical access handlers in a mapping.)
614	*/
615	if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
616	{
617	PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
618	for ( ; pMapping; pMapping = pMapping->CTX_SUFF(pNext))
619	{
620	if (pvFault < pMapping->GCPtr)
621	break;
622	if (pvFault - pMapping->GCPtr < pMapping->cb)
623	{
624	/*
625	* The first thing we check is if we've got an undetected conflict.
626	*/
627	if (pgmMapAreMappingsFloating(&pVM->pgm.s))
628	{
629	unsigned iPT = pMapping->cb >> GST_PD_SHIFT;
630	while (iPT-- > 0)
631	if (GstWalk.pPde[iPT].n.u1Present)
632	{
633	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eConflicts);
634	Log(("Trap0e: Detected Conflict %RGv-%RGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
635	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
636	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
637	return VINF_PGM_SYNC_CR3;
638	}
639	}
640
641	/*
642	* Check if the fault address is in a virtual page access handler range.
643	*/
644	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->HyperVirtHandlers, pvFault);
645	if ( pCur
646	&& pvFault - pCur->Core.Key < pCur->cb
647	&& uErr & X86_TRAP_PF_RW)
648	{
649	# ifdef IN_RC
650	STAM_PROFILE_START(&pCur->Stat, h);
651	pgmUnlock(pVM);
652	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
653	pgmLock(pVM);
654	STAM_PROFILE_STOP(&pCur->Stat, h);
655	# else
656	AssertFailed();
657	rc = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
658	# endif
659	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersMapping);
660	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
661	return rc;
662	}
663
664	/*
665	* Pretend we're not here and let the guest handle the trap.
666	*/
667	TRPMSetErrorCode(pVCpu, uErr & ~X86_TRAP_PF_P);
668	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eGuestPFMapping);
669	LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
670	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
671	return VINF_EM_RAW_GUEST_TRAP;
672	}
673	}
674	} /* pgmAreMappingsEnabled(&pVM->pgm.s) */
675	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
676
677	/*
678	* Check if this fault address is flagged for special treatment,
679	* which means we'll have to figure out the physical address and
680	* check flags associated with it.
681	*
682	* ASSUME that we can limit any special access handling to pages
683	* in page tables which the guest believes to be present.
684	*/
685	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
686	RTGCPHYS GCPhys = GstWalk.Core.GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
687	# else
688	RTGCPHYS GCPhys = (RTGCPHYS)pvFault & ~(RTGCPHYS)PAGE_OFFSET_MASK;
689	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
690	PPGMPAGE pPage;
691	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
692	if (RT_FAILURE(rc))
693	{
694	/*
695	* When the guest accesses invalid physical memory (e.g. probing
696	* of RAM or accessing a remapped MMIO range), then we'll fall
697	* back to the recompiler to emulate the instruction.
698	*/
699	LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhys, rc));
700	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersInvalid);
701	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2InvalidPhys; });
702	return VINF_EM_RAW_EMULATE_INSTR;
703	}
704
705	/*
706	* Any handlers for this page?
707	*/
708	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
709	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
710	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, &GstWalk));
711	# else
712	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage));
713	# endif
714
715	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTimeOutOfSync, c);
716
717	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
718	if (uErr & X86_TRAP_PF_P)
719	{
720	/*
721	* The page isn't marked, but it might still be monitored by a virtual page access handler.
722	* (ASSUMES no temporary disabling of virtual handlers.)
723	*/
724	/** @todo r=bird: Since the purpose is to catch out of sync pages with virtual handler(s) here,
725	* we should correct both the shadow page table and physical memory flags, and not only check for
726	* accesses within the handler region but for access to pages with virtual handlers. */
727	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
728	if (pCur)
729	{
730	AssertMsg( !(pvFault - pCur->Core.Key < pCur->cb)
731	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
732	\|\| !(uErr & X86_TRAP_PF_P)
733	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
734	("Unexpected trap for virtual handler: %08X (phys=%08x) %R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
735
736	if ( pvFault - pCur->Core.Key < pCur->cb
737	&& ( uErr & X86_TRAP_PF_RW
738	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
739	{
740	# ifdef IN_RC
741	STAM_PROFILE_START(&pCur->Stat, h);
742	pgmUnlock(pVM);
743	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
744	pgmLock(pVM);
745	STAM_PROFILE_STOP(&pCur->Stat, h);
746	# else
747	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
748	# endif
749	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
750	return rc;
751	}
752	}
753	}
754	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
755
756	/*
757	* We are here only if page is present in Guest page tables and
758	* trap is not handled by our handlers.
759	*
760	* Check it for page out-of-sync situation.
761	*/
762	if (!(uErr & X86_TRAP_PF_P))
763	{
764	/*
765	* Page is not present in our page tables. Try to sync it!
766	*/
767	if (uErr & X86_TRAP_PF_US)
768	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUser));
769	else /* supervisor */
770	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
771
772	if (PGM_PAGE_IS_BALLOONED(pPage))
773	{
774	/* Emulate reads from ballooned pages as they are not present in
775	our shadow page tables. (Required for e.g. Solaris guests; soft
776	ecc, random nr generator.) */
777	rc = PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault);
778	LogFlow(("PGM: PGMInterpretInstruction balloon -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
779	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncBallloon));
780	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Ballooned; });
781	return rc;
782	}
783
784	# if defined(LOG_ENABLED) && !defined(IN_RING0)
785	RTGCPHYS GCPhys2;
786	uint64_t fPageGst2;
787	PGMGstGetPage(pVCpu, pvFault, &fPageGst2, &GCPhys2);
788	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
789	Log(("Page out of sync: %RGv eip=%08x PdeSrc.US=%d fPageGst2=%08llx GCPhys2=%RGp scan=%d\n",
790	pvFault, pRegFrame->eip, GstWalk.Pde.n.u1User, fPageGst2, GCPhys2, CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)));
791	# else
792	Log(("Page out of sync: %RGv eip=%08x fPageGst2=%08llx GCPhys2=%RGp scan=%d\n",
793	pvFault, pRegFrame->eip, fPageGst2, GCPhys2, CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)));
794	# endif
795	# endif /* LOG_ENABLED */
796
797	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
798	if ( !GstWalk.Core.fEffectiveUS
799	&& CPUMGetGuestCPL(pVCpu, pRegFrame) == 0)
800	{
801	/* Note: Can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU. */
802	if ( pvFault == (RTGCPTR)pRegFrame->eip
803	\|\| pvFault - pRegFrame->eip < 8 /* instruction crossing a page boundary */
804	# ifdef CSAM_DETECT_NEW_CODE_PAGES
805	\|\| ( !PATMIsPatchGCAddr(pVM, pRegFrame->eip)
806	&& CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)) /* any new code we encounter here */
807	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
808	)
809	{
810	LogFlow(("CSAMExecFault %RX32\n", pRegFrame->eip));
811	rc = CSAMExecFault(pVM, (RTRCPTR)pRegFrame->eip);
812	if (rc != VINF_SUCCESS)
813	{
814	/*
815	* CSAM needs to perform a job in ring 3.
816	*
817	* Sync the page before going to the host context; otherwise we'll end up in a loop if
818	* CSAM fails (e.g. instruction crosses a page boundary and the next page is not present)
819	*/
820	LogFlow(("CSAM ring 3 job\n"));
821	int rc2 = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
822	AssertRC(rc2);
823
824	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2CSAM; });
825	return rc;
826	}
827	}
828	# ifdef CSAM_DETECT_NEW_CODE_PAGES
829	else if ( uErr == X86_TRAP_PF_RW
830	&& pRegFrame->ecx >= 0x100 /* early check for movswd count */
831	&& pRegFrame->ecx < 0x10000)
832	{
833	/* In case of a write to a non-present supervisor shadow page, we'll take special precautions
834	* to detect loading of new code pages.
835	*/
836
837	/*
838	* Decode the instruction.
839	*/
840	RTGCPTR PC;
841	rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs,
842	&pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &PC);
843	if (rc == VINF_SUCCESS)
844	{
845	PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
846	uint32_t cbOp;
847	rc = EMInterpretDisasOneEx(pVM, pVCpu, PC, pRegFrame, pDis, &cbOp);
848
849	/* For now we'll restrict this to rep movsw/d instructions */
850	if ( rc == VINF_SUCCESS
851	&& pDis->pCurInstr->opcode == OP_MOVSWD
852	&& (pDis->prefix & PREFIX_REP))
853	{
854	CSAMMarkPossibleCodePage(pVM, pvFault);
855	}
856	}
857	}
858	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
859
860	/*
861	* Mark this page as safe.
862	*/
863	/** @todo not correct for pages that contain both code and data!! */
864	Log2(("CSAMMarkPage %RGv; scanned=%d\n", pvFault, true));
865	CSAMMarkPage(pVM, pvFault, true);
866	}
867	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0) */
868	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
869	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
870	# else
871	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
872	# endif
873	if (RT_SUCCESS(rc))
874	{
875	/* The page was successfully synced, return to the guest. */
876	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSync; });
877	return VINF_SUCCESS;
878	}
879	}
880	else /* uErr & X86_TRAP_PF_P: */
881	{
882	/*
883	* Write protected pages are made writable when the guest makes the
884	* first write to it. This happens for pages that are shared, write
885	* monitored or not yet allocated.
886	*
887	* We may also end up here when CR0.WP=0 in the guest.
888	*
889	* Also, a side effect of not flushing global PDEs are out of sync
890	* pages due to physical monitored regions, that are no longer valid.
891	* Assume for now it only applies to the read/write flag.
892	*/
893	if (uErr & X86_TRAP_PF_RW)
894	{
895	/*
896	* Check if it is a read-only page.
897	*/
898	if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
899	{
900	Log(("PGM #PF: Make writable: %RGp %R[pgmpage] pvFault=%RGp uErr=%#x\n", GCPhys, pPage, pvFault, uErr));
901	Assert(!PGM_PAGE_IS_ZERO(pPage));
902	AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
903	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2MakeWritable; });
904
905	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
906	if (rc != VINF_SUCCESS)
907	{
908	AssertMsg(rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("%Rrc\n", rc));
909	return rc;
910	}
911	if (RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)))
912	return VINF_EM_NO_MEMORY;
913	}
914
915	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
916	/*
917	* Check to see if we need to emulate the instruction if CR0.WP=0.
918	*/
919	if ( !GstWalk.Core.fEffectiveRW
920	&& (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG
921	&& CPUMGetGuestCPL(pVCpu, pRegFrame) == 0)
922	{
923	Assert((uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RW \| X86_TRAP_PF_P));
924	rc = PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault);
925	if (RT_SUCCESS(rc))
926	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eWPEmulInRZ);
927	else
928	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eWPEmulToR3);
929	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2WPEmulation; });
930	return rc;
931	}
932	# endif
933	/// @todo count the above case; else
934	if (uErr & X86_TRAP_PF_US)
935	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUserWrite));
936	else /* supervisor */
937	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
938
939	/*
940	* Sync the page.
941	*
942	* Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
943	* page is not present, which is not true in this case.
944	*/
945	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
946	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
947	# else
948	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
949	# endif
950	if (RT_SUCCESS(rc))
951	{
952	/*
953	* Page was successfully synced, return to guest but invalidate
954	* the TLB first as the page is very likely to be in it.
955	*/
956	# if PGM_SHW_TYPE == PGM_TYPE_EPT
957	HWACCMInvalidatePhysPage(pVM, (RTGCPHYS)pvFault);
958	# else
959	PGM_INVL_PG(pVCpu, pvFault);
960	# endif
961	# ifdef VBOX_STRICT
962	RTGCPHYS GCPhys2;
963	uint64_t fPageGst;
964	if (!pVM->pgm.s.fNestedPaging)
965	{
966	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, &GCPhys2);
967	AssertMsg(RT_SUCCESS(rc) && (fPageGst & X86_PTE_RW), ("rc=%Rrc fPageGst=%RX64\n", rc, fPageGst));
968	LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GCPhys2, (uint64_t)fPageGst));
969	}
970	uint64_t fPageShw;
971	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
972	AssertMsg((RT_SUCCESS(rc) && (fPageShw & X86_PTE_RW)) \|\| pVM->cCpus > 1 /* new monitor can be installed/page table flushed between the trap exit and PGMTrap0eHandler */, ("rc=%Rrc fPageShw=%RX64\n", rc, fPageShw));
973	# endif /* VBOX_STRICT */
974	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndObs; });
975	return VINF_SUCCESS;
976	}
977	}
978	/** @todo else: WTF are we here? */
979
980	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && defined(VBOX_STRICT)
981	/*
982	* Check for VMM page flags vs. Guest page flags consistency.
983	* Currently only for debug purposes.
984	*/
985	if (RT_SUCCESS(rc))
986	{
987	/* Get guest page flags. */
988	uint64_t fPageGst;
989	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
990	if (RT_SUCCESS(rc))
991	{
992	uint64_t fPageShw;
993	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
994
995	/*
996	* Compare page flags.
997	* Note: we have AVL, A, D bits desynched.
998	*/
999	AssertMsg((fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)),
1000	("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64\n", pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst));
1001	}
1002	else
1003	AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc));
1004	}
1005	else
1006	AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc));
1007	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && VBOX_STRICT */
1008	}
1009
1010
1011	/*
1012	* If we get here it is because something failed above, i.e. most like guru
1013	* meditiation time.
1014	*/
1015	LogRel(("%s: returns rc=%Rrc pvFault=%RGv uErr=%RX64 cs:rip=%04x:%08RX64\n",
1016	__PRETTY_FUNCTION__, rc, pvFault, (uint64_t)uErr, pRegFrame->cs, pRegFrame->rip));
1017	return rc;
1018
1019	# else /* Nested paging, EPT except PGM_GST_TYPE = PROT */
1020	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
1021	return VERR_INTERNAL_ERROR;
1022	# endif
1023	}
1024	#endif /* !IN_RING3 */
1025
1026
1027	/**
1028	* Emulation of the invlpg instruction.
1029	*
1030	*
1031	* @returns VBox status code.
1032	*
1033	* @param pVCpu The VMCPU handle.
1034	* @param GCPtrPage Page to invalidate.
1035	*
1036	* @remark ASSUMES that the guest is updating before invalidating. This order
1037	* isn't required by the CPU, so this is speculative and could cause
1038	* trouble.
1039	* @remark No TLB shootdown is done on any other VCPU as we assume that
1040	* invlpg emulation is the only reason for calling this function.
1041	* (The guest has to shoot down TLB entries on other CPUs itself)
1042	* Currently true, but keep in mind!
1043	*
1044	* @todo Clean this up! Most of it is (or should be) no longer necessary as we catch all page table accesses.
1045	*/
1046	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
1047	{
1048	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1049	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
1050	&& PGM_SHW_TYPE != PGM_TYPE_EPT
1051	int rc;
1052	PVM pVM = pVCpu->CTX_SUFF(pVM);
1053	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1054
1055	Assert(PGMIsLockOwner(pVM));
1056
1057	LogFlow(("InvalidatePage %RGv\n", GCPtrPage));
1058
1059	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1060	if (pPool->cDirtyPages)
1061	pgmPoolResetDirtyPages(pVM);
1062	# endif
1063
1064	/*
1065	* Get the shadow PD entry and skip out if this PD isn't present.
1066	* (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
1067	*/
1068	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1069	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1070	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1071
1072	/* Fetch the pgm pool shadow descriptor. */
1073	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1074	Assert(pShwPde);
1075
1076	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1077	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT);
1078	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
1079
1080	/* If the shadow PDPE isn't present, then skip the invalidate. */
1081	if (!pPdptDst->a[iPdpt].n.u1Present)
1082	{
1083	Assert(!(pPdptDst->a[iPdpt].u & PGM_PLXFLAGS_MAPPING));
1084	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1085	return VINF_SUCCESS;
1086	}
1087
1088	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1089	PPGMPOOLPAGE pShwPde = NULL;
1090	PX86PDPAE pPDDst;
1091
1092	/* Fetch the pgm pool shadow descriptor. */
1093	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
1094	AssertRCSuccessReturn(rc, rc);
1095	Assert(pShwPde);
1096
1097	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR(pVM, pShwPde);
1098	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1099
1100	# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1101	/* PML4 */
1102	const unsigned iPml4 = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;
1103	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1104	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1105	PX86PDPAE pPDDst;
1106	PX86PDPT pPdptDst;
1107	PX86PML4E pPml4eDst;
1108	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eDst, &pPdptDst, &pPDDst);
1109	if (rc != VINF_SUCCESS)
1110	{
1111	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT \|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
1112	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1113	return VINF_SUCCESS;
1114	}
1115	Assert(pPDDst);
1116
1117	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1118	PX86PDPE pPdpeDst = &pPdptDst->a[iPdpt];
1119
1120	if (!pPdpeDst->n.u1Present)
1121	{
1122	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1123	return VINF_SUCCESS;
1124	}
1125
1126	/* Fetch the pgm pool shadow descriptor. */
1127	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & SHW_PDPE_PG_MASK);
1128	Assert(pShwPde);
1129
1130	# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1131
1132	const SHWPDE PdeDst = *pPdeDst;
1133	if (!PdeDst.n.u1Present)
1134	{
1135	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1136	return VINF_SUCCESS;
1137	}
1138
1139	# if defined(IN_RC)
1140	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1141	PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
1142	# endif
1143
1144	/*
1145	* Get the guest PD entry and calc big page.
1146	*/
1147	# if PGM_GST_TYPE == PGM_TYPE_32BIT
1148	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
1149	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
1150	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1151	# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1152	unsigned iPDSrc = 0;
1153	# if PGM_GST_TYPE == PGM_TYPE_PAE
1154	X86PDPE PdpeSrcIgn;
1155	PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrcIgn);
1156	# else /* AMD64 */
1157	PX86PML4E pPml4eSrcIgn;
1158	X86PDPE PdpeSrcIgn;
1159	PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrcIgn, &PdpeSrcIgn, &iPDSrc);
1160	# endif
1161	GSTPDE PdeSrc;
1162
1163	if (pPDSrc)
1164	PdeSrc = pPDSrc->a[iPDSrc];
1165	else
1166	PdeSrc.u = 0;
1167	# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1168	const bool fIsBigPage = PdeSrc.b.u1Size && GST_IS_PSE_ACTIVE(pVCpu);
1169
1170	# ifdef IN_RING3
1171	/*
1172	* If a CR3 Sync is pending we may ignore the invalidate page operation
1173	* depending on the kind of sync and if it's a global page or not.
1174	* This doesn't make sense in GC/R0 so we'll skip it entirely there.
1175	*/
1176	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
1177	if ( VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
1178	\|\| ( VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
1179	&& fIsBigPage
1180	&& PdeSrc.b.u1Global
1181	)
1182	)
1183	# else
1184	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
1185	# endif
1186	{
1187	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1188	return VINF_SUCCESS;
1189	}
1190	# endif /* IN_RING3 */
1191
1192	/*
1193	* Deal with the Guest PDE.
1194	*/
1195	rc = VINF_SUCCESS;
1196	if (PdeSrc.n.u1Present)
1197	{
1198	Assert( PdeSrc.n.u1User == PdeDst.n.u1User
1199	&& (PdeSrc.n.u1Write \|\| !PdeDst.n.u1Write));
1200	# ifndef PGM_WITHOUT_MAPPING
1201	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
1202	{
1203	/*
1204	* Conflict - Let SyncPT deal with it to avoid duplicate code.
1205	*/
1206	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1207	Assert(PGMGetGuestMode(pVCpu) <= PGMMODE_PAE);
1208	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
1209	}
1210	else
1211	# endif /* !PGM_WITHOUT_MAPPING */
1212	if (!fIsBigPage)
1213	{
1214	/*
1215	* 4KB - page.
1216	*/
1217	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1218	RTGCPHYS GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1219
1220	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1221	/* Reset the modification counter (OpenSolaris trashes tlb entries very often) */
1222	if (pShwPage->cModifications)
1223	pShwPage->cModifications = 1;
1224	# endif
1225
1226	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1227	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1228	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE/2);
1229	# endif
1230	if (pShwPage->GCPhys == GCPhys)
1231	{
1232	# if 0 /* likely cause of a major performance regression; must be SyncPageWorkerTrackDeref then */
1233	const unsigned iPTEDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1234	PSHWPT pPT = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1235	if (pPT->a[iPTEDst].n.u1Present)
1236	{
1237	/* This is very unlikely with caching/monitoring enabled. */
1238	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pShwPage, pPT->a[iPTEDst].u & SHW_PTE_PG_MASK, iPTEDst);
1239	ASMAtomicWriteSize(&pPT->a[iPTEDst], 0);
1240	}
1241	# else /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1242	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
1243	if (RT_SUCCESS(rc))
1244	rc = VINF_SUCCESS;
1245	# endif
1246	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4KBPages));
1247	PGM_INVL_PG(pVCpu, GCPtrPage);
1248	}
1249	else
1250	{
1251	/*
1252	* The page table address changed.
1253	*/
1254	LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
1255	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1256	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1257	ASMAtomicWriteSize(pPdeDst, 0);
1258	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1259	PGM_INVL_VCPU_TLBS(pVCpu);
1260	}
1261	}
1262	else
1263	{
1264	/*
1265	* 2/4MB - page.
1266	*/
1267	/* Before freeing the page, check if anything really changed. */
1268	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1269	RTGCPHYS GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(pVM, PdeSrc);
1270	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1271	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1272	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1273	# endif
1274	if ( pShwPage->GCPhys == GCPhys
1275	&& pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1276	{
1277	/* ASSUMES a the given bits are identical for 4M and normal PDEs */
1278	/** @todo PAT */
1279	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
1280	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
1281	&& ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1282	\|\| (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1283	{
1284	LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1285	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip));
1286	# if defined(IN_RC)
1287	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1288	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
1289	# endif
1290	return VINF_SUCCESS;
1291	}
1292	}
1293
1294	/*
1295	* Ok, the page table is present and it's been changed in the guest.
1296	* If we're in host context, we'll just mark it as not present taking the lazy approach.
1297	* We could do this for some flushes in GC too, but we need an algorithm for
1298	* deciding which 4MB pages containing code likely to be executed very soon.
1299	*/
1300	LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1301	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1302	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1303	ASMAtomicWriteSize(pPdeDst, 0);
1304	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4MBPages));
1305	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
1306	}
1307	}
1308	else
1309	{
1310	/*
1311	* Page directory is not present, mark shadow PDE not present.
1312	*/
1313	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
1314	{
1315	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1316	ASMAtomicWriteSize(pPdeDst, 0);
1317	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDNPs));
1318	PGM_INVL_PG(pVCpu, GCPtrPage);
1319	}
1320	else
1321	{
1322	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1323	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDMappings));
1324	}
1325	}
1326	# if defined(IN_RC)
1327	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1328	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
1329	# endif
1330	return rc;
1331
1332	#else /* guest real and protected mode */
1333	/* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1334	return VINF_SUCCESS;
1335	#endif
1336	}
1337
1338
1339	/**
1340	* Update the tracking of shadowed pages.
1341	*
1342	* @param pVCpu The VMCPU handle.
1343	* @param pShwPage The shadow page.
1344	* @param HCPhys The physical page we is being dereferenced.
1345	* @param iPte Shadow PTE index
1346	*/
1347	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys, uint16_t iPte)
1348	{
1349	PVM pVM = pVCpu->CTX_SUFF(pVM);
1350
1351	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackDeref, a);
1352	LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1353
1354	/** @todo If this turns out to be a bottle neck (very likely) two things can be done:
1355	* 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1356	* 2. write protect all shadowed pages. I.e. implement caching.
1357	*/
1358	/** @todo duplicated in the 2nd half of pgmPoolTracDerefGCPhysHint */
1359
1360	/*
1361	* Find the guest address.
1362	*/
1363	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
1364	pRam;
1365	pRam = pRam->CTX_SUFF(pNext))
1366	{
1367	unsigned iPage = pRam->cb >> PAGE_SHIFT;
1368	while (iPage-- > 0)
1369	{
1370	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1371	{
1372	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1373
1374	Assert(pShwPage->cPresent);
1375	Assert(pPool->cPresent);
1376	pShwPage->cPresent--;
1377	pPool->cPresent--;
1378
1379	pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage], iPte);
1380	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackDeref, a);
1381	return;
1382	}
1383	}
1384	}
1385
1386	for (;;)
1387	AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys));
1388	}
1389
1390
1391	/**
1392	* Update the tracking of shadowed pages.
1393	*
1394	* @param pVCpu The VMCPU handle.
1395	* @param pShwPage The shadow page.
1396	* @param u16 The top 16-bit of the pPage->HCPhys.
1397	* @param pPage Pointer to the guest page. this will be modified.
1398	* @param iPTDst The index into the shadow table.
1399	*/
1400	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
1401	{
1402	PVM pVM = pVCpu->CTX_SUFF(pVM);
1403	/*
1404	* Just deal with the simple first time here.
1405	*/
1406	if (!u16)
1407	{
1408	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackVirgin);
1409	u16 = PGMPOOL_TD_MAKE(1, pShwPage->idx);
1410	/* Save the page table index. */
1411	PGM_PAGE_SET_PTE_INDEX(pPage, iPTDst);
1412	}
1413	else
1414	u16 = pgmPoolTrackPhysExtAddref(pVM, pPage, u16, pShwPage->idx, iPTDst);
1415
1416	/* write back */
1417	Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x iPTDst=%#x\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst));
1418	PGM_PAGE_SET_TRACKING(pPage, u16);
1419
1420	/* update statistics. */
1421	pVM->pgm.s.CTX_SUFF(pPool)->cPresent++;
1422	pShwPage->cPresent++;
1423	if (pShwPage->iFirstPresent > iPTDst)
1424	pShwPage->iFirstPresent = iPTDst;
1425	}
1426
1427
1428	/**
1429	* Modifies a shadow PTE to account for access handlers.
1430	*
1431	* @param pVM The VM handle.
1432	* @param pPage The page in question.
1433	* @param fPteSrc The flags of the source PTE.
1434	* @param pPteDst The shadow PTE (output).
1435	*/
1436	DECLINLINE(void) PGM_BTH_NAME(SyncHandlerPte)(PVM pVM, PCPGMPAGE pPage, uint32_t fPteSrc, PSHWPTE pPteDst)
1437	{
1438	/** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No.
1439	* Update: \#PF should deal with this before or after calling the handlers. It has all the info to do the job efficiently. */
1440	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1441	{
1442	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1443	pPteDst->u = PGM_PAGE_GET_HCPHYS(pPage);
1444	pPteDst->n.u1Present = 1;
1445	pPteDst->n.u1Execute = 1;
1446	pPteDst->n.u1IgnorePAT = 1;
1447	pPteDst->n.u3EMT = VMX_EPT_MEMTYPE_WB;
1448	/* PteDst.n.u1Write = 0 && PteDst.n.u1Size = 0 */
1449	#else
1450	pPteDst->u = (fPteSrc & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1451	\| PGM_PAGE_GET_HCPHYS(pPage);
1452	#endif
1453	}
1454	#ifdef PGM_WITH_MMIO_OPTIMIZATIONS
1455	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
1456	else if ( PGM_PAGE_IS_MMIO(pPage)
1457	# if PGM_SHW_TYPE != PGM_TYPE_EPT
1458	&& ( (fPteSrc & (X86_PTE_RW /\| X86_PTE_D \| X86_PTE_A/ \| X86_PTE_US )) /* #PF handles D & A first. */
1459	== (X86_PTE_RW /\| X86_PTE_D \| X86_PTE_A/)
1460	\|\| BTH_IS_NP_ACTIVE(pVM) )
1461	# endif
1462	# if PGM_SHW_TYPE == PGM_TYPE_AMD64
1463	&& pVM->pgm.s.fLessThan52PhysicalAddressBits
1464	# endif
1465	)
1466	{
1467	LogFlow(("SyncHandlerPte: MMIO page -> invalid \n"));
1468	# if PGM_SHW_TYPE == PGM_TYPE_EPT
1469	/* 25.2.3.1: Reserved physical address bit -> EPT Misconfiguration (exit 49) */
1470	pPteDst->u = pVM->pgm.s.HCPhysInvMmioPg;
1471	/* 25.2.3.1: bits 2:0 = 010b -> EPT Misconfiguration (exit 49) */
1472	pPteDst->n.u1Present = 0;
1473	pPteDst->n.u1Write = 1;
1474	pPteDst->n.u1Execute = 0;
1475	/* 25.2.3.1: leaf && 2:0 != 0 && u3Emt in {2, 3, 7} -> EPT Misconfiguration */
1476	pPteDst->n.u3EMT = 7;
1477	# else
1478	/* Set high page frame bits that MBZ (bankers on PAE, CPU dependent on AMD64). */
1479	pPteDst->u = pVM->pgm.s.HCPhysInvMmioPg \| X86_PTE_PAE_MBZ_MASK_NO_NX \| X86_PTE_P;
1480	# endif
1481	}
1482	# endif
1483	#endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
1484	else
1485	{
1486	LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark not present\n", pPage));
1487	pPteDst->u = 0;
1488	}
1489	/** @todo count these kinds of entries. */
1490	}
1491
1492
1493	/**
1494	* Creates a 4K shadow page for a guest page.
1495	*
1496	* For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1497	* physical address. The PdeSrc argument only the flags are used. No page
1498	* structured will be mapped in this function.
1499	*
1500	* @param pVCpu The VMCPU handle.
1501	* @param pPteDst Destination page table entry.
1502	* @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1503	* Can safely assume that only the flags are being used.
1504	* @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1505	* @param pShwPage Pointer to the shadow page.
1506	* @param iPTDst The index into the shadow table.
1507	*
1508	* @remark Not used for 2/4MB pages!
1509	*/
1510	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc,
1511	PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1512	{
1513	if ( PteSrc.n.u1Present
1514	&& GST_IS_PTE_VALID(pVCpu, PteSrc))
1515	{
1516	PVM pVM = pVCpu->CTX_SUFF(pVM);
1517
1518	# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1519	&& PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1520	&& (PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64)
1521	if (pShwPage->fDirty)
1522	{
1523	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1524	PX86PTPAE pGstPT;
1525
1526	pGstPT = (PX86PTPAE)&pPool->aDirtyPages[pShwPage->idxDirty][0];
1527	pGstPT->a[iPTDst].u = PteSrc.u;
1528	}
1529	# endif
1530	/*
1531	* Find the ram range.
1532	*/
1533	PPGMPAGE pPage;
1534	int rc = pgmPhysGetPageEx(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK, &pPage);
1535	if (RT_SUCCESS(rc))
1536	{
1537	/* Ignore ballooned pages.
1538	Don't return errors or use a fatal assert here as part of a
1539	shadow sync range might included ballooned pages. */
1540	if (PGM_PAGE_IS_BALLOONED(pPage))
1541	{
1542	Assert(!pPteDst->n.u1Present); /** @todo user tracking needs updating if this triggers. */
1543	return;
1544	}
1545
1546	#ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1547	/* Make the page writable if necessary. */
1548	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1549	&& ( PGM_PAGE_IS_ZERO(pPage)
1550	\|\| ( PteSrc.n.u1Write
1551	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1552	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1553	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1554	# endif
1555	# ifdef VBOX_WITH_PAGE_SHARING
1556	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
1557	# endif
1558	)
1559	)
1560	)
1561	{
1562	rc = pgmPhysPageMakeWritable(pVM, pPage, PteSrc.u & GST_PTE_PG_MASK);
1563	AssertRC(rc);
1564	}
1565	#endif
1566
1567	/*
1568	* Make page table entry.
1569	*/
1570	SHWPTE PteDst;
1571	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1572	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage,
1573	PteSrc.u & ~( X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT
1574	\| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW),
1575	&PteDst);
1576	else
1577	{
1578	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1579	/*
1580	* If the page or page directory entry is not marked accessed,
1581	* we mark the page not present.
1582	*/
1583	if (!PteSrc.n.u1Accessed \|\| !PdeSrc.n.u1Accessed)
1584	{
1585	LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1586	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,AccessedPage));
1587	PteDst.u = 0;
1588	}
1589	/*
1590	* If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1591	* when the page is modified.
1592	*/
1593	else if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1594	{
1595	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPage));
1596	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1597	\| PGM_PAGE_GET_HCPHYS(pPage)
1598	\| PGM_PTFLAGS_TRACK_DIRTY;
1599	}
1600	else
1601	#endif
1602	{
1603	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageSkipped));
1604	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1605	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage);
1606	PteDst.n.u1Present = 1;
1607	PteDst.n.u1Write = 1;
1608	PteDst.n.u1Execute = 1;
1609	PteDst.n.u1IgnorePAT = 1;
1610	PteDst.n.u3EMT = VMX_EPT_MEMTYPE_WB;
1611	/* PteDst.n.u1Size = 0 */
1612	#else
1613	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1614	\| PGM_PAGE_GET_HCPHYS(pPage);
1615	#endif
1616	}
1617
1618	/*
1619	* Make sure only allocated pages are mapped writable.
1620	*/
1621	if ( PteDst.n.u1Write
1622	&& PteDst.n.u1Present
1623	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1624	{
1625	/* Still applies to shared pages. */
1626	Assert(!PGM_PAGE_IS_ZERO(pPage));
1627	PteDst.n.u1Write = 0; /** @todo this isn't quite working yet. Why, isn't it? */
1628	Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", (RTGCPHYS)(PteSrc.u & X86_PTE_PAE_PG_MASK), pPage, iPTDst));
1629	}
1630	}
1631
1632	/*
1633	* Keep user track up to date.
1634	*/
1635	if (PteDst.n.u1Present)
1636	{
1637	if (!pPteDst->n.u1Present)
1638	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1639	else if ((pPteDst->u & SHW_PTE_PG_MASK) != (PteDst.u & SHW_PTE_PG_MASK))
1640	{
1641	Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", (uint64_t)pPteDst->u, (uint64_t)PteDst.u));
1642	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK, iPTDst);
1643	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1644	}
1645	}
1646	else if (pPteDst->n.u1Present)
1647	{
1648	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1649	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK, iPTDst);
1650	}
1651
1652	/*
1653	* Update statistics and commit the entry.
1654	*/
1655	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1656	if (!PteSrc.n.u1Global)
1657	pShwPage->fSeenNonGlobal = true;
1658	#endif
1659	ASMAtomicWriteSize(pPteDst, PteDst.u);
1660	return;
1661	}
1662
1663	/** @todo count these three different kinds. */
1664	Log2(("SyncPageWorker: invalid address in Pte\n"));
1665	}
1666	else if (!PteSrc.n.u1Present)
1667	Log2(("SyncPageWorker: page not present in Pte\n"));
1668	else
1669	Log2(("SyncPageWorker: invalid Pte\n"));
1670
1671	/*
1672	* The page is not present or the PTE is bad. Replace the shadow PTE by
1673	* an empty entry, making sure to keep the user tracking up to date.
1674	*/
1675	if (pPteDst->n.u1Present)
1676	{
1677	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1678	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK, iPTDst);
1679	}
1680	ASMAtomicWriteSize(pPteDst, 0);
1681	}
1682
1683
1684	/**
1685	* Syncs a guest OS page.
1686	*
1687	* There are no conflicts at this point, neither is there any need for
1688	* page table allocations.
1689	*
1690	* When called in PAE or AMD64 guest mode, the guest PDPE shall be valid.
1691	* When called in AMD64 guest mode, the guest PML4E shall be valid.
1692	*
1693	* @returns VBox status code.
1694	* @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1695	* @param pVCpu The VMCPU handle.
1696	* @param PdeSrc Page directory entry of the guest.
1697	* @param GCPtrPage Guest context page address.
1698	* @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1699	* @param uErr Fault error (X86_TRAP_PF_*).
1700	*/
1701	static int PGM_BTH_NAME(SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr)
1702	{
1703	PVM pVM = pVCpu->CTX_SUFF(pVM);
1704	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1705	LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr));
1706
1707	Assert(PGMIsLockOwner(pVM));
1708
1709	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
1710	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
1711	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
1712	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
1713	&& PGM_SHW_TYPE != PGM_TYPE_EPT
1714
1715	/*
1716	* Assert preconditions.
1717	*/
1718	Assert(PdeSrc.n.u1Present);
1719	Assert(cPages);
1720	# if 0 /* rarely useful; leave for debugging. */
1721	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
1722	# endif
1723
1724	/*
1725	* Get the shadow PDE, find the shadow page table in the pool.
1726	*/
1727	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1728	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1729	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1730
1731	/* Fetch the pgm pool shadow descriptor. */
1732	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1733	Assert(pShwPde);
1734
1735	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1736	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1737	PPGMPOOLPAGE pShwPde = NULL;
1738	PX86PDPAE pPDDst;
1739
1740	/* Fetch the pgm pool shadow descriptor. */
1741	int rc2 = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
1742	AssertRCSuccessReturn(rc2, rc2);
1743	Assert(pShwPde);
1744
1745	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR(pVM, pShwPde);
1746	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1747
1748	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1749	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1750	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1751	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
1752	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
1753
1754	int rc2 = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
1755	AssertRCSuccessReturn(rc2, rc2);
1756	Assert(pPDDst && pPdptDst);
1757	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1758	# endif
1759	SHWPDE PdeDst = *pPdeDst;
1760
1761	/* In the guest SMP case we could have blocked while another VCPU reused this page table. */
1762	if (!PdeDst.n.u1Present)
1763	{
1764	AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE p=%p/%RX64\n", pPdeDst, (uint64_t)PdeDst.u));
1765	Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", pVCpu->idCpu, GCPtrPage));
1766	return VINF_SUCCESS; /* force the instruction to be executed again. */
1767	}
1768
1769	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1770	Assert(pShwPage);
1771
1772	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1773	/* Fetch the pgm pool shadow descriptor. */
1774	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
1775	Assert(pShwPde);
1776	# endif
1777
1778	# if defined(IN_RC)
1779	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1780	PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
1781	# endif
1782
1783	/*
1784	* Check that the page is present and that the shadow PDE isn't out of sync.
1785	*/
1786	const bool fBigPage = PdeSrc.b.u1Size && GST_IS_PSE_ACTIVE(pVCpu);
1787	const bool fPdeValid = !fBigPage ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc);
1788	RTGCPHYS GCPhys;
1789	if (!fBigPage)
1790	{
1791	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1792	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1793	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1794	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE / 2);
1795	# endif
1796	}
1797	else
1798	{
1799	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(pVM, PdeSrc);
1800	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1801	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1802	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1803	# endif
1804	}
1805	if ( fPdeValid
1806	&& pShwPage->GCPhys == GCPhys
1807	&& PdeSrc.n.u1Present
1808	&& PdeSrc.n.u1User == PdeDst.n.u1User
1809	&& (PdeSrc.n.u1Write == PdeDst.n.u1Write \|\| !PdeDst.n.u1Write)
1810	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
1811	&& (PdeSrc.n.u1NoExecute == PdeDst.n.u1NoExecute \|\| !GST_IS_NX_ACTIVE(pVCpu))
1812	# endif
1813	)
1814	{
1815	/*
1816	* Check that the PDE is marked accessed already.
1817	* Since we set the accessed bit before getting here on a #PF, this
1818	* check is only meant for dealing with non-#PF'ing paths.
1819	*/
1820	if (PdeSrc.n.u1Accessed)
1821	{
1822	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1823	if (!fBigPage)
1824	{
1825	/*
1826	* 4KB Page - Map the guest page table.
1827	*/
1828	PGSTPT pPTSrc;
1829	int rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
1830	if (RT_SUCCESS(rc))
1831	{
1832	# ifdef PGM_SYNC_N_PAGES
1833	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
1834	if ( cPages > 1
1835	&& !(uErr & X86_TRAP_PF_P)
1836	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
1837	{
1838	/*
1839	* This code path is currently only taken when the caller is PGMTrap0eHandler
1840	* for non-present pages!
1841	*
1842	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1843	* deal with locality.
1844	*/
1845	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1846	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1847	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1848	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1849	# else
1850	const unsigned offPTSrc = 0;
1851	# endif
1852	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
1853	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1854	iPTDst = 0;
1855	else
1856	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1857	for (; iPTDst < iPTDstEnd; iPTDst++)
1858	{
1859	if (!pPTDst->a[iPTDst].n.u1Present)
1860	{
1861	GSTPTE PteSrc = pPTSrc->a[offPTSrc + iPTDst];
1862	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
1863	NOREF(GCPtrCurPage);
1864	#ifndef IN_RING0
1865	/*
1866	* Assuming kernel code will be marked as supervisor - and not as user level
1867	* and executed using a conforming code selector - And marked as readonly.
1868	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
1869	*/
1870	PPGMPAGE pPage;
1871	if ( ((PdeSrc.u & PteSrc.u) & (X86_PTE_RW \| X86_PTE_US))
1872	\|\| iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK) /* always sync GCPtrPage */
1873	\|\| !CSAMDoesPageNeedScanning(pVM, GCPtrCurPage)
1874	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
1875	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1876	)
1877	#endif /* else: CSAM not active */
1878	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1879	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1880	GCPtrCurPage, PteSrc.n.u1Present,
1881	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1882	PteSrc.n.u1User & PdeSrc.n.u1User,
1883	(uint64_t)PteSrc.u,
1884	(uint64_t)pPTDst->a[iPTDst].u,
1885	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1886	}
1887	}
1888	}
1889	else
1890	# endif /* PGM_SYNC_N_PAGES */
1891	{
1892	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1893	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1894	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1895	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1896	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
1897	GCPtrPage, PteSrc.n.u1Present,
1898	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1899	PteSrc.n.u1User & PdeSrc.n.u1User,
1900	(uint64_t)PteSrc.u,
1901	(uint64_t)pPTDst->a[iPTDst].u,
1902	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1903	}
1904	}
1905	else /* MMIO or invalid page: emulated in #PF handler. */
1906	{
1907	LogFlow(("PGM_GCPHYS_2_PTR %RGp failed with %Rrc\n", GCPhys, rc));
1908	Assert(!pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK].n.u1Present);
1909	}
1910	}
1911	else
1912	{
1913	/*
1914	* 4/2MB page - lazy syncing shadow 4K pages.
1915	* (There are many causes of getting here, it's no longer only CSAM.)
1916	*/
1917	/* Calculate the GC physical address of this 4KB shadow page. */
1918	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(pVM, PdeSrc) \| (GCPtrPage & GST_BIG_PAGE_OFFSET_MASK);
1919	/* Find ram range. */
1920	PPGMPAGE pPage;
1921	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
1922	if (RT_SUCCESS(rc))
1923	{
1924	AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
1925
1926	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1927	/* Try to make the page writable if necessary. */
1928	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1929	&& ( PGM_PAGE_IS_ZERO(pPage)
1930	\|\| ( PdeSrc.n.u1Write
1931	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1932	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1933	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1934	# endif
1935	# ifdef VBOX_WITH_PAGE_SHARING
1936	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
1937	# endif
1938	)
1939	)
1940	)
1941	{
1942	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1943	AssertRC(rc);
1944	}
1945	# endif
1946
1947	/*
1948	* Make shadow PTE entry.
1949	*/
1950	SHWPTE PteDst;
1951	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1952	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage,
1953	PdeSrc.u & ~( X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK
1954	\| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT),
1955	&PteDst);
1956	else
1957	PteDst.u = (PdeSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1958	\| PGM_PAGE_GET_HCPHYS(pPage);
1959
1960	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1961	if ( PteDst.n.u1Present
1962	&& !pPTDst->a[iPTDst].n.u1Present)
1963	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1964
1965	/* Make sure only allocated pages are mapped writable. */
1966	if ( PteDst.n.u1Write
1967	&& PteDst.n.u1Present
1968	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1969	{
1970	/* Still applies to shared pages. */
1971	Assert(!PGM_PAGE_IS_ZERO(pPage));
1972	PteDst.n.u1Write = 0; /** @todo this isn't quite working yet... */
1973	Log3(("SyncPage: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, GCPtrPage));
1974	}
1975
1976	ASMAtomicWriteSize(&pPTDst->a[iPTDst], PteDst.u);
1977
1978	/*
1979	* If the page is not flagged as dirty and is writable, then make it read-only
1980	* at PD level, so we can set the dirty bit when the page is modified.
1981	*
1982	* ASSUMES that page access handlers are implemented on page table entry level.
1983	* Thus we will first catch the dirty access and set PDE.D and restart. If
1984	* there is an access handler, we'll trap again and let it work on the problem.
1985	*/
1986	/** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
1987	* As for invlpg, it simply frees the whole shadow PT.
1988	* ...It's possibly because the guest clears it and the guest doesn't really tell us... */
1989	if ( !PdeSrc.b.u1Dirty
1990	&& PdeSrc.b.u1Write)
1991	{
1992	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
1993	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
1994	PdeDst.n.u1Write = 0;
1995	}
1996	else
1997	{
1998	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
1999	PdeDst.n.u1Write = PdeSrc.n.u1Write;
2000	}
2001	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2002	Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n",
2003	GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
2004	PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2005	}
2006	else
2007	{
2008	LogFlow(("PGM_GCPHYS_2_PTR %RGp (big) failed with %Rrc\n", GCPhys, rc));
2009	/** @todo must wipe the shadow page table in this case. */
2010	}
2011	}
2012	# if defined(IN_RC)
2013	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
2014	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2015	# endif
2016	return VINF_SUCCESS;
2017	}
2018
2019	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPagePDNAs));
2020	}
2021	else if (fPdeValid)
2022	{
2023	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPagePDOutOfSync));
2024	Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2025	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2026	}
2027	else
2028	{
2029	/// @todo STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSync));
2030	Log2(("SyncPage: Bad PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2031	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2032	}
2033
2034	/*
2035	* Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
2036	* Yea, I'm lazy.
2037	*/
2038	pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
2039	ASMAtomicWriteSize(pPdeDst, 0);
2040
2041	# if defined(IN_RC)
2042	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
2043	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2044	# endif
2045	PGM_INVL_VCPU_TLBS(pVCpu);
2046	return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
2047
2048
2049	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2050	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2051	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2052	&& !defined(IN_RC)
2053
2054	# ifdef PGM_SYNC_N_PAGES
2055	/*
2056	* Get the shadow PDE, find the shadow page table in the pool.
2057	*/
2058	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2059	X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
2060
2061	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2062	X86PDEPAE PdeDst = pgmShwGetPaePDE(pVCpu, GCPtrPage);
2063
2064	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2065	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2066	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpt);
2067	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2068	X86PDEPAE PdeDst;
2069	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2070
2071	int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2072	AssertRCSuccessReturn(rc, rc);
2073	Assert(pPDDst && pPdptDst);
2074	PdeDst = pPDDst->a[iPDDst];
2075	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2076	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2077	PEPTPD pPDDst;
2078	EPTPDE PdeDst;
2079
2080	int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, NULL, &pPDDst);
2081	if (rc != VINF_SUCCESS)
2082	{
2083	AssertRC(rc);
2084	return rc;
2085	}
2086	Assert(pPDDst);
2087	PdeDst = pPDDst->a[iPDDst];
2088	# endif
2089	/* In the guest SMP case we could have blocked while another VCPU reused this page table. */
2090	if (!PdeDst.n.u1Present)
2091	{
2092	AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)PdeDst.u));
2093	Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", pVCpu->idCpu, GCPtrPage));
2094	return VINF_SUCCESS; /* force the instruction to be executed again. */
2095	}
2096
2097	/* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */
2098	if (PdeDst.n.u1Size)
2099	{
2100	Assert(pVM->pgm.s.fNestedPaging);
2101	Log(("CPU%d: SyncPage: Pde (big:%RX64) at %RGv changed behind our back!\n", pVCpu->idCpu, PdeDst.u, GCPtrPage));
2102	return VINF_SUCCESS;
2103	}
2104
2105	/* Mask away the page offset. */
2106	GCPtrPage &= ~((RTGCPTR)0xfff);
2107
2108	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2109	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2110
2111	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
2112	if ( cPages > 1
2113	&& !(uErr & X86_TRAP_PF_P)
2114	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2115	{
2116	/*
2117	* This code path is currently only taken when the caller is PGMTrap0eHandler
2118	* for non-present pages!
2119	*
2120	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2121	* deal with locality.
2122	*/
2123	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2124	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2125	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2126	iPTDst = 0;
2127	else
2128	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2129	for (; iPTDst < iPTDstEnd; iPTDst++)
2130	{
2131	if (!pPTDst->a[iPTDst].n.u1Present)
2132	{
2133	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT);
2134	GSTPTE PteSrc;
2135
2136	/* Fake the page table entry */
2137	PteSrc.u = GCPtrCurPage;
2138	PteSrc.n.u1Present = 1;
2139	PteSrc.n.u1Dirty = 1;
2140	PteSrc.n.u1Accessed = 1;
2141	PteSrc.n.u1Write = 1;
2142	PteSrc.n.u1User = 1;
2143
2144	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2145
2146	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
2147	GCPtrCurPage, PteSrc.n.u1Present,
2148	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2149	PteSrc.n.u1User & PdeSrc.n.u1User,
2150	(uint64_t)PteSrc.u,
2151	(uint64_t)pPTDst->a[iPTDst].u,
2152	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2153
2154	if (RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)))
2155	break;
2156	}
2157	else
2158	Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n", (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT), iPTDst, pPTDst->a[iPTDst].u));
2159	}
2160	}
2161	else
2162	# endif /* PGM_SYNC_N_PAGES */
2163	{
2164	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2165	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT);
2166	GSTPTE PteSrc;
2167
2168	/* Fake the page table entry */
2169	PteSrc.u = GCPtrCurPage;
2170	PteSrc.n.u1Present = 1;
2171	PteSrc.n.u1Dirty = 1;
2172	PteSrc.n.u1Accessed = 1;
2173	PteSrc.n.u1Write = 1;
2174	PteSrc.n.u1User = 1;
2175	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2176
2177	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}PteDst=%08llx%s\n",
2178	GCPtrPage, PteSrc.n.u1Present,
2179	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2180	PteSrc.n.u1User & PdeSrc.n.u1User,
2181	(uint64_t)PteSrc.u,
2182	(uint64_t)pPTDst->a[iPTDst].u,
2183	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2184	}
2185	return VINF_SUCCESS;
2186
2187	#else
2188	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2189	return VERR_INTERNAL_ERROR;
2190	#endif
2191	}
2192
2193
2194	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2195
2196	/**
2197	* CheckPageFault helper for returning a page fault indicating a non-present
2198	* (NP) entry in the page translation structures.
2199	*
2200	* @returns VINF_EM_RAW_GUEST_TRAP.
2201	* @param pVCpu The virtual CPU to operate on.
2202	* @param uErr The error code of the shadow fault. Corrections to
2203	* TRPM's copy will be made if necessary.
2204	* @param GCPtrPage For logging.
2205	* @param uPageFaultLevel For logging.
2206	*/
2207	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnNP)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2208	{
2209	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2210	AssertMsg(!(uErr & X86_TRAP_PF_P), ("%#x\n", uErr));
2211	AssertMsg(!(uErr & X86_TRAP_PF_RSVD), ("%#x\n", uErr));
2212	if (uErr & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
2213	TRPMSetErrorCode(pVCpu, uErr & ~(X86_TRAP_PF_RSVD \| X86_TRAP_PF_P));
2214
2215	Log(("CheckPageFault: real page fault (notp) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2216	return VINF_EM_RAW_GUEST_TRAP;
2217	}
2218
2219
2220	/**
2221	* CheckPageFault helper for returning a page fault indicating a reserved bit
2222	* (RSVD) error in the page translation structures.
2223	*
2224	* @returns VINF_EM_RAW_GUEST_TRAP.
2225	* @param pVCpu The virtual CPU to operate on.
2226	* @param uErr The error code of the shadow fault. Corrections to
2227	* TRPM's copy will be made if necessary.
2228	* @param GCPtrPage For logging.
2229	* @param uPageFaultLevel For logging.
2230	*/
2231	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnRSVD)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2232	{
2233	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2234	if ((uErr & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) != (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
2235	TRPMSetErrorCode(pVCpu, uErr \| X86_TRAP_PF_RSVD \| X86_TRAP_PF_P);
2236
2237	Log(("CheckPageFault: real page fault (rsvd) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2238	return VINF_EM_RAW_GUEST_TRAP;
2239	}
2240
2241
2242	/**
2243	* CheckPageFault helper for returning a page protection fault (P).
2244	*
2245	* @returns VINF_EM_RAW_GUEST_TRAP.
2246	* @param pVCpu The virtual CPU to operate on.
2247	* @param uErr The error code of the shadow fault. Corrections to
2248	* TRPM's copy will be made if necessary.
2249	* @param GCPtrPage For logging.
2250	* @param uPageFaultLevel For logging.
2251	*/
2252	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnProt)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2253	{
2254	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2255	AssertMsg(uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID), ("%#x\n", uErr));
2256	if ((uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) != X86_TRAP_PF_P)
2257	TRPMSetErrorCode(pVCpu, (uErr & ~X86_TRAP_PF_RSVD) \| X86_TRAP_PF_P);
2258
2259	Log(("CheckPageFault: real page fault (prot) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2260	return VINF_EM_RAW_GUEST_TRAP;
2261	}
2262
2263
2264	/**
2265	* Handle dirty bit tracking faults.
2266	*
2267	* @returns VBox status code.
2268	* @param pVCpu The VMCPU handle.
2269	* @param uErr Page fault error code.
2270	* @param pPdeSrc Guest page directory entry.
2271	* @param pPdeDst Shadow page directory entry.
2272	* @param GCPtrPage Guest context page address.
2273	*/
2274	static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, RTGCPTR GCPtrPage)
2275	{
2276	PVM pVM = pVCpu->CTX_SUFF(pVM);
2277	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2278
2279	Assert(PGMIsLockOwner(pVM));
2280
2281	/*
2282	* Handle big page.
2283	*/
2284	if (pPdeSrc->b.u1Size && GST_IS_PSE_ACTIVE(pVCpu))
2285	{
2286	if ( pPdeDst->n.u1Present
2287	&& (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY))
2288	{
2289	SHWPDE PdeDst = *pPdeDst;
2290
2291	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageTrap));
2292	Assert(pPdeSrc->b.u1Write);
2293
2294	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
2295	* fault again and take this path to only invalidate the entry (see below).
2296	*/
2297	PdeDst.n.u1Write = 1;
2298	PdeDst.n.u1Accessed = 1;
2299	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2300	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2301	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
2302	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2303	}
2304
2305	# ifdef IN_RING0
2306	/* Check for stale TLB entry; only applies to the SMP guest case. */
2307	if ( pVM->cCpus > 1
2308	&& pPdeDst->n.u1Write
2309	&& pPdeDst->n.u1Accessed)
2310	{
2311	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2312	if (pShwPage)
2313	{
2314	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2315	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2316	if ( pPteDst->n.u1Present
2317	&& pPteDst->n.u1Write)
2318	{
2319	/* Stale TLB entry. */
2320	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageStale));
2321	PGM_INVL_PG(pVCpu, GCPtrPage);
2322	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2323	}
2324	}
2325	}
2326	# endif /* IN_RING0 */
2327	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2328	}
2329
2330	/*
2331	* Map the guest page table.
2332	*/
2333	PGSTPT pPTSrc;
2334	int rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
2335	if (RT_FAILURE(rc))
2336	{
2337	AssertRC(rc);
2338	return rc;
2339	}
2340
2341	if (pPdeDst->n.u1Present)
2342	{
2343	GSTPTE const *pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2344	const GSTPTE PteSrc = *pPteSrc;
2345
2346	#ifndef IN_RING0
2347	/* Bail out here as pgmPoolGetPage will return NULL and we'll crash below.
2348	* Our individual shadow handlers will provide more information and force a fatal exit.
2349	*/
2350	if (MMHyperIsInsideArea(pVM, (RTGCPTR)GCPtrPage))
2351	{
2352	LogRel(("CheckPageFault: write to hypervisor region %RGv\n", GCPtrPage));
2353	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2354	}
2355	#endif
2356	/*
2357	* Map shadow page table.
2358	*/
2359	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2360	if (pShwPage)
2361	{
2362	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2363	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2364	if (pPteDst->n.u1Present) /** @todo Optimize accessed bit emulation? */
2365	{
2366	if (pPteDst->u & PGM_PTFLAGS_TRACK_DIRTY)
2367	{
2368	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, pPteSrc->u & GST_PTE_PG_MASK);
2369	SHWPTE PteDst = *pPteDst;
2370
2371	LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage));
2372	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageTrap));
2373
2374	Assert(pPteSrc->n.u1Write);
2375
2376	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB
2377	* entry will not harm; write access will simply fault again and
2378	* take this path to only invalidate the entry.
2379	*/
2380	if (RT_LIKELY(pPage))
2381	{
2382	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2383	{
2384	AssertMsgFailed(("%R[pgmpage] - we don't set PGM_PTFLAGS_TRACK_DIRTY for these pages\n", pPage));
2385	Assert(!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage));
2386	/* Assuming write handlers here as the PTE is present (otherwise we wouldn't be here). */
2387	PteDst.n.u1Write = 0;
2388	}
2389	else
2390	{
2391	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
2392	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
2393	{
2394	rc = pgmPhysPageMakeWritable(pVM, pPage, pPteSrc->u & GST_PTE_PG_MASK);
2395	AssertRC(rc);
2396	}
2397	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED)
2398	PteDst.n.u1Write = 1;
2399	else
2400	{
2401	/* Still applies to shared pages. */
2402	Assert(!PGM_PAGE_IS_ZERO(pPage));
2403	PteDst.n.u1Write = 0;
2404	}
2405	}
2406	}
2407	else
2408	PteDst.n.u1Write = 1; /** @todo r=bird: This doesn't make sense to me. */
2409
2410	PteDst.n.u1Dirty = 1;
2411	PteDst.n.u1Accessed = 1;
2412	PteDst.au32[0] &= ~PGM_PTFLAGS_TRACK_DIRTY;
2413	ASMAtomicWriteSize(pPteDst, PteDst.u);
2414	PGM_INVL_PG(pVCpu, GCPtrPage);
2415	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2416	}
2417
2418	# ifdef IN_RING0
2419	/* Check for stale TLB entry; only applies to the SMP guest case. */
2420	if ( pVM->cCpus > 1
2421	&& pPteDst->n.u1Write == 1
2422	&& pPteDst->n.u1Accessed == 1)
2423	{
2424	/* Stale TLB entry. */
2425	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageStale));
2426	PGM_INVL_PG(pVCpu, GCPtrPage);
2427	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2428	}
2429	# endif
2430	}
2431	}
2432	else
2433	AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
2434	}
2435
2436	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2437	}
2438
2439	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
2440
2441
2442	/**
2443	* Sync a shadow page table.
2444	*
2445	* The shadow page table is not present. This includes the case where
2446	* there is a conflict with a mapping.
2447	*
2448	* @returns VBox status code.
2449	* @param pVCpu The VMCPU handle.
2450	* @param iPD Page directory index.
2451	* @param pPDSrc Source page directory (i.e. Guest OS page directory).
2452	* Assume this is a temporary mapping.
2453	* @param GCPtrPage GC Pointer of the page that caused the fault
2454	*/
2455	static int PGM_BTH_NAME(SyncPT)(PVMCPU pVCpu, unsigned iPDSrc, PGSTPD pPDSrc, RTGCPTR GCPtrPage)
2456	{
2457	PVM pVM = pVCpu->CTX_SUFF(pVM);
2458	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2459
2460	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2461	#if 0 /* rarely useful; leave for debugging. */
2462	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPtPD[iPDSrc]);
2463	#endif
2464	LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage));
2465
2466	Assert(PGMIsLocked(pVM));
2467
2468	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2469	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
2470	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
2471	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2472	&& PGM_SHW_TYPE != PGM_TYPE_EPT
2473
2474	int rc = VINF_SUCCESS;
2475
2476	/*
2477	* Validate input a little bit.
2478	*/
2479	AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
2480	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2481	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2482	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
2483
2484	/* Fetch the pgm pool shadow descriptor. */
2485	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2486	Assert(pShwPde);
2487
2488	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2489	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2490	PPGMPOOLPAGE pShwPde = NULL;
2491	PX86PDPAE pPDDst;
2492	PSHWPDE pPdeDst;
2493
2494	/* Fetch the pgm pool shadow descriptor. */
2495	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
2496	AssertRCSuccessReturn(rc, rc);
2497	Assert(pShwPde);
2498
2499	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR(pVM, pShwPde);
2500	pPdeDst = &pPDDst->a[iPDDst];
2501
2502	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2503	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2504	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2505	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2506	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2507	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2508	AssertRCSuccessReturn(rc, rc);
2509	Assert(pPDDst);
2510	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2511	# endif
2512	SHWPDE PdeDst = *pPdeDst;
2513
2514	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2515	/* Fetch the pgm pool shadow descriptor. */
2516	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2517	Assert(pShwPde);
2518	# endif
2519
2520	# ifndef PGM_WITHOUT_MAPPINGS
2521	/*
2522	* Check for conflicts.
2523	* RC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
2524	* R3: Simply resolve the conflict.
2525	*/
2526	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2527	{
2528	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
2529	# ifndef IN_RING3
2530	Log(("SyncPT: Conflict at %RGv\n", GCPtrPage));
2531	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2532	return VERR_ADDRESS_CONFLICT;
2533
2534	# else /* IN_RING3 */
2535	PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
2536	Assert(pMapping);
2537	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2538	rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2539	# elif PGM_GST_TYPE == PGM_TYPE_PAE
2540	rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2541	# else
2542	AssertFailed(); /* can't happen for amd64 */
2543	# endif
2544	if (RT_FAILURE(rc))
2545	{
2546	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2547	return rc;
2548	}
2549	PdeDst = *pPdeDst;
2550	# endif /* IN_RING3 */
2551	}
2552	# endif /* !PGM_WITHOUT_MAPPINGS */
2553	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2554
2555	# if defined(IN_RC)
2556	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
2557	PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
2558	# endif
2559
2560	/*
2561	* Sync page directory entry.
2562	*/
2563	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2564	if (PdeSrc.n.u1Present)
2565	{
2566	/*
2567	* Allocate & map the page table.
2568	*/
2569	PSHWPT pPTDst;
2570	const bool fPageTable = !PdeSrc.b.u1Size \|\| !GST_IS_PSE_ACTIVE(pVCpu);
2571	PPGMPOOLPAGE pShwPage;
2572	RTGCPHYS GCPhys;
2573	if (fPageTable)
2574	{
2575	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
2576	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2577	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2578	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE / 2);
2579	# endif
2580	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
2581	}
2582	else
2583	{
2584	PGMPOOLACCESS enmAccess;
2585	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2586	const bool fNoExecute = PdeSrc.n.u1NoExecute && GST_IS_NX_ACTIVE(pVCpu);
2587	# else
2588	const bool fNoExecute = false;
2589	# endif
2590
2591	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(pVM, PdeSrc);
2592	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2593	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2594	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
2595	# endif
2596	/* Determine the right kind of large page to avoid incorrect cached entry reuse. */
2597	if (PdeSrc.n.u1User)
2598	{
2599	if (PdeSrc.n.u1Write)
2600	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_RW_NX : PGMPOOLACCESS_USER_RW;
2601	else
2602	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_R_NX : PGMPOOLACCESS_USER_R;
2603	}
2604	else
2605	{
2606	if (PdeSrc.n.u1Write)
2607	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW;
2608	else
2609	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_R_NX : PGMPOOLACCESS_SUPERVISOR_R;
2610	}
2611	rc = pgmPoolAllocEx(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, enmAccess, pShwPde->idx, iPDDst, &pShwPage);
2612	}
2613	if (rc == VINF_SUCCESS)
2614	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2615	else if (rc == VINF_PGM_CACHED_PAGE)
2616	{
2617	/*
2618	* The PT was cached, just hook it up.
2619	*/
2620	if (fPageTable)
2621	PdeDst.u = pShwPage->Core.Key
2622	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2623	else
2624	{
2625	PdeDst.u = pShwPage->Core.Key
2626	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2627	/* (see explanation and assumptions further down.) */
2628	if ( !PdeSrc.b.u1Dirty
2629	&& PdeSrc.b.u1Write)
2630	{
2631	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2632	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2633	PdeDst.b.u1Write = 0;
2634	}
2635	}
2636	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2637	# if defined(IN_RC)
2638	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2639	# endif
2640	return VINF_SUCCESS;
2641	}
2642	else if (rc == VERR_PGM_POOL_FLUSHED)
2643	{
2644	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2645	# if defined(IN_RC)
2646	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2647	# endif
2648	return VINF_PGM_SYNC_CR3;
2649	}
2650	else
2651	AssertMsgFailedReturn(("rc=%Rrc\n", rc), VERR_INTERNAL_ERROR);
2652	PdeDst.u &= X86_PDE_AVL_MASK;
2653	PdeDst.u \|= pShwPage->Core.Key;
2654
2655	/*
2656	* Page directory has been accessed (this is a fault situation, remember).
2657	*/
2658	pPDSrc->a[iPDSrc].n.u1Accessed = 1;
2659	if (fPageTable)
2660	{
2661	/*
2662	* Page table - 4KB.
2663	*
2664	* Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
2665	*/
2666	Log2(("SyncPT: 4K %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
2667	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
2668	PGSTPT pPTSrc;
2669	rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
2670	if (RT_SUCCESS(rc))
2671	{
2672	/*
2673	* Start by syncing the page directory entry so CSAM's TLB trick works.
2674	*/
2675	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| X86_PDE_AVL_MASK))
2676	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2677	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2678	# if defined(IN_RC)
2679	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2680	# endif
2681
2682	/*
2683	* Directory/page user or supervisor privilege: (same goes for read/write)
2684	*
2685	* Directory Page Combined
2686	* U/S U/S U/S
2687	* 0 0 0
2688	* 0 1 0
2689	* 1 0 0
2690	* 1 1 1
2691	*
2692	* Simple AND operation. Table listed for completeness.
2693	*
2694	*/
2695	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT4K));
2696	# ifdef PGM_SYNC_N_PAGES
2697	unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2698	unsigned iPTDst = iPTBase;
2699	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2700	if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
2701	iPTDst = 0;
2702	else
2703	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2704	# else /* !PGM_SYNC_N_PAGES */
2705	unsigned iPTDst = 0;
2706	const unsigned iPTDstEnd = RT_ELEMENTS(pPTDst->a);
2707	# endif /* !PGM_SYNC_N_PAGES */
2708	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2709	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2710	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2711	# else
2712	const unsigned offPTSrc = 0;
2713	# endif
2714	for (; iPTDst < iPTDstEnd; iPTDst++)
2715	{
2716	const unsigned iPTSrc = iPTDst + offPTSrc;
2717	const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2718
2719	if (PteSrc.n.u1Present)
2720	{
2721	# ifndef IN_RING0
2722	/*
2723	* Assuming kernel code will be marked as supervisor - and not as user level
2724	* and executed using a conforming code selector - And marked as readonly.
2725	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
2726	*/
2727	PPGMPAGE pPage;
2728	if ( ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW \| X86_PTE_US))
2729	\|\| !CSAMDoesPageNeedScanning(pVM, (iPDSrc << GST_PD_SHIFT) \| (iPTSrc << PAGE_SHIFT))
2730	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
2731	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2732	)
2733	# endif
2734	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2735	Log2(("SyncPT: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
2736	(RTGCPTR)(((RTGCPTR)iPDSrc << GST_PD_SHIFT) \| ((RTGCPTR)iPTSrc << PAGE_SHIFT)),
2737	PteSrc.n.u1Present,
2738	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2739	PteSrc.n.u1User & PdeSrc.n.u1User,
2740	(uint64_t)PteSrc.u,
2741	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : "", pPTDst->a[iPTDst].u, iPTSrc, PdeSrc.au32[0],
2742	(RTGCPHYS)((PdeSrc.u & GST_PDE_PG_MASK) + iPTSrc*sizeof(PteSrc)) ));
2743	}
2744	/* else: the page table was cleared by the pool */
2745	} /* for PTEs */
2746	}
2747	}
2748	else
2749	{
2750	/*
2751	* Big page - 2/4MB.
2752	*
2753	* We'll walk the ram range list in parallel and optimize lookups.
2754	* We will only sync on shadow page table at a time.
2755	*/
2756	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT4M));
2757
2758	/**
2759	* @todo It might be more efficient to sync only a part of the 4MB page (similar to what we do for 4kb PDs).
2760	*/
2761
2762	/*
2763	* Start by syncing the page directory entry.
2764	*/
2765	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
2766	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2767
2768	/*
2769	* If the page is not flagged as dirty and is writable, then make it read-only
2770	* at PD level, so we can set the dirty bit when the page is modified.
2771	*
2772	* ASSUMES that page access handlers are implemented on page table entry level.
2773	* Thus we will first catch the dirty access and set PDE.D and restart. If
2774	* there is an access handler, we'll trap again and let it work on the problem.
2775	*/
2776	/** @todo move the above stuff to a section in the PGM documentation. */
2777	Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
2778	if ( !PdeSrc.b.u1Dirty
2779	&& PdeSrc.b.u1Write)
2780	{
2781	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2782	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2783	PdeDst.b.u1Write = 0;
2784	}
2785	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2786	# if defined(IN_RC)
2787	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2788	# endif
2789
2790	/*
2791	* Fill the shadow page table.
2792	*/
2793	/* Get address and flags from the source PDE. */
2794	SHWPTE PteDstBase;
2795	PteDstBase.u = PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT);
2796
2797	/* Loop thru the entries in the shadow PT. */
2798	const RTGCPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
2799	Log2(("SyncPT: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n",
2800	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
2801	GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2802	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
2803	unsigned iPTDst = 0;
2804	while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2805	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2806	{
2807	/* Advance ram range list. */
2808	while (pRam && GCPhys > pRam->GCPhysLast)
2809	pRam = pRam->CTX_SUFF(pNext);
2810	if (pRam && GCPhys >= pRam->GCPhys)
2811	{
2812	unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2813	do
2814	{
2815	/* Make shadow PTE. */
2816	PPGMPAGE pPage = &pRam->aPages[iHCPage];
2817	SHWPTE PteDst;
2818
2819	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2820	/* Try to make the page writable if necessary. */
2821	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2822	&& ( PGM_PAGE_IS_ZERO(pPage)
2823	\|\| ( PteDstBase.n.u1Write
2824	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2825	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
2826	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
2827	# endif
2828	# ifdef VBOX_WITH_PAGE_SHARING
2829	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
2830	# endif
2831	&& !PGM_PAGE_IS_BALLOONED(pPage))
2832	)
2833	)
2834	{
2835	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
2836	AssertRCReturn(rc, rc);
2837	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2838	break;
2839	}
2840	# endif
2841
2842	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2843	{
2844	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
2845	{
2846	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) \| PteDstBase.u;
2847	PteDst.n.u1Write = 0;
2848	}
2849	else
2850	PteDst.u = 0;
2851	}
2852	else if (PGM_PAGE_IS_BALLOONED(pPage))
2853	{
2854	/* Skip ballooned pages. */
2855	PteDst.u = 0;
2856	}
2857	# ifndef IN_RING0
2858	/*
2859	* Assuming kernel code will be marked as supervisor and not as user level and executed
2860	* using a conforming code selector. Don't check for readonly, as that implies the whole
2861	* 4MB can be code or readonly data. Linux enables write access for its large pages.
2862	*/
2863	else if ( !PdeSrc.n.u1User
2864	&& CSAMDoesPageNeedScanning(pVM, GCPtr \| (iPTDst << SHW_PT_SHIFT)))
2865	PteDst.u = 0;
2866	# endif
2867	else
2868	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) \| PteDstBase.u;
2869
2870	/* Only map writable pages writable. */
2871	if ( PteDst.n.u1Write
2872	&& PteDst.n.u1Present
2873	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2874	{
2875	/* Still applies to shared pages. */
2876	Assert(!PGM_PAGE_IS_ZERO(pPage));
2877	PteDst.n.u1Write = 0; /** @todo this isn't quite working yet... */
2878	Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))));
2879	}
2880
2881	if (PteDst.n.u1Present)
2882	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2883
2884	/* commit it */
2885	pPTDst->a[iPTDst] = PteDst;
2886	Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
2887	(RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT)), PteDst.n.u1Present, PteDst.n.u1Write, PteDst.n.u1User, (uint64_t)PteDst.u,
2888	PteDst.u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2889
2890	/* advance */
2891	GCPhys += PAGE_SIZE;
2892	iHCPage++;
2893	iPTDst++;
2894	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2895	&& GCPhys <= pRam->GCPhysLast);
2896	}
2897	else if (pRam)
2898	{
2899	Log(("Invalid pages at %RGp\n", GCPhys));
2900	do
2901	{
2902	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2903	GCPhys += PAGE_SIZE;
2904	iPTDst++;
2905	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2906	&& GCPhys < pRam->GCPhys);
2907	}
2908	else
2909	{
2910	Log(("Invalid pages at %RGp (2)\n", GCPhys));
2911	for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
2912	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2913	}
2914	} /* while more PTEs */
2915	} /* 4KB / 4MB */
2916	}
2917	else
2918	AssertRelease(!PdeDst.n.u1Present);
2919
2920	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2921	if (RT_FAILURE(rc))
2922	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPTFailed));
2923	return rc;
2924
2925	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2926	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2927	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2928	&& !defined(IN_RC)
2929
2930	/*
2931	* Validate input a little bit.
2932	*/
2933	int rc = VINF_SUCCESS;
2934	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2935	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2936	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
2937
2938	/* Fetch the pgm pool shadow descriptor. */
2939	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2940	Assert(pShwPde);
2941
2942	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2943	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2944	PPGMPOOLPAGE pShwPde = NULL; /* initialized to shut up gcc */
2945	PX86PDPAE pPDDst;
2946	PSHWPDE pPdeDst;
2947
2948	/* Fetch the pgm pool shadow descriptor. */
2949	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
2950	AssertRCSuccessReturn(rc, rc);
2951	Assert(pShwPde);
2952
2953	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR(pVM, pShwPde);
2954	pPdeDst = &pPDDst->a[iPDDst];
2955
2956	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2957	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2958	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2959	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2960	PX86PDPT pPdptDst= NULL; /* initialized to shut up gcc */
2961	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2962	AssertRCSuccessReturn(rc, rc);
2963	Assert(pPDDst);
2964	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2965
2966	/* Fetch the pgm pool shadow descriptor. */
2967	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2968	Assert(pShwPde);
2969
2970	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2971	const unsigned iPdpt = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
2972	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2973	PEPTPD pPDDst;
2974	PEPTPDPT pPdptDst;
2975
2976	rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, &pPdptDst, &pPDDst);
2977	if (rc != VINF_SUCCESS)
2978	{
2979	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2980	AssertRC(rc);
2981	return rc;
2982	}
2983	Assert(pPDDst);
2984	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2985
2986	/* Fetch the pgm pool shadow descriptor. */
2987	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & EPT_PDPTE_PG_MASK);
2988	Assert(pShwPde);
2989	# endif
2990	SHWPDE PdeDst = *pPdeDst;
2991
2992	Assert(!(PdeDst.u & PGM_PDFLAGS_MAPPING));
2993	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2994
2995	# if defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
2996	if (BTH_IS_NP_ACTIVE(pVM))
2997	{
2998	PPGMPAGE pPage;
2999
3000	/* Check if we allocated a big page before for this 2 MB range. */
3001	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPtrPage & X86_PDE2M_PAE_PG_MASK, &pPage);
3002	if (RT_SUCCESS(rc))
3003	{
3004	RTHCPHYS HCPhys = NIL_RTHCPHYS;
3005
3006	if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
3007	{
3008	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageReused);
3009	AssertRelease(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3010	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3011	}
3012	else if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED)
3013	{
3014	/* Recheck the entire 2 MB range to see if we can use it again as a large page. */
3015	rc = pgmPhysIsValidLargePage(pVM, GCPtrPage, pPage);
3016	if (RT_SUCCESS(rc))
3017	{
3018	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3019	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3020	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3021	}
3022	}
3023	else if (PGMIsUsingLargePages(pVM))
3024	{
3025	rc = pgmPhysAllocLargePage(pVM, GCPtrPage);
3026	if (RT_SUCCESS(rc))
3027	{
3028	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3029	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3030	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3031	}
3032	else
3033	LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
3034	}
3035
3036	if (HCPhys != NIL_RTHCPHYS)
3037	{
3038	PdeDst.u &= X86_PDE_AVL_MASK;
3039	PdeDst.u \|= HCPhys;
3040	PdeDst.n.u1Present = 1;
3041	PdeDst.n.u1Write = 1;
3042	PdeDst.b.u1Size = 1;
3043	# if PGM_SHW_TYPE == PGM_TYPE_EPT
3044	PdeDst.n.u1Execute = 1;
3045	PdeDst.b.u1IgnorePAT = 1;
3046	PdeDst.b.u3EMT = VMX_EPT_MEMTYPE_WB;
3047	# else
3048	PdeDst.n.u1User = 1;
3049	# endif
3050	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
3051
3052	Log(("SyncPT: Use large page at %RGp PDE=%RX64\n", GCPtrPage, PdeDst.u));
3053	/* Add a reference to the first page only. */
3054	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPde, PGM_PAGE_GET_TRACKING(pPage), pPage, iPDDst);
3055
3056	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3057	return VINF_SUCCESS;
3058	}
3059	}
3060	}
3061	# endif /* HC_ARCH_BITS == 64 */
3062
3063	GSTPDE PdeSrc;
3064	PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3065	PdeSrc.n.u1Present = 1;
3066	PdeSrc.n.u1Write = 1;
3067	PdeSrc.n.u1Accessed = 1;
3068	PdeSrc.n.u1User = 1;
3069
3070	/*
3071	* Allocate & map the page table.
3072	*/
3073	PSHWPT pPTDst;
3074	PPGMPOOLPAGE pShwPage;
3075	RTGCPHYS GCPhys;
3076
3077	/* Virtual address = physical address */
3078	GCPhys = GCPtrPage & X86_PAGE_4K_BASE_MASK;
3079	rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
3080
3081	if ( rc == VINF_SUCCESS
3082	\|\| rc == VINF_PGM_CACHED_PAGE)
3083	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
3084	else
3085	AssertMsgFailedReturn(("rc=%Rrc\n", rc), VERR_INTERNAL_ERROR);
3086
3087	PdeDst.u &= X86_PDE_AVL_MASK;
3088	PdeDst.u \|= pShwPage->Core.Key;
3089	PdeDst.n.u1Present = 1;
3090	PdeDst.n.u1Write = 1;
3091	# if PGM_SHW_TYPE == PGM_TYPE_EPT
3092	PdeDst.n.u1Execute = 1;
3093	# else
3094	PdeDst.n.u1User = 1;
3095	PdeDst.n.u1Accessed = 1;
3096	# endif
3097	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
3098
3099	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, PGM_SYNC_NR_PAGES, 0 /* page not present */);
3100	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3101	return rc;
3102
3103	#else
3104	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
3105	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3106	return VERR_INTERNAL_ERROR;
3107	#endif
3108	}
3109
3110
3111
3112	/**
3113	* Prefetch a page/set of pages.
3114	*
3115	* Typically used to sync commonly used pages before entering raw mode
3116	* after a CR3 reload.
3117	*
3118	* @returns VBox status code.
3119	* @param pVCpu The VMCPU handle.
3120	* @param GCPtrPage Page to invalidate.
3121	*/
3122	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
3123	{
3124	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3125	\|\| PGM_GST_TYPE == PGM_TYPE_REAL \
3126	\|\| PGM_GST_TYPE == PGM_TYPE_PROT \
3127	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3128	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3129	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
3130	&& PGM_SHW_TYPE != PGM_TYPE_EPT
3131
3132	/*
3133	* Check that all Guest levels thru the PDE are present, getting the
3134	* PD and PDE in the processes.
3135	*/
3136	int rc = VINF_SUCCESS;
3137	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3138	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3139	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
3140	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3141	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3142	unsigned iPDSrc;
3143	X86PDPE PdpeSrc;
3144	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3145	if (!pPDSrc)
3146	return VINF_SUCCESS; /* not present */
3147	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3148	unsigned iPDSrc;
3149	PX86PML4E pPml4eSrc;
3150	X86PDPE PdpeSrc;
3151	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3152	if (!pPDSrc)
3153	return VINF_SUCCESS; /* not present */
3154	# endif
3155	const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3156	# else
3157	PGSTPD pPDSrc = NULL;
3158	const unsigned iPDSrc = 0;
3159	GSTPDE PdeSrc;
3160
3161	PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3162	PdeSrc.n.u1Present = 1;
3163	PdeSrc.n.u1Write = 1;
3164	PdeSrc.n.u1Accessed = 1;
3165	PdeSrc.n.u1User = 1;
3166	# endif
3167
3168	if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
3169	{
3170	PVM pVM = pVCpu->CTX_SUFF(pVM);
3171	pgmLock(pVM);
3172
3173	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3174	const X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
3175	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3176	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3177	PX86PDPAE pPDDst;
3178	X86PDEPAE PdeDst;
3179	# if PGM_GST_TYPE != PGM_TYPE_PAE
3180	X86PDPE PdpeSrc;
3181
3182	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3183	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3184	# endif
3185	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3186	if (rc != VINF_SUCCESS)
3187	{
3188	pgmUnlock(pVM);
3189	AssertRC(rc);
3190	return rc;
3191	}
3192	Assert(pPDDst);
3193	PdeDst = pPDDst->a[iPDDst];
3194
3195	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3196	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3197	PX86PDPAE pPDDst;
3198	X86PDEPAE PdeDst;
3199
3200	# if PGM_GST_TYPE == PGM_TYPE_PROT
3201	/* AMD-V nested paging */
3202	X86PML4E Pml4eSrc;
3203	X86PDPE PdpeSrc;
3204	PX86PML4E pPml4eSrc = &Pml4eSrc;
3205
3206	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3207	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
3208	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
3209	# endif
3210
3211	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3212	if (rc != VINF_SUCCESS)
3213	{
3214	pgmUnlock(pVM);
3215	AssertRC(rc);
3216	return rc;
3217	}
3218	Assert(pPDDst);
3219	PdeDst = pPDDst->a[iPDDst];
3220	# endif
3221	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
3222	{
3223	if (!PdeDst.n.u1Present)
3224	{
3225	/** @todo r=bird: This guy will set the A bit on the PDE,
3226	* probably harmless. */
3227	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3228	}
3229	else
3230	{
3231	/* Note! We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
3232	* R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
3233	* makes no sense to prefetch more than one page.
3234	*/
3235	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3236	if (RT_SUCCESS(rc))
3237	rc = VINF_SUCCESS;
3238	}
3239	}
3240	pgmUnlock(pVM);
3241	}
3242	return rc;
3243
3244	#elif PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3245	return VINF_SUCCESS; /* ignore */
3246	#else
3247	AssertCompile(0);
3248	#endif
3249	}
3250
3251
3252
3253
3254	/**
3255	* Syncs a page during a PGMVerifyAccess() call.
3256	*
3257	* @returns VBox status code (informational included).
3258	* @param pVCpu The VMCPU handle.
3259	* @param GCPtrPage The address of the page to sync.
3260	* @param fPage The effective guest page flags.
3261	* @param uErr The trap error code.
3262	* @remarks This will normally never be called on invalid guest page
3263	* translation entries.
3264	*/
3265	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fPage, unsigned uErr)
3266	{
3267	PVM pVM = pVCpu->CTX_SUFF(pVM);
3268
3269	LogFlow(("VerifyAccessSyncPage: GCPtrPage=%RGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
3270
3271	Assert(!pVM->pgm.s.fNestedPaging);
3272	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3273	\|\| PGM_GST_TYPE == PGM_TYPE_REAL \
3274	\|\| PGM_GST_TYPE == PGM_TYPE_PROT \
3275	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3276	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3277	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
3278	&& PGM_SHW_TYPE != PGM_TYPE_EPT
3279
3280	# ifndef IN_RING0
3281	if (!(fPage & X86_PTE_US))
3282	{
3283	/*
3284	* Mark this page as safe.
3285	*/
3286	/** @todo not correct for pages that contain both code and data!! */
3287	Log(("CSAMMarkPage %RGv; scanned=%d\n", GCPtrPage, true));
3288	CSAMMarkPage(pVM, GCPtrPage, true);
3289	}
3290	# endif
3291
3292	/*
3293	* Get guest PD and index.
3294	*/
3295	/** @todo Performance: We've done all this a jiffy ago in the
3296	* PGMGstGetPage call. */
3297	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3298	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3299	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
3300	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3301
3302	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3303	unsigned iPDSrc = 0;
3304	X86PDPE PdpeSrc;
3305	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3306	if (RT_UNLIKELY(!pPDSrc))
3307	{
3308	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3309	return VINF_EM_RAW_GUEST_TRAP;
3310	}
3311
3312	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3313	unsigned iPDSrc = 0; /* shut up gcc */
3314	PX86PML4E pPml4eSrc = NULL; /* ditto */
3315	X86PDPE PdpeSrc;
3316	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3317	if (RT_UNLIKELY(!pPDSrc))
3318	{
3319	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3320	return VINF_EM_RAW_GUEST_TRAP;
3321	}
3322	# endif
3323
3324	# else /* !PGM_WITH_PAGING */
3325	PGSTPD pPDSrc = NULL;
3326	const unsigned iPDSrc = 0;
3327	# endif /* !PGM_WITH_PAGING */
3328	int rc = VINF_SUCCESS;
3329
3330	pgmLock(pVM);
3331
3332	/*
3333	* First check if the shadow pd is present.
3334	*/
3335	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3336	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3337
3338	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3339	PX86PDEPAE pPdeDst;
3340	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3341	PX86PDPAE pPDDst;
3342	# if PGM_GST_TYPE != PGM_TYPE_PAE
3343	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3344	X86PDPE PdpeSrc;
3345	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3346	# endif
3347	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3348	if (rc != VINF_SUCCESS)
3349	{
3350	pgmUnlock(pVM);
3351	AssertRC(rc);
3352	return rc;
3353	}
3354	Assert(pPDDst);
3355	pPdeDst = &pPDDst->a[iPDDst];
3356
3357	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3358	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3359	PX86PDPAE pPDDst;
3360	PX86PDEPAE pPdeDst;
3361
3362	# if PGM_GST_TYPE == PGM_TYPE_PROT
3363	/* AMD-V nested paging: Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3364	X86PML4E Pml4eSrc;
3365	X86PDPE PdpeSrc;
3366	PX86PML4E pPml4eSrc = &Pml4eSrc;
3367	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
3368	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
3369	# endif
3370
3371	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3372	if (rc != VINF_SUCCESS)
3373	{
3374	pgmUnlock(pVM);
3375	AssertRC(rc);
3376	return rc;
3377	}
3378	Assert(pPDDst);
3379	pPdeDst = &pPDDst->a[iPDDst];
3380	# endif
3381
3382	# if defined(IN_RC)
3383	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
3384	PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
3385	# endif
3386
3387	if (!pPdeDst->n.u1Present)
3388	{
3389	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3390	if (rc != VINF_SUCCESS)
3391	{
3392	# if defined(IN_RC)
3393	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
3394	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
3395	# endif
3396	pgmUnlock(pVM);
3397	AssertRC(rc);
3398	return rc;
3399	}
3400	}
3401
3402	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3403	/* Check for dirty bit fault */
3404	rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
3405	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
3406	Log(("PGMVerifyAccess: success (dirty)\n"));
3407	else
3408	# endif
3409	{
3410	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3411	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3412	# else
3413	GSTPDE PdeSrc;
3414	PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3415	PdeSrc.n.u1Present = 1;
3416	PdeSrc.n.u1Write = 1;
3417	PdeSrc.n.u1Accessed = 1;
3418	PdeSrc.n.u1User = 1;
3419	# endif
3420
3421	Assert(rc != VINF_EM_RAW_GUEST_TRAP);
3422	if (uErr & X86_TRAP_PF_US)
3423	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUser));
3424	else /* supervisor */
3425	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
3426
3427	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3428	if (RT_SUCCESS(rc))
3429	{
3430	/* Page was successfully synced */
3431	Log2(("PGMVerifyAccess: success (sync)\n"));
3432	rc = VINF_SUCCESS;
3433	}
3434	else
3435	{
3436	Log(("PGMVerifyAccess: access violation for %RGv rc=%Rrc\n", GCPtrPage, rc));
3437	rc = VINF_EM_RAW_GUEST_TRAP;
3438	}
3439	}
3440	# if defined(IN_RC)
3441	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
3442	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
3443	# endif
3444	pgmUnlock(pVM);
3445	return rc;
3446
3447	#else /* PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED */
3448
3449	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
3450	return VERR_INTERNAL_ERROR;
3451	#endif /* PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED */
3452	}
3453
3454
3455	/**
3456	* Syncs the paging hierarchy starting at CR3.
3457	*
3458	* @returns VBox status code, no specials.
3459	* @param pVCpu The VMCPU handle.
3460	* @param cr0 Guest context CR0 register
3461	* @param cr3 Guest context CR3 register
3462	* @param cr4 Guest context CR4 register
3463	* @param fGlobal Including global page directories or not
3464	*/
3465	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
3466	{
3467	PVM pVM = pVCpu->CTX_SUFF(pVM);
3468
3469	LogFlow(("SyncCR3 fGlobal=%d\n", !!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)));
3470
3471	#if PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
3472
3473	pgmLock(pVM);
3474
3475	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
3476	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3477	if (pPool->cDirtyPages)
3478	pgmPoolResetDirtyPages(pVM);
3479	# endif
3480
3481	/*
3482	* Update page access handlers.
3483	* The virtual are always flushed, while the physical are only on demand.
3484	* WARNING: We are incorrectly not doing global flushing on Virtual Handler updates. We'll
3485	* have to look into that later because it will have a bad influence on the performance.
3486	* @note SvL: There's no need for that. Just invalidate the virtual range(s).
3487	* bird: Yes, but that won't work for aliases.
3488	*/
3489	/** @todo this MUST go away. See #1557. */
3490	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3Handlers), h);
3491	PGM_GST_NAME(HandlerVirtualUpdate)(pVM, cr4);
3492	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3Handlers), h);
3493	pgmUnlock(pVM);
3494	#endif /* !NESTED && !EPT */
3495
3496	#if PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3497	/*
3498	* Nested / EPT - almost no work.
3499	*/
3500	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
3501	return VINF_SUCCESS;
3502
3503	#elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3504	/*
3505	* AMD64 (Shw & Gst) - No need to check all paging levels; we zero
3506	* out the shadow parts when the guest modifies its tables.
3507	*/
3508	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
3509	return VINF_SUCCESS;
3510
3511	#else /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3512
3513	# ifndef PGM_WITHOUT_MAPPINGS
3514	/*
3515	* Check for and resolve conflicts with our guest mappings if they
3516	* are enabled and not fixed.
3517	*/
3518	if (pgmMapAreMappingsFloating(&pVM->pgm.s))
3519	{
3520	int rc = pgmMapResolveConflicts(pVM);
3521	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
3522	if (rc == VINF_PGM_SYNC_CR3)
3523	{
3524	LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
3525	return VINF_PGM_SYNC_CR3;
3526	}
3527	}
3528	# else
3529	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
3530	# endif
3531	return VINF_SUCCESS;
3532	#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3533	}
3534
3535
3536
3537
3538	#ifdef VBOX_STRICT
3539	#ifdef IN_RC
3540	# undef AssertMsgFailed
3541	# define AssertMsgFailed Log
3542	#endif
3543	#ifdef IN_RING3
3544	# include <VBox/dbgf.h>
3545
3546	/**
3547	* Dumps a page table hierarchy use only physical addresses and cr4/lm flags.
3548	*
3549	* @returns VBox status code (VINF_SUCCESS).
3550	* @param cr3 The root of the hierarchy.
3551	* @param crr The cr4, only PAE and PSE is currently used.
3552	* @param fLongMode Set if long mode, false if not long mode.
3553	* @param cMaxDepth Number of levels to dump.
3554	* @param pHlp Pointer to the output functions.
3555	*/
3556	RT_C_DECLS_BEGIN
3557	VMMR3DECL(int) PGMR3DumpHierarchyHC(PVM pVM, uint32_t cr3, uint32_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp);
3558	RT_C_DECLS_END
3559
3560	#endif
3561
3562	/**
3563	* Checks that the shadow page table is in sync with the guest one.
3564	*
3565	* @returns The number of errors.
3566	* @param pVM The virtual machine.
3567	* @param pVCpu The VMCPU handle.
3568	* @param cr3 Guest context CR3 register
3569	* @param cr4 Guest context CR4 register
3570	* @param GCPtr Where to start. Defaults to 0.
3571	* @param cb How much to check. Defaults to everything.
3572	*/
3573	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)
3574	{
3575	#if PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3576	return 0;
3577	#else
3578	unsigned cErrors = 0;
3579	PVM pVM = pVCpu->CTX_SUFF(pVM);
3580	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3581
3582	#if PGM_GST_TYPE == PGM_TYPE_PAE
3583	/** @todo currently broken; crashes below somewhere */
3584	AssertFailed();
3585	#endif
3586
3587	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
3588	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3589	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
3590
3591	bool fBigPagesSupported = GST_IS_PSE_ACTIVE(pVCpu);
3592	PPGMCPU pPGM = &pVCpu->pgm.s;
3593	RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
3594	RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
3595	# ifndef IN_RING0
3596	RTHCPHYS HCPhys; /* general usage. */
3597	# endif
3598	int rc;
3599
3600	/*
3601	* Check that the Guest CR3 and all its mappings are correct.
3602	*/
3603	AssertMsgReturn(pPGM->GCPhysCR3 == (cr3 & GST_CR3_PAGE_MASK),
3604	("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
3605	false);
3606	# if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64
3607	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3608	rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGst32BitPdRC, NULL, &HCPhysShw);
3609	# else
3610	rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGstPaePdptRC, NULL, &HCPhysShw);
3611	# endif
3612	AssertRCReturn(rc, 1);
3613	HCPhys = NIL_RTHCPHYS;
3614	rc = pgmRamGCPhys2HCPhys(&pVM->pgm.s, cr3 & GST_CR3_PAGE_MASK, &HCPhys);
3615	AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
3616	# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
3617	pgmGstGet32bitPDPtr(pVCpu);
3618	RTGCPHYS GCPhys;
3619	rc = PGMR3DbgR3Ptr2GCPhys(pVM, pPGM->pGst32BitPdR3, &GCPhys);
3620	AssertRCReturn(rc, 1);
3621	AssertMsgReturn((cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
3622	# endif
3623	# endif /* !IN_RING0 */
3624
3625	/*
3626	* Get and check the Shadow CR3.
3627	*/
3628	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3629	unsigned cPDEs = X86_PG_ENTRIES;
3630	unsigned cIncrement = X86_PG_ENTRIES * PAGE_SIZE;
3631	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3632	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3633	unsigned cPDEs = X86_PG_PAE_ENTRIES * 4; /* treat it as a 2048 entry table. */
3634	# else
3635	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3636	# endif
3637	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3638	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3639	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3640	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3641	# endif
3642	if (cb != ~(RTGCPTR)0)
3643	cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
3644
3645	/** @todo call the other two PGMAssert() functions. /
3646
3647	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3648	unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3649
3650	for (; iPml4 < X86_PG_PAE_ENTRIES; iPml4++)
3651	{
3652	PPGMPOOLPAGE pShwPdpt = NULL;
3653	PX86PML4E pPml4eSrc;
3654	PX86PML4E pPml4eDst;
3655	RTGCPHYS GCPhysPdptSrc;
3656
3657	pPml4eSrc = pgmGstGetLongModePML4EPtr(pVCpu, iPml4);
3658	pPml4eDst = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
3659
3660	/* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
3661	if (!pPml4eDst->n.u1Present)
3662	{
3663	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3664	continue;
3665	}
3666
3667	pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
3668	GCPhysPdptSrc = pPml4eSrc->u & X86_PML4E_PG_MASK_FULL;
3669
3670	if (pPml4eSrc->n.u1Present != pPml4eDst->n.u1Present)
3671	{
3672	AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3673	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3674	cErrors++;
3675	continue;
3676	}
3677
3678	if (GCPhysPdptSrc != pShwPdpt->GCPhys)
3679	{
3680	AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
3681	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3682	cErrors++;
3683	continue;
3684	}
3685
3686	if ( pPml4eDst->n.u1User != pPml4eSrc->n.u1User
3687	\|\| pPml4eDst->n.u1Write != pPml4eSrc->n.u1Write
3688	\|\| pPml4eDst->n.u1NoExecute != pPml4eSrc->n.u1NoExecute)
3689	{
3690	AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3691	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3692	cErrors++;
3693	continue;
3694	}
3695	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3696	{
3697	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3698
3699	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
3700	/*
3701	* Check the PDPTEs too.
3702	*/
3703	unsigned iPdpt = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
3704
3705	for (;iPdpt <= SHW_PDPT_MASK; iPdpt++)
3706	{
3707	unsigned iPDSrc = 0; /* initialized to shut up gcc */
3708	PPGMPOOLPAGE pShwPde = NULL;
3709	PX86PDPE pPdpeDst;
3710	RTGCPHYS GCPhysPdeSrc;
3711	# if PGM_GST_TYPE == PGM_TYPE_PAE
3712	X86PDPE PdpeSrc;
3713	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtr, &iPDSrc, &PdpeSrc);
3714	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
3715	# else
3716	PX86PML4E pPml4eSrcIgn;
3717	X86PDPE PdpeSrc;
3718	PX86PDPT pPdptDst;
3719	PX86PDPAE pPDDst;
3720	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtr, &pPml4eSrcIgn, &PdpeSrc, &iPDSrc);
3721
3722	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtr, NULL, &pPdptDst, &pPDDst);
3723	if (rc != VINF_SUCCESS)
3724	{
3725	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
3726	GCPtr += 512 * _2M;
3727	continue; /* next PDPTE */
3728	}
3729	Assert(pPDDst);
3730	# endif
3731	Assert(iPDSrc == 0);
3732
3733	pPdpeDst = &pPdptDst->a[iPdpt];
3734
3735	if (!pPdpeDst->n.u1Present)
3736	{
3737	GCPtr += 512 * _2M;
3738	continue; /* next PDPTE */
3739	}
3740
3741	pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
3742	GCPhysPdeSrc = PdpeSrc.u & X86_PDPE_PG_MASK;
3743
3744	if (pPdpeDst->n.u1Present != PdpeSrc.n.u1Present)
3745	{
3746	AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3747	GCPtr += 512 * _2M;
3748	cErrors++;
3749	continue;
3750	}
3751
3752	if (GCPhysPdeSrc != pShwPde->GCPhys)
3753	{
3754	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3755	AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3756	# else
3757	AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3758	# endif
3759	GCPtr += 512 * _2M;
3760	cErrors++;
3761	continue;
3762	}
3763
3764	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3765	if ( pPdpeDst->lm.u1User != PdpeSrc.lm.u1User
3766	\|\| pPdpeDst->lm.u1Write != PdpeSrc.lm.u1Write
3767	\|\| pPdpeDst->lm.u1NoExecute != PdpeSrc.lm.u1NoExecute)
3768	{
3769	AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3770	GCPtr += 512 * _2M;
3771	cErrors++;
3772	continue;
3773	}
3774	# endif
3775
3776	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3777	{
3778	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3779	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3780	GSTPD const *pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3781	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3782	PCX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
3783	# endif
3784	# endif /* PGM_GST_TYPE == PGM_TYPE_32BIT */
3785	/*
3786	* Iterate the shadow page directory.
3787	*/
3788	GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
3789	unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
3790
3791	for (;
3792	iPDDst < cPDEs;
3793	iPDDst++, GCPtr += cIncrement)
3794	{
3795	# if PGM_SHW_TYPE == PGM_TYPE_PAE
3796	const SHWPDE PdeDst = *pgmShwGetPaePDEPtr(pVCpu, GCPtr);
3797	# else
3798	const SHWPDE PdeDst = pPDDst->a[iPDDst];
3799	# endif
3800	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
3801	{
3802	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
3803	if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
3804	{
3805	AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
3806	cErrors++;
3807	continue;
3808	}
3809	}
3810	else if ( (PdeDst.u & X86_PDE_P)
3811	\|\| ((PdeDst.u & (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY))
3812	)
3813	{
3814	HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
3815	PPGMPOOLPAGE pPoolPage = pgmPoolGetPage(pPool, HCPhysShw);
3816	if (!pPoolPage)
3817	{
3818	AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n",
3819	HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
3820	cErrors++;
3821	continue;
3822	}
3823	const SHWPT pPTDst = (const SHWPT )PGMPOOL_PAGE_2_PTR(pVM, pPoolPage);
3824
3825	if (PdeDst.u & (X86_PDE4M_PWT \| X86_PDE4M_PCD))
3826	{
3827	AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
3828	GCPtr, (uint64_t)PdeDst.u));
3829	cErrors++;
3830	}
3831
3832	if (PdeDst.u & (X86_PDE4M_G \| X86_PDE4M_D))
3833	{
3834	AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n",
3835	GCPtr, (uint64_t)PdeDst.u));
3836	cErrors++;
3837	}
3838
3839	const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
3840	if (!PdeSrc.n.u1Present)
3841	{
3842	AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
3843	GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
3844	cErrors++;
3845	continue;
3846	}
3847
3848	if ( !PdeSrc.b.u1Size
3849	\|\| !fBigPagesSupported)
3850	{
3851	GCPhysGst = PdeSrc.u & GST_PDE_PG_MASK;
3852	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3853	GCPhysGst \|= (iPDDst & 1) * (PAGE_SIZE / 2);
3854	# endif
3855	}
3856	else
3857	{
3858	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3859	if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
3860	{
3861	AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n",
3862	GCPtr, (uint64_t)PdeSrc.u));
3863	cErrors++;
3864	continue;
3865	}
3866	# endif
3867	GCPhysGst = GST_GET_PDE_BIG_PG_GCPHYS(pVM, PdeSrc);
3868	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3869	GCPhysGst \|= GCPtr & RT_BIT(X86_PAGE_2M_SHIFT);
3870	# endif
3871	}
3872
3873	if ( pPoolPage->enmKind
3874	!= (!PdeSrc.b.u1Size \|\| !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
3875	{
3876	AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n",
3877	pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
3878	cErrors++;
3879	}
3880
3881	PPGMPAGE pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
3882	if (!pPhysPage)
3883	{
3884	AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3885	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3886	cErrors++;
3887	continue;
3888	}
3889
3890	if (GCPhysGst != pPoolPage->GCPhys)
3891	{
3892	AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n",
3893	GCPhysGst, pPoolPage->GCPhys, GCPtr));
3894	cErrors++;
3895	continue;
3896	}
3897
3898	if ( !PdeSrc.b.u1Size
3899	\|\| !fBigPagesSupported)
3900	{
3901	/*
3902	* Page Table.
3903	*/
3904	const GSTPT *pPTSrc;
3905	rc = PGM_GCPHYS_2_PTR(pVM, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1), &pPTSrc);
3906	if (RT_FAILURE(rc))
3907	{
3908	AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3909	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3910	cErrors++;
3911	continue;
3912	}
3913	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/))
3914	!= (PdeDst.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/)))
3915	{
3916	/// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
3917	// (This problem will go away when/if we shadow multiple CR3s.)
3918	AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3919	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3920	cErrors++;
3921	continue;
3922	}
3923	if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3924	{
3925	AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n",
3926	GCPtr, (uint64_t)PdeDst.u));
3927	cErrors++;
3928	continue;
3929	}
3930
3931	/* iterate the page table. */
3932	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3933	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3934	const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
3935	# else
3936	const unsigned offPTSrc = 0;
3937	# endif
3938	for (unsigned iPT = 0, off = 0;
3939	iPT < RT_ELEMENTS(pPTDst->a);
3940	iPT++, off += PAGE_SIZE)
3941	{
3942	const SHWPTE PteDst = pPTDst->a[iPT];
3943
3944	/* skip not-present entries. */
3945	if (!(PteDst.u & (X86_PTE_P \| PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
3946	continue;
3947	Assert(PteDst.n.u1Present);
3948
3949	const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
3950	if (!PteSrc.n.u1Present)
3951	{
3952	# ifdef IN_RING3
3953	PGMAssertHandlerAndFlagsInSync(pVM);
3954	PGMR3DumpHierarchyGC(pVM, cr3, cr4, (PdeSrc.u & GST_PDE_PG_MASK));
3955	# endif
3956	AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
3957	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u, pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
3958	(PdeSrc.u & GST_PDE_PG_MASK) + (iPT + offPTSrc)*sizeof(PteSrc)));
3959	cErrors++;
3960	continue;
3961	}
3962
3963	uint64_t fIgnoreFlags = GST_PTE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_G \| X86_PTE_D \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
3964	# if 1 /** @todo sync accessed bit properly... */
3965	fIgnoreFlags \|= X86_PTE_A;
3966	# endif
3967
3968	/* match the physical addresses */
3969	HCPhysShw = PteDst.u & SHW_PTE_PG_MASK;
3970	GCPhysGst = PteSrc.u & GST_PTE_PG_MASK;
3971
3972	# ifdef IN_RING3
3973	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3974	if (RT_FAILURE(rc))
3975	{
3976	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
3977	{
3978	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3979	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3980	cErrors++;
3981	continue;
3982	}
3983	}
3984	else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
3985	{
3986	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3987	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3988	cErrors++;
3989	continue;
3990	}
3991	# endif
3992
3993	pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
3994	if (!pPhysPage)
3995	{
3996	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
3997	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
3998	{
3999	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
4000	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4001	cErrors++;
4002	continue;
4003	}
4004	# endif
4005	if (PteDst.n.u1Write)
4006	{
4007	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4008	GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4009	cErrors++;
4010	}
4011	fIgnoreFlags \|= X86_PTE_RW;
4012	}
4013	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4014	{
4015	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4016	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4017	cErrors++;
4018	continue;
4019	}
4020
4021	/* flags */
4022	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4023	{
4024	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4025	{
4026	if (PteDst.n.u1Write)
4027	{
4028	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4029	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4030	cErrors++;
4031	continue;
4032	}
4033	fIgnoreFlags \|= X86_PTE_RW;
4034	}
4035	else
4036	{
4037	if ( PteDst.n.u1Present
4038	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
4039	&& !PGM_PAGE_IS_MMIO(pPhysPage)
4040	# endif
4041	)
4042	{
4043	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4044	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4045	cErrors++;
4046	continue;
4047	}
4048	fIgnoreFlags \|= X86_PTE_P;
4049	}
4050	}
4051	else
4052	{
4053	if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
4054	{
4055	if (PteDst.n.u1Write)
4056	{
4057	AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
4058	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4059	cErrors++;
4060	continue;
4061	}
4062	if (!(PteDst.u & PGM_PTFLAGS_TRACK_DIRTY))
4063	{
4064	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4065	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4066	cErrors++;
4067	continue;
4068	}
4069	if (PteDst.n.u1Dirty)
4070	{
4071	AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4072	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4073	cErrors++;
4074	}
4075	# if 0 /** @todo sync access bit properly... */
4076	if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
4077	{
4078	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4079	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4080	cErrors++;
4081	}
4082	fIgnoreFlags \|= X86_PTE_RW;
4083	# else
4084	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
4085	# endif
4086	}
4087	else if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
4088	{
4089	/* access bit emulation (not implemented). */
4090	if (PteSrc.n.u1Accessed \|\| PteDst.n.u1Present)
4091	{
4092	AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
4093	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4094	cErrors++;
4095	continue;
4096	}
4097	if (!PteDst.n.u1Accessed)
4098	{
4099	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
4100	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4101	cErrors++;
4102	}
4103	fIgnoreFlags \|= X86_PTE_P;
4104	}
4105	# ifdef DEBUG_sandervl
4106	fIgnoreFlags \|= X86_PTE_D \| X86_PTE_A;
4107	# endif
4108	}
4109
4110	if ( (PteSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
4111	&& (PteSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags)
4112	)
4113	{
4114	AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
4115	GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
4116	fIgnoreFlags, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4117	cErrors++;
4118	continue;
4119	}
4120	} /* foreach PTE */
4121	}
4122	else
4123	{
4124	/*
4125	* Big Page.
4126	*/
4127	uint64_t fIgnoreFlags = X86_PDE_AVL_MASK \| GST_PDE_PG_MASK \| X86_PDE4M_G \| X86_PDE4M_D \| X86_PDE4M_PS \| X86_PDE4M_PWT \| X86_PDE4M_PCD;
4128	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
4129	{
4130	if (PdeDst.n.u1Write)
4131	{
4132	AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4133	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4134	cErrors++;
4135	continue;
4136	}
4137	if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
4138	{
4139	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4140	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4141	cErrors++;
4142	continue;
4143	}
4144	# if 0 /** @todo sync access bit properly... */
4145	if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
4146	{
4147	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4148	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4149	cErrors++;
4150	}
4151	fIgnoreFlags \|= X86_PTE_RW;
4152	# else
4153	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
4154	# endif
4155	}
4156	else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4157	{
4158	/* access bit emulation (not implemented). */
4159	if (PdeSrc.b.u1Accessed \|\| PdeDst.n.u1Present)
4160	{
4161	AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4162	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4163	cErrors++;
4164	continue;
4165	}
4166	if (!PdeDst.n.u1Accessed)
4167	{
4168	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4169	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4170	cErrors++;
4171	}
4172	fIgnoreFlags \|= X86_PTE_P;
4173	}
4174
4175	if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
4176	{
4177	AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
4178	GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
4179	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4180	cErrors++;
4181	}
4182
4183	/* iterate the page table. */
4184	for (unsigned iPT = 0, off = 0;
4185	iPT < RT_ELEMENTS(pPTDst->a);
4186	iPT++, off += PAGE_SIZE, GCPhysGst += PAGE_SIZE)
4187	{
4188	const SHWPTE PteDst = pPTDst->a[iPT];
4189
4190	if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
4191	{
4192	AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
4193	GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4194	cErrors++;
4195	}
4196
4197	/* skip not-present entries. */
4198	if (!PteDst.n.u1Present) /** @todo deal with ALL handlers and CSAM !P pages! */
4199	continue;
4200
4201	fIgnoreFlags = X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT \| X86_PTE_D \| X86_PTE_A \| X86_PTE_G \| X86_PTE_PAE_NX;
4202
4203	/* match the physical addresses */
4204	HCPhysShw = PteDst.u & X86_PTE_PAE_PG_MASK;
4205
4206	# ifdef IN_RING3
4207	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4208	if (RT_FAILURE(rc))
4209	{
4210	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4211	{
4212	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4213	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4214	cErrors++;
4215	}
4216	}
4217	else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
4218	{
4219	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4220	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4221	cErrors++;
4222	continue;
4223	}
4224	# endif
4225	pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
4226	if (!pPhysPage)
4227	{
4228	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4229	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4230	{
4231	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4232	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4233	cErrors++;
4234	continue;
4235	}
4236	# endif
4237	if (PteDst.n.u1Write)
4238	{
4239	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4240	GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4241	cErrors++;
4242	}
4243	fIgnoreFlags \|= X86_PTE_RW;
4244	}
4245	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4246	{
4247	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4248	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4249	cErrors++;
4250	continue;
4251	}
4252
4253	/* flags */
4254	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4255	{
4256	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4257	{
4258	if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
4259	{
4260	if (PteDst.n.u1Write)
4261	{
4262	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4263	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4264	cErrors++;
4265	continue;
4266	}
4267	fIgnoreFlags \|= X86_PTE_RW;
4268	}
4269	}
4270	else
4271	{
4272	if ( PteDst.n.u1Present
4273	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
4274	&& !PGM_PAGE_IS_MMIO(pPhysPage)
4275	# endif
4276	)
4277	{
4278	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4279	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4280	cErrors++;
4281	continue;
4282	}
4283	fIgnoreFlags \|= X86_PTE_P;
4284	}
4285	}
4286
4287	if ( (PdeSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
4288	&& (PdeSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags) /* lazy phys handler dereg. */
4289	)
4290	{
4291	AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
4292	GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
4293	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4294	cErrors++;
4295	continue;
4296	}
4297	} /* for each PTE */
4298	}
4299	}
4300	/* not present */
4301
4302	} /* for each PDE */
4303
4304	} /* for each PDPTE */
4305
4306	} /* for each PML4E */
4307
4308	# ifdef DEBUG
4309	if (cErrors)
4310	LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
4311	# endif
4312
4313	#endif /* GST == 32BIT, PAE or AMD64 */
4314	return cErrors;
4315
4316	#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT */
4317	}
4318	#endif /* VBOX_STRICT */
4319
4320
4321	/**
4322	* Sets up the CR3 for shadow paging
4323	*
4324	* @returns Strict VBox status code.
4325	* @retval VINF_SUCCESS.
4326	*
4327	* @param pVCpu The VMCPU handle.
4328	* @param GCPhysCR3 The physical address in the CR3 register.
4329	*/
4330	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3)
4331	{
4332	PVM pVM = pVCpu->CTX_SUFF(pVM);
4333
4334	/* Update guest paging info. */
4335	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
4336	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
4337	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
4338
4339	LogFlow(("MapCR3: %RGp\n", GCPhysCR3));
4340
4341	/*
4342	* Map the page CR3 points at.
4343	*/
4344	RTHCPTR HCPtrGuestCR3;
4345	RTHCPHYS HCPhysGuestCR3;
4346	pgmLock(pVM);
4347	PPGMPAGE pPageCR3 = pgmPhysGetPage(&pVM->pgm.s, GCPhysCR3);
4348	AssertReturn(pPageCR3, VERR_INTERNAL_ERROR_2);
4349	HCPhysGuestCR3 = PGM_PAGE_GET_HCPHYS(pPageCR3);
4350	/** @todo this needs some reworking wrt. locking. */
4351	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4352	HCPtrGuestCR3 = NIL_RTHCPTR;
4353	int rc = VINF_SUCCESS;
4354	# else
4355	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPageCR3, GCPhysCR3 & GST_CR3_PAGE_MASK, (void )&HCPtrGuestCR3); / @todo r=bird: This GCPhysCR3 masking isn't necessary. */
4356	# endif
4357	pgmUnlock(pVM);
4358	if (RT_SUCCESS(rc))
4359	{
4360	rc = PGMMap(pVM, (RTGCPTR)pVM->pgm.s.GCPtrCR3Mapping, HCPhysGuestCR3, PAGE_SIZE, 0);
4361	if (RT_SUCCESS(rc))
4362	{
4363	# ifdef IN_RC
4364	PGM_INVL_PG(pVCpu, pVM->pgm.s.GCPtrCR3Mapping);
4365	# endif
4366	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4367	pVCpu->pgm.s.pGst32BitPdR3 = (R3PTRTYPE(PX86PD))HCPtrGuestCR3;
4368	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4369	pVCpu->pgm.s.pGst32BitPdR0 = (R0PTRTYPE(PX86PD))HCPtrGuestCR3;
4370	# endif
4371	pVCpu->pgm.s.pGst32BitPdRC = (RCPTRTYPE(PX86PD))(RTRCUINTPTR)pVM->pgm.s.GCPtrCR3Mapping;
4372
4373	# elif PGM_GST_TYPE == PGM_TYPE_PAE
4374	unsigned off = GCPhysCR3 & GST_CR3_PAGE_MASK & PAGE_OFFSET_MASK;
4375	pVCpu->pgm.s.pGstPaePdptR3 = (R3PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4376	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4377	pVCpu->pgm.s.pGstPaePdptR0 = (R0PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4378	# endif
4379	pVCpu->pgm.s.pGstPaePdptRC = (RCPTRTYPE(PX86PDPT))((RTRCUINTPTR)pVM->pgm.s.GCPtrCR3Mapping + off);
4380	LogFlow(("Cached mapping %RRv\n", pVCpu->pgm.s.pGstPaePdptRC));
4381
4382	/*
4383	* Map the 4 PDs too.
4384	*/
4385	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pVCpu);
4386	RTGCPTR GCPtr = pVM->pgm.s.GCPtrCR3Mapping + PAGE_SIZE;
4387	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++, GCPtr += PAGE_SIZE)
4388	{
4389	if (pGuestPDPT->a[i].n.u1Present)
4390	{
4391	RTHCPTR HCPtr;
4392	RTHCPHYS HCPhys;
4393	RTGCPHYS GCPhys = pGuestPDPT->a[i].u & X86_PDPE_PG_MASK;
4394	pgmLock(pVM);
4395	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
4396	AssertReturn(pPage, VERR_INTERNAL_ERROR_2);
4397	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
4398	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4399	HCPtr = NIL_RTHCPTR;
4400	int rc2 = VINF_SUCCESS;
4401	# else
4402	int rc2 = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, (void **)&HCPtr);
4403	# endif
4404	pgmUnlock(pVM);
4405	if (RT_SUCCESS(rc2))
4406	{
4407	rc = PGMMap(pVM, GCPtr, HCPhys, PAGE_SIZE, 0);
4408	AssertRCReturn(rc, rc);
4409
4410	pVCpu->pgm.s.apGstPaePDsR3[i] = (R3PTRTYPE(PX86PDPAE))HCPtr;
4411	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4412	pVCpu->pgm.s.apGstPaePDsR0[i] = (R0PTRTYPE(PX86PDPAE))HCPtr;
4413	# endif
4414	pVCpu->pgm.s.apGstPaePDsRC[i] = (RCPTRTYPE(PX86PDPAE))(RTRCUINTPTR)GCPtr;
4415	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
4416	# ifdef IN_RC
4417	PGM_INVL_PG(pVCpu, GCPtr);
4418	# endif
4419	continue;
4420	}
4421	AssertMsgFailed(("pgmR3Gst32BitMapCR3: rc2=%d GCPhys=%RGp i=%d\n", rc2, GCPhys, i));
4422	}
4423
4424	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4425	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4426	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4427	# endif
4428	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4429	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4430	# ifdef IN_RC
4431	PGM_INVL_PG(pVCpu, GCPtr); /** @todo this shouldn't be necessary? */
4432	# endif
4433	}
4434
4435	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
4436	pVCpu->pgm.s.pGstAmd64Pml4R3 = (R3PTRTYPE(PX86PML4))HCPtrGuestCR3;
4437	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4438	pVCpu->pgm.s.pGstAmd64Pml4R0 = (R0PTRTYPE(PX86PML4))HCPtrGuestCR3;
4439	# endif
4440	# endif
4441	}
4442	else
4443	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4444	}
4445	else
4446	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4447
4448	#else /* prot/real stub */
4449	int rc = VINF_SUCCESS;
4450	#endif
4451
4452	/* Update shadow paging info for guest modes with paging (32, pae, 64). */
4453	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4454	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4455	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64) \
4456	&& ( PGM_GST_TYPE != PGM_TYPE_REAL \
4457	&& PGM_GST_TYPE != PGM_TYPE_PROT))
4458
4459	Assert(!pVM->pgm.s.fNestedPaging);
4460
4461	/*
4462	* Update the shadow root page as well since that's not fixed.
4463	*/
4464	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4465	PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
4466	uint32_t iOldShwUserTable = pVCpu->pgm.s.iShwUserTable;
4467	uint32_t iOldShwUser = pVCpu->pgm.s.iShwUser;
4468	PPGMPOOLPAGE pNewShwPageCR3;
4469
4470	pgmLock(pVM);
4471
4472	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4473	if (pPool->cDirtyPages)
4474	pgmPoolResetDirtyPages(pVM);
4475	# endif
4476
4477	Assert(!(GCPhysCR3 >> (PAGE_SHIFT + 32)));
4478	rc = pgmPoolAlloc(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, SHW_POOL_ROOT_IDX, GCPhysCR3 >> PAGE_SHIFT, &pNewShwPageCR3, true /* lock page */);
4479	AssertFatalRC(rc);
4480	rc = VINF_SUCCESS;
4481
4482	# ifdef IN_RC
4483	/*
4484	* WARNING! We can't deal with jumps to ring 3 in the code below as the
4485	* state will be inconsistent! Flush important things now while
4486	* we still can and then make sure there are no ring-3 calls.
4487	*/
4488	REMNotifyHandlerPhysicalFlushIfAlmostFull(pVM, pVCpu);
4489	VMMRZCallRing3Disable(pVCpu);
4490	# endif
4491
4492	pVCpu->pgm.s.iShwUser = SHW_POOL_ROOT_IDX;
4493	pVCpu->pgm.s.iShwUserTable = GCPhysCR3 >> PAGE_SHIFT;
4494	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3) = pNewShwPageCR3;
4495	# ifdef IN_RING0
4496	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4497	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4498	# elif defined(IN_RC)
4499	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4500	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4501	# else
4502	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4503	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4504	# endif
4505
4506	# ifndef PGM_WITHOUT_MAPPINGS
4507	/*
4508	* Apply all hypervisor mappings to the new CR3.
4509	* Note that SyncCR3 will be executed in case CR3 is changed in a guest paging mode; this will
4510	* make sure we check for conflicts in the new CR3 root.
4511	*/
4512	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4513	Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL) \|\| VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
4514	# endif
4515	rc = pgmMapActivateCR3(pVM, pNewShwPageCR3);
4516	AssertRCReturn(rc, rc);
4517	# endif
4518
4519	/* Set the current hypervisor CR3. */
4520	CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
4521	SELMShadowCR3Changed(pVM, pVCpu);
4522
4523	# ifdef IN_RC
4524	/* NOTE: The state is consistent again. */
4525	VMMRZCallRing3Enable(pVCpu);
4526	# endif
4527
4528	/* Clean up the old CR3 root. */
4529	if ( pOldShwPageCR3
4530	&& pOldShwPageCR3 != pNewShwPageCR3 /* @todo can happen due to incorrect syncing between REM & PGM; find the real cause */)
4531	{
4532	Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
4533	# ifndef PGM_WITHOUT_MAPPINGS
4534	/* Remove the hypervisor mappings from the shadow page table. */
4535	pgmMapDeactivateCR3(pVM, pOldShwPageCR3);
4536	# endif
4537	/* Mark the page as unlocked; allow flushing again. */
4538	pgmPoolUnlockPage(pPool, pOldShwPageCR3);
4539
4540	pgmPoolFreeByPage(pPool, pOldShwPageCR3, iOldShwUser, iOldShwUserTable);
4541	}
4542	pgmUnlock(pVM);
4543	# endif
4544
4545	return rc;
4546	}
4547
4548	/**
4549	* Unmaps the shadow CR3.
4550	*
4551	* @returns VBox status, no specials.
4552	* @param pVCpu The VMCPU handle.
4553	*/
4554	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu)
4555	{
4556	LogFlow(("UnmapCR3\n"));
4557
4558	int rc = VINF_SUCCESS;
4559	PVM pVM = pVCpu->CTX_SUFF(pVM);
4560
4561	/*
4562	* Update guest paging info.
4563	*/
4564	#if PGM_GST_TYPE == PGM_TYPE_32BIT
4565	pVCpu->pgm.s.pGst32BitPdR3 = 0;
4566	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4567	pVCpu->pgm.s.pGst32BitPdR0 = 0;
4568	# endif
4569	pVCpu->pgm.s.pGst32BitPdRC = 0;
4570
4571	#elif PGM_GST_TYPE == PGM_TYPE_PAE
4572	pVCpu->pgm.s.pGstPaePdptR3 = 0;
4573	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4574	pVCpu->pgm.s.pGstPaePdptR0 = 0;
4575	# endif
4576	pVCpu->pgm.s.pGstPaePdptRC = 0;
4577	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4578	{
4579	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4580	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4581	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4582	# endif
4583	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4584	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4585	}
4586
4587	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
4588	pVCpu->pgm.s.pGstAmd64Pml4R3 = 0;
4589	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4590	pVCpu->pgm.s.pGstAmd64Pml4R0 = 0;
4591	# endif
4592
4593	#else /* prot/real mode stub */
4594	/* nothing to do */
4595	#endif
4596
4597	#if !defined(IN_RC) /* In RC we rely on MapCR3 to do the shadow part for us at a safe time */
4598	/*
4599	* Update shadow paging info.
4600	*/
4601	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4602	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4603	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64))
4604
4605	# if PGM_GST_TYPE != PGM_TYPE_REAL
4606	Assert(!pVM->pgm.s.fNestedPaging);
4607	# endif
4608
4609	pgmLock(pVM);
4610
4611	# ifndef PGM_WITHOUT_MAPPINGS
4612	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4613	/* Remove the hypervisor mappings from the shadow page table. */
4614	pgmMapDeactivateCR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4615	# endif
4616
4617	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4618	{
4619	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4620
4621	Assert(pVCpu->pgm.s.iShwUser != PGMPOOL_IDX_NESTED_ROOT);
4622
4623	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4624	if (pPool->cDirtyPages)
4625	pgmPoolResetDirtyPages(pVM);
4626	# endif
4627
4628	/* Mark the page as unlocked; allow flushing again. */
4629	pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4630
4631	pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), pVCpu->pgm.s.iShwUser, pVCpu->pgm.s.iShwUserTable);
4632	pVCpu->pgm.s.pShwPageCR3R3 = 0;
4633	pVCpu->pgm.s.pShwPageCR3R0 = 0;
4634	pVCpu->pgm.s.pShwPageCR3RC = 0;
4635	pVCpu->pgm.s.iShwUser = 0;
4636	pVCpu->pgm.s.iShwUserTable = 0;
4637	}
4638	pgmUnlock(pVM);
4639	# endif
4640	#endif /* !IN_RC*/
4641
4642	return rc;
4643	}

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 31167

Download in other formats: