PGMAllBth.h@ 31636

Last change on this file since 31636 was 31636, checked in by vboxsync, 14 years ago
EM: Addressed VBOXSTRICTRC todo from r64673. Decided to only use VBOXSTRICTRC on the methods that would actually return VINF_EM_* stuff.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 197.9 KB

Line
1	/* $Id: PGMAllBth.h 31636 2010-08-13 12:03:15Z vboxsync $ */
2	/** @file
3	* VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4	*
5	* @remarks The nested page tables on AMD makes use of PGM_SHW_TYPE in
6	* {PGM_TYPE_AMD64, PGM_TYPE_PAE and PGM_TYPE_32BIT} and PGM_GST_TYPE
7	* set to PGM_TYPE_PROT. Half of the code in this file is not
8	* exercised with PGM_SHW_TYPE set to PGM_TYPE_NESTED.
9	*
10	* @remarks Extended page tables (intel) are built with PGM_GST_TYPE set to
11	* PGM_TYPE_PROT (and PGM_SHW_TYPE set to PGM_TYPE_EPT).
12	*
13	* @remarks This file is one big \#ifdef-orgy!
14	*
15	*/
16
17	/*
18	* Copyright (C) 2006-2010 Oracle Corporation
19	*
20	* This file is part of VirtualBox Open Source Edition (OSE), as
21	* available from http://www.virtualbox.org. This file is free software;
22	* you can redistribute it and/or modify it under the terms of the GNU
23	* General Public License (GPL) as published by the Free Software
24	* Foundation, in version 2 as it comes in the "COPYING" file of the
25	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
26	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
27	*/
28
29
30	/*******************************************************************************
31	* Internal Functions *
32	*******************************************************************************/
33	RT_C_DECLS_BEGIN
34	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken);
35	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
36	static int PGM_BTH_NAME(SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr);
37	static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, RTGCPTR GCPtrPage);
38	static int PGM_BTH_NAME(SyncPT)(PVMCPU pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage);
39	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uErr);
40	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
41	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
42	#ifdef VBOX_STRICT
43	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
44	#endif
45	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3);
46	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu);
47	RT_C_DECLS_END
48
49
50	/*
51	* Filter out some illegal combinations of guest and shadow paging, so we can
52	* remove redundant checks inside functions.
53	*/
54	#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
55	# error "Invalid combination; PAE guest implies PAE shadow"
56	#endif
57
58	#if (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
59	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT)
60	# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
61	#endif
62
63	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) \
64	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT)
65	# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
66	#endif
67
68	#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT) \
69	\|\| (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
70	# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
71	#endif
72
73	#ifndef IN_RING3
74
75	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
76	/**
77	* Deal with a guest page fault.
78	*
79	* @returns Strict VBox status code.
80	* @retval VINF_EM_RAW_GUEST_TRAP
81	* @retval VINF_EM_RAW_EMULATE_INSTR
82	*
83	* @param pVCpu The current CPU.
84	* @param pGstWalk The guest page table walk result.
85	* @param uErr The error code.
86	*/
87	PGM_BTH_DECL(VBOXSTRICTRC, Trap0eHandlerGuestFault)(PVMCPU pVCpu, PGSTPTWALK pGstWalk, RTGCUINT uErr)
88	{
89	# if !defined(PGM_WITHOUT_MAPPINGS) && (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE)
90	/*
91	* Check for write conflicts with our hypervisor mapping.
92	*
93	* If the guest happens to access a non-present page, where our hypervisor
94	* is currently mapped, then we'll create a #PF storm in the guest.
95	*/
96	if ( (uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RW)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RW)
97	&& MMHyperIsInsideArea(pVCpu->CTX_SUFF(pVM), pGstWalk->Core.GCPtr))
98	{
99	/* Force a CR3 sync to check for conflicts and emulate the instruction. */
100	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
101	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
102	return VINF_EM_RAW_EMULATE_INSTR;
103	}
104	# endif
105
106	/*
107	* Calc the error code for the guest trap.
108	*/
109	uint32_t uNewErr = GST_IS_NX_ACTIVE(pVCpu)
110	? uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID)
111	: uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US);
112	if (pGstWalk->Core.fBadPhysAddr)
113	{
114	uNewErr \|= X86_TRAP_PF_RSVD \| X86_TRAP_PF_P;
115	Assert(!pGstWalk->Core.fNotPresent);
116	}
117	else if (!pGstWalk->Core.fNotPresent)
118	uNewErr \|= X86_TRAP_PF_P;
119	TRPMSetErrorCode(pVCpu, uNewErr);
120
121	LogFlow(("Guest trap; cr2=%RGv uErr=%RGv lvl=%d\n", pGstWalk->Core.GCPtr, uErr, pGstWalk->Core.uLevel));
122	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
123	return VINF_EM_RAW_GUEST_TRAP;
124	}
125	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
126
127
128	/**
129	* Deal with a guest page fault.
130	*
131	* The caller has taken the PGM lock.
132	*
133	* @returns Strict VBox status code.
134	*
135	* @param pVCpu The current CPU.
136	* @param uErr The error code.
137	* @param pRegFrame The register frame.
138	* @param pvFault The fault address.
139	* @param pPage The guest page at @a pvFault.
140	* @param pGstWalk The guest page table walk result.
141	* @param pfLockTaken PGM lock taken here or not (out). This is true
142	* when we're called.
143	*/
144	static VBOXSTRICTRC PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame,
145	RTGCPTR pvFault, PPGMPAGE pPage, bool *pfLockTaken
146	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
147	, PGSTPTWALK pGstWalk
148	# endif
149	)
150	{
151	# if !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
152	GSTPDE const PdeSrcDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
153	#endif
154	PVM pVM = pVCpu->CTX_SUFF(pVM);
155	int rc;
156
157	if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
158	{
159	/*
160	* Physical page access handler.
161	*/
162	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
163	const RTGCPHYS GCPhysFault = pGstWalk->Core.GCPhys;
164	# else
165	const RTGCPHYS GCPhysFault = (RTGCPHYS)pvFault;
166	# endif
167	PPGMPHYSHANDLER pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
168	if (pCur)
169	{
170	# ifdef PGM_SYNC_N_PAGES
171	/*
172	* If the region is write protected and we got a page not present fault, then sync
173	* the pages. If the fault was caused by a read, then restart the instruction.
174	* In case of write access continue to the GC write handler.
175	*
176	* ASSUMES that there is only one handler per page or that they have similar write properties.
177	*/
178	if ( pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
179	&& !(uErr & X86_TRAP_PF_P))
180	{
181	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
182	rc = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
183	# else
184	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
185	# endif
186	if ( RT_FAILURE(rc)
187	\|\| !(uErr & X86_TRAP_PF_RW)
188	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
189	{
190	AssertRC(rc);
191	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
192	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
193	return rc;
194	}
195	}
196	# endif
197
198	AssertMsg( pCur->enmType != PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
199	\|\| (pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE && (uErr & X86_TRAP_PF_RW)),
200	("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enum=%d\n",
201	pvFault, GCPhysFault, pPage, uErr, pCur->enmType));
202
203	if (pCur->CTX_SUFF(pfnHandler))
204	{
205	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
206	void *pvUser = pCur->CTX_SUFF(pvUser);
207	# ifdef IN_RING0
208	PFNPGMR0PHYSHANDLER pfnHandler = pCur->CTX_SUFF(pfnHandler);
209	# else
210	PFNPGMRCPHYSHANDLER pfnHandler = pCur->CTX_SUFF(pfnHandler);
211	# endif
212
213	STAM_PROFILE_START(&pCur->Stat, h);
214	if (pfnHandler != pPool->CTX_SUFF(pfnAccessHandler))
215	{
216	pgmUnlock(pVM);
217	*pfLockTaken = false;
218	}
219
220	rc = pfnHandler(pVM, uErr, pRegFrame, pvFault, GCPhysFault, pvUser);
221
222	# ifdef VBOX_WITH_STATISTICS
223	pgmLock(pVM);
224	pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
225	if (pCur)
226	STAM_PROFILE_STOP(&pCur->Stat, h);
227	pgmUnlock(pVM);
228	# endif
229	}
230	else
231	rc = VINF_EM_RAW_EMULATE_INSTR;
232
233	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysical);
234	if (uErr & X86_TRAP_PF_RSVD) STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysical);
235	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndPhys; });
236	return rc;
237	}
238	}
239	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
240	else
241	{
242	# ifdef PGM_SYNC_N_PAGES
243	/*
244	* If the region is write protected and we got a page not present fault, then sync
245	* the pages. If the fault was caused by a read, then restart the instruction.
246	* In case of write access continue to the GC write handler.
247	*/
248	if ( PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) < PGM_PAGE_HNDL_PHYS_STATE_ALL
249	&& !(uErr & X86_TRAP_PF_P))
250	{
251	rc = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
252	if ( RT_FAILURE(rc)
253	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
254	\|\| !(uErr & X86_TRAP_PF_RW))
255	{
256	AssertRC(rc);
257	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
258	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndVirt; });
259	return rc;
260	}
261	}
262	# endif
263	/*
264	* Ok, it's an virtual page access handler.
265	*
266	* Since it's faster to search by address, we'll do that first
267	* and then retry by GCPhys if that fails.
268	*/
269	/** @todo r=bird: perhaps we should consider looking up by physical address directly now?
270	* r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be
271	* out of sync, because the page was changed without us noticing it (not-present -> present
272	* without invlpg or mov cr3, xxx).
273	*/
274	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
275	if (pCur)
276	{
277	AssertMsg(!(pvFault - pCur->Core.Key < pCur->cb)
278	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
279	\|\| !(uErr & X86_TRAP_PF_P)
280	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
281	("Unexpected trap for virtual handler: %RGv (phys=%RGp) pPage=%R[pgmpage] uErr=%X, enum=%d\n",
282	pvFault, pGstWalk->Core.GCPhys, pPage, uErr, pCur->enmType));
283
284	if ( pvFault - pCur->Core.Key < pCur->cb
285	&& ( uErr & X86_TRAP_PF_RW
286	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
287	{
288	# ifdef IN_RC
289	STAM_PROFILE_START(&pCur->Stat, h);
290	RTGCPTR GCPtrStart = pCur->Core.Key;
291	CTX_MID(PFNPGM,VIRTHANDLER) pfnHandler = pCur->CTX_SUFF(pfnHandler);
292	pgmUnlock(pVM);
293	*pfLockTaken = false;
294
295	rc = pfnHandler(pVM, uErr, pRegFrame, pvFault, GCPtrStart, pvFault - GCPtrStart);
296
297	# ifdef VBOX_WITH_STATISTICS
298	pgmLock(pVM);
299	pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
300	if (pCur)
301	STAM_PROFILE_STOP(&pCur->Stat, h);
302	pgmUnlock(pVM);
303	# endif
304	# else
305	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
306	# endif
307	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersVirtual);
308	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
309	return rc;
310	}
311	/* Unhandled part of a monitored page */
312	}
313	else
314	{
315	/* Check by physical address. */
316	unsigned iPage;
317	rc = pgmHandlerVirtualFindByPhysAddr(pVM, pGstWalk->Core.GCPhys, &pCur, &iPage);
318	Assert(RT_SUCCESS(rc) \|\| !pCur);
319	if ( pCur
320	&& ( uErr & X86_TRAP_PF_RW
321	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
322	{
323	Assert((pCur->aPhysToVirt[iPage].Core.Key & X86_PTE_PAE_PG_MASK) == (pGstWalk->Core.GCPhys & X86_PTE_PAE_PG_MASK));
324	# ifdef IN_RC
325	STAM_PROFILE_START(&pCur->Stat, h);
326	RTGCPTR GCPtrStart = pCur->Core.Key;
327	CTX_MID(PFNPGM,VIRTHANDLER) pfnHandler = pCur->CTX_SUFF(pfnHandler);
328	pgmUnlock(pVM);
329	*pfLockTaken = false;
330
331	RTGCPTR off = (iPage << PAGE_SHIFT)
332	+ (pvFault & PAGE_OFFSET_MASK)
333	- (GCPtrStart & PAGE_OFFSET_MASK);
334	Assert(off < pCur->cb);
335	rc = pfnHandler(pVM, uErr, pRegFrame, pvFault, GCPtrStart, off);
336
337	# ifdef VBOX_WITH_STATISTICS
338	pgmLock(pVM);
339	pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, GCPtrStart);
340	if (pCur)
341	STAM_PROFILE_STOP(&pCur->Stat, h);
342	pgmUnlock(pVM);
343	# endif
344	# else
345	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
346	# endif
347	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersVirtualByPhys);
348	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
349	return rc;
350	}
351	}
352	}
353	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
354
355	/*
356	* There is a handled area of the page, but this fault doesn't belong to it.
357	* We must emulate the instruction.
358	*
359	* To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
360	* we first check if this was a page-not-present fault for a page with only
361	* write access handlers. Restart the instruction if it wasn't a write access.
362	*/
363	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersUnhandled);
364
365	if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
366	&& !(uErr & X86_TRAP_PF_P))
367	{
368	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
369	rc = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
370	# else
371	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
372	# endif
373	if ( RT_FAILURE(rc)
374	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
375	\|\| !(uErr & X86_TRAP_PF_RW))
376	{
377	AssertRC(rc);
378	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
379	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
380	return rc;
381	}
382	}
383
384	/** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
385	* It's writing to an unhandled part of the LDT page several million times.
386	*/
387	rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
388	LogFlow(("PGM: PGMInterpretInstruction -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
389	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndUnhandled; });
390	return rc;
391	} /* if any kind of handler */
392
393
394	/**
395	* #PF Handler for raw-mode guest execution.
396	*
397	* @returns VBox status code (appropriate for trap handling and GC return).
398	*
399	* @param pVCpu VMCPU Handle.
400	* @param uErr The trap error code.
401	* @param pRegFrame Trap register frame.
402	* @param pvFault The fault address.
403	* @param pfLockTaken PGM lock taken here or not (out)
404	*/
405	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken)
406	{
407	PVM pVM = pVCpu->CTX_SUFF(pVM);
408
409	*pfLockTaken = false;
410
411	# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \
412	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
413	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
414	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT)
415	int rc;
416
417	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
418	/*
419	* Walk the guest page translation tables and check if it's a guest fault.
420	*/
421	GSTPTWALK GstWalk;
422	rc = PGM_GST_NAME(Walk)(pVCpu, pvFault, &GstWalk);
423	if (RT_FAILURE_NP(rc))
424	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &GstWalk, uErr));
425
426	/* assert some GstWalk sanity. */
427	# if PGM_GST_TYPE == PGM_TYPE_AMD64
428	AssertMsg(GstWalk.Pml4e.u == GstWalk.pPml4e->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pml4e.u, (uint64_t)GstWalk.pPml4e->u));
429	# endif
430	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
431	AssertMsg(GstWalk.Pdpe.u == GstWalk.pPdpe->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pdpe.u, (uint64_t)GstWalk.pPdpe->u));
432	# endif
433	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u));
434	AssertMsg(GstWalk.Core.fBigPage \|\| GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u));
435	Assert(GstWalk.Core.fSucceeded);
436
437	if (uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID))
438	{
439	if ( ( (uErr & X86_TRAP_PF_RW)
440	&& !GstWalk.Core.fEffectiveRW
441	&& ( (uErr & X86_TRAP_PF_US)
442	\|\| CPUMIsGuestR0WriteProtEnabled(pVCpu)) )
443	\|\| ((uErr & X86_TRAP_PF_US) && !GstWalk.Core.fEffectiveUS)
444	\|\| ((uErr & X86_TRAP_PF_ID) && GstWalk.Core.fEffectiveNX)
445	)
446	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &GstWalk, uErr));
447	}
448
449	/*
450	* Set the accessed and dirty flags.
451	*/
452	# if PGM_GST_TYPE == PGM_TYPE_AMD64
453	GstWalk.Pml4e.u \|= X86_PML4E_A;
454	GstWalk.pPml4e->u \|= X86_PML4E_A;
455	GstWalk.Pdpe.u \|= X86_PDPE_A;
456	GstWalk.pPdpe->u \|= X86_PDPE_A;
457	# endif
458	if (GstWalk.Core.fBigPage)
459	{
460	Assert(GstWalk.Pde.b.u1Size);
461	if (uErr & X86_TRAP_PF_RW)
462	{
463	GstWalk.Pde.u \|= X86_PDE4M_A \| X86_PDE4M_D;
464	GstWalk.pPde->u \|= X86_PDE4M_A \| X86_PDE4M_D;
465	}
466	else
467	{
468	GstWalk.Pde.u \|= X86_PDE4M_A;
469	GstWalk.pPde->u \|= X86_PDE4M_A;
470	}
471	}
472	else
473	{
474	Assert(!GstWalk.Pde.b.u1Size);
475	GstWalk.Pde.u \|= X86_PDE_A;
476	GstWalk.pPde->u \|= X86_PDE_A;
477	if (uErr & X86_TRAP_PF_RW)
478	{
479	# ifdef VBOX_WITH_STATISTICS
480	if (!GstWalk.Pte.n.u1Dirty)
481	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtiedPage));
482	else
483	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageAlreadyDirty));
484	# endif
485	GstWalk.Pte.u \|= X86_PTE_A \| X86_PTE_D;
486	GstWalk.pPte->u \|= X86_PTE_A \| X86_PTE_D;
487	}
488	else
489	{
490	GstWalk.Pte.u \|= X86_PTE_A;
491	GstWalk.pPte->u \|= X86_PTE_A;
492	}
493	Assert(GstWalk.Pte.u == GstWalk.pPte->u);
494	}
495	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u \|\| GstWalk.pPte->u == GstWalk.pPde->u,
496	("%RX64 %RX64 pPte=%p pPde=%p Pte=%RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u, GstWalk.pPte, GstWalk.pPde, (uint64_t)GstWalk.pPte->u));
497	# else /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
498	GSTPDE const PdeSrcDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A}; /** @todo eliminate this */
499	# endif /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
500
501	/* Take the big lock now. */
502	*pfLockTaken = true;
503	pgmLock(pVM);
504
505	# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
506	/*
507	* If it is a reserved bit fault we know that it is an MMIO (access
508	* handler) related fault and can skip some 200 lines of code.
509	*/
510	if (uErr & X86_TRAP_PF_RSVD)
511	{
512	Assert(uErr & X86_TRAP_PF_P);
513	PPGMPAGE pPage;
514	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
515	rc = pgmPhysGetPageEx(&pVM->pgm.s, GstWalk.Core.GCPhys, &pPage);
516	if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
517	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
518	pfLockTaken, &GstWalk));
519	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
520	# else
521	rc = pgmPhysGetPageEx(&pVM->pgm.s, (RTGCPHYS)pvFault, &pPage);
522	if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
523	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
524	pfLockTaken));
525	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
526	# endif
527	AssertRC(rc);
528	}
529	# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
530
531	/*
532	* Fetch the guest PDE, PDPE and PML4E.
533	*/
534	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
535	const unsigned iPDDst = pvFault >> SHW_PD_SHIFT;
536	PX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
537
538	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
539	const unsigned iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK; /* pPDDst index, not used with the pool. */
540	PX86PDPAE pPDDst;
541	# if PGM_GST_TYPE == PGM_TYPE_PAE
542	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, GstWalk.Pdpe.u, &pPDDst);
543	# else
544	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, X86_PDPE_P, &pPDDst); /* RW, US and A are reserved in PAE mode. */
545	# endif
546	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_INTERNAL_ERROR_4);
547
548	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
549	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
550	PX86PDPAE pPDDst;
551	# if PGM_GST_TYPE == PGM_TYPE_PROT /* (AMD-V nested paging) */
552	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A,
553	X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A, &pPDDst);
554	# else
555	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, GstWalk.Pml4e.u, GstWalk.Pdpe.u, &pPDDst);
556	# endif
557	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_INTERNAL_ERROR_4);
558
559	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
560	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
561	PEPTPD pPDDst;
562	rc = pgmShwGetEPTPDPtr(pVCpu, pvFault, NULL, &pPDDst);
563	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_INTERNAL_ERROR_4);
564	# endif
565	Assert(pPDDst);
566
567	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
568	/*
569	* Dirty page handling.
570	*
571	* If we successfully correct the write protection fault due to dirty bit
572	* tracking, then return immediately.
573	*/
574	if (uErr & X86_TRAP_PF_RW) /* write fault? */
575	{
576	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyBitTracking), a);
577	rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, &pPDDst->a[iPDDst], GstWalk.pPde, pvFault);
578	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyBitTracking), a);
579	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
580	{
581	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution)
582	= rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
583	? &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2DirtyAndAccessed
584	: &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
585	LogBird(("Trap0eHandler: returns VINF_SUCCESS\n"));
586	return VINF_SUCCESS;
587	}
588	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u \|\| GstWalk.pPte->u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u));
589	AssertMsg(GstWalk.Core.fBigPage \|\| GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u));
590	}
591
592	# if 0 /* rarely useful; leave for debugging. */
593	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0ePD[iPDSrc]);
594	# endif
595	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
596
597	/*
598	* A common case is the not-present error caused by lazy page table syncing.
599	*
600	* It is IMPORTANT that we weed out any access to non-present shadow PDEs
601	* here so we can safely assume that the shadow PT is present when calling
602	* SyncPage later.
603	*
604	* On failure, we ASSUME that SyncPT is out of memory or detected some kind
605	* of mapping conflict and defer to SyncCR3 in R3.
606	* (Again, we do NOT support access handlers for non-present guest pages.)
607	*
608	*/
609	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
610	Assert(GstWalk.Pde.n.u1Present);
611	# endif
612	if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
613	&& !pPDDst->a[iPDDst].n.u1Present)
614	{
615	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2SyncPT; });
616	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
617	LogFlow(("=>SyncPT %04x = %08RX64\n", (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, (uint64_t)GstWalk.Pde.u));
618	rc = PGM_BTH_NAME(SyncPT)(pVCpu, (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, GstWalk.pPd, pvFault);
619	# else
620	LogFlow(("=>SyncPT pvFault=%RGv\n", pvFault));
621	rc = PGM_BTH_NAME(SyncPT)(pVCpu, 0, NULL, pvFault);
622	# endif
623	if (RT_SUCCESS(rc))
624	return rc;
625	Log(("SyncPT: %RGv failed!! rc=%Rrc\n", pvFault, rc));
626	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
627	return VINF_PGM_SYNC_CR3;
628	}
629
630	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(PGM_WITHOUT_MAPPINGS)
631	/*
632	* Check if this address is within any of our mappings.
633	*
634	* This is very fast and it's gonna save us a bit of effort below and prevent
635	* us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
636	* (BTW, it's impossible to have physical access handlers in a mapping.)
637	*/
638	if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
639	{
640	PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
641	for ( ; pMapping; pMapping = pMapping->CTX_SUFF(pNext))
642	{
643	if (pvFault < pMapping->GCPtr)
644	break;
645	if (pvFault - pMapping->GCPtr < pMapping->cb)
646	{
647	/*
648	* The first thing we check is if we've got an undetected conflict.
649	*/
650	if (pgmMapAreMappingsFloating(&pVM->pgm.s))
651	{
652	unsigned iPT = pMapping->cb >> GST_PD_SHIFT;
653	while (iPT-- > 0)
654	if (GstWalk.pPde[iPT].n.u1Present)
655	{
656	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eConflicts);
657	Log(("Trap0e: Detected Conflict %RGv-%RGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
658	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
659	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
660	return VINF_PGM_SYNC_CR3;
661	}
662	}
663
664	/*
665	* Check if the fault address is in a virtual page access handler range.
666	*/
667	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->HyperVirtHandlers, pvFault);
668	if ( pCur
669	&& pvFault - pCur->Core.Key < pCur->cb
670	&& uErr & X86_TRAP_PF_RW)
671	{
672	# ifdef IN_RC
673	STAM_PROFILE_START(&pCur->Stat, h);
674	pgmUnlock(pVM);
675	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
676	pgmLock(pVM);
677	STAM_PROFILE_STOP(&pCur->Stat, h);
678	# else
679	AssertFailed();
680	rc = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
681	# endif
682	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersMapping);
683	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
684	return rc;
685	}
686
687	/*
688	* Pretend we're not here and let the guest handle the trap.
689	*/
690	TRPMSetErrorCode(pVCpu, uErr & ~X86_TRAP_PF_P);
691	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eGuestPFMapping);
692	LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
693	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
694	return VINF_EM_RAW_GUEST_TRAP;
695	}
696	}
697	} /* pgmAreMappingsEnabled(&pVM->pgm.s) */
698	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
699
700	/*
701	* Check if this fault address is flagged for special treatment,
702	* which means we'll have to figure out the physical address and
703	* check flags associated with it.
704	*
705	* ASSUME that we can limit any special access handling to pages
706	* in page tables which the guest believes to be present.
707	*/
708	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
709	RTGCPHYS GCPhys = GstWalk.Core.GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
710	# else
711	RTGCPHYS GCPhys = (RTGCPHYS)pvFault & ~(RTGCPHYS)PAGE_OFFSET_MASK;
712	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
713	PPGMPAGE pPage;
714	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
715	if (RT_FAILURE(rc))
716	{
717	/*
718	* When the guest accesses invalid physical memory (e.g. probing
719	* of RAM or accessing a remapped MMIO range), then we'll fall
720	* back to the recompiler to emulate the instruction.
721	*/
722	LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhys, rc));
723	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersInvalid);
724	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2InvalidPhys; });
725	return VINF_EM_RAW_EMULATE_INSTR;
726	}
727
728	/*
729	* Any handlers for this page?
730	*/
731	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
732	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
733	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, pfLockTaken,
734	&GstWalk));
735	# else
736	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, pfLockTaken));
737	# endif
738
739	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTimeOutOfSync, c);
740
741	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
742	if (uErr & X86_TRAP_PF_P)
743	{
744	/*
745	* The page isn't marked, but it might still be monitored by a virtual page access handler.
746	* (ASSUMES no temporary disabling of virtual handlers.)
747	*/
748	/** @todo r=bird: Since the purpose is to catch out of sync pages with virtual handler(s) here,
749	* we should correct both the shadow page table and physical memory flags, and not only check for
750	* accesses within the handler region but for access to pages with virtual handlers. */
751	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
752	if (pCur)
753	{
754	AssertMsg( !(pvFault - pCur->Core.Key < pCur->cb)
755	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
756	\|\| !(uErr & X86_TRAP_PF_P)
757	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
758	("Unexpected trap for virtual handler: %08X (phys=%08x) %R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
759
760	if ( pvFault - pCur->Core.Key < pCur->cb
761	&& ( uErr & X86_TRAP_PF_RW
762	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
763	{
764	# ifdef IN_RC
765	STAM_PROFILE_START(&pCur->Stat, h);
766	pgmUnlock(pVM);
767	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
768	pgmLock(pVM);
769	STAM_PROFILE_STOP(&pCur->Stat, h);
770	# else
771	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
772	# endif
773	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
774	return rc;
775	}
776	}
777	}
778	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
779
780	/*
781	* We are here only if page is present in Guest page tables and
782	* trap is not handled by our handlers.
783	*
784	* Check it for page out-of-sync situation.
785	*/
786	if (!(uErr & X86_TRAP_PF_P))
787	{
788	/*
789	* Page is not present in our page tables. Try to sync it!
790	*/
791	if (uErr & X86_TRAP_PF_US)
792	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUser));
793	else /* supervisor */
794	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
795
796	if (PGM_PAGE_IS_BALLOONED(pPage))
797	{
798	/* Emulate reads from ballooned pages as they are not present in
799	our shadow page tables. (Required for e.g. Solaris guests; soft
800	ecc, random nr generator.) */
801	rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
802	LogFlow(("PGM: PGMInterpretInstruction balloon -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
803	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncBallloon));
804	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Ballooned; });
805	return rc;
806	}
807
808	# if defined(LOG_ENABLED) && !defined(IN_RING0)
809	RTGCPHYS GCPhys2;
810	uint64_t fPageGst2;
811	PGMGstGetPage(pVCpu, pvFault, &fPageGst2, &GCPhys2);
812	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
813	Log(("Page out of sync: %RGv eip=%08x PdeSrc.US=%d fPageGst2=%08llx GCPhys2=%RGp scan=%d\n",
814	pvFault, pRegFrame->eip, GstWalk.Pde.n.u1User, fPageGst2, GCPhys2, CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)));
815	# else
816	Log(("Page out of sync: %RGv eip=%08x fPageGst2=%08llx GCPhys2=%RGp scan=%d\n",
817	pvFault, pRegFrame->eip, fPageGst2, GCPhys2, CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)));
818	# endif
819	# endif /* LOG_ENABLED */
820
821	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
822	if ( !GstWalk.Core.fEffectiveUS
823	&& CPUMGetGuestCPL(pVCpu, pRegFrame) == 0)
824	{
825	/* Note: Can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU. */
826	if ( pvFault == (RTGCPTR)pRegFrame->eip
827	\|\| pvFault - pRegFrame->eip < 8 /* instruction crossing a page boundary */
828	# ifdef CSAM_DETECT_NEW_CODE_PAGES
829	\|\| ( !PATMIsPatchGCAddr(pVM, pRegFrame->eip)
830	&& CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)) /* any new code we encounter here */
831	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
832	)
833	{
834	LogFlow(("CSAMExecFault %RX32\n", pRegFrame->eip));
835	rc = CSAMExecFault(pVM, (RTRCPTR)pRegFrame->eip);
836	if (rc != VINF_SUCCESS)
837	{
838	/*
839	* CSAM needs to perform a job in ring 3.
840	*
841	* Sync the page before going to the host context; otherwise we'll end up in a loop if
842	* CSAM fails (e.g. instruction crosses a page boundary and the next page is not present)
843	*/
844	LogFlow(("CSAM ring 3 job\n"));
845	int rc2 = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
846	AssertRC(rc2);
847
848	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2CSAM; });
849	return rc;
850	}
851	}
852	# ifdef CSAM_DETECT_NEW_CODE_PAGES
853	else if ( uErr == X86_TRAP_PF_RW
854	&& pRegFrame->ecx >= 0x100 /* early check for movswd count */
855	&& pRegFrame->ecx < 0x10000)
856	{
857	/* In case of a write to a non-present supervisor shadow page, we'll take special precautions
858	* to detect loading of new code pages.
859	*/
860
861	/*
862	* Decode the instruction.
863	*/
864	RTGCPTR PC;
865	rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs,
866	&pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &PC);
867	if (rc == VINF_SUCCESS)
868	{
869	PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
870	uint32_t cbOp;
871	rc = EMInterpretDisasOneEx(pVM, pVCpu, PC, pRegFrame, pDis, &cbOp);
872
873	/* For now we'll restrict this to rep movsw/d instructions */
874	if ( rc == VINF_SUCCESS
875	&& pDis->pCurInstr->opcode == OP_MOVSWD
876	&& (pDis->prefix & PREFIX_REP))
877	{
878	CSAMMarkPossibleCodePage(pVM, pvFault);
879	}
880	}
881	}
882	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
883
884	/*
885	* Mark this page as safe.
886	*/
887	/** @todo not correct for pages that contain both code and data!! */
888	Log2(("CSAMMarkPage %RGv; scanned=%d\n", pvFault, true));
889	CSAMMarkPage(pVM, pvFault, true);
890	}
891	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0) */
892	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
893	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
894	# else
895	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
896	# endif
897	if (RT_SUCCESS(rc))
898	{
899	/* The page was successfully synced, return to the guest. */
900	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSync; });
901	return VINF_SUCCESS;
902	}
903	}
904	else /* uErr & X86_TRAP_PF_P: */
905	{
906	/*
907	* Write protected pages are made writable when the guest makes the
908	* first write to it. This happens for pages that are shared, write
909	* monitored or not yet allocated.
910	*
911	* We may also end up here when CR0.WP=0 in the guest.
912	*
913	* Also, a side effect of not flushing global PDEs are out of sync
914	* pages due to physical monitored regions, that are no longer valid.
915	* Assume for now it only applies to the read/write flag.
916	*/
917	if (uErr & X86_TRAP_PF_RW)
918	{
919	/*
920	* Check if it is a read-only page.
921	*/
922	if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
923	{
924	Log(("PGM #PF: Make writable: %RGp %R[pgmpage] pvFault=%RGp uErr=%#x\n", GCPhys, pPage, pvFault, uErr));
925	Assert(!PGM_PAGE_IS_ZERO(pPage));
926	AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
927	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2MakeWritable; });
928
929	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
930	if (rc != VINF_SUCCESS)
931	{
932	AssertMsg(rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("%Rrc\n", rc));
933	return rc;
934	}
935	if (RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)))
936	return VINF_EM_NO_MEMORY;
937	}
938
939	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
940	/*
941	* Check to see if we need to emulate the instruction if CR0.WP=0.
942	*/
943	if ( !GstWalk.Core.fEffectiveRW
944	&& (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG
945	&& CPUMGetGuestCPL(pVCpu, pRegFrame) == 0)
946	{
947	Assert((uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RW \| X86_TRAP_PF_P));
948	rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
949	if (RT_SUCCESS(rc))
950	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eWPEmulInRZ);
951	else
952	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eWPEmulToR3);
953	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2WPEmulation; });
954	return rc;
955	}
956	# endif
957	/// @todo count the above case; else
958	if (uErr & X86_TRAP_PF_US)
959	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUserWrite));
960	else /* supervisor */
961	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
962
963	/*
964	* Sync the page.
965	*
966	* Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
967	* page is not present, which is not true in this case.
968	*/
969	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
970	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
971	# else
972	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
973	# endif
974	if (RT_SUCCESS(rc))
975	{
976	/*
977	* Page was successfully synced, return to guest but invalidate
978	* the TLB first as the page is very likely to be in it.
979	*/
980	# if PGM_SHW_TYPE == PGM_TYPE_EPT
981	HWACCMInvalidatePhysPage(pVM, (RTGCPHYS)pvFault);
982	# else
983	PGM_INVL_PG(pVCpu, pvFault);
984	# endif
985	# ifdef VBOX_STRICT
986	RTGCPHYS GCPhys2;
987	uint64_t fPageGst;
988	if (!pVM->pgm.s.fNestedPaging)
989	{
990	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, &GCPhys2);
991	AssertMsg(RT_SUCCESS(rc) && (fPageGst & X86_PTE_RW), ("rc=%Rrc fPageGst=%RX64\n", rc, fPageGst));
992	LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GCPhys2, (uint64_t)fPageGst));
993	}
994	uint64_t fPageShw;
995	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
996	AssertMsg((RT_SUCCESS(rc) && (fPageShw & X86_PTE_RW)) \|\| pVM->cCpus > 1 /* new monitor can be installed/page table flushed between the trap exit and PGMTrap0eHandler */, ("rc=%Rrc fPageShw=%RX64\n", rc, fPageShw));
997	# endif /* VBOX_STRICT */
998	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndObs; });
999	return VINF_SUCCESS;
1000	}
1001	}
1002	/** @todo else: why are we here? */
1003
1004	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && defined(VBOX_STRICT)
1005	/*
1006	* Check for VMM page flags vs. Guest page flags consistency.
1007	* Currently only for debug purposes.
1008	*/
1009	if (RT_SUCCESS(rc))
1010	{
1011	/* Get guest page flags. */
1012	uint64_t fPageGst;
1013	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
1014	if (RT_SUCCESS(rc))
1015	{
1016	uint64_t fPageShw;
1017	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
1018
1019	/*
1020	* Compare page flags.
1021	* Note: we have AVL, A, D bits desynched.
1022	*/
1023	AssertMsg((fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)),
1024	("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64\n", pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst));
1025	}
1026	else
1027	AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc));
1028	}
1029	else
1030	AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc));
1031	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && VBOX_STRICT */
1032	}
1033
1034
1035	/*
1036	* If we get here it is because something failed above, i.e. most like guru
1037	* meditiation time.
1038	*/
1039	LogRel(("%s: returns rc=%Rrc pvFault=%RGv uErr=%RX64 cs:rip=%04x:%08RX64\n",
1040	__PRETTY_FUNCTION__, rc, pvFault, (uint64_t)uErr, pRegFrame->cs, pRegFrame->rip));
1041	return rc;
1042
1043	# else /* Nested paging, EPT except PGM_GST_TYPE = PROT */
1044	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
1045	return VERR_INTERNAL_ERROR;
1046	# endif
1047	}
1048	#endif /* !IN_RING3 */
1049
1050
1051	/**
1052	* Emulation of the invlpg instruction.
1053	*
1054	*
1055	* @returns VBox status code.
1056	*
1057	* @param pVCpu The VMCPU handle.
1058	* @param GCPtrPage Page to invalidate.
1059	*
1060	* @remark ASSUMES that the guest is updating before invalidating. This order
1061	* isn't required by the CPU, so this is speculative and could cause
1062	* trouble.
1063	* @remark No TLB shootdown is done on any other VCPU as we assume that
1064	* invlpg emulation is the only reason for calling this function.
1065	* (The guest has to shoot down TLB entries on other CPUs itself)
1066	* Currently true, but keep in mind!
1067	*
1068	* @todo Clean this up! Most of it is (or should be) no longer necessary as we catch all page table accesses.
1069	*/
1070	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
1071	{
1072	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1073	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
1074	&& PGM_SHW_TYPE != PGM_TYPE_EPT
1075	int rc;
1076	PVM pVM = pVCpu->CTX_SUFF(pVM);
1077	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1078
1079	Assert(PGMIsLockOwner(pVM));
1080
1081	LogFlow(("InvalidatePage %RGv\n", GCPtrPage));
1082
1083	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1084	if (pPool->cDirtyPages)
1085	pgmPoolResetDirtyPages(pVM);
1086	# endif
1087
1088	/*
1089	* Get the shadow PD entry and skip out if this PD isn't present.
1090	* (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
1091	*/
1092	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1093	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1094	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1095
1096	/* Fetch the pgm pool shadow descriptor. */
1097	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1098	Assert(pShwPde);
1099
1100	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1101	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT);
1102	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
1103
1104	/* If the shadow PDPE isn't present, then skip the invalidate. */
1105	if (!pPdptDst->a[iPdpt].n.u1Present)
1106	{
1107	Assert(!(pPdptDst->a[iPdpt].u & PGM_PLXFLAGS_MAPPING));
1108	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1109	return VINF_SUCCESS;
1110	}
1111
1112	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1113	PPGMPOOLPAGE pShwPde = NULL;
1114	PX86PDPAE pPDDst;
1115
1116	/* Fetch the pgm pool shadow descriptor. */
1117	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
1118	AssertRCSuccessReturn(rc, rc);
1119	Assert(pShwPde);
1120
1121	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1122	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1123
1124	# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1125	/* PML4 */
1126	const unsigned iPml4 = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;
1127	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1128	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1129	PX86PDPAE pPDDst;
1130	PX86PDPT pPdptDst;
1131	PX86PML4E pPml4eDst;
1132	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eDst, &pPdptDst, &pPDDst);
1133	if (rc != VINF_SUCCESS)
1134	{
1135	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT \|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
1136	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1137	return VINF_SUCCESS;
1138	}
1139	Assert(pPDDst);
1140
1141	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1142	PX86PDPE pPdpeDst = &pPdptDst->a[iPdpt];
1143
1144	if (!pPdpeDst->n.u1Present)
1145	{
1146	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1147	return VINF_SUCCESS;
1148	}
1149
1150	/* Fetch the pgm pool shadow descriptor. */
1151	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & SHW_PDPE_PG_MASK);
1152	Assert(pShwPde);
1153
1154	# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1155
1156	const SHWPDE PdeDst = *pPdeDst;
1157	if (!PdeDst.n.u1Present)
1158	{
1159	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1160	return VINF_SUCCESS;
1161	}
1162
1163	/*
1164	* Get the guest PD entry and calc big page.
1165	*/
1166	# if PGM_GST_TYPE == PGM_TYPE_32BIT
1167	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
1168	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
1169	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1170	# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1171	unsigned iPDSrc = 0;
1172	# if PGM_GST_TYPE == PGM_TYPE_PAE
1173	X86PDPE PdpeSrcIgn;
1174	PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrcIgn);
1175	# else /* AMD64 */
1176	PX86PML4E pPml4eSrcIgn;
1177	X86PDPE PdpeSrcIgn;
1178	PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrcIgn, &PdpeSrcIgn, &iPDSrc);
1179	# endif
1180	GSTPDE PdeSrc;
1181
1182	if (pPDSrc)
1183	PdeSrc = pPDSrc->a[iPDSrc];
1184	else
1185	PdeSrc.u = 0;
1186	# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1187	const bool fIsBigPage = PdeSrc.b.u1Size && GST_IS_PSE_ACTIVE(pVCpu);
1188
1189	# ifdef IN_RING3
1190	/*
1191	* If a CR3 Sync is pending we may ignore the invalidate page operation
1192	* depending on the kind of sync and if it's a global page or not.
1193	* This doesn't make sense in GC/R0 so we'll skip it entirely there.
1194	*/
1195	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
1196	if ( VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
1197	\|\| ( VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
1198	&& fIsBigPage
1199	&& PdeSrc.b.u1Global
1200	)
1201	)
1202	# else
1203	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
1204	# endif
1205	{
1206	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1207	return VINF_SUCCESS;
1208	}
1209	# endif /* IN_RING3 */
1210
1211	/*
1212	* Deal with the Guest PDE.
1213	*/
1214	rc = VINF_SUCCESS;
1215	if (PdeSrc.n.u1Present)
1216	{
1217	Assert( PdeSrc.n.u1User == PdeDst.n.u1User
1218	&& (PdeSrc.n.u1Write \|\| !PdeDst.n.u1Write));
1219	# ifndef PGM_WITHOUT_MAPPING
1220	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
1221	{
1222	/*
1223	* Conflict - Let SyncPT deal with it to avoid duplicate code.
1224	*/
1225	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1226	Assert(PGMGetGuestMode(pVCpu) <= PGMMODE_PAE);
1227	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
1228	}
1229	else
1230	# endif /* !PGM_WITHOUT_MAPPING */
1231	if (!fIsBigPage)
1232	{
1233	/*
1234	* 4KB - page.
1235	*/
1236	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1237	RTGCPHYS GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1238
1239	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1240	/* Reset the modification counter (OpenSolaris trashes tlb entries very often) */
1241	if (pShwPage->cModifications)
1242	pShwPage->cModifications = 1;
1243	# endif
1244
1245	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1246	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1247	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE/2);
1248	# endif
1249	if (pShwPage->GCPhys == GCPhys)
1250	{
1251	# if 0 /* likely cause of a major performance regression; must be SyncPageWorkerTrackDeref then */
1252	const unsigned iPTEDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1253	PSHWPT pPT = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1254	if (pPT->a[iPTEDst].n.u1Present)
1255	{
1256	/* This is very unlikely with caching/monitoring enabled. */
1257	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pShwPage, pPT->a[iPTEDst].u & SHW_PTE_PG_MASK, iPTEDst);
1258	ASMAtomicWriteSize(&pPT->a[iPTEDst], 0);
1259	}
1260	# else /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1261	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
1262	if (RT_SUCCESS(rc))
1263	rc = VINF_SUCCESS;
1264	# endif
1265	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4KBPages));
1266	PGM_INVL_PG(pVCpu, GCPtrPage);
1267	}
1268	else
1269	{
1270	/*
1271	* The page table address changed.
1272	*/
1273	LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
1274	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1275	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1276	ASMAtomicWriteSize(pPdeDst, 0);
1277	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1278	PGM_INVL_VCPU_TLBS(pVCpu);
1279	}
1280	}
1281	else
1282	{
1283	/*
1284	* 2/4MB - page.
1285	*/
1286	/* Before freeing the page, check if anything really changed. */
1287	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1288	RTGCPHYS GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(pVM, PdeSrc);
1289	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1290	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1291	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1292	# endif
1293	if ( pShwPage->GCPhys == GCPhys
1294	&& pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1295	{
1296	/* ASSUMES a the given bits are identical for 4M and normal PDEs */
1297	/** @todo PAT */
1298	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
1299	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
1300	&& ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1301	\|\| (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1302	{
1303	LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1304	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip));
1305	return VINF_SUCCESS;
1306	}
1307	}
1308
1309	/*
1310	* Ok, the page table is present and it's been changed in the guest.
1311	* If we're in host context, we'll just mark it as not present taking the lazy approach.
1312	* We could do this for some flushes in GC too, but we need an algorithm for
1313	* deciding which 4MB pages containing code likely to be executed very soon.
1314	*/
1315	LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1316	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1317	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1318	ASMAtomicWriteSize(pPdeDst, 0);
1319	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4MBPages));
1320	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
1321	}
1322	}
1323	else
1324	{
1325	/*
1326	* Page directory is not present, mark shadow PDE not present.
1327	*/
1328	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
1329	{
1330	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1331	ASMAtomicWriteSize(pPdeDst, 0);
1332	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDNPs));
1333	PGM_INVL_PG(pVCpu, GCPtrPage);
1334	}
1335	else
1336	{
1337	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1338	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDMappings));
1339	}
1340	}
1341	return rc;
1342
1343	#else /* guest real and protected mode */
1344	/* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1345	return VINF_SUCCESS;
1346	#endif
1347	}
1348
1349
1350	/**
1351	* Update the tracking of shadowed pages.
1352	*
1353	* @param pVCpu The VMCPU handle.
1354	* @param pShwPage The shadow page.
1355	* @param HCPhys The physical page we is being dereferenced.
1356	* @param iPte Shadow PTE index
1357	*/
1358	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys, uint16_t iPte)
1359	{
1360	PVM pVM = pVCpu->CTX_SUFF(pVM);
1361
1362	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackDeref, a);
1363	LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1364
1365	/** @todo If this turns out to be a bottle neck (very likely) two things can be done:
1366	* 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1367	* 2. write protect all shadowed pages. I.e. implement caching.
1368	*/
1369	/** @todo duplicated in the 2nd half of pgmPoolTracDerefGCPhysHint */
1370
1371	/*
1372	* Find the guest address.
1373	*/
1374	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
1375	pRam;
1376	pRam = pRam->CTX_SUFF(pNext))
1377	{
1378	unsigned iPage = pRam->cb >> PAGE_SHIFT;
1379	while (iPage-- > 0)
1380	{
1381	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1382	{
1383	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1384
1385	Assert(pShwPage->cPresent);
1386	Assert(pPool->cPresent);
1387	pShwPage->cPresent--;
1388	pPool->cPresent--;
1389
1390	pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage], iPte);
1391	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackDeref, a);
1392	return;
1393	}
1394	}
1395	}
1396
1397	for (;;)
1398	AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys));
1399	}
1400
1401
1402	/**
1403	* Update the tracking of shadowed pages.
1404	*
1405	* @param pVCpu The VMCPU handle.
1406	* @param pShwPage The shadow page.
1407	* @param u16 The top 16-bit of the pPage->HCPhys.
1408	* @param pPage Pointer to the guest page. this will be modified.
1409	* @param iPTDst The index into the shadow table.
1410	*/
1411	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
1412	{
1413	PVM pVM = pVCpu->CTX_SUFF(pVM);
1414	/*
1415	* Just deal with the simple first time here.
1416	*/
1417	if (!u16)
1418	{
1419	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackVirgin);
1420	u16 = PGMPOOL_TD_MAKE(1, pShwPage->idx);
1421	/* Save the page table index. */
1422	PGM_PAGE_SET_PTE_INDEX(pPage, iPTDst);
1423	}
1424	else
1425	u16 = pgmPoolTrackPhysExtAddref(pVM, pPage, u16, pShwPage->idx, iPTDst);
1426
1427	/* write back */
1428	Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x iPTDst=%#x\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst));
1429	PGM_PAGE_SET_TRACKING(pPage, u16);
1430
1431	/* update statistics. */
1432	pVM->pgm.s.CTX_SUFF(pPool)->cPresent++;
1433	pShwPage->cPresent++;
1434	if (pShwPage->iFirstPresent > iPTDst)
1435	pShwPage->iFirstPresent = iPTDst;
1436	}
1437
1438
1439	/**
1440	* Modifies a shadow PTE to account for access handlers.
1441	*
1442	* @param pVM The VM handle.
1443	* @param pPage The page in question.
1444	* @param fPteSrc The flags of the source PTE.
1445	* @param pPteDst The shadow PTE (output).
1446	*/
1447	DECLINLINE(void) PGM_BTH_NAME(SyncHandlerPte)(PVM pVM, PCPGMPAGE pPage, uint32_t fPteSrc, PSHWPTE pPteDst)
1448	{
1449	/** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No.
1450	* Update: \#PF should deal with this before or after calling the handlers. It has all the info to do the job efficiently. */
1451	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1452	{
1453	LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark read-only\n", pPage));
1454	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1455	pPteDst->u = PGM_PAGE_GET_HCPHYS(pPage);
1456	pPteDst->n.u1Present = 1;
1457	pPteDst->n.u1Execute = 1;
1458	pPteDst->n.u1IgnorePAT = 1;
1459	pPteDst->n.u3EMT = VMX_EPT_MEMTYPE_WB;
1460	/* PteDst.n.u1Write = 0 && PteDst.n.u1Size = 0 */
1461	#else
1462	pPteDst->u = (fPteSrc & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1463	\| PGM_PAGE_GET_HCPHYS(pPage);
1464	#endif
1465	}
1466	#ifdef PGM_WITH_MMIO_OPTIMIZATIONS
1467	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
1468	else if ( PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
1469	&& ( BTH_IS_NP_ACTIVE(pVM)
1470	\|\| (fPteSrc & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW) /** @todo remove X86_PTE_US */
1471	# if PGM_SHW_TYPE == PGM_TYPE_AMD64
1472	&& pVM->pgm.s.fLessThan52PhysicalAddressBits
1473	# endif
1474	)
1475	{
1476	LogFlow(("SyncHandlerPte: MMIO page -> invalid \n"));
1477	# if PGM_SHW_TYPE == PGM_TYPE_EPT
1478	/* 25.2.3.1: Reserved physical address bit -> EPT Misconfiguration (exit 49) */
1479	pPteDst->u = pVM->pgm.s.HCPhysInvMmioPg;
1480	/* 25.2.3.1: bits 2:0 = 010b -> EPT Misconfiguration (exit 49) */
1481	pPteDst->n.u1Present = 0;
1482	pPteDst->n.u1Write = 1;
1483	pPteDst->n.u1Execute = 0;
1484	/* 25.2.3.1: leaf && 2:0 != 0 && u3Emt in {2, 3, 7} -> EPT Misconfiguration */
1485	pPteDst->n.u3EMT = 7;
1486	# else
1487	/* Set high page frame bits that MBZ (bankers on PAE, CPU dependent on AMD64). */
1488	pPteDst->u = pVM->pgm.s.HCPhysInvMmioPg \| X86_PTE_PAE_MBZ_MASK_NO_NX \| X86_PTE_P;
1489	# endif
1490	}
1491	# endif
1492	#endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
1493	else
1494	{
1495	LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark not present\n", pPage));
1496	pPteDst->u = 0;
1497	}
1498	/** @todo count these kinds of entries. */
1499	}
1500
1501
1502	/**
1503	* Creates a 4K shadow page for a guest page.
1504	*
1505	* For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1506	* physical address. The PdeSrc argument only the flags are used. No page
1507	* structured will be mapped in this function.
1508	*
1509	* @param pVCpu The VMCPU handle.
1510	* @param pPteDst Destination page table entry.
1511	* @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1512	* Can safely assume that only the flags are being used.
1513	* @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1514	* @param pShwPage Pointer to the shadow page.
1515	* @param iPTDst The index into the shadow table.
1516	*
1517	* @remark Not used for 2/4MB pages!
1518	*/
1519	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc,
1520	PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1521	{
1522	if ( PteSrc.n.u1Present
1523	&& GST_IS_PTE_VALID(pVCpu, PteSrc))
1524	{
1525	PVM pVM = pVCpu->CTX_SUFF(pVM);
1526
1527	# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1528	&& PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1529	&& (PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64)
1530	if (pShwPage->fDirty)
1531	{
1532	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1533	PX86PTPAE pGstPT;
1534
1535	pGstPT = (PX86PTPAE)&pPool->aDirtyPages[pShwPage->idxDirty][0];
1536	pGstPT->a[iPTDst].u = PteSrc.u;
1537	}
1538	# endif
1539	/*
1540	* Find the ram range.
1541	*/
1542	PPGMPAGE pPage;
1543	int rc = pgmPhysGetPageEx(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK, &pPage);
1544	if (RT_SUCCESS(rc))
1545	{
1546	/* Ignore ballooned pages.
1547	Don't return errors or use a fatal assert here as part of a
1548	shadow sync range might included ballooned pages. */
1549	if (PGM_PAGE_IS_BALLOONED(pPage))
1550	{
1551	Assert(!pPteDst->n.u1Present); /** @todo user tracking needs updating if this triggers. */
1552	return;
1553	}
1554
1555	#ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1556	/* Make the page writable if necessary. */
1557	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1558	&& ( PGM_PAGE_IS_ZERO(pPage)
1559	\|\| ( PteSrc.n.u1Write
1560	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1561	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1562	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1563	# endif
1564	# ifdef VBOX_WITH_PAGE_SHARING
1565	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
1566	# endif
1567	)
1568	)
1569	)
1570	{
1571	rc = pgmPhysPageMakeWritable(pVM, pPage, PteSrc.u & GST_PTE_PG_MASK);
1572	AssertRC(rc);
1573	}
1574	#endif
1575
1576	/*
1577	* Make page table entry.
1578	*/
1579	SHWPTE PteDst;
1580	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1581	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage,
1582	PteSrc.u & ~( X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK
1583	\| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT),
1584	&PteDst);
1585	else
1586	{
1587	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1588	/*
1589	* If the page or page directory entry is not marked accessed,
1590	* we mark the page not present.
1591	*/
1592	if (!PteSrc.n.u1Accessed \|\| !PdeSrc.n.u1Accessed)
1593	{
1594	LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1595	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,AccessedPage));
1596	PteDst.u = 0;
1597	}
1598	/*
1599	* If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1600	* when the page is modified.
1601	*/
1602	else if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1603	{
1604	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPage));
1605	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1606	\| PGM_PAGE_GET_HCPHYS(pPage)
1607	\| PGM_PTFLAGS_TRACK_DIRTY;
1608	}
1609	else
1610	#endif
1611	{
1612	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageSkipped));
1613	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1614	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage);
1615	PteDst.n.u1Present = 1;
1616	PteDst.n.u1Write = 1;
1617	PteDst.n.u1Execute = 1;
1618	PteDst.n.u1IgnorePAT = 1;
1619	PteDst.n.u3EMT = VMX_EPT_MEMTYPE_WB;
1620	/* PteDst.n.u1Size = 0 */
1621	#else
1622	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1623	\| PGM_PAGE_GET_HCPHYS(pPage);
1624	#endif
1625	}
1626
1627	/*
1628	* Make sure only allocated pages are mapped writable.
1629	*/
1630	if ( PteDst.n.u1Write
1631	&& PteDst.n.u1Present
1632	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1633	{
1634	/* Still applies to shared pages. */
1635	Assert(!PGM_PAGE_IS_ZERO(pPage));
1636	PteDst.n.u1Write = 0; /** @todo this isn't quite working yet. Why, isn't it? */
1637	Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", (RTGCPHYS)(PteSrc.u & X86_PTE_PAE_PG_MASK), pPage, iPTDst));
1638	}
1639	}
1640
1641	/*
1642	* Keep user track up to date.
1643	*/
1644	if (PteDst.n.u1Present)
1645	{
1646	if (!pPteDst->n.u1Present)
1647	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1648	else if ((pPteDst->u & SHW_PTE_PG_MASK) != (PteDst.u & SHW_PTE_PG_MASK))
1649	{
1650	Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", (uint64_t)pPteDst->u, (uint64_t)PteDst.u));
1651	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK, iPTDst);
1652	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1653	}
1654	}
1655	else if (pPteDst->n.u1Present)
1656	{
1657	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1658	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK, iPTDst);
1659	}
1660
1661	/*
1662	* Update statistics and commit the entry.
1663	*/
1664	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1665	if (!PteSrc.n.u1Global)
1666	pShwPage->fSeenNonGlobal = true;
1667	#endif
1668	ASMAtomicWriteSize(pPteDst, PteDst.u);
1669	return;
1670	}
1671
1672	/** @todo count these three different kinds. */
1673	Log2(("SyncPageWorker: invalid address in Pte\n"));
1674	}
1675	else if (!PteSrc.n.u1Present)
1676	Log2(("SyncPageWorker: page not present in Pte\n"));
1677	else
1678	Log2(("SyncPageWorker: invalid Pte\n"));
1679
1680	/*
1681	* The page is not present or the PTE is bad. Replace the shadow PTE by
1682	* an empty entry, making sure to keep the user tracking up to date.
1683	*/
1684	if (pPteDst->n.u1Present)
1685	{
1686	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1687	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK, iPTDst);
1688	}
1689	ASMAtomicWriteSize(pPteDst, 0);
1690	}
1691
1692
1693	/**
1694	* Syncs a guest OS page.
1695	*
1696	* There are no conflicts at this point, neither is there any need for
1697	* page table allocations.
1698	*
1699	* When called in PAE or AMD64 guest mode, the guest PDPE shall be valid.
1700	* When called in AMD64 guest mode, the guest PML4E shall be valid.
1701	*
1702	* @returns VBox status code.
1703	* @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1704	* @param pVCpu The VMCPU handle.
1705	* @param PdeSrc Page directory entry of the guest.
1706	* @param GCPtrPage Guest context page address.
1707	* @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1708	* @param uErr Fault error (X86_TRAP_PF_*).
1709	*/
1710	static int PGM_BTH_NAME(SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr)
1711	{
1712	PVM pVM = pVCpu->CTX_SUFF(pVM);
1713	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1714	LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr));
1715
1716	Assert(PGMIsLockOwner(pVM));
1717
1718	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
1719	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
1720	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
1721	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
1722	&& PGM_SHW_TYPE != PGM_TYPE_EPT
1723
1724	/*
1725	* Assert preconditions.
1726	*/
1727	Assert(PdeSrc.n.u1Present);
1728	Assert(cPages);
1729	# if 0 /* rarely useful; leave for debugging. */
1730	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
1731	# endif
1732
1733	/*
1734	* Get the shadow PDE, find the shadow page table in the pool.
1735	*/
1736	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1737	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1738	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1739
1740	/* Fetch the pgm pool shadow descriptor. */
1741	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1742	Assert(pShwPde);
1743
1744	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1745	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1746	PPGMPOOLPAGE pShwPde = NULL;
1747	PX86PDPAE pPDDst;
1748
1749	/* Fetch the pgm pool shadow descriptor. */
1750	int rc2 = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
1751	AssertRCSuccessReturn(rc2, rc2);
1752	Assert(pShwPde);
1753
1754	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1755	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1756
1757	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1758	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1759	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1760	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
1761	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
1762
1763	int rc2 = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
1764	AssertRCSuccessReturn(rc2, rc2);
1765	Assert(pPDDst && pPdptDst);
1766	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1767	# endif
1768	SHWPDE PdeDst = *pPdeDst;
1769
1770	/* In the guest SMP case we could have blocked while another VCPU reused this page table. */
1771	if (!PdeDst.n.u1Present)
1772	{
1773	AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE p=%p/%RX64\n", pPdeDst, (uint64_t)PdeDst.u));
1774	Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", pVCpu->idCpu, GCPtrPage));
1775	return VINF_SUCCESS; /* force the instruction to be executed again. */
1776	}
1777
1778	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1779	Assert(pShwPage);
1780
1781	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1782	/* Fetch the pgm pool shadow descriptor. */
1783	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
1784	Assert(pShwPde);
1785	# endif
1786
1787	/*
1788	* Check that the page is present and that the shadow PDE isn't out of sync.
1789	*/
1790	const bool fBigPage = PdeSrc.b.u1Size && GST_IS_PSE_ACTIVE(pVCpu);
1791	const bool fPdeValid = !fBigPage ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc);
1792	RTGCPHYS GCPhys;
1793	if (!fBigPage)
1794	{
1795	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1796	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1797	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1798	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE / 2);
1799	# endif
1800	}
1801	else
1802	{
1803	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(pVM, PdeSrc);
1804	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1805	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1806	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1807	# endif
1808	}
1809	if ( fPdeValid
1810	&& pShwPage->GCPhys == GCPhys
1811	&& PdeSrc.n.u1Present
1812	&& PdeSrc.n.u1User == PdeDst.n.u1User
1813	&& (PdeSrc.n.u1Write == PdeDst.n.u1Write \|\| !PdeDst.n.u1Write)
1814	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
1815	&& (PdeSrc.n.u1NoExecute == PdeDst.n.u1NoExecute \|\| !GST_IS_NX_ACTIVE(pVCpu))
1816	# endif
1817	)
1818	{
1819	/*
1820	* Check that the PDE is marked accessed already.
1821	* Since we set the accessed bit before getting here on a #PF, this
1822	* check is only meant for dealing with non-#PF'ing paths.
1823	*/
1824	if (PdeSrc.n.u1Accessed)
1825	{
1826	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1827	if (!fBigPage)
1828	{
1829	/*
1830	* 4KB Page - Map the guest page table.
1831	*/
1832	PGSTPT pPTSrc;
1833	int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
1834	if (RT_SUCCESS(rc))
1835	{
1836	# ifdef PGM_SYNC_N_PAGES
1837	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
1838	if ( cPages > 1
1839	&& !(uErr & X86_TRAP_PF_P)
1840	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
1841	{
1842	/*
1843	* This code path is currently only taken when the caller is PGMTrap0eHandler
1844	* for non-present pages!
1845	*
1846	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1847	* deal with locality.
1848	*/
1849	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1850	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1851	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1852	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1853	# else
1854	const unsigned offPTSrc = 0;
1855	# endif
1856	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
1857	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1858	iPTDst = 0;
1859	else
1860	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1861	for (; iPTDst < iPTDstEnd; iPTDst++)
1862	{
1863	if (!pPTDst->a[iPTDst].n.u1Present)
1864	{
1865	GSTPTE PteSrc = pPTSrc->a[offPTSrc + iPTDst];
1866	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
1867	NOREF(GCPtrCurPage);
1868	#ifndef IN_RING0
1869	/*
1870	* Assuming kernel code will be marked as supervisor - and not as user level
1871	* and executed using a conforming code selector - And marked as readonly.
1872	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
1873	*/
1874	PPGMPAGE pPage;
1875	if ( ((PdeSrc.u & PteSrc.u) & (X86_PTE_RW \| X86_PTE_US))
1876	\|\| iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK) /* always sync GCPtrPage */
1877	\|\| !CSAMDoesPageNeedScanning(pVM, GCPtrCurPage)
1878	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
1879	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1880	)
1881	#endif /* else: CSAM not active */
1882	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1883	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1884	GCPtrCurPage, PteSrc.n.u1Present,
1885	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1886	PteSrc.n.u1User & PdeSrc.n.u1User,
1887	(uint64_t)PteSrc.u,
1888	(uint64_t)pPTDst->a[iPTDst].u,
1889	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1890	}
1891	}
1892	}
1893	else
1894	# endif /* PGM_SYNC_N_PAGES */
1895	{
1896	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1897	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1898	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1899	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1900	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
1901	GCPtrPage, PteSrc.n.u1Present,
1902	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1903	PteSrc.n.u1User & PdeSrc.n.u1User,
1904	(uint64_t)PteSrc.u,
1905	(uint64_t)pPTDst->a[iPTDst].u,
1906	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1907	}
1908	}
1909	else /* MMIO or invalid page: emulated in #PF handler. */
1910	{
1911	LogFlow(("PGM_GCPHYS_2_PTR %RGp failed with %Rrc\n", GCPhys, rc));
1912	Assert(!pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK].n.u1Present);
1913	}
1914	}
1915	else
1916	{
1917	/*
1918	* 4/2MB page - lazy syncing shadow 4K pages.
1919	* (There are many causes of getting here, it's no longer only CSAM.)
1920	*/
1921	/* Calculate the GC physical address of this 4KB shadow page. */
1922	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(pVM, PdeSrc) \| (GCPtrPage & GST_BIG_PAGE_OFFSET_MASK);
1923	/* Find ram range. */
1924	PPGMPAGE pPage;
1925	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
1926	if (RT_SUCCESS(rc))
1927	{
1928	AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
1929
1930	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1931	/* Try to make the page writable if necessary. */
1932	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1933	&& ( PGM_PAGE_IS_ZERO(pPage)
1934	\|\| ( PdeSrc.n.u1Write
1935	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1936	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1937	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1938	# endif
1939	# ifdef VBOX_WITH_PAGE_SHARING
1940	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
1941	# endif
1942	)
1943	)
1944	)
1945	{
1946	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1947	AssertRC(rc);
1948	}
1949	# endif
1950
1951	/*
1952	* Make shadow PTE entry.
1953	*/
1954	SHWPTE PteDst;
1955	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1956	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage,
1957	PdeSrc.u & ~( X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK
1958	\| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT),
1959	&PteDst);
1960	else
1961	PteDst.u = (PdeSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1962	\| PGM_PAGE_GET_HCPHYS(pPage);
1963
1964	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1965	if ( PteDst.n.u1Present
1966	&& !pPTDst->a[iPTDst].n.u1Present)
1967	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1968
1969	/* Make sure only allocated pages are mapped writable. */
1970	if ( PteDst.n.u1Write
1971	&& PteDst.n.u1Present
1972	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1973	{
1974	/* Still applies to shared pages. */
1975	Assert(!PGM_PAGE_IS_ZERO(pPage));
1976	PteDst.n.u1Write = 0; /** @todo this isn't quite working yet... */
1977	Log3(("SyncPage: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, GCPtrPage));
1978	}
1979
1980	ASMAtomicWriteSize(&pPTDst->a[iPTDst], PteDst.u);
1981
1982	/*
1983	* If the page is not flagged as dirty and is writable, then make it read-only
1984	* at PD level, so we can set the dirty bit when the page is modified.
1985	*
1986	* ASSUMES that page access handlers are implemented on page table entry level.
1987	* Thus we will first catch the dirty access and set PDE.D and restart. If
1988	* there is an access handler, we'll trap again and let it work on the problem.
1989	*/
1990	/** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
1991	* As for invlpg, it simply frees the whole shadow PT.
1992	* ...It's possibly because the guest clears it and the guest doesn't really tell us... */
1993	if ( !PdeSrc.b.u1Dirty
1994	&& PdeSrc.b.u1Write)
1995	{
1996	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
1997	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
1998	PdeDst.n.u1Write = 0;
1999	}
2000	else
2001	{
2002	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2003	PdeDst.n.u1Write = PdeSrc.n.u1Write;
2004	}
2005	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2006	Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n",
2007	GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
2008	PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2009	}
2010	else
2011	{
2012	LogFlow(("PGM_GCPHYS_2_PTR %RGp (big) failed with %Rrc\n", GCPhys, rc));
2013	/** @todo must wipe the shadow page table in this case. */
2014	}
2015	}
2016	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2017	return VINF_SUCCESS;
2018	}
2019
2020	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPagePDNAs));
2021	}
2022	else if (fPdeValid)
2023	{
2024	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPagePDOutOfSync));
2025	Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2026	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2027	}
2028	else
2029	{
2030	/// @todo STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSync));
2031	Log2(("SyncPage: Bad PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2032	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2033	}
2034
2035	/*
2036	* Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
2037	* Yea, I'm lazy.
2038	*/
2039	pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
2040	ASMAtomicWriteSize(pPdeDst, 0);
2041
2042	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2043	PGM_INVL_VCPU_TLBS(pVCpu);
2044	return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
2045
2046
2047	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2048	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2049	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2050	&& !defined(IN_RC)
2051
2052	# ifdef PGM_SYNC_N_PAGES
2053	/*
2054	* Get the shadow PDE, find the shadow page table in the pool.
2055	*/
2056	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2057	X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
2058
2059	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2060	X86PDEPAE PdeDst = pgmShwGetPaePDE(pVCpu, GCPtrPage);
2061
2062	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2063	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2064	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpt);
2065	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2066	X86PDEPAE PdeDst;
2067	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2068
2069	int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2070	AssertRCSuccessReturn(rc, rc);
2071	Assert(pPDDst && pPdptDst);
2072	PdeDst = pPDDst->a[iPDDst];
2073	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2074	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2075	PEPTPD pPDDst;
2076	EPTPDE PdeDst;
2077
2078	int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, NULL, &pPDDst);
2079	if (rc != VINF_SUCCESS)
2080	{
2081	AssertRC(rc);
2082	return rc;
2083	}
2084	Assert(pPDDst);
2085	PdeDst = pPDDst->a[iPDDst];
2086	# endif
2087	/* In the guest SMP case we could have blocked while another VCPU reused this page table. */
2088	if (!PdeDst.n.u1Present)
2089	{
2090	AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)PdeDst.u));
2091	Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", pVCpu->idCpu, GCPtrPage));
2092	return VINF_SUCCESS; /* force the instruction to be executed again. */
2093	}
2094
2095	/* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */
2096	if (PdeDst.n.u1Size)
2097	{
2098	Assert(pVM->pgm.s.fNestedPaging);
2099	Log(("CPU%d: SyncPage: Pde (big:%RX64) at %RGv changed behind our back!\n", pVCpu->idCpu, PdeDst.u, GCPtrPage));
2100	return VINF_SUCCESS;
2101	}
2102
2103	/* Mask away the page offset. */
2104	GCPtrPage &= ~((RTGCPTR)0xfff);
2105
2106	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2107	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2108
2109	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
2110	if ( cPages > 1
2111	&& !(uErr & X86_TRAP_PF_P)
2112	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2113	{
2114	/*
2115	* This code path is currently only taken when the caller is PGMTrap0eHandler
2116	* for non-present pages!
2117	*
2118	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2119	* deal with locality.
2120	*/
2121	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2122	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2123	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2124	iPTDst = 0;
2125	else
2126	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2127	for (; iPTDst < iPTDstEnd; iPTDst++)
2128	{
2129	if (!pPTDst->a[iPTDst].n.u1Present)
2130	{
2131	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT);
2132	GSTPTE PteSrc;
2133
2134	/* Fake the page table entry */
2135	PteSrc.u = GCPtrCurPage;
2136	PteSrc.n.u1Present = 1;
2137	PteSrc.n.u1Dirty = 1;
2138	PteSrc.n.u1Accessed = 1;
2139	PteSrc.n.u1Write = 1;
2140	PteSrc.n.u1User = 1;
2141
2142	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2143	#ifdef DEBUG_sandervl
2144	if (pVM->pgm.s.fCountingPhysWrites)
2145	pPTDst->a[iPTDst].n.u1Write = 0;
2146	#endif
2147
2148	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
2149	GCPtrCurPage, PteSrc.n.u1Present,
2150	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2151	PteSrc.n.u1User & PdeSrc.n.u1User,
2152	(uint64_t)PteSrc.u,
2153	(uint64_t)pPTDst->a[iPTDst].u,
2154	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2155
2156	if (RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)))
2157	break;
2158	}
2159	else
2160	Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n", (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT), iPTDst, pPTDst->a[iPTDst].u));
2161	}
2162	}
2163	else
2164	# endif /* PGM_SYNC_N_PAGES */
2165	{
2166	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2167	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT);
2168	GSTPTE PteSrc;
2169
2170	#ifdef DEBUG_sandervl
2171	if ( pVM->pgm.s.fCountingPhysWrites
2172	&& ((uErr & (X86_TRAP_PF_RW\|X86_TRAP_PF_P)) == (X86_TRAP_PF_RW\|X86_TRAP_PF_P)))
2173	{
2174	STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat, FTPhysPageWrite));
2175	}
2176	#endif
2177
2178	/* Fake the page table entry */
2179	PteSrc.u = GCPtrCurPage;
2180	PteSrc.n.u1Present = 1;
2181	PteSrc.n.u1Dirty = 1;
2182	PteSrc.n.u1Accessed = 1;
2183	PteSrc.n.u1Write = 1;
2184	PteSrc.n.u1User = 1;
2185	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2186
2187	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}PteDst=%08llx%s\n",
2188	GCPtrPage, PteSrc.n.u1Present,
2189	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2190	PteSrc.n.u1User & PdeSrc.n.u1User,
2191	(uint64_t)PteSrc.u,
2192	(uint64_t)pPTDst->a[iPTDst].u,
2193	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2194	}
2195	return VINF_SUCCESS;
2196
2197	#else
2198	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2199	return VERR_INTERNAL_ERROR;
2200	#endif
2201	}
2202
2203
2204	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2205
2206	/**
2207	* CheckPageFault helper for returning a page fault indicating a non-present
2208	* (NP) entry in the page translation structures.
2209	*
2210	* @returns VINF_EM_RAW_GUEST_TRAP.
2211	* @param pVCpu The virtual CPU to operate on.
2212	* @param uErr The error code of the shadow fault. Corrections to
2213	* TRPM's copy will be made if necessary.
2214	* @param GCPtrPage For logging.
2215	* @param uPageFaultLevel For logging.
2216	*/
2217	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnNP)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2218	{
2219	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2220	AssertMsg(!(uErr & X86_TRAP_PF_P), ("%#x\n", uErr));
2221	AssertMsg(!(uErr & X86_TRAP_PF_RSVD), ("%#x\n", uErr));
2222	if (uErr & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
2223	TRPMSetErrorCode(pVCpu, uErr & ~(X86_TRAP_PF_RSVD \| X86_TRAP_PF_P));
2224
2225	Log(("CheckPageFault: real page fault (notp) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2226	return VINF_EM_RAW_GUEST_TRAP;
2227	}
2228
2229
2230	/**
2231	* CheckPageFault helper for returning a page fault indicating a reserved bit
2232	* (RSVD) error in the page translation structures.
2233	*
2234	* @returns VINF_EM_RAW_GUEST_TRAP.
2235	* @param pVCpu The virtual CPU to operate on.
2236	* @param uErr The error code of the shadow fault. Corrections to
2237	* TRPM's copy will be made if necessary.
2238	* @param GCPtrPage For logging.
2239	* @param uPageFaultLevel For logging.
2240	*/
2241	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnRSVD)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2242	{
2243	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2244	if ((uErr & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) != (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
2245	TRPMSetErrorCode(pVCpu, uErr \| X86_TRAP_PF_RSVD \| X86_TRAP_PF_P);
2246
2247	Log(("CheckPageFault: real page fault (rsvd) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2248	return VINF_EM_RAW_GUEST_TRAP;
2249	}
2250
2251
2252	/**
2253	* CheckPageFault helper for returning a page protection fault (P).
2254	*
2255	* @returns VINF_EM_RAW_GUEST_TRAP.
2256	* @param pVCpu The virtual CPU to operate on.
2257	* @param uErr The error code of the shadow fault. Corrections to
2258	* TRPM's copy will be made if necessary.
2259	* @param GCPtrPage For logging.
2260	* @param uPageFaultLevel For logging.
2261	*/
2262	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnProt)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2263	{
2264	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2265	AssertMsg(uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID), ("%#x\n", uErr));
2266	if ((uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) != X86_TRAP_PF_P)
2267	TRPMSetErrorCode(pVCpu, (uErr & ~X86_TRAP_PF_RSVD) \| X86_TRAP_PF_P);
2268
2269	Log(("CheckPageFault: real page fault (prot) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2270	return VINF_EM_RAW_GUEST_TRAP;
2271	}
2272
2273
2274	/**
2275	* Handle dirty bit tracking faults.
2276	*
2277	* @returns VBox status code.
2278	* @param pVCpu The VMCPU handle.
2279	* @param uErr Page fault error code.
2280	* @param pPdeSrc Guest page directory entry.
2281	* @param pPdeDst Shadow page directory entry.
2282	* @param GCPtrPage Guest context page address.
2283	*/
2284	static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, RTGCPTR GCPtrPage)
2285	{
2286	PVM pVM = pVCpu->CTX_SUFF(pVM);
2287	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2288
2289	Assert(PGMIsLockOwner(pVM));
2290
2291	/*
2292	* Handle big page.
2293	*/
2294	if (pPdeSrc->b.u1Size && GST_IS_PSE_ACTIVE(pVCpu))
2295	{
2296	if ( pPdeDst->n.u1Present
2297	&& (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY))
2298	{
2299	SHWPDE PdeDst = *pPdeDst;
2300
2301	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageTrap));
2302	Assert(pPdeSrc->b.u1Write);
2303
2304	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
2305	* fault again and take this path to only invalidate the entry (see below).
2306	*/
2307	PdeDst.n.u1Write = 1;
2308	PdeDst.n.u1Accessed = 1;
2309	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2310	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2311	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
2312	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2313	}
2314
2315	# ifdef IN_RING0
2316	/* Check for stale TLB entry; only applies to the SMP guest case. */
2317	if ( pVM->cCpus > 1
2318	&& pPdeDst->n.u1Write
2319	&& pPdeDst->n.u1Accessed)
2320	{
2321	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2322	if (pShwPage)
2323	{
2324	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2325	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2326	if ( pPteDst->n.u1Present
2327	&& pPteDst->n.u1Write)
2328	{
2329	/* Stale TLB entry. */
2330	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageStale));
2331	PGM_INVL_PG(pVCpu, GCPtrPage);
2332	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2333	}
2334	}
2335	}
2336	# endif /* IN_RING0 */
2337	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2338	}
2339
2340	/*
2341	* Map the guest page table.
2342	*/
2343	PGSTPT pPTSrc;
2344	int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
2345	if (RT_FAILURE(rc))
2346	{
2347	AssertRC(rc);
2348	return rc;
2349	}
2350
2351	if (pPdeDst->n.u1Present)
2352	{
2353	GSTPTE const *pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2354	const GSTPTE PteSrc = *pPteSrc;
2355
2356	#ifndef IN_RING0
2357	/* Bail out here as pgmPoolGetPage will return NULL and we'll crash below.
2358	* Our individual shadow handlers will provide more information and force a fatal exit.
2359	*/
2360	if (MMHyperIsInsideArea(pVM, (RTGCPTR)GCPtrPage))
2361	{
2362	LogRel(("CheckPageFault: write to hypervisor region %RGv\n", GCPtrPage));
2363	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2364	}
2365	#endif
2366	/*
2367	* Map shadow page table.
2368	*/
2369	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2370	if (pShwPage)
2371	{
2372	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2373	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2374	if (pPteDst->n.u1Present) /** @todo Optimize accessed bit emulation? */
2375	{
2376	if (pPteDst->u & PGM_PTFLAGS_TRACK_DIRTY)
2377	{
2378	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, pPteSrc->u & GST_PTE_PG_MASK);
2379	SHWPTE PteDst = *pPteDst;
2380
2381	LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage));
2382	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageTrap));
2383
2384	Assert(pPteSrc->n.u1Write);
2385
2386	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB
2387	* entry will not harm; write access will simply fault again and
2388	* take this path to only invalidate the entry.
2389	*/
2390	if (RT_LIKELY(pPage))
2391	{
2392	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2393	{
2394	AssertMsgFailed(("%R[pgmpage] - we don't set PGM_PTFLAGS_TRACK_DIRTY for these pages\n", pPage));
2395	Assert(!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage));
2396	/* Assuming write handlers here as the PTE is present (otherwise we wouldn't be here). */
2397	PteDst.n.u1Write = 0;
2398	}
2399	else
2400	{
2401	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
2402	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
2403	{
2404	rc = pgmPhysPageMakeWritable(pVM, pPage, pPteSrc->u & GST_PTE_PG_MASK);
2405	AssertRC(rc);
2406	}
2407	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED)
2408	PteDst.n.u1Write = 1;
2409	else
2410	{
2411	/* Still applies to shared pages. */
2412	Assert(!PGM_PAGE_IS_ZERO(pPage));
2413	PteDst.n.u1Write = 0;
2414	}
2415	}
2416	}
2417	else
2418	PteDst.n.u1Write = 1; /** @todo r=bird: This doesn't make sense to me. */
2419
2420	PteDst.n.u1Dirty = 1;
2421	PteDst.n.u1Accessed = 1;
2422	PteDst.au32[0] &= ~PGM_PTFLAGS_TRACK_DIRTY;
2423	ASMAtomicWriteSize(pPteDst, PteDst.u);
2424	PGM_INVL_PG(pVCpu, GCPtrPage);
2425	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2426	}
2427
2428	# ifdef IN_RING0
2429	/* Check for stale TLB entry; only applies to the SMP guest case. */
2430	if ( pVM->cCpus > 1
2431	&& pPteDst->n.u1Write == 1
2432	&& pPteDst->n.u1Accessed == 1)
2433	{
2434	/* Stale TLB entry. */
2435	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageStale));
2436	PGM_INVL_PG(pVCpu, GCPtrPage);
2437	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2438	}
2439	# endif
2440	}
2441	}
2442	else
2443	AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
2444	}
2445
2446	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2447	}
2448
2449	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
2450
2451
2452	/**
2453	* Sync a shadow page table.
2454	*
2455	* The shadow page table is not present. This includes the case where
2456	* there is a conflict with a mapping.
2457	*
2458	* @returns VBox status code.
2459	* @param pVCpu The VMCPU handle.
2460	* @param iPD Page directory index.
2461	* @param pPDSrc Source page directory (i.e. Guest OS page directory).
2462	* Assume this is a temporary mapping.
2463	* @param GCPtrPage GC Pointer of the page that caused the fault
2464	*/
2465	static int PGM_BTH_NAME(SyncPT)(PVMCPU pVCpu, unsigned iPDSrc, PGSTPD pPDSrc, RTGCPTR GCPtrPage)
2466	{
2467	PVM pVM = pVCpu->CTX_SUFF(pVM);
2468	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2469
2470	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2471	#if 0 /* rarely useful; leave for debugging. */
2472	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPtPD[iPDSrc]);
2473	#endif
2474	LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage));
2475
2476	Assert(PGMIsLocked(pVM));
2477
2478	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2479	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
2480	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
2481	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2482	&& PGM_SHW_TYPE != PGM_TYPE_EPT
2483
2484	int rc = VINF_SUCCESS;
2485
2486	/*
2487	* Validate input a little bit.
2488	*/
2489	AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
2490	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2491	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2492	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
2493
2494	/* Fetch the pgm pool shadow descriptor. */
2495	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2496	Assert(pShwPde);
2497
2498	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2499	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2500	PPGMPOOLPAGE pShwPde = NULL;
2501	PX86PDPAE pPDDst;
2502	PSHWPDE pPdeDst;
2503
2504	/* Fetch the pgm pool shadow descriptor. */
2505	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
2506	AssertRCSuccessReturn(rc, rc);
2507	Assert(pShwPde);
2508
2509	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
2510	pPdeDst = &pPDDst->a[iPDDst];
2511
2512	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2513	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2514	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2515	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2516	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2517	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2518	AssertRCSuccessReturn(rc, rc);
2519	Assert(pPDDst);
2520	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2521	# endif
2522	SHWPDE PdeDst = *pPdeDst;
2523
2524	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2525	/* Fetch the pgm pool shadow descriptor. */
2526	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2527	Assert(pShwPde);
2528	# endif
2529
2530	# ifndef PGM_WITHOUT_MAPPINGS
2531	/*
2532	* Check for conflicts.
2533	* RC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
2534	* R3: Simply resolve the conflict.
2535	*/
2536	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2537	{
2538	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
2539	# ifndef IN_RING3
2540	Log(("SyncPT: Conflict at %RGv\n", GCPtrPage));
2541	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2542	return VERR_ADDRESS_CONFLICT;
2543
2544	# else /* IN_RING3 */
2545	PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
2546	Assert(pMapping);
2547	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2548	rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2549	# elif PGM_GST_TYPE == PGM_TYPE_PAE
2550	rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2551	# else
2552	AssertFailed(); /* can't happen for amd64 */
2553	# endif
2554	if (RT_FAILURE(rc))
2555	{
2556	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2557	return rc;
2558	}
2559	PdeDst = *pPdeDst;
2560	# endif /* IN_RING3 */
2561	}
2562	# endif /* !PGM_WITHOUT_MAPPINGS */
2563	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2564
2565	/*
2566	* Sync page directory entry.
2567	*/
2568	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2569	if (PdeSrc.n.u1Present)
2570	{
2571	/*
2572	* Allocate & map the page table.
2573	*/
2574	PSHWPT pPTDst;
2575	const bool fPageTable = !PdeSrc.b.u1Size \|\| !GST_IS_PSE_ACTIVE(pVCpu);
2576	PPGMPOOLPAGE pShwPage;
2577	RTGCPHYS GCPhys;
2578	if (fPageTable)
2579	{
2580	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
2581	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2582	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2583	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE / 2);
2584	# endif
2585	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
2586	}
2587	else
2588	{
2589	PGMPOOLACCESS enmAccess;
2590	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2591	const bool fNoExecute = PdeSrc.n.u1NoExecute && GST_IS_NX_ACTIVE(pVCpu);
2592	# else
2593	const bool fNoExecute = false;
2594	# endif
2595
2596	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(pVM, PdeSrc);
2597	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2598	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2599	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
2600	# endif
2601	/* Determine the right kind of large page to avoid incorrect cached entry reuse. */
2602	if (PdeSrc.n.u1User)
2603	{
2604	if (PdeSrc.n.u1Write)
2605	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_RW_NX : PGMPOOLACCESS_USER_RW;
2606	else
2607	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_R_NX : PGMPOOLACCESS_USER_R;
2608	}
2609	else
2610	{
2611	if (PdeSrc.n.u1Write)
2612	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW;
2613	else
2614	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_R_NX : PGMPOOLACCESS_SUPERVISOR_R;
2615	}
2616	rc = pgmPoolAllocEx(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, enmAccess, pShwPde->idx, iPDDst, &pShwPage);
2617	}
2618	if (rc == VINF_SUCCESS)
2619	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2620	else if (rc == VINF_PGM_CACHED_PAGE)
2621	{
2622	/*
2623	* The PT was cached, just hook it up.
2624	*/
2625	if (fPageTable)
2626	PdeDst.u = pShwPage->Core.Key
2627	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2628	else
2629	{
2630	PdeDst.u = pShwPage->Core.Key
2631	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2632	/* (see explanation and assumptions further down.) */
2633	if ( !PdeSrc.b.u1Dirty
2634	&& PdeSrc.b.u1Write)
2635	{
2636	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2637	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2638	PdeDst.b.u1Write = 0;
2639	}
2640	}
2641	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2642	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2643	return VINF_SUCCESS;
2644	}
2645	else if (rc == VERR_PGM_POOL_FLUSHED)
2646	{
2647	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2648	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2649	return VINF_PGM_SYNC_CR3;
2650	}
2651	else
2652	AssertMsgFailedReturn(("rc=%Rrc\n", rc), VERR_INTERNAL_ERROR);
2653	PdeDst.u &= X86_PDE_AVL_MASK;
2654	PdeDst.u \|= pShwPage->Core.Key;
2655
2656	/*
2657	* Page directory has been accessed (this is a fault situation, remember).
2658	*/
2659	pPDSrc->a[iPDSrc].n.u1Accessed = 1;
2660	if (fPageTable)
2661	{
2662	/*
2663	* Page table - 4KB.
2664	*
2665	* Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
2666	*/
2667	Log2(("SyncPT: 4K %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
2668	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
2669	PGSTPT pPTSrc;
2670	rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
2671	if (RT_SUCCESS(rc))
2672	{
2673	/*
2674	* Start by syncing the page directory entry so CSAM's TLB trick works.
2675	*/
2676	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| X86_PDE_AVL_MASK))
2677	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2678	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2679	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2680
2681	/*
2682	* Directory/page user or supervisor privilege: (same goes for read/write)
2683	*
2684	* Directory Page Combined
2685	* U/S U/S U/S
2686	* 0 0 0
2687	* 0 1 0
2688	* 1 0 0
2689	* 1 1 1
2690	*
2691	* Simple AND operation. Table listed for completeness.
2692	*
2693	*/
2694	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT4K));
2695	# ifdef PGM_SYNC_N_PAGES
2696	unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2697	unsigned iPTDst = iPTBase;
2698	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2699	if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
2700	iPTDst = 0;
2701	else
2702	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2703	# else /* !PGM_SYNC_N_PAGES */
2704	unsigned iPTDst = 0;
2705	const unsigned iPTDstEnd = RT_ELEMENTS(pPTDst->a);
2706	# endif /* !PGM_SYNC_N_PAGES */
2707	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2708	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2709	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2710	# else
2711	const unsigned offPTSrc = 0;
2712	# endif
2713	for (; iPTDst < iPTDstEnd; iPTDst++)
2714	{
2715	const unsigned iPTSrc = iPTDst + offPTSrc;
2716	const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2717
2718	if (PteSrc.n.u1Present)
2719	{
2720	# ifndef IN_RING0
2721	/*
2722	* Assuming kernel code will be marked as supervisor - and not as user level
2723	* and executed using a conforming code selector - And marked as readonly.
2724	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
2725	*/
2726	PPGMPAGE pPage;
2727	if ( ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW \| X86_PTE_US))
2728	\|\| !CSAMDoesPageNeedScanning(pVM, (iPDSrc << GST_PD_SHIFT) \| (iPTSrc << PAGE_SHIFT))
2729	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
2730	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2731	)
2732	# endif
2733	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2734	Log2(("SyncPT: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
2735	(RTGCPTR)(((RTGCPTR)iPDSrc << GST_PD_SHIFT) \| ((RTGCPTR)iPTSrc << PAGE_SHIFT)),
2736	PteSrc.n.u1Present,
2737	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2738	PteSrc.n.u1User & PdeSrc.n.u1User,
2739	(uint64_t)PteSrc.u,
2740	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : "", pPTDst->a[iPTDst].u, iPTSrc, PdeSrc.au32[0],
2741	(RTGCPHYS)((PdeSrc.u & GST_PDE_PG_MASK) + iPTSrc*sizeof(PteSrc)) ));
2742	}
2743	/* else: the page table was cleared by the pool */
2744	} /* for PTEs */
2745	}
2746	}
2747	else
2748	{
2749	/*
2750	* Big page - 2/4MB.
2751	*
2752	* We'll walk the ram range list in parallel and optimize lookups.
2753	* We will only sync on shadow page table at a time.
2754	*/
2755	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT4M));
2756
2757	/**
2758	* @todo It might be more efficient to sync only a part of the 4MB
2759	* page (similar to what we do for 4KB PDs).
2760	*/
2761
2762	/*
2763	* Start by syncing the page directory entry.
2764	*/
2765	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
2766	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2767
2768	/*
2769	* If the page is not flagged as dirty and is writable, then make it read-only
2770	* at PD level, so we can set the dirty bit when the page is modified.
2771	*
2772	* ASSUMES that page access handlers are implemented on page table entry level.
2773	* Thus we will first catch the dirty access and set PDE.D and restart. If
2774	* there is an access handler, we'll trap again and let it work on the problem.
2775	*/
2776	/** @todo move the above stuff to a section in the PGM documentation. */
2777	Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
2778	if ( !PdeSrc.b.u1Dirty
2779	&& PdeSrc.b.u1Write)
2780	{
2781	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2782	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2783	PdeDst.b.u1Write = 0;
2784	}
2785	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2786	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2787
2788	/*
2789	* Fill the shadow page table.
2790	*/
2791	/* Get address and flags from the source PDE. */
2792	SHWPTE PteDstBase;
2793	PteDstBase.u = PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT);
2794
2795	/* Loop thru the entries in the shadow PT. */
2796	const RTGCPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
2797	Log2(("SyncPT: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n",
2798	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
2799	GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2800	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
2801	unsigned iPTDst = 0;
2802	while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2803	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2804	{
2805	/* Advance ram range list. */
2806	while (pRam && GCPhys > pRam->GCPhysLast)
2807	pRam = pRam->CTX_SUFF(pNext);
2808	if (pRam && GCPhys >= pRam->GCPhys)
2809	{
2810	unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2811	do
2812	{
2813	/* Make shadow PTE. */
2814	PPGMPAGE pPage = &pRam->aPages[iHCPage];
2815	SHWPTE PteDst;
2816
2817	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2818	/* Try to make the page writable if necessary. */
2819	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2820	&& ( PGM_PAGE_IS_ZERO(pPage)
2821	\|\| ( PteDstBase.n.u1Write
2822	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2823	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
2824	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
2825	# endif
2826	# ifdef VBOX_WITH_PAGE_SHARING
2827	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
2828	# endif
2829	&& !PGM_PAGE_IS_BALLOONED(pPage))
2830	)
2831	)
2832	{
2833	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
2834	AssertRCReturn(rc, rc);
2835	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2836	break;
2837	}
2838	# endif
2839
2840	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2841	{
2842	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
2843	{
2844	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) \| PteDstBase.u;
2845	PteDst.n.u1Write = 0;
2846	}
2847	else
2848	PteDst.u = 0;
2849	}
2850	else if (PGM_PAGE_IS_BALLOONED(pPage))
2851	{
2852	/* Skip ballooned pages. */
2853	PteDst.u = 0;
2854	}
2855	# ifndef IN_RING0
2856	/*
2857	* Assuming kernel code will be marked as supervisor and not as user level and executed
2858	* using a conforming code selector. Don't check for readonly, as that implies the whole
2859	* 4MB can be code or readonly data. Linux enables write access for its large pages.
2860	*/
2861	else if ( !PdeSrc.n.u1User
2862	&& CSAMDoesPageNeedScanning(pVM, GCPtr \| (iPTDst << SHW_PT_SHIFT)))
2863	PteDst.u = 0;
2864	# endif
2865	else
2866	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) \| PteDstBase.u;
2867
2868	/* Only map writable pages writable. */
2869	if ( PteDst.n.u1Write
2870	&& PteDst.n.u1Present
2871	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2872	{
2873	/* Still applies to shared pages. */
2874	Assert(!PGM_PAGE_IS_ZERO(pPage));
2875	PteDst.n.u1Write = 0; /** @todo this isn't quite working yet... */
2876	Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))));
2877	}
2878
2879	if (PteDst.n.u1Present)
2880	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2881
2882	/* commit it */
2883	pPTDst->a[iPTDst] = PteDst;
2884	Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
2885	(RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT)), PteDst.n.u1Present, PteDst.n.u1Write, PteDst.n.u1User, (uint64_t)PteDst.u,
2886	PteDst.u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2887
2888	/* advance */
2889	GCPhys += PAGE_SIZE;
2890	iHCPage++;
2891	iPTDst++;
2892	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2893	&& GCPhys <= pRam->GCPhysLast);
2894	}
2895	else if (pRam)
2896	{
2897	Log(("Invalid pages at %RGp\n", GCPhys));
2898	do
2899	{
2900	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2901	GCPhys += PAGE_SIZE;
2902	iPTDst++;
2903	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2904	&& GCPhys < pRam->GCPhys);
2905	}
2906	else
2907	{
2908	Log(("Invalid pages at %RGp (2)\n", GCPhys));
2909	for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
2910	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2911	}
2912	} /* while more PTEs */
2913	} /* 4KB / 4MB */
2914	}
2915	else
2916	AssertRelease(!PdeDst.n.u1Present);
2917
2918	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2919	if (RT_FAILURE(rc))
2920	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPTFailed));
2921	return rc;
2922
2923	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2924	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2925	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2926	&& !defined(IN_RC)
2927
2928	/*
2929	* Validate input a little bit.
2930	*/
2931	int rc = VINF_SUCCESS;
2932	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2933	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2934	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
2935
2936	/* Fetch the pgm pool shadow descriptor. */
2937	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2938	Assert(pShwPde);
2939
2940	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2941	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2942	PPGMPOOLPAGE pShwPde = NULL; /* initialized to shut up gcc */
2943	PX86PDPAE pPDDst;
2944	PSHWPDE pPdeDst;
2945
2946	/* Fetch the pgm pool shadow descriptor. */
2947	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
2948	AssertRCSuccessReturn(rc, rc);
2949	Assert(pShwPde);
2950
2951	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
2952	pPdeDst = &pPDDst->a[iPDDst];
2953
2954	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2955	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2956	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2957	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2958	PX86PDPT pPdptDst= NULL; /* initialized to shut up gcc */
2959	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2960	AssertRCSuccessReturn(rc, rc);
2961	Assert(pPDDst);
2962	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2963
2964	/* Fetch the pgm pool shadow descriptor. */
2965	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2966	Assert(pShwPde);
2967
2968	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2969	const unsigned iPdpt = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
2970	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2971	PEPTPD pPDDst;
2972	PEPTPDPT pPdptDst;
2973
2974	rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, &pPdptDst, &pPDDst);
2975	if (rc != VINF_SUCCESS)
2976	{
2977	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2978	AssertRC(rc);
2979	return rc;
2980	}
2981	Assert(pPDDst);
2982	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2983
2984	/* Fetch the pgm pool shadow descriptor. */
2985	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & EPT_PDPTE_PG_MASK);
2986	Assert(pShwPde);
2987	# endif
2988	SHWPDE PdeDst = *pPdeDst;
2989
2990	Assert(!(PdeDst.u & PGM_PDFLAGS_MAPPING));
2991	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2992
2993	# if defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
2994	if (BTH_IS_NP_ACTIVE(pVM))
2995	{
2996	PPGMPAGE pPage;
2997
2998	/* Check if we allocated a big page before for this 2 MB range. */
2999	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPtrPage & X86_PDE2M_PAE_PG_MASK, &pPage);
3000	if (RT_SUCCESS(rc))
3001	{
3002	RTHCPHYS HCPhys = NIL_RTHCPHYS;
3003
3004	if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
3005	{
3006	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageReused);
3007	AssertRelease(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3008	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3009	}
3010	else if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED)
3011	{
3012	/* Recheck the entire 2 MB range to see if we can use it again as a large page. */
3013	rc = pgmPhysIsValidLargePage(pVM, GCPtrPage, pPage);
3014	if (RT_SUCCESS(rc))
3015	{
3016	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3017	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3018	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3019	}
3020	}
3021	else if (PGMIsUsingLargePages(pVM))
3022	{
3023	rc = pgmPhysAllocLargePage(pVM, GCPtrPage);
3024	if (RT_SUCCESS(rc))
3025	{
3026	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3027	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3028	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3029	}
3030	else
3031	LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
3032	}
3033
3034	if (HCPhys != NIL_RTHCPHYS)
3035	{
3036	PdeDst.u &= X86_PDE_AVL_MASK;
3037	PdeDst.u \|= HCPhys;
3038	PdeDst.n.u1Present = 1;
3039	PdeDst.n.u1Write = 1;
3040	PdeDst.b.u1Size = 1;
3041	# if PGM_SHW_TYPE == PGM_TYPE_EPT
3042	PdeDst.n.u1Execute = 1;
3043	PdeDst.b.u1IgnorePAT = 1;
3044	PdeDst.b.u3EMT = VMX_EPT_MEMTYPE_WB;
3045	# else
3046	PdeDst.n.u1User = 1;
3047	# endif
3048	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
3049
3050	Log(("SyncPT: Use large page at %RGp PDE=%RX64\n", GCPtrPage, PdeDst.u));
3051	/* Add a reference to the first page only. */
3052	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPde, PGM_PAGE_GET_TRACKING(pPage), pPage, iPDDst);
3053
3054	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3055	return VINF_SUCCESS;
3056	}
3057	}
3058	}
3059	# endif /* HC_ARCH_BITS == 64 */
3060
3061	GSTPDE PdeSrc;
3062	PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3063	PdeSrc.n.u1Present = 1;
3064	PdeSrc.n.u1Write = 1;
3065	PdeSrc.n.u1Accessed = 1;
3066	PdeSrc.n.u1User = 1;
3067
3068	/*
3069	* Allocate & map the page table.
3070	*/
3071	PSHWPT pPTDst;
3072	PPGMPOOLPAGE pShwPage;
3073	RTGCPHYS GCPhys;
3074
3075	/* Virtual address = physical address */
3076	GCPhys = GCPtrPage & X86_PAGE_4K_BASE_MASK;
3077	rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
3078
3079	if ( rc == VINF_SUCCESS
3080	\|\| rc == VINF_PGM_CACHED_PAGE)
3081	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3082	else
3083	AssertMsgFailedReturn(("rc=%Rrc\n", rc), VERR_INTERNAL_ERROR);
3084
3085	PdeDst.u &= X86_PDE_AVL_MASK;
3086	PdeDst.u \|= pShwPage->Core.Key;
3087	PdeDst.n.u1Present = 1;
3088	PdeDst.n.u1Write = 1;
3089	# if PGM_SHW_TYPE == PGM_TYPE_EPT
3090	PdeDst.n.u1Execute = 1;
3091	# else
3092	PdeDst.n.u1User = 1;
3093	PdeDst.n.u1Accessed = 1;
3094	# endif
3095	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
3096
3097	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, PGM_SYNC_NR_PAGES, 0 /* page not present */);
3098	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3099	return rc;
3100
3101	#else
3102	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
3103	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3104	return VERR_INTERNAL_ERROR;
3105	#endif
3106	}
3107
3108
3109
3110	/**
3111	* Prefetch a page/set of pages.
3112	*
3113	* Typically used to sync commonly used pages before entering raw mode
3114	* after a CR3 reload.
3115	*
3116	* @returns VBox status code.
3117	* @param pVCpu The VMCPU handle.
3118	* @param GCPtrPage Page to invalidate.
3119	*/
3120	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
3121	{
3122	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3123	\|\| PGM_GST_TYPE == PGM_TYPE_REAL \
3124	\|\| PGM_GST_TYPE == PGM_TYPE_PROT \
3125	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3126	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3127	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
3128	&& PGM_SHW_TYPE != PGM_TYPE_EPT
3129
3130	/*
3131	* Check that all Guest levels thru the PDE are present, getting the
3132	* PD and PDE in the processes.
3133	*/
3134	int rc = VINF_SUCCESS;
3135	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3136	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3137	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
3138	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3139	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3140	unsigned iPDSrc;
3141	X86PDPE PdpeSrc;
3142	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3143	if (!pPDSrc)
3144	return VINF_SUCCESS; /* not present */
3145	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3146	unsigned iPDSrc;
3147	PX86PML4E pPml4eSrc;
3148	X86PDPE PdpeSrc;
3149	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3150	if (!pPDSrc)
3151	return VINF_SUCCESS; /* not present */
3152	# endif
3153	const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3154	# else
3155	PGSTPD pPDSrc = NULL;
3156	const unsigned iPDSrc = 0;
3157	GSTPDE PdeSrc;
3158
3159	PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3160	PdeSrc.n.u1Present = 1;
3161	PdeSrc.n.u1Write = 1;
3162	PdeSrc.n.u1Accessed = 1;
3163	PdeSrc.n.u1User = 1;
3164	# endif
3165
3166	if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
3167	{
3168	PVM pVM = pVCpu->CTX_SUFF(pVM);
3169	pgmLock(pVM);
3170
3171	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3172	const X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
3173	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3174	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3175	PX86PDPAE pPDDst;
3176	X86PDEPAE PdeDst;
3177	# if PGM_GST_TYPE != PGM_TYPE_PAE
3178	X86PDPE PdpeSrc;
3179
3180	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3181	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3182	# endif
3183	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3184	if (rc != VINF_SUCCESS)
3185	{
3186	pgmUnlock(pVM);
3187	AssertRC(rc);
3188	return rc;
3189	}
3190	Assert(pPDDst);
3191	PdeDst = pPDDst->a[iPDDst];
3192
3193	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3194	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3195	PX86PDPAE pPDDst;
3196	X86PDEPAE PdeDst;
3197
3198	# if PGM_GST_TYPE == PGM_TYPE_PROT
3199	/* AMD-V nested paging */
3200	X86PML4E Pml4eSrc;
3201	X86PDPE PdpeSrc;
3202	PX86PML4E pPml4eSrc = &Pml4eSrc;
3203
3204	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3205	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
3206	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
3207	# endif
3208
3209	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3210	if (rc != VINF_SUCCESS)
3211	{
3212	pgmUnlock(pVM);
3213	AssertRC(rc);
3214	return rc;
3215	}
3216	Assert(pPDDst);
3217	PdeDst = pPDDst->a[iPDDst];
3218	# endif
3219	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
3220	{
3221	if (!PdeDst.n.u1Present)
3222	{
3223	/** @todo r=bird: This guy will set the A bit on the PDE,
3224	* probably harmless. */
3225	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3226	}
3227	else
3228	{
3229	/* Note! We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
3230	* R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
3231	* makes no sense to prefetch more than one page.
3232	*/
3233	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3234	if (RT_SUCCESS(rc))
3235	rc = VINF_SUCCESS;
3236	}
3237	}
3238	pgmUnlock(pVM);
3239	}
3240	return rc;
3241
3242	#elif PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3243	return VINF_SUCCESS; /* ignore */
3244	#else
3245	AssertCompile(0);
3246	#endif
3247	}
3248
3249
3250
3251
3252	/**
3253	* Syncs a page during a PGMVerifyAccess() call.
3254	*
3255	* @returns VBox status code (informational included).
3256	* @param pVCpu The VMCPU handle.
3257	* @param GCPtrPage The address of the page to sync.
3258	* @param fPage The effective guest page flags.
3259	* @param uErr The trap error code.
3260	* @remarks This will normally never be called on invalid guest page
3261	* translation entries.
3262	*/
3263	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fPage, unsigned uErr)
3264	{
3265	PVM pVM = pVCpu->CTX_SUFF(pVM);
3266
3267	LogFlow(("VerifyAccessSyncPage: GCPtrPage=%RGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
3268
3269	Assert(!pVM->pgm.s.fNestedPaging);
3270	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3271	\|\| PGM_GST_TYPE == PGM_TYPE_REAL \
3272	\|\| PGM_GST_TYPE == PGM_TYPE_PROT \
3273	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3274	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3275	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
3276	&& PGM_SHW_TYPE != PGM_TYPE_EPT
3277
3278	# ifndef IN_RING0
3279	if (!(fPage & X86_PTE_US))
3280	{
3281	/*
3282	* Mark this page as safe.
3283	*/
3284	/** @todo not correct for pages that contain both code and data!! */
3285	Log(("CSAMMarkPage %RGv; scanned=%d\n", GCPtrPage, true));
3286	CSAMMarkPage(pVM, GCPtrPage, true);
3287	}
3288	# endif
3289
3290	/*
3291	* Get guest PD and index.
3292	*/
3293	/** @todo Performance: We've done all this a jiffy ago in the
3294	* PGMGstGetPage call. */
3295	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3296	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3297	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
3298	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3299
3300	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3301	unsigned iPDSrc = 0;
3302	X86PDPE PdpeSrc;
3303	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3304	if (RT_UNLIKELY(!pPDSrc))
3305	{
3306	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3307	return VINF_EM_RAW_GUEST_TRAP;
3308	}
3309
3310	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3311	unsigned iPDSrc = 0; /* shut up gcc */
3312	PX86PML4E pPml4eSrc = NULL; /* ditto */
3313	X86PDPE PdpeSrc;
3314	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3315	if (RT_UNLIKELY(!pPDSrc))
3316	{
3317	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3318	return VINF_EM_RAW_GUEST_TRAP;
3319	}
3320	# endif
3321
3322	# else /* !PGM_WITH_PAGING */
3323	PGSTPD pPDSrc = NULL;
3324	const unsigned iPDSrc = 0;
3325	# endif /* !PGM_WITH_PAGING */
3326	int rc = VINF_SUCCESS;
3327
3328	pgmLock(pVM);
3329
3330	/*
3331	* First check if the shadow pd is present.
3332	*/
3333	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3334	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3335
3336	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3337	PX86PDEPAE pPdeDst;
3338	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3339	PX86PDPAE pPDDst;
3340	# if PGM_GST_TYPE != PGM_TYPE_PAE
3341	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3342	X86PDPE PdpeSrc;
3343	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3344	# endif
3345	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3346	if (rc != VINF_SUCCESS)
3347	{
3348	pgmUnlock(pVM);
3349	AssertRC(rc);
3350	return rc;
3351	}
3352	Assert(pPDDst);
3353	pPdeDst = &pPDDst->a[iPDDst];
3354
3355	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3356	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3357	PX86PDPAE pPDDst;
3358	PX86PDEPAE pPdeDst;
3359
3360	# if PGM_GST_TYPE == PGM_TYPE_PROT
3361	/* AMD-V nested paging: Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3362	X86PML4E Pml4eSrc;
3363	X86PDPE PdpeSrc;
3364	PX86PML4E pPml4eSrc = &Pml4eSrc;
3365	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
3366	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
3367	# endif
3368
3369	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3370	if (rc != VINF_SUCCESS)
3371	{
3372	pgmUnlock(pVM);
3373	AssertRC(rc);
3374	return rc;
3375	}
3376	Assert(pPDDst);
3377	pPdeDst = &pPDDst->a[iPDDst];
3378	# endif
3379
3380	if (!pPdeDst->n.u1Present)
3381	{
3382	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3383	if (rc != VINF_SUCCESS)
3384	{
3385	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3386	pgmUnlock(pVM);
3387	AssertRC(rc);
3388	return rc;
3389	}
3390	}
3391
3392	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3393	/* Check for dirty bit fault */
3394	rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
3395	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
3396	Log(("PGMVerifyAccess: success (dirty)\n"));
3397	else
3398	# endif
3399	{
3400	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3401	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3402	# else
3403	GSTPDE PdeSrc;
3404	PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3405	PdeSrc.n.u1Present = 1;
3406	PdeSrc.n.u1Write = 1;
3407	PdeSrc.n.u1Accessed = 1;
3408	PdeSrc.n.u1User = 1;
3409	# endif
3410
3411	Assert(rc != VINF_EM_RAW_GUEST_TRAP);
3412	if (uErr & X86_TRAP_PF_US)
3413	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUser));
3414	else /* supervisor */
3415	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
3416
3417	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3418	if (RT_SUCCESS(rc))
3419	{
3420	/* Page was successfully synced */
3421	Log2(("PGMVerifyAccess: success (sync)\n"));
3422	rc = VINF_SUCCESS;
3423	}
3424	else
3425	{
3426	Log(("PGMVerifyAccess: access violation for %RGv rc=%Rrc\n", GCPtrPage, rc));
3427	rc = VINF_EM_RAW_GUEST_TRAP;
3428	}
3429	}
3430	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3431	pgmUnlock(pVM);
3432	return rc;
3433
3434	#else /* PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED */
3435
3436	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
3437	return VERR_INTERNAL_ERROR;
3438	#endif /* PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED */
3439	}
3440
3441
3442	/**
3443	* Syncs the paging hierarchy starting at CR3.
3444	*
3445	* @returns VBox status code, no specials.
3446	* @param pVCpu The VMCPU handle.
3447	* @param cr0 Guest context CR0 register
3448	* @param cr3 Guest context CR3 register
3449	* @param cr4 Guest context CR4 register
3450	* @param fGlobal Including global page directories or not
3451	*/
3452	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
3453	{
3454	PVM pVM = pVCpu->CTX_SUFF(pVM);
3455
3456	LogFlow(("SyncCR3 fGlobal=%d\n", !!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)));
3457
3458	#if PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
3459
3460	pgmLock(pVM);
3461
3462	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
3463	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3464	if (pPool->cDirtyPages)
3465	pgmPoolResetDirtyPages(pVM);
3466	# endif
3467
3468	/*
3469	* Update page access handlers.
3470	* The virtual are always flushed, while the physical are only on demand.
3471	* WARNING: We are incorrectly not doing global flushing on Virtual Handler updates. We'll
3472	* have to look into that later because it will have a bad influence on the performance.
3473	* @note SvL: There's no need for that. Just invalidate the virtual range(s).
3474	* bird: Yes, but that won't work for aliases.
3475	*/
3476	/** @todo this MUST go away. See #1557. */
3477	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3Handlers), h);
3478	PGM_GST_NAME(HandlerVirtualUpdate)(pVM, cr4);
3479	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3Handlers), h);
3480	pgmUnlock(pVM);
3481	#endif /* !NESTED && !EPT */
3482
3483	#if PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3484	/*
3485	* Nested / EPT - almost no work.
3486	*/
3487	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
3488	return VINF_SUCCESS;
3489
3490	#elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3491	/*
3492	* AMD64 (Shw & Gst) - No need to check all paging levels; we zero
3493	* out the shadow parts when the guest modifies its tables.
3494	*/
3495	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
3496	return VINF_SUCCESS;
3497
3498	#else /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3499
3500	# ifndef PGM_WITHOUT_MAPPINGS
3501	/*
3502	* Check for and resolve conflicts with our guest mappings if they
3503	* are enabled and not fixed.
3504	*/
3505	if (pgmMapAreMappingsFloating(&pVM->pgm.s))
3506	{
3507	int rc = pgmMapResolveConflicts(pVM);
3508	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
3509	if (rc == VINF_PGM_SYNC_CR3)
3510	{
3511	LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
3512	return VINF_PGM_SYNC_CR3;
3513	}
3514	}
3515	# else
3516	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
3517	# endif
3518	return VINF_SUCCESS;
3519	#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3520	}
3521
3522
3523
3524
3525	#ifdef VBOX_STRICT
3526	#ifdef IN_RC
3527	# undef AssertMsgFailed
3528	# define AssertMsgFailed Log
3529	#endif
3530	#ifdef IN_RING3
3531	# include <VBox/dbgf.h>
3532
3533	/**
3534	* Dumps a page table hierarchy use only physical addresses and cr4/lm flags.
3535	*
3536	* @returns VBox status code (VINF_SUCCESS).
3537	* @param cr3 The root of the hierarchy.
3538	* @param crr The cr4, only PAE and PSE is currently used.
3539	* @param fLongMode Set if long mode, false if not long mode.
3540	* @param cMaxDepth Number of levels to dump.
3541	* @param pHlp Pointer to the output functions.
3542	*/
3543	RT_C_DECLS_BEGIN
3544	VMMR3DECL(int) PGMR3DumpHierarchyHC(PVM pVM, uint32_t cr3, uint32_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp);
3545	RT_C_DECLS_END
3546
3547	#endif
3548
3549	/**
3550	* Checks that the shadow page table is in sync with the guest one.
3551	*
3552	* @returns The number of errors.
3553	* @param pVM The virtual machine.
3554	* @param pVCpu The VMCPU handle.
3555	* @param cr3 Guest context CR3 register
3556	* @param cr4 Guest context CR4 register
3557	* @param GCPtr Where to start. Defaults to 0.
3558	* @param cb How much to check. Defaults to everything.
3559	*/
3560	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)
3561	{
3562	#if PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3563	return 0;
3564	#else
3565	unsigned cErrors = 0;
3566	PVM pVM = pVCpu->CTX_SUFF(pVM);
3567	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3568
3569	#if PGM_GST_TYPE == PGM_TYPE_PAE
3570	/** @todo currently broken; crashes below somewhere */
3571	AssertFailed();
3572	#endif
3573
3574	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
3575	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3576	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
3577
3578	bool fBigPagesSupported = GST_IS_PSE_ACTIVE(pVCpu);
3579	PPGMCPU pPGM = &pVCpu->pgm.s;
3580	RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
3581	RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
3582	# ifndef IN_RING0
3583	RTHCPHYS HCPhys; /* general usage. */
3584	# endif
3585	int rc;
3586
3587	/*
3588	* Check that the Guest CR3 and all its mappings are correct.
3589	*/
3590	AssertMsgReturn(pPGM->GCPhysCR3 == (cr3 & GST_CR3_PAGE_MASK),
3591	("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
3592	false);
3593	# if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64
3594	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3595	rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGst32BitPdRC, NULL, &HCPhysShw);
3596	# else
3597	rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGstPaePdptRC, NULL, &HCPhysShw);
3598	# endif
3599	AssertRCReturn(rc, 1);
3600	HCPhys = NIL_RTHCPHYS;
3601	rc = pgmRamGCPhys2HCPhys(&pVM->pgm.s, cr3 & GST_CR3_PAGE_MASK, &HCPhys);
3602	AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
3603	# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
3604	pgmGstGet32bitPDPtr(pVCpu);
3605	RTGCPHYS GCPhys;
3606	rc = PGMR3DbgR3Ptr2GCPhys(pVM, pPGM->pGst32BitPdR3, &GCPhys);
3607	AssertRCReturn(rc, 1);
3608	AssertMsgReturn((cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
3609	# endif
3610	# endif /* !IN_RING0 */
3611
3612	/*
3613	* Get and check the Shadow CR3.
3614	*/
3615	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3616	unsigned cPDEs = X86_PG_ENTRIES;
3617	unsigned cIncrement = X86_PG_ENTRIES * PAGE_SIZE;
3618	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3619	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3620	unsigned cPDEs = X86_PG_PAE_ENTRIES * 4; /* treat it as a 2048 entry table. */
3621	# else
3622	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3623	# endif
3624	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3625	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3626	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3627	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3628	# endif
3629	if (cb != ~(RTGCPTR)0)
3630	cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
3631
3632	/** @todo call the other two PGMAssert() functions. /
3633
3634	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3635	unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3636
3637	for (; iPml4 < X86_PG_PAE_ENTRIES; iPml4++)
3638	{
3639	PPGMPOOLPAGE pShwPdpt = NULL;
3640	PX86PML4E pPml4eSrc;
3641	PX86PML4E pPml4eDst;
3642	RTGCPHYS GCPhysPdptSrc;
3643
3644	pPml4eSrc = pgmGstGetLongModePML4EPtr(pVCpu, iPml4);
3645	pPml4eDst = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
3646
3647	/* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
3648	if (!pPml4eDst->n.u1Present)
3649	{
3650	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3651	continue;
3652	}
3653
3654	pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
3655	GCPhysPdptSrc = pPml4eSrc->u & X86_PML4E_PG_MASK_FULL;
3656
3657	if (pPml4eSrc->n.u1Present != pPml4eDst->n.u1Present)
3658	{
3659	AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3660	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3661	cErrors++;
3662	continue;
3663	}
3664
3665	if (GCPhysPdptSrc != pShwPdpt->GCPhys)
3666	{
3667	AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
3668	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3669	cErrors++;
3670	continue;
3671	}
3672
3673	if ( pPml4eDst->n.u1User != pPml4eSrc->n.u1User
3674	\|\| pPml4eDst->n.u1Write != pPml4eSrc->n.u1Write
3675	\|\| pPml4eDst->n.u1NoExecute != pPml4eSrc->n.u1NoExecute)
3676	{
3677	AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3678	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3679	cErrors++;
3680	continue;
3681	}
3682	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3683	{
3684	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3685
3686	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
3687	/*
3688	* Check the PDPTEs too.
3689	*/
3690	unsigned iPdpt = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
3691
3692	for (;iPdpt <= SHW_PDPT_MASK; iPdpt++)
3693	{
3694	unsigned iPDSrc = 0; /* initialized to shut up gcc */
3695	PPGMPOOLPAGE pShwPde = NULL;
3696	PX86PDPE pPdpeDst;
3697	RTGCPHYS GCPhysPdeSrc;
3698	# if PGM_GST_TYPE == PGM_TYPE_PAE
3699	X86PDPE PdpeSrc;
3700	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtr, &iPDSrc, &PdpeSrc);
3701	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
3702	# else
3703	PX86PML4E pPml4eSrcIgn;
3704	X86PDPE PdpeSrc;
3705	PX86PDPT pPdptDst;
3706	PX86PDPAE pPDDst;
3707	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtr, &pPml4eSrcIgn, &PdpeSrc, &iPDSrc);
3708
3709	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtr, NULL, &pPdptDst, &pPDDst);
3710	if (rc != VINF_SUCCESS)
3711	{
3712	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
3713	GCPtr += 512 * _2M;
3714	continue; /* next PDPTE */
3715	}
3716	Assert(pPDDst);
3717	# endif
3718	Assert(iPDSrc == 0);
3719
3720	pPdpeDst = &pPdptDst->a[iPdpt];
3721
3722	if (!pPdpeDst->n.u1Present)
3723	{
3724	GCPtr += 512 * _2M;
3725	continue; /* next PDPTE */
3726	}
3727
3728	pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
3729	GCPhysPdeSrc = PdpeSrc.u & X86_PDPE_PG_MASK;
3730
3731	if (pPdpeDst->n.u1Present != PdpeSrc.n.u1Present)
3732	{
3733	AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3734	GCPtr += 512 * _2M;
3735	cErrors++;
3736	continue;
3737	}
3738
3739	if (GCPhysPdeSrc != pShwPde->GCPhys)
3740	{
3741	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3742	AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3743	# else
3744	AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3745	# endif
3746	GCPtr += 512 * _2M;
3747	cErrors++;
3748	continue;
3749	}
3750
3751	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3752	if ( pPdpeDst->lm.u1User != PdpeSrc.lm.u1User
3753	\|\| pPdpeDst->lm.u1Write != PdpeSrc.lm.u1Write
3754	\|\| pPdpeDst->lm.u1NoExecute != PdpeSrc.lm.u1NoExecute)
3755	{
3756	AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3757	GCPtr += 512 * _2M;
3758	cErrors++;
3759	continue;
3760	}
3761	# endif
3762
3763	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3764	{
3765	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3766	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3767	GSTPD const *pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3768	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3769	PCX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
3770	# endif
3771	# endif /* PGM_GST_TYPE == PGM_TYPE_32BIT */
3772	/*
3773	* Iterate the shadow page directory.
3774	*/
3775	GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
3776	unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
3777
3778	for (;
3779	iPDDst < cPDEs;
3780	iPDDst++, GCPtr += cIncrement)
3781	{
3782	# if PGM_SHW_TYPE == PGM_TYPE_PAE
3783	const SHWPDE PdeDst = *pgmShwGetPaePDEPtr(pVCpu, GCPtr);
3784	# else
3785	const SHWPDE PdeDst = pPDDst->a[iPDDst];
3786	# endif
3787	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
3788	{
3789	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
3790	if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
3791	{
3792	AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
3793	cErrors++;
3794	continue;
3795	}
3796	}
3797	else if ( (PdeDst.u & X86_PDE_P)
3798	\|\| ((PdeDst.u & (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY))
3799	)
3800	{
3801	HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
3802	PPGMPOOLPAGE pPoolPage = pgmPoolGetPage(pPool, HCPhysShw);
3803	if (!pPoolPage)
3804	{
3805	AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n",
3806	HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
3807	cErrors++;
3808	continue;
3809	}
3810	const SHWPT pPTDst = (const SHWPT )PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pPoolPage);
3811
3812	if (PdeDst.u & (X86_PDE4M_PWT \| X86_PDE4M_PCD))
3813	{
3814	AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
3815	GCPtr, (uint64_t)PdeDst.u));
3816	cErrors++;
3817	}
3818
3819	if (PdeDst.u & (X86_PDE4M_G \| X86_PDE4M_D))
3820	{
3821	AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n",
3822	GCPtr, (uint64_t)PdeDst.u));
3823	cErrors++;
3824	}
3825
3826	const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
3827	if (!PdeSrc.n.u1Present)
3828	{
3829	AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
3830	GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
3831	cErrors++;
3832	continue;
3833	}
3834
3835	if ( !PdeSrc.b.u1Size
3836	\|\| !fBigPagesSupported)
3837	{
3838	GCPhysGst = PdeSrc.u & GST_PDE_PG_MASK;
3839	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3840	GCPhysGst \|= (iPDDst & 1) * (PAGE_SIZE / 2);
3841	# endif
3842	}
3843	else
3844	{
3845	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3846	if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
3847	{
3848	AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n",
3849	GCPtr, (uint64_t)PdeSrc.u));
3850	cErrors++;
3851	continue;
3852	}
3853	# endif
3854	GCPhysGst = GST_GET_PDE_BIG_PG_GCPHYS(pVM, PdeSrc);
3855	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3856	GCPhysGst \|= GCPtr & RT_BIT(X86_PAGE_2M_SHIFT);
3857	# endif
3858	}
3859
3860	if ( pPoolPage->enmKind
3861	!= (!PdeSrc.b.u1Size \|\| !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
3862	{
3863	AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n",
3864	pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
3865	cErrors++;
3866	}
3867
3868	PPGMPAGE pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
3869	if (!pPhysPage)
3870	{
3871	AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3872	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3873	cErrors++;
3874	continue;
3875	}
3876
3877	if (GCPhysGst != pPoolPage->GCPhys)
3878	{
3879	AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n",
3880	GCPhysGst, pPoolPage->GCPhys, GCPtr));
3881	cErrors++;
3882	continue;
3883	}
3884
3885	if ( !PdeSrc.b.u1Size
3886	\|\| !fBigPagesSupported)
3887	{
3888	/*
3889	* Page Table.
3890	*/
3891	const GSTPT *pPTSrc;
3892	rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1), &pPTSrc);
3893	if (RT_FAILURE(rc))
3894	{
3895	AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3896	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3897	cErrors++;
3898	continue;
3899	}
3900	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/))
3901	!= (PdeDst.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/)))
3902	{
3903	/// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
3904	// (This problem will go away when/if we shadow multiple CR3s.)
3905	AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3906	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3907	cErrors++;
3908	continue;
3909	}
3910	if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3911	{
3912	AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n",
3913	GCPtr, (uint64_t)PdeDst.u));
3914	cErrors++;
3915	continue;
3916	}
3917
3918	/* iterate the page table. */
3919	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3920	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3921	const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
3922	# else
3923	const unsigned offPTSrc = 0;
3924	# endif
3925	for (unsigned iPT = 0, off = 0;
3926	iPT < RT_ELEMENTS(pPTDst->a);
3927	iPT++, off += PAGE_SIZE)
3928	{
3929	const SHWPTE PteDst = pPTDst->a[iPT];
3930
3931	/* skip not-present entries. */
3932	if (!(PteDst.u & (X86_PTE_P \| PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
3933	continue;
3934	Assert(PteDst.n.u1Present);
3935
3936	const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
3937	if (!PteSrc.n.u1Present)
3938	{
3939	# ifdef IN_RING3
3940	PGMAssertHandlerAndFlagsInSync(pVM);
3941	PGMR3DumpHierarchyGC(pVM, cr3, cr4, (PdeSrc.u & GST_PDE_PG_MASK));
3942	# endif
3943	AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
3944	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u, pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
3945	(PdeSrc.u & GST_PDE_PG_MASK) + (iPT + offPTSrc)*sizeof(PteSrc)));
3946	cErrors++;
3947	continue;
3948	}
3949
3950	uint64_t fIgnoreFlags = GST_PTE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_G \| X86_PTE_D \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
3951	# if 1 /** @todo sync accessed bit properly... */
3952	fIgnoreFlags \|= X86_PTE_A;
3953	# endif
3954
3955	/* match the physical addresses */
3956	HCPhysShw = PteDst.u & SHW_PTE_PG_MASK;
3957	GCPhysGst = PteSrc.u & GST_PTE_PG_MASK;
3958
3959	# ifdef IN_RING3
3960	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3961	if (RT_FAILURE(rc))
3962	{
3963	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
3964	{
3965	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3966	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3967	cErrors++;
3968	continue;
3969	}
3970	}
3971	else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
3972	{
3973	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3974	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3975	cErrors++;
3976	continue;
3977	}
3978	# endif
3979
3980	pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
3981	if (!pPhysPage)
3982	{
3983	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
3984	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
3985	{
3986	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3987	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3988	cErrors++;
3989	continue;
3990	}
3991	# endif
3992	if (PteDst.n.u1Write)
3993	{
3994	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3995	GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3996	cErrors++;
3997	}
3998	fIgnoreFlags \|= X86_PTE_RW;
3999	}
4000	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4001	{
4002	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4003	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4004	cErrors++;
4005	continue;
4006	}
4007
4008	/* flags */
4009	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4010	{
4011	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4012	{
4013	if (PteDst.n.u1Write)
4014	{
4015	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4016	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4017	cErrors++;
4018	continue;
4019	}
4020	fIgnoreFlags \|= X86_PTE_RW;
4021	}
4022	else
4023	{
4024	if ( PteDst.n.u1Present
4025	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
4026	&& !PGM_PAGE_IS_MMIO(pPhysPage)
4027	# endif
4028	)
4029	{
4030	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4031	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4032	cErrors++;
4033	continue;
4034	}
4035	fIgnoreFlags \|= X86_PTE_P;
4036	}
4037	}
4038	else
4039	{
4040	if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
4041	{
4042	if (PteDst.n.u1Write)
4043	{
4044	AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
4045	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4046	cErrors++;
4047	continue;
4048	}
4049	if (!(PteDst.u & PGM_PTFLAGS_TRACK_DIRTY))
4050	{
4051	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4052	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4053	cErrors++;
4054	continue;
4055	}
4056	if (PteDst.n.u1Dirty)
4057	{
4058	AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4059	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4060	cErrors++;
4061	}
4062	# if 0 /** @todo sync access bit properly... */
4063	if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
4064	{
4065	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4066	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4067	cErrors++;
4068	}
4069	fIgnoreFlags \|= X86_PTE_RW;
4070	# else
4071	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
4072	# endif
4073	}
4074	else if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
4075	{
4076	/* access bit emulation (not implemented). */
4077	if (PteSrc.n.u1Accessed \|\| PteDst.n.u1Present)
4078	{
4079	AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
4080	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4081	cErrors++;
4082	continue;
4083	}
4084	if (!PteDst.n.u1Accessed)
4085	{
4086	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
4087	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4088	cErrors++;
4089	}
4090	fIgnoreFlags \|= X86_PTE_P;
4091	}
4092	# ifdef DEBUG_sandervl
4093	fIgnoreFlags \|= X86_PTE_D \| X86_PTE_A;
4094	# endif
4095	}
4096
4097	if ( (PteSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
4098	&& (PteSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags)
4099	)
4100	{
4101	AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
4102	GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
4103	fIgnoreFlags, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4104	cErrors++;
4105	continue;
4106	}
4107	} /* foreach PTE */
4108	}
4109	else
4110	{
4111	/*
4112	* Big Page.
4113	*/
4114	uint64_t fIgnoreFlags = X86_PDE_AVL_MASK \| GST_PDE_PG_MASK \| X86_PDE4M_G \| X86_PDE4M_D \| X86_PDE4M_PS \| X86_PDE4M_PWT \| X86_PDE4M_PCD;
4115	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
4116	{
4117	if (PdeDst.n.u1Write)
4118	{
4119	AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4120	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4121	cErrors++;
4122	continue;
4123	}
4124	if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
4125	{
4126	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4127	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4128	cErrors++;
4129	continue;
4130	}
4131	# if 0 /** @todo sync access bit properly... */
4132	if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
4133	{
4134	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4135	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4136	cErrors++;
4137	}
4138	fIgnoreFlags \|= X86_PTE_RW;
4139	# else
4140	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
4141	# endif
4142	}
4143	else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4144	{
4145	/* access bit emulation (not implemented). */
4146	if (PdeSrc.b.u1Accessed \|\| PdeDst.n.u1Present)
4147	{
4148	AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4149	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4150	cErrors++;
4151	continue;
4152	}
4153	if (!PdeDst.n.u1Accessed)
4154	{
4155	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4156	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4157	cErrors++;
4158	}
4159	fIgnoreFlags \|= X86_PTE_P;
4160	}
4161
4162	if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
4163	{
4164	AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
4165	GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
4166	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4167	cErrors++;
4168	}
4169
4170	/* iterate the page table. */
4171	for (unsigned iPT = 0, off = 0;
4172	iPT < RT_ELEMENTS(pPTDst->a);
4173	iPT++, off += PAGE_SIZE, GCPhysGst += PAGE_SIZE)
4174	{
4175	const SHWPTE PteDst = pPTDst->a[iPT];
4176
4177	if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
4178	{
4179	AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
4180	GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4181	cErrors++;
4182	}
4183
4184	/* skip not-present entries. */
4185	if (!PteDst.n.u1Present) /** @todo deal with ALL handlers and CSAM !P pages! */
4186	continue;
4187
4188	fIgnoreFlags = X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT \| X86_PTE_D \| X86_PTE_A \| X86_PTE_G \| X86_PTE_PAE_NX;
4189
4190	/* match the physical addresses */
4191	HCPhysShw = PteDst.u & X86_PTE_PAE_PG_MASK;
4192
4193	# ifdef IN_RING3
4194	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4195	if (RT_FAILURE(rc))
4196	{
4197	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4198	{
4199	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4200	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4201	cErrors++;
4202	}
4203	}
4204	else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
4205	{
4206	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4207	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4208	cErrors++;
4209	continue;
4210	}
4211	# endif
4212	pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
4213	if (!pPhysPage)
4214	{
4215	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4216	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4217	{
4218	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4219	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4220	cErrors++;
4221	continue;
4222	}
4223	# endif
4224	if (PteDst.n.u1Write)
4225	{
4226	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4227	GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4228	cErrors++;
4229	}
4230	fIgnoreFlags \|= X86_PTE_RW;
4231	}
4232	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4233	{
4234	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4235	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4236	cErrors++;
4237	continue;
4238	}
4239
4240	/* flags */
4241	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4242	{
4243	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4244	{
4245	if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
4246	{
4247	if (PteDst.n.u1Write)
4248	{
4249	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4250	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4251	cErrors++;
4252	continue;
4253	}
4254	fIgnoreFlags \|= X86_PTE_RW;
4255	}
4256	}
4257	else
4258	{
4259	if ( PteDst.n.u1Present
4260	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
4261	&& !PGM_PAGE_IS_MMIO(pPhysPage)
4262	# endif
4263	)
4264	{
4265	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4266	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4267	cErrors++;
4268	continue;
4269	}
4270	fIgnoreFlags \|= X86_PTE_P;
4271	}
4272	}
4273
4274	if ( (PdeSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
4275	&& (PdeSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags) /* lazy phys handler dereg. */
4276	)
4277	{
4278	AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
4279	GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
4280	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4281	cErrors++;
4282	continue;
4283	}
4284	} /* for each PTE */
4285	}
4286	}
4287	/* not present */
4288
4289	} /* for each PDE */
4290
4291	} /* for each PDPTE */
4292
4293	} /* for each PML4E */
4294
4295	# ifdef DEBUG
4296	if (cErrors)
4297	LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
4298	# endif
4299
4300	#endif /* GST == 32BIT, PAE or AMD64 */
4301	return cErrors;
4302
4303	#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT */
4304	}
4305	#endif /* VBOX_STRICT */
4306
4307
4308	/**
4309	* Sets up the CR3 for shadow paging
4310	*
4311	* @returns Strict VBox status code.
4312	* @retval VINF_SUCCESS.
4313	*
4314	* @param pVCpu The VMCPU handle.
4315	* @param GCPhysCR3 The physical address in the CR3 register.
4316	*/
4317	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3)
4318	{
4319	PVM pVM = pVCpu->CTX_SUFF(pVM);
4320
4321	/* Update guest paging info. */
4322	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
4323	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
4324	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
4325
4326	LogFlow(("MapCR3: %RGp\n", GCPhysCR3));
4327
4328	/*
4329	* Map the page CR3 points at.
4330	*/
4331	RTHCPTR HCPtrGuestCR3;
4332	RTHCPHYS HCPhysGuestCR3;
4333	pgmLock(pVM);
4334	PPGMPAGE pPageCR3 = pgmPhysGetPage(&pVM->pgm.s, GCPhysCR3);
4335	AssertReturn(pPageCR3, VERR_INTERNAL_ERROR_2);
4336	HCPhysGuestCR3 = PGM_PAGE_GET_HCPHYS(pPageCR3);
4337	/** @todo this needs some reworking wrt. locking? */
4338	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4339	HCPtrGuestCR3 = NIL_RTHCPTR;
4340	int rc = VINF_SUCCESS;
4341	# else
4342	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPageCR3, GCPhysCR3 & GST_CR3_PAGE_MASK, (void )&HCPtrGuestCR3); / @todo r=bird: This GCPhysCR3 masking isn't necessary. */
4343	# endif
4344	pgmUnlock(pVM);
4345	if (RT_SUCCESS(rc))
4346	{
4347	rc = PGMMap(pVM, (RTGCPTR)pVM->pgm.s.GCPtrCR3Mapping, HCPhysGuestCR3, PAGE_SIZE, 0);
4348	if (RT_SUCCESS(rc))
4349	{
4350	# ifdef IN_RC
4351	PGM_INVL_PG(pVCpu, pVM->pgm.s.GCPtrCR3Mapping);
4352	# endif
4353	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4354	pVCpu->pgm.s.pGst32BitPdR3 = (R3PTRTYPE(PX86PD))HCPtrGuestCR3;
4355	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4356	pVCpu->pgm.s.pGst32BitPdR0 = (R0PTRTYPE(PX86PD))HCPtrGuestCR3;
4357	# endif
4358	pVCpu->pgm.s.pGst32BitPdRC = (RCPTRTYPE(PX86PD))(RTRCUINTPTR)pVM->pgm.s.GCPtrCR3Mapping;
4359
4360	# elif PGM_GST_TYPE == PGM_TYPE_PAE
4361	unsigned off = GCPhysCR3 & GST_CR3_PAGE_MASK & PAGE_OFFSET_MASK;
4362	pVCpu->pgm.s.pGstPaePdptR3 = (R3PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4363	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4364	pVCpu->pgm.s.pGstPaePdptR0 = (R0PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4365	# endif
4366	pVCpu->pgm.s.pGstPaePdptRC = (RCPTRTYPE(PX86PDPT))((RTRCUINTPTR)pVM->pgm.s.GCPtrCR3Mapping + off);
4367	LogFlow(("Cached mapping %RRv\n", pVCpu->pgm.s.pGstPaePdptRC));
4368
4369	/*
4370	* Map the 4 PDs too.
4371	*/
4372	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pVCpu);
4373	RTGCPTR GCPtr = pVM->pgm.s.GCPtrCR3Mapping + PAGE_SIZE;
4374	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++, GCPtr += PAGE_SIZE)
4375	{
4376	if (pGuestPDPT->a[i].n.u1Present)
4377	{
4378	RTHCPTR HCPtr;
4379	RTHCPHYS HCPhys;
4380	RTGCPHYS GCPhys = pGuestPDPT->a[i].u & X86_PDPE_PG_MASK;
4381	pgmLock(pVM);
4382	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
4383	AssertReturn(pPage, VERR_INTERNAL_ERROR_2);
4384	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
4385	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4386	HCPtr = NIL_RTHCPTR;
4387	int rc2 = VINF_SUCCESS;
4388	# else
4389	int rc2 = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, (void **)&HCPtr);
4390	# endif
4391	pgmUnlock(pVM);
4392	if (RT_SUCCESS(rc2))
4393	{
4394	rc = PGMMap(pVM, GCPtr, HCPhys, PAGE_SIZE, 0);
4395	AssertRCReturn(rc, rc);
4396
4397	pVCpu->pgm.s.apGstPaePDsR3[i] = (R3PTRTYPE(PX86PDPAE))HCPtr;
4398	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4399	pVCpu->pgm.s.apGstPaePDsR0[i] = (R0PTRTYPE(PX86PDPAE))HCPtr;
4400	# endif
4401	pVCpu->pgm.s.apGstPaePDsRC[i] = (RCPTRTYPE(PX86PDPAE))(RTRCUINTPTR)GCPtr;
4402	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
4403	# ifdef IN_RC
4404	PGM_INVL_PG(pVCpu, GCPtr);
4405	# endif
4406	continue;
4407	}
4408	AssertMsgFailed(("pgmR3Gst32BitMapCR3: rc2=%d GCPhys=%RGp i=%d\n", rc2, GCPhys, i));
4409	}
4410
4411	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4412	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4413	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4414	# endif
4415	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4416	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4417	# ifdef IN_RC
4418	PGM_INVL_PG(pVCpu, GCPtr); /** @todo this shouldn't be necessary? */
4419	# endif
4420	}
4421
4422	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
4423	pVCpu->pgm.s.pGstAmd64Pml4R3 = (R3PTRTYPE(PX86PML4))HCPtrGuestCR3;
4424	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4425	pVCpu->pgm.s.pGstAmd64Pml4R0 = (R0PTRTYPE(PX86PML4))HCPtrGuestCR3;
4426	# endif
4427	# endif
4428	}
4429	else
4430	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4431	}
4432	else
4433	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4434
4435	#else /* prot/real stub */
4436	int rc = VINF_SUCCESS;
4437	#endif
4438
4439	/* Update shadow paging info for guest modes with paging (32, pae, 64). */
4440	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4441	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4442	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64) \
4443	&& ( PGM_GST_TYPE != PGM_TYPE_REAL \
4444	&& PGM_GST_TYPE != PGM_TYPE_PROT))
4445
4446	Assert(!pVM->pgm.s.fNestedPaging);
4447
4448	/*
4449	* Update the shadow root page as well since that's not fixed.
4450	*/
4451	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4452	PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
4453	uint32_t iOldShwUserTable = pVCpu->pgm.s.iShwUserTable;
4454	uint32_t iOldShwUser = pVCpu->pgm.s.iShwUser;
4455	PPGMPOOLPAGE pNewShwPageCR3;
4456
4457	pgmLock(pVM);
4458
4459	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4460	if (pPool->cDirtyPages)
4461	pgmPoolResetDirtyPages(pVM);
4462	# endif
4463
4464	Assert(!(GCPhysCR3 >> (PAGE_SHIFT + 32)));
4465	rc = pgmPoolAlloc(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, SHW_POOL_ROOT_IDX, GCPhysCR3 >> PAGE_SHIFT, &pNewShwPageCR3, true /* lock page */);
4466	AssertFatalRC(rc);
4467	rc = VINF_SUCCESS;
4468
4469	# ifdef IN_RC
4470	/*
4471	* WARNING! We can't deal with jumps to ring 3 in the code below as the
4472	* state will be inconsistent! Flush important things now while
4473	* we still can and then make sure there are no ring-3 calls.
4474	*/
4475	REMNotifyHandlerPhysicalFlushIfAlmostFull(pVM, pVCpu);
4476	VMMRZCallRing3Disable(pVCpu);
4477	# endif
4478
4479	pVCpu->pgm.s.iShwUser = SHW_POOL_ROOT_IDX;
4480	pVCpu->pgm.s.iShwUserTable = GCPhysCR3 >> PAGE_SHIFT;
4481	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3) = pNewShwPageCR3;
4482	# ifdef IN_RING0
4483	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4484	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4485	# elif defined(IN_RC)
4486	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4487	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4488	# else
4489	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4490	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4491	# endif
4492
4493	# ifndef PGM_WITHOUT_MAPPINGS
4494	/*
4495	* Apply all hypervisor mappings to the new CR3.
4496	* Note that SyncCR3 will be executed in case CR3 is changed in a guest paging mode; this will
4497	* make sure we check for conflicts in the new CR3 root.
4498	*/
4499	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4500	Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL) \|\| VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
4501	# endif
4502	rc = pgmMapActivateCR3(pVM, pNewShwPageCR3);
4503	AssertRCReturn(rc, rc);
4504	# endif
4505
4506	/* Set the current hypervisor CR3. */
4507	CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
4508	SELMShadowCR3Changed(pVM, pVCpu);
4509
4510	# ifdef IN_RC
4511	/* NOTE: The state is consistent again. */
4512	VMMRZCallRing3Enable(pVCpu);
4513	# endif
4514
4515	/* Clean up the old CR3 root. */
4516	if ( pOldShwPageCR3
4517	&& pOldShwPageCR3 != pNewShwPageCR3 /* @todo can happen due to incorrect syncing between REM & PGM; find the real cause */)
4518	{
4519	Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
4520	# ifndef PGM_WITHOUT_MAPPINGS
4521	/* Remove the hypervisor mappings from the shadow page table. */
4522	pgmMapDeactivateCR3(pVM, pOldShwPageCR3);
4523	# endif
4524	/* Mark the page as unlocked; allow flushing again. */
4525	pgmPoolUnlockPage(pPool, pOldShwPageCR3);
4526
4527	pgmPoolFreeByPage(pPool, pOldShwPageCR3, iOldShwUser, iOldShwUserTable);
4528	}
4529	pgmUnlock(pVM);
4530	# endif
4531
4532	return rc;
4533	}
4534
4535	/**
4536	* Unmaps the shadow CR3.
4537	*
4538	* @returns VBox status, no specials.
4539	* @param pVCpu The VMCPU handle.
4540	*/
4541	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu)
4542	{
4543	LogFlow(("UnmapCR3\n"));
4544
4545	int rc = VINF_SUCCESS;
4546	PVM pVM = pVCpu->CTX_SUFF(pVM);
4547
4548	/*
4549	* Update guest paging info.
4550	*/
4551	#if PGM_GST_TYPE == PGM_TYPE_32BIT
4552	pVCpu->pgm.s.pGst32BitPdR3 = 0;
4553	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4554	pVCpu->pgm.s.pGst32BitPdR0 = 0;
4555	# endif
4556	pVCpu->pgm.s.pGst32BitPdRC = 0;
4557
4558	#elif PGM_GST_TYPE == PGM_TYPE_PAE
4559	pVCpu->pgm.s.pGstPaePdptR3 = 0;
4560	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4561	pVCpu->pgm.s.pGstPaePdptR0 = 0;
4562	# endif
4563	pVCpu->pgm.s.pGstPaePdptRC = 0;
4564	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4565	{
4566	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4567	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4568	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4569	# endif
4570	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4571	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4572	}
4573
4574	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
4575	pVCpu->pgm.s.pGstAmd64Pml4R3 = 0;
4576	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4577	pVCpu->pgm.s.pGstAmd64Pml4R0 = 0;
4578	# endif
4579
4580	#else /* prot/real mode stub */
4581	/* nothing to do */
4582	#endif
4583
4584	#if !defined(IN_RC) /* In RC we rely on MapCR3 to do the shadow part for us at a safe time */
4585	/*
4586	* Update shadow paging info.
4587	*/
4588	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4589	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4590	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64))
4591
4592	# if PGM_GST_TYPE != PGM_TYPE_REAL
4593	Assert(!pVM->pgm.s.fNestedPaging);
4594	# endif
4595
4596	pgmLock(pVM);
4597
4598	# ifndef PGM_WITHOUT_MAPPINGS
4599	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4600	/* Remove the hypervisor mappings from the shadow page table. */
4601	pgmMapDeactivateCR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4602	# endif
4603
4604	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4605	{
4606	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4607
4608	Assert(pVCpu->pgm.s.iShwUser != PGMPOOL_IDX_NESTED_ROOT);
4609
4610	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4611	if (pPool->cDirtyPages)
4612	pgmPoolResetDirtyPages(pVM);
4613	# endif
4614
4615	/* Mark the page as unlocked; allow flushing again. */
4616	pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4617
4618	pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), pVCpu->pgm.s.iShwUser, pVCpu->pgm.s.iShwUserTable);
4619	pVCpu->pgm.s.pShwPageCR3R3 = 0;
4620	pVCpu->pgm.s.pShwPageCR3R0 = 0;
4621	pVCpu->pgm.s.pShwPageCR3RC = 0;
4622	pVCpu->pgm.s.iShwUser = 0;
4623	pVCpu->pgm.s.iShwUserTable = 0;
4624	}
4625	pgmUnlock(pVM);
4626	# endif
4627	#endif /* !IN_RC*/
4628
4629	return rc;
4630	}

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 31636

Download in other formats: