PGMAllBth.h@ 32521

Last change on this file since 32521 was 32521, checked in by vboxsync, 14 years ago
Optimization
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 201.0 KB

Line
1	/* $Id: PGMAllBth.h 32521 2010-09-15 13:46:22Z vboxsync $ */
2	/** @file
3	* VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4	*
5	* @remarks The nested page tables on AMD makes use of PGM_SHW_TYPE in
6	* {PGM_TYPE_AMD64, PGM_TYPE_PAE and PGM_TYPE_32BIT} and PGM_GST_TYPE
7	* set to PGM_TYPE_PROT. Half of the code in this file is not
8	* exercised with PGM_SHW_TYPE set to PGM_TYPE_NESTED.
9	*
10	* @remarks Extended page tables (intel) are built with PGM_GST_TYPE set to
11	* PGM_TYPE_PROT (and PGM_SHW_TYPE set to PGM_TYPE_EPT).
12	*
13	* @remarks This file is one big \#ifdef-orgy!
14	*
15	*/
16
17	/*
18	* Copyright (C) 2006-2010 Oracle Corporation
19	*
20	* This file is part of VirtualBox Open Source Edition (OSE), as
21	* available from http://www.virtualbox.org. This file is free software;
22	* you can redistribute it and/or modify it under the terms of the GNU
23	* General Public License (GPL) as published by the Free Software
24	* Foundation, in version 2 as it comes in the "COPYING" file of the
25	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
26	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
27	*/
28
29
30	/*******************************************************************************
31	* Internal Functions *
32	*******************************************************************************/
33	RT_C_DECLS_BEGIN
34	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken);
35	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
36	static int PGM_BTH_NAME(SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr);
37	static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, RTGCPTR GCPtrPage);
38	static int PGM_BTH_NAME(SyncPT)(PVMCPU pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage);
39	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uErr);
40	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
41	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
42	#ifdef VBOX_STRICT
43	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
44	#endif
45	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3);
46	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu);
47	RT_C_DECLS_END
48
49
50	/*
51	* Filter out some illegal combinations of guest and shadow paging, so we can
52	* remove redundant checks inside functions.
53	*/
54	#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
55	# error "Invalid combination; PAE guest implies PAE shadow"
56	#endif
57
58	#if (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
59	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT)
60	# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
61	#endif
62
63	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) \
64	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT)
65	# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
66	#endif
67
68	#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT) \
69	\|\| (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
70	# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
71	#endif
72
73	#ifndef IN_RING3
74
75	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
76	/**
77	* Deal with a guest page fault.
78	*
79	* @returns Strict VBox status code.
80	* @retval VINF_EM_RAW_GUEST_TRAP
81	* @retval VINF_EM_RAW_EMULATE_INSTR
82	*
83	* @param pVCpu The current CPU.
84	* @param pGstWalk The guest page table walk result.
85	* @param uErr The error code.
86	*/
87	PGM_BTH_DECL(VBOXSTRICTRC, Trap0eHandlerGuestFault)(PVMCPU pVCpu, PGSTPTWALK pGstWalk, RTGCUINT uErr)
88	{
89	# if !defined(PGM_WITHOUT_MAPPINGS) && (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE)
90	/*
91	* Check for write conflicts with our hypervisor mapping.
92	*
93	* If the guest happens to access a non-present page, where our hypervisor
94	* is currently mapped, then we'll create a #PF storm in the guest.
95	*/
96	if ( (uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RW)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RW)
97	&& MMHyperIsInsideArea(pVCpu->CTX_SUFF(pVM), pGstWalk->Core.GCPtr))
98	{
99	/* Force a CR3 sync to check for conflicts and emulate the instruction. */
100	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
101	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
102	return VINF_EM_RAW_EMULATE_INSTR;
103	}
104	# endif
105
106	/*
107	* Calc the error code for the guest trap.
108	*/
109	uint32_t uNewErr = GST_IS_NX_ACTIVE(pVCpu)
110	? uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID)
111	: uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US);
112	if (pGstWalk->Core.fBadPhysAddr)
113	{
114	uNewErr \|= X86_TRAP_PF_RSVD \| X86_TRAP_PF_P;
115	Assert(!pGstWalk->Core.fNotPresent);
116	}
117	else if (!pGstWalk->Core.fNotPresent)
118	uNewErr \|= X86_TRAP_PF_P;
119	TRPMSetErrorCode(pVCpu, uNewErr);
120
121	LogFlow(("Guest trap; cr2=%RGv uErr=%RGv lvl=%d\n", pGstWalk->Core.GCPtr, uErr, pGstWalk->Core.uLevel));
122	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
123	return VINF_EM_RAW_GUEST_TRAP;
124	}
125	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
126
127
128	/**
129	* Deal with a guest page fault.
130	*
131	* The caller has taken the PGM lock.
132	*
133	* @returns Strict VBox status code.
134	*
135	* @param pVCpu The current CPU.
136	* @param uErr The error code.
137	* @param pRegFrame The register frame.
138	* @param pvFault The fault address.
139	* @param pPage The guest page at @a pvFault.
140	* @param pGstWalk The guest page table walk result.
141	* @param pfLockTaken PGM lock taken here or not (out). This is true
142	* when we're called.
143	*/
144	static VBOXSTRICTRC PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame,
145	RTGCPTR pvFault, PPGMPAGE pPage, bool *pfLockTaken
146	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
147	, PGSTPTWALK pGstWalk
148	# endif
149	)
150	{
151	# if !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
152	GSTPDE const PdeSrcDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
153	#endif
154	PVM pVM = pVCpu->CTX_SUFF(pVM);
155	int rc;
156
157	if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
158	{
159	/*
160	* Physical page access handler.
161	*/
162	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
163	const RTGCPHYS GCPhysFault = pGstWalk->Core.GCPhys;
164	# else
165	const RTGCPHYS GCPhysFault = (RTGCPHYS)pvFault;
166	# endif
167	PPGMPHYSHANDLER pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
168	if (pCur)
169	{
170	# ifdef PGM_SYNC_N_PAGES
171	/*
172	* If the region is write protected and we got a page not present fault, then sync
173	* the pages. If the fault was caused by a read, then restart the instruction.
174	* In case of write access continue to the GC write handler.
175	*
176	* ASSUMES that there is only one handler per page or that they have similar write properties.
177	*/
178	if ( !(uErr & X86_TRAP_PF_P)
179	&& pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE)
180	{
181	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
182	rc = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
183	# else
184	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
185	# endif
186	if ( RT_FAILURE(rc)
187	\|\| !(uErr & X86_TRAP_PF_RW)
188	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
189	{
190	AssertRC(rc);
191	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
192	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
193	return rc;
194	}
195	}
196	# endif
197	# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
198	/*
199	* If the access was not thru a #PF(RSVD\|...) resync the page.
200	*/
201	if ( !(uErr & X86_TRAP_PF_RSVD)
202	&& pCur->enmType != PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
203	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
204	&& pGstWalk->Core.fEffectiveRW
205	&& !pGstWalk->Core.fEffectiveUS /** @todo Remove pGstWalk->Core.fEffectiveUS and X86_PTE_US further down in the sync code. */
206	# endif
207	)
208	{
209	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
210	rc = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
211	# else
212	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
213	# endif
214	if ( RT_FAILURE(rc)
215	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
216	{
217	AssertRC(rc);
218	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
219	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
220	return rc;
221	}
222	}
223	# endif
224
225	AssertMsg( pCur->enmType != PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
226	\|\| (pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE && (uErr & X86_TRAP_PF_RW)),
227	("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enum=%d\n",
228	pvFault, GCPhysFault, pPage, uErr, pCur->enmType));
229	if (pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE)
230	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysWrite);
231	else
232	{
233	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysAll);
234	if (uErr & X86_TRAP_PF_RSVD) STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysAllOpt);
235	}
236
237	if (pCur->CTX_SUFF(pfnHandler))
238	{
239	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
240	void *pvUser = pCur->CTX_SUFF(pvUser);
241	# ifdef IN_RING0
242	PFNPGMR0PHYSHANDLER pfnHandler = pCur->CTX_SUFF(pfnHandler);
243	# else
244	PFNPGMRCPHYSHANDLER pfnHandler = pCur->CTX_SUFF(pfnHandler);
245	# endif
246
247	STAM_PROFILE_START(&pCur->Stat, h);
248	if (pfnHandler != pPool->CTX_SUFF(pfnAccessHandler))
249	{
250	pgmUnlock(pVM);
251	*pfLockTaken = false;
252	}
253
254	rc = pfnHandler(pVM, uErr, pRegFrame, pvFault, GCPhysFault, pvUser);
255
256	# ifdef VBOX_WITH_STATISTICS
257	pgmLock(pVM);
258	pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
259	if (pCur)
260	STAM_PROFILE_STOP(&pCur->Stat, h);
261	pgmUnlock(pVM);
262	# endif
263	}
264	else
265	rc = VINF_EM_RAW_EMULATE_INSTR;
266
267	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndPhys; });
268	return rc;
269	}
270	}
271	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
272	else
273	{
274	# ifdef PGM_SYNC_N_PAGES
275	/*
276	* If the region is write protected and we got a page not present fault, then sync
277	* the pages. If the fault was caused by a read, then restart the instruction.
278	* In case of write access continue to the GC write handler.
279	*/
280	if ( PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) < PGM_PAGE_HNDL_PHYS_STATE_ALL
281	&& !(uErr & X86_TRAP_PF_P))
282	{
283	rc = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
284	if ( RT_FAILURE(rc)
285	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
286	\|\| !(uErr & X86_TRAP_PF_RW))
287	{
288	AssertRC(rc);
289	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
290	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndVirt; });
291	return rc;
292	}
293	}
294	# endif
295	/*
296	* Ok, it's an virtual page access handler.
297	*
298	* Since it's faster to search by address, we'll do that first
299	* and then retry by GCPhys if that fails.
300	*/
301	/** @todo r=bird: perhaps we should consider looking up by physical address directly now?
302	* r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be
303	* out of sync, because the page was changed without us noticing it (not-present -> present
304	* without invlpg or mov cr3, xxx).
305	*/
306	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
307	if (pCur)
308	{
309	AssertMsg(!(pvFault - pCur->Core.Key < pCur->cb)
310	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
311	\|\| !(uErr & X86_TRAP_PF_P)
312	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
313	("Unexpected trap for virtual handler: %RGv (phys=%RGp) pPage=%R[pgmpage] uErr=%X, enum=%d\n",
314	pvFault, pGstWalk->Core.GCPhys, pPage, uErr, pCur->enmType));
315
316	if ( pvFault - pCur->Core.Key < pCur->cb
317	&& ( uErr & X86_TRAP_PF_RW
318	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
319	{
320	# ifdef IN_RC
321	STAM_PROFILE_START(&pCur->Stat, h);
322	RTGCPTR GCPtrStart = pCur->Core.Key;
323	CTX_MID(PFNPGM,VIRTHANDLER) pfnHandler = pCur->CTX_SUFF(pfnHandler);
324	pgmUnlock(pVM);
325	*pfLockTaken = false;
326
327	rc = pfnHandler(pVM, uErr, pRegFrame, pvFault, GCPtrStart, pvFault - GCPtrStart);
328
329	# ifdef VBOX_WITH_STATISTICS
330	pgmLock(pVM);
331	pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
332	if (pCur)
333	STAM_PROFILE_STOP(&pCur->Stat, h);
334	pgmUnlock(pVM);
335	# endif
336	# else
337	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
338	# endif
339	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersVirtual);
340	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
341	return rc;
342	}
343	/* Unhandled part of a monitored page */
344	}
345	else
346	{
347	/* Check by physical address. */
348	unsigned iPage;
349	rc = pgmHandlerVirtualFindByPhysAddr(pVM, pGstWalk->Core.GCPhys, &pCur, &iPage);
350	Assert(RT_SUCCESS(rc) \|\| !pCur);
351	if ( pCur
352	&& ( uErr & X86_TRAP_PF_RW
353	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
354	{
355	Assert((pCur->aPhysToVirt[iPage].Core.Key & X86_PTE_PAE_PG_MASK) == (pGstWalk->Core.GCPhys & X86_PTE_PAE_PG_MASK));
356	# ifdef IN_RC
357	STAM_PROFILE_START(&pCur->Stat, h);
358	RTGCPTR GCPtrStart = pCur->Core.Key;
359	CTX_MID(PFNPGM,VIRTHANDLER) pfnHandler = pCur->CTX_SUFF(pfnHandler);
360	pgmUnlock(pVM);
361	*pfLockTaken = false;
362
363	RTGCPTR off = (iPage << PAGE_SHIFT)
364	+ (pvFault & PAGE_OFFSET_MASK)
365	- (GCPtrStart & PAGE_OFFSET_MASK);
366	Assert(off < pCur->cb);
367	rc = pfnHandler(pVM, uErr, pRegFrame, pvFault, GCPtrStart, off);
368
369	# ifdef VBOX_WITH_STATISTICS
370	pgmLock(pVM);
371	pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, GCPtrStart);
372	if (pCur)
373	STAM_PROFILE_STOP(&pCur->Stat, h);
374	pgmUnlock(pVM);
375	# endif
376	# else
377	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
378	# endif
379	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersVirtualByPhys);
380	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
381	return rc;
382	}
383	}
384	}
385	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
386
387	/*
388	* There is a handled area of the page, but this fault doesn't belong to it.
389	* We must emulate the instruction.
390	*
391	* To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
392	* we first check if this was a page-not-present fault for a page with only
393	* write access handlers. Restart the instruction if it wasn't a write access.
394	*/
395	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersUnhandled);
396
397	if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
398	&& !(uErr & X86_TRAP_PF_P))
399	{
400	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
401	rc = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
402	# else
403	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
404	# endif
405	if ( RT_FAILURE(rc)
406	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
407	\|\| !(uErr & X86_TRAP_PF_RW))
408	{
409	AssertRC(rc);
410	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
411	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
412	return rc;
413	}
414	}
415
416	/** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
417	* It's writing to an unhandled part of the LDT page several million times.
418	*/
419	rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
420	LogFlow(("PGM: PGMInterpretInstruction -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
421	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndUnhandled; });
422	return rc;
423	} /* if any kind of handler */
424
425
426	/**
427	* #PF Handler for raw-mode guest execution.
428	*
429	* @returns VBox status code (appropriate for trap handling and GC return).
430	*
431	* @param pVCpu VMCPU Handle.
432	* @param uErr The trap error code.
433	* @param pRegFrame Trap register frame.
434	* @param pvFault The fault address.
435	* @param pfLockTaken PGM lock taken here or not (out)
436	*/
437	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken)
438	{
439	PVM pVM = pVCpu->CTX_SUFF(pVM);
440
441	*pfLockTaken = false;
442
443	# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \
444	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
445	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
446	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT)
447	int rc;
448
449	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
450	/*
451	* Walk the guest page translation tables and check if it's a guest fault.
452	*/
453	GSTPTWALK GstWalk;
454	rc = PGM_GST_NAME(Walk)(pVCpu, pvFault, &GstWalk);
455	if (RT_FAILURE_NP(rc))
456	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &GstWalk, uErr));
457
458	/* assert some GstWalk sanity. */
459	# if PGM_GST_TYPE == PGM_TYPE_AMD64
460	AssertMsg(GstWalk.Pml4e.u == GstWalk.pPml4e->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pml4e.u, (uint64_t)GstWalk.pPml4e->u));
461	# endif
462	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
463	AssertMsg(GstWalk.Pdpe.u == GstWalk.pPdpe->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pdpe.u, (uint64_t)GstWalk.pPdpe->u));
464	# endif
465	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u));
466	AssertMsg(GstWalk.Core.fBigPage \|\| GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u));
467	Assert(GstWalk.Core.fSucceeded);
468
469	if (uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID))
470	{
471	if ( ( (uErr & X86_TRAP_PF_RW)
472	&& !GstWalk.Core.fEffectiveRW
473	&& ( (uErr & X86_TRAP_PF_US)
474	\|\| CPUMIsGuestR0WriteProtEnabled(pVCpu)) )
475	\|\| ((uErr & X86_TRAP_PF_US) && !GstWalk.Core.fEffectiveUS)
476	\|\| ((uErr & X86_TRAP_PF_ID) && GstWalk.Core.fEffectiveNX)
477	)
478	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &GstWalk, uErr));
479	}
480
481	/*
482	* Set the accessed and dirty flags.
483	*/
484	# if PGM_GST_TYPE == PGM_TYPE_AMD64
485	GstWalk.Pml4e.u \|= X86_PML4E_A;
486	GstWalk.pPml4e->u \|= X86_PML4E_A;
487	GstWalk.Pdpe.u \|= X86_PDPE_A;
488	GstWalk.pPdpe->u \|= X86_PDPE_A;
489	# endif
490	if (GstWalk.Core.fBigPage)
491	{
492	Assert(GstWalk.Pde.b.u1Size);
493	if (uErr & X86_TRAP_PF_RW)
494	{
495	GstWalk.Pde.u \|= X86_PDE4M_A \| X86_PDE4M_D;
496	GstWalk.pPde->u \|= X86_PDE4M_A \| X86_PDE4M_D;
497	}
498	else
499	{
500	GstWalk.Pde.u \|= X86_PDE4M_A;
501	GstWalk.pPde->u \|= X86_PDE4M_A;
502	}
503	}
504	else
505	{
506	Assert(!GstWalk.Pde.b.u1Size);
507	GstWalk.Pde.u \|= X86_PDE_A;
508	GstWalk.pPde->u \|= X86_PDE_A;
509	if (uErr & X86_TRAP_PF_RW)
510	{
511	# ifdef VBOX_WITH_STATISTICS
512	if (!GstWalk.Pte.n.u1Dirty)
513	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtiedPage));
514	else
515	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageAlreadyDirty));
516	# endif
517	GstWalk.Pte.u \|= X86_PTE_A \| X86_PTE_D;
518	GstWalk.pPte->u \|= X86_PTE_A \| X86_PTE_D;
519	}
520	else
521	{
522	GstWalk.Pte.u \|= X86_PTE_A;
523	GstWalk.pPte->u \|= X86_PTE_A;
524	}
525	Assert(GstWalk.Pte.u == GstWalk.pPte->u);
526	}
527	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u \|\| GstWalk.pPte->u == GstWalk.pPde->u,
528	("%RX64 %RX64 pPte=%p pPde=%p Pte=%RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u, GstWalk.pPte, GstWalk.pPde, (uint64_t)GstWalk.pPte->u));
529	# else /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
530	GSTPDE const PdeSrcDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A}; /** @todo eliminate this */
531	# endif /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
532
533	/* Take the big lock now. */
534	*pfLockTaken = true;
535	pgmLock(pVM);
536
537	# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
538	/*
539	* If it is a reserved bit fault we know that it is an MMIO (access
540	* handler) related fault and can skip some 200 lines of code.
541	*/
542	if (uErr & X86_TRAP_PF_RSVD)
543	{
544	Assert(uErr & X86_TRAP_PF_P);
545	PPGMPAGE pPage;
546	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
547	rc = pgmPhysGetPageEx(&pVM->pgm.s, GstWalk.Core.GCPhys, &pPage);
548	if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
549	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
550	pfLockTaken, &GstWalk));
551	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
552	# else
553	rc = pgmPhysGetPageEx(&pVM->pgm.s, (RTGCPHYS)pvFault, &pPage);
554	if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
555	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
556	pfLockTaken));
557	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
558	# endif
559	AssertRC(rc);
560	PGM_INVL_PG(pVCpu, pvFault);
561	return rc; /* Restart with the corrected entry. */
562	}
563	# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
564
565	/*
566	* Fetch the guest PDE, PDPE and PML4E.
567	*/
568	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
569	const unsigned iPDDst = pvFault >> SHW_PD_SHIFT;
570	PX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
571
572	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
573	const unsigned iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK; /* pPDDst index, not used with the pool. */
574	PX86PDPAE pPDDst;
575	# if PGM_GST_TYPE == PGM_TYPE_PAE
576	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, GstWalk.Pdpe.u, &pPDDst);
577	# else
578	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, X86_PDPE_P, &pPDDst); /* RW, US and A are reserved in PAE mode. */
579	# endif
580	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_INTERNAL_ERROR_4);
581
582	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
583	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
584	PX86PDPAE pPDDst;
585	# if PGM_GST_TYPE == PGM_TYPE_PROT /* (AMD-V nested paging) */
586	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A,
587	X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A, &pPDDst);
588	# else
589	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, GstWalk.Pml4e.u, GstWalk.Pdpe.u, &pPDDst);
590	# endif
591	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_INTERNAL_ERROR_4);
592
593	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
594	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
595	PEPTPD pPDDst;
596	rc = pgmShwGetEPTPDPtr(pVCpu, pvFault, NULL, &pPDDst);
597	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_INTERNAL_ERROR_4);
598	# endif
599	Assert(pPDDst);
600
601	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
602	/*
603	* Dirty page handling.
604	*
605	* If we successfully correct the write protection fault due to dirty bit
606	* tracking, then return immediately.
607	*/
608	if (uErr & X86_TRAP_PF_RW) /* write fault? */
609	{
610	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyBitTracking), a);
611	rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, &pPDDst->a[iPDDst], GstWalk.pPde, pvFault);
612	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyBitTracking), a);
613	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
614	{
615	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution)
616	= rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
617	? &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2DirtyAndAccessed
618	: &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
619	LogBird(("Trap0eHandler: returns VINF_SUCCESS\n"));
620	return VINF_SUCCESS;
621	}
622	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u \|\| GstWalk.pPte->u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u));
623	AssertMsg(GstWalk.Core.fBigPage \|\| GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u));
624	}
625
626	# if 0 /* rarely useful; leave for debugging. */
627	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0ePD[iPDSrc]);
628	# endif
629	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
630
631	/*
632	* A common case is the not-present error caused by lazy page table syncing.
633	*
634	* It is IMPORTANT that we weed out any access to non-present shadow PDEs
635	* here so we can safely assume that the shadow PT is present when calling
636	* SyncPage later.
637	*
638	* On failure, we ASSUME that SyncPT is out of memory or detected some kind
639	* of mapping conflict and defer to SyncCR3 in R3.
640	* (Again, we do NOT support access handlers for non-present guest pages.)
641	*
642	*/
643	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
644	Assert(GstWalk.Pde.n.u1Present);
645	# endif
646	if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
647	&& !pPDDst->a[iPDDst].n.u1Present)
648	{
649	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2SyncPT; });
650	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
651	LogFlow(("=>SyncPT %04x = %08RX64\n", (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, (uint64_t)GstWalk.Pde.u));
652	rc = PGM_BTH_NAME(SyncPT)(pVCpu, (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, GstWalk.pPd, pvFault);
653	# else
654	LogFlow(("=>SyncPT pvFault=%RGv\n", pvFault));
655	rc = PGM_BTH_NAME(SyncPT)(pVCpu, 0, NULL, pvFault);
656	# endif
657	if (RT_SUCCESS(rc))
658	return rc;
659	Log(("SyncPT: %RGv failed!! rc=%Rrc\n", pvFault, rc));
660	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
661	return VINF_PGM_SYNC_CR3;
662	}
663
664	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(PGM_WITHOUT_MAPPINGS)
665	/*
666	* Check if this address is within any of our mappings.
667	*
668	* This is very fast and it's gonna save us a bit of effort below and prevent
669	* us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
670	* (BTW, it's impossible to have physical access handlers in a mapping.)
671	*/
672	if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
673	{
674	PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
675	for ( ; pMapping; pMapping = pMapping->CTX_SUFF(pNext))
676	{
677	if (pvFault < pMapping->GCPtr)
678	break;
679	if (pvFault - pMapping->GCPtr < pMapping->cb)
680	{
681	/*
682	* The first thing we check is if we've got an undetected conflict.
683	*/
684	if (pgmMapAreMappingsFloating(&pVM->pgm.s))
685	{
686	unsigned iPT = pMapping->cb >> GST_PD_SHIFT;
687	while (iPT-- > 0)
688	if (GstWalk.pPde[iPT].n.u1Present)
689	{
690	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eConflicts);
691	Log(("Trap0e: Detected Conflict %RGv-%RGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
692	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
693	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
694	return VINF_PGM_SYNC_CR3;
695	}
696	}
697
698	/*
699	* Check if the fault address is in a virtual page access handler range.
700	*/
701	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->HyperVirtHandlers, pvFault);
702	if ( pCur
703	&& pvFault - pCur->Core.Key < pCur->cb
704	&& uErr & X86_TRAP_PF_RW)
705	{
706	# ifdef IN_RC
707	STAM_PROFILE_START(&pCur->Stat, h);
708	pgmUnlock(pVM);
709	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
710	pgmLock(pVM);
711	STAM_PROFILE_STOP(&pCur->Stat, h);
712	# else
713	AssertFailed();
714	rc = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
715	# endif
716	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersMapping);
717	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
718	return rc;
719	}
720
721	/*
722	* Pretend we're not here and let the guest handle the trap.
723	*/
724	TRPMSetErrorCode(pVCpu, uErr & ~X86_TRAP_PF_P);
725	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eGuestPFMapping);
726	LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
727	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
728	return VINF_EM_RAW_GUEST_TRAP;
729	}
730	}
731	} /* pgmAreMappingsEnabled(&pVM->pgm.s) */
732	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
733
734	/*
735	* Check if this fault address is flagged for special treatment,
736	* which means we'll have to figure out the physical address and
737	* check flags associated with it.
738	*
739	* ASSUME that we can limit any special access handling to pages
740	* in page tables which the guest believes to be present.
741	*/
742	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
743	RTGCPHYS GCPhys = GstWalk.Core.GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
744	# else
745	RTGCPHYS GCPhys = (RTGCPHYS)pvFault & ~(RTGCPHYS)PAGE_OFFSET_MASK;
746	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
747	PPGMPAGE pPage;
748	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
749	if (RT_FAILURE(rc))
750	{
751	/*
752	* When the guest accesses invalid physical memory (e.g. probing
753	* of RAM or accessing a remapped MMIO range), then we'll fall
754	* back to the recompiler to emulate the instruction.
755	*/
756	LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhys, rc));
757	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersInvalid);
758	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2InvalidPhys; });
759	return VINF_EM_RAW_EMULATE_INSTR;
760	}
761
762	/*
763	* Any handlers for this page?
764	*/
765	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
766	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
767	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, pfLockTaken,
768	&GstWalk));
769	# else
770	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, pfLockTaken));
771	# endif
772
773	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTimeOutOfSync, c);
774
775	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
776	if (uErr & X86_TRAP_PF_P)
777	{
778	/*
779	* The page isn't marked, but it might still be monitored by a virtual page access handler.
780	* (ASSUMES no temporary disabling of virtual handlers.)
781	*/
782	/** @todo r=bird: Since the purpose is to catch out of sync pages with virtual handler(s) here,
783	* we should correct both the shadow page table and physical memory flags, and not only check for
784	* accesses within the handler region but for access to pages with virtual handlers. */
785	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
786	if (pCur)
787	{
788	AssertMsg( !(pvFault - pCur->Core.Key < pCur->cb)
789	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
790	\|\| !(uErr & X86_TRAP_PF_P)
791	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
792	("Unexpected trap for virtual handler: %08X (phys=%08x) %R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
793
794	if ( pvFault - pCur->Core.Key < pCur->cb
795	&& ( uErr & X86_TRAP_PF_RW
796	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
797	{
798	# ifdef IN_RC
799	STAM_PROFILE_START(&pCur->Stat, h);
800	pgmUnlock(pVM);
801	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
802	pgmLock(pVM);
803	STAM_PROFILE_STOP(&pCur->Stat, h);
804	# else
805	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
806	# endif
807	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
808	return rc;
809	}
810	}
811	}
812	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
813
814	/*
815	* We are here only if page is present in Guest page tables and
816	* trap is not handled by our handlers.
817	*
818	* Check it for page out-of-sync situation.
819	*/
820	if (!(uErr & X86_TRAP_PF_P))
821	{
822	/*
823	* Page is not present in our page tables. Try to sync it!
824	*/
825	if (uErr & X86_TRAP_PF_US)
826	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUser));
827	else /* supervisor */
828	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
829
830	if (PGM_PAGE_IS_BALLOONED(pPage))
831	{
832	/* Emulate reads from ballooned pages as they are not present in
833	our shadow page tables. (Required for e.g. Solaris guests; soft
834	ecc, random nr generator.) */
835	rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
836	LogFlow(("PGM: PGMInterpretInstruction balloon -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
837	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncBallloon));
838	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Ballooned; });
839	return rc;
840	}
841
842	# if defined(LOG_ENABLED) && !defined(IN_RING0)
843	RTGCPHYS GCPhys2;
844	uint64_t fPageGst2;
845	PGMGstGetPage(pVCpu, pvFault, &fPageGst2, &GCPhys2);
846	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
847	Log(("Page out of sync: %RGv eip=%08x PdeSrc.US=%d fPageGst2=%08llx GCPhys2=%RGp scan=%d\n",
848	pvFault, pRegFrame->eip, GstWalk.Pde.n.u1User, fPageGst2, GCPhys2, CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)));
849	# else
850	Log(("Page out of sync: %RGv eip=%08x fPageGst2=%08llx GCPhys2=%RGp scan=%d\n",
851	pvFault, pRegFrame->eip, fPageGst2, GCPhys2, CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)));
852	# endif
853	# endif /* LOG_ENABLED */
854
855	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
856	if ( !GstWalk.Core.fEffectiveUS
857	&& CPUMGetGuestCPL(pVCpu, pRegFrame) == 0)
858	{
859	/* Note: Can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU. */
860	if ( pvFault == (RTGCPTR)pRegFrame->eip
861	\|\| pvFault - pRegFrame->eip < 8 /* instruction crossing a page boundary */
862	# ifdef CSAM_DETECT_NEW_CODE_PAGES
863	\|\| ( !PATMIsPatchGCAddr(pVM, pRegFrame->eip)
864	&& CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)) /* any new code we encounter here */
865	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
866	)
867	{
868	LogFlow(("CSAMExecFault %RX32\n", pRegFrame->eip));
869	rc = CSAMExecFault(pVM, (RTRCPTR)pRegFrame->eip);
870	if (rc != VINF_SUCCESS)
871	{
872	/*
873	* CSAM needs to perform a job in ring 3.
874	*
875	* Sync the page before going to the host context; otherwise we'll end up in a loop if
876	* CSAM fails (e.g. instruction crosses a page boundary and the next page is not present)
877	*/
878	LogFlow(("CSAM ring 3 job\n"));
879	int rc2 = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
880	AssertRC(rc2);
881
882	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2CSAM; });
883	return rc;
884	}
885	}
886	# ifdef CSAM_DETECT_NEW_CODE_PAGES
887	else if ( uErr == X86_TRAP_PF_RW
888	&& pRegFrame->ecx >= 0x100 /* early check for movswd count */
889	&& pRegFrame->ecx < 0x10000)
890	{
891	/* In case of a write to a non-present supervisor shadow page, we'll take special precautions
892	* to detect loading of new code pages.
893	*/
894
895	/*
896	* Decode the instruction.
897	*/
898	RTGCPTR PC;
899	rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs,
900	&pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &PC);
901	if (rc == VINF_SUCCESS)
902	{
903	PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
904	uint32_t cbOp;
905	rc = EMInterpretDisasOneEx(pVM, pVCpu, PC, pRegFrame, pDis, &cbOp);
906
907	/* For now we'll restrict this to rep movsw/d instructions */
908	if ( rc == VINF_SUCCESS
909	&& pDis->pCurInstr->opcode == OP_MOVSWD
910	&& (pDis->prefix & PREFIX_REP))
911	{
912	CSAMMarkPossibleCodePage(pVM, pvFault);
913	}
914	}
915	}
916	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
917
918	/*
919	* Mark this page as safe.
920	*/
921	/** @todo not correct for pages that contain both code and data!! */
922	Log2(("CSAMMarkPage %RGv; scanned=%d\n", pvFault, true));
923	CSAMMarkPage(pVM, pvFault, true);
924	}
925	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0) */
926	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
927	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
928	# else
929	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
930	# endif
931	if (RT_SUCCESS(rc))
932	{
933	/* The page was successfully synced, return to the guest. */
934	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSync; });
935	return VINF_SUCCESS;
936	}
937	}
938	else /* uErr & X86_TRAP_PF_P: */
939	{
940	/*
941	* Write protected pages are made writable when the guest makes the
942	* first write to it. This happens for pages that are shared, write
943	* monitored or not yet allocated.
944	*
945	* We may also end up here when CR0.WP=0 in the guest.
946	*
947	* Also, a side effect of not flushing global PDEs are out of sync
948	* pages due to physical monitored regions, that are no longer valid.
949	* Assume for now it only applies to the read/write flag.
950	*/
951	if (uErr & X86_TRAP_PF_RW)
952	{
953	/*
954	* Check if it is a read-only page.
955	*/
956	if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
957	{
958	Log(("PGM #PF: Make writable: %RGp %R[pgmpage] pvFault=%RGp uErr=%#x\n", GCPhys, pPage, pvFault, uErr));
959	Assert(!PGM_PAGE_IS_ZERO(pPage));
960	AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
961	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2MakeWritable; });
962
963	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
964	if (rc != VINF_SUCCESS)
965	{
966	AssertMsg(rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("%Rrc\n", rc));
967	return rc;
968	}
969	if (RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)))
970	return VINF_EM_NO_MEMORY;
971	}
972
973	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
974	/*
975	* Check to see if we need to emulate the instruction if CR0.WP=0.
976	*/
977	if ( !GstWalk.Core.fEffectiveRW
978	&& (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG
979	&& CPUMGetGuestCPL(pVCpu, pRegFrame) == 0)
980	{
981	Assert((uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RW \| X86_TRAP_PF_P));
982	rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
983	if (RT_SUCCESS(rc))
984	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eWPEmulInRZ);
985	else
986	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eWPEmulToR3);
987	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2WPEmulation; });
988	return rc;
989	}
990	# endif
991	/// @todo count the above case; else
992	if (uErr & X86_TRAP_PF_US)
993	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUserWrite));
994	else /* supervisor */
995	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
996
997	/*
998	* Sync the page.
999	*
1000	* Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
1001	* page is not present, which is not true in this case.
1002	*/
1003	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1004	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
1005	# else
1006	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
1007	# endif
1008	if (RT_SUCCESS(rc))
1009	{
1010	/*
1011	* Page was successfully synced, return to guest but invalidate
1012	* the TLB first as the page is very likely to be in it.
1013	*/
1014	# if PGM_SHW_TYPE == PGM_TYPE_EPT
1015	HWACCMInvalidatePhysPage(pVM, (RTGCPHYS)pvFault);
1016	# else
1017	PGM_INVL_PG(pVCpu, pvFault);
1018	# endif
1019	# ifdef VBOX_STRICT
1020	RTGCPHYS GCPhys2;
1021	uint64_t fPageGst;
1022	if (!pVM->pgm.s.fNestedPaging)
1023	{
1024	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, &GCPhys2);
1025	AssertMsg(RT_SUCCESS(rc) && (fPageGst & X86_PTE_RW), ("rc=%Rrc fPageGst=%RX64\n", rc, fPageGst));
1026	LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GCPhys2, (uint64_t)fPageGst));
1027	}
1028	uint64_t fPageShw;
1029	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
1030	AssertMsg((RT_SUCCESS(rc) && (fPageShw & X86_PTE_RW)) \|\| pVM->cCpus > 1 /* new monitor can be installed/page table flushed between the trap exit and PGMTrap0eHandler */,
1031	("rc=%Rrc fPageShw=%RX64\n", rc, fPageShw));
1032	# endif /* VBOX_STRICT */
1033	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndObs; });
1034	return VINF_SUCCESS;
1035	}
1036	}
1037	/** @todo else: why are we here? */
1038
1039	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && defined(VBOX_STRICT)
1040	/*
1041	* Check for VMM page flags vs. Guest page flags consistency.
1042	* Currently only for debug purposes.
1043	*/
1044	if (RT_SUCCESS(rc))
1045	{
1046	/* Get guest page flags. */
1047	uint64_t fPageGst;
1048	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
1049	if (RT_SUCCESS(rc))
1050	{
1051	uint64_t fPageShw;
1052	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
1053
1054	/*
1055	* Compare page flags.
1056	* Note: we have AVL, A, D bits desynched.
1057	*/
1058	AssertMsg( (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK))
1059	== (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)),
1060	("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64\n",
1061	pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst));
1062	}
1063	else
1064	AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc));
1065	}
1066	else
1067	AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc));
1068	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && VBOX_STRICT */
1069	}
1070
1071
1072	/*
1073	* If we get here it is because something failed above, i.e. most like guru
1074	* meditiation time.
1075	*/
1076	LogRel(("%s: returns rc=%Rrc pvFault=%RGv uErr=%RX64 cs:rip=%04x:%08RX64\n",
1077	__PRETTY_FUNCTION__, rc, pvFault, (uint64_t)uErr, pRegFrame->cs, pRegFrame->rip));
1078	return rc;
1079
1080	# else /* Nested paging, EPT except PGM_GST_TYPE = PROT */
1081	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
1082	return VERR_INTERNAL_ERROR;
1083	# endif
1084	}
1085	#endif /* !IN_RING3 */
1086
1087
1088	/**
1089	* Emulation of the invlpg instruction.
1090	*
1091	*
1092	* @returns VBox status code.
1093	*
1094	* @param pVCpu The VMCPU handle.
1095	* @param GCPtrPage Page to invalidate.
1096	*
1097	* @remark ASSUMES that the guest is updating before invalidating. This order
1098	* isn't required by the CPU, so this is speculative and could cause
1099	* trouble.
1100	* @remark No TLB shootdown is done on any other VCPU as we assume that
1101	* invlpg emulation is the only reason for calling this function.
1102	* (The guest has to shoot down TLB entries on other CPUs itself)
1103	* Currently true, but keep in mind!
1104	*
1105	* @todo Clean this up! Most of it is (or should be) no longer necessary as we catch all page table accesses.
1106	* Should only be required when PGMPOOL_WITH_OPTIMIZED_DIRTY_PT is active (PAE or AMD64 (for now))
1107	*/
1108	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
1109	{
1110	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1111	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
1112	&& PGM_SHW_TYPE != PGM_TYPE_EPT
1113	int rc;
1114	PVM pVM = pVCpu->CTX_SUFF(pVM);
1115	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1116
1117	Assert(PGMIsLockOwner(pVM));
1118
1119	LogFlow(("InvalidatePage %RGv\n", GCPtrPage));
1120
1121	/*
1122	* Get the shadow PD entry and skip out if this PD isn't present.
1123	* (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
1124	*/
1125	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1126	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1127	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1128
1129	/* Fetch the pgm pool shadow descriptor. */
1130	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1131	Assert(pShwPde);
1132
1133	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1134	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT);
1135	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
1136
1137	/* If the shadow PDPE isn't present, then skip the invalidate. */
1138	if (!pPdptDst->a[iPdpt].n.u1Present)
1139	{
1140	Assert(!(pPdptDst->a[iPdpt].u & PGM_PLXFLAGS_MAPPING));
1141	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1142	return VINF_SUCCESS;
1143	}
1144
1145	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1146	PPGMPOOLPAGE pShwPde = NULL;
1147	PX86PDPAE pPDDst;
1148
1149	/* Fetch the pgm pool shadow descriptor. */
1150	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
1151	AssertRCSuccessReturn(rc, rc);
1152	Assert(pShwPde);
1153
1154	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1155	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1156
1157	# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1158	/* PML4 */
1159	const unsigned iPml4 = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;
1160	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1161	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1162	PX86PDPAE pPDDst;
1163	PX86PDPT pPdptDst;
1164	PX86PML4E pPml4eDst;
1165	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eDst, &pPdptDst, &pPDDst);
1166	if (rc != VINF_SUCCESS)
1167	{
1168	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT \|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
1169	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1170	return VINF_SUCCESS;
1171	}
1172	Assert(pPDDst);
1173
1174	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1175	PX86PDPE pPdpeDst = &pPdptDst->a[iPdpt];
1176
1177	if (!pPdpeDst->n.u1Present)
1178	{
1179	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1180	return VINF_SUCCESS;
1181	}
1182
1183	/* Fetch the pgm pool shadow descriptor. */
1184	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & SHW_PDPE_PG_MASK);
1185	Assert(pShwPde);
1186
1187	# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1188
1189	const SHWPDE PdeDst = *pPdeDst;
1190	if (!PdeDst.n.u1Present)
1191	{
1192	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1193	return VINF_SUCCESS;
1194	}
1195
1196	/*
1197	* Get the guest PD entry and calc big page.
1198	*/
1199	# if PGM_GST_TYPE == PGM_TYPE_32BIT
1200	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
1201	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
1202	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1203	# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1204	unsigned iPDSrc = 0;
1205	# if PGM_GST_TYPE == PGM_TYPE_PAE
1206	X86PDPE PdpeSrcIgn;
1207	PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrcIgn);
1208	# else /* AMD64 */
1209	PX86PML4E pPml4eSrcIgn;
1210	X86PDPE PdpeSrcIgn;
1211	PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrcIgn, &PdpeSrcIgn, &iPDSrc);
1212	# endif
1213	GSTPDE PdeSrc;
1214
1215	if (pPDSrc)
1216	PdeSrc = pPDSrc->a[iPDSrc];
1217	else
1218	PdeSrc.u = 0;
1219	# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1220	const bool fIsBigPage = PdeSrc.b.u1Size && GST_IS_PSE_ACTIVE(pVCpu);
1221
1222	# ifdef IN_RING3
1223	/*
1224	* If a CR3 Sync is pending we may ignore the invalidate page operation
1225	* depending on the kind of sync and if it's a global page or not.
1226	* This doesn't make sense in GC/R0 so we'll skip it entirely there.
1227	*/
1228	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
1229	if ( VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
1230	\|\| ( VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
1231	&& fIsBigPage
1232	&& PdeSrc.b.u1Global
1233	)
1234	)
1235	# else
1236	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
1237	# endif
1238	{
1239	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1240	return VINF_SUCCESS;
1241	}
1242	# endif /* IN_RING3 */
1243
1244	/*
1245	* Deal with the Guest PDE.
1246	*/
1247	rc = VINF_SUCCESS;
1248	if (PdeSrc.n.u1Present)
1249	{
1250	Assert( PdeSrc.n.u1User == PdeDst.n.u1User
1251	&& (PdeSrc.n.u1Write \|\| !PdeDst.n.u1Write));
1252	# ifndef PGM_WITHOUT_MAPPING
1253	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
1254	{
1255	/*
1256	* Conflict - Let SyncPT deal with it to avoid duplicate code.
1257	*/
1258	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1259	Assert(PGMGetGuestMode(pVCpu) <= PGMMODE_PAE);
1260	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
1261	}
1262	else
1263	# endif /* !PGM_WITHOUT_MAPPING */
1264	if (!fIsBigPage)
1265	{
1266	/*
1267	* 4KB - page.
1268	*/
1269	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1270	RTGCPHYS GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
1271
1272	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1273	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1274	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE/2);
1275	# endif
1276	if (pShwPage->GCPhys == GCPhys)
1277	{
1278	# if 0 /* likely cause of a major performance regression; must be SyncPageWorkerTrackDeref then */
1279	const unsigned iPTEDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1280	PSHWPT pPT = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1281	if (pPT->a[iPTEDst].n.u1Present)
1282	{
1283	/* This is very unlikely with caching/monitoring enabled. */
1284	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pShwPage, pPT->a[iPTEDst].u & SHW_PTE_PG_MASK, iPTEDst);
1285	ASMAtomicWriteSize(&pPT->a[iPTEDst], 0);
1286	}
1287	# else /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1288	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
1289	if (RT_SUCCESS(rc))
1290	rc = VINF_SUCCESS;
1291	# endif
1292	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4KBPages));
1293	PGM_INVL_PG(pVCpu, GCPtrPage);
1294	}
1295	else
1296	{
1297	/*
1298	* The page table address changed.
1299	*/
1300	LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
1301	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1302	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1303	ASMAtomicWriteSize(pPdeDst, 0);
1304	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1305	PGM_INVL_VCPU_TLBS(pVCpu);
1306	}
1307	}
1308	else
1309	{
1310	/*
1311	* 2/4MB - page.
1312	*/
1313	/* Before freeing the page, check if anything really changed. */
1314	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1315	RTGCPHYS GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
1316	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1317	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1318	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1319	# endif
1320	if ( pShwPage->GCPhys == GCPhys
1321	&& pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1322	{
1323	/* ASSUMES a the given bits are identical for 4M and normal PDEs */
1324	/** @todo This test is wrong as it cannot check the G bit!
1325	* FIXME */
1326	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US))
1327	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US))
1328	&& ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1329	\|\| (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1330	{
1331	LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1332	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip));
1333	return VINF_SUCCESS;
1334	}
1335	}
1336
1337	/*
1338	* Ok, the page table is present and it's been changed in the guest.
1339	* If we're in host context, we'll just mark it as not present taking the lazy approach.
1340	* We could do this for some flushes in GC too, but we need an algorithm for
1341	* deciding which 4MB pages containing code likely to be executed very soon.
1342	*/
1343	LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1344	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1345	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1346	ASMAtomicWriteSize(pPdeDst, 0);
1347	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4MBPages));
1348	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
1349	}
1350	}
1351	else
1352	{
1353	/*
1354	* Page directory is not present, mark shadow PDE not present.
1355	*/
1356	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
1357	{
1358	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1359	ASMAtomicWriteSize(pPdeDst, 0);
1360	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDNPs));
1361	PGM_INVL_PG(pVCpu, GCPtrPage);
1362	}
1363	else
1364	{
1365	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1366	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDMappings));
1367	}
1368	}
1369	return rc;
1370
1371	#else /* guest real and protected mode */
1372	/* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1373	return VINF_SUCCESS;
1374	#endif
1375	}
1376
1377
1378	/**
1379	* Update the tracking of shadowed pages.
1380	*
1381	* @param pVCpu The VMCPU handle.
1382	* @param pShwPage The shadow page.
1383	* @param HCPhys The physical page we is being dereferenced.
1384	* @param iPte Shadow PTE index
1385	* @param GCPhysPage Guest physical address (only valid if pShwPage->fDirty is set)
1386	*/
1387	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys, uint16_t iPte, RTGCPHYS GCPhysPage)
1388	{
1389	PVM pVM = pVCpu->CTX_SUFF(pVM);
1390
1391	# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1392	&& PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1393	&& (PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
1394
1395	/* Use the hint we retrieved from the cached guest PT. */
1396	if (pShwPage->fDirty)
1397	{
1398	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1399
1400	Assert(pShwPage->cPresent);
1401	Assert(pPool->cPresent);
1402	pShwPage->cPresent--;
1403	pPool->cPresent--;
1404
1405	PPGMPAGE pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysPage);
1406	AssertRelease(pPhysPage);
1407	pgmTrackDerefGCPhys(pPool, pShwPage, pPhysPage, iPte);
1408	return;
1409	}
1410	# endif
1411
1412	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackDeref, a);
1413	LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1414
1415	/** @todo If this turns out to be a bottle neck (very likely) two things can be done:
1416	* 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1417	* 2. write protect all shadowed pages. I.e. implement caching.
1418	*/
1419	/** @todo duplicated in the 2nd half of pgmPoolTracDerefGCPhysHint */
1420
1421	/*
1422	* Find the guest address.
1423	*/
1424	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
1425	pRam;
1426	pRam = pRam->CTX_SUFF(pNext))
1427	{
1428	unsigned iPage = pRam->cb >> PAGE_SHIFT;
1429	while (iPage-- > 0)
1430	{
1431	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1432	{
1433	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1434
1435	Assert(pShwPage->cPresent);
1436	Assert(pPool->cPresent);
1437	pShwPage->cPresent--;
1438	pPool->cPresent--;
1439
1440	pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage], iPte);
1441	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackDeref, a);
1442	return;
1443	}
1444	}
1445	}
1446
1447	for (;;)
1448	AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys));
1449	}
1450
1451
1452	/**
1453	* Update the tracking of shadowed pages.
1454	*
1455	* @param pVCpu The VMCPU handle.
1456	* @param pShwPage The shadow page.
1457	* @param u16 The top 16-bit of the pPage->HCPhys.
1458	* @param pPage Pointer to the guest page. this will be modified.
1459	* @param iPTDst The index into the shadow table.
1460	*/
1461	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
1462	{
1463	PVM pVM = pVCpu->CTX_SUFF(pVM);
1464
1465	/*
1466	* Just deal with the simple first time here.
1467	*/
1468	if (!u16)
1469	{
1470	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackVirgin);
1471	u16 = PGMPOOL_TD_MAKE(1, pShwPage->idx);
1472	/* Save the page table index. */
1473	PGM_PAGE_SET_PTE_INDEX(pPage, iPTDst);
1474	}
1475	else
1476	u16 = pgmPoolTrackPhysExtAddref(pVM, pPage, u16, pShwPage->idx, iPTDst);
1477
1478	/* write back */
1479	Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x iPTDst=%#x\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst));
1480	PGM_PAGE_SET_TRACKING(pPage, u16);
1481
1482	/* update statistics. */
1483	pVM->pgm.s.CTX_SUFF(pPool)->cPresent++;
1484	pShwPage->cPresent++;
1485	if (pShwPage->iFirstPresent > iPTDst)
1486	pShwPage->iFirstPresent = iPTDst;
1487	}
1488
1489
1490	/**
1491	* Modifies a shadow PTE to account for access handlers.
1492	*
1493	* @param pVM The VM handle.
1494	* @param pPage The page in question.
1495	* @param fPteSrc The shadowed flags of the source PTE. Must include the
1496	* A (accessed) bit so it can be emulated correctly.
1497	* @param pPteDst The shadow PTE (output). This is temporary storage and
1498	* does not need to be set atomically.
1499	*/
1500	DECLINLINE(void) PGM_BTH_NAME(SyncHandlerPte)(PVM pVM, PCPGMPAGE pPage, uint64_t fPteSrc, PSHWPTE pPteDst)
1501	{
1502	/** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No.
1503	* Update: \#PF should deal with this before or after calling the handlers. It has all the info to do the job efficiently. */
1504	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1505	{
1506	LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark read-only\n", pPage));
1507	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1508	pPteDst->u = PGM_PAGE_GET_HCPHYS(pPage);
1509	pPteDst->n.u1Present = 1;
1510	pPteDst->n.u1Execute = 1;
1511	pPteDst->n.u1IgnorePAT = 1;
1512	pPteDst->n.u3EMT = VMX_EPT_MEMTYPE_WB;
1513	/* PteDst.n.u1Write = 0 && PteDst.n.u1Size = 0 */
1514	#else
1515	if (fPteSrc & X86_PTE_A)
1516	{
1517	SHW_PTE_SET(*pPteDst, fPteSrc \| PGM_PAGE_GET_HCPHYS(pPage));
1518	SHW_PTE_SET_RO(*pPteDst);
1519	}
1520	else
1521	SHW_PTE_SET(*pPteDst, 0);
1522	#endif
1523	}
1524	#ifdef PGM_WITH_MMIO_OPTIMIZATIONS
1525	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
1526	else if ( PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
1527	&& ( BTH_IS_NP_ACTIVE(pVM)
1528	\|\| (fPteSrc & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW) /** @todo Remove X86_PTE_US here and pGstWalk->Core.fEffectiveUS before the sync page test. */
1529	# if PGM_SHW_TYPE == PGM_TYPE_AMD64
1530	&& pVM->pgm.s.fLessThan52PhysicalAddressBits
1531	# endif
1532	)
1533	{
1534	LogFlow(("SyncHandlerPte: MMIO page -> invalid \n"));
1535	# if PGM_SHW_TYPE == PGM_TYPE_EPT
1536	/* 25.2.3.1: Reserved physical address bit -> EPT Misconfiguration (exit 49) */
1537	pPteDst->u = pVM->pgm.s.HCPhysInvMmioPg;
1538	/* 25.2.3.1: bits 2:0 = 010b -> EPT Misconfiguration (exit 49) */
1539	pPteDst->n.u1Present = 0;
1540	pPteDst->n.u1Write = 1;
1541	pPteDst->n.u1Execute = 0;
1542	/* 25.2.3.1: leaf && 2:0 != 0 && u3Emt in {2, 3, 7} -> EPT Misconfiguration */
1543	pPteDst->n.u3EMT = 7;
1544	# else
1545	/* Set high page frame bits that MBZ (bankers on PAE, CPU dependent on AMD64). */
1546	SHW_PTE_SET(*pPteDst, pVM->pgm.s.HCPhysInvMmioPg \| X86_PTE_PAE_MBZ_MASK_NO_NX \| X86_PTE_P);
1547	# endif
1548	}
1549	# endif
1550	#endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
1551	else
1552	{
1553	LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark not present\n", pPage));
1554	SHW_PTE_SET(*pPteDst, 0);
1555	}
1556	/** @todo count these kinds of entries. */
1557	}
1558
1559
1560	/**
1561	* Creates a 4K shadow page for a guest page.
1562	*
1563	* For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1564	* physical address. The PdeSrc argument only the flags are used. No page
1565	* structured will be mapped in this function.
1566	*
1567	* @param pVCpu The VMCPU handle.
1568	* @param pPteDst Destination page table entry.
1569	* @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1570	* Can safely assume that only the flags are being used.
1571	* @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1572	* @param pShwPage Pointer to the shadow page.
1573	* @param iPTDst The index into the shadow table.
1574	*
1575	* @remark Not used for 2/4MB pages!
1576	*/
1577	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc,
1578	PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1579	{
1580	PVM pVM = pVCpu->CTX_SUFF(pVM);
1581	RTGCPHYS GCPhysOldPage = NIL_RTGCPHYS;
1582
1583	#if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1584	&& PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1585	&& (PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
1586
1587	if (pShwPage->fDirty)
1588	{
1589	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1590	PGSTPT pGstPT;
1591
1592	pGstPT = (PGSTPT)&pPool->aDirtyPages[pShwPage->idxDirty].aPage[0];
1593	GCPhysOldPage = GST_GET_PTE_GCPHYS(pGstPT->a[iPTDst]);
1594	pGstPT->a[iPTDst].u = PteSrc.u;
1595	}
1596	#else
1597	Assert(!pShwPage->fDirty);
1598	#endif
1599
1600	if ( PteSrc.n.u1Present
1601	&& GST_IS_PTE_VALID(pVCpu, PteSrc))
1602	{
1603	/*
1604	* Find the ram range.
1605	*/
1606	PPGMPAGE pPage;
1607	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GST_GET_PTE_GCPHYS(PteSrc), &pPage);
1608	if (RT_SUCCESS(rc))
1609	{
1610	/* Ignore ballooned pages.
1611	Don't return errors or use a fatal assert here as part of a
1612	shadow sync range might included ballooned pages. */
1613	if (PGM_PAGE_IS_BALLOONED(pPage))
1614	{
1615	Assert(!SHW_PTE_IS_P(pPteDst)); /* @todo user tracking needs updating if this triggers. */
1616	return;
1617	}
1618
1619	#ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1620	/* Make the page writable if necessary. */
1621	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1622	&& ( PGM_PAGE_IS_ZERO(pPage)
1623	\|\| ( PteSrc.n.u1Write
1624	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1625	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1626	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1627	# endif
1628	# ifdef VBOX_WITH_PAGE_SHARING
1629	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
1630	# endif
1631	)
1632	)
1633	)
1634	{
1635	rc = pgmPhysPageMakeWritable(pVM, pPage, GST_GET_PTE_GCPHYS(PteSrc));
1636	AssertRC(rc);
1637	}
1638	#endif
1639
1640	/*
1641	* Make page table entry.
1642	*/
1643	SHWPTE PteDst;
1644	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1645	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, GST_GET_PTE_SHW_FLAGS(pVCpu, PteSrc), &PteDst);
1646	else
1647	{
1648	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1649	/*
1650	* If the page or page directory entry is not marked accessed,
1651	* we mark the page not present.
1652	*/
1653	if (!PteSrc.n.u1Accessed \|\| !PdeSrc.n.u1Accessed)
1654	{
1655	LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1656	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,AccessedPage));
1657	SHW_PTE_SET(PteDst, 0);
1658	}
1659	/*
1660	* If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1661	* when the page is modified.
1662	*/
1663	else if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1664	{
1665	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPage));
1666	SHW_PTE_SET(PteDst,
1667	GST_GET_PTE_SHW_FLAGS(pVCpu, PteSrc)
1668	\| PGM_PAGE_GET_HCPHYS(pPage)
1669	\| PGM_PTFLAGS_TRACK_DIRTY);
1670	SHW_PTE_SET_RO(PteDst);
1671	}
1672	else
1673	#endif
1674	{
1675	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageSkipped));
1676	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1677	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage);
1678	PteDst.n.u1Present = 1;
1679	PteDst.n.u1Write = 1;
1680	PteDst.n.u1Execute = 1;
1681	PteDst.n.u1IgnorePAT = 1;
1682	PteDst.n.u3EMT = VMX_EPT_MEMTYPE_WB;
1683	/* PteDst.n.u1Size = 0 */
1684	#else
1685	SHW_PTE_SET(PteDst, GST_GET_PTE_SHW_FLAGS(pVCpu, PteSrc) \| PGM_PAGE_GET_HCPHYS(pPage));
1686	#endif
1687	}
1688
1689	/*
1690	* Make sure only allocated pages are mapped writable.
1691	*/
1692	if ( SHW_PTE_IS_P_RW(PteDst)
1693	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1694	{
1695	/* Still applies to shared pages. */
1696	Assert(!PGM_PAGE_IS_ZERO(pPage));
1697	SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet. Why, isn't it? */
1698	Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", (RTGCPHYS)GST_GET_PTE_GCPHYS(PteSrc), pPage, iPTDst));
1699	}
1700	}
1701
1702	/*
1703	* Keep user track up to date.
1704	*/
1705	if (SHW_PTE_IS_P(PteDst))
1706	{
1707	if (!SHW_PTE_IS_P(*pPteDst))
1708	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1709	else if (SHW_PTE_GET_HCPHYS(*pPteDst) != SHW_PTE_GET_HCPHYS(PteDst))
1710	{
1711	Log2(("SyncPageWorker: deref! pPteDst=%RX64 PteDst=%RX64\n", SHW_PTE_LOG64(pPteDst), SHW_PTE_LOG64(PteDst)));
1712	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
1713	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1714	}
1715	}
1716	else if (SHW_PTE_IS_P(*pPteDst))
1717	{
1718	Log2(("SyncPageWorker: deref! pPteDst=%RX64\n", SHW_PTE_LOG64(pPteDst)));
1719	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
1720	}
1721
1722	/*
1723	* Update statistics and commit the entry.
1724	*/
1725	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1726	if (!PteSrc.n.u1Global)
1727	pShwPage->fSeenNonGlobal = true;
1728	#endif
1729	SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
1730	return;
1731	}
1732
1733	/** @todo count these three different kinds. */
1734	Log2(("SyncPageWorker: invalid address in Pte\n"));
1735	}
1736	else if (!PteSrc.n.u1Present)
1737	Log2(("SyncPageWorker: page not present in Pte\n"));
1738	else
1739	Log2(("SyncPageWorker: invalid Pte\n"));
1740
1741	/*
1742	* The page is not present or the PTE is bad. Replace the shadow PTE by
1743	* an empty entry, making sure to keep the user tracking up to date.
1744	*/
1745	if (SHW_PTE_IS_P(*pPteDst))
1746	{
1747	Log2(("SyncPageWorker: deref! pPteDst=%RX64\n", SHW_PTE_LOG64(pPteDst)));
1748	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
1749	}
1750	SHW_PTE_ATOMIC_SET(*pPteDst, 0);
1751	}
1752
1753
1754	/**
1755	* Syncs a guest OS page.
1756	*
1757	* There are no conflicts at this point, neither is there any need for
1758	* page table allocations.
1759	*
1760	* When called in PAE or AMD64 guest mode, the guest PDPE shall be valid.
1761	* When called in AMD64 guest mode, the guest PML4E shall be valid.
1762	*
1763	* @returns VBox status code.
1764	* @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1765	* @param pVCpu The VMCPU handle.
1766	* @param PdeSrc Page directory entry of the guest.
1767	* @param GCPtrPage Guest context page address.
1768	* @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1769	* @param uErr Fault error (X86_TRAP_PF_*).
1770	*/
1771	static int PGM_BTH_NAME(SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr)
1772	{
1773	PVM pVM = pVCpu->CTX_SUFF(pVM);
1774	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1775	LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr));
1776
1777	Assert(PGMIsLockOwner(pVM));
1778
1779	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
1780	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
1781	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
1782	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
1783	&& PGM_SHW_TYPE != PGM_TYPE_EPT
1784
1785	/*
1786	* Assert preconditions.
1787	*/
1788	Assert(PdeSrc.n.u1Present);
1789	Assert(cPages);
1790	# if 0 /* rarely useful; leave for debugging. */
1791	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
1792	# endif
1793
1794	/*
1795	* Get the shadow PDE, find the shadow page table in the pool.
1796	*/
1797	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1798	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1799	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1800
1801	/* Fetch the pgm pool shadow descriptor. */
1802	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1803	Assert(pShwPde);
1804
1805	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1806	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1807	PPGMPOOLPAGE pShwPde = NULL;
1808	PX86PDPAE pPDDst;
1809
1810	/* Fetch the pgm pool shadow descriptor. */
1811	int rc2 = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
1812	AssertRCSuccessReturn(rc2, rc2);
1813	Assert(pShwPde);
1814
1815	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1816	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1817
1818	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1819	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1820	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1821	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
1822	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
1823
1824	int rc2 = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
1825	AssertRCSuccessReturn(rc2, rc2);
1826	Assert(pPDDst && pPdptDst);
1827	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1828	# endif
1829	SHWPDE PdeDst = *pPdeDst;
1830
1831	/*
1832	* - In the guest SMP case we could have blocked while another VCPU reused
1833	* this page table.
1834	* - With W7-64 we may also take this path when the the A bit is cleared on
1835	* higher level tables (PDPE/PML4E). The guest does not invalidate the
1836	* relevant TLB entries. If we're write monitoring any page mapped by
1837	* the modified entry, we may end up here with a "stale" TLB entry.
1838	*/
1839	if (!PdeDst.n.u1Present)
1840	{
1841	Log(("CPU%u: SyncPage: Pde at %RGv changed behind our back? (pPdeDst=%p/%RX64) uErr=%#x\n", pVCpu->idCpu, GCPtrPage, pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
1842	AssertMsg(pVM->cCpus > 1 \|\| (uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RW)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RW),
1843	("Unexpected missing PDE p=%p/%RX64 uErr=%#x\n", pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
1844	if (uErr & X86_TRAP_PF_P)
1845	PGM_INVL_PG(pVCpu, GCPtrPage);
1846	return VINF_SUCCESS; /* force the instruction to be executed again. */
1847	}
1848
1849	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1850	Assert(pShwPage);
1851
1852	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1853	/* Fetch the pgm pool shadow descriptor. */
1854	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
1855	Assert(pShwPde);
1856	# endif
1857
1858	/*
1859	* Check that the page is present and that the shadow PDE isn't out of sync.
1860	*/
1861	const bool fBigPage = PdeSrc.b.u1Size && GST_IS_PSE_ACTIVE(pVCpu);
1862	const bool fPdeValid = !fBigPage ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc);
1863	RTGCPHYS GCPhys;
1864	if (!fBigPage)
1865	{
1866	GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
1867	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1868	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1869	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE / 2);
1870	# endif
1871	}
1872	else
1873	{
1874	GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
1875	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1876	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1877	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1878	# endif
1879	}
1880	/** @todo This doesn't check the G bit of 2/4MB pages. FIXME */
1881	if ( fPdeValid
1882	&& pShwPage->GCPhys == GCPhys
1883	&& PdeSrc.n.u1Present
1884	&& PdeSrc.n.u1User == PdeDst.n.u1User
1885	&& (PdeSrc.n.u1Write == PdeDst.n.u1Write \|\| !PdeDst.n.u1Write)
1886	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
1887	&& (PdeSrc.n.u1NoExecute == PdeDst.n.u1NoExecute \|\| !GST_IS_NX_ACTIVE(pVCpu))
1888	# endif
1889	)
1890	{
1891	/*
1892	* Check that the PDE is marked accessed already.
1893	* Since we set the accessed bit before getting here on a #PF, this
1894	* check is only meant for dealing with non-#PF'ing paths.
1895	*/
1896	if (PdeSrc.n.u1Accessed)
1897	{
1898	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1899	if (!fBigPage)
1900	{
1901	/*
1902	* 4KB Page - Map the guest page table.
1903	*/
1904	PGSTPT pPTSrc;
1905	int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
1906	if (RT_SUCCESS(rc))
1907	{
1908	# ifdef PGM_SYNC_N_PAGES
1909	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
1910	if ( cPages > 1
1911	&& !(uErr & X86_TRAP_PF_P)
1912	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
1913	{
1914	/*
1915	* This code path is currently only taken when the caller is PGMTrap0eHandler
1916	* for non-present pages!
1917	*
1918	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1919	* deal with locality.
1920	*/
1921	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1922	const unsigned iPTDstPage = iPTDst;
1923	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1924	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1925	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1926	# else
1927	const unsigned offPTSrc = 0;
1928	# endif
1929	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
1930	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1931	iPTDst = 0;
1932	else
1933	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1934
1935	/* Paranoia: the faulting page can't be present. We assume that below. */
1936	Assert(!SHW_PTE_IS_P(pPTDst->a[iPTDstPage]));
1937
1938	for (; iPTDst < iPTDstEnd; iPTDst++)
1939	{
1940	const PGSTPTE pPteSrc = &pPTSrc->a[offPTSrc + iPTDst];
1941
1942	if ( pPteSrc->n.u1Present
1943	&& !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
1944	{
1945	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
1946	NOREF(GCPtrCurPage);
1947	#ifndef IN_RING0
1948	/*
1949	* Assuming kernel code will be marked as supervisor - and not as user level
1950	* and executed using a conforming code selector - And marked as readonly.
1951	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
1952	*/
1953	PPGMPAGE pPage;
1954	if ( ((PdeSrc.u & pPteSrc->u) & (X86_PTE_RW \| X86_PTE_US))
1955	\|\| iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK) /* always sync GCPtrPage */
1956	\|\| !CSAMDoesPageNeedScanning(pVM, GCPtrCurPage)
1957	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, pPteSrc->u & GST_PTE_PG_MASK))
1958	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1959	)
1960	#endif /* else: CSAM not active */
1961	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, *pPteSrc, pShwPage, iPTDst);
1962	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1963	GCPtrCurPage, pPteSrc->n.u1Present,
1964	pPteSrc->n.u1Write & PdeSrc.n.u1Write,
1965	pPteSrc->n.u1User & PdeSrc.n.u1User,
1966	(uint64_t)pPteSrc->u,
1967	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
1968	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
1969	}
1970	}
1971	}
1972	else
1973	# endif /* PGM_SYNC_N_PAGES */
1974	{
1975	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1976	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1977	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1978	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1979	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
1980	GCPtrPage, PteSrc.n.u1Present,
1981	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1982	PteSrc.n.u1User & PdeSrc.n.u1User,
1983	(uint64_t)PteSrc.u,
1984	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
1985	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
1986	}
1987	}
1988	else /* MMIO or invalid page: emulated in #PF handler. */
1989	{
1990	LogFlow(("PGM_GCPHYS_2_PTR %RGp failed with %Rrc\n", GCPhys, rc));
1991	Assert(!SHW_PTE_IS_P(pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK]));
1992	}
1993	}
1994	else
1995	{
1996	/*
1997	* 4/2MB page - lazy syncing shadow 4K pages.
1998	* (There are many causes of getting here, it's no longer only CSAM.)
1999	*/
2000	/* Calculate the GC physical address of this 4KB shadow page. */
2001	GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc) \| (GCPtrPage & GST_BIG_PAGE_OFFSET_MASK);
2002	/* Find ram range. */
2003	PPGMPAGE pPage;
2004	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
2005	if (RT_SUCCESS(rc))
2006	{
2007	AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
2008
2009	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2010	/* Try to make the page writable if necessary. */
2011	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2012	&& ( PGM_PAGE_IS_ZERO(pPage)
2013	\|\| ( PdeSrc.n.u1Write
2014	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2015	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
2016	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
2017	# endif
2018	# ifdef VBOX_WITH_PAGE_SHARING
2019	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
2020	# endif
2021	)
2022	)
2023	)
2024	{
2025	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
2026	AssertRC(rc);
2027	}
2028	# endif
2029
2030	/*
2031	* Make shadow PTE entry.
2032	*/
2033	SHWPTE PteDst;
2034	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2035	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc), &PteDst);
2036	else
2037	SHW_PTE_SET(PteDst, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc) \| PGM_PAGE_GET_HCPHYS(pPage));
2038
2039	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2040	if ( SHW_PTE_IS_P(PteDst)
2041	&& !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2042	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2043
2044	/* Make sure only allocated pages are mapped writable. */
2045	if ( SHW_PTE_IS_P_RW(PteDst)
2046	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2047	{
2048	/* Still applies to shared pages. */
2049	Assert(!PGM_PAGE_IS_ZERO(pPage));
2050	SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
2051	Log3(("SyncPage: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, GCPtrPage));
2052	}
2053
2054	SHW_PTE_ATOMIC_SET2(pPTDst->a[iPTDst], PteDst);
2055
2056	/*
2057	* If the page is not flagged as dirty and is writable, then make it read-only
2058	* at PD level, so we can set the dirty bit when the page is modified.
2059	*
2060	* ASSUMES that page access handlers are implemented on page table entry level.
2061	* Thus we will first catch the dirty access and set PDE.D and restart. If
2062	* there is an access handler, we'll trap again and let it work on the problem.
2063	*/
2064	/** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
2065	* As for invlpg, it simply frees the whole shadow PT.
2066	* ...It's possibly because the guest clears it and the guest doesn't really tell us... */
2067	if ( !PdeSrc.b.u1Dirty
2068	&& PdeSrc.b.u1Write)
2069	{
2070	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2071	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2072	PdeDst.n.u1Write = 0;
2073	}
2074	else
2075	{
2076	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2077	PdeDst.n.u1Write = PdeSrc.n.u1Write;
2078	}
2079	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2080	Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n",
2081	GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
2082	PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2083	}
2084	else
2085	{
2086	LogFlow(("PGM_GCPHYS_2_PTR %RGp (big) failed with %Rrc\n", GCPhys, rc));
2087	/** @todo must wipe the shadow page table entry in this
2088	* case. */
2089	}
2090	}
2091	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2092	return VINF_SUCCESS;
2093	}
2094
2095	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPagePDNAs));
2096	}
2097	else if (fPdeValid)
2098	{
2099	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPagePDOutOfSync));
2100	Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2101	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2102	}
2103	else
2104	{
2105	/// @todo STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSyncAndInvalid));
2106	Log2(("SyncPage: Bad PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2107	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2108	}
2109
2110	/*
2111	* Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
2112	* Yea, I'm lazy.
2113	*/
2114	pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
2115	ASMAtomicWriteSize(pPdeDst, 0);
2116
2117	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2118	PGM_INVL_VCPU_TLBS(pVCpu);
2119	return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
2120
2121
2122	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2123	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2124	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2125	&& !defined(IN_RC)
2126
2127	# ifdef PGM_SYNC_N_PAGES
2128	/*
2129	* Get the shadow PDE, find the shadow page table in the pool.
2130	*/
2131	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2132	X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
2133
2134	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2135	X86PDEPAE PdeDst = pgmShwGetPaePDE(pVCpu, GCPtrPage);
2136
2137	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2138	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2139	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpt);
2140	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2141	X86PDEPAE PdeDst;
2142	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2143
2144	int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2145	AssertRCSuccessReturn(rc, rc);
2146	Assert(pPDDst && pPdptDst);
2147	PdeDst = pPDDst->a[iPDDst];
2148	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2149	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2150	PEPTPD pPDDst;
2151	EPTPDE PdeDst;
2152
2153	int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, NULL, &pPDDst);
2154	if (rc != VINF_SUCCESS)
2155	{
2156	AssertRC(rc);
2157	return rc;
2158	}
2159	Assert(pPDDst);
2160	PdeDst = pPDDst->a[iPDDst];
2161	# endif
2162	/* In the guest SMP case we could have blocked while another VCPU reused this page table. */
2163	if (!PdeDst.n.u1Present)
2164	{
2165	AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)PdeDst.u));
2166	Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", pVCpu->idCpu, GCPtrPage));
2167	return VINF_SUCCESS; /* force the instruction to be executed again. */
2168	}
2169
2170	/* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */
2171	if (PdeDst.n.u1Size)
2172	{
2173	Assert(pVM->pgm.s.fNestedPaging);
2174	Log(("CPU%d: SyncPage: Pde (big:%RX64) at %RGv changed behind our back!\n", pVCpu->idCpu, PdeDst.u, GCPtrPage));
2175	return VINF_SUCCESS;
2176	}
2177
2178	/* Mask away the page offset. */
2179	GCPtrPage &= ~((RTGCPTR)0xfff);
2180
2181	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2182	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2183
2184	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
2185	if ( cPages > 1
2186	&& !(uErr & X86_TRAP_PF_P)
2187	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2188	{
2189	/*
2190	* This code path is currently only taken when the caller is PGMTrap0eHandler
2191	* for non-present pages!
2192	*
2193	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2194	* deal with locality.
2195	*/
2196	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2197	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2198	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2199	iPTDst = 0;
2200	else
2201	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2202	for (; iPTDst < iPTDstEnd; iPTDst++)
2203	{
2204	if (!SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2205	{
2206	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT);
2207	GSTPTE PteSrc;
2208
2209	/* Fake the page table entry */
2210	PteSrc.u = GCPtrCurPage;
2211	PteSrc.n.u1Present = 1;
2212	PteSrc.n.u1Dirty = 1;
2213	PteSrc.n.u1Accessed = 1;
2214	PteSrc.n.u1Write = 1;
2215	PteSrc.n.u1User = 1;
2216
2217	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2218	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
2219	GCPtrCurPage, PteSrc.n.u1Present,
2220	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2221	PteSrc.n.u1User & PdeSrc.n.u1User,
2222	(uint64_t)PteSrc.u,
2223	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2224	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2225
2226	if (RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)))
2227	break;
2228	}
2229	else
2230	Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n", (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT), iPTDst, SHW_PTE_LOG64(pPTDst->a[iPTDst]) ));
2231	}
2232	}
2233	else
2234	# endif /* PGM_SYNC_N_PAGES */
2235	{
2236	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2237	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT);
2238	GSTPTE PteSrc;
2239
2240	/* Fake the page table entry */
2241	PteSrc.u = GCPtrCurPage;
2242	PteSrc.n.u1Present = 1;
2243	PteSrc.n.u1Dirty = 1;
2244	PteSrc.n.u1Accessed = 1;
2245	PteSrc.n.u1Write = 1;
2246	PteSrc.n.u1User = 1;
2247	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2248
2249	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}PteDst=%08llx%s\n",
2250	GCPtrPage, PteSrc.n.u1Present,
2251	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2252	PteSrc.n.u1User & PdeSrc.n.u1User,
2253	(uint64_t)PteSrc.u,
2254	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2255	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2256	}
2257	return VINF_SUCCESS;
2258
2259	#else
2260	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2261	return VERR_INTERNAL_ERROR;
2262	#endif
2263	}
2264
2265
2266	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2267
2268	/**
2269	* CheckPageFault helper for returning a page fault indicating a non-present
2270	* (NP) entry in the page translation structures.
2271	*
2272	* @returns VINF_EM_RAW_GUEST_TRAP.
2273	* @param pVCpu The virtual CPU to operate on.
2274	* @param uErr The error code of the shadow fault. Corrections to
2275	* TRPM's copy will be made if necessary.
2276	* @param GCPtrPage For logging.
2277	* @param uPageFaultLevel For logging.
2278	*/
2279	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnNP)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2280	{
2281	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2282	AssertMsg(!(uErr & X86_TRAP_PF_P), ("%#x\n", uErr));
2283	AssertMsg(!(uErr & X86_TRAP_PF_RSVD), ("%#x\n", uErr));
2284	if (uErr & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
2285	TRPMSetErrorCode(pVCpu, uErr & ~(X86_TRAP_PF_RSVD \| X86_TRAP_PF_P));
2286
2287	Log(("CheckPageFault: real page fault (notp) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2288	return VINF_EM_RAW_GUEST_TRAP;
2289	}
2290
2291
2292	/**
2293	* CheckPageFault helper for returning a page fault indicating a reserved bit
2294	* (RSVD) error in the page translation structures.
2295	*
2296	* @returns VINF_EM_RAW_GUEST_TRAP.
2297	* @param pVCpu The virtual CPU to operate on.
2298	* @param uErr The error code of the shadow fault. Corrections to
2299	* TRPM's copy will be made if necessary.
2300	* @param GCPtrPage For logging.
2301	* @param uPageFaultLevel For logging.
2302	*/
2303	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnRSVD)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2304	{
2305	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2306	if ((uErr & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) != (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
2307	TRPMSetErrorCode(pVCpu, uErr \| X86_TRAP_PF_RSVD \| X86_TRAP_PF_P);
2308
2309	Log(("CheckPageFault: real page fault (rsvd) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2310	return VINF_EM_RAW_GUEST_TRAP;
2311	}
2312
2313
2314	/**
2315	* CheckPageFault helper for returning a page protection fault (P).
2316	*
2317	* @returns VINF_EM_RAW_GUEST_TRAP.
2318	* @param pVCpu The virtual CPU to operate on.
2319	* @param uErr The error code of the shadow fault. Corrections to
2320	* TRPM's copy will be made if necessary.
2321	* @param GCPtrPage For logging.
2322	* @param uPageFaultLevel For logging.
2323	*/
2324	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnProt)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2325	{
2326	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2327	AssertMsg(uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID), ("%#x\n", uErr));
2328	if ((uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) != X86_TRAP_PF_P)
2329	TRPMSetErrorCode(pVCpu, (uErr & ~X86_TRAP_PF_RSVD) \| X86_TRAP_PF_P);
2330
2331	Log(("CheckPageFault: real page fault (prot) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2332	return VINF_EM_RAW_GUEST_TRAP;
2333	}
2334
2335
2336	/**
2337	* Handle dirty bit tracking faults.
2338	*
2339	* @returns VBox status code.
2340	* @param pVCpu The VMCPU handle.
2341	* @param uErr Page fault error code.
2342	* @param pPdeSrc Guest page directory entry.
2343	* @param pPdeDst Shadow page directory entry.
2344	* @param GCPtrPage Guest context page address.
2345	*/
2346	static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, RTGCPTR GCPtrPage)
2347	{
2348	PVM pVM = pVCpu->CTX_SUFF(pVM);
2349	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2350
2351	Assert(PGMIsLockOwner(pVM));
2352
2353	/*
2354	* Handle big page.
2355	*/
2356	if (pPdeSrc->b.u1Size && GST_IS_PSE_ACTIVE(pVCpu))
2357	{
2358	if ( pPdeDst->n.u1Present
2359	&& (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY))
2360	{
2361	SHWPDE PdeDst = *pPdeDst;
2362
2363	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageTrap));
2364	Assert(pPdeSrc->b.u1Write);
2365
2366	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
2367	* fault again and take this path to only invalidate the entry (see below).
2368	*/
2369	PdeDst.n.u1Write = 1;
2370	PdeDst.n.u1Accessed = 1;
2371	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2372	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2373	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
2374	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2375	}
2376
2377	# ifdef IN_RING0
2378	/* Check for stale TLB entry; only applies to the SMP guest case. */
2379	if ( pVM->cCpus > 1
2380	&& pPdeDst->n.u1Write
2381	&& pPdeDst->n.u1Accessed)
2382	{
2383	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2384	if (pShwPage)
2385	{
2386	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2387	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2388	if (SHW_PTE_IS_P_RW(*pPteDst))
2389	{
2390	/* Stale TLB entry. */
2391	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageStale));
2392	PGM_INVL_PG(pVCpu, GCPtrPage);
2393	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2394	}
2395	}
2396	}
2397	# endif /* IN_RING0 */
2398	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2399	}
2400
2401	/*
2402	* Map the guest page table.
2403	*/
2404	PGSTPT pPTSrc;
2405	int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(*pPdeSrc), &pPTSrc);
2406	if (RT_FAILURE(rc))
2407	{
2408	AssertRC(rc);
2409	return rc;
2410	}
2411
2412	if (pPdeDst->n.u1Present)
2413	{
2414	GSTPTE const *pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2415	const GSTPTE PteSrc = *pPteSrc;
2416
2417	#ifndef IN_RING0
2418	/* Bail out here as pgmPoolGetPage will return NULL and we'll crash below.
2419	* Our individual shadow handlers will provide more information and force a fatal exit.
2420	*/
2421	if (MMHyperIsInsideArea(pVM, (RTGCPTR)GCPtrPage))
2422	{
2423	LogRel(("CheckPageFault: write to hypervisor region %RGv\n", GCPtrPage));
2424	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2425	}
2426	#endif
2427	/*
2428	* Map shadow page table.
2429	*/
2430	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2431	if (pShwPage)
2432	{
2433	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2434	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2435	if (SHW_PTE_IS_P(pPteDst)) /* @todo Optimize accessed bit emulation? */
2436	{
2437	if (SHW_PTE_IS_TRACK_DIRTY(*pPteDst))
2438	{
2439	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GST_GET_PTE_GCPHYS(*pPteSrc));
2440	SHWPTE PteDst = *pPteDst;
2441
2442	LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage));
2443	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageTrap));
2444
2445	Assert(pPteSrc->n.u1Write);
2446
2447	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB
2448	* entry will not harm; write access will simply fault again and
2449	* take this path to only invalidate the entry.
2450	*/
2451	if (RT_LIKELY(pPage))
2452	{
2453	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2454	{
2455	AssertMsgFailed(("%R[pgmpage] - we don't set PGM_PTFLAGS_TRACK_DIRTY for these pages\n", pPage));
2456	Assert(!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage));
2457	/* Assuming write handlers here as the PTE is present (otherwise we wouldn't be here). */
2458	SHW_PTE_SET_RO(PteDst);
2459	}
2460	else
2461	{
2462	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
2463	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
2464	{
2465	rc = pgmPhysPageMakeWritable(pVM, pPage, GST_GET_PTE_GCPHYS(*pPteSrc));
2466	AssertRC(rc);
2467	}
2468	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED)
2469	SHW_PTE_SET_RW(PteDst);
2470	else
2471	{
2472	/* Still applies to shared pages. */
2473	Assert(!PGM_PAGE_IS_ZERO(pPage));
2474	SHW_PTE_SET_RO(PteDst);
2475	}
2476	}
2477	}
2478	else
2479	SHW_PTE_SET_RW(PteDst); /** @todo r=bird: This doesn't make sense to me. */
2480
2481	SHW_PTE_SET(PteDst, (SHW_PTE_GET_U(PteDst) \| X86_PTE_D \| X86_PTE_A) & ~(uint64_t)PGM_PTFLAGS_TRACK_DIRTY);
2482	SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
2483	PGM_INVL_PG(pVCpu, GCPtrPage);
2484	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2485	}
2486
2487	# ifdef IN_RING0
2488	/* Check for stale TLB entry; only applies to the SMP guest case. */
2489	if ( pVM->cCpus > 1
2490	&& SHW_PTE_IS_RW(*pPteDst)
2491	&& SHW_PTE_IS_A(*pPteDst))
2492	{
2493	/* Stale TLB entry. */
2494	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageStale));
2495	PGM_INVL_PG(pVCpu, GCPtrPage);
2496	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2497	}
2498	# endif
2499	}
2500	}
2501	else
2502	AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
2503	}
2504
2505	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2506	}
2507
2508	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
2509
2510
2511	/**
2512	* Sync a shadow page table.
2513	*
2514	* The shadow page table is not present in the shadow PDE.
2515	*
2516	* Handles mapping conflicts.
2517	*
2518	* This is called by VerifyAccessSyncPage, PrefetchPage, InvalidatePage (on
2519	* conflict), and Trap0eHandler.
2520	*
2521	* A precodition for this method is that the shadow PDE is not present. The
2522	* caller must take the PGM lock before checking this and continue to hold it
2523	* when calling this method.
2524	*
2525	* @returns VBox status code.
2526	* @param pVCpu The VMCPU handle.
2527	* @param iPD Page directory index.
2528	* @param pPDSrc Source page directory (i.e. Guest OS page directory).
2529	* Assume this is a temporary mapping.
2530	* @param GCPtrPage GC Pointer of the page that caused the fault
2531	*/
2532	static int PGM_BTH_NAME(SyncPT)(PVMCPU pVCpu, unsigned iPDSrc, PGSTPD pPDSrc, RTGCPTR GCPtrPage)
2533	{
2534	PVM pVM = pVCpu->CTX_SUFF(pVM);
2535	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2536
2537	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2538	#if 0 /* rarely useful; leave for debugging. */
2539	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPtPD[iPDSrc]);
2540	#endif
2541	LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage));
2542
2543	Assert(PGMIsLocked(pVM));
2544
2545	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2546	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
2547	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
2548	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2549	&& PGM_SHW_TYPE != PGM_TYPE_EPT
2550
2551	int rc = VINF_SUCCESS;
2552
2553	/*
2554	* Some input validation first.
2555	*/
2556	AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
2557
2558	/*
2559	* Get the relevant shadow PDE entry.
2560	*/
2561	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2562	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2563	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
2564
2565	/* Fetch the pgm pool shadow descriptor. */
2566	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2567	Assert(pShwPde);
2568
2569	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2570	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2571	PPGMPOOLPAGE pShwPde = NULL;
2572	PX86PDPAE pPDDst;
2573	PSHWPDE pPdeDst;
2574
2575	/* Fetch the pgm pool shadow descriptor. */
2576	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
2577	AssertRCSuccessReturn(rc, rc);
2578	Assert(pShwPde);
2579
2580	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
2581	pPdeDst = &pPDDst->a[iPDDst];
2582
2583	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2584	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2585	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2586	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2587	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2588	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2589	AssertRCSuccessReturn(rc, rc);
2590	Assert(pPDDst);
2591	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2592	# endif
2593	SHWPDE PdeDst = *pPdeDst;
2594
2595	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2596	/* Fetch the pgm pool shadow descriptor. */
2597	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2598	Assert(pShwPde);
2599	# endif
2600
2601	# ifndef PGM_WITHOUT_MAPPINGS
2602	/*
2603	* Check for conflicts.
2604	* RC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
2605	* R3: Simply resolve the conflict.
2606	*/
2607	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2608	{
2609	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
2610	# ifndef IN_RING3
2611	Log(("SyncPT: Conflict at %RGv\n", GCPtrPage));
2612	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2613	return VERR_ADDRESS_CONFLICT;
2614
2615	# else /* IN_RING3 */
2616	PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
2617	Assert(pMapping);
2618	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2619	rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2620	# elif PGM_GST_TYPE == PGM_TYPE_PAE
2621	rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2622	# else
2623	AssertFailed(); /* can't happen for amd64 */
2624	# endif
2625	if (RT_FAILURE(rc))
2626	{
2627	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2628	return rc;
2629	}
2630	PdeDst = *pPdeDst;
2631	# endif /* IN_RING3 */
2632	}
2633	# endif /* !PGM_WITHOUT_MAPPINGS */
2634	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2635
2636	/*
2637	* Sync the page directory entry.
2638	*/
2639	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2640	const bool fPageTable = !PdeSrc.b.u1Size \|\| !GST_IS_PSE_ACTIVE(pVCpu);
2641	if ( PdeSrc.n.u1Present
2642	&& (fPageTable ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc)) )
2643	{
2644	/*
2645	* Allocate & map the page table.
2646	*/
2647	PSHWPT pPTDst;
2648	PPGMPOOLPAGE pShwPage;
2649	RTGCPHYS GCPhys;
2650	if (fPageTable)
2651	{
2652	GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
2653	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2654	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2655	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE / 2);
2656	# endif
2657	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
2658	}
2659	else
2660	{
2661	PGMPOOLACCESS enmAccess;
2662	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2663	const bool fNoExecute = PdeSrc.n.u1NoExecute && GST_IS_NX_ACTIVE(pVCpu);
2664	# else
2665	const bool fNoExecute = false;
2666	# endif
2667
2668	GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
2669	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2670	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2671	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
2672	# endif
2673	/* Determine the right kind of large page to avoid incorrect cached entry reuse. */
2674	if (PdeSrc.n.u1User)
2675	{
2676	if (PdeSrc.n.u1Write)
2677	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_RW_NX : PGMPOOLACCESS_USER_RW;
2678	else
2679	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_R_NX : PGMPOOLACCESS_USER_R;
2680	}
2681	else
2682	{
2683	if (PdeSrc.n.u1Write)
2684	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW;
2685	else
2686	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_R_NX : PGMPOOLACCESS_SUPERVISOR_R;
2687	}
2688	rc = pgmPoolAllocEx(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, enmAccess, pShwPde->idx, iPDDst, false /fLockPage/,
2689	&pShwPage);
2690	}
2691	if (rc == VINF_SUCCESS)
2692	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2693	else if (rc == VINF_PGM_CACHED_PAGE)
2694	{
2695	/*
2696	* The PT was cached, just hook it up.
2697	*/
2698	if (fPageTable)
2699	PdeDst.u = pShwPage->Core.Key \| GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2700	else
2701	{
2702	PdeDst.u = pShwPage->Core.Key \| GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2703	/* (see explanation and assumptions further down.) */
2704	if ( !PdeSrc.b.u1Dirty
2705	&& PdeSrc.b.u1Write)
2706	{
2707	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2708	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2709	PdeDst.b.u1Write = 0;
2710	}
2711	}
2712	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2713	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2714	return VINF_SUCCESS;
2715	}
2716	else if (rc == VERR_PGM_POOL_FLUSHED)
2717	{
2718	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2719	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2720	return VINF_PGM_SYNC_CR3;
2721	}
2722	else
2723	AssertMsgFailedReturn(("rc=%Rrc\n", rc), VERR_INTERNAL_ERROR);
2724	/** @todo Why do we bother preserving X86_PDE_AVL_MASK here?
2725	* Both PGM_PDFLAGS_MAPPING and PGM_PDFLAGS_TRACK_DIRTY should be
2726	* irrelevant at this point. */
2727	PdeDst.u &= X86_PDE_AVL_MASK;
2728	PdeDst.u \|= pShwPage->Core.Key;
2729
2730	/*
2731	* Page directory has been accessed (this is a fault situation, remember).
2732	*/
2733	/** @todo
2734	* Well, when the caller is PrefetchPage or InvalidatePage is isn't a
2735	* fault situation. What's more, the Trap0eHandler has already set the
2736	* accessed bit. So, it's actually just VerifyAccessSyncPage which
2737	* might need setting the accessed flag.
2738	*
2739	* The best idea is to leave this change to the caller and add an
2740	* assertion that it's set already. */
2741	pPDSrc->a[iPDSrc].n.u1Accessed = 1;
2742	if (fPageTable)
2743	{
2744	/*
2745	* Page table - 4KB.
2746	*
2747	* Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
2748	*/
2749	Log2(("SyncPT: 4K %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
2750	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
2751	PGSTPT pPTSrc;
2752	rc = PGM_GCPHYS_2_PTR(pVM, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
2753	if (RT_SUCCESS(rc))
2754	{
2755	/*
2756	* Start by syncing the page directory entry so CSAM's TLB trick works.
2757	*/
2758	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| X86_PDE_AVL_MASK))
2759	\| GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2760	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2761	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2762
2763	/*
2764	* Directory/page user or supervisor privilege: (same goes for read/write)
2765	*
2766	* Directory Page Combined
2767	* U/S U/S U/S
2768	* 0 0 0
2769	* 0 1 0
2770	* 1 0 0
2771	* 1 1 1
2772	*
2773	* Simple AND operation. Table listed for completeness.
2774	*
2775	*/
2776	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT4K));
2777	# ifdef PGM_SYNC_N_PAGES
2778	unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2779	unsigned iPTDst = iPTBase;
2780	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2781	if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
2782	iPTDst = 0;
2783	else
2784	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2785	# else /* !PGM_SYNC_N_PAGES */
2786	unsigned iPTDst = 0;
2787	const unsigned iPTDstEnd = RT_ELEMENTS(pPTDst->a);
2788	# endif /* !PGM_SYNC_N_PAGES */
2789	RTGCPTR GCPtrCur = (GCPtrPage & ~(RTGCPTR)((1 << SHW_PD_SHIFT) - 1))
2790	\| ((RTGCPTR)iPTDst << PAGE_SHIFT);
2791	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2792	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2793	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2794	# else
2795	const unsigned offPTSrc = 0;
2796	# endif
2797	for (; iPTDst < iPTDstEnd; iPTDst++, GCPtrCur += PAGE_SIZE)
2798	{
2799	const unsigned iPTSrc = iPTDst + offPTSrc;
2800	const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2801
2802	if (PteSrc.n.u1Present)
2803	{
2804	# ifndef IN_RING0
2805	/*
2806	* Assuming kernel code will be marked as supervisor - and not as user level
2807	* and executed using a conforming code selector - And marked as readonly.
2808	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
2809	*/
2810	PPGMPAGE pPage;
2811	if ( ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW \| X86_PTE_US))
2812	\|\| !CSAMDoesPageNeedScanning(pVM, GCPtrCur)
2813	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, GST_GET_PTE_GCPHYS(PteSrc)))
2814	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2815	)
2816	# endif
2817	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2818	Log2(("SyncPT: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
2819	GCPtrCur,
2820	PteSrc.n.u1Present,
2821	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2822	PteSrc.n.u1User & PdeSrc.n.u1User,
2823	(uint64_t)PteSrc.u,
2824	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : "", SHW_PTE_LOG64(pPTDst->a[iPTDst]), iPTSrc, PdeSrc.au32[0],
2825	(RTGCPHYS)(GST_GET_PDE_GCPHYS(PdeSrc) + iPTSrc*sizeof(PteSrc)) ));
2826	}
2827	/* else: the page table was cleared by the pool */
2828	} /* for PTEs */
2829	}
2830	}
2831	else
2832	{
2833	/*
2834	* Big page - 2/4MB.
2835	*
2836	* We'll walk the ram range list in parallel and optimize lookups.
2837	* We will only sync on shadow page table at a time.
2838	*/
2839	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT4M));
2840
2841	/**
2842	* @todo It might be more efficient to sync only a part of the 4MB
2843	* page (similar to what we do for 4KB PDs).
2844	*/
2845
2846	/*
2847	* Start by syncing the page directory entry.
2848	*/
2849	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
2850	\| GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2851
2852	/*
2853	* If the page is not flagged as dirty and is writable, then make it read-only
2854	* at PD level, so we can set the dirty bit when the page is modified.
2855	*
2856	* ASSUMES that page access handlers are implemented on page table entry level.
2857	* Thus we will first catch the dirty access and set PDE.D and restart. If
2858	* there is an access handler, we'll trap again and let it work on the problem.
2859	*/
2860	/** @todo move the above stuff to a section in the PGM documentation. */
2861	Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
2862	if ( !PdeSrc.b.u1Dirty
2863	&& PdeSrc.b.u1Write)
2864	{
2865	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2866	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2867	PdeDst.b.u1Write = 0;
2868	}
2869	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2870	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2871
2872	/*
2873	* Fill the shadow page table.
2874	*/
2875	/* Get address and flags from the source PDE. */
2876	SHWPTE PteDstBase;
2877	SHW_PTE_SET(PteDstBase, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc));
2878
2879	/* Loop thru the entries in the shadow PT. */
2880	const RTGCPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
2881	Log2(("SyncPT: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n",
2882	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
2883	GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2884	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
2885	unsigned iPTDst = 0;
2886	while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2887	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2888	{
2889	/* Advance ram range list. */
2890	while (pRam && GCPhys > pRam->GCPhysLast)
2891	pRam = pRam->CTX_SUFF(pNext);
2892	if (pRam && GCPhys >= pRam->GCPhys)
2893	{
2894	unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2895	do
2896	{
2897	/* Make shadow PTE. */
2898	PPGMPAGE pPage = &pRam->aPages[iHCPage];
2899	SHWPTE PteDst;
2900
2901	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2902	/* Try to make the page writable if necessary. */
2903	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2904	&& ( PGM_PAGE_IS_ZERO(pPage)
2905	\|\| ( SHW_PTE_IS_RW(PteDstBase)
2906	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2907	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
2908	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
2909	# endif
2910	# ifdef VBOX_WITH_PAGE_SHARING
2911	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
2912	# endif
2913	&& !PGM_PAGE_IS_BALLOONED(pPage))
2914	)
2915	)
2916	{
2917	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
2918	AssertRCReturn(rc, rc);
2919	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2920	break;
2921	}
2922	# endif
2923
2924	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2925	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, SHW_PTE_GET_U(PteDstBase), &PteDst);
2926	else if (PGM_PAGE_IS_BALLOONED(pPage))
2927	SHW_PTE_SET(PteDst, 0); /* Handle ballooned pages at #PF time. */
2928	# ifndef IN_RING0
2929	/*
2930	* Assuming kernel code will be marked as supervisor and not as user level and executed
2931	* using a conforming code selector. Don't check for readonly, as that implies the whole
2932	* 4MB can be code or readonly data. Linux enables write access for its large pages.
2933	*/
2934	else if ( !PdeSrc.n.u1User
2935	&& CSAMDoesPageNeedScanning(pVM, GCPtr \| (iPTDst << SHW_PT_SHIFT)))
2936	SHW_PTE_SET(PteDst, 0);
2937	# endif
2938	else
2939	SHW_PTE_SET(PteDst, PGM_PAGE_GET_HCPHYS(pPage) \| SHW_PTE_GET_U(PteDstBase));
2940
2941	/* Only map writable pages writable. */
2942	if ( SHW_PTE_IS_P_RW(PteDst)
2943	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2944	{
2945	/* Still applies to shared pages. */
2946	Assert(!PGM_PAGE_IS_ZERO(pPage));
2947	SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
2948	Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))));
2949	}
2950
2951	if (SHW_PTE_IS_P(PteDst))
2952	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2953
2954	/* commit it (not atomic, new table) */
2955	pPTDst->a[iPTDst] = PteDst;
2956	Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
2957	(RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT)), SHW_PTE_IS_P(PteDst), SHW_PTE_IS_RW(PteDst), SHW_PTE_IS_US(PteDst), SHW_PTE_LOG64(PteDst),
2958	SHW_PTE_IS_TRACK_DIRTY(PteDst) ? " Track-Dirty" : ""));
2959
2960	/* advance */
2961	GCPhys += PAGE_SIZE;
2962	iHCPage++;
2963	iPTDst++;
2964	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2965	&& GCPhys <= pRam->GCPhysLast);
2966	}
2967	else if (pRam)
2968	{
2969	Log(("Invalid pages at %RGp\n", GCPhys));
2970	do
2971	{
2972	SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
2973	GCPhys += PAGE_SIZE;
2974	iPTDst++;
2975	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2976	&& GCPhys < pRam->GCPhys);
2977	}
2978	else
2979	{
2980	Log(("Invalid pages at %RGp (2)\n", GCPhys));
2981	for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
2982	SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
2983	}
2984	} /* while more PTEs */
2985	} /* 4KB / 4MB */
2986	}
2987	else
2988	AssertRelease(!PdeDst.n.u1Present);
2989
2990	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2991	if (RT_FAILURE(rc))
2992	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPTFailed));
2993	return rc;
2994
2995	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2996	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2997	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2998	&& !defined(IN_RC)
2999
3000	/*
3001	* Validate input a little bit.
3002	*/
3003	int rc = VINF_SUCCESS;
3004	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3005	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3006	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3007
3008	/* Fetch the pgm pool shadow descriptor. */
3009	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
3010	Assert(pShwPde);
3011
3012	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3013	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3014	PPGMPOOLPAGE pShwPde = NULL; /* initialized to shut up gcc */
3015	PX86PDPAE pPDDst;
3016	PSHWPDE pPdeDst;
3017
3018	/* Fetch the pgm pool shadow descriptor. */
3019	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
3020	AssertRCSuccessReturn(rc, rc);
3021	Assert(pShwPde);
3022
3023	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
3024	pPdeDst = &pPDDst->a[iPDDst];
3025
3026	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3027	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3028	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3029	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
3030	PX86PDPT pPdptDst= NULL; /* initialized to shut up gcc */
3031	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
3032	AssertRCSuccessReturn(rc, rc);
3033	Assert(pPDDst);
3034	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3035
3036	/* Fetch the pgm pool shadow descriptor. */
3037	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
3038	Assert(pShwPde);
3039
3040	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
3041	const unsigned iPdpt = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
3042	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3043	PEPTPD pPDDst;
3044	PEPTPDPT pPdptDst;
3045
3046	rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, &pPdptDst, &pPDDst);
3047	if (rc != VINF_SUCCESS)
3048	{
3049	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3050	AssertRC(rc);
3051	return rc;
3052	}
3053	Assert(pPDDst);
3054	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3055
3056	/* Fetch the pgm pool shadow descriptor. */
3057	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & EPT_PDPTE_PG_MASK);
3058	Assert(pShwPde);
3059	# endif
3060	SHWPDE PdeDst = *pPdeDst;
3061
3062	Assert(!(PdeDst.u & PGM_PDFLAGS_MAPPING));
3063	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
3064
3065	# if defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
3066	if (BTH_IS_NP_ACTIVE(pVM))
3067	{
3068	PPGMPAGE pPage;
3069
3070	/* Check if we allocated a big page before for this 2 MB range. */
3071	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPtrPage & X86_PDE2M_PAE_PG_MASK, &pPage);
3072	if (RT_SUCCESS(rc))
3073	{
3074	RTHCPHYS HCPhys = NIL_RTHCPHYS;
3075
3076	if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
3077	{
3078	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageReused);
3079	AssertRelease(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3080	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3081	}
3082	else if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED)
3083	{
3084	/* Recheck the entire 2 MB range to see if we can use it again as a large page. */
3085	rc = pgmPhysIsValidLargePage(pVM, GCPtrPage, pPage);
3086	if (RT_SUCCESS(rc))
3087	{
3088	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3089	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3090	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3091	}
3092	}
3093	else if (PGMIsUsingLargePages(pVM))
3094	{
3095	rc = pgmPhysAllocLargePage(pVM, GCPtrPage);
3096	if (RT_SUCCESS(rc))
3097	{
3098	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3099	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3100	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3101	}
3102	else
3103	LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
3104	}
3105
3106	if (HCPhys != NIL_RTHCPHYS)
3107	{
3108	PdeDst.u &= X86_PDE_AVL_MASK;
3109	PdeDst.u \|= HCPhys;
3110	PdeDst.n.u1Present = 1;
3111	PdeDst.n.u1Write = 1;
3112	PdeDst.b.u1Size = 1;
3113	# if PGM_SHW_TYPE == PGM_TYPE_EPT
3114	PdeDst.n.u1Execute = 1;
3115	PdeDst.b.u1IgnorePAT = 1;
3116	PdeDst.b.u3EMT = VMX_EPT_MEMTYPE_WB;
3117	# else
3118	PdeDst.n.u1User = 1;
3119	# endif
3120	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
3121
3122	Log(("SyncPT: Use large page at %RGp PDE=%RX64\n", GCPtrPage, PdeDst.u));
3123	/* Add a reference to the first page only. */
3124	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPde, PGM_PAGE_GET_TRACKING(pPage), pPage, iPDDst);
3125
3126	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3127	return VINF_SUCCESS;
3128	}
3129	}
3130	}
3131	# endif /* HC_ARCH_BITS == 64 */
3132
3133	GSTPDE PdeSrc;
3134	PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3135	PdeSrc.n.u1Present = 1;
3136	PdeSrc.n.u1Write = 1;
3137	PdeSrc.n.u1Accessed = 1;
3138	PdeSrc.n.u1User = 1;
3139
3140	/*
3141	* Allocate & map the page table.
3142	*/
3143	PSHWPT pPTDst;
3144	PPGMPOOLPAGE pShwPage;
3145	RTGCPHYS GCPhys;
3146
3147	/* Virtual address = physical address */
3148	GCPhys = GCPtrPage & X86_PAGE_4K_BASE_MASK;
3149	rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
3150
3151	if ( rc == VINF_SUCCESS
3152	\|\| rc == VINF_PGM_CACHED_PAGE)
3153	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3154	else
3155	AssertMsgFailedReturn(("rc=%Rrc\n", rc), VERR_INTERNAL_ERROR);
3156
3157	PdeDst.u &= X86_PDE_AVL_MASK;
3158	PdeDst.u \|= pShwPage->Core.Key;
3159	PdeDst.n.u1Present = 1;
3160	PdeDst.n.u1Write = 1;
3161	# if PGM_SHW_TYPE == PGM_TYPE_EPT
3162	PdeDst.n.u1Execute = 1;
3163	# else
3164	PdeDst.n.u1User = 1;
3165	PdeDst.n.u1Accessed = 1;
3166	# endif
3167	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
3168
3169	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, PGM_SYNC_NR_PAGES, 0 /* page not present */);
3170	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3171	return rc;
3172
3173	#else
3174	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
3175	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3176	return VERR_INTERNAL_ERROR;
3177	#endif
3178	}
3179
3180
3181
3182	/**
3183	* Prefetch a page/set of pages.
3184	*
3185	* Typically used to sync commonly used pages before entering raw mode
3186	* after a CR3 reload.
3187	*
3188	* @returns VBox status code.
3189	* @param pVCpu The VMCPU handle.
3190	* @param GCPtrPage Page to invalidate.
3191	*/
3192	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
3193	{
3194	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3195	\|\| PGM_GST_TYPE == PGM_TYPE_REAL \
3196	\|\| PGM_GST_TYPE == PGM_TYPE_PROT \
3197	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3198	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3199	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
3200	&& PGM_SHW_TYPE != PGM_TYPE_EPT
3201
3202	/*
3203	* Check that all Guest levels thru the PDE are present, getting the
3204	* PD and PDE in the processes.
3205	*/
3206	int rc = VINF_SUCCESS;
3207	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3208	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3209	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
3210	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3211	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3212	unsigned iPDSrc;
3213	X86PDPE PdpeSrc;
3214	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3215	if (!pPDSrc)
3216	return VINF_SUCCESS; /* not present */
3217	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3218	unsigned iPDSrc;
3219	PX86PML4E pPml4eSrc;
3220	X86PDPE PdpeSrc;
3221	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3222	if (!pPDSrc)
3223	return VINF_SUCCESS; /* not present */
3224	# endif
3225	const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3226	# else
3227	PGSTPD pPDSrc = NULL;
3228	const unsigned iPDSrc = 0;
3229	GSTPDE PdeSrc;
3230
3231	PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3232	PdeSrc.n.u1Present = 1;
3233	PdeSrc.n.u1Write = 1;
3234	PdeSrc.n.u1Accessed = 1;
3235	PdeSrc.n.u1User = 1;
3236	# endif
3237
3238	if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
3239	{
3240	PVM pVM = pVCpu->CTX_SUFF(pVM);
3241	pgmLock(pVM);
3242
3243	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3244	const X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
3245	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3246	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3247	PX86PDPAE pPDDst;
3248	X86PDEPAE PdeDst;
3249	# if PGM_GST_TYPE != PGM_TYPE_PAE
3250	X86PDPE PdpeSrc;
3251
3252	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3253	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3254	# endif
3255	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3256	if (rc != VINF_SUCCESS)
3257	{
3258	pgmUnlock(pVM);
3259	AssertRC(rc);
3260	return rc;
3261	}
3262	Assert(pPDDst);
3263	PdeDst = pPDDst->a[iPDDst];
3264
3265	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3266	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3267	PX86PDPAE pPDDst;
3268	X86PDEPAE PdeDst;
3269
3270	# if PGM_GST_TYPE == PGM_TYPE_PROT
3271	/* AMD-V nested paging */
3272	X86PML4E Pml4eSrc;
3273	X86PDPE PdpeSrc;
3274	PX86PML4E pPml4eSrc = &Pml4eSrc;
3275
3276	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3277	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
3278	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
3279	# endif
3280
3281	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3282	if (rc != VINF_SUCCESS)
3283	{
3284	pgmUnlock(pVM);
3285	AssertRC(rc);
3286	return rc;
3287	}
3288	Assert(pPDDst);
3289	PdeDst = pPDDst->a[iPDDst];
3290	# endif
3291	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
3292	{
3293	if (!PdeDst.n.u1Present)
3294	{
3295	/** @todo r=bird: This guy will set the A bit on the PDE,
3296	* probably harmless. */
3297	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3298	}
3299	else
3300	{
3301	/* Note! We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
3302	* R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
3303	* makes no sense to prefetch more than one page.
3304	*/
3305	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3306	if (RT_SUCCESS(rc))
3307	rc = VINF_SUCCESS;
3308	}
3309	}
3310	pgmUnlock(pVM);
3311	}
3312	return rc;
3313
3314	#elif PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3315	return VINF_SUCCESS; /* ignore */
3316	#else
3317	AssertCompile(0);
3318	#endif
3319	}
3320
3321
3322
3323
3324	/**
3325	* Syncs a page during a PGMVerifyAccess() call.
3326	*
3327	* @returns VBox status code (informational included).
3328	* @param pVCpu The VMCPU handle.
3329	* @param GCPtrPage The address of the page to sync.
3330	* @param fPage The effective guest page flags.
3331	* @param uErr The trap error code.
3332	* @remarks This will normally never be called on invalid guest page
3333	* translation entries.
3334	*/
3335	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fPage, unsigned uErr)
3336	{
3337	PVM pVM = pVCpu->CTX_SUFF(pVM);
3338
3339	LogFlow(("VerifyAccessSyncPage: GCPtrPage=%RGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
3340
3341	Assert(!pVM->pgm.s.fNestedPaging);
3342	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3343	\|\| PGM_GST_TYPE == PGM_TYPE_REAL \
3344	\|\| PGM_GST_TYPE == PGM_TYPE_PROT \
3345	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3346	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3347	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
3348	&& PGM_SHW_TYPE != PGM_TYPE_EPT
3349
3350	# ifndef IN_RING0
3351	if (!(fPage & X86_PTE_US))
3352	{
3353	/*
3354	* Mark this page as safe.
3355	*/
3356	/** @todo not correct for pages that contain both code and data!! */
3357	Log(("CSAMMarkPage %RGv; scanned=%d\n", GCPtrPage, true));
3358	CSAMMarkPage(pVM, GCPtrPage, true);
3359	}
3360	# endif
3361
3362	/*
3363	* Get guest PD and index.
3364	*/
3365	/** @todo Performance: We've done all this a jiffy ago in the
3366	* PGMGstGetPage call. */
3367	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3368	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3369	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
3370	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3371
3372	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3373	unsigned iPDSrc = 0;
3374	X86PDPE PdpeSrc;
3375	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3376	if (RT_UNLIKELY(!pPDSrc))
3377	{
3378	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3379	return VINF_EM_RAW_GUEST_TRAP;
3380	}
3381
3382	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3383	unsigned iPDSrc = 0; /* shut up gcc */
3384	PX86PML4E pPml4eSrc = NULL; /* ditto */
3385	X86PDPE PdpeSrc;
3386	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3387	if (RT_UNLIKELY(!pPDSrc))
3388	{
3389	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3390	return VINF_EM_RAW_GUEST_TRAP;
3391	}
3392	# endif
3393
3394	# else /* !PGM_WITH_PAGING */
3395	PGSTPD pPDSrc = NULL;
3396	const unsigned iPDSrc = 0;
3397	# endif /* !PGM_WITH_PAGING */
3398	int rc = VINF_SUCCESS;
3399
3400	pgmLock(pVM);
3401
3402	/*
3403	* First check if the shadow pd is present.
3404	*/
3405	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3406	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3407
3408	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3409	PX86PDEPAE pPdeDst;
3410	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3411	PX86PDPAE pPDDst;
3412	# if PGM_GST_TYPE != PGM_TYPE_PAE
3413	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3414	X86PDPE PdpeSrc;
3415	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3416	# endif
3417	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3418	if (rc != VINF_SUCCESS)
3419	{
3420	pgmUnlock(pVM);
3421	AssertRC(rc);
3422	return rc;
3423	}
3424	Assert(pPDDst);
3425	pPdeDst = &pPDDst->a[iPDDst];
3426
3427	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3428	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3429	PX86PDPAE pPDDst;
3430	PX86PDEPAE pPdeDst;
3431
3432	# if PGM_GST_TYPE == PGM_TYPE_PROT
3433	/* AMD-V nested paging: Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3434	X86PML4E Pml4eSrc;
3435	X86PDPE PdpeSrc;
3436	PX86PML4E pPml4eSrc = &Pml4eSrc;
3437	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
3438	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
3439	# endif
3440
3441	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3442	if (rc != VINF_SUCCESS)
3443	{
3444	pgmUnlock(pVM);
3445	AssertRC(rc);
3446	return rc;
3447	}
3448	Assert(pPDDst);
3449	pPdeDst = &pPDDst->a[iPDDst];
3450	# endif
3451
3452	if (!pPdeDst->n.u1Present)
3453	{
3454	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3455	if (rc != VINF_SUCCESS)
3456	{
3457	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3458	pgmUnlock(pVM);
3459	AssertRC(rc);
3460	return rc;
3461	}
3462	}
3463
3464	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3465	/* Check for dirty bit fault */
3466	rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
3467	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
3468	Log(("PGMVerifyAccess: success (dirty)\n"));
3469	else
3470	# endif
3471	{
3472	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3473	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3474	# else
3475	GSTPDE PdeSrc;
3476	PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3477	PdeSrc.n.u1Present = 1;
3478	PdeSrc.n.u1Write = 1;
3479	PdeSrc.n.u1Accessed = 1;
3480	PdeSrc.n.u1User = 1;
3481	# endif
3482
3483	Assert(rc != VINF_EM_RAW_GUEST_TRAP);
3484	if (uErr & X86_TRAP_PF_US)
3485	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUser));
3486	else /* supervisor */
3487	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
3488
3489	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3490	if (RT_SUCCESS(rc))
3491	{
3492	/* Page was successfully synced */
3493	Log2(("PGMVerifyAccess: success (sync)\n"));
3494	rc = VINF_SUCCESS;
3495	}
3496	else
3497	{
3498	Log(("PGMVerifyAccess: access violation for %RGv rc=%Rrc\n", GCPtrPage, rc));
3499	rc = VINF_EM_RAW_GUEST_TRAP;
3500	}
3501	}
3502	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3503	pgmUnlock(pVM);
3504	return rc;
3505
3506	#else /* PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED */
3507
3508	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
3509	return VERR_INTERNAL_ERROR;
3510	#endif /* PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED */
3511	}
3512
3513
3514	/**
3515	* Syncs the paging hierarchy starting at CR3.
3516	*
3517	* @returns VBox status code, no specials.
3518	* @param pVCpu The VMCPU handle.
3519	* @param cr0 Guest context CR0 register
3520	* @param cr3 Guest context CR3 register
3521	* @param cr4 Guest context CR4 register
3522	* @param fGlobal Including global page directories or not
3523	*/
3524	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
3525	{
3526	PVM pVM = pVCpu->CTX_SUFF(pVM);
3527
3528	LogFlow(("SyncCR3 fGlobal=%d\n", !!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)));
3529
3530	#if PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
3531
3532	pgmLock(pVM);
3533
3534	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
3535	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3536	if (pPool->cDirtyPages)
3537	pgmPoolResetDirtyPages(pVM);
3538	# endif
3539
3540	/*
3541	* Update page access handlers.
3542	* The virtual are always flushed, while the physical are only on demand.
3543	* WARNING: We are incorrectly not doing global flushing on Virtual Handler updates. We'll
3544	* have to look into that later because it will have a bad influence on the performance.
3545	* @note SvL: There's no need for that. Just invalidate the virtual range(s).
3546	* bird: Yes, but that won't work for aliases.
3547	*/
3548	/** @todo this MUST go away. See #1557. */
3549	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3Handlers), h);
3550	PGM_GST_NAME(HandlerVirtualUpdate)(pVM, cr4);
3551	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3Handlers), h);
3552	pgmUnlock(pVM);
3553	#endif /* !NESTED && !EPT */
3554
3555	#if PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3556	/*
3557	* Nested / EPT - almost no work.
3558	*/
3559	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
3560	return VINF_SUCCESS;
3561
3562	#elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3563	/*
3564	* AMD64 (Shw & Gst) - No need to check all paging levels; we zero
3565	* out the shadow parts when the guest modifies its tables.
3566	*/
3567	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
3568	return VINF_SUCCESS;
3569
3570	#else /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3571
3572	# ifndef PGM_WITHOUT_MAPPINGS
3573	/*
3574	* Check for and resolve conflicts with our guest mappings if they
3575	* are enabled and not fixed.
3576	*/
3577	if (pgmMapAreMappingsFloating(&pVM->pgm.s))
3578	{
3579	int rc = pgmMapResolveConflicts(pVM);
3580	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
3581	if (rc == VINF_PGM_SYNC_CR3)
3582	{
3583	LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
3584	return VINF_PGM_SYNC_CR3;
3585	}
3586	}
3587	# else
3588	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
3589	# endif
3590	return VINF_SUCCESS;
3591	#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3592	}
3593
3594
3595
3596
3597	#ifdef VBOX_STRICT
3598	# ifdef IN_RC
3599	# undef AssertMsgFailed
3600	# define AssertMsgFailed Log
3601	# endif
3602
3603	/**
3604	* Checks that the shadow page table is in sync with the guest one.
3605	*
3606	* @returns The number of errors.
3607	* @param pVM The virtual machine.
3608	* @param pVCpu The VMCPU handle.
3609	* @param cr3 Guest context CR3 register
3610	* @param cr4 Guest context CR4 register
3611	* @param GCPtr Where to start. Defaults to 0.
3612	* @param cb How much to check. Defaults to everything.
3613	*/
3614	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)
3615	{
3616	#if PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3617	return 0;
3618	#else
3619	unsigned cErrors = 0;
3620	PVM pVM = pVCpu->CTX_SUFF(pVM);
3621	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3622
3623	#if PGM_GST_TYPE == PGM_TYPE_PAE
3624	/** @todo currently broken; crashes below somewhere */
3625	AssertFailed();
3626	#endif
3627
3628	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
3629	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3630	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
3631
3632	bool fBigPagesSupported = GST_IS_PSE_ACTIVE(pVCpu);
3633	PPGMCPU pPGM = &pVCpu->pgm.s;
3634	RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
3635	RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
3636	# ifndef IN_RING0
3637	RTHCPHYS HCPhys; /* general usage. */
3638	# endif
3639	int rc;
3640
3641	/*
3642	* Check that the Guest CR3 and all its mappings are correct.
3643	*/
3644	AssertMsgReturn(pPGM->GCPhysCR3 == (cr3 & GST_CR3_PAGE_MASK),
3645	("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
3646	false);
3647	# if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64
3648	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3649	rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGst32BitPdRC, NULL, &HCPhysShw);
3650	# else
3651	rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGstPaePdptRC, NULL, &HCPhysShw);
3652	# endif
3653	AssertRCReturn(rc, 1);
3654	HCPhys = NIL_RTHCPHYS;
3655	rc = pgmRamGCPhys2HCPhys(&pVM->pgm.s, cr3 & GST_CR3_PAGE_MASK, &HCPhys);
3656	AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
3657	# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
3658	pgmGstGet32bitPDPtr(pVCpu);
3659	RTGCPHYS GCPhys;
3660	rc = PGMR3DbgR3Ptr2GCPhys(pVM, pPGM->pGst32BitPdR3, &GCPhys);
3661	AssertRCReturn(rc, 1);
3662	AssertMsgReturn((cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
3663	# endif
3664	# endif /* !IN_RING0 */
3665
3666	/*
3667	* Get and check the Shadow CR3.
3668	*/
3669	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3670	unsigned cPDEs = X86_PG_ENTRIES;
3671	unsigned cIncrement = X86_PG_ENTRIES * PAGE_SIZE;
3672	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3673	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3674	unsigned cPDEs = X86_PG_PAE_ENTRIES * 4; /* treat it as a 2048 entry table. */
3675	# else
3676	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3677	# endif
3678	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3679	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3680	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3681	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3682	# endif
3683	if (cb != ~(RTGCPTR)0)
3684	cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
3685
3686	/** @todo call the other two PGMAssert() functions. /
3687
3688	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3689	unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3690
3691	for (; iPml4 < X86_PG_PAE_ENTRIES; iPml4++)
3692	{
3693	PPGMPOOLPAGE pShwPdpt = NULL;
3694	PX86PML4E pPml4eSrc;
3695	PX86PML4E pPml4eDst;
3696	RTGCPHYS GCPhysPdptSrc;
3697
3698	pPml4eSrc = pgmGstGetLongModePML4EPtr(pVCpu, iPml4);
3699	pPml4eDst = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
3700
3701	/* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
3702	if (!pPml4eDst->n.u1Present)
3703	{
3704	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3705	continue;
3706	}
3707
3708	pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
3709	GCPhysPdptSrc = pPml4eSrc->u & X86_PML4E_PG_MASK;
3710
3711	if (pPml4eSrc->n.u1Present != pPml4eDst->n.u1Present)
3712	{
3713	AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3714	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3715	cErrors++;
3716	continue;
3717	}
3718
3719	if (GCPhysPdptSrc != pShwPdpt->GCPhys)
3720	{
3721	AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
3722	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3723	cErrors++;
3724	continue;
3725	}
3726
3727	if ( pPml4eDst->n.u1User != pPml4eSrc->n.u1User
3728	\|\| pPml4eDst->n.u1Write != pPml4eSrc->n.u1Write
3729	\|\| pPml4eDst->n.u1NoExecute != pPml4eSrc->n.u1NoExecute)
3730	{
3731	AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3732	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3733	cErrors++;
3734	continue;
3735	}
3736	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3737	{
3738	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3739
3740	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
3741	/*
3742	* Check the PDPTEs too.
3743	*/
3744	unsigned iPdpt = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
3745
3746	for (;iPdpt <= SHW_PDPT_MASK; iPdpt++)
3747	{
3748	unsigned iPDSrc = 0; /* initialized to shut up gcc */
3749	PPGMPOOLPAGE pShwPde = NULL;
3750	PX86PDPE pPdpeDst;
3751	RTGCPHYS GCPhysPdeSrc;
3752	# if PGM_GST_TYPE == PGM_TYPE_PAE
3753	X86PDPE PdpeSrc;
3754	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtr, &iPDSrc, &PdpeSrc);
3755	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
3756	# else
3757	PX86PML4E pPml4eSrcIgn;
3758	X86PDPE PdpeSrc;
3759	PX86PDPT pPdptDst;
3760	PX86PDPAE pPDDst;
3761	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtr, &pPml4eSrcIgn, &PdpeSrc, &iPDSrc);
3762
3763	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtr, NULL, &pPdptDst, &pPDDst);
3764	if (rc != VINF_SUCCESS)
3765	{
3766	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
3767	GCPtr += 512 * _2M;
3768	continue; /* next PDPTE */
3769	}
3770	Assert(pPDDst);
3771	# endif
3772	Assert(iPDSrc == 0);
3773
3774	pPdpeDst = &pPdptDst->a[iPdpt];
3775
3776	if (!pPdpeDst->n.u1Present)
3777	{
3778	GCPtr += 512 * _2M;
3779	continue; /* next PDPTE */
3780	}
3781
3782	pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
3783	GCPhysPdeSrc = PdpeSrc.u & X86_PDPE_PG_MASK;
3784
3785	if (pPdpeDst->n.u1Present != PdpeSrc.n.u1Present)
3786	{
3787	AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3788	GCPtr += 512 * _2M;
3789	cErrors++;
3790	continue;
3791	}
3792
3793	if (GCPhysPdeSrc != pShwPde->GCPhys)
3794	{
3795	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3796	AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3797	# else
3798	AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3799	# endif
3800	GCPtr += 512 * _2M;
3801	cErrors++;
3802	continue;
3803	}
3804
3805	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3806	if ( pPdpeDst->lm.u1User != PdpeSrc.lm.u1User
3807	\|\| pPdpeDst->lm.u1Write != PdpeSrc.lm.u1Write
3808	\|\| pPdpeDst->lm.u1NoExecute != PdpeSrc.lm.u1NoExecute)
3809	{
3810	AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3811	GCPtr += 512 * _2M;
3812	cErrors++;
3813	continue;
3814	}
3815	# endif
3816
3817	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3818	{
3819	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3820	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3821	GSTPD const *pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3822	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3823	PCX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
3824	# endif
3825	# endif /* PGM_GST_TYPE == PGM_TYPE_32BIT */
3826	/*
3827	* Iterate the shadow page directory.
3828	*/
3829	GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
3830	unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
3831
3832	for (;
3833	iPDDst < cPDEs;
3834	iPDDst++, GCPtr += cIncrement)
3835	{
3836	# if PGM_SHW_TYPE == PGM_TYPE_PAE
3837	const SHWPDE PdeDst = *pgmShwGetPaePDEPtr(pVCpu, GCPtr);
3838	# else
3839	const SHWPDE PdeDst = pPDDst->a[iPDDst];
3840	# endif
3841	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
3842	{
3843	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
3844	if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
3845	{
3846	AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
3847	cErrors++;
3848	continue;
3849	}
3850	}
3851	else if ( (PdeDst.u & X86_PDE_P)
3852	\|\| ((PdeDst.u & (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY))
3853	)
3854	{
3855	HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
3856	PPGMPOOLPAGE pPoolPage = pgmPoolGetPage(pPool, HCPhysShw);
3857	if (!pPoolPage)
3858	{
3859	AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n",
3860	HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
3861	cErrors++;
3862	continue;
3863	}
3864	const SHWPT pPTDst = (const SHWPT )PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pPoolPage);
3865
3866	if (PdeDst.u & (X86_PDE4M_PWT \| X86_PDE4M_PCD))
3867	{
3868	AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
3869	GCPtr, (uint64_t)PdeDst.u));
3870	cErrors++;
3871	}
3872
3873	if (PdeDst.u & (X86_PDE4M_G \| X86_PDE4M_D))
3874	{
3875	AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n",
3876	GCPtr, (uint64_t)PdeDst.u));
3877	cErrors++;
3878	}
3879
3880	const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
3881	if (!PdeSrc.n.u1Present)
3882	{
3883	AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
3884	GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
3885	cErrors++;
3886	continue;
3887	}
3888
3889	if ( !PdeSrc.b.u1Size
3890	\|\| !fBigPagesSupported)
3891	{
3892	GCPhysGst = GST_GET_PDE_GCPHYS(PdeSrc);
3893	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3894	GCPhysGst \|= (iPDDst & 1) * (PAGE_SIZE / 2);
3895	# endif
3896	}
3897	else
3898	{
3899	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3900	if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
3901	{
3902	AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n",
3903	GCPtr, (uint64_t)PdeSrc.u));
3904	cErrors++;
3905	continue;
3906	}
3907	# endif
3908	GCPhysGst = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
3909	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3910	GCPhysGst \|= GCPtr & RT_BIT(X86_PAGE_2M_SHIFT);
3911	# endif
3912	}
3913
3914	if ( pPoolPage->enmKind
3915	!= (!PdeSrc.b.u1Size \|\| !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
3916	{
3917	AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n",
3918	pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
3919	cErrors++;
3920	}
3921
3922	PPGMPAGE pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
3923	if (!pPhysPage)
3924	{
3925	AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3926	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3927	cErrors++;
3928	continue;
3929	}
3930
3931	if (GCPhysGst != pPoolPage->GCPhys)
3932	{
3933	AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n",
3934	GCPhysGst, pPoolPage->GCPhys, GCPtr));
3935	cErrors++;
3936	continue;
3937	}
3938
3939	if ( !PdeSrc.b.u1Size
3940	\|\| !fBigPagesSupported)
3941	{
3942	/*
3943	* Page Table.
3944	*/
3945	const GSTPT *pPTSrc;
3946	rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1), &pPTSrc);
3947	if (RT_FAILURE(rc))
3948	{
3949	AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3950	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3951	cErrors++;
3952	continue;
3953	}
3954	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/))
3955	!= (PdeDst.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/)))
3956	{
3957	/// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
3958	// (This problem will go away when/if we shadow multiple CR3s.)
3959	AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3960	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3961	cErrors++;
3962	continue;
3963	}
3964	if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3965	{
3966	AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n",
3967	GCPtr, (uint64_t)PdeDst.u));
3968	cErrors++;
3969	continue;
3970	}
3971
3972	/* iterate the page table. */
3973	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3974	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3975	const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
3976	# else
3977	const unsigned offPTSrc = 0;
3978	# endif
3979	for (unsigned iPT = 0, off = 0;
3980	iPT < RT_ELEMENTS(pPTDst->a);
3981	iPT++, off += PAGE_SIZE)
3982	{
3983	const SHWPTE PteDst = pPTDst->a[iPT];
3984
3985	/* skip not-present and dirty tracked entries. */
3986	if (!(SHW_PTE_GET_U(PteDst) & (X86_PTE_P \| PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
3987	continue;
3988	Assert(SHW_PTE_IS_P(PteDst));
3989
3990	const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
3991	if (!PteSrc.n.u1Present)
3992	{
3993	# ifdef IN_RING3
3994	PGMAssertHandlerAndFlagsInSync(pVM);
3995	DBGFR3PagingDumpEx(pVM, pVCpu->idCpu, DBGFPGDMP_FLAGS_CURRENT_CR3 \| DBGFPGDMP_FLAGS_CURRENT_MODE
3996	\| DBGFPGDMP_FLAGS_GUEST \| DBGFPGDMP_FLAGS_HEADER \| DBGFPGDMP_FLAGS_PRINT_CR3,
3997	0, 0, UINT64_MAX, 99, NULL);
3998	# endif
3999	AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
4000	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst), pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
4001	(uint64_t)GST_GET_PDE_GCPHYS(PdeSrc) + (iPT + offPTSrc)*sizeof(PteSrc)));
4002	cErrors++;
4003	continue;
4004	}
4005
4006	uint64_t fIgnoreFlags = GST_PTE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_G \| X86_PTE_D \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
4007	# if 1 /** @todo sync accessed bit properly... */
4008	fIgnoreFlags \|= X86_PTE_A;
4009	# endif
4010
4011	/* match the physical addresses */
4012	HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
4013	GCPhysGst = GST_GET_PTE_GCPHYS(PteSrc);
4014
4015	# ifdef IN_RING3
4016	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4017	if (RT_FAILURE(rc))
4018	{
4019	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4020	{
4021	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
4022	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4023	cErrors++;
4024	continue;
4025	}
4026	}
4027	else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
4028	{
4029	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4030	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4031	cErrors++;
4032	continue;
4033	}
4034	# endif
4035
4036	pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
4037	if (!pPhysPage)
4038	{
4039	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4040	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4041	{
4042	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
4043	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4044	cErrors++;
4045	continue;
4046	}
4047	# endif
4048	if (SHW_PTE_IS_RW(PteDst))
4049	{
4050	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4051	GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4052	cErrors++;
4053	}
4054	fIgnoreFlags \|= X86_PTE_RW;
4055	}
4056	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4057	{
4058	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4059	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4060	cErrors++;
4061	continue;
4062	}
4063
4064	/* flags */
4065	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4066	{
4067	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4068	{
4069	if (SHW_PTE_IS_RW(PteDst))
4070	{
4071	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4072	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4073	cErrors++;
4074	continue;
4075	}
4076	fIgnoreFlags \|= X86_PTE_RW;
4077	}
4078	else
4079	{
4080	if ( SHW_PTE_IS_P(PteDst)
4081	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
4082	&& !PGM_PAGE_IS_MMIO(pPhysPage)
4083	# endif
4084	)
4085	{
4086	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4087	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4088	cErrors++;
4089	continue;
4090	}
4091	fIgnoreFlags \|= X86_PTE_P;
4092	}
4093	}
4094	else
4095	{
4096	if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
4097	{
4098	if (SHW_PTE_IS_RW(PteDst))
4099	{
4100	AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
4101	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4102	cErrors++;
4103	continue;
4104	}
4105	if (!SHW_PTE_IS_TRACK_DIRTY(PteDst))
4106	{
4107	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4108	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4109	cErrors++;
4110	continue;
4111	}
4112	if (SHW_PTE_IS_D(PteDst))
4113	{
4114	AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4115	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4116	cErrors++;
4117	}
4118	# if 0 /** @todo sync access bit properly... */
4119	if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
4120	{
4121	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4122	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4123	cErrors++;
4124	}
4125	fIgnoreFlags \|= X86_PTE_RW;
4126	# else
4127	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
4128	# endif
4129	}
4130	else if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
4131	{
4132	/* access bit emulation (not implemented). */
4133	if (PteSrc.n.u1Accessed \|\| SHW_PTE_IS_P(PteDst))
4134	{
4135	AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
4136	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4137	cErrors++;
4138	continue;
4139	}
4140	if (!SHW_PTE_IS_A(PteDst))
4141	{
4142	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
4143	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4144	cErrors++;
4145	}
4146	fIgnoreFlags \|= X86_PTE_P;
4147	}
4148	# ifdef DEBUG_sandervl
4149	fIgnoreFlags \|= X86_PTE_D \| X86_PTE_A;
4150	# endif
4151	}
4152
4153	if ( (PteSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4154	&& (PteSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4155	)
4156	{
4157	AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
4158	GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
4159	fIgnoreFlags, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4160	cErrors++;
4161	continue;
4162	}
4163	} /* foreach PTE */
4164	}
4165	else
4166	{
4167	/*
4168	* Big Page.
4169	*/
4170	uint64_t fIgnoreFlags = X86_PDE_AVL_MASK \| GST_PDE_PG_MASK \| X86_PDE4M_G \| X86_PDE4M_D \| X86_PDE4M_PS \| X86_PDE4M_PWT \| X86_PDE4M_PCD;
4171	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
4172	{
4173	if (PdeDst.n.u1Write)
4174	{
4175	AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4176	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4177	cErrors++;
4178	continue;
4179	}
4180	if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
4181	{
4182	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4183	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4184	cErrors++;
4185	continue;
4186	}
4187	# if 0 /** @todo sync access bit properly... */
4188	if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
4189	{
4190	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4191	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4192	cErrors++;
4193	}
4194	fIgnoreFlags \|= X86_PTE_RW;
4195	# else
4196	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
4197	# endif
4198	}
4199	else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4200	{
4201	/* access bit emulation (not implemented). */
4202	if (PdeSrc.b.u1Accessed \|\| PdeDst.n.u1Present)
4203	{
4204	AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4205	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4206	cErrors++;
4207	continue;
4208	}
4209	if (!PdeDst.n.u1Accessed)
4210	{
4211	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4212	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4213	cErrors++;
4214	}
4215	fIgnoreFlags \|= X86_PTE_P;
4216	}
4217
4218	if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
4219	{
4220	AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
4221	GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
4222	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4223	cErrors++;
4224	}
4225
4226	/* iterate the page table. */
4227	for (unsigned iPT = 0, off = 0;
4228	iPT < RT_ELEMENTS(pPTDst->a);
4229	iPT++, off += PAGE_SIZE, GCPhysGst += PAGE_SIZE)
4230	{
4231	const SHWPTE PteDst = pPTDst->a[iPT];
4232
4233	if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
4234	{
4235	AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
4236	GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4237	cErrors++;
4238	}
4239
4240	/* skip not-present entries. */
4241	if (!SHW_PTE_IS_P(PteDst)) /** @todo deal with ALL handlers and CSAM !P pages! */
4242	continue;
4243
4244	fIgnoreFlags = X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT \| X86_PTE_D \| X86_PTE_A \| X86_PTE_G \| X86_PTE_PAE_NX;
4245
4246	/* match the physical addresses */
4247	HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
4248
4249	# ifdef IN_RING3
4250	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4251	if (RT_FAILURE(rc))
4252	{
4253	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4254	{
4255	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4256	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4257	cErrors++;
4258	}
4259	}
4260	else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
4261	{
4262	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4263	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4264	cErrors++;
4265	continue;
4266	}
4267	# endif
4268	pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
4269	if (!pPhysPage)
4270	{
4271	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4272	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4273	{
4274	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4275	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4276	cErrors++;
4277	continue;
4278	}
4279	# endif
4280	if (SHW_PTE_IS_RW(PteDst))
4281	{
4282	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4283	GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4284	cErrors++;
4285	}
4286	fIgnoreFlags \|= X86_PTE_RW;
4287	}
4288	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4289	{
4290	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4291	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4292	cErrors++;
4293	continue;
4294	}
4295
4296	/* flags */
4297	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4298	{
4299	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4300	{
4301	if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
4302	{
4303	if (SHW_PTE_IS_RW(PteDst))
4304	{
4305	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4306	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4307	cErrors++;
4308	continue;
4309	}
4310	fIgnoreFlags \|= X86_PTE_RW;
4311	}
4312	}
4313	else
4314	{
4315	if ( SHW_PTE_IS_P(PteDst)
4316	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
4317	&& !PGM_PAGE_IS_MMIO(pPhysPage)
4318	# endif
4319	)
4320	{
4321	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4322	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4323	cErrors++;
4324	continue;
4325	}
4326	fIgnoreFlags \|= X86_PTE_P;
4327	}
4328	}
4329
4330	if ( (PdeSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4331	&& (PdeSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags) /* lazy phys handler dereg. */
4332	)
4333	{
4334	AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
4335	GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
4336	fIgnoreFlags, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4337	cErrors++;
4338	continue;
4339	}
4340	} /* for each PTE */
4341	}
4342	}
4343	/* not present */
4344
4345	} /* for each PDE */
4346
4347	} /* for each PDPTE */
4348
4349	} /* for each PML4E */
4350
4351	# ifdef DEBUG
4352	if (cErrors)
4353	LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
4354	# endif
4355
4356	#endif /* GST == 32BIT, PAE or AMD64 */
4357	return cErrors;
4358
4359	#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT */
4360	}
4361	#endif /* VBOX_STRICT */
4362
4363
4364	/**
4365	* Sets up the CR3 for shadow paging
4366	*
4367	* @returns Strict VBox status code.
4368	* @retval VINF_SUCCESS.
4369	*
4370	* @param pVCpu The VMCPU handle.
4371	* @param GCPhysCR3 The physical address in the CR3 register.
4372	*/
4373	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3)
4374	{
4375	PVM pVM = pVCpu->CTX_SUFF(pVM);
4376
4377	/* Update guest paging info. */
4378	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
4379	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
4380	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
4381
4382	LogFlow(("MapCR3: %RGp\n", GCPhysCR3));
4383
4384	/*
4385	* Map the page CR3 points at.
4386	*/
4387	RTHCPTR HCPtrGuestCR3;
4388	RTHCPHYS HCPhysGuestCR3;
4389	pgmLock(pVM);
4390	PPGMPAGE pPageCR3 = pgmPhysGetPage(&pVM->pgm.s, GCPhysCR3);
4391	AssertReturn(pPageCR3, VERR_INTERNAL_ERROR_2);
4392	HCPhysGuestCR3 = PGM_PAGE_GET_HCPHYS(pPageCR3);
4393	/** @todo this needs some reworking wrt. locking? */
4394	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4395	HCPtrGuestCR3 = NIL_RTHCPTR;
4396	int rc = VINF_SUCCESS;
4397	# else
4398	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPageCR3, GCPhysCR3 & GST_CR3_PAGE_MASK, (void )&HCPtrGuestCR3); / @todo r=bird: This GCPhysCR3 masking isn't necessary. */
4399	# endif
4400	pgmUnlock(pVM);
4401	if (RT_SUCCESS(rc))
4402	{
4403	rc = PGMMap(pVM, (RTGCPTR)pVM->pgm.s.GCPtrCR3Mapping, HCPhysGuestCR3, PAGE_SIZE, 0);
4404	if (RT_SUCCESS(rc))
4405	{
4406	# ifdef IN_RC
4407	PGM_INVL_PG(pVCpu, pVM->pgm.s.GCPtrCR3Mapping);
4408	# endif
4409	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4410	pVCpu->pgm.s.pGst32BitPdR3 = (R3PTRTYPE(PX86PD))HCPtrGuestCR3;
4411	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4412	pVCpu->pgm.s.pGst32BitPdR0 = (R0PTRTYPE(PX86PD))HCPtrGuestCR3;
4413	# endif
4414	pVCpu->pgm.s.pGst32BitPdRC = (RCPTRTYPE(PX86PD))(RTRCUINTPTR)pVM->pgm.s.GCPtrCR3Mapping;
4415
4416	# elif PGM_GST_TYPE == PGM_TYPE_PAE
4417	unsigned off = GCPhysCR3 & GST_CR3_PAGE_MASK & PAGE_OFFSET_MASK;
4418	pVCpu->pgm.s.pGstPaePdptR3 = (R3PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4419	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4420	pVCpu->pgm.s.pGstPaePdptR0 = (R0PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4421	# endif
4422	pVCpu->pgm.s.pGstPaePdptRC = (RCPTRTYPE(PX86PDPT))((RTRCUINTPTR)pVM->pgm.s.GCPtrCR3Mapping + off);
4423	LogFlow(("Cached mapping %RRv\n", pVCpu->pgm.s.pGstPaePdptRC));
4424
4425	/*
4426	* Map the 4 PDs too.
4427	*/
4428	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pVCpu);
4429	RTGCPTR GCPtr = pVM->pgm.s.GCPtrCR3Mapping + PAGE_SIZE;
4430	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++, GCPtr += PAGE_SIZE)
4431	{
4432	if (pGuestPDPT->a[i].n.u1Present)
4433	{
4434	RTHCPTR HCPtr;
4435	RTHCPHYS HCPhys;
4436	RTGCPHYS GCPhys = pGuestPDPT->a[i].u & X86_PDPE_PG_MASK;
4437	pgmLock(pVM);
4438	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
4439	AssertReturn(pPage, VERR_INTERNAL_ERROR_2);
4440	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
4441	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4442	HCPtr = NIL_RTHCPTR;
4443	int rc2 = VINF_SUCCESS;
4444	# else
4445	int rc2 = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, (void **)&HCPtr);
4446	# endif
4447	pgmUnlock(pVM);
4448	if (RT_SUCCESS(rc2))
4449	{
4450	rc = PGMMap(pVM, GCPtr, HCPhys, PAGE_SIZE, 0);
4451	AssertRCReturn(rc, rc);
4452
4453	pVCpu->pgm.s.apGstPaePDsR3[i] = (R3PTRTYPE(PX86PDPAE))HCPtr;
4454	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4455	pVCpu->pgm.s.apGstPaePDsR0[i] = (R0PTRTYPE(PX86PDPAE))HCPtr;
4456	# endif
4457	pVCpu->pgm.s.apGstPaePDsRC[i] = (RCPTRTYPE(PX86PDPAE))(RTRCUINTPTR)GCPtr;
4458	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
4459	# ifdef IN_RC
4460	PGM_INVL_PG(pVCpu, GCPtr);
4461	# endif
4462	continue;
4463	}
4464	AssertMsgFailed(("pgmR3Gst32BitMapCR3: rc2=%d GCPhys=%RGp i=%d\n", rc2, GCPhys, i));
4465	}
4466
4467	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4468	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4469	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4470	# endif
4471	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4472	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4473	# ifdef IN_RC
4474	PGM_INVL_PG(pVCpu, GCPtr); /** @todo this shouldn't be necessary? */
4475	# endif
4476	}
4477
4478	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
4479	pVCpu->pgm.s.pGstAmd64Pml4R3 = (R3PTRTYPE(PX86PML4))HCPtrGuestCR3;
4480	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4481	pVCpu->pgm.s.pGstAmd64Pml4R0 = (R0PTRTYPE(PX86PML4))HCPtrGuestCR3;
4482	# endif
4483	# endif
4484	}
4485	else
4486	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4487	}
4488	else
4489	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4490
4491	#else /* prot/real stub */
4492	int rc = VINF_SUCCESS;
4493	#endif
4494
4495	/* Update shadow paging info for guest modes with paging (32, pae, 64). */
4496	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4497	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4498	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64) \
4499	&& ( PGM_GST_TYPE != PGM_TYPE_REAL \
4500	&& PGM_GST_TYPE != PGM_TYPE_PROT))
4501
4502	Assert(!pVM->pgm.s.fNestedPaging);
4503
4504	/*
4505	* Update the shadow root page as well since that's not fixed.
4506	*/
4507	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4508	PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
4509	uint32_t iOldShwUserTable = pVCpu->pgm.s.iShwUserTable;
4510	uint32_t iOldShwUser = pVCpu->pgm.s.iShwUser;
4511	PPGMPOOLPAGE pNewShwPageCR3;
4512
4513	pgmLock(pVM);
4514
4515	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4516	if (pPool->cDirtyPages)
4517	pgmPoolResetDirtyPages(pVM);
4518	# endif
4519
4520	Assert(!(GCPhysCR3 >> (PAGE_SHIFT + 32)));
4521	rc = pgmPoolAllocEx(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE, SHW_POOL_ROOT_IDX,
4522	GCPhysCR3 >> PAGE_SHIFT, true /fLockPage/, &pNewShwPageCR3);
4523	AssertFatalRC(rc);
4524	rc = VINF_SUCCESS;
4525
4526	# ifdef IN_RC
4527	/*
4528	* WARNING! We can't deal with jumps to ring 3 in the code below as the
4529	* state will be inconsistent! Flush important things now while
4530	* we still can and then make sure there are no ring-3 calls.
4531	*/
4532	REMNotifyHandlerPhysicalFlushIfAlmostFull(pVM, pVCpu);
4533	VMMRZCallRing3Disable(pVCpu);
4534	# endif
4535
4536	pVCpu->pgm.s.iShwUser = SHW_POOL_ROOT_IDX;
4537	pVCpu->pgm.s.iShwUserTable = GCPhysCR3 >> PAGE_SHIFT;
4538	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3) = pNewShwPageCR3;
4539	# ifdef IN_RING0
4540	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4541	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4542	# elif defined(IN_RC)
4543	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4544	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4545	# else
4546	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4547	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4548	# endif
4549
4550	# ifndef PGM_WITHOUT_MAPPINGS
4551	/*
4552	* Apply all hypervisor mappings to the new CR3.
4553	* Note that SyncCR3 will be executed in case CR3 is changed in a guest paging mode; this will
4554	* make sure we check for conflicts in the new CR3 root.
4555	*/
4556	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4557	Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL) \|\| VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
4558	# endif
4559	rc = pgmMapActivateCR3(pVM, pNewShwPageCR3);
4560	AssertRCReturn(rc, rc);
4561	# endif
4562
4563	/* Set the current hypervisor CR3. */
4564	CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
4565	SELMShadowCR3Changed(pVM, pVCpu);
4566
4567	# ifdef IN_RC
4568	/* NOTE: The state is consistent again. */
4569	VMMRZCallRing3Enable(pVCpu);
4570	# endif
4571
4572	/* Clean up the old CR3 root. */
4573	if ( pOldShwPageCR3
4574	&& pOldShwPageCR3 != pNewShwPageCR3 /* @todo can happen due to incorrect syncing between REM & PGM; find the real cause */)
4575	{
4576	Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
4577	# ifndef PGM_WITHOUT_MAPPINGS
4578	/* Remove the hypervisor mappings from the shadow page table. */
4579	pgmMapDeactivateCR3(pVM, pOldShwPageCR3);
4580	# endif
4581	/* Mark the page as unlocked; allow flushing again. */
4582	pgmPoolUnlockPage(pPool, pOldShwPageCR3);
4583
4584	pgmPoolFreeByPage(pPool, pOldShwPageCR3, iOldShwUser, iOldShwUserTable);
4585	}
4586	pgmUnlock(pVM);
4587	# endif
4588
4589	return rc;
4590	}
4591
4592	/**
4593	* Unmaps the shadow CR3.
4594	*
4595	* @returns VBox status, no specials.
4596	* @param pVCpu The VMCPU handle.
4597	*/
4598	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu)
4599	{
4600	LogFlow(("UnmapCR3\n"));
4601
4602	int rc = VINF_SUCCESS;
4603	PVM pVM = pVCpu->CTX_SUFF(pVM);
4604
4605	/*
4606	* Update guest paging info.
4607	*/
4608	#if PGM_GST_TYPE == PGM_TYPE_32BIT
4609	pVCpu->pgm.s.pGst32BitPdR3 = 0;
4610	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4611	pVCpu->pgm.s.pGst32BitPdR0 = 0;
4612	# endif
4613	pVCpu->pgm.s.pGst32BitPdRC = 0;
4614
4615	#elif PGM_GST_TYPE == PGM_TYPE_PAE
4616	pVCpu->pgm.s.pGstPaePdptR3 = 0;
4617	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4618	pVCpu->pgm.s.pGstPaePdptR0 = 0;
4619	# endif
4620	pVCpu->pgm.s.pGstPaePdptRC = 0;
4621	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4622	{
4623	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4624	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4625	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4626	# endif
4627	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4628	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4629	}
4630
4631	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
4632	pVCpu->pgm.s.pGstAmd64Pml4R3 = 0;
4633	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4634	pVCpu->pgm.s.pGstAmd64Pml4R0 = 0;
4635	# endif
4636
4637	#else /* prot/real mode stub */
4638	/* nothing to do */
4639	#endif
4640
4641	#if !defined(IN_RC) /* In RC we rely on MapCR3 to do the shadow part for us at a safe time */
4642	/*
4643	* Update shadow paging info.
4644	*/
4645	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4646	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4647	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64))
4648
4649	# if PGM_GST_TYPE != PGM_TYPE_REAL
4650	Assert(!pVM->pgm.s.fNestedPaging);
4651	# endif
4652
4653	pgmLock(pVM);
4654
4655	# ifndef PGM_WITHOUT_MAPPINGS
4656	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4657	/* Remove the hypervisor mappings from the shadow page table. */
4658	pgmMapDeactivateCR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4659	# endif
4660
4661	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4662	{
4663	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4664
4665	Assert(pVCpu->pgm.s.iShwUser != PGMPOOL_IDX_NESTED_ROOT);
4666
4667	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4668	if (pPool->cDirtyPages)
4669	pgmPoolResetDirtyPages(pVM);
4670	# endif
4671
4672	/* Mark the page as unlocked; allow flushing again. */
4673	pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4674
4675	pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), pVCpu->pgm.s.iShwUser, pVCpu->pgm.s.iShwUserTable);
4676	pVCpu->pgm.s.pShwPageCR3R3 = 0;
4677	pVCpu->pgm.s.pShwPageCR3R0 = 0;
4678	pVCpu->pgm.s.pShwPageCR3RC = 0;
4679	pVCpu->pgm.s.iShwUser = 0;
4680	pVCpu->pgm.s.iShwUserTable = 0;
4681	}
4682	pgmUnlock(pVM);
4683	# endif
4684	#endif /* !IN_RC*/
4685
4686	return rc;
4687	}

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 32521

Download in other formats: