PGMAllBth.h@ 32362

Last change on this file since 32362 was 32362, checked in by vboxsync, 14 years ago
Some cleanup
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 200.2 KB

Line
1	/* $Id: PGMAllBth.h 32362 2010-09-09 15:55:20Z vboxsync $ */
2	/** @file
3	* VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4	*
5	* @remarks The nested page tables on AMD makes use of PGM_SHW_TYPE in
6	* {PGM_TYPE_AMD64, PGM_TYPE_PAE and PGM_TYPE_32BIT} and PGM_GST_TYPE
7	* set to PGM_TYPE_PROT. Half of the code in this file is not
8	* exercised with PGM_SHW_TYPE set to PGM_TYPE_NESTED.
9	*
10	* @remarks Extended page tables (intel) are built with PGM_GST_TYPE set to
11	* PGM_TYPE_PROT (and PGM_SHW_TYPE set to PGM_TYPE_EPT).
12	*
13	* @remarks This file is one big \#ifdef-orgy!
14	*
15	*/
16
17	/*
18	* Copyright (C) 2006-2010 Oracle Corporation
19	*
20	* This file is part of VirtualBox Open Source Edition (OSE), as
21	* available from http://www.virtualbox.org. This file is free software;
22	* you can redistribute it and/or modify it under the terms of the GNU
23	* General Public License (GPL) as published by the Free Software
24	* Foundation, in version 2 as it comes in the "COPYING" file of the
25	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
26	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
27	*/
28
29
30	/*******************************************************************************
31	* Internal Functions *
32	*******************************************************************************/
33	RT_C_DECLS_BEGIN
34	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken);
35	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
36	static int PGM_BTH_NAME(SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr);
37	static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, RTGCPTR GCPtrPage);
38	static int PGM_BTH_NAME(SyncPT)(PVMCPU pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage);
39	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uErr);
40	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
41	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
42	#ifdef VBOX_STRICT
43	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
44	#endif
45	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3);
46	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu);
47	RT_C_DECLS_END
48
49
50	/*
51	* Filter out some illegal combinations of guest and shadow paging, so we can
52	* remove redundant checks inside functions.
53	*/
54	#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
55	# error "Invalid combination; PAE guest implies PAE shadow"
56	#endif
57
58	#if (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
59	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT)
60	# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
61	#endif
62
63	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) \
64	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT)
65	# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
66	#endif
67
68	#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT) \
69	\|\| (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
70	# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
71	#endif
72
73	#ifndef IN_RING3
74
75	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
76	/**
77	* Deal with a guest page fault.
78	*
79	* @returns Strict VBox status code.
80	* @retval VINF_EM_RAW_GUEST_TRAP
81	* @retval VINF_EM_RAW_EMULATE_INSTR
82	*
83	* @param pVCpu The current CPU.
84	* @param pGstWalk The guest page table walk result.
85	* @param uErr The error code.
86	*/
87	PGM_BTH_DECL(VBOXSTRICTRC, Trap0eHandlerGuestFault)(PVMCPU pVCpu, PGSTPTWALK pGstWalk, RTGCUINT uErr)
88	{
89	# if !defined(PGM_WITHOUT_MAPPINGS) && (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE)
90	/*
91	* Check for write conflicts with our hypervisor mapping.
92	*
93	* If the guest happens to access a non-present page, where our hypervisor
94	* is currently mapped, then we'll create a #PF storm in the guest.
95	*/
96	if ( (uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RW)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RW)
97	&& MMHyperIsInsideArea(pVCpu->CTX_SUFF(pVM), pGstWalk->Core.GCPtr))
98	{
99	/* Force a CR3 sync to check for conflicts and emulate the instruction. */
100	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
101	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
102	return VINF_EM_RAW_EMULATE_INSTR;
103	}
104	# endif
105
106	/*
107	* Calc the error code for the guest trap.
108	*/
109	uint32_t uNewErr = GST_IS_NX_ACTIVE(pVCpu)
110	? uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID)
111	: uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US);
112	if (pGstWalk->Core.fBadPhysAddr)
113	{
114	uNewErr \|= X86_TRAP_PF_RSVD \| X86_TRAP_PF_P;
115	Assert(!pGstWalk->Core.fNotPresent);
116	}
117	else if (!pGstWalk->Core.fNotPresent)
118	uNewErr \|= X86_TRAP_PF_P;
119	TRPMSetErrorCode(pVCpu, uNewErr);
120
121	LogFlow(("Guest trap; cr2=%RGv uErr=%RGv lvl=%d\n", pGstWalk->Core.GCPtr, uErr, pGstWalk->Core.uLevel));
122	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
123	return VINF_EM_RAW_GUEST_TRAP;
124	}
125	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
126
127
128	/**
129	* Deal with a guest page fault.
130	*
131	* The caller has taken the PGM lock.
132	*
133	* @returns Strict VBox status code.
134	*
135	* @param pVCpu The current CPU.
136	* @param uErr The error code.
137	* @param pRegFrame The register frame.
138	* @param pvFault The fault address.
139	* @param pPage The guest page at @a pvFault.
140	* @param pGstWalk The guest page table walk result.
141	* @param pfLockTaken PGM lock taken here or not (out). This is true
142	* when we're called.
143	*/
144	static VBOXSTRICTRC PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame,
145	RTGCPTR pvFault, PPGMPAGE pPage, bool *pfLockTaken
146	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
147	, PGSTPTWALK pGstWalk
148	# endif
149	)
150	{
151	# if !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
152	GSTPDE const PdeSrcDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
153	#endif
154	PVM pVM = pVCpu->CTX_SUFF(pVM);
155	int rc;
156
157	if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
158	{
159	/*
160	* Physical page access handler.
161	*/
162	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
163	const RTGCPHYS GCPhysFault = pGstWalk->Core.GCPhys;
164	# else
165	const RTGCPHYS GCPhysFault = (RTGCPHYS)pvFault;
166	# endif
167	PPGMPHYSHANDLER pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
168	if (pCur)
169	{
170	# ifdef PGM_SYNC_N_PAGES
171	/*
172	* If the region is write protected and we got a page not present fault, then sync
173	* the pages. If the fault was caused by a read, then restart the instruction.
174	* In case of write access continue to the GC write handler.
175	*
176	* ASSUMES that there is only one handler per page or that they have similar write properties.
177	*/
178	if ( !(uErr & X86_TRAP_PF_P)
179	&& pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE)
180	{
181	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
182	rc = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
183	# else
184	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
185	# endif
186	if ( RT_FAILURE(rc)
187	\|\| !(uErr & X86_TRAP_PF_RW)
188	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
189	{
190	AssertRC(rc);
191	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
192	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
193	return rc;
194	}
195	}
196	# endif
197	# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
198	/*
199	* If the access was not thru a #PF(RSVD\|...) resync the page.
200	*/
201	if ( !(uErr & X86_TRAP_PF_RSVD)
202	&& pCur->enmType != PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
203	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
204	&& pGstWalk->Core.fEffectiveRW
205	&& !pGstWalk->Core.fEffectiveUS /** @todo Remove pGstWalk->Core.fEffectiveUS and X86_PTE_US further down in the sync code. */
206	# endif
207	)
208	{
209	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
210	rc = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
211	# else
212	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
213	# endif
214	if ( RT_FAILURE(rc)
215	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
216	{
217	AssertRC(rc);
218	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
219	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
220	return rc;
221	}
222	}
223	# endif
224
225	AssertMsg( pCur->enmType != PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
226	\|\| (pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE && (uErr & X86_TRAP_PF_RW)),
227	("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enum=%d\n",
228	pvFault, GCPhysFault, pPage, uErr, pCur->enmType));
229	if (pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE)
230	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysWrite);
231	else
232	{
233	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysAll);
234	if (uErr & X86_TRAP_PF_RSVD) STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysAllOpt);
235	}
236
237	if (pCur->CTX_SUFF(pfnHandler))
238	{
239	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
240	void *pvUser = pCur->CTX_SUFF(pvUser);
241	# ifdef IN_RING0
242	PFNPGMR0PHYSHANDLER pfnHandler = pCur->CTX_SUFF(pfnHandler);
243	# else
244	PFNPGMRCPHYSHANDLER pfnHandler = pCur->CTX_SUFF(pfnHandler);
245	# endif
246
247	STAM_PROFILE_START(&pCur->Stat, h);
248	if (pfnHandler != pPool->CTX_SUFF(pfnAccessHandler))
249	{
250	pgmUnlock(pVM);
251	*pfLockTaken = false;
252	}
253
254	rc = pfnHandler(pVM, uErr, pRegFrame, pvFault, GCPhysFault, pvUser);
255
256	# ifdef VBOX_WITH_STATISTICS
257	pgmLock(pVM);
258	pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
259	if (pCur)
260	STAM_PROFILE_STOP(&pCur->Stat, h);
261	pgmUnlock(pVM);
262	# endif
263	}
264	else
265	rc = VINF_EM_RAW_EMULATE_INSTR;
266
267	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndPhys; });
268	return rc;
269	}
270	}
271	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
272	else
273	{
274	# ifdef PGM_SYNC_N_PAGES
275	/*
276	* If the region is write protected and we got a page not present fault, then sync
277	* the pages. If the fault was caused by a read, then restart the instruction.
278	* In case of write access continue to the GC write handler.
279	*/
280	if ( PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) < PGM_PAGE_HNDL_PHYS_STATE_ALL
281	&& !(uErr & X86_TRAP_PF_P))
282	{
283	rc = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
284	if ( RT_FAILURE(rc)
285	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
286	\|\| !(uErr & X86_TRAP_PF_RW))
287	{
288	AssertRC(rc);
289	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
290	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndVirt; });
291	return rc;
292	}
293	}
294	# endif
295	/*
296	* Ok, it's an virtual page access handler.
297	*
298	* Since it's faster to search by address, we'll do that first
299	* and then retry by GCPhys if that fails.
300	*/
301	/** @todo r=bird: perhaps we should consider looking up by physical address directly now?
302	* r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be
303	* out of sync, because the page was changed without us noticing it (not-present -> present
304	* without invlpg or mov cr3, xxx).
305	*/
306	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
307	if (pCur)
308	{
309	AssertMsg(!(pvFault - pCur->Core.Key < pCur->cb)
310	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
311	\|\| !(uErr & X86_TRAP_PF_P)
312	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
313	("Unexpected trap for virtual handler: %RGv (phys=%RGp) pPage=%R[pgmpage] uErr=%X, enum=%d\n",
314	pvFault, pGstWalk->Core.GCPhys, pPage, uErr, pCur->enmType));
315
316	if ( pvFault - pCur->Core.Key < pCur->cb
317	&& ( uErr & X86_TRAP_PF_RW
318	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
319	{
320	# ifdef IN_RC
321	STAM_PROFILE_START(&pCur->Stat, h);
322	RTGCPTR GCPtrStart = pCur->Core.Key;
323	CTX_MID(PFNPGM,VIRTHANDLER) pfnHandler = pCur->CTX_SUFF(pfnHandler);
324	pgmUnlock(pVM);
325	*pfLockTaken = false;
326
327	rc = pfnHandler(pVM, uErr, pRegFrame, pvFault, GCPtrStart, pvFault - GCPtrStart);
328
329	# ifdef VBOX_WITH_STATISTICS
330	pgmLock(pVM);
331	pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
332	if (pCur)
333	STAM_PROFILE_STOP(&pCur->Stat, h);
334	pgmUnlock(pVM);
335	# endif
336	# else
337	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
338	# endif
339	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersVirtual);
340	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
341	return rc;
342	}
343	/* Unhandled part of a monitored page */
344	}
345	else
346	{
347	/* Check by physical address. */
348	unsigned iPage;
349	rc = pgmHandlerVirtualFindByPhysAddr(pVM, pGstWalk->Core.GCPhys, &pCur, &iPage);
350	Assert(RT_SUCCESS(rc) \|\| !pCur);
351	if ( pCur
352	&& ( uErr & X86_TRAP_PF_RW
353	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
354	{
355	Assert((pCur->aPhysToVirt[iPage].Core.Key & X86_PTE_PAE_PG_MASK) == (pGstWalk->Core.GCPhys & X86_PTE_PAE_PG_MASK));
356	# ifdef IN_RC
357	STAM_PROFILE_START(&pCur->Stat, h);
358	RTGCPTR GCPtrStart = pCur->Core.Key;
359	CTX_MID(PFNPGM,VIRTHANDLER) pfnHandler = pCur->CTX_SUFF(pfnHandler);
360	pgmUnlock(pVM);
361	*pfLockTaken = false;
362
363	RTGCPTR off = (iPage << PAGE_SHIFT)
364	+ (pvFault & PAGE_OFFSET_MASK)
365	- (GCPtrStart & PAGE_OFFSET_MASK);
366	Assert(off < pCur->cb);
367	rc = pfnHandler(pVM, uErr, pRegFrame, pvFault, GCPtrStart, off);
368
369	# ifdef VBOX_WITH_STATISTICS
370	pgmLock(pVM);
371	pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, GCPtrStart);
372	if (pCur)
373	STAM_PROFILE_STOP(&pCur->Stat, h);
374	pgmUnlock(pVM);
375	# endif
376	# else
377	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
378	# endif
379	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersVirtualByPhys);
380	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
381	return rc;
382	}
383	}
384	}
385	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
386
387	/*
388	* There is a handled area of the page, but this fault doesn't belong to it.
389	* We must emulate the instruction.
390	*
391	* To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
392	* we first check if this was a page-not-present fault for a page with only
393	* write access handlers. Restart the instruction if it wasn't a write access.
394	*/
395	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersUnhandled);
396
397	if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
398	&& !(uErr & X86_TRAP_PF_P))
399	{
400	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
401	rc = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
402	# else
403	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
404	# endif
405	if ( RT_FAILURE(rc)
406	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
407	\|\| !(uErr & X86_TRAP_PF_RW))
408	{
409	AssertRC(rc);
410	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
411	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
412	return rc;
413	}
414	}
415
416	/** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
417	* It's writing to an unhandled part of the LDT page several million times.
418	*/
419	rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
420	LogFlow(("PGM: PGMInterpretInstruction -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
421	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndUnhandled; });
422	return rc;
423	} /* if any kind of handler */
424
425
426	/**
427	* #PF Handler for raw-mode guest execution.
428	*
429	* @returns VBox status code (appropriate for trap handling and GC return).
430	*
431	* @param pVCpu VMCPU Handle.
432	* @param uErr The trap error code.
433	* @param pRegFrame Trap register frame.
434	* @param pvFault The fault address.
435	* @param pfLockTaken PGM lock taken here or not (out)
436	*/
437	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken)
438	{
439	PVM pVM = pVCpu->CTX_SUFF(pVM);
440
441	*pfLockTaken = false;
442
443	# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \
444	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
445	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
446	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT)
447	int rc;
448
449	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
450	/*
451	* Walk the guest page translation tables and check if it's a guest fault.
452	*/
453	GSTPTWALK GstWalk;
454	rc = PGM_GST_NAME(Walk)(pVCpu, pvFault, &GstWalk);
455	if (RT_FAILURE_NP(rc))
456	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &GstWalk, uErr));
457
458	/* assert some GstWalk sanity. */
459	# if PGM_GST_TYPE == PGM_TYPE_AMD64
460	AssertMsg(GstWalk.Pml4e.u == GstWalk.pPml4e->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pml4e.u, (uint64_t)GstWalk.pPml4e->u));
461	# endif
462	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
463	AssertMsg(GstWalk.Pdpe.u == GstWalk.pPdpe->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pdpe.u, (uint64_t)GstWalk.pPdpe->u));
464	# endif
465	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u));
466	AssertMsg(GstWalk.Core.fBigPage \|\| GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u));
467	Assert(GstWalk.Core.fSucceeded);
468
469	if (uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID))
470	{
471	if ( ( (uErr & X86_TRAP_PF_RW)
472	&& !GstWalk.Core.fEffectiveRW
473	&& ( (uErr & X86_TRAP_PF_US)
474	\|\| CPUMIsGuestR0WriteProtEnabled(pVCpu)) )
475	\|\| ((uErr & X86_TRAP_PF_US) && !GstWalk.Core.fEffectiveUS)
476	\|\| ((uErr & X86_TRAP_PF_ID) && GstWalk.Core.fEffectiveNX)
477	)
478	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &GstWalk, uErr));
479	}
480
481	/*
482	* Set the accessed and dirty flags.
483	*/
484	# if PGM_GST_TYPE == PGM_TYPE_AMD64
485	GstWalk.Pml4e.u \|= X86_PML4E_A;
486	GstWalk.pPml4e->u \|= X86_PML4E_A;
487	GstWalk.Pdpe.u \|= X86_PDPE_A;
488	GstWalk.pPdpe->u \|= X86_PDPE_A;
489	# endif
490	if (GstWalk.Core.fBigPage)
491	{
492	Assert(GstWalk.Pde.b.u1Size);
493	if (uErr & X86_TRAP_PF_RW)
494	{
495	GstWalk.Pde.u \|= X86_PDE4M_A \| X86_PDE4M_D;
496	GstWalk.pPde->u \|= X86_PDE4M_A \| X86_PDE4M_D;
497	}
498	else
499	{
500	GstWalk.Pde.u \|= X86_PDE4M_A;
501	GstWalk.pPde->u \|= X86_PDE4M_A;
502	}
503	}
504	else
505	{
506	Assert(!GstWalk.Pde.b.u1Size);
507	GstWalk.Pde.u \|= X86_PDE_A;
508	GstWalk.pPde->u \|= X86_PDE_A;
509	if (uErr & X86_TRAP_PF_RW)
510	{
511	# ifdef VBOX_WITH_STATISTICS
512	if (!GstWalk.Pte.n.u1Dirty)
513	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtiedPage));
514	else
515	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageAlreadyDirty));
516	# endif
517	GstWalk.Pte.u \|= X86_PTE_A \| X86_PTE_D;
518	GstWalk.pPte->u \|= X86_PTE_A \| X86_PTE_D;
519	}
520	else
521	{
522	GstWalk.Pte.u \|= X86_PTE_A;
523	GstWalk.pPte->u \|= X86_PTE_A;
524	}
525	Assert(GstWalk.Pte.u == GstWalk.pPte->u);
526	}
527	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u \|\| GstWalk.pPte->u == GstWalk.pPde->u,
528	("%RX64 %RX64 pPte=%p pPde=%p Pte=%RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u, GstWalk.pPte, GstWalk.pPde, (uint64_t)GstWalk.pPte->u));
529	# else /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
530	GSTPDE const PdeSrcDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A}; /** @todo eliminate this */
531	# endif /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
532
533	/* Take the big lock now. */
534	*pfLockTaken = true;
535	pgmLock(pVM);
536
537	# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
538	/*
539	* If it is a reserved bit fault we know that it is an MMIO (access
540	* handler) related fault and can skip some 200 lines of code.
541	*/
542	if (uErr & X86_TRAP_PF_RSVD)
543	{
544	Assert(uErr & X86_TRAP_PF_P);
545	PPGMPAGE pPage;
546	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
547	rc = pgmPhysGetPageEx(&pVM->pgm.s, GstWalk.Core.GCPhys, &pPage);
548	if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
549	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
550	pfLockTaken, &GstWalk));
551	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
552	# else
553	rc = pgmPhysGetPageEx(&pVM->pgm.s, (RTGCPHYS)pvFault, &pPage);
554	if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
555	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
556	pfLockTaken));
557	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
558	# endif
559	AssertRC(rc);
560	PGM_INVL_PG(pVCpu, pvFault);
561	return rc; /* Restart with the corrected entry. */
562	}
563	# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
564
565	/*
566	* Fetch the guest PDE, PDPE and PML4E.
567	*/
568	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
569	const unsigned iPDDst = pvFault >> SHW_PD_SHIFT;
570	PX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
571
572	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
573	const unsigned iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK; /* pPDDst index, not used with the pool. */
574	PX86PDPAE pPDDst;
575	# if PGM_GST_TYPE == PGM_TYPE_PAE
576	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, GstWalk.Pdpe.u, &pPDDst);
577	# else
578	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, X86_PDPE_P, &pPDDst); /* RW, US and A are reserved in PAE mode. */
579	# endif
580	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_INTERNAL_ERROR_4);
581
582	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
583	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
584	PX86PDPAE pPDDst;
585	# if PGM_GST_TYPE == PGM_TYPE_PROT /* (AMD-V nested paging) */
586	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A,
587	X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A, &pPDDst);
588	# else
589	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, GstWalk.Pml4e.u, GstWalk.Pdpe.u, &pPDDst);
590	# endif
591	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_INTERNAL_ERROR_4);
592
593	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
594	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
595	PEPTPD pPDDst;
596	rc = pgmShwGetEPTPDPtr(pVCpu, pvFault, NULL, &pPDDst);
597	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_INTERNAL_ERROR_4);
598	# endif
599	Assert(pPDDst);
600
601	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
602	/*
603	* Dirty page handling.
604	*
605	* If we successfully correct the write protection fault due to dirty bit
606	* tracking, then return immediately.
607	*/
608	if (uErr & X86_TRAP_PF_RW) /* write fault? */
609	{
610	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyBitTracking), a);
611	rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, &pPDDst->a[iPDDst], GstWalk.pPde, pvFault);
612	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyBitTracking), a);
613	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
614	{
615	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution)
616	= rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
617	? &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2DirtyAndAccessed
618	: &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
619	LogBird(("Trap0eHandler: returns VINF_SUCCESS\n"));
620	return VINF_SUCCESS;
621	}
622	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u \|\| GstWalk.pPte->u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u));
623	AssertMsg(GstWalk.Core.fBigPage \|\| GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u));
624	}
625
626	# if 0 /* rarely useful; leave for debugging. */
627	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0ePD[iPDSrc]);
628	# endif
629	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
630
631	/*
632	* A common case is the not-present error caused by lazy page table syncing.
633	*
634	* It is IMPORTANT that we weed out any access to non-present shadow PDEs
635	* here so we can safely assume that the shadow PT is present when calling
636	* SyncPage later.
637	*
638	* On failure, we ASSUME that SyncPT is out of memory or detected some kind
639	* of mapping conflict and defer to SyncCR3 in R3.
640	* (Again, we do NOT support access handlers for non-present guest pages.)
641	*
642	*/
643	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
644	Assert(GstWalk.Pde.n.u1Present);
645	# endif
646	if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
647	&& !pPDDst->a[iPDDst].n.u1Present)
648	{
649	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2SyncPT; });
650	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
651	LogFlow(("=>SyncPT %04x = %08RX64\n", (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, (uint64_t)GstWalk.Pde.u));
652	rc = PGM_BTH_NAME(SyncPT)(pVCpu, (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, GstWalk.pPd, pvFault);
653	# else
654	LogFlow(("=>SyncPT pvFault=%RGv\n", pvFault));
655	rc = PGM_BTH_NAME(SyncPT)(pVCpu, 0, NULL, pvFault);
656	# endif
657	if (RT_SUCCESS(rc))
658	return rc;
659	Log(("SyncPT: %RGv failed!! rc=%Rrc\n", pvFault, rc));
660	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
661	return VINF_PGM_SYNC_CR3;
662	}
663
664	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(PGM_WITHOUT_MAPPINGS)
665	/*
666	* Check if this address is within any of our mappings.
667	*
668	* This is very fast and it's gonna save us a bit of effort below and prevent
669	* us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
670	* (BTW, it's impossible to have physical access handlers in a mapping.)
671	*/
672	if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
673	{
674	PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
675	for ( ; pMapping; pMapping = pMapping->CTX_SUFF(pNext))
676	{
677	if (pvFault < pMapping->GCPtr)
678	break;
679	if (pvFault - pMapping->GCPtr < pMapping->cb)
680	{
681	/*
682	* The first thing we check is if we've got an undetected conflict.
683	*/
684	if (pgmMapAreMappingsFloating(&pVM->pgm.s))
685	{
686	unsigned iPT = pMapping->cb >> GST_PD_SHIFT;
687	while (iPT-- > 0)
688	if (GstWalk.pPde[iPT].n.u1Present)
689	{
690	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eConflicts);
691	Log(("Trap0e: Detected Conflict %RGv-%RGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
692	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
693	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
694	return VINF_PGM_SYNC_CR3;
695	}
696	}
697
698	/*
699	* Check if the fault address is in a virtual page access handler range.
700	*/
701	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->HyperVirtHandlers, pvFault);
702	if ( pCur
703	&& pvFault - pCur->Core.Key < pCur->cb
704	&& uErr & X86_TRAP_PF_RW)
705	{
706	# ifdef IN_RC
707	STAM_PROFILE_START(&pCur->Stat, h);
708	pgmUnlock(pVM);
709	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
710	pgmLock(pVM);
711	STAM_PROFILE_STOP(&pCur->Stat, h);
712	# else
713	AssertFailed();
714	rc = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
715	# endif
716	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersMapping);
717	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
718	return rc;
719	}
720
721	/*
722	* Pretend we're not here and let the guest handle the trap.
723	*/
724	TRPMSetErrorCode(pVCpu, uErr & ~X86_TRAP_PF_P);
725	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eGuestPFMapping);
726	LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
727	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
728	return VINF_EM_RAW_GUEST_TRAP;
729	}
730	}
731	} /* pgmAreMappingsEnabled(&pVM->pgm.s) */
732	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
733
734	/*
735	* Check if this fault address is flagged for special treatment,
736	* which means we'll have to figure out the physical address and
737	* check flags associated with it.
738	*
739	* ASSUME that we can limit any special access handling to pages
740	* in page tables which the guest believes to be present.
741	*/
742	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
743	RTGCPHYS GCPhys = GstWalk.Core.GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
744	# else
745	RTGCPHYS GCPhys = (RTGCPHYS)pvFault & ~(RTGCPHYS)PAGE_OFFSET_MASK;
746	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
747	PPGMPAGE pPage;
748	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
749	if (RT_FAILURE(rc))
750	{
751	/*
752	* When the guest accesses invalid physical memory (e.g. probing
753	* of RAM or accessing a remapped MMIO range), then we'll fall
754	* back to the recompiler to emulate the instruction.
755	*/
756	LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhys, rc));
757	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersInvalid);
758	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2InvalidPhys; });
759	return VINF_EM_RAW_EMULATE_INSTR;
760	}
761
762	/*
763	* Any handlers for this page?
764	*/
765	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
766	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
767	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, pfLockTaken,
768	&GstWalk));
769	# else
770	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, pfLockTaken));
771	# endif
772
773	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTimeOutOfSync, c);
774
775	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
776	if (uErr & X86_TRAP_PF_P)
777	{
778	/*
779	* The page isn't marked, but it might still be monitored by a virtual page access handler.
780	* (ASSUMES no temporary disabling of virtual handlers.)
781	*/
782	/** @todo r=bird: Since the purpose is to catch out of sync pages with virtual handler(s) here,
783	* we should correct both the shadow page table and physical memory flags, and not only check for
784	* accesses within the handler region but for access to pages with virtual handlers. */
785	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
786	if (pCur)
787	{
788	AssertMsg( !(pvFault - pCur->Core.Key < pCur->cb)
789	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
790	\|\| !(uErr & X86_TRAP_PF_P)
791	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
792	("Unexpected trap for virtual handler: %08X (phys=%08x) %R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
793
794	if ( pvFault - pCur->Core.Key < pCur->cb
795	&& ( uErr & X86_TRAP_PF_RW
796	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
797	{
798	# ifdef IN_RC
799	STAM_PROFILE_START(&pCur->Stat, h);
800	pgmUnlock(pVM);
801	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
802	pgmLock(pVM);
803	STAM_PROFILE_STOP(&pCur->Stat, h);
804	# else
805	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
806	# endif
807	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
808	return rc;
809	}
810	}
811	}
812	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
813
814	/*
815	* We are here only if page is present in Guest page tables and
816	* trap is not handled by our handlers.
817	*
818	* Check it for page out-of-sync situation.
819	*/
820	if (!(uErr & X86_TRAP_PF_P))
821	{
822	/*
823	* Page is not present in our page tables. Try to sync it!
824	*/
825	if (uErr & X86_TRAP_PF_US)
826	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUser));
827	else /* supervisor */
828	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
829
830	if (PGM_PAGE_IS_BALLOONED(pPage))
831	{
832	/* Emulate reads from ballooned pages as they are not present in
833	our shadow page tables. (Required for e.g. Solaris guests; soft
834	ecc, random nr generator.) */
835	rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
836	LogFlow(("PGM: PGMInterpretInstruction balloon -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
837	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncBallloon));
838	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Ballooned; });
839	return rc;
840	}
841
842	# if defined(LOG_ENABLED) && !defined(IN_RING0)
843	RTGCPHYS GCPhys2;
844	uint64_t fPageGst2;
845	PGMGstGetPage(pVCpu, pvFault, &fPageGst2, &GCPhys2);
846	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
847	Log(("Page out of sync: %RGv eip=%08x PdeSrc.US=%d fPageGst2=%08llx GCPhys2=%RGp scan=%d\n",
848	pvFault, pRegFrame->eip, GstWalk.Pde.n.u1User, fPageGst2, GCPhys2, CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)));
849	# else
850	Log(("Page out of sync: %RGv eip=%08x fPageGst2=%08llx GCPhys2=%RGp scan=%d\n",
851	pvFault, pRegFrame->eip, fPageGst2, GCPhys2, CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)));
852	# endif
853	# endif /* LOG_ENABLED */
854
855	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
856	if ( !GstWalk.Core.fEffectiveUS
857	&& CPUMGetGuestCPL(pVCpu, pRegFrame) == 0)
858	{
859	/* Note: Can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU. */
860	if ( pvFault == (RTGCPTR)pRegFrame->eip
861	\|\| pvFault - pRegFrame->eip < 8 /* instruction crossing a page boundary */
862	# ifdef CSAM_DETECT_NEW_CODE_PAGES
863	\|\| ( !PATMIsPatchGCAddr(pVM, pRegFrame->eip)
864	&& CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)) /* any new code we encounter here */
865	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
866	)
867	{
868	LogFlow(("CSAMExecFault %RX32\n", pRegFrame->eip));
869	rc = CSAMExecFault(pVM, (RTRCPTR)pRegFrame->eip);
870	if (rc != VINF_SUCCESS)
871	{
872	/*
873	* CSAM needs to perform a job in ring 3.
874	*
875	* Sync the page before going to the host context; otherwise we'll end up in a loop if
876	* CSAM fails (e.g. instruction crosses a page boundary and the next page is not present)
877	*/
878	LogFlow(("CSAM ring 3 job\n"));
879	int rc2 = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
880	AssertRC(rc2);
881
882	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2CSAM; });
883	return rc;
884	}
885	}
886	# ifdef CSAM_DETECT_NEW_CODE_PAGES
887	else if ( uErr == X86_TRAP_PF_RW
888	&& pRegFrame->ecx >= 0x100 /* early check for movswd count */
889	&& pRegFrame->ecx < 0x10000)
890	{
891	/* In case of a write to a non-present supervisor shadow page, we'll take special precautions
892	* to detect loading of new code pages.
893	*/
894
895	/*
896	* Decode the instruction.
897	*/
898	RTGCPTR PC;
899	rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs,
900	&pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &PC);
901	if (rc == VINF_SUCCESS)
902	{
903	PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
904	uint32_t cbOp;
905	rc = EMInterpretDisasOneEx(pVM, pVCpu, PC, pRegFrame, pDis, &cbOp);
906
907	/* For now we'll restrict this to rep movsw/d instructions */
908	if ( rc == VINF_SUCCESS
909	&& pDis->pCurInstr->opcode == OP_MOVSWD
910	&& (pDis->prefix & PREFIX_REP))
911	{
912	CSAMMarkPossibleCodePage(pVM, pvFault);
913	}
914	}
915	}
916	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
917
918	/*
919	* Mark this page as safe.
920	*/
921	/** @todo not correct for pages that contain both code and data!! */
922	Log2(("CSAMMarkPage %RGv; scanned=%d\n", pvFault, true));
923	CSAMMarkPage(pVM, pvFault, true);
924	}
925	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0) */
926	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
927	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
928	# else
929	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
930	# endif
931	if (RT_SUCCESS(rc))
932	{
933	/* The page was successfully synced, return to the guest. */
934	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSync; });
935	return VINF_SUCCESS;
936	}
937	}
938	else /* uErr & X86_TRAP_PF_P: */
939	{
940	/*
941	* Write protected pages are made writable when the guest makes the
942	* first write to it. This happens for pages that are shared, write
943	* monitored or not yet allocated.
944	*
945	* We may also end up here when CR0.WP=0 in the guest.
946	*
947	* Also, a side effect of not flushing global PDEs are out of sync
948	* pages due to physical monitored regions, that are no longer valid.
949	* Assume for now it only applies to the read/write flag.
950	*/
951	if (uErr & X86_TRAP_PF_RW)
952	{
953	/*
954	* Check if it is a read-only page.
955	*/
956	if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
957	{
958	Log(("PGM #PF: Make writable: %RGp %R[pgmpage] pvFault=%RGp uErr=%#x\n", GCPhys, pPage, pvFault, uErr));
959	Assert(!PGM_PAGE_IS_ZERO(pPage));
960	AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
961	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2MakeWritable; });
962
963	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
964	if (rc != VINF_SUCCESS)
965	{
966	AssertMsg(rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("%Rrc\n", rc));
967	return rc;
968	}
969	if (RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)))
970	return VINF_EM_NO_MEMORY;
971	}
972
973	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
974	/*
975	* Check to see if we need to emulate the instruction if CR0.WP=0.
976	*/
977	if ( !GstWalk.Core.fEffectiveRW
978	&& (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG
979	&& CPUMGetGuestCPL(pVCpu, pRegFrame) == 0)
980	{
981	Assert((uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RW \| X86_TRAP_PF_P));
982	rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
983	if (RT_SUCCESS(rc))
984	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eWPEmulInRZ);
985	else
986	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eWPEmulToR3);
987	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2WPEmulation; });
988	return rc;
989	}
990	# endif
991	/// @todo count the above case; else
992	if (uErr & X86_TRAP_PF_US)
993	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUserWrite));
994	else /* supervisor */
995	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
996
997	/*
998	* Sync the page.
999	*
1000	* Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
1001	* page is not present, which is not true in this case.
1002	*/
1003	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1004	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
1005	# else
1006	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
1007	# endif
1008	if (RT_SUCCESS(rc))
1009	{
1010	/*
1011	* Page was successfully synced, return to guest but invalidate
1012	* the TLB first as the page is very likely to be in it.
1013	*/
1014	# if PGM_SHW_TYPE == PGM_TYPE_EPT
1015	HWACCMInvalidatePhysPage(pVM, (RTGCPHYS)pvFault);
1016	# else
1017	PGM_INVL_PG(pVCpu, pvFault);
1018	# endif
1019	# ifdef VBOX_STRICT
1020	RTGCPHYS GCPhys2;
1021	uint64_t fPageGst;
1022	if (!pVM->pgm.s.fNestedPaging)
1023	{
1024	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, &GCPhys2);
1025	AssertMsg(RT_SUCCESS(rc) && (fPageGst & X86_PTE_RW), ("rc=%Rrc fPageGst=%RX64\n", rc, fPageGst));
1026	LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GCPhys2, (uint64_t)fPageGst));
1027	}
1028	uint64_t fPageShw;
1029	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
1030	AssertMsg((RT_SUCCESS(rc) && (fPageShw & X86_PTE_RW)) \|\| pVM->cCpus > 1 /* new monitor can be installed/page table flushed between the trap exit and PGMTrap0eHandler */,
1031	("rc=%Rrc fPageShw=%RX64\n", rc, fPageShw));
1032	# endif /* VBOX_STRICT */
1033	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndObs; });
1034	return VINF_SUCCESS;
1035	}
1036	}
1037	/** @todo else: why are we here? */
1038
1039	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && defined(VBOX_STRICT)
1040	/*
1041	* Check for VMM page flags vs. Guest page flags consistency.
1042	* Currently only for debug purposes.
1043	*/
1044	if (RT_SUCCESS(rc))
1045	{
1046	/* Get guest page flags. */
1047	uint64_t fPageGst;
1048	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
1049	if (RT_SUCCESS(rc))
1050	{
1051	uint64_t fPageShw;
1052	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
1053
1054	/*
1055	* Compare page flags.
1056	* Note: we have AVL, A, D bits desynched.
1057	*/
1058	AssertMsg( (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK))
1059	== (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)),
1060	("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64\n",
1061	pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst));
1062	}
1063	else
1064	AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc));
1065	}
1066	else
1067	AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc));
1068	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && VBOX_STRICT */
1069	}
1070
1071
1072	/*
1073	* If we get here it is because something failed above, i.e. most like guru
1074	* meditiation time.
1075	*/
1076	LogRel(("%s: returns rc=%Rrc pvFault=%RGv uErr=%RX64 cs:rip=%04x:%08RX64\n",
1077	__PRETTY_FUNCTION__, rc, pvFault, (uint64_t)uErr, pRegFrame->cs, pRegFrame->rip));
1078	return rc;
1079
1080	# else /* Nested paging, EPT except PGM_GST_TYPE = PROT */
1081	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
1082	return VERR_INTERNAL_ERROR;
1083	# endif
1084	}
1085	#endif /* !IN_RING3 */
1086
1087
1088	/**
1089	* Emulation of the invlpg instruction.
1090	*
1091	*
1092	* @returns VBox status code.
1093	*
1094	* @param pVCpu The VMCPU handle.
1095	* @param GCPtrPage Page to invalidate.
1096	*
1097	* @remark ASSUMES that the guest is updating before invalidating. This order
1098	* isn't required by the CPU, so this is speculative and could cause
1099	* trouble.
1100	* @remark No TLB shootdown is done on any other VCPU as we assume that
1101	* invlpg emulation is the only reason for calling this function.
1102	* (The guest has to shoot down TLB entries on other CPUs itself)
1103	* Currently true, but keep in mind!
1104	*
1105	* @todo Clean this up! Most of it is (or should be) no longer necessary as we catch all page table accesses.
1106	* Should only required when PGMPOOL_WITH_OPTIMIZED_DIRTY_PT is active (PAE or AMD64 (for now))
1107	*/
1108	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
1109	{
1110	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1111	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
1112	&& PGM_SHW_TYPE != PGM_TYPE_EPT
1113	int rc;
1114	PVM pVM = pVCpu->CTX_SUFF(pVM);
1115	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1116
1117	Assert(PGMIsLockOwner(pVM));
1118
1119	LogFlow(("InvalidatePage %RGv\n", GCPtrPage));
1120
1121	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1122	/** @todo this shouldn't be necessary. */
1123	if (pPool->cDirtyPages)
1124	pgmPoolResetDirtyPages(pVM);
1125	# endif
1126
1127	/*
1128	* Get the shadow PD entry and skip out if this PD isn't present.
1129	* (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
1130	*/
1131	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1132	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1133	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1134
1135	/* Fetch the pgm pool shadow descriptor. */
1136	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1137	Assert(pShwPde);
1138
1139	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1140	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT);
1141	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
1142
1143	/* If the shadow PDPE isn't present, then skip the invalidate. */
1144	if (!pPdptDst->a[iPdpt].n.u1Present)
1145	{
1146	Assert(!(pPdptDst->a[iPdpt].u & PGM_PLXFLAGS_MAPPING));
1147	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1148	return VINF_SUCCESS;
1149	}
1150
1151	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1152	PPGMPOOLPAGE pShwPde = NULL;
1153	PX86PDPAE pPDDst;
1154
1155	/* Fetch the pgm pool shadow descriptor. */
1156	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
1157	AssertRCSuccessReturn(rc, rc);
1158	Assert(pShwPde);
1159
1160	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1161	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1162
1163	# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1164	/* PML4 */
1165	const unsigned iPml4 = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;
1166	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1167	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1168	PX86PDPAE pPDDst;
1169	PX86PDPT pPdptDst;
1170	PX86PML4E pPml4eDst;
1171	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eDst, &pPdptDst, &pPDDst);
1172	if (rc != VINF_SUCCESS)
1173	{
1174	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT \|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
1175	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1176	return VINF_SUCCESS;
1177	}
1178	Assert(pPDDst);
1179
1180	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1181	PX86PDPE pPdpeDst = &pPdptDst->a[iPdpt];
1182
1183	if (!pPdpeDst->n.u1Present)
1184	{
1185	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1186	return VINF_SUCCESS;
1187	}
1188
1189	/* Fetch the pgm pool shadow descriptor. */
1190	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & SHW_PDPE_PG_MASK);
1191	Assert(pShwPde);
1192
1193	# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1194
1195	const SHWPDE PdeDst = *pPdeDst;
1196	if (!PdeDst.n.u1Present)
1197	{
1198	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1199	return VINF_SUCCESS;
1200	}
1201
1202	/*
1203	* Get the guest PD entry and calc big page.
1204	*/
1205	# if PGM_GST_TYPE == PGM_TYPE_32BIT
1206	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
1207	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
1208	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1209	# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1210	unsigned iPDSrc = 0;
1211	# if PGM_GST_TYPE == PGM_TYPE_PAE
1212	X86PDPE PdpeSrcIgn;
1213	PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrcIgn);
1214	# else /* AMD64 */
1215	PX86PML4E pPml4eSrcIgn;
1216	X86PDPE PdpeSrcIgn;
1217	PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrcIgn, &PdpeSrcIgn, &iPDSrc);
1218	# endif
1219	GSTPDE PdeSrc;
1220
1221	if (pPDSrc)
1222	PdeSrc = pPDSrc->a[iPDSrc];
1223	else
1224	PdeSrc.u = 0;
1225	# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1226	const bool fIsBigPage = PdeSrc.b.u1Size && GST_IS_PSE_ACTIVE(pVCpu);
1227
1228	# ifdef IN_RING3
1229	/*
1230	* If a CR3 Sync is pending we may ignore the invalidate page operation
1231	* depending on the kind of sync and if it's a global page or not.
1232	* This doesn't make sense in GC/R0 so we'll skip it entirely there.
1233	*/
1234	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
1235	if ( VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
1236	\|\| ( VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
1237	&& fIsBigPage
1238	&& PdeSrc.b.u1Global
1239	)
1240	)
1241	# else
1242	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
1243	# endif
1244	{
1245	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1246	return VINF_SUCCESS;
1247	}
1248	# endif /* IN_RING3 */
1249
1250	/*
1251	* Deal with the Guest PDE.
1252	*/
1253	rc = VINF_SUCCESS;
1254	if (PdeSrc.n.u1Present)
1255	{
1256	Assert( PdeSrc.n.u1User == PdeDst.n.u1User
1257	&& (PdeSrc.n.u1Write \|\| !PdeDst.n.u1Write));
1258	# ifndef PGM_WITHOUT_MAPPING
1259	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
1260	{
1261	/*
1262	* Conflict - Let SyncPT deal with it to avoid duplicate code.
1263	*/
1264	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1265	Assert(PGMGetGuestMode(pVCpu) <= PGMMODE_PAE);
1266	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
1267	}
1268	else
1269	# endif /* !PGM_WITHOUT_MAPPING */
1270	if (!fIsBigPage)
1271	{
1272	/*
1273	* 4KB - page.
1274	*/
1275	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1276	RTGCPHYS GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
1277
1278	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1279	/* Reset the modification counter (OpenSolaris trashes tlb entries very often) */
1280	if (pShwPage->cModifications)
1281	pShwPage->cModifications = 1;
1282	# endif
1283
1284	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1285	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1286	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE/2);
1287	# endif
1288	if (pShwPage->GCPhys == GCPhys)
1289	{
1290	# if 0 /* likely cause of a major performance regression; must be SyncPageWorkerTrackDeref then */
1291	const unsigned iPTEDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1292	PSHWPT pPT = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1293	if (pPT->a[iPTEDst].n.u1Present)
1294	{
1295	/* This is very unlikely with caching/monitoring enabled. */
1296	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pShwPage, pPT->a[iPTEDst].u & SHW_PTE_PG_MASK, iPTEDst);
1297	ASMAtomicWriteSize(&pPT->a[iPTEDst], 0);
1298	}
1299	# else /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1300	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
1301	if (RT_SUCCESS(rc))
1302	rc = VINF_SUCCESS;
1303	# endif
1304	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4KBPages));
1305	PGM_INVL_PG(pVCpu, GCPtrPage);
1306	}
1307	else
1308	{
1309	/*
1310	* The page table address changed.
1311	*/
1312	LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
1313	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1314	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1315	ASMAtomicWriteSize(pPdeDst, 0);
1316	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1317	PGM_INVL_VCPU_TLBS(pVCpu);
1318	}
1319	}
1320	else
1321	{
1322	/*
1323	* 2/4MB - page.
1324	*/
1325	/* Before freeing the page, check if anything really changed. */
1326	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1327	RTGCPHYS GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
1328	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1329	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1330	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1331	# endif
1332	if ( pShwPage->GCPhys == GCPhys
1333	&& pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1334	{
1335	/* ASSUMES a the given bits are identical for 4M and normal PDEs */
1336	/** @todo This test is wrong as it cannot check the G bit!
1337	* FIXME */
1338	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US))
1339	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US))
1340	&& ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1341	\|\| (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1342	{
1343	LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1344	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip));
1345	return VINF_SUCCESS;
1346	}
1347	}
1348
1349	/*
1350	* Ok, the page table is present and it's been changed in the guest.
1351	* If we're in host context, we'll just mark it as not present taking the lazy approach.
1352	* We could do this for some flushes in GC too, but we need an algorithm for
1353	* deciding which 4MB pages containing code likely to be executed very soon.
1354	*/
1355	LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1356	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1357	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1358	ASMAtomicWriteSize(pPdeDst, 0);
1359	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4MBPages));
1360	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
1361	}
1362	}
1363	else
1364	{
1365	/*
1366	* Page directory is not present, mark shadow PDE not present.
1367	*/
1368	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
1369	{
1370	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1371	ASMAtomicWriteSize(pPdeDst, 0);
1372	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDNPs));
1373	PGM_INVL_PG(pVCpu, GCPtrPage);
1374	}
1375	else
1376	{
1377	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1378	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDMappings));
1379	}
1380	}
1381	return rc;
1382
1383	#else /* guest real and protected mode */
1384	/* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1385	return VINF_SUCCESS;
1386	#endif
1387	}
1388
1389
1390	/**
1391	* Update the tracking of shadowed pages.
1392	*
1393	* @param pVCpu The VMCPU handle.
1394	* @param pShwPage The shadow page.
1395	* @param HCPhys The physical page we is being dereferenced.
1396	* @param iPte Shadow PTE index
1397	*/
1398	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys, uint16_t iPte)
1399	{
1400	PVM pVM = pVCpu->CTX_SUFF(pVM);
1401
1402	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackDeref, a);
1403	LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1404
1405	/** @todo If this turns out to be a bottle neck (very likely) two things can be done:
1406	* 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1407	* 2. write protect all shadowed pages. I.e. implement caching.
1408	*/
1409	/** @todo duplicated in the 2nd half of pgmPoolTracDerefGCPhysHint */
1410
1411	/*
1412	* Find the guest address.
1413	*/
1414	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
1415	pRam;
1416	pRam = pRam->CTX_SUFF(pNext))
1417	{
1418	unsigned iPage = pRam->cb >> PAGE_SHIFT;
1419	while (iPage-- > 0)
1420	{
1421	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1422	{
1423	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1424
1425	Assert(pShwPage->cPresent);
1426	Assert(pPool->cPresent);
1427	pShwPage->cPresent--;
1428	pPool->cPresent--;
1429
1430	pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage], iPte);
1431	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackDeref, a);
1432	return;
1433	}
1434	}
1435	}
1436
1437	for (;;)
1438	AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys));
1439	}
1440
1441
1442	/**
1443	* Update the tracking of shadowed pages.
1444	*
1445	* @param pVCpu The VMCPU handle.
1446	* @param pShwPage The shadow page.
1447	* @param u16 The top 16-bit of the pPage->HCPhys.
1448	* @param pPage Pointer to the guest page. this will be modified.
1449	* @param iPTDst The index into the shadow table.
1450	*/
1451	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
1452	{
1453	PVM pVM = pVCpu->CTX_SUFF(pVM);
1454
1455	/*
1456	* Just deal with the simple first time here.
1457	*/
1458	if (!u16)
1459	{
1460	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackVirgin);
1461	u16 = PGMPOOL_TD_MAKE(1, pShwPage->idx);
1462	/* Save the page table index. */
1463	PGM_PAGE_SET_PTE_INDEX(pPage, iPTDst);
1464	}
1465	else
1466	u16 = pgmPoolTrackPhysExtAddref(pVM, pPage, u16, pShwPage->idx, iPTDst);
1467
1468	/* write back */
1469	Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x iPTDst=%#x\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst));
1470	PGM_PAGE_SET_TRACKING(pPage, u16);
1471
1472	/* update statistics. */
1473	pVM->pgm.s.CTX_SUFF(pPool)->cPresent++;
1474	pShwPage->cPresent++;
1475	if (pShwPage->iFirstPresent > iPTDst)
1476	pShwPage->iFirstPresent = iPTDst;
1477	}
1478
1479
1480	/**
1481	* Modifies a shadow PTE to account for access handlers.
1482	*
1483	* @param pVM The VM handle.
1484	* @param pPage The page in question.
1485	* @param fPteSrc The shadowed flags of the source PTE. Must include the
1486	* A (accessed) bit so it can be emulated correctly.
1487	* @param pPteDst The shadow PTE (output). This is temporary storage and
1488	* does not need to be set atomically.
1489	*/
1490	DECLINLINE(void) PGM_BTH_NAME(SyncHandlerPte)(PVM pVM, PCPGMPAGE pPage, uint64_t fPteSrc, PSHWPTE pPteDst)
1491	{
1492	/** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No.
1493	* Update: \#PF should deal with this before or after calling the handlers. It has all the info to do the job efficiently. */
1494	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1495	{
1496	LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark read-only\n", pPage));
1497	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1498	pPteDst->u = PGM_PAGE_GET_HCPHYS(pPage);
1499	pPteDst->n.u1Present = 1;
1500	pPteDst->n.u1Execute = 1;
1501	pPteDst->n.u1IgnorePAT = 1;
1502	pPteDst->n.u3EMT = VMX_EPT_MEMTYPE_WB;
1503	/* PteDst.n.u1Write = 0 && PteDst.n.u1Size = 0 */
1504	#else
1505	if (fPteSrc & X86_PTE_A)
1506	{
1507	SHW_PTE_SET(*pPteDst, fPteSrc \| PGM_PAGE_GET_HCPHYS(pPage));
1508	SHW_PTE_SET_RO(*pPteDst);
1509	}
1510	else
1511	SHW_PTE_SET(*pPteDst, 0);
1512	#endif
1513	}
1514	#ifdef PGM_WITH_MMIO_OPTIMIZATIONS
1515	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
1516	else if ( PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
1517	&& ( BTH_IS_NP_ACTIVE(pVM)
1518	\|\| (fPteSrc & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW) /** @todo Remove X86_PTE_US here and pGstWalk->Core.fEffectiveUS before the sync page test. */
1519	# if PGM_SHW_TYPE == PGM_TYPE_AMD64
1520	&& pVM->pgm.s.fLessThan52PhysicalAddressBits
1521	# endif
1522	)
1523	{
1524	LogFlow(("SyncHandlerPte: MMIO page -> invalid \n"));
1525	# if PGM_SHW_TYPE == PGM_TYPE_EPT
1526	/* 25.2.3.1: Reserved physical address bit -> EPT Misconfiguration (exit 49) */
1527	pPteDst->u = pVM->pgm.s.HCPhysInvMmioPg;
1528	/* 25.2.3.1: bits 2:0 = 010b -> EPT Misconfiguration (exit 49) */
1529	pPteDst->n.u1Present = 0;
1530	pPteDst->n.u1Write = 1;
1531	pPteDst->n.u1Execute = 0;
1532	/* 25.2.3.1: leaf && 2:0 != 0 && u3Emt in {2, 3, 7} -> EPT Misconfiguration */
1533	pPteDst->n.u3EMT = 7;
1534	# else
1535	/* Set high page frame bits that MBZ (bankers on PAE, CPU dependent on AMD64). */
1536	SHW_PTE_SET(*pPteDst, pVM->pgm.s.HCPhysInvMmioPg \| X86_PTE_PAE_MBZ_MASK_NO_NX \| X86_PTE_P);
1537	# endif
1538	}
1539	# endif
1540	#endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
1541	else
1542	{
1543	LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark not present\n", pPage));
1544	SHW_PTE_SET(*pPteDst, 0);
1545	}
1546	/** @todo count these kinds of entries. */
1547	}
1548
1549
1550	/**
1551	* Creates a 4K shadow page for a guest page.
1552	*
1553	* For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1554	* physical address. The PdeSrc argument only the flags are used. No page
1555	* structured will be mapped in this function.
1556	*
1557	* @param pVCpu The VMCPU handle.
1558	* @param pPteDst Destination page table entry.
1559	* @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1560	* Can safely assume that only the flags are being used.
1561	* @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1562	* @param pShwPage Pointer to the shadow page.
1563	* @param iPTDst The index into the shadow table.
1564	*
1565	* @remark Not used for 2/4MB pages!
1566	*/
1567	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc,
1568	PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1569	{
1570	PVM pVM = pVCpu->CTX_SUFF(pVM);
1571
1572	# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1573	&& PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1574	&& (PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64)
1575	if (pShwPage->fDirty)
1576	{
1577	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1578	PX86PTPAE pGstPT;
1579
1580	pGstPT = (PX86PTPAE)&pPool->aDirtyPages[pShwPage->idxDirty].aPage[0];
1581	pGstPT->a[iPTDst].u = PteSrc.u;
1582	}
1583	# endif
1584
1585	if ( PteSrc.n.u1Present
1586	&& GST_IS_PTE_VALID(pVCpu, PteSrc))
1587	{
1588	/*
1589	* Find the ram range.
1590	*/
1591	PPGMPAGE pPage;
1592	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GST_GET_PTE_GCPHYS(PteSrc), &pPage);
1593	if (RT_SUCCESS(rc))
1594	{
1595	/* Ignore ballooned pages.
1596	Don't return errors or use a fatal assert here as part of a
1597	shadow sync range might included ballooned pages. */
1598	if (PGM_PAGE_IS_BALLOONED(pPage))
1599	{
1600	Assert(!SHW_PTE_IS_P(pPteDst)); /* @todo user tracking needs updating if this triggers. */
1601	return;
1602	}
1603
1604	#ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1605	/* Make the page writable if necessary. */
1606	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1607	&& ( PGM_PAGE_IS_ZERO(pPage)
1608	\|\| ( PteSrc.n.u1Write
1609	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1610	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1611	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1612	# endif
1613	# ifdef VBOX_WITH_PAGE_SHARING
1614	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
1615	# endif
1616	)
1617	)
1618	)
1619	{
1620	rc = pgmPhysPageMakeWritable(pVM, pPage, GST_GET_PTE_GCPHYS(PteSrc));
1621	AssertRC(rc);
1622	}
1623	#endif
1624
1625	/*
1626	* Make page table entry.
1627	*/
1628	SHWPTE PteDst;
1629	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1630	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, GST_GET_PTE_SHW_FLAGS(pVCpu, PteSrc), &PteDst);
1631	else
1632	{
1633	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1634	/*
1635	* If the page or page directory entry is not marked accessed,
1636	* we mark the page not present.
1637	*/
1638	if (!PteSrc.n.u1Accessed \|\| !PdeSrc.n.u1Accessed)
1639	{
1640	LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1641	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,AccessedPage));
1642	SHW_PTE_SET(PteDst, 0);
1643	}
1644	/*
1645	* If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1646	* when the page is modified.
1647	*/
1648	else if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1649	{
1650	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPage));
1651	SHW_PTE_SET(PteDst,
1652	GST_GET_PTE_SHW_FLAGS(pVCpu, PteSrc)
1653	\| PGM_PAGE_GET_HCPHYS(pPage)
1654	\| PGM_PTFLAGS_TRACK_DIRTY);
1655	SHW_PTE_SET_RO(PteDst);
1656	}
1657	else
1658	#endif
1659	{
1660	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageSkipped));
1661	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1662	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage);
1663	PteDst.n.u1Present = 1;
1664	PteDst.n.u1Write = 1;
1665	PteDst.n.u1Execute = 1;
1666	PteDst.n.u1IgnorePAT = 1;
1667	PteDst.n.u3EMT = VMX_EPT_MEMTYPE_WB;
1668	/* PteDst.n.u1Size = 0 */
1669	#else
1670	SHW_PTE_SET(PteDst, GST_GET_PTE_SHW_FLAGS(pVCpu, PteSrc) \| PGM_PAGE_GET_HCPHYS(pPage));
1671	#endif
1672	}
1673
1674	/*
1675	* Make sure only allocated pages are mapped writable.
1676	*/
1677	if ( SHW_PTE_IS_P_RW(PteDst)
1678	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1679	{
1680	/* Still applies to shared pages. */
1681	Assert(!PGM_PAGE_IS_ZERO(pPage));
1682	SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet. Why, isn't it? */
1683	Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", (RTGCPHYS)GST_GET_PTE_GCPHYS(PteSrc), pPage, iPTDst));
1684	}
1685	}
1686
1687	/*
1688	* Keep user track up to date.
1689	*/
1690	if (SHW_PTE_IS_P(PteDst))
1691	{
1692	if (!SHW_PTE_IS_P(*pPteDst))
1693	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1694	else if (SHW_PTE_GET_HCPHYS(*pPteDst) != SHW_PTE_GET_HCPHYS(PteDst))
1695	{
1696	Log2(("SyncPageWorker: deref! pPteDst=%RX64 PteDst=%RX64\n", SHW_PTE_LOG64(pPteDst), SHW_PTE_LOG64(PteDst)));
1697	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst);
1698	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1699	}
1700	}
1701	else if (SHW_PTE_IS_P(*pPteDst))
1702	{
1703	Log2(("SyncPageWorker: deref! pPteDst=%RX64\n", SHW_PTE_LOG64(pPteDst)));
1704	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst);
1705	}
1706
1707	/*
1708	* Update statistics and commit the entry.
1709	*/
1710	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1711	if (!PteSrc.n.u1Global)
1712	pShwPage->fSeenNonGlobal = true;
1713	#endif
1714	SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
1715	return;
1716	}
1717
1718	/** @todo count these three different kinds. */
1719	Log2(("SyncPageWorker: invalid address in Pte\n"));
1720	}
1721	else if (!PteSrc.n.u1Present)
1722	Log2(("SyncPageWorker: page not present in Pte\n"));
1723	else
1724	Log2(("SyncPageWorker: invalid Pte\n"));
1725
1726	/*
1727	* The page is not present or the PTE is bad. Replace the shadow PTE by
1728	* an empty entry, making sure to keep the user tracking up to date.
1729	*/
1730	if (SHW_PTE_IS_P(*pPteDst))
1731	{
1732	Log2(("SyncPageWorker: deref! pPteDst=%RX64\n", SHW_PTE_LOG64(pPteDst)));
1733	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst);
1734	}
1735	SHW_PTE_ATOMIC_SET(*pPteDst, 0);
1736	}
1737
1738
1739	/**
1740	* Syncs a guest OS page.
1741	*
1742	* There are no conflicts at this point, neither is there any need for
1743	* page table allocations.
1744	*
1745	* When called in PAE or AMD64 guest mode, the guest PDPE shall be valid.
1746	* When called in AMD64 guest mode, the guest PML4E shall be valid.
1747	*
1748	* @returns VBox status code.
1749	* @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1750	* @param pVCpu The VMCPU handle.
1751	* @param PdeSrc Page directory entry of the guest.
1752	* @param GCPtrPage Guest context page address.
1753	* @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1754	* @param uErr Fault error (X86_TRAP_PF_*).
1755	*/
1756	static int PGM_BTH_NAME(SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr)
1757	{
1758	PVM pVM = pVCpu->CTX_SUFF(pVM);
1759	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1760	LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr));
1761
1762	Assert(PGMIsLockOwner(pVM));
1763
1764	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
1765	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
1766	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
1767	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
1768	&& PGM_SHW_TYPE != PGM_TYPE_EPT
1769
1770	/*
1771	* Assert preconditions.
1772	*/
1773	Assert(PdeSrc.n.u1Present);
1774	Assert(cPages);
1775	# if 0 /* rarely useful; leave for debugging. */
1776	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
1777	# endif
1778
1779	/*
1780	* Get the shadow PDE, find the shadow page table in the pool.
1781	*/
1782	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1783	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1784	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1785
1786	/* Fetch the pgm pool shadow descriptor. */
1787	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1788	Assert(pShwPde);
1789
1790	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1791	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1792	PPGMPOOLPAGE pShwPde = NULL;
1793	PX86PDPAE pPDDst;
1794
1795	/* Fetch the pgm pool shadow descriptor. */
1796	int rc2 = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
1797	AssertRCSuccessReturn(rc2, rc2);
1798	Assert(pShwPde);
1799
1800	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1801	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1802
1803	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1804	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1805	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1806	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
1807	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
1808
1809	int rc2 = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
1810	AssertRCSuccessReturn(rc2, rc2);
1811	Assert(pPDDst && pPdptDst);
1812	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1813	# endif
1814	SHWPDE PdeDst = *pPdeDst;
1815
1816	/*
1817	* - In the guest SMP case we could have blocked while another VCPU reused
1818	* this page table.
1819	* - With W7-64 we may also take this path when the the A bit is cleared on
1820	* higher level tables (PDPE/PML4E). The guest does not invalidate the
1821	* relevant TLB entries. If we're write monitoring any page mapped by
1822	* the modified entry, we may end up here with a "stale" TLB entry.
1823	*/
1824	if (!PdeDst.n.u1Present)
1825	{
1826	Log(("CPU%u: SyncPage: Pde at %RGv changed behind our back? (pPdeDst=%p/%RX64) uErr=%#x\n", pVCpu->idCpu, GCPtrPage, pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
1827	AssertMsg(pVM->cCpus > 1 \|\| (uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RW)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RW),
1828	("Unexpected missing PDE p=%p/%RX64 uErr=%#x\n", pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
1829	if (uErr & X86_TRAP_PF_P)
1830	PGM_INVL_PG(pVCpu, GCPtrPage);
1831	return VINF_SUCCESS; /* force the instruction to be executed again. */
1832	}
1833
1834	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1835	Assert(pShwPage);
1836
1837	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1838	/* Fetch the pgm pool shadow descriptor. */
1839	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
1840	Assert(pShwPde);
1841	# endif
1842
1843	/*
1844	* Check that the page is present and that the shadow PDE isn't out of sync.
1845	*/
1846	const bool fBigPage = PdeSrc.b.u1Size && GST_IS_PSE_ACTIVE(pVCpu);
1847	const bool fPdeValid = !fBigPage ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc);
1848	RTGCPHYS GCPhys;
1849	if (!fBigPage)
1850	{
1851	GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
1852	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1853	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1854	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE / 2);
1855	# endif
1856	}
1857	else
1858	{
1859	GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
1860	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1861	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1862	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1863	# endif
1864	}
1865	/** @todo This doesn't check the G bit of 2/4MB pages. FIXME */
1866	if ( fPdeValid
1867	&& pShwPage->GCPhys == GCPhys
1868	&& PdeSrc.n.u1Present
1869	&& PdeSrc.n.u1User == PdeDst.n.u1User
1870	&& (PdeSrc.n.u1Write == PdeDst.n.u1Write \|\| !PdeDst.n.u1Write)
1871	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
1872	&& (PdeSrc.n.u1NoExecute == PdeDst.n.u1NoExecute \|\| !GST_IS_NX_ACTIVE(pVCpu))
1873	# endif
1874	)
1875	{
1876	/*
1877	* Check that the PDE is marked accessed already.
1878	* Since we set the accessed bit before getting here on a #PF, this
1879	* check is only meant for dealing with non-#PF'ing paths.
1880	*/
1881	if (PdeSrc.n.u1Accessed)
1882	{
1883	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1884	if (!fBigPage)
1885	{
1886	/*
1887	* 4KB Page - Map the guest page table.
1888	*/
1889	PGSTPT pPTSrc;
1890	int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
1891	if (RT_SUCCESS(rc))
1892	{
1893	# ifdef PGM_SYNC_N_PAGES
1894	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
1895	if ( cPages > 1
1896	&& !(uErr & X86_TRAP_PF_P)
1897	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
1898	{
1899	/*
1900	* This code path is currently only taken when the caller is PGMTrap0eHandler
1901	* for non-present pages!
1902	*
1903	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1904	* deal with locality.
1905	*/
1906	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1907	const unsigned iPTDstPage = iPTDst;
1908	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1909	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1910	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1911	# else
1912	const unsigned offPTSrc = 0;
1913	# endif
1914	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
1915	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1916	iPTDst = 0;
1917	else
1918	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1919	for (; iPTDst < iPTDstEnd; iPTDst++)
1920	{
1921	if ( !SHW_PTE_IS_P(pPTDst->a[iPTDst])
1922	\|\| iPTDst == iPTDstPage) /* always sync GCPtrPage */
1923	{
1924	GSTPTE PteSrc = pPTSrc->a[offPTSrc + iPTDst];
1925	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
1926	NOREF(GCPtrCurPage);
1927	#ifndef IN_RING0
1928	/*
1929	* Assuming kernel code will be marked as supervisor - and not as user level
1930	* and executed using a conforming code selector - And marked as readonly.
1931	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
1932	*/
1933	PPGMPAGE pPage;
1934	if ( ((PdeSrc.u & PteSrc.u) & (X86_PTE_RW \| X86_PTE_US))
1935	\|\| iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK) /* always sync GCPtrPage */
1936	\|\| !CSAMDoesPageNeedScanning(pVM, GCPtrCurPage)
1937	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
1938	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1939	)
1940	#endif /* else: CSAM not active */
1941	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1942	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1943	GCPtrCurPage, PteSrc.n.u1Present,
1944	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1945	PteSrc.n.u1User & PdeSrc.n.u1User,
1946	(uint64_t)PteSrc.u,
1947	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
1948	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
1949	}
1950	}
1951	}
1952	else
1953	# endif /* PGM_SYNC_N_PAGES */
1954	{
1955	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1956	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1957	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1958	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1959	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
1960	GCPtrPage, PteSrc.n.u1Present,
1961	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1962	PteSrc.n.u1User & PdeSrc.n.u1User,
1963	(uint64_t)PteSrc.u,
1964	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
1965	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
1966	}
1967	}
1968	else /* MMIO or invalid page: emulated in #PF handler. */
1969	{
1970	LogFlow(("PGM_GCPHYS_2_PTR %RGp failed with %Rrc\n", GCPhys, rc));
1971	Assert(!SHW_PTE_IS_P(pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK]));
1972	}
1973	}
1974	else
1975	{
1976	/*
1977	* 4/2MB page - lazy syncing shadow 4K pages.
1978	* (There are many causes of getting here, it's no longer only CSAM.)
1979	*/
1980	/* Calculate the GC physical address of this 4KB shadow page. */
1981	GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc) \| (GCPtrPage & GST_BIG_PAGE_OFFSET_MASK);
1982	/* Find ram range. */
1983	PPGMPAGE pPage;
1984	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
1985	if (RT_SUCCESS(rc))
1986	{
1987	AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
1988
1989	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1990	/* Try to make the page writable if necessary. */
1991	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1992	&& ( PGM_PAGE_IS_ZERO(pPage)
1993	\|\| ( PdeSrc.n.u1Write
1994	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1995	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1996	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1997	# endif
1998	# ifdef VBOX_WITH_PAGE_SHARING
1999	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
2000	# endif
2001	)
2002	)
2003	)
2004	{
2005	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
2006	AssertRC(rc);
2007	}
2008	# endif
2009
2010	/*
2011	* Make shadow PTE entry.
2012	*/
2013	SHWPTE PteDst;
2014	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2015	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc), &PteDst);
2016	else
2017	SHW_PTE_SET(PteDst, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc) \| PGM_PAGE_GET_HCPHYS(pPage));
2018
2019	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2020	if ( SHW_PTE_IS_P(PteDst)
2021	&& !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2022	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2023
2024	/* Make sure only allocated pages are mapped writable. */
2025	if ( SHW_PTE_IS_P_RW(PteDst)
2026	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2027	{
2028	/* Still applies to shared pages. */
2029	Assert(!PGM_PAGE_IS_ZERO(pPage));
2030	SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
2031	Log3(("SyncPage: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, GCPtrPage));
2032	}
2033
2034	SHW_PTE_ATOMIC_SET2(pPTDst->a[iPTDst], PteDst);
2035
2036	/*
2037	* If the page is not flagged as dirty and is writable, then make it read-only
2038	* at PD level, so we can set the dirty bit when the page is modified.
2039	*
2040	* ASSUMES that page access handlers are implemented on page table entry level.
2041	* Thus we will first catch the dirty access and set PDE.D and restart. If
2042	* there is an access handler, we'll trap again and let it work on the problem.
2043	*/
2044	/** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
2045	* As for invlpg, it simply frees the whole shadow PT.
2046	* ...It's possibly because the guest clears it and the guest doesn't really tell us... */
2047	if ( !PdeSrc.b.u1Dirty
2048	&& PdeSrc.b.u1Write)
2049	{
2050	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2051	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2052	PdeDst.n.u1Write = 0;
2053	}
2054	else
2055	{
2056	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2057	PdeDst.n.u1Write = PdeSrc.n.u1Write;
2058	}
2059	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2060	Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n",
2061	GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
2062	PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2063	}
2064	else
2065	{
2066	LogFlow(("PGM_GCPHYS_2_PTR %RGp (big) failed with %Rrc\n", GCPhys, rc));
2067	/** @todo must wipe the shadow page table entry in this
2068	* case. */
2069	}
2070	}
2071	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2072	return VINF_SUCCESS;
2073	}
2074
2075	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPagePDNAs));
2076	}
2077	else if (fPdeValid)
2078	{
2079	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPagePDOutOfSync));
2080	Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2081	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2082	}
2083	else
2084	{
2085	/// @todo STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSyncAndInvalid));
2086	Log2(("SyncPage: Bad PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2087	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2088	}
2089
2090	/*
2091	* Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
2092	* Yea, I'm lazy.
2093	*/
2094	pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
2095	ASMAtomicWriteSize(pPdeDst, 0);
2096
2097	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2098	PGM_INVL_VCPU_TLBS(pVCpu);
2099	return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
2100
2101
2102	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2103	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2104	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2105	&& !defined(IN_RC)
2106
2107	# ifdef PGM_SYNC_N_PAGES
2108	/*
2109	* Get the shadow PDE, find the shadow page table in the pool.
2110	*/
2111	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2112	X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
2113
2114	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2115	X86PDEPAE PdeDst = pgmShwGetPaePDE(pVCpu, GCPtrPage);
2116
2117	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2118	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2119	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpt);
2120	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2121	X86PDEPAE PdeDst;
2122	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2123
2124	int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2125	AssertRCSuccessReturn(rc, rc);
2126	Assert(pPDDst && pPdptDst);
2127	PdeDst = pPDDst->a[iPDDst];
2128	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2129	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2130	PEPTPD pPDDst;
2131	EPTPDE PdeDst;
2132
2133	int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, NULL, &pPDDst);
2134	if (rc != VINF_SUCCESS)
2135	{
2136	AssertRC(rc);
2137	return rc;
2138	}
2139	Assert(pPDDst);
2140	PdeDst = pPDDst->a[iPDDst];
2141	# endif
2142	/* In the guest SMP case we could have blocked while another VCPU reused this page table. */
2143	if (!PdeDst.n.u1Present)
2144	{
2145	AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)PdeDst.u));
2146	Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", pVCpu->idCpu, GCPtrPage));
2147	return VINF_SUCCESS; /* force the instruction to be executed again. */
2148	}
2149
2150	/* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */
2151	if (PdeDst.n.u1Size)
2152	{
2153	Assert(pVM->pgm.s.fNestedPaging);
2154	Log(("CPU%d: SyncPage: Pde (big:%RX64) at %RGv changed behind our back!\n", pVCpu->idCpu, PdeDst.u, GCPtrPage));
2155	return VINF_SUCCESS;
2156	}
2157
2158	/* Mask away the page offset. */
2159	GCPtrPage &= ~((RTGCPTR)0xfff);
2160
2161	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2162	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2163
2164	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
2165	if ( cPages > 1
2166	&& !(uErr & X86_TRAP_PF_P)
2167	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2168	{
2169	/*
2170	* This code path is currently only taken when the caller is PGMTrap0eHandler
2171	* for non-present pages!
2172	*
2173	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2174	* deal with locality.
2175	*/
2176	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2177	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2178	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2179	iPTDst = 0;
2180	else
2181	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2182	for (; iPTDst < iPTDstEnd; iPTDst++)
2183	{
2184	if (!SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2185	{
2186	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT);
2187	GSTPTE PteSrc;
2188
2189	/* Fake the page table entry */
2190	PteSrc.u = GCPtrCurPage;
2191	PteSrc.n.u1Present = 1;
2192	PteSrc.n.u1Dirty = 1;
2193	PteSrc.n.u1Accessed = 1;
2194	PteSrc.n.u1Write = 1;
2195	PteSrc.n.u1User = 1;
2196
2197	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2198	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
2199	GCPtrCurPage, PteSrc.n.u1Present,
2200	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2201	PteSrc.n.u1User & PdeSrc.n.u1User,
2202	(uint64_t)PteSrc.u,
2203	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2204	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2205
2206	if (RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)))
2207	break;
2208	}
2209	else
2210	Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n", (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT), iPTDst, SHW_PTE_LOG64(pPTDst->a[iPTDst]) ));
2211	}
2212	}
2213	else
2214	# endif /* PGM_SYNC_N_PAGES */
2215	{
2216	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2217	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT);
2218	GSTPTE PteSrc;
2219
2220	/* Fake the page table entry */
2221	PteSrc.u = GCPtrCurPage;
2222	PteSrc.n.u1Present = 1;
2223	PteSrc.n.u1Dirty = 1;
2224	PteSrc.n.u1Accessed = 1;
2225	PteSrc.n.u1Write = 1;
2226	PteSrc.n.u1User = 1;
2227	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2228
2229	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}PteDst=%08llx%s\n",
2230	GCPtrPage, PteSrc.n.u1Present,
2231	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2232	PteSrc.n.u1User & PdeSrc.n.u1User,
2233	(uint64_t)PteSrc.u,
2234	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2235	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2236	}
2237	return VINF_SUCCESS;
2238
2239	#else
2240	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2241	return VERR_INTERNAL_ERROR;
2242	#endif
2243	}
2244
2245
2246	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2247
2248	/**
2249	* CheckPageFault helper for returning a page fault indicating a non-present
2250	* (NP) entry in the page translation structures.
2251	*
2252	* @returns VINF_EM_RAW_GUEST_TRAP.
2253	* @param pVCpu The virtual CPU to operate on.
2254	* @param uErr The error code of the shadow fault. Corrections to
2255	* TRPM's copy will be made if necessary.
2256	* @param GCPtrPage For logging.
2257	* @param uPageFaultLevel For logging.
2258	*/
2259	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnNP)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2260	{
2261	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2262	AssertMsg(!(uErr & X86_TRAP_PF_P), ("%#x\n", uErr));
2263	AssertMsg(!(uErr & X86_TRAP_PF_RSVD), ("%#x\n", uErr));
2264	if (uErr & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
2265	TRPMSetErrorCode(pVCpu, uErr & ~(X86_TRAP_PF_RSVD \| X86_TRAP_PF_P));
2266
2267	Log(("CheckPageFault: real page fault (notp) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2268	return VINF_EM_RAW_GUEST_TRAP;
2269	}
2270
2271
2272	/**
2273	* CheckPageFault helper for returning a page fault indicating a reserved bit
2274	* (RSVD) error in the page translation structures.
2275	*
2276	* @returns VINF_EM_RAW_GUEST_TRAP.
2277	* @param pVCpu The virtual CPU to operate on.
2278	* @param uErr The error code of the shadow fault. Corrections to
2279	* TRPM's copy will be made if necessary.
2280	* @param GCPtrPage For logging.
2281	* @param uPageFaultLevel For logging.
2282	*/
2283	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnRSVD)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2284	{
2285	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2286	if ((uErr & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) != (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
2287	TRPMSetErrorCode(pVCpu, uErr \| X86_TRAP_PF_RSVD \| X86_TRAP_PF_P);
2288
2289	Log(("CheckPageFault: real page fault (rsvd) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2290	return VINF_EM_RAW_GUEST_TRAP;
2291	}
2292
2293
2294	/**
2295	* CheckPageFault helper for returning a page protection fault (P).
2296	*
2297	* @returns VINF_EM_RAW_GUEST_TRAP.
2298	* @param pVCpu The virtual CPU to operate on.
2299	* @param uErr The error code of the shadow fault. Corrections to
2300	* TRPM's copy will be made if necessary.
2301	* @param GCPtrPage For logging.
2302	* @param uPageFaultLevel For logging.
2303	*/
2304	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnProt)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2305	{
2306	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2307	AssertMsg(uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID), ("%#x\n", uErr));
2308	if ((uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) != X86_TRAP_PF_P)
2309	TRPMSetErrorCode(pVCpu, (uErr & ~X86_TRAP_PF_RSVD) \| X86_TRAP_PF_P);
2310
2311	Log(("CheckPageFault: real page fault (prot) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2312	return VINF_EM_RAW_GUEST_TRAP;
2313	}
2314
2315
2316	/**
2317	* Handle dirty bit tracking faults.
2318	*
2319	* @returns VBox status code.
2320	* @param pVCpu The VMCPU handle.
2321	* @param uErr Page fault error code.
2322	* @param pPdeSrc Guest page directory entry.
2323	* @param pPdeDst Shadow page directory entry.
2324	* @param GCPtrPage Guest context page address.
2325	*/
2326	static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, RTGCPTR GCPtrPage)
2327	{
2328	PVM pVM = pVCpu->CTX_SUFF(pVM);
2329	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2330
2331	Assert(PGMIsLockOwner(pVM));
2332
2333	/*
2334	* Handle big page.
2335	*/
2336	if (pPdeSrc->b.u1Size && GST_IS_PSE_ACTIVE(pVCpu))
2337	{
2338	if ( pPdeDst->n.u1Present
2339	&& (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY))
2340	{
2341	SHWPDE PdeDst = *pPdeDst;
2342
2343	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageTrap));
2344	Assert(pPdeSrc->b.u1Write);
2345
2346	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
2347	* fault again and take this path to only invalidate the entry (see below).
2348	*/
2349	PdeDst.n.u1Write = 1;
2350	PdeDst.n.u1Accessed = 1;
2351	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2352	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2353	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
2354	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2355	}
2356
2357	# ifdef IN_RING0
2358	/* Check for stale TLB entry; only applies to the SMP guest case. */
2359	if ( pVM->cCpus > 1
2360	&& pPdeDst->n.u1Write
2361	&& pPdeDst->n.u1Accessed)
2362	{
2363	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2364	if (pShwPage)
2365	{
2366	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2367	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2368	if (SHW_PTE_IS_P_RW(*pPteDst))
2369	{
2370	/* Stale TLB entry. */
2371	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageStale));
2372	PGM_INVL_PG(pVCpu, GCPtrPage);
2373	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2374	}
2375	}
2376	}
2377	# endif /* IN_RING0 */
2378	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2379	}
2380
2381	/*
2382	* Map the guest page table.
2383	*/
2384	PGSTPT pPTSrc;
2385	int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(*pPdeSrc), &pPTSrc);
2386	if (RT_FAILURE(rc))
2387	{
2388	AssertRC(rc);
2389	return rc;
2390	}
2391
2392	if (pPdeDst->n.u1Present)
2393	{
2394	GSTPTE const *pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2395	const GSTPTE PteSrc = *pPteSrc;
2396
2397	#ifndef IN_RING0
2398	/* Bail out here as pgmPoolGetPage will return NULL and we'll crash below.
2399	* Our individual shadow handlers will provide more information and force a fatal exit.
2400	*/
2401	if (MMHyperIsInsideArea(pVM, (RTGCPTR)GCPtrPage))
2402	{
2403	LogRel(("CheckPageFault: write to hypervisor region %RGv\n", GCPtrPage));
2404	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2405	}
2406	#endif
2407	/*
2408	* Map shadow page table.
2409	*/
2410	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2411	if (pShwPage)
2412	{
2413	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2414	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2415	if (SHW_PTE_IS_P(pPteDst)) /* @todo Optimize accessed bit emulation? */
2416	{
2417	if (SHW_PTE_IS_TRACK_DIRTY(*pPteDst))
2418	{
2419	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GST_GET_PTE_GCPHYS(*pPteSrc));
2420	SHWPTE PteDst = *pPteDst;
2421
2422	LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage));
2423	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageTrap));
2424
2425	Assert(pPteSrc->n.u1Write);
2426
2427	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB
2428	* entry will not harm; write access will simply fault again and
2429	* take this path to only invalidate the entry.
2430	*/
2431	if (RT_LIKELY(pPage))
2432	{
2433	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2434	{
2435	AssertMsgFailed(("%R[pgmpage] - we don't set PGM_PTFLAGS_TRACK_DIRTY for these pages\n", pPage));
2436	Assert(!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage));
2437	/* Assuming write handlers here as the PTE is present (otherwise we wouldn't be here). */
2438	SHW_PTE_SET_RO(PteDst);
2439	}
2440	else
2441	{
2442	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
2443	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
2444	{
2445	rc = pgmPhysPageMakeWritable(pVM, pPage, GST_GET_PTE_GCPHYS(*pPteSrc));
2446	AssertRC(rc);
2447	}
2448	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED)
2449	SHW_PTE_SET_RW(PteDst);
2450	else
2451	{
2452	/* Still applies to shared pages. */
2453	Assert(!PGM_PAGE_IS_ZERO(pPage));
2454	SHW_PTE_SET_RO(PteDst);
2455	}
2456	}
2457	}
2458	else
2459	SHW_PTE_SET_RW(PteDst); /** @todo r=bird: This doesn't make sense to me. */
2460
2461	SHW_PTE_SET(PteDst, (SHW_PTE_GET_U(PteDst) \| X86_PTE_D \| X86_PTE_A) & ~(uint64_t)PGM_PTFLAGS_TRACK_DIRTY);
2462	SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
2463	PGM_INVL_PG(pVCpu, GCPtrPage);
2464	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2465	}
2466
2467	# ifdef IN_RING0
2468	/* Check for stale TLB entry; only applies to the SMP guest case. */
2469	if ( pVM->cCpus > 1
2470	&& SHW_PTE_IS_RW(*pPteDst)
2471	&& SHW_PTE_IS_A(*pPteDst))
2472	{
2473	/* Stale TLB entry. */
2474	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageStale));
2475	PGM_INVL_PG(pVCpu, GCPtrPage);
2476	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2477	}
2478	# endif
2479	}
2480	}
2481	else
2482	AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
2483	}
2484
2485	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2486	}
2487
2488	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
2489
2490
2491	/**
2492	* Sync a shadow page table.
2493	*
2494	* The shadow page table is not present in the shadow PDE.
2495	*
2496	* Handles mapping conflicts.
2497	*
2498	* This is called by VerifyAccessSyncPage, PrefetchPage, InvalidatePage (on
2499	* conflict), and Trap0eHandler.
2500	*
2501	* A precodition for this method is that the shadow PDE is not present. The
2502	* caller must take the PGM lock before checking this and continue to hold it
2503	* when calling this method.
2504	*
2505	* @returns VBox status code.
2506	* @param pVCpu The VMCPU handle.
2507	* @param iPD Page directory index.
2508	* @param pPDSrc Source page directory (i.e. Guest OS page directory).
2509	* Assume this is a temporary mapping.
2510	* @param GCPtrPage GC Pointer of the page that caused the fault
2511	*/
2512	static int PGM_BTH_NAME(SyncPT)(PVMCPU pVCpu, unsigned iPDSrc, PGSTPD pPDSrc, RTGCPTR GCPtrPage)
2513	{
2514	PVM pVM = pVCpu->CTX_SUFF(pVM);
2515	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2516
2517	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2518	#if 0 /* rarely useful; leave for debugging. */
2519	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPtPD[iPDSrc]);
2520	#endif
2521	LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage));
2522
2523	Assert(PGMIsLocked(pVM));
2524
2525	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2526	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
2527	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
2528	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2529	&& PGM_SHW_TYPE != PGM_TYPE_EPT
2530
2531	int rc = VINF_SUCCESS;
2532
2533	/*
2534	* Some input validation first.
2535	*/
2536	AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
2537
2538	/*
2539	* Get the relevant shadow PDE entry.
2540	*/
2541	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2542	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2543	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
2544
2545	/* Fetch the pgm pool shadow descriptor. */
2546	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2547	Assert(pShwPde);
2548
2549	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2550	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2551	PPGMPOOLPAGE pShwPde = NULL;
2552	PX86PDPAE pPDDst;
2553	PSHWPDE pPdeDst;
2554
2555	/* Fetch the pgm pool shadow descriptor. */
2556	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
2557	AssertRCSuccessReturn(rc, rc);
2558	Assert(pShwPde);
2559
2560	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
2561	pPdeDst = &pPDDst->a[iPDDst];
2562
2563	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2564	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2565	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2566	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2567	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2568	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2569	AssertRCSuccessReturn(rc, rc);
2570	Assert(pPDDst);
2571	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2572	# endif
2573	SHWPDE PdeDst = *pPdeDst;
2574
2575	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2576	/* Fetch the pgm pool shadow descriptor. */
2577	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2578	Assert(pShwPde);
2579	# endif
2580
2581	# ifndef PGM_WITHOUT_MAPPINGS
2582	/*
2583	* Check for conflicts.
2584	* RC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
2585	* R3: Simply resolve the conflict.
2586	*/
2587	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2588	{
2589	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
2590	# ifndef IN_RING3
2591	Log(("SyncPT: Conflict at %RGv\n", GCPtrPage));
2592	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2593	return VERR_ADDRESS_CONFLICT;
2594
2595	# else /* IN_RING3 */
2596	PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
2597	Assert(pMapping);
2598	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2599	rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2600	# elif PGM_GST_TYPE == PGM_TYPE_PAE
2601	rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2602	# else
2603	AssertFailed(); /* can't happen for amd64 */
2604	# endif
2605	if (RT_FAILURE(rc))
2606	{
2607	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2608	return rc;
2609	}
2610	PdeDst = *pPdeDst;
2611	# endif /* IN_RING3 */
2612	}
2613	# endif /* !PGM_WITHOUT_MAPPINGS */
2614	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2615
2616	/*
2617	* Sync the page directory entry.
2618	*/
2619	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2620	const bool fPageTable = !PdeSrc.b.u1Size \|\| !GST_IS_PSE_ACTIVE(pVCpu);
2621	if ( PdeSrc.n.u1Present
2622	&& (fPageTable ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc)) )
2623	{
2624	/*
2625	* Allocate & map the page table.
2626	*/
2627	PSHWPT pPTDst;
2628	PPGMPOOLPAGE pShwPage;
2629	RTGCPHYS GCPhys;
2630	if (fPageTable)
2631	{
2632	GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
2633	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2634	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2635	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE / 2);
2636	# endif
2637	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
2638	}
2639	else
2640	{
2641	PGMPOOLACCESS enmAccess;
2642	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2643	const bool fNoExecute = PdeSrc.n.u1NoExecute && GST_IS_NX_ACTIVE(pVCpu);
2644	# else
2645	const bool fNoExecute = false;
2646	# endif
2647
2648	GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
2649	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2650	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2651	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
2652	# endif
2653	/* Determine the right kind of large page to avoid incorrect cached entry reuse. */
2654	if (PdeSrc.n.u1User)
2655	{
2656	if (PdeSrc.n.u1Write)
2657	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_RW_NX : PGMPOOLACCESS_USER_RW;
2658	else
2659	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_R_NX : PGMPOOLACCESS_USER_R;
2660	}
2661	else
2662	{
2663	if (PdeSrc.n.u1Write)
2664	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW;
2665	else
2666	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_R_NX : PGMPOOLACCESS_SUPERVISOR_R;
2667	}
2668	rc = pgmPoolAllocEx(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, enmAccess, pShwPde->idx, iPDDst, false /fLockPage/,
2669	&pShwPage);
2670	}
2671	if (rc == VINF_SUCCESS)
2672	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2673	else if (rc == VINF_PGM_CACHED_PAGE)
2674	{
2675	/*
2676	* The PT was cached, just hook it up.
2677	*/
2678	if (fPageTable)
2679	PdeDst.u = pShwPage->Core.Key \| GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2680	else
2681	{
2682	PdeDst.u = pShwPage->Core.Key \| GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2683	/* (see explanation and assumptions further down.) */
2684	if ( !PdeSrc.b.u1Dirty
2685	&& PdeSrc.b.u1Write)
2686	{
2687	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2688	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2689	PdeDst.b.u1Write = 0;
2690	}
2691	}
2692	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2693	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2694	return VINF_SUCCESS;
2695	}
2696	else if (rc == VERR_PGM_POOL_FLUSHED)
2697	{
2698	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2699	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2700	return VINF_PGM_SYNC_CR3;
2701	}
2702	else
2703	AssertMsgFailedReturn(("rc=%Rrc\n", rc), VERR_INTERNAL_ERROR);
2704	/** @todo Why do we bother preserving X86_PDE_AVL_MASK here?
2705	* Both PGM_PDFLAGS_MAPPING and PGM_PDFLAGS_TRACK_DIRTY should be
2706	* irrelevant at this point. */
2707	PdeDst.u &= X86_PDE_AVL_MASK;
2708	PdeDst.u \|= pShwPage->Core.Key;
2709
2710	/*
2711	* Page directory has been accessed (this is a fault situation, remember).
2712	*/
2713	/** @todo
2714	* Well, when the caller is PrefetchPage or InvalidatePage is isn't a
2715	* fault situation. What's more, the Trap0eHandler has already set the
2716	* accessed bit. So, it's actually just VerifyAccessSyncPage which
2717	* might need setting the accessed flag.
2718	*
2719	* The best idea is to leave this change to the caller and add an
2720	* assertion that it's set already. */
2721	pPDSrc->a[iPDSrc].n.u1Accessed = 1;
2722	if (fPageTable)
2723	{
2724	/*
2725	* Page table - 4KB.
2726	*
2727	* Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
2728	*/
2729	Log2(("SyncPT: 4K %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
2730	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
2731	PGSTPT pPTSrc;
2732	rc = PGM_GCPHYS_2_PTR(pVM, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
2733	if (RT_SUCCESS(rc))
2734	{
2735	/*
2736	* Start by syncing the page directory entry so CSAM's TLB trick works.
2737	*/
2738	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| X86_PDE_AVL_MASK))
2739	\| GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2740	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2741	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2742
2743	/*
2744	* Directory/page user or supervisor privilege: (same goes for read/write)
2745	*
2746	* Directory Page Combined
2747	* U/S U/S U/S
2748	* 0 0 0
2749	* 0 1 0
2750	* 1 0 0
2751	* 1 1 1
2752	*
2753	* Simple AND operation. Table listed for completeness.
2754	*
2755	*/
2756	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT4K));
2757	# ifdef PGM_SYNC_N_PAGES
2758	unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2759	unsigned iPTDst = iPTBase;
2760	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2761	if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
2762	iPTDst = 0;
2763	else
2764	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2765	# else /* !PGM_SYNC_N_PAGES */
2766	unsigned iPTDst = 0;
2767	const unsigned iPTDstEnd = RT_ELEMENTS(pPTDst->a);
2768	# endif /* !PGM_SYNC_N_PAGES */
2769	RTGCPTR GCPtrCur = (GCPtrPage & ~(RTGCPTR)((1 << SHW_PD_SHIFT) - 1))
2770	\| ((RTGCPTR)iPTDst << PAGE_SHIFT);
2771	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2772	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2773	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2774	# else
2775	const unsigned offPTSrc = 0;
2776	# endif
2777	for (; iPTDst < iPTDstEnd; iPTDst++, GCPtrCur += PAGE_SIZE)
2778	{
2779	const unsigned iPTSrc = iPTDst + offPTSrc;
2780	const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2781
2782	if (PteSrc.n.u1Present)
2783	{
2784	# ifndef IN_RING0
2785	/*
2786	* Assuming kernel code will be marked as supervisor - and not as user level
2787	* and executed using a conforming code selector - And marked as readonly.
2788	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
2789	*/
2790	PPGMPAGE pPage;
2791	if ( ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW \| X86_PTE_US))
2792	\|\| !CSAMDoesPageNeedScanning(pVM, GCPtrCur)
2793	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, GST_GET_PTE_GCPHYS(PteSrc)))
2794	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2795	)
2796	# endif
2797	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2798	Log2(("SyncPT: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
2799	GCPtrCur,
2800	PteSrc.n.u1Present,
2801	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2802	PteSrc.n.u1User & PdeSrc.n.u1User,
2803	(uint64_t)PteSrc.u,
2804	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : "", SHW_PTE_LOG64(pPTDst->a[iPTDst]), iPTSrc, PdeSrc.au32[0],
2805	(RTGCPHYS)(GST_GET_PDE_GCPHYS(PdeSrc) + iPTSrc*sizeof(PteSrc)) ));
2806	}
2807	/* else: the page table was cleared by the pool */
2808	} /* for PTEs */
2809	}
2810	}
2811	else
2812	{
2813	/*
2814	* Big page - 2/4MB.
2815	*
2816	* We'll walk the ram range list in parallel and optimize lookups.
2817	* We will only sync on shadow page table at a time.
2818	*/
2819	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT4M));
2820
2821	/**
2822	* @todo It might be more efficient to sync only a part of the 4MB
2823	* page (similar to what we do for 4KB PDs).
2824	*/
2825
2826	/*
2827	* Start by syncing the page directory entry.
2828	*/
2829	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
2830	\| GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2831
2832	/*
2833	* If the page is not flagged as dirty and is writable, then make it read-only
2834	* at PD level, so we can set the dirty bit when the page is modified.
2835	*
2836	* ASSUMES that page access handlers are implemented on page table entry level.
2837	* Thus we will first catch the dirty access and set PDE.D and restart. If
2838	* there is an access handler, we'll trap again and let it work on the problem.
2839	*/
2840	/** @todo move the above stuff to a section in the PGM documentation. */
2841	Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
2842	if ( !PdeSrc.b.u1Dirty
2843	&& PdeSrc.b.u1Write)
2844	{
2845	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2846	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2847	PdeDst.b.u1Write = 0;
2848	}
2849	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2850	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2851
2852	/*
2853	* Fill the shadow page table.
2854	*/
2855	/* Get address and flags from the source PDE. */
2856	SHWPTE PteDstBase;
2857	SHW_PTE_SET(PteDstBase, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc));
2858
2859	/* Loop thru the entries in the shadow PT. */
2860	const RTGCPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
2861	Log2(("SyncPT: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n",
2862	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
2863	GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2864	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
2865	unsigned iPTDst = 0;
2866	while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2867	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2868	{
2869	/* Advance ram range list. */
2870	while (pRam && GCPhys > pRam->GCPhysLast)
2871	pRam = pRam->CTX_SUFF(pNext);
2872	if (pRam && GCPhys >= pRam->GCPhys)
2873	{
2874	unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2875	do
2876	{
2877	/* Make shadow PTE. */
2878	PPGMPAGE pPage = &pRam->aPages[iHCPage];
2879	SHWPTE PteDst;
2880
2881	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2882	/* Try to make the page writable if necessary. */
2883	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2884	&& ( PGM_PAGE_IS_ZERO(pPage)
2885	\|\| ( SHW_PTE_IS_RW(PteDstBase)
2886	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2887	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
2888	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
2889	# endif
2890	# ifdef VBOX_WITH_PAGE_SHARING
2891	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
2892	# endif
2893	&& !PGM_PAGE_IS_BALLOONED(pPage))
2894	)
2895	)
2896	{
2897	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
2898	AssertRCReturn(rc, rc);
2899	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2900	break;
2901	}
2902	# endif
2903
2904	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2905	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, SHW_PTE_GET_U(PteDstBase), &PteDst);
2906	else if (PGM_PAGE_IS_BALLOONED(pPage))
2907	SHW_PTE_SET(PteDst, 0); /* Handle ballooned pages at #PF time. */
2908	# ifndef IN_RING0
2909	/*
2910	* Assuming kernel code will be marked as supervisor and not as user level and executed
2911	* using a conforming code selector. Don't check for readonly, as that implies the whole
2912	* 4MB can be code or readonly data. Linux enables write access for its large pages.
2913	*/
2914	else if ( !PdeSrc.n.u1User
2915	&& CSAMDoesPageNeedScanning(pVM, GCPtr \| (iPTDst << SHW_PT_SHIFT)))
2916	SHW_PTE_SET(PteDst, 0);
2917	# endif
2918	else
2919	SHW_PTE_SET(PteDst, PGM_PAGE_GET_HCPHYS(pPage) \| SHW_PTE_GET_U(PteDstBase));
2920
2921	/* Only map writable pages writable. */
2922	if ( SHW_PTE_IS_P_RW(PteDst)
2923	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2924	{
2925	/* Still applies to shared pages. */
2926	Assert(!PGM_PAGE_IS_ZERO(pPage));
2927	SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
2928	Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))));
2929	}
2930
2931	if (SHW_PTE_IS_P(PteDst))
2932	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2933
2934	/* commit it (not atomic, new table) */
2935	pPTDst->a[iPTDst] = PteDst;
2936	Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
2937	(RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT)), SHW_PTE_IS_P(PteDst), SHW_PTE_IS_RW(PteDst), SHW_PTE_IS_US(PteDst), SHW_PTE_LOG64(PteDst),
2938	SHW_PTE_IS_TRACK_DIRTY(PteDst) ? " Track-Dirty" : ""));
2939
2940	/* advance */
2941	GCPhys += PAGE_SIZE;
2942	iHCPage++;
2943	iPTDst++;
2944	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2945	&& GCPhys <= pRam->GCPhysLast);
2946	}
2947	else if (pRam)
2948	{
2949	Log(("Invalid pages at %RGp\n", GCPhys));
2950	do
2951	{
2952	SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
2953	GCPhys += PAGE_SIZE;
2954	iPTDst++;
2955	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2956	&& GCPhys < pRam->GCPhys);
2957	}
2958	else
2959	{
2960	Log(("Invalid pages at %RGp (2)\n", GCPhys));
2961	for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
2962	SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
2963	}
2964	} /* while more PTEs */
2965	} /* 4KB / 4MB */
2966	}
2967	else
2968	AssertRelease(!PdeDst.n.u1Present);
2969
2970	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2971	if (RT_FAILURE(rc))
2972	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPTFailed));
2973	return rc;
2974
2975	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2976	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2977	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2978	&& !defined(IN_RC)
2979
2980	/*
2981	* Validate input a little bit.
2982	*/
2983	int rc = VINF_SUCCESS;
2984	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2985	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2986	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
2987
2988	/* Fetch the pgm pool shadow descriptor. */
2989	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2990	Assert(pShwPde);
2991
2992	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2993	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2994	PPGMPOOLPAGE pShwPde = NULL; /* initialized to shut up gcc */
2995	PX86PDPAE pPDDst;
2996	PSHWPDE pPdeDst;
2997
2998	/* Fetch the pgm pool shadow descriptor. */
2999	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
3000	AssertRCSuccessReturn(rc, rc);
3001	Assert(pShwPde);
3002
3003	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
3004	pPdeDst = &pPDDst->a[iPDDst];
3005
3006	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3007	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3008	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3009	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
3010	PX86PDPT pPdptDst= NULL; /* initialized to shut up gcc */
3011	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
3012	AssertRCSuccessReturn(rc, rc);
3013	Assert(pPDDst);
3014	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3015
3016	/* Fetch the pgm pool shadow descriptor. */
3017	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
3018	Assert(pShwPde);
3019
3020	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
3021	const unsigned iPdpt = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
3022	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3023	PEPTPD pPDDst;
3024	PEPTPDPT pPdptDst;
3025
3026	rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, &pPdptDst, &pPDDst);
3027	if (rc != VINF_SUCCESS)
3028	{
3029	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3030	AssertRC(rc);
3031	return rc;
3032	}
3033	Assert(pPDDst);
3034	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3035
3036	/* Fetch the pgm pool shadow descriptor. */
3037	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & EPT_PDPTE_PG_MASK);
3038	Assert(pShwPde);
3039	# endif
3040	SHWPDE PdeDst = *pPdeDst;
3041
3042	Assert(!(PdeDst.u & PGM_PDFLAGS_MAPPING));
3043	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
3044
3045	# if defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
3046	if (BTH_IS_NP_ACTIVE(pVM))
3047	{
3048	PPGMPAGE pPage;
3049
3050	/* Check if we allocated a big page before for this 2 MB range. */
3051	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPtrPage & X86_PDE2M_PAE_PG_MASK, &pPage);
3052	if (RT_SUCCESS(rc))
3053	{
3054	RTHCPHYS HCPhys = NIL_RTHCPHYS;
3055
3056	if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
3057	{
3058	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageReused);
3059	AssertRelease(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3060	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3061	}
3062	else if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED)
3063	{
3064	/* Recheck the entire 2 MB range to see if we can use it again as a large page. */
3065	rc = pgmPhysIsValidLargePage(pVM, GCPtrPage, pPage);
3066	if (RT_SUCCESS(rc))
3067	{
3068	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3069	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3070	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3071	}
3072	}
3073	else if (PGMIsUsingLargePages(pVM))
3074	{
3075	rc = pgmPhysAllocLargePage(pVM, GCPtrPage);
3076	if (RT_SUCCESS(rc))
3077	{
3078	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3079	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3080	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3081	}
3082	else
3083	LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
3084	}
3085
3086	if (HCPhys != NIL_RTHCPHYS)
3087	{
3088	PdeDst.u &= X86_PDE_AVL_MASK;
3089	PdeDst.u \|= HCPhys;
3090	PdeDst.n.u1Present = 1;
3091	PdeDst.n.u1Write = 1;
3092	PdeDst.b.u1Size = 1;
3093	# if PGM_SHW_TYPE == PGM_TYPE_EPT
3094	PdeDst.n.u1Execute = 1;
3095	PdeDst.b.u1IgnorePAT = 1;
3096	PdeDst.b.u3EMT = VMX_EPT_MEMTYPE_WB;
3097	# else
3098	PdeDst.n.u1User = 1;
3099	# endif
3100	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
3101
3102	Log(("SyncPT: Use large page at %RGp PDE=%RX64\n", GCPtrPage, PdeDst.u));
3103	/* Add a reference to the first page only. */
3104	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPde, PGM_PAGE_GET_TRACKING(pPage), pPage, iPDDst);
3105
3106	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3107	return VINF_SUCCESS;
3108	}
3109	}
3110	}
3111	# endif /* HC_ARCH_BITS == 64 */
3112
3113	GSTPDE PdeSrc;
3114	PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3115	PdeSrc.n.u1Present = 1;
3116	PdeSrc.n.u1Write = 1;
3117	PdeSrc.n.u1Accessed = 1;
3118	PdeSrc.n.u1User = 1;
3119
3120	/*
3121	* Allocate & map the page table.
3122	*/
3123	PSHWPT pPTDst;
3124	PPGMPOOLPAGE pShwPage;
3125	RTGCPHYS GCPhys;
3126
3127	/* Virtual address = physical address */
3128	GCPhys = GCPtrPage & X86_PAGE_4K_BASE_MASK;
3129	rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
3130
3131	if ( rc == VINF_SUCCESS
3132	\|\| rc == VINF_PGM_CACHED_PAGE)
3133	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3134	else
3135	AssertMsgFailedReturn(("rc=%Rrc\n", rc), VERR_INTERNAL_ERROR);
3136
3137	PdeDst.u &= X86_PDE_AVL_MASK;
3138	PdeDst.u \|= pShwPage->Core.Key;
3139	PdeDst.n.u1Present = 1;
3140	PdeDst.n.u1Write = 1;
3141	# if PGM_SHW_TYPE == PGM_TYPE_EPT
3142	PdeDst.n.u1Execute = 1;
3143	# else
3144	PdeDst.n.u1User = 1;
3145	PdeDst.n.u1Accessed = 1;
3146	# endif
3147	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
3148
3149	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, PGM_SYNC_NR_PAGES, 0 /* page not present */);
3150	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3151	return rc;
3152
3153	#else
3154	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
3155	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3156	return VERR_INTERNAL_ERROR;
3157	#endif
3158	}
3159
3160
3161
3162	/**
3163	* Prefetch a page/set of pages.
3164	*
3165	* Typically used to sync commonly used pages before entering raw mode
3166	* after a CR3 reload.
3167	*
3168	* @returns VBox status code.
3169	* @param pVCpu The VMCPU handle.
3170	* @param GCPtrPage Page to invalidate.
3171	*/
3172	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
3173	{
3174	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3175	\|\| PGM_GST_TYPE == PGM_TYPE_REAL \
3176	\|\| PGM_GST_TYPE == PGM_TYPE_PROT \
3177	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3178	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3179	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
3180	&& PGM_SHW_TYPE != PGM_TYPE_EPT
3181
3182	/*
3183	* Check that all Guest levels thru the PDE are present, getting the
3184	* PD and PDE in the processes.
3185	*/
3186	int rc = VINF_SUCCESS;
3187	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3188	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3189	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
3190	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3191	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3192	unsigned iPDSrc;
3193	X86PDPE PdpeSrc;
3194	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3195	if (!pPDSrc)
3196	return VINF_SUCCESS; /* not present */
3197	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3198	unsigned iPDSrc;
3199	PX86PML4E pPml4eSrc;
3200	X86PDPE PdpeSrc;
3201	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3202	if (!pPDSrc)
3203	return VINF_SUCCESS; /* not present */
3204	# endif
3205	const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3206	# else
3207	PGSTPD pPDSrc = NULL;
3208	const unsigned iPDSrc = 0;
3209	GSTPDE PdeSrc;
3210
3211	PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3212	PdeSrc.n.u1Present = 1;
3213	PdeSrc.n.u1Write = 1;
3214	PdeSrc.n.u1Accessed = 1;
3215	PdeSrc.n.u1User = 1;
3216	# endif
3217
3218	if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
3219	{
3220	PVM pVM = pVCpu->CTX_SUFF(pVM);
3221	pgmLock(pVM);
3222
3223	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3224	const X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
3225	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3226	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3227	PX86PDPAE pPDDst;
3228	X86PDEPAE PdeDst;
3229	# if PGM_GST_TYPE != PGM_TYPE_PAE
3230	X86PDPE PdpeSrc;
3231
3232	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3233	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3234	# endif
3235	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3236	if (rc != VINF_SUCCESS)
3237	{
3238	pgmUnlock(pVM);
3239	AssertRC(rc);
3240	return rc;
3241	}
3242	Assert(pPDDst);
3243	PdeDst = pPDDst->a[iPDDst];
3244
3245	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3246	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3247	PX86PDPAE pPDDst;
3248	X86PDEPAE PdeDst;
3249
3250	# if PGM_GST_TYPE == PGM_TYPE_PROT
3251	/* AMD-V nested paging */
3252	X86PML4E Pml4eSrc;
3253	X86PDPE PdpeSrc;
3254	PX86PML4E pPml4eSrc = &Pml4eSrc;
3255
3256	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3257	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
3258	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
3259	# endif
3260
3261	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3262	if (rc != VINF_SUCCESS)
3263	{
3264	pgmUnlock(pVM);
3265	AssertRC(rc);
3266	return rc;
3267	}
3268	Assert(pPDDst);
3269	PdeDst = pPDDst->a[iPDDst];
3270	# endif
3271	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
3272	{
3273	if (!PdeDst.n.u1Present)
3274	{
3275	/** @todo r=bird: This guy will set the A bit on the PDE,
3276	* probably harmless. */
3277	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3278	}
3279	else
3280	{
3281	/* Note! We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
3282	* R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
3283	* makes no sense to prefetch more than one page.
3284	*/
3285	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3286	if (RT_SUCCESS(rc))
3287	rc = VINF_SUCCESS;
3288	}
3289	}
3290	pgmUnlock(pVM);
3291	}
3292	return rc;
3293
3294	#elif PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3295	return VINF_SUCCESS; /* ignore */
3296	#else
3297	AssertCompile(0);
3298	#endif
3299	}
3300
3301
3302
3303
3304	/**
3305	* Syncs a page during a PGMVerifyAccess() call.
3306	*
3307	* @returns VBox status code (informational included).
3308	* @param pVCpu The VMCPU handle.
3309	* @param GCPtrPage The address of the page to sync.
3310	* @param fPage The effective guest page flags.
3311	* @param uErr The trap error code.
3312	* @remarks This will normally never be called on invalid guest page
3313	* translation entries.
3314	*/
3315	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fPage, unsigned uErr)
3316	{
3317	PVM pVM = pVCpu->CTX_SUFF(pVM);
3318
3319	LogFlow(("VerifyAccessSyncPage: GCPtrPage=%RGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
3320
3321	Assert(!pVM->pgm.s.fNestedPaging);
3322	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3323	\|\| PGM_GST_TYPE == PGM_TYPE_REAL \
3324	\|\| PGM_GST_TYPE == PGM_TYPE_PROT \
3325	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3326	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3327	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
3328	&& PGM_SHW_TYPE != PGM_TYPE_EPT
3329
3330	# ifndef IN_RING0
3331	if (!(fPage & X86_PTE_US))
3332	{
3333	/*
3334	* Mark this page as safe.
3335	*/
3336	/** @todo not correct for pages that contain both code and data!! */
3337	Log(("CSAMMarkPage %RGv; scanned=%d\n", GCPtrPage, true));
3338	CSAMMarkPage(pVM, GCPtrPage, true);
3339	}
3340	# endif
3341
3342	/*
3343	* Get guest PD and index.
3344	*/
3345	/** @todo Performance: We've done all this a jiffy ago in the
3346	* PGMGstGetPage call. */
3347	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3348	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3349	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
3350	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3351
3352	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3353	unsigned iPDSrc = 0;
3354	X86PDPE PdpeSrc;
3355	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3356	if (RT_UNLIKELY(!pPDSrc))
3357	{
3358	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3359	return VINF_EM_RAW_GUEST_TRAP;
3360	}
3361
3362	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3363	unsigned iPDSrc = 0; /* shut up gcc */
3364	PX86PML4E pPml4eSrc = NULL; /* ditto */
3365	X86PDPE PdpeSrc;
3366	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3367	if (RT_UNLIKELY(!pPDSrc))
3368	{
3369	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3370	return VINF_EM_RAW_GUEST_TRAP;
3371	}
3372	# endif
3373
3374	# else /* !PGM_WITH_PAGING */
3375	PGSTPD pPDSrc = NULL;
3376	const unsigned iPDSrc = 0;
3377	# endif /* !PGM_WITH_PAGING */
3378	int rc = VINF_SUCCESS;
3379
3380	pgmLock(pVM);
3381
3382	/*
3383	* First check if the shadow pd is present.
3384	*/
3385	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3386	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3387
3388	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3389	PX86PDEPAE pPdeDst;
3390	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3391	PX86PDPAE pPDDst;
3392	# if PGM_GST_TYPE != PGM_TYPE_PAE
3393	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3394	X86PDPE PdpeSrc;
3395	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3396	# endif
3397	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3398	if (rc != VINF_SUCCESS)
3399	{
3400	pgmUnlock(pVM);
3401	AssertRC(rc);
3402	return rc;
3403	}
3404	Assert(pPDDst);
3405	pPdeDst = &pPDDst->a[iPDDst];
3406
3407	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3408	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3409	PX86PDPAE pPDDst;
3410	PX86PDEPAE pPdeDst;
3411
3412	# if PGM_GST_TYPE == PGM_TYPE_PROT
3413	/* AMD-V nested paging: Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3414	X86PML4E Pml4eSrc;
3415	X86PDPE PdpeSrc;
3416	PX86PML4E pPml4eSrc = &Pml4eSrc;
3417	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
3418	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
3419	# endif
3420
3421	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3422	if (rc != VINF_SUCCESS)
3423	{
3424	pgmUnlock(pVM);
3425	AssertRC(rc);
3426	return rc;
3427	}
3428	Assert(pPDDst);
3429	pPdeDst = &pPDDst->a[iPDDst];
3430	# endif
3431
3432	if (!pPdeDst->n.u1Present)
3433	{
3434	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3435	if (rc != VINF_SUCCESS)
3436	{
3437	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3438	pgmUnlock(pVM);
3439	AssertRC(rc);
3440	return rc;
3441	}
3442	}
3443
3444	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3445	/* Check for dirty bit fault */
3446	rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
3447	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
3448	Log(("PGMVerifyAccess: success (dirty)\n"));
3449	else
3450	# endif
3451	{
3452	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3453	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3454	# else
3455	GSTPDE PdeSrc;
3456	PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3457	PdeSrc.n.u1Present = 1;
3458	PdeSrc.n.u1Write = 1;
3459	PdeSrc.n.u1Accessed = 1;
3460	PdeSrc.n.u1User = 1;
3461	# endif
3462
3463	Assert(rc != VINF_EM_RAW_GUEST_TRAP);
3464	if (uErr & X86_TRAP_PF_US)
3465	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUser));
3466	else /* supervisor */
3467	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
3468
3469	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3470	if (RT_SUCCESS(rc))
3471	{
3472	/* Page was successfully synced */
3473	Log2(("PGMVerifyAccess: success (sync)\n"));
3474	rc = VINF_SUCCESS;
3475	}
3476	else
3477	{
3478	Log(("PGMVerifyAccess: access violation for %RGv rc=%Rrc\n", GCPtrPage, rc));
3479	rc = VINF_EM_RAW_GUEST_TRAP;
3480	}
3481	}
3482	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3483	pgmUnlock(pVM);
3484	return rc;
3485
3486	#else /* PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED */
3487
3488	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
3489	return VERR_INTERNAL_ERROR;
3490	#endif /* PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED */
3491	}
3492
3493
3494	/**
3495	* Syncs the paging hierarchy starting at CR3.
3496	*
3497	* @returns VBox status code, no specials.
3498	* @param pVCpu The VMCPU handle.
3499	* @param cr0 Guest context CR0 register
3500	* @param cr3 Guest context CR3 register
3501	* @param cr4 Guest context CR4 register
3502	* @param fGlobal Including global page directories or not
3503	*/
3504	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
3505	{
3506	PVM pVM = pVCpu->CTX_SUFF(pVM);
3507
3508	LogFlow(("SyncCR3 fGlobal=%d\n", !!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)));
3509
3510	#if PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
3511
3512	pgmLock(pVM);
3513
3514	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
3515	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3516	if (pPool->cDirtyPages)
3517	pgmPoolResetDirtyPages(pVM);
3518	# endif
3519
3520	/*
3521	* Update page access handlers.
3522	* The virtual are always flushed, while the physical are only on demand.
3523	* WARNING: We are incorrectly not doing global flushing on Virtual Handler updates. We'll
3524	* have to look into that later because it will have a bad influence on the performance.
3525	* @note SvL: There's no need for that. Just invalidate the virtual range(s).
3526	* bird: Yes, but that won't work for aliases.
3527	*/
3528	/** @todo this MUST go away. See #1557. */
3529	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3Handlers), h);
3530	PGM_GST_NAME(HandlerVirtualUpdate)(pVM, cr4);
3531	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3Handlers), h);
3532	pgmUnlock(pVM);
3533	#endif /* !NESTED && !EPT */
3534
3535	#if PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3536	/*
3537	* Nested / EPT - almost no work.
3538	*/
3539	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
3540	return VINF_SUCCESS;
3541
3542	#elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3543	/*
3544	* AMD64 (Shw & Gst) - No need to check all paging levels; we zero
3545	* out the shadow parts when the guest modifies its tables.
3546	*/
3547	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
3548	return VINF_SUCCESS;
3549
3550	#else /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3551
3552	# ifndef PGM_WITHOUT_MAPPINGS
3553	/*
3554	* Check for and resolve conflicts with our guest mappings if they
3555	* are enabled and not fixed.
3556	*/
3557	if (pgmMapAreMappingsFloating(&pVM->pgm.s))
3558	{
3559	int rc = pgmMapResolveConflicts(pVM);
3560	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
3561	if (rc == VINF_PGM_SYNC_CR3)
3562	{
3563	LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
3564	return VINF_PGM_SYNC_CR3;
3565	}
3566	}
3567	# else
3568	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
3569	# endif
3570	return VINF_SUCCESS;
3571	#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3572	}
3573
3574
3575
3576
3577	#ifdef VBOX_STRICT
3578	# ifdef IN_RC
3579	# undef AssertMsgFailed
3580	# define AssertMsgFailed Log
3581	# endif
3582
3583	/**
3584	* Checks that the shadow page table is in sync with the guest one.
3585	*
3586	* @returns The number of errors.
3587	* @param pVM The virtual machine.
3588	* @param pVCpu The VMCPU handle.
3589	* @param cr3 Guest context CR3 register
3590	* @param cr4 Guest context CR4 register
3591	* @param GCPtr Where to start. Defaults to 0.
3592	* @param cb How much to check. Defaults to everything.
3593	*/
3594	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)
3595	{
3596	#if PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3597	return 0;
3598	#else
3599	unsigned cErrors = 0;
3600	PVM pVM = pVCpu->CTX_SUFF(pVM);
3601	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3602
3603	#if PGM_GST_TYPE == PGM_TYPE_PAE
3604	/** @todo currently broken; crashes below somewhere */
3605	AssertFailed();
3606	#endif
3607
3608	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
3609	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3610	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
3611
3612	bool fBigPagesSupported = GST_IS_PSE_ACTIVE(pVCpu);
3613	PPGMCPU pPGM = &pVCpu->pgm.s;
3614	RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
3615	RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
3616	# ifndef IN_RING0
3617	RTHCPHYS HCPhys; /* general usage. */
3618	# endif
3619	int rc;
3620
3621	/*
3622	* Check that the Guest CR3 and all its mappings are correct.
3623	*/
3624	AssertMsgReturn(pPGM->GCPhysCR3 == (cr3 & GST_CR3_PAGE_MASK),
3625	("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
3626	false);
3627	# if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64
3628	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3629	rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGst32BitPdRC, NULL, &HCPhysShw);
3630	# else
3631	rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGstPaePdptRC, NULL, &HCPhysShw);
3632	# endif
3633	AssertRCReturn(rc, 1);
3634	HCPhys = NIL_RTHCPHYS;
3635	rc = pgmRamGCPhys2HCPhys(&pVM->pgm.s, cr3 & GST_CR3_PAGE_MASK, &HCPhys);
3636	AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
3637	# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
3638	pgmGstGet32bitPDPtr(pVCpu);
3639	RTGCPHYS GCPhys;
3640	rc = PGMR3DbgR3Ptr2GCPhys(pVM, pPGM->pGst32BitPdR3, &GCPhys);
3641	AssertRCReturn(rc, 1);
3642	AssertMsgReturn((cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
3643	# endif
3644	# endif /* !IN_RING0 */
3645
3646	/*
3647	* Get and check the Shadow CR3.
3648	*/
3649	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3650	unsigned cPDEs = X86_PG_ENTRIES;
3651	unsigned cIncrement = X86_PG_ENTRIES * PAGE_SIZE;
3652	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3653	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3654	unsigned cPDEs = X86_PG_PAE_ENTRIES * 4; /* treat it as a 2048 entry table. */
3655	# else
3656	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3657	# endif
3658	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3659	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3660	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3661	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3662	# endif
3663	if (cb != ~(RTGCPTR)0)
3664	cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
3665
3666	/** @todo call the other two PGMAssert() functions. /
3667
3668	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3669	unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3670
3671	for (; iPml4 < X86_PG_PAE_ENTRIES; iPml4++)
3672	{
3673	PPGMPOOLPAGE pShwPdpt = NULL;
3674	PX86PML4E pPml4eSrc;
3675	PX86PML4E pPml4eDst;
3676	RTGCPHYS GCPhysPdptSrc;
3677
3678	pPml4eSrc = pgmGstGetLongModePML4EPtr(pVCpu, iPml4);
3679	pPml4eDst = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
3680
3681	/* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
3682	if (!pPml4eDst->n.u1Present)
3683	{
3684	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3685	continue;
3686	}
3687
3688	pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
3689	GCPhysPdptSrc = pPml4eSrc->u & X86_PML4E_PG_MASK;
3690
3691	if (pPml4eSrc->n.u1Present != pPml4eDst->n.u1Present)
3692	{
3693	AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3694	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3695	cErrors++;
3696	continue;
3697	}
3698
3699	if (GCPhysPdptSrc != pShwPdpt->GCPhys)
3700	{
3701	AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
3702	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3703	cErrors++;
3704	continue;
3705	}
3706
3707	if ( pPml4eDst->n.u1User != pPml4eSrc->n.u1User
3708	\|\| pPml4eDst->n.u1Write != pPml4eSrc->n.u1Write
3709	\|\| pPml4eDst->n.u1NoExecute != pPml4eSrc->n.u1NoExecute)
3710	{
3711	AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3712	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3713	cErrors++;
3714	continue;
3715	}
3716	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3717	{
3718	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3719
3720	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
3721	/*
3722	* Check the PDPTEs too.
3723	*/
3724	unsigned iPdpt = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
3725
3726	for (;iPdpt <= SHW_PDPT_MASK; iPdpt++)
3727	{
3728	unsigned iPDSrc = 0; /* initialized to shut up gcc */
3729	PPGMPOOLPAGE pShwPde = NULL;
3730	PX86PDPE pPdpeDst;
3731	RTGCPHYS GCPhysPdeSrc;
3732	# if PGM_GST_TYPE == PGM_TYPE_PAE
3733	X86PDPE PdpeSrc;
3734	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtr, &iPDSrc, &PdpeSrc);
3735	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
3736	# else
3737	PX86PML4E pPml4eSrcIgn;
3738	X86PDPE PdpeSrc;
3739	PX86PDPT pPdptDst;
3740	PX86PDPAE pPDDst;
3741	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtr, &pPml4eSrcIgn, &PdpeSrc, &iPDSrc);
3742
3743	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtr, NULL, &pPdptDst, &pPDDst);
3744	if (rc != VINF_SUCCESS)
3745	{
3746	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
3747	GCPtr += 512 * _2M;
3748	continue; /* next PDPTE */
3749	}
3750	Assert(pPDDst);
3751	# endif
3752	Assert(iPDSrc == 0);
3753
3754	pPdpeDst = &pPdptDst->a[iPdpt];
3755
3756	if (!pPdpeDst->n.u1Present)
3757	{
3758	GCPtr += 512 * _2M;
3759	continue; /* next PDPTE */
3760	}
3761
3762	pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
3763	GCPhysPdeSrc = PdpeSrc.u & X86_PDPE_PG_MASK;
3764
3765	if (pPdpeDst->n.u1Present != PdpeSrc.n.u1Present)
3766	{
3767	AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3768	GCPtr += 512 * _2M;
3769	cErrors++;
3770	continue;
3771	}
3772
3773	if (GCPhysPdeSrc != pShwPde->GCPhys)
3774	{
3775	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3776	AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3777	# else
3778	AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3779	# endif
3780	GCPtr += 512 * _2M;
3781	cErrors++;
3782	continue;
3783	}
3784
3785	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3786	if ( pPdpeDst->lm.u1User != PdpeSrc.lm.u1User
3787	\|\| pPdpeDst->lm.u1Write != PdpeSrc.lm.u1Write
3788	\|\| pPdpeDst->lm.u1NoExecute != PdpeSrc.lm.u1NoExecute)
3789	{
3790	AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3791	GCPtr += 512 * _2M;
3792	cErrors++;
3793	continue;
3794	}
3795	# endif
3796
3797	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3798	{
3799	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3800	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3801	GSTPD const *pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3802	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3803	PCX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
3804	# endif
3805	# endif /* PGM_GST_TYPE == PGM_TYPE_32BIT */
3806	/*
3807	* Iterate the shadow page directory.
3808	*/
3809	GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
3810	unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
3811
3812	for (;
3813	iPDDst < cPDEs;
3814	iPDDst++, GCPtr += cIncrement)
3815	{
3816	# if PGM_SHW_TYPE == PGM_TYPE_PAE
3817	const SHWPDE PdeDst = *pgmShwGetPaePDEPtr(pVCpu, GCPtr);
3818	# else
3819	const SHWPDE PdeDst = pPDDst->a[iPDDst];
3820	# endif
3821	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
3822	{
3823	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
3824	if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
3825	{
3826	AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
3827	cErrors++;
3828	continue;
3829	}
3830	}
3831	else if ( (PdeDst.u & X86_PDE_P)
3832	\|\| ((PdeDst.u & (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY))
3833	)
3834	{
3835	HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
3836	PPGMPOOLPAGE pPoolPage = pgmPoolGetPage(pPool, HCPhysShw);
3837	if (!pPoolPage)
3838	{
3839	AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n",
3840	HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
3841	cErrors++;
3842	continue;
3843	}
3844	const SHWPT pPTDst = (const SHWPT )PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pPoolPage);
3845
3846	if (PdeDst.u & (X86_PDE4M_PWT \| X86_PDE4M_PCD))
3847	{
3848	AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
3849	GCPtr, (uint64_t)PdeDst.u));
3850	cErrors++;
3851	}
3852
3853	if (PdeDst.u & (X86_PDE4M_G \| X86_PDE4M_D))
3854	{
3855	AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n",
3856	GCPtr, (uint64_t)PdeDst.u));
3857	cErrors++;
3858	}
3859
3860	const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
3861	if (!PdeSrc.n.u1Present)
3862	{
3863	AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
3864	GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
3865	cErrors++;
3866	continue;
3867	}
3868
3869	if ( !PdeSrc.b.u1Size
3870	\|\| !fBigPagesSupported)
3871	{
3872	GCPhysGst = GST_GET_PDE_GCPHYS(PdeSrc);
3873	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3874	GCPhysGst \|= (iPDDst & 1) * (PAGE_SIZE / 2);
3875	# endif
3876	}
3877	else
3878	{
3879	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3880	if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
3881	{
3882	AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n",
3883	GCPtr, (uint64_t)PdeSrc.u));
3884	cErrors++;
3885	continue;
3886	}
3887	# endif
3888	GCPhysGst = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
3889	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3890	GCPhysGst \|= GCPtr & RT_BIT(X86_PAGE_2M_SHIFT);
3891	# endif
3892	}
3893
3894	if ( pPoolPage->enmKind
3895	!= (!PdeSrc.b.u1Size \|\| !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
3896	{
3897	AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n",
3898	pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
3899	cErrors++;
3900	}
3901
3902	PPGMPAGE pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
3903	if (!pPhysPage)
3904	{
3905	AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3906	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3907	cErrors++;
3908	continue;
3909	}
3910
3911	if (GCPhysGst != pPoolPage->GCPhys)
3912	{
3913	AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n",
3914	GCPhysGst, pPoolPage->GCPhys, GCPtr));
3915	cErrors++;
3916	continue;
3917	}
3918
3919	if ( !PdeSrc.b.u1Size
3920	\|\| !fBigPagesSupported)
3921	{
3922	/*
3923	* Page Table.
3924	*/
3925	const GSTPT *pPTSrc;
3926	rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1), &pPTSrc);
3927	if (RT_FAILURE(rc))
3928	{
3929	AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3930	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3931	cErrors++;
3932	continue;
3933	}
3934	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/))
3935	!= (PdeDst.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/)))
3936	{
3937	/// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
3938	// (This problem will go away when/if we shadow multiple CR3s.)
3939	AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3940	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3941	cErrors++;
3942	continue;
3943	}
3944	if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3945	{
3946	AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n",
3947	GCPtr, (uint64_t)PdeDst.u));
3948	cErrors++;
3949	continue;
3950	}
3951
3952	/* iterate the page table. */
3953	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3954	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3955	const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
3956	# else
3957	const unsigned offPTSrc = 0;
3958	# endif
3959	for (unsigned iPT = 0, off = 0;
3960	iPT < RT_ELEMENTS(pPTDst->a);
3961	iPT++, off += PAGE_SIZE)
3962	{
3963	const SHWPTE PteDst = pPTDst->a[iPT];
3964
3965	/* skip not-present and dirty tracked entries. */
3966	if (!(SHW_PTE_GET_U(PteDst) & (X86_PTE_P \| PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
3967	continue;
3968	Assert(SHW_PTE_IS_P(PteDst));
3969
3970	const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
3971	if (!PteSrc.n.u1Present)
3972	{
3973	# ifdef IN_RING3
3974	PGMAssertHandlerAndFlagsInSync(pVM);
3975	DBGFR3PagingDumpEx(pVM, pVCpu->idCpu, DBGFPGDMP_FLAGS_CURRENT_CR3 \| DBGFPGDMP_FLAGS_CURRENT_MODE
3976	\| DBGFPGDMP_FLAGS_GUEST \| DBGFPGDMP_FLAGS_HEADER \| DBGFPGDMP_FLAGS_PRINT_CR3,
3977	0, 0, UINT64_MAX, 99, NULL);
3978	# endif
3979	AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
3980	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst), pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
3981	(uint64_t)GST_GET_PDE_GCPHYS(PdeSrc) + (iPT + offPTSrc)*sizeof(PteSrc)));
3982	cErrors++;
3983	continue;
3984	}
3985
3986	uint64_t fIgnoreFlags = GST_PTE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_G \| X86_PTE_D \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
3987	# if 1 /** @todo sync accessed bit properly... */
3988	fIgnoreFlags \|= X86_PTE_A;
3989	# endif
3990
3991	/* match the physical addresses */
3992	HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
3993	GCPhysGst = GST_GET_PTE_GCPHYS(PteSrc);
3994
3995	# ifdef IN_RING3
3996	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3997	if (RT_FAILURE(rc))
3998	{
3999	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4000	{
4001	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
4002	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4003	cErrors++;
4004	continue;
4005	}
4006	}
4007	else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
4008	{
4009	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4010	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4011	cErrors++;
4012	continue;
4013	}
4014	# endif
4015
4016	pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
4017	if (!pPhysPage)
4018	{
4019	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4020	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4021	{
4022	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
4023	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4024	cErrors++;
4025	continue;
4026	}
4027	# endif
4028	if (SHW_PTE_IS_RW(PteDst))
4029	{
4030	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4031	GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4032	cErrors++;
4033	}
4034	fIgnoreFlags \|= X86_PTE_RW;
4035	}
4036	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4037	{
4038	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4039	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4040	cErrors++;
4041	continue;
4042	}
4043
4044	/* flags */
4045	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4046	{
4047	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4048	{
4049	if (SHW_PTE_IS_RW(PteDst))
4050	{
4051	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4052	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4053	cErrors++;
4054	continue;
4055	}
4056	fIgnoreFlags \|= X86_PTE_RW;
4057	}
4058	else
4059	{
4060	if ( SHW_PTE_IS_P(PteDst)
4061	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
4062	&& !PGM_PAGE_IS_MMIO(pPhysPage)
4063	# endif
4064	)
4065	{
4066	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4067	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4068	cErrors++;
4069	continue;
4070	}
4071	fIgnoreFlags \|= X86_PTE_P;
4072	}
4073	}
4074	else
4075	{
4076	if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
4077	{
4078	if (SHW_PTE_IS_RW(PteDst))
4079	{
4080	AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
4081	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4082	cErrors++;
4083	continue;
4084	}
4085	if (!SHW_PTE_IS_TRACK_DIRTY(PteDst))
4086	{
4087	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4088	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4089	cErrors++;
4090	continue;
4091	}
4092	if (SHW_PTE_IS_D(PteDst))
4093	{
4094	AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4095	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4096	cErrors++;
4097	}
4098	# if 0 /** @todo sync access bit properly... */
4099	if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
4100	{
4101	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4102	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4103	cErrors++;
4104	}
4105	fIgnoreFlags \|= X86_PTE_RW;
4106	# else
4107	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
4108	# endif
4109	}
4110	else if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
4111	{
4112	/* access bit emulation (not implemented). */
4113	if (PteSrc.n.u1Accessed \|\| SHW_PTE_IS_P(PteDst))
4114	{
4115	AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
4116	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4117	cErrors++;
4118	continue;
4119	}
4120	if (!SHW_PTE_IS_A(PteDst))
4121	{
4122	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
4123	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4124	cErrors++;
4125	}
4126	fIgnoreFlags \|= X86_PTE_P;
4127	}
4128	# ifdef DEBUG_sandervl
4129	fIgnoreFlags \|= X86_PTE_D \| X86_PTE_A;
4130	# endif
4131	}
4132
4133	if ( (PteSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4134	&& (PteSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4135	)
4136	{
4137	AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
4138	GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
4139	fIgnoreFlags, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4140	cErrors++;
4141	continue;
4142	}
4143	} /* foreach PTE */
4144	}
4145	else
4146	{
4147	/*
4148	* Big Page.
4149	*/
4150	uint64_t fIgnoreFlags = X86_PDE_AVL_MASK \| GST_PDE_PG_MASK \| X86_PDE4M_G \| X86_PDE4M_D \| X86_PDE4M_PS \| X86_PDE4M_PWT \| X86_PDE4M_PCD;
4151	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
4152	{
4153	if (PdeDst.n.u1Write)
4154	{
4155	AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4156	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4157	cErrors++;
4158	continue;
4159	}
4160	if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
4161	{
4162	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4163	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4164	cErrors++;
4165	continue;
4166	}
4167	# if 0 /** @todo sync access bit properly... */
4168	if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
4169	{
4170	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4171	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4172	cErrors++;
4173	}
4174	fIgnoreFlags \|= X86_PTE_RW;
4175	# else
4176	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
4177	# endif
4178	}
4179	else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4180	{
4181	/* access bit emulation (not implemented). */
4182	if (PdeSrc.b.u1Accessed \|\| PdeDst.n.u1Present)
4183	{
4184	AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4185	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4186	cErrors++;
4187	continue;
4188	}
4189	if (!PdeDst.n.u1Accessed)
4190	{
4191	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4192	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4193	cErrors++;
4194	}
4195	fIgnoreFlags \|= X86_PTE_P;
4196	}
4197
4198	if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
4199	{
4200	AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
4201	GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
4202	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4203	cErrors++;
4204	}
4205
4206	/* iterate the page table. */
4207	for (unsigned iPT = 0, off = 0;
4208	iPT < RT_ELEMENTS(pPTDst->a);
4209	iPT++, off += PAGE_SIZE, GCPhysGst += PAGE_SIZE)
4210	{
4211	const SHWPTE PteDst = pPTDst->a[iPT];
4212
4213	if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
4214	{
4215	AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
4216	GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4217	cErrors++;
4218	}
4219
4220	/* skip not-present entries. */
4221	if (!SHW_PTE_IS_P(PteDst)) /** @todo deal with ALL handlers and CSAM !P pages! */
4222	continue;
4223
4224	fIgnoreFlags = X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT \| X86_PTE_D \| X86_PTE_A \| X86_PTE_G \| X86_PTE_PAE_NX;
4225
4226	/* match the physical addresses */
4227	HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
4228
4229	# ifdef IN_RING3
4230	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4231	if (RT_FAILURE(rc))
4232	{
4233	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4234	{
4235	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4236	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4237	cErrors++;
4238	}
4239	}
4240	else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
4241	{
4242	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4243	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4244	cErrors++;
4245	continue;
4246	}
4247	# endif
4248	pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
4249	if (!pPhysPage)
4250	{
4251	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4252	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4253	{
4254	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4255	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4256	cErrors++;
4257	continue;
4258	}
4259	# endif
4260	if (SHW_PTE_IS_RW(PteDst))
4261	{
4262	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4263	GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4264	cErrors++;
4265	}
4266	fIgnoreFlags \|= X86_PTE_RW;
4267	}
4268	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4269	{
4270	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4271	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4272	cErrors++;
4273	continue;
4274	}
4275
4276	/* flags */
4277	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4278	{
4279	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4280	{
4281	if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
4282	{
4283	if (SHW_PTE_IS_RW(PteDst))
4284	{
4285	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4286	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4287	cErrors++;
4288	continue;
4289	}
4290	fIgnoreFlags \|= X86_PTE_RW;
4291	}
4292	}
4293	else
4294	{
4295	if ( SHW_PTE_IS_P(PteDst)
4296	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
4297	&& !PGM_PAGE_IS_MMIO(pPhysPage)
4298	# endif
4299	)
4300	{
4301	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4302	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4303	cErrors++;
4304	continue;
4305	}
4306	fIgnoreFlags \|= X86_PTE_P;
4307	}
4308	}
4309
4310	if ( (PdeSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4311	&& (PdeSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags) /* lazy phys handler dereg. */
4312	)
4313	{
4314	AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
4315	GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
4316	fIgnoreFlags, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4317	cErrors++;
4318	continue;
4319	}
4320	} /* for each PTE */
4321	}
4322	}
4323	/* not present */
4324
4325	} /* for each PDE */
4326
4327	} /* for each PDPTE */
4328
4329	} /* for each PML4E */
4330
4331	# ifdef DEBUG
4332	if (cErrors)
4333	LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
4334	# endif
4335
4336	#endif /* GST == 32BIT, PAE or AMD64 */
4337	return cErrors;
4338
4339	#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT */
4340	}
4341	#endif /* VBOX_STRICT */
4342
4343
4344	/**
4345	* Sets up the CR3 for shadow paging
4346	*
4347	* @returns Strict VBox status code.
4348	* @retval VINF_SUCCESS.
4349	*
4350	* @param pVCpu The VMCPU handle.
4351	* @param GCPhysCR3 The physical address in the CR3 register.
4352	*/
4353	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3)
4354	{
4355	PVM pVM = pVCpu->CTX_SUFF(pVM);
4356
4357	/* Update guest paging info. */
4358	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
4359	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
4360	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
4361
4362	LogFlow(("MapCR3: %RGp\n", GCPhysCR3));
4363
4364	/*
4365	* Map the page CR3 points at.
4366	*/
4367	RTHCPTR HCPtrGuestCR3;
4368	RTHCPHYS HCPhysGuestCR3;
4369	pgmLock(pVM);
4370	PPGMPAGE pPageCR3 = pgmPhysGetPage(&pVM->pgm.s, GCPhysCR3);
4371	AssertReturn(pPageCR3, VERR_INTERNAL_ERROR_2);
4372	HCPhysGuestCR3 = PGM_PAGE_GET_HCPHYS(pPageCR3);
4373	/** @todo this needs some reworking wrt. locking? */
4374	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4375	HCPtrGuestCR3 = NIL_RTHCPTR;
4376	int rc = VINF_SUCCESS;
4377	# else
4378	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPageCR3, GCPhysCR3 & GST_CR3_PAGE_MASK, (void )&HCPtrGuestCR3); / @todo r=bird: This GCPhysCR3 masking isn't necessary. */
4379	# endif
4380	pgmUnlock(pVM);
4381	if (RT_SUCCESS(rc))
4382	{
4383	rc = PGMMap(pVM, (RTGCPTR)pVM->pgm.s.GCPtrCR3Mapping, HCPhysGuestCR3, PAGE_SIZE, 0);
4384	if (RT_SUCCESS(rc))
4385	{
4386	# ifdef IN_RC
4387	PGM_INVL_PG(pVCpu, pVM->pgm.s.GCPtrCR3Mapping);
4388	# endif
4389	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4390	pVCpu->pgm.s.pGst32BitPdR3 = (R3PTRTYPE(PX86PD))HCPtrGuestCR3;
4391	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4392	pVCpu->pgm.s.pGst32BitPdR0 = (R0PTRTYPE(PX86PD))HCPtrGuestCR3;
4393	# endif
4394	pVCpu->pgm.s.pGst32BitPdRC = (RCPTRTYPE(PX86PD))(RTRCUINTPTR)pVM->pgm.s.GCPtrCR3Mapping;
4395
4396	# elif PGM_GST_TYPE == PGM_TYPE_PAE
4397	unsigned off = GCPhysCR3 & GST_CR3_PAGE_MASK & PAGE_OFFSET_MASK;
4398	pVCpu->pgm.s.pGstPaePdptR3 = (R3PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4399	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4400	pVCpu->pgm.s.pGstPaePdptR0 = (R0PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4401	# endif
4402	pVCpu->pgm.s.pGstPaePdptRC = (RCPTRTYPE(PX86PDPT))((RTRCUINTPTR)pVM->pgm.s.GCPtrCR3Mapping + off);
4403	LogFlow(("Cached mapping %RRv\n", pVCpu->pgm.s.pGstPaePdptRC));
4404
4405	/*
4406	* Map the 4 PDs too.
4407	*/
4408	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pVCpu);
4409	RTGCPTR GCPtr = pVM->pgm.s.GCPtrCR3Mapping + PAGE_SIZE;
4410	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++, GCPtr += PAGE_SIZE)
4411	{
4412	if (pGuestPDPT->a[i].n.u1Present)
4413	{
4414	RTHCPTR HCPtr;
4415	RTHCPHYS HCPhys;
4416	RTGCPHYS GCPhys = pGuestPDPT->a[i].u & X86_PDPE_PG_MASK;
4417	pgmLock(pVM);
4418	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
4419	AssertReturn(pPage, VERR_INTERNAL_ERROR_2);
4420	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
4421	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4422	HCPtr = NIL_RTHCPTR;
4423	int rc2 = VINF_SUCCESS;
4424	# else
4425	int rc2 = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, (void **)&HCPtr);
4426	# endif
4427	pgmUnlock(pVM);
4428	if (RT_SUCCESS(rc2))
4429	{
4430	rc = PGMMap(pVM, GCPtr, HCPhys, PAGE_SIZE, 0);
4431	AssertRCReturn(rc, rc);
4432
4433	pVCpu->pgm.s.apGstPaePDsR3[i] = (R3PTRTYPE(PX86PDPAE))HCPtr;
4434	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4435	pVCpu->pgm.s.apGstPaePDsR0[i] = (R0PTRTYPE(PX86PDPAE))HCPtr;
4436	# endif
4437	pVCpu->pgm.s.apGstPaePDsRC[i] = (RCPTRTYPE(PX86PDPAE))(RTRCUINTPTR)GCPtr;
4438	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
4439	# ifdef IN_RC
4440	PGM_INVL_PG(pVCpu, GCPtr);
4441	# endif
4442	continue;
4443	}
4444	AssertMsgFailed(("pgmR3Gst32BitMapCR3: rc2=%d GCPhys=%RGp i=%d\n", rc2, GCPhys, i));
4445	}
4446
4447	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4448	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4449	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4450	# endif
4451	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4452	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4453	# ifdef IN_RC
4454	PGM_INVL_PG(pVCpu, GCPtr); /** @todo this shouldn't be necessary? */
4455	# endif
4456	}
4457
4458	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
4459	pVCpu->pgm.s.pGstAmd64Pml4R3 = (R3PTRTYPE(PX86PML4))HCPtrGuestCR3;
4460	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4461	pVCpu->pgm.s.pGstAmd64Pml4R0 = (R0PTRTYPE(PX86PML4))HCPtrGuestCR3;
4462	# endif
4463	# endif
4464	}
4465	else
4466	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4467	}
4468	else
4469	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4470
4471	#else /* prot/real stub */
4472	int rc = VINF_SUCCESS;
4473	#endif
4474
4475	/* Update shadow paging info for guest modes with paging (32, pae, 64). */
4476	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4477	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4478	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64) \
4479	&& ( PGM_GST_TYPE != PGM_TYPE_REAL \
4480	&& PGM_GST_TYPE != PGM_TYPE_PROT))
4481
4482	Assert(!pVM->pgm.s.fNestedPaging);
4483
4484	/*
4485	* Update the shadow root page as well since that's not fixed.
4486	*/
4487	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4488	PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
4489	uint32_t iOldShwUserTable = pVCpu->pgm.s.iShwUserTable;
4490	uint32_t iOldShwUser = pVCpu->pgm.s.iShwUser;
4491	PPGMPOOLPAGE pNewShwPageCR3;
4492
4493	pgmLock(pVM);
4494
4495	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4496	if (pPool->cDirtyPages)
4497	pgmPoolResetDirtyPages(pVM);
4498	# endif
4499
4500	Assert(!(GCPhysCR3 >> (PAGE_SHIFT + 32)));
4501	rc = pgmPoolAllocEx(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE, SHW_POOL_ROOT_IDX,
4502	GCPhysCR3 >> PAGE_SHIFT, true /fLockPage/, &pNewShwPageCR3);
4503	AssertFatalRC(rc);
4504	rc = VINF_SUCCESS;
4505
4506	# ifdef IN_RC
4507	/*
4508	* WARNING! We can't deal with jumps to ring 3 in the code below as the
4509	* state will be inconsistent! Flush important things now while
4510	* we still can and then make sure there are no ring-3 calls.
4511	*/
4512	REMNotifyHandlerPhysicalFlushIfAlmostFull(pVM, pVCpu);
4513	VMMRZCallRing3Disable(pVCpu);
4514	# endif
4515
4516	pVCpu->pgm.s.iShwUser = SHW_POOL_ROOT_IDX;
4517	pVCpu->pgm.s.iShwUserTable = GCPhysCR3 >> PAGE_SHIFT;
4518	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3) = pNewShwPageCR3;
4519	# ifdef IN_RING0
4520	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4521	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4522	# elif defined(IN_RC)
4523	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4524	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4525	# else
4526	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4527	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4528	# endif
4529
4530	# ifndef PGM_WITHOUT_MAPPINGS
4531	/*
4532	* Apply all hypervisor mappings to the new CR3.
4533	* Note that SyncCR3 will be executed in case CR3 is changed in a guest paging mode; this will
4534	* make sure we check for conflicts in the new CR3 root.
4535	*/
4536	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4537	Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL) \|\| VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
4538	# endif
4539	rc = pgmMapActivateCR3(pVM, pNewShwPageCR3);
4540	AssertRCReturn(rc, rc);
4541	# endif
4542
4543	/* Set the current hypervisor CR3. */
4544	CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
4545	SELMShadowCR3Changed(pVM, pVCpu);
4546
4547	# ifdef IN_RC
4548	/* NOTE: The state is consistent again. */
4549	VMMRZCallRing3Enable(pVCpu);
4550	# endif
4551
4552	/* Clean up the old CR3 root. */
4553	if ( pOldShwPageCR3
4554	&& pOldShwPageCR3 != pNewShwPageCR3 /* @todo can happen due to incorrect syncing between REM & PGM; find the real cause */)
4555	{
4556	Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
4557	# ifndef PGM_WITHOUT_MAPPINGS
4558	/* Remove the hypervisor mappings from the shadow page table. */
4559	pgmMapDeactivateCR3(pVM, pOldShwPageCR3);
4560	# endif
4561	/* Mark the page as unlocked; allow flushing again. */
4562	pgmPoolUnlockPage(pPool, pOldShwPageCR3);
4563
4564	pgmPoolFreeByPage(pPool, pOldShwPageCR3, iOldShwUser, iOldShwUserTable);
4565	}
4566	pgmUnlock(pVM);
4567	# endif
4568
4569	return rc;
4570	}
4571
4572	/**
4573	* Unmaps the shadow CR3.
4574	*
4575	* @returns VBox status, no specials.
4576	* @param pVCpu The VMCPU handle.
4577	*/
4578	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu)
4579	{
4580	LogFlow(("UnmapCR3\n"));
4581
4582	int rc = VINF_SUCCESS;
4583	PVM pVM = pVCpu->CTX_SUFF(pVM);
4584
4585	/*
4586	* Update guest paging info.
4587	*/
4588	#if PGM_GST_TYPE == PGM_TYPE_32BIT
4589	pVCpu->pgm.s.pGst32BitPdR3 = 0;
4590	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4591	pVCpu->pgm.s.pGst32BitPdR0 = 0;
4592	# endif
4593	pVCpu->pgm.s.pGst32BitPdRC = 0;
4594
4595	#elif PGM_GST_TYPE == PGM_TYPE_PAE
4596	pVCpu->pgm.s.pGstPaePdptR3 = 0;
4597	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4598	pVCpu->pgm.s.pGstPaePdptR0 = 0;
4599	# endif
4600	pVCpu->pgm.s.pGstPaePdptRC = 0;
4601	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4602	{
4603	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4604	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4605	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4606	# endif
4607	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4608	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4609	}
4610
4611	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
4612	pVCpu->pgm.s.pGstAmd64Pml4R3 = 0;
4613	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4614	pVCpu->pgm.s.pGstAmd64Pml4R0 = 0;
4615	# endif
4616
4617	#else /* prot/real mode stub */
4618	/* nothing to do */
4619	#endif
4620
4621	#if !defined(IN_RC) /* In RC we rely on MapCR3 to do the shadow part for us at a safe time */
4622	/*
4623	* Update shadow paging info.
4624	*/
4625	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4626	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4627	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64))
4628
4629	# if PGM_GST_TYPE != PGM_TYPE_REAL
4630	Assert(!pVM->pgm.s.fNestedPaging);
4631	# endif
4632
4633	pgmLock(pVM);
4634
4635	# ifndef PGM_WITHOUT_MAPPINGS
4636	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4637	/* Remove the hypervisor mappings from the shadow page table. */
4638	pgmMapDeactivateCR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4639	# endif
4640
4641	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4642	{
4643	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4644
4645	Assert(pVCpu->pgm.s.iShwUser != PGMPOOL_IDX_NESTED_ROOT);
4646
4647	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4648	if (pPool->cDirtyPages)
4649	pgmPoolResetDirtyPages(pVM);
4650	# endif
4651
4652	/* Mark the page as unlocked; allow flushing again. */
4653	pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4654
4655	pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), pVCpu->pgm.s.iShwUser, pVCpu->pgm.s.iShwUserTable);
4656	pVCpu->pgm.s.pShwPageCR3R3 = 0;
4657	pVCpu->pgm.s.pShwPageCR3R0 = 0;
4658	pVCpu->pgm.s.pShwPageCR3RC = 0;
4659	pVCpu->pgm.s.iShwUser = 0;
4660	pVCpu->pgm.s.iShwUserTable = 0;
4661	}
4662	pgmUnlock(pVM);
4663	# endif
4664	#endif /* !IN_RC*/
4665
4666	return rc;
4667	}

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 32362

Download in other formats: