PGMAllBth.h@ 66650

Last change on this file since 66650 was 66649, checked in by vboxsync, 8 years ago
Backed the PGMAllBoth.h and IEMR3.cpp parts of r111975 since these are unrelated changes and should've been comitted separately.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 212.4 KB

Line
1	/* $Id: PGMAllBth.h 66649 2017-04-22 09:27:06Z vboxsync $ */
2	/** @file
3	* VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4	*
5	* @remarks The nested page tables on AMD makes use of PGM_SHW_TYPE in
6	* {PGM_TYPE_AMD64, PGM_TYPE_PAE and PGM_TYPE_32BIT} and PGM_GST_TYPE
7	* set to PGM_TYPE_PROT. Half of the code in this file is not
8	* exercised with PGM_SHW_TYPE set to PGM_TYPE_NESTED.
9	*
10	* @remarks Extended page tables (intel) are built with PGM_GST_TYPE set to
11	* PGM_TYPE_PROT (and PGM_SHW_TYPE set to PGM_TYPE_EPT).
12	*
13	* @remarks This file is one big \#ifdef-orgy!
14	*
15	*/
16
17	/*
18	* Copyright (C) 2006-2016 Oracle Corporation
19	*
20	* This file is part of VirtualBox Open Source Edition (OSE), as
21	* available from http://www.virtualbox.org. This file is free software;
22	* you can redistribute it and/or modify it under the terms of the GNU
23	* General Public License (GPL) as published by the Free Software
24	* Foundation, in version 2 as it comes in the "COPYING" file of the
25	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
26	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
27	*/
28
29	#ifdef _MSC_VER
30	/** @todo we're generating unnecessary code in nested/ept shadow mode and for
31	* real/prot-guest+RC mode. */
32	# pragma warning(disable: 4505)
33	#endif
34
35	/*******************************************************************************
36	* Internal Functions *
37	*******************************************************************************/
38	RT_C_DECLS_BEGIN
39	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken);
40	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
41	static int PGM_BTH_NAME(SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr);
42	static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, RTGCPTR GCPtrPage);
43	static int PGM_BTH_NAME(SyncPT)(PVMCPU pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage);
44	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
45	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
46	# else
47	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
48	#endif
49	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uErr);
50	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
51	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
52	#ifdef VBOX_STRICT
53	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
54	#endif
55	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3);
56	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu);
57	RT_C_DECLS_END
58
59
60	/*
61	* Filter out some illegal combinations of guest and shadow paging, so we can
62	* remove redundant checks inside functions.
63	*/
64	#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
65	# error "Invalid combination; PAE guest implies PAE shadow"
66	#endif
67
68	#if (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
69	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT)
70	# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
71	#endif
72
73	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) \
74	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT)
75	# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
76	#endif
77
78	#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT) \
79	\|\| (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
80	# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
81	#endif
82
83	#ifndef IN_RING3
84
85	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
86	/**
87	* Deal with a guest page fault.
88	*
89	* @returns Strict VBox status code.
90	* @retval VINF_EM_RAW_GUEST_TRAP
91	* @retval VINF_EM_RAW_EMULATE_INSTR
92	*
93	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
94	* @param pGstWalk The guest page table walk result.
95	* @param uErr The error code.
96	*/
97	PGM_BTH_DECL(VBOXSTRICTRC, Trap0eHandlerGuestFault)(PVMCPU pVCpu, PGSTPTWALK pGstWalk, RTGCUINT uErr)
98	{
99	# if !defined(PGM_WITHOUT_MAPPINGS) && (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE)
100	/*
101	* Check for write conflicts with our hypervisor mapping.
102	*
103	* If the guest happens to access a non-present page, where our hypervisor
104	* is currently mapped, then we'll create a #PF storm in the guest.
105	*/
106	if ( (uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RW)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RW)
107	&& pgmMapAreMappingsEnabled(pVCpu->CTX_SUFF(pVM))
108	&& MMHyperIsInsideArea(pVCpu->CTX_SUFF(pVM), pGstWalk->Core.GCPtr))
109	{
110	/* Force a CR3 sync to check for conflicts and emulate the instruction. */
111	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
112	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
113	return VINF_EM_RAW_EMULATE_INSTR;
114	}
115	# endif
116
117	/*
118	* Calc the error code for the guest trap.
119	*/
120	uint32_t uNewErr = GST_IS_NX_ACTIVE(pVCpu)
121	? uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID)
122	: uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US);
123	if ( pGstWalk->Core.fRsvdError
124	\|\| pGstWalk->Core.fBadPhysAddr)
125	{
126	uNewErr \|= X86_TRAP_PF_RSVD \| X86_TRAP_PF_P;
127	Assert(!pGstWalk->Core.fNotPresent);
128	}
129	else if (!pGstWalk->Core.fNotPresent)
130	uNewErr \|= X86_TRAP_PF_P;
131	TRPMSetErrorCode(pVCpu, uNewErr);
132
133	LogFlow(("Guest trap; cr2=%RGv uErr=%RGv lvl=%d\n", pGstWalk->Core.GCPtr, uErr, pGstWalk->Core.uLevel));
134	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
135	return VINF_EM_RAW_GUEST_TRAP;
136	}
137	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
138
139
140	/**
141	* Deal with a guest page fault.
142	*
143	* The caller has taken the PGM lock.
144	*
145	* @returns Strict VBox status code.
146	*
147	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
148	* @param uErr The error code.
149	* @param pRegFrame The register frame.
150	* @param pvFault The fault address.
151	* @param pPage The guest page at @a pvFault.
152	* @param pGstWalk The guest page table walk result.
153	* @param pfLockTaken PGM lock taken here or not (out). This is true
154	* when we're called.
155	*/
156	static VBOXSTRICTRC PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame,
157	RTGCPTR pvFault, PPGMPAGE pPage, bool *pfLockTaken
158	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \|\| defined(DOXYGEN_RUNNING)
159	, PGSTPTWALK pGstWalk
160	# endif
161	)
162	{
163	# if !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
164	GSTPDE const PdeSrcDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
165	#endif
166	PVM pVM = pVCpu->CTX_SUFF(pVM);
167	VBOXSTRICTRC rcStrict;
168
169	if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
170	{
171	/*
172	* Physical page access handler.
173	*/
174	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
175	const RTGCPHYS GCPhysFault = pGstWalk->Core.GCPhys;
176	# else
177	const RTGCPHYS GCPhysFault = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault);
178	# endif
179	PPGMPHYSHANDLER pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
180	if (pCur)
181	{
182	PPGMPHYSHANDLERTYPEINT pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
183
184	# ifdef PGM_SYNC_N_PAGES
185	/*
186	* If the region is write protected and we got a page not present fault, then sync
187	* the pages. If the fault was caused by a read, then restart the instruction.
188	* In case of write access continue to the GC write handler.
189	*
190	* ASSUMES that there is only one handler per page or that they have similar write properties.
191	*/
192	if ( !(uErr & X86_TRAP_PF_P)
193	&& pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
194	{
195	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
196	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
197	# else
198	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
199	# endif
200	if ( RT_FAILURE(rcStrict)
201	\|\| !(uErr & X86_TRAP_PF_RW)
202	\|\| rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
203	{
204	AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
205	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
206	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
207	return rcStrict;
208	}
209	}
210	# endif
211	# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
212	/*
213	* If the access was not thru a #PF(RSVD\|...) resync the page.
214	*/
215	if ( !(uErr & X86_TRAP_PF_RSVD)
216	&& pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
217	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
218	&& pGstWalk->Core.fEffectiveRW
219	&& !pGstWalk->Core.fEffectiveUS /** @todo Remove pGstWalk->Core.fEffectiveUS and X86_PTE_US further down in the sync code. */
220	# endif
221	)
222	{
223	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
224	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
225	# else
226	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
227	# endif
228	if ( RT_FAILURE(rcStrict)
229	\|\| rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
230	{
231	AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
232	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
233	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
234	return rcStrict;
235	}
236	}
237	# endif
238
239	AssertMsg( pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
240	\|\| (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW)),
241	("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enmKind=%d\n",
242	pvFault, GCPhysFault, pPage, uErr, pCurType->enmKind));
243	if (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
244	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysWrite);
245	else
246	{
247	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysAll);
248	if (uErr & X86_TRAP_PF_RSVD) STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysAllOpt);
249	}
250
251	if (pCurType->CTX_SUFF(pfnPfHandler))
252	{
253	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
254	void *pvUser = pCur->CTX_SUFF(pvUser);
255
256	STAM_PROFILE_START(&pCur->Stat, h);
257	if (pCur->hType != pPool->hAccessHandlerType)
258	{
259	pgmUnlock(pVM);
260	*pfLockTaken = false;
261	}
262
263	rcStrict = pCurType->CTX_SUFF(pfnPfHandler)(pVM, pVCpu, uErr, pRegFrame, pvFault, GCPhysFault, pvUser);
264
265	# ifdef VBOX_WITH_STATISTICS
266	pgmLock(pVM);
267	pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
268	if (pCur)
269	STAM_PROFILE_STOP(&pCur->Stat, h);
270	pgmUnlock(pVM);
271	# endif
272	}
273	else
274	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
275
276	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndPhys; });
277	return rcStrict;
278	}
279	}
280	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
281	else
282	{
283	# ifdef PGM_SYNC_N_PAGES
284	/*
285	* If the region is write protected and we got a page not present fault, then sync
286	* the pages. If the fault was caused by a read, then restart the instruction.
287	* In case of write access continue to the GC write handler.
288	*/
289	if ( PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) < PGM_PAGE_HNDL_PHYS_STATE_ALL
290	&& !(uErr & X86_TRAP_PF_P))
291	{
292	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
293	if ( RT_FAILURE(rcStrict)
294	\|\| rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE
295	\|\| !(uErr & X86_TRAP_PF_RW))
296	{
297	AssertRC(rcStrict);
298	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
299	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndVirt; });
300	return rcStrict;
301	}
302	}
303	# endif
304	/*
305	* Ok, it's an virtual page access handler.
306	*
307	* Since it's faster to search by address, we'll do that first
308	* and then retry by GCPhys if that fails.
309	*/
310	/** @todo r=bird: perhaps we should consider looking up by physical address directly now?
311	* r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be
312	* out of sync, because the page was changed without us noticing it (not-present -> present
313	* without invlpg or mov cr3, xxx).
314	*/
315	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
316	if (pCur)
317	{
318	PPGMVIRTHANDLERTYPEINT pCurType = PGMVIRTANDLER_GET_TYPE(pVM, pCur);
319	AssertMsg(!(pvFault - pCur->Core.Key < pCur->cb)
320	\|\| ( pCurType->enmKind != PGMVIRTHANDLERKIND_WRITE
321	\|\| !(uErr & X86_TRAP_PF_P)
322	\|\| (pCurType->enmKind == PGMVIRTHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW))),
323	("Unexpected trap for virtual handler: %RGv (phys=%RGp) pPage=%R[pgmpage] uErr=%X, enumKind=%d\n",
324	pvFault, pGstWalk->Core.GCPhys, pPage, uErr, pCurType->enmKind));
325
326	if ( pvFault - pCur->Core.Key < pCur->cb
327	&& ( uErr & X86_TRAP_PF_RW
328	\|\| pCurType->enmKind != PGMVIRTHANDLERKIND_WRITE ) )
329	{
330	# ifdef IN_RC
331	STAM_PROFILE_START(&pCur->Stat, h);
332	RTGCPTR GCPtrStart = pCur->Core.Key;
333	void *pvUser = pCur->CTX_SUFF(pvUser);
334	pgmUnlock(pVM);
335	*pfLockTaken = false;
336
337	rcStrict = pCurType->CTX_SUFF(pfnPfHandler)(pVM, pVCpu, uErr, pRegFrame, pvFault, GCPtrStart,
338	pvFault - GCPtrStart, pvUser);
339
340	# ifdef VBOX_WITH_STATISTICS
341	pgmLock(pVM);
342	pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
343	if (pCur)
344	STAM_PROFILE_STOP(&pCur->Stat, h);
345	pgmUnlock(pVM);
346	# endif
347	# else
348	rcStrict = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
349	# endif
350	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersVirtual);
351	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
352	return rcStrict;
353	}
354	/* Unhandled part of a monitored page */
355	Log(("Unhandled part of monitored page %RGv\n", pvFault));
356	}
357	else
358	{
359	/* Check by physical address. */
360	unsigned iPage;
361	pCur = pgmHandlerVirtualFindByPhysAddr(pVM, pGstWalk->Core.GCPhys, &iPage);
362	if (pCur)
363	{
364	PPGMVIRTHANDLERTYPEINT pCurType = PGMVIRTANDLER_GET_TYPE(pVM, pCur);
365	if ( uErr & X86_TRAP_PF_RW
366	\|\| pCurType->enmKind != PGMVIRTHANDLERKIND_WRITE )
367	{
368	Assert( (pCur->aPhysToVirt[iPage].Core.Key & X86_PTE_PAE_PG_MASK)
369	== (pGstWalk->Core.GCPhys & X86_PTE_PAE_PG_MASK));
370	# ifdef IN_RC
371	STAM_PROFILE_START(&pCur->Stat, h);
372	RTGCPTR GCPtrStart = pCur->Core.Key;
373	void *pvUser = pCur->CTX_SUFF(pvUser);
374	pgmUnlock(pVM);
375	*pfLockTaken = false;
376
377	RTGCPTR off = (iPage << PAGE_SHIFT)
378	+ (pvFault & PAGE_OFFSET_MASK)
379	- (GCPtrStart & PAGE_OFFSET_MASK);
380	Assert(off < pCur->cb);
381	rcStrict = pCurType->CTX_SUFF(pfnPfHandler)(pVM, pVCpu, uErr, pRegFrame, pvFault, GCPtrStart, off, pvUser);
382
383	# ifdef VBOX_WITH_STATISTICS
384	pgmLock(pVM);
385	pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, GCPtrStart);
386	if (pCur)
387	STAM_PROFILE_STOP(&pCur->Stat, h);
388	pgmUnlock(pVM);
389	# endif
390	# else
391	rcStrict = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
392	# endif
393	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersVirtualByPhys);
394	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
395	return rcStrict;
396	}
397	}
398	}
399	}
400	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
401
402	/*
403	* There is a handled area of the page, but this fault doesn't belong to it.
404	* We must emulate the instruction.
405	*
406	* To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
407	* we first check if this was a page-not-present fault for a page with only
408	* write access handlers. Restart the instruction if it wasn't a write access.
409	*/
410	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersUnhandled);
411
412	if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
413	&& !(uErr & X86_TRAP_PF_P))
414	{
415	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
416	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
417	# else
418	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
419	# endif
420	if ( RT_FAILURE(rcStrict)
421	\|\| rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE
422	\|\| !(uErr & X86_TRAP_PF_RW))
423	{
424	AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
425	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
426	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
427	return rcStrict;
428	}
429	}
430
431	/** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
432	* It's writing to an unhandled part of the LDT page several million times.
433	*/
434	rcStrict = PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault);
435	LogFlow(("PGM: PGMInterpretInstruction -> rcStrict=%d pPage=%R[pgmpage]\n", VBOXSTRICTRC_VAL(rcStrict), pPage));
436	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndUnhandled; });
437	return rcStrict;
438	} /* if any kind of handler */
439
440
441	/**
442	* \#PF Handler for raw-mode guest execution.
443	*
444	* @returns VBox status code (appropriate for trap handling and GC return).
445	*
446	* @param pVCpu The cross context virtual CPU structure.
447	* @param uErr The trap error code.
448	* @param pRegFrame Trap register frame.
449	* @param pvFault The fault address.
450	* @param pfLockTaken PGM lock taken here or not (out)
451	*/
452	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken)
453	{
454	PVM pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
455
456	*pfLockTaken = false;
457
458	# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \
459	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
460	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
461	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT)
462	int rc;
463
464	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
465	/*
466	* Walk the guest page translation tables and check if it's a guest fault.
467	*/
468	GSTPTWALK GstWalk;
469	rc = PGM_GST_NAME(Walk)(pVCpu, pvFault, &GstWalk);
470	if (RT_FAILURE_NP(rc))
471	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &GstWalk, uErr));
472
473	/* assert some GstWalk sanity. */
474	# if PGM_GST_TYPE == PGM_TYPE_AMD64
475	/AssertMsg(GstWalk.Pml4e.u == GstWalk.pPml4e->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pml4e.u, (uint64_t)GstWalk.pPml4e->u)); - not always true with SMP guests. /
476	# endif
477	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
478	/AssertMsg(GstWalk.Pdpe.u == GstWalk.pPdpe->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pdpe.u, (uint64_t)GstWalk.pPdpe->u)); - ditto /
479	# endif
480	/AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); - ditto /
481	/AssertMsg(GstWalk.Core.fBigPage \|\| GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); - ditto /
482	Assert(GstWalk.Core.fSucceeded);
483
484	if (uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID))
485	{
486	if ( ( (uErr & X86_TRAP_PF_RW)
487	&& !GstWalk.Core.fEffectiveRW
488	&& ( (uErr & X86_TRAP_PF_US)
489	\|\| CPUMIsGuestR0WriteProtEnabled(pVCpu)) )
490	\|\| ((uErr & X86_TRAP_PF_US) && !GstWalk.Core.fEffectiveUS)
491	\|\| ((uErr & X86_TRAP_PF_ID) && GstWalk.Core.fEffectiveNX)
492	)
493	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &GstWalk, uErr));
494	}
495
496	/*
497	* Set the accessed and dirty flags.
498	*/
499	# if PGM_GST_TYPE == PGM_TYPE_AMD64
500	GstWalk.Pml4e.u \|= X86_PML4E_A;
501	GstWalk.pPml4e->u \|= X86_PML4E_A;
502	GstWalk.Pdpe.u \|= X86_PDPE_A;
503	GstWalk.pPdpe->u \|= X86_PDPE_A;
504	# endif
505	if (GstWalk.Core.fBigPage)
506	{
507	Assert(GstWalk.Pde.b.u1Size);
508	if (uErr & X86_TRAP_PF_RW)
509	{
510	GstWalk.Pde.u \|= X86_PDE4M_A \| X86_PDE4M_D;
511	GstWalk.pPde->u \|= X86_PDE4M_A \| X86_PDE4M_D;
512	}
513	else
514	{
515	GstWalk.Pde.u \|= X86_PDE4M_A;
516	GstWalk.pPde->u \|= X86_PDE4M_A;
517	}
518	}
519	else
520	{
521	Assert(!GstWalk.Pde.b.u1Size);
522	GstWalk.Pde.u \|= X86_PDE_A;
523	GstWalk.pPde->u \|= X86_PDE_A;
524	if (uErr & X86_TRAP_PF_RW)
525	{
526	# ifdef VBOX_WITH_STATISTICS
527	if (!GstWalk.Pte.n.u1Dirty)
528	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtiedPage));
529	else
530	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageAlreadyDirty));
531	# endif
532	GstWalk.Pte.u \|= X86_PTE_A \| X86_PTE_D;
533	GstWalk.pPte->u \|= X86_PTE_A \| X86_PTE_D;
534	}
535	else
536	{
537	GstWalk.Pte.u \|= X86_PTE_A;
538	GstWalk.pPte->u \|= X86_PTE_A;
539	}
540	Assert(GstWalk.Pte.u == GstWalk.pPte->u);
541	}
542	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u \|\| GstWalk.pPte->u == GstWalk.pPde->u,
543	("%RX64 %RX64 pPte=%p pPde=%p Pte=%RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u, GstWalk.pPte, GstWalk.pPde, (uint64_t)GstWalk.pPte->u));
544	# else /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
545	GSTPDE const PdeSrcDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A}; /** @todo eliminate this */
546	# endif /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
547
548	/* Take the big lock now. */
549	*pfLockTaken = true;
550	pgmLock(pVM);
551
552	# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
553	/*
554	* If it is a reserved bit fault we know that it is an MMIO (access
555	* handler) related fault and can skip some 200 lines of code.
556	*/
557	if (uErr & X86_TRAP_PF_RSVD)
558	{
559	Assert(uErr & X86_TRAP_PF_P);
560	PPGMPAGE pPage;
561	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
562	rc = pgmPhysGetPageEx(pVM, GstWalk.Core.GCPhys, &pPage);
563	if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
564	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
565	pfLockTaken, &GstWalk));
566	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
567	# else
568	rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault), &pPage);
569	if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
570	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
571	pfLockTaken));
572	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
573	#endif
574	AssertRC(rc);
575	PGM_INVL_PG(pVCpu, pvFault);
576	return rc; /* Restart with the corrected entry. */
577	}
578	# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
579
580	/*
581	* Fetch the guest PDE, PDPE and PML4E.
582	*/
583	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
584	const unsigned iPDDst = pvFault >> SHW_PD_SHIFT;
585	PX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
586
587	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
588	const unsigned iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK; /* pPDDst index, not used with the pool. */
589	PX86PDPAE pPDDst;
590	# if PGM_GST_TYPE == PGM_TYPE_PAE
591	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, GstWalk.Pdpe.u, &pPDDst);
592	# else
593	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, X86_PDPE_P, &pPDDst); /* RW, US and A are reserved in PAE mode. */
594	# endif
595	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
596
597	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
598	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
599	PX86PDPAE pPDDst;
600	# if PGM_GST_TYPE == PGM_TYPE_PROT /* (AMD-V nested paging) */
601	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A,
602	X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A, &pPDDst);
603	# else
604	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, GstWalk.Pml4e.u, GstWalk.Pdpe.u, &pPDDst);
605	# endif
606	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
607
608	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
609	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
610	PEPTPD pPDDst;
611	rc = pgmShwGetEPTPDPtr(pVCpu, pvFault, NULL, &pPDDst);
612	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
613	# endif
614	Assert(pPDDst);
615
616	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
617	/*
618	* Dirty page handling.
619	*
620	* If we successfully correct the write protection fault due to dirty bit
621	* tracking, then return immediately.
622	*/
623	if (uErr & X86_TRAP_PF_RW) /* write fault? */
624	{
625	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyBitTracking), a);
626	rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, &pPDDst->a[iPDDst], GstWalk.pPde, pvFault);
627	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyBitTracking), a);
628	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
629	{
630	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution)
631	= rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
632	? &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2DirtyAndAccessed
633	: &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
634	Log8(("Trap0eHandler: returns VINF_SUCCESS\n"));
635	return VINF_SUCCESS;
636	}
637	//AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u \|\| GstWalk.pPte->u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); - triggers with smp w7 guests.
638	//AssertMsg(GstWalk.Core.fBigPage \|\| GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); - ditto.
639	}
640
641	# if 0 /* rarely useful; leave for debugging. */
642	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0ePD[iPDSrc]);
643	# endif
644	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
645
646	/*
647	* A common case is the not-present error caused by lazy page table syncing.
648	*
649	* It is IMPORTANT that we weed out any access to non-present shadow PDEs
650	* here so we can safely assume that the shadow PT is present when calling
651	* SyncPage later.
652	*
653	* On failure, we ASSUME that SyncPT is out of memory or detected some kind
654	* of mapping conflict and defer to SyncCR3 in R3.
655	* (Again, we do NOT support access handlers for non-present guest pages.)
656	*
657	*/
658	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
659	Assert(GstWalk.Pde.n.u1Present);
660	# endif
661	if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
662	&& !pPDDst->a[iPDDst].n.u1Present)
663	{
664	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2SyncPT; });
665	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
666	LogFlow(("=>SyncPT %04x = %08RX64\n", (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, (uint64_t)GstWalk.Pde.u));
667	rc = PGM_BTH_NAME(SyncPT)(pVCpu, (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, GstWalk.pPd, pvFault);
668	# else
669	LogFlow(("=>SyncPT pvFault=%RGv\n", pvFault));
670	rc = PGM_BTH_NAME(SyncPT)(pVCpu, 0, NULL, pvFault);
671	# endif
672	if (RT_SUCCESS(rc))
673	return rc;
674	Log(("SyncPT: %RGv failed!! rc=%Rrc\n", pvFault, rc));
675	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
676	return VINF_PGM_SYNC_CR3;
677	}
678
679	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(PGM_WITHOUT_MAPPINGS)
680	/*
681	* Check if this address is within any of our mappings.
682	*
683	* This is very fast and it's gonna save us a bit of effort below and prevent
684	* us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
685	* (BTW, it's impossible to have physical access handlers in a mapping.)
686	*/
687	if (pgmMapAreMappingsEnabled(pVM))
688	{
689	PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
690	for ( ; pMapping; pMapping = pMapping->CTX_SUFF(pNext))
691	{
692	if (pvFault < pMapping->GCPtr)
693	break;
694	if (pvFault - pMapping->GCPtr < pMapping->cb)
695	{
696	/*
697	* The first thing we check is if we've got an undetected conflict.
698	*/
699	if (pgmMapAreMappingsFloating(pVM))
700	{
701	unsigned iPT = pMapping->cb >> GST_PD_SHIFT;
702	while (iPT-- > 0)
703	if (GstWalk.pPde[iPT].n.u1Present)
704	{
705	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eConflicts);
706	Log(("Trap0e: Detected Conflict %RGv-%RGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
707	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
708	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
709	return VINF_PGM_SYNC_CR3;
710	}
711	}
712
713	/*
714	* Check if the fault address is in a virtual page access handler range.
715	*/
716	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->HyperVirtHandlers,
717	pvFault);
718	if ( pCur
719	&& pvFault - pCur->Core.Key < pCur->cb
720	&& uErr & X86_TRAP_PF_RW)
721	{
722	VBOXSTRICTRC rcStrict;
723	# ifdef IN_RC
724	STAM_PROFILE_START(&pCur->Stat, h);
725	PPGMVIRTHANDLERTYPEINT pCurType = PGMVIRTANDLER_GET_TYPE(pVM, pCur);
726	void *pvUser = pCur->CTX_SUFF(pvUser);
727	pgmUnlock(pVM);
728	rcStrict = pCurType->CTX_SUFF(pfnPfHandler)(pVM, pVCpu, uErr, pRegFrame, pvFault, pCur->Core.Key,
729	pvFault - pCur->Core.Key, pvUser);
730	pgmLock(pVM);
731	STAM_PROFILE_STOP(&pCur->Stat, h);
732	# else
733	AssertFailed();
734	rcStrict = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
735	# endif
736	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersMapping);
737	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
738	return VBOXSTRICTRC_TODO(rcStrict);
739	}
740
741	/*
742	* Pretend we're not here and let the guest handle the trap.
743	*/
744	TRPMSetErrorCode(pVCpu, uErr & ~X86_TRAP_PF_P);
745	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eGuestPFMapping);
746	LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
747	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
748	return VINF_EM_RAW_GUEST_TRAP;
749	}
750	}
751	} /* pgmAreMappingsEnabled(&pVM->pgm.s) */
752	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
753
754	/*
755	* Check if this fault address is flagged for special treatment,
756	* which means we'll have to figure out the physical address and
757	* check flags associated with it.
758	*
759	* ASSUME that we can limit any special access handling to pages
760	* in page tables which the guest believes to be present.
761	*/
762	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
763	RTGCPHYS GCPhys = GstWalk.Core.GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
764	# else
765	RTGCPHYS GCPhys = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault & ~(RTGCPHYS)PAGE_OFFSET_MASK);
766	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
767	PPGMPAGE pPage;
768	rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
769	if (RT_FAILURE(rc))
770	{
771	/*
772	* When the guest accesses invalid physical memory (e.g. probing
773	* of RAM or accessing a remapped MMIO range), then we'll fall
774	* back to the recompiler to emulate the instruction.
775	*/
776	LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhys, rc));
777	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersInvalid);
778	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2InvalidPhys; });
779	return VINF_EM_RAW_EMULATE_INSTR;
780	}
781
782	/*
783	* Any handlers for this page?
784	*/
785	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
786	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
787	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, pfLockTaken,
788	&GstWalk));
789	# else
790	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, pfLockTaken));
791	# endif
792
793	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
794	if (uErr & X86_TRAP_PF_P)
795	{
796	/*
797	* The page isn't marked, but it might still be monitored by a virtual page access handler.
798	* (ASSUMES no temporary disabling of virtual handlers.)
799	*/
800	/** @todo r=bird: Since the purpose is to catch out of sync pages with virtual handler(s) here,
801	* we should correct both the shadow page table and physical memory flags, and not only check for
802	* accesses within the handler region but for access to pages with virtual handlers. */
803	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
804	if (pCur)
805	{
806	PPGMVIRTHANDLERTYPEINT pCurType = PGMVIRTANDLER_GET_TYPE(pVM, pCur);
807	AssertMsg( !(pvFault - pCur->Core.Key < pCur->cb)
808	\|\| ( pCurType->enmKind != PGMVIRTHANDLERKIND_WRITE
809	\|\| !(uErr & X86_TRAP_PF_P)
810	\|\| (pCurType->enmKind == PGMVIRTHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW))),
811	("Unexpected trap for virtual handler: %08X (phys=%08x) %R[pgmpage] uErr=%X, enumKind=%d\n",
812	pvFault, GCPhys, pPage, uErr, pCurType->enmKind));
813
814	if ( pvFault - pCur->Core.Key < pCur->cb
815	&& ( uErr & X86_TRAP_PF_RW
816	\|\| pCurType->enmKind != PGMVIRTHANDLERKIND_WRITE ) )
817	{
818	VBOXSTRICTRC rcStrict;
819	# ifdef IN_RC
820	STAM_PROFILE_START(&pCur->Stat, h);
821	void *pvUser = pCur->CTX_SUFF(pvUser);
822	pgmUnlock(pVM);
823	rcStrict = pCurType->CTX_SUFF(pfnPfHandler)(pVM, pVCpu, uErr, pRegFrame, pvFault, pCur->Core.Key,
824	pvFault - pCur->Core.Key, pvUser);
825	pgmLock(pVM);
826	STAM_PROFILE_STOP(&pCur->Stat, h);
827	# else
828	rcStrict = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
829	# endif
830	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
831	return VBOXSTRICTRC_TODO(rcStrict);
832	}
833	}
834	}
835	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
836
837	/*
838	* We are here only if page is present in Guest page tables and
839	* trap is not handled by our handlers.
840	*
841	* Check it for page out-of-sync situation.
842	*/
843	if (!(uErr & X86_TRAP_PF_P))
844	{
845	/*
846	* Page is not present in our page tables. Try to sync it!
847	*/
848	if (uErr & X86_TRAP_PF_US)
849	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUser));
850	else /* supervisor */
851	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
852
853	if (PGM_PAGE_IS_BALLOONED(pPage))
854	{
855	/* Emulate reads from ballooned pages as they are not present in
856	our shadow page tables. (Required for e.g. Solaris guests; soft
857	ecc, random nr generator.) */
858	rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
859	LogFlow(("PGM: PGMInterpretInstruction balloon -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
860	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncBallloon));
861	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Ballooned; });
862	return rc;
863	}
864
865	# if defined(LOG_ENABLED) && !defined(IN_RING0)
866	RTGCPHYS GCPhys2;
867	uint64_t fPageGst2;
868	PGMGstGetPage(pVCpu, pvFault, &fPageGst2, &GCPhys2);
869	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
870	Log(("Page out of sync: %RGv eip=%08x PdeSrc.US=%d fPageGst2=%08llx GCPhys2=%RGp scan=%d\n",
871	pvFault, pRegFrame->eip, GstWalk.Pde.n.u1User, fPageGst2, GCPhys2, CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)));
872	# else
873	Log(("Page out of sync: %RGv eip=%08x fPageGst2=%08llx GCPhys2=%RGp scan=%d\n",
874	pvFault, pRegFrame->eip, fPageGst2, GCPhys2, CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)));
875	# endif
876	# endif /* LOG_ENABLED */
877
878	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
879	if ( !GstWalk.Core.fEffectiveUS
880	&& CSAMIsEnabled(pVM)
881	&& CPUMGetGuestCPL(pVCpu) == 0)
882	{
883	/* Note: Can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU. */
884	if ( pvFault == (RTGCPTR)pRegFrame->eip
885	\|\| pvFault - pRegFrame->eip < 8 /* instruction crossing a page boundary */
886	# ifdef CSAM_DETECT_NEW_CODE_PAGES
887	\|\| ( !PATMIsPatchGCAddr(pVM, pRegFrame->eip)
888	&& CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)) /* any new code we encounter here */
889	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
890	)
891	{
892	LogFlow(("CSAMExecFault %RX32\n", pRegFrame->eip));
893	rc = CSAMExecFault(pVM, (RTRCPTR)pRegFrame->eip);
894	if (rc != VINF_SUCCESS)
895	{
896	/*
897	* CSAM needs to perform a job in ring 3.
898	*
899	* Sync the page before going to the host context; otherwise we'll end up in a loop if
900	* CSAM fails (e.g. instruction crosses a page boundary and the next page is not present)
901	*/
902	LogFlow(("CSAM ring 3 job\n"));
903	int rc2 = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
904	AssertRC(rc2);
905
906	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2CSAM; });
907	return rc;
908	}
909	}
910	# ifdef CSAM_DETECT_NEW_CODE_PAGES
911	else if ( uErr == X86_TRAP_PF_RW
912	&& pRegFrame->ecx >= 0x100 /* early check for movswd count */
913	&& pRegFrame->ecx < 0x10000)
914	{
915	/* In case of a write to a non-present supervisor shadow page, we'll take special precautions
916	* to detect loading of new code pages.
917	*/
918
919	/*
920	* Decode the instruction.
921	*/
922	PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
923	uint32_t cbOp;
924	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
925
926	/* For now we'll restrict this to rep movsw/d instructions */
927	if ( rc == VINF_SUCCESS
928	&& pDis->pCurInstr->opcode == OP_MOVSWD
929	&& (pDis->prefix & DISPREFIX_REP))
930	{
931	CSAMMarkPossibleCodePage(pVM, pvFault);
932	}
933	}
934	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
935
936	/*
937	* Mark this page as safe.
938	*/
939	/** @todo not correct for pages that contain both code and data!! */
940	Log2(("CSAMMarkPage %RGv; scanned=%d\n", pvFault, true));
941	CSAMMarkPage(pVM, pvFault, true);
942	}
943	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0) */
944	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
945	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
946	# else
947	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
948	# endif
949	if (RT_SUCCESS(rc))
950	{
951	/* The page was successfully synced, return to the guest. */
952	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSync; });
953	return VINF_SUCCESS;
954	}
955	}
956	else /* uErr & X86_TRAP_PF_P: */
957	{
958	/*
959	* Write protected pages are made writable when the guest makes the
960	* first write to it. This happens for pages that are shared, write
961	* monitored or not yet allocated.
962	*
963	* We may also end up here when CR0.WP=0 in the guest.
964	*
965	* Also, a side effect of not flushing global PDEs are out of sync
966	* pages due to physical monitored regions, that are no longer valid.
967	* Assume for now it only applies to the read/write flag.
968	*/
969	if (uErr & X86_TRAP_PF_RW)
970	{
971	/*
972	* Check if it is a read-only page.
973	*/
974	if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
975	{
976	Log(("PGM #PF: Make writable: %RGp %R[pgmpage] pvFault=%RGp uErr=%#x\n", GCPhys, pPage, pvFault, uErr));
977	Assert(!PGM_PAGE_IS_ZERO(pPage));
978	AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
979	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2MakeWritable; });
980
981	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
982	if (rc != VINF_SUCCESS)
983	{
984	AssertMsg(rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("%Rrc\n", rc));
985	return rc;
986	}
987	if (RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)))
988	return VINF_EM_NO_MEMORY;
989	}
990
991	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
992	/*
993	* Check to see if we need to emulate the instruction if CR0.WP=0.
994	*/
995	if ( !GstWalk.Core.fEffectiveRW
996	&& (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG
997	&& CPUMGetGuestCPL(pVCpu) < 3)
998	{
999	Assert((uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RW \| X86_TRAP_PF_P));
1000
1001	/*
1002	* The Netware WP0+RO+US hack.
1003	*
1004	* Netware sometimes(/always?) runs with WP0. It has been observed doing
1005	* excessive write accesses to pages which are mapped with US=1 and RW=0
1006	* while WP=0. This causes a lot of exits and extremely slow execution.
1007	* To avoid trapping and emulating every write here, we change the shadow
1008	* page table entry to map it as US=0 and RW=1 until user mode tries to
1009	* access it again (see further below). We count these shadow page table
1010	* changes so we can avoid having to clear the page pool every time the WP
1011	* bit changes to 1 (see PGMCr0WpEnabled()).
1012	*/
1013	# if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) && 1
1014	if ( GstWalk.Core.fEffectiveUS
1015	&& !GstWalk.Core.fEffectiveRW
1016	&& (GstWalk.Core.fBigPage \|\| GstWalk.Pde.n.u1Write)
1017	&& pVM->cCpus == 1 /* Sorry, no go on SMP. Add CFGM option? */)
1018	{
1019	Log(("PGM #PF: Netware WP0+RO+US hack: pvFault=%RGp uErr=%#x (big=%d)\n", pvFault, uErr, GstWalk.Core.fBigPage));
1020	rc = pgmShwMakePageSupervisorAndWritable(pVCpu, pvFault, GstWalk.Core.fBigPage, PGM_MK_PG_IS_WRITE_FAULT);
1021	if (rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3)
1022	{
1023	PGM_INVL_PG(pVCpu, pvFault);
1024	pVCpu->pgm.s.cNetwareWp0Hacks++;
1025	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Wp0RoUsHack; });
1026	return rc;
1027	}
1028	AssertMsg(RT_FAILURE_NP(rc), ("%Rrc\n", rc));
1029	Log(("pgmShwMakePageSupervisorAndWritable(%RGv) failed with rc=%Rrc - ignored\n", pvFault, rc));
1030	}
1031	# endif
1032
1033	/* Interpret the access. */
1034	rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
1035	Log(("PGM #PF: WP0 emulation (pvFault=%RGp uErr=%#x cpl=%d fBig=%d fEffUs=%d)\n", pvFault, uErr, CPUMGetGuestCPL(pVCpu), GstWalk.Core.fBigPage, GstWalk.Core.fEffectiveUS));
1036	if (RT_SUCCESS(rc))
1037	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eWPEmulInRZ);
1038	else
1039	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eWPEmulToR3);
1040	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2WPEmulation; });
1041	return rc;
1042	}
1043	# endif
1044	/// @todo count the above case; else
1045	if (uErr & X86_TRAP_PF_US)
1046	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUserWrite));
1047	else /* supervisor */
1048	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
1049
1050	/*
1051	* Sync the page.
1052	*
1053	* Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
1054	* page is not present, which is not true in this case.
1055	*/
1056	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1057	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
1058	# else
1059	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
1060	# endif
1061	if (RT_SUCCESS(rc))
1062	{
1063	/*
1064	* Page was successfully synced, return to guest but invalidate
1065	* the TLB first as the page is very likely to be in it.
1066	*/
1067	# if PGM_SHW_TYPE == PGM_TYPE_EPT
1068	HMInvalidatePhysPage(pVM, (RTGCPHYS)pvFault);
1069	# else
1070	PGM_INVL_PG(pVCpu, pvFault);
1071	# endif
1072	# ifdef VBOX_STRICT
1073	RTGCPHYS GCPhys2 = RTGCPHYS_MAX;
1074	uint64_t fPageGst = UINT64_MAX;
1075	if (!pVM->pgm.s.fNestedPaging)
1076	{
1077	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, &GCPhys2);
1078	AssertMsg(RT_SUCCESS(rc) && ((fPageGst & X86_PTE_RW) \|\| ((CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG && CPUMGetGuestCPL(pVCpu) < 3)), ("rc=%Rrc fPageGst=%RX64\n", rc, fPageGst));
1079	LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GCPhys2, (uint64_t)fPageGst));
1080	}
1081	# if 0 /* Bogus! Triggers incorrectly with w7-64 and later for the SyncPage case: "Pde at %RGv changed behind our back?" */
1082	uint64_t fPageShw = 0;
1083	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
1084	AssertMsg((RT_SUCCESS(rc) && (fPageShw & X86_PTE_RW)) \|\| pVM->cCpus > 1 /* new monitor can be installed/page table flushed between the trap exit and PGMTrap0eHandler */,
1085	("rc=%Rrc fPageShw=%RX64 GCPhys2=%RGp fPageGst=%RX64 pvFault=%RGv\n", rc, fPageShw, GCPhys2, fPageGst, pvFault));
1086	# endif
1087	# endif /* VBOX_STRICT */
1088	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndObs; });
1089	return VINF_SUCCESS;
1090	}
1091	}
1092	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1093	/*
1094	* Check for Netware WP0+RO+US hack from above and undo it when user
1095	* mode accesses the page again.
1096	*/
1097	else if ( GstWalk.Core.fEffectiveUS
1098	&& !GstWalk.Core.fEffectiveRW
1099	&& (GstWalk.Core.fBigPage \|\| GstWalk.Pde.n.u1Write)
1100	&& pVCpu->pgm.s.cNetwareWp0Hacks > 0
1101	&& (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG
1102	&& CPUMGetGuestCPL(pVCpu) == 3
1103	&& pVM->cCpus == 1
1104	)
1105	{
1106	Log(("PGM #PF: Undo netware WP0+RO+US hack: pvFault=%RGp uErr=%#x\n", pvFault, uErr));
1107	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
1108	if (RT_SUCCESS(rc))
1109	{
1110	PGM_INVL_PG(pVCpu, pvFault);
1111	pVCpu->pgm.s.cNetwareWp0Hacks--;
1112	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Wp0RoUsUnhack; });
1113	return VINF_SUCCESS;
1114	}
1115	}
1116	# endif /* PGM_WITH_PAGING */
1117
1118	/** @todo else: why are we here? */
1119
1120	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && defined(VBOX_STRICT)
1121	/*
1122	* Check for VMM page flags vs. Guest page flags consistency.
1123	* Currently only for debug purposes.
1124	*/
1125	if (RT_SUCCESS(rc))
1126	{
1127	/* Get guest page flags. */
1128	uint64_t fPageGst;
1129	int rc2 = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
1130	if (RT_SUCCESS(rc2))
1131	{
1132	uint64_t fPageShw = 0;
1133	rc2 = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
1134
1135	/*
1136	* Compare page flags.
1137	* Note: we have AVL, A, D bits desynced.
1138	*/
1139	AssertMsg( (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK))
1140	== (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK))
1141	\|\| ( pVCpu->pgm.s.cNetwareWp0Hacks > 0
1142	&& (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US))
1143	== (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US))
1144	&& (fPageShw & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW
1145	&& (fPageGst & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_US),
1146	("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64 rc=%d\n",
1147	pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst, rc));
1148	}
1149	else
1150	AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc));
1151	}
1152	else
1153	AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc));
1154	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && VBOX_STRICT */
1155	}
1156
1157
1158	/*
1159	* If we get here it is because something failed above, i.e. most like guru
1160	* meditiation time.
1161	*/
1162	LogRel(("%s: returns rc=%Rrc pvFault=%RGv uErr=%RX64 cs:rip=%04x:%08RX64\n",
1163	__PRETTY_FUNCTION__, rc, pvFault, (uint64_t)uErr, pRegFrame->cs.Sel, pRegFrame->rip));
1164	return rc;
1165
1166	# else /* Nested paging, EPT except PGM_GST_TYPE = PROT */
1167	NOREF(uErr); NOREF(pRegFrame); NOREF(pvFault);
1168	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
1169	return VERR_PGM_NOT_USED_IN_MODE;
1170	# endif
1171	}
1172	#endif /* !IN_RING3 */
1173
1174
1175	/**
1176	* Emulation of the invlpg instruction.
1177	*
1178	*
1179	* @returns VBox status code.
1180	*
1181	* @param pVCpu The cross context virtual CPU structure.
1182	* @param GCPtrPage Page to invalidate.
1183	*
1184	* @remark ASSUMES that the guest is updating before invalidating. This order
1185	* isn't required by the CPU, so this is speculative and could cause
1186	* trouble.
1187	* @remark No TLB shootdown is done on any other VCPU as we assume that
1188	* invlpg emulation is the only reason for calling this function.
1189	* (The guest has to shoot down TLB entries on other CPUs itself)
1190	* Currently true, but keep in mind!
1191	*
1192	* @todo Clean this up! Most of it is (or should be) no longer necessary as we catch all page table accesses.
1193	* Should only be required when PGMPOOL_WITH_OPTIMIZED_DIRTY_PT is active (PAE or AMD64 (for now))
1194	*/
1195	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
1196	{
1197	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1198	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
1199	&& PGM_SHW_TYPE != PGM_TYPE_EPT
1200	int rc;
1201	PVM pVM = pVCpu->CTX_SUFF(pVM);
1202	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1203
1204	PGM_LOCK_ASSERT_OWNER(pVM);
1205
1206	LogFlow(("InvalidatePage %RGv\n", GCPtrPage));
1207
1208	/*
1209	* Get the shadow PD entry and skip out if this PD isn't present.
1210	* (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
1211	*/
1212	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1213	const unsigned iPDDst = (uint32_t)GCPtrPage >> SHW_PD_SHIFT;
1214	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1215
1216	/* Fetch the pgm pool shadow descriptor. */
1217	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1218	# ifdef IN_RING3 /* Possible we didn't resync yet when called from REM. */
1219	if (!pShwPde)
1220	{
1221	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1222	return VINF_SUCCESS;
1223	}
1224	# else
1225	Assert(pShwPde);
1226	# endif
1227
1228	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1229	const unsigned iPdpt = (uint32_t)GCPtrPage >> X86_PDPT_SHIFT;
1230	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
1231
1232	/* If the shadow PDPE isn't present, then skip the invalidate. */
1233	# ifdef IN_RING3 /* Possible we didn't resync yet when called from REM. */
1234	if (!pPdptDst \|\| !pPdptDst->a[iPdpt].n.u1Present)
1235	# else
1236	if (!pPdptDst->a[iPdpt].n.u1Present)
1237	# endif
1238	{
1239	Assert(!pPdptDst \|\| !(pPdptDst->a[iPdpt].u & PGM_PLXFLAGS_MAPPING));
1240	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1241	PGM_INVL_PG(pVCpu, GCPtrPage);
1242	return VINF_SUCCESS;
1243	}
1244
1245	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1246	PPGMPOOLPAGE pShwPde = NULL;
1247	PX86PDPAE pPDDst;
1248
1249	/* Fetch the pgm pool shadow descriptor. */
1250	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
1251	AssertRCSuccessReturn(rc, rc);
1252	Assert(pShwPde);
1253
1254	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1255	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1256
1257	# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1258	/* PML4 */
1259	/const unsigned iPml4 = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;/
1260	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1261	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1262	PX86PDPAE pPDDst;
1263	PX86PDPT pPdptDst;
1264	PX86PML4E pPml4eDst;
1265	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eDst, &pPdptDst, &pPDDst);
1266	if (rc != VINF_SUCCESS)
1267	{
1268	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT \|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
1269	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1270	PGM_INVL_PG(pVCpu, GCPtrPage);
1271	return VINF_SUCCESS;
1272	}
1273	Assert(pPDDst);
1274
1275	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1276	PX86PDPE pPdpeDst = &pPdptDst->a[iPdpt];
1277
1278	if (!pPdpeDst->n.u1Present)
1279	{
1280	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1281	PGM_INVL_PG(pVCpu, GCPtrPage);
1282	return VINF_SUCCESS;
1283	}
1284
1285	/* Fetch the pgm pool shadow descriptor. */
1286	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & SHW_PDPE_PG_MASK);
1287	Assert(pShwPde);
1288
1289	# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1290
1291	const SHWPDE PdeDst = *pPdeDst;
1292	if (!PdeDst.n.u1Present)
1293	{
1294	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1295	PGM_INVL_PG(pVCpu, GCPtrPage);
1296	return VINF_SUCCESS;
1297	}
1298
1299	/*
1300	* Get the guest PD entry and calc big page.
1301	*/
1302	# if PGM_GST_TYPE == PGM_TYPE_32BIT
1303	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
1304	const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
1305	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1306	# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1307	unsigned iPDSrc = 0;
1308	# if PGM_GST_TYPE == PGM_TYPE_PAE
1309	X86PDPE PdpeSrcIgn;
1310	PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrcIgn);
1311	# else /* AMD64 */
1312	PX86PML4E pPml4eSrcIgn;
1313	X86PDPE PdpeSrcIgn;
1314	PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrcIgn, &PdpeSrcIgn, &iPDSrc);
1315	# endif
1316	GSTPDE PdeSrc;
1317
1318	if (pPDSrc)
1319	PdeSrc = pPDSrc->a[iPDSrc];
1320	else
1321	PdeSrc.u = 0;
1322	# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1323	const bool fWasBigPage = RT_BOOL(PdeDst.u & PGM_PDFLAGS_BIG_PAGE);
1324	const bool fIsBigPage = PdeSrc.b.u1Size && GST_IS_PSE_ACTIVE(pVCpu);
1325	if (fWasBigPage != fIsBigPage)
1326	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1327
1328	# ifdef IN_RING3
1329	/*
1330	* If a CR3 Sync is pending we may ignore the invalidate page operation
1331	* depending on the kind of sync and if it's a global page or not.
1332	* This doesn't make sense in GC/R0 so we'll skip it entirely there.
1333	*/
1334	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
1335	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
1336	\|\| ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
1337	&& fIsBigPage
1338	&& PdeSrc.b.u1Global
1339	)
1340	)
1341	# else
1342	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
1343	# endif
1344	{
1345	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1346	return VINF_SUCCESS;
1347	}
1348	# endif /* IN_RING3 */
1349
1350	/*
1351	* Deal with the Guest PDE.
1352	*/
1353	rc = VINF_SUCCESS;
1354	if (PdeSrc.n.u1Present)
1355	{
1356	Assert( PdeSrc.n.u1User == PdeDst.n.u1User
1357	&& (PdeSrc.n.u1Write \|\| !PdeDst.n.u1Write \|\| pVCpu->pgm.s.cNetwareWp0Hacks > 0));
1358	# ifndef PGM_WITHOUT_MAPPING
1359	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
1360	{
1361	/*
1362	* Conflict - Let SyncPT deal with it to avoid duplicate code.
1363	*/
1364	Assert(pgmMapAreMappingsEnabled(pVM));
1365	Assert(PGMGetGuestMode(pVCpu) <= PGMMODE_PAE);
1366	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
1367	}
1368	else
1369	# endif /* !PGM_WITHOUT_MAPPING */
1370	if (!fIsBigPage)
1371	{
1372	/*
1373	* 4KB - page.
1374	*/
1375	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1376	RTGCPHYS GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
1377
1378	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1379	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1380	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| ((iPDDst & 1) * (PAGE_SIZE / 2)));
1381	# endif
1382	if (pShwPage->GCPhys == GCPhys)
1383	{
1384	/* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1385	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1386
1387	PGSTPT pPTSrc;
1388	rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
1389	if (RT_SUCCESS(rc))
1390	{
1391	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1392	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1393	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1394	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1395	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
1396	GCPtrPage, PteSrc.n.u1Present,
1397	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1398	PteSrc.n.u1User & PdeSrc.n.u1User,
1399	(uint64_t)PteSrc.u,
1400	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
1401	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
1402	}
1403	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4KBPages));
1404	PGM_INVL_PG(pVCpu, GCPtrPage);
1405	}
1406	else
1407	{
1408	/*
1409	* The page table address changed.
1410	*/
1411	LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
1412	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1413	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1414	ASMAtomicWriteSize(pPdeDst, 0);
1415	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1416	PGM_INVL_VCPU_TLBS(pVCpu);
1417	}
1418	}
1419	else
1420	{
1421	/*
1422	* 2/4MB - page.
1423	*/
1424	/* Before freeing the page, check if anything really changed. */
1425	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1426	RTGCPHYS GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
1427	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1428	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1429	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
1430	# endif
1431	if ( pShwPage->GCPhys == GCPhys
1432	&& pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1433	{
1434	/* ASSUMES a the given bits are identical for 4M and normal PDEs */
1435	/** @todo This test is wrong as it cannot check the G bit!
1436	* FIXME */
1437	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US))
1438	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US))
1439	&& ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1440	\|\| (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1441	{
1442	LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1443	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip));
1444	return VINF_SUCCESS;
1445	}
1446	}
1447
1448	/*
1449	* Ok, the page table is present and it's been changed in the guest.
1450	* If we're in host context, we'll just mark it as not present taking the lazy approach.
1451	* We could do this for some flushes in GC too, but we need an algorithm for
1452	* deciding which 4MB pages containing code likely to be executed very soon.
1453	*/
1454	LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1455	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1456	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1457	ASMAtomicWriteSize(pPdeDst, 0);
1458	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4MBPages));
1459	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
1460	}
1461	}
1462	else
1463	{
1464	/*
1465	* Page directory is not present, mark shadow PDE not present.
1466	*/
1467	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
1468	{
1469	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1470	ASMAtomicWriteSize(pPdeDst, 0);
1471	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDNPs));
1472	PGM_INVL_PG(pVCpu, GCPtrPage);
1473	}
1474	else
1475	{
1476	Assert(pgmMapAreMappingsEnabled(pVM));
1477	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDMappings));
1478	}
1479	}
1480	return rc;
1481
1482	#else /* guest real and protected mode */
1483	/* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1484	NOREF(pVCpu); NOREF(GCPtrPage);
1485	return VINF_SUCCESS;
1486	#endif
1487	}
1488
1489
1490	/**
1491	* Update the tracking of shadowed pages.
1492	*
1493	* @param pVCpu The cross context virtual CPU structure.
1494	* @param pShwPage The shadow page.
1495	* @param HCPhys The physical page we is being dereferenced.
1496	* @param iPte Shadow PTE index
1497	* @param GCPhysPage Guest physical address (only valid if pShwPage->fDirty is set)
1498	*/
1499	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys, uint16_t iPte,
1500	RTGCPHYS GCPhysPage)
1501	{
1502	PVM pVM = pVCpu->CTX_SUFF(pVM);
1503
1504	# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1505	&& PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1506	&& (PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
1507
1508	/* Use the hint we retrieved from the cached guest PT. */
1509	if (pShwPage->fDirty)
1510	{
1511	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1512
1513	Assert(pShwPage->cPresent);
1514	Assert(pPool->cPresent);
1515	pShwPage->cPresent--;
1516	pPool->cPresent--;
1517
1518	PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysPage);
1519	AssertRelease(pPhysPage);
1520	pgmTrackDerefGCPhys(pPool, pShwPage, pPhysPage, iPte);
1521	return;
1522	}
1523	# else
1524	NOREF(GCPhysPage);
1525	# endif
1526
1527	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackDeref, a);
1528	LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1529
1530	/** @todo If this turns out to be a bottle neck (very likely) two things can be done:
1531	* 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1532	* 2. write protect all shadowed pages. I.e. implement caching.
1533	*/
1534	/** @todo duplicated in the 2nd half of pgmPoolTracDerefGCPhysHint */
1535
1536	/*
1537	* Find the guest address.
1538	*/
1539	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1540	pRam;
1541	pRam = pRam->CTX_SUFF(pNext))
1542	{
1543	unsigned iPage = pRam->cb >> PAGE_SHIFT;
1544	while (iPage-- > 0)
1545	{
1546	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1547	{
1548	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1549
1550	Assert(pShwPage->cPresent);
1551	Assert(pPool->cPresent);
1552	pShwPage->cPresent--;
1553	pPool->cPresent--;
1554
1555	pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage], iPte);
1556	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackDeref, a);
1557	return;
1558	}
1559	}
1560	}
1561
1562	for (;;)
1563	AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys));
1564	}
1565
1566
1567	/**
1568	* Update the tracking of shadowed pages.
1569	*
1570	* @param pVCpu The cross context virtual CPU structure.
1571	* @param pShwPage The shadow page.
1572	* @param u16 The top 16-bit of the pPage->HCPhys.
1573	* @param pPage Pointer to the guest page. this will be modified.
1574	* @param iPTDst The index into the shadow table.
1575	*/
1576	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
1577	{
1578	PVM pVM = pVCpu->CTX_SUFF(pVM);
1579
1580	/*
1581	* Just deal with the simple first time here.
1582	*/
1583	if (!u16)
1584	{
1585	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackVirgin);
1586	u16 = PGMPOOL_TD_MAKE(1, pShwPage->idx);
1587	/* Save the page table index. */
1588	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, iPTDst);
1589	}
1590	else
1591	u16 = pgmPoolTrackPhysExtAddref(pVM, pPage, u16, pShwPage->idx, iPTDst);
1592
1593	/* write back */
1594	Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x iPTDst=%#x\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst));
1595	PGM_PAGE_SET_TRACKING(pVM, pPage, u16);
1596
1597	/* update statistics. */
1598	pVM->pgm.s.CTX_SUFF(pPool)->cPresent++;
1599	pShwPage->cPresent++;
1600	if (pShwPage->iFirstPresent > iPTDst)
1601	pShwPage->iFirstPresent = iPTDst;
1602	}
1603
1604
1605	/**
1606	* Modifies a shadow PTE to account for access handlers.
1607	*
1608	* @param pVM The cross context VM structure.
1609	* @param pPage The page in question.
1610	* @param fPteSrc The shadowed flags of the source PTE. Must include the
1611	* A (accessed) bit so it can be emulated correctly.
1612	* @param pPteDst The shadow PTE (output). This is temporary storage and
1613	* does not need to be set atomically.
1614	*/
1615	DECLINLINE(void) PGM_BTH_NAME(SyncHandlerPte)(PVM pVM, PCPGMPAGE pPage, uint64_t fPteSrc, PSHWPTE pPteDst)
1616	{
1617	NOREF(pVM); RT_NOREF_PV(fPteSrc);
1618
1619	/** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No.
1620	* Update: \#PF should deal with this before or after calling the handlers. It has all the info to do the job efficiently. */
1621	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1622	{
1623	LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark read-only\n", pPage));
1624	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1625	pPteDst->u = PGM_PAGE_GET_HCPHYS(pPage);
1626	pPteDst->n.u1Present = 1;
1627	pPteDst->n.u1Execute = 1;
1628	pPteDst->n.u1IgnorePAT = 1;
1629	pPteDst->n.u3EMT = VMX_EPT_MEMTYPE_WB;
1630	/* PteDst.n.u1Write = 0 && PteDst.n.u1Size = 0 */
1631	#else
1632	if (fPteSrc & X86_PTE_A)
1633	{
1634	SHW_PTE_SET(*pPteDst, fPteSrc \| PGM_PAGE_GET_HCPHYS(pPage));
1635	SHW_PTE_SET_RO(*pPteDst);
1636	}
1637	else
1638	SHW_PTE_SET(*pPteDst, 0);
1639	#endif
1640	}
1641	#ifdef PGM_WITH_MMIO_OPTIMIZATIONS
1642	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
1643	else if ( PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
1644	&& ( BTH_IS_NP_ACTIVE(pVM)
1645	\|\| (fPteSrc & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW) /** @todo Remove X86_PTE_US here and pGstWalk->Core.fEffectiveUS before the sync page test. */
1646	# if PGM_SHW_TYPE == PGM_TYPE_AMD64
1647	&& pVM->pgm.s.fLessThan52PhysicalAddressBits
1648	# endif
1649	)
1650	{
1651	LogFlow(("SyncHandlerPte: MMIO page -> invalid \n"));
1652	# if PGM_SHW_TYPE == PGM_TYPE_EPT
1653	/* 25.2.3.1: Reserved physical address bit -> EPT Misconfiguration (exit 49) */
1654	pPteDst->u = pVM->pgm.s.HCPhysInvMmioPg;
1655	/* 25.2.3.1: bits 2:0 = 010b -> EPT Misconfiguration (exit 49) */
1656	pPteDst->n.u1Present = 0;
1657	pPteDst->n.u1Write = 1;
1658	pPteDst->n.u1Execute = 0;
1659	/* 25.2.3.1: leaf && 2:0 != 0 && u3Emt in {2, 3, 7} -> EPT Misconfiguration */
1660	pPteDst->n.u3EMT = 7;
1661	# else
1662	/* Set high page frame bits that MBZ (bankers on PAE, CPU dependent on AMD64). */
1663	SHW_PTE_SET(*pPteDst, pVM->pgm.s.HCPhysInvMmioPg \| X86_PTE_PAE_MBZ_MASK_NO_NX \| X86_PTE_P);
1664	# endif
1665	}
1666	# endif
1667	#endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
1668	else
1669	{
1670	LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark not present\n", pPage));
1671	SHW_PTE_SET(*pPteDst, 0);
1672	}
1673	/** @todo count these kinds of entries. */
1674	}
1675
1676
1677	/**
1678	* Creates a 4K shadow page for a guest page.
1679	*
1680	* For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1681	* physical address. The PdeSrc argument only the flags are used. No page
1682	* structured will be mapped in this function.
1683	*
1684	* @param pVCpu The cross context virtual CPU structure.
1685	* @param pPteDst Destination page table entry.
1686	* @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1687	* Can safely assume that only the flags are being used.
1688	* @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1689	* @param pShwPage Pointer to the shadow page.
1690	* @param iPTDst The index into the shadow table.
1691	*
1692	* @remark Not used for 2/4MB pages!
1693	*/
1694	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \|\| defined(DOXYGEN_RUNNING)
1695	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc,
1696	PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1697	#else
1698	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage,
1699	PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1700	#endif
1701	{
1702	PVM pVM = pVCpu->CTX_SUFF(pVM);
1703	RTGCPHYS GCPhysOldPage = NIL_RTGCPHYS;
1704
1705	#if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1706	&& PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1707	&& (PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
1708
1709	if (pShwPage->fDirty)
1710	{
1711	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1712	PGSTPT pGstPT;
1713
1714	/* Note that iPTDst can be used to index the guest PT even in the pae/32bit combo as we copy only half the table; see pgmPoolAddDirtyPage. */
1715	pGstPT = (PGSTPT)&pPool->aDirtyPages[pShwPage->idxDirtyEntry].aPage[0];
1716	GCPhysOldPage = GST_GET_PTE_GCPHYS(pGstPT->a[iPTDst]);
1717	pGstPT->a[iPTDst].u = PteSrc.u;
1718	}
1719	#else
1720	Assert(!pShwPage->fDirty);
1721	#endif
1722
1723	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1724	if ( PteSrc.n.u1Present
1725	&& GST_IS_PTE_VALID(pVCpu, PteSrc))
1726	#endif
1727	{
1728	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1729	RTGCPHYS GCPhysPage = GST_GET_PTE_GCPHYS(PteSrc);
1730	# endif
1731	PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
1732
1733	/*
1734	* Find the ram range.
1735	*/
1736	PPGMPAGE pPage;
1737	int rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
1738	if (RT_SUCCESS(rc))
1739	{
1740	/* Ignore ballooned pages.
1741	Don't return errors or use a fatal assert here as part of a
1742	shadow sync range might included ballooned pages. */
1743	if (PGM_PAGE_IS_BALLOONED(pPage))
1744	{
1745	Assert(!SHW_PTE_IS_P(pPteDst)); /* @todo user tracking needs updating if this triggers. */
1746	return;
1747	}
1748
1749	#ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1750	/* Make the page writable if necessary. */
1751	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1752	&& ( PGM_PAGE_IS_ZERO(pPage)
1753	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1754	\|\| ( PteSrc.n.u1Write
1755	# else
1756	\|\| ( 1
1757	# endif
1758	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1759	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1760	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1761	# endif
1762	# ifdef VBOX_WITH_PAGE_SHARING
1763	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
1764	# endif
1765	)
1766	)
1767	)
1768	{
1769	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhysPage);
1770	AssertRC(rc);
1771	}
1772	#endif
1773
1774	/*
1775	* Make page table entry.
1776	*/
1777	SHWPTE PteDst;
1778	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1779	uint64_t fGstShwPteFlags = GST_GET_PTE_SHW_FLAGS(pVCpu, PteSrc);
1780	# else
1781	uint64_t fGstShwPteFlags = X86_PTE_P \| X86_PTE_RW \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D;
1782	# endif
1783	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1784	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, fGstShwPteFlags, &PteDst);
1785	else
1786	{
1787	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1788	/*
1789	* If the page or page directory entry is not marked accessed,
1790	* we mark the page not present.
1791	*/
1792	if (!PteSrc.n.u1Accessed \|\| !PdeSrc.n.u1Accessed)
1793	{
1794	LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1795	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,AccessedPage));
1796	SHW_PTE_SET(PteDst, 0);
1797	}
1798	/*
1799	* If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1800	* when the page is modified.
1801	*/
1802	else if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1803	{
1804	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPage));
1805	SHW_PTE_SET(PteDst,
1806	fGstShwPteFlags
1807	\| PGM_PAGE_GET_HCPHYS(pPage)
1808	\| PGM_PTFLAGS_TRACK_DIRTY);
1809	SHW_PTE_SET_RO(PteDst);
1810	}
1811	else
1812	#endif
1813	{
1814	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageSkipped));
1815	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1816	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage);
1817	PteDst.n.u1Present = 1;
1818	PteDst.n.u1Write = 1;
1819	PteDst.n.u1Execute = 1;
1820	PteDst.n.u1IgnorePAT = 1;
1821	PteDst.n.u3EMT = VMX_EPT_MEMTYPE_WB;
1822	/* PteDst.n.u1Size = 0 */
1823	#else
1824	SHW_PTE_SET(PteDst, fGstShwPteFlags \| PGM_PAGE_GET_HCPHYS(pPage));
1825	#endif
1826	}
1827
1828	/*
1829	* Make sure only allocated pages are mapped writable.
1830	*/
1831	if ( SHW_PTE_IS_P_RW(PteDst)
1832	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1833	{
1834	/* Still applies to shared pages. */
1835	Assert(!PGM_PAGE_IS_ZERO(pPage));
1836	SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet. Why, isn't it? */
1837	Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", GCPhysPage, pPage, iPTDst));
1838	}
1839	}
1840
1841	/*
1842	* Keep user track up to date.
1843	*/
1844	if (SHW_PTE_IS_P(PteDst))
1845	{
1846	if (!SHW_PTE_IS_P(*pPteDst))
1847	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1848	else if (SHW_PTE_GET_HCPHYS(*pPteDst) != SHW_PTE_GET_HCPHYS(PteDst))
1849	{
1850	Log2(("SyncPageWorker: deref! pPteDst=%RX64 PteDst=%RX64\n", SHW_PTE_LOG64(pPteDst), SHW_PTE_LOG64(PteDst)));
1851	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
1852	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1853	}
1854	}
1855	else if (SHW_PTE_IS_P(*pPteDst))
1856	{
1857	Log2(("SyncPageWorker: deref! pPteDst=%RX64\n", SHW_PTE_LOG64(pPteDst)));
1858	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
1859	}
1860
1861	/*
1862	* Update statistics and commit the entry.
1863	*/
1864	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1865	if (!PteSrc.n.u1Global)
1866	pShwPage->fSeenNonGlobal = true;
1867	#endif
1868	SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
1869	return;
1870	}
1871
1872	/** @todo count these three different kinds. */
1873	Log2(("SyncPageWorker: invalid address in Pte\n"));
1874	}
1875	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1876	else if (!PteSrc.n.u1Present)
1877	Log2(("SyncPageWorker: page not present in Pte\n"));
1878	else
1879	Log2(("SyncPageWorker: invalid Pte\n"));
1880	#endif
1881
1882	/*
1883	* The page is not present or the PTE is bad. Replace the shadow PTE by
1884	* an empty entry, making sure to keep the user tracking up to date.
1885	*/
1886	if (SHW_PTE_IS_P(*pPteDst))
1887	{
1888	Log2(("SyncPageWorker: deref! pPteDst=%RX64\n", SHW_PTE_LOG64(pPteDst)));
1889	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
1890	}
1891	SHW_PTE_ATOMIC_SET(*pPteDst, 0);
1892	}
1893
1894
1895	/**
1896	* Syncs a guest OS page.
1897	*
1898	* There are no conflicts at this point, neither is there any need for
1899	* page table allocations.
1900	*
1901	* When called in PAE or AMD64 guest mode, the guest PDPE shall be valid.
1902	* When called in AMD64 guest mode, the guest PML4E shall be valid.
1903	*
1904	* @returns VBox status code.
1905	* @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1906	* @param pVCpu The cross context virtual CPU structure.
1907	* @param PdeSrc Page directory entry of the guest.
1908	* @param GCPtrPage Guest context page address.
1909	* @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1910	* @param uErr Fault error (X86_TRAP_PF_*).
1911	*/
1912	static int PGM_BTH_NAME(SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr)
1913	{
1914	PVM pVM = pVCpu->CTX_SUFF(pVM);
1915	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
1916	LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr));
1917	RT_NOREF_PV(uErr); RT_NOREF_PV(cPages); RT_NOREF_PV(GCPtrPage);
1918
1919	PGM_LOCK_ASSERT_OWNER(pVM);
1920
1921	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
1922	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
1923	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
1924	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
1925	&& PGM_SHW_TYPE != PGM_TYPE_EPT
1926
1927	/*
1928	* Assert preconditions.
1929	*/
1930	Assert(PdeSrc.n.u1Present);
1931	Assert(cPages);
1932	# if 0 /* rarely useful; leave for debugging. */
1933	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
1934	# endif
1935
1936	/*
1937	* Get the shadow PDE, find the shadow page table in the pool.
1938	*/
1939	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1940	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1941	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1942
1943	/* Fetch the pgm pool shadow descriptor. */
1944	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1945	Assert(pShwPde);
1946
1947	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1948	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1949	PPGMPOOLPAGE pShwPde = NULL;
1950	PX86PDPAE pPDDst;
1951
1952	/* Fetch the pgm pool shadow descriptor. */
1953	int rc2 = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
1954	AssertRCSuccessReturn(rc2, rc2);
1955	Assert(pShwPde);
1956
1957	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1958	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1959
1960	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1961	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1962	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1963	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
1964	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
1965
1966	int rc2 = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
1967	AssertRCSuccessReturn(rc2, rc2);
1968	Assert(pPDDst && pPdptDst);
1969	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1970	# endif
1971	SHWPDE PdeDst = *pPdeDst;
1972
1973	/*
1974	* - In the guest SMP case we could have blocked while another VCPU reused
1975	* this page table.
1976	* - With W7-64 we may also take this path when the A bit is cleared on
1977	* higher level tables (PDPE/PML4E). The guest does not invalidate the
1978	* relevant TLB entries. If we're write monitoring any page mapped by
1979	* the modified entry, we may end up here with a "stale" TLB entry.
1980	*/
1981	if (!PdeDst.n.u1Present)
1982	{
1983	Log(("CPU%u: SyncPage: Pde at %RGv changed behind our back? (pPdeDst=%p/%RX64) uErr=%#x\n", pVCpu->idCpu, GCPtrPage, pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
1984	AssertMsg(pVM->cCpus > 1 \|\| (uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RW)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RW),
1985	("Unexpected missing PDE p=%p/%RX64 uErr=%#x\n", pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
1986	if (uErr & X86_TRAP_PF_P)
1987	PGM_INVL_PG(pVCpu, GCPtrPage);
1988	return VINF_SUCCESS; /* force the instruction to be executed again. */
1989	}
1990
1991	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1992	Assert(pShwPage);
1993
1994	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1995	/* Fetch the pgm pool shadow descriptor. */
1996	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
1997	Assert(pShwPde);
1998	# endif
1999
2000	/*
2001	* Check that the page is present and that the shadow PDE isn't out of sync.
2002	*/
2003	const bool fBigPage = PdeSrc.b.u1Size && GST_IS_PSE_ACTIVE(pVCpu);
2004	const bool fPdeValid = !fBigPage ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc);
2005	RTGCPHYS GCPhys;
2006	if (!fBigPage)
2007	{
2008	GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
2009	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2010	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2011	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| ((iPDDst & 1) * (PAGE_SIZE / 2)));
2012	# endif
2013	}
2014	else
2015	{
2016	GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
2017	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2018	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2019	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
2020	# endif
2021	}
2022	/** @todo This doesn't check the G bit of 2/4MB pages. FIXME */
2023	if ( fPdeValid
2024	&& pShwPage->GCPhys == GCPhys
2025	&& PdeSrc.n.u1Present
2026	&& PdeSrc.n.u1User == PdeDst.n.u1User
2027	&& (PdeSrc.n.u1Write == PdeDst.n.u1Write \|\| !PdeDst.n.u1Write)
2028	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2029	&& (PdeSrc.n.u1NoExecute == PdeDst.n.u1NoExecute \|\| !GST_IS_NX_ACTIVE(pVCpu))
2030	# endif
2031	)
2032	{
2033	/*
2034	* Check that the PDE is marked accessed already.
2035	* Since we set the accessed bit before getting here on a #PF, this
2036	* check is only meant for dealing with non-#PF'ing paths.
2037	*/
2038	if (PdeSrc.n.u1Accessed)
2039	{
2040	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2041	if (!fBigPage)
2042	{
2043	/*
2044	* 4KB Page - Map the guest page table.
2045	*/
2046	PGSTPT pPTSrc;
2047	int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
2048	if (RT_SUCCESS(rc))
2049	{
2050	# ifdef PGM_SYNC_N_PAGES
2051	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
2052	if ( cPages > 1
2053	&& !(uErr & X86_TRAP_PF_P)
2054	&& !VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY))
2055	{
2056	/*
2057	* This code path is currently only taken when the caller is PGMTrap0eHandler
2058	* for non-present pages!
2059	*
2060	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2061	* deal with locality.
2062	*/
2063	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2064	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2065	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2066	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2067	# else
2068	const unsigned offPTSrc = 0;
2069	# endif
2070	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2071	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2072	iPTDst = 0;
2073	else
2074	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2075
2076	for (; iPTDst < iPTDstEnd; iPTDst++)
2077	{
2078	const PGSTPTE pPteSrc = &pPTSrc->a[offPTSrc + iPTDst];
2079
2080	if ( pPteSrc->n.u1Present
2081	&& !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2082	{
2083	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
2084	NOREF(GCPtrCurPage);
2085	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
2086	/*
2087	* Assuming kernel code will be marked as supervisor - and not as user level
2088	* and executed using a conforming code selector - And marked as readonly.
2089	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
2090	*/
2091	PPGMPAGE pPage;
2092	if ( ((PdeSrc.u & pPteSrc->u) & (X86_PTE_RW \| X86_PTE_US))
2093	\|\| iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK) /* always sync GCPtrPage */
2094	\|\| !CSAMDoesPageNeedScanning(pVM, GCPtrCurPage)
2095	\|\| ( (pPage = pgmPhysGetPage(pVM, pPteSrc->u & GST_PTE_PG_MASK))
2096	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2097	)
2098	# endif /* else: CSAM not active */
2099	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, *pPteSrc, pShwPage, iPTDst);
2100	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
2101	GCPtrCurPage, pPteSrc->n.u1Present,
2102	pPteSrc->n.u1Write & PdeSrc.n.u1Write,
2103	pPteSrc->n.u1User & PdeSrc.n.u1User,
2104	(uint64_t)pPteSrc->u,
2105	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2106	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2107	}
2108	}
2109	}
2110	else
2111	# endif /* PGM_SYNC_N_PAGES */
2112	{
2113	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
2114	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2115	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2116	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2117	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
2118	GCPtrPage, PteSrc.n.u1Present,
2119	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2120	PteSrc.n.u1User & PdeSrc.n.u1User,
2121	(uint64_t)PteSrc.u,
2122	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2123	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2124	}
2125	}
2126	else /* MMIO or invalid page: emulated in #PF handler. */
2127	{
2128	LogFlow(("PGM_GCPHYS_2_PTR %RGp failed with %Rrc\n", GCPhys, rc));
2129	Assert(!SHW_PTE_IS_P(pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK]));
2130	}
2131	}
2132	else
2133	{
2134	/*
2135	* 4/2MB page - lazy syncing shadow 4K pages.
2136	* (There are many causes of getting here, it's no longer only CSAM.)
2137	*/
2138	/* Calculate the GC physical address of this 4KB shadow page. */
2139	GCPhys = PGM_A20_APPLY(pVCpu, GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc) \| (GCPtrPage & GST_BIG_PAGE_OFFSET_MASK));
2140	/* Find ram range. */
2141	PPGMPAGE pPage;
2142	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
2143	if (RT_SUCCESS(rc))
2144	{
2145	AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
2146
2147	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2148	/* Try to make the page writable if necessary. */
2149	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2150	&& ( PGM_PAGE_IS_ZERO(pPage)
2151	\|\| ( PdeSrc.n.u1Write
2152	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2153	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
2154	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
2155	# endif
2156	# ifdef VBOX_WITH_PAGE_SHARING
2157	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
2158	# endif
2159	)
2160	)
2161	)
2162	{
2163	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
2164	AssertRC(rc);
2165	}
2166	# endif
2167
2168	/*
2169	* Make shadow PTE entry.
2170	*/
2171	SHWPTE PteDst;
2172	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2173	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc), &PteDst);
2174	else
2175	SHW_PTE_SET(PteDst, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc) \| PGM_PAGE_GET_HCPHYS(pPage));
2176
2177	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2178	if ( SHW_PTE_IS_P(PteDst)
2179	&& !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2180	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2181
2182	/* Make sure only allocated pages are mapped writable. */
2183	if ( SHW_PTE_IS_P_RW(PteDst)
2184	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2185	{
2186	/* Still applies to shared pages. */
2187	Assert(!PGM_PAGE_IS_ZERO(pPage));
2188	SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
2189	Log3(("SyncPage: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, GCPtrPage));
2190	}
2191
2192	SHW_PTE_ATOMIC_SET2(pPTDst->a[iPTDst], PteDst);
2193
2194	/*
2195	* If the page is not flagged as dirty and is writable, then make it read-only
2196	* at PD level, so we can set the dirty bit when the page is modified.
2197	*
2198	* ASSUMES that page access handlers are implemented on page table entry level.
2199	* Thus we will first catch the dirty access and set PDE.D and restart. If
2200	* there is an access handler, we'll trap again and let it work on the problem.
2201	*/
2202	/** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
2203	* As for invlpg, it simply frees the whole shadow PT.
2204	* ...It's possibly because the guest clears it and the guest doesn't really tell us... */
2205	if ( !PdeSrc.b.u1Dirty
2206	&& PdeSrc.b.u1Write)
2207	{
2208	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2209	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2210	PdeDst.n.u1Write = 0;
2211	}
2212	else
2213	{
2214	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2215	PdeDst.n.u1Write = PdeSrc.n.u1Write;
2216	}
2217	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2218	Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n",
2219	GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
2220	PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2221	}
2222	else
2223	{
2224	LogFlow(("PGM_GCPHYS_2_PTR %RGp (big) failed with %Rrc\n", GCPhys, rc));
2225	/** @todo must wipe the shadow page table entry in this
2226	* case. */
2227	}
2228	}
2229	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2230	return VINF_SUCCESS;
2231	}
2232
2233	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPagePDNAs));
2234	}
2235	else if (fPdeValid)
2236	{
2237	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPagePDOutOfSync));
2238	Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2239	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2240	}
2241	else
2242	{
2243	/// @todo STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSyncAndInvalid));
2244	Log2(("SyncPage: Bad PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2245	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2246	}
2247
2248	/*
2249	* Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
2250	* Yea, I'm lazy.
2251	*/
2252	pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
2253	ASMAtomicWriteSize(pPdeDst, 0);
2254
2255	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2256	PGM_INVL_VCPU_TLBS(pVCpu);
2257	return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
2258
2259
2260	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2261	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2262	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2263	&& !defined(IN_RC)
2264	NOREF(PdeSrc);
2265
2266	# ifdef PGM_SYNC_N_PAGES
2267	/*
2268	* Get the shadow PDE, find the shadow page table in the pool.
2269	*/
2270	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2271	X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
2272
2273	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2274	X86PDEPAE PdeDst = pgmShwGetPaePDE(pVCpu, GCPtrPage);
2275
2276	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2277	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2278	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpt);
2279	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2280	X86PDEPAE PdeDst;
2281	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2282
2283	int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2284	AssertRCSuccessReturn(rc, rc);
2285	Assert(pPDDst && pPdptDst);
2286	PdeDst = pPDDst->a[iPDDst];
2287	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2288	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2289	PEPTPD pPDDst;
2290	EPTPDE PdeDst;
2291
2292	int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, NULL, &pPDDst);
2293	if (rc != VINF_SUCCESS)
2294	{
2295	AssertRC(rc);
2296	return rc;
2297	}
2298	Assert(pPDDst);
2299	PdeDst = pPDDst->a[iPDDst];
2300	# endif
2301	/* In the guest SMP case we could have blocked while another VCPU reused this page table. */
2302	if (!PdeDst.n.u1Present)
2303	{
2304	AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)PdeDst.u));
2305	Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", pVCpu->idCpu, GCPtrPage));
2306	return VINF_SUCCESS; /* force the instruction to be executed again. */
2307	}
2308
2309	/* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */
2310	if (PdeDst.n.u1Size)
2311	{
2312	Assert(pVM->pgm.s.fNestedPaging);
2313	Log(("CPU%d: SyncPage: Pde (big:%RX64) at %RGv changed behind our back!\n", pVCpu->idCpu, PdeDst.u, GCPtrPage));
2314	return VINF_SUCCESS;
2315	}
2316
2317	/* Mask away the page offset. */
2318	GCPtrPage &= ~((RTGCPTR)0xfff);
2319
2320	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2321	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2322
2323	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
2324	if ( cPages > 1
2325	&& !(uErr & X86_TRAP_PF_P)
2326	&& !VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY))
2327	{
2328	/*
2329	* This code path is currently only taken when the caller is PGMTrap0eHandler
2330	* for non-present pages!
2331	*
2332	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2333	* deal with locality.
2334	*/
2335	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2336	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2337	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2338	iPTDst = 0;
2339	else
2340	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2341	for (; iPTDst < iPTDstEnd; iPTDst++)
2342	{
2343	if (!SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2344	{
2345	RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
2346	\| (iPTDst << PAGE_SHIFT));
2347
2348	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
2349	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
2350	GCPtrCurPage,
2351	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2352	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2353
2354	if (RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)))
2355	break;
2356	}
2357	else
2358	Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n", (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT), iPTDst, SHW_PTE_LOG64(pPTDst->a[iPTDst]) ));
2359	}
2360	}
2361	else
2362	# endif /* PGM_SYNC_N_PAGES */
2363	{
2364	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2365	RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
2366	\| (iPTDst << PAGE_SHIFT));
2367
2368	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
2369
2370	Log2(("SyncPage: 4K %RGv PteSrc:{P=1 RW=1 U=1}PteDst=%08llx%s\n",
2371	GCPtrPage,
2372	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2373	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2374	}
2375	return VINF_SUCCESS;
2376
2377	#else
2378	NOREF(PdeSrc);
2379	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2380	return VERR_PGM_NOT_USED_IN_MODE;
2381	#endif
2382	}
2383
2384
2385	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2386
2387	/**
2388	* CheckPageFault helper for returning a page fault indicating a non-present
2389	* (NP) entry in the page translation structures.
2390	*
2391	* @returns VINF_EM_RAW_GUEST_TRAP.
2392	* @param pVCpu The cross context virtual CPU structure.
2393	* @param uErr The error code of the shadow fault. Corrections to
2394	* TRPM's copy will be made if necessary.
2395	* @param GCPtrPage For logging.
2396	* @param uPageFaultLevel For logging.
2397	*/
2398	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnNP)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2399	{
2400	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2401	AssertMsg(!(uErr & X86_TRAP_PF_P), ("%#x\n", uErr));
2402	AssertMsg(!(uErr & X86_TRAP_PF_RSVD), ("%#x\n", uErr));
2403	if (uErr & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
2404	TRPMSetErrorCode(pVCpu, uErr & ~(X86_TRAP_PF_RSVD \| X86_TRAP_PF_P));
2405
2406	Log(("CheckPageFault: real page fault (notp) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2407	RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(uPageFaultLevel);
2408	return VINF_EM_RAW_GUEST_TRAP;
2409	}
2410
2411
2412	/**
2413	* CheckPageFault helper for returning a page fault indicating a reserved bit
2414	* (RSVD) error in the page translation structures.
2415	*
2416	* @returns VINF_EM_RAW_GUEST_TRAP.
2417	* @param pVCpu The cross context virtual CPU structure.
2418	* @param uErr The error code of the shadow fault. Corrections to
2419	* TRPM's copy will be made if necessary.
2420	* @param GCPtrPage For logging.
2421	* @param uPageFaultLevel For logging.
2422	*/
2423	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnRSVD)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2424	{
2425	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2426	if ((uErr & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) != (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
2427	TRPMSetErrorCode(pVCpu, uErr \| X86_TRAP_PF_RSVD \| X86_TRAP_PF_P);
2428
2429	Log(("CheckPageFault: real page fault (rsvd) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2430	RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(uPageFaultLevel);
2431	return VINF_EM_RAW_GUEST_TRAP;
2432	}
2433
2434
2435	/**
2436	* CheckPageFault helper for returning a page protection fault (P).
2437	*
2438	* @returns VINF_EM_RAW_GUEST_TRAP.
2439	* @param pVCpu The cross context virtual CPU structure.
2440	* @param uErr The error code of the shadow fault. Corrections to
2441	* TRPM's copy will be made if necessary.
2442	* @param GCPtrPage For logging.
2443	* @param uPageFaultLevel For logging.
2444	*/
2445	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnProt)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2446	{
2447	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2448	AssertMsg(uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID), ("%#x\n", uErr));
2449	if ((uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) != X86_TRAP_PF_P)
2450	TRPMSetErrorCode(pVCpu, (uErr & ~X86_TRAP_PF_RSVD) \| X86_TRAP_PF_P);
2451
2452	Log(("CheckPageFault: real page fault (prot) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2453	RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(uPageFaultLevel);
2454	return VINF_EM_RAW_GUEST_TRAP;
2455	}
2456
2457
2458	/**
2459	* Handle dirty bit tracking faults.
2460	*
2461	* @returns VBox status code.
2462	* @param pVCpu The cross context virtual CPU structure.
2463	* @param uErr Page fault error code.
2464	* @param pPdeSrc Guest page directory entry.
2465	* @param pPdeDst Shadow page directory entry.
2466	* @param GCPtrPage Guest context page address.
2467	*/
2468	static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc,
2469	RTGCPTR GCPtrPage)
2470	{
2471	PVM pVM = pVCpu->CTX_SUFF(pVM);
2472	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2473	NOREF(uErr);
2474
2475	PGM_LOCK_ASSERT_OWNER(pVM);
2476
2477	/*
2478	* Handle big page.
2479	*/
2480	if (pPdeSrc->b.u1Size && GST_IS_PSE_ACTIVE(pVCpu))
2481	{
2482	if ( pPdeDst->n.u1Present
2483	&& (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY))
2484	{
2485	SHWPDE PdeDst = *pPdeDst;
2486
2487	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageTrap));
2488	Assert(pPdeSrc->b.u1Write);
2489
2490	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
2491	* fault again and take this path to only invalidate the entry (see below).
2492	*/
2493	PdeDst.n.u1Write = 1;
2494	PdeDst.n.u1Accessed = 1;
2495	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2496	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2497	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
2498	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2499	}
2500
2501	# ifdef IN_RING0
2502	/* Check for stale TLB entry; only applies to the SMP guest case. */
2503	if ( pVM->cCpus > 1
2504	&& pPdeDst->n.u1Write
2505	&& pPdeDst->n.u1Accessed)
2506	{
2507	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2508	if (pShwPage)
2509	{
2510	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2511	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2512	if (SHW_PTE_IS_P_RW(*pPteDst))
2513	{
2514	/* Stale TLB entry. */
2515	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageStale));
2516	PGM_INVL_PG(pVCpu, GCPtrPage);
2517	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2518	}
2519	}
2520	}
2521	# endif /* IN_RING0 */
2522	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2523	}
2524
2525	/*
2526	* Map the guest page table.
2527	*/
2528	PGSTPT pPTSrc;
2529	int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(*pPdeSrc), &pPTSrc);
2530	if (RT_FAILURE(rc))
2531	{
2532	AssertRC(rc);
2533	return rc;
2534	}
2535
2536	if (pPdeDst->n.u1Present)
2537	{
2538	GSTPTE const *pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2539	const GSTPTE PteSrc = *pPteSrc;
2540
2541	#ifdef VBOX_WITH_RAW_MODE_NOT_R0
2542	/* Bail out here as pgmPoolGetPage will return NULL and we'll crash below.
2543	* Our individual shadow handlers will provide more information and force a fatal exit.
2544	*/
2545	if ( !HMIsEnabled(pVM)
2546	&& MMHyperIsInsideArea(pVM, (RTGCPTR)GCPtrPage))
2547	{
2548	LogRel(("CheckPageFault: write to hypervisor region %RGv\n", GCPtrPage));
2549	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2550	}
2551	#endif
2552	/*
2553	* Map shadow page table.
2554	*/
2555	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2556	if (pShwPage)
2557	{
2558	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2559	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2560	if (SHW_PTE_IS_P(pPteDst)) /* @todo Optimize accessed bit emulation? */
2561	{
2562	if (SHW_PTE_IS_TRACK_DIRTY(*pPteDst))
2563	{
2564	PPGMPAGE pPage = pgmPhysGetPage(pVM, GST_GET_PTE_GCPHYS(PteSrc));
2565	SHWPTE PteDst = *pPteDst;
2566
2567	LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage));
2568	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageTrap));
2569
2570	Assert(PteSrc.n.u1Write);
2571
2572	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB
2573	* entry will not harm; write access will simply fault again and
2574	* take this path to only invalidate the entry.
2575	*/
2576	if (RT_LIKELY(pPage))
2577	{
2578	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2579	{
2580	//AssertMsgFailed(("%R[pgmpage] - we don't set PGM_PTFLAGS_TRACK_DIRTY for these pages\n", pPage));
2581	Assert(!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage));
2582	/* Assuming write handlers here as the PTE is present (otherwise we wouldn't be here). */
2583	SHW_PTE_SET_RO(PteDst);
2584	}
2585	else
2586	{
2587	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
2588	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
2589	{
2590	rc = pgmPhysPageMakeWritable(pVM, pPage, GST_GET_PTE_GCPHYS(PteSrc));
2591	AssertRC(rc);
2592	}
2593	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED)
2594	SHW_PTE_SET_RW(PteDst);
2595	else
2596	{
2597	/* Still applies to shared pages. */
2598	Assert(!PGM_PAGE_IS_ZERO(pPage));
2599	SHW_PTE_SET_RO(PteDst);
2600	}
2601	}
2602	}
2603	else
2604	SHW_PTE_SET_RW(PteDst); /** @todo r=bird: This doesn't make sense to me. */
2605
2606	SHW_PTE_SET(PteDst, (SHW_PTE_GET_U(PteDst) \| X86_PTE_D \| X86_PTE_A) & ~(uint64_t)PGM_PTFLAGS_TRACK_DIRTY);
2607	SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
2608	PGM_INVL_PG(pVCpu, GCPtrPage);
2609	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2610	}
2611
2612	# ifdef IN_RING0
2613	/* Check for stale TLB entry; only applies to the SMP guest case. */
2614	if ( pVM->cCpus > 1
2615	&& SHW_PTE_IS_RW(*pPteDst)
2616	&& SHW_PTE_IS_A(*pPteDst))
2617	{
2618	/* Stale TLB entry. */
2619	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageStale));
2620	PGM_INVL_PG(pVCpu, GCPtrPage);
2621	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2622	}
2623	# endif
2624	}
2625	}
2626	else
2627	AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
2628	}
2629
2630	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2631	}
2632
2633	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
2634
2635
2636	/**
2637	* Sync a shadow page table.
2638	*
2639	* The shadow page table is not present in the shadow PDE.
2640	*
2641	* Handles mapping conflicts.
2642	*
2643	* This is called by VerifyAccessSyncPage, PrefetchPage, InvalidatePage (on
2644	* conflict), and Trap0eHandler.
2645	*
2646	* A precondition for this method is that the shadow PDE is not present. The
2647	* caller must take the PGM lock before checking this and continue to hold it
2648	* when calling this method.
2649	*
2650	* @returns VBox status code.
2651	* @param pVCpu The cross context virtual CPU structure.
2652	* @param iPDSrc Page directory index.
2653	* @param pPDSrc Source page directory (i.e. Guest OS page directory).
2654	* Assume this is a temporary mapping.
2655	* @param GCPtrPage GC Pointer of the page that caused the fault
2656	*/
2657	static int PGM_BTH_NAME(SyncPT)(PVMCPU pVCpu, unsigned iPDSrc, PGSTPD pPDSrc, RTGCPTR GCPtrPage)
2658	{
2659	PVM pVM = pVCpu->CTX_SUFF(pVM);
2660	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
2661
2662	#if 0 /* rarely useful; leave for debugging. */
2663	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPtPD[iPDSrc]);
2664	#endif
2665	LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage)); RT_NOREF_PV(GCPtrPage);
2666
2667	PGM_LOCK_ASSERT_OWNER(pVM);
2668
2669	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2670	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
2671	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
2672	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2673	&& PGM_SHW_TYPE != PGM_TYPE_EPT
2674
2675	int rc = VINF_SUCCESS;
2676
2677	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2678
2679	/*
2680	* Some input validation first.
2681	*/
2682	AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
2683
2684	/*
2685	* Get the relevant shadow PDE entry.
2686	*/
2687	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2688	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2689	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
2690
2691	/* Fetch the pgm pool shadow descriptor. */
2692	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2693	Assert(pShwPde);
2694
2695	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2696	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2697	PPGMPOOLPAGE pShwPde = NULL;
2698	PX86PDPAE pPDDst;
2699	PSHWPDE pPdeDst;
2700
2701	/* Fetch the pgm pool shadow descriptor. */
2702	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
2703	AssertRCSuccessReturn(rc, rc);
2704	Assert(pShwPde);
2705
2706	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
2707	pPdeDst = &pPDDst->a[iPDDst];
2708
2709	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2710	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2711	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2712	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2713	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2714	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2715	AssertRCSuccessReturn(rc, rc);
2716	Assert(pPDDst);
2717	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2718	# endif
2719	SHWPDE PdeDst = *pPdeDst;
2720
2721	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2722	/* Fetch the pgm pool shadow descriptor. */
2723	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2724	Assert(pShwPde);
2725	# endif
2726
2727	# ifndef PGM_WITHOUT_MAPPINGS
2728	/*
2729	* Check for conflicts.
2730	* RC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
2731	* R3: Simply resolve the conflict.
2732	*/
2733	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2734	{
2735	Assert(pgmMapAreMappingsEnabled(pVM));
2736	# ifndef IN_RING3
2737	Log(("SyncPT: Conflict at %RGv\n", GCPtrPage));
2738	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2739	return VERR_ADDRESS_CONFLICT;
2740
2741	# else /* IN_RING3 */
2742	PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
2743	Assert(pMapping);
2744	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2745	rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2746	# elif PGM_GST_TYPE == PGM_TYPE_PAE
2747	rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2748	# else
2749	AssertFailed(); NOREF(pMapping); /* can't happen for amd64 */
2750	# endif
2751	if (RT_FAILURE(rc))
2752	{
2753	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2754	return rc;
2755	}
2756	PdeDst = *pPdeDst;
2757	# endif /* IN_RING3 */
2758	}
2759	# endif /* !PGM_WITHOUT_MAPPINGS */
2760	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2761
2762	/*
2763	* Sync the page directory entry.
2764	*/
2765	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2766	const bool fPageTable = !PdeSrc.b.u1Size \|\| !GST_IS_PSE_ACTIVE(pVCpu);
2767	if ( PdeSrc.n.u1Present
2768	&& (fPageTable ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc)) )
2769	{
2770	/*
2771	* Allocate & map the page table.
2772	*/
2773	PSHWPT pPTDst;
2774	PPGMPOOLPAGE pShwPage;
2775	RTGCPHYS GCPhys;
2776	if (fPageTable)
2777	{
2778	GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
2779	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2780	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2781	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| ((iPDDst & 1) * (PAGE_SIZE / 2)));
2782	# endif
2783	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
2784	pShwPde->idx, iPDDst, false /fLockPage/,
2785	&pShwPage);
2786	}
2787	else
2788	{
2789	PGMPOOLACCESS enmAccess;
2790	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2791	const bool fNoExecute = PdeSrc.n.u1NoExecute && GST_IS_NX_ACTIVE(pVCpu);
2792	# else
2793	const bool fNoExecute = false;
2794	# endif
2795
2796	GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
2797	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2798	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2799	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
2800	# endif
2801	/* Determine the right kind of large page to avoid incorrect cached entry reuse. */
2802	if (PdeSrc.n.u1User)
2803	{
2804	if (PdeSrc.n.u1Write)
2805	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_RW_NX : PGMPOOLACCESS_USER_RW;
2806	else
2807	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_R_NX : PGMPOOLACCESS_USER_R;
2808	}
2809	else
2810	{
2811	if (PdeSrc.n.u1Write)
2812	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW;
2813	else
2814	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_R_NX : PGMPOOLACCESS_SUPERVISOR_R;
2815	}
2816	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, enmAccess, PGM_A20_IS_ENABLED(pVCpu),
2817	pShwPde->idx, iPDDst, false /fLockPage/,
2818	&pShwPage);
2819	}
2820	if (rc == VINF_SUCCESS)
2821	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2822	else if (rc == VINF_PGM_CACHED_PAGE)
2823	{
2824	/*
2825	* The PT was cached, just hook it up.
2826	*/
2827	if (fPageTable)
2828	PdeDst.u = pShwPage->Core.Key \| GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2829	else
2830	{
2831	PdeDst.u = pShwPage->Core.Key \| GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2832	/* (see explanation and assumptions further down.) */
2833	if ( !PdeSrc.b.u1Dirty
2834	&& PdeSrc.b.u1Write)
2835	{
2836	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2837	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2838	PdeDst.b.u1Write = 0;
2839	}
2840	}
2841	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2842	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2843	return VINF_SUCCESS;
2844	}
2845	else if (rc == VERR_PGM_POOL_FLUSHED)
2846	{
2847	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2848	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2849	return VINF_PGM_SYNC_CR3;
2850	}
2851	else
2852	AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
2853	/** @todo Why do we bother preserving X86_PDE_AVL_MASK here?
2854	* Both PGM_PDFLAGS_MAPPING and PGM_PDFLAGS_TRACK_DIRTY should be
2855	* irrelevant at this point. */
2856	PdeDst.u &= X86_PDE_AVL_MASK;
2857	PdeDst.u \|= pShwPage->Core.Key;
2858
2859	/*
2860	* Page directory has been accessed (this is a fault situation, remember).
2861	*/
2862	/** @todo
2863	* Well, when the caller is PrefetchPage or InvalidatePage is isn't a
2864	* fault situation. What's more, the Trap0eHandler has already set the
2865	* accessed bit. So, it's actually just VerifyAccessSyncPage which
2866	* might need setting the accessed flag.
2867	*
2868	* The best idea is to leave this change to the caller and add an
2869	* assertion that it's set already. */
2870	pPDSrc->a[iPDSrc].n.u1Accessed = 1;
2871	if (fPageTable)
2872	{
2873	/*
2874	* Page table - 4KB.
2875	*
2876	* Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
2877	*/
2878	Log2(("SyncPT: 4K %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
2879	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
2880	PGSTPT pPTSrc;
2881	rc = PGM_GCPHYS_2_PTR(pVM, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
2882	if (RT_SUCCESS(rc))
2883	{
2884	/*
2885	* Start by syncing the page directory entry so CSAM's TLB trick works.
2886	*/
2887	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| X86_PDE_AVL_MASK))
2888	\| GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2889	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2890	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2891
2892	/*
2893	* Directory/page user or supervisor privilege: (same goes for read/write)
2894	*
2895	* Directory Page Combined
2896	* U/S U/S U/S
2897	* 0 0 0
2898	* 0 1 0
2899	* 1 0 0
2900	* 1 1 1
2901	*
2902	* Simple AND operation. Table listed for completeness.
2903	*
2904	*/
2905	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT4K));
2906	# ifdef PGM_SYNC_N_PAGES
2907	unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2908	unsigned iPTDst = iPTBase;
2909	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2910	if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
2911	iPTDst = 0;
2912	else
2913	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2914	# else /* !PGM_SYNC_N_PAGES */
2915	unsigned iPTDst = 0;
2916	const unsigned iPTDstEnd = RT_ELEMENTS(pPTDst->a);
2917	# endif /* !PGM_SYNC_N_PAGES */
2918	RTGCPTR GCPtrCur = (GCPtrPage & ~(RTGCPTR)((1 << SHW_PD_SHIFT) - 1))
2919	\| ((RTGCPTR)iPTDst << PAGE_SHIFT);
2920	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2921	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2922	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2923	# else
2924	const unsigned offPTSrc = 0;
2925	# endif
2926	for (; iPTDst < iPTDstEnd; iPTDst++, GCPtrCur += PAGE_SIZE)
2927	{
2928	const unsigned iPTSrc = iPTDst + offPTSrc;
2929	const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2930
2931	if (PteSrc.n.u1Present)
2932	{
2933	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
2934	/*
2935	* Assuming kernel code will be marked as supervisor - and not as user level
2936	* and executed using a conforming code selector - And marked as readonly.
2937	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
2938	*/
2939	PPGMPAGE pPage;
2940	if ( ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW \| X86_PTE_US))
2941	\|\| !CSAMDoesPageNeedScanning(pVM, GCPtrCur)
2942	\|\| ( (pPage = pgmPhysGetPage(pVM, GST_GET_PTE_GCPHYS(PteSrc)))
2943	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2944	)
2945	# endif
2946	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2947	Log2(("SyncPT: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
2948	GCPtrCur,
2949	PteSrc.n.u1Present,
2950	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2951	PteSrc.n.u1User & PdeSrc.n.u1User,
2952	(uint64_t)PteSrc.u,
2953	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : "", SHW_PTE_LOG64(pPTDst->a[iPTDst]), iPTSrc, PdeSrc.au32[0],
2954	(RTGCPHYS)(GST_GET_PDE_GCPHYS(PdeSrc) + iPTSrc*sizeof(PteSrc)) ));
2955	}
2956	/* else: the page table was cleared by the pool */
2957	} /* for PTEs */
2958	}
2959	}
2960	else
2961	{
2962	/*
2963	* Big page - 2/4MB.
2964	*
2965	* We'll walk the ram range list in parallel and optimize lookups.
2966	* We will only sync one shadow page table at a time.
2967	*/
2968	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT4M));
2969
2970	/**
2971	* @todo It might be more efficient to sync only a part of the 4MB
2972	* page (similar to what we do for 4KB PDs).
2973	*/
2974
2975	/*
2976	* Start by syncing the page directory entry.
2977	*/
2978	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
2979	\| GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2980
2981	/*
2982	* If the page is not flagged as dirty and is writable, then make it read-only
2983	* at PD level, so we can set the dirty bit when the page is modified.
2984	*
2985	* ASSUMES that page access handlers are implemented on page table entry level.
2986	* Thus we will first catch the dirty access and set PDE.D and restart. If
2987	* there is an access handler, we'll trap again and let it work on the problem.
2988	*/
2989	/** @todo move the above stuff to a section in the PGM documentation. */
2990	Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
2991	if ( !PdeSrc.b.u1Dirty
2992	&& PdeSrc.b.u1Write)
2993	{
2994	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2995	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2996	PdeDst.b.u1Write = 0;
2997	}
2998	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2999	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3000
3001	/*
3002	* Fill the shadow page table.
3003	*/
3004	/* Get address and flags from the source PDE. */
3005	SHWPTE PteDstBase;
3006	SHW_PTE_SET(PteDstBase, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc));
3007
3008	/* Loop thru the entries in the shadow PT. */
3009	const RTGCPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
3010	Log2(("SyncPT: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n",
3011	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
3012	GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
3013	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
3014	unsigned iPTDst = 0;
3015	while ( iPTDst < RT_ELEMENTS(pPTDst->a)
3016	&& !VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY))
3017	{
3018	if (pRam && GCPhys >= pRam->GCPhys)
3019	{
3020	# ifndef PGM_WITH_A20
3021	unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
3022	# endif
3023	do
3024	{
3025	/* Make shadow PTE. */
3026	# ifdef PGM_WITH_A20
3027	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
3028	# else
3029	PPGMPAGE pPage = &pRam->aPages[iHCPage];
3030	# endif
3031	SHWPTE PteDst;
3032
3033	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
3034	/* Try to make the page writable if necessary. */
3035	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
3036	&& ( PGM_PAGE_IS_ZERO(pPage)
3037	\|\| ( SHW_PTE_IS_RW(PteDstBase)
3038	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
3039	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
3040	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
3041	# endif
3042	# ifdef VBOX_WITH_PAGE_SHARING
3043	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
3044	# endif
3045	&& !PGM_PAGE_IS_BALLOONED(pPage))
3046	)
3047	)
3048	{
3049	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
3050	AssertRCReturn(rc, rc);
3051	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY))
3052	break;
3053	}
3054	# endif
3055
3056	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
3057	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, SHW_PTE_GET_U(PteDstBase), &PteDst);
3058	else if (PGM_PAGE_IS_BALLOONED(pPage))
3059	SHW_PTE_SET(PteDst, 0); /* Handle ballooned pages at #PF time. */
3060	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
3061	/*
3062	* Assuming kernel code will be marked as supervisor and not as user level and executed
3063	* using a conforming code selector. Don't check for readonly, as that implies the whole
3064	* 4MB can be code or readonly data. Linux enables write access for its large pages.
3065	*/
3066	else if ( !PdeSrc.n.u1User
3067	&& CSAMDoesPageNeedScanning(pVM, GCPtr \| (iPTDst << SHW_PT_SHIFT)))
3068	SHW_PTE_SET(PteDst, 0);
3069	# endif
3070	else
3071	SHW_PTE_SET(PteDst, PGM_PAGE_GET_HCPHYS(pPage) \| SHW_PTE_GET_U(PteDstBase));
3072
3073	/* Only map writable pages writable. */
3074	if ( SHW_PTE_IS_P_RW(PteDst)
3075	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
3076	{
3077	/* Still applies to shared pages. */
3078	Assert(!PGM_PAGE_IS_ZERO(pPage));
3079	SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
3080	Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))));
3081	}
3082
3083	if (SHW_PTE_IS_P(PteDst))
3084	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
3085
3086	/* commit it (not atomic, new table) */
3087	pPTDst->a[iPTDst] = PteDst;
3088	Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
3089	(RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT)), SHW_PTE_IS_P(PteDst), SHW_PTE_IS_RW(PteDst), SHW_PTE_IS_US(PteDst), SHW_PTE_LOG64(PteDst),
3090	SHW_PTE_IS_TRACK_DIRTY(PteDst) ? " Track-Dirty" : ""));
3091
3092	/* advance */
3093	GCPhys += PAGE_SIZE;
3094	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
3095	# ifndef PGM_WITH_A20
3096	iHCPage++;
3097	# endif
3098	iPTDst++;
3099	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
3100	&& GCPhys <= pRam->GCPhysLast);
3101
3102	/* Advance ram range list. */
3103	while (pRam && GCPhys > pRam->GCPhysLast)
3104	pRam = pRam->CTX_SUFF(pNext);
3105	}
3106	else if (pRam)
3107	{
3108	Log(("Invalid pages at %RGp\n", GCPhys));
3109	do
3110	{
3111	SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
3112	GCPhys += PAGE_SIZE;
3113	iPTDst++;
3114	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
3115	&& GCPhys < pRam->GCPhys);
3116	PGM_A20_APPLY_TO_VAR(pVCpu,GCPhys);
3117	}
3118	else
3119	{
3120	Log(("Invalid pages at %RGp (2)\n", GCPhys));
3121	for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
3122	SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
3123	}
3124	} /* while more PTEs */
3125	} /* 4KB / 4MB */
3126	}
3127	else
3128	AssertRelease(!PdeDst.n.u1Present);
3129
3130	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3131	if (RT_FAILURE(rc))
3132	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPTFailed));
3133	return rc;
3134
3135	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
3136	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
3137	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
3138	&& !defined(IN_RC)
3139	NOREF(iPDSrc); NOREF(pPDSrc);
3140
3141	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3142
3143	/*
3144	* Validate input a little bit.
3145	*/
3146	int rc = VINF_SUCCESS;
3147	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3148	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3149	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3150
3151	/* Fetch the pgm pool shadow descriptor. */
3152	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
3153	Assert(pShwPde);
3154
3155	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3156	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3157	PPGMPOOLPAGE pShwPde = NULL; /* initialized to shut up gcc */
3158	PX86PDPAE pPDDst;
3159	PSHWPDE pPdeDst;
3160
3161	/* Fetch the pgm pool shadow descriptor. */
3162	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
3163	AssertRCSuccessReturn(rc, rc);
3164	Assert(pShwPde);
3165
3166	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
3167	pPdeDst = &pPDDst->a[iPDDst];
3168
3169	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3170	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3171	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3172	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
3173	PX86PDPT pPdptDst= NULL; /* initialized to shut up gcc */
3174	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
3175	AssertRCSuccessReturn(rc, rc);
3176	Assert(pPDDst);
3177	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3178
3179	/* Fetch the pgm pool shadow descriptor. */
3180	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
3181	Assert(pShwPde);
3182
3183	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
3184	const unsigned iPdpt = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
3185	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3186	PEPTPD pPDDst;
3187	PEPTPDPT pPdptDst;
3188
3189	rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, &pPdptDst, &pPDDst);
3190	if (rc != VINF_SUCCESS)
3191	{
3192	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3193	AssertRC(rc);
3194	return rc;
3195	}
3196	Assert(pPDDst);
3197	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3198
3199	/* Fetch the pgm pool shadow descriptor. */
3200	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & EPT_PDPTE_PG_MASK);
3201	Assert(pShwPde);
3202	# endif
3203	SHWPDE PdeDst = *pPdeDst;
3204
3205	Assert(!(PdeDst.u & PGM_PDFLAGS_MAPPING));
3206	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
3207
3208	# if defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
3209	if (BTH_IS_NP_ACTIVE(pVM))
3210	{
3211	/* Check if we allocated a big page before for this 2 MB range. */
3212	PPGMPAGE pPage;
3213	rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PDE2M_PAE_PG_MASK), &pPage);
3214	if (RT_SUCCESS(rc))
3215	{
3216	RTHCPHYS HCPhys = NIL_RTHCPHYS;
3217	if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
3218	{
3219	if (PGM_A20_IS_ENABLED(pVCpu))
3220	{
3221	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageReused);
3222	AssertRelease(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3223	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3224	}
3225	else
3226	{
3227	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PDE_DISABLED);
3228	pVM->pgm.s.cLargePagesDisabled++;
3229	}
3230	}
3231	else if ( PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED
3232	&& PGM_A20_IS_ENABLED(pVCpu))
3233	{
3234	/* Recheck the entire 2 MB range to see if we can use it again as a large page. */
3235	rc = pgmPhysRecheckLargePage(pVM, GCPtrPage, pPage);
3236	if (RT_SUCCESS(rc))
3237	{
3238	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3239	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3240	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3241	}
3242	}
3243	else if ( PGMIsUsingLargePages(pVM)
3244	&& PGM_A20_IS_ENABLED(pVCpu))
3245	{
3246	rc = pgmPhysAllocLargePage(pVM, GCPtrPage);
3247	if (RT_SUCCESS(rc))
3248	{
3249	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3250	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3251	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3252	}
3253	else
3254	LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
3255	}
3256
3257	if (HCPhys != NIL_RTHCPHYS)
3258	{
3259	PdeDst.u &= X86_PDE_AVL_MASK;
3260	PdeDst.u \|= HCPhys;
3261	PdeDst.n.u1Present = 1;
3262	PdeDst.n.u1Write = 1;
3263	PdeDst.b.u1Size = 1;
3264	# if PGM_SHW_TYPE == PGM_TYPE_EPT
3265	PdeDst.n.u1Execute = 1;
3266	PdeDst.b.u1IgnorePAT = 1;
3267	PdeDst.b.u3EMT = VMX_EPT_MEMTYPE_WB;
3268	# else
3269	PdeDst.n.u1User = 1;
3270	# endif
3271	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
3272
3273	Log(("SyncPT: Use large page at %RGp PDE=%RX64\n", GCPtrPage, PdeDst.u));
3274	/* Add a reference to the first page only. */
3275	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPde, PGM_PAGE_GET_TRACKING(pPage), pPage, iPDDst);
3276
3277	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3278	return VINF_SUCCESS;
3279	}
3280	}
3281	}
3282	# endif /* HC_ARCH_BITS == 64 */
3283
3284	/*
3285	* Allocate & map the page table.
3286	*/
3287	PSHWPT pPTDst;
3288	PPGMPOOLPAGE pShwPage;
3289	RTGCPHYS GCPhys;
3290
3291	/* Virtual address = physical address */
3292	GCPhys = PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PAGE_4K_BASE_MASK);
3293	rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE,
3294	PGM_A20_IS_ENABLED(pVCpu), pShwPde->idx, iPDDst, false /fLockPage/,
3295	&pShwPage);
3296	if ( rc == VINF_SUCCESS
3297	\|\| rc == VINF_PGM_CACHED_PAGE)
3298	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3299	else
3300	{
3301	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3302	AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
3303	}
3304
3305	if (rc == VINF_SUCCESS)
3306	{
3307	/* New page table; fully set it up. */
3308	Assert(pPTDst);
3309
3310	/* Mask away the page offset. */
3311	GCPtrPage &= ~(RTGCPTR)PAGE_OFFSET_MASK;
3312
3313	for (unsigned iPTDst = 0; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
3314	{
3315	RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
3316	\| (iPTDst << PAGE_SHIFT));
3317
3318	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
3319	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
3320	GCPtrCurPage,
3321	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
3322	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
3323
3324	if (RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)))
3325	break;
3326	}
3327	}
3328	else
3329	rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */
3330
3331	/* Save the new PDE. */
3332	PdeDst.u &= X86_PDE_AVL_MASK;
3333	PdeDst.u \|= pShwPage->Core.Key;
3334	PdeDst.n.u1Present = 1;
3335	PdeDst.n.u1Write = 1;
3336	# if PGM_SHW_TYPE == PGM_TYPE_EPT
3337	PdeDst.n.u1Execute = 1;
3338	# else
3339	PdeDst.n.u1User = 1;
3340	PdeDst.n.u1Accessed = 1;
3341	# endif
3342	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
3343
3344	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3345	if (RT_FAILURE(rc))
3346	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPTFailed));
3347	return rc;
3348
3349	#else
3350	NOREF(iPDSrc); NOREF(pPDSrc);
3351	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
3352	return VERR_PGM_NOT_USED_IN_MODE;
3353	#endif
3354	}
3355
3356
3357
3358	/**
3359	* Prefetch a page/set of pages.
3360	*
3361	* Typically used to sync commonly used pages before entering raw mode
3362	* after a CR3 reload.
3363	*
3364	* @returns VBox status code.
3365	* @param pVCpu The cross context virtual CPU structure.
3366	* @param GCPtrPage Page to invalidate.
3367	*/
3368	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
3369	{
3370	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3371	\|\| PGM_GST_TYPE == PGM_TYPE_REAL \
3372	\|\| PGM_GST_TYPE == PGM_TYPE_PROT \
3373	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3374	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3375	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
3376	&& PGM_SHW_TYPE != PGM_TYPE_EPT
3377
3378	/*
3379	* Check that all Guest levels thru the PDE are present, getting the
3380	* PD and PDE in the processes.
3381	*/
3382	int rc = VINF_SUCCESS;
3383	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3384	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3385	const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
3386	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3387	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3388	unsigned iPDSrc;
3389	X86PDPE PdpeSrc;
3390	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3391	if (!pPDSrc)
3392	return VINF_SUCCESS; /* not present */
3393	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3394	unsigned iPDSrc;
3395	PX86PML4E pPml4eSrc;
3396	X86PDPE PdpeSrc;
3397	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3398	if (!pPDSrc)
3399	return VINF_SUCCESS; /* not present */
3400	# endif
3401	const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3402	# else
3403	PGSTPD pPDSrc = NULL;
3404	const unsigned iPDSrc = 0;
3405	GSTPDE PdeSrc;
3406
3407	PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3408	PdeSrc.n.u1Present = 1;
3409	PdeSrc.n.u1Write = 1;
3410	PdeSrc.n.u1Accessed = 1;
3411	PdeSrc.n.u1User = 1;
3412	# endif
3413
3414	if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
3415	{
3416	PVM pVM = pVCpu->CTX_SUFF(pVM);
3417	pgmLock(pVM);
3418
3419	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3420	const X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
3421	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3422	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3423	PX86PDPAE pPDDst;
3424	X86PDEPAE PdeDst;
3425	# if PGM_GST_TYPE != PGM_TYPE_PAE
3426	X86PDPE PdpeSrc;
3427
3428	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3429	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3430	# endif
3431	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3432	if (rc != VINF_SUCCESS)
3433	{
3434	pgmUnlock(pVM);
3435	AssertRC(rc);
3436	return rc;
3437	}
3438	Assert(pPDDst);
3439	PdeDst = pPDDst->a[iPDDst];
3440
3441	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3442	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3443	PX86PDPAE pPDDst;
3444	X86PDEPAE PdeDst;
3445
3446	# if PGM_GST_TYPE == PGM_TYPE_PROT
3447	/* AMD-V nested paging */
3448	X86PML4E Pml4eSrc;
3449	X86PDPE PdpeSrc;
3450	PX86PML4E pPml4eSrc = &Pml4eSrc;
3451
3452	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3453	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
3454	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
3455	# endif
3456
3457	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3458	if (rc != VINF_SUCCESS)
3459	{
3460	pgmUnlock(pVM);
3461	AssertRC(rc);
3462	return rc;
3463	}
3464	Assert(pPDDst);
3465	PdeDst = pPDDst->a[iPDDst];
3466	# endif
3467	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
3468	{
3469	if (!PdeDst.n.u1Present)
3470	{
3471	/** @todo r=bird: This guy will set the A bit on the PDE,
3472	* probably harmless. */
3473	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3474	}
3475	else
3476	{
3477	/* Note! We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
3478	* R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
3479	* makes no sense to prefetch more than one page.
3480	*/
3481	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3482	if (RT_SUCCESS(rc))
3483	rc = VINF_SUCCESS;
3484	}
3485	}
3486	pgmUnlock(pVM);
3487	}
3488	return rc;
3489
3490	#elif PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3491	NOREF(pVCpu); NOREF(GCPtrPage);
3492	return VINF_SUCCESS; /* ignore */
3493	#else
3494	AssertCompile(0);
3495	#endif
3496	}
3497
3498
3499
3500
3501	/**
3502	* Syncs a page during a PGMVerifyAccess() call.
3503	*
3504	* @returns VBox status code (informational included).
3505	* @param pVCpu The cross context virtual CPU structure.
3506	* @param GCPtrPage The address of the page to sync.
3507	* @param fPage The effective guest page flags.
3508	* @param uErr The trap error code.
3509	* @remarks This will normally never be called on invalid guest page
3510	* translation entries.
3511	*/
3512	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fPage, unsigned uErr)
3513	{
3514	PVM pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
3515
3516	LogFlow(("VerifyAccessSyncPage: GCPtrPage=%RGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
3517	RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(fPage); RT_NOREF_PV(uErr);
3518
3519	Assert(!pVM->pgm.s.fNestedPaging);
3520	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3521	\|\| PGM_GST_TYPE == PGM_TYPE_REAL \
3522	\|\| PGM_GST_TYPE == PGM_TYPE_PROT \
3523	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3524	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3525	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
3526	&& PGM_SHW_TYPE != PGM_TYPE_EPT
3527
3528	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
3529	if (!(fPage & X86_PTE_US))
3530	{
3531	/*
3532	* Mark this page as safe.
3533	*/
3534	/** @todo not correct for pages that contain both code and data!! */
3535	Log(("CSAMMarkPage %RGv; scanned=%d\n", GCPtrPage, true));
3536	CSAMMarkPage(pVM, GCPtrPage, true);
3537	}
3538	# endif
3539
3540	/*
3541	* Get guest PD and index.
3542	*/
3543	/** @todo Performance: We've done all this a jiffy ago in the
3544	* PGMGstGetPage call. */
3545	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3546	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3547	const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
3548	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3549
3550	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3551	unsigned iPDSrc = 0;
3552	X86PDPE PdpeSrc;
3553	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3554	if (RT_UNLIKELY(!pPDSrc))
3555	{
3556	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3557	return VINF_EM_RAW_GUEST_TRAP;
3558	}
3559
3560	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3561	unsigned iPDSrc = 0; /* shut up gcc */
3562	PX86PML4E pPml4eSrc = NULL; /* ditto */
3563	X86PDPE PdpeSrc;
3564	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3565	if (RT_UNLIKELY(!pPDSrc))
3566	{
3567	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3568	return VINF_EM_RAW_GUEST_TRAP;
3569	}
3570	# endif
3571
3572	# else /* !PGM_WITH_PAGING */
3573	PGSTPD pPDSrc = NULL;
3574	const unsigned iPDSrc = 0;
3575	# endif /* !PGM_WITH_PAGING */
3576	int rc = VINF_SUCCESS;
3577
3578	pgmLock(pVM);
3579
3580	/*
3581	* First check if the shadow pd is present.
3582	*/
3583	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3584	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3585
3586	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3587	PX86PDEPAE pPdeDst;
3588	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3589	PX86PDPAE pPDDst;
3590	# if PGM_GST_TYPE != PGM_TYPE_PAE
3591	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3592	X86PDPE PdpeSrc;
3593	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3594	# endif
3595	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3596	if (rc != VINF_SUCCESS)
3597	{
3598	pgmUnlock(pVM);
3599	AssertRC(rc);
3600	return rc;
3601	}
3602	Assert(pPDDst);
3603	pPdeDst = &pPDDst->a[iPDDst];
3604
3605	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3606	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3607	PX86PDPAE pPDDst;
3608	PX86PDEPAE pPdeDst;
3609
3610	# if PGM_GST_TYPE == PGM_TYPE_PROT
3611	/* AMD-V nested paging: Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3612	X86PML4E Pml4eSrc;
3613	X86PDPE PdpeSrc;
3614	PX86PML4E pPml4eSrc = &Pml4eSrc;
3615	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
3616	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
3617	# endif
3618
3619	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3620	if (rc != VINF_SUCCESS)
3621	{
3622	pgmUnlock(pVM);
3623	AssertRC(rc);
3624	return rc;
3625	}
3626	Assert(pPDDst);
3627	pPdeDst = &pPDDst->a[iPDDst];
3628	# endif
3629
3630	if (!pPdeDst->n.u1Present)
3631	{
3632	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3633	if (rc != VINF_SUCCESS)
3634	{
3635	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3636	pgmUnlock(pVM);
3637	AssertRC(rc);
3638	return rc;
3639	}
3640	}
3641
3642	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3643	/* Check for dirty bit fault */
3644	rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
3645	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
3646	Log(("PGMVerifyAccess: success (dirty)\n"));
3647	else
3648	# endif
3649	{
3650	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3651	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3652	# else
3653	GSTPDE PdeSrc;
3654	PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3655	PdeSrc.n.u1Present = 1;
3656	PdeSrc.n.u1Write = 1;
3657	PdeSrc.n.u1Accessed = 1;
3658	PdeSrc.n.u1User = 1;
3659	# endif
3660
3661	Assert(rc != VINF_EM_RAW_GUEST_TRAP);
3662	if (uErr & X86_TRAP_PF_US)
3663	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUser));
3664	else /* supervisor */
3665	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
3666
3667	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3668	if (RT_SUCCESS(rc))
3669	{
3670	/* Page was successfully synced */
3671	Log2(("PGMVerifyAccess: success (sync)\n"));
3672	rc = VINF_SUCCESS;
3673	}
3674	else
3675	{
3676	Log(("PGMVerifyAccess: access violation for %RGv rc=%Rrc\n", GCPtrPage, rc));
3677	rc = VINF_EM_RAW_GUEST_TRAP;
3678	}
3679	}
3680	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3681	pgmUnlock(pVM);
3682	return rc;
3683
3684	#else /* PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED */
3685
3686	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
3687	return VERR_PGM_NOT_USED_IN_MODE;
3688	#endif /* PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED */
3689	}
3690
3691
3692	/**
3693	* Syncs the paging hierarchy starting at CR3.
3694	*
3695	* @returns VBox status code, R0/RC may return VINF_PGM_SYNC_CR3, no other
3696	* informational status codes.
3697	* @retval VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map
3698	* the VMM into guest context.
3699	* @param pVCpu The cross context virtual CPU structure.
3700	* @param cr0 Guest context CR0 register.
3701	* @param cr3 Guest context CR3 register. Not subjected to the A20
3702	* mask.
3703	* @param cr4 Guest context CR4 register.
3704	* @param fGlobal Including global page directories or not
3705	*/
3706	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
3707	{
3708	PVM pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
3709	NOREF(cr0); NOREF(cr3); NOREF(cr4); NOREF(fGlobal);
3710
3711	LogFlow(("SyncCR3 FF=%d fGlobal=%d\n", !!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), fGlobal));
3712
3713	#if PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
3714
3715	pgmLock(pVM);
3716
3717	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
3718	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3719	if (pPool->cDirtyPages)
3720	pgmPoolResetDirtyPages(pVM);
3721	# endif
3722
3723	/*
3724	* Update page access handlers.
3725	* The virtual are always flushed, while the physical are only on demand.
3726	* WARNING: We are incorrectly not doing global flushing on Virtual Handler updates. We'll
3727	* have to look into that later because it will have a bad influence on the performance.
3728	* @note SvL: There's no need for that. Just invalidate the virtual range(s).
3729	* bird: Yes, but that won't work for aliases.
3730	*/
3731	/** @todo this MUST go away. See @bugref{1557}. */
3732	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3Handlers), h);
3733	PGM_GST_NAME(HandlerVirtualUpdate)(pVM, cr4);
3734	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3Handlers), h);
3735	pgmUnlock(pVM);
3736	#endif /* !NESTED && !EPT */
3737
3738	#if PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3739	/*
3740	* Nested / EPT - almost no work.
3741	*/
3742	Assert(!pgmMapAreMappingsEnabled(pVM));
3743	return VINF_SUCCESS;
3744
3745	#elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3746	/*
3747	* AMD64 (Shw & Gst) - No need to check all paging levels; we zero
3748	* out the shadow parts when the guest modifies its tables.
3749	*/
3750	Assert(!pgmMapAreMappingsEnabled(pVM));
3751	return VINF_SUCCESS;
3752
3753	#else /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3754
3755	# ifndef PGM_WITHOUT_MAPPINGS
3756	/*
3757	* Check for and resolve conflicts with our guest mappings if they
3758	* are enabled and not fixed.
3759	*/
3760	if (pgmMapAreMappingsFloating(pVM))
3761	{
3762	int rc = pgmMapResolveConflicts(pVM);
3763	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
3764	if (rc == VINF_SUCCESS)
3765	{ /* likely */ }
3766	else if (rc == VINF_PGM_SYNC_CR3)
3767	{
3768	LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
3769	return VINF_PGM_SYNC_CR3;
3770	}
3771	else if (RT_FAILURE(rc))
3772	return rc;
3773	else
3774	AssertMsgFailed(("%Rrc\n", rc));
3775	}
3776	# else
3777	Assert(!pgmMapAreMappingsEnabled(pVM));
3778	# endif
3779	return VINF_SUCCESS;
3780	#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3781	}
3782
3783
3784
3785
3786	#ifdef VBOX_STRICT
3787	# ifdef IN_RC
3788	# undef AssertMsgFailed
3789	# define AssertMsgFailed Log
3790	# endif
3791
3792	/**
3793	* Checks that the shadow page table is in sync with the guest one.
3794	*
3795	* @returns The number of errors.
3796	* @param pVCpu The cross context virtual CPU structure.
3797	* @param cr3 Guest context CR3 register.
3798	* @param cr4 Guest context CR4 register.
3799	* @param GCPtr Where to start. Defaults to 0.
3800	* @param cb How much to check. Defaults to everything.
3801	*/
3802	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)
3803	{
3804	NOREF(pVCpu); NOREF(cr3); NOREF(cr4); NOREF(GCPtr); NOREF(cb);
3805	#if PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3806	return 0;
3807	#else
3808	unsigned cErrors = 0;
3809	PVM pVM = pVCpu->CTX_SUFF(pVM);
3810	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
3811
3812	# if PGM_GST_TYPE == PGM_TYPE_PAE
3813	/** @todo currently broken; crashes below somewhere */
3814	AssertFailed();
3815	# endif
3816
3817	# if PGM_GST_TYPE == PGM_TYPE_32BIT \
3818	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3819	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
3820
3821	bool fBigPagesSupported = GST_IS_PSE_ACTIVE(pVCpu);
3822	PPGMCPU pPGM = &pVCpu->pgm.s;
3823	RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
3824	RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
3825	# ifndef IN_RING0
3826	RTHCPHYS HCPhys; /* general usage. */
3827	# endif
3828	int rc;
3829
3830	/*
3831	* Check that the Guest CR3 and all its mappings are correct.
3832	*/
3833	AssertMsgReturn(pPGM->GCPhysCR3 == PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK),
3834	("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
3835	false);
3836	# if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64
3837	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3838	rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGst32BitPdRC, NULL, &HCPhysShw);
3839	# else
3840	rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGstPaePdptRC, NULL, &HCPhysShw);
3841	# endif
3842	AssertRCReturn(rc, 1);
3843	HCPhys = NIL_RTHCPHYS;
3844	rc = pgmRamGCPhys2HCPhys(pVM, PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK), &HCPhys);
3845	AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
3846	# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
3847	pgmGstGet32bitPDPtr(pVCpu);
3848	RTGCPHYS GCPhys;
3849	rc = PGMR3DbgR3Ptr2GCPhys(pVM->pUVM, pPGM->pGst32BitPdR3, &GCPhys);
3850	AssertRCReturn(rc, 1);
3851	AssertMsgReturn(PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
3852	# endif
3853	# endif /* !IN_RING0 */
3854
3855	/*
3856	* Get and check the Shadow CR3.
3857	*/
3858	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3859	unsigned cPDEs = X86_PG_ENTRIES;
3860	unsigned cIncrement = X86_PG_ENTRIES * PAGE_SIZE;
3861	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3862	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3863	unsigned cPDEs = X86_PG_PAE_ENTRIES * 4; /* treat it as a 2048 entry table. */
3864	# else
3865	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3866	# endif
3867	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3868	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3869	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3870	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3871	# endif
3872	if (cb != ~(RTGCPTR)0)
3873	cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
3874
3875	/** @todo call the other two PGMAssert() functions. /
3876
3877	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3878	unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3879
3880	for (; iPml4 < X86_PG_PAE_ENTRIES; iPml4++)
3881	{
3882	PPGMPOOLPAGE pShwPdpt = NULL;
3883	PX86PML4E pPml4eSrc;
3884	PX86PML4E pPml4eDst;
3885	RTGCPHYS GCPhysPdptSrc;
3886
3887	pPml4eSrc = pgmGstGetLongModePML4EPtr(pVCpu, iPml4);
3888	pPml4eDst = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
3889
3890	/* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
3891	if (!pPml4eDst->n.u1Present)
3892	{
3893	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3894	continue;
3895	}
3896
3897	pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
3898	GCPhysPdptSrc = PGM_A20_APPLY(pVCpu, pPml4eSrc->u & X86_PML4E_PG_MASK);
3899
3900	if (pPml4eSrc->n.u1Present != pPml4eDst->n.u1Present)
3901	{
3902	AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3903	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3904	cErrors++;
3905	continue;
3906	}
3907
3908	if (GCPhysPdptSrc != pShwPdpt->GCPhys)
3909	{
3910	AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
3911	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3912	cErrors++;
3913	continue;
3914	}
3915
3916	if ( pPml4eDst->n.u1User != pPml4eSrc->n.u1User
3917	\|\| pPml4eDst->n.u1Write != pPml4eSrc->n.u1Write
3918	\|\| pPml4eDst->n.u1NoExecute != pPml4eSrc->n.u1NoExecute)
3919	{
3920	AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3921	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3922	cErrors++;
3923	continue;
3924	}
3925	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3926	{
3927	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3928
3929	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
3930	/*
3931	* Check the PDPTEs too.
3932	*/
3933	unsigned iPdpt = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
3934
3935	for (;iPdpt <= SHW_PDPT_MASK; iPdpt++)
3936	{
3937	unsigned iPDSrc = 0; /* initialized to shut up gcc */
3938	PPGMPOOLPAGE pShwPde = NULL;
3939	PX86PDPE pPdpeDst;
3940	RTGCPHYS GCPhysPdeSrc;
3941	X86PDPE PdpeSrc;
3942	PdpeSrc.u = 0; /* initialized to shut up gcc 4.5 */
3943	# if PGM_GST_TYPE == PGM_TYPE_PAE
3944	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtr, &iPDSrc, &PdpeSrc);
3945	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
3946	# else
3947	PX86PML4E pPml4eSrcIgn;
3948	PX86PDPT pPdptDst;
3949	PX86PDPAE pPDDst;
3950	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtr, &pPml4eSrcIgn, &PdpeSrc, &iPDSrc);
3951
3952	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtr, NULL, &pPdptDst, &pPDDst);
3953	if (rc != VINF_SUCCESS)
3954	{
3955	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
3956	GCPtr += 512 * _2M;
3957	continue; /* next PDPTE */
3958	}
3959	Assert(pPDDst);
3960	# endif
3961	Assert(iPDSrc == 0);
3962
3963	pPdpeDst = &pPdptDst->a[iPdpt];
3964
3965	if (!pPdpeDst->n.u1Present)
3966	{
3967	GCPtr += 512 * _2M;
3968	continue; /* next PDPTE */
3969	}
3970
3971	pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
3972	GCPhysPdeSrc = PGM_A20_APPLY(pVCpu, PdpeSrc.u & X86_PDPE_PG_MASK);
3973
3974	if (pPdpeDst->n.u1Present != PdpeSrc.n.u1Present)
3975	{
3976	AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3977	GCPtr += 512 * _2M;
3978	cErrors++;
3979	continue;
3980	}
3981
3982	if (GCPhysPdeSrc != pShwPde->GCPhys)
3983	{
3984	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3985	AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3986	# else
3987	AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3988	# endif
3989	GCPtr += 512 * _2M;
3990	cErrors++;
3991	continue;
3992	}
3993
3994	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3995	if ( pPdpeDst->lm.u1User != PdpeSrc.lm.u1User
3996	\|\| pPdpeDst->lm.u1Write != PdpeSrc.lm.u1Write
3997	\|\| pPdpeDst->lm.u1NoExecute != PdpeSrc.lm.u1NoExecute)
3998	{
3999	AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
4000	GCPtr += 512 * _2M;
4001	cErrors++;
4002	continue;
4003	}
4004	# endif
4005
4006	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
4007	{
4008	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
4009	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4010	GSTPD const *pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
4011	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
4012	PCX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
4013	# endif
4014	# endif /* PGM_GST_TYPE == PGM_TYPE_32BIT */
4015	/*
4016	* Iterate the shadow page directory.
4017	*/
4018	GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
4019	unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
4020
4021	for (;
4022	iPDDst < cPDEs;
4023	iPDDst++, GCPtr += cIncrement)
4024	{
4025	# if PGM_SHW_TYPE == PGM_TYPE_PAE
4026	const SHWPDE PdeDst = *pgmShwGetPaePDEPtr(pVCpu, GCPtr);
4027	# else
4028	const SHWPDE PdeDst = pPDDst->a[iPDDst];
4029	# endif
4030	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
4031	{
4032	Assert(pgmMapAreMappingsEnabled(pVM));
4033	if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
4034	{
4035	AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
4036	cErrors++;
4037	continue;
4038	}
4039	}
4040	else if ( (PdeDst.u & X86_PDE_P)
4041	\|\| ((PdeDst.u & (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY))
4042	)
4043	{
4044	HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
4045	PPGMPOOLPAGE pPoolPage = pgmPoolGetPage(pPool, HCPhysShw);
4046	if (!pPoolPage)
4047	{
4048	AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n",
4049	HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
4050	cErrors++;
4051	continue;
4052	}
4053	const SHWPT pPTDst = (const SHWPT )PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pPoolPage);
4054
4055	if (PdeDst.u & (X86_PDE4M_PWT \| X86_PDE4M_PCD))
4056	{
4057	AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
4058	GCPtr, (uint64_t)PdeDst.u));
4059	cErrors++;
4060	}
4061
4062	if (PdeDst.u & (X86_PDE4M_G \| X86_PDE4M_D))
4063	{
4064	AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n",
4065	GCPtr, (uint64_t)PdeDst.u));
4066	cErrors++;
4067	}
4068
4069	const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
4070	if (!PdeSrc.n.u1Present)
4071	{
4072	AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
4073	GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
4074	cErrors++;
4075	continue;
4076	}
4077
4078	if ( !PdeSrc.b.u1Size
4079	\|\| !fBigPagesSupported)
4080	{
4081	GCPhysGst = GST_GET_PDE_GCPHYS(PdeSrc);
4082	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
4083	GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst \| ((iPDDst & 1) * (PAGE_SIZE / 2)));
4084	# endif
4085	}
4086	else
4087	{
4088	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4089	if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
4090	{
4091	AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n",
4092	GCPtr, (uint64_t)PdeSrc.u));
4093	cErrors++;
4094	continue;
4095	}
4096	# endif
4097	GCPhysGst = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
4098	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
4099	GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst \| (GCPtr & RT_BIT(X86_PAGE_2M_SHIFT)));
4100	# endif
4101	}
4102
4103	if ( pPoolPage->enmKind
4104	!= (!PdeSrc.b.u1Size \|\| !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
4105	{
4106	AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n",
4107	pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
4108	cErrors++;
4109	}
4110
4111	PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4112	if (!pPhysPage)
4113	{
4114	AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
4115	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
4116	cErrors++;
4117	continue;
4118	}
4119
4120	if (GCPhysGst != pPoolPage->GCPhys)
4121	{
4122	AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n",
4123	GCPhysGst, pPoolPage->GCPhys, GCPtr));
4124	cErrors++;
4125	continue;
4126	}
4127
4128	if ( !PdeSrc.b.u1Size
4129	\|\| !fBigPagesSupported)
4130	{
4131	/*
4132	* Page Table.
4133	*/
4134	const GSTPT *pPTSrc;
4135	rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, PGM_A20_APPLY(pVCpu, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1)),
4136	&pPTSrc);
4137	if (RT_FAILURE(rc))
4138	{
4139	AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
4140	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
4141	cErrors++;
4142	continue;
4143	}
4144	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/))
4145	!= (PdeDst.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/)))
4146	{
4147	/// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
4148	// (This problem will go away when/if we shadow multiple CR3s.)
4149	AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4150	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4151	cErrors++;
4152	continue;
4153	}
4154	if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4155	{
4156	AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n",
4157	GCPtr, (uint64_t)PdeDst.u));
4158	cErrors++;
4159	continue;
4160	}
4161
4162	/* iterate the page table. */
4163	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
4164	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
4165	const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
4166	# else
4167	const unsigned offPTSrc = 0;
4168	# endif
4169	for (unsigned iPT = 0, off = 0;
4170	iPT < RT_ELEMENTS(pPTDst->a);
4171	iPT++, off += PAGE_SIZE)
4172	{
4173	const SHWPTE PteDst = pPTDst->a[iPT];
4174
4175	/* skip not-present and dirty tracked entries. */
4176	if (!(SHW_PTE_GET_U(PteDst) & (X86_PTE_P \| PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
4177	continue;
4178	Assert(SHW_PTE_IS_P(PteDst));
4179
4180	const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
4181	if (!PteSrc.n.u1Present)
4182	{
4183	# ifdef IN_RING3
4184	PGMAssertHandlerAndFlagsInSync(pVM);
4185	DBGFR3PagingDumpEx(pVM->pUVM, pVCpu->idCpu, DBGFPGDMP_FLAGS_CURRENT_CR3 \| DBGFPGDMP_FLAGS_CURRENT_MODE
4186	\| DBGFPGDMP_FLAGS_GUEST \| DBGFPGDMP_FLAGS_HEADER \| DBGFPGDMP_FLAGS_PRINT_CR3,
4187	0, 0, UINT64_MAX, 99, NULL);
4188	# endif
4189	AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
4190	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst), pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
4191	(uint64_t)GST_GET_PDE_GCPHYS(PdeSrc) + (iPT + offPTSrc) * sizeof(PteSrc)));
4192	cErrors++;
4193	continue;
4194	}
4195
4196	uint64_t fIgnoreFlags = GST_PTE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_G \| X86_PTE_D \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
4197	# if 1 /** @todo sync accessed bit properly... */
4198	fIgnoreFlags \|= X86_PTE_A;
4199	# endif
4200
4201	/* match the physical addresses */
4202	HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
4203	GCPhysGst = GST_GET_PTE_GCPHYS(PteSrc);
4204
4205	# ifdef IN_RING3
4206	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4207	if (RT_FAILURE(rc))
4208	{
4209	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4210	{
4211	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
4212	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4213	cErrors++;
4214	continue;
4215	}
4216	}
4217	else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
4218	{
4219	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4220	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4221	cErrors++;
4222	continue;
4223	}
4224	# endif
4225
4226	pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4227	if (!pPhysPage)
4228	{
4229	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4230	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4231	{
4232	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
4233	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4234	cErrors++;
4235	continue;
4236	}
4237	# endif
4238	if (SHW_PTE_IS_RW(PteDst))
4239	{
4240	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4241	GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4242	cErrors++;
4243	}
4244	fIgnoreFlags \|= X86_PTE_RW;
4245	}
4246	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4247	{
4248	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4249	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4250	cErrors++;
4251	continue;
4252	}
4253
4254	/* flags */
4255	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4256	{
4257	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4258	{
4259	if (SHW_PTE_IS_RW(PteDst))
4260	{
4261	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4262	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4263	cErrors++;
4264	continue;
4265	}
4266	fIgnoreFlags \|= X86_PTE_RW;
4267	}
4268	else
4269	{
4270	if ( SHW_PTE_IS_P(PteDst)
4271	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
4272	&& !PGM_PAGE_IS_MMIO(pPhysPage)
4273	# endif
4274	)
4275	{
4276	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4277	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4278	cErrors++;
4279	continue;
4280	}
4281	fIgnoreFlags \|= X86_PTE_P;
4282	}
4283	}
4284	else
4285	{
4286	if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
4287	{
4288	if (SHW_PTE_IS_RW(PteDst))
4289	{
4290	AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
4291	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4292	cErrors++;
4293	continue;
4294	}
4295	if (!SHW_PTE_IS_TRACK_DIRTY(PteDst))
4296	{
4297	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4298	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4299	cErrors++;
4300	continue;
4301	}
4302	if (SHW_PTE_IS_D(PteDst))
4303	{
4304	AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4305	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4306	cErrors++;
4307	}
4308	# if 0 /** @todo sync access bit properly... */
4309	if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
4310	{
4311	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4312	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4313	cErrors++;
4314	}
4315	fIgnoreFlags \|= X86_PTE_RW;
4316	# else
4317	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
4318	# endif
4319	}
4320	else if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
4321	{
4322	/* access bit emulation (not implemented). */
4323	if (PteSrc.n.u1Accessed \|\| SHW_PTE_IS_P(PteDst))
4324	{
4325	AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
4326	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4327	cErrors++;
4328	continue;
4329	}
4330	if (!SHW_PTE_IS_A(PteDst))
4331	{
4332	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
4333	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4334	cErrors++;
4335	}
4336	fIgnoreFlags \|= X86_PTE_P;
4337	}
4338	# ifdef DEBUG_sandervl
4339	fIgnoreFlags \|= X86_PTE_D \| X86_PTE_A;
4340	# endif
4341	}
4342
4343	if ( (PteSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4344	&& (PteSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4345	)
4346	{
4347	AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
4348	GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
4349	fIgnoreFlags, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4350	cErrors++;
4351	continue;
4352	}
4353	} /* foreach PTE */
4354	}
4355	else
4356	{
4357	/*
4358	* Big Page.
4359	*/
4360	uint64_t fIgnoreFlags = X86_PDE_AVL_MASK \| GST_PDE_PG_MASK \| X86_PDE4M_G \| X86_PDE4M_D \| X86_PDE4M_PS \| X86_PDE4M_PWT \| X86_PDE4M_PCD;
4361	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
4362	{
4363	if (PdeDst.n.u1Write)
4364	{
4365	AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4366	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4367	cErrors++;
4368	continue;
4369	}
4370	if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
4371	{
4372	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4373	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4374	cErrors++;
4375	continue;
4376	}
4377	# if 0 /** @todo sync access bit properly... */
4378	if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
4379	{
4380	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4381	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4382	cErrors++;
4383	}
4384	fIgnoreFlags \|= X86_PTE_RW;
4385	# else
4386	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
4387	# endif
4388	}
4389	else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4390	{
4391	/* access bit emulation (not implemented). */
4392	if (PdeSrc.b.u1Accessed \|\| PdeDst.n.u1Present)
4393	{
4394	AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4395	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4396	cErrors++;
4397	continue;
4398	}
4399	if (!PdeDst.n.u1Accessed)
4400	{
4401	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4402	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4403	cErrors++;
4404	}
4405	fIgnoreFlags \|= X86_PTE_P;
4406	}
4407
4408	if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
4409	{
4410	AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
4411	GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
4412	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4413	cErrors++;
4414	}
4415
4416	/* iterate the page table. */
4417	for (unsigned iPT = 0, off = 0;
4418	iPT < RT_ELEMENTS(pPTDst->a);
4419	iPT++, off += PAGE_SIZE, GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst + PAGE_SIZE))
4420	{
4421	const SHWPTE PteDst = pPTDst->a[iPT];
4422
4423	if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
4424	{
4425	AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
4426	GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4427	cErrors++;
4428	}
4429
4430	/* skip not-present entries. */
4431	if (!SHW_PTE_IS_P(PteDst)) /** @todo deal with ALL handlers and CSAM !P pages! */
4432	continue;
4433
4434	fIgnoreFlags = X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT \| X86_PTE_D \| X86_PTE_A \| X86_PTE_G \| X86_PTE_PAE_NX;
4435
4436	/* match the physical addresses */
4437	HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
4438
4439	# ifdef IN_RING3
4440	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4441	if (RT_FAILURE(rc))
4442	{
4443	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4444	{
4445	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4446	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4447	cErrors++;
4448	}
4449	}
4450	else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
4451	{
4452	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4453	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4454	cErrors++;
4455	continue;
4456	}
4457	# endif
4458	pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4459	if (!pPhysPage)
4460	{
4461	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4462	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4463	{
4464	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4465	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4466	cErrors++;
4467	continue;
4468	}
4469	# endif
4470	if (SHW_PTE_IS_RW(PteDst))
4471	{
4472	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4473	GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4474	cErrors++;
4475	}
4476	fIgnoreFlags \|= X86_PTE_RW;
4477	}
4478	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4479	{
4480	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4481	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4482	cErrors++;
4483	continue;
4484	}
4485
4486	/* flags */
4487	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4488	{
4489	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4490	{
4491	if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
4492	{
4493	if (SHW_PTE_IS_RW(PteDst))
4494	{
4495	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4496	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4497	cErrors++;
4498	continue;
4499	}
4500	fIgnoreFlags \|= X86_PTE_RW;
4501	}
4502	}
4503	else
4504	{
4505	if ( SHW_PTE_IS_P(PteDst)
4506	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
4507	&& !PGM_PAGE_IS_MMIO(pPhysPage)
4508	# endif
4509	)
4510	{
4511	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4512	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4513	cErrors++;
4514	continue;
4515	}
4516	fIgnoreFlags \|= X86_PTE_P;
4517	}
4518	}
4519
4520	if ( (PdeSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4521	&& (PdeSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags) /* lazy phys handler dereg. */
4522	)
4523	{
4524	AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
4525	GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
4526	fIgnoreFlags, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4527	cErrors++;
4528	continue;
4529	}
4530	} /* for each PTE */
4531	}
4532	}
4533	/* not present */
4534
4535	} /* for each PDE */
4536
4537	} /* for each PDPTE */
4538
4539	} /* for each PML4E */
4540
4541	# ifdef DEBUG
4542	if (cErrors)
4543	LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
4544	# endif
4545	# endif /* GST is in {32BIT, PAE, AMD64} */
4546	return cErrors;
4547	#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT */
4548	}
4549	#endif /* VBOX_STRICT */
4550
4551
4552	/**
4553	* Sets up the CR3 for shadow paging
4554	*
4555	* @returns Strict VBox status code.
4556	* @retval VINF_SUCCESS.
4557	*
4558	* @param pVCpu The cross context virtual CPU structure.
4559	* @param GCPhysCR3 The physical address in the CR3 register. (A20
4560	* mask already applied.)
4561	*/
4562	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3)
4563	{
4564	PVM pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
4565
4566	/* Update guest paging info. */
4567	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
4568	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
4569	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
4570
4571	LogFlow(("MapCR3: %RGp\n", GCPhysCR3));
4572	PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
4573
4574	/*
4575	* Map the page CR3 points at.
4576	*/
4577	RTHCPTR HCPtrGuestCR3;
4578	RTHCPHYS HCPhysGuestCR3;
4579	pgmLock(pVM);
4580	PPGMPAGE pPageCR3 = pgmPhysGetPage(pVM, GCPhysCR3);
4581	AssertReturn(pPageCR3, VERR_PGM_INVALID_CR3_ADDR);
4582	HCPhysGuestCR3 = PGM_PAGE_GET_HCPHYS(pPageCR3);
4583	/** @todo this needs some reworking wrt. locking? */
4584	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4585	HCPtrGuestCR3 = NIL_RTHCPTR;
4586	int rc = VINF_SUCCESS;
4587	# else
4588	int rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPageCR3, GCPhysCR3 & GST_CR3_PAGE_MASK, (void )&HCPtrGuestCR3); / @todo r=bird: This GCPhysCR3 masking isn't necessary. */
4589	# endif
4590	pgmUnlock(pVM);
4591	if (RT_SUCCESS(rc))
4592	{
4593	rc = PGMMap(pVM, (RTGCPTR)pVM->pgm.s.GCPtrCR3Mapping, HCPhysGuestCR3, PAGE_SIZE, 0);
4594	if (RT_SUCCESS(rc))
4595	{
4596	# ifdef IN_RC
4597	PGM_INVL_PG(pVCpu, pVM->pgm.s.GCPtrCR3Mapping);
4598	# endif
4599	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4600	pVCpu->pgm.s.pGst32BitPdR3 = (R3PTRTYPE(PX86PD))HCPtrGuestCR3;
4601	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4602	pVCpu->pgm.s.pGst32BitPdR0 = (R0PTRTYPE(PX86PD))HCPtrGuestCR3;
4603	# endif
4604	pVCpu->pgm.s.pGst32BitPdRC = (RCPTRTYPE(PX86PD))(RTRCUINTPTR)pVM->pgm.s.GCPtrCR3Mapping;
4605
4606	# elif PGM_GST_TYPE == PGM_TYPE_PAE
4607	unsigned off = GCPhysCR3 & GST_CR3_PAGE_MASK & PAGE_OFFSET_MASK;
4608	pVCpu->pgm.s.pGstPaePdptR3 = (R3PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4609	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4610	pVCpu->pgm.s.pGstPaePdptR0 = (R0PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4611	# endif
4612	pVCpu->pgm.s.pGstPaePdptRC = (RCPTRTYPE(PX86PDPT))((RTRCUINTPTR)pVM->pgm.s.GCPtrCR3Mapping + off);
4613	LogFlow(("Cached mapping %RRv\n", pVCpu->pgm.s.pGstPaePdptRC));
4614
4615	/*
4616	* Map the 4 PDs too.
4617	*/
4618	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pVCpu);
4619	RTGCPTR GCPtr = pVM->pgm.s.GCPtrCR3Mapping + PAGE_SIZE;
4620	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++, GCPtr += PAGE_SIZE)
4621	{
4622	pVCpu->pgm.s.aGstPaePdpeRegs[i].u = pGuestPDPT->a[i].u;
4623	if (pGuestPDPT->a[i].n.u1Present)
4624	{
4625	RTHCPTR HCPtr;
4626	RTHCPHYS HCPhys;
4627	RTGCPHYS GCPhys = PGM_A20_APPLY(pVCpu, pGuestPDPT->a[i].u & X86_PDPE_PG_MASK);
4628	pgmLock(pVM);
4629	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
4630	AssertReturn(pPage, VERR_PGM_INVALID_PDPE_ADDR);
4631	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
4632	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4633	HCPtr = NIL_RTHCPTR;
4634	int rc2 = VINF_SUCCESS;
4635	# else
4636	int rc2 = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)&HCPtr);
4637	# endif
4638	pgmUnlock(pVM);
4639	if (RT_SUCCESS(rc2))
4640	{
4641	rc = PGMMap(pVM, GCPtr, HCPhys, PAGE_SIZE, 0);
4642	AssertRCReturn(rc, rc);
4643
4644	pVCpu->pgm.s.apGstPaePDsR3[i] = (R3PTRTYPE(PX86PDPAE))HCPtr;
4645	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4646	pVCpu->pgm.s.apGstPaePDsR0[i] = (R0PTRTYPE(PX86PDPAE))HCPtr;
4647	# endif
4648	pVCpu->pgm.s.apGstPaePDsRC[i] = (RCPTRTYPE(PX86PDPAE))(RTRCUINTPTR)GCPtr;
4649	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
4650	# ifdef IN_RC
4651	PGM_INVL_PG(pVCpu, GCPtr);
4652	# endif
4653	continue;
4654	}
4655	AssertMsgFailed(("pgmR3Gst32BitMapCR3: rc2=%d GCPhys=%RGp i=%d\n", rc2, GCPhys, i));
4656	}
4657
4658	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4659	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4660	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4661	# endif
4662	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4663	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4664	# ifdef IN_RC
4665	PGM_INVL_PG(pVCpu, GCPtr); /** @todo this shouldn't be necessary? */
4666	# endif
4667	}
4668
4669	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
4670	pVCpu->pgm.s.pGstAmd64Pml4R3 = (R3PTRTYPE(PX86PML4))HCPtrGuestCR3;
4671	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4672	pVCpu->pgm.s.pGstAmd64Pml4R0 = (R0PTRTYPE(PX86PML4))HCPtrGuestCR3;
4673	# endif
4674	# endif
4675	}
4676	else
4677	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4678	}
4679	else
4680	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4681
4682	#else /* prot/real stub */
4683	int rc = VINF_SUCCESS;
4684	#endif
4685
4686	/* Update shadow paging info for guest modes with paging (32, pae, 64). */
4687	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4688	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4689	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64) \
4690	&& ( PGM_GST_TYPE != PGM_TYPE_REAL \
4691	&& PGM_GST_TYPE != PGM_TYPE_PROT))
4692
4693	Assert(!pVM->pgm.s.fNestedPaging);
4694	PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
4695
4696	/*
4697	* Update the shadow root page as well since that's not fixed.
4698	*/
4699	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4700	PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
4701	PPGMPOOLPAGE pNewShwPageCR3;
4702
4703	pgmLock(pVM);
4704
4705	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4706	if (pPool->cDirtyPages)
4707	pgmPoolResetDirtyPages(pVM);
4708	# endif
4709
4710	Assert(!(GCPhysCR3 >> (PAGE_SHIFT + 32)));
4711	rc = pgmPoolAlloc(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
4712	NIL_PGMPOOL_IDX, UINT32_MAX, true /fLockPage/,
4713	&pNewShwPageCR3);
4714	AssertFatalRC(rc);
4715	rc = VINF_SUCCESS;
4716
4717	# ifdef IN_RC
4718	/*
4719	* WARNING! We can't deal with jumps to ring 3 in the code below as the
4720	* state will be inconsistent! Flush important things now while
4721	* we still can and then make sure there are no ring-3 calls.
4722	*/
4723	# ifdef VBOX_WITH_REM
4724	REMNotifyHandlerPhysicalFlushIfAlmostFull(pVM, pVCpu);
4725	# endif
4726	VMMRZCallRing3Disable(pVCpu);
4727	# endif
4728
4729	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3) = pNewShwPageCR3;
4730	# ifdef IN_RING0
4731	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4732	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4733	# elif defined(IN_RC)
4734	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4735	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4736	# else
4737	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4738	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4739	# endif
4740
4741	# ifndef PGM_WITHOUT_MAPPINGS
4742	/*
4743	* Apply all hypervisor mappings to the new CR3.
4744	* Note that SyncCR3 will be executed in case CR3 is changed in a guest paging mode; this will
4745	* make sure we check for conflicts in the new CR3 root.
4746	*/
4747	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4748	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL) \|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
4749	# endif
4750	rc = pgmMapActivateCR3(pVM, pNewShwPageCR3);
4751	AssertRCReturn(rc, rc);
4752	# endif
4753
4754	/* Set the current hypervisor CR3. */
4755	CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
4756	SELMShadowCR3Changed(pVM, pVCpu);
4757
4758	# ifdef IN_RC
4759	/* NOTE: The state is consistent again. */
4760	VMMRZCallRing3Enable(pVCpu);
4761	# endif
4762
4763	/* Clean up the old CR3 root. */
4764	if ( pOldShwPageCR3
4765	&& pOldShwPageCR3 != pNewShwPageCR3 /* @todo can happen due to incorrect syncing between REM & PGM; find the real cause */)
4766	{
4767	Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
4768	# ifndef PGM_WITHOUT_MAPPINGS
4769	/* Remove the hypervisor mappings from the shadow page table. */
4770	pgmMapDeactivateCR3(pVM, pOldShwPageCR3);
4771	# endif
4772	/* Mark the page as unlocked; allow flushing again. */
4773	pgmPoolUnlockPage(pPool, pOldShwPageCR3);
4774
4775	pgmPoolFreeByPage(pPool, pOldShwPageCR3, NIL_PGMPOOL_IDX, UINT32_MAX);
4776	}
4777	pgmUnlock(pVM);
4778	# else
4779	NOREF(GCPhysCR3);
4780	# endif
4781
4782	return rc;
4783	}
4784
4785	/**
4786	* Unmaps the shadow CR3.
4787	*
4788	* @returns VBox status, no specials.
4789	* @param pVCpu The cross context virtual CPU structure.
4790	*/
4791	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu)
4792	{
4793	LogFlow(("UnmapCR3\n"));
4794
4795	int rc = VINF_SUCCESS;
4796	PVM pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
4797
4798	/*
4799	* Update guest paging info.
4800	*/
4801	#if PGM_GST_TYPE == PGM_TYPE_32BIT
4802	pVCpu->pgm.s.pGst32BitPdR3 = 0;
4803	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4804	pVCpu->pgm.s.pGst32BitPdR0 = 0;
4805	# endif
4806	pVCpu->pgm.s.pGst32BitPdRC = 0;
4807
4808	#elif PGM_GST_TYPE == PGM_TYPE_PAE
4809	pVCpu->pgm.s.pGstPaePdptR3 = 0;
4810	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4811	pVCpu->pgm.s.pGstPaePdptR0 = 0;
4812	# endif
4813	pVCpu->pgm.s.pGstPaePdptRC = 0;
4814	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4815	{
4816	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4817	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4818	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4819	# endif
4820	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4821	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4822	}
4823
4824	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
4825	pVCpu->pgm.s.pGstAmd64Pml4R3 = 0;
4826	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4827	pVCpu->pgm.s.pGstAmd64Pml4R0 = 0;
4828	# endif
4829
4830	#else /* prot/real mode stub */
4831	/* nothing to do */
4832	#endif
4833
4834	#if !defined(IN_RC) /* In RC we rely on MapCR3 to do the shadow part for us at a safe time */
4835	/*
4836	* Update shadow paging info.
4837	*/
4838	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4839	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4840	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64))
4841
4842	# if PGM_GST_TYPE != PGM_TYPE_REAL
4843	Assert(!pVM->pgm.s.fNestedPaging);
4844	# endif
4845
4846	pgmLock(pVM);
4847
4848	# ifndef PGM_WITHOUT_MAPPINGS
4849	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4850	/* Remove the hypervisor mappings from the shadow page table. */
4851	pgmMapDeactivateCR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4852	# endif
4853
4854	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4855	{
4856	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4857
4858	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4859	if (pPool->cDirtyPages)
4860	pgmPoolResetDirtyPages(pVM);
4861	# endif
4862
4863	/* Mark the page as unlocked; allow flushing again. */
4864	pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4865
4866	pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), NIL_PGMPOOL_IDX, UINT32_MAX);
4867	pVCpu->pgm.s.pShwPageCR3R3 = 0;
4868	pVCpu->pgm.s.pShwPageCR3R0 = 0;
4869	pVCpu->pgm.s.pShwPageCR3RC = 0;
4870	}
4871	pgmUnlock(pVM);
4872	# endif
4873	#endif /* !IN_RC*/
4874
4875	return rc;
4876	}

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 66650

Download in other formats: