PGMAllBth.h@ 93666

Last change on this file since 93666 was 93650, checked in by vboxsync, 3 years ago
VMM/PGM,*: Split the physical access handler type registration into separate ring-0 and ring-3 steps, expanding the type to 64-bit. bugref:10094
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 189.3 KB

Line
1	/* $Id: PGMAllBth.h 93650 2022-02-08 10:43:53Z vboxsync $ */
2	/** @file
3	* VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4	*
5	* @remarks Extended page tables (intel) are built with PGM_GST_TYPE set to
6	* PGM_TYPE_PROT (and PGM_SHW_TYPE set to PGM_TYPE_EPT).
7	* bird: WTF does this mean these days? Looking at PGMAll.cpp it's
8	*
9	* @remarks This file is one big \#ifdef-orgy!
10	*
11	*/
12
13	/*
14	* Copyright (C) 2006-2022 Oracle Corporation
15	*
16	* This file is part of VirtualBox Open Source Edition (OSE), as
17	* available from http://www.virtualbox.org. This file is free software;
18	* you can redistribute it and/or modify it under the terms of the GNU
19	* General Public License (GPL) as published by the Free Software
20	* Foundation, in version 2 as it comes in the "COPYING" file of the
21	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
22	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
23	*/
24
25	#ifdef _MSC_VER
26	/** @todo we're generating unnecessary code in nested/ept shadow mode and for
27	* real/prot-guest+RC mode. */
28	# pragma warning(disable: 4505)
29	#endif
30
31
32	/*********************************************************************************************************************************
33	* Internal Functions *
34	*********************************************************************************************************************************/
35	RT_C_DECLS_BEGIN
36	PGM_BTH_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3);
37	#ifndef IN_RING3
38	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken);
39	#endif
40	PGM_BTH_DECL(int, InvalidatePage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage);
41	static int PGM_BTH_NAME(SyncPage)(PVMCPUCC pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr);
42	static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPUCC pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, RTGCPTR GCPtrPage);
43	static int PGM_BTH_NAME(SyncPT)(PVMCPUCC pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage);
44	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
45	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
46	#else
47	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
48	#endif
49	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPUCC pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uErr);
50	PGM_BTH_DECL(int, PrefetchPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage);
51	PGM_BTH_DECL(int, SyncCR3)(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
52	#ifdef VBOX_STRICT
53	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
54	#endif
55	PGM_BTH_DECL(int, MapCR3)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3);
56	PGM_BTH_DECL(int, UnmapCR3)(PVMCPUCC pVCpu);
57
58	#ifdef IN_RING3
59	PGM_BTH_DECL(int, Relocate)(PVMCPUCC pVCpu, RTGCPTR offDelta);
60	#endif
61	RT_C_DECLS_END
62
63
64
65
66	/*
67	* Filter out some illegal combinations of guest and shadow paging, so we can
68	* remove redundant checks inside functions.
69	*/
70	#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE \
71	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
72	# error "Invalid combination; PAE guest implies PAE shadow"
73	#endif
74
75	#if (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
76	&& !( PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64 \
77	\|\| PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \|\| PGM_SHW_TYPE == PGM_TYPE_NONE)
78	# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
79	#endif
80
81	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) \
82	&& !( PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
83	\|\| PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \|\| PGM_SHW_TYPE == PGM_TYPE_NONE)
84	# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
85	#endif
86
87	#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE) \
88	\|\| (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
89	# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
90	#endif
91
92
93	/**
94	* Enters the shadow+guest mode.
95	*
96	* @returns VBox status code.
97	* @param pVCpu The cross context virtual CPU structure.
98	* @param GCPhysCR3 The physical address from the CR3 register.
99	*/
100	PGM_BTH_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3)
101	{
102	/* Here we deal with allocation of the root shadow page table for real and protected mode during mode switches;
103	* Other modes rely on MapCR3/UnmapCR3 to setup the shadow root page tables.
104	*/
105	#if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
106	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
107	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64) \
108	&& ( PGM_GST_TYPE == PGM_TYPE_REAL \
109	\|\| PGM_GST_TYPE == PGM_TYPE_PROT))
110
111	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
112
113	Assert((HMIsNestedPagingActive(pVM) \|\| VM_IS_NEM_ENABLED(pVM)) == pVM->pgm.s.fNestedPaging);
114	Assert(!pVM->pgm.s.fNestedPaging);
115
116	PGM_LOCK_VOID(pVM);
117	/* Note: we only really need shadow paging in real and protected mode for VT-x and AMD-V (excluding nested paging/EPT modes),
118	* but any calls to GC need a proper shadow page setup as well.
119	*/
120	/* Free the previous root mapping if still active. */
121	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
122	PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
123	if (pOldShwPageCR3)
124	{
125	Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
126
127	/* Mark the page as unlocked; allow flushing again. */
128	pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
129
130	pgmPoolFreeByPage(pPool, pOldShwPageCR3, NIL_PGMPOOL_IDX, UINT32_MAX);
131	pVCpu->pgm.s.pShwPageCR3R3 = NIL_RTR3PTR;
132	pVCpu->pgm.s.pShwPageCR3R0 = NIL_RTR0PTR;
133	}
134
135	/* construct a fake address. */
136	GCPhysCR3 = RT_BIT_64(63);
137	PPGMPOOLPAGE pNewShwPageCR3;
138	int rc = pgmPoolAlloc(pVM, GCPhysCR3, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
139	NIL_PGMPOOL_IDX, UINT32_MAX, false /fLockPage/,
140	&pNewShwPageCR3);
141	AssertRCReturn(rc, rc);
142
143	pVCpu->pgm.s.pShwPageCR3R3 = pgmPoolConvertPageToR3(pPool, pNewShwPageCR3);
144	pVCpu->pgm.s.pShwPageCR3R0 = pgmPoolConvertPageToR0(pPool, pNewShwPageCR3);
145
146	/* Mark the page as locked; disallow flushing. */
147	pgmPoolLockPage(pPool, pNewShwPageCR3);
148
149	/* Set the current hypervisor CR3. */
150	CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
151
152	PGM_UNLOCK(pVM);
153	return rc;
154	#else
155	NOREF(pVCpu); NOREF(GCPhysCR3);
156	return VINF_SUCCESS;
157	#endif
158	}
159
160
161	#ifndef IN_RING3
162
163	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
164	/**
165	* Deal with a guest page fault.
166	*
167	* @returns Strict VBox status code.
168	* @retval VINF_EM_RAW_GUEST_TRAP
169	* @retval VINF_EM_RAW_EMULATE_INSTR
170	*
171	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
172	* @param pWalk The guest page table walk result.
173	* @param uErr The error code.
174	*/
175	PGM_BTH_DECL(VBOXSTRICTRC, Trap0eHandlerGuestFault)(PVMCPUCC pVCpu, PPGMPTWALK pWalk, RTGCUINT uErr)
176	{
177	/*
178	* Calc the error code for the guest trap.
179	*/
180	uint32_t uNewErr = GST_IS_NX_ACTIVE(pVCpu)
181	? uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID)
182	: uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US);
183	if ( pWalk->fRsvdError
184	\|\| pWalk->fBadPhysAddr)
185	{
186	uNewErr \|= X86_TRAP_PF_RSVD \| X86_TRAP_PF_P;
187	Assert(!pWalk->fNotPresent);
188	}
189	else if (!pWalk->fNotPresent)
190	uNewErr \|= X86_TRAP_PF_P;
191	TRPMSetErrorCode(pVCpu, uNewErr);
192
193	LogFlow(("Guest trap; cr2=%RGv uErr=%RGv lvl=%d\n", pWalk->GCPtr, uErr, pWalk->uLevel));
194	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2GuestTrap; });
195	return VINF_EM_RAW_GUEST_TRAP;
196	}
197	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
198
199
200	#if !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
201	/**
202	* Deal with a guest page fault.
203	*
204	* The caller has taken the PGM lock.
205	*
206	* @returns Strict VBox status code.
207	*
208	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
209	* @param uErr The error code.
210	* @param pRegFrame The register frame.
211	* @param pvFault The fault address.
212	* @param pPage The guest page at @a pvFault.
213	* @param pWalk The guest page table walk result.
214	* @param pGstWalk The guest paging-mode specific walk information.
215	* @param pfLockTaken PGM lock taken here or not (out). This is true
216	* when we're called.
217	*/
218	static VBOXSTRICTRC PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame,
219	RTGCPTR pvFault, PPGMPAGE pPage, bool *pfLockTaken
220	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \|\| defined(DOXYGEN_RUNNING)
221	, PPGMPTWALK pWalk
222	, PGSTPTWALK pGstWalk
223	# endif
224	)
225	{
226	# if !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
227	GSTPDE const PdeSrcDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
228	# endif
229	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
230	VBOXSTRICTRC rcStrict;
231
232	if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
233	{
234	/*
235	* Physical page access handler.
236	*/
237	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
238	const RTGCPHYS GCPhysFault = pWalk->GCPhys;
239	# else
240	const RTGCPHYS GCPhysFault = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault);
241	# endif
242	PPGMPHYSHANDLER pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
243	if (pCur)
244	{
245	PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
246
247	# ifdef PGM_SYNC_N_PAGES
248	/*
249	* If the region is write protected and we got a page not present fault, then sync
250	* the pages. If the fault was caused by a read, then restart the instruction.
251	* In case of write access continue to the GC write handler.
252	*
253	* ASSUMES that there is only one handler per page or that they have similar write properties.
254	*/
255	if ( !(uErr & X86_TRAP_PF_P)
256	&& pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
257	{
258	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
259	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
260	# else
261	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
262	# endif
263	if ( RT_FAILURE(rcStrict)
264	\|\| !(uErr & X86_TRAP_PF_RW)
265	\|\| rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
266	{
267	AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
268	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
269	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
270	return rcStrict;
271	}
272	}
273	# endif
274	# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
275	/*
276	* If the access was not thru a #PF(RSVD\|...) resync the page.
277	*/
278	if ( !(uErr & X86_TRAP_PF_RSVD)
279	&& pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
280	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
281	&& (pWalk->fEffective & (PGM_PTATTRS_W_MASK \| PGM_PTATTRS_US_MASK))
282	== PGM_PTATTRS_W_MASK /** @todo Remove pGstWalk->Core.fEffectiveUS and X86_PTE_US further down in the sync code. */
283	# endif
284	)
285	{
286	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
287	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
288	# else
289	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
290	# endif
291	if ( RT_FAILURE(rcStrict)
292	\|\| rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
293	{
294	AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
295	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
296	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
297	return rcStrict;
298	}
299	}
300	# endif
301
302	AssertMsg( pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
303	\|\| (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW)),
304	("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enmKind=%d\n",
305	pvFault, GCPhysFault, pPage, uErr, pCurType->enmKind));
306	if (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
307	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysWrite);
308	else
309	{
310	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAll);
311	if (uErr & X86_TRAP_PF_RSVD) STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAllOpt);
312	}
313
314	if (pCurType->pfnPfHandler)
315	{
316	STAM_PROFILE_START(&pCur->Stat, h);
317
318	if (pCurType->fKeepPgmLock)
319	{
320	rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pRegFrame, pvFault, GCPhysFault,
321	!pCurType->fRing0DevInsIdx ? pCur->uUser
322	: (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser));
323
324	# ifdef VBOX_WITH_STATISTICS
325	pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault); /* paranoia in case the handler deregistered itself */
326	if (pCur)
327	STAM_PROFILE_STOP(&pCur->Stat, h);
328	# endif
329	}
330	else
331	{
332	uint64_t const uUser = !pCurType->fRing0DevInsIdx ? pCur->uUser
333	: (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser);
334	PGM_UNLOCK(pVM);
335	*pfLockTaken = false;
336
337	rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pRegFrame, pvFault, GCPhysFault, uUser);
338
339	# ifdef VBOX_WITH_STATISTICS
340	PGM_LOCK_VOID(pVM);
341	pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
342	if (pCur)
343	STAM_PROFILE_STOP(&pCur->Stat, h);
344	PGM_UNLOCK(pVM);
345	# endif
346	}
347	}
348	else
349	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
350
351	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2HndPhys; });
352	return rcStrict;
353	}
354	}
355
356	/*
357	* There is a handled area of the page, but this fault doesn't belong to it.
358	* We must emulate the instruction.
359	*
360	* To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
361	* we first check if this was a page-not-present fault for a page with only
362	* write access handlers. Restart the instruction if it wasn't a write access.
363	*/
364	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersUnhandled);
365
366	if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
367	&& !(uErr & X86_TRAP_PF_P))
368	{
369	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
370	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
371	# else
372	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
373	# endif
374	if ( RT_FAILURE(rcStrict)
375	\|\| rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE
376	\|\| !(uErr & X86_TRAP_PF_RW))
377	{
378	AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
379	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
380	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
381	return rcStrict;
382	}
383	}
384
385	/** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
386	* It's writing to an unhandled part of the LDT page several million times.
387	*/
388	rcStrict = PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault);
389	LogFlow(("PGM: PGMInterpretInstruction -> rcStrict=%d pPage=%R[pgmpage]\n", VBOXSTRICTRC_VAL(rcStrict), pPage));
390	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2HndUnhandled; });
391	return rcStrict;
392	} /* if any kind of handler */
393	# endif /* !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE*/
394
395
396	/**
397	* \#PF Handler for raw-mode guest execution.
398	*
399	* @returns VBox status code (appropriate for trap handling and GC return).
400	*
401	* @param pVCpu The cross context virtual CPU structure.
402	* @param uErr The trap error code.
403	* @param pRegFrame Trap register frame.
404	* @param pvFault The fault address.
405	* @param pfLockTaken PGM lock taken here or not (out)
406	*/
407	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken)
408	{
409	PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
410
411	*pfLockTaken = false;
412
413	# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \
414	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
415	&& !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
416	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
417	&& PGM_SHW_TYPE != PGM_TYPE_NONE
418	int rc;
419
420	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
421	/*
422	* Walk the guest page translation tables and check if it's a guest fault.
423	*/
424	PGMPTWALK Walk;
425	GSTPTWALK GstWalk;
426	rc = PGM_GST_NAME(Walk)(pVCpu, pvFault, &Walk, &GstWalk);
427	if (RT_FAILURE_NP(rc))
428	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &Walk, uErr));
429
430	/* assert some GstWalk sanity. */
431	# if PGM_GST_TYPE == PGM_TYPE_AMD64
432	/AssertMsg(GstWalk.Pml4e.u == GstWalk.pPml4e->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pml4e.u, (uint64_t)GstWalk.pPml4e->u)); - not always true with SMP guests. /
433	# endif
434	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
435	/AssertMsg(GstWalk.Pdpe.u == GstWalk.pPdpe->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pdpe.u, (uint64_t)GstWalk.pPdpe->u)); - ditto /
436	# endif
437	/AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); - ditto /
438	/AssertMsg(GstWalk.Core.fBigPage \|\| GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); - ditto /
439	Assert(Walk.fSucceeded);
440	Assert(Walk.fEffective & PGM_PTATTRS_R_MASK);
441
442	if (uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID))
443	{
444	if ( ( (uErr & X86_TRAP_PF_RW)
445	&& !(Walk.fEffective & PGM_PTATTRS_W_MASK)
446	&& ( (uErr & X86_TRAP_PF_US)
447	\|\| CPUMIsGuestR0WriteProtEnabled(pVCpu)) )
448	\|\| ((uErr & X86_TRAP_PF_US) && !(Walk.fEffective & PGM_PTATTRS_US_MASK))
449	\|\| ((uErr & X86_TRAP_PF_ID) && (Walk.fEffective & PGM_PTATTRS_NX_MASK))
450	)
451	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &Walk, uErr));
452	}
453
454	/* Take the big lock now before we update flags. */
455	*pfLockTaken = true;
456	PGM_LOCK_VOID(pVM);
457
458	/*
459	* Set the accessed and dirty flags.
460	*/
461	/** @todo Should probably use cmpxchg logic here as we're potentially racing
462	* other CPUs in SMP configs. (the lock isn't enough, since we take it
463	* after walking and the page tables could be stale already) */
464	# if PGM_GST_TYPE == PGM_TYPE_AMD64
465	if (!(GstWalk.Pml4e.u & X86_PML4E_A))
466	{
467	GstWalk.Pml4e.u \|= X86_PML4E_A;
468	GST_ATOMIC_OR(&GstWalk.pPml4e->u, X86_PML4E_A);
469	}
470	if (!(GstWalk.Pdpe.u & X86_PDPE_A))
471	{
472	GstWalk.Pdpe.u \|= X86_PDPE_A;
473	GST_ATOMIC_OR(&GstWalk.pPdpe->u, X86_PDPE_A);
474	}
475	# endif
476	if (Walk.fBigPage)
477	{
478	Assert(GstWalk.Pde.u & X86_PDE_PS);
479	if (uErr & X86_TRAP_PF_RW)
480	{
481	if ((GstWalk.Pde.u & (X86_PDE4M_A \| X86_PDE4M_D)) != (X86_PDE4M_A \| X86_PDE4M_D))
482	{
483	GstWalk.Pde.u \|= X86_PDE4M_A \| X86_PDE4M_D;
484	GST_ATOMIC_OR(&GstWalk.pPde->u, X86_PDE4M_A \| X86_PDE4M_D);
485	}
486	}
487	else
488	{
489	if (!(GstWalk.Pde.u & X86_PDE4M_A))
490	{
491	GstWalk.Pde.u \|= X86_PDE4M_A;
492	GST_ATOMIC_OR(&GstWalk.pPde->u, X86_PDE4M_A);
493	}
494	}
495	}
496	else
497	{
498	Assert(!(GstWalk.Pde.u & X86_PDE_PS));
499	if (!(GstWalk.Pde.u & X86_PDE_A))
500	{
501	GstWalk.Pde.u \|= X86_PDE_A;
502	GST_ATOMIC_OR(&GstWalk.pPde->u, X86_PDE_A);
503	}
504
505	if (uErr & X86_TRAP_PF_RW)
506	{
507	# ifdef VBOX_WITH_STATISTICS
508	if (GstWalk.Pte.u & X86_PTE_D)
509	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageAlreadyDirty));
510	else
511	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtiedPage));
512	# endif
513	if ((GstWalk.Pte.u & (X86_PTE_A \| X86_PTE_D)) != (X86_PTE_A \| X86_PTE_D))
514	{
515	GstWalk.Pte.u \|= X86_PTE_A \| X86_PTE_D;
516	GST_ATOMIC_OR(&GstWalk.pPte->u, X86_PTE_A \| X86_PTE_D);
517	}
518	}
519	else
520	{
521	if (!(GstWalk.Pte.u & X86_PTE_A))
522	{
523	GstWalk.Pte.u \|= X86_PTE_A;
524	GST_ATOMIC_OR(&GstWalk.pPte->u, X86_PTE_A);
525	}
526	}
527	Assert(GstWalk.Pte.u == GstWalk.pPte->u);
528	}
529	#if 0
530	/* Disabling this since it's not reliable for SMP, see @bugref{10092#c22}. */
531	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u \|\| GstWalk.pPte->u == GstWalk.pPde->u,
532	("%RX64 %RX64 pPte=%p pPde=%p Pte=%RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u, GstWalk.pPte, GstWalk.pPde, (uint64_t)GstWalk.pPte->u));
533	#endif
534
535	# else /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
536	GSTPDE const PdeSrcDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A}; /** @todo eliminate this */
537
538	/* Take the big lock now. */
539	*pfLockTaken = true;
540	PGM_LOCK_VOID(pVM);
541	# endif /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
542
543	# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
544	/*
545	* If it is a reserved bit fault we know that it is an MMIO (access
546	* handler) related fault and can skip some 200 lines of code.
547	*/
548	if (uErr & X86_TRAP_PF_RSVD)
549	{
550	Assert(uErr & X86_TRAP_PF_P);
551	PPGMPAGE pPage;
552	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
553	rc = pgmPhysGetPageEx(pVM, Walk.GCPhys, &pPage);
554	if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
555	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
556	pfLockTaken, &Walk, &GstWalk));
557	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
558	# else
559	rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault), &pPage);
560	if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
561	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
562	pfLockTaken));
563	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
564	# endif
565	AssertRC(rc);
566	PGM_INVL_PG(pVCpu, pvFault);
567	return rc; /* Restart with the corrected entry. */
568	}
569	# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
570
571	/*
572	* Fetch the guest PDE, PDPE and PML4E.
573	*/
574	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
575	const unsigned iPDDst = pvFault >> SHW_PD_SHIFT;
576	PX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
577
578	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
579	const unsigned iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK; /* pPDDst index, not used with the pool. */
580	PX86PDPAE pPDDst;
581	# if PGM_GST_TYPE == PGM_TYPE_PAE
582	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, GstWalk.Pdpe.u, &pPDDst);
583	# else
584	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, X86_PDPE_P, &pPDDst); /* RW, US and A are reserved in PAE mode. */
585	# endif
586	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
587
588	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
589	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
590	PX86PDPAE pPDDst;
591	# if PGM_GST_TYPE == PGM_TYPE_PROT /* (AMD-V nested paging) */
592	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A,
593	X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A, &pPDDst);
594	# else
595	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, GstWalk.Pml4e.u, GstWalk.Pdpe.u, &pPDDst);
596	# endif
597	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
598
599	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
600	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
601	PEPTPD pPDDst;
602	rc = pgmShwGetEPTPDPtr(pVCpu, pvFault, NULL, &pPDDst);
603	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
604	# endif
605	Assert(pPDDst);
606
607	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
608	/*
609	* Dirty page handling.
610	*
611	* If we successfully correct the write protection fault due to dirty bit
612	* tracking, then return immediately.
613	*/
614	if (uErr & X86_TRAP_PF_RW) /* write fault? */
615	{
616	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyBitTracking), a);
617	rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, &pPDDst->a[iPDDst], GstWalk.pPde, pvFault);
618	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyBitTracking), a);
619	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
620	{
621	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0
622	= rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
623	? &pVCpu->pgm.s.Stats.StatRZTrap0eTime2DirtyAndAccessed
624	: &pVCpu->pgm.s.Stats.StatRZTrap0eTime2GuestTrap; });
625	Log8(("Trap0eHandler: returns VINF_SUCCESS\n"));
626	return VINF_SUCCESS;
627	}
628	#ifdef DEBUG_bird
629	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u \|\| GstWalk.pPte->u == GstWalk.pPde->u \|\| pVM->cCpus > 1, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); // - triggers with smp w7 guests.
630	AssertMsg(Walk.fBigPage \|\| GstWalk.Pte.u == GstWalk.pPte->u \|\| pVM->cCpus > 1, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); // - ditto.
631	#endif
632	}
633
634	# if 0 /* rarely useful; leave for debugging. */
635	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0ePD[iPDSrc]);
636	# endif
637	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
638
639	/*
640	* A common case is the not-present error caused by lazy page table syncing.
641	*
642	* It is IMPORTANT that we weed out any access to non-present shadow PDEs
643	* here so we can safely assume that the shadow PT is present when calling
644	* SyncPage later.
645	*
646	* On failure, we ASSUME that SyncPT is out of memory or detected some kind
647	* of mapping conflict and defer to SyncCR3 in R3.
648	* (Again, we do NOT support access handlers for non-present guest pages.)
649	*
650	*/
651	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
652	Assert(GstWalk.Pde.u & X86_PDE_P);
653	# endif
654	if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
655	&& !SHW_PDE_IS_P(pPDDst->a[iPDDst]))
656	{
657	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2SyncPT; });
658	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
659	LogFlow(("=>SyncPT %04x = %08RX64\n", (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, (uint64_t)GstWalk.Pde.u));
660	rc = PGM_BTH_NAME(SyncPT)(pVCpu, (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, GstWalk.pPd, pvFault);
661	# else
662	LogFlow(("=>SyncPT pvFault=%RGv\n", pvFault));
663	rc = PGM_BTH_NAME(SyncPT)(pVCpu, 0, NULL, pvFault);
664	# endif
665	if (RT_SUCCESS(rc))
666	return rc;
667	Log(("SyncPT: %RGv failed!! rc=%Rrc\n", pvFault, rc));
668	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
669	return VINF_PGM_SYNC_CR3;
670	}
671
672	/*
673	* Check if this fault address is flagged for special treatment,
674	* which means we'll have to figure out the physical address and
675	* check flags associated with it.
676	*
677	* ASSUME that we can limit any special access handling to pages
678	* in page tables which the guest believes to be present.
679	*/
680	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
681	RTGCPHYS GCPhys = Walk.GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
682	# else
683	RTGCPHYS GCPhys = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
684	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
685	PPGMPAGE pPage;
686	rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
687	if (RT_FAILURE(rc))
688	{
689	/*
690	* When the guest accesses invalid physical memory (e.g. probing
691	* of RAM or accessing a remapped MMIO range), then we'll fall
692	* back to the recompiler to emulate the instruction.
693	*/
694	LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhys, rc));
695	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersInvalid);
696	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2InvalidPhys; });
697	return VINF_EM_RAW_EMULATE_INSTR;
698	}
699
700	/*
701	* Any handlers for this page?
702	*/
703	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
704	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
705	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, pfLockTaken,
706	&Walk, &GstWalk));
707	# else
708	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, pfLockTaken));
709	# endif
710
711	/*
712	* We are here only if page is present in Guest page tables and
713	* trap is not handled by our handlers.
714	*
715	* Check it for page out-of-sync situation.
716	*/
717	if (!(uErr & X86_TRAP_PF_P))
718	{
719	/*
720	* Page is not present in our page tables. Try to sync it!
721	*/
722	if (uErr & X86_TRAP_PF_US)
723	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncUser));
724	else /* supervisor */
725	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
726
727	if (PGM_PAGE_IS_BALLOONED(pPage))
728	{
729	/* Emulate reads from ballooned pages as they are not present in
730	our shadow page tables. (Required for e.g. Solaris guests; soft
731	ecc, random nr generator.) */
732	rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
733	LogFlow(("PGM: PGMInterpretInstruction balloon -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
734	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncBallloon));
735	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Ballooned; });
736	return rc;
737	}
738
739	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
740	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
741	# else
742	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
743	# endif
744	if (RT_SUCCESS(rc))
745	{
746	/* The page was successfully synced, return to the guest. */
747	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSync; });
748	return VINF_SUCCESS;
749	}
750	}
751	else /* uErr & X86_TRAP_PF_P: */
752	{
753	/*
754	* Write protected pages are made writable when the guest makes the
755	* first write to it. This happens for pages that are shared, write
756	* monitored or not yet allocated.
757	*
758	* We may also end up here when CR0.WP=0 in the guest.
759	*
760	* Also, a side effect of not flushing global PDEs are out of sync
761	* pages due to physical monitored regions, that are no longer valid.
762	* Assume for now it only applies to the read/write flag.
763	*/
764	if (uErr & X86_TRAP_PF_RW)
765	{
766	/*
767	* Check if it is a read-only page.
768	*/
769	if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
770	{
771	Log(("PGM #PF: Make writable: %RGp %R[pgmpage] pvFault=%RGp uErr=%#x\n", GCPhys, pPage, pvFault, uErr));
772	Assert(!PGM_PAGE_IS_ZERO(pPage));
773	AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
774	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2MakeWritable; });
775
776	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
777	if (rc != VINF_SUCCESS)
778	{
779	AssertMsg(rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("%Rrc\n", rc));
780	return rc;
781	}
782	if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
783	return VINF_EM_NO_MEMORY;
784	}
785
786	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
787	/*
788	* Check to see if we need to emulate the instruction if CR0.WP=0.
789	*/
790	if ( !(Walk.fEffective & PGM_PTATTRS_W_MASK)
791	&& (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG
792	&& CPUMGetGuestCPL(pVCpu) < 3)
793	{
794	Assert((uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RW \| X86_TRAP_PF_P));
795
796	/*
797	* The Netware WP0+RO+US hack.
798	*
799	* Netware sometimes(/always?) runs with WP0. It has been observed doing
800	* excessive write accesses to pages which are mapped with US=1 and RW=0
801	* while WP=0. This causes a lot of exits and extremely slow execution.
802	* To avoid trapping and emulating every write here, we change the shadow
803	* page table entry to map it as US=0 and RW=1 until user mode tries to
804	* access it again (see further below). We count these shadow page table
805	* changes so we can avoid having to clear the page pool every time the WP
806	* bit changes to 1 (see PGMCr0WpEnabled()).
807	*/
808	# if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) && 1
809	if ( (Walk.fEffective & (PGM_PTATTRS_W_MASK \| PGM_PTATTRS_US_MASK)) == PGM_PTATTRS_US_MASK
810	&& (Walk.fBigPage \|\| (GstWalk.Pde.u & X86_PDE_RW))
811	&& pVM->cCpus == 1 /* Sorry, no go on SMP. Add CFGM option? */)
812	{
813	Log(("PGM #PF: Netware WP0+RO+US hack: pvFault=%RGp uErr=%#x (big=%d)\n", pvFault, uErr, Walk.fBigPage));
814	rc = pgmShwMakePageSupervisorAndWritable(pVCpu, pvFault, Walk.fBigPage, PGM_MK_PG_IS_WRITE_FAULT);
815	if (rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3)
816	{
817	PGM_INVL_PG(pVCpu, pvFault);
818	pVCpu->pgm.s.cNetwareWp0Hacks++;
819	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Wp0RoUsHack; });
820	return rc;
821	}
822	AssertMsg(RT_FAILURE_NP(rc), ("%Rrc\n", rc));
823	Log(("pgmShwMakePageSupervisorAndWritable(%RGv) failed with rc=%Rrc - ignored\n", pvFault, rc));
824	}
825	# endif
826
827	/* Interpret the access. */
828	rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
829	Log(("PGM #PF: WP0 emulation (pvFault=%RGp uErr=%#x cpl=%d fBig=%d fEffUs=%d)\n", pvFault, uErr, CPUMGetGuestCPL(pVCpu), Walk.fBigPage, !!(Walk.fEffective & PGM_PTATTRS_US_MASK)));
830	if (RT_SUCCESS(rc))
831	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eWPEmulInRZ);
832	else
833	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eWPEmulToR3);
834	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2WPEmulation; });
835	return rc;
836	}
837	# endif
838	/// @todo count the above case; else
839	if (uErr & X86_TRAP_PF_US)
840	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncUserWrite));
841	else /* supervisor */
842	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
843
844	/*
845	* Sync the page.
846	*
847	* Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
848	* page is not present, which is not true in this case.
849	*/
850	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
851	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
852	# else
853	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
854	# endif
855	if (RT_SUCCESS(rc))
856	{
857	/*
858	* Page was successfully synced, return to guest but invalidate
859	* the TLB first as the page is very likely to be in it.
860	*/
861	# if PGM_SHW_TYPE == PGM_TYPE_EPT
862	HMInvalidatePhysPage(pVM, (RTGCPHYS)pvFault);
863	# else
864	PGM_INVL_PG(pVCpu, pvFault);
865	# endif
866	# ifdef VBOX_STRICT
867	PGMPTWALK GstPageWalk;
868	GstPageWalk.GCPhys = RTGCPHYS_MAX;
869	if (!pVM->pgm.s.fNestedPaging)
870	{
871	rc = PGMGstGetPage(pVCpu, pvFault, &GstPageWalk);
872	AssertMsg(RT_SUCCESS(rc) && ((GstPageWalk.fEffective & X86_PTE_RW) \|\| ((CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG && CPUMGetGuestCPL(pVCpu) < 3)), ("rc=%Rrc fPageGst=%RX64\n", rc, GstPageWalk.fEffective));
873	LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GstPageWalk.GCPhys, GstPageWalk.fEffective));
874	}
875	# if 0 /* Bogus! Triggers incorrectly with w7-64 and later for the SyncPage case: "Pde at %RGv changed behind our back?" */
876	uint64_t fPageShw = 0;
877	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
878	AssertMsg((RT_SUCCESS(rc) && (fPageShw & X86_PTE_RW)) \|\| pVM->cCpus > 1 /* new monitor can be installed/page table flushed between the trap exit and PGMTrap0eHandler */,
879	("rc=%Rrc fPageShw=%RX64 GCPhys2=%RGp fPageGst=%RX64 pvFault=%RGv\n", rc, fPageShw, GstPageWalk.GCPhys, fPageGst, pvFault));
880	# endif
881	# endif /* VBOX_STRICT */
882	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndObs; });
883	return VINF_SUCCESS;
884	}
885	}
886	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
887	/*
888	* Check for Netware WP0+RO+US hack from above and undo it when user
889	* mode accesses the page again.
890	*/
891	else if ( (Walk.fEffective & (PGM_PTATTRS_W_MASK \| PGM_PTATTRS_US_MASK)) == PGM_PTATTRS_US_MASK
892	&& (Walk.fBigPage \|\| (GstWalk.Pde.u & X86_PDE_RW))
893	&& pVCpu->pgm.s.cNetwareWp0Hacks > 0
894	&& (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG
895	&& CPUMGetGuestCPL(pVCpu) == 3
896	&& pVM->cCpus == 1
897	)
898	{
899	Log(("PGM #PF: Undo netware WP0+RO+US hack: pvFault=%RGp uErr=%#x\n", pvFault, uErr));
900	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
901	if (RT_SUCCESS(rc))
902	{
903	PGM_INVL_PG(pVCpu, pvFault);
904	pVCpu->pgm.s.cNetwareWp0Hacks--;
905	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Wp0RoUsUnhack; });
906	return VINF_SUCCESS;
907	}
908	}
909	# endif /* PGM_WITH_PAGING */
910
911	/** @todo else: why are we here? */
912
913	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && defined(VBOX_STRICT)
914	/*
915	* Check for VMM page flags vs. Guest page flags consistency.
916	* Currently only for debug purposes.
917	*/
918	if (RT_SUCCESS(rc))
919	{
920	/* Get guest page flags. */
921	PGMPTWALK GstPageWalk;
922	int rc2 = PGMGstGetPage(pVCpu, pvFault, &GstPageWalk);
923	if (RT_SUCCESS(rc2))
924	{
925	uint64_t fPageShw = 0;
926	rc2 = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
927
928	#if 0
929	/*
930	* Compare page flags.
931	* Note: we have AVL, A, D bits desynced.
932	*/
933	AssertMsg( (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK))
934	== (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK))
935	\|\| ( pVCpu->pgm.s.cNetwareWp0Hacks > 0
936	&& (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US))
937	== (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US))
938	&& (fPageShw & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW
939	&& (fPageGst & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_US),
940	("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64 rc=%d\n",
941	pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst, rc));
942	01:01:15.623511 00:08:43.266063 Expression: (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) \|\| ( pVCpu->pgm.s.cNetwareWp0Hacks > 0 && (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US)) && (fPageShw & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW && (fPageGst & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_US)
943	01:01:15.623511 00:08:43.266064 Location : e:\vbox\svn\trunk\srcPage flags mismatch! pvFault=fffff801b0d7b000 uErr=11 GCPhys=0000000019b52000 fPageShw=0 fPageGst=77b0000000000121 rc=0
944
945	01:01:15.625516 00:08:43.268051 Expression: (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) \|\| ( pVCpu->pgm.s.cNetwareWp0Hacks > 0 && (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US)) && (fPageShw & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW && (fPageGst & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_US)
946	01:01:15.625516 00:08:43.268051 Location :
947	e:\vbox\svn\trunk\srcPage flags mismatch!
948	pvFault=fffff801b0d7b000
949	uErr=11 X86_TRAP_PF_ID \| X86_TRAP_PF_P
950	GCPhys=0000000019b52000
951	fPageShw=0
952	fPageGst=77b0000000000121
953	rc=0
954	#endif
955
956	}
957	else
958	AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc));
959	}
960	else
961	AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc));
962	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && VBOX_STRICT */
963	}
964
965
966	/*
967	* If we get here it is because something failed above, i.e. most like guru
968	* meditiation time.
969	*/
970	LogRel(("%s: returns rc=%Rrc pvFault=%RGv uErr=%RX64 cs:rip=%04x:%08RX64\n",
971	__PRETTY_FUNCTION__, rc, pvFault, (uint64_t)uErr, pRegFrame->cs.Sel, pRegFrame->rip));
972	return rc;
973
974	# else /* Nested paging, EPT except PGM_GST_TYPE = PROT, NONE. */
975	NOREF(uErr); NOREF(pRegFrame); NOREF(pvFault);
976	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
977	return VERR_PGM_NOT_USED_IN_MODE;
978	# endif
979	}
980
981	#endif /* !IN_RING3 */
982
983
984	/**
985	* Emulation of the invlpg instruction.
986	*
987	*
988	* @returns VBox status code.
989	*
990	* @param pVCpu The cross context virtual CPU structure.
991	* @param GCPtrPage Page to invalidate.
992	*
993	* @remark ASSUMES that the guest is updating before invalidating. This order
994	* isn't required by the CPU, so this is speculative and could cause
995	* trouble.
996	* @remark No TLB shootdown is done on any other VCPU as we assume that
997	* invlpg emulation is the only reason for calling this function.
998	* (The guest has to shoot down TLB entries on other CPUs itself)
999	* Currently true, but keep in mind!
1000	*
1001	* @todo Clean this up! Most of it is (or should be) no longer necessary as we catch all page table accesses.
1002	* Should only be required when PGMPOOL_WITH_OPTIMIZED_DIRTY_PT is active (PAE or AMD64 (for now))
1003	*/
1004	PGM_BTH_DECL(int, InvalidatePage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
1005	{
1006	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1007	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
1008	&& PGM_SHW_TYPE != PGM_TYPE_NONE
1009	int rc;
1010	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1011	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1012
1013	PGM_LOCK_ASSERT_OWNER(pVM);
1014
1015	LogFlow(("InvalidatePage %RGv\n", GCPtrPage));
1016
1017	/*
1018	* Get the shadow PD entry and skip out if this PD isn't present.
1019	* (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
1020	*/
1021	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1022	const unsigned iPDDst = (uint32_t)GCPtrPage >> SHW_PD_SHIFT;
1023	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1024
1025	/* Fetch the pgm pool shadow descriptor. */
1026	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1027	# ifdef IN_RING3 /* Possible we didn't resync yet when called from REM. */
1028	if (!pShwPde)
1029	{
1030	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1031	return VINF_SUCCESS;
1032	}
1033	# else
1034	Assert(pShwPde);
1035	# endif
1036
1037	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1038	const unsigned iPdpt = (uint32_t)GCPtrPage >> X86_PDPT_SHIFT;
1039	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
1040
1041	/* If the shadow PDPE isn't present, then skip the invalidate. */
1042	# ifdef IN_RING3 /* Possible we didn't resync yet when called from REM. */
1043	if (!pPdptDst \|\| !(pPdptDst->a[iPdpt].u & X86_PDPE_P))
1044	# else
1045	if (!(pPdptDst->a[iPdpt].u & X86_PDPE_P))
1046	# endif
1047	{
1048	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1049	PGM_INVL_PG(pVCpu, GCPtrPage);
1050	return VINF_SUCCESS;
1051	}
1052
1053	/* Fetch the pgm pool shadow descriptor. */
1054	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
1055	AssertReturn(pShwPde, VERR_PGM_POOL_GET_PAGE_FAILED);
1056
1057	PX86PDPAE pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1058	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1059	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1060
1061	# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1062	/* PML4 */
1063	/const unsigned iPml4 = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;/
1064	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1065	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1066	PX86PDPAE pPDDst;
1067	PX86PDPT pPdptDst;
1068	PX86PML4E pPml4eDst;
1069	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eDst, &pPdptDst, &pPDDst);
1070	if (rc != VINF_SUCCESS)
1071	{
1072	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT \|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
1073	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1074	PGM_INVL_PG(pVCpu, GCPtrPage);
1075	return VINF_SUCCESS;
1076	}
1077	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1078	Assert(pPDDst);
1079	Assert(pPdptDst->a[iPdpt].u & X86_PDPE_P);
1080
1081	/* Fetch the pgm pool shadow descriptor. */
1082	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & SHW_PDPE_PG_MASK);
1083	Assert(pShwPde);
1084
1085	# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1086
1087	const SHWPDE PdeDst = *pPdeDst;
1088	if (!(PdeDst.u & X86_PDE_P))
1089	{
1090	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1091	PGM_INVL_PG(pVCpu, GCPtrPage);
1092	return VINF_SUCCESS;
1093	}
1094
1095	/*
1096	* Get the guest PD entry and calc big page.
1097	*/
1098	# if PGM_GST_TYPE == PGM_TYPE_32BIT
1099	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
1100	const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
1101	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1102	# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1103	unsigned iPDSrc = 0;
1104	# if PGM_GST_TYPE == PGM_TYPE_PAE
1105	X86PDPE PdpeSrcIgn;
1106	PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrcIgn);
1107	# else /* AMD64 */
1108	PX86PML4E pPml4eSrcIgn;
1109	X86PDPE PdpeSrcIgn;
1110	PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrcIgn, &PdpeSrcIgn, &iPDSrc);
1111	# endif
1112	GSTPDE PdeSrc;
1113
1114	if (pPDSrc)
1115	PdeSrc = pPDSrc->a[iPDSrc];
1116	else
1117	PdeSrc.u = 0;
1118	# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1119	const bool fWasBigPage = RT_BOOL(PdeDst.u & PGM_PDFLAGS_BIG_PAGE);
1120	const bool fIsBigPage = (PdeSrc.u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu);
1121	if (fWasBigPage != fIsBigPage)
1122	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1123
1124	# ifdef IN_RING3
1125	/*
1126	* If a CR3 Sync is pending we may ignore the invalidate page operation
1127	* depending on the kind of sync and if it's a global page or not.
1128	* This doesn't make sense in GC/R0 so we'll skip it entirely there.
1129	*/
1130	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
1131	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
1132	\|\| ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
1133	&& fIsBigPage
1134	&& (PdeSrc.u & X86_PDE4M_G)
1135	)
1136	)
1137	# else
1138	if (VM_FF_IS_ANY_SET(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
1139	# endif
1140	{
1141	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1142	return VINF_SUCCESS;
1143	}
1144	# endif /* IN_RING3 */
1145
1146	/*
1147	* Deal with the Guest PDE.
1148	*/
1149	rc = VINF_SUCCESS;
1150	if (PdeSrc.u & X86_PDE_P)
1151	{
1152	Assert( (PdeSrc.u & X86_PDE_US) == (PdeDst.u & X86_PDE_US)
1153	&& ((PdeSrc.u & X86_PDE_RW) \|\| !(PdeDst.u & X86_PDE_RW) \|\| pVCpu->pgm.s.cNetwareWp0Hacks > 0));
1154	if (!fIsBigPage)
1155	{
1156	/*
1157	* 4KB - page.
1158	*/
1159	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1160	RTGCPHYS GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
1161
1162	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1163	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1164	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
1165	# endif
1166	if (pShwPage->GCPhys == GCPhys)
1167	{
1168	/* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1169	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1170
1171	PGSTPT pPTSrc;
1172	rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
1173	if (RT_SUCCESS(rc))
1174	{
1175	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1176	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1177	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1178	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1179	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
1180	GCPtrPage, PteSrc.u & X86_PTE_P,
1181	(PteSrc.u & PdeSrc.u & X86_PTE_RW),
1182	(PteSrc.u & PdeSrc.u & X86_PTE_US),
1183	(uint64_t)PteSrc.u,
1184	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
1185	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
1186	}
1187	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage4KBPages));
1188	PGM_INVL_PG(pVCpu, GCPtrPage);
1189	}
1190	else
1191	{
1192	/*
1193	* The page table address changed.
1194	*/
1195	LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
1196	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1197	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1198	SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
1199	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1200	PGM_INVL_VCPU_TLBS(pVCpu);
1201	}
1202	}
1203	else
1204	{
1205	/*
1206	* 2/4MB - page.
1207	*/
1208	/* Before freeing the page, check if anything really changed. */
1209	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1210	RTGCPHYS GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
1211	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1212	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1213	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
1214	# endif
1215	if ( pShwPage->GCPhys == GCPhys
1216	&& pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1217	{
1218	/* ASSUMES a the given bits are identical for 4M and normal PDEs */
1219	/** @todo This test is wrong as it cannot check the G bit!
1220	* FIXME */
1221	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US))
1222	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US))
1223	&& ( (PdeSrc.u & X86_PDE4M_D) /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1224	\|\| (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1225	{
1226	LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1227	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip));
1228	return VINF_SUCCESS;
1229	}
1230	}
1231
1232	/*
1233	* Ok, the page table is present and it's been changed in the guest.
1234	* If we're in host context, we'll just mark it as not present taking the lazy approach.
1235	* We could do this for some flushes in GC too, but we need an algorithm for
1236	* deciding which 4MB pages containing code likely to be executed very soon.
1237	*/
1238	LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1239	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1240	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1241	SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
1242	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage4MBPages));
1243	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
1244	}
1245	}
1246	else
1247	{
1248	/*
1249	* Page directory is not present, mark shadow PDE not present.
1250	*/
1251	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1252	SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
1253	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePagePDNPs));
1254	PGM_INVL_PG(pVCpu, GCPtrPage);
1255	}
1256	return rc;
1257
1258	#else /* guest real and protected mode, nested + ept, none. */
1259	/* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1260	NOREF(pVCpu); NOREF(GCPtrPage);
1261	return VINF_SUCCESS;
1262	#endif
1263	}
1264
1265	#if PGM_SHW_TYPE != PGM_TYPE_NONE
1266
1267	/**
1268	* Update the tracking of shadowed pages.
1269	*
1270	* @param pVCpu The cross context virtual CPU structure.
1271	* @param pShwPage The shadow page.
1272	* @param HCPhys The physical page we is being dereferenced.
1273	* @param iPte Shadow PTE index
1274	* @param GCPhysPage Guest physical address (only valid if pShwPage->fDirty is set)
1275	*/
1276	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPUCC pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys, uint16_t iPte,
1277	RTGCPHYS GCPhysPage)
1278	{
1279	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1280
1281	# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1282	&& PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1283	&& (PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
1284
1285	/* Use the hint we retrieved from the cached guest PT. */
1286	if (pShwPage->fDirty)
1287	{
1288	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1289
1290	Assert(pShwPage->cPresent);
1291	Assert(pPool->cPresent);
1292	pShwPage->cPresent--;
1293	pPool->cPresent--;
1294
1295	PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysPage);
1296	AssertRelease(pPhysPage);
1297	pgmTrackDerefGCPhys(pPool, pShwPage, pPhysPage, iPte);
1298	return;
1299	}
1300	# else
1301	NOREF(GCPhysPage);
1302	# endif
1303
1304	STAM_PROFILE_START(&pVM->pgm.s.Stats.StatTrackDeref, a);
1305	LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1306
1307	/** @todo If this turns out to be a bottle neck (very likely) two things can be done:
1308	* 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1309	* 2. write protect all shadowed pages. I.e. implement caching.
1310	*/
1311	/** @todo duplicated in the 2nd half of pgmPoolTracDerefGCPhysHint */
1312
1313	/*
1314	* Find the guest address.
1315	*/
1316	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1317	pRam;
1318	pRam = pRam->CTX_SUFF(pNext))
1319	{
1320	unsigned iPage = pRam->cb >> GUEST_PAGE_SHIFT;
1321	while (iPage-- > 0)
1322	{
1323	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1324	{
1325	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1326
1327	Assert(pShwPage->cPresent);
1328	Assert(pPool->cPresent);
1329	pShwPage->cPresent--;
1330	pPool->cPresent--;
1331
1332	pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage], iPte);
1333	STAM_PROFILE_STOP(&pVM->pgm.s.Stats.StatTrackDeref, a);
1334	return;
1335	}
1336	}
1337	}
1338
1339	for (;;)
1340	AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys));
1341	}
1342
1343
1344	/**
1345	* Update the tracking of shadowed pages.
1346	*
1347	* @param pVCpu The cross context virtual CPU structure.
1348	* @param pShwPage The shadow page.
1349	* @param u16 The top 16-bit of the pPage->HCPhys.
1350	* @param pPage Pointer to the guest page. this will be modified.
1351	* @param iPTDst The index into the shadow table.
1352	*/
1353	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPUCC pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16,
1354	PPGMPAGE pPage, const unsigned iPTDst)
1355	{
1356	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1357
1358	/*
1359	* Just deal with the simple first time here.
1360	*/
1361	if (!u16)
1362	{
1363	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackVirgin);
1364	u16 = PGMPOOL_TD_MAKE(1, pShwPage->idx);
1365	/* Save the page table index. */
1366	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, iPTDst);
1367	}
1368	else
1369	u16 = pgmPoolTrackPhysExtAddref(pVM, pPage, u16, pShwPage->idx, iPTDst);
1370
1371	/* write back */
1372	Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x iPTDst=%#x\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst));
1373	PGM_PAGE_SET_TRACKING(pVM, pPage, u16);
1374
1375	/* update statistics. */
1376	pVM->pgm.s.CTX_SUFF(pPool)->cPresent++;
1377	pShwPage->cPresent++;
1378	if (pShwPage->iFirstPresent > iPTDst)
1379	pShwPage->iFirstPresent = iPTDst;
1380	}
1381
1382
1383	/**
1384	* Modifies a shadow PTE to account for access handlers.
1385	*
1386	* @param pVM The cross context VM structure.
1387	* @param pPage The page in question.
1388	* @param fPteSrc The shadowed flags of the source PTE. Must include the
1389	* A (accessed) bit so it can be emulated correctly.
1390	* @param pPteDst The shadow PTE (output). This is temporary storage and
1391	* does not need to be set atomically.
1392	*/
1393	DECLINLINE(void) PGM_BTH_NAME(SyncHandlerPte)(PVMCC pVM, PCPGMPAGE pPage, uint64_t fPteSrc, PSHWPTE pPteDst)
1394	{
1395	NOREF(pVM); RT_NOREF_PV(fPteSrc);
1396
1397	/** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No.
1398	* Update: \#PF should deal with this before or after calling the handlers. It has all the info to do the job efficiently. */
1399	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1400	{
1401	LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark read-only\n", pPage));
1402	# if PGM_SHW_TYPE == PGM_TYPE_EPT
1403	pPteDst->u = PGM_PAGE_GET_HCPHYS(pPage) \| EPT_E_READ \| EPT_E_EXECUTE \| EPT_E_MEMTYPE_WB \| EPT_E_IGNORE_PAT;
1404	# else
1405	if (fPteSrc & X86_PTE_A)
1406	{
1407	SHW_PTE_SET(*pPteDst, fPteSrc \| PGM_PAGE_GET_HCPHYS(pPage));
1408	SHW_PTE_SET_RO(*pPteDst);
1409	}
1410	else
1411	SHW_PTE_SET(*pPteDst, 0);
1412	# endif
1413	}
1414	# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
1415	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
1416	else if ( PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
1417	&& ( BTH_IS_NP_ACTIVE(pVM)
1418	\|\| (fPteSrc & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW) /** @todo Remove X86_PTE_US here and pGstWalk->Core.fEffectiveUS before the sync page test. */
1419	# if PGM_SHW_TYPE == PGM_TYPE_AMD64
1420	&& pVM->pgm.s.fLessThan52PhysicalAddressBits
1421	# endif
1422	)
1423	{
1424	LogFlow(("SyncHandlerPte: MMIO page -> invalid \n"));
1425	# if PGM_SHW_TYPE == PGM_TYPE_EPT
1426	/* 25.2.3.1: Reserved physical address bit -> EPT Misconfiguration (exit 49) */
1427	pPteDst->u = pVM->pgm.s.HCPhysInvMmioPg
1428	/* 25.2.3.1: bits 2:0 = 010b -> EPT Misconfiguration (exit 49) */
1429	\| EPT_E_WRITE
1430	/* 25.2.3.1: leaf && 2:0 != 0 && u3Emt in {2, 3, 7} -> EPT Misconfiguration */
1431	\| EPT_E_MEMTYPE_INVALID_3;
1432	# else
1433	/* Set high page frame bits that MBZ (bankers on PAE, CPU dependent on AMD64). */
1434	SHW_PTE_SET(*pPteDst, pVM->pgm.s.HCPhysInvMmioPg \| X86_PTE_PAE_MBZ_MASK_NO_NX \| X86_PTE_P);
1435	# endif
1436	}
1437	# endif
1438	# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
1439	else
1440	{
1441	LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark not present\n", pPage));
1442	SHW_PTE_SET(*pPteDst, 0);
1443	}
1444	/** @todo count these kinds of entries. */
1445	}
1446
1447
1448	/**
1449	* Creates a 4K shadow page for a guest page.
1450	*
1451	* For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1452	* physical address. The PdeSrc argument only the flags are used. No page
1453	* structured will be mapped in this function.
1454	*
1455	* @param pVCpu The cross context virtual CPU structure.
1456	* @param pPteDst Destination page table entry.
1457	* @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1458	* Can safely assume that only the flags are being used.
1459	* @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1460	* @param pShwPage Pointer to the shadow page.
1461	* @param iPTDst The index into the shadow table.
1462	*
1463	* @remark Not used for 2/4MB pages!
1464	*/
1465	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \|\| defined(DOXYGEN_RUNNING)
1466	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc,
1467	PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1468	# else
1469	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage,
1470	PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1471	# endif
1472	{
1473	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1474	RTGCPHYS GCPhysOldPage = NIL_RTGCPHYS;
1475
1476	# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1477	&& PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1478	&& (PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
1479
1480	if (pShwPage->fDirty)
1481	{
1482	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1483	PGSTPT pGstPT;
1484
1485	/* Note that iPTDst can be used to index the guest PT even in the pae/32bit combo as we copy only half the table; see pgmPoolAddDirtyPage. */
1486	pGstPT = (PGSTPT)&pPool->aDirtyPages[pShwPage->idxDirtyEntry].aPage[0];
1487	GCPhysOldPage = GST_GET_PTE_GCPHYS(pGstPT->a[iPTDst]);
1488	pGstPT->a[iPTDst].u = PteSrc.u;
1489	}
1490	# else
1491	Assert(!pShwPage->fDirty);
1492	# endif
1493
1494	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1495	if ( (PteSrc.u & X86_PTE_P)
1496	&& GST_IS_PTE_VALID(pVCpu, PteSrc))
1497	# endif
1498	{
1499	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1500	RTGCPHYS GCPhysPage = GST_GET_PTE_GCPHYS(PteSrc);
1501	# endif
1502	PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
1503
1504	/*
1505	* Find the ram range.
1506	*/
1507	PPGMPAGE pPage;
1508	int rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
1509	if (RT_SUCCESS(rc))
1510	{
1511	/* Ignore ballooned pages.
1512	Don't return errors or use a fatal assert here as part of a
1513	shadow sync range might included ballooned pages. */
1514	if (PGM_PAGE_IS_BALLOONED(pPage))
1515	{
1516	Assert(!SHW_PTE_IS_P(pPteDst)); /* @todo user tracking needs updating if this triggers. */
1517	return;
1518	}
1519
1520	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1521	/* Make the page writable if necessary. */
1522	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1523	&& ( PGM_PAGE_IS_ZERO(pPage)
1524	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1525	\|\| ( (PteSrc.u & X86_PTE_RW)
1526	# else
1527	\|\| ( 1
1528	# endif
1529	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1530	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1531	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1532	# endif
1533	# ifdef VBOX_WITH_PAGE_SHARING
1534	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
1535	# endif
1536	)
1537	)
1538	)
1539	{
1540	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhysPage);
1541	AssertRC(rc);
1542	}
1543	# endif
1544
1545	/*
1546	* Make page table entry.
1547	*/
1548	SHWPTE PteDst;
1549	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1550	uint64_t fGstShwPteFlags = GST_GET_PTE_SHW_FLAGS(pVCpu, PteSrc);
1551	# else
1552	uint64_t fGstShwPteFlags = X86_PTE_P \| X86_PTE_RW \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D;
1553	# endif
1554	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1555	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, fGstShwPteFlags, &PteDst);
1556	else
1557	{
1558	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1559	/*
1560	* If the page or page directory entry is not marked accessed,
1561	* we mark the page not present.
1562	*/
1563	if (!(PteSrc.u & X86_PTE_A) \|\| !(PdeSrc.u & X86_PDE_A))
1564	{
1565	LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1566	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,AccessedPage));
1567	SHW_PTE_SET(PteDst, 0);
1568	}
1569	/*
1570	* If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1571	* when the page is modified.
1572	*/
1573	else if (!(PteSrc.u & X86_PTE_D) && (PdeSrc.u & PteSrc.u & X86_PTE_RW))
1574	{
1575	AssertCompile(X86_PTE_RW == X86_PDE_RW);
1576	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPage));
1577	SHW_PTE_SET(PteDst,
1578	fGstShwPteFlags
1579	\| PGM_PAGE_GET_HCPHYS(pPage)
1580	\| PGM_PTFLAGS_TRACK_DIRTY);
1581	SHW_PTE_SET_RO(PteDst);
1582	}
1583	else
1584	# endif
1585	{
1586	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageSkipped));
1587	# if PGM_SHW_TYPE == PGM_TYPE_EPT
1588	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage)
1589	\| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE \| EPT_E_MEMTYPE_WB \| EPT_E_IGNORE_PAT;
1590	# else
1591	SHW_PTE_SET(PteDst, fGstShwPteFlags \| PGM_PAGE_GET_HCPHYS(pPage));
1592	# endif
1593	}
1594
1595	/*
1596	* Make sure only allocated pages are mapped writable.
1597	*/
1598	if ( SHW_PTE_IS_P_RW(PteDst)
1599	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1600	{
1601	/* Still applies to shared pages. */
1602	Assert(!PGM_PAGE_IS_ZERO(pPage));
1603	SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet. Why, isn't it? */
1604	Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", GCPhysPage, pPage, iPTDst));
1605	}
1606	}
1607
1608	/*
1609	* Keep user track up to date.
1610	*/
1611	if (SHW_PTE_IS_P(PteDst))
1612	{
1613	if (!SHW_PTE_IS_P(*pPteDst))
1614	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1615	else if (SHW_PTE_GET_HCPHYS(*pPteDst) != SHW_PTE_GET_HCPHYS(PteDst))
1616	{
1617	Log2(("SyncPageWorker: deref! pPteDst=%RX64 PteDst=%RX64\n", SHW_PTE_LOG64(pPteDst), SHW_PTE_LOG64(PteDst)));
1618	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
1619	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1620	}
1621	}
1622	else if (SHW_PTE_IS_P(*pPteDst))
1623	{
1624	Log2(("SyncPageWorker: deref! pPteDst=%RX64\n", SHW_PTE_LOG64(pPteDst)));
1625	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
1626	}
1627
1628	/*
1629	* Update statistics and commit the entry.
1630	*/
1631	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1632	if (!(PteSrc.u & X86_PTE_G))
1633	pShwPage->fSeenNonGlobal = true;
1634	# endif
1635	SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
1636	return;
1637	}
1638
1639	/** @todo count these three different kinds. */
1640	Log2(("SyncPageWorker: invalid address in Pte\n"));
1641	}
1642	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1643	else if (!(PteSrc.u & X86_PTE_P))
1644	Log2(("SyncPageWorker: page not present in Pte\n"));
1645	else
1646	Log2(("SyncPageWorker: invalid Pte\n"));
1647	# endif
1648
1649	/*
1650	* The page is not present or the PTE is bad. Replace the shadow PTE by
1651	* an empty entry, making sure to keep the user tracking up to date.
1652	*/
1653	if (SHW_PTE_IS_P(*pPteDst))
1654	{
1655	Log2(("SyncPageWorker: deref! pPteDst=%RX64\n", SHW_PTE_LOG64(pPteDst)));
1656	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
1657	}
1658	SHW_PTE_ATOMIC_SET(*pPteDst, 0);
1659	}
1660
1661
1662	/**
1663	* Syncs a guest OS page.
1664	*
1665	* There are no conflicts at this point, neither is there any need for
1666	* page table allocations.
1667	*
1668	* When called in PAE or AMD64 guest mode, the guest PDPE shall be valid.
1669	* When called in AMD64 guest mode, the guest PML4E shall be valid.
1670	*
1671	* @returns VBox status code.
1672	* @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1673	* @param pVCpu The cross context virtual CPU structure.
1674	* @param PdeSrc Page directory entry of the guest.
1675	* @param GCPtrPage Guest context page address.
1676	* @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1677	* @param uErr Fault error (X86_TRAP_PF_*).
1678	*/
1679	static int PGM_BTH_NAME(SyncPage)(PVMCPUCC pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr)
1680	{
1681	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1682	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
1683	LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr));
1684	RT_NOREF_PV(uErr); RT_NOREF_PV(cPages); RT_NOREF_PV(GCPtrPage);
1685
1686	PGM_LOCK_ASSERT_OWNER(pVM);
1687
1688	# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
1689	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
1690	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
1691	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE)
1692
1693	/*
1694	* Assert preconditions.
1695	*/
1696	Assert(PdeSrc.u & X86_PDE_P);
1697	Assert(cPages);
1698	# if 0 /* rarely useful; leave for debugging. */
1699	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
1700	# endif
1701
1702	/*
1703	* Get the shadow PDE, find the shadow page table in the pool.
1704	*/
1705	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1706	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1707	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1708
1709	/* Fetch the pgm pool shadow descriptor. */
1710	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1711	Assert(pShwPde);
1712
1713	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1714	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1715	PPGMPOOLPAGE pShwPde = NULL;
1716	PX86PDPAE pPDDst;
1717
1718	/* Fetch the pgm pool shadow descriptor. */
1719	int rc2 = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
1720	AssertRCSuccessReturn(rc2, rc2);
1721	Assert(pShwPde);
1722
1723	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1724	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1725
1726	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1727	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1728	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1729	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
1730	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
1731
1732	int rc2 = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
1733	AssertRCSuccessReturn(rc2, rc2);
1734	Assert(pPDDst && pPdptDst);
1735	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1736	# endif
1737	SHWPDE PdeDst = *pPdeDst;
1738
1739	/*
1740	* - In the guest SMP case we could have blocked while another VCPU reused
1741	* this page table.
1742	* - With W7-64 we may also take this path when the A bit is cleared on
1743	* higher level tables (PDPE/PML4E). The guest does not invalidate the
1744	* relevant TLB entries. If we're write monitoring any page mapped by
1745	* the modified entry, we may end up here with a "stale" TLB entry.
1746	*/
1747	if (!(PdeDst.u & X86_PDE_P))
1748	{
1749	Log(("CPU%u: SyncPage: Pde at %RGv changed behind our back? (pPdeDst=%p/%RX64) uErr=%#x\n", pVCpu->idCpu, GCPtrPage, pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
1750	AssertMsg(pVM->cCpus > 1 \|\| (uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RW)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RW),
1751	("Unexpected missing PDE p=%p/%RX64 uErr=%#x\n", pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
1752	if (uErr & X86_TRAP_PF_P)
1753	PGM_INVL_PG(pVCpu, GCPtrPage);
1754	return VINF_SUCCESS; /* force the instruction to be executed again. */
1755	}
1756
1757	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1758	Assert(pShwPage);
1759
1760	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1761	/* Fetch the pgm pool shadow descriptor. */
1762	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
1763	Assert(pShwPde);
1764	# endif
1765
1766	/*
1767	* Check that the page is present and that the shadow PDE isn't out of sync.
1768	*/
1769	const bool fBigPage = (PdeSrc.u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu);
1770	const bool fPdeValid = !fBigPage ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc);
1771	RTGCPHYS GCPhys;
1772	if (!fBigPage)
1773	{
1774	GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
1775	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1776	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1777	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
1778	# endif
1779	}
1780	else
1781	{
1782	GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
1783	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1784	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1785	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
1786	# endif
1787	}
1788	/** @todo This doesn't check the G bit of 2/4MB pages. FIXME */
1789	if ( fPdeValid
1790	&& pShwPage->GCPhys == GCPhys
1791	&& (PdeSrc.u & X86_PDE_P)
1792	&& (PdeSrc.u & X86_PDE_US) == (PdeDst.u & X86_PDE_US)
1793	&& ((PdeSrc.u & X86_PDE_RW) == (PdeDst.u & X86_PDE_RW) \|\| !(PdeDst.u & X86_PDE_RW))
1794	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
1795	&& ((PdeSrc.u & X86_PDE_PAE_NX) == (PdeDst.u & X86_PDE_PAE_NX) \|\| !GST_IS_NX_ACTIVE(pVCpu))
1796	# endif
1797	)
1798	{
1799	/*
1800	* Check that the PDE is marked accessed already.
1801	* Since we set the accessed bit before getting here on a #PF, this
1802	* check is only meant for dealing with non-#PF'ing paths.
1803	*/
1804	if (PdeSrc.u & X86_PDE_A)
1805	{
1806	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1807	if (!fBigPage)
1808	{
1809	/*
1810	* 4KB Page - Map the guest page table.
1811	*/
1812	PGSTPT pPTSrc;
1813	int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
1814	if (RT_SUCCESS(rc))
1815	{
1816	# ifdef PGM_SYNC_N_PAGES
1817	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
1818	if ( cPages > 1
1819	&& !(uErr & X86_TRAP_PF_P)
1820	&& !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
1821	{
1822	/*
1823	* This code path is currently only taken when the caller is PGMTrap0eHandler
1824	* for non-present pages!
1825	*
1826	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1827	* deal with locality.
1828	*/
1829	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1830	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1831	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1832	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1833	# else
1834	const unsigned offPTSrc = 0;
1835	# endif
1836	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
1837	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1838	iPTDst = 0;
1839	else
1840	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1841
1842	for (; iPTDst < iPTDstEnd; iPTDst++)
1843	{
1844	const PGSTPTE pPteSrc = &pPTSrc->a[offPTSrc + iPTDst];
1845
1846	if ( (pPteSrc->u & X86_PTE_P)
1847	&& !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
1848	{
1849	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT))
1850	\| ((offPTSrc + iPTDst) << GUEST_PAGE_SHIFT);
1851	NOREF(GCPtrCurPage);
1852	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, *pPteSrc, pShwPage, iPTDst);
1853	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1854	GCPtrCurPage, pPteSrc->u & X86_PTE_P,
1855	!!(pPteSrc->u & PdeSrc.u & X86_PTE_RW),
1856	!!(pPteSrc->u & PdeSrc.u & X86_PTE_US),
1857	(uint64_t)pPteSrc->u,
1858	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
1859	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
1860	}
1861	}
1862	}
1863	else
1864	# endif /* PGM_SYNC_N_PAGES */
1865	{
1866	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1867	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1868	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1869	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1870	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
1871	GCPtrPage, PteSrc.u & X86_PTE_P,
1872	!!(PteSrc.u & PdeSrc.u & X86_PTE_RW),
1873	!!(PteSrc.u & PdeSrc.u & X86_PTE_US),
1874	(uint64_t)PteSrc.u,
1875	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
1876	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
1877	}
1878	}
1879	else /* MMIO or invalid page: emulated in #PF handler. */
1880	{
1881	LogFlow(("PGM_GCPHYS_2_PTR %RGp failed with %Rrc\n", GCPhys, rc));
1882	Assert(!SHW_PTE_IS_P(pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK]));
1883	}
1884	}
1885	else
1886	{
1887	/*
1888	* 4/2MB page - lazy syncing shadow 4K pages.
1889	* (There are many causes of getting here, it's no longer only CSAM.)
1890	*/
1891	/* Calculate the GC physical address of this 4KB shadow page. */
1892	GCPhys = PGM_A20_APPLY(pVCpu, GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc) \| (GCPtrPage & GST_BIG_PAGE_OFFSET_MASK));
1893	/* Find ram range. */
1894	PPGMPAGE pPage;
1895	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
1896	if (RT_SUCCESS(rc))
1897	{
1898	AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
1899
1900	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1901	/* Try to make the page writable if necessary. */
1902	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1903	&& ( PGM_PAGE_IS_ZERO(pPage)
1904	\|\| ( (PdeSrc.u & X86_PDE_RW)
1905	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1906	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1907	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1908	# endif
1909	# ifdef VBOX_WITH_PAGE_SHARING
1910	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
1911	# endif
1912	)
1913	)
1914	)
1915	{
1916	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1917	AssertRC(rc);
1918	}
1919	# endif
1920
1921	/*
1922	* Make shadow PTE entry.
1923	*/
1924	SHWPTE PteDst;
1925	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1926	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc), &PteDst);
1927	else
1928	SHW_PTE_SET(PteDst, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc) \| PGM_PAGE_GET_HCPHYS(pPage));
1929
1930	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1931	if ( SHW_PTE_IS_P(PteDst)
1932	&& !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
1933	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1934
1935	/* Make sure only allocated pages are mapped writable. */
1936	if ( SHW_PTE_IS_P_RW(PteDst)
1937	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1938	{
1939	/* Still applies to shared pages. */
1940	Assert(!PGM_PAGE_IS_ZERO(pPage));
1941	SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
1942	Log3(("SyncPage: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, GCPtrPage));
1943	}
1944
1945	SHW_PTE_ATOMIC_SET2(pPTDst->a[iPTDst], PteDst);
1946
1947	/*
1948	* If the page is not flagged as dirty and is writable, then make it read-only
1949	* at PD level, so we can set the dirty bit when the page is modified.
1950	*
1951	* ASSUMES that page access handlers are implemented on page table entry level.
1952	* Thus we will first catch the dirty access and set PDE.D and restart. If
1953	* there is an access handler, we'll trap again and let it work on the problem.
1954	*/
1955	/** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
1956	* As for invlpg, it simply frees the whole shadow PT.
1957	* ...It's possibly because the guest clears it and the guest doesn't really tell us... */
1958	if ((PdeSrc.u & (X86_PDE4M_D \| X86_PDE_RW)) == X86_PDE_RW)
1959	{
1960	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageBig));
1961	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
1962	PdeDst.u &= ~(SHWUINT)X86_PDE_RW;
1963	}
1964	else
1965	{
1966	PdeDst.u &= ~(SHWUINT)(PGM_PDFLAGS_TRACK_DIRTY \| X86_PDE_RW);
1967	PdeDst.u \|= PdeSrc.u & X86_PDE_RW;
1968	}
1969	SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
1970	Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n",
1971	GCPtrPage, PdeSrc.u & X86_PDE_P, !!(PdeSrc.u & X86_PDE_RW), !!(PdeSrc.u & X86_PDE_US),
1972	(uint64_t)PdeSrc.u, GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1973	}
1974	else
1975	{
1976	LogFlow(("PGM_GCPHYS_2_PTR %RGp (big) failed with %Rrc\n", GCPhys, rc));
1977	/** @todo must wipe the shadow page table entry in this
1978	* case. */
1979	}
1980	}
1981	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
1982	return VINF_SUCCESS;
1983	}
1984
1985	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPagePDNAs));
1986	}
1987	else if (fPdeValid)
1988	{
1989	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPagePDOutOfSync));
1990	Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
1991	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
1992	}
1993	else
1994	{
1995	/// @todo STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSyncAndInvalid));
1996	Log2(("SyncPage: Bad PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
1997	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
1998	}
1999
2000	/*
2001	* Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
2002	* Yea, I'm lazy.
2003	*/
2004	pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
2005	SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
2006
2007	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2008	PGM_INVL_VCPU_TLBS(pVCpu);
2009	return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
2010
2011
2012	# elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2013	&& !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
2014	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT)
2015	NOREF(PdeSrc);
2016
2017	# ifdef PGM_SYNC_N_PAGES
2018	/*
2019	* Get the shadow PDE, find the shadow page table in the pool.
2020	*/
2021	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2022	X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
2023
2024	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2025	X86PDEPAE PdeDst = pgmShwGetPaePDE(pVCpu, GCPtrPage);
2026
2027	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2028	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2029	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpt);
2030	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2031	X86PDEPAE PdeDst;
2032	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2033
2034	int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2035	AssertRCSuccessReturn(rc, rc);
2036	Assert(pPDDst && pPdptDst);
2037	PdeDst = pPDDst->a[iPDDst];
2038
2039	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2040	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2041	PEPTPD pPDDst;
2042	EPTPDE PdeDst;
2043
2044	int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, NULL, &pPDDst);
2045	if (rc != VINF_SUCCESS)
2046	{
2047	AssertRC(rc);
2048	return rc;
2049	}
2050	Assert(pPDDst);
2051	PdeDst = pPDDst->a[iPDDst];
2052	# endif
2053	/* In the guest SMP case we could have blocked while another VCPU reused this page table. */
2054	if (!SHW_PDE_IS_P(PdeDst))
2055	{
2056	AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)PdeDst.u));
2057	Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", pVCpu->idCpu, GCPtrPage));
2058	return VINF_SUCCESS; /* force the instruction to be executed again. */
2059	}
2060
2061	/* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */
2062	if (SHW_PDE_IS_BIG(PdeDst))
2063	{
2064	Assert(pVM->pgm.s.fNestedPaging);
2065	Log(("CPU%d: SyncPage: Pde (big:%RX64) at %RGv changed behind our back!\n", pVCpu->idCpu, PdeDst.u, GCPtrPage));
2066	return VINF_SUCCESS;
2067	}
2068
2069	/* Mask away the page offset. */
2070	GCPtrPage &= ~((RTGCPTR)0xfff);
2071
2072	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2073	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2074
2075	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
2076	if ( cPages > 1
2077	&& !(uErr & X86_TRAP_PF_P)
2078	&& !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
2079	{
2080	/*
2081	* This code path is currently only taken when the caller is PGMTrap0eHandler
2082	* for non-present pages!
2083	*
2084	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2085	* deal with locality.
2086	*/
2087	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2088	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2089	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2090	iPTDst = 0;
2091	else
2092	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2093	for (; iPTDst < iPTDstEnd; iPTDst++)
2094	{
2095	if (!SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2096	{
2097	RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
2098	\| (iPTDst << GUEST_PAGE_SHIFT));
2099
2100	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
2101	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
2102	GCPtrCurPage,
2103	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2104	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2105
2106	if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
2107	break;
2108	}
2109	else
2110	Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n",
2111	(GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << GUEST_PAGE_SHIFT), iPTDst, SHW_PTE_LOG64(pPTDst->a[iPTDst]) ));
2112	}
2113	}
2114	else
2115	# endif /* PGM_SYNC_N_PAGES */
2116	{
2117	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2118	RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
2119	\| (iPTDst << GUEST_PAGE_SHIFT));
2120
2121	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
2122
2123	Log2(("SyncPage: 4K %RGv PteSrc:{P=1 RW=1 U=1}PteDst=%08llx%s\n",
2124	GCPtrPage,
2125	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2126	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2127	}
2128	return VINF_SUCCESS;
2129
2130	# else
2131	NOREF(PdeSrc);
2132	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2133	return VERR_PGM_NOT_USED_IN_MODE;
2134	# endif
2135	}
2136
2137	#endif /* PGM_SHW_TYPE != PGM_TYPE_NONE */
2138	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
2139
2140	/**
2141	* CheckPageFault helper for returning a page fault indicating a non-present
2142	* (NP) entry in the page translation structures.
2143	*
2144	* @returns VINF_EM_RAW_GUEST_TRAP.
2145	* @param pVCpu The cross context virtual CPU structure.
2146	* @param uErr The error code of the shadow fault. Corrections to
2147	* TRPM's copy will be made if necessary.
2148	* @param GCPtrPage For logging.
2149	* @param uPageFaultLevel For logging.
2150	*/
2151	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnNP)(PVMCPUCC pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2152	{
2153	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyTrackRealPF));
2154	AssertMsg(!(uErr & X86_TRAP_PF_P), ("%#x\n", uErr));
2155	AssertMsg(!(uErr & X86_TRAP_PF_RSVD), ("%#x\n", uErr));
2156	if (uErr & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
2157	TRPMSetErrorCode(pVCpu, uErr & ~(X86_TRAP_PF_RSVD \| X86_TRAP_PF_P));
2158
2159	Log(("CheckPageFault: real page fault (notp) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2160	RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(uPageFaultLevel);
2161	return VINF_EM_RAW_GUEST_TRAP;
2162	}
2163
2164
2165	/**
2166	* CheckPageFault helper for returning a page fault indicating a reserved bit
2167	* (RSVD) error in the page translation structures.
2168	*
2169	* @returns VINF_EM_RAW_GUEST_TRAP.
2170	* @param pVCpu The cross context virtual CPU structure.
2171	* @param uErr The error code of the shadow fault. Corrections to
2172	* TRPM's copy will be made if necessary.
2173	* @param GCPtrPage For logging.
2174	* @param uPageFaultLevel For logging.
2175	*/
2176	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnRSVD)(PVMCPUCC pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2177	{
2178	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyTrackRealPF));
2179	if ((uErr & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) != (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
2180	TRPMSetErrorCode(pVCpu, uErr \| X86_TRAP_PF_RSVD \| X86_TRAP_PF_P);
2181
2182	Log(("CheckPageFault: real page fault (rsvd) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2183	RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(uPageFaultLevel);
2184	return VINF_EM_RAW_GUEST_TRAP;
2185	}
2186
2187
2188	/**
2189	* CheckPageFault helper for returning a page protection fault (P).
2190	*
2191	* @returns VINF_EM_RAW_GUEST_TRAP.
2192	* @param pVCpu The cross context virtual CPU structure.
2193	* @param uErr The error code of the shadow fault. Corrections to
2194	* TRPM's copy will be made if necessary.
2195	* @param GCPtrPage For logging.
2196	* @param uPageFaultLevel For logging.
2197	*/
2198	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnProt)(PVMCPUCC pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2199	{
2200	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyTrackRealPF));
2201	AssertMsg(uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID), ("%#x\n", uErr));
2202	if ((uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) != X86_TRAP_PF_P)
2203	TRPMSetErrorCode(pVCpu, (uErr & ~X86_TRAP_PF_RSVD) \| X86_TRAP_PF_P);
2204
2205	Log(("CheckPageFault: real page fault (prot) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2206	RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(uPageFaultLevel);
2207	return VINF_EM_RAW_GUEST_TRAP;
2208	}
2209
2210
2211	/**
2212	* Handle dirty bit tracking faults.
2213	*
2214	* @returns VBox status code.
2215	* @param pVCpu The cross context virtual CPU structure.
2216	* @param uErr Page fault error code.
2217	* @param pPdeSrc Guest page directory entry.
2218	* @param pPdeDst Shadow page directory entry.
2219	* @param GCPtrPage Guest context page address.
2220	*/
2221	static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPUCC pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc,
2222	RTGCPTR GCPtrPage)
2223	{
2224	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2225	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2226	NOREF(uErr);
2227
2228	PGM_LOCK_ASSERT_OWNER(pVM);
2229
2230	/*
2231	* Handle big page.
2232	*/
2233	if ((pPdeSrc->u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu))
2234	{
2235	if ((pPdeDst->u & (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY))
2236	{
2237	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageTrap));
2238	Assert(pPdeSrc->u & X86_PDE_RW);
2239
2240	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
2241	* fault again and take this path to only invalidate the entry (see below). */
2242	SHWPDE PdeDst = *pPdeDst;
2243	PdeDst.u &= ~(SHWUINT)PGM_PDFLAGS_TRACK_DIRTY;
2244	PdeDst.u \|= X86_PDE_RW \| X86_PDE_A;
2245	SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
2246	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
2247	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2248	}
2249
2250	# ifdef IN_RING0
2251	/* Check for stale TLB entry; only applies to the SMP guest case. */
2252	if ( pVM->cCpus > 1
2253	&& (pPdeDst->u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_A)) == (X86_PDE_P \| X86_PDE_RW \| X86_PDE_A))
2254	{
2255	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2256	if (pShwPage)
2257	{
2258	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2259	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2260	if (SHW_PTE_IS_P_RW(*pPteDst))
2261	{
2262	/* Stale TLB entry. */
2263	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageStale));
2264	PGM_INVL_PG(pVCpu, GCPtrPage);
2265	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2266	}
2267	}
2268	}
2269	# endif /* IN_RING0 */
2270	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2271	}
2272
2273	/*
2274	* Map the guest page table.
2275	*/
2276	PGSTPT pPTSrc;
2277	int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(*pPdeSrc), &pPTSrc);
2278	AssertRCReturn(rc, rc);
2279
2280	if (SHW_PDE_IS_P(*pPdeDst))
2281	{
2282	GSTPTE const *pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2283	const GSTPTE PteSrc = *pPteSrc;
2284
2285	/*
2286	* Map shadow page table.
2287	*/
2288	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2289	if (pShwPage)
2290	{
2291	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2292	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2293	if (SHW_PTE_IS_P(pPteDst)) /* @todo Optimize accessed bit emulation? */
2294	{
2295	if (SHW_PTE_IS_TRACK_DIRTY(*pPteDst))
2296	{
2297	PPGMPAGE pPage = pgmPhysGetPage(pVM, GST_GET_PTE_GCPHYS(PteSrc));
2298	SHWPTE PteDst = *pPteDst;
2299
2300	LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage));
2301	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageTrap));
2302
2303	Assert(PteSrc.u & X86_PTE_RW);
2304
2305	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB
2306	* entry will not harm; write access will simply fault again and
2307	* take this path to only invalidate the entry.
2308	*/
2309	if (RT_LIKELY(pPage))
2310	{
2311	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2312	{
2313	//AssertMsgFailed(("%R[pgmpage] - we don't set PGM_PTFLAGS_TRACK_DIRTY for these pages\n", pPage));
2314	Assert(!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage));
2315	/* Assuming write handlers here as the PTE is present (otherwise we wouldn't be here). */
2316	SHW_PTE_SET_RO(PteDst);
2317	}
2318	else
2319	{
2320	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
2321	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
2322	{
2323	rc = pgmPhysPageMakeWritable(pVM, pPage, GST_GET_PTE_GCPHYS(PteSrc));
2324	AssertRC(rc);
2325	}
2326	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED)
2327	SHW_PTE_SET_RW(PteDst);
2328	else
2329	{
2330	/* Still applies to shared pages. */
2331	Assert(!PGM_PAGE_IS_ZERO(pPage));
2332	SHW_PTE_SET_RO(PteDst);
2333	}
2334	}
2335	}
2336	else
2337	SHW_PTE_SET_RW(PteDst); /** @todo r=bird: This doesn't make sense to me. */
2338
2339	SHW_PTE_SET(PteDst, (SHW_PTE_GET_U(PteDst) \| X86_PTE_D \| X86_PTE_A) & ~(uint64_t)PGM_PTFLAGS_TRACK_DIRTY);
2340	SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
2341	PGM_INVL_PG(pVCpu, GCPtrPage);
2342	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2343	}
2344
2345	# ifdef IN_RING0
2346	/* Check for stale TLB entry; only applies to the SMP guest case. */
2347	if ( pVM->cCpus > 1
2348	&& SHW_PTE_IS_RW(*pPteDst)
2349	&& SHW_PTE_IS_A(*pPteDst))
2350	{
2351	/* Stale TLB entry. */
2352	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageStale));
2353	PGM_INVL_PG(pVCpu, GCPtrPage);
2354	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2355	}
2356	# endif
2357	}
2358	}
2359	else
2360	AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
2361	}
2362
2363	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2364	}
2365
2366	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE */
2367
2368	/**
2369	* Sync a shadow page table.
2370	*
2371	* The shadow page table is not present in the shadow PDE.
2372	*
2373	* Handles mapping conflicts.
2374	*
2375	* This is called by VerifyAccessSyncPage, PrefetchPage, InvalidatePage (on
2376	* conflict), and Trap0eHandler.
2377	*
2378	* A precondition for this method is that the shadow PDE is not present. The
2379	* caller must take the PGM lock before checking this and continue to hold it
2380	* when calling this method.
2381	*
2382	* @returns VBox status code.
2383	* @param pVCpu The cross context virtual CPU structure.
2384	* @param iPDSrc Page directory index.
2385	* @param pPDSrc Source page directory (i.e. Guest OS page directory).
2386	* Assume this is a temporary mapping.
2387	* @param GCPtrPage GC Pointer of the page that caused the fault
2388	*/
2389	static int PGM_BTH_NAME(SyncPT)(PVMCPUCC pVCpu, unsigned iPDSrc, PGSTPD pPDSrc, RTGCPTR GCPtrPage)
2390	{
2391	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2392	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
2393
2394	#if 0 /* rarely useful; leave for debugging. */
2395	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPtPD[iPDSrc]);
2396	#endif
2397	LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage)); RT_NOREF_PV(GCPtrPage);
2398
2399	PGM_LOCK_ASSERT_OWNER(pVM);
2400
2401	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2402	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
2403	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
2404	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
2405	&& PGM_SHW_TYPE != PGM_TYPE_NONE
2406	int rc = VINF_SUCCESS;
2407
2408	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
2409
2410	/*
2411	* Some input validation first.
2412	*/
2413	AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
2414
2415	/*
2416	* Get the relevant shadow PDE entry.
2417	*/
2418	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2419	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2420	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
2421
2422	/* Fetch the pgm pool shadow descriptor. */
2423	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2424	Assert(pShwPde);
2425
2426	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2427	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2428	PPGMPOOLPAGE pShwPde = NULL;
2429	PX86PDPAE pPDDst;
2430	PSHWPDE pPdeDst;
2431
2432	/* Fetch the pgm pool shadow descriptor. */
2433	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
2434	AssertRCSuccessReturn(rc, rc);
2435	Assert(pShwPde);
2436
2437	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
2438	pPdeDst = &pPDDst->a[iPDDst];
2439
2440	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2441	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2442	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2443	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2444	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2445	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2446	AssertRCSuccessReturn(rc, rc);
2447	Assert(pPDDst);
2448	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2449
2450	# endif
2451	SHWPDE PdeDst = *pPdeDst;
2452
2453	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2454	/* Fetch the pgm pool shadow descriptor. */
2455	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2456	Assert(pShwPde);
2457	# endif
2458
2459	Assert(!SHW_PDE_IS_P(PdeDst)); /* We're only supposed to call SyncPT on PDE!P.*/
2460
2461	/*
2462	* Sync the page directory entry.
2463	*/
2464	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2465	const bool fPageTable = !(PdeSrc.u & X86_PDE_PS) \|\| !GST_IS_PSE_ACTIVE(pVCpu);
2466	if ( (PdeSrc.u & X86_PDE_P)
2467	&& (fPageTable ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc)) )
2468	{
2469	/*
2470	* Allocate & map the page table.
2471	*/
2472	PSHWPT pPTDst;
2473	PPGMPOOLPAGE pShwPage;
2474	RTGCPHYS GCPhys;
2475	if (fPageTable)
2476	{
2477	GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
2478	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2479	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2480	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
2481	# endif
2482	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
2483	pShwPde->idx, iPDDst, false /fLockPage/,
2484	&pShwPage);
2485	}
2486	else
2487	{
2488	PGMPOOLACCESS enmAccess;
2489	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2490	const bool fNoExecute = (PdeSrc.u & X86_PDE_PAE_NX) && GST_IS_NX_ACTIVE(pVCpu);
2491	# else
2492	const bool fNoExecute = false;
2493	# endif
2494
2495	GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
2496	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2497	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2498	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
2499	# endif
2500	/* Determine the right kind of large page to avoid incorrect cached entry reuse. */
2501	if (PdeSrc.u & X86_PDE_US)
2502	{
2503	if (PdeSrc.u & X86_PDE_RW)
2504	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_RW_NX : PGMPOOLACCESS_USER_RW;
2505	else
2506	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_R_NX : PGMPOOLACCESS_USER_R;
2507	}
2508	else
2509	{
2510	if (PdeSrc.u & X86_PDE_RW)
2511	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW;
2512	else
2513	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_R_NX : PGMPOOLACCESS_SUPERVISOR_R;
2514	}
2515	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, enmAccess, PGM_A20_IS_ENABLED(pVCpu),
2516	pShwPde->idx, iPDDst, false /fLockPage/,
2517	&pShwPage);
2518	}
2519	if (rc == VINF_SUCCESS)
2520	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2521	else if (rc == VINF_PGM_CACHED_PAGE)
2522	{
2523	/*
2524	* The PT was cached, just hook it up.
2525	*/
2526	if (fPageTable)
2527	PdeDst.u = pShwPage->Core.Key \| GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2528	else
2529	{
2530	PdeDst.u = pShwPage->Core.Key \| GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2531	/* (see explanation and assumptions further down.) */
2532	if ((PdeSrc.u & (X86_PDE_RW \| X86_PDE4M_D)) == X86_PDE_RW)
2533	{
2534	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageBig));
2535	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2536	PdeDst.u &= ~(SHWUINT)X86_PDE_RW;
2537	}
2538	}
2539	SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
2540	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2541	return VINF_SUCCESS;
2542	}
2543	else
2544	AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
2545	/** @todo Why do we bother preserving X86_PDE_AVL_MASK here?
2546	* Both PGM_PDFLAGS_MAPPING and PGM_PDFLAGS_TRACK_DIRTY should be
2547	* irrelevant at this point. */
2548	PdeDst.u &= X86_PDE_AVL_MASK;
2549	PdeDst.u \|= pShwPage->Core.Key;
2550
2551	/*
2552	* Page directory has been accessed (this is a fault situation, remember).
2553	*/
2554	/** @todo
2555	* Well, when the caller is PrefetchPage or InvalidatePage is isn't a
2556	* fault situation. What's more, the Trap0eHandler has already set the
2557	* accessed bit. So, it's actually just VerifyAccessSyncPage which
2558	* might need setting the accessed flag.
2559	*
2560	* The best idea is to leave this change to the caller and add an
2561	* assertion that it's set already. */
2562	pPDSrc->a[iPDSrc].u \|= X86_PDE_A;
2563	if (fPageTable)
2564	{
2565	/*
2566	* Page table - 4KB.
2567	*
2568	* Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
2569	*/
2570	Log2(("SyncPT: 4K %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
2571	GCPtrPage, PdeSrc.u & X86_PTE_P, !!(PdeSrc.u & X86_PTE_RW), !!(PdeSrc.u & X86_PDE_US), (uint64_t)PdeSrc.u));
2572	PGSTPT pPTSrc;
2573	rc = PGM_GCPHYS_2_PTR(pVM, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
2574	if (RT_SUCCESS(rc))
2575	{
2576	/*
2577	* Start by syncing the page directory entry so CSAM's TLB trick works.
2578	*/
2579	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| X86_PDE_AVL_MASK))
2580	\| GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2581	SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
2582	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2583
2584	/*
2585	* Directory/page user or supervisor privilege: (same goes for read/write)
2586	*
2587	* Directory Page Combined
2588	* U/S U/S U/S
2589	* 0 0 0
2590	* 0 1 0
2591	* 1 0 0
2592	* 1 1 1
2593	*
2594	* Simple AND operation. Table listed for completeness.
2595	*
2596	*/
2597	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT4K));
2598	# ifdef PGM_SYNC_N_PAGES
2599	unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2600	unsigned iPTDst = iPTBase;
2601	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2602	if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
2603	iPTDst = 0;
2604	else
2605	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2606	# else /* !PGM_SYNC_N_PAGES */
2607	unsigned iPTDst = 0;
2608	const unsigned iPTDstEnd = RT_ELEMENTS(pPTDst->a);
2609	# endif /* !PGM_SYNC_N_PAGES */
2610	RTGCPTR GCPtrCur = (GCPtrPage & ~(RTGCPTR)((1 << SHW_PD_SHIFT) - 1))
2611	\| ((RTGCPTR)iPTDst << GUEST_PAGE_SHIFT);
2612	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2613	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2614	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2615	# else
2616	const unsigned offPTSrc = 0;
2617	# endif
2618	for (; iPTDst < iPTDstEnd; iPTDst++, GCPtrCur += GUEST_PAGE_SIZE)
2619	{
2620	const unsigned iPTSrc = iPTDst + offPTSrc;
2621	const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2622	if (PteSrc.u & X86_PTE_P)
2623	{
2624	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2625	Log2(("SyncPT: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
2626	GCPtrCur,
2627	PteSrc.u & X86_PTE_P,
2628	!!(PteSrc.u & PdeSrc.u & X86_PTE_RW),
2629	!!(PteSrc.u & PdeSrc.u & X86_PTE_US),
2630	(uint64_t)PteSrc.u,
2631	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : "", SHW_PTE_LOG64(pPTDst->a[iPTDst]), iPTSrc, PdeSrc.au32[0],
2632	(RTGCPHYS)(GST_GET_PDE_GCPHYS(PdeSrc) + iPTSrc*sizeof(PteSrc)) ));
2633	}
2634	/* else: the page table was cleared by the pool */
2635	} /* for PTEs */
2636	}
2637	}
2638	else
2639	{
2640	/*
2641	* Big page - 2/4MB.
2642	*
2643	* We'll walk the ram range list in parallel and optimize lookups.
2644	* We will only sync one shadow page table at a time.
2645	*/
2646	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT4M));
2647
2648	/**
2649	* @todo It might be more efficient to sync only a part of the 4MB
2650	* page (similar to what we do for 4KB PDs).
2651	*/
2652
2653	/*
2654	* Start by syncing the page directory entry.
2655	*/
2656	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
2657	\| GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2658
2659	/*
2660	* If the page is not flagged as dirty and is writable, then make it read-only
2661	* at PD level, so we can set the dirty bit when the page is modified.
2662	*
2663	* ASSUMES that page access handlers are implemented on page table entry level.
2664	* Thus we will first catch the dirty access and set PDE.D and restart. If
2665	* there is an access handler, we'll trap again and let it work on the problem.
2666	*/
2667	/** @todo move the above stuff to a section in the PGM documentation. */
2668	Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
2669	if ((PdeSrc.u & (X86_PDE_RW \| X86_PDE4M_D)) == X86_PDE_RW)
2670	{
2671	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageBig));
2672	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2673	PdeDst.u &= ~(SHWUINT)X86_PDE_RW;
2674	}
2675	SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
2676	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2677
2678	/*
2679	* Fill the shadow page table.
2680	*/
2681	/* Get address and flags from the source PDE. */
2682	SHWPTE PteDstBase;
2683	SHW_PTE_SET(PteDstBase, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc));
2684
2685	/* Loop thru the entries in the shadow PT. */
2686	const RTGCPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
2687	Log2(("SyncPT: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n",
2688	GCPtrPage, PdeSrc.u & X86_PDE_P, !!(PdeSrc.u & X86_PDE_RW), !!(PdeSrc.u & X86_PDE_US), (uint64_t)PdeSrc.u, GCPtr,
2689	GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2690	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
2691	unsigned iPTDst = 0;
2692	while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2693	&& !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
2694	{
2695	if (pRam && GCPhys >= pRam->GCPhys)
2696	{
2697	# ifndef PGM_WITH_A20
2698	unsigned iHCPage = (GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT;
2699	# endif
2700	do
2701	{
2702	/* Make shadow PTE. */
2703	# ifdef PGM_WITH_A20
2704	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT];
2705	# else
2706	PPGMPAGE pPage = &pRam->aPages[iHCPage];
2707	# endif
2708	SHWPTE PteDst;
2709
2710	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2711	/* Try to make the page writable if necessary. */
2712	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2713	&& ( PGM_PAGE_IS_ZERO(pPage)
2714	\|\| ( SHW_PTE_IS_RW(PteDstBase)
2715	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2716	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
2717	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
2718	# endif
2719	# ifdef VBOX_WITH_PAGE_SHARING
2720	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
2721	# endif
2722	&& !PGM_PAGE_IS_BALLOONED(pPage))
2723	)
2724	)
2725	{
2726	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
2727	AssertRCReturn(rc, rc);
2728	if (VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
2729	break;
2730	}
2731	# endif
2732
2733	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2734	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, SHW_PTE_GET_U(PteDstBase), &PteDst);
2735	else if (PGM_PAGE_IS_BALLOONED(pPage))
2736	SHW_PTE_SET(PteDst, 0); /* Handle ballooned pages at #PF time. */
2737	else
2738	SHW_PTE_SET(PteDst, PGM_PAGE_GET_HCPHYS(pPage) \| SHW_PTE_GET_U(PteDstBase));
2739
2740	/* Only map writable pages writable. */
2741	if ( SHW_PTE_IS_P_RW(PteDst)
2742	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2743	{
2744	/* Still applies to shared pages. */
2745	Assert(!PGM_PAGE_IS_ZERO(pPage));
2746	SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
2747	Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))));
2748	}
2749
2750	if (SHW_PTE_IS_P(PteDst))
2751	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2752
2753	/* commit it (not atomic, new table) */
2754	pPTDst->a[iPTDst] = PteDst;
2755	Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
2756	(RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT)), SHW_PTE_IS_P(PteDst), SHW_PTE_IS_RW(PteDst), SHW_PTE_IS_US(PteDst), SHW_PTE_LOG64(PteDst),
2757	SHW_PTE_IS_TRACK_DIRTY(PteDst) ? " Track-Dirty" : ""));
2758
2759	/* advance */
2760	GCPhys += GUEST_PAGE_SIZE;
2761	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
2762	# ifndef PGM_WITH_A20
2763	iHCPage++;
2764	# endif
2765	iPTDst++;
2766	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2767	&& GCPhys <= pRam->GCPhysLast);
2768
2769	/* Advance ram range list. */
2770	while (pRam && GCPhys > pRam->GCPhysLast)
2771	pRam = pRam->CTX_SUFF(pNext);
2772	}
2773	else if (pRam)
2774	{
2775	Log(("Invalid pages at %RGp\n", GCPhys));
2776	do
2777	{
2778	SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
2779	GCPhys += GUEST_PAGE_SIZE;
2780	iPTDst++;
2781	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2782	&& GCPhys < pRam->GCPhys);
2783	PGM_A20_APPLY_TO_VAR(pVCpu,GCPhys);
2784	}
2785	else
2786	{
2787	Log(("Invalid pages at %RGp (2)\n", GCPhys));
2788	for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
2789	SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
2790	}
2791	} /* while more PTEs */
2792	} /* 4KB / 4MB */
2793	}
2794	else
2795	AssertRelease(!SHW_PDE_IS_P(PdeDst));
2796
2797	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
2798	if (RT_FAILURE(rc))
2799	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPTFailed));
2800	return rc;
2801
2802	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2803	&& !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
2804	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2805	&& PGM_SHW_TYPE != PGM_TYPE_NONE
2806	NOREF(iPDSrc); NOREF(pPDSrc);
2807
2808	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
2809
2810	/*
2811	* Validate input a little bit.
2812	*/
2813	int rc = VINF_SUCCESS;
2814	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2815	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2816	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
2817
2818	/* Fetch the pgm pool shadow descriptor. */
2819	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2820	Assert(pShwPde);
2821
2822	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2823	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2824	PPGMPOOLPAGE pShwPde = NULL; /* initialized to shut up gcc */
2825	PX86PDPAE pPDDst;
2826	PSHWPDE pPdeDst;
2827
2828	/* Fetch the pgm pool shadow descriptor. */
2829	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
2830	AssertRCSuccessReturn(rc, rc);
2831	Assert(pShwPde);
2832
2833	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
2834	pPdeDst = &pPDDst->a[iPDDst];
2835
2836	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2837	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2838	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2839	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2840	PX86PDPT pPdptDst= NULL; /* initialized to shut up gcc */
2841	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2842	AssertRCSuccessReturn(rc, rc);
2843	Assert(pPDDst);
2844	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2845
2846	/* Fetch the pgm pool shadow descriptor. */
2847	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2848	Assert(pShwPde);
2849
2850	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2851	const unsigned iPdpt = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
2852	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2853	PEPTPD pPDDst;
2854	PEPTPDPT pPdptDst;
2855
2856	rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, &pPdptDst, &pPDDst);
2857	if (rc != VINF_SUCCESS)
2858	{
2859	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
2860	AssertRC(rc);
2861	return rc;
2862	}
2863	Assert(pPDDst);
2864	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2865
2866	/* Fetch the pgm pool shadow descriptor. */
2867	/** @todo r=bird: didn't pgmShwGetEPTPDPtr just do this lookup already? */
2868	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & EPT_PDPTE_PG_MASK);
2869	Assert(pShwPde);
2870	# endif
2871	SHWPDE PdeDst = *pPdeDst;
2872
2873	Assert(!SHW_PDE_IS_P(PdeDst)); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2874
2875	# if defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
2876	if ( BTH_IS_NP_ACTIVE(pVM)
2877	&& !VM_IS_NEM_ENABLED(pVM)) /** @todo NEM: Large page support. */
2878	{
2879	/* Check if we allocated a big page before for this 2 MB range. */
2880	PPGMPAGE pPage;
2881	rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PDE2M_PAE_PG_MASK), &pPage);
2882	if (RT_SUCCESS(rc))
2883	{
2884	RTHCPHYS HCPhys = NIL_RTHCPHYS;
2885	if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
2886	{
2887	if (PGM_A20_IS_ENABLED(pVCpu))
2888	{
2889	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageReused);
2890	AssertRelease(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
2891	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2892	}
2893	else
2894	{
2895	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PDE_DISABLED);
2896	pVM->pgm.s.cLargePagesDisabled++;
2897	}
2898	}
2899	else if ( PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED
2900	&& PGM_A20_IS_ENABLED(pVCpu))
2901	{
2902	/* Recheck the entire 2 MB range to see if we can use it again as a large page. */
2903	rc = pgmPhysRecheckLargePage(pVM, GCPtrPage, pPage);
2904	if (RT_SUCCESS(rc))
2905	{
2906	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
2907	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
2908	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2909	}
2910	}
2911	else if ( PGMIsUsingLargePages(pVM)
2912	&& PGM_A20_IS_ENABLED(pVCpu))
2913	{
2914	rc = pgmPhysAllocLargePage(pVM, GCPtrPage);
2915	if (RT_SUCCESS(rc))
2916	{
2917	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
2918	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
2919	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2920	}
2921	else
2922	LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
2923	}
2924
2925	if (HCPhys != NIL_RTHCPHYS)
2926	{
2927	# if PGM_SHW_TYPE == PGM_TYPE_EPT
2928	PdeDst.u = HCPhys \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE \| EPT_E_LEAF \| EPT_E_IGNORE_PAT \| EPT_E_MEMTYPE_WB
2929	\| (PdeDst.u & X86_PDE_AVL_MASK) /** @todo do we need this? */;
2930	# else
2931	PdeDst.u = HCPhys \| X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PS
2932	\| (PdeDst.u & X86_PDE_AVL_MASK) /** @todo PGM_PD_FLAGS? */;
2933	# endif
2934	SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
2935
2936	Log(("SyncPT: Use large page at %RGp PDE=%RX64\n", GCPtrPage, PdeDst.u));
2937	/* Add a reference to the first page only. */
2938	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPde, PGM_PAGE_GET_TRACKING(pPage), pPage, iPDDst);
2939
2940	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
2941	return VINF_SUCCESS;
2942	}
2943	}
2944	}
2945	# endif /* defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE */
2946
2947	/*
2948	* Allocate & map the page table.
2949	*/
2950	PSHWPT pPTDst;
2951	PPGMPOOLPAGE pShwPage;
2952	RTGCPHYS GCPhys;
2953
2954	/* Virtual address = physical address */
2955	GCPhys = PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PAGE_4K_BASE_MASK);
2956	rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE,
2957	PGM_A20_IS_ENABLED(pVCpu), pShwPde->idx, iPDDst, false /fLockPage/,
2958	&pShwPage);
2959	if ( rc == VINF_SUCCESS
2960	\|\| rc == VINF_PGM_CACHED_PAGE)
2961	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2962	else
2963	{
2964	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
2965	AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
2966	}
2967
2968	if (rc == VINF_SUCCESS)
2969	{
2970	/* New page table; fully set it up. */
2971	Assert(pPTDst);
2972
2973	/* Mask away the page offset. */
2974	GCPtrPage &= ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK;
2975
2976	for (unsigned iPTDst = 0; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
2977	{
2978	RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
2979	\| (iPTDst << GUEST_PAGE_SHIFT));
2980
2981	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
2982	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
2983	GCPtrCurPage,
2984	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2985	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2986
2987	if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
2988	break;
2989	}
2990	}
2991	else
2992	rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */
2993
2994	/* Save the new PDE. */
2995	# if PGM_SHW_TYPE == PGM_TYPE_EPT
2996	PdeDst.u = pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE
2997	\| (PdeDst.u & X86_PDE_AVL_MASK /** @todo do we really need this? */);
2998	# else
2999	PdeDst.u = pShwPage->Core.Key \| X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_A
3000	\| (PdeDst.u & X86_PDE_AVL_MASK /** @todo use a PGM_PD_FLAGS define */);
3001	# endif
3002	SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
3003
3004	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
3005	if (RT_FAILURE(rc))
3006	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPTFailed));
3007	return rc;
3008
3009	#else
3010	NOREF(iPDSrc); NOREF(pPDSrc);
3011	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
3012	return VERR_PGM_NOT_USED_IN_MODE;
3013	#endif
3014	}
3015
3016
3017
3018	/**
3019	* Prefetch a page/set of pages.
3020	*
3021	* Typically used to sync commonly used pages before entering raw mode
3022	* after a CR3 reload.
3023	*
3024	* @returns VBox status code.
3025	* @param pVCpu The cross context virtual CPU structure.
3026	* @param GCPtrPage Page to invalidate.
3027	*/
3028	PGM_BTH_DECL(int, PrefetchPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
3029	{
3030	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3031	\|\| PGM_GST_TYPE == PGM_TYPE_REAL \
3032	\|\| PGM_GST_TYPE == PGM_TYPE_PROT \
3033	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3034	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3035	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
3036	&& PGM_SHW_TYPE != PGM_TYPE_NONE
3037	/*
3038	* Check that all Guest levels thru the PDE are present, getting the
3039	* PD and PDE in the processes.
3040	*/
3041	int rc = VINF_SUCCESS;
3042	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3043	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3044	const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
3045	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3046	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3047	unsigned iPDSrc;
3048	X86PDPE PdpeSrc;
3049	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3050	if (!pPDSrc)
3051	return VINF_SUCCESS; /* not present */
3052	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3053	unsigned iPDSrc;
3054	PX86PML4E pPml4eSrc;
3055	X86PDPE PdpeSrc;
3056	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3057	if (!pPDSrc)
3058	return VINF_SUCCESS; /* not present */
3059	# endif
3060	const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3061	# else
3062	PGSTPD pPDSrc = NULL;
3063	const unsigned iPDSrc = 0;
3064	GSTPDE const PdeSrc = { X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_A }; /* faked so we don't have to #ifdef everything */
3065	# endif
3066
3067	if ((PdeSrc.u & (X86_PDE_P \| X86_PDE_A)) == (X86_PDE_P \| X86_PDE_A))
3068	{
3069	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3070	PGM_LOCK_VOID(pVM);
3071
3072	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3073	const X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
3074	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3075	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3076	PX86PDPAE pPDDst;
3077	X86PDEPAE PdeDst;
3078	# if PGM_GST_TYPE != PGM_TYPE_PAE
3079	X86PDPE PdpeSrc;
3080
3081	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3082	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3083	# endif
3084	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3085	if (rc != VINF_SUCCESS)
3086	{
3087	PGM_UNLOCK(pVM);
3088	AssertRC(rc);
3089	return rc;
3090	}
3091	Assert(pPDDst);
3092	PdeDst = pPDDst->a[iPDDst];
3093
3094	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3095	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3096	PX86PDPAE pPDDst;
3097	X86PDEPAE PdeDst;
3098
3099	# if PGM_GST_TYPE == PGM_TYPE_PROT
3100	/* AMD-V nested paging */
3101	X86PML4E Pml4eSrc;
3102	X86PDPE PdpeSrc;
3103	PX86PML4E pPml4eSrc = &Pml4eSrc;
3104
3105	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3106	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
3107	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
3108	# endif
3109
3110	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3111	if (rc != VINF_SUCCESS)
3112	{
3113	PGM_UNLOCK(pVM);
3114	AssertRC(rc);
3115	return rc;
3116	}
3117	Assert(pPDDst);
3118	PdeDst = pPDDst->a[iPDDst];
3119	# endif
3120	if (!(PdeDst.u & X86_PDE_P))
3121	{
3122	/** @todo r=bird: This guy will set the A bit on the PDE,
3123	* probably harmless. */
3124	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3125	}
3126	else
3127	{
3128	/* Note! We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
3129	* R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
3130	* makes no sense to prefetch more than one page.
3131	*/
3132	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3133	if (RT_SUCCESS(rc))
3134	rc = VINF_SUCCESS;
3135	}
3136	PGM_UNLOCK(pVM);
3137	}
3138	return rc;
3139
3140	#elif PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \|\| PGM_SHW_TYPE == PGM_TYPE_NONE
3141	NOREF(pVCpu); NOREF(GCPtrPage);
3142	return VINF_SUCCESS; /* ignore */
3143	#else
3144	AssertCompile(0);
3145	#endif
3146	}
3147
3148
3149
3150
3151	/**
3152	* Syncs a page during a PGMVerifyAccess() call.
3153	*
3154	* @returns VBox status code (informational included).
3155	* @param pVCpu The cross context virtual CPU structure.
3156	* @param GCPtrPage The address of the page to sync.
3157	* @param fPage The effective guest page flags.
3158	* @param uErr The trap error code.
3159	* @remarks This will normally never be called on invalid guest page
3160	* translation entries.
3161	*/
3162	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage, unsigned fPage, unsigned uErr)
3163	{
3164	PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
3165
3166	LogFlow(("VerifyAccessSyncPage: GCPtrPage=%RGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
3167	RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(fPage); RT_NOREF_PV(uErr);
3168
3169	Assert(!pVM->pgm.s.fNestedPaging);
3170	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3171	\|\| PGM_GST_TYPE == PGM_TYPE_REAL \
3172	\|\| PGM_GST_TYPE == PGM_TYPE_PROT \
3173	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3174	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3175	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
3176	&& PGM_SHW_TYPE != PGM_TYPE_NONE
3177
3178	/*
3179	* Get guest PD and index.
3180	*/
3181	/** @todo Performance: We've done all this a jiffy ago in the
3182	* PGMGstGetPage call. */
3183	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3184	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3185	const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
3186	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3187
3188	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3189	unsigned iPDSrc = 0;
3190	X86PDPE PdpeSrc;
3191	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3192	if (RT_UNLIKELY(!pPDSrc))
3193	{
3194	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3195	return VINF_EM_RAW_GUEST_TRAP;
3196	}
3197
3198	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3199	unsigned iPDSrc = 0; /* shut up gcc */
3200	PX86PML4E pPml4eSrc = NULL; /* ditto */
3201	X86PDPE PdpeSrc;
3202	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3203	if (RT_UNLIKELY(!pPDSrc))
3204	{
3205	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3206	return VINF_EM_RAW_GUEST_TRAP;
3207	}
3208	# endif
3209
3210	# else /* !PGM_WITH_PAGING */
3211	PGSTPD pPDSrc = NULL;
3212	const unsigned iPDSrc = 0;
3213	# endif /* !PGM_WITH_PAGING */
3214	int rc = VINF_SUCCESS;
3215
3216	PGM_LOCK_VOID(pVM);
3217
3218	/*
3219	* First check if the shadow pd is present.
3220	*/
3221	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3222	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3223
3224	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3225	PX86PDEPAE pPdeDst;
3226	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3227	PX86PDPAE pPDDst;
3228	# if PGM_GST_TYPE != PGM_TYPE_PAE
3229	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3230	X86PDPE PdpeSrc;
3231	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3232	# endif
3233	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3234	if (rc != VINF_SUCCESS)
3235	{
3236	PGM_UNLOCK(pVM);
3237	AssertRC(rc);
3238	return rc;
3239	}
3240	Assert(pPDDst);
3241	pPdeDst = &pPDDst->a[iPDDst];
3242
3243	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3244	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3245	PX86PDPAE pPDDst;
3246	PX86PDEPAE pPdeDst;
3247
3248	# if PGM_GST_TYPE == PGM_TYPE_PROT
3249	/* AMD-V nested paging: Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3250	X86PML4E Pml4eSrc;
3251	X86PDPE PdpeSrc;
3252	PX86PML4E pPml4eSrc = &Pml4eSrc;
3253	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
3254	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
3255	# endif
3256
3257	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3258	if (rc != VINF_SUCCESS)
3259	{
3260	PGM_UNLOCK(pVM);
3261	AssertRC(rc);
3262	return rc;
3263	}
3264	Assert(pPDDst);
3265	pPdeDst = &pPDDst->a[iPDDst];
3266	# endif
3267
3268	if (!(pPdeDst->u & X86_PDE_P))
3269	{
3270	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3271	if (rc != VINF_SUCCESS)
3272	{
3273	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3274	PGM_UNLOCK(pVM);
3275	AssertRC(rc);
3276	return rc;
3277	}
3278	}
3279
3280	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3281	/* Check for dirty bit fault */
3282	rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
3283	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
3284	Log(("PGMVerifyAccess: success (dirty)\n"));
3285	else
3286	# endif
3287	{
3288	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3289	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3290	# else
3291	GSTPDE const PdeSrc = { X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_A }; /* faked so we don't have to #ifdef everything */
3292	# endif
3293
3294	Assert(rc != VINF_EM_RAW_GUEST_TRAP);
3295	if (uErr & X86_TRAP_PF_US)
3296	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncUser));
3297	else /* supervisor */
3298	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
3299
3300	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3301	if (RT_SUCCESS(rc))
3302	{
3303	/* Page was successfully synced */
3304	Log2(("PGMVerifyAccess: success (sync)\n"));
3305	rc = VINF_SUCCESS;
3306	}
3307	else
3308	{
3309	Log(("PGMVerifyAccess: access violation for %RGv rc=%Rrc\n", GCPtrPage, rc));
3310	rc = VINF_EM_RAW_GUEST_TRAP;
3311	}
3312	}
3313	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3314	PGM_UNLOCK(pVM);
3315	return rc;
3316
3317	#else /* PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) */
3318
3319	AssertLogRelMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
3320	return VERR_PGM_NOT_USED_IN_MODE;
3321	#endif /* PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) */
3322	}
3323
3324
3325	/**
3326	* Syncs the paging hierarchy starting at CR3.
3327	*
3328	* @returns VBox status code, R0/RC may return VINF_PGM_SYNC_CR3, no other
3329	* informational status codes.
3330	* @retval VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map
3331	* the VMM into guest context.
3332	* @param pVCpu The cross context virtual CPU structure.
3333	* @param cr0 Guest context CR0 register.
3334	* @param cr3 Guest context CR3 register. Not subjected to the A20
3335	* mask.
3336	* @param cr4 Guest context CR4 register.
3337	* @param fGlobal Including global page directories or not
3338	*/
3339	PGM_BTH_DECL(int, SyncCR3)(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
3340	{
3341	PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
3342	NOREF(cr0); NOREF(cr3); NOREF(cr4); NOREF(fGlobal);
3343
3344	LogFlow(("SyncCR3 FF=%d fGlobal=%d\n", !!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), fGlobal));
3345
3346	#if !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
3347	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
3348	PGM_LOCK_VOID(pVM);
3349	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3350	if (pPool->cDirtyPages)
3351	pgmPoolResetDirtyPages(pVM);
3352	PGM_UNLOCK(pVM);
3353	# endif
3354	#endif /* !NESTED && !EPT */
3355
3356	#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \|\| PGM_SHW_TYPE == PGM_TYPE_NONE
3357	/*
3358	* Nested / EPT / None - No work.
3359	*/
3360	return VINF_SUCCESS;
3361
3362	#elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3363	/*
3364	* AMD64 (Shw & Gst) - No need to check all paging levels; we zero
3365	* out the shadow parts when the guest modifies its tables.
3366	*/
3367	return VINF_SUCCESS;
3368
3369	#else /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3370
3371	return VINF_SUCCESS;
3372	#endif /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3373	}
3374
3375
3376
3377
3378	#ifdef VBOX_STRICT
3379
3380	/**
3381	* Checks that the shadow page table is in sync with the guest one.
3382	*
3383	* @returns The number of errors.
3384	* @param pVCpu The cross context virtual CPU structure.
3385	* @param cr3 Guest context CR3 register.
3386	* @param cr4 Guest context CR4 register.
3387	* @param GCPtr Where to start. Defaults to 0.
3388	* @param cb How much to check. Defaults to everything.
3389	*/
3390	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)
3391	{
3392	NOREF(pVCpu); NOREF(cr3); NOREF(cr4); NOREF(GCPtr); NOREF(cb);
3393	#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \|\| PGM_SHW_TYPE == PGM_TYPE_NONE
3394	return 0;
3395	#else
3396	unsigned cErrors = 0;
3397	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3398	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
3399
3400	# if PGM_GST_TYPE == PGM_TYPE_PAE
3401	/** @todo currently broken; crashes below somewhere */
3402	AssertFailed();
3403	# endif
3404
3405	# if PGM_GST_TYPE == PGM_TYPE_32BIT \
3406	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3407	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
3408
3409	bool fBigPagesSupported = GST_IS_PSE_ACTIVE(pVCpu);
3410	PPGMCPU pPGM = &pVCpu->pgm.s;
3411	RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
3412	RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
3413	# ifndef IN_RING0
3414	RTHCPHYS HCPhys; /* general usage. */
3415	# endif
3416	int rc;
3417
3418	/*
3419	* Check that the Guest CR3 and all its mappings are correct.
3420	*/
3421	AssertMsgReturn(pPGM->GCPhysCR3 == PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK),
3422	("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
3423	false);
3424	# if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64
3425	# if 0
3426	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3427	rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGst32BitPdRC, NULL, &HCPhysShw);
3428	# else
3429	rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGstPaePdptRC, NULL, &HCPhysShw);
3430	# endif
3431	AssertRCReturn(rc, 1);
3432	HCPhys = NIL_RTHCPHYS;
3433	rc = pgmRamGCPhys2HCPhys(pVM, PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK), &HCPhys);
3434	AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
3435	# endif
3436	# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
3437	pgmGstGet32bitPDPtr(pVCpu);
3438	RTGCPHYS GCPhys;
3439	rc = PGMR3DbgR3Ptr2GCPhys(pVM->pUVM, pPGM->pGst32BitPdR3, &GCPhys);
3440	AssertRCReturn(rc, 1);
3441	AssertMsgReturn(PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
3442	# endif
3443	# endif /* !IN_RING0 */
3444
3445	/*
3446	* Get and check the Shadow CR3.
3447	*/
3448	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3449	unsigned cPDEs = X86_PG_ENTRIES;
3450	unsigned cIncrement = X86_PG_ENTRIES * GUEST_PAGE_SIZE;
3451	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3452	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3453	unsigned cPDEs = X86_PG_PAE_ENTRIES * 4; /* treat it as a 2048 entry table. */
3454	# else
3455	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3456	# endif
3457	unsigned cIncrement = X86_PG_PAE_ENTRIES * GUEST_PAGE_SIZE;
3458	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3459	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3460	unsigned cIncrement = X86_PG_PAE_ENTRIES * GUEST_PAGE_SIZE;
3461	# endif
3462	if (cb != ~(RTGCPTR)0)
3463	cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
3464
3465	/** @todo call the other two PGMAssert() functions. /
3466
3467	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3468	unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3469
3470	for (; iPml4 < X86_PG_PAE_ENTRIES; iPml4++)
3471	{
3472	PPGMPOOLPAGE pShwPdpt = NULL;
3473	PX86PML4E pPml4eSrc;
3474	PX86PML4E pPml4eDst;
3475	RTGCPHYS GCPhysPdptSrc;
3476
3477	pPml4eSrc = pgmGstGetLongModePML4EPtr(pVCpu, iPml4);
3478	pPml4eDst = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
3479
3480	/* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
3481	if (!(pPml4eDst->u & X86_PML4E_P))
3482	{
3483	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3484	continue;
3485	}
3486
3487	pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
3488	GCPhysPdptSrc = PGM_A20_APPLY(pVCpu, pPml4eSrc->u & X86_PML4E_PG_MASK);
3489
3490	if ((pPml4eSrc->u & X86_PML4E_P) != (pPml4eDst->u & X86_PML4E_P))
3491	{
3492	AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3493	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3494	cErrors++;
3495	continue;
3496	}
3497
3498	if (GCPhysPdptSrc != pShwPdpt->GCPhys)
3499	{
3500	AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
3501	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3502	cErrors++;
3503	continue;
3504	}
3505
3506	if ( (pPml4eDst->u & (X86_PML4E_US \| X86_PML4E_RW \| X86_PML4E_NX))
3507	!= (pPml4eSrc->u & (X86_PML4E_US \| X86_PML4E_RW \| X86_PML4E_NX)))
3508	{
3509	AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3510	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3511	cErrors++;
3512	continue;
3513	}
3514	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3515	{
3516	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3517
3518	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
3519	/*
3520	* Check the PDPTEs too.
3521	*/
3522	unsigned iPdpt = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
3523
3524	for (;iPdpt <= SHW_PDPT_MASK; iPdpt++)
3525	{
3526	unsigned iPDSrc = 0; /* initialized to shut up gcc */
3527	PPGMPOOLPAGE pShwPde = NULL;
3528	PX86PDPE pPdpeDst;
3529	RTGCPHYS GCPhysPdeSrc;
3530	X86PDPE PdpeSrc;
3531	PdpeSrc.u = 0; /* initialized to shut up gcc 4.5 */
3532	# if PGM_GST_TYPE == PGM_TYPE_PAE
3533	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtr, &iPDSrc, &PdpeSrc);
3534	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
3535	# else
3536	PX86PML4E pPml4eSrcIgn;
3537	PX86PDPT pPdptDst;
3538	PX86PDPAE pPDDst;
3539	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtr, &pPml4eSrcIgn, &PdpeSrc, &iPDSrc);
3540
3541	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtr, NULL, &pPdptDst, &pPDDst);
3542	if (rc != VINF_SUCCESS)
3543	{
3544	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
3545	GCPtr += 512 * _2M;
3546	continue; /* next PDPTE */
3547	}
3548	Assert(pPDDst);
3549	# endif
3550	Assert(iPDSrc == 0);
3551
3552	pPdpeDst = &pPdptDst->a[iPdpt];
3553
3554	if (!(pPdpeDst->u & X86_PDPE_P))
3555	{
3556	GCPtr += 512 * _2M;
3557	continue; /* next PDPTE */
3558	}
3559
3560	pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
3561	GCPhysPdeSrc = PGM_A20_APPLY(pVCpu, PdpeSrc.u & X86_PDPE_PG_MASK);
3562
3563	if ((pPdpeDst->u & X86_PDPE_P) != (PdpeSrc.u & X86_PDPE_P))
3564	{
3565	AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3566	GCPtr += 512 * _2M;
3567	cErrors++;
3568	continue;
3569	}
3570
3571	if (GCPhysPdeSrc != pShwPde->GCPhys)
3572	{
3573	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3574	AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3575	# else
3576	AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3577	# endif
3578	GCPtr += 512 * _2M;
3579	cErrors++;
3580	continue;
3581	}
3582
3583	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3584	if ( (pPdpeDst->u & (X86_PDPE_US \| X86_PDPE_RW \| X86_PDPE_LM_NX))
3585	!= (PdpeSrc.u & (X86_PDPE_US \| X86_PDPE_RW \| X86_PDPE_LM_NX)))
3586	{
3587	AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3588	GCPtr += 512 * _2M;
3589	cErrors++;
3590	continue;
3591	}
3592	# endif
3593
3594	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3595	{
3596	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3597	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3598	GSTPD const *pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3599	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3600	PCX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
3601	# endif
3602	# endif /* PGM_GST_TYPE == PGM_TYPE_32BIT */
3603	/*
3604	* Iterate the shadow page directory.
3605	*/
3606	GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
3607	unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
3608
3609	for (;
3610	iPDDst < cPDEs;
3611	iPDDst++, GCPtr += cIncrement)
3612	{
3613	# if PGM_SHW_TYPE == PGM_TYPE_PAE
3614	const SHWPDE PdeDst = *pgmShwGetPaePDEPtr(pVCpu, GCPtr);
3615	# else
3616	const SHWPDE PdeDst = pPDDst->a[iPDDst];
3617	# endif
3618	if ( (PdeDst.u & X86_PDE_P)
3619	\|\| ((PdeDst.u & (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) )
3620	{
3621	HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
3622	PPGMPOOLPAGE pPoolPage = pgmPoolGetPage(pPool, HCPhysShw);
3623	if (!pPoolPage)
3624	{
3625	AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n",
3626	HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
3627	cErrors++;
3628	continue;
3629	}
3630	const SHWPT pPTDst = (const SHWPT )PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pPoolPage);
3631
3632	if (PdeDst.u & (X86_PDE4M_PWT \| X86_PDE4M_PCD))
3633	{
3634	AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
3635	GCPtr, (uint64_t)PdeDst.u));
3636	cErrors++;
3637	}
3638
3639	if (PdeDst.u & (X86_PDE4M_G \| X86_PDE4M_D))
3640	{
3641	AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n",
3642	GCPtr, (uint64_t)PdeDst.u));
3643	cErrors++;
3644	}
3645
3646	const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
3647	if (!(PdeSrc.u & X86_PDE_P))
3648	{
3649	AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
3650	GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
3651	cErrors++;
3652	continue;
3653	}
3654
3655	if ( !(PdeSrc.u & X86_PDE_PS)
3656	\|\| !fBigPagesSupported)
3657	{
3658	GCPhysGst = GST_GET_PDE_GCPHYS(PdeSrc);
3659	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3660	GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst \| ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
3661	# endif
3662	}
3663	else
3664	{
3665	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3666	if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
3667	{
3668	AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n",
3669	GCPtr, (uint64_t)PdeSrc.u));
3670	cErrors++;
3671	continue;
3672	}
3673	# endif
3674	GCPhysGst = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
3675	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3676	GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst \| (GCPtr & RT_BIT(X86_PAGE_2M_SHIFT)));
3677	# endif
3678	}
3679
3680	if ( pPoolPage->enmKind
3681	!= (!(PdeSrc.u & X86_PDE_PS) \|\| !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
3682	{
3683	AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n",
3684	pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
3685	cErrors++;
3686	}
3687
3688	PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
3689	if (!pPhysPage)
3690	{
3691	AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3692	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3693	cErrors++;
3694	continue;
3695	}
3696
3697	if (GCPhysGst != pPoolPage->GCPhys)
3698	{
3699	AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n",
3700	GCPhysGst, pPoolPage->GCPhys, GCPtr));
3701	cErrors++;
3702	continue;
3703	}
3704
3705	if ( !(PdeSrc.u & X86_PDE_PS)
3706	\|\| !fBigPagesSupported)
3707	{
3708	/*
3709	* Page Table.
3710	*/
3711	const GSTPT *pPTSrc;
3712	rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, PGM_A20_APPLY(pVCpu, GCPhysGst & ~(RTGCPHYS)(GUEST_PAGE_SIZE - 1)),
3713	&pPTSrc);
3714	if (RT_FAILURE(rc))
3715	{
3716	AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3717	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3718	cErrors++;
3719	continue;
3720	}
3721	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/))
3722	!= (PdeDst.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/)))
3723	{
3724	/// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
3725	// (This problem will go away when/if we shadow multiple CR3s.)
3726	AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3727	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3728	cErrors++;
3729	continue;
3730	}
3731	if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3732	{
3733	AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n",
3734	GCPtr, (uint64_t)PdeDst.u));
3735	cErrors++;
3736	continue;
3737	}
3738
3739	/* iterate the page table. */
3740	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3741	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3742	const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
3743	# else
3744	const unsigned offPTSrc = 0;
3745	# endif
3746	for (unsigned iPT = 0, off = 0;
3747	iPT < RT_ELEMENTS(pPTDst->a);
3748	iPT++, off += GUEST_PAGE_SIZE)
3749	{
3750	const SHWPTE PteDst = pPTDst->a[iPT];
3751
3752	/* skip not-present and dirty tracked entries. */
3753	if (!(SHW_PTE_GET_U(PteDst) & (X86_PTE_P \| PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
3754	continue;
3755	Assert(SHW_PTE_IS_P(PteDst));
3756
3757	const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
3758	if (!(PteSrc.u & X86_PTE_P))
3759	{
3760	# ifdef IN_RING3
3761	PGMAssertHandlerAndFlagsInSync(pVM);
3762	DBGFR3PagingDumpEx(pVM->pUVM, pVCpu->idCpu, DBGFPGDMP_FLAGS_CURRENT_CR3 \| DBGFPGDMP_FLAGS_CURRENT_MODE
3763	\| DBGFPGDMP_FLAGS_GUEST \| DBGFPGDMP_FLAGS_HEADER \| DBGFPGDMP_FLAGS_PRINT_CR3,
3764	0, 0, UINT64_MAX, 99, NULL);
3765	# endif
3766	AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
3767	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst), pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
3768	(uint64_t)GST_GET_PDE_GCPHYS(PdeSrc) + (iPT + offPTSrc) * sizeof(PteSrc)));
3769	cErrors++;
3770	continue;
3771	}
3772
3773	uint64_t fIgnoreFlags = GST_PTE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_G \| X86_PTE_D \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
3774	# if 1 /** @todo sync accessed bit properly... */
3775	fIgnoreFlags \|= X86_PTE_A;
3776	# endif
3777
3778	/* match the physical addresses */
3779	HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
3780	GCPhysGst = GST_GET_PTE_GCPHYS(PteSrc);
3781
3782	# ifdef IN_RING3
3783	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3784	if (RT_FAILURE(rc))
3785	{
3786	# if 0
3787	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
3788	{
3789	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3790	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3791	cErrors++;
3792	continue;
3793	}
3794	# endif
3795	}
3796	else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
3797	{
3798	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3799	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3800	cErrors++;
3801	continue;
3802	}
3803	# endif
3804
3805	pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
3806	if (!pPhysPage)
3807	{
3808	# if 0
3809	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
3810	{
3811	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3812	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3813	cErrors++;
3814	continue;
3815	}
3816	# endif
3817	if (SHW_PTE_IS_RW(PteDst))
3818	{
3819	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3820	GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3821	cErrors++;
3822	}
3823	fIgnoreFlags \|= X86_PTE_RW;
3824	}
3825	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
3826	{
3827	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3828	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3829	cErrors++;
3830	continue;
3831	}
3832
3833	/* flags */
3834	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
3835	{
3836	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
3837	{
3838	if (SHW_PTE_IS_RW(PteDst))
3839	{
3840	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
3841	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3842	cErrors++;
3843	continue;
3844	}
3845	fIgnoreFlags \|= X86_PTE_RW;
3846	}
3847	else
3848	{
3849	if ( SHW_PTE_IS_P(PteDst)
3850	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
3851	&& !PGM_PAGE_IS_MMIO(pPhysPage)
3852	# endif
3853	)
3854	{
3855	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
3856	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3857	cErrors++;
3858	continue;
3859	}
3860	fIgnoreFlags \|= X86_PTE_P;
3861	}
3862	}
3863	else
3864	{
3865	if ((PteSrc.u & (X86_PTE_RW \| X86_PTE_D)) == X86_PTE_RW)
3866	{
3867	if (SHW_PTE_IS_RW(PteDst))
3868	{
3869	AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
3870	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3871	cErrors++;
3872	continue;
3873	}
3874	if (!SHW_PTE_IS_TRACK_DIRTY(PteDst))
3875	{
3876	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3877	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3878	cErrors++;
3879	continue;
3880	}
3881	if (SHW_PTE_IS_D(PteDst))
3882	{
3883	AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3884	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3885	cErrors++;
3886	}
3887	# if 0 /** @todo sync access bit properly... */
3888	if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
3889	{
3890	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
3891	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3892	cErrors++;
3893	}
3894	fIgnoreFlags \|= X86_PTE_RW;
3895	# else
3896	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
3897	# endif
3898	}
3899	else if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
3900	{
3901	/* access bit emulation (not implemented). */
3902	if ((PteSrc.u & X86_PTE_A) \|\| SHW_PTE_IS_P(PteDst))
3903	{
3904	AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
3905	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3906	cErrors++;
3907	continue;
3908	}
3909	if (!SHW_PTE_IS_A(PteDst))
3910	{
3911	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
3912	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3913	cErrors++;
3914	}
3915	fIgnoreFlags \|= X86_PTE_P;
3916	}
3917	# ifdef DEBUG_sandervl
3918	fIgnoreFlags \|= X86_PTE_D \| X86_PTE_A;
3919	# endif
3920	}
3921
3922	if ( (PteSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
3923	&& (PteSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
3924	)
3925	{
3926	AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
3927	GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
3928	fIgnoreFlags, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3929	cErrors++;
3930	continue;
3931	}
3932	} /* foreach PTE */
3933	}
3934	else
3935	{
3936	/*
3937	* Big Page.
3938	*/
3939	uint64_t fIgnoreFlags = X86_PDE_AVL_MASK \| GST_PDE_PG_MASK \| X86_PDE4M_G \| X86_PDE4M_D \| X86_PDE4M_PS \| X86_PDE4M_PWT \| X86_PDE4M_PCD;
3940	if ((PdeSrc.u & (X86_PDE_RW \| X86_PDE4M_D)) == X86_PDE_RW)
3941	{
3942	if (PdeDst.u & X86_PDE_RW)
3943	{
3944	AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3945	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3946	cErrors++;
3947	continue;
3948	}
3949	if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
3950	{
3951	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3952	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3953	cErrors++;
3954	continue;
3955	}
3956	# if 0 /** @todo sync access bit properly... */
3957	if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
3958	{
3959	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
3960	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3961	cErrors++;
3962	}
3963	fIgnoreFlags \|= X86_PTE_RW;
3964	# else
3965	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
3966	# endif
3967	}
3968	else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3969	{
3970	/* access bit emulation (not implemented). */
3971	if ((PdeSrc.u & X86_PDE_A) \|\| SHW_PDE_IS_P(PdeDst))
3972	{
3973	AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3974	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3975	cErrors++;
3976	continue;
3977	}
3978	if (!SHW_PDE_IS_A(PdeDst))
3979	{
3980	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3981	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3982	cErrors++;
3983	}
3984	fIgnoreFlags \|= X86_PTE_P;
3985	}
3986
3987	if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
3988	{
3989	AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
3990	GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
3991	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3992	cErrors++;
3993	}
3994
3995	/* iterate the page table. */
3996	for (unsigned iPT = 0, off = 0;
3997	iPT < RT_ELEMENTS(pPTDst->a);
3998	iPT++, off += GUEST_PAGE_SIZE, GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst + GUEST_PAGE_SIZE))
3999	{
4000	const SHWPTE PteDst = pPTDst->a[iPT];
4001
4002	if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
4003	{
4004	AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
4005	GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4006	cErrors++;
4007	}
4008
4009	/* skip not-present entries. */
4010	if (!SHW_PTE_IS_P(PteDst)) /** @todo deal with ALL handlers and CSAM !P pages! */
4011	continue;
4012
4013	fIgnoreFlags = X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT \| X86_PTE_D \| X86_PTE_A \| X86_PTE_G \| X86_PTE_PAE_NX;
4014
4015	/* match the physical addresses */
4016	HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
4017
4018	# ifdef IN_RING3
4019	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4020	if (RT_FAILURE(rc))
4021	{
4022	# if 0
4023	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4024	{
4025	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4026	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4027	cErrors++;
4028	}
4029	# endif
4030	}
4031	else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
4032	{
4033	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4034	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4035	cErrors++;
4036	continue;
4037	}
4038	# endif
4039	pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4040	if (!pPhysPage)
4041	{
4042	# if 0 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4043	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4044	{
4045	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4046	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4047	cErrors++;
4048	continue;
4049	}
4050	# endif
4051	if (SHW_PTE_IS_RW(PteDst))
4052	{
4053	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4054	GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4055	cErrors++;
4056	}
4057	fIgnoreFlags \|= X86_PTE_RW;
4058	}
4059	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4060	{
4061	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4062	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4063	cErrors++;
4064	continue;
4065	}
4066
4067	/* flags */
4068	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4069	{
4070	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4071	{
4072	if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
4073	{
4074	if (SHW_PTE_IS_RW(PteDst))
4075	{
4076	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4077	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4078	cErrors++;
4079	continue;
4080	}
4081	fIgnoreFlags \|= X86_PTE_RW;
4082	}
4083	}
4084	else
4085	{
4086	if ( SHW_PTE_IS_P(PteDst)
4087	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
4088	&& !PGM_PAGE_IS_MMIO(pPhysPage)
4089	# endif
4090	)
4091	{
4092	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4093	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4094	cErrors++;
4095	continue;
4096	}
4097	fIgnoreFlags \|= X86_PTE_P;
4098	}
4099	}
4100
4101	if ( (PdeSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4102	&& (PdeSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags) /* lazy phys handler dereg. */
4103	)
4104	{
4105	AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
4106	GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
4107	fIgnoreFlags, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4108	cErrors++;
4109	continue;
4110	}
4111	} /* for each PTE */
4112	}
4113	}
4114	/* not present */
4115
4116	} /* for each PDE */
4117
4118	} /* for each PDPTE */
4119
4120	} /* for each PML4E */
4121
4122	# ifdef DEBUG
4123	if (cErrors)
4124	LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
4125	# endif
4126	# endif /* GST is in {32BIT, PAE, AMD64} */
4127	return cErrors;
4128	#endif /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE */
4129	}
4130	#endif /* VBOX_STRICT */
4131
4132
4133	/**
4134	* Sets up the CR3 for shadow paging
4135	*
4136	* @returns Strict VBox status code.
4137	* @retval VINF_SUCCESS.
4138	*
4139	* @param pVCpu The cross context virtual CPU structure.
4140	* @param GCPhysCR3 The physical address in the CR3 register. (A20 mask
4141	* already applied.)
4142	*/
4143	PGM_BTH_DECL(int, MapCR3)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3)
4144	{
4145	PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
4146	int rc = VINF_SUCCESS;
4147
4148	/* Update guest paging info. */
4149	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
4150	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
4151	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
4152
4153	LogFlow(("MapCR3: %RGp\n", GCPhysCR3));
4154	PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
4155
4156	# if PGM_GST_TYPE == PGM_TYPE_PAE
4157	if (!pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped))
4158	# endif
4159	{
4160	/*
4161	* Map the page CR3 points at.
4162	*/
4163	RTHCPTR HCPtrGuestCR3;
4164	rc = pgmGstMapCr3(pVCpu, GCPhysCR3, &HCPtrGuestCR3);
4165	if (RT_SUCCESS(rc))
4166	{
4167	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4168	# ifdef IN_RING3
4169	pVCpu->pgm.s.pGst32BitPdR3 = (PX86PD)HCPtrGuestCR3;
4170	pVCpu->pgm.s.pGst32BitPdR0 = NIL_RTR0PTR;
4171	# else
4172	pVCpu->pgm.s.pGst32BitPdR3 = NIL_RTR3PTR;
4173	pVCpu->pgm.s.pGst32BitPdR0 = (PX86PD)HCPtrGuestCR3;
4174	# endif
4175
4176	# elif PGM_GST_TYPE == PGM_TYPE_PAE
4177	# ifdef IN_RING3
4178	pVCpu->pgm.s.pGstPaePdptR3 = (PX86PDPT)HCPtrGuestCR3;
4179	pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
4180	# else
4181	pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
4182	pVCpu->pgm.s.pGstPaePdptR0 = (PX86PDPT)HCPtrGuestCR3;
4183	# endif
4184
4185	/*
4186	* Update CPUM and map the 4 PDs too.
4187	*/
4188	X86PDPE aGstPaePdpes[X86_PG_PAE_PDPE_ENTRIES];
4189	memcpy(&aGstPaePdpes, HCPtrGuestCR3, sizeof(aGstPaePdpes));
4190	CPUMSetGuestPaePdpes(pVCpu, &aGstPaePdpes[0]);
4191	PGMGstMapPaePdpes(pVCpu, &aGstPaePdpes[0]);
4192
4193	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
4194	# ifdef IN_RING3
4195	pVCpu->pgm.s.pGstAmd64Pml4R3 = (PX86PML4)HCPtrGuestCR3;
4196	pVCpu->pgm.s.pGstAmd64Pml4R0 = NIL_RTR0PTR;
4197	# else
4198	pVCpu->pgm.s.pGstAmd64Pml4R3 = NIL_RTR3PTR;
4199	pVCpu->pgm.s.pGstAmd64Pml4R0 = (PX86PML4)HCPtrGuestCR3;
4200	# endif
4201	# endif
4202	}
4203	else
4204	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4205	}
4206
4207	/*
4208	* Reset fPaePdpesAndCr3Mapped for all modes as there's no guarantee that
4209	* we were called in the correct sequence of PAE followed by other modes
4210	* without CR3 changing in between.
4211	*/
4212	pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = false;
4213	pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = false;
4214	#endif
4215
4216	/*
4217	* Update shadow paging info for guest modes with paging (32-bit, PAE, AMD64).
4218	*/
4219	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4220	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4221	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64) \
4222	&& ( PGM_GST_TYPE != PGM_TYPE_REAL \
4223	&& PGM_GST_TYPE != PGM_TYPE_PROT))
4224
4225	Assert(!pVM->pgm.s.fNestedPaging);
4226	PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
4227
4228	/*
4229	* Update the shadow root page as well since that's not fixed.
4230	*/
4231	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4232	PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
4233	PPGMPOOLPAGE pNewShwPageCR3;
4234
4235	PGM_LOCK_VOID(pVM);
4236
4237	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4238	if (pPool->cDirtyPages)
4239	pgmPoolResetDirtyPages(pVM);
4240	# endif
4241
4242	Assert(!(GCPhysCR3 >> (GUEST_PAGE_SHIFT + 32))); /** @todo what is this for? */
4243	int const rc2 = pgmPoolAlloc(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE,
4244	PGM_A20_IS_ENABLED(pVCpu), NIL_PGMPOOL_IDX, UINT32_MAX, true /fLockPage/, &pNewShwPageCR3);
4245	AssertFatalRC(rc2);
4246
4247	pVCpu->pgm.s.pShwPageCR3R3 = pgmPoolConvertPageToR3(pPool, pNewShwPageCR3);
4248	pVCpu->pgm.s.pShwPageCR3R0 = pgmPoolConvertPageToR0(pPool, pNewShwPageCR3);
4249
4250	/* Set the current hypervisor CR3. */
4251	CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
4252
4253	/* Clean up the old CR3 root. */
4254	if ( pOldShwPageCR3
4255	&& pOldShwPageCR3 != pNewShwPageCR3 /* @todo can happen due to incorrect syncing between REM & PGM; find the real cause */)
4256	{
4257	Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
4258
4259	/* Mark the page as unlocked; allow flushing again. */
4260	pgmPoolUnlockPage(pPool, pOldShwPageCR3);
4261
4262	pgmPoolFreeByPage(pPool, pOldShwPageCR3, NIL_PGMPOOL_IDX, UINT32_MAX);
4263	}
4264	PGM_UNLOCK(pVM);
4265	# else
4266	NOREF(GCPhysCR3);
4267	# endif
4268
4269	return rc;
4270	}
4271
4272	/**
4273	* Unmaps the shadow CR3.
4274	*
4275	* @returns VBox status, no specials.
4276	* @param pVCpu The cross context virtual CPU structure.
4277	*/
4278	PGM_BTH_DECL(int, UnmapCR3)(PVMCPUCC pVCpu)
4279	{
4280	LogFlow(("UnmapCR3\n"));
4281
4282	int rc = VINF_SUCCESS;
4283	PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
4284
4285	/*
4286	* Update guest paging info.
4287	*/
4288	#if PGM_GST_TYPE == PGM_TYPE_32BIT
4289	pVCpu->pgm.s.pGst32BitPdR3 = 0;
4290	pVCpu->pgm.s.pGst32BitPdR0 = 0;
4291
4292	#elif PGM_GST_TYPE == PGM_TYPE_PAE
4293	pVCpu->pgm.s.pGstPaePdptR3 = 0;
4294	pVCpu->pgm.s.pGstPaePdptR0 = 0;
4295	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4296	{
4297	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4298	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4299	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4300	}
4301
4302	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
4303	pVCpu->pgm.s.pGstAmd64Pml4R3 = 0;
4304	pVCpu->pgm.s.pGstAmd64Pml4R0 = 0;
4305
4306	#else /* prot/real mode stub */
4307	/* nothing to do */
4308	#endif
4309
4310	/*
4311	* Update second-level address translation info.
4312	*/
4313	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
4314	pVCpu->pgm.s.pGstEptPml4R3 = 0;
4315	pVCpu->pgm.s.pGstEptPml4R0 = 0;
4316	#endif
4317
4318	pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = false;
4319	pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = false;
4320
4321	/*
4322	* Update shadow paging info.
4323	*/
4324	#if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4325	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4326	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64))
4327	# if PGM_GST_TYPE != PGM_TYPE_REAL
4328	Assert(!pVM->pgm.s.fNestedPaging);
4329	# endif
4330	PGM_LOCK_VOID(pVM);
4331
4332	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4333	{
4334	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4335
4336	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4337	if (pPool->cDirtyPages)
4338	pgmPoolResetDirtyPages(pVM);
4339	# endif
4340
4341	/* Mark the page as unlocked; allow flushing again. */
4342	pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4343
4344	pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), NIL_PGMPOOL_IDX, UINT32_MAX);
4345	pVCpu->pgm.s.pShwPageCR3R3 = 0;
4346	pVCpu->pgm.s.pShwPageCR3R0 = 0;
4347	}
4348
4349	PGM_UNLOCK(pVM);
4350	#endif
4351
4352	return rc;
4353	}
4354

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 93666

Download in other formats: