PGMAllBth.h@ 74791

Last change on this file since 74791 was 74791, checked in by vboxsync, 6 years ago
vm.h,VMM,REM: s/VM_FF_IS_PENDING/VM_FF_IS_ANY_SET/g to emphasize the plurality of the flags argument and encourage using VM_FF_IS_SET. bugref:9180
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 217.2 KB

Line
1	/* $Id: PGMAllBth.h 74791 2018-10-12 10:44:17Z vboxsync $ */
2	/** @file
3	* VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4	*
5	* @remarks Extended page tables (intel) are built with PGM_GST_TYPE set to
6	* PGM_TYPE_PROT (and PGM_SHW_TYPE set to PGM_TYPE_EPT).
7	* bird: WTF does this mean these days? Looking at PGMAll.cpp it's
8	*
9	* @remarks This file is one big \#ifdef-orgy!
10	*
11	*/
12
13	/*
14	* Copyright (C) 2006-2017 Oracle Corporation
15	*
16	* This file is part of VirtualBox Open Source Edition (OSE), as
17	* available from http://www.virtualbox.org. This file is free software;
18	* you can redistribute it and/or modify it under the terms of the GNU
19	* General Public License (GPL) as published by the Free Software
20	* Foundation, in version 2 as it comes in the "COPYING" file of the
21	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
22	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
23	*/
24
25	#ifdef _MSC_VER
26	/** @todo we're generating unnecessary code in nested/ept shadow mode and for
27	* real/prot-guest+RC mode. */
28	# pragma warning(disable: 4505)
29	#endif
30
31
32	/*********************************************************************************************************************************
33	* Internal Functions *
34	*********************************************************************************************************************************/
35	RT_C_DECLS_BEGIN
36	PGM_BTH_DECL(int, Enter)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3);
37	#ifndef IN_RING3
38	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken);
39	#endif
40	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
41	static int PGM_BTH_NAME(SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr);
42	static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, RTGCPTR GCPtrPage);
43	static int PGM_BTH_NAME(SyncPT)(PVMCPU pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage);
44	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
45	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
46	# else
47	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
48	#endif
49	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uErr);
50	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
51	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
52	#ifdef VBOX_STRICT
53	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
54	#endif
55	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3);
56	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu);
57
58	#ifdef IN_RING3
59	PGM_BTH_DECL(int, Relocate)(PVMCPU pVCpu, RTGCPTR offDelta);
60	#endif
61	RT_C_DECLS_END
62
63
64
65
66	/*
67	* Filter out some illegal combinations of guest and shadow paging, so we can
68	* remove redundant checks inside functions.
69	*/
70	#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE \
71	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
72	# error "Invalid combination; PAE guest implies PAE shadow"
73	#endif
74
75	#if (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
76	&& !( PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64 \
77	\|\| PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \|\| PGM_SHW_TYPE == PGM_TYPE_NONE)
78	# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
79	#endif
80
81	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) \
82	&& !( PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
83	\|\| PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \|\| PGM_SHW_TYPE == PGM_TYPE_NONE)
84	# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
85	#endif
86
87	#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE) \
88	\|\| (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
89	# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
90	#endif
91
92
93	/**
94	* Enters the shadow+guest mode.
95	*
96	* @returns VBox status code.
97	* @param pVCpu The cross context virtual CPU structure.
98	* @param GCPhysCR3 The physical address from the CR3 register.
99	*/
100	PGM_BTH_DECL(int, Enter)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3)
101	{
102	/* Here we deal with allocation of the root shadow page table for real and protected mode during mode switches;
103	* Other modes rely on MapCR3/UnmapCR3 to setup the shadow root page tables.
104	*/
105	#if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
106	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
107	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64) \
108	&& ( PGM_GST_TYPE == PGM_TYPE_REAL \
109	\|\| PGM_GST_TYPE == PGM_TYPE_PROT))
110
111	PVM pVM = pVCpu->CTX_SUFF(pVM);
112
113	Assert((HMIsNestedPagingActive(pVM) \|\| VM_IS_NEM_ENABLED(pVM)) == pVM->pgm.s.fNestedPaging);
114	Assert(!pVM->pgm.s.fNestedPaging);
115
116	pgmLock(pVM);
117	/* Note: we only really need shadow paging in real and protected mode for VT-x and AMD-V (excluding nested paging/EPT modes),
118	* but any calls to GC need a proper shadow page setup as well.
119	*/
120	/* Free the previous root mapping if still active. */
121	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
122	PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
123	if (pOldShwPageCR3)
124	{
125	Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
126
127	/* Mark the page as unlocked; allow flushing again. */
128	pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
129
130	# ifndef PGM_WITHOUT_MAPPINGS
131	/* Remove the hypervisor mappings from the shadow page table. */
132	pgmMapDeactivateCR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
133	# endif
134
135	pgmPoolFreeByPage(pPool, pOldShwPageCR3, NIL_PGMPOOL_IDX, UINT32_MAX);
136	pVCpu->pgm.s.pShwPageCR3R3 = NIL_RTR3PTR;
137	pVCpu->pgm.s.pShwPageCR3RC = NIL_RTRCPTR;
138	pVCpu->pgm.s.pShwPageCR3R0 = NIL_RTR0PTR;
139	}
140
141	/* construct a fake address. */
142	GCPhysCR3 = RT_BIT_64(63);
143	PPGMPOOLPAGE pNewShwPageCR3;
144	int rc = pgmPoolAlloc(pVM, GCPhysCR3, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
145	NIL_PGMPOOL_IDX, UINT32_MAX, false /fLockPage/,
146	&pNewShwPageCR3);
147	AssertRCReturn(rc, rc);
148
149	pVCpu->pgm.s.pShwPageCR3R3 = (R3PTRTYPE(PPGMPOOLPAGE))MMHyperCCToR3(pVM, pNewShwPageCR3);
150	pVCpu->pgm.s.pShwPageCR3RC = (RCPTRTYPE(PPGMPOOLPAGE))MMHyperCCToRC(pVM, pNewShwPageCR3);
151	pVCpu->pgm.s.pShwPageCR3R0 = (R0PTRTYPE(PPGMPOOLPAGE))MMHyperCCToR0(pVM, pNewShwPageCR3);
152
153	/* Mark the page as locked; disallow flushing. */
154	pgmPoolLockPage(pPool, pNewShwPageCR3);
155
156	/* Set the current hypervisor CR3. */
157	CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
158
159	# ifndef PGM_WITHOUT_MAPPINGS
160	/* Apply all hypervisor mappings to the new CR3. */
161	rc = pgmMapActivateCR3(pVM, pNewShwPageCR3);
162	# endif
163
164	pgmUnlock(pVM);
165	return rc;
166	#else
167	NOREF(pVCpu); NOREF(GCPhysCR3);
168	return VINF_SUCCESS;
169	#endif
170	}
171
172
173	#ifndef IN_RING3
174
175	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
176	/**
177	* Deal with a guest page fault.
178	*
179	* @returns Strict VBox status code.
180	* @retval VINF_EM_RAW_GUEST_TRAP
181	* @retval VINF_EM_RAW_EMULATE_INSTR
182	*
183	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
184	* @param pGstWalk The guest page table walk result.
185	* @param uErr The error code.
186	*/
187	PGM_BTH_DECL(VBOXSTRICTRC, Trap0eHandlerGuestFault)(PVMCPU pVCpu, PGSTPTWALK pGstWalk, RTGCUINT uErr)
188	{
189	# if !defined(PGM_WITHOUT_MAPPINGS) && (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE)
190	/*
191	* Check for write conflicts with our hypervisor mapping.
192	*
193	* If the guest happens to access a non-present page, where our hypervisor
194	* is currently mapped, then we'll create a #PF storm in the guest.
195	*/
196	if ( (uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RW)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RW)
197	&& pgmMapAreMappingsEnabled(pVCpu->CTX_SUFF(pVM))
198	&& MMHyperIsInsideArea(pVCpu->CTX_SUFF(pVM), pGstWalk->Core.GCPtr))
199	{
200	/* Force a CR3 sync to check for conflicts and emulate the instruction. */
201	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
202	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
203	return VINF_EM_RAW_EMULATE_INSTR;
204	}
205	# endif
206
207	/*
208	* Calc the error code for the guest trap.
209	*/
210	uint32_t uNewErr = GST_IS_NX_ACTIVE(pVCpu)
211	? uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID)
212	: uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US);
213	if ( pGstWalk->Core.fRsvdError
214	\|\| pGstWalk->Core.fBadPhysAddr)
215	{
216	uNewErr \|= X86_TRAP_PF_RSVD \| X86_TRAP_PF_P;
217	Assert(!pGstWalk->Core.fNotPresent);
218	}
219	else if (!pGstWalk->Core.fNotPresent)
220	uNewErr \|= X86_TRAP_PF_P;
221	TRPMSetErrorCode(pVCpu, uNewErr);
222
223	LogFlow(("Guest trap; cr2=%RGv uErr=%RGv lvl=%d\n", pGstWalk->Core.GCPtr, uErr, pGstWalk->Core.uLevel));
224	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
225	return VINF_EM_RAW_GUEST_TRAP;
226	}
227	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
228
229
230	#if !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
231	/**
232	* Deal with a guest page fault.
233	*
234	* The caller has taken the PGM lock.
235	*
236	* @returns Strict VBox status code.
237	*
238	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
239	* @param uErr The error code.
240	* @param pRegFrame The register frame.
241	* @param pvFault The fault address.
242	* @param pPage The guest page at @a pvFault.
243	* @param pGstWalk The guest page table walk result.
244	* @param pfLockTaken PGM lock taken here or not (out). This is true
245	* when we're called.
246	*/
247	static VBOXSTRICTRC PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame,
248	RTGCPTR pvFault, PPGMPAGE pPage, bool *pfLockTaken
249	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \|\| defined(DOXYGEN_RUNNING)
250	, PGSTPTWALK pGstWalk
251	# endif
252	)
253	{
254	# if !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
255	GSTPDE const PdeSrcDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
256	# endif
257	PVM pVM = pVCpu->CTX_SUFF(pVM);
258	VBOXSTRICTRC rcStrict;
259
260	if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
261	{
262	/*
263	* Physical page access handler.
264	*/
265	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
266	const RTGCPHYS GCPhysFault = pGstWalk->Core.GCPhys;
267	# else
268	const RTGCPHYS GCPhysFault = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault);
269	# endif
270	PPGMPHYSHANDLER pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
271	if (pCur)
272	{
273	PPGMPHYSHANDLERTYPEINT pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
274
275	# ifdef PGM_SYNC_N_PAGES
276	/*
277	* If the region is write protected and we got a page not present fault, then sync
278	* the pages. If the fault was caused by a read, then restart the instruction.
279	* In case of write access continue to the GC write handler.
280	*
281	* ASSUMES that there is only one handler per page or that they have similar write properties.
282	*/
283	if ( !(uErr & X86_TRAP_PF_P)
284	&& pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
285	{
286	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
287	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
288	# else
289	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
290	# endif
291	if ( RT_FAILURE(rcStrict)
292	\|\| !(uErr & X86_TRAP_PF_RW)
293	\|\| rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
294	{
295	AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
296	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
297	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
298	return rcStrict;
299	}
300	}
301	# endif
302	# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
303	/*
304	* If the access was not thru a #PF(RSVD\|...) resync the page.
305	*/
306	if ( !(uErr & X86_TRAP_PF_RSVD)
307	&& pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
308	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
309	&& pGstWalk->Core.fEffectiveRW
310	&& !pGstWalk->Core.fEffectiveUS /** @todo Remove pGstWalk->Core.fEffectiveUS and X86_PTE_US further down in the sync code. */
311	# endif
312	)
313	{
314	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
315	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
316	# else
317	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
318	# endif
319	if ( RT_FAILURE(rcStrict)
320	\|\| rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
321	{
322	AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
323	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
324	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
325	return rcStrict;
326	}
327	}
328	# endif
329
330	AssertMsg( pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
331	\|\| (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW)),
332	("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enmKind=%d\n",
333	pvFault, GCPhysFault, pPage, uErr, pCurType->enmKind));
334	if (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
335	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysWrite);
336	else
337	{
338	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysAll);
339	if (uErr & X86_TRAP_PF_RSVD) STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysAllOpt);
340	}
341
342	if (pCurType->CTX_SUFF(pfnPfHandler))
343	{
344	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
345	void *pvUser = pCur->CTX_SUFF(pvUser);
346
347	STAM_PROFILE_START(&pCur->Stat, h);
348	if (pCur->hType != pPool->hAccessHandlerType)
349	{
350	pgmUnlock(pVM);
351	*pfLockTaken = false;
352	}
353
354	rcStrict = pCurType->CTX_SUFF(pfnPfHandler)(pVM, pVCpu, uErr, pRegFrame, pvFault, GCPhysFault, pvUser);
355
356	# ifdef VBOX_WITH_STATISTICS
357	pgmLock(pVM);
358	pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
359	if (pCur)
360	STAM_PROFILE_STOP(&pCur->Stat, h);
361	pgmUnlock(pVM);
362	# endif
363	}
364	else
365	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
366
367	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndPhys; });
368	return rcStrict;
369	}
370	}
371	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
372	else
373	{
374	# ifdef PGM_SYNC_N_PAGES
375	/*
376	* If the region is write protected and we got a page not present fault, then sync
377	* the pages. If the fault was caused by a read, then restart the instruction.
378	* In case of write access continue to the GC write handler.
379	*/
380	if ( PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) < PGM_PAGE_HNDL_PHYS_STATE_ALL
381	&& !(uErr & X86_TRAP_PF_P))
382	{
383	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
384	if ( RT_FAILURE(rcStrict)
385	\|\| rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE
386	\|\| !(uErr & X86_TRAP_PF_RW))
387	{
388	AssertRC(rcStrict);
389	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
390	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndVirt; });
391	return rcStrict;
392	}
393	}
394	# endif
395	/*
396	* Ok, it's an virtual page access handler.
397	*
398	* Since it's faster to search by address, we'll do that first
399	* and then retry by GCPhys if that fails.
400	*/
401	/** @todo r=bird: perhaps we should consider looking up by physical address directly now?
402	* r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be
403	* out of sync, because the page was changed without us noticing it (not-present -> present
404	* without invlpg or mov cr3, xxx).
405	*/
406	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
407	if (pCur)
408	{
409	PPGMVIRTHANDLERTYPEINT pCurType = PGMVIRTANDLER_GET_TYPE(pVM, pCur);
410	AssertMsg(!(pvFault - pCur->Core.Key < pCur->cb)
411	\|\| ( pCurType->enmKind != PGMVIRTHANDLERKIND_WRITE
412	\|\| !(uErr & X86_TRAP_PF_P)
413	\|\| (pCurType->enmKind == PGMVIRTHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW))),
414	("Unexpected trap for virtual handler: %RGv (phys=%RGp) pPage=%R[pgmpage] uErr=%X, enumKind=%d\n",
415	pvFault, pGstWalk->Core.GCPhys, pPage, uErr, pCurType->enmKind));
416
417	if ( pvFault - pCur->Core.Key < pCur->cb
418	&& ( uErr & X86_TRAP_PF_RW
419	\|\| pCurType->enmKind != PGMVIRTHANDLERKIND_WRITE ) )
420	{
421	# ifdef IN_RC
422	STAM_PROFILE_START(&pCur->Stat, h);
423	RTGCPTR GCPtrStart = pCur->Core.Key;
424	void *pvUser = pCur->CTX_SUFF(pvUser);
425	pgmUnlock(pVM);
426	*pfLockTaken = false;
427
428	rcStrict = pCurType->CTX_SUFF(pfnPfHandler)(pVM, pVCpu, uErr, pRegFrame, pvFault, GCPtrStart,
429	pvFault - GCPtrStart, pvUser);
430
431	# ifdef VBOX_WITH_STATISTICS
432	pgmLock(pVM);
433	pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
434	if (pCur)
435	STAM_PROFILE_STOP(&pCur->Stat, h);
436	pgmUnlock(pVM);
437	# endif
438	# else
439	rcStrict = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
440	# endif
441	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersVirtual);
442	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
443	return rcStrict;
444	}
445	/* Unhandled part of a monitored page */
446	Log(("Unhandled part of monitored page %RGv\n", pvFault));
447	}
448	else
449	{
450	/* Check by physical address. */
451	unsigned iPage;
452	pCur = pgmHandlerVirtualFindByPhysAddr(pVM, pGstWalk->Core.GCPhys, &iPage);
453	if (pCur)
454	{
455	PPGMVIRTHANDLERTYPEINT pCurType = PGMVIRTANDLER_GET_TYPE(pVM, pCur);
456	if ( uErr & X86_TRAP_PF_RW
457	\|\| pCurType->enmKind != PGMVIRTHANDLERKIND_WRITE )
458	{
459	Assert( (pCur->aPhysToVirt[iPage].Core.Key & X86_PTE_PAE_PG_MASK)
460	== (pGstWalk->Core.GCPhys & X86_PTE_PAE_PG_MASK));
461	# ifdef IN_RC
462	STAM_PROFILE_START(&pCur->Stat, h);
463	RTGCPTR GCPtrStart = pCur->Core.Key;
464	void *pvUser = pCur->CTX_SUFF(pvUser);
465	pgmUnlock(pVM);
466	*pfLockTaken = false;
467
468	RTGCPTR off = (iPage << PAGE_SHIFT)
469	+ (pvFault & PAGE_OFFSET_MASK)
470	- (GCPtrStart & PAGE_OFFSET_MASK);
471	Assert(off < pCur->cb);
472	rcStrict = pCurType->CTX_SUFF(pfnPfHandler)(pVM, pVCpu, uErr, pRegFrame, pvFault, GCPtrStart, off, pvUser);
473
474	# ifdef VBOX_WITH_STATISTICS
475	pgmLock(pVM);
476	pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, GCPtrStart);
477	if (pCur)
478	STAM_PROFILE_STOP(&pCur->Stat, h);
479	pgmUnlock(pVM);
480	# endif
481	# else
482	rcStrict = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
483	# endif
484	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersVirtualByPhys);
485	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
486	return rcStrict;
487	}
488	}
489	}
490	}
491	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
492
493	/*
494	* There is a handled area of the page, but this fault doesn't belong to it.
495	* We must emulate the instruction.
496	*
497	* To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
498	* we first check if this was a page-not-present fault for a page with only
499	* write access handlers. Restart the instruction if it wasn't a write access.
500	*/
501	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersUnhandled);
502
503	if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
504	&& !(uErr & X86_TRAP_PF_P))
505	{
506	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
507	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
508	# else
509	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
510	# endif
511	if ( RT_FAILURE(rcStrict)
512	\|\| rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE
513	\|\| !(uErr & X86_TRAP_PF_RW))
514	{
515	AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
516	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
517	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
518	return rcStrict;
519	}
520	}
521
522	/** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
523	* It's writing to an unhandled part of the LDT page several million times.
524	*/
525	rcStrict = PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault);
526	LogFlow(("PGM: PGMInterpretInstruction -> rcStrict=%d pPage=%R[pgmpage]\n", VBOXSTRICTRC_VAL(rcStrict), pPage));
527	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndUnhandled; });
528	return rcStrict;
529	} /* if any kind of handler */
530	# endif /* !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE*/
531
532
533	/**
534	* \#PF Handler for raw-mode guest execution.
535	*
536	* @returns VBox status code (appropriate for trap handling and GC return).
537	*
538	* @param pVCpu The cross context virtual CPU structure.
539	* @param uErr The trap error code.
540	* @param pRegFrame Trap register frame.
541	* @param pvFault The fault address.
542	* @param pfLockTaken PGM lock taken here or not (out)
543	*/
544	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken)
545	{
546	PVM pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
547
548	*pfLockTaken = false;
549
550	# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \
551	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
552	&& !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
553	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
554	&& PGM_SHW_TYPE != PGM_TYPE_NONE
555	int rc;
556
557	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
558	/*
559	* Walk the guest page translation tables and check if it's a guest fault.
560	*/
561	GSTPTWALK GstWalk;
562	rc = PGM_GST_NAME(Walk)(pVCpu, pvFault, &GstWalk);
563	if (RT_FAILURE_NP(rc))
564	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &GstWalk, uErr));
565
566	/* assert some GstWalk sanity. */
567	# if PGM_GST_TYPE == PGM_TYPE_AMD64
568	/AssertMsg(GstWalk.Pml4e.u == GstWalk.pPml4e->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pml4e.u, (uint64_t)GstWalk.pPml4e->u)); - not always true with SMP guests. /
569	# endif
570	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
571	/AssertMsg(GstWalk.Pdpe.u == GstWalk.pPdpe->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pdpe.u, (uint64_t)GstWalk.pPdpe->u)); - ditto /
572	# endif
573	/AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); - ditto /
574	/AssertMsg(GstWalk.Core.fBigPage \|\| GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); - ditto /
575	Assert(GstWalk.Core.fSucceeded);
576
577	if (uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID))
578	{
579	if ( ( (uErr & X86_TRAP_PF_RW)
580	&& !GstWalk.Core.fEffectiveRW
581	&& ( (uErr & X86_TRAP_PF_US)
582	\|\| CPUMIsGuestR0WriteProtEnabled(pVCpu)) )
583	\|\| ((uErr & X86_TRAP_PF_US) && !GstWalk.Core.fEffectiveUS)
584	\|\| ((uErr & X86_TRAP_PF_ID) && GstWalk.Core.fEffectiveNX)
585	)
586	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &GstWalk, uErr));
587	}
588
589	/*
590	* Set the accessed and dirty flags.
591	*/
592	# if PGM_GST_TYPE == PGM_TYPE_AMD64
593	GstWalk.Pml4e.u \|= X86_PML4E_A;
594	GstWalk.pPml4e->u \|= X86_PML4E_A;
595	GstWalk.Pdpe.u \|= X86_PDPE_A;
596	GstWalk.pPdpe->u \|= X86_PDPE_A;
597	# endif
598	if (GstWalk.Core.fBigPage)
599	{
600	Assert(GstWalk.Pde.b.u1Size);
601	if (uErr & X86_TRAP_PF_RW)
602	{
603	GstWalk.Pde.u \|= X86_PDE4M_A \| X86_PDE4M_D;
604	GstWalk.pPde->u \|= X86_PDE4M_A \| X86_PDE4M_D;
605	}
606	else
607	{
608	GstWalk.Pde.u \|= X86_PDE4M_A;
609	GstWalk.pPde->u \|= X86_PDE4M_A;
610	}
611	}
612	else
613	{
614	Assert(!GstWalk.Pde.b.u1Size);
615	GstWalk.Pde.u \|= X86_PDE_A;
616	GstWalk.pPde->u \|= X86_PDE_A;
617	if (uErr & X86_TRAP_PF_RW)
618	{
619	# ifdef VBOX_WITH_STATISTICS
620	if (!GstWalk.Pte.n.u1Dirty)
621	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtiedPage));
622	else
623	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageAlreadyDirty));
624	# endif
625	GstWalk.Pte.u \|= X86_PTE_A \| X86_PTE_D;
626	GstWalk.pPte->u \|= X86_PTE_A \| X86_PTE_D;
627	}
628	else
629	{
630	GstWalk.Pte.u \|= X86_PTE_A;
631	GstWalk.pPte->u \|= X86_PTE_A;
632	}
633	Assert(GstWalk.Pte.u == GstWalk.pPte->u);
634	}
635	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u \|\| GstWalk.pPte->u == GstWalk.pPde->u,
636	("%RX64 %RX64 pPte=%p pPde=%p Pte=%RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u, GstWalk.pPte, GstWalk.pPde, (uint64_t)GstWalk.pPte->u));
637	# else /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
638	GSTPDE const PdeSrcDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A}; /** @todo eliminate this */
639	# endif /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
640
641	/* Take the big lock now. */
642	*pfLockTaken = true;
643	pgmLock(pVM);
644
645	# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
646	/*
647	* If it is a reserved bit fault we know that it is an MMIO (access
648	* handler) related fault and can skip some 200 lines of code.
649	*/
650	if (uErr & X86_TRAP_PF_RSVD)
651	{
652	Assert(uErr & X86_TRAP_PF_P);
653	PPGMPAGE pPage;
654	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
655	rc = pgmPhysGetPageEx(pVM, GstWalk.Core.GCPhys, &pPage);
656	if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
657	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
658	pfLockTaken, &GstWalk));
659	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
660	# else
661	rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault), &pPage);
662	if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
663	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
664	pfLockTaken));
665	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
666	# endif
667	AssertRC(rc);
668	PGM_INVL_PG(pVCpu, pvFault);
669	return rc; /* Restart with the corrected entry. */
670	}
671	# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
672
673	/*
674	* Fetch the guest PDE, PDPE and PML4E.
675	*/
676	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
677	const unsigned iPDDst = pvFault >> SHW_PD_SHIFT;
678	PX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
679
680	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
681	const unsigned iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK; /* pPDDst index, not used with the pool. */
682	PX86PDPAE pPDDst;
683	# if PGM_GST_TYPE == PGM_TYPE_PAE
684	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, GstWalk.Pdpe.u, &pPDDst);
685	# else
686	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, X86_PDPE_P, &pPDDst); /* RW, US and A are reserved in PAE mode. */
687	# endif
688	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
689
690	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
691	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
692	PX86PDPAE pPDDst;
693	# if PGM_GST_TYPE == PGM_TYPE_PROT /* (AMD-V nested paging) */
694	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A,
695	X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A, &pPDDst);
696	# else
697	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, GstWalk.Pml4e.u, GstWalk.Pdpe.u, &pPDDst);
698	# endif
699	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
700
701	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
702	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
703	PEPTPD pPDDst;
704	rc = pgmShwGetEPTPDPtr(pVCpu, pvFault, NULL, &pPDDst);
705	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
706	# endif
707	Assert(pPDDst);
708
709	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
710	/*
711	* Dirty page handling.
712	*
713	* If we successfully correct the write protection fault due to dirty bit
714	* tracking, then return immediately.
715	*/
716	if (uErr & X86_TRAP_PF_RW) /* write fault? */
717	{
718	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyBitTracking), a);
719	rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, &pPDDst->a[iPDDst], GstWalk.pPde, pvFault);
720	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyBitTracking), a);
721	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
722	{
723	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution)
724	= rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
725	? &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2DirtyAndAccessed
726	: &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
727	Log8(("Trap0eHandler: returns VINF_SUCCESS\n"));
728	return VINF_SUCCESS;
729	}
730	#ifdef DEBUG_bird
731	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u \|\| GstWalk.pPte->u == GstWalk.pPde->u \|\| pVM->cCpus > 1, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); // - triggers with smp w7 guests.
732	AssertMsg(GstWalk.Core.fBigPage \|\| GstWalk.Pte.u == GstWalk.pPte->u \|\| pVM->cCpus > 1, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); // - ditto.
733	#endif
734	}
735
736	# if 0 /* rarely useful; leave for debugging. */
737	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0ePD[iPDSrc]);
738	# endif
739	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
740
741	/*
742	* A common case is the not-present error caused by lazy page table syncing.
743	*
744	* It is IMPORTANT that we weed out any access to non-present shadow PDEs
745	* here so we can safely assume that the shadow PT is present when calling
746	* SyncPage later.
747	*
748	* On failure, we ASSUME that SyncPT is out of memory or detected some kind
749	* of mapping conflict and defer to SyncCR3 in R3.
750	* (Again, we do NOT support access handlers for non-present guest pages.)
751	*
752	*/
753	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
754	Assert(GstWalk.Pde.n.u1Present);
755	# endif
756	if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
757	&& !pPDDst->a[iPDDst].n.u1Present)
758	{
759	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2SyncPT; });
760	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
761	LogFlow(("=>SyncPT %04x = %08RX64\n", (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, (uint64_t)GstWalk.Pde.u));
762	rc = PGM_BTH_NAME(SyncPT)(pVCpu, (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, GstWalk.pPd, pvFault);
763	# else
764	LogFlow(("=>SyncPT pvFault=%RGv\n", pvFault));
765	rc = PGM_BTH_NAME(SyncPT)(pVCpu, 0, NULL, pvFault);
766	# endif
767	if (RT_SUCCESS(rc))
768	return rc;
769	Log(("SyncPT: %RGv failed!! rc=%Rrc\n", pvFault, rc));
770	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
771	return VINF_PGM_SYNC_CR3;
772	}
773
774	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(PGM_WITHOUT_MAPPINGS)
775	/*
776	* Check if this address is within any of our mappings.
777	*
778	* This is very fast and it's gonna save us a bit of effort below and prevent
779	* us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
780	* (BTW, it's impossible to have physical access handlers in a mapping.)
781	*/
782	if (pgmMapAreMappingsEnabled(pVM))
783	{
784	PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
785	for ( ; pMapping; pMapping = pMapping->CTX_SUFF(pNext))
786	{
787	if (pvFault < pMapping->GCPtr)
788	break;
789	if (pvFault - pMapping->GCPtr < pMapping->cb)
790	{
791	/*
792	* The first thing we check is if we've got an undetected conflict.
793	*/
794	if (pgmMapAreMappingsFloating(pVM))
795	{
796	unsigned iPT = pMapping->cb >> GST_PD_SHIFT;
797	while (iPT-- > 0)
798	if (GstWalk.pPde[iPT].n.u1Present)
799	{
800	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eConflicts);
801	Log(("Trap0e: Detected Conflict %RGv-%RGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
802	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
803	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
804	return VINF_PGM_SYNC_CR3;
805	}
806	}
807
808	/*
809	* Check if the fault address is in a virtual page access handler range.
810	*/
811	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->HyperVirtHandlers,
812	pvFault);
813	if ( pCur
814	&& pvFault - pCur->Core.Key < pCur->cb
815	&& uErr & X86_TRAP_PF_RW)
816	{
817	VBOXSTRICTRC rcStrict;
818	# ifdef IN_RC
819	STAM_PROFILE_START(&pCur->Stat, h);
820	PPGMVIRTHANDLERTYPEINT pCurType = PGMVIRTANDLER_GET_TYPE(pVM, pCur);
821	void *pvUser = pCur->CTX_SUFF(pvUser);
822	pgmUnlock(pVM);
823	rcStrict = pCurType->CTX_SUFF(pfnPfHandler)(pVM, pVCpu, uErr, pRegFrame, pvFault, pCur->Core.Key,
824	pvFault - pCur->Core.Key, pvUser);
825	pgmLock(pVM);
826	STAM_PROFILE_STOP(&pCur->Stat, h);
827	# else
828	AssertFailed();
829	rcStrict = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
830	# endif
831	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersMapping);
832	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
833	return VBOXSTRICTRC_TODO(rcStrict);
834	}
835
836	/*
837	* Pretend we're not here and let the guest handle the trap.
838	*/
839	TRPMSetErrorCode(pVCpu, uErr & ~X86_TRAP_PF_P);
840	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eGuestPFMapping);
841	LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
842	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
843	return VINF_EM_RAW_GUEST_TRAP;
844	}
845	}
846	} /* pgmAreMappingsEnabled(&pVM->pgm.s) */
847	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
848
849	/*
850	* Check if this fault address is flagged for special treatment,
851	* which means we'll have to figure out the physical address and
852	* check flags associated with it.
853	*
854	* ASSUME that we can limit any special access handling to pages
855	* in page tables which the guest believes to be present.
856	*/
857	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
858	RTGCPHYS GCPhys = GstWalk.Core.GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
859	# else
860	RTGCPHYS GCPhys = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault & ~(RTGCPHYS)PAGE_OFFSET_MASK);
861	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
862	PPGMPAGE pPage;
863	rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
864	if (RT_FAILURE(rc))
865	{
866	/*
867	* When the guest accesses invalid physical memory (e.g. probing
868	* of RAM or accessing a remapped MMIO range), then we'll fall
869	* back to the recompiler to emulate the instruction.
870	*/
871	LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhys, rc));
872	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersInvalid);
873	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2InvalidPhys; });
874	return VINF_EM_RAW_EMULATE_INSTR;
875	}
876
877	/*
878	* Any handlers for this page?
879	*/
880	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
881	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
882	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, pfLockTaken,
883	&GstWalk));
884	# else
885	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, pfLockTaken));
886	# endif
887
888	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
889	if (uErr & X86_TRAP_PF_P)
890	{
891	/*
892	* The page isn't marked, but it might still be monitored by a virtual page access handler.
893	* (ASSUMES no temporary disabling of virtual handlers.)
894	*/
895	/** @todo r=bird: Since the purpose is to catch out of sync pages with virtual handler(s) here,
896	* we should correct both the shadow page table and physical memory flags, and not only check for
897	* accesses within the handler region but for access to pages with virtual handlers. */
898	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
899	if (pCur)
900	{
901	PPGMVIRTHANDLERTYPEINT pCurType = PGMVIRTANDLER_GET_TYPE(pVM, pCur);
902	AssertMsg( !(pvFault - pCur->Core.Key < pCur->cb)
903	\|\| ( pCurType->enmKind != PGMVIRTHANDLERKIND_WRITE
904	\|\| !(uErr & X86_TRAP_PF_P)
905	\|\| (pCurType->enmKind == PGMVIRTHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW))),
906	("Unexpected trap for virtual handler: %08X (phys=%08x) %R[pgmpage] uErr=%X, enumKind=%d\n",
907	pvFault, GCPhys, pPage, uErr, pCurType->enmKind));
908
909	if ( pvFault - pCur->Core.Key < pCur->cb
910	&& ( uErr & X86_TRAP_PF_RW
911	\|\| pCurType->enmKind != PGMVIRTHANDLERKIND_WRITE ) )
912	{
913	VBOXSTRICTRC rcStrict;
914	# ifdef IN_RC
915	STAM_PROFILE_START(&pCur->Stat, h);
916	void *pvUser = pCur->CTX_SUFF(pvUser);
917	pgmUnlock(pVM);
918	rcStrict = pCurType->CTX_SUFF(pfnPfHandler)(pVM, pVCpu, uErr, pRegFrame, pvFault, pCur->Core.Key,
919	pvFault - pCur->Core.Key, pvUser);
920	pgmLock(pVM);
921	STAM_PROFILE_STOP(&pCur->Stat, h);
922	# else
923	rcStrict = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
924	# endif
925	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
926	return VBOXSTRICTRC_TODO(rcStrict);
927	}
928	}
929	}
930	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
931
932	/*
933	* We are here only if page is present in Guest page tables and
934	* trap is not handled by our handlers.
935	*
936	* Check it for page out-of-sync situation.
937	*/
938	if (!(uErr & X86_TRAP_PF_P))
939	{
940	/*
941	* Page is not present in our page tables. Try to sync it!
942	*/
943	if (uErr & X86_TRAP_PF_US)
944	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUser));
945	else /* supervisor */
946	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
947
948	if (PGM_PAGE_IS_BALLOONED(pPage))
949	{
950	/* Emulate reads from ballooned pages as they are not present in
951	our shadow page tables. (Required for e.g. Solaris guests; soft
952	ecc, random nr generator.) */
953	rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
954	LogFlow(("PGM: PGMInterpretInstruction balloon -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
955	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncBallloon));
956	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Ballooned; });
957	return rc;
958	}
959
960	# if defined(LOG_ENABLED) && !defined(IN_RING0)
961	RTGCPHYS GCPhys2;
962	uint64_t fPageGst2;
963	PGMGstGetPage(pVCpu, pvFault, &fPageGst2, &GCPhys2);
964	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
965	Log(("Page out of sync: %RGv eip=%08x PdeSrc.US=%d fPageGst2=%08llx GCPhys2=%RGp scan=%d\n",
966	pvFault, pRegFrame->eip, GstWalk.Pde.n.u1User, fPageGst2, GCPhys2, CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)));
967	# else
968	Log(("Page out of sync: %RGv eip=%08x fPageGst2=%08llx GCPhys2=%RGp scan=%d\n",
969	pvFault, pRegFrame->eip, fPageGst2, GCPhys2, CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)));
970	# endif
971	# endif /* LOG_ENABLED */
972
973	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
974	if ( !GstWalk.Core.fEffectiveUS
975	&& CSAMIsEnabled(pVM)
976	&& CPUMGetGuestCPL(pVCpu) == 0)
977	{
978	/* Note: Can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU. */
979	if ( pvFault == (RTGCPTR)pRegFrame->eip
980	\|\| pvFault - pRegFrame->eip < 8 /* instruction crossing a page boundary */
981	# ifdef CSAM_DETECT_NEW_CODE_PAGES
982	\|\| ( !PATMIsPatchGCAddr(pVM, pRegFrame->eip)
983	&& CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)) /* any new code we encounter here */
984	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
985	)
986	{
987	LogFlow(("CSAMExecFault %RX32\n", pRegFrame->eip));
988	rc = CSAMExecFault(pVM, (RTRCPTR)pRegFrame->eip);
989	if (rc != VINF_SUCCESS)
990	{
991	/*
992	* CSAM needs to perform a job in ring 3.
993	*
994	* Sync the page before going to the host context; otherwise we'll end up in a loop if
995	* CSAM fails (e.g. instruction crosses a page boundary and the next page is not present)
996	*/
997	LogFlow(("CSAM ring 3 job\n"));
998	int rc2 = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
999	AssertRC(rc2);
1000
1001	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2CSAM; });
1002	return rc;
1003	}
1004	}
1005	# ifdef CSAM_DETECT_NEW_CODE_PAGES
1006	else if ( uErr == X86_TRAP_PF_RW
1007	&& pRegFrame->ecx >= 0x100 /* early check for movswd count */
1008	&& pRegFrame->ecx < 0x10000)
1009	{
1010	/* In case of a write to a non-present supervisor shadow page, we'll take special precautions
1011	* to detect loading of new code pages.
1012	*/
1013
1014	/*
1015	* Decode the instruction.
1016	*/
1017	PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
1018	uint32_t cbOp;
1019	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
1020
1021	/* For now we'll restrict this to rep movsw/d instructions */
1022	if ( rc == VINF_SUCCESS
1023	&& pDis->pCurInstr->opcode == OP_MOVSWD
1024	&& (pDis->prefix & DISPREFIX_REP))
1025	{
1026	CSAMMarkPossibleCodePage(pVM, pvFault);
1027	}
1028	}
1029	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
1030
1031	/*
1032	* Mark this page as safe.
1033	*/
1034	/** @todo not correct for pages that contain both code and data!! */
1035	Log2(("CSAMMarkPage %RGv; scanned=%d\n", pvFault, true));
1036	CSAMMarkPage(pVM, pvFault, true);
1037	}
1038	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0) */
1039	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1040	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
1041	# else
1042	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
1043	# endif
1044	if (RT_SUCCESS(rc))
1045	{
1046	/* The page was successfully synced, return to the guest. */
1047	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSync; });
1048	return VINF_SUCCESS;
1049	}
1050	}
1051	else /* uErr & X86_TRAP_PF_P: */
1052	{
1053	/*
1054	* Write protected pages are made writable when the guest makes the
1055	* first write to it. This happens for pages that are shared, write
1056	* monitored or not yet allocated.
1057	*
1058	* We may also end up here when CR0.WP=0 in the guest.
1059	*
1060	* Also, a side effect of not flushing global PDEs are out of sync
1061	* pages due to physical monitored regions, that are no longer valid.
1062	* Assume for now it only applies to the read/write flag.
1063	*/
1064	if (uErr & X86_TRAP_PF_RW)
1065	{
1066	/*
1067	* Check if it is a read-only page.
1068	*/
1069	if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1070	{
1071	Log(("PGM #PF: Make writable: %RGp %R[pgmpage] pvFault=%RGp uErr=%#x\n", GCPhys, pPage, pvFault, uErr));
1072	Assert(!PGM_PAGE_IS_ZERO(pPage));
1073	AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
1074	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2MakeWritable; });
1075
1076	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1077	if (rc != VINF_SUCCESS)
1078	{
1079	AssertMsg(rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("%Rrc\n", rc));
1080	return rc;
1081	}
1082	if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
1083	return VINF_EM_NO_MEMORY;
1084	}
1085
1086	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1087	/*
1088	* Check to see if we need to emulate the instruction if CR0.WP=0.
1089	*/
1090	if ( !GstWalk.Core.fEffectiveRW
1091	&& (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG
1092	&& CPUMGetGuestCPL(pVCpu) < 3)
1093	{
1094	Assert((uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RW \| X86_TRAP_PF_P));
1095
1096	/*
1097	* The Netware WP0+RO+US hack.
1098	*
1099	* Netware sometimes(/always?) runs with WP0. It has been observed doing
1100	* excessive write accesses to pages which are mapped with US=1 and RW=0
1101	* while WP=0. This causes a lot of exits and extremely slow execution.
1102	* To avoid trapping and emulating every write here, we change the shadow
1103	* page table entry to map it as US=0 and RW=1 until user mode tries to
1104	* access it again (see further below). We count these shadow page table
1105	* changes so we can avoid having to clear the page pool every time the WP
1106	* bit changes to 1 (see PGMCr0WpEnabled()).
1107	*/
1108	# if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) && 1
1109	if ( GstWalk.Core.fEffectiveUS
1110	&& !GstWalk.Core.fEffectiveRW
1111	&& (GstWalk.Core.fBigPage \|\| GstWalk.Pde.n.u1Write)
1112	&& pVM->cCpus == 1 /* Sorry, no go on SMP. Add CFGM option? */)
1113	{
1114	Log(("PGM #PF: Netware WP0+RO+US hack: pvFault=%RGp uErr=%#x (big=%d)\n", pvFault, uErr, GstWalk.Core.fBigPage));
1115	rc = pgmShwMakePageSupervisorAndWritable(pVCpu, pvFault, GstWalk.Core.fBigPage, PGM_MK_PG_IS_WRITE_FAULT);
1116	if (rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3)
1117	{
1118	PGM_INVL_PG(pVCpu, pvFault);
1119	pVCpu->pgm.s.cNetwareWp0Hacks++;
1120	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Wp0RoUsHack; });
1121	return rc;
1122	}
1123	AssertMsg(RT_FAILURE_NP(rc), ("%Rrc\n", rc));
1124	Log(("pgmShwMakePageSupervisorAndWritable(%RGv) failed with rc=%Rrc - ignored\n", pvFault, rc));
1125	}
1126	# endif
1127
1128	/* Interpret the access. */
1129	rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
1130	Log(("PGM #PF: WP0 emulation (pvFault=%RGp uErr=%#x cpl=%d fBig=%d fEffUs=%d)\n", pvFault, uErr, CPUMGetGuestCPL(pVCpu), GstWalk.Core.fBigPage, GstWalk.Core.fEffectiveUS));
1131	if (RT_SUCCESS(rc))
1132	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eWPEmulInRZ);
1133	else
1134	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eWPEmulToR3);
1135	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2WPEmulation; });
1136	return rc;
1137	}
1138	# endif
1139	/// @todo count the above case; else
1140	if (uErr & X86_TRAP_PF_US)
1141	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUserWrite));
1142	else /* supervisor */
1143	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
1144
1145	/*
1146	* Sync the page.
1147	*
1148	* Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
1149	* page is not present, which is not true in this case.
1150	*/
1151	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1152	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
1153	# else
1154	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
1155	# endif
1156	if (RT_SUCCESS(rc))
1157	{
1158	/*
1159	* Page was successfully synced, return to guest but invalidate
1160	* the TLB first as the page is very likely to be in it.
1161	*/
1162	# if PGM_SHW_TYPE == PGM_TYPE_EPT
1163	HMInvalidatePhysPage(pVM, (RTGCPHYS)pvFault);
1164	# else
1165	PGM_INVL_PG(pVCpu, pvFault);
1166	# endif
1167	# ifdef VBOX_STRICT
1168	RTGCPHYS GCPhys2 = RTGCPHYS_MAX;
1169	uint64_t fPageGst = UINT64_MAX;
1170	if (!pVM->pgm.s.fNestedPaging)
1171	{
1172	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, &GCPhys2);
1173	AssertMsg(RT_SUCCESS(rc) && ((fPageGst & X86_PTE_RW) \|\| ((CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG && CPUMGetGuestCPL(pVCpu) < 3)), ("rc=%Rrc fPageGst=%RX64\n", rc, fPageGst));
1174	LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GCPhys2, (uint64_t)fPageGst));
1175	}
1176	# if 0 /* Bogus! Triggers incorrectly with w7-64 and later for the SyncPage case: "Pde at %RGv changed behind our back?" */
1177	uint64_t fPageShw = 0;
1178	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
1179	AssertMsg((RT_SUCCESS(rc) && (fPageShw & X86_PTE_RW)) \|\| pVM->cCpus > 1 /* new monitor can be installed/page table flushed between the trap exit and PGMTrap0eHandler */,
1180	("rc=%Rrc fPageShw=%RX64 GCPhys2=%RGp fPageGst=%RX64 pvFault=%RGv\n", rc, fPageShw, GCPhys2, fPageGst, pvFault));
1181	# endif
1182	# endif /* VBOX_STRICT */
1183	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndObs; });
1184	return VINF_SUCCESS;
1185	}
1186	}
1187	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1188	/*
1189	* Check for Netware WP0+RO+US hack from above and undo it when user
1190	* mode accesses the page again.
1191	*/
1192	else if ( GstWalk.Core.fEffectiveUS
1193	&& !GstWalk.Core.fEffectiveRW
1194	&& (GstWalk.Core.fBigPage \|\| GstWalk.Pde.n.u1Write)
1195	&& pVCpu->pgm.s.cNetwareWp0Hacks > 0
1196	&& (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG
1197	&& CPUMGetGuestCPL(pVCpu) == 3
1198	&& pVM->cCpus == 1
1199	)
1200	{
1201	Log(("PGM #PF: Undo netware WP0+RO+US hack: pvFault=%RGp uErr=%#x\n", pvFault, uErr));
1202	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
1203	if (RT_SUCCESS(rc))
1204	{
1205	PGM_INVL_PG(pVCpu, pvFault);
1206	pVCpu->pgm.s.cNetwareWp0Hacks--;
1207	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Wp0RoUsUnhack; });
1208	return VINF_SUCCESS;
1209	}
1210	}
1211	# endif /* PGM_WITH_PAGING */
1212
1213	/** @todo else: why are we here? */
1214
1215	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && defined(VBOX_STRICT)
1216	/*
1217	* Check for VMM page flags vs. Guest page flags consistency.
1218	* Currently only for debug purposes.
1219	*/
1220	if (RT_SUCCESS(rc))
1221	{
1222	/* Get guest page flags. */
1223	uint64_t fPageGst;
1224	int rc2 = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
1225	if (RT_SUCCESS(rc2))
1226	{
1227	uint64_t fPageShw = 0;
1228	rc2 = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
1229
1230	#if 0
1231	/*
1232	* Compare page flags.
1233	* Note: we have AVL, A, D bits desynced.
1234	*/
1235	AssertMsg( (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK))
1236	== (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK))
1237	\|\| ( pVCpu->pgm.s.cNetwareWp0Hacks > 0
1238	&& (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US))
1239	== (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US))
1240	&& (fPageShw & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW
1241	&& (fPageGst & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_US),
1242	("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64 rc=%d\n",
1243	pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst, rc));
1244	01:01:15.623511 00:08:43.266063 Expression: (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) \|\| ( pVCpu->pgm.s.cNetwareWp0Hacks > 0 && (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US)) && (fPageShw & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW && (fPageGst & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_US)
1245	01:01:15.623511 00:08:43.266064 Location : e:\vbox\svn\trunk\srcPage flags mismatch! pvFault=fffff801b0d7b000 uErr=11 GCPhys=0000000019b52000 fPageShw=0 fPageGst=77b0000000000121 rc=0
1246
1247	01:01:15.625516 00:08:43.268051 Expression: (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) \|\| ( pVCpu->pgm.s.cNetwareWp0Hacks > 0 && (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US)) && (fPageShw & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW && (fPageGst & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_US)
1248	01:01:15.625516 00:08:43.268051 Location :
1249	e:\vbox\svn\trunk\srcPage flags mismatch!
1250	pvFault=fffff801b0d7b000
1251	uErr=11 X86_TRAP_PF_ID \| X86_TRAP_PF_P
1252	GCPhys=0000000019b52000
1253	fPageShw=0
1254	fPageGst=77b0000000000121
1255	rc=0
1256	#endif
1257
1258	}
1259	else
1260	AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc));
1261	}
1262	else
1263	AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc));
1264	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && VBOX_STRICT */
1265	}
1266
1267
1268	/*
1269	* If we get here it is because something failed above, i.e. most like guru
1270	* meditiation time.
1271	*/
1272	LogRel(("%s: returns rc=%Rrc pvFault=%RGv uErr=%RX64 cs:rip=%04x:%08RX64\n",
1273	__PRETTY_FUNCTION__, rc, pvFault, (uint64_t)uErr, pRegFrame->cs.Sel, pRegFrame->rip));
1274	return rc;
1275
1276	# else /* Nested paging, EPT except PGM_GST_TYPE = PROT, NONE. */
1277	NOREF(uErr); NOREF(pRegFrame); NOREF(pvFault);
1278	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
1279	return VERR_PGM_NOT_USED_IN_MODE;
1280	# endif
1281	}
1282	#endif /* !IN_RING3 */
1283
1284
1285	/**
1286	* Emulation of the invlpg instruction.
1287	*
1288	*
1289	* @returns VBox status code.
1290	*
1291	* @param pVCpu The cross context virtual CPU structure.
1292	* @param GCPtrPage Page to invalidate.
1293	*
1294	* @remark ASSUMES that the guest is updating before invalidating. This order
1295	* isn't required by the CPU, so this is speculative and could cause
1296	* trouble.
1297	* @remark No TLB shootdown is done on any other VCPU as we assume that
1298	* invlpg emulation is the only reason for calling this function.
1299	* (The guest has to shoot down TLB entries on other CPUs itself)
1300	* Currently true, but keep in mind!
1301	*
1302	* @todo Clean this up! Most of it is (or should be) no longer necessary as we catch all page table accesses.
1303	* Should only be required when PGMPOOL_WITH_OPTIMIZED_DIRTY_PT is active (PAE or AMD64 (for now))
1304	*/
1305	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
1306	{
1307	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1308	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
1309	&& PGM_SHW_TYPE != PGM_TYPE_NONE
1310	int rc;
1311	PVM pVM = pVCpu->CTX_SUFF(pVM);
1312	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1313
1314	PGM_LOCK_ASSERT_OWNER(pVM);
1315
1316	LogFlow(("InvalidatePage %RGv\n", GCPtrPage));
1317
1318	/*
1319	* Get the shadow PD entry and skip out if this PD isn't present.
1320	* (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
1321	*/
1322	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1323	const unsigned iPDDst = (uint32_t)GCPtrPage >> SHW_PD_SHIFT;
1324	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1325
1326	/* Fetch the pgm pool shadow descriptor. */
1327	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1328	# ifdef IN_RING3 /* Possible we didn't resync yet when called from REM. */
1329	if (!pShwPde)
1330	{
1331	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1332	return VINF_SUCCESS;
1333	}
1334	# else
1335	Assert(pShwPde);
1336	# endif
1337
1338	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1339	const unsigned iPdpt = (uint32_t)GCPtrPage >> X86_PDPT_SHIFT;
1340	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
1341
1342	/* If the shadow PDPE isn't present, then skip the invalidate. */
1343	# ifdef IN_RING3 /* Possible we didn't resync yet when called from REM. */
1344	if (!pPdptDst \|\| !pPdptDst->a[iPdpt].n.u1Present)
1345	# else
1346	if (!pPdptDst->a[iPdpt].n.u1Present)
1347	# endif
1348	{
1349	Assert(!pPdptDst \|\| !(pPdptDst->a[iPdpt].u & PGM_PLXFLAGS_MAPPING));
1350	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1351	PGM_INVL_PG(pVCpu, GCPtrPage);
1352	return VINF_SUCCESS;
1353	}
1354
1355	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1356	PPGMPOOLPAGE pShwPde = NULL;
1357	PX86PDPAE pPDDst;
1358
1359	/* Fetch the pgm pool shadow descriptor. */
1360	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
1361	AssertRCSuccessReturn(rc, rc);
1362	Assert(pShwPde);
1363
1364	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1365	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1366
1367	# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1368	/* PML4 */
1369	/const unsigned iPml4 = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;/
1370	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1371	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1372	PX86PDPAE pPDDst;
1373	PX86PDPT pPdptDst;
1374	PX86PML4E pPml4eDst;
1375	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eDst, &pPdptDst, &pPDDst);
1376	if (rc != VINF_SUCCESS)
1377	{
1378	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT \|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
1379	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1380	PGM_INVL_PG(pVCpu, GCPtrPage);
1381	return VINF_SUCCESS;
1382	}
1383	Assert(pPDDst);
1384
1385	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1386	PX86PDPE pPdpeDst = &pPdptDst->a[iPdpt];
1387
1388	if (!pPdpeDst->n.u1Present)
1389	{
1390	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1391	PGM_INVL_PG(pVCpu, GCPtrPage);
1392	return VINF_SUCCESS;
1393	}
1394
1395	/* Fetch the pgm pool shadow descriptor. */
1396	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & SHW_PDPE_PG_MASK);
1397	Assert(pShwPde);
1398
1399	# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1400
1401	const SHWPDE PdeDst = *pPdeDst;
1402	if (!PdeDst.n.u1Present)
1403	{
1404	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1405	PGM_INVL_PG(pVCpu, GCPtrPage);
1406	return VINF_SUCCESS;
1407	}
1408
1409	/*
1410	* Get the guest PD entry and calc big page.
1411	*/
1412	# if PGM_GST_TYPE == PGM_TYPE_32BIT
1413	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
1414	const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
1415	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1416	# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1417	unsigned iPDSrc = 0;
1418	# if PGM_GST_TYPE == PGM_TYPE_PAE
1419	X86PDPE PdpeSrcIgn;
1420	PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrcIgn);
1421	# else /* AMD64 */
1422	PX86PML4E pPml4eSrcIgn;
1423	X86PDPE PdpeSrcIgn;
1424	PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrcIgn, &PdpeSrcIgn, &iPDSrc);
1425	# endif
1426	GSTPDE PdeSrc;
1427
1428	if (pPDSrc)
1429	PdeSrc = pPDSrc->a[iPDSrc];
1430	else
1431	PdeSrc.u = 0;
1432	# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1433	const bool fWasBigPage = RT_BOOL(PdeDst.u & PGM_PDFLAGS_BIG_PAGE);
1434	const bool fIsBigPage = PdeSrc.b.u1Size && GST_IS_PSE_ACTIVE(pVCpu);
1435	if (fWasBigPage != fIsBigPage)
1436	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1437
1438	# ifdef IN_RING3
1439	/*
1440	* If a CR3 Sync is pending we may ignore the invalidate page operation
1441	* depending on the kind of sync and if it's a global page or not.
1442	* This doesn't make sense in GC/R0 so we'll skip it entirely there.
1443	*/
1444	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
1445	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
1446	\|\| ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
1447	&& fIsBigPage
1448	&& PdeSrc.b.u1Global
1449	)
1450	)
1451	# else
1452	if (VM_FF_IS_ANY_SET(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
1453	# endif
1454	{
1455	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1456	return VINF_SUCCESS;
1457	}
1458	# endif /* IN_RING3 */
1459
1460	/*
1461	* Deal with the Guest PDE.
1462	*/
1463	rc = VINF_SUCCESS;
1464	if (PdeSrc.n.u1Present)
1465	{
1466	Assert( PdeSrc.n.u1User == PdeDst.n.u1User
1467	&& (PdeSrc.n.u1Write \|\| !PdeDst.n.u1Write \|\| pVCpu->pgm.s.cNetwareWp0Hacks > 0));
1468	# ifndef PGM_WITHOUT_MAPPING
1469	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
1470	{
1471	/*
1472	* Conflict - Let SyncPT deal with it to avoid duplicate code.
1473	*/
1474	Assert(pgmMapAreMappingsEnabled(pVM));
1475	Assert(PGMGetGuestMode(pVCpu) <= PGMMODE_PAE);
1476	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
1477	}
1478	else
1479	# endif /* !PGM_WITHOUT_MAPPING */
1480	if (!fIsBigPage)
1481	{
1482	/*
1483	* 4KB - page.
1484	*/
1485	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1486	RTGCPHYS GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
1487
1488	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1489	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1490	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| ((iPDDst & 1) * (PAGE_SIZE / 2)));
1491	# endif
1492	if (pShwPage->GCPhys == GCPhys)
1493	{
1494	/* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1495	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1496
1497	PGSTPT pPTSrc;
1498	rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
1499	if (RT_SUCCESS(rc))
1500	{
1501	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1502	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1503	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1504	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1505	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
1506	GCPtrPage, PteSrc.n.u1Present,
1507	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1508	PteSrc.n.u1User & PdeSrc.n.u1User,
1509	(uint64_t)PteSrc.u,
1510	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
1511	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
1512	}
1513	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4KBPages));
1514	PGM_INVL_PG(pVCpu, GCPtrPage);
1515	}
1516	else
1517	{
1518	/*
1519	* The page table address changed.
1520	*/
1521	LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
1522	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1523	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1524	ASMAtomicWriteSize(pPdeDst, 0);
1525	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1526	PGM_INVL_VCPU_TLBS(pVCpu);
1527	}
1528	}
1529	else
1530	{
1531	/*
1532	* 2/4MB - page.
1533	*/
1534	/* Before freeing the page, check if anything really changed. */
1535	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1536	RTGCPHYS GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
1537	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1538	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1539	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
1540	# endif
1541	if ( pShwPage->GCPhys == GCPhys
1542	&& pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1543	{
1544	/* ASSUMES a the given bits are identical for 4M and normal PDEs */
1545	/** @todo This test is wrong as it cannot check the G bit!
1546	* FIXME */
1547	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US))
1548	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US))
1549	&& ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1550	\|\| (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1551	{
1552	LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1553	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip));
1554	return VINF_SUCCESS;
1555	}
1556	}
1557
1558	/*
1559	* Ok, the page table is present and it's been changed in the guest.
1560	* If we're in host context, we'll just mark it as not present taking the lazy approach.
1561	* We could do this for some flushes in GC too, but we need an algorithm for
1562	* deciding which 4MB pages containing code likely to be executed very soon.
1563	*/
1564	LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1565	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1566	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1567	ASMAtomicWriteSize(pPdeDst, 0);
1568	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4MBPages));
1569	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
1570	}
1571	}
1572	else
1573	{
1574	/*
1575	* Page directory is not present, mark shadow PDE not present.
1576	*/
1577	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
1578	{
1579	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1580	ASMAtomicWriteSize(pPdeDst, 0);
1581	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDNPs));
1582	PGM_INVL_PG(pVCpu, GCPtrPage);
1583	}
1584	else
1585	{
1586	Assert(pgmMapAreMappingsEnabled(pVM));
1587	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDMappings));
1588	}
1589	}
1590	return rc;
1591
1592	#else /* guest real and protected mode, nested + ept, none. */
1593	/* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1594	NOREF(pVCpu); NOREF(GCPtrPage);
1595	return VINF_SUCCESS;
1596	#endif
1597	}
1598
1599	#if PGM_SHW_TYPE != PGM_TYPE_NONE
1600
1601	/**
1602	* Update the tracking of shadowed pages.
1603	*
1604	* @param pVCpu The cross context virtual CPU structure.
1605	* @param pShwPage The shadow page.
1606	* @param HCPhys The physical page we is being dereferenced.
1607	* @param iPte Shadow PTE index
1608	* @param GCPhysPage Guest physical address (only valid if pShwPage->fDirty is set)
1609	*/
1610	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys, uint16_t iPte,
1611	RTGCPHYS GCPhysPage)
1612	{
1613	PVM pVM = pVCpu->CTX_SUFF(pVM);
1614
1615	# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1616	&& PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1617	&& (PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
1618
1619	/* Use the hint we retrieved from the cached guest PT. */
1620	if (pShwPage->fDirty)
1621	{
1622	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1623
1624	Assert(pShwPage->cPresent);
1625	Assert(pPool->cPresent);
1626	pShwPage->cPresent--;
1627	pPool->cPresent--;
1628
1629	PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysPage);
1630	AssertRelease(pPhysPage);
1631	pgmTrackDerefGCPhys(pPool, pShwPage, pPhysPage, iPte);
1632	return;
1633	}
1634	# else
1635	NOREF(GCPhysPage);
1636	# endif
1637
1638	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackDeref, a);
1639	LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1640
1641	/** @todo If this turns out to be a bottle neck (very likely) two things can be done:
1642	* 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1643	* 2. write protect all shadowed pages. I.e. implement caching.
1644	*/
1645	/** @todo duplicated in the 2nd half of pgmPoolTracDerefGCPhysHint */
1646
1647	/*
1648	* Find the guest address.
1649	*/
1650	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1651	pRam;
1652	pRam = pRam->CTX_SUFF(pNext))
1653	{
1654	unsigned iPage = pRam->cb >> PAGE_SHIFT;
1655	while (iPage-- > 0)
1656	{
1657	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1658	{
1659	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1660
1661	Assert(pShwPage->cPresent);
1662	Assert(pPool->cPresent);
1663	pShwPage->cPresent--;
1664	pPool->cPresent--;
1665
1666	pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage], iPte);
1667	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackDeref, a);
1668	return;
1669	}
1670	}
1671	}
1672
1673	for (;;)
1674	AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys));
1675	}
1676
1677
1678	/**
1679	* Update the tracking of shadowed pages.
1680	*
1681	* @param pVCpu The cross context virtual CPU structure.
1682	* @param pShwPage The shadow page.
1683	* @param u16 The top 16-bit of the pPage->HCPhys.
1684	* @param pPage Pointer to the guest page. this will be modified.
1685	* @param iPTDst The index into the shadow table.
1686	*/
1687	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
1688	{
1689	PVM pVM = pVCpu->CTX_SUFF(pVM);
1690
1691	/*
1692	* Just deal with the simple first time here.
1693	*/
1694	if (!u16)
1695	{
1696	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackVirgin);
1697	u16 = PGMPOOL_TD_MAKE(1, pShwPage->idx);
1698	/* Save the page table index. */
1699	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, iPTDst);
1700	}
1701	else
1702	u16 = pgmPoolTrackPhysExtAddref(pVM, pPage, u16, pShwPage->idx, iPTDst);
1703
1704	/* write back */
1705	Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x iPTDst=%#x\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst));
1706	PGM_PAGE_SET_TRACKING(pVM, pPage, u16);
1707
1708	/* update statistics. */
1709	pVM->pgm.s.CTX_SUFF(pPool)->cPresent++;
1710	pShwPage->cPresent++;
1711	if (pShwPage->iFirstPresent > iPTDst)
1712	pShwPage->iFirstPresent = iPTDst;
1713	}
1714
1715
1716	/**
1717	* Modifies a shadow PTE to account for access handlers.
1718	*
1719	* @param pVM The cross context VM structure.
1720	* @param pPage The page in question.
1721	* @param fPteSrc The shadowed flags of the source PTE. Must include the
1722	* A (accessed) bit so it can be emulated correctly.
1723	* @param pPteDst The shadow PTE (output). This is temporary storage and
1724	* does not need to be set atomically.
1725	*/
1726	DECLINLINE(void) PGM_BTH_NAME(SyncHandlerPte)(PVM pVM, PCPGMPAGE pPage, uint64_t fPteSrc, PSHWPTE pPteDst)
1727	{
1728	NOREF(pVM); RT_NOREF_PV(fPteSrc);
1729
1730	/** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No.
1731	* Update: \#PF should deal with this before or after calling the handlers. It has all the info to do the job efficiently. */
1732	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1733	{
1734	LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark read-only\n", pPage));
1735	# if PGM_SHW_TYPE == PGM_TYPE_EPT
1736	pPteDst->u = PGM_PAGE_GET_HCPHYS(pPage);
1737	pPteDst->n.u1Present = 1;
1738	pPteDst->n.u1Execute = 1;
1739	pPteDst->n.u1IgnorePAT = 1;
1740	pPteDst->n.u3EMT = VMX_EPT_MEMTYPE_WB;
1741	/* PteDst.n.u1Write = 0 && PteDst.n.u1Size = 0 */
1742	# else
1743	if (fPteSrc & X86_PTE_A)
1744	{
1745	SHW_PTE_SET(*pPteDst, fPteSrc \| PGM_PAGE_GET_HCPHYS(pPage));
1746	SHW_PTE_SET_RO(*pPteDst);
1747	}
1748	else
1749	SHW_PTE_SET(*pPteDst, 0);
1750	# endif
1751	}
1752	# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
1753	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
1754	else if ( PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
1755	&& ( BTH_IS_NP_ACTIVE(pVM)
1756	\|\| (fPteSrc & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW) /** @todo Remove X86_PTE_US here and pGstWalk->Core.fEffectiveUS before the sync page test. */
1757	# if PGM_SHW_TYPE == PGM_TYPE_AMD64
1758	&& pVM->pgm.s.fLessThan52PhysicalAddressBits
1759	# endif
1760	)
1761	{
1762	LogFlow(("SyncHandlerPte: MMIO page -> invalid \n"));
1763	# if PGM_SHW_TYPE == PGM_TYPE_EPT
1764	/* 25.2.3.1: Reserved physical address bit -> EPT Misconfiguration (exit 49) */
1765	pPteDst->u = pVM->pgm.s.HCPhysInvMmioPg;
1766	/* 25.2.3.1: bits 2:0 = 010b -> EPT Misconfiguration (exit 49) */
1767	pPteDst->n.u1Present = 0;
1768	pPteDst->n.u1Write = 1;
1769	pPteDst->n.u1Execute = 0;
1770	/* 25.2.3.1: leaf && 2:0 != 0 && u3Emt in {2, 3, 7} -> EPT Misconfiguration */
1771	pPteDst->n.u3EMT = 7;
1772	# else
1773	/* Set high page frame bits that MBZ (bankers on PAE, CPU dependent on AMD64). */
1774	SHW_PTE_SET(*pPteDst, pVM->pgm.s.HCPhysInvMmioPg \| X86_PTE_PAE_MBZ_MASK_NO_NX \| X86_PTE_P);
1775	# endif
1776	}
1777	# endif
1778	# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
1779	else
1780	{
1781	LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark not present\n", pPage));
1782	SHW_PTE_SET(*pPteDst, 0);
1783	}
1784	/** @todo count these kinds of entries. */
1785	}
1786
1787
1788	/**
1789	* Creates a 4K shadow page for a guest page.
1790	*
1791	* For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1792	* physical address. The PdeSrc argument only the flags are used. No page
1793	* structured will be mapped in this function.
1794	*
1795	* @param pVCpu The cross context virtual CPU structure.
1796	* @param pPteDst Destination page table entry.
1797	* @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1798	* Can safely assume that only the flags are being used.
1799	* @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1800	* @param pShwPage Pointer to the shadow page.
1801	* @param iPTDst The index into the shadow table.
1802	*
1803	* @remark Not used for 2/4MB pages!
1804	*/
1805	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \|\| defined(DOXYGEN_RUNNING)
1806	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc,
1807	PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1808	# else
1809	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage,
1810	PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1811	# endif
1812	{
1813	PVM pVM = pVCpu->CTX_SUFF(pVM);
1814	RTGCPHYS GCPhysOldPage = NIL_RTGCPHYS;
1815
1816	# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1817	&& PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1818	&& (PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
1819
1820	if (pShwPage->fDirty)
1821	{
1822	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1823	PGSTPT pGstPT;
1824
1825	/* Note that iPTDst can be used to index the guest PT even in the pae/32bit combo as we copy only half the table; see pgmPoolAddDirtyPage. */
1826	pGstPT = (PGSTPT)&pPool->aDirtyPages[pShwPage->idxDirtyEntry].aPage[0];
1827	GCPhysOldPage = GST_GET_PTE_GCPHYS(pGstPT->a[iPTDst]);
1828	pGstPT->a[iPTDst].u = PteSrc.u;
1829	}
1830	# else
1831	Assert(!pShwPage->fDirty);
1832	# endif
1833
1834	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1835	if ( PteSrc.n.u1Present
1836	&& GST_IS_PTE_VALID(pVCpu, PteSrc))
1837	# endif
1838	{
1839	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1840	RTGCPHYS GCPhysPage = GST_GET_PTE_GCPHYS(PteSrc);
1841	# endif
1842	PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
1843
1844	/*
1845	* Find the ram range.
1846	*/
1847	PPGMPAGE pPage;
1848	int rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
1849	if (RT_SUCCESS(rc))
1850	{
1851	/* Ignore ballooned pages.
1852	Don't return errors or use a fatal assert here as part of a
1853	shadow sync range might included ballooned pages. */
1854	if (PGM_PAGE_IS_BALLOONED(pPage))
1855	{
1856	Assert(!SHW_PTE_IS_P(pPteDst)); /* @todo user tracking needs updating if this triggers. */
1857	return;
1858	}
1859
1860	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1861	/* Make the page writable if necessary. */
1862	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1863	&& ( PGM_PAGE_IS_ZERO(pPage)
1864	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1865	\|\| ( PteSrc.n.u1Write
1866	# else
1867	\|\| ( 1
1868	# endif
1869	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1870	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1871	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1872	# endif
1873	# ifdef VBOX_WITH_PAGE_SHARING
1874	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
1875	# endif
1876	)
1877	)
1878	)
1879	{
1880	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhysPage);
1881	AssertRC(rc);
1882	}
1883	# endif
1884
1885	/*
1886	* Make page table entry.
1887	*/
1888	SHWPTE PteDst;
1889	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1890	uint64_t fGstShwPteFlags = GST_GET_PTE_SHW_FLAGS(pVCpu, PteSrc);
1891	# else
1892	uint64_t fGstShwPteFlags = X86_PTE_P \| X86_PTE_RW \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D;
1893	# endif
1894	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1895	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, fGstShwPteFlags, &PteDst);
1896	else
1897	{
1898	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1899	/*
1900	* If the page or page directory entry is not marked accessed,
1901	* we mark the page not present.
1902	*/
1903	if (!PteSrc.n.u1Accessed \|\| !PdeSrc.n.u1Accessed)
1904	{
1905	LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1906	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,AccessedPage));
1907	SHW_PTE_SET(PteDst, 0);
1908	}
1909	/*
1910	* If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1911	* when the page is modified.
1912	*/
1913	else if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1914	{
1915	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPage));
1916	SHW_PTE_SET(PteDst,
1917	fGstShwPteFlags
1918	\| PGM_PAGE_GET_HCPHYS(pPage)
1919	\| PGM_PTFLAGS_TRACK_DIRTY);
1920	SHW_PTE_SET_RO(PteDst);
1921	}
1922	else
1923	# endif
1924	{
1925	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageSkipped));
1926	# if PGM_SHW_TYPE == PGM_TYPE_EPT
1927	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage);
1928	PteDst.n.u1Present = 1;
1929	PteDst.n.u1Write = 1;
1930	PteDst.n.u1Execute = 1;
1931	PteDst.n.u1IgnorePAT = 1;
1932	PteDst.n.u3EMT = VMX_EPT_MEMTYPE_WB;
1933	/* PteDst.n.u1Size = 0 */
1934	# else
1935	SHW_PTE_SET(PteDst, fGstShwPteFlags \| PGM_PAGE_GET_HCPHYS(pPage));
1936	# endif
1937	}
1938
1939	/*
1940	* Make sure only allocated pages are mapped writable.
1941	*/
1942	if ( SHW_PTE_IS_P_RW(PteDst)
1943	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1944	{
1945	/* Still applies to shared pages. */
1946	Assert(!PGM_PAGE_IS_ZERO(pPage));
1947	SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet. Why, isn't it? */
1948	Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", GCPhysPage, pPage, iPTDst));
1949	}
1950	}
1951
1952	/*
1953	* Keep user track up to date.
1954	*/
1955	if (SHW_PTE_IS_P(PteDst))
1956	{
1957	if (!SHW_PTE_IS_P(*pPteDst))
1958	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1959	else if (SHW_PTE_GET_HCPHYS(*pPteDst) != SHW_PTE_GET_HCPHYS(PteDst))
1960	{
1961	Log2(("SyncPageWorker: deref! pPteDst=%RX64 PteDst=%RX64\n", SHW_PTE_LOG64(pPteDst), SHW_PTE_LOG64(PteDst)));
1962	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
1963	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1964	}
1965	}
1966	else if (SHW_PTE_IS_P(*pPteDst))
1967	{
1968	Log2(("SyncPageWorker: deref! pPteDst=%RX64\n", SHW_PTE_LOG64(pPteDst)));
1969	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
1970	}
1971
1972	/*
1973	* Update statistics and commit the entry.
1974	*/
1975	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1976	if (!PteSrc.n.u1Global)
1977	pShwPage->fSeenNonGlobal = true;
1978	# endif
1979	SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
1980	return;
1981	}
1982
1983	/** @todo count these three different kinds. */
1984	Log2(("SyncPageWorker: invalid address in Pte\n"));
1985	}
1986	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1987	else if (!PteSrc.n.u1Present)
1988	Log2(("SyncPageWorker: page not present in Pte\n"));
1989	else
1990	Log2(("SyncPageWorker: invalid Pte\n"));
1991	# endif
1992
1993	/*
1994	* The page is not present or the PTE is bad. Replace the shadow PTE by
1995	* an empty entry, making sure to keep the user tracking up to date.
1996	*/
1997	if (SHW_PTE_IS_P(*pPteDst))
1998	{
1999	Log2(("SyncPageWorker: deref! pPteDst=%RX64\n", SHW_PTE_LOG64(pPteDst)));
2000	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
2001	}
2002	SHW_PTE_ATOMIC_SET(*pPteDst, 0);
2003	}
2004
2005
2006	/**
2007	* Syncs a guest OS page.
2008	*
2009	* There are no conflicts at this point, neither is there any need for
2010	* page table allocations.
2011	*
2012	* When called in PAE or AMD64 guest mode, the guest PDPE shall be valid.
2013	* When called in AMD64 guest mode, the guest PML4E shall be valid.
2014	*
2015	* @returns VBox status code.
2016	* @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
2017	* @param pVCpu The cross context virtual CPU structure.
2018	* @param PdeSrc Page directory entry of the guest.
2019	* @param GCPtrPage Guest context page address.
2020	* @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
2021	* @param uErr Fault error (X86_TRAP_PF_*).
2022	*/
2023	static int PGM_BTH_NAME(SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr)
2024	{
2025	PVM pVM = pVCpu->CTX_SUFF(pVM);
2026	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
2027	LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr));
2028	RT_NOREF_PV(uErr); RT_NOREF_PV(cPages); RT_NOREF_PV(GCPtrPage);
2029
2030	PGM_LOCK_ASSERT_OWNER(pVM);
2031
2032	# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2033	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
2034	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
2035	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE)
2036
2037	/*
2038	* Assert preconditions.
2039	*/
2040	Assert(PdeSrc.n.u1Present);
2041	Assert(cPages);
2042	# if 0 /* rarely useful; leave for debugging. */
2043	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
2044	# endif
2045
2046	/*
2047	* Get the shadow PDE, find the shadow page table in the pool.
2048	*/
2049	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2050	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2051	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
2052
2053	/* Fetch the pgm pool shadow descriptor. */
2054	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2055	Assert(pShwPde);
2056
2057	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2058	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2059	PPGMPOOLPAGE pShwPde = NULL;
2060	PX86PDPAE pPDDst;
2061
2062	/* Fetch the pgm pool shadow descriptor. */
2063	int rc2 = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
2064	AssertRCSuccessReturn(rc2, rc2);
2065	Assert(pShwPde);
2066
2067	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
2068	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
2069
2070	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2071	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2072	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2073	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2074	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2075
2076	int rc2 = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2077	AssertRCSuccessReturn(rc2, rc2);
2078	Assert(pPDDst && pPdptDst);
2079	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
2080	# endif
2081	SHWPDE PdeDst = *pPdeDst;
2082
2083	/*
2084	* - In the guest SMP case we could have blocked while another VCPU reused
2085	* this page table.
2086	* - With W7-64 we may also take this path when the A bit is cleared on
2087	* higher level tables (PDPE/PML4E). The guest does not invalidate the
2088	* relevant TLB entries. If we're write monitoring any page mapped by
2089	* the modified entry, we may end up here with a "stale" TLB entry.
2090	*/
2091	if (!PdeDst.n.u1Present)
2092	{
2093	Log(("CPU%u: SyncPage: Pde at %RGv changed behind our back? (pPdeDst=%p/%RX64) uErr=%#x\n", pVCpu->idCpu, GCPtrPage, pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
2094	AssertMsg(pVM->cCpus > 1 \|\| (uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RW)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RW),
2095	("Unexpected missing PDE p=%p/%RX64 uErr=%#x\n", pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
2096	if (uErr & X86_TRAP_PF_P)
2097	PGM_INVL_PG(pVCpu, GCPtrPage);
2098	return VINF_SUCCESS; /* force the instruction to be executed again. */
2099	}
2100
2101	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2102	Assert(pShwPage);
2103
2104	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2105	/* Fetch the pgm pool shadow descriptor. */
2106	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2107	Assert(pShwPde);
2108	# endif
2109
2110	/*
2111	* Check that the page is present and that the shadow PDE isn't out of sync.
2112	*/
2113	const bool fBigPage = PdeSrc.b.u1Size && GST_IS_PSE_ACTIVE(pVCpu);
2114	const bool fPdeValid = !fBigPage ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc);
2115	RTGCPHYS GCPhys;
2116	if (!fBigPage)
2117	{
2118	GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
2119	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2120	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2121	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| ((iPDDst & 1) * (PAGE_SIZE / 2)));
2122	# endif
2123	}
2124	else
2125	{
2126	GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
2127	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2128	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2129	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
2130	# endif
2131	}
2132	/** @todo This doesn't check the G bit of 2/4MB pages. FIXME */
2133	if ( fPdeValid
2134	&& pShwPage->GCPhys == GCPhys
2135	&& PdeSrc.n.u1Present
2136	&& PdeSrc.n.u1User == PdeDst.n.u1User
2137	&& (PdeSrc.n.u1Write == PdeDst.n.u1Write \|\| !PdeDst.n.u1Write)
2138	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2139	&& (PdeSrc.n.u1NoExecute == PdeDst.n.u1NoExecute \|\| !GST_IS_NX_ACTIVE(pVCpu))
2140	# endif
2141	)
2142	{
2143	/*
2144	* Check that the PDE is marked accessed already.
2145	* Since we set the accessed bit before getting here on a #PF, this
2146	* check is only meant for dealing with non-#PF'ing paths.
2147	*/
2148	if (PdeSrc.n.u1Accessed)
2149	{
2150	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2151	if (!fBigPage)
2152	{
2153	/*
2154	* 4KB Page - Map the guest page table.
2155	*/
2156	PGSTPT pPTSrc;
2157	int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
2158	if (RT_SUCCESS(rc))
2159	{
2160	# ifdef PGM_SYNC_N_PAGES
2161	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
2162	if ( cPages > 1
2163	&& !(uErr & X86_TRAP_PF_P)
2164	&& !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
2165	{
2166	/*
2167	* This code path is currently only taken when the caller is PGMTrap0eHandler
2168	* for non-present pages!
2169	*
2170	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2171	* deal with locality.
2172	*/
2173	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2174	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2175	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2176	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2177	# else
2178	const unsigned offPTSrc = 0;
2179	# endif
2180	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2181	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2182	iPTDst = 0;
2183	else
2184	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2185
2186	for (; iPTDst < iPTDstEnd; iPTDst++)
2187	{
2188	const PGSTPTE pPteSrc = &pPTSrc->a[offPTSrc + iPTDst];
2189
2190	if ( pPteSrc->n.u1Present
2191	&& !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2192	{
2193	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
2194	NOREF(GCPtrCurPage);
2195	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
2196	/*
2197	* Assuming kernel code will be marked as supervisor - and not as user level
2198	* and executed using a conforming code selector - And marked as readonly.
2199	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
2200	*/
2201	PPGMPAGE pPage;
2202	if ( ((PdeSrc.u & pPteSrc->u) & (X86_PTE_RW \| X86_PTE_US))
2203	\|\| iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK) /* always sync GCPtrPage */
2204	\|\| !CSAMDoesPageNeedScanning(pVM, GCPtrCurPage)
2205	\|\| ( (pPage = pgmPhysGetPage(pVM, pPteSrc->u & GST_PTE_PG_MASK))
2206	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2207	)
2208	# endif /* else: CSAM not active */
2209	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, *pPteSrc, pShwPage, iPTDst);
2210	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
2211	GCPtrCurPage, pPteSrc->n.u1Present,
2212	pPteSrc->n.u1Write & PdeSrc.n.u1Write,
2213	pPteSrc->n.u1User & PdeSrc.n.u1User,
2214	(uint64_t)pPteSrc->u,
2215	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2216	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2217	}
2218	}
2219	}
2220	else
2221	# endif /* PGM_SYNC_N_PAGES */
2222	{
2223	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
2224	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2225	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2226	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2227	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
2228	GCPtrPage, PteSrc.n.u1Present,
2229	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2230	PteSrc.n.u1User & PdeSrc.n.u1User,
2231	(uint64_t)PteSrc.u,
2232	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2233	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2234	}
2235	}
2236	else /* MMIO or invalid page: emulated in #PF handler. */
2237	{
2238	LogFlow(("PGM_GCPHYS_2_PTR %RGp failed with %Rrc\n", GCPhys, rc));
2239	Assert(!SHW_PTE_IS_P(pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK]));
2240	}
2241	}
2242	else
2243	{
2244	/*
2245	* 4/2MB page - lazy syncing shadow 4K pages.
2246	* (There are many causes of getting here, it's no longer only CSAM.)
2247	*/
2248	/* Calculate the GC physical address of this 4KB shadow page. */
2249	GCPhys = PGM_A20_APPLY(pVCpu, GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc) \| (GCPtrPage & GST_BIG_PAGE_OFFSET_MASK));
2250	/* Find ram range. */
2251	PPGMPAGE pPage;
2252	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
2253	if (RT_SUCCESS(rc))
2254	{
2255	AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
2256
2257	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2258	/* Try to make the page writable if necessary. */
2259	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2260	&& ( PGM_PAGE_IS_ZERO(pPage)
2261	\|\| ( PdeSrc.n.u1Write
2262	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2263	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
2264	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
2265	# endif
2266	# ifdef VBOX_WITH_PAGE_SHARING
2267	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
2268	# endif
2269	)
2270	)
2271	)
2272	{
2273	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
2274	AssertRC(rc);
2275	}
2276	# endif
2277
2278	/*
2279	* Make shadow PTE entry.
2280	*/
2281	SHWPTE PteDst;
2282	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2283	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc), &PteDst);
2284	else
2285	SHW_PTE_SET(PteDst, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc) \| PGM_PAGE_GET_HCPHYS(pPage));
2286
2287	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2288	if ( SHW_PTE_IS_P(PteDst)
2289	&& !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2290	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2291
2292	/* Make sure only allocated pages are mapped writable. */
2293	if ( SHW_PTE_IS_P_RW(PteDst)
2294	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2295	{
2296	/* Still applies to shared pages. */
2297	Assert(!PGM_PAGE_IS_ZERO(pPage));
2298	SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
2299	Log3(("SyncPage: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, GCPtrPage));
2300	}
2301
2302	SHW_PTE_ATOMIC_SET2(pPTDst->a[iPTDst], PteDst);
2303
2304	/*
2305	* If the page is not flagged as dirty and is writable, then make it read-only
2306	* at PD level, so we can set the dirty bit when the page is modified.
2307	*
2308	* ASSUMES that page access handlers are implemented on page table entry level.
2309	* Thus we will first catch the dirty access and set PDE.D and restart. If
2310	* there is an access handler, we'll trap again and let it work on the problem.
2311	*/
2312	/** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
2313	* As for invlpg, it simply frees the whole shadow PT.
2314	* ...It's possibly because the guest clears it and the guest doesn't really tell us... */
2315	if ( !PdeSrc.b.u1Dirty
2316	&& PdeSrc.b.u1Write)
2317	{
2318	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2319	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2320	PdeDst.n.u1Write = 0;
2321	}
2322	else
2323	{
2324	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2325	PdeDst.n.u1Write = PdeSrc.n.u1Write;
2326	}
2327	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2328	Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n",
2329	GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
2330	PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2331	}
2332	else
2333	{
2334	LogFlow(("PGM_GCPHYS_2_PTR %RGp (big) failed with %Rrc\n", GCPhys, rc));
2335	/** @todo must wipe the shadow page table entry in this
2336	* case. */
2337	}
2338	}
2339	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2340	return VINF_SUCCESS;
2341	}
2342
2343	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPagePDNAs));
2344	}
2345	else if (fPdeValid)
2346	{
2347	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPagePDOutOfSync));
2348	Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2349	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2350	}
2351	else
2352	{
2353	/// @todo STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSyncAndInvalid));
2354	Log2(("SyncPage: Bad PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2355	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2356	}
2357
2358	/*
2359	* Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
2360	* Yea, I'm lazy.
2361	*/
2362	pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
2363	ASMAtomicWriteSize(pPdeDst, 0);
2364
2365	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2366	PGM_INVL_VCPU_TLBS(pVCpu);
2367	return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
2368
2369
2370	# elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2371	&& !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
2372	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2373	&& !defined(IN_RC)
2374	NOREF(PdeSrc);
2375
2376	# ifdef PGM_SYNC_N_PAGES
2377	/*
2378	* Get the shadow PDE, find the shadow page table in the pool.
2379	*/
2380	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2381	X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
2382
2383	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2384	X86PDEPAE PdeDst = pgmShwGetPaePDE(pVCpu, GCPtrPage);
2385
2386	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2387	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2388	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpt);
2389	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2390	X86PDEPAE PdeDst;
2391	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2392
2393	int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2394	AssertRCSuccessReturn(rc, rc);
2395	Assert(pPDDst && pPdptDst);
2396	PdeDst = pPDDst->a[iPDDst];
2397	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2398	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2399	PEPTPD pPDDst;
2400	EPTPDE PdeDst;
2401
2402	int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, NULL, &pPDDst);
2403	if (rc != VINF_SUCCESS)
2404	{
2405	AssertRC(rc);
2406	return rc;
2407	}
2408	Assert(pPDDst);
2409	PdeDst = pPDDst->a[iPDDst];
2410	# endif
2411	/* In the guest SMP case we could have blocked while another VCPU reused this page table. */
2412	if (!PdeDst.n.u1Present)
2413	{
2414	AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)PdeDst.u));
2415	Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", pVCpu->idCpu, GCPtrPage));
2416	return VINF_SUCCESS; /* force the instruction to be executed again. */
2417	}
2418
2419	/* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */
2420	if (PdeDst.n.u1Size)
2421	{
2422	Assert(pVM->pgm.s.fNestedPaging);
2423	Log(("CPU%d: SyncPage: Pde (big:%RX64) at %RGv changed behind our back!\n", pVCpu->idCpu, PdeDst.u, GCPtrPage));
2424	return VINF_SUCCESS;
2425	}
2426
2427	/* Mask away the page offset. */
2428	GCPtrPage &= ~((RTGCPTR)0xfff);
2429
2430	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2431	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2432
2433	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
2434	if ( cPages > 1
2435	&& !(uErr & X86_TRAP_PF_P)
2436	&& !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
2437	{
2438	/*
2439	* This code path is currently only taken when the caller is PGMTrap0eHandler
2440	* for non-present pages!
2441	*
2442	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2443	* deal with locality.
2444	*/
2445	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2446	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2447	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2448	iPTDst = 0;
2449	else
2450	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2451	for (; iPTDst < iPTDstEnd; iPTDst++)
2452	{
2453	if (!SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2454	{
2455	RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
2456	\| (iPTDst << PAGE_SHIFT));
2457
2458	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
2459	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
2460	GCPtrCurPage,
2461	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2462	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2463
2464	if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
2465	break;
2466	}
2467	else
2468	Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n", (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT), iPTDst, SHW_PTE_LOG64(pPTDst->a[iPTDst]) ));
2469	}
2470	}
2471	else
2472	# endif /* PGM_SYNC_N_PAGES */
2473	{
2474	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2475	RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
2476	\| (iPTDst << PAGE_SHIFT));
2477
2478	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
2479
2480	Log2(("SyncPage: 4K %RGv PteSrc:{P=1 RW=1 U=1}PteDst=%08llx%s\n",
2481	GCPtrPage,
2482	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2483	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2484	}
2485	return VINF_SUCCESS;
2486
2487	# else
2488	NOREF(PdeSrc);
2489	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2490	return VERR_PGM_NOT_USED_IN_MODE;
2491	# endif
2492	}
2493
2494	#endif /* PGM_SHW_TYPE != PGM_TYPE_NONE */
2495	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
2496
2497	/**
2498	* CheckPageFault helper for returning a page fault indicating a non-present
2499	* (NP) entry in the page translation structures.
2500	*
2501	* @returns VINF_EM_RAW_GUEST_TRAP.
2502	* @param pVCpu The cross context virtual CPU structure.
2503	* @param uErr The error code of the shadow fault. Corrections to
2504	* TRPM's copy will be made if necessary.
2505	* @param GCPtrPage For logging.
2506	* @param uPageFaultLevel For logging.
2507	*/
2508	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnNP)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2509	{
2510	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2511	AssertMsg(!(uErr & X86_TRAP_PF_P), ("%#x\n", uErr));
2512	AssertMsg(!(uErr & X86_TRAP_PF_RSVD), ("%#x\n", uErr));
2513	if (uErr & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
2514	TRPMSetErrorCode(pVCpu, uErr & ~(X86_TRAP_PF_RSVD \| X86_TRAP_PF_P));
2515
2516	Log(("CheckPageFault: real page fault (notp) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2517	RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(uPageFaultLevel);
2518	return VINF_EM_RAW_GUEST_TRAP;
2519	}
2520
2521
2522	/**
2523	* CheckPageFault helper for returning a page fault indicating a reserved bit
2524	* (RSVD) error in the page translation structures.
2525	*
2526	* @returns VINF_EM_RAW_GUEST_TRAP.
2527	* @param pVCpu The cross context virtual CPU structure.
2528	* @param uErr The error code of the shadow fault. Corrections to
2529	* TRPM's copy will be made if necessary.
2530	* @param GCPtrPage For logging.
2531	* @param uPageFaultLevel For logging.
2532	*/
2533	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnRSVD)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2534	{
2535	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2536	if ((uErr & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) != (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
2537	TRPMSetErrorCode(pVCpu, uErr \| X86_TRAP_PF_RSVD \| X86_TRAP_PF_P);
2538
2539	Log(("CheckPageFault: real page fault (rsvd) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2540	RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(uPageFaultLevel);
2541	return VINF_EM_RAW_GUEST_TRAP;
2542	}
2543
2544
2545	/**
2546	* CheckPageFault helper for returning a page protection fault (P).
2547	*
2548	* @returns VINF_EM_RAW_GUEST_TRAP.
2549	* @param pVCpu The cross context virtual CPU structure.
2550	* @param uErr The error code of the shadow fault. Corrections to
2551	* TRPM's copy will be made if necessary.
2552	* @param GCPtrPage For logging.
2553	* @param uPageFaultLevel For logging.
2554	*/
2555	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnProt)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2556	{
2557	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2558	AssertMsg(uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID), ("%#x\n", uErr));
2559	if ((uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) != X86_TRAP_PF_P)
2560	TRPMSetErrorCode(pVCpu, (uErr & ~X86_TRAP_PF_RSVD) \| X86_TRAP_PF_P);
2561
2562	Log(("CheckPageFault: real page fault (prot) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2563	RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(uPageFaultLevel);
2564	return VINF_EM_RAW_GUEST_TRAP;
2565	}
2566
2567
2568	/**
2569	* Handle dirty bit tracking faults.
2570	*
2571	* @returns VBox status code.
2572	* @param pVCpu The cross context virtual CPU structure.
2573	* @param uErr Page fault error code.
2574	* @param pPdeSrc Guest page directory entry.
2575	* @param pPdeDst Shadow page directory entry.
2576	* @param GCPtrPage Guest context page address.
2577	*/
2578	static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc,
2579	RTGCPTR GCPtrPage)
2580	{
2581	PVM pVM = pVCpu->CTX_SUFF(pVM);
2582	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2583	NOREF(uErr);
2584
2585	PGM_LOCK_ASSERT_OWNER(pVM);
2586
2587	/*
2588	* Handle big page.
2589	*/
2590	if (pPdeSrc->b.u1Size && GST_IS_PSE_ACTIVE(pVCpu))
2591	{
2592	if ( pPdeDst->n.u1Present
2593	&& (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY))
2594	{
2595	SHWPDE PdeDst = *pPdeDst;
2596
2597	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageTrap));
2598	Assert(pPdeSrc->b.u1Write);
2599
2600	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
2601	* fault again and take this path to only invalidate the entry (see below).
2602	*/
2603	PdeDst.n.u1Write = 1;
2604	PdeDst.n.u1Accessed = 1;
2605	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2606	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2607	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
2608	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2609	}
2610
2611	# ifdef IN_RING0
2612	/* Check for stale TLB entry; only applies to the SMP guest case. */
2613	if ( pVM->cCpus > 1
2614	&& pPdeDst->n.u1Write
2615	&& pPdeDst->n.u1Accessed)
2616	{
2617	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2618	if (pShwPage)
2619	{
2620	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2621	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2622	if (SHW_PTE_IS_P_RW(*pPteDst))
2623	{
2624	/* Stale TLB entry. */
2625	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageStale));
2626	PGM_INVL_PG(pVCpu, GCPtrPage);
2627	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2628	}
2629	}
2630	}
2631	# endif /* IN_RING0 */
2632	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2633	}
2634
2635	/*
2636	* Map the guest page table.
2637	*/
2638	PGSTPT pPTSrc;
2639	int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(*pPdeSrc), &pPTSrc);
2640	if (RT_FAILURE(rc))
2641	{
2642	AssertRC(rc);
2643	return rc;
2644	}
2645
2646	if (pPdeDst->n.u1Present)
2647	{
2648	GSTPTE const *pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2649	const GSTPTE PteSrc = *pPteSrc;
2650
2651	#ifdef VBOX_WITH_RAW_MODE_NOT_R0
2652	/* Bail out here as pgmPoolGetPage will return NULL and we'll crash below.
2653	* Our individual shadow handlers will provide more information and force a fatal exit.
2654	*/
2655	if ( VM_IS_RAW_MODE_ENABLED(pVM)
2656	&& MMHyperIsInsideArea(pVM, (RTGCPTR)GCPtrPage))
2657	{
2658	LogRel(("CheckPageFault: write to hypervisor region %RGv\n", GCPtrPage));
2659	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2660	}
2661	#endif
2662	/*
2663	* Map shadow page table.
2664	*/
2665	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2666	if (pShwPage)
2667	{
2668	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2669	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2670	if (SHW_PTE_IS_P(pPteDst)) /* @todo Optimize accessed bit emulation? */
2671	{
2672	if (SHW_PTE_IS_TRACK_DIRTY(*pPteDst))
2673	{
2674	PPGMPAGE pPage = pgmPhysGetPage(pVM, GST_GET_PTE_GCPHYS(PteSrc));
2675	SHWPTE PteDst = *pPteDst;
2676
2677	LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage));
2678	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageTrap));
2679
2680	Assert(PteSrc.n.u1Write);
2681
2682	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB
2683	* entry will not harm; write access will simply fault again and
2684	* take this path to only invalidate the entry.
2685	*/
2686	if (RT_LIKELY(pPage))
2687	{
2688	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2689	{
2690	//AssertMsgFailed(("%R[pgmpage] - we don't set PGM_PTFLAGS_TRACK_DIRTY for these pages\n", pPage));
2691	Assert(!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage));
2692	/* Assuming write handlers here as the PTE is present (otherwise we wouldn't be here). */
2693	SHW_PTE_SET_RO(PteDst);
2694	}
2695	else
2696	{
2697	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
2698	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
2699	{
2700	rc = pgmPhysPageMakeWritable(pVM, pPage, GST_GET_PTE_GCPHYS(PteSrc));
2701	AssertRC(rc);
2702	}
2703	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED)
2704	SHW_PTE_SET_RW(PteDst);
2705	else
2706	{
2707	/* Still applies to shared pages. */
2708	Assert(!PGM_PAGE_IS_ZERO(pPage));
2709	SHW_PTE_SET_RO(PteDst);
2710	}
2711	}
2712	}
2713	else
2714	SHW_PTE_SET_RW(PteDst); /** @todo r=bird: This doesn't make sense to me. */
2715
2716	SHW_PTE_SET(PteDst, (SHW_PTE_GET_U(PteDst) \| X86_PTE_D \| X86_PTE_A) & ~(uint64_t)PGM_PTFLAGS_TRACK_DIRTY);
2717	SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
2718	PGM_INVL_PG(pVCpu, GCPtrPage);
2719	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2720	}
2721
2722	# ifdef IN_RING0
2723	/* Check for stale TLB entry; only applies to the SMP guest case. */
2724	if ( pVM->cCpus > 1
2725	&& SHW_PTE_IS_RW(*pPteDst)
2726	&& SHW_PTE_IS_A(*pPteDst))
2727	{
2728	/* Stale TLB entry. */
2729	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageStale));
2730	PGM_INVL_PG(pVCpu, GCPtrPage);
2731	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2732	}
2733	# endif
2734	}
2735	}
2736	else
2737	AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
2738	}
2739
2740	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2741	}
2742
2743	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE */
2744
2745	/**
2746	* Sync a shadow page table.
2747	*
2748	* The shadow page table is not present in the shadow PDE.
2749	*
2750	* Handles mapping conflicts.
2751	*
2752	* This is called by VerifyAccessSyncPage, PrefetchPage, InvalidatePage (on
2753	* conflict), and Trap0eHandler.
2754	*
2755	* A precondition for this method is that the shadow PDE is not present. The
2756	* caller must take the PGM lock before checking this and continue to hold it
2757	* when calling this method.
2758	*
2759	* @returns VBox status code.
2760	* @param pVCpu The cross context virtual CPU structure.
2761	* @param iPDSrc Page directory index.
2762	* @param pPDSrc Source page directory (i.e. Guest OS page directory).
2763	* Assume this is a temporary mapping.
2764	* @param GCPtrPage GC Pointer of the page that caused the fault
2765	*/
2766	static int PGM_BTH_NAME(SyncPT)(PVMCPU pVCpu, unsigned iPDSrc, PGSTPD pPDSrc, RTGCPTR GCPtrPage)
2767	{
2768	PVM pVM = pVCpu->CTX_SUFF(pVM);
2769	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
2770
2771	#if 0 /* rarely useful; leave for debugging. */
2772	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPtPD[iPDSrc]);
2773	#endif
2774	LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage)); RT_NOREF_PV(GCPtrPage);
2775
2776	PGM_LOCK_ASSERT_OWNER(pVM);
2777
2778	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2779	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
2780	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
2781	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
2782	&& PGM_SHW_TYPE != PGM_TYPE_NONE
2783	int rc = VINF_SUCCESS;
2784
2785	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2786
2787	/*
2788	* Some input validation first.
2789	*/
2790	AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
2791
2792	/*
2793	* Get the relevant shadow PDE entry.
2794	*/
2795	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2796	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2797	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
2798
2799	/* Fetch the pgm pool shadow descriptor. */
2800	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2801	Assert(pShwPde);
2802
2803	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2804	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2805	PPGMPOOLPAGE pShwPde = NULL;
2806	PX86PDPAE pPDDst;
2807	PSHWPDE pPdeDst;
2808
2809	/* Fetch the pgm pool shadow descriptor. */
2810	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
2811	AssertRCSuccessReturn(rc, rc);
2812	Assert(pShwPde);
2813
2814	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
2815	pPdeDst = &pPDDst->a[iPDDst];
2816
2817	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2818	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2819	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2820	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2821	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2822	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2823	AssertRCSuccessReturn(rc, rc);
2824	Assert(pPDDst);
2825	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2826	# endif
2827	SHWPDE PdeDst = *pPdeDst;
2828
2829	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2830	/* Fetch the pgm pool shadow descriptor. */
2831	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2832	Assert(pShwPde);
2833	# endif
2834
2835	# ifndef PGM_WITHOUT_MAPPINGS
2836	/*
2837	* Check for conflicts.
2838	* RC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
2839	* R3: Simply resolve the conflict.
2840	*/
2841	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2842	{
2843	Assert(pgmMapAreMappingsEnabled(pVM));
2844	# ifndef IN_RING3
2845	Log(("SyncPT: Conflict at %RGv\n", GCPtrPage));
2846	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2847	return VERR_ADDRESS_CONFLICT;
2848
2849	# else /* IN_RING3 */
2850	PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
2851	Assert(pMapping);
2852	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2853	rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2854	# elif PGM_GST_TYPE == PGM_TYPE_PAE
2855	rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2856	# else
2857	AssertFailed(); NOREF(pMapping); /* can't happen for amd64 */
2858	# endif
2859	if (RT_FAILURE(rc))
2860	{
2861	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2862	return rc;
2863	}
2864	PdeDst = *pPdeDst;
2865	# endif /* IN_RING3 */
2866	}
2867	# endif /* !PGM_WITHOUT_MAPPINGS */
2868	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2869
2870	/*
2871	* Sync the page directory entry.
2872	*/
2873	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2874	const bool fPageTable = !PdeSrc.b.u1Size \|\| !GST_IS_PSE_ACTIVE(pVCpu);
2875	if ( PdeSrc.n.u1Present
2876	&& (fPageTable ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc)) )
2877	{
2878	/*
2879	* Allocate & map the page table.
2880	*/
2881	PSHWPT pPTDst;
2882	PPGMPOOLPAGE pShwPage;
2883	RTGCPHYS GCPhys;
2884	if (fPageTable)
2885	{
2886	GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
2887	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2888	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2889	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| ((iPDDst & 1) * (PAGE_SIZE / 2)));
2890	# endif
2891	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
2892	pShwPde->idx, iPDDst, false /fLockPage/,
2893	&pShwPage);
2894	}
2895	else
2896	{
2897	PGMPOOLACCESS enmAccess;
2898	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2899	const bool fNoExecute = PdeSrc.n.u1NoExecute && GST_IS_NX_ACTIVE(pVCpu);
2900	# else
2901	const bool fNoExecute = false;
2902	# endif
2903
2904	GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
2905	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2906	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2907	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
2908	# endif
2909	/* Determine the right kind of large page to avoid incorrect cached entry reuse. */
2910	if (PdeSrc.n.u1User)
2911	{
2912	if (PdeSrc.n.u1Write)
2913	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_RW_NX : PGMPOOLACCESS_USER_RW;
2914	else
2915	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_R_NX : PGMPOOLACCESS_USER_R;
2916	}
2917	else
2918	{
2919	if (PdeSrc.n.u1Write)
2920	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW;
2921	else
2922	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_R_NX : PGMPOOLACCESS_SUPERVISOR_R;
2923	}
2924	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, enmAccess, PGM_A20_IS_ENABLED(pVCpu),
2925	pShwPde->idx, iPDDst, false /fLockPage/,
2926	&pShwPage);
2927	}
2928	if (rc == VINF_SUCCESS)
2929	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2930	else if (rc == VINF_PGM_CACHED_PAGE)
2931	{
2932	/*
2933	* The PT was cached, just hook it up.
2934	*/
2935	if (fPageTable)
2936	PdeDst.u = pShwPage->Core.Key \| GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2937	else
2938	{
2939	PdeDst.u = pShwPage->Core.Key \| GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2940	/* (see explanation and assumptions further down.) */
2941	if ( !PdeSrc.b.u1Dirty
2942	&& PdeSrc.b.u1Write)
2943	{
2944	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2945	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2946	PdeDst.b.u1Write = 0;
2947	}
2948	}
2949	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2950	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2951	return VINF_SUCCESS;
2952	}
2953	else
2954	AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
2955	/** @todo Why do we bother preserving X86_PDE_AVL_MASK here?
2956	* Both PGM_PDFLAGS_MAPPING and PGM_PDFLAGS_TRACK_DIRTY should be
2957	* irrelevant at this point. */
2958	PdeDst.u &= X86_PDE_AVL_MASK;
2959	PdeDst.u \|= pShwPage->Core.Key;
2960
2961	/*
2962	* Page directory has been accessed (this is a fault situation, remember).
2963	*/
2964	/** @todo
2965	* Well, when the caller is PrefetchPage or InvalidatePage is isn't a
2966	* fault situation. What's more, the Trap0eHandler has already set the
2967	* accessed bit. So, it's actually just VerifyAccessSyncPage which
2968	* might need setting the accessed flag.
2969	*
2970	* The best idea is to leave this change to the caller and add an
2971	* assertion that it's set already. */
2972	pPDSrc->a[iPDSrc].n.u1Accessed = 1;
2973	if (fPageTable)
2974	{
2975	/*
2976	* Page table - 4KB.
2977	*
2978	* Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
2979	*/
2980	Log2(("SyncPT: 4K %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
2981	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
2982	PGSTPT pPTSrc;
2983	rc = PGM_GCPHYS_2_PTR(pVM, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
2984	if (RT_SUCCESS(rc))
2985	{
2986	/*
2987	* Start by syncing the page directory entry so CSAM's TLB trick works.
2988	*/
2989	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| X86_PDE_AVL_MASK))
2990	\| GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2991	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2992	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2993
2994	/*
2995	* Directory/page user or supervisor privilege: (same goes for read/write)
2996	*
2997	* Directory Page Combined
2998	* U/S U/S U/S
2999	* 0 0 0
3000	* 0 1 0
3001	* 1 0 0
3002	* 1 1 1
3003	*
3004	* Simple AND operation. Table listed for completeness.
3005	*
3006	*/
3007	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT4K));
3008	# ifdef PGM_SYNC_N_PAGES
3009	unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
3010	unsigned iPTDst = iPTBase;
3011	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
3012	if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
3013	iPTDst = 0;
3014	else
3015	iPTDst -= PGM_SYNC_NR_PAGES / 2;
3016	# else /* !PGM_SYNC_N_PAGES */
3017	unsigned iPTDst = 0;
3018	const unsigned iPTDstEnd = RT_ELEMENTS(pPTDst->a);
3019	# endif /* !PGM_SYNC_N_PAGES */
3020	RTGCPTR GCPtrCur = (GCPtrPage & ~(RTGCPTR)((1 << SHW_PD_SHIFT) - 1))
3021	\| ((RTGCPTR)iPTDst << PAGE_SHIFT);
3022	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3023	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3024	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
3025	# else
3026	const unsigned offPTSrc = 0;
3027	# endif
3028	for (; iPTDst < iPTDstEnd; iPTDst++, GCPtrCur += PAGE_SIZE)
3029	{
3030	const unsigned iPTSrc = iPTDst + offPTSrc;
3031	const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
3032
3033	if (PteSrc.n.u1Present)
3034	{
3035	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
3036	/*
3037	* Assuming kernel code will be marked as supervisor - and not as user level
3038	* and executed using a conforming code selector - And marked as readonly.
3039	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
3040	*/
3041	PPGMPAGE pPage;
3042	if ( ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW \| X86_PTE_US))
3043	\|\| !CSAMDoesPageNeedScanning(pVM, GCPtrCur)
3044	\|\| ( (pPage = pgmPhysGetPage(pVM, GST_GET_PTE_GCPHYS(PteSrc)))
3045	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
3046	)
3047	# endif
3048	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
3049	Log2(("SyncPT: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
3050	GCPtrCur,
3051	PteSrc.n.u1Present,
3052	PteSrc.n.u1Write & PdeSrc.n.u1Write,
3053	PteSrc.n.u1User & PdeSrc.n.u1User,
3054	(uint64_t)PteSrc.u,
3055	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : "", SHW_PTE_LOG64(pPTDst->a[iPTDst]), iPTSrc, PdeSrc.au32[0],
3056	(RTGCPHYS)(GST_GET_PDE_GCPHYS(PdeSrc) + iPTSrc*sizeof(PteSrc)) ));
3057	}
3058	/* else: the page table was cleared by the pool */
3059	} /* for PTEs */
3060	}
3061	}
3062	else
3063	{
3064	/*
3065	* Big page - 2/4MB.
3066	*
3067	* We'll walk the ram range list in parallel and optimize lookups.
3068	* We will only sync one shadow page table at a time.
3069	*/
3070	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT4M));
3071
3072	/**
3073	* @todo It might be more efficient to sync only a part of the 4MB
3074	* page (similar to what we do for 4KB PDs).
3075	*/
3076
3077	/*
3078	* Start by syncing the page directory entry.
3079	*/
3080	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
3081	\| GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
3082
3083	/*
3084	* If the page is not flagged as dirty and is writable, then make it read-only
3085	* at PD level, so we can set the dirty bit when the page is modified.
3086	*
3087	* ASSUMES that page access handlers are implemented on page table entry level.
3088	* Thus we will first catch the dirty access and set PDE.D and restart. If
3089	* there is an access handler, we'll trap again and let it work on the problem.
3090	*/
3091	/** @todo move the above stuff to a section in the PGM documentation. */
3092	Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
3093	if ( !PdeSrc.b.u1Dirty
3094	&& PdeSrc.b.u1Write)
3095	{
3096	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
3097	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
3098	PdeDst.b.u1Write = 0;
3099	}
3100	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
3101	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3102
3103	/*
3104	* Fill the shadow page table.
3105	*/
3106	/* Get address and flags from the source PDE. */
3107	SHWPTE PteDstBase;
3108	SHW_PTE_SET(PteDstBase, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc));
3109
3110	/* Loop thru the entries in the shadow PT. */
3111	const RTGCPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
3112	Log2(("SyncPT: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n",
3113	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
3114	GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
3115	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
3116	unsigned iPTDst = 0;
3117	while ( iPTDst < RT_ELEMENTS(pPTDst->a)
3118	&& !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
3119	{
3120	if (pRam && GCPhys >= pRam->GCPhys)
3121	{
3122	# ifndef PGM_WITH_A20
3123	unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
3124	# endif
3125	do
3126	{
3127	/* Make shadow PTE. */
3128	# ifdef PGM_WITH_A20
3129	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
3130	# else
3131	PPGMPAGE pPage = &pRam->aPages[iHCPage];
3132	# endif
3133	SHWPTE PteDst;
3134
3135	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
3136	/* Try to make the page writable if necessary. */
3137	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
3138	&& ( PGM_PAGE_IS_ZERO(pPage)
3139	\|\| ( SHW_PTE_IS_RW(PteDstBase)
3140	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
3141	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
3142	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
3143	# endif
3144	# ifdef VBOX_WITH_PAGE_SHARING
3145	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
3146	# endif
3147	&& !PGM_PAGE_IS_BALLOONED(pPage))
3148	)
3149	)
3150	{
3151	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
3152	AssertRCReturn(rc, rc);
3153	if (VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
3154	break;
3155	}
3156	# endif
3157
3158	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
3159	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, SHW_PTE_GET_U(PteDstBase), &PteDst);
3160	else if (PGM_PAGE_IS_BALLOONED(pPage))
3161	SHW_PTE_SET(PteDst, 0); /* Handle ballooned pages at #PF time. */
3162	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
3163	/*
3164	* Assuming kernel code will be marked as supervisor and not as user level and executed
3165	* using a conforming code selector. Don't check for readonly, as that implies the whole
3166	* 4MB can be code or readonly data. Linux enables write access for its large pages.
3167	*/
3168	else if ( !PdeSrc.n.u1User
3169	&& CSAMDoesPageNeedScanning(pVM, GCPtr \| (iPTDst << SHW_PT_SHIFT)))
3170	SHW_PTE_SET(PteDst, 0);
3171	# endif
3172	else
3173	SHW_PTE_SET(PteDst, PGM_PAGE_GET_HCPHYS(pPage) \| SHW_PTE_GET_U(PteDstBase));
3174
3175	/* Only map writable pages writable. */
3176	if ( SHW_PTE_IS_P_RW(PteDst)
3177	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
3178	{
3179	/* Still applies to shared pages. */
3180	Assert(!PGM_PAGE_IS_ZERO(pPage));
3181	SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
3182	Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))));
3183	}
3184
3185	if (SHW_PTE_IS_P(PteDst))
3186	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
3187
3188	/* commit it (not atomic, new table) */
3189	pPTDst->a[iPTDst] = PteDst;
3190	Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
3191	(RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT)), SHW_PTE_IS_P(PteDst), SHW_PTE_IS_RW(PteDst), SHW_PTE_IS_US(PteDst), SHW_PTE_LOG64(PteDst),
3192	SHW_PTE_IS_TRACK_DIRTY(PteDst) ? " Track-Dirty" : ""));
3193
3194	/* advance */
3195	GCPhys += PAGE_SIZE;
3196	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
3197	# ifndef PGM_WITH_A20
3198	iHCPage++;
3199	# endif
3200	iPTDst++;
3201	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
3202	&& GCPhys <= pRam->GCPhysLast);
3203
3204	/* Advance ram range list. */
3205	while (pRam && GCPhys > pRam->GCPhysLast)
3206	pRam = pRam->CTX_SUFF(pNext);
3207	}
3208	else if (pRam)
3209	{
3210	Log(("Invalid pages at %RGp\n", GCPhys));
3211	do
3212	{
3213	SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
3214	GCPhys += PAGE_SIZE;
3215	iPTDst++;
3216	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
3217	&& GCPhys < pRam->GCPhys);
3218	PGM_A20_APPLY_TO_VAR(pVCpu,GCPhys);
3219	}
3220	else
3221	{
3222	Log(("Invalid pages at %RGp (2)\n", GCPhys));
3223	for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
3224	SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
3225	}
3226	} /* while more PTEs */
3227	} /* 4KB / 4MB */
3228	}
3229	else
3230	AssertRelease(!PdeDst.n.u1Present);
3231
3232	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3233	if (RT_FAILURE(rc))
3234	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPTFailed));
3235	return rc;
3236
3237	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
3238	&& !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
3239	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
3240	&& PGM_SHW_TYPE != PGM_TYPE_NONE \
3241	&& !defined(IN_RC)
3242	NOREF(iPDSrc); NOREF(pPDSrc);
3243
3244	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3245
3246	/*
3247	* Validate input a little bit.
3248	*/
3249	int rc = VINF_SUCCESS;
3250	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3251	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3252	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3253
3254	/* Fetch the pgm pool shadow descriptor. */
3255	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
3256	Assert(pShwPde);
3257
3258	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3259	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3260	PPGMPOOLPAGE pShwPde = NULL; /* initialized to shut up gcc */
3261	PX86PDPAE pPDDst;
3262	PSHWPDE pPdeDst;
3263
3264	/* Fetch the pgm pool shadow descriptor. */
3265	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
3266	AssertRCSuccessReturn(rc, rc);
3267	Assert(pShwPde);
3268
3269	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
3270	pPdeDst = &pPDDst->a[iPDDst];
3271
3272	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3273	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3274	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3275	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
3276	PX86PDPT pPdptDst= NULL; /* initialized to shut up gcc */
3277	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
3278	AssertRCSuccessReturn(rc, rc);
3279	Assert(pPDDst);
3280	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3281
3282	/* Fetch the pgm pool shadow descriptor. */
3283	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
3284	Assert(pShwPde);
3285
3286	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
3287	const unsigned iPdpt = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
3288	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3289	PEPTPD pPDDst;
3290	PEPTPDPT pPdptDst;
3291
3292	rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, &pPdptDst, &pPDDst);
3293	if (rc != VINF_SUCCESS)
3294	{
3295	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3296	AssertRC(rc);
3297	return rc;
3298	}
3299	Assert(pPDDst);
3300	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3301
3302	/* Fetch the pgm pool shadow descriptor. */
3303	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & EPT_PDPTE_PG_MASK);
3304	Assert(pShwPde);
3305	# endif
3306	SHWPDE PdeDst = *pPdeDst;
3307
3308	Assert(!(PdeDst.u & PGM_PDFLAGS_MAPPING));
3309	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
3310
3311	# if defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
3312	if ( BTH_IS_NP_ACTIVE(pVM)
3313	&& !VM_IS_NEM_ENABLED(pVM)) /** @todo NEM: Large page support. */
3314	{
3315	/* Check if we allocated a big page before for this 2 MB range. */
3316	PPGMPAGE pPage;
3317	rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PDE2M_PAE_PG_MASK), &pPage);
3318	if (RT_SUCCESS(rc))
3319	{
3320	RTHCPHYS HCPhys = NIL_RTHCPHYS;
3321	if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
3322	{
3323	if (PGM_A20_IS_ENABLED(pVCpu))
3324	{
3325	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageReused);
3326	AssertRelease(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3327	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3328	}
3329	else
3330	{
3331	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PDE_DISABLED);
3332	pVM->pgm.s.cLargePagesDisabled++;
3333	}
3334	}
3335	else if ( PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED
3336	&& PGM_A20_IS_ENABLED(pVCpu))
3337	{
3338	/* Recheck the entire 2 MB range to see if we can use it again as a large page. */
3339	rc = pgmPhysRecheckLargePage(pVM, GCPtrPage, pPage);
3340	if (RT_SUCCESS(rc))
3341	{
3342	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3343	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3344	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3345	}
3346	}
3347	else if ( PGMIsUsingLargePages(pVM)
3348	&& PGM_A20_IS_ENABLED(pVCpu))
3349	{
3350	rc = pgmPhysAllocLargePage(pVM, GCPtrPage);
3351	if (RT_SUCCESS(rc))
3352	{
3353	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3354	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3355	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3356	}
3357	else
3358	LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
3359	}
3360
3361	if (HCPhys != NIL_RTHCPHYS)
3362	{
3363	PdeDst.u &= X86_PDE_AVL_MASK;
3364	PdeDst.u \|= HCPhys;
3365	PdeDst.n.u1Present = 1;
3366	PdeDst.n.u1Write = 1;
3367	PdeDst.b.u1Size = 1;
3368	# if PGM_SHW_TYPE == PGM_TYPE_EPT
3369	PdeDst.n.u1Execute = 1;
3370	PdeDst.b.u1IgnorePAT = 1;
3371	PdeDst.b.u3EMT = VMX_EPT_MEMTYPE_WB;
3372	# else
3373	PdeDst.n.u1User = 1;
3374	# endif
3375	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
3376
3377	Log(("SyncPT: Use large page at %RGp PDE=%RX64\n", GCPtrPage, PdeDst.u));
3378	/* Add a reference to the first page only. */
3379	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPde, PGM_PAGE_GET_TRACKING(pPage), pPage, iPDDst);
3380
3381	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3382	return VINF_SUCCESS;
3383	}
3384	}
3385	}
3386	# endif /* HC_ARCH_BITS == 64 */
3387
3388	/*
3389	* Allocate & map the page table.
3390	*/
3391	PSHWPT pPTDst;
3392	PPGMPOOLPAGE pShwPage;
3393	RTGCPHYS GCPhys;
3394
3395	/* Virtual address = physical address */
3396	GCPhys = PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PAGE_4K_BASE_MASK);
3397	rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE,
3398	PGM_A20_IS_ENABLED(pVCpu), pShwPde->idx, iPDDst, false /fLockPage/,
3399	&pShwPage);
3400	if ( rc == VINF_SUCCESS
3401	\|\| rc == VINF_PGM_CACHED_PAGE)
3402	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3403	else
3404	{
3405	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3406	AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
3407	}
3408
3409	if (rc == VINF_SUCCESS)
3410	{
3411	/* New page table; fully set it up. */
3412	Assert(pPTDst);
3413
3414	/* Mask away the page offset. */
3415	GCPtrPage &= ~(RTGCPTR)PAGE_OFFSET_MASK;
3416
3417	for (unsigned iPTDst = 0; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
3418	{
3419	RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
3420	\| (iPTDst << PAGE_SHIFT));
3421
3422	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
3423	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
3424	GCPtrCurPage,
3425	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
3426	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
3427
3428	if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
3429	break;
3430	}
3431	}
3432	else
3433	rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */
3434
3435	/* Save the new PDE. */
3436	PdeDst.u &= X86_PDE_AVL_MASK;
3437	PdeDst.u \|= pShwPage->Core.Key;
3438	PdeDst.n.u1Present = 1;
3439	PdeDst.n.u1Write = 1;
3440	# if PGM_SHW_TYPE == PGM_TYPE_EPT
3441	PdeDst.n.u1Execute = 1;
3442	# else
3443	PdeDst.n.u1User = 1;
3444	PdeDst.n.u1Accessed = 1;
3445	# endif
3446	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
3447
3448	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3449	if (RT_FAILURE(rc))
3450	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPTFailed));
3451	return rc;
3452
3453	#else
3454	NOREF(iPDSrc); NOREF(pPDSrc);
3455	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
3456	return VERR_PGM_NOT_USED_IN_MODE;
3457	#endif
3458	}
3459
3460
3461
3462	/**
3463	* Prefetch a page/set of pages.
3464	*
3465	* Typically used to sync commonly used pages before entering raw mode
3466	* after a CR3 reload.
3467	*
3468	* @returns VBox status code.
3469	* @param pVCpu The cross context virtual CPU structure.
3470	* @param GCPtrPage Page to invalidate.
3471	*/
3472	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
3473	{
3474	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3475	\|\| PGM_GST_TYPE == PGM_TYPE_REAL \
3476	\|\| PGM_GST_TYPE == PGM_TYPE_PROT \
3477	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3478	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3479	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
3480	&& PGM_SHW_TYPE != PGM_TYPE_NONE
3481	/*
3482	* Check that all Guest levels thru the PDE are present, getting the
3483	* PD and PDE in the processes.
3484	*/
3485	int rc = VINF_SUCCESS;
3486	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3487	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3488	const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
3489	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3490	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3491	unsigned iPDSrc;
3492	X86PDPE PdpeSrc;
3493	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3494	if (!pPDSrc)
3495	return VINF_SUCCESS; /* not present */
3496	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3497	unsigned iPDSrc;
3498	PX86PML4E pPml4eSrc;
3499	X86PDPE PdpeSrc;
3500	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3501	if (!pPDSrc)
3502	return VINF_SUCCESS; /* not present */
3503	# endif
3504	const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3505	# else
3506	PGSTPD pPDSrc = NULL;
3507	const unsigned iPDSrc = 0;
3508	GSTPDE PdeSrc;
3509
3510	PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3511	PdeSrc.n.u1Present = 1;
3512	PdeSrc.n.u1Write = 1;
3513	PdeSrc.n.u1Accessed = 1;
3514	PdeSrc.n.u1User = 1;
3515	# endif
3516
3517	if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
3518	{
3519	PVM pVM = pVCpu->CTX_SUFF(pVM);
3520	pgmLock(pVM);
3521
3522	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3523	const X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
3524	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3525	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3526	PX86PDPAE pPDDst;
3527	X86PDEPAE PdeDst;
3528	# if PGM_GST_TYPE != PGM_TYPE_PAE
3529	X86PDPE PdpeSrc;
3530
3531	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3532	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3533	# endif
3534	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3535	if (rc != VINF_SUCCESS)
3536	{
3537	pgmUnlock(pVM);
3538	AssertRC(rc);
3539	return rc;
3540	}
3541	Assert(pPDDst);
3542	PdeDst = pPDDst->a[iPDDst];
3543
3544	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3545	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3546	PX86PDPAE pPDDst;
3547	X86PDEPAE PdeDst;
3548
3549	# if PGM_GST_TYPE == PGM_TYPE_PROT
3550	/* AMD-V nested paging */
3551	X86PML4E Pml4eSrc;
3552	X86PDPE PdpeSrc;
3553	PX86PML4E pPml4eSrc = &Pml4eSrc;
3554
3555	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3556	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
3557	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
3558	# endif
3559
3560	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3561	if (rc != VINF_SUCCESS)
3562	{
3563	pgmUnlock(pVM);
3564	AssertRC(rc);
3565	return rc;
3566	}
3567	Assert(pPDDst);
3568	PdeDst = pPDDst->a[iPDDst];
3569	# endif
3570	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
3571	{
3572	if (!PdeDst.n.u1Present)
3573	{
3574	/** @todo r=bird: This guy will set the A bit on the PDE,
3575	* probably harmless. */
3576	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3577	}
3578	else
3579	{
3580	/* Note! We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
3581	* R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
3582	* makes no sense to prefetch more than one page.
3583	*/
3584	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3585	if (RT_SUCCESS(rc))
3586	rc = VINF_SUCCESS;
3587	}
3588	}
3589	pgmUnlock(pVM);
3590	}
3591	return rc;
3592
3593	#elif PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \|\| PGM_SHW_TYPE == PGM_TYPE_NONE
3594	NOREF(pVCpu); NOREF(GCPtrPage);
3595	return VINF_SUCCESS; /* ignore */
3596	#else
3597	AssertCompile(0);
3598	#endif
3599	}
3600
3601
3602
3603
3604	/**
3605	* Syncs a page during a PGMVerifyAccess() call.
3606	*
3607	* @returns VBox status code (informational included).
3608	* @param pVCpu The cross context virtual CPU structure.
3609	* @param GCPtrPage The address of the page to sync.
3610	* @param fPage The effective guest page flags.
3611	* @param uErr The trap error code.
3612	* @remarks This will normally never be called on invalid guest page
3613	* translation entries.
3614	*/
3615	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fPage, unsigned uErr)
3616	{
3617	PVM pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
3618
3619	LogFlow(("VerifyAccessSyncPage: GCPtrPage=%RGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
3620	RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(fPage); RT_NOREF_PV(uErr);
3621
3622	Assert(!pVM->pgm.s.fNestedPaging);
3623	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3624	\|\| PGM_GST_TYPE == PGM_TYPE_REAL \
3625	\|\| PGM_GST_TYPE == PGM_TYPE_PROT \
3626	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3627	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3628	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
3629	&& PGM_SHW_TYPE != PGM_TYPE_NONE
3630
3631	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
3632	if (!(fPage & X86_PTE_US))
3633	{
3634	/*
3635	* Mark this page as safe.
3636	*/
3637	/** @todo not correct for pages that contain both code and data!! */
3638	Log(("CSAMMarkPage %RGv; scanned=%d\n", GCPtrPage, true));
3639	CSAMMarkPage(pVM, GCPtrPage, true);
3640	}
3641	# endif
3642
3643	/*
3644	* Get guest PD and index.
3645	*/
3646	/** @todo Performance: We've done all this a jiffy ago in the
3647	* PGMGstGetPage call. */
3648	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3649	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3650	const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
3651	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3652
3653	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3654	unsigned iPDSrc = 0;
3655	X86PDPE PdpeSrc;
3656	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3657	if (RT_UNLIKELY(!pPDSrc))
3658	{
3659	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3660	return VINF_EM_RAW_GUEST_TRAP;
3661	}
3662
3663	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3664	unsigned iPDSrc = 0; /* shut up gcc */
3665	PX86PML4E pPml4eSrc = NULL; /* ditto */
3666	X86PDPE PdpeSrc;
3667	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3668	if (RT_UNLIKELY(!pPDSrc))
3669	{
3670	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3671	return VINF_EM_RAW_GUEST_TRAP;
3672	}
3673	# endif
3674
3675	# else /* !PGM_WITH_PAGING */
3676	PGSTPD pPDSrc = NULL;
3677	const unsigned iPDSrc = 0;
3678	# endif /* !PGM_WITH_PAGING */
3679	int rc = VINF_SUCCESS;
3680
3681	pgmLock(pVM);
3682
3683	/*
3684	* First check if the shadow pd is present.
3685	*/
3686	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3687	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3688
3689	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3690	PX86PDEPAE pPdeDst;
3691	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3692	PX86PDPAE pPDDst;
3693	# if PGM_GST_TYPE != PGM_TYPE_PAE
3694	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3695	X86PDPE PdpeSrc;
3696	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3697	# endif
3698	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3699	if (rc != VINF_SUCCESS)
3700	{
3701	pgmUnlock(pVM);
3702	AssertRC(rc);
3703	return rc;
3704	}
3705	Assert(pPDDst);
3706	pPdeDst = &pPDDst->a[iPDDst];
3707
3708	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3709	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3710	PX86PDPAE pPDDst;
3711	PX86PDEPAE pPdeDst;
3712
3713	# if PGM_GST_TYPE == PGM_TYPE_PROT
3714	/* AMD-V nested paging: Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3715	X86PML4E Pml4eSrc;
3716	X86PDPE PdpeSrc;
3717	PX86PML4E pPml4eSrc = &Pml4eSrc;
3718	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
3719	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
3720	# endif
3721
3722	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3723	if (rc != VINF_SUCCESS)
3724	{
3725	pgmUnlock(pVM);
3726	AssertRC(rc);
3727	return rc;
3728	}
3729	Assert(pPDDst);
3730	pPdeDst = &pPDDst->a[iPDDst];
3731	# endif
3732
3733	if (!pPdeDst->n.u1Present)
3734	{
3735	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3736	if (rc != VINF_SUCCESS)
3737	{
3738	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3739	pgmUnlock(pVM);
3740	AssertRC(rc);
3741	return rc;
3742	}
3743	}
3744
3745	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3746	/* Check for dirty bit fault */
3747	rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
3748	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
3749	Log(("PGMVerifyAccess: success (dirty)\n"));
3750	else
3751	# endif
3752	{
3753	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3754	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3755	# else
3756	GSTPDE PdeSrc;
3757	PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3758	PdeSrc.n.u1Present = 1;
3759	PdeSrc.n.u1Write = 1;
3760	PdeSrc.n.u1Accessed = 1;
3761	PdeSrc.n.u1User = 1;
3762	# endif
3763
3764	Assert(rc != VINF_EM_RAW_GUEST_TRAP);
3765	if (uErr & X86_TRAP_PF_US)
3766	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUser));
3767	else /* supervisor */
3768	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
3769
3770	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3771	if (RT_SUCCESS(rc))
3772	{
3773	/* Page was successfully synced */
3774	Log2(("PGMVerifyAccess: success (sync)\n"));
3775	rc = VINF_SUCCESS;
3776	}
3777	else
3778	{
3779	Log(("PGMVerifyAccess: access violation for %RGv rc=%Rrc\n", GCPtrPage, rc));
3780	rc = VINF_EM_RAW_GUEST_TRAP;
3781	}
3782	}
3783	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3784	pgmUnlock(pVM);
3785	return rc;
3786
3787	#else /* PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) */
3788
3789	AssertLogRelMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
3790	return VERR_PGM_NOT_USED_IN_MODE;
3791	#endif /* PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) */
3792	}
3793
3794
3795	/**
3796	* Syncs the paging hierarchy starting at CR3.
3797	*
3798	* @returns VBox status code, R0/RC may return VINF_PGM_SYNC_CR3, no other
3799	* informational status codes.
3800	* @retval VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map
3801	* the VMM into guest context.
3802	* @param pVCpu The cross context virtual CPU structure.
3803	* @param cr0 Guest context CR0 register.
3804	* @param cr3 Guest context CR3 register. Not subjected to the A20
3805	* mask.
3806	* @param cr4 Guest context CR4 register.
3807	* @param fGlobal Including global page directories or not
3808	*/
3809	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
3810	{
3811	PVM pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
3812	NOREF(cr0); NOREF(cr3); NOREF(cr4); NOREF(fGlobal);
3813
3814	LogFlow(("SyncCR3 FF=%d fGlobal=%d\n", !!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), fGlobal));
3815
3816	#if !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
3817
3818	pgmLock(pVM);
3819
3820	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
3821	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3822	if (pPool->cDirtyPages)
3823	pgmPoolResetDirtyPages(pVM);
3824	# endif
3825
3826	/*
3827	* Update page access handlers.
3828	* The virtual are always flushed, while the physical are only on demand.
3829	* WARNING: We are incorrectly not doing global flushing on Virtual Handler updates. We'll
3830	* have to look into that later because it will have a bad influence on the performance.
3831	* @note SvL: There's no need for that. Just invalidate the virtual range(s).
3832	* bird: Yes, but that won't work for aliases.
3833	*/
3834	/** @todo this MUST go away. See @bugref{1557}. */
3835	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3Handlers), h);
3836	PGM_GST_NAME(HandlerVirtualUpdate)(pVM, cr4);
3837	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3Handlers), h);
3838	pgmUnlock(pVM);
3839	#endif /* !NESTED && !EPT */
3840
3841	#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \|\| PGM_SHW_TYPE == PGM_TYPE_NONE
3842	/*
3843	* Nested / EPT / None - No work.
3844	*/
3845	Assert(!pgmMapAreMappingsEnabled(pVM));
3846	return VINF_SUCCESS;
3847
3848	#elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3849	/*
3850	* AMD64 (Shw & Gst) - No need to check all paging levels; we zero
3851	* out the shadow parts when the guest modifies its tables.
3852	*/
3853	Assert(!pgmMapAreMappingsEnabled(pVM));
3854	return VINF_SUCCESS;
3855
3856	#else /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3857
3858	# ifndef PGM_WITHOUT_MAPPINGS
3859	/*
3860	* Check for and resolve conflicts with our guest mappings if they
3861	* are enabled and not fixed.
3862	*/
3863	if (pgmMapAreMappingsFloating(pVM))
3864	{
3865	int rc = pgmMapResolveConflicts(pVM);
3866	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
3867	if (rc == VINF_SUCCESS)
3868	{ /* likely */ }
3869	else if (rc == VINF_PGM_SYNC_CR3)
3870	{
3871	LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
3872	return VINF_PGM_SYNC_CR3;
3873	}
3874	else if (RT_FAILURE(rc))
3875	return rc;
3876	else
3877	AssertMsgFailed(("%Rrc\n", rc));
3878	}
3879	# else
3880	Assert(!pgmMapAreMappingsEnabled(pVM));
3881	# endif
3882	return VINF_SUCCESS;
3883	#endif /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3884	}
3885
3886
3887
3888
3889	#ifdef VBOX_STRICT
3890	# ifdef IN_RC
3891	# undef AssertMsgFailed
3892	# define AssertMsgFailed Log
3893	# endif
3894
3895	/**
3896	* Checks that the shadow page table is in sync with the guest one.
3897	*
3898	* @returns The number of errors.
3899	* @param pVCpu The cross context virtual CPU structure.
3900	* @param cr3 Guest context CR3 register.
3901	* @param cr4 Guest context CR4 register.
3902	* @param GCPtr Where to start. Defaults to 0.
3903	* @param cb How much to check. Defaults to everything.
3904	*/
3905	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)
3906	{
3907	NOREF(pVCpu); NOREF(cr3); NOREF(cr4); NOREF(GCPtr); NOREF(cb);
3908	#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \|\| PGM_SHW_TYPE == PGM_TYPE_NONE
3909	return 0;
3910	#else
3911	unsigned cErrors = 0;
3912	PVM pVM = pVCpu->CTX_SUFF(pVM);
3913	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
3914
3915	# if PGM_GST_TYPE == PGM_TYPE_PAE
3916	/** @todo currently broken; crashes below somewhere */
3917	AssertFailed();
3918	# endif
3919
3920	# if PGM_GST_TYPE == PGM_TYPE_32BIT \
3921	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3922	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
3923
3924	bool fBigPagesSupported = GST_IS_PSE_ACTIVE(pVCpu);
3925	PPGMCPU pPGM = &pVCpu->pgm.s;
3926	RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
3927	RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
3928	# ifndef IN_RING0
3929	RTHCPHYS HCPhys; /* general usage. */
3930	# endif
3931	int rc;
3932
3933	/*
3934	* Check that the Guest CR3 and all its mappings are correct.
3935	*/
3936	AssertMsgReturn(pPGM->GCPhysCR3 == PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK),
3937	("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
3938	false);
3939	# if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64
3940	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3941	rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGst32BitPdRC, NULL, &HCPhysShw);
3942	# else
3943	rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGstPaePdptRC, NULL, &HCPhysShw);
3944	# endif
3945	AssertRCReturn(rc, 1);
3946	HCPhys = NIL_RTHCPHYS;
3947	rc = pgmRamGCPhys2HCPhys(pVM, PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK), &HCPhys);
3948	AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
3949	# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
3950	pgmGstGet32bitPDPtr(pVCpu);
3951	RTGCPHYS GCPhys;
3952	rc = PGMR3DbgR3Ptr2GCPhys(pVM->pUVM, pPGM->pGst32BitPdR3, &GCPhys);
3953	AssertRCReturn(rc, 1);
3954	AssertMsgReturn(PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
3955	# endif
3956	# endif /* !IN_RING0 */
3957
3958	/*
3959	* Get and check the Shadow CR3.
3960	*/
3961	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3962	unsigned cPDEs = X86_PG_ENTRIES;
3963	unsigned cIncrement = X86_PG_ENTRIES * PAGE_SIZE;
3964	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3965	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3966	unsigned cPDEs = X86_PG_PAE_ENTRIES * 4; /* treat it as a 2048 entry table. */
3967	# else
3968	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3969	# endif
3970	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3971	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3972	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3973	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3974	# endif
3975	if (cb != ~(RTGCPTR)0)
3976	cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
3977
3978	/** @todo call the other two PGMAssert() functions. /
3979
3980	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3981	unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3982
3983	for (; iPml4 < X86_PG_PAE_ENTRIES; iPml4++)
3984	{
3985	PPGMPOOLPAGE pShwPdpt = NULL;
3986	PX86PML4E pPml4eSrc;
3987	PX86PML4E pPml4eDst;
3988	RTGCPHYS GCPhysPdptSrc;
3989
3990	pPml4eSrc = pgmGstGetLongModePML4EPtr(pVCpu, iPml4);
3991	pPml4eDst = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
3992
3993	/* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
3994	if (!pPml4eDst->n.u1Present)
3995	{
3996	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3997	continue;
3998	}
3999
4000	pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
4001	GCPhysPdptSrc = PGM_A20_APPLY(pVCpu, pPml4eSrc->u & X86_PML4E_PG_MASK);
4002
4003	if (pPml4eSrc->n.u1Present != pPml4eDst->n.u1Present)
4004	{
4005	AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
4006	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
4007	cErrors++;
4008	continue;
4009	}
4010
4011	if (GCPhysPdptSrc != pShwPdpt->GCPhys)
4012	{
4013	AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
4014	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
4015	cErrors++;
4016	continue;
4017	}
4018
4019	if ( pPml4eDst->n.u1User != pPml4eSrc->n.u1User
4020	\|\| pPml4eDst->n.u1Write != pPml4eSrc->n.u1Write
4021	\|\| pPml4eDst->n.u1NoExecute != pPml4eSrc->n.u1NoExecute)
4022	{
4023	AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
4024	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
4025	cErrors++;
4026	continue;
4027	}
4028	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
4029	{
4030	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
4031
4032	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
4033	/*
4034	* Check the PDPTEs too.
4035	*/
4036	unsigned iPdpt = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
4037
4038	for (;iPdpt <= SHW_PDPT_MASK; iPdpt++)
4039	{
4040	unsigned iPDSrc = 0; /* initialized to shut up gcc */
4041	PPGMPOOLPAGE pShwPde = NULL;
4042	PX86PDPE pPdpeDst;
4043	RTGCPHYS GCPhysPdeSrc;
4044	X86PDPE PdpeSrc;
4045	PdpeSrc.u = 0; /* initialized to shut up gcc 4.5 */
4046	# if PGM_GST_TYPE == PGM_TYPE_PAE
4047	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtr, &iPDSrc, &PdpeSrc);
4048	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
4049	# else
4050	PX86PML4E pPml4eSrcIgn;
4051	PX86PDPT pPdptDst;
4052	PX86PDPAE pPDDst;
4053	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtr, &pPml4eSrcIgn, &PdpeSrc, &iPDSrc);
4054
4055	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtr, NULL, &pPdptDst, &pPDDst);
4056	if (rc != VINF_SUCCESS)
4057	{
4058	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
4059	GCPtr += 512 * _2M;
4060	continue; /* next PDPTE */
4061	}
4062	Assert(pPDDst);
4063	# endif
4064	Assert(iPDSrc == 0);
4065
4066	pPdpeDst = &pPdptDst->a[iPdpt];
4067
4068	if (!pPdpeDst->n.u1Present)
4069	{
4070	GCPtr += 512 * _2M;
4071	continue; /* next PDPTE */
4072	}
4073
4074	pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
4075	GCPhysPdeSrc = PGM_A20_APPLY(pVCpu, PdpeSrc.u & X86_PDPE_PG_MASK);
4076
4077	if (pPdpeDst->n.u1Present != PdpeSrc.n.u1Present)
4078	{
4079	AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
4080	GCPtr += 512 * _2M;
4081	cErrors++;
4082	continue;
4083	}
4084
4085	if (GCPhysPdeSrc != pShwPde->GCPhys)
4086	{
4087	# if PGM_GST_TYPE == PGM_TYPE_AMD64
4088	AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
4089	# else
4090	AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
4091	# endif
4092	GCPtr += 512 * _2M;
4093	cErrors++;
4094	continue;
4095	}
4096
4097	# if PGM_GST_TYPE == PGM_TYPE_AMD64
4098	if ( pPdpeDst->lm.u1User != PdpeSrc.lm.u1User
4099	\|\| pPdpeDst->lm.u1Write != PdpeSrc.lm.u1Write
4100	\|\| pPdpeDst->lm.u1NoExecute != PdpeSrc.lm.u1NoExecute)
4101	{
4102	AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
4103	GCPtr += 512 * _2M;
4104	cErrors++;
4105	continue;
4106	}
4107	# endif
4108
4109	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
4110	{
4111	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
4112	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4113	GSTPD const *pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
4114	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
4115	PCX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
4116	# endif
4117	# endif /* PGM_GST_TYPE == PGM_TYPE_32BIT */
4118	/*
4119	* Iterate the shadow page directory.
4120	*/
4121	GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
4122	unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
4123
4124	for (;
4125	iPDDst < cPDEs;
4126	iPDDst++, GCPtr += cIncrement)
4127	{
4128	# if PGM_SHW_TYPE == PGM_TYPE_PAE
4129	const SHWPDE PdeDst = *pgmShwGetPaePDEPtr(pVCpu, GCPtr);
4130	# else
4131	const SHWPDE PdeDst = pPDDst->a[iPDDst];
4132	# endif
4133	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
4134	{
4135	Assert(pgmMapAreMappingsEnabled(pVM));
4136	if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
4137	{
4138	AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
4139	cErrors++;
4140	continue;
4141	}
4142	}
4143	else if ( (PdeDst.u & X86_PDE_P)
4144	\|\| ((PdeDst.u & (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY))
4145	)
4146	{
4147	HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
4148	PPGMPOOLPAGE pPoolPage = pgmPoolGetPage(pPool, HCPhysShw);
4149	if (!pPoolPage)
4150	{
4151	AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n",
4152	HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
4153	cErrors++;
4154	continue;
4155	}
4156	const SHWPT pPTDst = (const SHWPT )PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pPoolPage);
4157
4158	if (PdeDst.u & (X86_PDE4M_PWT \| X86_PDE4M_PCD))
4159	{
4160	AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
4161	GCPtr, (uint64_t)PdeDst.u));
4162	cErrors++;
4163	}
4164
4165	if (PdeDst.u & (X86_PDE4M_G \| X86_PDE4M_D))
4166	{
4167	AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n",
4168	GCPtr, (uint64_t)PdeDst.u));
4169	cErrors++;
4170	}
4171
4172	const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
4173	if (!PdeSrc.n.u1Present)
4174	{
4175	AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
4176	GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
4177	cErrors++;
4178	continue;
4179	}
4180
4181	if ( !PdeSrc.b.u1Size
4182	\|\| !fBigPagesSupported)
4183	{
4184	GCPhysGst = GST_GET_PDE_GCPHYS(PdeSrc);
4185	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
4186	GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst \| ((iPDDst & 1) * (PAGE_SIZE / 2)));
4187	# endif
4188	}
4189	else
4190	{
4191	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4192	if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
4193	{
4194	AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n",
4195	GCPtr, (uint64_t)PdeSrc.u));
4196	cErrors++;
4197	continue;
4198	}
4199	# endif
4200	GCPhysGst = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
4201	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
4202	GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst \| (GCPtr & RT_BIT(X86_PAGE_2M_SHIFT)));
4203	# endif
4204	}
4205
4206	if ( pPoolPage->enmKind
4207	!= (!PdeSrc.b.u1Size \|\| !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
4208	{
4209	AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n",
4210	pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
4211	cErrors++;
4212	}
4213
4214	PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4215	if (!pPhysPage)
4216	{
4217	AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
4218	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
4219	cErrors++;
4220	continue;
4221	}
4222
4223	if (GCPhysGst != pPoolPage->GCPhys)
4224	{
4225	AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n",
4226	GCPhysGst, pPoolPage->GCPhys, GCPtr));
4227	cErrors++;
4228	continue;
4229	}
4230
4231	if ( !PdeSrc.b.u1Size
4232	\|\| !fBigPagesSupported)
4233	{
4234	/*
4235	* Page Table.
4236	*/
4237	const GSTPT *pPTSrc;
4238	rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, PGM_A20_APPLY(pVCpu, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1)),
4239	&pPTSrc);
4240	if (RT_FAILURE(rc))
4241	{
4242	AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
4243	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
4244	cErrors++;
4245	continue;
4246	}
4247	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/))
4248	!= (PdeDst.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/)))
4249	{
4250	/// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
4251	// (This problem will go away when/if we shadow multiple CR3s.)
4252	AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4253	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4254	cErrors++;
4255	continue;
4256	}
4257	if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4258	{
4259	AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n",
4260	GCPtr, (uint64_t)PdeDst.u));
4261	cErrors++;
4262	continue;
4263	}
4264
4265	/* iterate the page table. */
4266	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
4267	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
4268	const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
4269	# else
4270	const unsigned offPTSrc = 0;
4271	# endif
4272	for (unsigned iPT = 0, off = 0;
4273	iPT < RT_ELEMENTS(pPTDst->a);
4274	iPT++, off += PAGE_SIZE)
4275	{
4276	const SHWPTE PteDst = pPTDst->a[iPT];
4277
4278	/* skip not-present and dirty tracked entries. */
4279	if (!(SHW_PTE_GET_U(PteDst) & (X86_PTE_P \| PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
4280	continue;
4281	Assert(SHW_PTE_IS_P(PteDst));
4282
4283	const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
4284	if (!PteSrc.n.u1Present)
4285	{
4286	# ifdef IN_RING3
4287	PGMAssertHandlerAndFlagsInSync(pVM);
4288	DBGFR3PagingDumpEx(pVM->pUVM, pVCpu->idCpu, DBGFPGDMP_FLAGS_CURRENT_CR3 \| DBGFPGDMP_FLAGS_CURRENT_MODE
4289	\| DBGFPGDMP_FLAGS_GUEST \| DBGFPGDMP_FLAGS_HEADER \| DBGFPGDMP_FLAGS_PRINT_CR3,
4290	0, 0, UINT64_MAX, 99, NULL);
4291	# endif
4292	AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
4293	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst), pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
4294	(uint64_t)GST_GET_PDE_GCPHYS(PdeSrc) + (iPT + offPTSrc) * sizeof(PteSrc)));
4295	cErrors++;
4296	continue;
4297	}
4298
4299	uint64_t fIgnoreFlags = GST_PTE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_G \| X86_PTE_D \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
4300	# if 1 /** @todo sync accessed bit properly... */
4301	fIgnoreFlags \|= X86_PTE_A;
4302	# endif
4303
4304	/* match the physical addresses */
4305	HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
4306	GCPhysGst = GST_GET_PTE_GCPHYS(PteSrc);
4307
4308	# ifdef IN_RING3
4309	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4310	if (RT_FAILURE(rc))
4311	{
4312	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4313	{
4314	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
4315	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4316	cErrors++;
4317	continue;
4318	}
4319	}
4320	else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
4321	{
4322	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4323	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4324	cErrors++;
4325	continue;
4326	}
4327	# endif
4328
4329	pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4330	if (!pPhysPage)
4331	{
4332	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4333	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4334	{
4335	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
4336	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4337	cErrors++;
4338	continue;
4339	}
4340	# endif
4341	if (SHW_PTE_IS_RW(PteDst))
4342	{
4343	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4344	GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4345	cErrors++;
4346	}
4347	fIgnoreFlags \|= X86_PTE_RW;
4348	}
4349	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4350	{
4351	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4352	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4353	cErrors++;
4354	continue;
4355	}
4356
4357	/* flags */
4358	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4359	{
4360	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4361	{
4362	if (SHW_PTE_IS_RW(PteDst))
4363	{
4364	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4365	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4366	cErrors++;
4367	continue;
4368	}
4369	fIgnoreFlags \|= X86_PTE_RW;
4370	}
4371	else
4372	{
4373	if ( SHW_PTE_IS_P(PteDst)
4374	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
4375	&& !PGM_PAGE_IS_MMIO(pPhysPage)
4376	# endif
4377	)
4378	{
4379	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4380	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4381	cErrors++;
4382	continue;
4383	}
4384	fIgnoreFlags \|= X86_PTE_P;
4385	}
4386	}
4387	else
4388	{
4389	if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
4390	{
4391	if (SHW_PTE_IS_RW(PteDst))
4392	{
4393	AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
4394	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4395	cErrors++;
4396	continue;
4397	}
4398	if (!SHW_PTE_IS_TRACK_DIRTY(PteDst))
4399	{
4400	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4401	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4402	cErrors++;
4403	continue;
4404	}
4405	if (SHW_PTE_IS_D(PteDst))
4406	{
4407	AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4408	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4409	cErrors++;
4410	}
4411	# if 0 /** @todo sync access bit properly... */
4412	if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
4413	{
4414	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4415	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4416	cErrors++;
4417	}
4418	fIgnoreFlags \|= X86_PTE_RW;
4419	# else
4420	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
4421	# endif
4422	}
4423	else if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
4424	{
4425	/* access bit emulation (not implemented). */
4426	if (PteSrc.n.u1Accessed \|\| SHW_PTE_IS_P(PteDst))
4427	{
4428	AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
4429	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4430	cErrors++;
4431	continue;
4432	}
4433	if (!SHW_PTE_IS_A(PteDst))
4434	{
4435	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
4436	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4437	cErrors++;
4438	}
4439	fIgnoreFlags \|= X86_PTE_P;
4440	}
4441	# ifdef DEBUG_sandervl
4442	fIgnoreFlags \|= X86_PTE_D \| X86_PTE_A;
4443	# endif
4444	}
4445
4446	if ( (PteSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4447	&& (PteSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4448	)
4449	{
4450	AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
4451	GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
4452	fIgnoreFlags, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4453	cErrors++;
4454	continue;
4455	}
4456	} /* foreach PTE */
4457	}
4458	else
4459	{
4460	/*
4461	* Big Page.
4462	*/
4463	uint64_t fIgnoreFlags = X86_PDE_AVL_MASK \| GST_PDE_PG_MASK \| X86_PDE4M_G \| X86_PDE4M_D \| X86_PDE4M_PS \| X86_PDE4M_PWT \| X86_PDE4M_PCD;
4464	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
4465	{
4466	if (PdeDst.n.u1Write)
4467	{
4468	AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4469	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4470	cErrors++;
4471	continue;
4472	}
4473	if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
4474	{
4475	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4476	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4477	cErrors++;
4478	continue;
4479	}
4480	# if 0 /** @todo sync access bit properly... */
4481	if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
4482	{
4483	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4484	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4485	cErrors++;
4486	}
4487	fIgnoreFlags \|= X86_PTE_RW;
4488	# else
4489	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
4490	# endif
4491	}
4492	else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4493	{
4494	/* access bit emulation (not implemented). */
4495	if (PdeSrc.b.u1Accessed \|\| PdeDst.n.u1Present)
4496	{
4497	AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4498	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4499	cErrors++;
4500	continue;
4501	}
4502	if (!PdeDst.n.u1Accessed)
4503	{
4504	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4505	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4506	cErrors++;
4507	}
4508	fIgnoreFlags \|= X86_PTE_P;
4509	}
4510
4511	if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
4512	{
4513	AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
4514	GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
4515	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4516	cErrors++;
4517	}
4518
4519	/* iterate the page table. */
4520	for (unsigned iPT = 0, off = 0;
4521	iPT < RT_ELEMENTS(pPTDst->a);
4522	iPT++, off += PAGE_SIZE, GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst + PAGE_SIZE))
4523	{
4524	const SHWPTE PteDst = pPTDst->a[iPT];
4525
4526	if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
4527	{
4528	AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
4529	GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4530	cErrors++;
4531	}
4532
4533	/* skip not-present entries. */
4534	if (!SHW_PTE_IS_P(PteDst)) /** @todo deal with ALL handlers and CSAM !P pages! */
4535	continue;
4536
4537	fIgnoreFlags = X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT \| X86_PTE_D \| X86_PTE_A \| X86_PTE_G \| X86_PTE_PAE_NX;
4538
4539	/* match the physical addresses */
4540	HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
4541
4542	# ifdef IN_RING3
4543	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4544	if (RT_FAILURE(rc))
4545	{
4546	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4547	{
4548	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4549	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4550	cErrors++;
4551	}
4552	}
4553	else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
4554	{
4555	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4556	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4557	cErrors++;
4558	continue;
4559	}
4560	# endif
4561	pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4562	if (!pPhysPage)
4563	{
4564	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4565	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4566	{
4567	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4568	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4569	cErrors++;
4570	continue;
4571	}
4572	# endif
4573	if (SHW_PTE_IS_RW(PteDst))
4574	{
4575	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4576	GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4577	cErrors++;
4578	}
4579	fIgnoreFlags \|= X86_PTE_RW;
4580	}
4581	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4582	{
4583	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4584	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4585	cErrors++;
4586	continue;
4587	}
4588
4589	/* flags */
4590	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4591	{
4592	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4593	{
4594	if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
4595	{
4596	if (SHW_PTE_IS_RW(PteDst))
4597	{
4598	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4599	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4600	cErrors++;
4601	continue;
4602	}
4603	fIgnoreFlags \|= X86_PTE_RW;
4604	}
4605	}
4606	else
4607	{
4608	if ( SHW_PTE_IS_P(PteDst)
4609	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
4610	&& !PGM_PAGE_IS_MMIO(pPhysPage)
4611	# endif
4612	)
4613	{
4614	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4615	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4616	cErrors++;
4617	continue;
4618	}
4619	fIgnoreFlags \|= X86_PTE_P;
4620	}
4621	}
4622
4623	if ( (PdeSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4624	&& (PdeSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags) /* lazy phys handler dereg. */
4625	)
4626	{
4627	AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
4628	GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
4629	fIgnoreFlags, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4630	cErrors++;
4631	continue;
4632	}
4633	} /* for each PTE */
4634	}
4635	}
4636	/* not present */
4637
4638	} /* for each PDE */
4639
4640	} /* for each PDPTE */
4641
4642	} /* for each PML4E */
4643
4644	# ifdef DEBUG
4645	if (cErrors)
4646	LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
4647	# endif
4648	# endif /* GST is in {32BIT, PAE, AMD64} */
4649	return cErrors;
4650	#endif /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE */
4651	}
4652	#endif /* VBOX_STRICT */
4653
4654
4655	/**
4656	* Sets up the CR3 for shadow paging
4657	*
4658	* @returns Strict VBox status code.
4659	* @retval VINF_SUCCESS.
4660	*
4661	* @param pVCpu The cross context virtual CPU structure.
4662	* @param GCPhysCR3 The physical address in the CR3 register. (A20
4663	* mask already applied.)
4664	*/
4665	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3)
4666	{
4667	PVM pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
4668
4669	/* Update guest paging info. */
4670	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
4671	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
4672	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
4673
4674	LogFlow(("MapCR3: %RGp\n", GCPhysCR3));
4675	PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
4676
4677	/*
4678	* Map the page CR3 points at.
4679	*/
4680	RTHCPTR HCPtrGuestCR3;
4681	RTHCPHYS HCPhysGuestCR3;
4682	pgmLock(pVM);
4683	PPGMPAGE pPageCR3 = pgmPhysGetPage(pVM, GCPhysCR3);
4684	AssertReturn(pPageCR3, VERR_PGM_INVALID_CR3_ADDR);
4685	HCPhysGuestCR3 = PGM_PAGE_GET_HCPHYS(pPageCR3);
4686	/** @todo this needs some reworking wrt. locking? */
4687	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4688	HCPtrGuestCR3 = NIL_RTHCPTR;
4689	int rc = VINF_SUCCESS;
4690	# else
4691	int rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPageCR3, GCPhysCR3 & GST_CR3_PAGE_MASK, (void )&HCPtrGuestCR3); / @todo r=bird: This GCPhysCR3 masking isn't necessary. */
4692	# endif
4693	pgmUnlock(pVM);
4694	if (RT_SUCCESS(rc))
4695	{
4696	rc = PGMMap(pVM, (RTGCPTR)pVM->pgm.s.GCPtrCR3Mapping, HCPhysGuestCR3, PAGE_SIZE, 0);
4697	if (RT_SUCCESS(rc))
4698	{
4699	# ifdef IN_RC
4700	PGM_INVL_PG(pVCpu, pVM->pgm.s.GCPtrCR3Mapping);
4701	# endif
4702	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4703	pVCpu->pgm.s.pGst32BitPdR3 = (R3PTRTYPE(PX86PD))HCPtrGuestCR3;
4704	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4705	pVCpu->pgm.s.pGst32BitPdR0 = (R0PTRTYPE(PX86PD))HCPtrGuestCR3;
4706	# endif
4707	pVCpu->pgm.s.pGst32BitPdRC = (RCPTRTYPE(PX86PD))(RTRCUINTPTR)pVM->pgm.s.GCPtrCR3Mapping;
4708
4709	# elif PGM_GST_TYPE == PGM_TYPE_PAE
4710	unsigned off = GCPhysCR3 & GST_CR3_PAGE_MASK & PAGE_OFFSET_MASK;
4711	pVCpu->pgm.s.pGstPaePdptR3 = (R3PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4712	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4713	pVCpu->pgm.s.pGstPaePdptR0 = (R0PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4714	# endif
4715	pVCpu->pgm.s.pGstPaePdptRC = (RCPTRTYPE(PX86PDPT))((RTRCUINTPTR)pVM->pgm.s.GCPtrCR3Mapping + off);
4716	LogFlow(("Cached mapping %RRv\n", pVCpu->pgm.s.pGstPaePdptRC));
4717
4718	/*
4719	* Map the 4 PDs too.
4720	*/
4721	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pVCpu);
4722	RTGCPTR GCPtr = pVM->pgm.s.GCPtrCR3Mapping + PAGE_SIZE;
4723	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++, GCPtr += PAGE_SIZE)
4724	{
4725	pVCpu->pgm.s.aGstPaePdpeRegs[i].u = pGuestPDPT->a[i].u;
4726	if (pGuestPDPT->a[i].n.u1Present)
4727	{
4728	RTHCPTR HCPtr;
4729	RTHCPHYS HCPhys;
4730	RTGCPHYS GCPhys = PGM_A20_APPLY(pVCpu, pGuestPDPT->a[i].u & X86_PDPE_PG_MASK);
4731	pgmLock(pVM);
4732	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
4733	AssertReturn(pPage, VERR_PGM_INVALID_PDPE_ADDR);
4734	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
4735	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4736	HCPtr = NIL_RTHCPTR;
4737	int rc2 = VINF_SUCCESS;
4738	# else
4739	int rc2 = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)&HCPtr);
4740	# endif
4741	pgmUnlock(pVM);
4742	if (RT_SUCCESS(rc2))
4743	{
4744	rc = PGMMap(pVM, GCPtr, HCPhys, PAGE_SIZE, 0);
4745	AssertRCReturn(rc, rc);
4746
4747	pVCpu->pgm.s.apGstPaePDsR3[i] = (R3PTRTYPE(PX86PDPAE))HCPtr;
4748	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4749	pVCpu->pgm.s.apGstPaePDsR0[i] = (R0PTRTYPE(PX86PDPAE))HCPtr;
4750	# endif
4751	pVCpu->pgm.s.apGstPaePDsRC[i] = (RCPTRTYPE(PX86PDPAE))(RTRCUINTPTR)GCPtr;
4752	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
4753	# ifdef IN_RC
4754	PGM_INVL_PG(pVCpu, GCPtr);
4755	# endif
4756	continue;
4757	}
4758	AssertMsgFailed(("pgmR3Gst32BitMapCR3: rc2=%d GCPhys=%RGp i=%d\n", rc2, GCPhys, i));
4759	}
4760
4761	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4762	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4763	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4764	# endif
4765	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4766	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4767	# ifdef IN_RC
4768	PGM_INVL_PG(pVCpu, GCPtr); /** @todo this shouldn't be necessary? */
4769	# endif
4770	}
4771
4772	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
4773	pVCpu->pgm.s.pGstAmd64Pml4R3 = (R3PTRTYPE(PX86PML4))HCPtrGuestCR3;
4774	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4775	pVCpu->pgm.s.pGstAmd64Pml4R0 = (R0PTRTYPE(PX86PML4))HCPtrGuestCR3;
4776	# endif
4777	# endif
4778	}
4779	else
4780	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4781	}
4782	else
4783	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4784
4785	#else /* prot/real stub */
4786	int rc = VINF_SUCCESS;
4787	#endif
4788
4789	/*
4790	* Update shadow paging info for guest modes with paging (32-bit, PAE, AMD64).
4791	*/
4792	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4793	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4794	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64) \
4795	&& ( PGM_GST_TYPE != PGM_TYPE_REAL \
4796	&& PGM_GST_TYPE != PGM_TYPE_PROT))
4797
4798	Assert(!pVM->pgm.s.fNestedPaging);
4799	PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
4800
4801	/*
4802	* Update the shadow root page as well since that's not fixed.
4803	*/
4804	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4805	PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
4806	PPGMPOOLPAGE pNewShwPageCR3;
4807
4808	pgmLock(pVM);
4809
4810	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4811	if (pPool->cDirtyPages)
4812	pgmPoolResetDirtyPages(pVM);
4813	# endif
4814
4815	Assert(!(GCPhysCR3 >> (PAGE_SHIFT + 32)));
4816	rc = pgmPoolAlloc(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
4817	NIL_PGMPOOL_IDX, UINT32_MAX, true /fLockPage/,
4818	&pNewShwPageCR3);
4819	AssertFatalRC(rc);
4820	rc = VINF_SUCCESS;
4821
4822	# ifdef IN_RC
4823	/*
4824	* WARNING! We can't deal with jumps to ring 3 in the code below as the
4825	* state will be inconsistent! Flush important things now while
4826	* we still can and then make sure there are no ring-3 calls.
4827	*/
4828	# ifdef VBOX_WITH_REM
4829	REMNotifyHandlerPhysicalFlushIfAlmostFull(pVM, pVCpu);
4830	# endif
4831	VMMRZCallRing3Disable(pVCpu);
4832	# endif
4833
4834	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3) = pNewShwPageCR3;
4835	# ifdef IN_RING0
4836	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4837	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4838	# elif defined(IN_RC)
4839	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4840	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4841	# else
4842	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4843	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4844	# endif
4845
4846	# ifndef PGM_WITHOUT_MAPPINGS
4847	/*
4848	* Apply all hypervisor mappings to the new CR3.
4849	* Note that SyncCR3 will be executed in case CR3 is changed in a guest paging mode; this will
4850	* make sure we check for conflicts in the new CR3 root.
4851	*/
4852	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4853	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL) \|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
4854	# endif
4855	rc = pgmMapActivateCR3(pVM, pNewShwPageCR3);
4856	AssertRCReturn(rc, rc);
4857	# endif
4858
4859	/* Set the current hypervisor CR3. */
4860	CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
4861	SELMShadowCR3Changed(pVM, pVCpu);
4862
4863	# ifdef IN_RC
4864	/* NOTE: The state is consistent again. */
4865	VMMRZCallRing3Enable(pVCpu);
4866	# endif
4867
4868	/* Clean up the old CR3 root. */
4869	if ( pOldShwPageCR3
4870	&& pOldShwPageCR3 != pNewShwPageCR3 /* @todo can happen due to incorrect syncing between REM & PGM; find the real cause */)
4871	{
4872	Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
4873	# ifndef PGM_WITHOUT_MAPPINGS
4874	/* Remove the hypervisor mappings from the shadow page table. */
4875	pgmMapDeactivateCR3(pVM, pOldShwPageCR3);
4876	# endif
4877	/* Mark the page as unlocked; allow flushing again. */
4878	pgmPoolUnlockPage(pPool, pOldShwPageCR3);
4879
4880	pgmPoolFreeByPage(pPool, pOldShwPageCR3, NIL_PGMPOOL_IDX, UINT32_MAX);
4881	}
4882	pgmUnlock(pVM);
4883	# else
4884	NOREF(GCPhysCR3);
4885	# endif
4886
4887	return rc;
4888	}
4889
4890	/**
4891	* Unmaps the shadow CR3.
4892	*
4893	* @returns VBox status, no specials.
4894	* @param pVCpu The cross context virtual CPU structure.
4895	*/
4896	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu)
4897	{
4898	LogFlow(("UnmapCR3\n"));
4899
4900	int rc = VINF_SUCCESS;
4901	PVM pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
4902
4903	/*
4904	* Update guest paging info.
4905	*/
4906	#if PGM_GST_TYPE == PGM_TYPE_32BIT
4907	pVCpu->pgm.s.pGst32BitPdR3 = 0;
4908	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4909	pVCpu->pgm.s.pGst32BitPdR0 = 0;
4910	# endif
4911	pVCpu->pgm.s.pGst32BitPdRC = 0;
4912
4913	#elif PGM_GST_TYPE == PGM_TYPE_PAE
4914	pVCpu->pgm.s.pGstPaePdptR3 = 0;
4915	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4916	pVCpu->pgm.s.pGstPaePdptR0 = 0;
4917	# endif
4918	pVCpu->pgm.s.pGstPaePdptRC = 0;
4919	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4920	{
4921	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4922	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4923	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4924	# endif
4925	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4926	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4927	}
4928
4929	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
4930	pVCpu->pgm.s.pGstAmd64Pml4R3 = 0;
4931	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4932	pVCpu->pgm.s.pGstAmd64Pml4R0 = 0;
4933	# endif
4934
4935	#else /* prot/real mode stub */
4936	/* nothing to do */
4937	#endif
4938
4939	#if !defined(IN_RC) /* In RC we rely on MapCR3 to do the shadow part for us at a safe time */
4940	/*
4941	* Update shadow paging info.
4942	*/
4943	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4944	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4945	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64))
4946
4947	# if PGM_GST_TYPE != PGM_TYPE_REAL
4948	Assert(!pVM->pgm.s.fNestedPaging);
4949	# endif
4950
4951	pgmLock(pVM);
4952
4953	# ifndef PGM_WITHOUT_MAPPINGS
4954	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4955	/* Remove the hypervisor mappings from the shadow page table. */
4956	pgmMapDeactivateCR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4957	# endif
4958
4959	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4960	{
4961	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4962
4963	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4964	if (pPool->cDirtyPages)
4965	pgmPoolResetDirtyPages(pVM);
4966	# endif
4967
4968	/* Mark the page as unlocked; allow flushing again. */
4969	pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4970
4971	pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), NIL_PGMPOOL_IDX, UINT32_MAX);
4972	pVCpu->pgm.s.pShwPageCR3R3 = 0;
4973	pVCpu->pgm.s.pShwPageCR3R0 = 0;
4974	pVCpu->pgm.s.pShwPageCR3RC = 0;
4975	}
4976	pgmUnlock(pVM);
4977	# endif
4978	#endif /* !IN_RC*/
4979
4980	return rc;
4981	}
4982

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 74791

Download in other formats: