PGMAllBth.h@ 31443

Last change on this file since 31443 was 31443, checked in by vboxsync, 14 years ago
PGMAllBth.h: Must pass the RW flag to SyncHandlerPte. Some more logging.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 196.5 KB

Line
1	/* $Id: PGMAllBth.h 31443 2010-08-06 19:19:31Z vboxsync $ */
2	/** @file
3	* VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4	*
5	* @remarks The nested page tables on AMD makes use of PGM_SHW_TYPE in
6	* {PGM_TYPE_AMD64, PGM_TYPE_PAE and PGM_TYPE_32BIT} and PGM_GST_TYPE
7	* set to PGM_TYPE_PROT. Half of the code in this file is not
8	* exercised with PGM_SHW_TYPE set to PGM_TYPE_NESTED.
9	*
10	* @remarks Extended page tables (intel) are built with PGM_GST_TYPE set to
11	* PGM_TYPE_PROT (and PGM_SHW_TYPE set to PGM_TYPE_EPT).
12	*
13	* @remarks This file is one big \#ifdef-orgy!
14	*
15	*/
16
17	/*
18	* Copyright (C) 2006-2010 Oracle Corporation
19	*
20	* This file is part of VirtualBox Open Source Edition (OSE), as
21	* available from http://www.virtualbox.org. This file is free software;
22	* you can redistribute it and/or modify it under the terms of the GNU
23	* General Public License (GPL) as published by the Free Software
24	* Foundation, in version 2 as it comes in the "COPYING" file of the
25	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
26	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
27	*/
28
29
30	/*******************************************************************************
31	* Internal Functions *
32	*******************************************************************************/
33	RT_C_DECLS_BEGIN
34	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken);
35	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
36	static int PGM_BTH_NAME(SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr);
37	static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, RTGCPTR GCPtrPage);
38	static int PGM_BTH_NAME(SyncPT)(PVMCPU pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage);
39	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uErr);
40	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
41	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
42	#ifdef VBOX_STRICT
43	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
44	#endif
45	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3);
46	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu);
47	RT_C_DECLS_END
48
49
50	/*
51	* Filter out some illegal combinations of guest and shadow paging, so we can
52	* remove redundant checks inside functions.
53	*/
54	#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
55	# error "Invalid combination; PAE guest implies PAE shadow"
56	#endif
57
58	#if (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
59	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT)
60	# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
61	#endif
62
63	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) \
64	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT)
65	# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
66	#endif
67
68	#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT) \
69	\|\| (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
70	# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
71	#endif
72
73	#ifndef IN_RING3
74
75	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
76	/**
77	* Deal with a guest page fault.
78	*
79	* @returns Strict VBox status code.
80	* @retval VINF_EM_RAW_GUEST_TRAP
81	* @retval VINF_EM_RAW_EMULATE_INSTR
82	*
83	* @param pVCpu The current CPU.
84	* @param pGstWalk The guest page table walk result.
85	* @param uErr The error code.
86	*/
87	PGM_BTH_DECL(VBOXSTRICTRC, Trap0eHandlerGuestFault)(PVMCPU pVCpu, PGSTPTWALK pGstWalk, RTGCUINT uErr)
88	{
89	# if !defined(PGM_WITHOUT_MAPPINGS) && (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE)
90	/*
91	* Check for write conflicts with our hypervisor mapping.
92	*
93	* If the guest happens to access a non-present page, where our hypervisor
94	* is currently mapped, then we'll create a #PF storm in the guest.
95	*/
96	if ( (uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RW)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RW)
97	&& MMHyperIsInsideArea(pVCpu->CTX_SUFF(pVM), pGstWalk->Core.GCPtr))
98	{
99	/* Force a CR3 sync to check for conflicts and emulate the instruction. */
100	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
101	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
102	return VINF_EM_RAW_EMULATE_INSTR;
103	}
104	# endif
105
106	/*
107	* Calc the error code for the guest trap.
108	*/
109	uint32_t uNewErr = GST_IS_NX_ACTIVE(pVCpu)
110	? uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID)
111	: uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US);
112	if (pGstWalk->Core.fBadPhysAddr)
113	{
114	uNewErr \|= X86_TRAP_PF_RSVD \| X86_TRAP_PF_P;
115	Assert(!pGstWalk->Core.fNotPresent);
116	}
117	else if (!pGstWalk->Core.fNotPresent)
118	uNewErr \|= X86_TRAP_PF_P;
119	TRPMSetErrorCode(pVCpu, uNewErr);
120
121	LogFlow(("Guest trap; cr2=%RGv uErr=%RGv lvl=%d\n", pGstWalk->Core.GCPtr, uErr, pGstWalk->Core.uLevel));
122	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
123	return VINF_EM_RAW_GUEST_TRAP;
124	}
125	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
126
127
128	/**
129	* Deal with a guest page fault.
130	*
131	* @returns Strict VBox status code.
132	*
133	* @param pVCpu The current CPU.
134	* @param uErr The error code.
135	* @param pRegFrame The register frame.
136	* @param pvFault The fault address.
137	* @param pPage The guest page at @a pvFault.
138	* @param pGstWalk The guest page table walk result.
139	*/
140	static VBOXSTRICTRC PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame,
141	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
142	RTGCPTR pvFault, PPGMPAGE pPage, PGSTPTWALK pGstWalk)
143	# else
144	RTGCPTR pvFault, PPGMPAGE pPage)
145	# endif
146	{
147	# if !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
148	GSTPDE const PdeSrcDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A};
149	#endif
150	PVM pVM = pVCpu->CTX_SUFF(pVM);
151	int rc;
152
153	if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
154	{
155	/*
156	* Physical page access handler.
157	*/
158	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
159	const RTGCPHYS GCPhysFault = pGstWalk->Core.GCPhys;
160	# else
161	const RTGCPHYS GCPhysFault = (RTGCPHYS)pvFault;
162	# endif
163	PPGMPHYSHANDLER pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
164	if (pCur)
165	{
166	# ifdef PGM_SYNC_N_PAGES
167	/*
168	* If the region is write protected and we got a page not present fault, then sync
169	* the pages. If the fault was caused by a read, then restart the instruction.
170	* In case of write access continue to the GC write handler.
171	*
172	* ASSUMES that there is only one handler per page or that they have similar write properties.
173	*/
174	if ( pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
175	&& !(uErr & X86_TRAP_PF_P))
176	{
177	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
178	rc = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
179	# else
180	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
181	# endif
182	if ( RT_FAILURE(rc)
183	\|\| !(uErr & X86_TRAP_PF_RW)
184	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
185	{
186	AssertRC(rc);
187	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
188	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
189	return rc;
190	}
191	}
192	# endif
193
194	AssertMsg( pCur->enmType != PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
195	\|\| (pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE && (uErr & X86_TRAP_PF_RW)),
196	("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enum=%d\n",
197	pvFault, GCPhysFault, pPage, uErr, pCur->enmType));
198
199	# if defined(IN_RC) \|\| defined(IN_RING0) /** @todo remove this */
200	if (pCur->CTX_SUFF(pfnHandler))
201	{
202	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
203	# ifdef IN_RING0
204	PFNPGMR0PHYSHANDLER pfnHandler = pCur->CTX_SUFF(pfnHandler);
205	# else
206	PFNPGMRCPHYSHANDLER pfnHandler = pCur->CTX_SUFF(pfnHandler);
207	# endif
208	bool fLeaveLock = (pfnHandler != pPool->CTX_SUFF(pfnAccessHandler));
209	void *pvUser = pCur->CTX_SUFF(pvUser);
210
211	STAM_PROFILE_START(&pCur->Stat, h);
212	if (fLeaveLock)
213	pgmUnlock(pVM); /** @todo: Not entirely safe. */
214
215	rc = pfnHandler(pVM, uErr, pRegFrame, pvFault, GCPhysFault, pvUser);
216	if (fLeaveLock)
217	pgmLock(pVM);
218	# ifdef VBOX_WITH_STATISTICS
219	pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
220	if (pCur)
221	STAM_PROFILE_STOP(&pCur->Stat, h);
222	# else
223	pCur = NULL; /* might be invalid by now. */
224	# endif
225
226	}
227	else
228	# endif /* IN_RC \|\| IN_RING0 */
229	rc = VINF_EM_RAW_EMULATE_INSTR;
230
231	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysical);
232	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndPhys; });
233	return rc;
234	}
235	}
236	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
237	else
238	{
239	# ifdef PGM_SYNC_N_PAGES
240	/*
241	* If the region is write protected and we got a page not present fault, then sync
242	* the pages. If the fault was caused by a read, then restart the instruction.
243	* In case of write access continue to the GC write handler.
244	*/
245	if ( PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) < PGM_PAGE_HNDL_PHYS_STATE_ALL
246	&& !(uErr & X86_TRAP_PF_P))
247	{
248	rc = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
249	if ( RT_FAILURE(rc)
250	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
251	\|\| !(uErr & X86_TRAP_PF_RW))
252	{
253	AssertRC(rc);
254	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
255	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndVirt; });
256	return rc;
257	}
258	}
259	# endif
260	/*
261	* Ok, it's an virtual page access handler.
262	*
263	* Since it's faster to search by address, we'll do that first
264	* and then retry by GCPhys if that fails.
265	*/
266	/** @todo r=bird: perhaps we should consider looking up by physical address directly now?
267	* r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be
268	* out of sync, because the page was changed without us noticing it (not-present -> present
269	* without invlpg or mov cr3, xxx).
270	*/
271	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
272	if (pCur)
273	{
274	AssertMsg(!(pvFault - pCur->Core.Key < pCur->cb)
275	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
276	\|\| !(uErr & X86_TRAP_PF_P)
277	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
278	("Unexpected trap for virtual handler: %RGv (phys=%RGp) pPage=%R[pgmpage] uErr=%X, enum=%d\n",
279	pvFault, pGstWalk->Core.GCPhys, pPage, uErr, pCur->enmType));
280
281	if ( pvFault - pCur->Core.Key < pCur->cb
282	&& ( uErr & X86_TRAP_PF_RW
283	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
284	{
285	# ifdef IN_RC
286	STAM_PROFILE_START(&pCur->Stat, h);
287	pgmUnlock(pVM);
288	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
289	pgmLock(pVM);
290	STAM_PROFILE_STOP(&pCur->Stat, h);
291	# else
292	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
293	# endif
294	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersVirtual);
295	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
296	return rc;
297	}
298	/* Unhandled part of a monitored page */
299	}
300	else
301	{
302	/* Check by physical address. */
303	unsigned iPage;
304	rc = pgmHandlerVirtualFindByPhysAddr(pVM, pGstWalk->Core.GCPhys, &pCur, &iPage);
305	Assert(RT_SUCCESS(rc) \|\| !pCur);
306	if ( pCur
307	&& ( uErr & X86_TRAP_PF_RW
308	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
309	{
310	Assert((pCur->aPhysToVirt[iPage].Core.Key & X86_PTE_PAE_PG_MASK) == (pGstWalk->Core.GCPhys & X86_PTE_PAE_PG_MASK));
311	# ifdef IN_RC
312	RTGCPTR off = (iPage << PAGE_SHIFT) + (pvFault & PAGE_OFFSET_MASK) - (pCur->Core.Key & PAGE_OFFSET_MASK);
313	Assert(off < pCur->cb);
314	STAM_PROFILE_START(&pCur->Stat, h);
315	pgmUnlock(pVM);
316	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, off);
317	pgmLock(pVM);
318	STAM_PROFILE_STOP(&pCur->Stat, h);
319	# else
320	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
321	# endif
322	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersVirtualByPhys);
323	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
324	return rc;
325	}
326	}
327	}
328	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
329
330	/*
331	* There is a handled area of the page, but this fault doesn't belong to it.
332	* We must emulate the instruction.
333	*
334	* To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
335	* we first check if this was a page-not-present fault for a page with only
336	* write access handlers. Restart the instruction if it wasn't a write access.
337	*/
338	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersUnhandled);
339
340	if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
341	&& !(uErr & X86_TRAP_PF_P))
342	{
343	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
344	rc = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
345	# else
346	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
347	# endif
348	if ( RT_FAILURE(rc)
349	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
350	\|\| !(uErr & X86_TRAP_PF_RW))
351	{
352	AssertRC(rc);
353	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
354	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
355	return rc;
356	}
357	}
358
359	/** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
360	* It's writing to an unhandled part of the LDT page several million times.
361	*/
362	rc = PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault);
363	LogFlow(("PGM: PGMInterpretInstruction -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
364	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndUnhandled; });
365	return rc;
366	} /* if any kind of handler */
367
368
369	/**
370	* #PF Handler for raw-mode guest execution.
371	*
372	* @returns VBox status code (appropriate for trap handling and GC return).
373	*
374	* @param pVCpu VMCPU Handle.
375	* @param uErr The trap error code.
376	* @param pRegFrame Trap register frame.
377	* @param pvFault The fault address.
378	* @param pfLockTaken PGM lock taken here or not (out)
379	*/
380	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken)
381	{
382	PVM pVM = pVCpu->CTX_SUFF(pVM);
383
384	*pfLockTaken = false;
385
386	# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \
387	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
388	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
389	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT)
390	int rc;
391
392	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
393	/*
394	* Walk the guest page translation tables and check if it's a guest fault.
395	*/
396	GSTPTWALK GstWalk;
397	rc = PGM_GST_NAME(Walk)(pVCpu, pvFault, &GstWalk);
398	if (RT_FAILURE_NP(rc))
399	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &GstWalk, uErr));
400
401	/* assert some GstWalk sanity. */
402	# if PGM_GST_TYPE == PGM_TYPE_AMD64
403	AssertMsg(GstWalk.Pml4e.u == GstWalk.pPml4e->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pml4e.u, (uint64_t)GstWalk.pPml4e->u));
404	# endif
405	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
406	AssertMsg(GstWalk.Pdpe.u == GstWalk.pPdpe->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pdpe.u, (uint64_t)GstWalk.pPdpe->u));
407	# endif
408	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u));
409	AssertMsg(GstWalk.Core.fBigPage \|\| GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u));
410	Assert(GstWalk.Core.fSucceeded);
411
412	if (uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID))
413	{
414	if ( ( (uErr & X86_TRAP_PF_RW)
415	&& !GstWalk.Core.fEffectiveRW
416	&& ( (uErr & X86_TRAP_PF_US)
417	\|\| CPUMIsGuestR0WriteProtEnabled(pVCpu)) )
418	\|\| ((uErr & X86_TRAP_PF_US) && !GstWalk.Core.fEffectiveUS)
419	\|\| ((uErr & X86_TRAP_PF_ID) && GstWalk.Core.fEffectiveNX)
420	)
421	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &GstWalk, uErr));
422	}
423	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
424
425	# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
426	/*
427	* If it is a reserved bit fault we know that it is an MMIO or access
428	* handler related fault and can skip the dirty page stuff below.
429	*/
430	if (uErr & X86_TRAP_PF_RSVD)
431	{
432	/** @todo This is not complete code. take locks */
433	Assert(uErr & X86_TRAP_PF_P);
434	PPGMPAGE pPage;
435	/** @todo Only all physical access handlers here, so optimize further. */
436	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
437	rc = pgmPhysGetPageEx(&pVM->pgm.s, GstWalk.Core.GCPhys, &pPage);
438	if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
439	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
440	&GstWalk));
441	# else
442	rc = pgmPhysGetPageEx(&pVM->pgm.s, (RTGCPHYS)pvFault, &pPage);
443	if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
444	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage));
445	# endif
446	}
447	# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
448
449	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
450	/*
451	* Set the accessed and dirty flags.
452	*/
453	# if PGM_GST_TYPE == PGM_TYPE_AMD64
454	GstWalk.Pml4e.u \|= X86_PML4E_A;
455	GstWalk.pPml4e->u \|= X86_PML4E_A;
456	GstWalk.Pdpe.u \|= X86_PDPE_A;
457	GstWalk.pPdpe->u \|= X86_PDPE_A;
458	# endif
459	if (GstWalk.Core.fBigPage)
460	{
461	Assert(GstWalk.Pde.b.u1Size);
462	if (uErr & X86_TRAP_PF_RW)
463	{
464	GstWalk.Pde.u \|= X86_PDE4M_A \| X86_PDE4M_D;
465	GstWalk.pPde->u \|= X86_PDE4M_A \| X86_PDE4M_D;
466	}
467	else
468	{
469	GstWalk.Pde.u \|= X86_PDE4M_A;
470	GstWalk.pPde->u \|= X86_PDE4M_A;
471	}
472	}
473	else
474	{
475	Assert(!GstWalk.Pde.b.u1Size);
476	GstWalk.Pde.u \|= X86_PDE_A;
477	GstWalk.pPde->u \|= X86_PDE_A;
478	if (uErr & X86_TRAP_PF_RW)
479	{
480	# ifdef VBOX_WITH_STATISTICS
481	if (!GstWalk.Pte.n.u1Dirty)
482	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtiedPage));
483	else
484	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageAlreadyDirty));
485	# endif
486	GstWalk.Pte.u \|= X86_PTE_A \| X86_PTE_D;
487	GstWalk.pPte->u \|= X86_PTE_A \| X86_PTE_D;
488	}
489	else
490	{
491	GstWalk.Pte.u \|= X86_PTE_A;
492	GstWalk.pPte->u \|= X86_PTE_A;
493	}
494	Assert(GstWalk.Pte.u == GstWalk.pPte->u);
495	}
496	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u \|\| GstWalk.pPte->u == GstWalk.pPde->u,
497	("%RX64 %RX64 pPte=%p pPde=%p Pte=%RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u, GstWalk.pPte, GstWalk.pPde, (uint64_t)GstWalk.pPte->u));
498	# else /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
499	GSTPDE const PdeSrcDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A}; /** @todo eliminate this */
500	# endif /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
501
502	/* Take the big lock now. */
503	*pfLockTaken = true;
504	pgmLock(pVM);
505
506	/*
507	* Fetch the guest PDE, PDPE and PML4E.
508	*/
509	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
510	const unsigned iPDDst = pvFault >> SHW_PD_SHIFT;
511	PX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
512
513	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
514	const unsigned iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK; /* pPDDst index, not used with the pool. */
515	PX86PDPAE pPDDst;
516	# if PGM_GST_TYPE == PGM_TYPE_PAE
517	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, GstWalk.Pdpe.u, &pPDDst);
518	# else
519	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, X86_PDPE_P, &pPDDst); /* RW, US and A are reserved in PAE mode. */
520	# endif
521	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_INTERNAL_ERROR_4);
522
523	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
524	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
525	PX86PDPAE pPDDst;
526	# if PGM_GST_TYPE == PGM_TYPE_PROT /* (AMD-V nested paging) */
527	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A,
528	X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A, &pPDDst);
529	# else
530	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, GstWalk.Pml4e.u, GstWalk.Pdpe.u, &pPDDst);
531	# endif
532	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_INTERNAL_ERROR_4);
533
534	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
535	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
536	PEPTPD pPDDst;
537	rc = pgmShwGetEPTPDPtr(pVCpu, pvFault, NULL, &pPDDst);
538	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_INTERNAL_ERROR_4);
539	# endif
540	Assert(pPDDst);
541
542	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
543	/*
544	* Dirty page handling.
545	*
546	* If we successfully correct the write protection fault due to dirty bit
547	* tracking, then return immediately.
548	*/
549	if (uErr & X86_TRAP_PF_RW) /* write fault? */
550	{
551	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyBitTracking), a);
552	rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, &pPDDst->a[iPDDst], GstWalk.pPde, pvFault);
553	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyBitTracking), a);
554	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
555	{
556	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution)
557	= rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
558	? &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2DirtyAndAccessed
559	: &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
560	LogBird(("Trap0eHandler: returns VINF_SUCCESS\n"));
561	return VINF_SUCCESS;
562	}
563	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u \|\| GstWalk.pPte->u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u));
564	AssertMsg(GstWalk.Core.fBigPage \|\| GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u));
565	}
566
567	# if 0 /* rarely useful; leave for debugging. */
568	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0ePD[iPDSrc]);
569	# endif
570	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
571
572	/*
573	* A common case is the not-present error caused by lazy page table syncing.
574	*
575	* It is IMPORTANT that we weed out any access to non-present shadow PDEs
576	* here so we can safely assume that the shadow PT is present when calling
577	* SyncPage later.
578	*
579	* On failure, we ASSUME that SyncPT is out of memory or detected some kind
580	* of mapping conflict and defer to SyncCR3 in R3.
581	* (Again, we do NOT support access handlers for non-present guest pages.)
582	*
583	*/
584	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
585	Assert(GstWalk.Pde.n.u1Present);
586	# endif
587	if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
588	&& !pPDDst->a[iPDDst].n.u1Present)
589	{
590	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2SyncPT; });
591	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
592	LogFlow(("=>SyncPT %04x = %08RX64\n", (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, (uint64_t)GstWalk.Pde.u));
593	rc = PGM_BTH_NAME(SyncPT)(pVCpu, (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, GstWalk.pPd, pvFault);
594	# else
595	LogFlow(("=>SyncPT pvFault=%RGv\n", pvFault));
596	rc = PGM_BTH_NAME(SyncPT)(pVCpu, 0, NULL, pvFault);
597	# endif
598	if (RT_SUCCESS(rc))
599	return rc;
600	Log(("SyncPT: %RGv failed!! rc=%Rrc\n", pvFault, rc));
601	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
602	return VINF_PGM_SYNC_CR3;
603	}
604
605	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(PGM_WITHOUT_MAPPINGS)
606	/*
607	* Check if this address is within any of our mappings.
608	*
609	* This is very fast and it's gonna save us a bit of effort below and prevent
610	* us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
611	* (BTW, it's impossible to have physical access handlers in a mapping.)
612	*/
613	if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
614	{
615	PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
616	for ( ; pMapping; pMapping = pMapping->CTX_SUFF(pNext))
617	{
618	if (pvFault < pMapping->GCPtr)
619	break;
620	if (pvFault - pMapping->GCPtr < pMapping->cb)
621	{
622	/*
623	* The first thing we check is if we've got an undetected conflict.
624	*/
625	if (pgmMapAreMappingsFloating(&pVM->pgm.s))
626	{
627	unsigned iPT = pMapping->cb >> GST_PD_SHIFT;
628	while (iPT-- > 0)
629	if (GstWalk.pPde[iPT].n.u1Present)
630	{
631	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eConflicts);
632	Log(("Trap0e: Detected Conflict %RGv-%RGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
633	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
634	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
635	return VINF_PGM_SYNC_CR3;
636	}
637	}
638
639	/*
640	* Check if the fault address is in a virtual page access handler range.
641	*/
642	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->HyperVirtHandlers, pvFault);
643	if ( pCur
644	&& pvFault - pCur->Core.Key < pCur->cb
645	&& uErr & X86_TRAP_PF_RW)
646	{
647	# ifdef IN_RC
648	STAM_PROFILE_START(&pCur->Stat, h);
649	pgmUnlock(pVM);
650	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
651	pgmLock(pVM);
652	STAM_PROFILE_STOP(&pCur->Stat, h);
653	# else
654	AssertFailed();
655	rc = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
656	# endif
657	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersMapping);
658	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
659	return rc;
660	}
661
662	/*
663	* Pretend we're not here and let the guest handle the trap.
664	*/
665	TRPMSetErrorCode(pVCpu, uErr & ~X86_TRAP_PF_P);
666	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eGuestPFMapping);
667	LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
668	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
669	return VINF_EM_RAW_GUEST_TRAP;
670	}
671	}
672	} /* pgmAreMappingsEnabled(&pVM->pgm.s) */
673	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
674
675	/*
676	* Check if this fault address is flagged for special treatment,
677	* which means we'll have to figure out the physical address and
678	* check flags associated with it.
679	*
680	* ASSUME that we can limit any special access handling to pages
681	* in page tables which the guest believes to be present.
682	*/
683	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
684	RTGCPHYS GCPhys = GstWalk.Core.GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
685	# else
686	RTGCPHYS GCPhys = (RTGCPHYS)pvFault & ~(RTGCPHYS)PAGE_OFFSET_MASK;
687	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
688	PPGMPAGE pPage;
689	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
690	if (RT_FAILURE(rc))
691	{
692	/*
693	* When the guest accesses invalid physical memory (e.g. probing
694	* of RAM or accessing a remapped MMIO range), then we'll fall
695	* back to the recompiler to emulate the instruction.
696	*/
697	LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhys, rc));
698	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersInvalid);
699	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2InvalidPhys; });
700	return VINF_EM_RAW_EMULATE_INSTR;
701	}
702
703	/*
704	* Any handlers for this page?
705	*/
706	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
707	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
708	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, &GstWalk));
709	# else
710	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage));
711	# endif
712
713	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTimeOutOfSync, c);
714
715	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
716	if (uErr & X86_TRAP_PF_P)
717	{
718	/*
719	* The page isn't marked, but it might still be monitored by a virtual page access handler.
720	* (ASSUMES no temporary disabling of virtual handlers.)
721	*/
722	/** @todo r=bird: Since the purpose is to catch out of sync pages with virtual handler(s) here,
723	* we should correct both the shadow page table and physical memory flags, and not only check for
724	* accesses within the handler region but for access to pages with virtual handlers. */
725	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
726	if (pCur)
727	{
728	AssertMsg( !(pvFault - pCur->Core.Key < pCur->cb)
729	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
730	\|\| !(uErr & X86_TRAP_PF_P)
731	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
732	("Unexpected trap for virtual handler: %08X (phys=%08x) %R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
733
734	if ( pvFault - pCur->Core.Key < pCur->cb
735	&& ( uErr & X86_TRAP_PF_RW
736	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
737	{
738	# ifdef IN_RC
739	STAM_PROFILE_START(&pCur->Stat, h);
740	pgmUnlock(pVM);
741	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
742	pgmLock(pVM);
743	STAM_PROFILE_STOP(&pCur->Stat, h);
744	# else
745	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
746	# endif
747	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
748	return rc;
749	}
750	}
751	}
752	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
753
754	/*
755	* We are here only if page is present in Guest page tables and
756	* trap is not handled by our handlers.
757	*
758	* Check it for page out-of-sync situation.
759	*/
760	if (!(uErr & X86_TRAP_PF_P))
761	{
762	/*
763	* Page is not present in our page tables. Try to sync it!
764	*/
765	if (uErr & X86_TRAP_PF_US)
766	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUser));
767	else /* supervisor */
768	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
769
770	if (PGM_PAGE_IS_BALLOONED(pPage))
771	{
772	/* Emulate reads from ballooned pages as they are not present in
773	our shadow page tables. (Required for e.g. Solaris guests; soft
774	ecc, random nr generator.) */
775	rc = PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault);
776	LogFlow(("PGM: PGMInterpretInstruction balloon -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
777	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncBallloon));
778	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Ballooned; });
779	return rc;
780	}
781
782	# if defined(LOG_ENABLED) && !defined(IN_RING0)
783	RTGCPHYS GCPhys2;
784	uint64_t fPageGst2;
785	PGMGstGetPage(pVCpu, pvFault, &fPageGst2, &GCPhys2);
786	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
787	Log(("Page out of sync: %RGv eip=%08x PdeSrc.US=%d fPageGst2=%08llx GCPhys2=%RGp scan=%d\n",
788	pvFault, pRegFrame->eip, GstWalk.Pde.n.u1User, fPageGst2, GCPhys2, CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)));
789	# else
790	Log(("Page out of sync: %RGv eip=%08x fPageGst2=%08llx GCPhys2=%RGp scan=%d\n",
791	pvFault, pRegFrame->eip, fPageGst2, GCPhys2, CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)));
792	# endif
793	# endif /* LOG_ENABLED */
794
795	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
796	if ( !GstWalk.Core.fEffectiveUS
797	&& CPUMGetGuestCPL(pVCpu, pRegFrame) == 0)
798	{
799	/* Note: Can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU. */
800	if ( pvFault == (RTGCPTR)pRegFrame->eip
801	\|\| pvFault - pRegFrame->eip < 8 /* instruction crossing a page boundary */
802	# ifdef CSAM_DETECT_NEW_CODE_PAGES
803	\|\| ( !PATMIsPatchGCAddr(pVM, pRegFrame->eip)
804	&& CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)) /* any new code we encounter here */
805	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
806	)
807	{
808	LogFlow(("CSAMExecFault %RX32\n", pRegFrame->eip));
809	rc = CSAMExecFault(pVM, (RTRCPTR)pRegFrame->eip);
810	if (rc != VINF_SUCCESS)
811	{
812	/*
813	* CSAM needs to perform a job in ring 3.
814	*
815	* Sync the page before going to the host context; otherwise we'll end up in a loop if
816	* CSAM fails (e.g. instruction crosses a page boundary and the next page is not present)
817	*/
818	LogFlow(("CSAM ring 3 job\n"));
819	int rc2 = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
820	AssertRC(rc2);
821
822	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2CSAM; });
823	return rc;
824	}
825	}
826	# ifdef CSAM_DETECT_NEW_CODE_PAGES
827	else if ( uErr == X86_TRAP_PF_RW
828	&& pRegFrame->ecx >= 0x100 /* early check for movswd count */
829	&& pRegFrame->ecx < 0x10000)
830	{
831	/* In case of a write to a non-present supervisor shadow page, we'll take special precautions
832	* to detect loading of new code pages.
833	*/
834
835	/*
836	* Decode the instruction.
837	*/
838	RTGCPTR PC;
839	rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs,
840	&pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &PC);
841	if (rc == VINF_SUCCESS)
842	{
843	PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
844	uint32_t cbOp;
845	rc = EMInterpretDisasOneEx(pVM, pVCpu, PC, pRegFrame, pDis, &cbOp);
846
847	/* For now we'll restrict this to rep movsw/d instructions */
848	if ( rc == VINF_SUCCESS
849	&& pDis->pCurInstr->opcode == OP_MOVSWD
850	&& (pDis->prefix & PREFIX_REP))
851	{
852	CSAMMarkPossibleCodePage(pVM, pvFault);
853	}
854	}
855	}
856	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
857
858	/*
859	* Mark this page as safe.
860	*/
861	/** @todo not correct for pages that contain both code and data!! */
862	Log2(("CSAMMarkPage %RGv; scanned=%d\n", pvFault, true));
863	CSAMMarkPage(pVM, pvFault, true);
864	}
865	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0) */
866	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
867	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
868	# else
869	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
870	# endif
871	if (RT_SUCCESS(rc))
872	{
873	/* The page was successfully synced, return to the guest. */
874	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSync; });
875	return VINF_SUCCESS;
876	}
877	}
878	else /* uErr & X86_TRAP_PF_P: */
879	{
880	/*
881	* Write protected pages are made writable when the guest makes the
882	* first write to it. This happens for pages that are shared, write
883	* monitored or not yet allocated.
884	*
885	* We may also end up here when CR0.WP=0 in the guest.
886	*
887	* Also, a side effect of not flushing global PDEs are out of sync
888	* pages due to physical monitored regions, that are no longer valid.
889	* Assume for now it only applies to the read/write flag.
890	*/
891	if (uErr & X86_TRAP_PF_RW)
892	{
893	/*
894	* Check if it is a read-only page.
895	*/
896	if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
897	{
898	Log(("PGM #PF: Make writable: %RGp %R[pgmpage] pvFault=%RGp uErr=%#x\n", GCPhys, pPage, pvFault, uErr));
899	Assert(!PGM_PAGE_IS_ZERO(pPage));
900	AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
901	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2MakeWritable; });
902
903	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
904	if (rc != VINF_SUCCESS)
905	{
906	AssertMsg(rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("%Rrc\n", rc));
907	return rc;
908	}
909	if (RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)))
910	return VINF_EM_NO_MEMORY;
911	}
912
913	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
914	/*
915	* Check to see if we need to emulate the instruction if CR0.WP=0.
916	*/
917	if ( !GstWalk.Core.fEffectiveRW
918	&& (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG
919	&& CPUMGetGuestCPL(pVCpu, pRegFrame) == 0)
920	{
921	Assert((uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RW \| X86_TRAP_PF_P));
922	rc = PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault);
923	if (RT_SUCCESS(rc))
924	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eWPEmulInRZ);
925	else
926	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eWPEmulToR3);
927	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2WPEmulation; });
928	return rc;
929	}
930	# endif
931	/// @todo count the above case; else
932	if (uErr & X86_TRAP_PF_US)
933	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUserWrite));
934	else /* supervisor */
935	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
936
937	/*
938	* Sync the page.
939	*
940	* Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
941	* page is not present, which is not true in this case.
942	*/
943	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
944	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
945	# else
946	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
947	# endif
948	if (RT_SUCCESS(rc))
949	{
950	/*
951	* Page was successfully synced, return to guest but invalidate
952	* the TLB first as the page is very likely to be in it.
953	*/
954	# if PGM_SHW_TYPE == PGM_TYPE_EPT
955	HWACCMInvalidatePhysPage(pVM, (RTGCPHYS)pvFault);
956	# else
957	PGM_INVL_PG(pVCpu, pvFault);
958	# endif
959	# ifdef VBOX_STRICT
960	RTGCPHYS GCPhys2;
961	uint64_t fPageGst;
962	if (!pVM->pgm.s.fNestedPaging)
963	{
964	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, &GCPhys2);
965	AssertMsg(RT_SUCCESS(rc) && (fPageGst & X86_PTE_RW), ("rc=%Rrc fPageGst=%RX64\n", rc, fPageGst));
966	LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GCPhys2, (uint64_t)fPageGst));
967	}
968	uint64_t fPageShw;
969	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
970	AssertMsg((RT_SUCCESS(rc) && (fPageShw & X86_PTE_RW)) \|\| pVM->cCpus > 1 /* new monitor can be installed/page table flushed between the trap exit and PGMTrap0eHandler */, ("rc=%Rrc fPageShw=%RX64\n", rc, fPageShw));
971	# endif /* VBOX_STRICT */
972	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndObs; });
973	return VINF_SUCCESS;
974	}
975	}
976	/** @todo else: WTF are we here? */
977
978	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && defined(VBOX_STRICT)
979	/*
980	* Check for VMM page flags vs. Guest page flags consistency.
981	* Currently only for debug purposes.
982	*/
983	if (RT_SUCCESS(rc))
984	{
985	/* Get guest page flags. */
986	uint64_t fPageGst;
987	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
988	if (RT_SUCCESS(rc))
989	{
990	uint64_t fPageShw;
991	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
992
993	/*
994	* Compare page flags.
995	* Note: we have AVL, A, D bits desynched.
996	*/
997	AssertMsg((fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)),
998	("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64\n", pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst));
999	}
1000	else
1001	AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc));
1002	}
1003	else
1004	AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc));
1005	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && VBOX_STRICT */
1006	}
1007
1008
1009	/*
1010	* If we get here it is because something failed above, i.e. most like guru
1011	* meditiation time.
1012	*/
1013	LogRel(("%s: returns rc=%Rrc pvFault=%RGv uErr=%RX64 cs:rip=%04x:%08RX64\n",
1014	__PRETTY_FUNCTION__, rc, pvFault, (uint64_t)uErr, pRegFrame->cs, pRegFrame->rip));
1015	return rc;
1016
1017	# else /* Nested paging, EPT except PGM_GST_TYPE = PROT */
1018	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
1019	return VERR_INTERNAL_ERROR;
1020	# endif
1021	}
1022	#endif /* !IN_RING3 */
1023
1024
1025	/**
1026	* Emulation of the invlpg instruction.
1027	*
1028	*
1029	* @returns VBox status code.
1030	*
1031	* @param pVCpu The VMCPU handle.
1032	* @param GCPtrPage Page to invalidate.
1033	*
1034	* @remark ASSUMES that the guest is updating before invalidating. This order
1035	* isn't required by the CPU, so this is speculative and could cause
1036	* trouble.
1037	* @remark No TLB shootdown is done on any other VCPU as we assume that
1038	* invlpg emulation is the only reason for calling this function.
1039	* (The guest has to shoot down TLB entries on other CPUs itself)
1040	* Currently true, but keep in mind!
1041	*
1042	* @todo Clean this up! Most of it is (or should be) no longer necessary as we catch all page table accesses.
1043	*/
1044	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
1045	{
1046	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1047	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
1048	&& PGM_SHW_TYPE != PGM_TYPE_EPT
1049	int rc;
1050	PVM pVM = pVCpu->CTX_SUFF(pVM);
1051	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1052
1053	Assert(PGMIsLockOwner(pVM));
1054
1055	LogFlow(("InvalidatePage %RGv\n", GCPtrPage));
1056
1057	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1058	if (pPool->cDirtyPages)
1059	pgmPoolResetDirtyPages(pVM);
1060	# endif
1061
1062	/*
1063	* Get the shadow PD entry and skip out if this PD isn't present.
1064	* (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
1065	*/
1066	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1067	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1068	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1069
1070	/* Fetch the pgm pool shadow descriptor. */
1071	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1072	Assert(pShwPde);
1073
1074	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1075	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT);
1076	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
1077
1078	/* If the shadow PDPE isn't present, then skip the invalidate. */
1079	if (!pPdptDst->a[iPdpt].n.u1Present)
1080	{
1081	Assert(!(pPdptDst->a[iPdpt].u & PGM_PLXFLAGS_MAPPING));
1082	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1083	return VINF_SUCCESS;
1084	}
1085
1086	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1087	PPGMPOOLPAGE pShwPde = NULL;
1088	PX86PDPAE pPDDst;
1089
1090	/* Fetch the pgm pool shadow descriptor. */
1091	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
1092	AssertRCSuccessReturn(rc, rc);
1093	Assert(pShwPde);
1094
1095	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1096	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1097
1098	# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1099	/* PML4 */
1100	const unsigned iPml4 = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;
1101	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1102	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1103	PX86PDPAE pPDDst;
1104	PX86PDPT pPdptDst;
1105	PX86PML4E pPml4eDst;
1106	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eDst, &pPdptDst, &pPDDst);
1107	if (rc != VINF_SUCCESS)
1108	{
1109	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT \|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
1110	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1111	return VINF_SUCCESS;
1112	}
1113	Assert(pPDDst);
1114
1115	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1116	PX86PDPE pPdpeDst = &pPdptDst->a[iPdpt];
1117
1118	if (!pPdpeDst->n.u1Present)
1119	{
1120	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1121	return VINF_SUCCESS;
1122	}
1123
1124	/* Fetch the pgm pool shadow descriptor. */
1125	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & SHW_PDPE_PG_MASK);
1126	Assert(pShwPde);
1127
1128	# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1129
1130	const SHWPDE PdeDst = *pPdeDst;
1131	if (!PdeDst.n.u1Present)
1132	{
1133	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1134	return VINF_SUCCESS;
1135	}
1136
1137	/*
1138	* Get the guest PD entry and calc big page.
1139	*/
1140	# if PGM_GST_TYPE == PGM_TYPE_32BIT
1141	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
1142	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
1143	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1144	# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1145	unsigned iPDSrc = 0;
1146	# if PGM_GST_TYPE == PGM_TYPE_PAE
1147	X86PDPE PdpeSrcIgn;
1148	PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrcIgn);
1149	# else /* AMD64 */
1150	PX86PML4E pPml4eSrcIgn;
1151	X86PDPE PdpeSrcIgn;
1152	PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrcIgn, &PdpeSrcIgn, &iPDSrc);
1153	# endif
1154	GSTPDE PdeSrc;
1155
1156	if (pPDSrc)
1157	PdeSrc = pPDSrc->a[iPDSrc];
1158	else
1159	PdeSrc.u = 0;
1160	# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1161	const bool fIsBigPage = PdeSrc.b.u1Size && GST_IS_PSE_ACTIVE(pVCpu);
1162
1163	# ifdef IN_RING3
1164	/*
1165	* If a CR3 Sync is pending we may ignore the invalidate page operation
1166	* depending on the kind of sync and if it's a global page or not.
1167	* This doesn't make sense in GC/R0 so we'll skip it entirely there.
1168	*/
1169	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
1170	if ( VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
1171	\|\| ( VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
1172	&& fIsBigPage
1173	&& PdeSrc.b.u1Global
1174	)
1175	)
1176	# else
1177	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
1178	# endif
1179	{
1180	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1181	return VINF_SUCCESS;
1182	}
1183	# endif /* IN_RING3 */
1184
1185	/*
1186	* Deal with the Guest PDE.
1187	*/
1188	rc = VINF_SUCCESS;
1189	if (PdeSrc.n.u1Present)
1190	{
1191	Assert( PdeSrc.n.u1User == PdeDst.n.u1User
1192	&& (PdeSrc.n.u1Write \|\| !PdeDst.n.u1Write));
1193	# ifndef PGM_WITHOUT_MAPPING
1194	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
1195	{
1196	/*
1197	* Conflict - Let SyncPT deal with it to avoid duplicate code.
1198	*/
1199	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1200	Assert(PGMGetGuestMode(pVCpu) <= PGMMODE_PAE);
1201	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
1202	}
1203	else
1204	# endif /* !PGM_WITHOUT_MAPPING */
1205	if (!fIsBigPage)
1206	{
1207	/*
1208	* 4KB - page.
1209	*/
1210	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1211	RTGCPHYS GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1212
1213	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1214	/* Reset the modification counter (OpenSolaris trashes tlb entries very often) */
1215	if (pShwPage->cModifications)
1216	pShwPage->cModifications = 1;
1217	# endif
1218
1219	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1220	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1221	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE/2);
1222	# endif
1223	if (pShwPage->GCPhys == GCPhys)
1224	{
1225	# if 0 /* likely cause of a major performance regression; must be SyncPageWorkerTrackDeref then */
1226	const unsigned iPTEDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1227	PSHWPT pPT = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1228	if (pPT->a[iPTEDst].n.u1Present)
1229	{
1230	/* This is very unlikely with caching/monitoring enabled. */
1231	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pShwPage, pPT->a[iPTEDst].u & SHW_PTE_PG_MASK, iPTEDst);
1232	ASMAtomicWriteSize(&pPT->a[iPTEDst], 0);
1233	}
1234	# else /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1235	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
1236	if (RT_SUCCESS(rc))
1237	rc = VINF_SUCCESS;
1238	# endif
1239	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4KBPages));
1240	PGM_INVL_PG(pVCpu, GCPtrPage);
1241	}
1242	else
1243	{
1244	/*
1245	* The page table address changed.
1246	*/
1247	LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
1248	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1249	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1250	ASMAtomicWriteSize(pPdeDst, 0);
1251	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1252	PGM_INVL_VCPU_TLBS(pVCpu);
1253	}
1254	}
1255	else
1256	{
1257	/*
1258	* 2/4MB - page.
1259	*/
1260	/* Before freeing the page, check if anything really changed. */
1261	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1262	RTGCPHYS GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(pVM, PdeSrc);
1263	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1264	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1265	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1266	# endif
1267	if ( pShwPage->GCPhys == GCPhys
1268	&& pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1269	{
1270	/* ASSUMES a the given bits are identical for 4M and normal PDEs */
1271	/** @todo PAT */
1272	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
1273	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
1274	&& ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1275	\|\| (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1276	{
1277	LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1278	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip));
1279	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
1280	return VINF_SUCCESS;
1281	}
1282	}
1283
1284	/*
1285	* Ok, the page table is present and it's been changed in the guest.
1286	* If we're in host context, we'll just mark it as not present taking the lazy approach.
1287	* We could do this for some flushes in GC too, but we need an algorithm for
1288	* deciding which 4MB pages containing code likely to be executed very soon.
1289	*/
1290	LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1291	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1292	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1293	ASMAtomicWriteSize(pPdeDst, 0);
1294	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4MBPages));
1295	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
1296	}
1297	}
1298	else
1299	{
1300	/*
1301	* Page directory is not present, mark shadow PDE not present.
1302	*/
1303	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
1304	{
1305	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1306	ASMAtomicWriteSize(pPdeDst, 0);
1307	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDNPs));
1308	PGM_INVL_PG(pVCpu, GCPtrPage);
1309	}
1310	else
1311	{
1312	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1313	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDMappings));
1314	}
1315	}
1316	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
1317	return rc;
1318
1319	#else /* guest real and protected mode */
1320	/* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1321	return VINF_SUCCESS;
1322	#endif
1323	}
1324
1325
1326	/**
1327	* Update the tracking of shadowed pages.
1328	*
1329	* @param pVCpu The VMCPU handle.
1330	* @param pShwPage The shadow page.
1331	* @param HCPhys The physical page we is being dereferenced.
1332	* @param iPte Shadow PTE index
1333	*/
1334	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys, uint16_t iPte)
1335	{
1336	PVM pVM = pVCpu->CTX_SUFF(pVM);
1337
1338	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackDeref, a);
1339	LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1340
1341	/** @todo If this turns out to be a bottle neck (very likely) two things can be done:
1342	* 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1343	* 2. write protect all shadowed pages. I.e. implement caching.
1344	*/
1345	/** @todo duplicated in the 2nd half of pgmPoolTracDerefGCPhysHint */
1346
1347	/*
1348	* Find the guest address.
1349	*/
1350	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
1351	pRam;
1352	pRam = pRam->CTX_SUFF(pNext))
1353	{
1354	unsigned iPage = pRam->cb >> PAGE_SHIFT;
1355	while (iPage-- > 0)
1356	{
1357	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1358	{
1359	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1360
1361	Assert(pShwPage->cPresent);
1362	Assert(pPool->cPresent);
1363	pShwPage->cPresent--;
1364	pPool->cPresent--;
1365
1366	pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage], iPte);
1367	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackDeref, a);
1368	return;
1369	}
1370	}
1371	}
1372
1373	for (;;)
1374	AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys));
1375	}
1376
1377
1378	/**
1379	* Update the tracking of shadowed pages.
1380	*
1381	* @param pVCpu The VMCPU handle.
1382	* @param pShwPage The shadow page.
1383	* @param u16 The top 16-bit of the pPage->HCPhys.
1384	* @param pPage Pointer to the guest page. this will be modified.
1385	* @param iPTDst The index into the shadow table.
1386	*/
1387	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
1388	{
1389	PVM pVM = pVCpu->CTX_SUFF(pVM);
1390	/*
1391	* Just deal with the simple first time here.
1392	*/
1393	if (!u16)
1394	{
1395	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackVirgin);
1396	u16 = PGMPOOL_TD_MAKE(1, pShwPage->idx);
1397	/* Save the page table index. */
1398	PGM_PAGE_SET_PTE_INDEX(pPage, iPTDst);
1399	}
1400	else
1401	u16 = pgmPoolTrackPhysExtAddref(pVM, pPage, u16, pShwPage->idx, iPTDst);
1402
1403	/* write back */
1404	Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x iPTDst=%#x\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst));
1405	PGM_PAGE_SET_TRACKING(pPage, u16);
1406
1407	/* update statistics. */
1408	pVM->pgm.s.CTX_SUFF(pPool)->cPresent++;
1409	pShwPage->cPresent++;
1410	if (pShwPage->iFirstPresent > iPTDst)
1411	pShwPage->iFirstPresent = iPTDst;
1412	}
1413
1414
1415	/**
1416	* Modifies a shadow PTE to account for access handlers.
1417	*
1418	* @param pVM The VM handle.
1419	* @param pPage The page in question.
1420	* @param fPteSrc The flags of the source PTE.
1421	* @param pPteDst The shadow PTE (output).
1422	*/
1423	DECLINLINE(void) PGM_BTH_NAME(SyncHandlerPte)(PVM pVM, PCPGMPAGE pPage, uint32_t fPteSrc, PSHWPTE pPteDst)
1424	{
1425	/** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No.
1426	* Update: \#PF should deal with this before or after calling the handlers. It has all the info to do the job efficiently. */
1427	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1428	{
1429	LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark read-only\n", pPage));
1430	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1431	pPteDst->u = PGM_PAGE_GET_HCPHYS(pPage);
1432	pPteDst->n.u1Present = 1;
1433	pPteDst->n.u1Execute = 1;
1434	pPteDst->n.u1IgnorePAT = 1;
1435	pPteDst->n.u3EMT = VMX_EPT_MEMTYPE_WB;
1436	/* PteDst.n.u1Write = 0 && PteDst.n.u1Size = 0 */
1437	#else
1438	pPteDst->u = (fPteSrc & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1439	\| PGM_PAGE_GET_HCPHYS(pPage);
1440	#endif
1441	}
1442	#ifdef PGM_WITH_MMIO_OPTIMIZATIONS
1443	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
1444	else if ( PGM_PAGE_IS_MMIO(pPage)
1445	# if PGM_SHW_TYPE != PGM_TYPE_EPT
1446	&& ( (fPteSrc & (X86_PTE_RW /\| X86_PTE_D \| X86_PTE_A/ \| X86_PTE_US )) /* #PF handles D & A first. */
1447	== (X86_PTE_RW /\| X86_PTE_D \| X86_PTE_A/)
1448	\|\| BTH_IS_NP_ACTIVE(pVM) )
1449	# endif
1450	# if PGM_SHW_TYPE == PGM_TYPE_AMD64
1451	&& pVM->pgm.s.fLessThan52PhysicalAddressBits
1452	# endif
1453	)
1454	{
1455	LogFlow(("SyncHandlerPte: MMIO page -> invalid \n"));
1456	# if PGM_SHW_TYPE == PGM_TYPE_EPT
1457	/* 25.2.3.1: Reserved physical address bit -> EPT Misconfiguration (exit 49) */
1458	pPteDst->u = pVM->pgm.s.HCPhysInvMmioPg;
1459	/* 25.2.3.1: bits 2:0 = 010b -> EPT Misconfiguration (exit 49) */
1460	pPteDst->n.u1Present = 0;
1461	pPteDst->n.u1Write = 1;
1462	pPteDst->n.u1Execute = 0;
1463	/* 25.2.3.1: leaf && 2:0 != 0 && u3Emt in {2, 3, 7} -> EPT Misconfiguration */
1464	pPteDst->n.u3EMT = 7;
1465	# else
1466	/* Set high page frame bits that MBZ (bankers on PAE, CPU dependent on AMD64). */
1467	pPteDst->u = pVM->pgm.s.HCPhysInvMmioPg \| X86_PTE_PAE_MBZ_MASK_NO_NX \| X86_PTE_P;
1468	# endif
1469	}
1470	# endif
1471	#endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
1472	else
1473	{
1474	LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark not present\n", pPage));
1475	pPteDst->u = 0;
1476	}
1477	/** @todo count these kinds of entries. */
1478	}
1479
1480
1481	/**
1482	* Creates a 4K shadow page for a guest page.
1483	*
1484	* For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1485	* physical address. The PdeSrc argument only the flags are used. No page
1486	* structured will be mapped in this function.
1487	*
1488	* @param pVCpu The VMCPU handle.
1489	* @param pPteDst Destination page table entry.
1490	* @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1491	* Can safely assume that only the flags are being used.
1492	* @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1493	* @param pShwPage Pointer to the shadow page.
1494	* @param iPTDst The index into the shadow table.
1495	*
1496	* @remark Not used for 2/4MB pages!
1497	*/
1498	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc,
1499	PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1500	{
1501	if ( PteSrc.n.u1Present
1502	&& GST_IS_PTE_VALID(pVCpu, PteSrc))
1503	{
1504	PVM pVM = pVCpu->CTX_SUFF(pVM);
1505
1506	# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1507	&& PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1508	&& (PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64)
1509	if (pShwPage->fDirty)
1510	{
1511	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1512	PX86PTPAE pGstPT;
1513
1514	pGstPT = (PX86PTPAE)&pPool->aDirtyPages[pShwPage->idxDirty][0];
1515	pGstPT->a[iPTDst].u = PteSrc.u;
1516	}
1517	# endif
1518	/*
1519	* Find the ram range.
1520	*/
1521	PPGMPAGE pPage;
1522	int rc = pgmPhysGetPageEx(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK, &pPage);
1523	if (RT_SUCCESS(rc))
1524	{
1525	/* Ignore ballooned pages.
1526	Don't return errors or use a fatal assert here as part of a
1527	shadow sync range might included ballooned pages. */
1528	if (PGM_PAGE_IS_BALLOONED(pPage))
1529	{
1530	Assert(!pPteDst->n.u1Present); /** @todo user tracking needs updating if this triggers. */
1531	return;
1532	}
1533
1534	#ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1535	/* Make the page writable if necessary. */
1536	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1537	&& ( PGM_PAGE_IS_ZERO(pPage)
1538	\|\| ( PteSrc.n.u1Write
1539	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1540	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1541	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1542	# endif
1543	# ifdef VBOX_WITH_PAGE_SHARING
1544	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
1545	# endif
1546	)
1547	)
1548	)
1549	{
1550	rc = pgmPhysPageMakeWritable(pVM, pPage, PteSrc.u & GST_PTE_PG_MASK);
1551	AssertRC(rc);
1552	}
1553	#endif
1554
1555	/*
1556	* Make page table entry.
1557	*/
1558	SHWPTE PteDst;
1559	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1560	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage,
1561	PteSrc.u & ~( X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK
1562	\| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT),
1563	&PteDst);
1564	else
1565	{
1566	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1567	/*
1568	* If the page or page directory entry is not marked accessed,
1569	* we mark the page not present.
1570	*/
1571	if (!PteSrc.n.u1Accessed \|\| !PdeSrc.n.u1Accessed)
1572	{
1573	LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1574	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,AccessedPage));
1575	PteDst.u = 0;
1576	}
1577	/*
1578	* If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1579	* when the page is modified.
1580	*/
1581	else if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1582	{
1583	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPage));
1584	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1585	\| PGM_PAGE_GET_HCPHYS(pPage)
1586	\| PGM_PTFLAGS_TRACK_DIRTY;
1587	}
1588	else
1589	#endif
1590	{
1591	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageSkipped));
1592	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1593	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage);
1594	PteDst.n.u1Present = 1;
1595	PteDst.n.u1Write = 1;
1596	PteDst.n.u1Execute = 1;
1597	PteDst.n.u1IgnorePAT = 1;
1598	PteDst.n.u3EMT = VMX_EPT_MEMTYPE_WB;
1599	/* PteDst.n.u1Size = 0 */
1600	#else
1601	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1602	\| PGM_PAGE_GET_HCPHYS(pPage);
1603	#endif
1604	}
1605
1606	/*
1607	* Make sure only allocated pages are mapped writable.
1608	*/
1609	if ( PteDst.n.u1Write
1610	&& PteDst.n.u1Present
1611	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1612	{
1613	/* Still applies to shared pages. */
1614	Assert(!PGM_PAGE_IS_ZERO(pPage));
1615	PteDst.n.u1Write = 0; /** @todo this isn't quite working yet. Why, isn't it? */
1616	Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", (RTGCPHYS)(PteSrc.u & X86_PTE_PAE_PG_MASK), pPage, iPTDst));
1617	}
1618	}
1619
1620	/*
1621	* Keep user track up to date.
1622	*/
1623	if (PteDst.n.u1Present)
1624	{
1625	if (!pPteDst->n.u1Present)
1626	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1627	else if ((pPteDst->u & SHW_PTE_PG_MASK) != (PteDst.u & SHW_PTE_PG_MASK))
1628	{
1629	Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", (uint64_t)pPteDst->u, (uint64_t)PteDst.u));
1630	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK, iPTDst);
1631	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1632	}
1633	}
1634	else if (pPteDst->n.u1Present)
1635	{
1636	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1637	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK, iPTDst);
1638	}
1639
1640	/*
1641	* Update statistics and commit the entry.
1642	*/
1643	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1644	if (!PteSrc.n.u1Global)
1645	pShwPage->fSeenNonGlobal = true;
1646	#endif
1647	ASMAtomicWriteSize(pPteDst, PteDst.u);
1648	return;
1649	}
1650
1651	/** @todo count these three different kinds. */
1652	Log2(("SyncPageWorker: invalid address in Pte\n"));
1653	}
1654	else if (!PteSrc.n.u1Present)
1655	Log2(("SyncPageWorker: page not present in Pte\n"));
1656	else
1657	Log2(("SyncPageWorker: invalid Pte\n"));
1658
1659	/*
1660	* The page is not present or the PTE is bad. Replace the shadow PTE by
1661	* an empty entry, making sure to keep the user tracking up to date.
1662	*/
1663	if (pPteDst->n.u1Present)
1664	{
1665	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1666	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK, iPTDst);
1667	}
1668	ASMAtomicWriteSize(pPteDst, 0);
1669	}
1670
1671
1672	/**
1673	* Syncs a guest OS page.
1674	*
1675	* There are no conflicts at this point, neither is there any need for
1676	* page table allocations.
1677	*
1678	* When called in PAE or AMD64 guest mode, the guest PDPE shall be valid.
1679	* When called in AMD64 guest mode, the guest PML4E shall be valid.
1680	*
1681	* @returns VBox status code.
1682	* @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1683	* @param pVCpu The VMCPU handle.
1684	* @param PdeSrc Page directory entry of the guest.
1685	* @param GCPtrPage Guest context page address.
1686	* @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1687	* @param uErr Fault error (X86_TRAP_PF_*).
1688	*/
1689	static int PGM_BTH_NAME(SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr)
1690	{
1691	PVM pVM = pVCpu->CTX_SUFF(pVM);
1692	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1693	LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr));
1694
1695	Assert(PGMIsLockOwner(pVM));
1696
1697	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
1698	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
1699	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
1700	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
1701	&& PGM_SHW_TYPE != PGM_TYPE_EPT
1702
1703	/*
1704	* Assert preconditions.
1705	*/
1706	Assert(PdeSrc.n.u1Present);
1707	Assert(cPages);
1708	# if 0 /* rarely useful; leave for debugging. */
1709	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
1710	# endif
1711
1712	/*
1713	* Get the shadow PDE, find the shadow page table in the pool.
1714	*/
1715	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1716	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1717	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1718
1719	/* Fetch the pgm pool shadow descriptor. */
1720	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1721	Assert(pShwPde);
1722
1723	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1724	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1725	PPGMPOOLPAGE pShwPde = NULL;
1726	PX86PDPAE pPDDst;
1727
1728	/* Fetch the pgm pool shadow descriptor. */
1729	int rc2 = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
1730	AssertRCSuccessReturn(rc2, rc2);
1731	Assert(pShwPde);
1732
1733	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1734	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1735
1736	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1737	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1738	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1739	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
1740	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
1741
1742	int rc2 = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
1743	AssertRCSuccessReturn(rc2, rc2);
1744	Assert(pPDDst && pPdptDst);
1745	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1746	# endif
1747	SHWPDE PdeDst = *pPdeDst;
1748
1749	/* In the guest SMP case we could have blocked while another VCPU reused this page table. */
1750	if (!PdeDst.n.u1Present)
1751	{
1752	AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE p=%p/%RX64\n", pPdeDst, (uint64_t)PdeDst.u));
1753	Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", pVCpu->idCpu, GCPtrPage));
1754	return VINF_SUCCESS; /* force the instruction to be executed again. */
1755	}
1756
1757	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1758	Assert(pShwPage);
1759
1760	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1761	/* Fetch the pgm pool shadow descriptor. */
1762	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
1763	Assert(pShwPde);
1764	# endif
1765
1766	/*
1767	* Check that the page is present and that the shadow PDE isn't out of sync.
1768	*/
1769	const bool fBigPage = PdeSrc.b.u1Size && GST_IS_PSE_ACTIVE(pVCpu);
1770	const bool fPdeValid = !fBigPage ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc);
1771	RTGCPHYS GCPhys;
1772	if (!fBigPage)
1773	{
1774	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1775	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1776	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1777	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE / 2);
1778	# endif
1779	}
1780	else
1781	{
1782	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(pVM, PdeSrc);
1783	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1784	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1785	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1786	# endif
1787	}
1788	if ( fPdeValid
1789	&& pShwPage->GCPhys == GCPhys
1790	&& PdeSrc.n.u1Present
1791	&& PdeSrc.n.u1User == PdeDst.n.u1User
1792	&& (PdeSrc.n.u1Write == PdeDst.n.u1Write \|\| !PdeDst.n.u1Write)
1793	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
1794	&& (PdeSrc.n.u1NoExecute == PdeDst.n.u1NoExecute \|\| !GST_IS_NX_ACTIVE(pVCpu))
1795	# endif
1796	)
1797	{
1798	/*
1799	* Check that the PDE is marked accessed already.
1800	* Since we set the accessed bit before getting here on a #PF, this
1801	* check is only meant for dealing with non-#PF'ing paths.
1802	*/
1803	if (PdeSrc.n.u1Accessed)
1804	{
1805	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1806	if (!fBigPage)
1807	{
1808	/*
1809	* 4KB Page - Map the guest page table.
1810	*/
1811	PGSTPT pPTSrc;
1812	int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
1813	if (RT_SUCCESS(rc))
1814	{
1815	# ifdef PGM_SYNC_N_PAGES
1816	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
1817	if ( cPages > 1
1818	&& !(uErr & X86_TRAP_PF_P)
1819	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
1820	{
1821	/*
1822	* This code path is currently only taken when the caller is PGMTrap0eHandler
1823	* for non-present pages!
1824	*
1825	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1826	* deal with locality.
1827	*/
1828	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1829	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1830	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1831	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1832	# else
1833	const unsigned offPTSrc = 0;
1834	# endif
1835	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
1836	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1837	iPTDst = 0;
1838	else
1839	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1840	for (; iPTDst < iPTDstEnd; iPTDst++)
1841	{
1842	if (!pPTDst->a[iPTDst].n.u1Present)
1843	{
1844	GSTPTE PteSrc = pPTSrc->a[offPTSrc + iPTDst];
1845	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
1846	NOREF(GCPtrCurPage);
1847	#ifndef IN_RING0
1848	/*
1849	* Assuming kernel code will be marked as supervisor - and not as user level
1850	* and executed using a conforming code selector - And marked as readonly.
1851	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
1852	*/
1853	PPGMPAGE pPage;
1854	if ( ((PdeSrc.u & PteSrc.u) & (X86_PTE_RW \| X86_PTE_US))
1855	\|\| iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK) /* always sync GCPtrPage */
1856	\|\| !CSAMDoesPageNeedScanning(pVM, GCPtrCurPage)
1857	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
1858	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1859	)
1860	#endif /* else: CSAM not active */
1861	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1862	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1863	GCPtrCurPage, PteSrc.n.u1Present,
1864	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1865	PteSrc.n.u1User & PdeSrc.n.u1User,
1866	(uint64_t)PteSrc.u,
1867	(uint64_t)pPTDst->a[iPTDst].u,
1868	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1869	}
1870	}
1871	}
1872	else
1873	# endif /* PGM_SYNC_N_PAGES */
1874	{
1875	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1876	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1877	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1878	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1879	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
1880	GCPtrPage, PteSrc.n.u1Present,
1881	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1882	PteSrc.n.u1User & PdeSrc.n.u1User,
1883	(uint64_t)PteSrc.u,
1884	(uint64_t)pPTDst->a[iPTDst].u,
1885	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1886	}
1887	}
1888	else /* MMIO or invalid page: emulated in #PF handler. */
1889	{
1890	LogFlow(("PGM_GCPHYS_2_PTR %RGp failed with %Rrc\n", GCPhys, rc));
1891	Assert(!pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK].n.u1Present);
1892	}
1893	}
1894	else
1895	{
1896	/*
1897	* 4/2MB page - lazy syncing shadow 4K pages.
1898	* (There are many causes of getting here, it's no longer only CSAM.)
1899	*/
1900	/* Calculate the GC physical address of this 4KB shadow page. */
1901	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(pVM, PdeSrc) \| (GCPtrPage & GST_BIG_PAGE_OFFSET_MASK);
1902	/* Find ram range. */
1903	PPGMPAGE pPage;
1904	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
1905	if (RT_SUCCESS(rc))
1906	{
1907	AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
1908
1909	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1910	/* Try to make the page writable if necessary. */
1911	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1912	&& ( PGM_PAGE_IS_ZERO(pPage)
1913	\|\| ( PdeSrc.n.u1Write
1914	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1915	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1916	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1917	# endif
1918	# ifdef VBOX_WITH_PAGE_SHARING
1919	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
1920	# endif
1921	)
1922	)
1923	)
1924	{
1925	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1926	AssertRC(rc);
1927	}
1928	# endif
1929
1930	/*
1931	* Make shadow PTE entry.
1932	*/
1933	SHWPTE PteDst;
1934	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1935	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage,
1936	PdeSrc.u & ~( X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK
1937	\| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT),
1938	&PteDst);
1939	else
1940	PteDst.u = (PdeSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1941	\| PGM_PAGE_GET_HCPHYS(pPage);
1942
1943	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1944	if ( PteDst.n.u1Present
1945	&& !pPTDst->a[iPTDst].n.u1Present)
1946	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1947
1948	/* Make sure only allocated pages are mapped writable. */
1949	if ( PteDst.n.u1Write
1950	&& PteDst.n.u1Present
1951	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1952	{
1953	/* Still applies to shared pages. */
1954	Assert(!PGM_PAGE_IS_ZERO(pPage));
1955	PteDst.n.u1Write = 0; /** @todo this isn't quite working yet... */
1956	Log3(("SyncPage: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, GCPtrPage));
1957	}
1958
1959	ASMAtomicWriteSize(&pPTDst->a[iPTDst], PteDst.u);
1960
1961	/*
1962	* If the page is not flagged as dirty and is writable, then make it read-only
1963	* at PD level, so we can set the dirty bit when the page is modified.
1964	*
1965	* ASSUMES that page access handlers are implemented on page table entry level.
1966	* Thus we will first catch the dirty access and set PDE.D and restart. If
1967	* there is an access handler, we'll trap again and let it work on the problem.
1968	*/
1969	/** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
1970	* As for invlpg, it simply frees the whole shadow PT.
1971	* ...It's possibly because the guest clears it and the guest doesn't really tell us... */
1972	if ( !PdeSrc.b.u1Dirty
1973	&& PdeSrc.b.u1Write)
1974	{
1975	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
1976	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
1977	PdeDst.n.u1Write = 0;
1978	}
1979	else
1980	{
1981	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
1982	PdeDst.n.u1Write = PdeSrc.n.u1Write;
1983	}
1984	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
1985	Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n",
1986	GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
1987	PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1988	}
1989	else
1990	{
1991	LogFlow(("PGM_GCPHYS_2_PTR %RGp (big) failed with %Rrc\n", GCPhys, rc));
1992	/** @todo must wipe the shadow page table in this case. */
1993	}
1994	}
1995	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
1996	return VINF_SUCCESS;
1997	}
1998
1999	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPagePDNAs));
2000	}
2001	else if (fPdeValid)
2002	{
2003	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPagePDOutOfSync));
2004	Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2005	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2006	}
2007	else
2008	{
2009	/// @todo STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSync));
2010	Log2(("SyncPage: Bad PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2011	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2012	}
2013
2014	/*
2015	* Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
2016	* Yea, I'm lazy.
2017	*/
2018	pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
2019	ASMAtomicWriteSize(pPdeDst, 0);
2020
2021	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2022	PGM_INVL_VCPU_TLBS(pVCpu);
2023	return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
2024
2025
2026	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2027	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2028	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2029	&& !defined(IN_RC)
2030
2031	# ifdef PGM_SYNC_N_PAGES
2032	/*
2033	* Get the shadow PDE, find the shadow page table in the pool.
2034	*/
2035	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2036	X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
2037
2038	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2039	X86PDEPAE PdeDst = pgmShwGetPaePDE(pVCpu, GCPtrPage);
2040
2041	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2042	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2043	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpt);
2044	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2045	X86PDEPAE PdeDst;
2046	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2047
2048	int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2049	AssertRCSuccessReturn(rc, rc);
2050	Assert(pPDDst && pPdptDst);
2051	PdeDst = pPDDst->a[iPDDst];
2052	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2053	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2054	PEPTPD pPDDst;
2055	EPTPDE PdeDst;
2056
2057	int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, NULL, &pPDDst);
2058	if (rc != VINF_SUCCESS)
2059	{
2060	AssertRC(rc);
2061	return rc;
2062	}
2063	Assert(pPDDst);
2064	PdeDst = pPDDst->a[iPDDst];
2065	# endif
2066	/* In the guest SMP case we could have blocked while another VCPU reused this page table. */
2067	if (!PdeDst.n.u1Present)
2068	{
2069	AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)PdeDst.u));
2070	Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", pVCpu->idCpu, GCPtrPage));
2071	return VINF_SUCCESS; /* force the instruction to be executed again. */
2072	}
2073
2074	/* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */
2075	if (PdeDst.n.u1Size)
2076	{
2077	Assert(pVM->pgm.s.fNestedPaging);
2078	Log(("CPU%d: SyncPage: Pde (big:%RX64) at %RGv changed behind our back!\n", pVCpu->idCpu, PdeDst.u, GCPtrPage));
2079	return VINF_SUCCESS;
2080	}
2081
2082	/* Mask away the page offset. */
2083	GCPtrPage &= ~((RTGCPTR)0xfff);
2084
2085	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2086	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2087
2088	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
2089	if ( cPages > 1
2090	&& !(uErr & X86_TRAP_PF_P)
2091	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2092	{
2093	/*
2094	* This code path is currently only taken when the caller is PGMTrap0eHandler
2095	* for non-present pages!
2096	*
2097	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2098	* deal with locality.
2099	*/
2100	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2101	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2102	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2103	iPTDst = 0;
2104	else
2105	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2106	for (; iPTDst < iPTDstEnd; iPTDst++)
2107	{
2108	if (!pPTDst->a[iPTDst].n.u1Present)
2109	{
2110	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT);
2111	GSTPTE PteSrc;
2112
2113	/* Fake the page table entry */
2114	PteSrc.u = GCPtrCurPage;
2115	PteSrc.n.u1Present = 1;
2116	PteSrc.n.u1Dirty = 1;
2117	PteSrc.n.u1Accessed = 1;
2118	PteSrc.n.u1Write = 1;
2119	PteSrc.n.u1User = 1;
2120
2121	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2122
2123	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
2124	GCPtrCurPage, PteSrc.n.u1Present,
2125	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2126	PteSrc.n.u1User & PdeSrc.n.u1User,
2127	(uint64_t)PteSrc.u,
2128	(uint64_t)pPTDst->a[iPTDst].u,
2129	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2130
2131	if (RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)))
2132	break;
2133	}
2134	else
2135	Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n", (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT), iPTDst, pPTDst->a[iPTDst].u));
2136	}
2137	}
2138	else
2139	# endif /* PGM_SYNC_N_PAGES */
2140	{
2141	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2142	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT);
2143	GSTPTE PteSrc;
2144
2145	/* Fake the page table entry */
2146	PteSrc.u = GCPtrCurPage;
2147	PteSrc.n.u1Present = 1;
2148	PteSrc.n.u1Dirty = 1;
2149	PteSrc.n.u1Accessed = 1;
2150	PteSrc.n.u1Write = 1;
2151	PteSrc.n.u1User = 1;
2152	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2153
2154	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}PteDst=%08llx%s\n",
2155	GCPtrPage, PteSrc.n.u1Present,
2156	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2157	PteSrc.n.u1User & PdeSrc.n.u1User,
2158	(uint64_t)PteSrc.u,
2159	(uint64_t)pPTDst->a[iPTDst].u,
2160	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2161	}
2162	return VINF_SUCCESS;
2163
2164	#else
2165	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2166	return VERR_INTERNAL_ERROR;
2167	#endif
2168	}
2169
2170
2171	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2172
2173	/**
2174	* CheckPageFault helper for returning a page fault indicating a non-present
2175	* (NP) entry in the page translation structures.
2176	*
2177	* @returns VINF_EM_RAW_GUEST_TRAP.
2178	* @param pVCpu The virtual CPU to operate on.
2179	* @param uErr The error code of the shadow fault. Corrections to
2180	* TRPM's copy will be made if necessary.
2181	* @param GCPtrPage For logging.
2182	* @param uPageFaultLevel For logging.
2183	*/
2184	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnNP)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2185	{
2186	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2187	AssertMsg(!(uErr & X86_TRAP_PF_P), ("%#x\n", uErr));
2188	AssertMsg(!(uErr & X86_TRAP_PF_RSVD), ("%#x\n", uErr));
2189	if (uErr & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
2190	TRPMSetErrorCode(pVCpu, uErr & ~(X86_TRAP_PF_RSVD \| X86_TRAP_PF_P));
2191
2192	Log(("CheckPageFault: real page fault (notp) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2193	return VINF_EM_RAW_GUEST_TRAP;
2194	}
2195
2196
2197	/**
2198	* CheckPageFault helper for returning a page fault indicating a reserved bit
2199	* (RSVD) error in the page translation structures.
2200	*
2201	* @returns VINF_EM_RAW_GUEST_TRAP.
2202	* @param pVCpu The virtual CPU to operate on.
2203	* @param uErr The error code of the shadow fault. Corrections to
2204	* TRPM's copy will be made if necessary.
2205	* @param GCPtrPage For logging.
2206	* @param uPageFaultLevel For logging.
2207	*/
2208	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnRSVD)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2209	{
2210	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2211	if ((uErr & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) != (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
2212	TRPMSetErrorCode(pVCpu, uErr \| X86_TRAP_PF_RSVD \| X86_TRAP_PF_P);
2213
2214	Log(("CheckPageFault: real page fault (rsvd) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2215	return VINF_EM_RAW_GUEST_TRAP;
2216	}
2217
2218
2219	/**
2220	* CheckPageFault helper for returning a page protection fault (P).
2221	*
2222	* @returns VINF_EM_RAW_GUEST_TRAP.
2223	* @param pVCpu The virtual CPU to operate on.
2224	* @param uErr The error code of the shadow fault. Corrections to
2225	* TRPM's copy will be made if necessary.
2226	* @param GCPtrPage For logging.
2227	* @param uPageFaultLevel For logging.
2228	*/
2229	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnProt)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2230	{
2231	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2232	AssertMsg(uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID), ("%#x\n", uErr));
2233	if ((uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) != X86_TRAP_PF_P)
2234	TRPMSetErrorCode(pVCpu, (uErr & ~X86_TRAP_PF_RSVD) \| X86_TRAP_PF_P);
2235
2236	Log(("CheckPageFault: real page fault (prot) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2237	return VINF_EM_RAW_GUEST_TRAP;
2238	}
2239
2240
2241	/**
2242	* Handle dirty bit tracking faults.
2243	*
2244	* @returns VBox status code.
2245	* @param pVCpu The VMCPU handle.
2246	* @param uErr Page fault error code.
2247	* @param pPdeSrc Guest page directory entry.
2248	* @param pPdeDst Shadow page directory entry.
2249	* @param GCPtrPage Guest context page address.
2250	*/
2251	static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, RTGCPTR GCPtrPage)
2252	{
2253	PVM pVM = pVCpu->CTX_SUFF(pVM);
2254	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2255
2256	Assert(PGMIsLockOwner(pVM));
2257
2258	/*
2259	* Handle big page.
2260	*/
2261	if (pPdeSrc->b.u1Size && GST_IS_PSE_ACTIVE(pVCpu))
2262	{
2263	if ( pPdeDst->n.u1Present
2264	&& (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY))
2265	{
2266	SHWPDE PdeDst = *pPdeDst;
2267
2268	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageTrap));
2269	Assert(pPdeSrc->b.u1Write);
2270
2271	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
2272	* fault again and take this path to only invalidate the entry (see below).
2273	*/
2274	PdeDst.n.u1Write = 1;
2275	PdeDst.n.u1Accessed = 1;
2276	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2277	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2278	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
2279	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2280	}
2281
2282	# ifdef IN_RING0
2283	/* Check for stale TLB entry; only applies to the SMP guest case. */
2284	if ( pVM->cCpus > 1
2285	&& pPdeDst->n.u1Write
2286	&& pPdeDst->n.u1Accessed)
2287	{
2288	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2289	if (pShwPage)
2290	{
2291	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2292	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2293	if ( pPteDst->n.u1Present
2294	&& pPteDst->n.u1Write)
2295	{
2296	/* Stale TLB entry. */
2297	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageStale));
2298	PGM_INVL_PG(pVCpu, GCPtrPage);
2299	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2300	}
2301	}
2302	}
2303	# endif /* IN_RING0 */
2304	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2305	}
2306
2307	/*
2308	* Map the guest page table.
2309	*/
2310	PGSTPT pPTSrc;
2311	int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
2312	if (RT_FAILURE(rc))
2313	{
2314	AssertRC(rc);
2315	return rc;
2316	}
2317
2318	if (pPdeDst->n.u1Present)
2319	{
2320	GSTPTE const *pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2321	const GSTPTE PteSrc = *pPteSrc;
2322
2323	#ifndef IN_RING0
2324	/* Bail out here as pgmPoolGetPage will return NULL and we'll crash below.
2325	* Our individual shadow handlers will provide more information and force a fatal exit.
2326	*/
2327	if (MMHyperIsInsideArea(pVM, (RTGCPTR)GCPtrPage))
2328	{
2329	LogRel(("CheckPageFault: write to hypervisor region %RGv\n", GCPtrPage));
2330	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2331	}
2332	#endif
2333	/*
2334	* Map shadow page table.
2335	*/
2336	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2337	if (pShwPage)
2338	{
2339	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2340	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2341	if (pPteDst->n.u1Present) /** @todo Optimize accessed bit emulation? */
2342	{
2343	if (pPteDst->u & PGM_PTFLAGS_TRACK_DIRTY)
2344	{
2345	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, pPteSrc->u & GST_PTE_PG_MASK);
2346	SHWPTE PteDst = *pPteDst;
2347
2348	LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage));
2349	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageTrap));
2350
2351	Assert(pPteSrc->n.u1Write);
2352
2353	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB
2354	* entry will not harm; write access will simply fault again and
2355	* take this path to only invalidate the entry.
2356	*/
2357	if (RT_LIKELY(pPage))
2358	{
2359	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2360	{
2361	AssertMsgFailed(("%R[pgmpage] - we don't set PGM_PTFLAGS_TRACK_DIRTY for these pages\n", pPage));
2362	Assert(!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage));
2363	/* Assuming write handlers here as the PTE is present (otherwise we wouldn't be here). */
2364	PteDst.n.u1Write = 0;
2365	}
2366	else
2367	{
2368	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
2369	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
2370	{
2371	rc = pgmPhysPageMakeWritable(pVM, pPage, pPteSrc->u & GST_PTE_PG_MASK);
2372	AssertRC(rc);
2373	}
2374	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED)
2375	PteDst.n.u1Write = 1;
2376	else
2377	{
2378	/* Still applies to shared pages. */
2379	Assert(!PGM_PAGE_IS_ZERO(pPage));
2380	PteDst.n.u1Write = 0;
2381	}
2382	}
2383	}
2384	else
2385	PteDst.n.u1Write = 1; /** @todo r=bird: This doesn't make sense to me. */
2386
2387	PteDst.n.u1Dirty = 1;
2388	PteDst.n.u1Accessed = 1;
2389	PteDst.au32[0] &= ~PGM_PTFLAGS_TRACK_DIRTY;
2390	ASMAtomicWriteSize(pPteDst, PteDst.u);
2391	PGM_INVL_PG(pVCpu, GCPtrPage);
2392	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2393	}
2394
2395	# ifdef IN_RING0
2396	/* Check for stale TLB entry; only applies to the SMP guest case. */
2397	if ( pVM->cCpus > 1
2398	&& pPteDst->n.u1Write == 1
2399	&& pPteDst->n.u1Accessed == 1)
2400	{
2401	/* Stale TLB entry. */
2402	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageStale));
2403	PGM_INVL_PG(pVCpu, GCPtrPage);
2404	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2405	}
2406	# endif
2407	}
2408	}
2409	else
2410	AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
2411	}
2412
2413	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2414	}
2415
2416	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
2417
2418
2419	/**
2420	* Sync a shadow page table.
2421	*
2422	* The shadow page table is not present. This includes the case where
2423	* there is a conflict with a mapping.
2424	*
2425	* @returns VBox status code.
2426	* @param pVCpu The VMCPU handle.
2427	* @param iPD Page directory index.
2428	* @param pPDSrc Source page directory (i.e. Guest OS page directory).
2429	* Assume this is a temporary mapping.
2430	* @param GCPtrPage GC Pointer of the page that caused the fault
2431	*/
2432	static int PGM_BTH_NAME(SyncPT)(PVMCPU pVCpu, unsigned iPDSrc, PGSTPD pPDSrc, RTGCPTR GCPtrPage)
2433	{
2434	PVM pVM = pVCpu->CTX_SUFF(pVM);
2435	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2436
2437	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2438	#if 0 /* rarely useful; leave for debugging. */
2439	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPtPD[iPDSrc]);
2440	#endif
2441	LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage));
2442
2443	Assert(PGMIsLocked(pVM));
2444
2445	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2446	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
2447	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
2448	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2449	&& PGM_SHW_TYPE != PGM_TYPE_EPT
2450
2451	int rc = VINF_SUCCESS;
2452
2453	/*
2454	* Validate input a little bit.
2455	*/
2456	AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
2457	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2458	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2459	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
2460
2461	/* Fetch the pgm pool shadow descriptor. */
2462	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2463	Assert(pShwPde);
2464
2465	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2466	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2467	PPGMPOOLPAGE pShwPde = NULL;
2468	PX86PDPAE pPDDst;
2469	PSHWPDE pPdeDst;
2470
2471	/* Fetch the pgm pool shadow descriptor. */
2472	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
2473	AssertRCSuccessReturn(rc, rc);
2474	Assert(pShwPde);
2475
2476	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
2477	pPdeDst = &pPDDst->a[iPDDst];
2478
2479	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2480	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2481	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2482	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2483	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2484	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2485	AssertRCSuccessReturn(rc, rc);
2486	Assert(pPDDst);
2487	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2488	# endif
2489	SHWPDE PdeDst = *pPdeDst;
2490
2491	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2492	/* Fetch the pgm pool shadow descriptor. */
2493	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2494	Assert(pShwPde);
2495	# endif
2496
2497	# ifndef PGM_WITHOUT_MAPPINGS
2498	/*
2499	* Check for conflicts.
2500	* RC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
2501	* R3: Simply resolve the conflict.
2502	*/
2503	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2504	{
2505	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
2506	# ifndef IN_RING3
2507	Log(("SyncPT: Conflict at %RGv\n", GCPtrPage));
2508	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2509	return VERR_ADDRESS_CONFLICT;
2510
2511	# else /* IN_RING3 */
2512	PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
2513	Assert(pMapping);
2514	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2515	rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2516	# elif PGM_GST_TYPE == PGM_TYPE_PAE
2517	rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2518	# else
2519	AssertFailed(); /* can't happen for amd64 */
2520	# endif
2521	if (RT_FAILURE(rc))
2522	{
2523	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2524	return rc;
2525	}
2526	PdeDst = *pPdeDst;
2527	# endif /* IN_RING3 */
2528	}
2529	# endif /* !PGM_WITHOUT_MAPPINGS */
2530	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2531
2532	/*
2533	* Sync page directory entry.
2534	*/
2535	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2536	if (PdeSrc.n.u1Present)
2537	{
2538	/*
2539	* Allocate & map the page table.
2540	*/
2541	PSHWPT pPTDst;
2542	const bool fPageTable = !PdeSrc.b.u1Size \|\| !GST_IS_PSE_ACTIVE(pVCpu);
2543	PPGMPOOLPAGE pShwPage;
2544	RTGCPHYS GCPhys;
2545	if (fPageTable)
2546	{
2547	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
2548	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2549	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2550	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE / 2);
2551	# endif
2552	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
2553	}
2554	else
2555	{
2556	PGMPOOLACCESS enmAccess;
2557	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2558	const bool fNoExecute = PdeSrc.n.u1NoExecute && GST_IS_NX_ACTIVE(pVCpu);
2559	# else
2560	const bool fNoExecute = false;
2561	# endif
2562
2563	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(pVM, PdeSrc);
2564	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2565	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2566	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
2567	# endif
2568	/* Determine the right kind of large page to avoid incorrect cached entry reuse. */
2569	if (PdeSrc.n.u1User)
2570	{
2571	if (PdeSrc.n.u1Write)
2572	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_RW_NX : PGMPOOLACCESS_USER_RW;
2573	else
2574	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_R_NX : PGMPOOLACCESS_USER_R;
2575	}
2576	else
2577	{
2578	if (PdeSrc.n.u1Write)
2579	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW;
2580	else
2581	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_R_NX : PGMPOOLACCESS_SUPERVISOR_R;
2582	}
2583	rc = pgmPoolAllocEx(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, enmAccess, pShwPde->idx, iPDDst, &pShwPage);
2584	}
2585	if (rc == VINF_SUCCESS)
2586	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2587	else if (rc == VINF_PGM_CACHED_PAGE)
2588	{
2589	/*
2590	* The PT was cached, just hook it up.
2591	*/
2592	if (fPageTable)
2593	PdeDst.u = pShwPage->Core.Key
2594	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2595	else
2596	{
2597	PdeDst.u = pShwPage->Core.Key
2598	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2599	/* (see explanation and assumptions further down.) */
2600	if ( !PdeSrc.b.u1Dirty
2601	&& PdeSrc.b.u1Write)
2602	{
2603	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2604	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2605	PdeDst.b.u1Write = 0;
2606	}
2607	}
2608	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2609	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2610	return VINF_SUCCESS;
2611	}
2612	else if (rc == VERR_PGM_POOL_FLUSHED)
2613	{
2614	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2615	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2616	return VINF_PGM_SYNC_CR3;
2617	}
2618	else
2619	AssertMsgFailedReturn(("rc=%Rrc\n", rc), VERR_INTERNAL_ERROR);
2620	PdeDst.u &= X86_PDE_AVL_MASK;
2621	PdeDst.u \|= pShwPage->Core.Key;
2622
2623	/*
2624	* Page directory has been accessed (this is a fault situation, remember).
2625	*/
2626	pPDSrc->a[iPDSrc].n.u1Accessed = 1;
2627	if (fPageTable)
2628	{
2629	/*
2630	* Page table - 4KB.
2631	*
2632	* Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
2633	*/
2634	Log2(("SyncPT: 4K %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
2635	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
2636	PGSTPT pPTSrc;
2637	rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
2638	if (RT_SUCCESS(rc))
2639	{
2640	/*
2641	* Start by syncing the page directory entry so CSAM's TLB trick works.
2642	*/
2643	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| X86_PDE_AVL_MASK))
2644	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2645	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2646	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2647
2648	/*
2649	* Directory/page user or supervisor privilege: (same goes for read/write)
2650	*
2651	* Directory Page Combined
2652	* U/S U/S U/S
2653	* 0 0 0
2654	* 0 1 0
2655	* 1 0 0
2656	* 1 1 1
2657	*
2658	* Simple AND operation. Table listed for completeness.
2659	*
2660	*/
2661	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT4K));
2662	# ifdef PGM_SYNC_N_PAGES
2663	unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2664	unsigned iPTDst = iPTBase;
2665	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2666	if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
2667	iPTDst = 0;
2668	else
2669	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2670	# else /* !PGM_SYNC_N_PAGES */
2671	unsigned iPTDst = 0;
2672	const unsigned iPTDstEnd = RT_ELEMENTS(pPTDst->a);
2673	# endif /* !PGM_SYNC_N_PAGES */
2674	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2675	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2676	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2677	# else
2678	const unsigned offPTSrc = 0;
2679	# endif
2680	for (; iPTDst < iPTDstEnd; iPTDst++)
2681	{
2682	const unsigned iPTSrc = iPTDst + offPTSrc;
2683	const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2684
2685	if (PteSrc.n.u1Present)
2686	{
2687	# ifndef IN_RING0
2688	/*
2689	* Assuming kernel code will be marked as supervisor - and not as user level
2690	* and executed using a conforming code selector - And marked as readonly.
2691	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
2692	*/
2693	PPGMPAGE pPage;
2694	if ( ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW \| X86_PTE_US))
2695	\|\| !CSAMDoesPageNeedScanning(pVM, (iPDSrc << GST_PD_SHIFT) \| (iPTSrc << PAGE_SHIFT))
2696	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
2697	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2698	)
2699	# endif
2700	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2701	Log2(("SyncPT: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
2702	(RTGCPTR)(((RTGCPTR)iPDSrc << GST_PD_SHIFT) \| ((RTGCPTR)iPTSrc << PAGE_SHIFT)),
2703	PteSrc.n.u1Present,
2704	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2705	PteSrc.n.u1User & PdeSrc.n.u1User,
2706	(uint64_t)PteSrc.u,
2707	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : "", pPTDst->a[iPTDst].u, iPTSrc, PdeSrc.au32[0],
2708	(RTGCPHYS)((PdeSrc.u & GST_PDE_PG_MASK) + iPTSrc*sizeof(PteSrc)) ));
2709	}
2710	/* else: the page table was cleared by the pool */
2711	} /* for PTEs */
2712	}
2713	}
2714	else
2715	{
2716	/*
2717	* Big page - 2/4MB.
2718	*
2719	* We'll walk the ram range list in parallel and optimize lookups.
2720	* We will only sync on shadow page table at a time.
2721	*/
2722	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT4M));
2723
2724	/**
2725	* @todo It might be more efficient to sync only a part of the 4MB
2726	* page (similar to what we do for 4KB PDs).
2727	*/
2728
2729	/*
2730	* Start by syncing the page directory entry.
2731	*/
2732	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
2733	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2734
2735	/*
2736	* If the page is not flagged as dirty and is writable, then make it read-only
2737	* at PD level, so we can set the dirty bit when the page is modified.
2738	*
2739	* ASSUMES that page access handlers are implemented on page table entry level.
2740	* Thus we will first catch the dirty access and set PDE.D and restart. If
2741	* there is an access handler, we'll trap again and let it work on the problem.
2742	*/
2743	/** @todo move the above stuff to a section in the PGM documentation. */
2744	Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
2745	if ( !PdeSrc.b.u1Dirty
2746	&& PdeSrc.b.u1Write)
2747	{
2748	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2749	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2750	PdeDst.b.u1Write = 0;
2751	}
2752	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2753	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2754
2755	/*
2756	* Fill the shadow page table.
2757	*/
2758	/* Get address and flags from the source PDE. */
2759	SHWPTE PteDstBase;
2760	PteDstBase.u = PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT);
2761
2762	/* Loop thru the entries in the shadow PT. */
2763	const RTGCPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
2764	Log2(("SyncPT: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n",
2765	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
2766	GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2767	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
2768	unsigned iPTDst = 0;
2769	while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2770	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2771	{
2772	/* Advance ram range list. */
2773	while (pRam && GCPhys > pRam->GCPhysLast)
2774	pRam = pRam->CTX_SUFF(pNext);
2775	if (pRam && GCPhys >= pRam->GCPhys)
2776	{
2777	unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2778	do
2779	{
2780	/* Make shadow PTE. */
2781	PPGMPAGE pPage = &pRam->aPages[iHCPage];
2782	SHWPTE PteDst;
2783
2784	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2785	/* Try to make the page writable if necessary. */
2786	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2787	&& ( PGM_PAGE_IS_ZERO(pPage)
2788	\|\| ( PteDstBase.n.u1Write
2789	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2790	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
2791	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
2792	# endif
2793	# ifdef VBOX_WITH_PAGE_SHARING
2794	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
2795	# endif
2796	&& !PGM_PAGE_IS_BALLOONED(pPage))
2797	)
2798	)
2799	{
2800	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
2801	AssertRCReturn(rc, rc);
2802	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2803	break;
2804	}
2805	# endif
2806
2807	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2808	{
2809	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
2810	{
2811	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) \| PteDstBase.u;
2812	PteDst.n.u1Write = 0;
2813	}
2814	else
2815	PteDst.u = 0;
2816	}
2817	else if (PGM_PAGE_IS_BALLOONED(pPage))
2818	{
2819	/* Skip ballooned pages. */
2820	PteDst.u = 0;
2821	}
2822	# ifndef IN_RING0
2823	/*
2824	* Assuming kernel code will be marked as supervisor and not as user level and executed
2825	* using a conforming code selector. Don't check for readonly, as that implies the whole
2826	* 4MB can be code or readonly data. Linux enables write access for its large pages.
2827	*/
2828	else if ( !PdeSrc.n.u1User
2829	&& CSAMDoesPageNeedScanning(pVM, GCPtr \| (iPTDst << SHW_PT_SHIFT)))
2830	PteDst.u = 0;
2831	# endif
2832	else
2833	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) \| PteDstBase.u;
2834
2835	/* Only map writable pages writable. */
2836	if ( PteDst.n.u1Write
2837	&& PteDst.n.u1Present
2838	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2839	{
2840	/* Still applies to shared pages. */
2841	Assert(!PGM_PAGE_IS_ZERO(pPage));
2842	PteDst.n.u1Write = 0; /** @todo this isn't quite working yet... */
2843	Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))));
2844	}
2845
2846	if (PteDst.n.u1Present)
2847	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2848
2849	/* commit it */
2850	pPTDst->a[iPTDst] = PteDst;
2851	Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
2852	(RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT)), PteDst.n.u1Present, PteDst.n.u1Write, PteDst.n.u1User, (uint64_t)PteDst.u,
2853	PteDst.u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2854
2855	/* advance */
2856	GCPhys += PAGE_SIZE;
2857	iHCPage++;
2858	iPTDst++;
2859	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2860	&& GCPhys <= pRam->GCPhysLast);
2861	}
2862	else if (pRam)
2863	{
2864	Log(("Invalid pages at %RGp\n", GCPhys));
2865	do
2866	{
2867	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2868	GCPhys += PAGE_SIZE;
2869	iPTDst++;
2870	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2871	&& GCPhys < pRam->GCPhys);
2872	}
2873	else
2874	{
2875	Log(("Invalid pages at %RGp (2)\n", GCPhys));
2876	for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
2877	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2878	}
2879	} /* while more PTEs */
2880	} /* 4KB / 4MB */
2881	}
2882	else
2883	AssertRelease(!PdeDst.n.u1Present);
2884
2885	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2886	if (RT_FAILURE(rc))
2887	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPTFailed));
2888	return rc;
2889
2890	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2891	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2892	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2893	&& !defined(IN_RC)
2894
2895	/*
2896	* Validate input a little bit.
2897	*/
2898	int rc = VINF_SUCCESS;
2899	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2900	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2901	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
2902
2903	/* Fetch the pgm pool shadow descriptor. */
2904	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2905	Assert(pShwPde);
2906
2907	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2908	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2909	PPGMPOOLPAGE pShwPde = NULL; /* initialized to shut up gcc */
2910	PX86PDPAE pPDDst;
2911	PSHWPDE pPdeDst;
2912
2913	/* Fetch the pgm pool shadow descriptor. */
2914	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
2915	AssertRCSuccessReturn(rc, rc);
2916	Assert(pShwPde);
2917
2918	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
2919	pPdeDst = &pPDDst->a[iPDDst];
2920
2921	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2922	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2923	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2924	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2925	PX86PDPT pPdptDst= NULL; /* initialized to shut up gcc */
2926	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2927	AssertRCSuccessReturn(rc, rc);
2928	Assert(pPDDst);
2929	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2930
2931	/* Fetch the pgm pool shadow descriptor. */
2932	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2933	Assert(pShwPde);
2934
2935	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2936	const unsigned iPdpt = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
2937	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2938	PEPTPD pPDDst;
2939	PEPTPDPT pPdptDst;
2940
2941	rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, &pPdptDst, &pPDDst);
2942	if (rc != VINF_SUCCESS)
2943	{
2944	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2945	AssertRC(rc);
2946	return rc;
2947	}
2948	Assert(pPDDst);
2949	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2950
2951	/* Fetch the pgm pool shadow descriptor. */
2952	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & EPT_PDPTE_PG_MASK);
2953	Assert(pShwPde);
2954	# endif
2955	SHWPDE PdeDst = *pPdeDst;
2956
2957	Assert(!(PdeDst.u & PGM_PDFLAGS_MAPPING));
2958	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2959
2960	# if defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
2961	if (BTH_IS_NP_ACTIVE(pVM))
2962	{
2963	PPGMPAGE pPage;
2964
2965	/* Check if we allocated a big page before for this 2 MB range. */
2966	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPtrPage & X86_PDE2M_PAE_PG_MASK, &pPage);
2967	if (RT_SUCCESS(rc))
2968	{
2969	RTHCPHYS HCPhys = NIL_RTHCPHYS;
2970
2971	if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
2972	{
2973	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageReused);
2974	AssertRelease(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
2975	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2976	}
2977	else if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED)
2978	{
2979	/* Recheck the entire 2 MB range to see if we can use it again as a large page. */
2980	rc = pgmPhysIsValidLargePage(pVM, GCPtrPage, pPage);
2981	if (RT_SUCCESS(rc))
2982	{
2983	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
2984	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
2985	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2986	}
2987	}
2988	else if (PGMIsUsingLargePages(pVM))
2989	{
2990	rc = pgmPhysAllocLargePage(pVM, GCPtrPage);
2991	if (RT_SUCCESS(rc))
2992	{
2993	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
2994	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
2995	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2996	}
2997	else
2998	LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
2999	}
3000
3001	if (HCPhys != NIL_RTHCPHYS)
3002	{
3003	PdeDst.u &= X86_PDE_AVL_MASK;
3004	PdeDst.u \|= HCPhys;
3005	PdeDst.n.u1Present = 1;
3006	PdeDst.n.u1Write = 1;
3007	PdeDst.b.u1Size = 1;
3008	# if PGM_SHW_TYPE == PGM_TYPE_EPT
3009	PdeDst.n.u1Execute = 1;
3010	PdeDst.b.u1IgnorePAT = 1;
3011	PdeDst.b.u3EMT = VMX_EPT_MEMTYPE_WB;
3012	# else
3013	PdeDst.n.u1User = 1;
3014	# endif
3015	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
3016
3017	Log(("SyncPT: Use large page at %RGp PDE=%RX64\n", GCPtrPage, PdeDst.u));
3018	/* Add a reference to the first page only. */
3019	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPde, PGM_PAGE_GET_TRACKING(pPage), pPage, iPDDst);
3020
3021	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3022	return VINF_SUCCESS;
3023	}
3024	}
3025	}
3026	# endif /* HC_ARCH_BITS == 64 */
3027
3028	GSTPDE PdeSrc;
3029	PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3030	PdeSrc.n.u1Present = 1;
3031	PdeSrc.n.u1Write = 1;
3032	PdeSrc.n.u1Accessed = 1;
3033	PdeSrc.n.u1User = 1;
3034
3035	/*
3036	* Allocate & map the page table.
3037	*/
3038	PSHWPT pPTDst;
3039	PPGMPOOLPAGE pShwPage;
3040	RTGCPHYS GCPhys;
3041
3042	/* Virtual address = physical address */
3043	GCPhys = GCPtrPage & X86_PAGE_4K_BASE_MASK;
3044	rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
3045
3046	if ( rc == VINF_SUCCESS
3047	\|\| rc == VINF_PGM_CACHED_PAGE)
3048	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3049	else
3050	AssertMsgFailedReturn(("rc=%Rrc\n", rc), VERR_INTERNAL_ERROR);
3051
3052	PdeDst.u &= X86_PDE_AVL_MASK;
3053	PdeDst.u \|= pShwPage->Core.Key;
3054	PdeDst.n.u1Present = 1;
3055	PdeDst.n.u1Write = 1;
3056	# if PGM_SHW_TYPE == PGM_TYPE_EPT
3057	PdeDst.n.u1Execute = 1;
3058	# else
3059	PdeDst.n.u1User = 1;
3060	PdeDst.n.u1Accessed = 1;
3061	# endif
3062	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
3063
3064	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, PGM_SYNC_NR_PAGES, 0 /* page not present */);
3065	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3066	return rc;
3067
3068	#else
3069	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
3070	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3071	return VERR_INTERNAL_ERROR;
3072	#endif
3073	}
3074
3075
3076
3077	/**
3078	* Prefetch a page/set of pages.
3079	*
3080	* Typically used to sync commonly used pages before entering raw mode
3081	* after a CR3 reload.
3082	*
3083	* @returns VBox status code.
3084	* @param pVCpu The VMCPU handle.
3085	* @param GCPtrPage Page to invalidate.
3086	*/
3087	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
3088	{
3089	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3090	\|\| PGM_GST_TYPE == PGM_TYPE_REAL \
3091	\|\| PGM_GST_TYPE == PGM_TYPE_PROT \
3092	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3093	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3094	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
3095	&& PGM_SHW_TYPE != PGM_TYPE_EPT
3096
3097	/*
3098	* Check that all Guest levels thru the PDE are present, getting the
3099	* PD and PDE in the processes.
3100	*/
3101	int rc = VINF_SUCCESS;
3102	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3103	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3104	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
3105	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3106	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3107	unsigned iPDSrc;
3108	X86PDPE PdpeSrc;
3109	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3110	if (!pPDSrc)
3111	return VINF_SUCCESS; /* not present */
3112	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3113	unsigned iPDSrc;
3114	PX86PML4E pPml4eSrc;
3115	X86PDPE PdpeSrc;
3116	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3117	if (!pPDSrc)
3118	return VINF_SUCCESS; /* not present */
3119	# endif
3120	const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3121	# else
3122	PGSTPD pPDSrc = NULL;
3123	const unsigned iPDSrc = 0;
3124	GSTPDE PdeSrc;
3125
3126	PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3127	PdeSrc.n.u1Present = 1;
3128	PdeSrc.n.u1Write = 1;
3129	PdeSrc.n.u1Accessed = 1;
3130	PdeSrc.n.u1User = 1;
3131	# endif
3132
3133	if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
3134	{
3135	PVM pVM = pVCpu->CTX_SUFF(pVM);
3136	pgmLock(pVM);
3137
3138	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3139	const X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
3140	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3141	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3142	PX86PDPAE pPDDst;
3143	X86PDEPAE PdeDst;
3144	# if PGM_GST_TYPE != PGM_TYPE_PAE
3145	X86PDPE PdpeSrc;
3146
3147	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3148	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3149	# endif
3150	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3151	if (rc != VINF_SUCCESS)
3152	{
3153	pgmUnlock(pVM);
3154	AssertRC(rc);
3155	return rc;
3156	}
3157	Assert(pPDDst);
3158	PdeDst = pPDDst->a[iPDDst];
3159
3160	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3161	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3162	PX86PDPAE pPDDst;
3163	X86PDEPAE PdeDst;
3164
3165	# if PGM_GST_TYPE == PGM_TYPE_PROT
3166	/* AMD-V nested paging */
3167	X86PML4E Pml4eSrc;
3168	X86PDPE PdpeSrc;
3169	PX86PML4E pPml4eSrc = &Pml4eSrc;
3170
3171	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3172	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
3173	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
3174	# endif
3175
3176	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3177	if (rc != VINF_SUCCESS)
3178	{
3179	pgmUnlock(pVM);
3180	AssertRC(rc);
3181	return rc;
3182	}
3183	Assert(pPDDst);
3184	PdeDst = pPDDst->a[iPDDst];
3185	# endif
3186	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
3187	{
3188	if (!PdeDst.n.u1Present)
3189	{
3190	/** @todo r=bird: This guy will set the A bit on the PDE,
3191	* probably harmless. */
3192	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3193	}
3194	else
3195	{
3196	/* Note! We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
3197	* R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
3198	* makes no sense to prefetch more than one page.
3199	*/
3200	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3201	if (RT_SUCCESS(rc))
3202	rc = VINF_SUCCESS;
3203	}
3204	}
3205	pgmUnlock(pVM);
3206	}
3207	return rc;
3208
3209	#elif PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3210	return VINF_SUCCESS; /* ignore */
3211	#else
3212	AssertCompile(0);
3213	#endif
3214	}
3215
3216
3217
3218
3219	/**
3220	* Syncs a page during a PGMVerifyAccess() call.
3221	*
3222	* @returns VBox status code (informational included).
3223	* @param pVCpu The VMCPU handle.
3224	* @param GCPtrPage The address of the page to sync.
3225	* @param fPage The effective guest page flags.
3226	* @param uErr The trap error code.
3227	* @remarks This will normally never be called on invalid guest page
3228	* translation entries.
3229	*/
3230	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fPage, unsigned uErr)
3231	{
3232	PVM pVM = pVCpu->CTX_SUFF(pVM);
3233
3234	LogFlow(("VerifyAccessSyncPage: GCPtrPage=%RGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
3235
3236	Assert(!pVM->pgm.s.fNestedPaging);
3237	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3238	\|\| PGM_GST_TYPE == PGM_TYPE_REAL \
3239	\|\| PGM_GST_TYPE == PGM_TYPE_PROT \
3240	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3241	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3242	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
3243	&& PGM_SHW_TYPE != PGM_TYPE_EPT
3244
3245	# ifndef IN_RING0
3246	if (!(fPage & X86_PTE_US))
3247	{
3248	/*
3249	* Mark this page as safe.
3250	*/
3251	/** @todo not correct for pages that contain both code and data!! */
3252	Log(("CSAMMarkPage %RGv; scanned=%d\n", GCPtrPage, true));
3253	CSAMMarkPage(pVM, GCPtrPage, true);
3254	}
3255	# endif
3256
3257	/*
3258	* Get guest PD and index.
3259	*/
3260	/** @todo Performance: We've done all this a jiffy ago in the
3261	* PGMGstGetPage call. */
3262	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3263	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3264	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
3265	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3266
3267	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3268	unsigned iPDSrc = 0;
3269	X86PDPE PdpeSrc;
3270	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3271	if (RT_UNLIKELY(!pPDSrc))
3272	{
3273	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3274	return VINF_EM_RAW_GUEST_TRAP;
3275	}
3276
3277	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3278	unsigned iPDSrc = 0; /* shut up gcc */
3279	PX86PML4E pPml4eSrc = NULL; /* ditto */
3280	X86PDPE PdpeSrc;
3281	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3282	if (RT_UNLIKELY(!pPDSrc))
3283	{
3284	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3285	return VINF_EM_RAW_GUEST_TRAP;
3286	}
3287	# endif
3288
3289	# else /* !PGM_WITH_PAGING */
3290	PGSTPD pPDSrc = NULL;
3291	const unsigned iPDSrc = 0;
3292	# endif /* !PGM_WITH_PAGING */
3293	int rc = VINF_SUCCESS;
3294
3295	pgmLock(pVM);
3296
3297	/*
3298	* First check if the shadow pd is present.
3299	*/
3300	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3301	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3302
3303	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3304	PX86PDEPAE pPdeDst;
3305	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3306	PX86PDPAE pPDDst;
3307	# if PGM_GST_TYPE != PGM_TYPE_PAE
3308	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3309	X86PDPE PdpeSrc;
3310	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3311	# endif
3312	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3313	if (rc != VINF_SUCCESS)
3314	{
3315	pgmUnlock(pVM);
3316	AssertRC(rc);
3317	return rc;
3318	}
3319	Assert(pPDDst);
3320	pPdeDst = &pPDDst->a[iPDDst];
3321
3322	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3323	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3324	PX86PDPAE pPDDst;
3325	PX86PDEPAE pPdeDst;
3326
3327	# if PGM_GST_TYPE == PGM_TYPE_PROT
3328	/* AMD-V nested paging: Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3329	X86PML4E Pml4eSrc;
3330	X86PDPE PdpeSrc;
3331	PX86PML4E pPml4eSrc = &Pml4eSrc;
3332	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
3333	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
3334	# endif
3335
3336	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3337	if (rc != VINF_SUCCESS)
3338	{
3339	pgmUnlock(pVM);
3340	AssertRC(rc);
3341	return rc;
3342	}
3343	Assert(pPDDst);
3344	pPdeDst = &pPDDst->a[iPDDst];
3345	# endif
3346
3347	if (!pPdeDst->n.u1Present)
3348	{
3349	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3350	if (rc != VINF_SUCCESS)
3351	{
3352	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3353	pgmUnlock(pVM);
3354	AssertRC(rc);
3355	return rc;
3356	}
3357	}
3358
3359	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3360	/* Check for dirty bit fault */
3361	rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
3362	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
3363	Log(("PGMVerifyAccess: success (dirty)\n"));
3364	else
3365	# endif
3366	{
3367	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3368	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3369	# else
3370	GSTPDE PdeSrc;
3371	PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3372	PdeSrc.n.u1Present = 1;
3373	PdeSrc.n.u1Write = 1;
3374	PdeSrc.n.u1Accessed = 1;
3375	PdeSrc.n.u1User = 1;
3376	# endif
3377
3378	Assert(rc != VINF_EM_RAW_GUEST_TRAP);
3379	if (uErr & X86_TRAP_PF_US)
3380	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUser));
3381	else /* supervisor */
3382	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
3383
3384	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3385	if (RT_SUCCESS(rc))
3386	{
3387	/* Page was successfully synced */
3388	Log2(("PGMVerifyAccess: success (sync)\n"));
3389	rc = VINF_SUCCESS;
3390	}
3391	else
3392	{
3393	Log(("PGMVerifyAccess: access violation for %RGv rc=%Rrc\n", GCPtrPage, rc));
3394	rc = VINF_EM_RAW_GUEST_TRAP;
3395	}
3396	}
3397	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3398	pgmUnlock(pVM);
3399	return rc;
3400
3401	#else /* PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED */
3402
3403	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
3404	return VERR_INTERNAL_ERROR;
3405	#endif /* PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED */
3406	}
3407
3408
3409	/**
3410	* Syncs the paging hierarchy starting at CR3.
3411	*
3412	* @returns VBox status code, no specials.
3413	* @param pVCpu The VMCPU handle.
3414	* @param cr0 Guest context CR0 register
3415	* @param cr3 Guest context CR3 register
3416	* @param cr4 Guest context CR4 register
3417	* @param fGlobal Including global page directories or not
3418	*/
3419	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
3420	{
3421	PVM pVM = pVCpu->CTX_SUFF(pVM);
3422
3423	LogFlow(("SyncCR3 fGlobal=%d\n", !!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)));
3424
3425	#if PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
3426
3427	pgmLock(pVM);
3428
3429	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
3430	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3431	if (pPool->cDirtyPages)
3432	pgmPoolResetDirtyPages(pVM);
3433	# endif
3434
3435	/*
3436	* Update page access handlers.
3437	* The virtual are always flushed, while the physical are only on demand.
3438	* WARNING: We are incorrectly not doing global flushing on Virtual Handler updates. We'll
3439	* have to look into that later because it will have a bad influence on the performance.
3440	* @note SvL: There's no need for that. Just invalidate the virtual range(s).
3441	* bird: Yes, but that won't work for aliases.
3442	*/
3443	/** @todo this MUST go away. See #1557. */
3444	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3Handlers), h);
3445	PGM_GST_NAME(HandlerVirtualUpdate)(pVM, cr4);
3446	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3Handlers), h);
3447	pgmUnlock(pVM);
3448	#endif /* !NESTED && !EPT */
3449
3450	#if PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3451	/*
3452	* Nested / EPT - almost no work.
3453	*/
3454	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
3455	return VINF_SUCCESS;
3456
3457	#elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3458	/*
3459	* AMD64 (Shw & Gst) - No need to check all paging levels; we zero
3460	* out the shadow parts when the guest modifies its tables.
3461	*/
3462	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
3463	return VINF_SUCCESS;
3464
3465	#else /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3466
3467	# ifndef PGM_WITHOUT_MAPPINGS
3468	/*
3469	* Check for and resolve conflicts with our guest mappings if they
3470	* are enabled and not fixed.
3471	*/
3472	if (pgmMapAreMappingsFloating(&pVM->pgm.s))
3473	{
3474	int rc = pgmMapResolveConflicts(pVM);
3475	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
3476	if (rc == VINF_PGM_SYNC_CR3)
3477	{
3478	LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
3479	return VINF_PGM_SYNC_CR3;
3480	}
3481	}
3482	# else
3483	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
3484	# endif
3485	return VINF_SUCCESS;
3486	#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3487	}
3488
3489
3490
3491
3492	#ifdef VBOX_STRICT
3493	#ifdef IN_RC
3494	# undef AssertMsgFailed
3495	# define AssertMsgFailed Log
3496	#endif
3497	#ifdef IN_RING3
3498	# include <VBox/dbgf.h>
3499
3500	/**
3501	* Dumps a page table hierarchy use only physical addresses and cr4/lm flags.
3502	*
3503	* @returns VBox status code (VINF_SUCCESS).
3504	* @param cr3 The root of the hierarchy.
3505	* @param crr The cr4, only PAE and PSE is currently used.
3506	* @param fLongMode Set if long mode, false if not long mode.
3507	* @param cMaxDepth Number of levels to dump.
3508	* @param pHlp Pointer to the output functions.
3509	*/
3510	RT_C_DECLS_BEGIN
3511	VMMR3DECL(int) PGMR3DumpHierarchyHC(PVM pVM, uint32_t cr3, uint32_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp);
3512	RT_C_DECLS_END
3513
3514	#endif
3515
3516	/**
3517	* Checks that the shadow page table is in sync with the guest one.
3518	*
3519	* @returns The number of errors.
3520	* @param pVM The virtual machine.
3521	* @param pVCpu The VMCPU handle.
3522	* @param cr3 Guest context CR3 register
3523	* @param cr4 Guest context CR4 register
3524	* @param GCPtr Where to start. Defaults to 0.
3525	* @param cb How much to check. Defaults to everything.
3526	*/
3527	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)
3528	{
3529	#if PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3530	return 0;
3531	#else
3532	unsigned cErrors = 0;
3533	PVM pVM = pVCpu->CTX_SUFF(pVM);
3534	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3535
3536	#if PGM_GST_TYPE == PGM_TYPE_PAE
3537	/** @todo currently broken; crashes below somewhere */
3538	AssertFailed();
3539	#endif
3540
3541	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
3542	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3543	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
3544
3545	bool fBigPagesSupported = GST_IS_PSE_ACTIVE(pVCpu);
3546	PPGMCPU pPGM = &pVCpu->pgm.s;
3547	RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
3548	RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
3549	# ifndef IN_RING0
3550	RTHCPHYS HCPhys; /* general usage. */
3551	# endif
3552	int rc;
3553
3554	/*
3555	* Check that the Guest CR3 and all its mappings are correct.
3556	*/
3557	AssertMsgReturn(pPGM->GCPhysCR3 == (cr3 & GST_CR3_PAGE_MASK),
3558	("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
3559	false);
3560	# if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64
3561	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3562	rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGst32BitPdRC, NULL, &HCPhysShw);
3563	# else
3564	rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGstPaePdptRC, NULL, &HCPhysShw);
3565	# endif
3566	AssertRCReturn(rc, 1);
3567	HCPhys = NIL_RTHCPHYS;
3568	rc = pgmRamGCPhys2HCPhys(&pVM->pgm.s, cr3 & GST_CR3_PAGE_MASK, &HCPhys);
3569	AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
3570	# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
3571	pgmGstGet32bitPDPtr(pVCpu);
3572	RTGCPHYS GCPhys;
3573	rc = PGMR3DbgR3Ptr2GCPhys(pVM, pPGM->pGst32BitPdR3, &GCPhys);
3574	AssertRCReturn(rc, 1);
3575	AssertMsgReturn((cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
3576	# endif
3577	# endif /* !IN_RING0 */
3578
3579	/*
3580	* Get and check the Shadow CR3.
3581	*/
3582	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3583	unsigned cPDEs = X86_PG_ENTRIES;
3584	unsigned cIncrement = X86_PG_ENTRIES * PAGE_SIZE;
3585	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3586	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3587	unsigned cPDEs = X86_PG_PAE_ENTRIES * 4; /* treat it as a 2048 entry table. */
3588	# else
3589	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3590	# endif
3591	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3592	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3593	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3594	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3595	# endif
3596	if (cb != ~(RTGCPTR)0)
3597	cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
3598
3599	/** @todo call the other two PGMAssert() functions. /
3600
3601	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3602	unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3603
3604	for (; iPml4 < X86_PG_PAE_ENTRIES; iPml4++)
3605	{
3606	PPGMPOOLPAGE pShwPdpt = NULL;
3607	PX86PML4E pPml4eSrc;
3608	PX86PML4E pPml4eDst;
3609	RTGCPHYS GCPhysPdptSrc;
3610
3611	pPml4eSrc = pgmGstGetLongModePML4EPtr(pVCpu, iPml4);
3612	pPml4eDst = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
3613
3614	/* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
3615	if (!pPml4eDst->n.u1Present)
3616	{
3617	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3618	continue;
3619	}
3620
3621	pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
3622	GCPhysPdptSrc = pPml4eSrc->u & X86_PML4E_PG_MASK_FULL;
3623
3624	if (pPml4eSrc->n.u1Present != pPml4eDst->n.u1Present)
3625	{
3626	AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3627	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3628	cErrors++;
3629	continue;
3630	}
3631
3632	if (GCPhysPdptSrc != pShwPdpt->GCPhys)
3633	{
3634	AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
3635	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3636	cErrors++;
3637	continue;
3638	}
3639
3640	if ( pPml4eDst->n.u1User != pPml4eSrc->n.u1User
3641	\|\| pPml4eDst->n.u1Write != pPml4eSrc->n.u1Write
3642	\|\| pPml4eDst->n.u1NoExecute != pPml4eSrc->n.u1NoExecute)
3643	{
3644	AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3645	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3646	cErrors++;
3647	continue;
3648	}
3649	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3650	{
3651	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3652
3653	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
3654	/*
3655	* Check the PDPTEs too.
3656	*/
3657	unsigned iPdpt = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
3658
3659	for (;iPdpt <= SHW_PDPT_MASK; iPdpt++)
3660	{
3661	unsigned iPDSrc = 0; /* initialized to shut up gcc */
3662	PPGMPOOLPAGE pShwPde = NULL;
3663	PX86PDPE pPdpeDst;
3664	RTGCPHYS GCPhysPdeSrc;
3665	# if PGM_GST_TYPE == PGM_TYPE_PAE
3666	X86PDPE PdpeSrc;
3667	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtr, &iPDSrc, &PdpeSrc);
3668	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
3669	# else
3670	PX86PML4E pPml4eSrcIgn;
3671	X86PDPE PdpeSrc;
3672	PX86PDPT pPdptDst;
3673	PX86PDPAE pPDDst;
3674	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtr, &pPml4eSrcIgn, &PdpeSrc, &iPDSrc);
3675
3676	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtr, NULL, &pPdptDst, &pPDDst);
3677	if (rc != VINF_SUCCESS)
3678	{
3679	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
3680	GCPtr += 512 * _2M;
3681	continue; /* next PDPTE */
3682	}
3683	Assert(pPDDst);
3684	# endif
3685	Assert(iPDSrc == 0);
3686
3687	pPdpeDst = &pPdptDst->a[iPdpt];
3688
3689	if (!pPdpeDst->n.u1Present)
3690	{
3691	GCPtr += 512 * _2M;
3692	continue; /* next PDPTE */
3693	}
3694
3695	pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
3696	GCPhysPdeSrc = PdpeSrc.u & X86_PDPE_PG_MASK;
3697
3698	if (pPdpeDst->n.u1Present != PdpeSrc.n.u1Present)
3699	{
3700	AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3701	GCPtr += 512 * _2M;
3702	cErrors++;
3703	continue;
3704	}
3705
3706	if (GCPhysPdeSrc != pShwPde->GCPhys)
3707	{
3708	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3709	AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3710	# else
3711	AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3712	# endif
3713	GCPtr += 512 * _2M;
3714	cErrors++;
3715	continue;
3716	}
3717
3718	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3719	if ( pPdpeDst->lm.u1User != PdpeSrc.lm.u1User
3720	\|\| pPdpeDst->lm.u1Write != PdpeSrc.lm.u1Write
3721	\|\| pPdpeDst->lm.u1NoExecute != PdpeSrc.lm.u1NoExecute)
3722	{
3723	AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3724	GCPtr += 512 * _2M;
3725	cErrors++;
3726	continue;
3727	}
3728	# endif
3729
3730	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3731	{
3732	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3733	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3734	GSTPD const *pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3735	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3736	PCX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
3737	# endif
3738	# endif /* PGM_GST_TYPE == PGM_TYPE_32BIT */
3739	/*
3740	* Iterate the shadow page directory.
3741	*/
3742	GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
3743	unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
3744
3745	for (;
3746	iPDDst < cPDEs;
3747	iPDDst++, GCPtr += cIncrement)
3748	{
3749	# if PGM_SHW_TYPE == PGM_TYPE_PAE
3750	const SHWPDE PdeDst = *pgmShwGetPaePDEPtr(pVCpu, GCPtr);
3751	# else
3752	const SHWPDE PdeDst = pPDDst->a[iPDDst];
3753	# endif
3754	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
3755	{
3756	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
3757	if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
3758	{
3759	AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
3760	cErrors++;
3761	continue;
3762	}
3763	}
3764	else if ( (PdeDst.u & X86_PDE_P)
3765	\|\| ((PdeDst.u & (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY))
3766	)
3767	{
3768	HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
3769	PPGMPOOLPAGE pPoolPage = pgmPoolGetPage(pPool, HCPhysShw);
3770	if (!pPoolPage)
3771	{
3772	AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n",
3773	HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
3774	cErrors++;
3775	continue;
3776	}
3777	const SHWPT pPTDst = (const SHWPT )PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pPoolPage);
3778
3779	if (PdeDst.u & (X86_PDE4M_PWT \| X86_PDE4M_PCD))
3780	{
3781	AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
3782	GCPtr, (uint64_t)PdeDst.u));
3783	cErrors++;
3784	}
3785
3786	if (PdeDst.u & (X86_PDE4M_G \| X86_PDE4M_D))
3787	{
3788	AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n",
3789	GCPtr, (uint64_t)PdeDst.u));
3790	cErrors++;
3791	}
3792
3793	const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
3794	if (!PdeSrc.n.u1Present)
3795	{
3796	AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
3797	GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
3798	cErrors++;
3799	continue;
3800	}
3801
3802	if ( !PdeSrc.b.u1Size
3803	\|\| !fBigPagesSupported)
3804	{
3805	GCPhysGst = PdeSrc.u & GST_PDE_PG_MASK;
3806	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3807	GCPhysGst \|= (iPDDst & 1) * (PAGE_SIZE / 2);
3808	# endif
3809	}
3810	else
3811	{
3812	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3813	if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
3814	{
3815	AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n",
3816	GCPtr, (uint64_t)PdeSrc.u));
3817	cErrors++;
3818	continue;
3819	}
3820	# endif
3821	GCPhysGst = GST_GET_PDE_BIG_PG_GCPHYS(pVM, PdeSrc);
3822	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3823	GCPhysGst \|= GCPtr & RT_BIT(X86_PAGE_2M_SHIFT);
3824	# endif
3825	}
3826
3827	if ( pPoolPage->enmKind
3828	!= (!PdeSrc.b.u1Size \|\| !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
3829	{
3830	AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n",
3831	pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
3832	cErrors++;
3833	}
3834
3835	PPGMPAGE pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
3836	if (!pPhysPage)
3837	{
3838	AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3839	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3840	cErrors++;
3841	continue;
3842	}
3843
3844	if (GCPhysGst != pPoolPage->GCPhys)
3845	{
3846	AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n",
3847	GCPhysGst, pPoolPage->GCPhys, GCPtr));
3848	cErrors++;
3849	continue;
3850	}
3851
3852	if ( !PdeSrc.b.u1Size
3853	\|\| !fBigPagesSupported)
3854	{
3855	/*
3856	* Page Table.
3857	*/
3858	const GSTPT *pPTSrc;
3859	rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1), &pPTSrc);
3860	if (RT_FAILURE(rc))
3861	{
3862	AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3863	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3864	cErrors++;
3865	continue;
3866	}
3867	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/))
3868	!= (PdeDst.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/)))
3869	{
3870	/// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
3871	// (This problem will go away when/if we shadow multiple CR3s.)
3872	AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3873	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3874	cErrors++;
3875	continue;
3876	}
3877	if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3878	{
3879	AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n",
3880	GCPtr, (uint64_t)PdeDst.u));
3881	cErrors++;
3882	continue;
3883	}
3884
3885	/* iterate the page table. */
3886	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3887	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3888	const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
3889	# else
3890	const unsigned offPTSrc = 0;
3891	# endif
3892	for (unsigned iPT = 0, off = 0;
3893	iPT < RT_ELEMENTS(pPTDst->a);
3894	iPT++, off += PAGE_SIZE)
3895	{
3896	const SHWPTE PteDst = pPTDst->a[iPT];
3897
3898	/* skip not-present entries. */
3899	if (!(PteDst.u & (X86_PTE_P \| PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
3900	continue;
3901	Assert(PteDst.n.u1Present);
3902
3903	const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
3904	if (!PteSrc.n.u1Present)
3905	{
3906	# ifdef IN_RING3
3907	PGMAssertHandlerAndFlagsInSync(pVM);
3908	PGMR3DumpHierarchyGC(pVM, cr3, cr4, (PdeSrc.u & GST_PDE_PG_MASK));
3909	# endif
3910	AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
3911	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u, pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
3912	(PdeSrc.u & GST_PDE_PG_MASK) + (iPT + offPTSrc)*sizeof(PteSrc)));
3913	cErrors++;
3914	continue;
3915	}
3916
3917	uint64_t fIgnoreFlags = GST_PTE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_G \| X86_PTE_D \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
3918	# if 1 /** @todo sync accessed bit properly... */
3919	fIgnoreFlags \|= X86_PTE_A;
3920	# endif
3921
3922	/* match the physical addresses */
3923	HCPhysShw = PteDst.u & SHW_PTE_PG_MASK;
3924	GCPhysGst = PteSrc.u & GST_PTE_PG_MASK;
3925
3926	# ifdef IN_RING3
3927	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3928	if (RT_FAILURE(rc))
3929	{
3930	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
3931	{
3932	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3933	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3934	cErrors++;
3935	continue;
3936	}
3937	}
3938	else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
3939	{
3940	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3941	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3942	cErrors++;
3943	continue;
3944	}
3945	# endif
3946
3947	pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
3948	if (!pPhysPage)
3949	{
3950	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
3951	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
3952	{
3953	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3954	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3955	cErrors++;
3956	continue;
3957	}
3958	# endif
3959	if (PteDst.n.u1Write)
3960	{
3961	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3962	GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3963	cErrors++;
3964	}
3965	fIgnoreFlags \|= X86_PTE_RW;
3966	}
3967	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
3968	{
3969	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3970	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3971	cErrors++;
3972	continue;
3973	}
3974
3975	/* flags */
3976	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
3977	{
3978	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
3979	{
3980	if (PteDst.n.u1Write)
3981	{
3982	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
3983	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3984	cErrors++;
3985	continue;
3986	}
3987	fIgnoreFlags \|= X86_PTE_RW;
3988	}
3989	else
3990	{
3991	if ( PteDst.n.u1Present
3992	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
3993	&& !PGM_PAGE_IS_MMIO(pPhysPage)
3994	# endif
3995	)
3996	{
3997	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
3998	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3999	cErrors++;
4000	continue;
4001	}
4002	fIgnoreFlags \|= X86_PTE_P;
4003	}
4004	}
4005	else
4006	{
4007	if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
4008	{
4009	if (PteDst.n.u1Write)
4010	{
4011	AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
4012	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4013	cErrors++;
4014	continue;
4015	}
4016	if (!(PteDst.u & PGM_PTFLAGS_TRACK_DIRTY))
4017	{
4018	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4019	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4020	cErrors++;
4021	continue;
4022	}
4023	if (PteDst.n.u1Dirty)
4024	{
4025	AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4026	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4027	cErrors++;
4028	}
4029	# if 0 /** @todo sync access bit properly... */
4030	if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
4031	{
4032	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4033	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4034	cErrors++;
4035	}
4036	fIgnoreFlags \|= X86_PTE_RW;
4037	# else
4038	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
4039	# endif
4040	}
4041	else if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
4042	{
4043	/* access bit emulation (not implemented). */
4044	if (PteSrc.n.u1Accessed \|\| PteDst.n.u1Present)
4045	{
4046	AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
4047	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4048	cErrors++;
4049	continue;
4050	}
4051	if (!PteDst.n.u1Accessed)
4052	{
4053	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
4054	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4055	cErrors++;
4056	}
4057	fIgnoreFlags \|= X86_PTE_P;
4058	}
4059	# ifdef DEBUG_sandervl
4060	fIgnoreFlags \|= X86_PTE_D \| X86_PTE_A;
4061	# endif
4062	}
4063
4064	if ( (PteSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
4065	&& (PteSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags)
4066	)
4067	{
4068	AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
4069	GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
4070	fIgnoreFlags, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4071	cErrors++;
4072	continue;
4073	}
4074	} /* foreach PTE */
4075	}
4076	else
4077	{
4078	/*
4079	* Big Page.
4080	*/
4081	uint64_t fIgnoreFlags = X86_PDE_AVL_MASK \| GST_PDE_PG_MASK \| X86_PDE4M_G \| X86_PDE4M_D \| X86_PDE4M_PS \| X86_PDE4M_PWT \| X86_PDE4M_PCD;
4082	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
4083	{
4084	if (PdeDst.n.u1Write)
4085	{
4086	AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4087	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4088	cErrors++;
4089	continue;
4090	}
4091	if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
4092	{
4093	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4094	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4095	cErrors++;
4096	continue;
4097	}
4098	# if 0 /** @todo sync access bit properly... */
4099	if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
4100	{
4101	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4102	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4103	cErrors++;
4104	}
4105	fIgnoreFlags \|= X86_PTE_RW;
4106	# else
4107	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
4108	# endif
4109	}
4110	else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4111	{
4112	/* access bit emulation (not implemented). */
4113	if (PdeSrc.b.u1Accessed \|\| PdeDst.n.u1Present)
4114	{
4115	AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4116	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4117	cErrors++;
4118	continue;
4119	}
4120	if (!PdeDst.n.u1Accessed)
4121	{
4122	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4123	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4124	cErrors++;
4125	}
4126	fIgnoreFlags \|= X86_PTE_P;
4127	}
4128
4129	if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
4130	{
4131	AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
4132	GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
4133	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4134	cErrors++;
4135	}
4136
4137	/* iterate the page table. */
4138	for (unsigned iPT = 0, off = 0;
4139	iPT < RT_ELEMENTS(pPTDst->a);
4140	iPT++, off += PAGE_SIZE, GCPhysGst += PAGE_SIZE)
4141	{
4142	const SHWPTE PteDst = pPTDst->a[iPT];
4143
4144	if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
4145	{
4146	AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
4147	GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4148	cErrors++;
4149	}
4150
4151	/* skip not-present entries. */
4152	if (!PteDst.n.u1Present) /** @todo deal with ALL handlers and CSAM !P pages! */
4153	continue;
4154
4155	fIgnoreFlags = X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT \| X86_PTE_D \| X86_PTE_A \| X86_PTE_G \| X86_PTE_PAE_NX;
4156
4157	/* match the physical addresses */
4158	HCPhysShw = PteDst.u & X86_PTE_PAE_PG_MASK;
4159
4160	# ifdef IN_RING3
4161	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4162	if (RT_FAILURE(rc))
4163	{
4164	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4165	{
4166	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4167	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4168	cErrors++;
4169	}
4170	}
4171	else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
4172	{
4173	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4174	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4175	cErrors++;
4176	continue;
4177	}
4178	# endif
4179	pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
4180	if (!pPhysPage)
4181	{
4182	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4183	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4184	{
4185	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4186	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4187	cErrors++;
4188	continue;
4189	}
4190	# endif
4191	if (PteDst.n.u1Write)
4192	{
4193	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4194	GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4195	cErrors++;
4196	}
4197	fIgnoreFlags \|= X86_PTE_RW;
4198	}
4199	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4200	{
4201	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4202	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4203	cErrors++;
4204	continue;
4205	}
4206
4207	/* flags */
4208	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4209	{
4210	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4211	{
4212	if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
4213	{
4214	if (PteDst.n.u1Write)
4215	{
4216	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4217	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4218	cErrors++;
4219	continue;
4220	}
4221	fIgnoreFlags \|= X86_PTE_RW;
4222	}
4223	}
4224	else
4225	{
4226	if ( PteDst.n.u1Present
4227	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
4228	&& !PGM_PAGE_IS_MMIO(pPhysPage)
4229	# endif
4230	)
4231	{
4232	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4233	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4234	cErrors++;
4235	continue;
4236	}
4237	fIgnoreFlags \|= X86_PTE_P;
4238	}
4239	}
4240
4241	if ( (PdeSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
4242	&& (PdeSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags) /* lazy phys handler dereg. */
4243	)
4244	{
4245	AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
4246	GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
4247	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4248	cErrors++;
4249	continue;
4250	}
4251	} /* for each PTE */
4252	}
4253	}
4254	/* not present */
4255
4256	} /* for each PDE */
4257
4258	} /* for each PDPTE */
4259
4260	} /* for each PML4E */
4261
4262	# ifdef DEBUG
4263	if (cErrors)
4264	LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
4265	# endif
4266
4267	#endif /* GST == 32BIT, PAE or AMD64 */
4268	return cErrors;
4269
4270	#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT */
4271	}
4272	#endif /* VBOX_STRICT */
4273
4274
4275	/**
4276	* Sets up the CR3 for shadow paging
4277	*
4278	* @returns Strict VBox status code.
4279	* @retval VINF_SUCCESS.
4280	*
4281	* @param pVCpu The VMCPU handle.
4282	* @param GCPhysCR3 The physical address in the CR3 register.
4283	*/
4284	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3)
4285	{
4286	PVM pVM = pVCpu->CTX_SUFF(pVM);
4287
4288	/* Update guest paging info. */
4289	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
4290	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
4291	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
4292
4293	LogFlow(("MapCR3: %RGp\n", GCPhysCR3));
4294
4295	/*
4296	* Map the page CR3 points at.
4297	*/
4298	RTHCPTR HCPtrGuestCR3;
4299	RTHCPHYS HCPhysGuestCR3;
4300	pgmLock(pVM);
4301	PPGMPAGE pPageCR3 = pgmPhysGetPage(&pVM->pgm.s, GCPhysCR3);
4302	AssertReturn(pPageCR3, VERR_INTERNAL_ERROR_2);
4303	HCPhysGuestCR3 = PGM_PAGE_GET_HCPHYS(pPageCR3);
4304	/** @todo this needs some reworking wrt. locking? */
4305	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4306	HCPtrGuestCR3 = NIL_RTHCPTR;
4307	int rc = VINF_SUCCESS;
4308	# else
4309	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPageCR3, GCPhysCR3 & GST_CR3_PAGE_MASK, (void )&HCPtrGuestCR3); / @todo r=bird: This GCPhysCR3 masking isn't necessary. */
4310	# endif
4311	pgmUnlock(pVM);
4312	if (RT_SUCCESS(rc))
4313	{
4314	rc = PGMMap(pVM, (RTGCPTR)pVM->pgm.s.GCPtrCR3Mapping, HCPhysGuestCR3, PAGE_SIZE, 0);
4315	if (RT_SUCCESS(rc))
4316	{
4317	# ifdef IN_RC
4318	PGM_INVL_PG(pVCpu, pVM->pgm.s.GCPtrCR3Mapping);
4319	# endif
4320	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4321	pVCpu->pgm.s.pGst32BitPdR3 = (R3PTRTYPE(PX86PD))HCPtrGuestCR3;
4322	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4323	pVCpu->pgm.s.pGst32BitPdR0 = (R0PTRTYPE(PX86PD))HCPtrGuestCR3;
4324	# endif
4325	pVCpu->pgm.s.pGst32BitPdRC = (RCPTRTYPE(PX86PD))(RTRCUINTPTR)pVM->pgm.s.GCPtrCR3Mapping;
4326
4327	# elif PGM_GST_TYPE == PGM_TYPE_PAE
4328	unsigned off = GCPhysCR3 & GST_CR3_PAGE_MASK & PAGE_OFFSET_MASK;
4329	pVCpu->pgm.s.pGstPaePdptR3 = (R3PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4330	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4331	pVCpu->pgm.s.pGstPaePdptR0 = (R0PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4332	# endif
4333	pVCpu->pgm.s.pGstPaePdptRC = (RCPTRTYPE(PX86PDPT))((RTRCUINTPTR)pVM->pgm.s.GCPtrCR3Mapping + off);
4334	LogFlow(("Cached mapping %RRv\n", pVCpu->pgm.s.pGstPaePdptRC));
4335
4336	/*
4337	* Map the 4 PDs too.
4338	*/
4339	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pVCpu);
4340	RTGCPTR GCPtr = pVM->pgm.s.GCPtrCR3Mapping + PAGE_SIZE;
4341	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++, GCPtr += PAGE_SIZE)
4342	{
4343	if (pGuestPDPT->a[i].n.u1Present)
4344	{
4345	RTHCPTR HCPtr;
4346	RTHCPHYS HCPhys;
4347	RTGCPHYS GCPhys = pGuestPDPT->a[i].u & X86_PDPE_PG_MASK;
4348	pgmLock(pVM);
4349	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
4350	AssertReturn(pPage, VERR_INTERNAL_ERROR_2);
4351	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
4352	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4353	HCPtr = NIL_RTHCPTR;
4354	int rc2 = VINF_SUCCESS;
4355	# else
4356	int rc2 = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, (void **)&HCPtr);
4357	# endif
4358	pgmUnlock(pVM);
4359	if (RT_SUCCESS(rc2))
4360	{
4361	rc = PGMMap(pVM, GCPtr, HCPhys, PAGE_SIZE, 0);
4362	AssertRCReturn(rc, rc);
4363
4364	pVCpu->pgm.s.apGstPaePDsR3[i] = (R3PTRTYPE(PX86PDPAE))HCPtr;
4365	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4366	pVCpu->pgm.s.apGstPaePDsR0[i] = (R0PTRTYPE(PX86PDPAE))HCPtr;
4367	# endif
4368	pVCpu->pgm.s.apGstPaePDsRC[i] = (RCPTRTYPE(PX86PDPAE))(RTRCUINTPTR)GCPtr;
4369	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
4370	# ifdef IN_RC
4371	PGM_INVL_PG(pVCpu, GCPtr);
4372	# endif
4373	continue;
4374	}
4375	AssertMsgFailed(("pgmR3Gst32BitMapCR3: rc2=%d GCPhys=%RGp i=%d\n", rc2, GCPhys, i));
4376	}
4377
4378	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4379	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4380	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4381	# endif
4382	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4383	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4384	# ifdef IN_RC
4385	PGM_INVL_PG(pVCpu, GCPtr); /** @todo this shouldn't be necessary? */
4386	# endif
4387	}
4388
4389	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
4390	pVCpu->pgm.s.pGstAmd64Pml4R3 = (R3PTRTYPE(PX86PML4))HCPtrGuestCR3;
4391	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4392	pVCpu->pgm.s.pGstAmd64Pml4R0 = (R0PTRTYPE(PX86PML4))HCPtrGuestCR3;
4393	# endif
4394	# endif
4395	}
4396	else
4397	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4398	}
4399	else
4400	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4401
4402	#else /* prot/real stub */
4403	int rc = VINF_SUCCESS;
4404	#endif
4405
4406	/* Update shadow paging info for guest modes with paging (32, pae, 64). */
4407	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4408	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4409	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64) \
4410	&& ( PGM_GST_TYPE != PGM_TYPE_REAL \
4411	&& PGM_GST_TYPE != PGM_TYPE_PROT))
4412
4413	Assert(!pVM->pgm.s.fNestedPaging);
4414
4415	/*
4416	* Update the shadow root page as well since that's not fixed.
4417	*/
4418	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4419	PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
4420	uint32_t iOldShwUserTable = pVCpu->pgm.s.iShwUserTable;
4421	uint32_t iOldShwUser = pVCpu->pgm.s.iShwUser;
4422	PPGMPOOLPAGE pNewShwPageCR3;
4423
4424	pgmLock(pVM);
4425
4426	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4427	if (pPool->cDirtyPages)
4428	pgmPoolResetDirtyPages(pVM);
4429	# endif
4430
4431	Assert(!(GCPhysCR3 >> (PAGE_SHIFT + 32)));
4432	rc = pgmPoolAlloc(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, SHW_POOL_ROOT_IDX, GCPhysCR3 >> PAGE_SHIFT, &pNewShwPageCR3, true /* lock page */);
4433	AssertFatalRC(rc);
4434	rc = VINF_SUCCESS;
4435
4436	# ifdef IN_RC
4437	/*
4438	* WARNING! We can't deal with jumps to ring 3 in the code below as the
4439	* state will be inconsistent! Flush important things now while
4440	* we still can and then make sure there are no ring-3 calls.
4441	*/
4442	REMNotifyHandlerPhysicalFlushIfAlmostFull(pVM, pVCpu);
4443	VMMRZCallRing3Disable(pVCpu);
4444	# endif
4445
4446	pVCpu->pgm.s.iShwUser = SHW_POOL_ROOT_IDX;
4447	pVCpu->pgm.s.iShwUserTable = GCPhysCR3 >> PAGE_SHIFT;
4448	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3) = pNewShwPageCR3;
4449	# ifdef IN_RING0
4450	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4451	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4452	# elif defined(IN_RC)
4453	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4454	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4455	# else
4456	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4457	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4458	# endif
4459
4460	# ifndef PGM_WITHOUT_MAPPINGS
4461	/*
4462	* Apply all hypervisor mappings to the new CR3.
4463	* Note that SyncCR3 will be executed in case CR3 is changed in a guest paging mode; this will
4464	* make sure we check for conflicts in the new CR3 root.
4465	*/
4466	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4467	Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL) \|\| VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
4468	# endif
4469	rc = pgmMapActivateCR3(pVM, pNewShwPageCR3);
4470	AssertRCReturn(rc, rc);
4471	# endif
4472
4473	/* Set the current hypervisor CR3. */
4474	CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
4475	SELMShadowCR3Changed(pVM, pVCpu);
4476
4477	# ifdef IN_RC
4478	/* NOTE: The state is consistent again. */
4479	VMMRZCallRing3Enable(pVCpu);
4480	# endif
4481
4482	/* Clean up the old CR3 root. */
4483	if ( pOldShwPageCR3
4484	&& pOldShwPageCR3 != pNewShwPageCR3 /* @todo can happen due to incorrect syncing between REM & PGM; find the real cause */)
4485	{
4486	Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
4487	# ifndef PGM_WITHOUT_MAPPINGS
4488	/* Remove the hypervisor mappings from the shadow page table. */
4489	pgmMapDeactivateCR3(pVM, pOldShwPageCR3);
4490	# endif
4491	/* Mark the page as unlocked; allow flushing again. */
4492	pgmPoolUnlockPage(pPool, pOldShwPageCR3);
4493
4494	pgmPoolFreeByPage(pPool, pOldShwPageCR3, iOldShwUser, iOldShwUserTable);
4495	}
4496	pgmUnlock(pVM);
4497	# endif
4498
4499	return rc;
4500	}
4501
4502	/**
4503	* Unmaps the shadow CR3.
4504	*
4505	* @returns VBox status, no specials.
4506	* @param pVCpu The VMCPU handle.
4507	*/
4508	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu)
4509	{
4510	LogFlow(("UnmapCR3\n"));
4511
4512	int rc = VINF_SUCCESS;
4513	PVM pVM = pVCpu->CTX_SUFF(pVM);
4514
4515	/*
4516	* Update guest paging info.
4517	*/
4518	#if PGM_GST_TYPE == PGM_TYPE_32BIT
4519	pVCpu->pgm.s.pGst32BitPdR3 = 0;
4520	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4521	pVCpu->pgm.s.pGst32BitPdR0 = 0;
4522	# endif
4523	pVCpu->pgm.s.pGst32BitPdRC = 0;
4524
4525	#elif PGM_GST_TYPE == PGM_TYPE_PAE
4526	pVCpu->pgm.s.pGstPaePdptR3 = 0;
4527	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4528	pVCpu->pgm.s.pGstPaePdptR0 = 0;
4529	# endif
4530	pVCpu->pgm.s.pGstPaePdptRC = 0;
4531	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4532	{
4533	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4534	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4535	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4536	# endif
4537	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4538	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4539	}
4540
4541	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
4542	pVCpu->pgm.s.pGstAmd64Pml4R3 = 0;
4543	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4544	pVCpu->pgm.s.pGstAmd64Pml4R0 = 0;
4545	# endif
4546
4547	#else /* prot/real mode stub */
4548	/* nothing to do */
4549	#endif
4550
4551	#if !defined(IN_RC) /* In RC we rely on MapCR3 to do the shadow part for us at a safe time */
4552	/*
4553	* Update shadow paging info.
4554	*/
4555	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4556	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4557	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64))
4558
4559	# if PGM_GST_TYPE != PGM_TYPE_REAL
4560	Assert(!pVM->pgm.s.fNestedPaging);
4561	# endif
4562
4563	pgmLock(pVM);
4564
4565	# ifndef PGM_WITHOUT_MAPPINGS
4566	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4567	/* Remove the hypervisor mappings from the shadow page table. */
4568	pgmMapDeactivateCR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4569	# endif
4570
4571	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4572	{
4573	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4574
4575	Assert(pVCpu->pgm.s.iShwUser != PGMPOOL_IDX_NESTED_ROOT);
4576
4577	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4578	if (pPool->cDirtyPages)
4579	pgmPoolResetDirtyPages(pVM);
4580	# endif
4581
4582	/* Mark the page as unlocked; allow flushing again. */
4583	pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4584
4585	pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), pVCpu->pgm.s.iShwUser, pVCpu->pgm.s.iShwUserTable);
4586	pVCpu->pgm.s.pShwPageCR3R3 = 0;
4587	pVCpu->pgm.s.pShwPageCR3R0 = 0;
4588	pVCpu->pgm.s.pShwPageCR3RC = 0;
4589	pVCpu->pgm.s.iShwUser = 0;
4590	pVCpu->pgm.s.iShwUserTable = 0;
4591	}
4592	pgmUnlock(pVM);
4593	# endif
4594	#endif /* !IN_RC*/
4595
4596	return rc;
4597	}

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 31443

Download in other formats: