PGMAllBth.h@ 31768

Last change on this file since 31768 was 31657, checked in by vboxsync, 14 years ago
PGMPool: Drop cRefs since it's always 1.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 197.9 KB

Line
1	/* $Id: PGMAllBth.h 31657 2010-08-13 14:53:34Z vboxsync $ */
2	/** @file
3	* VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4	*
5	* @remarks The nested page tables on AMD makes use of PGM_SHW_TYPE in
6	* {PGM_TYPE_AMD64, PGM_TYPE_PAE and PGM_TYPE_32BIT} and PGM_GST_TYPE
7	* set to PGM_TYPE_PROT. Half of the code in this file is not
8	* exercised with PGM_SHW_TYPE set to PGM_TYPE_NESTED.
9	*
10	* @remarks Extended page tables (intel) are built with PGM_GST_TYPE set to
11	* PGM_TYPE_PROT (and PGM_SHW_TYPE set to PGM_TYPE_EPT).
12	*
13	* @remarks This file is one big \#ifdef-orgy!
14	*
15	*/
16
17	/*
18	* Copyright (C) 2006-2010 Oracle Corporation
19	*
20	* This file is part of VirtualBox Open Source Edition (OSE), as
21	* available from http://www.virtualbox.org. This file is free software;
22	* you can redistribute it and/or modify it under the terms of the GNU
23	* General Public License (GPL) as published by the Free Software
24	* Foundation, in version 2 as it comes in the "COPYING" file of the
25	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
26	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
27	*/
28
29
30	/*******************************************************************************
31	* Internal Functions *
32	*******************************************************************************/
33	RT_C_DECLS_BEGIN
34	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken);
35	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
36	static int PGM_BTH_NAME(SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr);
37	static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, RTGCPTR GCPtrPage);
38	static int PGM_BTH_NAME(SyncPT)(PVMCPU pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage);
39	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uErr);
40	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
41	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
42	#ifdef VBOX_STRICT
43	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
44	#endif
45	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3);
46	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu);
47	RT_C_DECLS_END
48
49
50	/*
51	* Filter out some illegal combinations of guest and shadow paging, so we can
52	* remove redundant checks inside functions.
53	*/
54	#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
55	# error "Invalid combination; PAE guest implies PAE shadow"
56	#endif
57
58	#if (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
59	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT)
60	# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
61	#endif
62
63	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) \
64	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT)
65	# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
66	#endif
67
68	#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT) \
69	\|\| (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
70	# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
71	#endif
72
73	#ifndef IN_RING3
74
75	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
76	/**
77	* Deal with a guest page fault.
78	*
79	* @returns Strict VBox status code.
80	* @retval VINF_EM_RAW_GUEST_TRAP
81	* @retval VINF_EM_RAW_EMULATE_INSTR
82	*
83	* @param pVCpu The current CPU.
84	* @param pGstWalk The guest page table walk result.
85	* @param uErr The error code.
86	*/
87	PGM_BTH_DECL(VBOXSTRICTRC, Trap0eHandlerGuestFault)(PVMCPU pVCpu, PGSTPTWALK pGstWalk, RTGCUINT uErr)
88	{
89	# if !defined(PGM_WITHOUT_MAPPINGS) && (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE)
90	/*
91	* Check for write conflicts with our hypervisor mapping.
92	*
93	* If the guest happens to access a non-present page, where our hypervisor
94	* is currently mapped, then we'll create a #PF storm in the guest.
95	*/
96	if ( (uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RW)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RW)
97	&& MMHyperIsInsideArea(pVCpu->CTX_SUFF(pVM), pGstWalk->Core.GCPtr))
98	{
99	/* Force a CR3 sync to check for conflicts and emulate the instruction. */
100	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
101	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
102	return VINF_EM_RAW_EMULATE_INSTR;
103	}
104	# endif
105
106	/*
107	* Calc the error code for the guest trap.
108	*/
109	uint32_t uNewErr = GST_IS_NX_ACTIVE(pVCpu)
110	? uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID)
111	: uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US);
112	if (pGstWalk->Core.fBadPhysAddr)
113	{
114	uNewErr \|= X86_TRAP_PF_RSVD \| X86_TRAP_PF_P;
115	Assert(!pGstWalk->Core.fNotPresent);
116	}
117	else if (!pGstWalk->Core.fNotPresent)
118	uNewErr \|= X86_TRAP_PF_P;
119	TRPMSetErrorCode(pVCpu, uNewErr);
120
121	LogFlow(("Guest trap; cr2=%RGv uErr=%RGv lvl=%d\n", pGstWalk->Core.GCPtr, uErr, pGstWalk->Core.uLevel));
122	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
123	return VINF_EM_RAW_GUEST_TRAP;
124	}
125	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
126
127
128	/**
129	* Deal with a guest page fault.
130	*
131	* The caller has taken the PGM lock.
132	*
133	* @returns Strict VBox status code.
134	*
135	* @param pVCpu The current CPU.
136	* @param uErr The error code.
137	* @param pRegFrame The register frame.
138	* @param pvFault The fault address.
139	* @param pPage The guest page at @a pvFault.
140	* @param pGstWalk The guest page table walk result.
141	* @param pfLockTaken PGM lock taken here or not (out). This is true
142	* when we're called.
143	*/
144	static VBOXSTRICTRC PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame,
145	RTGCPTR pvFault, PPGMPAGE pPage, bool *pfLockTaken
146	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
147	, PGSTPTWALK pGstWalk
148	# endif
149	)
150	{
151	# if !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
152	GSTPDE const PdeSrcDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
153	#endif
154	PVM pVM = pVCpu->CTX_SUFF(pVM);
155	int rc;
156
157	if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
158	{
159	/*
160	* Physical page access handler.
161	*/
162	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
163	const RTGCPHYS GCPhysFault = pGstWalk->Core.GCPhys;
164	# else
165	const RTGCPHYS GCPhysFault = (RTGCPHYS)pvFault;
166	# endif
167	PPGMPHYSHANDLER pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
168	if (pCur)
169	{
170	# ifdef PGM_SYNC_N_PAGES
171	/*
172	* If the region is write protected and we got a page not present fault, then sync
173	* the pages. If the fault was caused by a read, then restart the instruction.
174	* In case of write access continue to the GC write handler.
175	*
176	* ASSUMES that there is only one handler per page or that they have similar write properties.
177	*/
178	if ( pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
179	&& !(uErr & X86_TRAP_PF_P))
180	{
181	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
182	rc = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
183	# else
184	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
185	# endif
186	if ( RT_FAILURE(rc)
187	\|\| !(uErr & X86_TRAP_PF_RW)
188	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
189	{
190	AssertRC(rc);
191	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
192	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
193	return rc;
194	}
195	}
196	# endif
197
198	AssertMsg( pCur->enmType != PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
199	\|\| (pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE && (uErr & X86_TRAP_PF_RW)),
200	("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enum=%d\n",
201	pvFault, GCPhysFault, pPage, uErr, pCur->enmType));
202
203	if (pCur->CTX_SUFF(pfnHandler))
204	{
205	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
206	void *pvUser = pCur->CTX_SUFF(pvUser);
207	# ifdef IN_RING0
208	PFNPGMR0PHYSHANDLER pfnHandler = pCur->CTX_SUFF(pfnHandler);
209	# else
210	PFNPGMRCPHYSHANDLER pfnHandler = pCur->CTX_SUFF(pfnHandler);
211	# endif
212
213	STAM_PROFILE_START(&pCur->Stat, h);
214	if (pfnHandler != pPool->CTX_SUFF(pfnAccessHandler))
215	{
216	pgmUnlock(pVM);
217	*pfLockTaken = false;
218	}
219
220	rc = pfnHandler(pVM, uErr, pRegFrame, pvFault, GCPhysFault, pvUser);
221
222	# ifdef VBOX_WITH_STATISTICS
223	pgmLock(pVM);
224	pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
225	if (pCur)
226	STAM_PROFILE_STOP(&pCur->Stat, h);
227	pgmUnlock(pVM);
228	# endif
229	}
230	else
231	rc = VINF_EM_RAW_EMULATE_INSTR;
232
233	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysical);
234	if (uErr & X86_TRAP_PF_RSVD) STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysical);
235	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndPhys; });
236	return rc;
237	}
238	}
239	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
240	else
241	{
242	# ifdef PGM_SYNC_N_PAGES
243	/*
244	* If the region is write protected and we got a page not present fault, then sync
245	* the pages. If the fault was caused by a read, then restart the instruction.
246	* In case of write access continue to the GC write handler.
247	*/
248	if ( PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) < PGM_PAGE_HNDL_PHYS_STATE_ALL
249	&& !(uErr & X86_TRAP_PF_P))
250	{
251	rc = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
252	if ( RT_FAILURE(rc)
253	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
254	\|\| !(uErr & X86_TRAP_PF_RW))
255	{
256	AssertRC(rc);
257	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
258	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndVirt; });
259	return rc;
260	}
261	}
262	# endif
263	/*
264	* Ok, it's an virtual page access handler.
265	*
266	* Since it's faster to search by address, we'll do that first
267	* and then retry by GCPhys if that fails.
268	*/
269	/** @todo r=bird: perhaps we should consider looking up by physical address directly now?
270	* r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be
271	* out of sync, because the page was changed without us noticing it (not-present -> present
272	* without invlpg or mov cr3, xxx).
273	*/
274	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
275	if (pCur)
276	{
277	AssertMsg(!(pvFault - pCur->Core.Key < pCur->cb)
278	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
279	\|\| !(uErr & X86_TRAP_PF_P)
280	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
281	("Unexpected trap for virtual handler: %RGv (phys=%RGp) pPage=%R[pgmpage] uErr=%X, enum=%d\n",
282	pvFault, pGstWalk->Core.GCPhys, pPage, uErr, pCur->enmType));
283
284	if ( pvFault - pCur->Core.Key < pCur->cb
285	&& ( uErr & X86_TRAP_PF_RW
286	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
287	{
288	# ifdef IN_RC
289	STAM_PROFILE_START(&pCur->Stat, h);
290	RTGCPTR GCPtrStart = pCur->Core.Key;
291	CTX_MID(PFNPGM,VIRTHANDLER) pfnHandler = pCur->CTX_SUFF(pfnHandler);
292	pgmUnlock(pVM);
293	*pfLockTaken = false;
294
295	rc = pfnHandler(pVM, uErr, pRegFrame, pvFault, GCPtrStart, pvFault - GCPtrStart);
296
297	# ifdef VBOX_WITH_STATISTICS
298	pgmLock(pVM);
299	pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
300	if (pCur)
301	STAM_PROFILE_STOP(&pCur->Stat, h);
302	pgmUnlock(pVM);
303	# endif
304	# else
305	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
306	# endif
307	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersVirtual);
308	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
309	return rc;
310	}
311	/* Unhandled part of a monitored page */
312	}
313	else
314	{
315	/* Check by physical address. */
316	unsigned iPage;
317	rc = pgmHandlerVirtualFindByPhysAddr(pVM, pGstWalk->Core.GCPhys, &pCur, &iPage);
318	Assert(RT_SUCCESS(rc) \|\| !pCur);
319	if ( pCur
320	&& ( uErr & X86_TRAP_PF_RW
321	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
322	{
323	Assert((pCur->aPhysToVirt[iPage].Core.Key & X86_PTE_PAE_PG_MASK) == (pGstWalk->Core.GCPhys & X86_PTE_PAE_PG_MASK));
324	# ifdef IN_RC
325	STAM_PROFILE_START(&pCur->Stat, h);
326	RTGCPTR GCPtrStart = pCur->Core.Key;
327	CTX_MID(PFNPGM,VIRTHANDLER) pfnHandler = pCur->CTX_SUFF(pfnHandler);
328	pgmUnlock(pVM);
329	*pfLockTaken = false;
330
331	RTGCPTR off = (iPage << PAGE_SHIFT)
332	+ (pvFault & PAGE_OFFSET_MASK)
333	- (GCPtrStart & PAGE_OFFSET_MASK);
334	Assert(off < pCur->cb);
335	rc = pfnHandler(pVM, uErr, pRegFrame, pvFault, GCPtrStart, off);
336
337	# ifdef VBOX_WITH_STATISTICS
338	pgmLock(pVM);
339	pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, GCPtrStart);
340	if (pCur)
341	STAM_PROFILE_STOP(&pCur->Stat, h);
342	pgmUnlock(pVM);
343	# endif
344	# else
345	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
346	# endif
347	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersVirtualByPhys);
348	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
349	return rc;
350	}
351	}
352	}
353	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
354
355	/*
356	* There is a handled area of the page, but this fault doesn't belong to it.
357	* We must emulate the instruction.
358	*
359	* To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
360	* we first check if this was a page-not-present fault for a page with only
361	* write access handlers. Restart the instruction if it wasn't a write access.
362	*/
363	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersUnhandled);
364
365	if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
366	&& !(uErr & X86_TRAP_PF_P))
367	{
368	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
369	rc = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
370	# else
371	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
372	# endif
373	if ( RT_FAILURE(rc)
374	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
375	\|\| !(uErr & X86_TRAP_PF_RW))
376	{
377	AssertRC(rc);
378	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
379	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
380	return rc;
381	}
382	}
383
384	/** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
385	* It's writing to an unhandled part of the LDT page several million times.
386	*/
387	rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
388	LogFlow(("PGM: PGMInterpretInstruction -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
389	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndUnhandled; });
390	return rc;
391	} /* if any kind of handler */
392
393
394	/**
395	* #PF Handler for raw-mode guest execution.
396	*
397	* @returns VBox status code (appropriate for trap handling and GC return).
398	*
399	* @param pVCpu VMCPU Handle.
400	* @param uErr The trap error code.
401	* @param pRegFrame Trap register frame.
402	* @param pvFault The fault address.
403	* @param pfLockTaken PGM lock taken here or not (out)
404	*/
405	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken)
406	{
407	PVM pVM = pVCpu->CTX_SUFF(pVM);
408
409	*pfLockTaken = false;
410
411	# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \
412	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
413	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
414	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT)
415	int rc;
416
417	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
418	/*
419	* Walk the guest page translation tables and check if it's a guest fault.
420	*/
421	GSTPTWALK GstWalk;
422	rc = PGM_GST_NAME(Walk)(pVCpu, pvFault, &GstWalk);
423	if (RT_FAILURE_NP(rc))
424	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &GstWalk, uErr));
425
426	/* assert some GstWalk sanity. */
427	# if PGM_GST_TYPE == PGM_TYPE_AMD64
428	AssertMsg(GstWalk.Pml4e.u == GstWalk.pPml4e->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pml4e.u, (uint64_t)GstWalk.pPml4e->u));
429	# endif
430	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
431	AssertMsg(GstWalk.Pdpe.u == GstWalk.pPdpe->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pdpe.u, (uint64_t)GstWalk.pPdpe->u));
432	# endif
433	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u));
434	AssertMsg(GstWalk.Core.fBigPage \|\| GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u));
435	Assert(GstWalk.Core.fSucceeded);
436
437	if (uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID))
438	{
439	if ( ( (uErr & X86_TRAP_PF_RW)
440	&& !GstWalk.Core.fEffectiveRW
441	&& ( (uErr & X86_TRAP_PF_US)
442	\|\| CPUMIsGuestR0WriteProtEnabled(pVCpu)) )
443	\|\| ((uErr & X86_TRAP_PF_US) && !GstWalk.Core.fEffectiveUS)
444	\|\| ((uErr & X86_TRAP_PF_ID) && GstWalk.Core.fEffectiveNX)
445	)
446	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &GstWalk, uErr));
447	}
448
449	/*
450	* Set the accessed and dirty flags.
451	*/
452	# if PGM_GST_TYPE == PGM_TYPE_AMD64
453	GstWalk.Pml4e.u \|= X86_PML4E_A;
454	GstWalk.pPml4e->u \|= X86_PML4E_A;
455	GstWalk.Pdpe.u \|= X86_PDPE_A;
456	GstWalk.pPdpe->u \|= X86_PDPE_A;
457	# endif
458	if (GstWalk.Core.fBigPage)
459	{
460	Assert(GstWalk.Pde.b.u1Size);
461	if (uErr & X86_TRAP_PF_RW)
462	{
463	GstWalk.Pde.u \|= X86_PDE4M_A \| X86_PDE4M_D;
464	GstWalk.pPde->u \|= X86_PDE4M_A \| X86_PDE4M_D;
465	}
466	else
467	{
468	GstWalk.Pde.u \|= X86_PDE4M_A;
469	GstWalk.pPde->u \|= X86_PDE4M_A;
470	}
471	}
472	else
473	{
474	Assert(!GstWalk.Pde.b.u1Size);
475	GstWalk.Pde.u \|= X86_PDE_A;
476	GstWalk.pPde->u \|= X86_PDE_A;
477	if (uErr & X86_TRAP_PF_RW)
478	{
479	# ifdef VBOX_WITH_STATISTICS
480	if (!GstWalk.Pte.n.u1Dirty)
481	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtiedPage));
482	else
483	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageAlreadyDirty));
484	# endif
485	GstWalk.Pte.u \|= X86_PTE_A \| X86_PTE_D;
486	GstWalk.pPte->u \|= X86_PTE_A \| X86_PTE_D;
487	}
488	else
489	{
490	GstWalk.Pte.u \|= X86_PTE_A;
491	GstWalk.pPte->u \|= X86_PTE_A;
492	}
493	Assert(GstWalk.Pte.u == GstWalk.pPte->u);
494	}
495	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u \|\| GstWalk.pPte->u == GstWalk.pPde->u,
496	("%RX64 %RX64 pPte=%p pPde=%p Pte=%RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u, GstWalk.pPte, GstWalk.pPde, (uint64_t)GstWalk.pPte->u));
497	# else /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
498	GSTPDE const PdeSrcDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A}; /** @todo eliminate this */
499	# endif /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
500
501	/* Take the big lock now. */
502	*pfLockTaken = true;
503	pgmLock(pVM);
504
505	# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
506	/*
507	* If it is a reserved bit fault we know that it is an MMIO (access
508	* handler) related fault and can skip some 200 lines of code.
509	*/
510	if (uErr & X86_TRAP_PF_RSVD)
511	{
512	Assert(uErr & X86_TRAP_PF_P);
513	PPGMPAGE pPage;
514	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
515	rc = pgmPhysGetPageEx(&pVM->pgm.s, GstWalk.Core.GCPhys, &pPage);
516	if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
517	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
518	pfLockTaken, &GstWalk));
519	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
520	# else
521	rc = pgmPhysGetPageEx(&pVM->pgm.s, (RTGCPHYS)pvFault, &pPage);
522	if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
523	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
524	pfLockTaken));
525	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
526	# endif
527	AssertRC(rc);
528	}
529	# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
530
531	/*
532	* Fetch the guest PDE, PDPE and PML4E.
533	*/
534	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
535	const unsigned iPDDst = pvFault >> SHW_PD_SHIFT;
536	PX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
537
538	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
539	const unsigned iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK; /* pPDDst index, not used with the pool. */
540	PX86PDPAE pPDDst;
541	# if PGM_GST_TYPE == PGM_TYPE_PAE
542	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, GstWalk.Pdpe.u, &pPDDst);
543	# else
544	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, X86_PDPE_P, &pPDDst); /* RW, US and A are reserved in PAE mode. */
545	# endif
546	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_INTERNAL_ERROR_4);
547
548	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
549	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
550	PX86PDPAE pPDDst;
551	# if PGM_GST_TYPE == PGM_TYPE_PROT /* (AMD-V nested paging) */
552	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A,
553	X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A, &pPDDst);
554	# else
555	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, GstWalk.Pml4e.u, GstWalk.Pdpe.u, &pPDDst);
556	# endif
557	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_INTERNAL_ERROR_4);
558
559	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
560	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
561	PEPTPD pPDDst;
562	rc = pgmShwGetEPTPDPtr(pVCpu, pvFault, NULL, &pPDDst);
563	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_INTERNAL_ERROR_4);
564	# endif
565	Assert(pPDDst);
566
567	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
568	/*
569	* Dirty page handling.
570	*
571	* If we successfully correct the write protection fault due to dirty bit
572	* tracking, then return immediately.
573	*/
574	if (uErr & X86_TRAP_PF_RW) /* write fault? */
575	{
576	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyBitTracking), a);
577	rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, &pPDDst->a[iPDDst], GstWalk.pPde, pvFault);
578	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyBitTracking), a);
579	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
580	{
581	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution)
582	= rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
583	? &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2DirtyAndAccessed
584	: &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
585	LogBird(("Trap0eHandler: returns VINF_SUCCESS\n"));
586	return VINF_SUCCESS;
587	}
588	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u \|\| GstWalk.pPte->u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u));
589	AssertMsg(GstWalk.Core.fBigPage \|\| GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u));
590	}
591
592	# if 0 /* rarely useful; leave for debugging. */
593	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0ePD[iPDSrc]);
594	# endif
595	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
596
597	/*
598	* A common case is the not-present error caused by lazy page table syncing.
599	*
600	* It is IMPORTANT that we weed out any access to non-present shadow PDEs
601	* here so we can safely assume that the shadow PT is present when calling
602	* SyncPage later.
603	*
604	* On failure, we ASSUME that SyncPT is out of memory or detected some kind
605	* of mapping conflict and defer to SyncCR3 in R3.
606	* (Again, we do NOT support access handlers for non-present guest pages.)
607	*
608	*/
609	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
610	Assert(GstWalk.Pde.n.u1Present);
611	# endif
612	if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
613	&& !pPDDst->a[iPDDst].n.u1Present)
614	{
615	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2SyncPT; });
616	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
617	LogFlow(("=>SyncPT %04x = %08RX64\n", (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, (uint64_t)GstWalk.Pde.u));
618	rc = PGM_BTH_NAME(SyncPT)(pVCpu, (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, GstWalk.pPd, pvFault);
619	# else
620	LogFlow(("=>SyncPT pvFault=%RGv\n", pvFault));
621	rc = PGM_BTH_NAME(SyncPT)(pVCpu, 0, NULL, pvFault);
622	# endif
623	if (RT_SUCCESS(rc))
624	return rc;
625	Log(("SyncPT: %RGv failed!! rc=%Rrc\n", pvFault, rc));
626	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
627	return VINF_PGM_SYNC_CR3;
628	}
629
630	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(PGM_WITHOUT_MAPPINGS)
631	/*
632	* Check if this address is within any of our mappings.
633	*
634	* This is very fast and it's gonna save us a bit of effort below and prevent
635	* us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
636	* (BTW, it's impossible to have physical access handlers in a mapping.)
637	*/
638	if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
639	{
640	PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
641	for ( ; pMapping; pMapping = pMapping->CTX_SUFF(pNext))
642	{
643	if (pvFault < pMapping->GCPtr)
644	break;
645	if (pvFault - pMapping->GCPtr < pMapping->cb)
646	{
647	/*
648	* The first thing we check is if we've got an undetected conflict.
649	*/
650	if (pgmMapAreMappingsFloating(&pVM->pgm.s))
651	{
652	unsigned iPT = pMapping->cb >> GST_PD_SHIFT;
653	while (iPT-- > 0)
654	if (GstWalk.pPde[iPT].n.u1Present)
655	{
656	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eConflicts);
657	Log(("Trap0e: Detected Conflict %RGv-%RGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
658	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
659	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
660	return VINF_PGM_SYNC_CR3;
661	}
662	}
663
664	/*
665	* Check if the fault address is in a virtual page access handler range.
666	*/
667	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->HyperVirtHandlers, pvFault);
668	if ( pCur
669	&& pvFault - pCur->Core.Key < pCur->cb
670	&& uErr & X86_TRAP_PF_RW)
671	{
672	# ifdef IN_RC
673	STAM_PROFILE_START(&pCur->Stat, h);
674	pgmUnlock(pVM);
675	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
676	pgmLock(pVM);
677	STAM_PROFILE_STOP(&pCur->Stat, h);
678	# else
679	AssertFailed();
680	rc = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
681	# endif
682	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersMapping);
683	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
684	return rc;
685	}
686
687	/*
688	* Pretend we're not here and let the guest handle the trap.
689	*/
690	TRPMSetErrorCode(pVCpu, uErr & ~X86_TRAP_PF_P);
691	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eGuestPFMapping);
692	LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
693	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
694	return VINF_EM_RAW_GUEST_TRAP;
695	}
696	}
697	} /* pgmAreMappingsEnabled(&pVM->pgm.s) */
698	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
699
700	/*
701	* Check if this fault address is flagged for special treatment,
702	* which means we'll have to figure out the physical address and
703	* check flags associated with it.
704	*
705	* ASSUME that we can limit any special access handling to pages
706	* in page tables which the guest believes to be present.
707	*/
708	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
709	RTGCPHYS GCPhys = GstWalk.Core.GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
710	# else
711	RTGCPHYS GCPhys = (RTGCPHYS)pvFault & ~(RTGCPHYS)PAGE_OFFSET_MASK;
712	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
713	PPGMPAGE pPage;
714	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
715	if (RT_FAILURE(rc))
716	{
717	/*
718	* When the guest accesses invalid physical memory (e.g. probing
719	* of RAM or accessing a remapped MMIO range), then we'll fall
720	* back to the recompiler to emulate the instruction.
721	*/
722	LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhys, rc));
723	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersInvalid);
724	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2InvalidPhys; });
725	return VINF_EM_RAW_EMULATE_INSTR;
726	}
727
728	/*
729	* Any handlers for this page?
730	*/
731	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
732	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
733	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, pfLockTaken,
734	&GstWalk));
735	# else
736	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, pfLockTaken));
737	# endif
738
739	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTimeOutOfSync, c);
740
741	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
742	if (uErr & X86_TRAP_PF_P)
743	{
744	/*
745	* The page isn't marked, but it might still be monitored by a virtual page access handler.
746	* (ASSUMES no temporary disabling of virtual handlers.)
747	*/
748	/** @todo r=bird: Since the purpose is to catch out of sync pages with virtual handler(s) here,
749	* we should correct both the shadow page table and physical memory flags, and not only check for
750	* accesses within the handler region but for access to pages with virtual handlers. */
751	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
752	if (pCur)
753	{
754	AssertMsg( !(pvFault - pCur->Core.Key < pCur->cb)
755	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
756	\|\| !(uErr & X86_TRAP_PF_P)
757	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
758	("Unexpected trap for virtual handler: %08X (phys=%08x) %R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
759
760	if ( pvFault - pCur->Core.Key < pCur->cb
761	&& ( uErr & X86_TRAP_PF_RW
762	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
763	{
764	# ifdef IN_RC
765	STAM_PROFILE_START(&pCur->Stat, h);
766	pgmUnlock(pVM);
767	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
768	pgmLock(pVM);
769	STAM_PROFILE_STOP(&pCur->Stat, h);
770	# else
771	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
772	# endif
773	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
774	return rc;
775	}
776	}
777	}
778	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
779
780	/*
781	* We are here only if page is present in Guest page tables and
782	* trap is not handled by our handlers.
783	*
784	* Check it for page out-of-sync situation.
785	*/
786	if (!(uErr & X86_TRAP_PF_P))
787	{
788	/*
789	* Page is not present in our page tables. Try to sync it!
790	*/
791	if (uErr & X86_TRAP_PF_US)
792	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUser));
793	else /* supervisor */
794	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
795
796	if (PGM_PAGE_IS_BALLOONED(pPage))
797	{
798	/* Emulate reads from ballooned pages as they are not present in
799	our shadow page tables. (Required for e.g. Solaris guests; soft
800	ecc, random nr generator.) */
801	rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
802	LogFlow(("PGM: PGMInterpretInstruction balloon -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
803	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncBallloon));
804	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Ballooned; });
805	return rc;
806	}
807
808	# if defined(LOG_ENABLED) && !defined(IN_RING0)
809	RTGCPHYS GCPhys2;
810	uint64_t fPageGst2;
811	PGMGstGetPage(pVCpu, pvFault, &fPageGst2, &GCPhys2);
812	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
813	Log(("Page out of sync: %RGv eip=%08x PdeSrc.US=%d fPageGst2=%08llx GCPhys2=%RGp scan=%d\n",
814	pvFault, pRegFrame->eip, GstWalk.Pde.n.u1User, fPageGst2, GCPhys2, CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)));
815	# else
816	Log(("Page out of sync: %RGv eip=%08x fPageGst2=%08llx GCPhys2=%RGp scan=%d\n",
817	pvFault, pRegFrame->eip, fPageGst2, GCPhys2, CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)));
818	# endif
819	# endif /* LOG_ENABLED */
820
821	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
822	if ( !GstWalk.Core.fEffectiveUS
823	&& CPUMGetGuestCPL(pVCpu, pRegFrame) == 0)
824	{
825	/* Note: Can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU. */
826	if ( pvFault == (RTGCPTR)pRegFrame->eip
827	\|\| pvFault - pRegFrame->eip < 8 /* instruction crossing a page boundary */
828	# ifdef CSAM_DETECT_NEW_CODE_PAGES
829	\|\| ( !PATMIsPatchGCAddr(pVM, pRegFrame->eip)
830	&& CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)) /* any new code we encounter here */
831	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
832	)
833	{
834	LogFlow(("CSAMExecFault %RX32\n", pRegFrame->eip));
835	rc = CSAMExecFault(pVM, (RTRCPTR)pRegFrame->eip);
836	if (rc != VINF_SUCCESS)
837	{
838	/*
839	* CSAM needs to perform a job in ring 3.
840	*
841	* Sync the page before going to the host context; otherwise we'll end up in a loop if
842	* CSAM fails (e.g. instruction crosses a page boundary and the next page is not present)
843	*/
844	LogFlow(("CSAM ring 3 job\n"));
845	int rc2 = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
846	AssertRC(rc2);
847
848	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2CSAM; });
849	return rc;
850	}
851	}
852	# ifdef CSAM_DETECT_NEW_CODE_PAGES
853	else if ( uErr == X86_TRAP_PF_RW
854	&& pRegFrame->ecx >= 0x100 /* early check for movswd count */
855	&& pRegFrame->ecx < 0x10000)
856	{
857	/* In case of a write to a non-present supervisor shadow page, we'll take special precautions
858	* to detect loading of new code pages.
859	*/
860
861	/*
862	* Decode the instruction.
863	*/
864	RTGCPTR PC;
865	rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs,
866	&pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &PC);
867	if (rc == VINF_SUCCESS)
868	{
869	PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
870	uint32_t cbOp;
871	rc = EMInterpretDisasOneEx(pVM, pVCpu, PC, pRegFrame, pDis, &cbOp);
872
873	/* For now we'll restrict this to rep movsw/d instructions */
874	if ( rc == VINF_SUCCESS
875	&& pDis->pCurInstr->opcode == OP_MOVSWD
876	&& (pDis->prefix & PREFIX_REP))
877	{
878	CSAMMarkPossibleCodePage(pVM, pvFault);
879	}
880	}
881	}
882	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
883
884	/*
885	* Mark this page as safe.
886	*/
887	/** @todo not correct for pages that contain both code and data!! */
888	Log2(("CSAMMarkPage %RGv; scanned=%d\n", pvFault, true));
889	CSAMMarkPage(pVM, pvFault, true);
890	}
891	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0) */
892	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
893	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
894	# else
895	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
896	# endif
897	if (RT_SUCCESS(rc))
898	{
899	/* The page was successfully synced, return to the guest. */
900	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSync; });
901	return VINF_SUCCESS;
902	}
903	}
904	else /* uErr & X86_TRAP_PF_P: */
905	{
906	/*
907	* Write protected pages are made writable when the guest makes the
908	* first write to it. This happens for pages that are shared, write
909	* monitored or not yet allocated.
910	*
911	* We may also end up here when CR0.WP=0 in the guest.
912	*
913	* Also, a side effect of not flushing global PDEs are out of sync
914	* pages due to physical monitored regions, that are no longer valid.
915	* Assume for now it only applies to the read/write flag.
916	*/
917	if (uErr & X86_TRAP_PF_RW)
918	{
919	/*
920	* Check if it is a read-only page.
921	*/
922	if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
923	{
924	Log(("PGM #PF: Make writable: %RGp %R[pgmpage] pvFault=%RGp uErr=%#x\n", GCPhys, pPage, pvFault, uErr));
925	Assert(!PGM_PAGE_IS_ZERO(pPage));
926	AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
927	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2MakeWritable; });
928
929	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
930	if (rc != VINF_SUCCESS)
931	{
932	AssertMsg(rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("%Rrc\n", rc));
933	return rc;
934	}
935	if (RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)))
936	return VINF_EM_NO_MEMORY;
937	}
938
939	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
940	/*
941	* Check to see if we need to emulate the instruction if CR0.WP=0.
942	*/
943	if ( !GstWalk.Core.fEffectiveRW
944	&& (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG
945	&& CPUMGetGuestCPL(pVCpu, pRegFrame) == 0)
946	{
947	Assert((uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RW \| X86_TRAP_PF_P));
948	rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
949	if (RT_SUCCESS(rc))
950	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eWPEmulInRZ);
951	else
952	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eWPEmulToR3);
953	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2WPEmulation; });
954	return rc;
955	}
956	# endif
957	/// @todo count the above case; else
958	if (uErr & X86_TRAP_PF_US)
959	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUserWrite));
960	else /* supervisor */
961	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
962
963	/*
964	* Sync the page.
965	*
966	* Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
967	* page is not present, which is not true in this case.
968	*/
969	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
970	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
971	# else
972	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
973	# endif
974	if (RT_SUCCESS(rc))
975	{
976	/*
977	* Page was successfully synced, return to guest but invalidate
978	* the TLB first as the page is very likely to be in it.
979	*/
980	# if PGM_SHW_TYPE == PGM_TYPE_EPT
981	HWACCMInvalidatePhysPage(pVM, (RTGCPHYS)pvFault);
982	# else
983	PGM_INVL_PG(pVCpu, pvFault);
984	# endif
985	# ifdef VBOX_STRICT
986	RTGCPHYS GCPhys2;
987	uint64_t fPageGst;
988	if (!pVM->pgm.s.fNestedPaging)
989	{
990	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, &GCPhys2);
991	AssertMsg(RT_SUCCESS(rc) && (fPageGst & X86_PTE_RW), ("rc=%Rrc fPageGst=%RX64\n", rc, fPageGst));
992	LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GCPhys2, (uint64_t)fPageGst));
993	}
994	uint64_t fPageShw;
995	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
996	AssertMsg((RT_SUCCESS(rc) && (fPageShw & X86_PTE_RW)) \|\| pVM->cCpus > 1 /* new monitor can be installed/page table flushed between the trap exit and PGMTrap0eHandler */, ("rc=%Rrc fPageShw=%RX64\n", rc, fPageShw));
997	# endif /* VBOX_STRICT */
998	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndObs; });
999	return VINF_SUCCESS;
1000	}
1001	}
1002	/** @todo else: why are we here? */
1003
1004	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && defined(VBOX_STRICT)
1005	/*
1006	* Check for VMM page flags vs. Guest page flags consistency.
1007	* Currently only for debug purposes.
1008	*/
1009	if (RT_SUCCESS(rc))
1010	{
1011	/* Get guest page flags. */
1012	uint64_t fPageGst;
1013	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
1014	if (RT_SUCCESS(rc))
1015	{
1016	uint64_t fPageShw;
1017	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
1018
1019	/*
1020	* Compare page flags.
1021	* Note: we have AVL, A, D bits desynched.
1022	*/
1023	AssertMsg((fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)),
1024	("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64\n", pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst));
1025	}
1026	else
1027	AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc));
1028	}
1029	else
1030	AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc));
1031	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && VBOX_STRICT */
1032	}
1033
1034
1035	/*
1036	* If we get here it is because something failed above, i.e. most like guru
1037	* meditiation time.
1038	*/
1039	LogRel(("%s: returns rc=%Rrc pvFault=%RGv uErr=%RX64 cs:rip=%04x:%08RX64\n",
1040	__PRETTY_FUNCTION__, rc, pvFault, (uint64_t)uErr, pRegFrame->cs, pRegFrame->rip));
1041	return rc;
1042
1043	# else /* Nested paging, EPT except PGM_GST_TYPE = PROT */
1044	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
1045	return VERR_INTERNAL_ERROR;
1046	# endif
1047	}
1048	#endif /* !IN_RING3 */
1049
1050
1051	/**
1052	* Emulation of the invlpg instruction.
1053	*
1054	*
1055	* @returns VBox status code.
1056	*
1057	* @param pVCpu The VMCPU handle.
1058	* @param GCPtrPage Page to invalidate.
1059	*
1060	* @remark ASSUMES that the guest is updating before invalidating. This order
1061	* isn't required by the CPU, so this is speculative and could cause
1062	* trouble.
1063	* @remark No TLB shootdown is done on any other VCPU as we assume that
1064	* invlpg emulation is the only reason for calling this function.
1065	* (The guest has to shoot down TLB entries on other CPUs itself)
1066	* Currently true, but keep in mind!
1067	*
1068	* @todo Clean this up! Most of it is (or should be) no longer necessary as we catch all page table accesses.
1069	*/
1070	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
1071	{
1072	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1073	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
1074	&& PGM_SHW_TYPE != PGM_TYPE_EPT
1075	int rc;
1076	PVM pVM = pVCpu->CTX_SUFF(pVM);
1077	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1078
1079	Assert(PGMIsLockOwner(pVM));
1080
1081	LogFlow(("InvalidatePage %RGv\n", GCPtrPage));
1082
1083	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1084	if (pPool->cDirtyPages)
1085	pgmPoolResetDirtyPages(pVM);
1086	# endif
1087
1088	/*
1089	* Get the shadow PD entry and skip out if this PD isn't present.
1090	* (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
1091	*/
1092	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1093	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1094	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1095
1096	/* Fetch the pgm pool shadow descriptor. */
1097	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1098	Assert(pShwPde);
1099
1100	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1101	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT);
1102	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
1103
1104	/* If the shadow PDPE isn't present, then skip the invalidate. */
1105	if (!pPdptDst->a[iPdpt].n.u1Present)
1106	{
1107	Assert(!(pPdptDst->a[iPdpt].u & PGM_PLXFLAGS_MAPPING));
1108	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1109	return VINF_SUCCESS;
1110	}
1111
1112	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1113	PPGMPOOLPAGE pShwPde = NULL;
1114	PX86PDPAE pPDDst;
1115
1116	/* Fetch the pgm pool shadow descriptor. */
1117	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
1118	AssertRCSuccessReturn(rc, rc);
1119	Assert(pShwPde);
1120
1121	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1122	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1123
1124	# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1125	/* PML4 */
1126	const unsigned iPml4 = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;
1127	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1128	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1129	PX86PDPAE pPDDst;
1130	PX86PDPT pPdptDst;
1131	PX86PML4E pPml4eDst;
1132	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eDst, &pPdptDst, &pPDDst);
1133	if (rc != VINF_SUCCESS)
1134	{
1135	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT \|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
1136	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1137	return VINF_SUCCESS;
1138	}
1139	Assert(pPDDst);
1140
1141	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1142	PX86PDPE pPdpeDst = &pPdptDst->a[iPdpt];
1143
1144	if (!pPdpeDst->n.u1Present)
1145	{
1146	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1147	return VINF_SUCCESS;
1148	}
1149
1150	/* Fetch the pgm pool shadow descriptor. */
1151	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & SHW_PDPE_PG_MASK);
1152	Assert(pShwPde);
1153
1154	# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1155
1156	const SHWPDE PdeDst = *pPdeDst;
1157	if (!PdeDst.n.u1Present)
1158	{
1159	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1160	return VINF_SUCCESS;
1161	}
1162
1163	/*
1164	* Get the guest PD entry and calc big page.
1165	*/
1166	# if PGM_GST_TYPE == PGM_TYPE_32BIT
1167	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
1168	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
1169	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1170	# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1171	unsigned iPDSrc = 0;
1172	# if PGM_GST_TYPE == PGM_TYPE_PAE
1173	X86PDPE PdpeSrcIgn;
1174	PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrcIgn);
1175	# else /* AMD64 */
1176	PX86PML4E pPml4eSrcIgn;
1177	X86PDPE PdpeSrcIgn;
1178	PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrcIgn, &PdpeSrcIgn, &iPDSrc);
1179	# endif
1180	GSTPDE PdeSrc;
1181
1182	if (pPDSrc)
1183	PdeSrc = pPDSrc->a[iPDSrc];
1184	else
1185	PdeSrc.u = 0;
1186	# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1187	const bool fIsBigPage = PdeSrc.b.u1Size && GST_IS_PSE_ACTIVE(pVCpu);
1188
1189	# ifdef IN_RING3
1190	/*
1191	* If a CR3 Sync is pending we may ignore the invalidate page operation
1192	* depending on the kind of sync and if it's a global page or not.
1193	* This doesn't make sense in GC/R0 so we'll skip it entirely there.
1194	*/
1195	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
1196	if ( VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
1197	\|\| ( VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
1198	&& fIsBigPage
1199	&& PdeSrc.b.u1Global
1200	)
1201	)
1202	# else
1203	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
1204	# endif
1205	{
1206	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1207	return VINF_SUCCESS;
1208	}
1209	# endif /* IN_RING3 */
1210
1211	/*
1212	* Deal with the Guest PDE.
1213	*/
1214	rc = VINF_SUCCESS;
1215	if (PdeSrc.n.u1Present)
1216	{
1217	Assert( PdeSrc.n.u1User == PdeDst.n.u1User
1218	&& (PdeSrc.n.u1Write \|\| !PdeDst.n.u1Write));
1219	# ifndef PGM_WITHOUT_MAPPING
1220	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
1221	{
1222	/*
1223	* Conflict - Let SyncPT deal with it to avoid duplicate code.
1224	*/
1225	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1226	Assert(PGMGetGuestMode(pVCpu) <= PGMMODE_PAE);
1227	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
1228	}
1229	else
1230	# endif /* !PGM_WITHOUT_MAPPING */
1231	if (!fIsBigPage)
1232	{
1233	/*
1234	* 4KB - page.
1235	*/
1236	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1237	RTGCPHYS GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1238
1239	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1240	/* Reset the modification counter (OpenSolaris trashes tlb entries very often) */
1241	if (pShwPage->cModifications)
1242	pShwPage->cModifications = 1;
1243	# endif
1244
1245	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1246	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1247	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE/2);
1248	# endif
1249	if (pShwPage->GCPhys == GCPhys)
1250	{
1251	# if 0 /* likely cause of a major performance regression; must be SyncPageWorkerTrackDeref then */
1252	const unsigned iPTEDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1253	PSHWPT pPT = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1254	if (pPT->a[iPTEDst].n.u1Present)
1255	{
1256	/* This is very unlikely with caching/monitoring enabled. */
1257	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pShwPage, pPT->a[iPTEDst].u & SHW_PTE_PG_MASK, iPTEDst);
1258	ASMAtomicWriteSize(&pPT->a[iPTEDst], 0);
1259	}
1260	# else /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1261	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
1262	if (RT_SUCCESS(rc))
1263	rc = VINF_SUCCESS;
1264	# endif
1265	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4KBPages));
1266	PGM_INVL_PG(pVCpu, GCPtrPage);
1267	}
1268	else
1269	{
1270	/*
1271	* The page table address changed.
1272	*/
1273	LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
1274	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1275	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1276	ASMAtomicWriteSize(pPdeDst, 0);
1277	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1278	PGM_INVL_VCPU_TLBS(pVCpu);
1279	}
1280	}
1281	else
1282	{
1283	/*
1284	* 2/4MB - page.
1285	*/
1286	/* Before freeing the page, check if anything really changed. */
1287	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1288	RTGCPHYS GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(pVM, PdeSrc);
1289	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1290	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1291	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1292	# endif
1293	if ( pShwPage->GCPhys == GCPhys
1294	&& pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1295	{
1296	/* ASSUMES a the given bits are identical for 4M and normal PDEs */
1297	/** @todo PAT */
1298	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
1299	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
1300	&& ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1301	\|\| (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1302	{
1303	LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1304	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip));
1305	return VINF_SUCCESS;
1306	}
1307	}
1308
1309	/*
1310	* Ok, the page table is present and it's been changed in the guest.
1311	* If we're in host context, we'll just mark it as not present taking the lazy approach.
1312	* We could do this for some flushes in GC too, but we need an algorithm for
1313	* deciding which 4MB pages containing code likely to be executed very soon.
1314	*/
1315	LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1316	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1317	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1318	ASMAtomicWriteSize(pPdeDst, 0);
1319	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4MBPages));
1320	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
1321	}
1322	}
1323	else
1324	{
1325	/*
1326	* Page directory is not present, mark shadow PDE not present.
1327	*/
1328	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
1329	{
1330	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1331	ASMAtomicWriteSize(pPdeDst, 0);
1332	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDNPs));
1333	PGM_INVL_PG(pVCpu, GCPtrPage);
1334	}
1335	else
1336	{
1337	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1338	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDMappings));
1339	}
1340	}
1341	return rc;
1342
1343	#else /* guest real and protected mode */
1344	/* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1345	return VINF_SUCCESS;
1346	#endif
1347	}
1348
1349
1350	/**
1351	* Update the tracking of shadowed pages.
1352	*
1353	* @param pVCpu The VMCPU handle.
1354	* @param pShwPage The shadow page.
1355	* @param HCPhys The physical page we is being dereferenced.
1356	* @param iPte Shadow PTE index
1357	*/
1358	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys, uint16_t iPte)
1359	{
1360	PVM pVM = pVCpu->CTX_SUFF(pVM);
1361
1362	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackDeref, a);
1363	LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1364
1365	/** @todo If this turns out to be a bottle neck (very likely) two things can be done:
1366	* 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1367	* 2. write protect all shadowed pages. I.e. implement caching.
1368	*/
1369	/** @todo duplicated in the 2nd half of pgmPoolTracDerefGCPhysHint */
1370
1371	/*
1372	* Find the guest address.
1373	*/
1374	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
1375	pRam;
1376	pRam = pRam->CTX_SUFF(pNext))
1377	{
1378	unsigned iPage = pRam->cb >> PAGE_SHIFT;
1379	while (iPage-- > 0)
1380	{
1381	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1382	{
1383	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1384
1385	Assert(pShwPage->cPresent);
1386	Assert(pPool->cPresent);
1387	pShwPage->cPresent--;
1388	pPool->cPresent--;
1389
1390	pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage], iPte);
1391	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackDeref, a);
1392	return;
1393	}
1394	}
1395	}
1396
1397	for (;;)
1398	AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys));
1399	}
1400
1401
1402	/**
1403	* Update the tracking of shadowed pages.
1404	*
1405	* @param pVCpu The VMCPU handle.
1406	* @param pShwPage The shadow page.
1407	* @param u16 The top 16-bit of the pPage->HCPhys.
1408	* @param pPage Pointer to the guest page. this will be modified.
1409	* @param iPTDst The index into the shadow table.
1410	*/
1411	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
1412	{
1413	PVM pVM = pVCpu->CTX_SUFF(pVM);
1414
1415	/*
1416	* Just deal with the simple first time here.
1417	*/
1418	if (!u16)
1419	{
1420	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackVirgin);
1421	u16 = PGMPOOL_TD_MAKE(1, pShwPage->idx);
1422	/* Save the page table index. */
1423	PGM_PAGE_SET_PTE_INDEX(pPage, iPTDst);
1424	}
1425	else
1426	u16 = pgmPoolTrackPhysExtAddref(pVM, pPage, u16, pShwPage->idx, iPTDst);
1427
1428	/* write back */
1429	Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x iPTDst=%#x\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst));
1430	PGM_PAGE_SET_TRACKING(pPage, u16);
1431
1432	/* update statistics. */
1433	pVM->pgm.s.CTX_SUFF(pPool)->cPresent++;
1434	pShwPage->cPresent++;
1435	if (pShwPage->iFirstPresent > iPTDst)
1436	pShwPage->iFirstPresent = iPTDst;
1437	}
1438
1439
1440	/**
1441	* Modifies a shadow PTE to account for access handlers.
1442	*
1443	* @param pVM The VM handle.
1444	* @param pPage The page in question.
1445	* @param fPteSrc The flags of the source PTE.
1446	* @param pPteDst The shadow PTE (output).
1447	*/
1448	DECLINLINE(void) PGM_BTH_NAME(SyncHandlerPte)(PVM pVM, PCPGMPAGE pPage, uint32_t fPteSrc, PSHWPTE pPteDst)
1449	{
1450	/** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No.
1451	* Update: \#PF should deal with this before or after calling the handlers. It has all the info to do the job efficiently. */
1452	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1453	{
1454	LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark read-only\n", pPage));
1455	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1456	pPteDst->u = PGM_PAGE_GET_HCPHYS(pPage);
1457	pPteDst->n.u1Present = 1;
1458	pPteDst->n.u1Execute = 1;
1459	pPteDst->n.u1IgnorePAT = 1;
1460	pPteDst->n.u3EMT = VMX_EPT_MEMTYPE_WB;
1461	/* PteDst.n.u1Write = 0 && PteDst.n.u1Size = 0 */
1462	#else
1463	pPteDst->u = (fPteSrc & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1464	\| PGM_PAGE_GET_HCPHYS(pPage);
1465	#endif
1466	}
1467	#ifdef PGM_WITH_MMIO_OPTIMIZATIONS
1468	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
1469	else if ( PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
1470	&& ( BTH_IS_NP_ACTIVE(pVM)
1471	\|\| (fPteSrc & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW) /** @todo remove X86_PTE_US */
1472	# if PGM_SHW_TYPE == PGM_TYPE_AMD64
1473	&& pVM->pgm.s.fLessThan52PhysicalAddressBits
1474	# endif
1475	)
1476	{
1477	LogFlow(("SyncHandlerPte: MMIO page -> invalid \n"));
1478	# if PGM_SHW_TYPE == PGM_TYPE_EPT
1479	/* 25.2.3.1: Reserved physical address bit -> EPT Misconfiguration (exit 49) */
1480	pPteDst->u = pVM->pgm.s.HCPhysInvMmioPg;
1481	/* 25.2.3.1: bits 2:0 = 010b -> EPT Misconfiguration (exit 49) */
1482	pPteDst->n.u1Present = 0;
1483	pPteDst->n.u1Write = 1;
1484	pPteDst->n.u1Execute = 0;
1485	/* 25.2.3.1: leaf && 2:0 != 0 && u3Emt in {2, 3, 7} -> EPT Misconfiguration */
1486	pPteDst->n.u3EMT = 7;
1487	# else
1488	/* Set high page frame bits that MBZ (bankers on PAE, CPU dependent on AMD64). */
1489	pPteDst->u = pVM->pgm.s.HCPhysInvMmioPg \| X86_PTE_PAE_MBZ_MASK_NO_NX \| X86_PTE_P;
1490	# endif
1491	}
1492	# endif
1493	#endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
1494	else
1495	{
1496	LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark not present\n", pPage));
1497	pPteDst->u = 0;
1498	}
1499	/** @todo count these kinds of entries. */
1500	}
1501
1502
1503	/**
1504	* Creates a 4K shadow page for a guest page.
1505	*
1506	* For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1507	* physical address. The PdeSrc argument only the flags are used. No page
1508	* structured will be mapped in this function.
1509	*
1510	* @param pVCpu The VMCPU handle.
1511	* @param pPteDst Destination page table entry.
1512	* @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1513	* Can safely assume that only the flags are being used.
1514	* @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1515	* @param pShwPage Pointer to the shadow page.
1516	* @param iPTDst The index into the shadow table.
1517	*
1518	* @remark Not used for 2/4MB pages!
1519	*/
1520	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc,
1521	PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1522	{
1523	if ( PteSrc.n.u1Present
1524	&& GST_IS_PTE_VALID(pVCpu, PteSrc))
1525	{
1526	PVM pVM = pVCpu->CTX_SUFF(pVM);
1527
1528	# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1529	&& PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1530	&& (PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64)
1531	if (pShwPage->fDirty)
1532	{
1533	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1534	PX86PTPAE pGstPT;
1535
1536	pGstPT = (PX86PTPAE)&pPool->aDirtyPages[pShwPage->idxDirty][0];
1537	pGstPT->a[iPTDst].u = PteSrc.u;
1538	}
1539	# endif
1540	/*
1541	* Find the ram range.
1542	*/
1543	PPGMPAGE pPage;
1544	int rc = pgmPhysGetPageEx(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK, &pPage);
1545	if (RT_SUCCESS(rc))
1546	{
1547	/* Ignore ballooned pages.
1548	Don't return errors or use a fatal assert here as part of a
1549	shadow sync range might included ballooned pages. */
1550	if (PGM_PAGE_IS_BALLOONED(pPage))
1551	{
1552	Assert(!pPteDst->n.u1Present); /** @todo user tracking needs updating if this triggers. */
1553	return;
1554	}
1555
1556	#ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1557	/* Make the page writable if necessary. */
1558	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1559	&& ( PGM_PAGE_IS_ZERO(pPage)
1560	\|\| ( PteSrc.n.u1Write
1561	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1562	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1563	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1564	# endif
1565	# ifdef VBOX_WITH_PAGE_SHARING
1566	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
1567	# endif
1568	)
1569	)
1570	)
1571	{
1572	rc = pgmPhysPageMakeWritable(pVM, pPage, PteSrc.u & GST_PTE_PG_MASK);
1573	AssertRC(rc);
1574	}
1575	#endif
1576
1577	/*
1578	* Make page table entry.
1579	*/
1580	SHWPTE PteDst;
1581	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1582	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage,
1583	PteSrc.u & ~( X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK
1584	\| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT),
1585	&PteDst);
1586	else
1587	{
1588	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1589	/*
1590	* If the page or page directory entry is not marked accessed,
1591	* we mark the page not present.
1592	*/
1593	if (!PteSrc.n.u1Accessed \|\| !PdeSrc.n.u1Accessed)
1594	{
1595	LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1596	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,AccessedPage));
1597	PteDst.u = 0;
1598	}
1599	/*
1600	* If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1601	* when the page is modified.
1602	*/
1603	else if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1604	{
1605	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPage));
1606	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1607	\| PGM_PAGE_GET_HCPHYS(pPage)
1608	\| PGM_PTFLAGS_TRACK_DIRTY;
1609	}
1610	else
1611	#endif
1612	{
1613	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageSkipped));
1614	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1615	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage);
1616	PteDst.n.u1Present = 1;
1617	PteDst.n.u1Write = 1;
1618	PteDst.n.u1Execute = 1;
1619	PteDst.n.u1IgnorePAT = 1;
1620	PteDst.n.u3EMT = VMX_EPT_MEMTYPE_WB;
1621	/* PteDst.n.u1Size = 0 */
1622	#else
1623	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1624	\| PGM_PAGE_GET_HCPHYS(pPage);
1625	#endif
1626	}
1627
1628	/*
1629	* Make sure only allocated pages are mapped writable.
1630	*/
1631	if ( PteDst.n.u1Write
1632	&& PteDst.n.u1Present
1633	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1634	{
1635	/* Still applies to shared pages. */
1636	Assert(!PGM_PAGE_IS_ZERO(pPage));
1637	PteDst.n.u1Write = 0; /** @todo this isn't quite working yet. Why, isn't it? */
1638	Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", (RTGCPHYS)(PteSrc.u & X86_PTE_PAE_PG_MASK), pPage, iPTDst));
1639	}
1640	}
1641
1642	/*
1643	* Keep user track up to date.
1644	*/
1645	if (PteDst.n.u1Present)
1646	{
1647	if (!pPteDst->n.u1Present)
1648	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1649	else if ((pPteDst->u & SHW_PTE_PG_MASK) != (PteDst.u & SHW_PTE_PG_MASK))
1650	{
1651	Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", (uint64_t)pPteDst->u, (uint64_t)PteDst.u));
1652	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK, iPTDst);
1653	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1654	}
1655	}
1656	else if (pPteDst->n.u1Present)
1657	{
1658	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1659	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK, iPTDst);
1660	}
1661
1662	/*
1663	* Update statistics and commit the entry.
1664	*/
1665	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1666	if (!PteSrc.n.u1Global)
1667	pShwPage->fSeenNonGlobal = true;
1668	#endif
1669	ASMAtomicWriteSize(pPteDst, PteDst.u);
1670	return;
1671	}
1672
1673	/** @todo count these three different kinds. */
1674	Log2(("SyncPageWorker: invalid address in Pte\n"));
1675	}
1676	else if (!PteSrc.n.u1Present)
1677	Log2(("SyncPageWorker: page not present in Pte\n"));
1678	else
1679	Log2(("SyncPageWorker: invalid Pte\n"));
1680
1681	/*
1682	* The page is not present or the PTE is bad. Replace the shadow PTE by
1683	* an empty entry, making sure to keep the user tracking up to date.
1684	*/
1685	if (pPteDst->n.u1Present)
1686	{
1687	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1688	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK, iPTDst);
1689	}
1690	ASMAtomicWriteSize(pPteDst, 0);
1691	}
1692
1693
1694	/**
1695	* Syncs a guest OS page.
1696	*
1697	* There are no conflicts at this point, neither is there any need for
1698	* page table allocations.
1699	*
1700	* When called in PAE or AMD64 guest mode, the guest PDPE shall be valid.
1701	* When called in AMD64 guest mode, the guest PML4E shall be valid.
1702	*
1703	* @returns VBox status code.
1704	* @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1705	* @param pVCpu The VMCPU handle.
1706	* @param PdeSrc Page directory entry of the guest.
1707	* @param GCPtrPage Guest context page address.
1708	* @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1709	* @param uErr Fault error (X86_TRAP_PF_*).
1710	*/
1711	static int PGM_BTH_NAME(SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr)
1712	{
1713	PVM pVM = pVCpu->CTX_SUFF(pVM);
1714	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1715	LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr));
1716
1717	Assert(PGMIsLockOwner(pVM));
1718
1719	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
1720	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
1721	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
1722	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
1723	&& PGM_SHW_TYPE != PGM_TYPE_EPT
1724
1725	/*
1726	* Assert preconditions.
1727	*/
1728	Assert(PdeSrc.n.u1Present);
1729	Assert(cPages);
1730	# if 0 /* rarely useful; leave for debugging. */
1731	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
1732	# endif
1733
1734	/*
1735	* Get the shadow PDE, find the shadow page table in the pool.
1736	*/
1737	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1738	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1739	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1740
1741	/* Fetch the pgm pool shadow descriptor. */
1742	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1743	Assert(pShwPde);
1744
1745	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1746	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1747	PPGMPOOLPAGE pShwPde = NULL;
1748	PX86PDPAE pPDDst;
1749
1750	/* Fetch the pgm pool shadow descriptor. */
1751	int rc2 = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
1752	AssertRCSuccessReturn(rc2, rc2);
1753	Assert(pShwPde);
1754
1755	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1756	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1757
1758	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1759	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1760	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1761	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
1762	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
1763
1764	int rc2 = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
1765	AssertRCSuccessReturn(rc2, rc2);
1766	Assert(pPDDst && pPdptDst);
1767	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1768	# endif
1769	SHWPDE PdeDst = *pPdeDst;
1770
1771	/* In the guest SMP case we could have blocked while another VCPU reused this page table. */
1772	if (!PdeDst.n.u1Present)
1773	{
1774	AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE p=%p/%RX64\n", pPdeDst, (uint64_t)PdeDst.u));
1775	Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", pVCpu->idCpu, GCPtrPage));
1776	return VINF_SUCCESS; /* force the instruction to be executed again. */
1777	}
1778
1779	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1780	Assert(pShwPage);
1781
1782	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1783	/* Fetch the pgm pool shadow descriptor. */
1784	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
1785	Assert(pShwPde);
1786	# endif
1787
1788	/*
1789	* Check that the page is present and that the shadow PDE isn't out of sync.
1790	*/
1791	const bool fBigPage = PdeSrc.b.u1Size && GST_IS_PSE_ACTIVE(pVCpu);
1792	const bool fPdeValid = !fBigPage ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc);
1793	RTGCPHYS GCPhys;
1794	if (!fBigPage)
1795	{
1796	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1797	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1798	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1799	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE / 2);
1800	# endif
1801	}
1802	else
1803	{
1804	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(pVM, PdeSrc);
1805	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1806	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1807	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1808	# endif
1809	}
1810	if ( fPdeValid
1811	&& pShwPage->GCPhys == GCPhys
1812	&& PdeSrc.n.u1Present
1813	&& PdeSrc.n.u1User == PdeDst.n.u1User
1814	&& (PdeSrc.n.u1Write == PdeDst.n.u1Write \|\| !PdeDst.n.u1Write)
1815	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
1816	&& (PdeSrc.n.u1NoExecute == PdeDst.n.u1NoExecute \|\| !GST_IS_NX_ACTIVE(pVCpu))
1817	# endif
1818	)
1819	{
1820	/*
1821	* Check that the PDE is marked accessed already.
1822	* Since we set the accessed bit before getting here on a #PF, this
1823	* check is only meant for dealing with non-#PF'ing paths.
1824	*/
1825	if (PdeSrc.n.u1Accessed)
1826	{
1827	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1828	if (!fBigPage)
1829	{
1830	/*
1831	* 4KB Page - Map the guest page table.
1832	*/
1833	PGSTPT pPTSrc;
1834	int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
1835	if (RT_SUCCESS(rc))
1836	{
1837	# ifdef PGM_SYNC_N_PAGES
1838	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
1839	if ( cPages > 1
1840	&& !(uErr & X86_TRAP_PF_P)
1841	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
1842	{
1843	/*
1844	* This code path is currently only taken when the caller is PGMTrap0eHandler
1845	* for non-present pages!
1846	*
1847	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1848	* deal with locality.
1849	*/
1850	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1851	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1852	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1853	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1854	# else
1855	const unsigned offPTSrc = 0;
1856	# endif
1857	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
1858	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1859	iPTDst = 0;
1860	else
1861	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1862	for (; iPTDst < iPTDstEnd; iPTDst++)
1863	{
1864	if (!pPTDst->a[iPTDst].n.u1Present)
1865	{
1866	GSTPTE PteSrc = pPTSrc->a[offPTSrc + iPTDst];
1867	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
1868	NOREF(GCPtrCurPage);
1869	#ifndef IN_RING0
1870	/*
1871	* Assuming kernel code will be marked as supervisor - and not as user level
1872	* and executed using a conforming code selector - And marked as readonly.
1873	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
1874	*/
1875	PPGMPAGE pPage;
1876	if ( ((PdeSrc.u & PteSrc.u) & (X86_PTE_RW \| X86_PTE_US))
1877	\|\| iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK) /* always sync GCPtrPage */
1878	\|\| !CSAMDoesPageNeedScanning(pVM, GCPtrCurPage)
1879	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
1880	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1881	)
1882	#endif /* else: CSAM not active */
1883	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1884	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1885	GCPtrCurPage, PteSrc.n.u1Present,
1886	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1887	PteSrc.n.u1User & PdeSrc.n.u1User,
1888	(uint64_t)PteSrc.u,
1889	(uint64_t)pPTDst->a[iPTDst].u,
1890	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1891	}
1892	}
1893	}
1894	else
1895	# endif /* PGM_SYNC_N_PAGES */
1896	{
1897	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1898	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1899	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1900	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1901	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
1902	GCPtrPage, PteSrc.n.u1Present,
1903	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1904	PteSrc.n.u1User & PdeSrc.n.u1User,
1905	(uint64_t)PteSrc.u,
1906	(uint64_t)pPTDst->a[iPTDst].u,
1907	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1908	}
1909	}
1910	else /* MMIO or invalid page: emulated in #PF handler. */
1911	{
1912	LogFlow(("PGM_GCPHYS_2_PTR %RGp failed with %Rrc\n", GCPhys, rc));
1913	Assert(!pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK].n.u1Present);
1914	}
1915	}
1916	else
1917	{
1918	/*
1919	* 4/2MB page - lazy syncing shadow 4K pages.
1920	* (There are many causes of getting here, it's no longer only CSAM.)
1921	*/
1922	/* Calculate the GC physical address of this 4KB shadow page. */
1923	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(pVM, PdeSrc) \| (GCPtrPage & GST_BIG_PAGE_OFFSET_MASK);
1924	/* Find ram range. */
1925	PPGMPAGE pPage;
1926	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
1927	if (RT_SUCCESS(rc))
1928	{
1929	AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
1930
1931	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1932	/* Try to make the page writable if necessary. */
1933	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1934	&& ( PGM_PAGE_IS_ZERO(pPage)
1935	\|\| ( PdeSrc.n.u1Write
1936	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1937	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1938	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1939	# endif
1940	# ifdef VBOX_WITH_PAGE_SHARING
1941	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
1942	# endif
1943	)
1944	)
1945	)
1946	{
1947	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1948	AssertRC(rc);
1949	}
1950	# endif
1951
1952	/*
1953	* Make shadow PTE entry.
1954	*/
1955	SHWPTE PteDst;
1956	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1957	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage,
1958	PdeSrc.u & ~( X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK
1959	\| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT),
1960	&PteDst);
1961	else
1962	PteDst.u = (PdeSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1963	\| PGM_PAGE_GET_HCPHYS(pPage);
1964
1965	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1966	if ( PteDst.n.u1Present
1967	&& !pPTDst->a[iPTDst].n.u1Present)
1968	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1969
1970	/* Make sure only allocated pages are mapped writable. */
1971	if ( PteDst.n.u1Write
1972	&& PteDst.n.u1Present
1973	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1974	{
1975	/* Still applies to shared pages. */
1976	Assert(!PGM_PAGE_IS_ZERO(pPage));
1977	PteDst.n.u1Write = 0; /** @todo this isn't quite working yet... */
1978	Log3(("SyncPage: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, GCPtrPage));
1979	}
1980
1981	ASMAtomicWriteSize(&pPTDst->a[iPTDst], PteDst.u);
1982
1983	/*
1984	* If the page is not flagged as dirty and is writable, then make it read-only
1985	* at PD level, so we can set the dirty bit when the page is modified.
1986	*
1987	* ASSUMES that page access handlers are implemented on page table entry level.
1988	* Thus we will first catch the dirty access and set PDE.D and restart. If
1989	* there is an access handler, we'll trap again and let it work on the problem.
1990	*/
1991	/** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
1992	* As for invlpg, it simply frees the whole shadow PT.
1993	* ...It's possibly because the guest clears it and the guest doesn't really tell us... */
1994	if ( !PdeSrc.b.u1Dirty
1995	&& PdeSrc.b.u1Write)
1996	{
1997	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
1998	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
1999	PdeDst.n.u1Write = 0;
2000	}
2001	else
2002	{
2003	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2004	PdeDst.n.u1Write = PdeSrc.n.u1Write;
2005	}
2006	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2007	Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n",
2008	GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
2009	PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2010	}
2011	else
2012	{
2013	LogFlow(("PGM_GCPHYS_2_PTR %RGp (big) failed with %Rrc\n", GCPhys, rc));
2014	/** @todo must wipe the shadow page table in this case. */
2015	}
2016	}
2017	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2018	return VINF_SUCCESS;
2019	}
2020
2021	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPagePDNAs));
2022	}
2023	else if (fPdeValid)
2024	{
2025	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPagePDOutOfSync));
2026	Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2027	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2028	}
2029	else
2030	{
2031	/// @todo STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSync));
2032	Log2(("SyncPage: Bad PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2033	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2034	}
2035
2036	/*
2037	* Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
2038	* Yea, I'm lazy.
2039	*/
2040	pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
2041	ASMAtomicWriteSize(pPdeDst, 0);
2042
2043	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2044	PGM_INVL_VCPU_TLBS(pVCpu);
2045	return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
2046
2047
2048	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2049	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2050	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2051	&& !defined(IN_RC)
2052
2053	# ifdef PGM_SYNC_N_PAGES
2054	/*
2055	* Get the shadow PDE, find the shadow page table in the pool.
2056	*/
2057	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2058	X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
2059
2060	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2061	X86PDEPAE PdeDst = pgmShwGetPaePDE(pVCpu, GCPtrPage);
2062
2063	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2064	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2065	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpt);
2066	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2067	X86PDEPAE PdeDst;
2068	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2069
2070	int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2071	AssertRCSuccessReturn(rc, rc);
2072	Assert(pPDDst && pPdptDst);
2073	PdeDst = pPDDst->a[iPDDst];
2074	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2075	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2076	PEPTPD pPDDst;
2077	EPTPDE PdeDst;
2078
2079	int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, NULL, &pPDDst);
2080	if (rc != VINF_SUCCESS)
2081	{
2082	AssertRC(rc);
2083	return rc;
2084	}
2085	Assert(pPDDst);
2086	PdeDst = pPDDst->a[iPDDst];
2087	# endif
2088	/* In the guest SMP case we could have blocked while another VCPU reused this page table. */
2089	if (!PdeDst.n.u1Present)
2090	{
2091	AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)PdeDst.u));
2092	Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", pVCpu->idCpu, GCPtrPage));
2093	return VINF_SUCCESS; /* force the instruction to be executed again. */
2094	}
2095
2096	/* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */
2097	if (PdeDst.n.u1Size)
2098	{
2099	Assert(pVM->pgm.s.fNestedPaging);
2100	Log(("CPU%d: SyncPage: Pde (big:%RX64) at %RGv changed behind our back!\n", pVCpu->idCpu, PdeDst.u, GCPtrPage));
2101	return VINF_SUCCESS;
2102	}
2103
2104	/* Mask away the page offset. */
2105	GCPtrPage &= ~((RTGCPTR)0xfff);
2106
2107	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2108	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2109
2110	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
2111	if ( cPages > 1
2112	&& !(uErr & X86_TRAP_PF_P)
2113	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2114	{
2115	/*
2116	* This code path is currently only taken when the caller is PGMTrap0eHandler
2117	* for non-present pages!
2118	*
2119	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2120	* deal with locality.
2121	*/
2122	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2123	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2124	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2125	iPTDst = 0;
2126	else
2127	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2128	for (; iPTDst < iPTDstEnd; iPTDst++)
2129	{
2130	if (!pPTDst->a[iPTDst].n.u1Present)
2131	{
2132	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT);
2133	GSTPTE PteSrc;
2134
2135	/* Fake the page table entry */
2136	PteSrc.u = GCPtrCurPage;
2137	PteSrc.n.u1Present = 1;
2138	PteSrc.n.u1Dirty = 1;
2139	PteSrc.n.u1Accessed = 1;
2140	PteSrc.n.u1Write = 1;
2141	PteSrc.n.u1User = 1;
2142
2143	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2144	#ifdef DEBUG_sandervl
2145	if (pVM->pgm.s.fCountingPhysWrites)
2146	pPTDst->a[iPTDst].n.u1Write = 0;
2147	#endif
2148
2149	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
2150	GCPtrCurPage, PteSrc.n.u1Present,
2151	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2152	PteSrc.n.u1User & PdeSrc.n.u1User,
2153	(uint64_t)PteSrc.u,
2154	(uint64_t)pPTDst->a[iPTDst].u,
2155	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2156
2157	if (RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)))
2158	break;
2159	}
2160	else
2161	Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n", (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT), iPTDst, pPTDst->a[iPTDst].u));
2162	}
2163	}
2164	else
2165	# endif /* PGM_SYNC_N_PAGES */
2166	{
2167	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2168	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT);
2169	GSTPTE PteSrc;
2170
2171	#ifdef DEBUG_sandervl
2172	if ( pVM->pgm.s.fCountingPhysWrites
2173	&& ((uErr & (X86_TRAP_PF_RW\|X86_TRAP_PF_P)) == (X86_TRAP_PF_RW\|X86_TRAP_PF_P)))
2174	{
2175	STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat, FTPhysPageWrite));
2176	}
2177	#endif
2178
2179	/* Fake the page table entry */
2180	PteSrc.u = GCPtrCurPage;
2181	PteSrc.n.u1Present = 1;
2182	PteSrc.n.u1Dirty = 1;
2183	PteSrc.n.u1Accessed = 1;
2184	PteSrc.n.u1Write = 1;
2185	PteSrc.n.u1User = 1;
2186	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2187
2188	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}PteDst=%08llx%s\n",
2189	GCPtrPage, PteSrc.n.u1Present,
2190	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2191	PteSrc.n.u1User & PdeSrc.n.u1User,
2192	(uint64_t)PteSrc.u,
2193	(uint64_t)pPTDst->a[iPTDst].u,
2194	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2195	}
2196	return VINF_SUCCESS;
2197
2198	#else
2199	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2200	return VERR_INTERNAL_ERROR;
2201	#endif
2202	}
2203
2204
2205	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2206
2207	/**
2208	* CheckPageFault helper for returning a page fault indicating a non-present
2209	* (NP) entry in the page translation structures.
2210	*
2211	* @returns VINF_EM_RAW_GUEST_TRAP.
2212	* @param pVCpu The virtual CPU to operate on.
2213	* @param uErr The error code of the shadow fault. Corrections to
2214	* TRPM's copy will be made if necessary.
2215	* @param GCPtrPage For logging.
2216	* @param uPageFaultLevel For logging.
2217	*/
2218	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnNP)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2219	{
2220	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2221	AssertMsg(!(uErr & X86_TRAP_PF_P), ("%#x\n", uErr));
2222	AssertMsg(!(uErr & X86_TRAP_PF_RSVD), ("%#x\n", uErr));
2223	if (uErr & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
2224	TRPMSetErrorCode(pVCpu, uErr & ~(X86_TRAP_PF_RSVD \| X86_TRAP_PF_P));
2225
2226	Log(("CheckPageFault: real page fault (notp) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2227	return VINF_EM_RAW_GUEST_TRAP;
2228	}
2229
2230
2231	/**
2232	* CheckPageFault helper for returning a page fault indicating a reserved bit
2233	* (RSVD) error in the page translation structures.
2234	*
2235	* @returns VINF_EM_RAW_GUEST_TRAP.
2236	* @param pVCpu The virtual CPU to operate on.
2237	* @param uErr The error code of the shadow fault. Corrections to
2238	* TRPM's copy will be made if necessary.
2239	* @param GCPtrPage For logging.
2240	* @param uPageFaultLevel For logging.
2241	*/
2242	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnRSVD)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2243	{
2244	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2245	if ((uErr & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) != (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
2246	TRPMSetErrorCode(pVCpu, uErr \| X86_TRAP_PF_RSVD \| X86_TRAP_PF_P);
2247
2248	Log(("CheckPageFault: real page fault (rsvd) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2249	return VINF_EM_RAW_GUEST_TRAP;
2250	}
2251
2252
2253	/**
2254	* CheckPageFault helper for returning a page protection fault (P).
2255	*
2256	* @returns VINF_EM_RAW_GUEST_TRAP.
2257	* @param pVCpu The virtual CPU to operate on.
2258	* @param uErr The error code of the shadow fault. Corrections to
2259	* TRPM's copy will be made if necessary.
2260	* @param GCPtrPage For logging.
2261	* @param uPageFaultLevel For logging.
2262	*/
2263	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnProt)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2264	{
2265	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2266	AssertMsg(uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID), ("%#x\n", uErr));
2267	if ((uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) != X86_TRAP_PF_P)
2268	TRPMSetErrorCode(pVCpu, (uErr & ~X86_TRAP_PF_RSVD) \| X86_TRAP_PF_P);
2269
2270	Log(("CheckPageFault: real page fault (prot) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2271	return VINF_EM_RAW_GUEST_TRAP;
2272	}
2273
2274
2275	/**
2276	* Handle dirty bit tracking faults.
2277	*
2278	* @returns VBox status code.
2279	* @param pVCpu The VMCPU handle.
2280	* @param uErr Page fault error code.
2281	* @param pPdeSrc Guest page directory entry.
2282	* @param pPdeDst Shadow page directory entry.
2283	* @param GCPtrPage Guest context page address.
2284	*/
2285	static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, RTGCPTR GCPtrPage)
2286	{
2287	PVM pVM = pVCpu->CTX_SUFF(pVM);
2288	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2289
2290	Assert(PGMIsLockOwner(pVM));
2291
2292	/*
2293	* Handle big page.
2294	*/
2295	if (pPdeSrc->b.u1Size && GST_IS_PSE_ACTIVE(pVCpu))
2296	{
2297	if ( pPdeDst->n.u1Present
2298	&& (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY))
2299	{
2300	SHWPDE PdeDst = *pPdeDst;
2301
2302	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageTrap));
2303	Assert(pPdeSrc->b.u1Write);
2304
2305	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
2306	* fault again and take this path to only invalidate the entry (see below).
2307	*/
2308	PdeDst.n.u1Write = 1;
2309	PdeDst.n.u1Accessed = 1;
2310	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2311	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2312	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
2313	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2314	}
2315
2316	# ifdef IN_RING0
2317	/* Check for stale TLB entry; only applies to the SMP guest case. */
2318	if ( pVM->cCpus > 1
2319	&& pPdeDst->n.u1Write
2320	&& pPdeDst->n.u1Accessed)
2321	{
2322	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2323	if (pShwPage)
2324	{
2325	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2326	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2327	if ( pPteDst->n.u1Present
2328	&& pPteDst->n.u1Write)
2329	{
2330	/* Stale TLB entry. */
2331	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageStale));
2332	PGM_INVL_PG(pVCpu, GCPtrPage);
2333	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2334	}
2335	}
2336	}
2337	# endif /* IN_RING0 */
2338	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2339	}
2340
2341	/*
2342	* Map the guest page table.
2343	*/
2344	PGSTPT pPTSrc;
2345	int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
2346	if (RT_FAILURE(rc))
2347	{
2348	AssertRC(rc);
2349	return rc;
2350	}
2351
2352	if (pPdeDst->n.u1Present)
2353	{
2354	GSTPTE const *pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2355	const GSTPTE PteSrc = *pPteSrc;
2356
2357	#ifndef IN_RING0
2358	/* Bail out here as pgmPoolGetPage will return NULL and we'll crash below.
2359	* Our individual shadow handlers will provide more information and force a fatal exit.
2360	*/
2361	if (MMHyperIsInsideArea(pVM, (RTGCPTR)GCPtrPage))
2362	{
2363	LogRel(("CheckPageFault: write to hypervisor region %RGv\n", GCPtrPage));
2364	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2365	}
2366	#endif
2367	/*
2368	* Map shadow page table.
2369	*/
2370	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2371	if (pShwPage)
2372	{
2373	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2374	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2375	if (pPteDst->n.u1Present) /** @todo Optimize accessed bit emulation? */
2376	{
2377	if (pPteDst->u & PGM_PTFLAGS_TRACK_DIRTY)
2378	{
2379	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, pPteSrc->u & GST_PTE_PG_MASK);
2380	SHWPTE PteDst = *pPteDst;
2381
2382	LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage));
2383	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageTrap));
2384
2385	Assert(pPteSrc->n.u1Write);
2386
2387	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB
2388	* entry will not harm; write access will simply fault again and
2389	* take this path to only invalidate the entry.
2390	*/
2391	if (RT_LIKELY(pPage))
2392	{
2393	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2394	{
2395	AssertMsgFailed(("%R[pgmpage] - we don't set PGM_PTFLAGS_TRACK_DIRTY for these pages\n", pPage));
2396	Assert(!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage));
2397	/* Assuming write handlers here as the PTE is present (otherwise we wouldn't be here). */
2398	PteDst.n.u1Write = 0;
2399	}
2400	else
2401	{
2402	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
2403	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
2404	{
2405	rc = pgmPhysPageMakeWritable(pVM, pPage, pPteSrc->u & GST_PTE_PG_MASK);
2406	AssertRC(rc);
2407	}
2408	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED)
2409	PteDst.n.u1Write = 1;
2410	else
2411	{
2412	/* Still applies to shared pages. */
2413	Assert(!PGM_PAGE_IS_ZERO(pPage));
2414	PteDst.n.u1Write = 0;
2415	}
2416	}
2417	}
2418	else
2419	PteDst.n.u1Write = 1; /** @todo r=bird: This doesn't make sense to me. */
2420
2421	PteDst.n.u1Dirty = 1;
2422	PteDst.n.u1Accessed = 1;
2423	PteDst.au32[0] &= ~PGM_PTFLAGS_TRACK_DIRTY;
2424	ASMAtomicWriteSize(pPteDst, PteDst.u);
2425	PGM_INVL_PG(pVCpu, GCPtrPage);
2426	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2427	}
2428
2429	# ifdef IN_RING0
2430	/* Check for stale TLB entry; only applies to the SMP guest case. */
2431	if ( pVM->cCpus > 1
2432	&& pPteDst->n.u1Write == 1
2433	&& pPteDst->n.u1Accessed == 1)
2434	{
2435	/* Stale TLB entry. */
2436	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageStale));
2437	PGM_INVL_PG(pVCpu, GCPtrPage);
2438	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2439	}
2440	# endif
2441	}
2442	}
2443	else
2444	AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
2445	}
2446
2447	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2448	}
2449
2450	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
2451
2452
2453	/**
2454	* Sync a shadow page table.
2455	*
2456	* The shadow page table is not present. This includes the case where
2457	* there is a conflict with a mapping.
2458	*
2459	* @returns VBox status code.
2460	* @param pVCpu The VMCPU handle.
2461	* @param iPD Page directory index.
2462	* @param pPDSrc Source page directory (i.e. Guest OS page directory).
2463	* Assume this is a temporary mapping.
2464	* @param GCPtrPage GC Pointer of the page that caused the fault
2465	*/
2466	static int PGM_BTH_NAME(SyncPT)(PVMCPU pVCpu, unsigned iPDSrc, PGSTPD pPDSrc, RTGCPTR GCPtrPage)
2467	{
2468	PVM pVM = pVCpu->CTX_SUFF(pVM);
2469	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2470
2471	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2472	#if 0 /* rarely useful; leave for debugging. */
2473	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPtPD[iPDSrc]);
2474	#endif
2475	LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage));
2476
2477	Assert(PGMIsLocked(pVM));
2478
2479	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2480	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
2481	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
2482	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2483	&& PGM_SHW_TYPE != PGM_TYPE_EPT
2484
2485	int rc = VINF_SUCCESS;
2486
2487	/*
2488	* Validate input a little bit.
2489	*/
2490	AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
2491	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2492	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2493	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
2494
2495	/* Fetch the pgm pool shadow descriptor. */
2496	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2497	Assert(pShwPde);
2498
2499	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2500	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2501	PPGMPOOLPAGE pShwPde = NULL;
2502	PX86PDPAE pPDDst;
2503	PSHWPDE pPdeDst;
2504
2505	/* Fetch the pgm pool shadow descriptor. */
2506	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
2507	AssertRCSuccessReturn(rc, rc);
2508	Assert(pShwPde);
2509
2510	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
2511	pPdeDst = &pPDDst->a[iPDDst];
2512
2513	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2514	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2515	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2516	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2517	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2518	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2519	AssertRCSuccessReturn(rc, rc);
2520	Assert(pPDDst);
2521	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2522	# endif
2523	SHWPDE PdeDst = *pPdeDst;
2524
2525	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2526	/* Fetch the pgm pool shadow descriptor. */
2527	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2528	Assert(pShwPde);
2529	# endif
2530
2531	# ifndef PGM_WITHOUT_MAPPINGS
2532	/*
2533	* Check for conflicts.
2534	* RC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
2535	* R3: Simply resolve the conflict.
2536	*/
2537	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2538	{
2539	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
2540	# ifndef IN_RING3
2541	Log(("SyncPT: Conflict at %RGv\n", GCPtrPage));
2542	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2543	return VERR_ADDRESS_CONFLICT;
2544
2545	# else /* IN_RING3 */
2546	PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
2547	Assert(pMapping);
2548	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2549	rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2550	# elif PGM_GST_TYPE == PGM_TYPE_PAE
2551	rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2552	# else
2553	AssertFailed(); /* can't happen for amd64 */
2554	# endif
2555	if (RT_FAILURE(rc))
2556	{
2557	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2558	return rc;
2559	}
2560	PdeDst = *pPdeDst;
2561	# endif /* IN_RING3 */
2562	}
2563	# endif /* !PGM_WITHOUT_MAPPINGS */
2564	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2565
2566	/*
2567	* Sync page directory entry.
2568	*/
2569	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2570	if (PdeSrc.n.u1Present)
2571	{
2572	/*
2573	* Allocate & map the page table.
2574	*/
2575	PSHWPT pPTDst;
2576	const bool fPageTable = !PdeSrc.b.u1Size \|\| !GST_IS_PSE_ACTIVE(pVCpu);
2577	PPGMPOOLPAGE pShwPage;
2578	RTGCPHYS GCPhys;
2579	if (fPageTable)
2580	{
2581	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
2582	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2583	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2584	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE / 2);
2585	# endif
2586	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
2587	}
2588	else
2589	{
2590	PGMPOOLACCESS enmAccess;
2591	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2592	const bool fNoExecute = PdeSrc.n.u1NoExecute && GST_IS_NX_ACTIVE(pVCpu);
2593	# else
2594	const bool fNoExecute = false;
2595	# endif
2596
2597	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(pVM, PdeSrc);
2598	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2599	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2600	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
2601	# endif
2602	/* Determine the right kind of large page to avoid incorrect cached entry reuse. */
2603	if (PdeSrc.n.u1User)
2604	{
2605	if (PdeSrc.n.u1Write)
2606	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_RW_NX : PGMPOOLACCESS_USER_RW;
2607	else
2608	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_R_NX : PGMPOOLACCESS_USER_R;
2609	}
2610	else
2611	{
2612	if (PdeSrc.n.u1Write)
2613	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW;
2614	else
2615	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_R_NX : PGMPOOLACCESS_SUPERVISOR_R;
2616	}
2617	rc = pgmPoolAllocEx(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, enmAccess, pShwPde->idx, iPDDst, &pShwPage);
2618	}
2619	if (rc == VINF_SUCCESS)
2620	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2621	else if (rc == VINF_PGM_CACHED_PAGE)
2622	{
2623	/*
2624	* The PT was cached, just hook it up.
2625	*/
2626	if (fPageTable)
2627	PdeDst.u = pShwPage->Core.Key
2628	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2629	else
2630	{
2631	PdeDst.u = pShwPage->Core.Key
2632	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2633	/* (see explanation and assumptions further down.) */
2634	if ( !PdeSrc.b.u1Dirty
2635	&& PdeSrc.b.u1Write)
2636	{
2637	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2638	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2639	PdeDst.b.u1Write = 0;
2640	}
2641	}
2642	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2643	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2644	return VINF_SUCCESS;
2645	}
2646	else if (rc == VERR_PGM_POOL_FLUSHED)
2647	{
2648	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2649	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2650	return VINF_PGM_SYNC_CR3;
2651	}
2652	else
2653	AssertMsgFailedReturn(("rc=%Rrc\n", rc), VERR_INTERNAL_ERROR);
2654	PdeDst.u &= X86_PDE_AVL_MASK;
2655	PdeDst.u \|= pShwPage->Core.Key;
2656
2657	/*
2658	* Page directory has been accessed (this is a fault situation, remember).
2659	*/
2660	pPDSrc->a[iPDSrc].n.u1Accessed = 1;
2661	if (fPageTable)
2662	{
2663	/*
2664	* Page table - 4KB.
2665	*
2666	* Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
2667	*/
2668	Log2(("SyncPT: 4K %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
2669	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
2670	PGSTPT pPTSrc;
2671	rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
2672	if (RT_SUCCESS(rc))
2673	{
2674	/*
2675	* Start by syncing the page directory entry so CSAM's TLB trick works.
2676	*/
2677	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| X86_PDE_AVL_MASK))
2678	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2679	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2680	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2681
2682	/*
2683	* Directory/page user or supervisor privilege: (same goes for read/write)
2684	*
2685	* Directory Page Combined
2686	* U/S U/S U/S
2687	* 0 0 0
2688	* 0 1 0
2689	* 1 0 0
2690	* 1 1 1
2691	*
2692	* Simple AND operation. Table listed for completeness.
2693	*
2694	*/
2695	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT4K));
2696	# ifdef PGM_SYNC_N_PAGES
2697	unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2698	unsigned iPTDst = iPTBase;
2699	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2700	if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
2701	iPTDst = 0;
2702	else
2703	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2704	# else /* !PGM_SYNC_N_PAGES */
2705	unsigned iPTDst = 0;
2706	const unsigned iPTDstEnd = RT_ELEMENTS(pPTDst->a);
2707	# endif /* !PGM_SYNC_N_PAGES */
2708	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2709	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2710	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2711	# else
2712	const unsigned offPTSrc = 0;
2713	# endif
2714	for (; iPTDst < iPTDstEnd; iPTDst++)
2715	{
2716	const unsigned iPTSrc = iPTDst + offPTSrc;
2717	const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2718
2719	if (PteSrc.n.u1Present)
2720	{
2721	# ifndef IN_RING0
2722	/*
2723	* Assuming kernel code will be marked as supervisor - and not as user level
2724	* and executed using a conforming code selector - And marked as readonly.
2725	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
2726	*/
2727	PPGMPAGE pPage;
2728	if ( ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW \| X86_PTE_US))
2729	\|\| !CSAMDoesPageNeedScanning(pVM, (iPDSrc << GST_PD_SHIFT) \| (iPTSrc << PAGE_SHIFT))
2730	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
2731	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2732	)
2733	# endif
2734	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2735	Log2(("SyncPT: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
2736	(RTGCPTR)(((RTGCPTR)iPDSrc << GST_PD_SHIFT) \| ((RTGCPTR)iPTSrc << PAGE_SHIFT)),
2737	PteSrc.n.u1Present,
2738	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2739	PteSrc.n.u1User & PdeSrc.n.u1User,
2740	(uint64_t)PteSrc.u,
2741	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : "", pPTDst->a[iPTDst].u, iPTSrc, PdeSrc.au32[0],
2742	(RTGCPHYS)((PdeSrc.u & GST_PDE_PG_MASK) + iPTSrc*sizeof(PteSrc)) ));
2743	}
2744	/* else: the page table was cleared by the pool */
2745	} /* for PTEs */
2746	}
2747	}
2748	else
2749	{
2750	/*
2751	* Big page - 2/4MB.
2752	*
2753	* We'll walk the ram range list in parallel and optimize lookups.
2754	* We will only sync on shadow page table at a time.
2755	*/
2756	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT4M));
2757
2758	/**
2759	* @todo It might be more efficient to sync only a part of the 4MB
2760	* page (similar to what we do for 4KB PDs).
2761	*/
2762
2763	/*
2764	* Start by syncing the page directory entry.
2765	*/
2766	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
2767	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2768
2769	/*
2770	* If the page is not flagged as dirty and is writable, then make it read-only
2771	* at PD level, so we can set the dirty bit when the page is modified.
2772	*
2773	* ASSUMES that page access handlers are implemented on page table entry level.
2774	* Thus we will first catch the dirty access and set PDE.D and restart. If
2775	* there is an access handler, we'll trap again and let it work on the problem.
2776	*/
2777	/** @todo move the above stuff to a section in the PGM documentation. */
2778	Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
2779	if ( !PdeSrc.b.u1Dirty
2780	&& PdeSrc.b.u1Write)
2781	{
2782	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2783	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2784	PdeDst.b.u1Write = 0;
2785	}
2786	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2787	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2788
2789	/*
2790	* Fill the shadow page table.
2791	*/
2792	/* Get address and flags from the source PDE. */
2793	SHWPTE PteDstBase;
2794	PteDstBase.u = PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT);
2795
2796	/* Loop thru the entries in the shadow PT. */
2797	const RTGCPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
2798	Log2(("SyncPT: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n",
2799	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
2800	GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2801	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
2802	unsigned iPTDst = 0;
2803	while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2804	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2805	{
2806	/* Advance ram range list. */
2807	while (pRam && GCPhys > pRam->GCPhysLast)
2808	pRam = pRam->CTX_SUFF(pNext);
2809	if (pRam && GCPhys >= pRam->GCPhys)
2810	{
2811	unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2812	do
2813	{
2814	/* Make shadow PTE. */
2815	PPGMPAGE pPage = &pRam->aPages[iHCPage];
2816	SHWPTE PteDst;
2817
2818	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2819	/* Try to make the page writable if necessary. */
2820	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2821	&& ( PGM_PAGE_IS_ZERO(pPage)
2822	\|\| ( PteDstBase.n.u1Write
2823	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2824	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
2825	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
2826	# endif
2827	# ifdef VBOX_WITH_PAGE_SHARING
2828	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
2829	# endif
2830	&& !PGM_PAGE_IS_BALLOONED(pPage))
2831	)
2832	)
2833	{
2834	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
2835	AssertRCReturn(rc, rc);
2836	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2837	break;
2838	}
2839	# endif
2840
2841	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2842	{
2843	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
2844	{
2845	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) \| PteDstBase.u;
2846	PteDst.n.u1Write = 0;
2847	}
2848	else
2849	PteDst.u = 0;
2850	}
2851	else if (PGM_PAGE_IS_BALLOONED(pPage))
2852	{
2853	/* Skip ballooned pages. */
2854	PteDst.u = 0;
2855	}
2856	# ifndef IN_RING0
2857	/*
2858	* Assuming kernel code will be marked as supervisor and not as user level and executed
2859	* using a conforming code selector. Don't check for readonly, as that implies the whole
2860	* 4MB can be code or readonly data. Linux enables write access for its large pages.
2861	*/
2862	else if ( !PdeSrc.n.u1User
2863	&& CSAMDoesPageNeedScanning(pVM, GCPtr \| (iPTDst << SHW_PT_SHIFT)))
2864	PteDst.u = 0;
2865	# endif
2866	else
2867	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) \| PteDstBase.u;
2868
2869	/* Only map writable pages writable. */
2870	if ( PteDst.n.u1Write
2871	&& PteDst.n.u1Present
2872	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2873	{
2874	/* Still applies to shared pages. */
2875	Assert(!PGM_PAGE_IS_ZERO(pPage));
2876	PteDst.n.u1Write = 0; /** @todo this isn't quite working yet... */
2877	Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))));
2878	}
2879
2880	if (PteDst.n.u1Present)
2881	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2882
2883	/* commit it */
2884	pPTDst->a[iPTDst] = PteDst;
2885	Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
2886	(RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT)), PteDst.n.u1Present, PteDst.n.u1Write, PteDst.n.u1User, (uint64_t)PteDst.u,
2887	PteDst.u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2888
2889	/* advance */
2890	GCPhys += PAGE_SIZE;
2891	iHCPage++;
2892	iPTDst++;
2893	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2894	&& GCPhys <= pRam->GCPhysLast);
2895	}
2896	else if (pRam)
2897	{
2898	Log(("Invalid pages at %RGp\n", GCPhys));
2899	do
2900	{
2901	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2902	GCPhys += PAGE_SIZE;
2903	iPTDst++;
2904	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2905	&& GCPhys < pRam->GCPhys);
2906	}
2907	else
2908	{
2909	Log(("Invalid pages at %RGp (2)\n", GCPhys));
2910	for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
2911	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2912	}
2913	} /* while more PTEs */
2914	} /* 4KB / 4MB */
2915	}
2916	else
2917	AssertRelease(!PdeDst.n.u1Present);
2918
2919	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2920	if (RT_FAILURE(rc))
2921	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPTFailed));
2922	return rc;
2923
2924	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2925	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2926	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2927	&& !defined(IN_RC)
2928
2929	/*
2930	* Validate input a little bit.
2931	*/
2932	int rc = VINF_SUCCESS;
2933	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2934	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2935	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
2936
2937	/* Fetch the pgm pool shadow descriptor. */
2938	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2939	Assert(pShwPde);
2940
2941	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2942	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2943	PPGMPOOLPAGE pShwPde = NULL; /* initialized to shut up gcc */
2944	PX86PDPAE pPDDst;
2945	PSHWPDE pPdeDst;
2946
2947	/* Fetch the pgm pool shadow descriptor. */
2948	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
2949	AssertRCSuccessReturn(rc, rc);
2950	Assert(pShwPde);
2951
2952	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
2953	pPdeDst = &pPDDst->a[iPDDst];
2954
2955	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2956	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2957	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2958	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2959	PX86PDPT pPdptDst= NULL; /* initialized to shut up gcc */
2960	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2961	AssertRCSuccessReturn(rc, rc);
2962	Assert(pPDDst);
2963	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2964
2965	/* Fetch the pgm pool shadow descriptor. */
2966	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2967	Assert(pShwPde);
2968
2969	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2970	const unsigned iPdpt = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
2971	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2972	PEPTPD pPDDst;
2973	PEPTPDPT pPdptDst;
2974
2975	rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, &pPdptDst, &pPDDst);
2976	if (rc != VINF_SUCCESS)
2977	{
2978	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2979	AssertRC(rc);
2980	return rc;
2981	}
2982	Assert(pPDDst);
2983	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2984
2985	/* Fetch the pgm pool shadow descriptor. */
2986	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & EPT_PDPTE_PG_MASK);
2987	Assert(pShwPde);
2988	# endif
2989	SHWPDE PdeDst = *pPdeDst;
2990
2991	Assert(!(PdeDst.u & PGM_PDFLAGS_MAPPING));
2992	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2993
2994	# if defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
2995	if (BTH_IS_NP_ACTIVE(pVM))
2996	{
2997	PPGMPAGE pPage;
2998
2999	/* Check if we allocated a big page before for this 2 MB range. */
3000	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPtrPage & X86_PDE2M_PAE_PG_MASK, &pPage);
3001	if (RT_SUCCESS(rc))
3002	{
3003	RTHCPHYS HCPhys = NIL_RTHCPHYS;
3004
3005	if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
3006	{
3007	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageReused);
3008	AssertRelease(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3009	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3010	}
3011	else if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED)
3012	{
3013	/* Recheck the entire 2 MB range to see if we can use it again as a large page. */
3014	rc = pgmPhysIsValidLargePage(pVM, GCPtrPage, pPage);
3015	if (RT_SUCCESS(rc))
3016	{
3017	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3018	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3019	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3020	}
3021	}
3022	else if (PGMIsUsingLargePages(pVM))
3023	{
3024	rc = pgmPhysAllocLargePage(pVM, GCPtrPage);
3025	if (RT_SUCCESS(rc))
3026	{
3027	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3028	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3029	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3030	}
3031	else
3032	LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
3033	}
3034
3035	if (HCPhys != NIL_RTHCPHYS)
3036	{
3037	PdeDst.u &= X86_PDE_AVL_MASK;
3038	PdeDst.u \|= HCPhys;
3039	PdeDst.n.u1Present = 1;
3040	PdeDst.n.u1Write = 1;
3041	PdeDst.b.u1Size = 1;
3042	# if PGM_SHW_TYPE == PGM_TYPE_EPT
3043	PdeDst.n.u1Execute = 1;
3044	PdeDst.b.u1IgnorePAT = 1;
3045	PdeDst.b.u3EMT = VMX_EPT_MEMTYPE_WB;
3046	# else
3047	PdeDst.n.u1User = 1;
3048	# endif
3049	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
3050
3051	Log(("SyncPT: Use large page at %RGp PDE=%RX64\n", GCPtrPage, PdeDst.u));
3052	/* Add a reference to the first page only. */
3053	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPde, PGM_PAGE_GET_TRACKING(pPage), pPage, iPDDst);
3054
3055	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3056	return VINF_SUCCESS;
3057	}
3058	}
3059	}
3060	# endif /* HC_ARCH_BITS == 64 */
3061
3062	GSTPDE PdeSrc;
3063	PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3064	PdeSrc.n.u1Present = 1;
3065	PdeSrc.n.u1Write = 1;
3066	PdeSrc.n.u1Accessed = 1;
3067	PdeSrc.n.u1User = 1;
3068
3069	/*
3070	* Allocate & map the page table.
3071	*/
3072	PSHWPT pPTDst;
3073	PPGMPOOLPAGE pShwPage;
3074	RTGCPHYS GCPhys;
3075
3076	/* Virtual address = physical address */
3077	GCPhys = GCPtrPage & X86_PAGE_4K_BASE_MASK;
3078	rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
3079
3080	if ( rc == VINF_SUCCESS
3081	\|\| rc == VINF_PGM_CACHED_PAGE)
3082	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3083	else
3084	AssertMsgFailedReturn(("rc=%Rrc\n", rc), VERR_INTERNAL_ERROR);
3085
3086	PdeDst.u &= X86_PDE_AVL_MASK;
3087	PdeDst.u \|= pShwPage->Core.Key;
3088	PdeDst.n.u1Present = 1;
3089	PdeDst.n.u1Write = 1;
3090	# if PGM_SHW_TYPE == PGM_TYPE_EPT
3091	PdeDst.n.u1Execute = 1;
3092	# else
3093	PdeDst.n.u1User = 1;
3094	PdeDst.n.u1Accessed = 1;
3095	# endif
3096	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
3097
3098	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, PGM_SYNC_NR_PAGES, 0 /* page not present */);
3099	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3100	return rc;
3101
3102	#else
3103	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
3104	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3105	return VERR_INTERNAL_ERROR;
3106	#endif
3107	}
3108
3109
3110
3111	/**
3112	* Prefetch a page/set of pages.
3113	*
3114	* Typically used to sync commonly used pages before entering raw mode
3115	* after a CR3 reload.
3116	*
3117	* @returns VBox status code.
3118	* @param pVCpu The VMCPU handle.
3119	* @param GCPtrPage Page to invalidate.
3120	*/
3121	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
3122	{
3123	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3124	\|\| PGM_GST_TYPE == PGM_TYPE_REAL \
3125	\|\| PGM_GST_TYPE == PGM_TYPE_PROT \
3126	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3127	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3128	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
3129	&& PGM_SHW_TYPE != PGM_TYPE_EPT
3130
3131	/*
3132	* Check that all Guest levels thru the PDE are present, getting the
3133	* PD and PDE in the processes.
3134	*/
3135	int rc = VINF_SUCCESS;
3136	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3137	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3138	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
3139	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3140	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3141	unsigned iPDSrc;
3142	X86PDPE PdpeSrc;
3143	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3144	if (!pPDSrc)
3145	return VINF_SUCCESS; /* not present */
3146	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3147	unsigned iPDSrc;
3148	PX86PML4E pPml4eSrc;
3149	X86PDPE PdpeSrc;
3150	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3151	if (!pPDSrc)
3152	return VINF_SUCCESS; /* not present */
3153	# endif
3154	const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3155	# else
3156	PGSTPD pPDSrc = NULL;
3157	const unsigned iPDSrc = 0;
3158	GSTPDE PdeSrc;
3159
3160	PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3161	PdeSrc.n.u1Present = 1;
3162	PdeSrc.n.u1Write = 1;
3163	PdeSrc.n.u1Accessed = 1;
3164	PdeSrc.n.u1User = 1;
3165	# endif
3166
3167	if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
3168	{
3169	PVM pVM = pVCpu->CTX_SUFF(pVM);
3170	pgmLock(pVM);
3171
3172	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3173	const X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
3174	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3175	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3176	PX86PDPAE pPDDst;
3177	X86PDEPAE PdeDst;
3178	# if PGM_GST_TYPE != PGM_TYPE_PAE
3179	X86PDPE PdpeSrc;
3180
3181	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3182	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3183	# endif
3184	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3185	if (rc != VINF_SUCCESS)
3186	{
3187	pgmUnlock(pVM);
3188	AssertRC(rc);
3189	return rc;
3190	}
3191	Assert(pPDDst);
3192	PdeDst = pPDDst->a[iPDDst];
3193
3194	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3195	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3196	PX86PDPAE pPDDst;
3197	X86PDEPAE PdeDst;
3198
3199	# if PGM_GST_TYPE == PGM_TYPE_PROT
3200	/* AMD-V nested paging */
3201	X86PML4E Pml4eSrc;
3202	X86PDPE PdpeSrc;
3203	PX86PML4E pPml4eSrc = &Pml4eSrc;
3204
3205	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3206	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
3207	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
3208	# endif
3209
3210	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3211	if (rc != VINF_SUCCESS)
3212	{
3213	pgmUnlock(pVM);
3214	AssertRC(rc);
3215	return rc;
3216	}
3217	Assert(pPDDst);
3218	PdeDst = pPDDst->a[iPDDst];
3219	# endif
3220	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
3221	{
3222	if (!PdeDst.n.u1Present)
3223	{
3224	/** @todo r=bird: This guy will set the A bit on the PDE,
3225	* probably harmless. */
3226	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3227	}
3228	else
3229	{
3230	/* Note! We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
3231	* R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
3232	* makes no sense to prefetch more than one page.
3233	*/
3234	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3235	if (RT_SUCCESS(rc))
3236	rc = VINF_SUCCESS;
3237	}
3238	}
3239	pgmUnlock(pVM);
3240	}
3241	return rc;
3242
3243	#elif PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3244	return VINF_SUCCESS; /* ignore */
3245	#else
3246	AssertCompile(0);
3247	#endif
3248	}
3249
3250
3251
3252
3253	/**
3254	* Syncs a page during a PGMVerifyAccess() call.
3255	*
3256	* @returns VBox status code (informational included).
3257	* @param pVCpu The VMCPU handle.
3258	* @param GCPtrPage The address of the page to sync.
3259	* @param fPage The effective guest page flags.
3260	* @param uErr The trap error code.
3261	* @remarks This will normally never be called on invalid guest page
3262	* translation entries.
3263	*/
3264	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fPage, unsigned uErr)
3265	{
3266	PVM pVM = pVCpu->CTX_SUFF(pVM);
3267
3268	LogFlow(("VerifyAccessSyncPage: GCPtrPage=%RGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
3269
3270	Assert(!pVM->pgm.s.fNestedPaging);
3271	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3272	\|\| PGM_GST_TYPE == PGM_TYPE_REAL \
3273	\|\| PGM_GST_TYPE == PGM_TYPE_PROT \
3274	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3275	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3276	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
3277	&& PGM_SHW_TYPE != PGM_TYPE_EPT
3278
3279	# ifndef IN_RING0
3280	if (!(fPage & X86_PTE_US))
3281	{
3282	/*
3283	* Mark this page as safe.
3284	*/
3285	/** @todo not correct for pages that contain both code and data!! */
3286	Log(("CSAMMarkPage %RGv; scanned=%d\n", GCPtrPage, true));
3287	CSAMMarkPage(pVM, GCPtrPage, true);
3288	}
3289	# endif
3290
3291	/*
3292	* Get guest PD and index.
3293	*/
3294	/** @todo Performance: We've done all this a jiffy ago in the
3295	* PGMGstGetPage call. */
3296	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3297	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3298	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
3299	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3300
3301	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3302	unsigned iPDSrc = 0;
3303	X86PDPE PdpeSrc;
3304	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3305	if (RT_UNLIKELY(!pPDSrc))
3306	{
3307	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3308	return VINF_EM_RAW_GUEST_TRAP;
3309	}
3310
3311	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3312	unsigned iPDSrc = 0; /* shut up gcc */
3313	PX86PML4E pPml4eSrc = NULL; /* ditto */
3314	X86PDPE PdpeSrc;
3315	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3316	if (RT_UNLIKELY(!pPDSrc))
3317	{
3318	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3319	return VINF_EM_RAW_GUEST_TRAP;
3320	}
3321	# endif
3322
3323	# else /* !PGM_WITH_PAGING */
3324	PGSTPD pPDSrc = NULL;
3325	const unsigned iPDSrc = 0;
3326	# endif /* !PGM_WITH_PAGING */
3327	int rc = VINF_SUCCESS;
3328
3329	pgmLock(pVM);
3330
3331	/*
3332	* First check if the shadow pd is present.
3333	*/
3334	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3335	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3336
3337	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3338	PX86PDEPAE pPdeDst;
3339	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3340	PX86PDPAE pPDDst;
3341	# if PGM_GST_TYPE != PGM_TYPE_PAE
3342	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3343	X86PDPE PdpeSrc;
3344	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3345	# endif
3346	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3347	if (rc != VINF_SUCCESS)
3348	{
3349	pgmUnlock(pVM);
3350	AssertRC(rc);
3351	return rc;
3352	}
3353	Assert(pPDDst);
3354	pPdeDst = &pPDDst->a[iPDDst];
3355
3356	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3357	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3358	PX86PDPAE pPDDst;
3359	PX86PDEPAE pPdeDst;
3360
3361	# if PGM_GST_TYPE == PGM_TYPE_PROT
3362	/* AMD-V nested paging: Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3363	X86PML4E Pml4eSrc;
3364	X86PDPE PdpeSrc;
3365	PX86PML4E pPml4eSrc = &Pml4eSrc;
3366	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
3367	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
3368	# endif
3369
3370	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3371	if (rc != VINF_SUCCESS)
3372	{
3373	pgmUnlock(pVM);
3374	AssertRC(rc);
3375	return rc;
3376	}
3377	Assert(pPDDst);
3378	pPdeDst = &pPDDst->a[iPDDst];
3379	# endif
3380
3381	if (!pPdeDst->n.u1Present)
3382	{
3383	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3384	if (rc != VINF_SUCCESS)
3385	{
3386	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3387	pgmUnlock(pVM);
3388	AssertRC(rc);
3389	return rc;
3390	}
3391	}
3392
3393	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3394	/* Check for dirty bit fault */
3395	rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
3396	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
3397	Log(("PGMVerifyAccess: success (dirty)\n"));
3398	else
3399	# endif
3400	{
3401	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3402	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3403	# else
3404	GSTPDE PdeSrc;
3405	PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3406	PdeSrc.n.u1Present = 1;
3407	PdeSrc.n.u1Write = 1;
3408	PdeSrc.n.u1Accessed = 1;
3409	PdeSrc.n.u1User = 1;
3410	# endif
3411
3412	Assert(rc != VINF_EM_RAW_GUEST_TRAP);
3413	if (uErr & X86_TRAP_PF_US)
3414	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUser));
3415	else /* supervisor */
3416	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
3417
3418	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3419	if (RT_SUCCESS(rc))
3420	{
3421	/* Page was successfully synced */
3422	Log2(("PGMVerifyAccess: success (sync)\n"));
3423	rc = VINF_SUCCESS;
3424	}
3425	else
3426	{
3427	Log(("PGMVerifyAccess: access violation for %RGv rc=%Rrc\n", GCPtrPage, rc));
3428	rc = VINF_EM_RAW_GUEST_TRAP;
3429	}
3430	}
3431	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3432	pgmUnlock(pVM);
3433	return rc;
3434
3435	#else /* PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED */
3436
3437	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
3438	return VERR_INTERNAL_ERROR;
3439	#endif /* PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED */
3440	}
3441
3442
3443	/**
3444	* Syncs the paging hierarchy starting at CR3.
3445	*
3446	* @returns VBox status code, no specials.
3447	* @param pVCpu The VMCPU handle.
3448	* @param cr0 Guest context CR0 register
3449	* @param cr3 Guest context CR3 register
3450	* @param cr4 Guest context CR4 register
3451	* @param fGlobal Including global page directories or not
3452	*/
3453	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
3454	{
3455	PVM pVM = pVCpu->CTX_SUFF(pVM);
3456
3457	LogFlow(("SyncCR3 fGlobal=%d\n", !!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)));
3458
3459	#if PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
3460
3461	pgmLock(pVM);
3462
3463	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
3464	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3465	if (pPool->cDirtyPages)
3466	pgmPoolResetDirtyPages(pVM);
3467	# endif
3468
3469	/*
3470	* Update page access handlers.
3471	* The virtual are always flushed, while the physical are only on demand.
3472	* WARNING: We are incorrectly not doing global flushing on Virtual Handler updates. We'll
3473	* have to look into that later because it will have a bad influence on the performance.
3474	* @note SvL: There's no need for that. Just invalidate the virtual range(s).
3475	* bird: Yes, but that won't work for aliases.
3476	*/
3477	/** @todo this MUST go away. See #1557. */
3478	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3Handlers), h);
3479	PGM_GST_NAME(HandlerVirtualUpdate)(pVM, cr4);
3480	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3Handlers), h);
3481	pgmUnlock(pVM);
3482	#endif /* !NESTED && !EPT */
3483
3484	#if PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3485	/*
3486	* Nested / EPT - almost no work.
3487	*/
3488	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
3489	return VINF_SUCCESS;
3490
3491	#elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3492	/*
3493	* AMD64 (Shw & Gst) - No need to check all paging levels; we zero
3494	* out the shadow parts when the guest modifies its tables.
3495	*/
3496	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
3497	return VINF_SUCCESS;
3498
3499	#else /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3500
3501	# ifndef PGM_WITHOUT_MAPPINGS
3502	/*
3503	* Check for and resolve conflicts with our guest mappings if they
3504	* are enabled and not fixed.
3505	*/
3506	if (pgmMapAreMappingsFloating(&pVM->pgm.s))
3507	{
3508	int rc = pgmMapResolveConflicts(pVM);
3509	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
3510	if (rc == VINF_PGM_SYNC_CR3)
3511	{
3512	LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
3513	return VINF_PGM_SYNC_CR3;
3514	}
3515	}
3516	# else
3517	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
3518	# endif
3519	return VINF_SUCCESS;
3520	#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3521	}
3522
3523
3524
3525
3526	#ifdef VBOX_STRICT
3527	#ifdef IN_RC
3528	# undef AssertMsgFailed
3529	# define AssertMsgFailed Log
3530	#endif
3531	#ifdef IN_RING3
3532	# include <VBox/dbgf.h>
3533
3534	/**
3535	* Dumps a page table hierarchy use only physical addresses and cr4/lm flags.
3536	*
3537	* @returns VBox status code (VINF_SUCCESS).
3538	* @param cr3 The root of the hierarchy.
3539	* @param crr The cr4, only PAE and PSE is currently used.
3540	* @param fLongMode Set if long mode, false if not long mode.
3541	* @param cMaxDepth Number of levels to dump.
3542	* @param pHlp Pointer to the output functions.
3543	*/
3544	RT_C_DECLS_BEGIN
3545	VMMR3DECL(int) PGMR3DumpHierarchyHC(PVM pVM, uint32_t cr3, uint32_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp);
3546	RT_C_DECLS_END
3547
3548	#endif
3549
3550	/**
3551	* Checks that the shadow page table is in sync with the guest one.
3552	*
3553	* @returns The number of errors.
3554	* @param pVM The virtual machine.
3555	* @param pVCpu The VMCPU handle.
3556	* @param cr3 Guest context CR3 register
3557	* @param cr4 Guest context CR4 register
3558	* @param GCPtr Where to start. Defaults to 0.
3559	* @param cb How much to check. Defaults to everything.
3560	*/
3561	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)
3562	{
3563	#if PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3564	return 0;
3565	#else
3566	unsigned cErrors = 0;
3567	PVM pVM = pVCpu->CTX_SUFF(pVM);
3568	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3569
3570	#if PGM_GST_TYPE == PGM_TYPE_PAE
3571	/** @todo currently broken; crashes below somewhere */
3572	AssertFailed();
3573	#endif
3574
3575	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
3576	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3577	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
3578
3579	bool fBigPagesSupported = GST_IS_PSE_ACTIVE(pVCpu);
3580	PPGMCPU pPGM = &pVCpu->pgm.s;
3581	RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
3582	RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
3583	# ifndef IN_RING0
3584	RTHCPHYS HCPhys; /* general usage. */
3585	# endif
3586	int rc;
3587
3588	/*
3589	* Check that the Guest CR3 and all its mappings are correct.
3590	*/
3591	AssertMsgReturn(pPGM->GCPhysCR3 == (cr3 & GST_CR3_PAGE_MASK),
3592	("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
3593	false);
3594	# if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64
3595	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3596	rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGst32BitPdRC, NULL, &HCPhysShw);
3597	# else
3598	rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGstPaePdptRC, NULL, &HCPhysShw);
3599	# endif
3600	AssertRCReturn(rc, 1);
3601	HCPhys = NIL_RTHCPHYS;
3602	rc = pgmRamGCPhys2HCPhys(&pVM->pgm.s, cr3 & GST_CR3_PAGE_MASK, &HCPhys);
3603	AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
3604	# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
3605	pgmGstGet32bitPDPtr(pVCpu);
3606	RTGCPHYS GCPhys;
3607	rc = PGMR3DbgR3Ptr2GCPhys(pVM, pPGM->pGst32BitPdR3, &GCPhys);
3608	AssertRCReturn(rc, 1);
3609	AssertMsgReturn((cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
3610	# endif
3611	# endif /* !IN_RING0 */
3612
3613	/*
3614	* Get and check the Shadow CR3.
3615	*/
3616	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3617	unsigned cPDEs = X86_PG_ENTRIES;
3618	unsigned cIncrement = X86_PG_ENTRIES * PAGE_SIZE;
3619	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3620	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3621	unsigned cPDEs = X86_PG_PAE_ENTRIES * 4; /* treat it as a 2048 entry table. */
3622	# else
3623	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3624	# endif
3625	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3626	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3627	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3628	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3629	# endif
3630	if (cb != ~(RTGCPTR)0)
3631	cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
3632
3633	/** @todo call the other two PGMAssert() functions. /
3634
3635	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3636	unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3637
3638	for (; iPml4 < X86_PG_PAE_ENTRIES; iPml4++)
3639	{
3640	PPGMPOOLPAGE pShwPdpt = NULL;
3641	PX86PML4E pPml4eSrc;
3642	PX86PML4E pPml4eDst;
3643	RTGCPHYS GCPhysPdptSrc;
3644
3645	pPml4eSrc = pgmGstGetLongModePML4EPtr(pVCpu, iPml4);
3646	pPml4eDst = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
3647
3648	/* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
3649	if (!pPml4eDst->n.u1Present)
3650	{
3651	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3652	continue;
3653	}
3654
3655	pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
3656	GCPhysPdptSrc = pPml4eSrc->u & X86_PML4E_PG_MASK_FULL;
3657
3658	if (pPml4eSrc->n.u1Present != pPml4eDst->n.u1Present)
3659	{
3660	AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3661	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3662	cErrors++;
3663	continue;
3664	}
3665
3666	if (GCPhysPdptSrc != pShwPdpt->GCPhys)
3667	{
3668	AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
3669	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3670	cErrors++;
3671	continue;
3672	}
3673
3674	if ( pPml4eDst->n.u1User != pPml4eSrc->n.u1User
3675	\|\| pPml4eDst->n.u1Write != pPml4eSrc->n.u1Write
3676	\|\| pPml4eDst->n.u1NoExecute != pPml4eSrc->n.u1NoExecute)
3677	{
3678	AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3679	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3680	cErrors++;
3681	continue;
3682	}
3683	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3684	{
3685	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3686
3687	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
3688	/*
3689	* Check the PDPTEs too.
3690	*/
3691	unsigned iPdpt = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
3692
3693	for (;iPdpt <= SHW_PDPT_MASK; iPdpt++)
3694	{
3695	unsigned iPDSrc = 0; /* initialized to shut up gcc */
3696	PPGMPOOLPAGE pShwPde = NULL;
3697	PX86PDPE pPdpeDst;
3698	RTGCPHYS GCPhysPdeSrc;
3699	# if PGM_GST_TYPE == PGM_TYPE_PAE
3700	X86PDPE PdpeSrc;
3701	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtr, &iPDSrc, &PdpeSrc);
3702	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
3703	# else
3704	PX86PML4E pPml4eSrcIgn;
3705	X86PDPE PdpeSrc;
3706	PX86PDPT pPdptDst;
3707	PX86PDPAE pPDDst;
3708	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtr, &pPml4eSrcIgn, &PdpeSrc, &iPDSrc);
3709
3710	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtr, NULL, &pPdptDst, &pPDDst);
3711	if (rc != VINF_SUCCESS)
3712	{
3713	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
3714	GCPtr += 512 * _2M;
3715	continue; /* next PDPTE */
3716	}
3717	Assert(pPDDst);
3718	# endif
3719	Assert(iPDSrc == 0);
3720
3721	pPdpeDst = &pPdptDst->a[iPdpt];
3722
3723	if (!pPdpeDst->n.u1Present)
3724	{
3725	GCPtr += 512 * _2M;
3726	continue; /* next PDPTE */
3727	}
3728
3729	pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
3730	GCPhysPdeSrc = PdpeSrc.u & X86_PDPE_PG_MASK;
3731
3732	if (pPdpeDst->n.u1Present != PdpeSrc.n.u1Present)
3733	{
3734	AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3735	GCPtr += 512 * _2M;
3736	cErrors++;
3737	continue;
3738	}
3739
3740	if (GCPhysPdeSrc != pShwPde->GCPhys)
3741	{
3742	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3743	AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3744	# else
3745	AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3746	# endif
3747	GCPtr += 512 * _2M;
3748	cErrors++;
3749	continue;
3750	}
3751
3752	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3753	if ( pPdpeDst->lm.u1User != PdpeSrc.lm.u1User
3754	\|\| pPdpeDst->lm.u1Write != PdpeSrc.lm.u1Write
3755	\|\| pPdpeDst->lm.u1NoExecute != PdpeSrc.lm.u1NoExecute)
3756	{
3757	AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3758	GCPtr += 512 * _2M;
3759	cErrors++;
3760	continue;
3761	}
3762	# endif
3763
3764	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3765	{
3766	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3767	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3768	GSTPD const *pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3769	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3770	PCX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
3771	# endif
3772	# endif /* PGM_GST_TYPE == PGM_TYPE_32BIT */
3773	/*
3774	* Iterate the shadow page directory.
3775	*/
3776	GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
3777	unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
3778
3779	for (;
3780	iPDDst < cPDEs;
3781	iPDDst++, GCPtr += cIncrement)
3782	{
3783	# if PGM_SHW_TYPE == PGM_TYPE_PAE
3784	const SHWPDE PdeDst = *pgmShwGetPaePDEPtr(pVCpu, GCPtr);
3785	# else
3786	const SHWPDE PdeDst = pPDDst->a[iPDDst];
3787	# endif
3788	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
3789	{
3790	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
3791	if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
3792	{
3793	AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
3794	cErrors++;
3795	continue;
3796	}
3797	}
3798	else if ( (PdeDst.u & X86_PDE_P)
3799	\|\| ((PdeDst.u & (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY))
3800	)
3801	{
3802	HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
3803	PPGMPOOLPAGE pPoolPage = pgmPoolGetPage(pPool, HCPhysShw);
3804	if (!pPoolPage)
3805	{
3806	AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n",
3807	HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
3808	cErrors++;
3809	continue;
3810	}
3811	const SHWPT pPTDst = (const SHWPT )PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pPoolPage);
3812
3813	if (PdeDst.u & (X86_PDE4M_PWT \| X86_PDE4M_PCD))
3814	{
3815	AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
3816	GCPtr, (uint64_t)PdeDst.u));
3817	cErrors++;
3818	}
3819
3820	if (PdeDst.u & (X86_PDE4M_G \| X86_PDE4M_D))
3821	{
3822	AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n",
3823	GCPtr, (uint64_t)PdeDst.u));
3824	cErrors++;
3825	}
3826
3827	const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
3828	if (!PdeSrc.n.u1Present)
3829	{
3830	AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
3831	GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
3832	cErrors++;
3833	continue;
3834	}
3835
3836	if ( !PdeSrc.b.u1Size
3837	\|\| !fBigPagesSupported)
3838	{
3839	GCPhysGst = PdeSrc.u & GST_PDE_PG_MASK;
3840	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3841	GCPhysGst \|= (iPDDst & 1) * (PAGE_SIZE / 2);
3842	# endif
3843	}
3844	else
3845	{
3846	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3847	if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
3848	{
3849	AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n",
3850	GCPtr, (uint64_t)PdeSrc.u));
3851	cErrors++;
3852	continue;
3853	}
3854	# endif
3855	GCPhysGst = GST_GET_PDE_BIG_PG_GCPHYS(pVM, PdeSrc);
3856	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3857	GCPhysGst \|= GCPtr & RT_BIT(X86_PAGE_2M_SHIFT);
3858	# endif
3859	}
3860
3861	if ( pPoolPage->enmKind
3862	!= (!PdeSrc.b.u1Size \|\| !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
3863	{
3864	AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n",
3865	pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
3866	cErrors++;
3867	}
3868
3869	PPGMPAGE pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
3870	if (!pPhysPage)
3871	{
3872	AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3873	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3874	cErrors++;
3875	continue;
3876	}
3877
3878	if (GCPhysGst != pPoolPage->GCPhys)
3879	{
3880	AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n",
3881	GCPhysGst, pPoolPage->GCPhys, GCPtr));
3882	cErrors++;
3883	continue;
3884	}
3885
3886	if ( !PdeSrc.b.u1Size
3887	\|\| !fBigPagesSupported)
3888	{
3889	/*
3890	* Page Table.
3891	*/
3892	const GSTPT *pPTSrc;
3893	rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1), &pPTSrc);
3894	if (RT_FAILURE(rc))
3895	{
3896	AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3897	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3898	cErrors++;
3899	continue;
3900	}
3901	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/))
3902	!= (PdeDst.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/)))
3903	{
3904	/// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
3905	// (This problem will go away when/if we shadow multiple CR3s.)
3906	AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3907	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3908	cErrors++;
3909	continue;
3910	}
3911	if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3912	{
3913	AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n",
3914	GCPtr, (uint64_t)PdeDst.u));
3915	cErrors++;
3916	continue;
3917	}
3918
3919	/* iterate the page table. */
3920	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3921	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3922	const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
3923	# else
3924	const unsigned offPTSrc = 0;
3925	# endif
3926	for (unsigned iPT = 0, off = 0;
3927	iPT < RT_ELEMENTS(pPTDst->a);
3928	iPT++, off += PAGE_SIZE)
3929	{
3930	const SHWPTE PteDst = pPTDst->a[iPT];
3931
3932	/* skip not-present entries. */
3933	if (!(PteDst.u & (X86_PTE_P \| PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
3934	continue;
3935	Assert(PteDst.n.u1Present);
3936
3937	const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
3938	if (!PteSrc.n.u1Present)
3939	{
3940	# ifdef IN_RING3
3941	PGMAssertHandlerAndFlagsInSync(pVM);
3942	PGMR3DumpHierarchyGC(pVM, cr3, cr4, (PdeSrc.u & GST_PDE_PG_MASK));
3943	# endif
3944	AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
3945	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u, pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
3946	(PdeSrc.u & GST_PDE_PG_MASK) + (iPT + offPTSrc)*sizeof(PteSrc)));
3947	cErrors++;
3948	continue;
3949	}
3950
3951	uint64_t fIgnoreFlags = GST_PTE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_G \| X86_PTE_D \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
3952	# if 1 /** @todo sync accessed bit properly... */
3953	fIgnoreFlags \|= X86_PTE_A;
3954	# endif
3955
3956	/* match the physical addresses */
3957	HCPhysShw = PteDst.u & SHW_PTE_PG_MASK;
3958	GCPhysGst = PteSrc.u & GST_PTE_PG_MASK;
3959
3960	# ifdef IN_RING3
3961	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3962	if (RT_FAILURE(rc))
3963	{
3964	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
3965	{
3966	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3967	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3968	cErrors++;
3969	continue;
3970	}
3971	}
3972	else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
3973	{
3974	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3975	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3976	cErrors++;
3977	continue;
3978	}
3979	# endif
3980
3981	pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
3982	if (!pPhysPage)
3983	{
3984	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
3985	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
3986	{
3987	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3988	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3989	cErrors++;
3990	continue;
3991	}
3992	# endif
3993	if (PteDst.n.u1Write)
3994	{
3995	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3996	GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3997	cErrors++;
3998	}
3999	fIgnoreFlags \|= X86_PTE_RW;
4000	}
4001	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4002	{
4003	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4004	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4005	cErrors++;
4006	continue;
4007	}
4008
4009	/* flags */
4010	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4011	{
4012	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4013	{
4014	if (PteDst.n.u1Write)
4015	{
4016	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4017	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4018	cErrors++;
4019	continue;
4020	}
4021	fIgnoreFlags \|= X86_PTE_RW;
4022	}
4023	else
4024	{
4025	if ( PteDst.n.u1Present
4026	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
4027	&& !PGM_PAGE_IS_MMIO(pPhysPage)
4028	# endif
4029	)
4030	{
4031	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4032	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4033	cErrors++;
4034	continue;
4035	}
4036	fIgnoreFlags \|= X86_PTE_P;
4037	}
4038	}
4039	else
4040	{
4041	if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
4042	{
4043	if (PteDst.n.u1Write)
4044	{
4045	AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
4046	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4047	cErrors++;
4048	continue;
4049	}
4050	if (!(PteDst.u & PGM_PTFLAGS_TRACK_DIRTY))
4051	{
4052	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4053	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4054	cErrors++;
4055	continue;
4056	}
4057	if (PteDst.n.u1Dirty)
4058	{
4059	AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4060	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4061	cErrors++;
4062	}
4063	# if 0 /** @todo sync access bit properly... */
4064	if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
4065	{
4066	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4067	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4068	cErrors++;
4069	}
4070	fIgnoreFlags \|= X86_PTE_RW;
4071	# else
4072	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
4073	# endif
4074	}
4075	else if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
4076	{
4077	/* access bit emulation (not implemented). */
4078	if (PteSrc.n.u1Accessed \|\| PteDst.n.u1Present)
4079	{
4080	AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
4081	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4082	cErrors++;
4083	continue;
4084	}
4085	if (!PteDst.n.u1Accessed)
4086	{
4087	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
4088	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4089	cErrors++;
4090	}
4091	fIgnoreFlags \|= X86_PTE_P;
4092	}
4093	# ifdef DEBUG_sandervl
4094	fIgnoreFlags \|= X86_PTE_D \| X86_PTE_A;
4095	# endif
4096	}
4097
4098	if ( (PteSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
4099	&& (PteSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags)
4100	)
4101	{
4102	AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
4103	GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
4104	fIgnoreFlags, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4105	cErrors++;
4106	continue;
4107	}
4108	} /* foreach PTE */
4109	}
4110	else
4111	{
4112	/*
4113	* Big Page.
4114	*/
4115	uint64_t fIgnoreFlags = X86_PDE_AVL_MASK \| GST_PDE_PG_MASK \| X86_PDE4M_G \| X86_PDE4M_D \| X86_PDE4M_PS \| X86_PDE4M_PWT \| X86_PDE4M_PCD;
4116	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
4117	{
4118	if (PdeDst.n.u1Write)
4119	{
4120	AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4121	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4122	cErrors++;
4123	continue;
4124	}
4125	if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
4126	{
4127	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4128	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4129	cErrors++;
4130	continue;
4131	}
4132	# if 0 /** @todo sync access bit properly... */
4133	if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
4134	{
4135	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4136	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4137	cErrors++;
4138	}
4139	fIgnoreFlags \|= X86_PTE_RW;
4140	# else
4141	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
4142	# endif
4143	}
4144	else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4145	{
4146	/* access bit emulation (not implemented). */
4147	if (PdeSrc.b.u1Accessed \|\| PdeDst.n.u1Present)
4148	{
4149	AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4150	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4151	cErrors++;
4152	continue;
4153	}
4154	if (!PdeDst.n.u1Accessed)
4155	{
4156	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4157	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4158	cErrors++;
4159	}
4160	fIgnoreFlags \|= X86_PTE_P;
4161	}
4162
4163	if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
4164	{
4165	AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
4166	GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
4167	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4168	cErrors++;
4169	}
4170
4171	/* iterate the page table. */
4172	for (unsigned iPT = 0, off = 0;
4173	iPT < RT_ELEMENTS(pPTDst->a);
4174	iPT++, off += PAGE_SIZE, GCPhysGst += PAGE_SIZE)
4175	{
4176	const SHWPTE PteDst = pPTDst->a[iPT];
4177
4178	if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
4179	{
4180	AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
4181	GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4182	cErrors++;
4183	}
4184
4185	/* skip not-present entries. */
4186	if (!PteDst.n.u1Present) /** @todo deal with ALL handlers and CSAM !P pages! */
4187	continue;
4188
4189	fIgnoreFlags = X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT \| X86_PTE_D \| X86_PTE_A \| X86_PTE_G \| X86_PTE_PAE_NX;
4190
4191	/* match the physical addresses */
4192	HCPhysShw = PteDst.u & X86_PTE_PAE_PG_MASK;
4193
4194	# ifdef IN_RING3
4195	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4196	if (RT_FAILURE(rc))
4197	{
4198	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4199	{
4200	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4201	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4202	cErrors++;
4203	}
4204	}
4205	else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
4206	{
4207	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4208	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4209	cErrors++;
4210	continue;
4211	}
4212	# endif
4213	pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
4214	if (!pPhysPage)
4215	{
4216	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4217	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4218	{
4219	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4220	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4221	cErrors++;
4222	continue;
4223	}
4224	# endif
4225	if (PteDst.n.u1Write)
4226	{
4227	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4228	GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4229	cErrors++;
4230	}
4231	fIgnoreFlags \|= X86_PTE_RW;
4232	}
4233	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4234	{
4235	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4236	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4237	cErrors++;
4238	continue;
4239	}
4240
4241	/* flags */
4242	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4243	{
4244	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4245	{
4246	if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
4247	{
4248	if (PteDst.n.u1Write)
4249	{
4250	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4251	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4252	cErrors++;
4253	continue;
4254	}
4255	fIgnoreFlags \|= X86_PTE_RW;
4256	}
4257	}
4258	else
4259	{
4260	if ( PteDst.n.u1Present
4261	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
4262	&& !PGM_PAGE_IS_MMIO(pPhysPage)
4263	# endif
4264	)
4265	{
4266	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4267	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4268	cErrors++;
4269	continue;
4270	}
4271	fIgnoreFlags \|= X86_PTE_P;
4272	}
4273	}
4274
4275	if ( (PdeSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
4276	&& (PdeSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags) /* lazy phys handler dereg. */
4277	)
4278	{
4279	AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
4280	GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
4281	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4282	cErrors++;
4283	continue;
4284	}
4285	} /* for each PTE */
4286	}
4287	}
4288	/* not present */
4289
4290	} /* for each PDE */
4291
4292	} /* for each PDPTE */
4293
4294	} /* for each PML4E */
4295
4296	# ifdef DEBUG
4297	if (cErrors)
4298	LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
4299	# endif
4300
4301	#endif /* GST == 32BIT, PAE or AMD64 */
4302	return cErrors;
4303
4304	#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT */
4305	}
4306	#endif /* VBOX_STRICT */
4307
4308
4309	/**
4310	* Sets up the CR3 for shadow paging
4311	*
4312	* @returns Strict VBox status code.
4313	* @retval VINF_SUCCESS.
4314	*
4315	* @param pVCpu The VMCPU handle.
4316	* @param GCPhysCR3 The physical address in the CR3 register.
4317	*/
4318	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3)
4319	{
4320	PVM pVM = pVCpu->CTX_SUFF(pVM);
4321
4322	/* Update guest paging info. */
4323	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
4324	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
4325	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
4326
4327	LogFlow(("MapCR3: %RGp\n", GCPhysCR3));
4328
4329	/*
4330	* Map the page CR3 points at.
4331	*/
4332	RTHCPTR HCPtrGuestCR3;
4333	RTHCPHYS HCPhysGuestCR3;
4334	pgmLock(pVM);
4335	PPGMPAGE pPageCR3 = pgmPhysGetPage(&pVM->pgm.s, GCPhysCR3);
4336	AssertReturn(pPageCR3, VERR_INTERNAL_ERROR_2);
4337	HCPhysGuestCR3 = PGM_PAGE_GET_HCPHYS(pPageCR3);
4338	/** @todo this needs some reworking wrt. locking? */
4339	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4340	HCPtrGuestCR3 = NIL_RTHCPTR;
4341	int rc = VINF_SUCCESS;
4342	# else
4343	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPageCR3, GCPhysCR3 & GST_CR3_PAGE_MASK, (void )&HCPtrGuestCR3); / @todo r=bird: This GCPhysCR3 masking isn't necessary. */
4344	# endif
4345	pgmUnlock(pVM);
4346	if (RT_SUCCESS(rc))
4347	{
4348	rc = PGMMap(pVM, (RTGCPTR)pVM->pgm.s.GCPtrCR3Mapping, HCPhysGuestCR3, PAGE_SIZE, 0);
4349	if (RT_SUCCESS(rc))
4350	{
4351	# ifdef IN_RC
4352	PGM_INVL_PG(pVCpu, pVM->pgm.s.GCPtrCR3Mapping);
4353	# endif
4354	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4355	pVCpu->pgm.s.pGst32BitPdR3 = (R3PTRTYPE(PX86PD))HCPtrGuestCR3;
4356	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4357	pVCpu->pgm.s.pGst32BitPdR0 = (R0PTRTYPE(PX86PD))HCPtrGuestCR3;
4358	# endif
4359	pVCpu->pgm.s.pGst32BitPdRC = (RCPTRTYPE(PX86PD))(RTRCUINTPTR)pVM->pgm.s.GCPtrCR3Mapping;
4360
4361	# elif PGM_GST_TYPE == PGM_TYPE_PAE
4362	unsigned off = GCPhysCR3 & GST_CR3_PAGE_MASK & PAGE_OFFSET_MASK;
4363	pVCpu->pgm.s.pGstPaePdptR3 = (R3PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4364	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4365	pVCpu->pgm.s.pGstPaePdptR0 = (R0PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4366	# endif
4367	pVCpu->pgm.s.pGstPaePdptRC = (RCPTRTYPE(PX86PDPT))((RTRCUINTPTR)pVM->pgm.s.GCPtrCR3Mapping + off);
4368	LogFlow(("Cached mapping %RRv\n", pVCpu->pgm.s.pGstPaePdptRC));
4369
4370	/*
4371	* Map the 4 PDs too.
4372	*/
4373	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pVCpu);
4374	RTGCPTR GCPtr = pVM->pgm.s.GCPtrCR3Mapping + PAGE_SIZE;
4375	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++, GCPtr += PAGE_SIZE)
4376	{
4377	if (pGuestPDPT->a[i].n.u1Present)
4378	{
4379	RTHCPTR HCPtr;
4380	RTHCPHYS HCPhys;
4381	RTGCPHYS GCPhys = pGuestPDPT->a[i].u & X86_PDPE_PG_MASK;
4382	pgmLock(pVM);
4383	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
4384	AssertReturn(pPage, VERR_INTERNAL_ERROR_2);
4385	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
4386	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4387	HCPtr = NIL_RTHCPTR;
4388	int rc2 = VINF_SUCCESS;
4389	# else
4390	int rc2 = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, (void **)&HCPtr);
4391	# endif
4392	pgmUnlock(pVM);
4393	if (RT_SUCCESS(rc2))
4394	{
4395	rc = PGMMap(pVM, GCPtr, HCPhys, PAGE_SIZE, 0);
4396	AssertRCReturn(rc, rc);
4397
4398	pVCpu->pgm.s.apGstPaePDsR3[i] = (R3PTRTYPE(PX86PDPAE))HCPtr;
4399	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4400	pVCpu->pgm.s.apGstPaePDsR0[i] = (R0PTRTYPE(PX86PDPAE))HCPtr;
4401	# endif
4402	pVCpu->pgm.s.apGstPaePDsRC[i] = (RCPTRTYPE(PX86PDPAE))(RTRCUINTPTR)GCPtr;
4403	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
4404	# ifdef IN_RC
4405	PGM_INVL_PG(pVCpu, GCPtr);
4406	# endif
4407	continue;
4408	}
4409	AssertMsgFailed(("pgmR3Gst32BitMapCR3: rc2=%d GCPhys=%RGp i=%d\n", rc2, GCPhys, i));
4410	}
4411
4412	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4413	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4414	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4415	# endif
4416	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4417	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4418	# ifdef IN_RC
4419	PGM_INVL_PG(pVCpu, GCPtr); /** @todo this shouldn't be necessary? */
4420	# endif
4421	}
4422
4423	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
4424	pVCpu->pgm.s.pGstAmd64Pml4R3 = (R3PTRTYPE(PX86PML4))HCPtrGuestCR3;
4425	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4426	pVCpu->pgm.s.pGstAmd64Pml4R0 = (R0PTRTYPE(PX86PML4))HCPtrGuestCR3;
4427	# endif
4428	# endif
4429	}
4430	else
4431	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4432	}
4433	else
4434	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4435
4436	#else /* prot/real stub */
4437	int rc = VINF_SUCCESS;
4438	#endif
4439
4440	/* Update shadow paging info for guest modes with paging (32, pae, 64). */
4441	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4442	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4443	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64) \
4444	&& ( PGM_GST_TYPE != PGM_TYPE_REAL \
4445	&& PGM_GST_TYPE != PGM_TYPE_PROT))
4446
4447	Assert(!pVM->pgm.s.fNestedPaging);
4448
4449	/*
4450	* Update the shadow root page as well since that's not fixed.
4451	*/
4452	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4453	PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
4454	uint32_t iOldShwUserTable = pVCpu->pgm.s.iShwUserTable;
4455	uint32_t iOldShwUser = pVCpu->pgm.s.iShwUser;
4456	PPGMPOOLPAGE pNewShwPageCR3;
4457
4458	pgmLock(pVM);
4459
4460	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4461	if (pPool->cDirtyPages)
4462	pgmPoolResetDirtyPages(pVM);
4463	# endif
4464
4465	Assert(!(GCPhysCR3 >> (PAGE_SHIFT + 32)));
4466	rc = pgmPoolAlloc(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, SHW_POOL_ROOT_IDX, GCPhysCR3 >> PAGE_SHIFT, &pNewShwPageCR3, true /* lock page */);
4467	AssertFatalRC(rc);
4468	rc = VINF_SUCCESS;
4469
4470	# ifdef IN_RC
4471	/*
4472	* WARNING! We can't deal with jumps to ring 3 in the code below as the
4473	* state will be inconsistent! Flush important things now while
4474	* we still can and then make sure there are no ring-3 calls.
4475	*/
4476	REMNotifyHandlerPhysicalFlushIfAlmostFull(pVM, pVCpu);
4477	VMMRZCallRing3Disable(pVCpu);
4478	# endif
4479
4480	pVCpu->pgm.s.iShwUser = SHW_POOL_ROOT_IDX;
4481	pVCpu->pgm.s.iShwUserTable = GCPhysCR3 >> PAGE_SHIFT;
4482	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3) = pNewShwPageCR3;
4483	# ifdef IN_RING0
4484	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4485	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4486	# elif defined(IN_RC)
4487	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4488	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4489	# else
4490	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4491	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4492	# endif
4493
4494	# ifndef PGM_WITHOUT_MAPPINGS
4495	/*
4496	* Apply all hypervisor mappings to the new CR3.
4497	* Note that SyncCR3 will be executed in case CR3 is changed in a guest paging mode; this will
4498	* make sure we check for conflicts in the new CR3 root.
4499	*/
4500	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4501	Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL) \|\| VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
4502	# endif
4503	rc = pgmMapActivateCR3(pVM, pNewShwPageCR3);
4504	AssertRCReturn(rc, rc);
4505	# endif
4506
4507	/* Set the current hypervisor CR3. */
4508	CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
4509	SELMShadowCR3Changed(pVM, pVCpu);
4510
4511	# ifdef IN_RC
4512	/* NOTE: The state is consistent again. */
4513	VMMRZCallRing3Enable(pVCpu);
4514	# endif
4515
4516	/* Clean up the old CR3 root. */
4517	if ( pOldShwPageCR3
4518	&& pOldShwPageCR3 != pNewShwPageCR3 /* @todo can happen due to incorrect syncing between REM & PGM; find the real cause */)
4519	{
4520	Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
4521	# ifndef PGM_WITHOUT_MAPPINGS
4522	/* Remove the hypervisor mappings from the shadow page table. */
4523	pgmMapDeactivateCR3(pVM, pOldShwPageCR3);
4524	# endif
4525	/* Mark the page as unlocked; allow flushing again. */
4526	pgmPoolUnlockPage(pPool, pOldShwPageCR3);
4527
4528	pgmPoolFreeByPage(pPool, pOldShwPageCR3, iOldShwUser, iOldShwUserTable);
4529	}
4530	pgmUnlock(pVM);
4531	# endif
4532
4533	return rc;
4534	}
4535
4536	/**
4537	* Unmaps the shadow CR3.
4538	*
4539	* @returns VBox status, no specials.
4540	* @param pVCpu The VMCPU handle.
4541	*/
4542	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu)
4543	{
4544	LogFlow(("UnmapCR3\n"));
4545
4546	int rc = VINF_SUCCESS;
4547	PVM pVM = pVCpu->CTX_SUFF(pVM);
4548
4549	/*
4550	* Update guest paging info.
4551	*/
4552	#if PGM_GST_TYPE == PGM_TYPE_32BIT
4553	pVCpu->pgm.s.pGst32BitPdR3 = 0;
4554	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4555	pVCpu->pgm.s.pGst32BitPdR0 = 0;
4556	# endif
4557	pVCpu->pgm.s.pGst32BitPdRC = 0;
4558
4559	#elif PGM_GST_TYPE == PGM_TYPE_PAE
4560	pVCpu->pgm.s.pGstPaePdptR3 = 0;
4561	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4562	pVCpu->pgm.s.pGstPaePdptR0 = 0;
4563	# endif
4564	pVCpu->pgm.s.pGstPaePdptRC = 0;
4565	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4566	{
4567	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4568	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4569	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4570	# endif
4571	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4572	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4573	}
4574
4575	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
4576	pVCpu->pgm.s.pGstAmd64Pml4R3 = 0;
4577	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4578	pVCpu->pgm.s.pGstAmd64Pml4R0 = 0;
4579	# endif
4580
4581	#else /* prot/real mode stub */
4582	/* nothing to do */
4583	#endif
4584
4585	#if !defined(IN_RC) /* In RC we rely on MapCR3 to do the shadow part for us at a safe time */
4586	/*
4587	* Update shadow paging info.
4588	*/
4589	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4590	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4591	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64))
4592
4593	# if PGM_GST_TYPE != PGM_TYPE_REAL
4594	Assert(!pVM->pgm.s.fNestedPaging);
4595	# endif
4596
4597	pgmLock(pVM);
4598
4599	# ifndef PGM_WITHOUT_MAPPINGS
4600	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4601	/* Remove the hypervisor mappings from the shadow page table. */
4602	pgmMapDeactivateCR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4603	# endif
4604
4605	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4606	{
4607	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4608
4609	Assert(pVCpu->pgm.s.iShwUser != PGMPOOL_IDX_NESTED_ROOT);
4610
4611	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4612	if (pPool->cDirtyPages)
4613	pgmPoolResetDirtyPages(pVM);
4614	# endif
4615
4616	/* Mark the page as unlocked; allow flushing again. */
4617	pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4618
4619	pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), pVCpu->pgm.s.iShwUser, pVCpu->pgm.s.iShwUserTable);
4620	pVCpu->pgm.s.pShwPageCR3R3 = 0;
4621	pVCpu->pgm.s.pShwPageCR3R0 = 0;
4622	pVCpu->pgm.s.pShwPageCR3RC = 0;
4623	pVCpu->pgm.s.iShwUser = 0;
4624	pVCpu->pgm.s.iShwUserTable = 0;
4625	}
4626	pgmUnlock(pVM);
4627	# endif
4628	#endif /* !IN_RC*/
4629
4630	return rc;
4631	}

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 31768

Download in other formats: