PGMAllBth.h@ 19784

Last change on this file since 19784 was 19784, checked in by vboxsync, 16 years ago
Use PDM lock for protecting pdm queue management.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 194.2 KB

Line
1	/* $Id: PGMAllBth.h 19784 2009-05-18 13:15:46Z vboxsync $ */
2	/** @file
3	* VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4	*
5	* This file is a big challenge!
6	*/
7
8	/*
9	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
10	*
11	* This file is part of VirtualBox Open Source Edition (OSE), as
12	* available from http://www.virtualbox.org. This file is free software;
13	* you can redistribute it and/or modify it under the terms of the GNU
14	* General Public License (GPL) as published by the Free Software
15	* Foundation, in version 2 as it comes in the "COPYING" file of the
16	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
17	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
18	*
19	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
20	* Clara, CA 95054 USA or visit http://www.sun.com if you need
21	* additional information or have any questions.
22	*/
23
24	/*******************************************************************************
25	* Internal Functions *
26	*******************************************************************************/
27	__BEGIN_DECLS
28	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault);
29	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
30	PGM_BTH_DECL(int, SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr);
31	PGM_BTH_DECL(int, CheckPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCPTR GCPtrPage);
32	PGM_BTH_DECL(int, SyncPT)(PVMCPU pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage);
33	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uErr);
34	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
35	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
36	#ifdef VBOX_STRICT
37	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
38	#endif
39	#ifdef PGMPOOL_WITH_USER_TRACKING
40	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys);
41	#endif
42	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3);
43	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu);
44	__END_DECLS
45
46
47	/* Filter out some illegal combinations of guest and shadow paging, so we can remove redundant checks inside functions. */
48	#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
49	# error "Invalid combination; PAE guest implies PAE shadow"
50	#endif
51
52	#if (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
53	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT)
54	# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
55	#endif
56
57	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) \
58	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT)
59	# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
60	#endif
61
62	#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT) \
63	\|\| (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
64	# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
65	#endif
66
67	#ifdef IN_RING0 /* no mappings in VT-x and AMD-V mode */
68	# define PGM_WITHOUT_MAPPINGS
69	#endif
70
71
72	#ifndef IN_RING3
73	/**
74	* #PF Handler for raw-mode guest execution.
75	*
76	* @returns VBox status code (appropriate for trap handling and GC return).
77	*
78	* @param pVCpu VMCPU Handle.
79	* @param uErr The trap error code.
80	* @param pRegFrame Trap register frame.
81	* @param pvFault The fault address.
82	*/
83	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
84	{
85	PVM pVM = pVCpu->CTX_SUFF(pVM);
86
87	# if defined(IN_RC) && defined(VBOX_STRICT)
88	PGMDynCheckLocks(pVM);
89	# endif
90
91	# if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
92	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
93	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT)
94
95	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE != PGM_TYPE_PAE
96	/*
97	* Hide the instruction fetch trap indicator for now.
98	*/
99	/** @todo NXE will change this and we must fix NXE in the switcher too! */
100	if (uErr & X86_TRAP_PF_ID)
101	{
102	uErr &= ~X86_TRAP_PF_ID;
103	TRPMSetErrorCode(pVCpu, uErr);
104	}
105	# endif
106
107	/*
108	* Get PDs.
109	*/
110	int rc;
111	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
112	# if PGM_GST_TYPE == PGM_TYPE_32BIT
113	const unsigned iPDSrc = pvFault >> GST_PD_SHIFT;
114	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
115
116	# elif PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64
117
118	# if PGM_GST_TYPE == PGM_TYPE_PAE
119	unsigned iPDSrc;
120	X86PDPE PdpeSrc;
121	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, pvFault, &iPDSrc, &PdpeSrc);
122
123	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
124	unsigned iPDSrc;
125	PX86PML4E pPml4eSrc;
126	X86PDPE PdpeSrc;
127	PGSTPD pPDSrc;
128
129	pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, pvFault, &pPml4eSrc, &PdpeSrc, &iPDSrc);
130	Assert(pPml4eSrc);
131	# endif
132
133	/* Quick check for a valid guest trap. (PAE & AMD64) */
134	if (!pPDSrc)
135	{
136	# if PGM_GST_TYPE == PGM_TYPE_AMD64 && GC_ARCH_BITS == 64
137	LogFlow(("Trap0eHandler: guest PML4 %d not present CR3=%RGp\n", (int)((pvFault >> X86_PML4_SHIFT) & X86_PML4_MASK), CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK));
138	# else
139	LogFlow(("Trap0eHandler: guest iPDSrc=%u not present CR3=%RGp\n", iPDSrc, CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK));
140	# endif
141	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2GuestTrap; });
142	TRPMSetErrorCode(pVCpu, uErr);
143	return VINF_EM_RAW_GUEST_TRAP;
144	}
145	# endif
146
147	# else /* !PGM_WITH_PAGING */
148	PGSTPD pPDSrc = NULL;
149	const unsigned iPDSrc = 0;
150	# endif /* !PGM_WITH_PAGING */
151
152	/* Fetch the guest PDE */
153	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
154	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
155	# else
156	GSTPDE PdeSrc;
157	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
158	PdeSrc.n.u1Present = 1;
159	PdeSrc.n.u1Write = 1;
160	PdeSrc.n.u1Accessed = 1;
161	PdeSrc.n.u1User = 1;
162	# endif
163
164	pgmLock(pVM);
165	{ /* Force the shadow pointers to go out of scope after releasing the lock. */
166	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
167	const unsigned iPDDst = pvFault >> SHW_PD_SHIFT;
168	PX86PD pPDDst = pgmShwGet32BitPDPtr(&pVCpu->pgm.s);
169
170	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
171	const unsigned iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK; /* pPDDst index, not used with the pool. */
172
173	PX86PDPAE pPDDst;
174	# if PGM_GST_TYPE != PGM_TYPE_PAE
175	X86PDPE PdpeSrc;
176
177	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
178	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
179	# endif
180	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, &PdpeSrc, &pPDDst);
181	if (rc != VINF_SUCCESS)
182	{
183	pgmUnlock(pVM);
184	AssertRC(rc);
185	return rc;
186	}
187	Assert(pPDDst);
188
189	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
190	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
191	PX86PDPAE pPDDst;
192	# if PGM_GST_TYPE == PGM_TYPE_PROT
193	/* AMD-V nested paging */
194	X86PML4E Pml4eSrc;
195	X86PDPE PdpeSrc;
196	PX86PML4E pPml4eSrc = &Pml4eSrc;
197
198	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
199	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
200	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
201	# endif
202
203	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, pPml4eSrc, &PdpeSrc, &pPDDst);
204	if (rc != VINF_SUCCESS)
205	{
206	pgmUnlock(pVM);
207	AssertRC(rc);
208	return rc;
209	}
210	Assert(pPDDst);
211
212	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
213	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
214	PEPTPD pPDDst;
215
216	rc = pgmShwGetEPTPDPtr(pVCpu, pvFault, NULL, &pPDDst);
217	if (rc != VINF_SUCCESS)
218	{
219	pgmUnlock(pVM);
220	AssertRC(rc);
221	return rc;
222	}
223	Assert(pPDDst);
224	# endif
225
226	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
227	/*
228	* If we successfully correct the write protection fault due to dirty bit
229	* tracking, or this page fault is a genuine one, then return immediately.
230	*/
231	STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeCheckPageFault, e);
232	rc = PGM_BTH_NAME(CheckPageFault)(pVCpu, uErr, &pPDDst->a[iPDDst], &pPDSrc->a[iPDSrc], pvFault);
233	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeCheckPageFault, e);
234	if ( rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
235	\|\| rc == VINF_EM_RAW_GUEST_TRAP)
236	{
237	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution)
238	= rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? &pVCpu->pgm.s.StatRZTrap0eTime2DirtyAndAccessed : &pVCpu->pgm.s.StatRZTrap0eTime2GuestTrap; });
239	LogBird(("Trap0eHandler: returns %s\n", rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? "VINF_SUCCESS" : "VINF_EM_RAW_GUEST_TRAP"));
240	pgmUnlock(pVM);
241	return rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? VINF_SUCCESS : rc;
242	}
243
244	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0ePD[iPDSrc]);
245	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
246
247	/*
248	* A common case is the not-present error caused by lazy page table syncing.
249	*
250	* It is IMPORTANT that we weed out any access to non-present shadow PDEs here
251	* so we can safely assume that the shadow PT is present when calling SyncPage later.
252	*
253	* On failure, we ASSUME that SyncPT is out of memory or detected some kind
254	* of mapping conflict and defer to SyncCR3 in R3.
255	* (Again, we do NOT support access handlers for non-present guest pages.)
256	*
257	*/
258	if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
259	&& !pPDDst->a[iPDDst].n.u1Present
260	&& PdeSrc.n.u1Present
261	)
262	{
263	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2SyncPT; });
264	STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeSyncPT, f);
265	LogFlow(("=>SyncPT %04x = %08x\n", iPDSrc, PdeSrc.au32[0]));
266	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, pvFault);
267	pgmUnlock(pVM);
268	if (RT_SUCCESS(rc))
269	{
270	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeSyncPT, f);
271	return rc;
272	}
273	Log(("SyncPT: %d failed!! rc=%d\n", iPDSrc, rc));
274	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
275	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeSyncPT, f);
276	return VINF_PGM_SYNC_CR3;
277	}
278	pgmUnlock(pVM);
279	}
280
281	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
282	/*
283	* Check if this address is within any of our mappings.
284	*
285	* This is very fast and it's gonna save us a bit of effort below and prevent
286	* us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
287	* (BTW, it's impossible to have physical access handlers in a mapping.)
288	*/
289	if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
290	{
291	STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
292	PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
293	for ( ; pMapping; pMapping = pMapping->CTX_SUFF(pNext))
294	{
295	if (pvFault < pMapping->GCPtr)
296	break;
297	if (pvFault - pMapping->GCPtr < pMapping->cb)
298	{
299	/*
300	* The first thing we check is if we've got an undetected conflict.
301	*/
302	if (!pVM->pgm.s.fMappingsFixed)
303	{
304	unsigned iPT = pMapping->cb >> GST_PD_SHIFT;
305	while (iPT-- > 0)
306	if (pPDSrc->a[iPDSrc + iPT].n.u1Present)
307	{
308	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eConflicts);
309	Log(("Trap0e: Detected Conflict %RGv-%RGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
310	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
311	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
312	return VINF_PGM_SYNC_CR3;
313	}
314	}
315
316	/*
317	* Check if the fault address is in a virtual page access handler range.
318	*/
319	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->HyperVirtHandlers, pvFault);
320	if ( pCur
321	&& pvFault - pCur->Core.Key < pCur->cb
322	&& uErr & X86_TRAP_PF_RW)
323	{
324	# ifdef IN_RC
325	STAM_PROFILE_START(&pCur->Stat, h);
326	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
327	STAM_PROFILE_STOP(&pCur->Stat, h);
328	# else
329	AssertFailed();
330	rc = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
331	# endif
332	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersMapping);
333	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
334	return rc;
335	}
336
337	/*
338	* Pretend we're not here and let the guest handle the trap.
339	*/
340	TRPMSetErrorCode(pVCpu, uErr & ~X86_TRAP_PF_P);
341	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eGuestPFMapping);
342	LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
343	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
344	return VINF_EM_RAW_GUEST_TRAP;
345	}
346	}
347	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
348	} /* pgmAreMappingsEnabled(&pVM->pgm.s) */
349	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
350
351	/*
352	* Check if this fault address is flagged for special treatment,
353	* which means we'll have to figure out the physical address and
354	* check flags associated with it.
355	*
356	* ASSUME that we can limit any special access handling to pages
357	* in page tables which the guest believes to be present.
358	*/
359	if (PdeSrc.n.u1Present)
360	{
361	RTGCPHYS GCPhys = NIL_RTGCPHYS;
362
363	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
364	# if PGM_GST_TYPE == PGM_TYPE_AMD64
365	bool fBigPagesSupported = true;
366	# else
367	bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
368	# endif
369	if ( PdeSrc.b.u1Size
370	&& fBigPagesSupported)
371	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc)
372	\| ((RTGCPHYS)pvFault & (GST_BIG_PAGE_OFFSET_MASK ^ PAGE_OFFSET_MASK));
373	else
374	{
375	PGSTPT pPTSrc;
376	rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
377	if (RT_SUCCESS(rc))
378	{
379	unsigned iPTESrc = (pvFault >> GST_PT_SHIFT) & GST_PT_MASK;
380	if (pPTSrc->a[iPTESrc].n.u1Present)
381	GCPhys = pPTSrc->a[iPTESrc].u & GST_PTE_PG_MASK;
382	}
383	}
384	# else
385	/* No paging so the fault address is the physical address */
386	GCPhys = (RTGCPHYS)(pvFault & ~PAGE_OFFSET_MASK);
387	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
388
389	/*
390	* If we have a GC address we'll check if it has any flags set.
391	*/
392	if (GCPhys != NIL_RTGCPHYS)
393	{
394	STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
395
396	PPGMPAGE pPage;
397	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
398	if (RT_SUCCESS(rc)) /** just handle the failure immediate (it returns) and make things easier to read. */
399	{
400	if ( PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage)
401	\|\| PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage))
402	{
403	if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
404	{
405	/*
406	* Physical page access handler.
407	*/
408	const RTGCPHYS GCPhysFault = GCPhys \| (pvFault & PAGE_OFFSET_MASK);
409	PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhysFault);
410	if (pCur)
411	{
412	# ifdef PGM_SYNC_N_PAGES
413	/*
414	* If the region is write protected and we got a page not present fault, then sync
415	* the pages. If the fault was caused by a read, then restart the instruction.
416	* In case of write access continue to the GC write handler.
417	*
418	* ASSUMES that there is only one handler per page or that they have similar write properties.
419	*/
420	if ( pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
421	&& !(uErr & X86_TRAP_PF_P))
422	{
423	pgmLock(pVM);
424	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, PGM_SYNC_NR_PAGES, uErr);
425	pgmUnlock(pVM);
426	if ( RT_FAILURE(rc)
427	\|\| !(uErr & X86_TRAP_PF_RW)
428	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
429	{
430	AssertRC(rc);
431	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersOutOfSync);
432	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
433	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndPhys; });
434	return rc;
435	}
436	}
437	# endif
438
439	AssertMsg( pCur->enmType != PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
440	\|\| (pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE && (uErr & X86_TRAP_PF_RW)),
441	("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
442
443	# if defined(IN_RC) \|\| defined(IN_RING0)
444	if (pCur->CTX_SUFF(pfnHandler))
445	{
446	STAM_PROFILE_START(&pCur->Stat, h);
447	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, GCPhysFault, pCur->CTX_SUFF(pvUser));
448	STAM_PROFILE_STOP(&pCur->Stat, h);
449	}
450	else
451	# endif
452	rc = VINF_EM_RAW_EMULATE_INSTR;
453	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersPhysical);
454	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
455	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndPhys; });
456	return rc;
457	}
458	}
459	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
460	else
461	{
462	# ifdef PGM_SYNC_N_PAGES
463	/*
464	* If the region is write protected and we got a page not present fault, then sync
465	* the pages. If the fault was caused by a read, then restart the instruction.
466	* In case of write access continue to the GC write handler.
467	*/
468	if ( PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) < PGM_PAGE_HNDL_PHYS_STATE_ALL
469	&& !(uErr & X86_TRAP_PF_P))
470	{
471	pgmLock(pVM);
472	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, PGM_SYNC_NR_PAGES, uErr);
473	pgmUnlock(pVM);
474	if ( RT_FAILURE(rc)
475	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
476	\|\| !(uErr & X86_TRAP_PF_RW))
477	{
478	AssertRC(rc);
479	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersOutOfSync);
480	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
481	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndVirt; });
482	return rc;
483	}
484	}
485	# endif
486	/*
487	* Ok, it's an virtual page access handler.
488	*
489	* Since it's faster to search by address, we'll do that first
490	* and then retry by GCPhys if that fails.
491	*/
492	/** @todo r=bird: perhaps we should consider looking up by physical address directly now? */
493	/** @note r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be out of sync, because the
494	* page was changed without us noticing it (not-present -> present without invlpg or mov cr3, xxx)
495	*/
496	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
497	if (pCur)
498	{
499	AssertMsg(!(pvFault - pCur->Core.Key < pCur->cb)
500	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
501	\|\| !(uErr & X86_TRAP_PF_P)
502	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
503	("Unexpected trap for virtual handler: %RGv (phys=%RGp) pPage=%R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
504
505	if ( pvFault - pCur->Core.Key < pCur->cb
506	&& ( uErr & X86_TRAP_PF_RW
507	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
508	{
509	# ifdef IN_RC
510	STAM_PROFILE_START(&pCur->Stat, h);
511	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
512	STAM_PROFILE_STOP(&pCur->Stat, h);
513	# else
514	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
515	# endif
516	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersVirtual);
517	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
518	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndVirt; });
519	return rc;
520	}
521	/* Unhandled part of a monitored page */
522	}
523	else
524	{
525	/* Check by physical address. */
526	PPGMVIRTHANDLER pCur;
527	unsigned iPage;
528	rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys + (pvFault & PAGE_OFFSET_MASK),
529	&pCur, &iPage);
530	Assert(RT_SUCCESS(rc) \|\| !pCur);
531	if ( pCur
532	&& ( uErr & X86_TRAP_PF_RW
533	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
534	{
535	Assert((pCur->aPhysToVirt[iPage].Core.Key & X86_PTE_PAE_PG_MASK) == GCPhys);
536	# ifdef IN_RC
537	RTGCPTR off = (iPage << PAGE_SHIFT) + (pvFault & PAGE_OFFSET_MASK) - (pCur->Core.Key & PAGE_OFFSET_MASK);
538	Assert(off < pCur->cb);
539	STAM_PROFILE_START(&pCur->Stat, h);
540	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, off);
541	STAM_PROFILE_STOP(&pCur->Stat, h);
542	# else
543	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
544	# endif
545	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersVirtualByPhys);
546	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
547	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndVirt; });
548	return rc;
549	}
550	}
551	}
552	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
553
554	/*
555	* There is a handled area of the page, but this fault doesn't belong to it.
556	* We must emulate the instruction.
557	*
558	* To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
559	* we first check if this was a page-not-present fault for a page with only
560	* write access handlers. Restart the instruction if it wasn't a write access.
561	*/
562	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersUnhandled);
563
564	if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
565	&& !(uErr & X86_TRAP_PF_P))
566	{
567	pgmLock(pVM);
568	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, PGM_SYNC_NR_PAGES, uErr);
569	pgmUnlock(pVM);
570	if ( RT_FAILURE(rc)
571	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
572	\|\| !(uErr & X86_TRAP_PF_RW))
573	{
574	AssertRC(rc);
575	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersOutOfSync);
576	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
577	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndPhys; });
578	return rc;
579	}
580	}
581
582	/** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
583	* It's writing to an unhandled part of the LDT page several million times.
584	*/
585	rc = PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault);
586	LogFlow(("PGM: PGMInterpretInstruction -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
587	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
588	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndUnhandled; });
589	return rc;
590	} /* if any kind of handler */
591
592	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
593	if (uErr & X86_TRAP_PF_P)
594	{
595	/*
596	* The page isn't marked, but it might still be monitored by a virtual page access handler.
597	* (ASSUMES no temporary disabling of virtual handlers.)
598	*/
599	/** @todo r=bird: Since the purpose is to catch out of sync pages with virtual handler(s) here,
600	* we should correct both the shadow page table and physical memory flags, and not only check for
601	* accesses within the handler region but for access to pages with virtual handlers. */
602	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
603	if (pCur)
604	{
605	AssertMsg( !(pvFault - pCur->Core.Key < pCur->cb)
606	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
607	\|\| !(uErr & X86_TRAP_PF_P)
608	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
609	("Unexpected trap for virtual handler: %08X (phys=%08x) %R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
610
611	if ( pvFault - pCur->Core.Key < pCur->cb
612	&& ( uErr & X86_TRAP_PF_RW
613	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
614	{
615	# ifdef IN_RC
616	STAM_PROFILE_START(&pCur->Stat, h);
617	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
618	STAM_PROFILE_STOP(&pCur->Stat, h);
619	# else
620	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
621	# endif
622	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersVirtualUnmarked);
623	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
624	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndVirt; });
625	return rc;
626	}
627	}
628	}
629	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
630	}
631	else
632	{
633	/*
634	* When the guest accesses invalid physical memory (e.g. probing
635	* of RAM or accessing a remapped MMIO range), then we'll fall
636	* back to the recompiler to emulate the instruction.
637	*/
638	LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhys, rc));
639	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersInvalid);
640	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
641	return VINF_EM_RAW_EMULATE_INSTR;
642	}
643
644	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
645
646	# ifdef PGM_OUT_OF_SYNC_IN_GC /** @todo remove this bugger. */
647	/*
648	* We are here only if page is present in Guest page tables and
649	* trap is not handled by our handlers.
650	*
651	* Check it for page out-of-sync situation.
652	*/
653	STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
654
655	if (!(uErr & X86_TRAP_PF_P))
656	{
657	/*
658	* Page is not present in our page tables.
659	* Try to sync it!
660	* BTW, fPageShw is invalid in this branch!
661	*/
662	if (uErr & X86_TRAP_PF_US)
663	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncUser));
664	else /* supervisor */
665	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
666
667	# if defined(LOG_ENABLED) && !defined(IN_RING0)
668	RTGCPHYS GCPhys;
669	uint64_t fPageGst;
670	PGMGstGetPage(pVCpu, pvFault, &fPageGst, &GCPhys);
671	Log(("Page out of sync: %RGv eip=%08x PdeSrc.n.u1User=%d fPageGst=%08llx GCPhys=%RGp scan=%d\n",
672	pvFault, pRegFrame->eip, PdeSrc.n.u1User, fPageGst, GCPhys, CSAMDoesPageNeedScanning(pVM, (RTRCPTR)pRegFrame->eip)));
673	# endif /* LOG_ENABLED */
674
675	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
676	if (CPUMGetGuestCPL(pVCpu, pRegFrame) == 0)
677	{
678	uint64_t fPageGst;
679	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
680	if ( RT_SUCCESS(rc)
681	&& !(fPageGst & X86_PTE_US))
682	{
683	/* Note: can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU */
684	if ( pvFault == (RTGCPTR)pRegFrame->eip
685	\|\| pvFault - pRegFrame->eip < 8 /* instruction crossing a page boundary */
686	# ifdef CSAM_DETECT_NEW_CODE_PAGES
687	\|\| ( !PATMIsPatchGCAddr(pVM, (RTGCPTR)pRegFrame->eip)
688	&& CSAMDoesPageNeedScanning(pVM, (RTRCPTR)pRegFrame->eip)) /* any new code we encounter here */
689	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
690	)
691	{
692	LogFlow(("CSAMExecFault %RX32\n", pRegFrame->eip));
693	rc = CSAMExecFault(pVM, (RTRCPTR)pRegFrame->eip);
694	if (rc != VINF_SUCCESS)
695	{
696	/*
697	* CSAM needs to perform a job in ring 3.
698	*
699	* Sync the page before going to the host context; otherwise we'll end up in a loop if
700	* CSAM fails (e.g. instruction crosses a page boundary and the next page is not present)
701	*/
702	LogFlow(("CSAM ring 3 job\n"));
703	pgmLock(pVM);
704	int rc2 = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, 1, uErr);
705	pgmUnlock(pVM);
706	AssertRC(rc2);
707
708	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
709	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2CSAM; });
710	return rc;
711	}
712	}
713	# ifdef CSAM_DETECT_NEW_CODE_PAGES
714	else if ( uErr == X86_TRAP_PF_RW
715	&& pRegFrame->ecx >= 0x100 /* early check for movswd count */
716	&& pRegFrame->ecx < 0x10000)
717	{
718	/* In case of a write to a non-present supervisor shadow page, we'll take special precautions
719	* to detect loading of new code pages.
720	*/
721
722	/*
723	* Decode the instruction.
724	*/
725	RTGCPTR PC;
726	rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &PC);
727	if (rc == VINF_SUCCESS)
728	{
729	DISCPUSTATE Cpu;
730	uint32_t cbOp;
731	rc = EMInterpretDisasOneEx(pVM, pVCpu, PC, pRegFrame, &Cpu, &cbOp);
732
733	/* For now we'll restrict this to rep movsw/d instructions */
734	if ( rc == VINF_SUCCESS
735	&& Cpu.pCurInstr->opcode == OP_MOVSWD
736	&& (Cpu.prefix & PREFIX_REP))
737	{
738	CSAMMarkPossibleCodePage(pVM, pvFault);
739	}
740	}
741	}
742	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
743
744	/*
745	* Mark this page as safe.
746	*/
747	/** @todo not correct for pages that contain both code and data!! */
748	Log2(("CSAMMarkPage %RGv; scanned=%d\n", pvFault, true));
749	CSAMMarkPage(pVM, (RTRCPTR)pvFault, true);
750	}
751	}
752	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0) */
753	pgmLock(pVM);
754	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, PGM_SYNC_NR_PAGES, uErr);
755	pgmUnlock(pVM);
756	if (RT_SUCCESS(rc))
757	{
758	/* The page was successfully synced, return to the guest. */
759	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
760	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSync; });
761	return VINF_SUCCESS;
762	}
763	}
764	else /* uErr & X86_TRAP_PF_P: */
765	{
766	/*
767	* Write protected pages are make writable when the guest makes the first
768	* write to it. This happens for pages that are shared, write monitored
769	* and not yet allocated.
770	*
771	* Also, a side effect of not flushing global PDEs are out of sync pages due
772	* to physical monitored regions, that are no longer valid.
773	* Assume for now it only applies to the read/write flag.
774	*/
775	if (RT_SUCCESS(rc) && (uErr & X86_TRAP_PF_RW))
776	{
777	if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
778	{
779	Log(("PGM #PF: Make writable: %RGp %R[pgmpage] pvFault=%RGp uErr=%#x\n",
780	GCPhys, pPage, pvFault, uErr));
781	rc = pgmPhysPageMakeWritableUnlocked(pVM, pPage, GCPhys);
782	if (rc != VINF_SUCCESS)
783	{
784	AssertMsg(rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("%Rrc\n", rc));
785	return rc;
786	}
787	if (RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)))
788	return VINF_EM_NO_MEMORY;
789	}
790	/// @todo count the above case; else
791	if (uErr & X86_TRAP_PF_US)
792	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncUser));
793	else /* supervisor */
794	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
795
796	/*
797	* Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
798	* page is not present, which is not true in this case.
799	*/
800	pgmLock(pVM);
801	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, 1, uErr);
802	pgmUnlock(pVM);
803	if (RT_SUCCESS(rc))
804	{
805	/*
806	* Page was successfully synced, return to guest.
807	*/
808	# ifdef VBOX_STRICT
809	RTGCPHYS GCPhys;
810	uint64_t fPageGst;
811	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, &GCPhys);
812	Assert(RT_SUCCESS(rc) && fPageGst & X86_PTE_RW);
813	LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GCPhys, (uint64_t)fPageGst));
814
815	uint64_t fPageShw;
816	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
817	AssertMsg(RT_SUCCESS(rc) && fPageShw & X86_PTE_RW, ("rc=%Rrc fPageShw=%RX64\n", rc, fPageShw));
818	# endif /* VBOX_STRICT */
819	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
820	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndObs; });
821	return VINF_SUCCESS;
822	}
823
824	/* Check to see if we need to emulate the instruction as X86_CR0_WP has been cleared. */
825	if ( CPUMGetGuestCPL(pVCpu, pRegFrame) == 0
826	&& ((CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG)
827	&& (uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RW \| X86_TRAP_PF_P))
828	{
829	uint64_t fPageGst;
830	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
831	if ( RT_SUCCESS(rc)
832	&& !(fPageGst & X86_PTE_RW))
833	{
834	rc = PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault);
835	if (RT_SUCCESS(rc))
836	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eWPEmulInRZ);
837	else
838	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eWPEmulToR3);
839	return rc;
840	}
841	AssertMsgFailed(("Unexpected r/w page %RGv flag=%x rc=%Rrc\n", pvFault, (uint32_t)fPageGst, rc));
842	}
843	}
844
845	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
846	# ifdef VBOX_STRICT
847	/*
848	* Check for VMM page flags vs. Guest page flags consistency.
849	* Currently only for debug purposes.
850	*/
851	if (RT_SUCCESS(rc))
852	{
853	/* Get guest page flags. */
854	uint64_t fPageGst;
855	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
856	if (RT_SUCCESS(rc))
857	{
858	uint64_t fPageShw;
859	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
860
861	/*
862	* Compare page flags.
863	* Note: we have AVL, A, D bits desynched.
864	*/
865	AssertMsg((fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)),
866	("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64\n", pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst));
867	}
868	else
869	AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc));
870	}
871	else
872	AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc));
873	# endif /* VBOX_STRICT */
874	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
875	}
876	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
877	# endif /* PGM_OUT_OF_SYNC_IN_GC */
878	}
879	else /* GCPhys == NIL_RTGCPHYS */
880	{
881	/*
882	* Page not present in Guest OS or invalid page table address.
883	* This is potential virtual page access handler food.
884	*
885	* For the present we'll say that our access handlers don't
886	* work for this case - we've already discarded the page table
887	* not present case which is identical to this.
888	*
889	* When we perchance find we need this, we will probably have AVL
890	* trees (offset based) to operate on and we can measure their speed
891	* agains mapping a page table and probably rearrange this handling
892	* a bit. (Like, searching virtual ranges before checking the
893	* physical address.)
894	*/
895	}
896	}
897	/* else: !present (guest) */
898
899
900	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
901	/*
902	* Conclusion, this is a guest trap.
903	*/
904	LogFlow(("PGM: Unhandled #PF -> route trap to recompiler!\n"));
905	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eGuestPFUnh);
906	return VINF_EM_RAW_GUEST_TRAP;
907	# else
908	/* present, but not a monitored page; perhaps the guest is probing physical memory */
909	return VINF_EM_RAW_EMULATE_INSTR;
910	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
911
912
913	# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
914
915	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
916	return VERR_INTERNAL_ERROR;
917	# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
918	}
919	#endif /* !IN_RING3 */
920
921
922	/**
923	* Emulation of the invlpg instruction.
924	*
925	*
926	* @returns VBox status code.
927	*
928	* @param pVCpu The VMCPU handle.
929	* @param GCPtrPage Page to invalidate.
930	*
931	* @remark ASSUMES that the guest is updating before invalidating. This order
932	* isn't required by the CPU, so this is speculative and could cause
933	* trouble.
934	*
935	* @todo Flush page or page directory only if necessary!
936	* @todo Add a #define for simply invalidating the page.
937	*/
938	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
939	{
940	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
941	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
942	&& PGM_SHW_TYPE != PGM_TYPE_EPT
943	int rc;
944	PVM pVM = pVCpu->CTX_SUFF(pVM);
945	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
946
947	LogFlow(("InvalidatePage %RGv\n", GCPtrPage));
948	/*
949	* Get the shadow PD entry and skip out if this PD isn't present.
950	* (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
951	*/
952	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
953	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
954	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
955
956	/* Fetch the pgm pool shadow descriptor. */
957	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
958	Assert(pShwPde);
959
960	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
961	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT);
962	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(&pVCpu->pgm.s);
963
964	/* If the shadow PDPE isn't present, then skip the invalidate. */
965	if (!pPdptDst->a[iPdpt].n.u1Present)
966	{
967	Assert(!(pPdptDst->a[iPdpt].u & PGM_PLXFLAGS_MAPPING));
968	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
969	return VINF_SUCCESS;
970	}
971
972	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
973	PPGMPOOLPAGE pShwPde = NULL;
974	PX86PDPAE pPDDst;
975
976	/* Fetch the pgm pool shadow descriptor. */
977	rc = pgmShwGetPaePoolPagePD(&pVCpu->pgm.s, GCPtrPage, &pShwPde);
978	AssertRCSuccessReturn(rc, rc);
979	Assert(pShwPde);
980
981	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGM(&pVM->pgm.s, pShwPde);
982	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
983
984	# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
985	/* PML4 */
986	const unsigned iPml4 = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;
987	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
988	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
989	PX86PDPAE pPDDst;
990	PX86PDPT pPdptDst;
991	PX86PML4E pPml4eDst;
992	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eDst, &pPdptDst, &pPDDst);
993	if (rc != VINF_SUCCESS)
994	{
995	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT \|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
996	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
997	if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
998	PGM_INVL_GUEST_TLBS();
999	return VINF_SUCCESS;
1000	}
1001	Assert(pPDDst);
1002
1003	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1004	PX86PDPE pPdpeDst = &pPdptDst->a[iPdpt];
1005
1006	if (!pPdpeDst->n.u1Present)
1007	{
1008	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
1009	if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
1010	PGM_INVL_GUEST_TLBS();
1011	return VINF_SUCCESS;
1012	}
1013
1014	# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1015
1016	const SHWPDE PdeDst = *pPdeDst;
1017	if (!PdeDst.n.u1Present)
1018	{
1019	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
1020	return VINF_SUCCESS;
1021	}
1022
1023	# if defined(IN_RC)
1024	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1025	PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
1026	# endif
1027
1028	/*
1029	* Get the guest PD entry and calc big page.
1030	*/
1031	# if PGM_GST_TYPE == PGM_TYPE_32BIT
1032	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
1033	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
1034	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1035	# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1036	unsigned iPDSrc = 0;
1037	# if PGM_GST_TYPE == PGM_TYPE_PAE
1038	X86PDPE PdpeSrc;
1039	PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, GCPtrPage, &iPDSrc, &PdpeSrc);
1040	# else /* AMD64 */
1041	PX86PML4E pPml4eSrc;
1042	X86PDPE PdpeSrc;
1043	PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
1044	# endif
1045	GSTPDE PdeSrc;
1046
1047	if (pPDSrc)
1048	PdeSrc = pPDSrc->a[iPDSrc];
1049	else
1050	PdeSrc.u = 0;
1051	# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1052
1053	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1054	const bool fIsBigPage = PdeSrc.b.u1Size;
1055	# else
1056	const bool fIsBigPage = PdeSrc.b.u1Size && (CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
1057	# endif
1058
1059	# ifdef IN_RING3
1060	/*
1061	* If a CR3 Sync is pending we may ignore the invalidate page operation
1062	* depending on the kind of sync and if it's a global page or not.
1063	* This doesn't make sense in GC/R0 so we'll skip it entirely there.
1064	*/
1065	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
1066	if ( VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
1067	\|\| ( VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
1068	&& fIsBigPage
1069	&& PdeSrc.b.u1Global
1070	)
1071	)
1072	# else
1073	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
1074	# endif
1075	{
1076	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
1077	return VINF_SUCCESS;
1078	}
1079	# endif /* IN_RING3 */
1080
1081	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1082	/* Fetch the pgm pool shadow descriptor. */
1083	PPGMPOOLPAGE pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
1084	Assert(pShwPdpt);
1085
1086	/* Fetch the pgm pool shadow descriptor. */
1087	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & SHW_PDPE_PG_MASK);
1088	Assert(pShwPde);
1089
1090	Assert(pPml4eDst->n.u1Present && (pPml4eDst->u & SHW_PDPT_MASK));
1091	RTGCPHYS GCPhysPdpt = pPml4eSrc->u & X86_PML4E_PG_MASK;
1092
1093	if ( !pPml4eSrc->n.u1Present
1094	\|\| pShwPdpt->GCPhys != GCPhysPdpt)
1095	{
1096	LogFlow(("InvalidatePage: Out-of-sync PML4E (P/GCPhys) at %RGv GCPhys=%RGp vs %RGp Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
1097	GCPtrPage, pShwPdpt->GCPhys, GCPhysPdpt, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
1098	pgmPoolFreeByPage(pPool, pShwPdpt, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4);
1099	ASMAtomicWriteSize(pPml4eDst, 0);
1100	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNPs));
1101	PGM_INVL_GUEST_TLBS();
1102	return VINF_SUCCESS;
1103	}
1104	if ( pPml4eSrc->n.u1User != pPml4eDst->n.u1User
1105	\|\| (!pPml4eSrc->n.u1Write && pPml4eDst->n.u1Write))
1106	{
1107	/*
1108	* Mark not present so we can resync the PML4E when it's used.
1109	*/
1110	LogFlow(("InvalidatePage: Out-of-sync PML4E at %RGv Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
1111	GCPtrPage, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
1112	pgmPoolFreeByPage(pPool, pShwPdpt, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4);
1113	ASMAtomicWriteSize(pPml4eDst, 0);
1114	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1115	PGM_INVL_GUEST_TLBS();
1116	}
1117	else if (!pPml4eSrc->n.u1Accessed)
1118	{
1119	/*
1120	* Mark not present so we can set the accessed bit.
1121	*/
1122	LogFlow(("InvalidatePage: Out-of-sync PML4E (A) at %RGv Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
1123	GCPtrPage, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
1124	pgmPoolFreeByPage(pPool, pShwPdpt, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4);
1125	ASMAtomicWriteSize(pPml4eDst, 0);
1126	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNAs));
1127	PGM_INVL_GUEST_TLBS();
1128	}
1129
1130	/* Check if the PDPT entry has changed. */
1131	Assert(pPdpeDst->n.u1Present && pPdpeDst->u & SHW_PDPT_MASK);
1132	RTGCPHYS GCPhysPd = PdpeSrc.u & GST_PDPE_PG_MASK;
1133	if ( !PdpeSrc.n.u1Present
1134	\|\| pShwPde->GCPhys != GCPhysPd)
1135	{
1136	LogFlow(("InvalidatePage: Out-of-sync PDPE (P/GCPhys) at %RGv GCPhys=%RGp vs %RGp PdpeSrc=%RX64 PdpeDst=%RX64\n",
1137	GCPtrPage, pShwPde->GCPhys, GCPhysPd, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
1138	pgmPoolFreeByPage(pPool, pShwPde, pShwPdpt->idx, iPdpt);
1139	ASMAtomicWriteSize(pPdpeDst, 0);
1140	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNPs));
1141	PGM_INVL_GUEST_TLBS();
1142	return VINF_SUCCESS;
1143	}
1144	if ( PdpeSrc.lm.u1User != pPdpeDst->lm.u1User
1145	\|\| (!PdpeSrc.lm.u1Write && pPdpeDst->lm.u1Write))
1146	{
1147	/*
1148	* Mark not present so we can resync the PDPTE when it's used.
1149	*/
1150	LogFlow(("InvalidatePage: Out-of-sync PDPE at %RGv PdpeSrc=%RX64 PdpeDst=%RX64\n",
1151	GCPtrPage, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
1152	pgmPoolFreeByPage(pPool, pShwPde, pShwPdpt->idx, iPdpt);
1153	ASMAtomicWriteSize(pPdpeDst, 0);
1154	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1155	PGM_INVL_GUEST_TLBS();
1156	}
1157	else if (!PdpeSrc.lm.u1Accessed)
1158	{
1159	/*
1160	* Mark not present so we can set the accessed bit.
1161	*/
1162	LogFlow(("InvalidatePage: Out-of-sync PDPE (A) at %RGv PdpeSrc=%RX64 PdpeDst=%RX64\n",
1163	GCPtrPage, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
1164	pgmPoolFreeByPage(pPool, pShwPde, pShwPdpt->idx, iPdpt);
1165	ASMAtomicWriteSize(pPdpeDst, 0);
1166	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNAs));
1167	PGM_INVL_GUEST_TLBS();
1168	}
1169	# endif /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
1170
1171	/*
1172	* Deal with the Guest PDE.
1173	*/
1174	rc = VINF_SUCCESS;
1175	if (PdeSrc.n.u1Present)
1176	{
1177	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
1178	{
1179	/*
1180	* Conflict - Let SyncPT deal with it to avoid duplicate code.
1181	*/
1182	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1183	Assert(PGMGetGuestMode(pVCpu) <= PGMMODE_PAE);
1184	pgmLock(pVM);
1185	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
1186	pgmUnlock(pVM);
1187	}
1188	else if ( PdeSrc.n.u1User != PdeDst.n.u1User
1189	\|\| (!PdeSrc.n.u1Write && PdeDst.n.u1Write))
1190	{
1191	/*
1192	* Mark not present so we can resync the PDE when it's used.
1193	*/
1194	LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1195	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1196	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1197	ASMAtomicWriteSize(pPdeDst, 0);
1198	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1199	PGM_INVL_GUEST_TLBS();
1200	}
1201	else if (!PdeSrc.n.u1Accessed)
1202	{
1203	/*
1204	* Mark not present so we can set the accessed bit.
1205	*/
1206	LogFlow(("InvalidatePage: Out-of-sync (A) at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1207	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1208	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1209	ASMAtomicWriteSize(pPdeDst, 0);
1210	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNAs));
1211	PGM_INVL_GUEST_TLBS();
1212	}
1213	else if (!fIsBigPage)
1214	{
1215	/*
1216	* 4KB - page.
1217	*/
1218	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1219	RTGCPHYS GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1220	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1221	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1222	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE/2);
1223	# endif
1224	if (pShwPage->GCPhys == GCPhys)
1225	{
1226	# if 0 /* likely cause of a major performance regression; must be SyncPageWorkerTrackDeref then */
1227	const unsigned iPTEDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1228	PSHWPT pPT = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1229	if (pPT->a[iPTEDst].n.u1Present)
1230	{
1231	# ifdef PGMPOOL_WITH_USER_TRACKING
1232	/* This is very unlikely with caching/monitoring enabled. */
1233	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pShwPage, pPT->a[iPTEDst].u & SHW_PTE_PG_MASK);
1234	# endif
1235	ASMAtomicWriteSize(&pPT->a[iPTEDst], 0);
1236	}
1237	# else /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1238	pgmLock(pVM);
1239	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
1240	pgmUnlock(pVM);
1241	if (RT_SUCCESS(rc))
1242	rc = VINF_SUCCESS;
1243	# endif
1244	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePage4KBPages));
1245	PGM_INVL_PG(GCPtrPage);
1246	}
1247	else
1248	{
1249	/*
1250	* The page table address changed.
1251	*/
1252	LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
1253	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1254	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1255	ASMAtomicWriteSize(pPdeDst, 0);
1256	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1257	PGM_INVL_GUEST_TLBS();
1258	}
1259	}
1260	else
1261	{
1262	/*
1263	* 2/4MB - page.
1264	*/
1265	/* Before freeing the page, check if anything really changed. */
1266	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1267	RTGCPHYS GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
1268	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1269	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1270	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1271	# endif
1272	if ( pShwPage->GCPhys == GCPhys
1273	&& pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1274	{
1275	/* ASSUMES a the given bits are identical for 4M and normal PDEs */
1276	/** @todo PAT */
1277	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
1278	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
1279	&& ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1280	\|\| (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1281	{
1282	LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1283	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip));
1284	# if defined(IN_RC)
1285	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1286	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
1287	# endif
1288	return VINF_SUCCESS;
1289	}
1290	}
1291
1292	/*
1293	* Ok, the page table is present and it's been changed in the guest.
1294	* If we're in host context, we'll just mark it as not present taking the lazy approach.
1295	* We could do this for some flushes in GC too, but we need an algorithm for
1296	* deciding which 4MB pages containing code likely to be executed very soon.
1297	*/
1298	LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1299	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1300	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1301	ASMAtomicWriteSize(pPdeDst, 0);
1302	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePage4MBPages));
1303	PGM_INVL_BIG_PG(GCPtrPage);
1304	}
1305	}
1306	else
1307	{
1308	/*
1309	* Page directory is not present, mark shadow PDE not present.
1310	*/
1311	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
1312	{
1313	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1314	ASMAtomicWriteSize(pPdeDst, 0);
1315	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNPs));
1316	PGM_INVL_PG(GCPtrPage);
1317	}
1318	else
1319	{
1320	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1321	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDMappings));
1322	}
1323	}
1324	# if defined(IN_RC)
1325	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1326	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
1327	# endif
1328	return rc;
1329
1330	#else /* guest real and protected mode */
1331	/* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1332	return VINF_SUCCESS;
1333	#endif
1334	}
1335
1336
1337	#ifdef PGMPOOL_WITH_USER_TRACKING
1338	/**
1339	* Update the tracking of shadowed pages.
1340	*
1341	* @param pVCpu The VMCPU handle.
1342	* @param pShwPage The shadow page.
1343	* @param HCPhys The physical page we is being dereferenced.
1344	*/
1345	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys)
1346	{
1347	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1348	PVM pVM = pVCpu->CTX_SUFF(pVM);
1349
1350	STAM_PROFILE_START(&pVM->pgm.s.StatTrackDeref, a);
1351	LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1352
1353	/** @todo If this turns out to be a bottle neck (very likely) two things can be done:
1354	* 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1355	* 2. write protect all shadowed pages. I.e. implement caching.
1356	*/
1357	/*
1358	* Find the guest address.
1359	*/
1360	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
1361	pRam;
1362	pRam = pRam->CTX_SUFF(pNext))
1363	{
1364	unsigned iPage = pRam->cb >> PAGE_SHIFT;
1365	while (iPage-- > 0)
1366	{
1367	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1368	{
1369	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1370	pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage]);
1371	pShwPage->cPresent--;
1372	pPool->cPresent--;
1373	STAM_PROFILE_STOP(&pVM->pgm.s.StatTrackDeref, a);
1374	return;
1375	}
1376	}
1377	}
1378
1379	for (;;)
1380	AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys));
1381	# else /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1382	pShwPage->cPresent--;
1383	pVM->pgm.s.CTX_SUFF(pPool)->cPresent--;
1384	# endif /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1385	}
1386
1387
1388	/**
1389	* Update the tracking of shadowed pages.
1390	*
1391	* @param pVCpu The VMCPU handle.
1392	* @param pShwPage The shadow page.
1393	* @param u16 The top 16-bit of the pPage->HCPhys.
1394	* @param pPage Pointer to the guest page. this will be modified.
1395	* @param iPTDst The index into the shadow table.
1396	*/
1397	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
1398	{
1399	PVM pVM = pVCpu->CTX_SUFF(pVM);
1400	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1401	/*
1402	* Just deal with the simple first time here.
1403	*/
1404	if (!u16)
1405	{
1406	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackVirgin);
1407	u16 = PGMPOOL_TD_MAKE(1, pShwPage->idx);
1408	}
1409	else
1410	u16 = pgmPoolTrackPhysExtAddref(pVM, u16, pShwPage->idx);
1411
1412	/* write back */
1413	Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x iPTDst=%#x\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst));
1414	PGM_PAGE_SET_TRACKING(pPage, u16);
1415
1416	# endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
1417
1418	/* update statistics. */
1419	pVM->pgm.s.CTX_SUFF(pPool)->cPresent++;
1420	pShwPage->cPresent++;
1421	if (pShwPage->iFirstPresent > iPTDst)
1422	pShwPage->iFirstPresent = iPTDst;
1423	}
1424	#endif /* PGMPOOL_WITH_USER_TRACKING */
1425
1426
1427	/**
1428	* Creates a 4K shadow page for a guest page.
1429	*
1430	* For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1431	* physical address. The PdeSrc argument only the flags are used. No page structured
1432	* will be mapped in this function.
1433	*
1434	* @param pVCpu The VMCPU handle.
1435	* @param pPteDst Destination page table entry.
1436	* @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1437	* Can safely assume that only the flags are being used.
1438	* @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1439	* @param pShwPage Pointer to the shadow page.
1440	* @param iPTDst The index into the shadow table.
1441	*
1442	* @remark Not used for 2/4MB pages!
1443	*/
1444	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1445	{
1446	if (PteSrc.n.u1Present)
1447	{
1448	PVM pVM = pVCpu->CTX_SUFF(pVM);
1449
1450	/*
1451	* Find the ram range.
1452	*/
1453	PPGMPAGE pPage;
1454	int rc = pgmPhysGetPageEx(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK, &pPage);
1455	if (RT_SUCCESS(rc))
1456	{
1457	#ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1458	/* Try make the page writable if necessary. */
1459	if ( PteSrc.n.u1Write
1460	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1461	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
1462	{
1463	rc = pgmPhysPageMakeWritableUnlocked(pVM, pPage, PteSrc.u & GST_PTE_PG_MASK);
1464	AssertRC(rc);
1465	}
1466	#endif
1467
1468	/** @todo investiage PWT, PCD and PAT. */
1469	/*
1470	* Make page table entry.
1471	*/
1472	SHWPTE PteDst;
1473	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1474	{
1475	/** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No. */
1476	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1477	{
1478	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1479	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage);
1480	PteDst.n.u1Present = 1;
1481	PteDst.n.u1Execute = 1;
1482	PteDst.n.u1IgnorePAT = 1;
1483	PteDst.n.u3EMT = VMX_EPT_MEMTYPE_WB;
1484	/* PteDst.n.u1Write = 0 && PteDst.n.u1Size = 0 */
1485	#else
1486	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1487	\| PGM_PAGE_GET_HCPHYS(pPage);
1488	#endif
1489	}
1490	else
1491	{
1492	LogFlow(("SyncPageWorker: monitored page (%RHp) -> mark not present\n", PGM_PAGE_GET_HCPHYS(pPage)));
1493	PteDst.u = 0;
1494	}
1495	/** @todo count these two kinds. */
1496	}
1497	else
1498	{
1499	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1500	/*
1501	* If the page or page directory entry is not marked accessed,
1502	* we mark the page not present.
1503	*/
1504	if (!PteSrc.n.u1Accessed \|\| !PdeSrc.n.u1Accessed)
1505	{
1506	LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1507	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,AccessedPage));
1508	PteDst.u = 0;
1509	}
1510	else
1511	/*
1512	* If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1513	* when the page is modified.
1514	*/
1515	if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1516	{
1517	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPage));
1518	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1519	\| PGM_PAGE_GET_HCPHYS(pPage)
1520	\| PGM_PTFLAGS_TRACK_DIRTY;
1521	}
1522	else
1523	#endif
1524	{
1525	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageSkipped));
1526	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1527	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage);
1528	PteDst.n.u1Present = 1;
1529	PteDst.n.u1Write = 1;
1530	PteDst.n.u1Execute = 1;
1531	PteDst.n.u1IgnorePAT = 1;
1532	PteDst.n.u3EMT = VMX_EPT_MEMTYPE_WB;
1533	/* PteDst.n.u1Size = 0 */
1534	#else
1535	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1536	\| PGM_PAGE_GET_HCPHYS(pPage);
1537	#endif
1538	}
1539	}
1540
1541	/*
1542	* Make sure only allocated pages are mapped writable.
1543	*/
1544	if ( PteDst.n.u1Write
1545	&& PteDst.n.u1Present
1546	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1547	{
1548	PteDst.n.u1Write = 0; /** @todo this isn't quite working yet. */
1549	Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", (RTGCPHYS)(PteSrc.u & X86_PTE_PAE_PG_MASK), pPage, iPTDst));
1550	}
1551
1552	#ifdef PGMPOOL_WITH_USER_TRACKING
1553	/*
1554	* Keep user track up to date.
1555	*/
1556	if (PteDst.n.u1Present)
1557	{
1558	if (!pPteDst->n.u1Present)
1559	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1560	else if ((pPteDst->u & SHW_PTE_PG_MASK) != (PteDst.u & SHW_PTE_PG_MASK))
1561	{
1562	Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", (uint64_t)pPteDst->u, (uint64_t)PteDst.u));
1563	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1564	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1565	}
1566	}
1567	else if (pPteDst->n.u1Present)
1568	{
1569	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1570	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1571	}
1572	#endif /* PGMPOOL_WITH_USER_TRACKING */
1573
1574	/*
1575	* Update statistics and commit the entry.
1576	*/
1577	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1578	if (!PteSrc.n.u1Global)
1579	pShwPage->fSeenNonGlobal = true;
1580	#endif
1581	ASMAtomicWriteSize(pPteDst, PteDst.u);
1582	}
1583	/* else MMIO or invalid page, we must handle them manually in the #PF handler. */
1584	/** @todo count these. */
1585	}
1586	else
1587	{
1588	/*
1589	* Page not-present.
1590	*/
1591	LogFlow(("SyncPageWorker: page not present in Pte\n"));
1592	#ifdef PGMPOOL_WITH_USER_TRACKING
1593	/* Keep user track up to date. */
1594	if (pPteDst->n.u1Present)
1595	{
1596	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1597	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1598	}
1599	#endif /* PGMPOOL_WITH_USER_TRACKING */
1600	ASMAtomicWriteSize(pPteDst, 0);
1601	/** @todo count these. */
1602	}
1603	}
1604
1605
1606	/**
1607	* Syncs a guest OS page.
1608	*
1609	* There are no conflicts at this point, neither is there any need for
1610	* page table allocations.
1611	*
1612	* @returns VBox status code.
1613	* @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1614	* @param pVCpu The VMCPU handle.
1615	* @param PdeSrc Page directory entry of the guest.
1616	* @param GCPtrPage Guest context page address.
1617	* @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1618	* @param uErr Fault error (X86_TRAP_PF_*).
1619	*/
1620	PGM_BTH_DECL(int, SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr)
1621	{
1622	PVM pVM = pVCpu->CTX_SUFF(pVM);
1623	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1624	LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr));
1625
1626	Assert(PGMIsLockOwner(pVM));
1627	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
1628	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
1629	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
1630	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
1631	&& PGM_SHW_TYPE != PGM_TYPE_EPT
1632
1633	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
1634	bool fNoExecuteBitValid = !!(CPUMGetGuestEFER(pVCpu) & MSR_K6_EFER_NXE);
1635	# endif
1636
1637	/*
1638	* Assert preconditions.
1639	*/
1640	Assert(PdeSrc.n.u1Present);
1641	Assert(cPages);
1642	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
1643
1644	/*
1645	* Get the shadow PDE, find the shadow page table in the pool.
1646	*/
1647	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1648	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1649	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
1650
1651	/* Fetch the pgm pool shadow descriptor. */
1652	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1653	Assert(pShwPde);
1654
1655	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1656	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1657	PPGMPOOLPAGE pShwPde = NULL;
1658	PX86PDPAE pPDDst;
1659
1660	/* Fetch the pgm pool shadow descriptor. */
1661	int rc = pgmShwGetPaePoolPagePD(&pVCpu->pgm.s, GCPtrPage, &pShwPde);
1662	AssertRCSuccessReturn(rc, rc);
1663	Assert(pShwPde);
1664
1665	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGM(&pVM->pgm.s, pShwPde);
1666	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1667
1668	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1669	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1670	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1671	PX86PDPAE pPDDst;
1672	PX86PDPT pPdptDst;
1673
1674	int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
1675	AssertRCSuccessReturn(rc, rc);
1676	Assert(pPDDst && pPdptDst);
1677	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1678	# endif
1679	SHWPDE PdeDst = *pPdeDst;
1680	AssertMsg(PdeDst.n.u1Present, ("%p=%llx\n", pPdeDst, (uint64_t)PdeDst.u));
1681	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1682
1683	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1684	/* Fetch the pgm pool shadow descriptor. */
1685	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
1686	Assert(pShwPde);
1687	# endif
1688
1689	# if defined(IN_RC)
1690	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1691	PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
1692	# endif
1693
1694	/*
1695	* Check that the page is present and that the shadow PDE isn't out of sync.
1696	*/
1697	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1698	const bool fBigPage = PdeSrc.b.u1Size;
1699	# else
1700	const bool fBigPage = PdeSrc.b.u1Size && (CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
1701	# endif
1702	RTGCPHYS GCPhys;
1703	if (!fBigPage)
1704	{
1705	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1706	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1707	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1708	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE/2);
1709	# endif
1710	}
1711	else
1712	{
1713	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
1714	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1715	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1716	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1717	# endif
1718	}
1719	if ( pShwPage->GCPhys == GCPhys
1720	&& PdeSrc.n.u1Present
1721	&& (PdeSrc.n.u1User == PdeDst.n.u1User)
1722	&& (PdeSrc.n.u1Write == PdeDst.n.u1Write \|\| !PdeDst.n.u1Write)
1723	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
1724	&& (!fNoExecuteBitValid \|\| PdeSrc.n.u1NoExecute == PdeDst.n.u1NoExecute)
1725	# endif
1726	)
1727	{
1728	/*
1729	* Check that the PDE is marked accessed already.
1730	* Since we set the accessed bit before getting here on a #PF, this
1731	* check is only meant for dealing with non-#PF'ing paths.
1732	*/
1733	if (PdeSrc.n.u1Accessed)
1734	{
1735	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1736	if (!fBigPage)
1737	{
1738	/*
1739	* 4KB Page - Map the guest page table.
1740	*/
1741	PGSTPT pPTSrc;
1742	int rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
1743	if (RT_SUCCESS(rc))
1744	{
1745	# ifdef PGM_SYNC_N_PAGES
1746	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
1747	if ( cPages > 1
1748	&& !(uErr & X86_TRAP_PF_P)
1749	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
1750	{
1751	/*
1752	* This code path is currently only taken when the caller is PGMTrap0eHandler
1753	* for non-present pages!
1754	*
1755	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1756	* deal with locality.
1757	*/
1758	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1759	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1760	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1761	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1762	# else
1763	const unsigned offPTSrc = 0;
1764	# endif
1765	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
1766	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1767	iPTDst = 0;
1768	else
1769	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1770	for (; iPTDst < iPTDstEnd; iPTDst++)
1771	{
1772	if (!pPTDst->a[iPTDst].n.u1Present)
1773	{
1774	GSTPTE PteSrc = pPTSrc->a[offPTSrc + iPTDst];
1775	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
1776	NOREF(GCPtrCurPage);
1777	#ifndef IN_RING0
1778	/*
1779	* Assuming kernel code will be marked as supervisor - and not as user level
1780	* and executed using a conforming code selector - And marked as readonly.
1781	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
1782	*/
1783	PPGMPAGE pPage;
1784	if ( ((PdeSrc.u & PteSrc.u) & (X86_PTE_RW \| X86_PTE_US))
1785	\|\| iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK) /* always sync GCPtrPage */
1786	\|\| !CSAMDoesPageNeedScanning(pVM, (RTRCPTR)GCPtrCurPage)
1787	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
1788	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1789	)
1790	#endif /* else: CSAM not active */
1791	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1792	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1793	GCPtrCurPage, PteSrc.n.u1Present,
1794	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1795	PteSrc.n.u1User & PdeSrc.n.u1User,
1796	(uint64_t)PteSrc.u,
1797	(uint64_t)pPTDst->a[iPTDst].u,
1798	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1799	}
1800	}
1801	}
1802	else
1803	# endif /* PGM_SYNC_N_PAGES */
1804	{
1805	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1806	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1807	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1808	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1809	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s\n",
1810	GCPtrPage, PteSrc.n.u1Present,
1811	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1812	PteSrc.n.u1User & PdeSrc.n.u1User,
1813	(uint64_t)PteSrc.u,
1814	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1815	}
1816	}
1817	else /* MMIO or invalid page: emulated in #PF handler. */
1818	{
1819	LogFlow(("PGM_GCPHYS_2_PTR %RGp failed with %Rrc\n", GCPhys, rc));
1820	Assert(!pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK].n.u1Present);
1821	}
1822	}
1823	else
1824	{
1825	/*
1826	* 4/2MB page - lazy syncing shadow 4K pages.
1827	* (There are many causes of getting here, it's no longer only CSAM.)
1828	*/
1829	/* Calculate the GC physical address of this 4KB shadow page. */
1830	RTGCPHYS GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc) \| (GCPtrPage & GST_BIG_PAGE_OFFSET_MASK);
1831	/* Find ram range. */
1832	PPGMPAGE pPage;
1833	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
1834	if (RT_SUCCESS(rc))
1835	{
1836	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1837	/* Try make the page writable if necessary. */
1838	if ( PdeSrc.n.u1Write
1839	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1840	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
1841	{
1842	rc = pgmPhysPageMakeWritableUnlocked(pVM, pPage, GCPhys);
1843	AssertRC(rc);
1844	}
1845	# endif
1846
1847	/*
1848	* Make shadow PTE entry.
1849	*/
1850	SHWPTE PteDst;
1851	PteDst.u = (PdeSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1852	\| PGM_PAGE_GET_HCPHYS(pPage);
1853	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1854	{
1855	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1856	PteDst.n.u1Write = 0;
1857	else
1858	PteDst.u = 0;
1859	}
1860	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1861	# ifdef PGMPOOL_WITH_USER_TRACKING
1862	if (PteDst.n.u1Present && !pPTDst->a[iPTDst].n.u1Present)
1863	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1864	# endif
1865	/* Make sure only allocated pages are mapped writable. */
1866	if ( PteDst.n.u1Write
1867	&& PteDst.n.u1Present
1868	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1869	{
1870	PteDst.n.u1Write = 0; /** @todo this isn't quite working yet... */
1871	Log3(("SyncPage: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, GCPtrPage));
1872	}
1873
1874	ASMAtomicWriteSize(&pPTDst->a[iPTDst], PteDst.u);
1875
1876	/*
1877	* If the page is not flagged as dirty and is writable, then make it read-only
1878	* at PD level, so we can set the dirty bit when the page is modified.
1879	*
1880	* ASSUMES that page access handlers are implemented on page table entry level.
1881	* Thus we will first catch the dirty access and set PDE.D and restart. If
1882	* there is an access handler, we'll trap again and let it work on the problem.
1883	*/
1884	/** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
1885	* As for invlpg, it simply frees the whole shadow PT.
1886	* ...It's possibly because the guest clears it and the guest doesn't really tell us... */
1887	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
1888	{
1889	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageBig));
1890	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
1891	PdeDst.n.u1Write = 0;
1892	}
1893	else
1894	{
1895	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
1896	PdeDst.n.u1Write = PdeSrc.n.u1Write;
1897	}
1898	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
1899	Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n",
1900	GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
1901	PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1902	}
1903	else
1904	LogFlow(("PGM_GCPHYS_2_PTR %RGp (big) failed with %Rrc\n", GCPhys, rc));
1905	}
1906	# if defined(IN_RC)
1907	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1908	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
1909	# endif
1910	return VINF_SUCCESS;
1911	}
1912	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDNAs));
1913	}
1914	else
1915	{
1916	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSync));
1917	Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
1918	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
1919	}
1920
1921	/*
1922	* Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
1923	* Yea, I'm lazy.
1924	*/
1925	pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
1926	ASMAtomicWriteSize(pPdeDst, 0);
1927
1928	# if defined(IN_RC)
1929	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1930	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
1931	# endif
1932	PGM_INVL_GUEST_TLBS();
1933	return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
1934
1935	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
1936	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
1937	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
1938	&& !defined(IN_RC)
1939
1940	# ifdef PGM_SYNC_N_PAGES
1941	/*
1942	* Get the shadow PDE, find the shadow page table in the pool.
1943	*/
1944	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1945	X86PDE PdeDst = pgmShwGet32BitPDE(&pVCpu->pgm.s, GCPtrPage);
1946
1947	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1948	X86PDEPAE PdeDst = pgmShwGetPaePDE(&pVCpu->pgm.s, GCPtrPage);
1949
1950	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1951	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
1952	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpt);
1953	PX86PDPAE pPDDst;
1954	X86PDEPAE PdeDst;
1955	PX86PDPT pPdptDst;
1956
1957	int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
1958	AssertRCSuccessReturn(rc, rc);
1959	Assert(pPDDst && pPdptDst);
1960	PdeDst = pPDDst->a[iPDDst];
1961	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
1962	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
1963	PEPTPD pPDDst;
1964	EPTPDE PdeDst;
1965
1966	int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, NULL, &pPDDst);
1967	if (rc != VINF_SUCCESS)
1968	{
1969	AssertRC(rc);
1970	return rc;
1971	}
1972	Assert(pPDDst);
1973	PdeDst = pPDDst->a[iPDDst];
1974	# endif
1975	AssertMsg(PdeDst.n.u1Present, ("%#llx\n", (uint64_t)PdeDst.u));
1976	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1977	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1978
1979	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
1980	if ( cPages > 1
1981	&& !(uErr & X86_TRAP_PF_P)
1982	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
1983	{
1984	/*
1985	* This code path is currently only taken when the caller is PGMTrap0eHandler
1986	* for non-present pages!
1987	*
1988	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1989	* deal with locality.
1990	*/
1991	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1992	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
1993	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1994	iPTDst = 0;
1995	else
1996	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1997	for (; iPTDst < iPTDstEnd; iPTDst++)
1998	{
1999	if (!pPTDst->a[iPTDst].n.u1Present)
2000	{
2001	GSTPTE PteSrc;
2002
2003	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT);
2004
2005	/* Fake the page table entry */
2006	PteSrc.u = GCPtrCurPage;
2007	PteSrc.n.u1Present = 1;
2008	PteSrc.n.u1Dirty = 1;
2009	PteSrc.n.u1Accessed = 1;
2010	PteSrc.n.u1Write = 1;
2011	PteSrc.n.u1User = 1;
2012
2013	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2014
2015	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
2016	GCPtrCurPage, PteSrc.n.u1Present,
2017	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2018	PteSrc.n.u1User & PdeSrc.n.u1User,
2019	(uint64_t)PteSrc.u,
2020	(uint64_t)pPTDst->a[iPTDst].u,
2021	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2022
2023	if (RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)))
2024	break;
2025	}
2026	else
2027	Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n", (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT), iPTDst, pPTDst->a[iPTDst].u));
2028	}
2029	}
2030	else
2031	# endif /* PGM_SYNC_N_PAGES */
2032	{
2033	GSTPTE PteSrc;
2034	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2035	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT);
2036
2037	/* Fake the page table entry */
2038	PteSrc.u = GCPtrCurPage;
2039	PteSrc.n.u1Present = 1;
2040	PteSrc.n.u1Dirty = 1;
2041	PteSrc.n.u1Accessed = 1;
2042	PteSrc.n.u1Write = 1;
2043	PteSrc.n.u1User = 1;
2044	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2045
2046	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}PteDst=%08llx%s\n",
2047	GCPtrPage, PteSrc.n.u1Present,
2048	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2049	PteSrc.n.u1User & PdeSrc.n.u1User,
2050	(uint64_t)PteSrc.u,
2051	(uint64_t)pPTDst->a[iPTDst].u,
2052	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2053	}
2054	return VINF_SUCCESS;
2055
2056	#else
2057	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2058	return VERR_INTERNAL_ERROR;
2059	#endif
2060	}
2061
2062
2063	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2064	/**
2065	* Investigate page fault and handle write protection page faults caused by
2066	* dirty bit tracking.
2067	*
2068	* @returns VBox status code.
2069	* @param pVCpu The VMCPU handle.
2070	* @param uErr Page fault error code.
2071	* @param pPdeDst Shadow page directory entry.
2072	* @param pPdeSrc Guest page directory entry.
2073	* @param GCPtrPage Guest context page address.
2074	*/
2075	PGM_BTH_DECL(int, CheckPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCPTR GCPtrPage)
2076	{
2077	bool fWriteProtect = !!(CPUMGetGuestCR0(pVCpu) & X86_CR0_WP);
2078	bool fUserLevelFault = !!(uErr & X86_TRAP_PF_US);
2079	bool fWriteFault = !!(uErr & X86_TRAP_PF_RW);
2080	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2081	bool fBigPagesSupported = true;
2082	# else
2083	bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
2084	# endif
2085	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2086	bool fNoExecuteBitValid = !!(CPUMGetGuestEFER(pVCpu) & MSR_K6_EFER_NXE);
2087	# endif
2088	unsigned uPageFaultLevel;
2089	int rc;
2090	PVM pVM = pVCpu->CTX_SUFF(pVM);
2091	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2092
2093	Assert(PGMIsLockOwner(pVM));
2094
2095	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2096	LogFlow(("CheckPageFault: GCPtrPage=%RGv uErr=%#x PdeSrc=%08x\n", GCPtrPage, uErr, pPdeSrc->u));
2097
2098	# if PGM_GST_TYPE == PGM_TYPE_PAE \
2099	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
2100
2101	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2102	PX86PML4E pPml4eSrc;
2103	PX86PDPE pPdpeSrc;
2104
2105	pPdpeSrc = pgmGstGetLongModePDPTPtr(&pVCpu->pgm.s, GCPtrPage, &pPml4eSrc);
2106	Assert(pPml4eSrc);
2107
2108	/*
2109	* Real page fault? (PML4E level)
2110	*/
2111	if ( (uErr & X86_TRAP_PF_RSVD)
2112	\|\| !pPml4eSrc->n.u1Present
2113	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPml4eSrc->n.u1NoExecute)
2114	\|\| (fWriteFault && !pPml4eSrc->n.u1Write && (fUserLevelFault \|\| fWriteProtect))
2115	\|\| (fUserLevelFault && !pPml4eSrc->n.u1User)
2116	)
2117	{
2118	uPageFaultLevel = 0;
2119	goto l_UpperLevelPageFault;
2120	}
2121	Assert(pPdpeSrc);
2122
2123	# else /* PAE */
2124	PX86PDPE pPdpeSrc = pgmGstGetPaePDPEPtr(&pVCpu->pgm.s, GCPtrPage);
2125	# endif /* PAE */
2126
2127	/*
2128	* Real page fault? (PDPE level)
2129	*/
2130	if ( (uErr & X86_TRAP_PF_RSVD)
2131	\|\| !pPdpeSrc->n.u1Present
2132	# if PGM_GST_TYPE == PGM_TYPE_AMD64 /* NX, r/w, u/s bits in the PDPE are long mode only */
2133	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPdpeSrc->lm.u1NoExecute)
2134	\|\| (fWriteFault && !pPdpeSrc->lm.u1Write && (fUserLevelFault \|\| fWriteProtect))
2135	\|\| (fUserLevelFault && !pPdpeSrc->lm.u1User)
2136	# endif
2137	)
2138	{
2139	uPageFaultLevel = 1;
2140	goto l_UpperLevelPageFault;
2141	}
2142	# endif
2143
2144	/*
2145	* Real page fault? (PDE level)
2146	*/
2147	if ( (uErr & X86_TRAP_PF_RSVD)
2148	\|\| !pPdeSrc->n.u1Present
2149	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2150	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPdeSrc->n.u1NoExecute)
2151	# endif
2152	\|\| (fWriteFault && !pPdeSrc->n.u1Write && (fUserLevelFault \|\| fWriteProtect))
2153	\|\| (fUserLevelFault && !pPdeSrc->n.u1User) )
2154	{
2155	uPageFaultLevel = 2;
2156	goto l_UpperLevelPageFault;
2157	}
2158
2159	/*
2160	* First check the easy case where the page directory has been marked read-only to track
2161	* the dirty bit of an emulated BIG page
2162	*/
2163	if (pPdeSrc->b.u1Size && fBigPagesSupported)
2164	{
2165	/* Mark guest page directory as accessed */
2166	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2167	pPml4eSrc->n.u1Accessed = 1;
2168	pPdpeSrc->lm.u1Accessed = 1;
2169	# endif
2170	pPdeSrc->b.u1Accessed = 1;
2171
2172	/*
2173	* Only write protection page faults are relevant here.
2174	*/
2175	if (fWriteFault)
2176	{
2177	/* Mark guest page directory as dirty (BIG page only). */
2178	pPdeSrc->b.u1Dirty = 1;
2179
2180	if (pPdeDst->n.u1Present)
2181	{
2182	if (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY)
2183	{
2184	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageTrap));
2185	Assert(pPdeSrc->b.u1Write);
2186
2187	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
2188	* fault again and take this path to only invalidate the entry.
2189	*/
2190	pPdeDst->n.u1Write = 1;
2191	pPdeDst->n.u1Accessed = 1;
2192	pPdeDst->au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2193	PGM_INVL_BIG_PG(GCPtrPage);
2194	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2195	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
2196	}
2197	# ifdef IN_RING0
2198	else
2199	/* Check for stale TLB entry; only applies to the SMP guest case. */
2200	if ( pVM->cCPUs > 1
2201	&& pPdeDst->n.u1Write
2202	&& pPdeDst->n.u1Accessed)
2203	{
2204	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2205	if (pShwPage)
2206	{
2207	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2208	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2209	if ( pPteDst->n.u1Present
2210	&& pPteDst->n.u1Write)
2211	{
2212	/* Stale TLB entry. */
2213	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageStale));
2214	PGM_INVL_PG(GCPtrPage);
2215
2216	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2217	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
2218	}
2219	}
2220	}
2221	# endif /* IN_RING0 */
2222	}
2223	}
2224	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2225	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2226	}
2227	/* else: 4KB page table */
2228
2229	/*
2230	* Map the guest page table.
2231	*/
2232	PGSTPT pPTSrc;
2233	rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
2234	if (RT_SUCCESS(rc))
2235	{
2236	/*
2237	* Real page fault?
2238	*/
2239	PGSTPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2240	const GSTPTE PteSrc = *pPteSrc;
2241	if ( !PteSrc.n.u1Present
2242	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2243	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && PteSrc.n.u1NoExecute)
2244	# endif
2245	\|\| (fWriteFault && !PteSrc.n.u1Write && (fUserLevelFault \|\| fWriteProtect))
2246	\|\| (fUserLevelFault && !PteSrc.n.u1User)
2247	)
2248	{
2249	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyTrackRealPF));
2250	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2251	LogFlow(("CheckPageFault: real page fault at %RGv PteSrc.u=%08x (2)\n", GCPtrPage, PteSrc.u));
2252
2253	/* Check the present bit as the shadow tables can cause different error codes by being out of sync.
2254	* See the 2nd case above as well.
2255	*/
2256	if (pPdeSrc->n.u1Present && pPteSrc->n.u1Present)
2257	TRPMSetErrorCode(pVCpu, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
2258
2259	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2260	return VINF_EM_RAW_GUEST_TRAP;
2261	}
2262	LogFlow(("CheckPageFault: page fault at %RGv PteSrc.u=%08x\n", GCPtrPage, PteSrc.u));
2263
2264	/*
2265	* Set the accessed bits in the page directory and the page table.
2266	*/
2267	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2268	pPml4eSrc->n.u1Accessed = 1;
2269	pPdpeSrc->lm.u1Accessed = 1;
2270	# endif
2271	pPdeSrc->n.u1Accessed = 1;
2272	pPteSrc->n.u1Accessed = 1;
2273
2274	/*
2275	* Only write protection page faults are relevant here.
2276	*/
2277	if (fWriteFault)
2278	{
2279	/* Write access, so mark guest entry as dirty. */
2280	# ifdef VBOX_WITH_STATISTICS
2281	if (!pPteSrc->n.u1Dirty)
2282	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtiedPage));
2283	else
2284	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageAlreadyDirty));
2285	# endif
2286
2287	pPteSrc->n.u1Dirty = 1;
2288
2289	if (pPdeDst->n.u1Present)
2290	{
2291	#ifndef IN_RING0
2292	/* Bail out here as pgmPoolGetPageByHCPhys will return NULL and we'll crash below.
2293	* Our individual shadow handlers will provide more information and force a fatal exit.
2294	*/
2295	if (MMHyperIsInsideArea(pVM, (RTGCPTR)GCPtrPage))
2296	{
2297	LogRel(("CheckPageFault: write to hypervisor region %RGv\n", GCPtrPage));
2298	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2299	return VINF_SUCCESS;
2300	}
2301	#endif
2302	/*
2303	* Map shadow page table.
2304	*/
2305	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2306	if (pShwPage)
2307	{
2308	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2309	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2310	if (pPteDst->n.u1Present) /** @todo Optimize accessed bit emulation? */
2311	{
2312	if (pPteDst->u & PGM_PTFLAGS_TRACK_DIRTY)
2313	{
2314	LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage));
2315	# ifdef VBOX_STRICT
2316	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, pPteSrc->u & GST_PTE_PG_MASK);
2317	if (pPage)
2318	AssertMsg(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage),
2319	("Unexpected dirty bit tracking on monitored page %RGv (phys %RGp)!!!!!!\n", GCPtrPage, pPteSrc->u & X86_PTE_PAE_PG_MASK));
2320	# endif
2321	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageTrap));
2322
2323	Assert(pPteSrc->n.u1Write);
2324
2325	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
2326	* fault again and take this path to only invalidate the entry.
2327	*/
2328	pPteDst->n.u1Write = 1;
2329	pPteDst->n.u1Dirty = 1;
2330	pPteDst->n.u1Accessed = 1;
2331	pPteDst->au32[0] &= ~PGM_PTFLAGS_TRACK_DIRTY;
2332	PGM_INVL_PG(GCPtrPage);
2333
2334	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2335	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
2336	}
2337	# ifdef IN_RING0
2338	else
2339	/* Check for stale TLB entry; only applies to the SMP guest case. */
2340	if ( pVM->cCPUs > 1
2341	&& pPteDst->n.u1Write == 1
2342	&& pPteDst->n.u1Accessed == 1)
2343	{
2344	/* Stale TLB entry. */
2345	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageStale));
2346	PGM_INVL_PG(GCPtrPage);
2347
2348	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2349	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
2350	}
2351	# endif
2352	}
2353	}
2354	else
2355	AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
2356	}
2357	}
2358	/** @todo Optimize accessed bit emulation? */
2359	# ifdef VBOX_STRICT
2360	/*
2361	* Sanity check.
2362	*/
2363	else if ( !pPteSrc->n.u1Dirty
2364	&& (pPdeSrc->n.u1Write & pPteSrc->n.u1Write)
2365	&& pPdeDst->n.u1Present)
2366	{
2367	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2368	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2369	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2370	if ( pPteDst->n.u1Present
2371	&& pPteDst->n.u1Write)
2372	LogFlow(("Writable present page %RGv not marked for dirty bit tracking!!!\n", GCPtrPage));
2373	}
2374	# endif /* VBOX_STRICT */
2375	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2376	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2377	}
2378	AssertRC(rc);
2379	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2380	return rc;
2381
2382
2383	l_UpperLevelPageFault:
2384	/*
2385	* Pagefault detected while checking the PML4E, PDPE or PDE.
2386	* Single exit handler to get rid of duplicate code paths.
2387	*/
2388	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyTrackRealPF));
2389	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2390	Log(("CheckPageFault: real page fault at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2391
2392	if (
2393	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2394	pPml4eSrc->n.u1Present &&
2395	# endif
2396	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
2397	pPdpeSrc->n.u1Present &&
2398	# endif
2399	pPdeSrc->n.u1Present)
2400	{
2401	/* Check the present bit as the shadow tables can cause different error codes by being out of sync. */
2402	if (pPdeSrc->b.u1Size && fBigPagesSupported)
2403	{
2404	TRPMSetErrorCode(pVCpu, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
2405	}
2406	else
2407	{
2408	/*
2409	* Map the guest page table.
2410	*/
2411	PGSTPT pPTSrc;
2412	rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
2413	if (RT_SUCCESS(rc))
2414	{
2415	PGSTPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2416	const GSTPTE PteSrc = *pPteSrc;
2417	if (pPteSrc->n.u1Present)
2418	TRPMSetErrorCode(pVCpu, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
2419	}
2420	AssertRC(rc);
2421	}
2422	}
2423	return VINF_EM_RAW_GUEST_TRAP;
2424	}
2425	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
2426
2427
2428	/**
2429	* Sync a shadow page table.
2430	*
2431	* The shadow page table is not present. This includes the case where
2432	* there is a conflict with a mapping.
2433	*
2434	* @returns VBox status code.
2435	* @param pVCpu The VMCPU handle.
2436	* @param iPD Page directory index.
2437	* @param pPDSrc Source page directory (i.e. Guest OS page directory).
2438	* Assume this is a temporary mapping.
2439	* @param GCPtrPage GC Pointer of the page that caused the fault
2440	*/
2441	PGM_BTH_DECL(int, SyncPT)(PVMCPU pVCpu, unsigned iPDSrc, PGSTPD pPDSrc, RTGCPTR GCPtrPage)
2442	{
2443	PVM pVM = pVCpu->CTX_SUFF(pVM);
2444	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2445
2446	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2447	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPtPD[iPDSrc]);
2448	LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage));
2449
2450	Assert(PGMIsLocked(pVM));
2451
2452	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2453	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
2454	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
2455	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2456	&& PGM_SHW_TYPE != PGM_TYPE_EPT
2457
2458	int rc = VINF_SUCCESS;
2459
2460	/*
2461	* Validate input a little bit.
2462	*/
2463	AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
2464	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2465	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2466	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
2467
2468	/* Fetch the pgm pool shadow descriptor. */
2469	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2470	Assert(pShwPde);
2471
2472	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2473	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2474	PPGMPOOLPAGE pShwPde = NULL;
2475	PX86PDPAE pPDDst;
2476	PSHWPDE pPdeDst;
2477
2478	/* Fetch the pgm pool shadow descriptor. */
2479	rc = pgmShwGetPaePoolPagePD(&pVCpu->pgm.s, GCPtrPage, &pShwPde);
2480	AssertRCSuccessReturn(rc, rc);
2481	Assert(pShwPde);
2482
2483	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGM(&pVM->pgm.s, pShwPde);
2484	pPdeDst = &pPDDst->a[iPDDst];
2485
2486	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2487	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2488	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2489	PX86PDPAE pPDDst;
2490	PX86PDPT pPdptDst;
2491	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2492	AssertRCSuccessReturn(rc, rc);
2493	Assert(pPDDst);
2494	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2495	# endif
2496	SHWPDE PdeDst = *pPdeDst;
2497
2498	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2499	/* Fetch the pgm pool shadow descriptor. */
2500	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2501	Assert(pShwPde);
2502	# endif
2503
2504	# ifndef PGM_WITHOUT_MAPPINGS
2505	/*
2506	* Check for conflicts.
2507	* GC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
2508	* HC: Simply resolve the conflict.
2509	*/
2510	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2511	{
2512	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
2513	# ifndef IN_RING3
2514	Log(("SyncPT: Conflict at %RGv\n", GCPtrPage));
2515	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2516	return VERR_ADDRESS_CONFLICT;
2517	# else
2518	PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
2519	Assert(pMapping);
2520	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2521	int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2522	# elif PGM_GST_TYPE == PGM_TYPE_PAE
2523	int rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2524	# else
2525	AssertFailed(); /* can't happen for amd64 */
2526	# endif
2527	if (RT_FAILURE(rc))
2528	{
2529	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2530	return rc;
2531	}
2532	PdeDst = *pPdeDst;
2533	# endif
2534	}
2535	# else /* PGM_WITHOUT_MAPPINGS */
2536	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
2537	# endif /* PGM_WITHOUT_MAPPINGS */
2538	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2539
2540	# if defined(IN_RC)
2541	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
2542	PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
2543	# endif
2544
2545	/*
2546	* Sync page directory entry.
2547	*/
2548	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2549	if (PdeSrc.n.u1Present)
2550	{
2551	/*
2552	* Allocate & map the page table.
2553	*/
2554	PSHWPT pPTDst;
2555	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2556	const bool fPageTable = !PdeSrc.b.u1Size;
2557	# else
2558	const bool fPageTable = !PdeSrc.b.u1Size \|\| !(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
2559	# endif
2560	PPGMPOOLPAGE pShwPage;
2561	RTGCPHYS GCPhys;
2562	if (fPageTable)
2563	{
2564	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
2565	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2566	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2567	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE / 2);
2568	# endif
2569	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
2570	}
2571	else
2572	{
2573	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
2574	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2575	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2576	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
2577	# endif
2578	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, pShwPde->idx, iPDDst, &pShwPage);
2579	}
2580	if (rc == VINF_SUCCESS)
2581	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2582	else if (rc == VINF_PGM_CACHED_PAGE)
2583	{
2584	/*
2585	* The PT was cached, just hook it up.
2586	*/
2587	if (fPageTable)
2588	PdeDst.u = pShwPage->Core.Key
2589	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2590	else
2591	{
2592	PdeDst.u = pShwPage->Core.Key
2593	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2594	/* (see explanation and assumptions further down.) */
2595	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
2596	{
2597	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageBig));
2598	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2599	PdeDst.b.u1Write = 0;
2600	}
2601	}
2602	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2603	# if defined(IN_RC)
2604	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2605	# endif
2606	return VINF_SUCCESS;
2607	}
2608	else if (rc == VERR_PGM_POOL_FLUSHED)
2609	{
2610	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2611	# if defined(IN_RC)
2612	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2613	# endif
2614	return VINF_PGM_SYNC_CR3;
2615	}
2616	else
2617	AssertMsgFailedReturn(("rc=%Rrc\n", rc), VERR_INTERNAL_ERROR);
2618	PdeDst.u &= X86_PDE_AVL_MASK;
2619	PdeDst.u \|= pShwPage->Core.Key;
2620
2621	/*
2622	* Page directory has been accessed (this is a fault situation, remember).
2623	*/
2624	pPDSrc->a[iPDSrc].n.u1Accessed = 1;
2625	if (fPageTable)
2626	{
2627	/*
2628	* Page table - 4KB.
2629	*
2630	* Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
2631	*/
2632	Log2(("SyncPT: 4K %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
2633	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
2634	PGSTPT pPTSrc;
2635	rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
2636	if (RT_SUCCESS(rc))
2637	{
2638	/*
2639	* Start by syncing the page directory entry so CSAM's TLB trick works.
2640	*/
2641	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| X86_PDE_AVL_MASK))
2642	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2643	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2644	# if defined(IN_RC)
2645	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2646	# endif
2647
2648	/*
2649	* Directory/page user or supervisor privilege: (same goes for read/write)
2650	*
2651	* Directory Page Combined
2652	* U/S U/S U/S
2653	* 0 0 0
2654	* 0 1 0
2655	* 1 0 0
2656	* 1 1 1
2657	*
2658	* Simple AND operation. Table listed for completeness.
2659	*
2660	*/
2661	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT4K));
2662	# ifdef PGM_SYNC_N_PAGES
2663	unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2664	unsigned iPTDst = iPTBase;
2665	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2666	if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
2667	iPTDst = 0;
2668	else
2669	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2670	# else /* !PGM_SYNC_N_PAGES */
2671	unsigned iPTDst = 0;
2672	const unsigned iPTDstEnd = RT_ELEMENTS(pPTDst->a);
2673	# endif /* !PGM_SYNC_N_PAGES */
2674	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2675	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2676	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2677	# else
2678	const unsigned offPTSrc = 0;
2679	# endif
2680	for (; iPTDst < iPTDstEnd; iPTDst++)
2681	{
2682	const unsigned iPTSrc = iPTDst + offPTSrc;
2683	const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2684
2685	if (PteSrc.n.u1Present) /* we've already cleared it above */
2686	{
2687	# ifndef IN_RING0
2688	/*
2689	* Assuming kernel code will be marked as supervisor - and not as user level
2690	* and executed using a conforming code selector - And marked as readonly.
2691	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
2692	*/
2693	PPGMPAGE pPage;
2694	if ( ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW \| X86_PTE_US))
2695	\|\| !CSAMDoesPageNeedScanning(pVM, (RTRCPTR)((iPDSrc << GST_PD_SHIFT) \| (iPTSrc << PAGE_SHIFT)))
2696	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
2697	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2698	)
2699	# endif
2700	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2701	Log2(("SyncPT: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
2702	(RTGCPTR)((iPDSrc << GST_PD_SHIFT) \| (iPTSrc << PAGE_SHIFT)),
2703	PteSrc.n.u1Present,
2704	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2705	PteSrc.n.u1User & PdeSrc.n.u1User,
2706	(uint64_t)PteSrc.u,
2707	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : "", pPTDst->a[iPTDst].u, iPTSrc, PdeSrc.au32[0],
2708	(RTGCPHYS)((PdeSrc.u & GST_PDE_PG_MASK) + iPTSrc*sizeof(PteSrc)) ));
2709	}
2710	} /* for PTEs */
2711	}
2712	}
2713	else
2714	{
2715	/*
2716	* Big page - 2/4MB.
2717	*
2718	* We'll walk the ram range list in parallel and optimize lookups.
2719	* We will only sync on shadow page table at a time.
2720	*/
2721	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT4M));
2722
2723	/**
2724	* @todo It might be more efficient to sync only a part of the 4MB page (similar to what we do for 4kb PDs).
2725	*/
2726
2727	/*
2728	* Start by syncing the page directory entry.
2729	*/
2730	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
2731	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2732
2733	/*
2734	* If the page is not flagged as dirty and is writable, then make it read-only
2735	* at PD level, so we can set the dirty bit when the page is modified.
2736	*
2737	* ASSUMES that page access handlers are implemented on page table entry level.
2738	* Thus we will first catch the dirty access and set PDE.D and restart. If
2739	* there is an access handler, we'll trap again and let it work on the problem.
2740	*/
2741	/** @todo move the above stuff to a section in the PGM documentation. */
2742	Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
2743	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
2744	{
2745	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageBig));
2746	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2747	PdeDst.b.u1Write = 0;
2748	}
2749	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2750	# if defined(IN_RC)
2751	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2752	# endif
2753
2754	/*
2755	* Fill the shadow page table.
2756	*/
2757	/* Get address and flags from the source PDE. */
2758	SHWPTE PteDstBase;
2759	PteDstBase.u = PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT);
2760
2761	/* Loop thru the entries in the shadow PT. */
2762	const RTGCPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
2763	Log2(("SyncPT: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n",
2764	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
2765	GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2766	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
2767	unsigned iPTDst = 0;
2768	while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2769	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2770	{
2771	/* Advance ram range list. */
2772	while (pRam && GCPhys > pRam->GCPhysLast)
2773	pRam = pRam->CTX_SUFF(pNext);
2774	if (pRam && GCPhys >= pRam->GCPhys)
2775	{
2776	unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2777	do
2778	{
2779	/* Make shadow PTE. */
2780	PPGMPAGE pPage = &pRam->aPages[iHCPage];
2781	SHWPTE PteDst;
2782
2783	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2784	/* Try make the page writable if necessary. */
2785	if ( PteDstBase.n.u1Write
2786	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2787	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
2788	{
2789	rc = pgmPhysPageMakeWritableUnlocked(pVM, pPage, GCPhys);
2790	AssertRCReturn(rc, rc);
2791	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2792	break;
2793	}
2794	# endif
2795
2796	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2797	{
2798	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
2799	{
2800	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) \| PteDstBase.u;
2801	PteDst.n.u1Write = 0;
2802	}
2803	else
2804	PteDst.u = 0;
2805	}
2806	# ifndef IN_RING0
2807	/*
2808	* Assuming kernel code will be marked as supervisor and not as user level and executed
2809	* using a conforming code selector. Don't check for readonly, as that implies the whole
2810	* 4MB can be code or readonly data. Linux enables write access for its large pages.
2811	*/
2812	else if ( !PdeSrc.n.u1User
2813	&& CSAMDoesPageNeedScanning(pVM, (RTRCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))))
2814	PteDst.u = 0;
2815	# endif
2816	else
2817	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) \| PteDstBase.u;
2818
2819	/* Only map writable pages writable. */
2820	if ( PteDst.n.u1Write
2821	&& PteDst.n.u1Present
2822	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2823	{
2824	PteDst.n.u1Write = 0; /** @todo this isn't quite working yet... */
2825	Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))));
2826	}
2827
2828	# ifdef PGMPOOL_WITH_USER_TRACKING
2829	if (PteDst.n.u1Present)
2830	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2831	# endif
2832	/* commit it */
2833	pPTDst->a[iPTDst] = PteDst;
2834	Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
2835	(RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT)), PteDst.n.u1Present, PteDst.n.u1Write, PteDst.n.u1User, (uint64_t)PteDst.u,
2836	PteDst.u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2837
2838	/* advance */
2839	GCPhys += PAGE_SIZE;
2840	iHCPage++;
2841	iPTDst++;
2842	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2843	&& GCPhys <= pRam->GCPhysLast);
2844	}
2845	else if (pRam)
2846	{
2847	Log(("Invalid pages at %RGp\n", GCPhys));
2848	do
2849	{
2850	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2851	GCPhys += PAGE_SIZE;
2852	iPTDst++;
2853	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2854	&& GCPhys < pRam->GCPhys);
2855	}
2856	else
2857	{
2858	Log(("Invalid pages at %RGp (2)\n", GCPhys));
2859	for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
2860	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2861	}
2862	} /* while more PTEs */
2863	} /* 4KB / 4MB */
2864	}
2865	else
2866	AssertRelease(!PdeDst.n.u1Present);
2867
2868	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2869	if (RT_FAILURE(rc))
2870	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPTFailed));
2871	return rc;
2872
2873	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2874	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2875	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2876	&& !defined(IN_RC)
2877
2878	/*
2879	* Validate input a little bit.
2880	*/
2881	int rc = VINF_SUCCESS;
2882	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2883	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2884	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
2885
2886	/* Fetch the pgm pool shadow descriptor. */
2887	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2888	Assert(pShwPde);
2889
2890	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2891	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2892	PPGMPOOLPAGE pShwPde;
2893	PX86PDPAE pPDDst;
2894	PSHWPDE pPdeDst;
2895
2896	/* Fetch the pgm pool shadow descriptor. */
2897	rc = pgmShwGetPaePoolPagePD(&pVCpu->pgm.s, GCPtrPage, &pShwPde);
2898	AssertRCSuccessReturn(rc, rc);
2899	Assert(pShwPde);
2900
2901	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGM(&pVM->pgm.s, pShwPde);
2902	pPdeDst = &pPDDst->a[iPDDst];
2903
2904	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2905	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2906	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2907	PX86PDPAE pPDDst;
2908	PX86PDPT pPdptDst;
2909	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2910	AssertRCSuccessReturn(rc, rc);
2911	Assert(pPDDst);
2912	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2913
2914	/* Fetch the pgm pool shadow descriptor. */
2915	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2916	Assert(pShwPde);
2917
2918	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2919	const unsigned iPdpt = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
2920	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2921	PEPTPD pPDDst;
2922	PEPTPDPT pPdptDst;
2923
2924	rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, &pPdptDst, &pPDDst);
2925	if (rc != VINF_SUCCESS)
2926	{
2927	AssertRC(rc);
2928	return rc;
2929	}
2930	Assert(pPDDst);
2931	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2932
2933	/* Fetch the pgm pool shadow descriptor. */
2934	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & EPT_PDPTE_PG_MASK);
2935	Assert(pShwPde);
2936	# endif
2937	SHWPDE PdeDst = *pPdeDst;
2938
2939	Assert(!(PdeDst.u & PGM_PDFLAGS_MAPPING));
2940	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2941
2942	GSTPDE PdeSrc;
2943	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2944	PdeSrc.n.u1Present = 1;
2945	PdeSrc.n.u1Write = 1;
2946	PdeSrc.n.u1Accessed = 1;
2947	PdeSrc.n.u1User = 1;
2948
2949	/*
2950	* Allocate & map the page table.
2951	*/
2952	PSHWPT pPTDst;
2953	PPGMPOOLPAGE pShwPage;
2954	RTGCPHYS GCPhys;
2955
2956	/* Virtual address = physical address */
2957	GCPhys = GCPtrPage & X86_PAGE_4K_BASE_MASK;
2958	rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
2959
2960	if ( rc == VINF_SUCCESS
2961	\|\| rc == VINF_PGM_CACHED_PAGE)
2962	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2963	else
2964	AssertMsgFailedReturn(("rc=%Rrc\n", rc), VERR_INTERNAL_ERROR);
2965
2966	PdeDst.u &= X86_PDE_AVL_MASK;
2967	PdeDst.u \|= pShwPage->Core.Key;
2968	PdeDst.n.u1Present = 1;
2969	PdeDst.n.u1Write = 1;
2970	# if PGM_SHW_TYPE == PGM_TYPE_EPT
2971	PdeDst.n.u1Execute = 1;
2972	# else
2973	PdeDst.n.u1User = 1;
2974	PdeDst.n.u1Accessed = 1;
2975	# endif
2976	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2977
2978	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, PGM_SYNC_NR_PAGES, 0 /* page not present */);
2979	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2980	return rc;
2981
2982	#else
2983	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
2984	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2985	return VERR_INTERNAL_ERROR;
2986	#endif
2987	}
2988
2989
2990
2991	/**
2992	* Prefetch a page/set of pages.
2993	*
2994	* Typically used to sync commonly used pages before entering raw mode
2995	* after a CR3 reload.
2996	*
2997	* @returns VBox status code.
2998	* @param pVCpu The VMCPU handle.
2999	* @param GCPtrPage Page to invalidate.
3000	*/
3001	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
3002	{
3003	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
3004	&& PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
3005	/*
3006	* Check that all Guest levels thru the PDE are present, getting the
3007	* PD and PDE in the processes.
3008	*/
3009	int rc = VINF_SUCCESS;
3010	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3011	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3012	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
3013	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
3014	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3015	unsigned iPDSrc;
3016	X86PDPE PdpeSrc;
3017	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, GCPtrPage, &iPDSrc, &PdpeSrc);
3018	if (!pPDSrc)
3019	return VINF_SUCCESS; /* not present */
3020	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3021	unsigned iPDSrc;
3022	PX86PML4E pPml4eSrc;
3023	X86PDPE PdpeSrc;
3024	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3025	if (!pPDSrc)
3026	return VINF_SUCCESS; /* not present */
3027	# endif
3028	const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3029	# else
3030	PGSTPD pPDSrc = NULL;
3031	const unsigned iPDSrc = 0;
3032	GSTPDE PdeSrc;
3033
3034	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
3035	PdeSrc.n.u1Present = 1;
3036	PdeSrc.n.u1Write = 1;
3037	PdeSrc.n.u1Accessed = 1;
3038	PdeSrc.n.u1User = 1;
3039	# endif
3040
3041	if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
3042	{
3043	PVM pVM = pVCpu->CTX_SUFF(pVM);
3044	pgmLock(pVM);
3045
3046	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3047	const X86PDE PdeDst = pgmShwGet32BitPDE(&pVCpu->pgm.s, GCPtrPage);
3048	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3049	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3050	PX86PDPAE pPDDst;
3051	X86PDEPAE PdeDst;
3052	# if PGM_GST_TYPE != PGM_TYPE_PAE
3053	X86PDPE PdpeSrc;
3054
3055	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3056	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3057	# endif
3058	int rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, &PdpeSrc, &pPDDst);
3059	if (rc != VINF_SUCCESS)
3060	{
3061	pgmUnlock(pVM);
3062	AssertRC(rc);
3063	return rc;
3064	}
3065	Assert(pPDDst);
3066	PdeDst = pPDDst->a[iPDDst];
3067
3068	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3069	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3070	PX86PDPAE pPDDst;
3071	X86PDEPAE PdeDst;
3072
3073	# if PGM_GST_TYPE == PGM_TYPE_PROT
3074	/* AMD-V nested paging */
3075	X86PML4E Pml4eSrc;
3076	X86PDPE PdpeSrc;
3077	PX86PML4E pPml4eSrc = &Pml4eSrc;
3078
3079	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3080	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_NX \| X86_PML4E_A;
3081	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_NX \| X86_PDPE_A;
3082	# endif
3083
3084	int rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc, &PdpeSrc, &pPDDst);
3085	if (rc != VINF_SUCCESS)
3086	{
3087	pgmUnlock(pVM);
3088	AssertRC(rc);
3089	return rc;
3090	}
3091	Assert(pPDDst);
3092	PdeDst = pPDDst->a[iPDDst];
3093	# endif
3094	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
3095	{
3096	if (!PdeDst.n.u1Present)
3097	{
3098	/** r=bird: This guy will set the A bit on the PDE, probably harmless. */
3099	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3100	}
3101	else
3102	{
3103	/** @note We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
3104	* R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
3105	* makes no sense to prefetch more than one page.
3106	*/
3107	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3108	if (RT_SUCCESS(rc))
3109	rc = VINF_SUCCESS;
3110	}
3111	}
3112	pgmUnlock(pVM);
3113	}
3114	return rc;
3115
3116	#elif PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3117	return VINF_SUCCESS; /* ignore */
3118	#endif
3119	}
3120
3121
3122
3123
3124	/**
3125	* Syncs a page during a PGMVerifyAccess() call.
3126	*
3127	* @returns VBox status code (informational included).
3128	* @param pVCpu The VMCPU handle.
3129	* @param GCPtrPage The address of the page to sync.
3130	* @param fPage The effective guest page flags.
3131	* @param uErr The trap error code.
3132	*/
3133	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fPage, unsigned uErr)
3134	{
3135	PVM pVM = pVCpu->CTX_SUFF(pVM);
3136
3137	LogFlow(("VerifyAccessSyncPage: GCPtrPage=%RGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
3138
3139	Assert(!HWACCMIsNestedPagingActive(pVM));
3140	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_TYPE_AMD64) \
3141	&& PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
3142
3143	# ifndef IN_RING0
3144	if (!(fPage & X86_PTE_US))
3145	{
3146	/*
3147	* Mark this page as safe.
3148	*/
3149	/** @todo not correct for pages that contain both code and data!! */
3150	Log(("CSAMMarkPage %RGv; scanned=%d\n", GCPtrPage, true));
3151	CSAMMarkPage(pVM, (RTRCPTR)GCPtrPage, true);
3152	}
3153	# endif
3154
3155	/*
3156	* Get guest PD and index.
3157	*/
3158	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3159	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3160	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
3161	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
3162	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3163	unsigned iPDSrc = 0;
3164	X86PDPE PdpeSrc;
3165	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, GCPtrPage, &iPDSrc, &PdpeSrc);
3166
3167	if (pPDSrc)
3168	{
3169	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3170	return VINF_EM_RAW_GUEST_TRAP;
3171	}
3172	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3173	unsigned iPDSrc;
3174	PX86PML4E pPml4eSrc;
3175	X86PDPE PdpeSrc;
3176	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3177	if (!pPDSrc)
3178	{
3179	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3180	return VINF_EM_RAW_GUEST_TRAP;
3181	}
3182	# endif
3183	# else
3184	PGSTPD pPDSrc = NULL;
3185	const unsigned iPDSrc = 0;
3186	# endif
3187	int rc = VINF_SUCCESS;
3188
3189	pgmLock(pVM);
3190
3191	/*
3192	* First check if the shadow pd is present.
3193	*/
3194	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3195	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
3196	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3197	PX86PDEPAE pPdeDst;
3198	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3199	PX86PDPAE pPDDst;
3200	# if PGM_GST_TYPE != PGM_TYPE_PAE
3201	X86PDPE PdpeSrc;
3202
3203	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3204	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3205	# endif
3206	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, &PdpeSrc, &pPDDst);
3207	if (rc != VINF_SUCCESS)
3208	{
3209	pgmUnlock(pVM);
3210	AssertRC(rc);
3211	return rc;
3212	}
3213	Assert(pPDDst);
3214	pPdeDst = &pPDDst->a[iPDDst];
3215
3216	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3217	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3218	PX86PDPAE pPDDst;
3219	PX86PDEPAE pPdeDst;
3220
3221	# if PGM_GST_TYPE == PGM_TYPE_PROT
3222	/* AMD-V nested paging */
3223	X86PML4E Pml4eSrc;
3224	X86PDPE PdpeSrc;
3225	PX86PML4E pPml4eSrc = &Pml4eSrc;
3226
3227	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3228	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_NX \| X86_PML4E_A;
3229	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_NX \| X86_PDPE_A;
3230	# endif
3231
3232	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc, &PdpeSrc, &pPDDst);
3233	if (rc != VINF_SUCCESS)
3234	{
3235	pgmUnlock(pVM);
3236	AssertRC(rc);
3237	return rc;
3238	}
3239	Assert(pPDDst);
3240	pPdeDst = &pPDDst->a[iPDDst];
3241	# endif
3242
3243	# if defined(IN_RC)
3244	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
3245	PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
3246	# endif
3247
3248	if (!pPdeDst->n.u1Present)
3249	{
3250	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3251	if (rc != VINF_SUCCESS)
3252	{
3253	# if defined(IN_RC)
3254	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
3255	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
3256	# endif
3257	pgmUnlock(pVM);
3258	AssertRC(rc);
3259	return rc;
3260	}
3261	}
3262
3263	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3264	/* Check for dirty bit fault */
3265	rc = PGM_BTH_NAME(CheckPageFault)(pVCpu, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
3266	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
3267	Log(("PGMVerifyAccess: success (dirty)\n"));
3268	else
3269	{
3270	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3271	# else
3272	{
3273	GSTPDE PdeSrc;
3274	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
3275	PdeSrc.n.u1Present = 1;
3276	PdeSrc.n.u1Write = 1;
3277	PdeSrc.n.u1Accessed = 1;
3278	PdeSrc.n.u1User = 1;
3279
3280	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
3281	Assert(rc != VINF_EM_RAW_GUEST_TRAP);
3282	if (uErr & X86_TRAP_PF_US)
3283	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncUser));
3284	else /* supervisor */
3285	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
3286
3287	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3288	if (RT_SUCCESS(rc))
3289	{
3290	/* Page was successfully synced */
3291	Log2(("PGMVerifyAccess: success (sync)\n"));
3292	rc = VINF_SUCCESS;
3293	}
3294	else
3295	{
3296	Log(("PGMVerifyAccess: access violation for %RGv rc=%d\n", GCPtrPage, rc));
3297	rc = VINF_EM_RAW_GUEST_TRAP;
3298	}
3299	}
3300	# if defined(IN_RC)
3301	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
3302	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
3303	# endif
3304	pgmUnlock(pVM);
3305	return rc;
3306
3307	#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
3308
3309	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
3310	return VERR_INTERNAL_ERROR;
3311	#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
3312	}
3313
3314
3315	#if PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64
3316	# if PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
3317	/**
3318	* Figures out which kind of shadow page this guest PDE warrants.
3319	*
3320	* @returns Shadow page kind.
3321	* @param pPdeSrc The guest PDE in question.
3322	* @param cr4 The current guest cr4 value.
3323	*/
3324	DECLINLINE(PGMPOOLKIND) PGM_BTH_NAME(CalcPageKind)(const GSTPDE *pPdeSrc, uint32_t cr4)
3325	{
3326	# if PMG_GST_TYPE == PGM_TYPE_AMD64
3327	if (!pPdeSrc->n.u1Size)
3328	# else
3329	if (!pPdeSrc->n.u1Size \|\| !(cr4 & X86_CR4_PSE))
3330	# endif
3331	return BTH_PGMPOOLKIND_PT_FOR_PT;
3332	//switch (pPdeSrc->u & (X86_PDE4M_RW \| X86_PDE4M_US /\| X86_PDE4M_PAE_NX/))
3333	//{
3334	// case 0:
3335	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RO;
3336	// case X86_PDE4M_RW:
3337	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW;
3338	// case X86_PDE4M_US:
3339	// return BTH_PGMPOOLKIND_PT_FOR_BIG_US;
3340	// case X86_PDE4M_RW \| X86_PDE4M_US:
3341	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US;
3342	# if 0
3343	// case X86_PDE4M_PAE_NX:
3344	// return BTH_PGMPOOLKIND_PT_FOR_BIG_NX;
3345	// case X86_PDE4M_RW \| X86_PDE4M_PAE_NX:
3346	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_NX;
3347	// case X86_PDE4M_US \| X86_PDE4M_PAE_NX:
3348	// return BTH_PGMPOOLKIND_PT_FOR_BIG_US_NX;
3349	// case X86_PDE4M_RW \| X86_PDE4M_US \| X86_PDE4M_PAE_NX:
3350	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US_NX;
3351	# endif
3352	return BTH_PGMPOOLKIND_PT_FOR_BIG;
3353	//}
3354	}
3355	# endif
3356	#endif
3357
3358	#undef MY_STAM_COUNTER_INC
3359	#define MY_STAM_COUNTER_INC(a) do { } while (0)
3360
3361
3362	/**
3363	* Syncs the paging hierarchy starting at CR3.
3364	*
3365	* @returns VBox status code, no specials.
3366	* @param pVCpu The VMCPU handle.
3367	* @param cr0 Guest context CR0 register
3368	* @param cr3 Guest context CR3 register
3369	* @param cr4 Guest context CR4 register
3370	* @param fGlobal Including global page directories or not
3371	*/
3372	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
3373	{
3374	PVM pVM = pVCpu->CTX_SUFF(pVM);
3375
3376	if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
3377	fGlobal = true; /* Change this CR3 reload to be a global one. */
3378
3379	LogFlow(("SyncCR3 %d\n", fGlobal));
3380
3381	#if PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
3382	/*
3383	* Update page access handlers.
3384	* The virtual are always flushed, while the physical are only on demand.
3385	* WARNING: We are incorrectly not doing global flushing on Virtual Handler updates. We'll
3386	* have to look into that later because it will have a bad influence on the performance.
3387	* @note SvL: There's no need for that. Just invalidate the virtual range(s).
3388	* bird: Yes, but that won't work for aliases.
3389	*/
3390	/** @todo this MUST go away. See #1557. */
3391	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncCR3Handlers), h);
3392	PGM_GST_NAME(HandlerVirtualUpdate)(pVM, cr4);
3393	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncCR3Handlers), h);
3394	#endif
3395
3396	#if PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3397	/*
3398	* Nested / EPT - almost no work.
3399	*/
3400	/** @todo check if this is really necessary; the call does it as well... */
3401	HWACCMFlushTLB(pVM);
3402	return VINF_SUCCESS;
3403
3404	#elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3405	/*
3406	* AMD64 (Shw & Gst) - No need to check all paging levels; we zero
3407	* out the shadow parts when the guest modifies its tables.
3408	*/
3409	return VINF_SUCCESS;
3410
3411	#else /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3412
3413	# ifdef PGM_WITHOUT_MAPPINGS
3414	Assert(pVM->pgm.s.fMappingsFixed);
3415	return VINF_SUCCESS;
3416	# else
3417	/* Nothing to do when mappings are fixed. */
3418	if (pVM->pgm.s.fMappingsFixed)
3419	return VINF_SUCCESS;
3420
3421	int rc = PGMMapResolveConflicts(pVM);
3422	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
3423	if (rc == VINF_PGM_SYNC_CR3)
3424	{
3425	LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
3426	return VINF_PGM_SYNC_CR3;
3427	}
3428	# endif
3429	return VINF_SUCCESS;
3430	#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3431	}
3432
3433
3434
3435
3436	#ifdef VBOX_STRICT
3437	#ifdef IN_RC
3438	# undef AssertMsgFailed
3439	# define AssertMsgFailed Log
3440	#endif
3441	#ifdef IN_RING3
3442	# include <VBox/dbgf.h>
3443
3444	/**
3445	* Dumps a page table hierarchy use only physical addresses and cr4/lm flags.
3446	*
3447	* @returns VBox status code (VINF_SUCCESS).
3448	* @param cr3 The root of the hierarchy.
3449	* @param crr The cr4, only PAE and PSE is currently used.
3450	* @param fLongMode Set if long mode, false if not long mode.
3451	* @param cMaxDepth Number of levels to dump.
3452	* @param pHlp Pointer to the output functions.
3453	*/
3454	__BEGIN_DECLS
3455	VMMR3DECL(int) PGMR3DumpHierarchyHC(PVM pVM, uint32_t cr3, uint32_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp);
3456	__END_DECLS
3457
3458	#endif
3459
3460	/**
3461	* Checks that the shadow page table is in sync with the guest one.
3462	*
3463	* @returns The number of errors.
3464	* @param pVM The virtual machine.
3465	* @param pVCpu The VMCPU handle.
3466	* @param cr3 Guest context CR3 register
3467	* @param cr4 Guest context CR4 register
3468	* @param GCPtr Where to start. Defaults to 0.
3469	* @param cb How much to check. Defaults to everything.
3470	*/
3471	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)
3472	{
3473	#if PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3474	return 0;
3475	#else
3476	unsigned cErrors = 0;
3477	PVM pVM = pVCpu->CTX_SUFF(pVM);
3478	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3479
3480	#if PGM_GST_TYPE == PGM_TYPE_PAE
3481	/** @todo currently broken; crashes below somewhere */
3482	AssertFailed();
3483	#endif
3484
3485	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
3486	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3487	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
3488
3489	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3490	bool fBigPagesSupported = true;
3491	# else
3492	bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
3493	# endif
3494	PPGMCPU pPGM = &pVCpu->pgm.s;
3495	RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
3496	RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
3497	# ifndef IN_RING0
3498	RTHCPHYS HCPhys; /* general usage. */
3499	# endif
3500	int rc;
3501
3502	/*
3503	* Check that the Guest CR3 and all its mappings are correct.
3504	*/
3505	AssertMsgReturn(pPGM->GCPhysCR3 == (cr3 & GST_CR3_PAGE_MASK),
3506	("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
3507	false);
3508	# if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64
3509	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3510	rc = PGMShwGetPage(pVCpu, (RTGCPTR)pPGM->pGst32BitPdRC, NULL, &HCPhysShw);
3511	# else
3512	rc = PGMShwGetPage(pVCpu, (RTGCPTR)pPGM->pGstPaePdptRC, NULL, &HCPhysShw);
3513	# endif
3514	AssertRCReturn(rc, 1);
3515	HCPhys = NIL_RTHCPHYS;
3516	rc = pgmRamGCPhys2HCPhys(&pVM->pgm.s, cr3 & GST_CR3_PAGE_MASK, &HCPhys);
3517	AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
3518	# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
3519	pgmGstGet32bitPDPtr(pPGM);
3520	RTGCPHYS GCPhys;
3521	rc = PGMR3DbgR3Ptr2GCPhys(pVM, pPGM->pGst32BitPdR3, &GCPhys);
3522	AssertRCReturn(rc, 1);
3523	AssertMsgReturn((cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
3524	# endif
3525	# endif /* !IN_RING0 */
3526
3527	/*
3528	* Get and check the Shadow CR3.
3529	*/
3530	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3531	unsigned cPDEs = X86_PG_ENTRIES;
3532	unsigned cIncrement = X86_PG_ENTRIES * PAGE_SIZE;
3533	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3534	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3535	unsigned cPDEs = X86_PG_PAE_ENTRIES * 4; /* treat it as a 2048 entry table. */
3536	# else
3537	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3538	# endif
3539	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3540	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3541	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3542	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3543	# endif
3544	if (cb != ~(RTGCPTR)0)
3545	cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
3546
3547	/** @todo call the other two PGMAssert() functions. /
3548
3549	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3550	unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3551
3552	for (; iPml4 < X86_PG_PAE_ENTRIES; iPml4++)
3553	{
3554	PPGMPOOLPAGE pShwPdpt = NULL;
3555	PX86PML4E pPml4eSrc;
3556	PX86PML4E pPml4eDst;
3557	RTGCPHYS GCPhysPdptSrc;
3558
3559	pPml4eSrc = pgmGstGetLongModePML4EPtr(&pVCpu->pgm.s, iPml4);
3560	pPml4eDst = pgmShwGetLongModePML4EPtr(&pVCpu->pgm.s, iPml4);
3561
3562	/* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
3563	if (!pPml4eDst->n.u1Present)
3564	{
3565	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3566	continue;
3567	}
3568
3569	pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
3570	GCPhysPdptSrc = pPml4eSrc->u & X86_PML4E_PG_MASK_FULL;
3571
3572	if (pPml4eSrc->n.u1Present != pPml4eDst->n.u1Present)
3573	{
3574	AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3575	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3576	cErrors++;
3577	continue;
3578	}
3579
3580	if (GCPhysPdptSrc != pShwPdpt->GCPhys)
3581	{
3582	AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
3583	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3584	cErrors++;
3585	continue;
3586	}
3587
3588	if ( pPml4eDst->n.u1User != pPml4eSrc->n.u1User
3589	\|\| pPml4eDst->n.u1Write != pPml4eSrc->n.u1Write
3590	\|\| pPml4eDst->n.u1NoExecute != pPml4eSrc->n.u1NoExecute)
3591	{
3592	AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3593	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3594	cErrors++;
3595	continue;
3596	}
3597	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3598	{
3599	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3600
3601	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
3602	/*
3603	* Check the PDPTEs too.
3604	*/
3605	unsigned iPdpt = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
3606
3607	for (;iPdpt <= SHW_PDPT_MASK; iPdpt++)
3608	{
3609	unsigned iPDSrc;
3610	PPGMPOOLPAGE pShwPde = NULL;
3611	PX86PDPE pPdpeDst;
3612	RTGCPHYS GCPhysPdeSrc;
3613	# if PGM_GST_TYPE == PGM_TYPE_PAE
3614	X86PDPE PdpeSrc;
3615	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, GCPtr, &iPDSrc, &PdpeSrc);
3616	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(&pVCpu->pgm.s);
3617	# else
3618	PX86PML4E pPml4eSrc;
3619	X86PDPE PdpeSrc;
3620	PX86PDPT pPdptDst;
3621	PX86PDPAE pPDDst;
3622	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, GCPtr, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3623
3624	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtr, NULL, &pPdptDst, &pPDDst);
3625	if (rc != VINF_SUCCESS)
3626	{
3627	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
3628	GCPtr += 512 * _2M;
3629	continue; /* next PDPTE */
3630	}
3631	Assert(pPDDst);
3632	# endif
3633	Assert(iPDSrc == 0);
3634
3635	pPdpeDst = &pPdptDst->a[iPdpt];
3636
3637	if (!pPdpeDst->n.u1Present)
3638	{
3639	GCPtr += 512 * _2M;
3640	continue; /* next PDPTE */
3641	}
3642
3643	pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
3644	GCPhysPdeSrc = PdpeSrc.u & X86_PDPE_PG_MASK;
3645
3646	if (pPdpeDst->n.u1Present != PdpeSrc.n.u1Present)
3647	{
3648	AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3649	GCPtr += 512 * _2M;
3650	cErrors++;
3651	continue;
3652	}
3653
3654	if (GCPhysPdeSrc != pShwPde->GCPhys)
3655	{
3656	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3657	AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3658	# else
3659	AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3660	# endif
3661	GCPtr += 512 * _2M;
3662	cErrors++;
3663	continue;
3664	}
3665
3666	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3667	if ( pPdpeDst->lm.u1User != PdpeSrc.lm.u1User
3668	\|\| pPdpeDst->lm.u1Write != PdpeSrc.lm.u1Write
3669	\|\| pPdpeDst->lm.u1NoExecute != PdpeSrc.lm.u1NoExecute)
3670	{
3671	AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3672	GCPtr += 512 * _2M;
3673	cErrors++;
3674	continue;
3675	}
3676	# endif
3677
3678	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3679	{
3680	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3681	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3682	GSTPD const *pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
3683	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3684	PCX86PD pPDDst = pgmShwGet32BitPDPtr(&pVCpu->pgm.s);
3685	# endif
3686	# endif /* PGM_GST_TYPE == PGM_TYPE_32BIT */
3687	/*
3688	* Iterate the shadow page directory.
3689	*/
3690	GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
3691	unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
3692
3693	for (;
3694	iPDDst < cPDEs;
3695	iPDDst++, GCPtr += cIncrement)
3696	{
3697	# if PGM_SHW_TYPE == PGM_TYPE_PAE
3698	const SHWPDE PdeDst = *pgmShwGetPaePDEPtr(pPGM, GCPtr);
3699	# else
3700	const SHWPDE PdeDst = pPDDst->a[iPDDst];
3701	# endif
3702	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
3703	{
3704	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
3705	if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
3706	{
3707	AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
3708	cErrors++;
3709	continue;
3710	}
3711	}
3712	else if ( (PdeDst.u & X86_PDE_P)
3713	\|\| ((PdeDst.u & (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY))
3714	)
3715	{
3716	HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
3717	PPGMPOOLPAGE pPoolPage = pgmPoolGetPage(pPool, HCPhysShw);
3718	if (!pPoolPage)
3719	{
3720	AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n",
3721	HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
3722	cErrors++;
3723	continue;
3724	}
3725	const SHWPT pPTDst = (const SHWPT )PGMPOOL_PAGE_2_PTR(pVM, pPoolPage);
3726
3727	if (PdeDst.u & (X86_PDE4M_PWT \| X86_PDE4M_PCD))
3728	{
3729	AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
3730	GCPtr, (uint64_t)PdeDst.u));
3731	cErrors++;
3732	}
3733
3734	if (PdeDst.u & (X86_PDE4M_G \| X86_PDE4M_D))
3735	{
3736	AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n",
3737	GCPtr, (uint64_t)PdeDst.u));
3738	cErrors++;
3739	}
3740
3741	const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
3742	if (!PdeSrc.n.u1Present)
3743	{
3744	AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
3745	GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
3746	cErrors++;
3747	continue;
3748	}
3749
3750	if ( !PdeSrc.b.u1Size
3751	\|\| !fBigPagesSupported)
3752	{
3753	GCPhysGst = PdeSrc.u & GST_PDE_PG_MASK;
3754	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3755	GCPhysGst \|= (iPDDst & 1) * (PAGE_SIZE / 2);
3756	# endif
3757	}
3758	else
3759	{
3760	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3761	if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
3762	{
3763	AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n",
3764	GCPtr, (uint64_t)PdeSrc.u));
3765	cErrors++;
3766	continue;
3767	}
3768	# endif
3769	GCPhysGst = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
3770	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3771	GCPhysGst \|= GCPtr & RT_BIT(X86_PAGE_2M_SHIFT);
3772	# endif
3773	}
3774
3775	if ( pPoolPage->enmKind
3776	!= (!PdeSrc.b.u1Size \|\| !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
3777	{
3778	AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n",
3779	pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
3780	cErrors++;
3781	}
3782
3783	PPGMPAGE pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
3784	if (!pPhysPage)
3785	{
3786	AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3787	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3788	cErrors++;
3789	continue;
3790	}
3791
3792	if (GCPhysGst != pPoolPage->GCPhys)
3793	{
3794	AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n",
3795	GCPhysGst, pPoolPage->GCPhys, GCPtr));
3796	cErrors++;
3797	continue;
3798	}
3799
3800	if ( !PdeSrc.b.u1Size
3801	\|\| !fBigPagesSupported)
3802	{
3803	/*
3804	* Page Table.
3805	*/
3806	const GSTPT *pPTSrc;
3807	rc = PGM_GCPHYS_2_PTR(pVM, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1), &pPTSrc);
3808	if (RT_FAILURE(rc))
3809	{
3810	AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3811	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3812	cErrors++;
3813	continue;
3814	}
3815	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/))
3816	!= (PdeDst.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/)))
3817	{
3818	/// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
3819	// (This problem will go away when/if we shadow multiple CR3s.)
3820	AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3821	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3822	cErrors++;
3823	continue;
3824	}
3825	if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3826	{
3827	AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n",
3828	GCPtr, (uint64_t)PdeDst.u));
3829	cErrors++;
3830	continue;
3831	}
3832
3833	/* iterate the page table. */
3834	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3835	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3836	const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
3837	# else
3838	const unsigned offPTSrc = 0;
3839	# endif
3840	for (unsigned iPT = 0, off = 0;
3841	iPT < RT_ELEMENTS(pPTDst->a);
3842	iPT++, off += PAGE_SIZE)
3843	{
3844	const SHWPTE PteDst = pPTDst->a[iPT];
3845
3846	/* skip not-present entries. */
3847	if (!(PteDst.u & (X86_PTE_P \| PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
3848	continue;
3849	Assert(PteDst.n.u1Present);
3850
3851	const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
3852	if (!PteSrc.n.u1Present)
3853	{
3854	# ifdef IN_RING3
3855	PGMAssertHandlerAndFlagsInSync(pVM);
3856	PGMR3DumpHierarchyGC(pVM, cr3, cr4, (PdeSrc.u & GST_PDE_PG_MASK));
3857	# endif
3858	AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
3859	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u, pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
3860	(PdeSrc.u & GST_PDE_PG_MASK) + (iPT + offPTSrc)*sizeof(PteSrc)));
3861	cErrors++;
3862	continue;
3863	}
3864
3865	uint64_t fIgnoreFlags = GST_PTE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_G \| X86_PTE_D \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
3866	# if 1 /** @todo sync accessed bit properly... */
3867	fIgnoreFlags \|= X86_PTE_A;
3868	# endif
3869
3870	/* match the physical addresses */
3871	HCPhysShw = PteDst.u & SHW_PTE_PG_MASK;
3872	GCPhysGst = PteSrc.u & GST_PTE_PG_MASK;
3873
3874	# ifdef IN_RING3
3875	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3876	if (RT_FAILURE(rc))
3877	{
3878	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
3879	{
3880	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3881	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3882	cErrors++;
3883	continue;
3884	}
3885	}
3886	else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
3887	{
3888	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3889	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3890	cErrors++;
3891	continue;
3892	}
3893	# endif
3894
3895	pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
3896	if (!pPhysPage)
3897	{
3898	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
3899	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
3900	{
3901	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3902	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3903	cErrors++;
3904	continue;
3905	}
3906	# endif
3907	if (PteDst.n.u1Write)
3908	{
3909	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3910	GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3911	cErrors++;
3912	}
3913	fIgnoreFlags \|= X86_PTE_RW;
3914	}
3915	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
3916	{
3917	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3918	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3919	cErrors++;
3920	continue;
3921	}
3922
3923	/* flags */
3924	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
3925	{
3926	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
3927	{
3928	if (PteDst.n.u1Write)
3929	{
3930	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
3931	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3932	cErrors++;
3933	continue;
3934	}
3935	fIgnoreFlags \|= X86_PTE_RW;
3936	}
3937	else
3938	{
3939	if (PteDst.n.u1Present)
3940	{
3941	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
3942	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3943	cErrors++;
3944	continue;
3945	}
3946	fIgnoreFlags \|= X86_PTE_P;
3947	}
3948	}
3949	else
3950	{
3951	if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
3952	{
3953	if (PteDst.n.u1Write)
3954	{
3955	AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
3956	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3957	cErrors++;
3958	continue;
3959	}
3960	if (!(PteDst.u & PGM_PTFLAGS_TRACK_DIRTY))
3961	{
3962	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3963	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3964	cErrors++;
3965	continue;
3966	}
3967	if (PteDst.n.u1Dirty)
3968	{
3969	AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3970	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3971	cErrors++;
3972	}
3973	# if 0 /** @todo sync access bit properly... */
3974	if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
3975	{
3976	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
3977	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3978	cErrors++;
3979	}
3980	fIgnoreFlags \|= X86_PTE_RW;
3981	# else
3982	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
3983	# endif
3984	}
3985	else if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
3986	{
3987	/* access bit emulation (not implemented). */
3988	if (PteSrc.n.u1Accessed \|\| PteDst.n.u1Present)
3989	{
3990	AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
3991	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3992	cErrors++;
3993	continue;
3994	}
3995	if (!PteDst.n.u1Accessed)
3996	{
3997	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
3998	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3999	cErrors++;
4000	}
4001	fIgnoreFlags \|= X86_PTE_P;
4002	}
4003	# ifdef DEBUG_sandervl
4004	fIgnoreFlags \|= X86_PTE_D \| X86_PTE_A;
4005	# endif
4006	}
4007
4008	if ( (PteSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
4009	&& (PteSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags)
4010	)
4011	{
4012	AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
4013	GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
4014	fIgnoreFlags, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4015	cErrors++;
4016	continue;
4017	}
4018	} /* foreach PTE */
4019	}
4020	else
4021	{
4022	/*
4023	* Big Page.
4024	*/
4025	uint64_t fIgnoreFlags = X86_PDE_AVL_MASK \| GST_PDE_PG_MASK \| X86_PDE4M_G \| X86_PDE4M_D \| X86_PDE4M_PS \| X86_PDE4M_PWT \| X86_PDE4M_PCD;
4026	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
4027	{
4028	if (PdeDst.n.u1Write)
4029	{
4030	AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4031	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4032	cErrors++;
4033	continue;
4034	}
4035	if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
4036	{
4037	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4038	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4039	cErrors++;
4040	continue;
4041	}
4042	# if 0 /** @todo sync access bit properly... */
4043	if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
4044	{
4045	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4046	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4047	cErrors++;
4048	}
4049	fIgnoreFlags \|= X86_PTE_RW;
4050	# else
4051	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
4052	# endif
4053	}
4054	else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4055	{
4056	/* access bit emulation (not implemented). */
4057	if (PdeSrc.b.u1Accessed \|\| PdeDst.n.u1Present)
4058	{
4059	AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4060	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4061	cErrors++;
4062	continue;
4063	}
4064	if (!PdeDst.n.u1Accessed)
4065	{
4066	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4067	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4068	cErrors++;
4069	}
4070	fIgnoreFlags \|= X86_PTE_P;
4071	}
4072
4073	if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
4074	{
4075	AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
4076	GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
4077	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4078	cErrors++;
4079	}
4080
4081	/* iterate the page table. */
4082	for (unsigned iPT = 0, off = 0;
4083	iPT < RT_ELEMENTS(pPTDst->a);
4084	iPT++, off += PAGE_SIZE, GCPhysGst += PAGE_SIZE)
4085	{
4086	const SHWPTE PteDst = pPTDst->a[iPT];
4087
4088	if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
4089	{
4090	AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
4091	GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4092	cErrors++;
4093	}
4094
4095	/* skip not-present entries. */
4096	if (!PteDst.n.u1Present) /** @todo deal with ALL handlers and CSAM !P pages! */
4097	continue;
4098
4099	fIgnoreFlags = X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT \| X86_PTE_D \| X86_PTE_A \| X86_PTE_G \| X86_PTE_PAE_NX;
4100
4101	/* match the physical addresses */
4102	HCPhysShw = PteDst.u & X86_PTE_PAE_PG_MASK;
4103
4104	# ifdef IN_RING3
4105	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4106	if (RT_FAILURE(rc))
4107	{
4108	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4109	{
4110	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4111	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4112	cErrors++;
4113	}
4114	}
4115	else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
4116	{
4117	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4118	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4119	cErrors++;
4120	continue;
4121	}
4122	# endif
4123	pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
4124	if (!pPhysPage)
4125	{
4126	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4127	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4128	{
4129	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4130	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4131	cErrors++;
4132	continue;
4133	}
4134	# endif
4135	if (PteDst.n.u1Write)
4136	{
4137	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4138	GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4139	cErrors++;
4140	}
4141	fIgnoreFlags \|= X86_PTE_RW;
4142	}
4143	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4144	{
4145	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4146	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4147	cErrors++;
4148	continue;
4149	}
4150
4151	/* flags */
4152	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4153	{
4154	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4155	{
4156	if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
4157	{
4158	if (PteDst.n.u1Write)
4159	{
4160	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4161	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4162	cErrors++;
4163	continue;
4164	}
4165	fIgnoreFlags \|= X86_PTE_RW;
4166	}
4167	}
4168	else
4169	{
4170	if (PteDst.n.u1Present)
4171	{
4172	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4173	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4174	cErrors++;
4175	continue;
4176	}
4177	fIgnoreFlags \|= X86_PTE_P;
4178	}
4179	}
4180
4181	if ( (PdeSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
4182	&& (PdeSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags) /* lazy phys handler dereg. */
4183	)
4184	{
4185	AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
4186	GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
4187	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4188	cErrors++;
4189	continue;
4190	}
4191	} /* for each PTE */
4192	}
4193	}
4194	/* not present */
4195
4196	} /* for each PDE */
4197
4198	} /* for each PDPTE */
4199
4200	} /* for each PML4E */
4201
4202	# ifdef DEBUG
4203	if (cErrors)
4204	LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
4205	# endif
4206
4207	#endif /* GST == 32BIT, PAE or AMD64 */
4208	return cErrors;
4209
4210	#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT */
4211	}
4212	#endif /* VBOX_STRICT */
4213
4214
4215	/**
4216	* Sets up the CR3 for shadow paging
4217	*
4218	* @returns Strict VBox status code.
4219	* @retval VINF_SUCCESS.
4220	*
4221	* @param pVCpu The VMCPU handle.
4222	* @param GCPhysCR3 The physical address in the CR3 register.
4223	*/
4224	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3)
4225	{
4226	PVM pVM = pVCpu->CTX_SUFF(pVM);
4227
4228	/* Update guest paging info. */
4229	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
4230	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
4231	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
4232
4233	LogFlow(("MapCR3: %RGp\n", GCPhysCR3));
4234
4235	/*
4236	* Map the page CR3 points at.
4237	*/
4238	RTHCPTR HCPtrGuestCR3;
4239	RTHCPHYS HCPhysGuestCR3;
4240	pgmLock(pVM);
4241	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysCR3);
4242	AssertReturn(pPage, VERR_INTERNAL_ERROR_2);
4243	HCPhysGuestCR3 = PGM_PAGE_GET_HCPHYS(pPage);
4244	/** @todo this needs some reworking wrt. locking. */
4245	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4246	HCPtrGuestCR3 = NIL_RTHCPTR;
4247	int rc = VINF_SUCCESS;
4248	# else
4249	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhysCR3 & GST_CR3_PAGE_MASK, (void **)&HCPtrGuestCR3);
4250	# endif
4251	pgmUnlock(pVM);
4252	if (RT_SUCCESS(rc))
4253	{
4254	rc = PGMMap(pVM, (RTGCPTR)pVM->pgm.s.GCPtrCR3Mapping, HCPhysGuestCR3, PAGE_SIZE, 0);
4255	if (RT_SUCCESS(rc))
4256	{
4257	# ifdef IN_RC
4258	PGM_INVL_PG(pVM->pgm.s.GCPtrCR3Mapping);
4259	# endif
4260	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4261	pVCpu->pgm.s.pGst32BitPdR3 = (R3PTRTYPE(PX86PD))HCPtrGuestCR3;
4262	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4263	pVCpu->pgm.s.pGst32BitPdR0 = (R0PTRTYPE(PX86PD))HCPtrGuestCR3;
4264	# endif
4265	pVCpu->pgm.s.pGst32BitPdRC = (RCPTRTYPE(PX86PD))pVM->pgm.s.GCPtrCR3Mapping;
4266
4267	# elif PGM_GST_TYPE == PGM_TYPE_PAE
4268	unsigned off = GCPhysCR3 & GST_CR3_PAGE_MASK & PAGE_OFFSET_MASK;
4269	pVCpu->pgm.s.pGstPaePdptR3 = (R3PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4270	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4271	pVCpu->pgm.s.pGstPaePdptR0 = (R0PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4272	# endif
4273	pVCpu->pgm.s.pGstPaePdptRC = (RCPTRTYPE(PX86PDPT))((RCPTRTYPE(uint8_t *))pVM->pgm.s.GCPtrCR3Mapping + off);
4274	Log(("Cached mapping %RRv\n", pVCpu->pgm.s.pGstPaePdptRC));
4275
4276	/*
4277	* Map the 4 PDs too.
4278	*/
4279	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(&pVCpu->pgm.s);
4280	RTGCPTR GCPtr = pVM->pgm.s.GCPtrCR3Mapping + PAGE_SIZE;
4281	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++, GCPtr += PAGE_SIZE)
4282	{
4283	if (pGuestPDPT->a[i].n.u1Present)
4284	{
4285	RTHCPTR HCPtr;
4286	RTHCPHYS HCPhys;
4287	RTGCPHYS GCPhys = pGuestPDPT->a[i].u & X86_PDPE_PG_MASK;
4288	pgmLock(pVM);
4289	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
4290	AssertReturn(pPage, VERR_INTERNAL_ERROR_2);
4291	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
4292	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4293	HCPtr = NIL_RTHCPTR;
4294	int rc2 = VINF_SUCCESS;
4295	# else
4296	int rc2 = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, (void **)&HCPtr);
4297	# endif
4298	pgmUnlock(pVM);
4299	if (RT_SUCCESS(rc2))
4300	{
4301	rc = PGMMap(pVM, GCPtr, HCPhys, PAGE_SIZE, 0);
4302	AssertRCReturn(rc, rc);
4303
4304	pVCpu->pgm.s.apGstPaePDsR3[i] = (R3PTRTYPE(PX86PDPAE))HCPtr;
4305	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4306	pVCpu->pgm.s.apGstPaePDsR0[i] = (R0PTRTYPE(PX86PDPAE))HCPtr;
4307	# endif
4308	pVCpu->pgm.s.apGstPaePDsRC[i] = (RCPTRTYPE(PX86PDPAE))GCPtr;
4309	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
4310	PGM_INVL_PG(GCPtr); /** @todo This ends up calling HWACCMInvalidatePage, is that correct? */
4311	continue;
4312	}
4313	AssertMsgFailed(("pgmR3Gst32BitMapCR3: rc2=%d GCPhys=%RGp i=%d\n", rc2, GCPhys, i));
4314	}
4315
4316	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4317	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4318	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4319	# endif
4320	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4321	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4322	PGM_INVL_PG(GCPtr); /** @todo this shouldn't be necessary? */
4323	}
4324
4325	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
4326	pVCpu->pgm.s.pGstAmd64Pml4R3 = (R3PTRTYPE(PX86PML4))HCPtrGuestCR3;
4327	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4328	pVCpu->pgm.s.pGstAmd64Pml4R0 = (R0PTRTYPE(PX86PML4))HCPtrGuestCR3;
4329	# endif
4330	# endif
4331	}
4332	else
4333	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4334	}
4335	else
4336	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4337
4338	#else /* prot/real stub */
4339	int rc = VINF_SUCCESS;
4340	#endif
4341
4342	/* Update shadow paging info for guest modes with paging (32, pae, 64). */
4343	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4344	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4345	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64) \
4346	&& ( PGM_GST_TYPE != PGM_TYPE_REAL \
4347	&& PGM_GST_TYPE != PGM_TYPE_PROT))
4348
4349	Assert(!HWACCMIsNestedPagingActive(pVM));
4350
4351	/*
4352	* Update the shadow root page as well since that's not fixed.
4353	*/
4354	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4355	PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
4356	uint32_t iOldShwUserTable = pVCpu->pgm.s.iShwUserTable;
4357	uint32_t iOldShwUser = pVCpu->pgm.s.iShwUser;
4358	PPGMPOOLPAGE pNewShwPageCR3;
4359
4360	Assert(!(GCPhysCR3 >> (PAGE_SHIFT + 32)));
4361	rc = pgmPoolAlloc(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, SHW_POOL_ROOT_IDX, GCPhysCR3 >> PAGE_SHIFT, &pNewShwPageCR3);
4362	AssertFatalRC(rc);
4363	rc = VINF_SUCCESS;
4364
4365	/* Mark the page as locked; disallow flushing. */
4366	pgmPoolLockPage(pPool, pNewShwPageCR3);
4367
4368	# ifdef IN_RC
4369	/* NOTE: We can't deal with jumps to ring 3 here as we're now in an inconsistent state! */
4370	bool fLog = VMMGCLogDisable(pVM);
4371	pgmLock(pVM);
4372	# endif
4373
4374	pVCpu->pgm.s.iShwUser = SHW_POOL_ROOT_IDX;
4375	pVCpu->pgm.s.iShwUserTable = GCPhysCR3 >> PAGE_SHIFT;
4376	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3) = pNewShwPageCR3;
4377	# ifdef IN_RING0
4378	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4379	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4380	# elif defined(IN_RC)
4381	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4382	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4383	# else
4384	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4385	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4386	# endif
4387
4388	# ifndef PGM_WITHOUT_MAPPINGS
4389	/*
4390	* Apply all hypervisor mappings to the new CR3.
4391	* Note that SyncCR3 will be executed in case CR3 is changed in a guest paging mode; this will
4392	* make sure we check for conflicts in the new CR3 root.
4393	*/
4394	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4395	Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL) \|\| VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
4396	# endif
4397	rc = pgmMapActivateCR3(pVM, pNewShwPageCR3);
4398	AssertRCReturn(rc, rc);
4399	# endif
4400
4401	/* Set the current hypervisor CR3. */
4402	CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
4403	SELMShadowCR3Changed(pVM, pVCpu);
4404
4405	# ifdef IN_RC
4406	pgmUnlock(pVM);
4407	VMMGCLogRestore(pVM, fLog);
4408	# endif
4409
4410	/* Clean up the old CR3 root. */
4411	if (pOldShwPageCR3)
4412	{
4413	Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
4414	# ifndef PGM_WITHOUT_MAPPINGS
4415	/* Remove the hypervisor mappings from the shadow page table. */
4416	pgmMapDeactivateCR3(pVM, pOldShwPageCR3);
4417	# endif
4418	/* Mark the page as unlocked; allow flushing again. */
4419	pgmPoolUnlockPage(pPool, pOldShwPageCR3);
4420
4421	pgmPoolFreeByPage(pPool, pOldShwPageCR3, iOldShwUser, iOldShwUserTable);
4422	}
4423
4424	# endif
4425
4426	return rc;
4427	}
4428
4429	/**
4430	* Unmaps the shadow CR3.
4431	*
4432	* @returns VBox status, no specials.
4433	* @param pVCpu The VMCPU handle.
4434	*/
4435	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu)
4436	{
4437	LogFlow(("UnmapCR3\n"));
4438
4439	int rc = VINF_SUCCESS;
4440	PVM pVM = pVCpu->CTX_SUFF(pVM);
4441
4442	/*
4443	* Update guest paging info.
4444	*/
4445	#if PGM_GST_TYPE == PGM_TYPE_32BIT
4446	pVCpu->pgm.s.pGst32BitPdR3 = 0;
4447	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4448	pVCpu->pgm.s.pGst32BitPdR0 = 0;
4449	# endif
4450	pVCpu->pgm.s.pGst32BitPdRC = 0;
4451
4452	#elif PGM_GST_TYPE == PGM_TYPE_PAE
4453	pVCpu->pgm.s.pGstPaePdptR3 = 0;
4454	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4455	pVCpu->pgm.s.pGstPaePdptR0 = 0;
4456	# endif
4457	pVCpu->pgm.s.pGstPaePdptRC = 0;
4458	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4459	{
4460	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4461	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4462	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4463	# endif
4464	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4465	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4466	}
4467
4468	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
4469	pVCpu->pgm.s.pGstAmd64Pml4R3 = 0;
4470	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4471	pVCpu->pgm.s.pGstAmd64Pml4R0 = 0;
4472	# endif
4473
4474	#else /* prot/real mode stub */
4475	/* nothing to do */
4476	#endif
4477
4478	#if !defined(IN_RC) /* In RC we rely on MapCR3 to do the shadow part for us at a safe time */
4479	/*
4480	* Update shadow paging info.
4481	*/
4482	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4483	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4484	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64))
4485
4486	# if PGM_GST_TYPE != PGM_TYPE_REAL
4487	Assert(!HWACCMIsNestedPagingActive(pVM));
4488	# endif
4489
4490	# ifndef PGM_WITHOUT_MAPPINGS
4491	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4492	/* Remove the hypervisor mappings from the shadow page table. */
4493	pgmMapDeactivateCR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4494	# endif
4495
4496	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4497	{
4498	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4499
4500	Assert(pVCpu->pgm.s.iShwUser != PGMPOOL_IDX_NESTED_ROOT);
4501
4502	/* Mark the page as unlocked; allow flushing again. */
4503	pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4504
4505	pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), pVCpu->pgm.s.iShwUser, pVCpu->pgm.s.iShwUserTable);
4506	pVCpu->pgm.s.pShwPageCR3R3 = 0;
4507	pVCpu->pgm.s.pShwPageCR3R0 = 0;
4508	pVCpu->pgm.s.pShwPageCR3RC = 0;
4509	pVCpu->pgm.s.iShwUser = 0;
4510	pVCpu->pgm.s.iShwUserTable = 0;
4511	}
4512	# endif
4513	#endif /* !IN_RC*/
4514
4515	return rc;
4516	}
4517

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 19784

Download in other formats: