PGMAllBth.h@ 19668

Last change on this file since 19668 was 19329, checked in by vboxsync, 16 years ago
Removed pgmPoolGetPageByHCPhys.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 189.7 KB

Line
1	/* $Id: PGMAllBth.h 19329 2009-05-04 14:59:32Z vboxsync $ */
2	/** @file
3	* VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4	*
5	* This file is a big challenge!
6	*/
7
8	/*
9	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
10	*
11	* This file is part of VirtualBox Open Source Edition (OSE), as
12	* available from http://www.virtualbox.org. This file is free software;
13	* you can redistribute it and/or modify it under the terms of the GNU
14	* General Public License (GPL) as published by the Free Software
15	* Foundation, in version 2 as it comes in the "COPYING" file of the
16	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
17	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
18	*
19	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
20	* Clara, CA 95054 USA or visit http://www.sun.com if you need
21	* additional information or have any questions.
22	*/
23
24	/*******************************************************************************
25	* Internal Functions *
26	*******************************************************************************/
27	__BEGIN_DECLS
28	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault);
29	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
30	PGM_BTH_DECL(int, SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr);
31	PGM_BTH_DECL(int, CheckPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCPTR GCPtrPage);
32	PGM_BTH_DECL(int, SyncPT)(PVMCPU pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage);
33	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uErr);
34	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
35	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
36	#ifdef VBOX_STRICT
37	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
38	#endif
39	#ifdef PGMPOOL_WITH_USER_TRACKING
40	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys);
41	#endif
42	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3);
43	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu);
44	__END_DECLS
45
46
47	/* Filter out some illegal combinations of guest and shadow paging, so we can remove redundant checks inside functions. */
48	#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
49	# error "Invalid combination; PAE guest implies PAE shadow"
50	#endif
51
52	#if (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
53	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT)
54	# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
55	#endif
56
57	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) \
58	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT)
59	# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
60	#endif
61
62	#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT) \
63	\|\| (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
64	# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
65	#endif
66
67	#ifdef IN_RING0 /* no mappings in VT-x and AMD-V mode */
68	# define PGM_WITHOUT_MAPPINGS
69	#endif
70
71
72	#ifndef IN_RING3
73	/**
74	* #PF Handler for raw-mode guest execution.
75	*
76	* @returns VBox status code (appropriate for trap handling and GC return).
77	*
78	* @param pVCpu VMCPU Handle.
79	* @param uErr The trap error code.
80	* @param pRegFrame Trap register frame.
81	* @param pvFault The fault address.
82	*/
83	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
84	{
85	PVM pVM = pVCpu->CTX_SUFF(pVM);
86
87	# if defined(IN_RC) && defined(VBOX_STRICT)
88	PGMDynCheckLocks(pVM);
89	# endif
90
91	# if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
92	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
93	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT)
94
95	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE != PGM_TYPE_PAE
96	/*
97	* Hide the instruction fetch trap indicator for now.
98	*/
99	/** @todo NXE will change this and we must fix NXE in the switcher too! */
100	if (uErr & X86_TRAP_PF_ID)
101	{
102	uErr &= ~X86_TRAP_PF_ID;
103	TRPMSetErrorCode(pVCpu, uErr);
104	}
105	# endif
106
107	/*
108	* Get PDs.
109	*/
110	int rc;
111	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
112	# if PGM_GST_TYPE == PGM_TYPE_32BIT
113	const unsigned iPDSrc = pvFault >> GST_PD_SHIFT;
114	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
115
116	# elif PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64
117
118	# if PGM_GST_TYPE == PGM_TYPE_PAE
119	unsigned iPDSrc;
120	X86PDPE PdpeSrc;
121	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, pvFault, &iPDSrc, &PdpeSrc);
122
123	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
124	unsigned iPDSrc;
125	PX86PML4E pPml4eSrc;
126	X86PDPE PdpeSrc;
127	PGSTPD pPDSrc;
128
129	pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, pvFault, &pPml4eSrc, &PdpeSrc, &iPDSrc);
130	Assert(pPml4eSrc);
131	# endif
132
133	/* Quick check for a valid guest trap. (PAE & AMD64) */
134	if (!pPDSrc)
135	{
136	# if PGM_GST_TYPE == PGM_TYPE_AMD64 && GC_ARCH_BITS == 64
137	LogFlow(("Trap0eHandler: guest PML4 %d not present CR3=%RGp\n", (int)((pvFault >> X86_PML4_SHIFT) & X86_PML4_MASK), CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK));
138	# else
139	LogFlow(("Trap0eHandler: guest iPDSrc=%u not present CR3=%RGp\n", iPDSrc, CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK));
140	# endif
141	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2GuestTrap; });
142	TRPMSetErrorCode(pVCpu, uErr);
143	return VINF_EM_RAW_GUEST_TRAP;
144	}
145	# endif
146
147	# else /* !PGM_WITH_PAGING */
148	PGSTPD pPDSrc = NULL;
149	const unsigned iPDSrc = 0;
150	# endif /* !PGM_WITH_PAGING */
151
152
153	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
154	const unsigned iPDDst = pvFault >> SHW_PD_SHIFT;
155	PX86PD pPDDst = pgmShwGet32BitPDPtr(&pVCpu->pgm.s);
156
157	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
158	const unsigned iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK; /* pPDDst index, not used with the pool. */
159
160	PX86PDPAE pPDDst;
161	# if PGM_GST_TYPE != PGM_TYPE_PAE
162	X86PDPE PdpeSrc;
163
164	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
165	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
166	# endif
167	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, &PdpeSrc, &pPDDst);
168	if (rc != VINF_SUCCESS)
169	{
170	AssertRC(rc);
171	return rc;
172	}
173	Assert(pPDDst);
174
175	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
176	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
177	PX86PDPAE pPDDst;
178	# if PGM_GST_TYPE == PGM_TYPE_PROT
179	/* AMD-V nested paging */
180	X86PML4E Pml4eSrc;
181	X86PDPE PdpeSrc;
182	PX86PML4E pPml4eSrc = &Pml4eSrc;
183
184	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
185	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
186	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
187	# endif
188
189	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, pPml4eSrc, &PdpeSrc, &pPDDst);
190	if (rc != VINF_SUCCESS)
191	{
192	AssertRC(rc);
193	return rc;
194	}
195	Assert(pPDDst);
196
197	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
198	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
199	PEPTPD pPDDst;
200
201	rc = pgmShwGetEPTPDPtr(pVCpu, pvFault, NULL, &pPDDst);
202	if (rc != VINF_SUCCESS)
203	{
204	AssertRC(rc);
205	return rc;
206	}
207	Assert(pPDDst);
208	# endif
209
210	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
211	/*
212	* If we successfully correct the write protection fault due to dirty bit
213	* tracking, or this page fault is a genuine one, then return immediately.
214	*/
215	STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeCheckPageFault, e);
216	rc = PGM_BTH_NAME(CheckPageFault)(pVCpu, uErr, &pPDDst->a[iPDDst], &pPDSrc->a[iPDSrc], pvFault);
217	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeCheckPageFault, e);
218	if ( rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
219	\|\| rc == VINF_EM_RAW_GUEST_TRAP)
220	{
221	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution)
222	= rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? &pVCpu->pgm.s.StatRZTrap0eTime2DirtyAndAccessed : &pVCpu->pgm.s.StatRZTrap0eTime2GuestTrap; });
223	LogBird(("Trap0eHandler: returns %s\n", rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? "VINF_SUCCESS" : "VINF_EM_RAW_GUEST_TRAP"));
224	return rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? VINF_SUCCESS : rc;
225	}
226
227	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0ePD[iPDSrc]);
228	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
229
230	/*
231	* A common case is the not-present error caused by lazy page table syncing.
232	*
233	* It is IMPORTANT that we weed out any access to non-present shadow PDEs here
234	* so we can safely assume that the shadow PT is present when calling SyncPage later.
235	*
236	* On failure, we ASSUME that SyncPT is out of memory or detected some kind
237	* of mapping conflict and defer to SyncCR3 in R3.
238	* (Again, we do NOT support access handlers for non-present guest pages.)
239	*
240	*/
241	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
242	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
243	# else
244	GSTPDE PdeSrc;
245	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
246	PdeSrc.n.u1Present = 1;
247	PdeSrc.n.u1Write = 1;
248	PdeSrc.n.u1Accessed = 1;
249	PdeSrc.n.u1User = 1;
250	# endif
251	if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
252	&& !pPDDst->a[iPDDst].n.u1Present
253	&& PdeSrc.n.u1Present
254	)
255
256	{
257	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2SyncPT; });
258	STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeSyncPT, f);
259	LogFlow(("=>SyncPT %04x = %08x\n", iPDSrc, PdeSrc.au32[0]));
260	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, pvFault);
261	if (RT_SUCCESS(rc))
262	{
263	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeSyncPT, f);
264	return rc;
265	}
266	Log(("SyncPT: %d failed!! rc=%d\n", iPDSrc, rc));
267	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
268	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeSyncPT, f);
269	return VINF_PGM_SYNC_CR3;
270	}
271
272	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
273	/*
274	* Check if this address is within any of our mappings.
275	*
276	* This is very fast and it's gonna save us a bit of effort below and prevent
277	* us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
278	* (BTW, it's impossible to have physical access handlers in a mapping.)
279	*/
280	if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
281	{
282	STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
283	PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
284	for ( ; pMapping; pMapping = pMapping->CTX_SUFF(pNext))
285	{
286	if (pvFault < pMapping->GCPtr)
287	break;
288	if (pvFault - pMapping->GCPtr < pMapping->cb)
289	{
290	/*
291	* The first thing we check is if we've got an undetected conflict.
292	*/
293	if (!pVM->pgm.s.fMappingsFixed)
294	{
295	unsigned iPT = pMapping->cb >> GST_PD_SHIFT;
296	while (iPT-- > 0)
297	if (pPDSrc->a[iPDSrc + iPT].n.u1Present)
298	{
299	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eConflicts);
300	Log(("Trap0e: Detected Conflict %RGv-%RGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
301	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
302	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
303	return VINF_PGM_SYNC_CR3;
304	}
305	}
306
307	/*
308	* Check if the fault address is in a virtual page access handler range.
309	*/
310	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->HyperVirtHandlers, pvFault);
311	if ( pCur
312	&& pvFault - pCur->Core.Key < pCur->cb
313	&& uErr & X86_TRAP_PF_RW)
314	{
315	# ifdef IN_RC
316	STAM_PROFILE_START(&pCur->Stat, h);
317	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
318	STAM_PROFILE_STOP(&pCur->Stat, h);
319	# else
320	AssertFailed();
321	rc = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
322	# endif
323	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersMapping);
324	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
325	return rc;
326	}
327
328	/*
329	* Pretend we're not here and let the guest handle the trap.
330	*/
331	TRPMSetErrorCode(pVCpu, uErr & ~X86_TRAP_PF_P);
332	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eGuestPFMapping);
333	LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
334	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
335	return VINF_EM_RAW_GUEST_TRAP;
336	}
337	}
338	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
339	} /* pgmAreMappingsEnabled(&pVM->pgm.s) */
340	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
341
342	/*
343	* Check if this fault address is flagged for special treatment,
344	* which means we'll have to figure out the physical address and
345	* check flags associated with it.
346	*
347	* ASSUME that we can limit any special access handling to pages
348	* in page tables which the guest believes to be present.
349	*/
350	if (PdeSrc.n.u1Present)
351	{
352	RTGCPHYS GCPhys = NIL_RTGCPHYS;
353
354	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
355	# if PGM_GST_TYPE == PGM_TYPE_AMD64
356	bool fBigPagesSupported = true;
357	# else
358	bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
359	# endif
360	if ( PdeSrc.b.u1Size
361	&& fBigPagesSupported)
362	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc)
363	\| ((RTGCPHYS)pvFault & (GST_BIG_PAGE_OFFSET_MASK ^ PAGE_OFFSET_MASK));
364	else
365	{
366	PGSTPT pPTSrc;
367	rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
368	if (RT_SUCCESS(rc))
369	{
370	unsigned iPTESrc = (pvFault >> GST_PT_SHIFT) & GST_PT_MASK;
371	if (pPTSrc->a[iPTESrc].n.u1Present)
372	GCPhys = pPTSrc->a[iPTESrc].u & GST_PTE_PG_MASK;
373	}
374	}
375	# else
376	/* No paging so the fault address is the physical address */
377	GCPhys = (RTGCPHYS)(pvFault & ~PAGE_OFFSET_MASK);
378	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
379
380	/*
381	* If we have a GC address we'll check if it has any flags set.
382	*/
383	if (GCPhys != NIL_RTGCPHYS)
384	{
385	STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
386
387	PPGMPAGE pPage;
388	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
389	if (RT_SUCCESS(rc)) /** just handle the failure immediate (it returns) and make things easier to read. */
390	{
391	if ( PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage)
392	\|\| PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage))
393	{
394	if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
395	{
396	/*
397	* Physical page access handler.
398	*/
399	const RTGCPHYS GCPhysFault = GCPhys \| (pvFault & PAGE_OFFSET_MASK);
400	PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhysFault);
401	if (pCur)
402	{
403	# ifdef PGM_SYNC_N_PAGES
404	/*
405	* If the region is write protected and we got a page not present fault, then sync
406	* the pages. If the fault was caused by a read, then restart the instruction.
407	* In case of write access continue to the GC write handler.
408	*
409	* ASSUMES that there is only one handler per page or that they have similar write properties.
410	*/
411	if ( pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
412	&& !(uErr & X86_TRAP_PF_P))
413	{
414	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, PGM_SYNC_NR_PAGES, uErr);
415	if ( RT_FAILURE(rc)
416	\|\| !(uErr & X86_TRAP_PF_RW)
417	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
418	{
419	AssertRC(rc);
420	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersOutOfSync);
421	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
422	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndPhys; });
423	return rc;
424	}
425	}
426	# endif
427
428	AssertMsg( pCur->enmType != PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
429	\|\| (pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE && (uErr & X86_TRAP_PF_RW)),
430	("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
431
432	# if defined(IN_RC) \|\| defined(IN_RING0)
433	if (pCur->CTX_SUFF(pfnHandler))
434	{
435	STAM_PROFILE_START(&pCur->Stat, h);
436	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, GCPhysFault, pCur->CTX_SUFF(pvUser));
437	STAM_PROFILE_STOP(&pCur->Stat, h);
438	}
439	else
440	# endif
441	rc = VINF_EM_RAW_EMULATE_INSTR;
442	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersPhysical);
443	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
444	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndPhys; });
445	return rc;
446	}
447	}
448	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
449	else
450	{
451	# ifdef PGM_SYNC_N_PAGES
452	/*
453	* If the region is write protected and we got a page not present fault, then sync
454	* the pages. If the fault was caused by a read, then restart the instruction.
455	* In case of write access continue to the GC write handler.
456	*/
457	if ( PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) < PGM_PAGE_HNDL_PHYS_STATE_ALL
458	&& !(uErr & X86_TRAP_PF_P))
459	{
460	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, PGM_SYNC_NR_PAGES, uErr);
461	if ( RT_FAILURE(rc)
462	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
463	\|\| !(uErr & X86_TRAP_PF_RW))
464	{
465	AssertRC(rc);
466	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersOutOfSync);
467	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
468	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndVirt; });
469	return rc;
470	}
471	}
472	# endif
473	/*
474	* Ok, it's an virtual page access handler.
475	*
476	* Since it's faster to search by address, we'll do that first
477	* and then retry by GCPhys if that fails.
478	*/
479	/** @todo r=bird: perhaps we should consider looking up by physical address directly now? */
480	/** @note r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be out of sync, because the
481	* page was changed without us noticing it (not-present -> present without invlpg or mov cr3, xxx)
482	*/
483	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
484	if (pCur)
485	{
486	AssertMsg(!(pvFault - pCur->Core.Key < pCur->cb)
487	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
488	\|\| !(uErr & X86_TRAP_PF_P)
489	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
490	("Unexpected trap for virtual handler: %RGv (phys=%RGp) pPage=%R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
491
492	if ( pvFault - pCur->Core.Key < pCur->cb
493	&& ( uErr & X86_TRAP_PF_RW
494	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
495	{
496	# ifdef IN_RC
497	STAM_PROFILE_START(&pCur->Stat, h);
498	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
499	STAM_PROFILE_STOP(&pCur->Stat, h);
500	# else
501	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
502	# endif
503	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersVirtual);
504	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
505	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndVirt; });
506	return rc;
507	}
508	/* Unhandled part of a monitored page */
509	}
510	else
511	{
512	/* Check by physical address. */
513	PPGMVIRTHANDLER pCur;
514	unsigned iPage;
515	rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys + (pvFault & PAGE_OFFSET_MASK),
516	&pCur, &iPage);
517	Assert(RT_SUCCESS(rc) \|\| !pCur);
518	if ( pCur
519	&& ( uErr & X86_TRAP_PF_RW
520	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
521	{
522	Assert((pCur->aPhysToVirt[iPage].Core.Key & X86_PTE_PAE_PG_MASK) == GCPhys);
523	# ifdef IN_RC
524	RTGCPTR off = (iPage << PAGE_SHIFT) + (pvFault & PAGE_OFFSET_MASK) - (pCur->Core.Key & PAGE_OFFSET_MASK);
525	Assert(off < pCur->cb);
526	STAM_PROFILE_START(&pCur->Stat, h);
527	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, off);
528	STAM_PROFILE_STOP(&pCur->Stat, h);
529	# else
530	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
531	# endif
532	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersVirtualByPhys);
533	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
534	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndVirt; });
535	return rc;
536	}
537	}
538	}
539	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
540
541	/*
542	* There is a handled area of the page, but this fault doesn't belong to it.
543	* We must emulate the instruction.
544	*
545	* To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
546	* we first check if this was a page-not-present fault for a page with only
547	* write access handlers. Restart the instruction if it wasn't a write access.
548	*/
549	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersUnhandled);
550
551	if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
552	&& !(uErr & X86_TRAP_PF_P))
553	{
554	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, PGM_SYNC_NR_PAGES, uErr);
555	if ( RT_FAILURE(rc)
556	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
557	\|\| !(uErr & X86_TRAP_PF_RW))
558	{
559	AssertRC(rc);
560	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersOutOfSync);
561	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
562	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndPhys; });
563	return rc;
564	}
565	}
566
567	/** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
568	* It's writing to an unhandled part of the LDT page several million times.
569	*/
570	rc = PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault);
571	LogFlow(("PGM: PGMInterpretInstruction -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
572	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
573	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndUnhandled; });
574	return rc;
575	} /* if any kind of handler */
576
577	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
578	if (uErr & X86_TRAP_PF_P)
579	{
580	/*
581	* The page isn't marked, but it might still be monitored by a virtual page access handler.
582	* (ASSUMES no temporary disabling of virtual handlers.)
583	*/
584	/** @todo r=bird: Since the purpose is to catch out of sync pages with virtual handler(s) here,
585	* we should correct both the shadow page table and physical memory flags, and not only check for
586	* accesses within the handler region but for access to pages with virtual handlers. */
587	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
588	if (pCur)
589	{
590	AssertMsg( !(pvFault - pCur->Core.Key < pCur->cb)
591	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
592	\|\| !(uErr & X86_TRAP_PF_P)
593	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
594	("Unexpected trap for virtual handler: %08X (phys=%08x) %R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
595
596	if ( pvFault - pCur->Core.Key < pCur->cb
597	&& ( uErr & X86_TRAP_PF_RW
598	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
599	{
600	# ifdef IN_RC
601	STAM_PROFILE_START(&pCur->Stat, h);
602	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
603	STAM_PROFILE_STOP(&pCur->Stat, h);
604	# else
605	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
606	# endif
607	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersVirtualUnmarked);
608	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
609	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndVirt; });
610	return rc;
611	}
612	}
613	}
614	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
615	}
616	else
617	{
618	/*
619	* When the guest accesses invalid physical memory (e.g. probing
620	* of RAM or accessing a remapped MMIO range), then we'll fall
621	* back to the recompiler to emulate the instruction.
622	*/
623	LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhys, rc));
624	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersInvalid);
625	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
626	return VINF_EM_RAW_EMULATE_INSTR;
627	}
628
629	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
630
631	# ifdef PGM_OUT_OF_SYNC_IN_GC /** @todo remove this bugger. */
632	/*
633	* We are here only if page is present in Guest page tables and
634	* trap is not handled by our handlers.
635	*
636	* Check it for page out-of-sync situation.
637	*/
638	STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
639
640	if (!(uErr & X86_TRAP_PF_P))
641	{
642	/*
643	* Page is not present in our page tables.
644	* Try to sync it!
645	* BTW, fPageShw is invalid in this branch!
646	*/
647	if (uErr & X86_TRAP_PF_US)
648	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncUser));
649	else /* supervisor */
650	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
651
652	# if defined(LOG_ENABLED) && !defined(IN_RING0)
653	RTGCPHYS GCPhys;
654	uint64_t fPageGst;
655	PGMGstGetPage(pVCpu, pvFault, &fPageGst, &GCPhys);
656	Log(("Page out of sync: %RGv eip=%08x PdeSrc.n.u1User=%d fPageGst=%08llx GCPhys=%RGp scan=%d\n",
657	pvFault, pRegFrame->eip, PdeSrc.n.u1User, fPageGst, GCPhys, CSAMDoesPageNeedScanning(pVM, (RTRCPTR)pRegFrame->eip)));
658	# endif /* LOG_ENABLED */
659
660	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
661	if (CPUMGetGuestCPL(pVCpu, pRegFrame) == 0)
662	{
663	uint64_t fPageGst;
664	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
665	if ( RT_SUCCESS(rc)
666	&& !(fPageGst & X86_PTE_US))
667	{
668	/* Note: can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU */
669	if ( pvFault == (RTGCPTR)pRegFrame->eip
670	\|\| pvFault - pRegFrame->eip < 8 /* instruction crossing a page boundary */
671	# ifdef CSAM_DETECT_NEW_CODE_PAGES
672	\|\| ( !PATMIsPatchGCAddr(pVM, (RTGCPTR)pRegFrame->eip)
673	&& CSAMDoesPageNeedScanning(pVM, (RTRCPTR)pRegFrame->eip)) /* any new code we encounter here */
674	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
675	)
676	{
677	LogFlow(("CSAMExecFault %RX32\n", pRegFrame->eip));
678	rc = CSAMExecFault(pVM, (RTRCPTR)pRegFrame->eip);
679	if (rc != VINF_SUCCESS)
680	{
681	/*
682	* CSAM needs to perform a job in ring 3.
683	*
684	* Sync the page before going to the host context; otherwise we'll end up in a loop if
685	* CSAM fails (e.g. instruction crosses a page boundary and the next page is not present)
686	*/
687	LogFlow(("CSAM ring 3 job\n"));
688	int rc2 = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, 1, uErr);
689	AssertRC(rc2);
690
691	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
692	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2CSAM; });
693	return rc;
694	}
695	}
696	# ifdef CSAM_DETECT_NEW_CODE_PAGES
697	else if ( uErr == X86_TRAP_PF_RW
698	&& pRegFrame->ecx >= 0x100 /* early check for movswd count */
699	&& pRegFrame->ecx < 0x10000)
700	{
701	/* In case of a write to a non-present supervisor shadow page, we'll take special precautions
702	* to detect loading of new code pages.
703	*/
704
705	/*
706	* Decode the instruction.
707	*/
708	RTGCPTR PC;
709	rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &PC);
710	if (rc == VINF_SUCCESS)
711	{
712	DISCPUSTATE Cpu;
713	uint32_t cbOp;
714	rc = EMInterpretDisasOneEx(pVM, pVCpu, PC, pRegFrame, &Cpu, &cbOp);
715
716	/* For now we'll restrict this to rep movsw/d instructions */
717	if ( rc == VINF_SUCCESS
718	&& Cpu.pCurInstr->opcode == OP_MOVSWD
719	&& (Cpu.prefix & PREFIX_REP))
720	{
721	CSAMMarkPossibleCodePage(pVM, pvFault);
722	}
723	}
724	}
725	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
726
727	/*
728	* Mark this page as safe.
729	*/
730	/** @todo not correct for pages that contain both code and data!! */
731	Log2(("CSAMMarkPage %RGv; scanned=%d\n", pvFault, true));
732	CSAMMarkPage(pVM, (RTRCPTR)pvFault, true);
733	}
734	}
735	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0) */
736	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, PGM_SYNC_NR_PAGES, uErr);
737	if (RT_SUCCESS(rc))
738	{
739	/* The page was successfully synced, return to the guest. */
740	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
741	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSync; });
742	return VINF_SUCCESS;
743	}
744	}
745	else /* uErr & X86_TRAP_PF_P: */
746	{
747	/*
748	* Write protected pages is make writable when the guest makes the first
749	* write to it. This happens for pages that are shared, write monitored
750	* and not yet allocated.
751	*
752	* Also, a side effect of not flushing global PDEs are out of sync pages due
753	* to physical monitored regions, that are no longer valid.
754	* Assume for now it only applies to the read/write flag.
755	*/
756	if (RT_SUCCESS(rc) && (uErr & X86_TRAP_PF_RW))
757	{
758	if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
759	{
760	Log(("PGM #PF: Make writable: %RGp %R[pgmpage] pvFault=%RGp uErr=%#x\n",
761	GCPhys, pPage, pvFault, uErr));
762	rc = pgmPhysPageMakeWritableUnlocked(pVM, pPage, GCPhys);
763	if (rc != VINF_SUCCESS)
764	{
765	AssertMsg(rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("%Rrc\n", rc));
766	return rc;
767	}
768	if (RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)))
769	return VINF_EM_NO_MEMORY;
770	}
771	/// @todo count the above case; else
772	if (uErr & X86_TRAP_PF_US)
773	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncUser));
774	else /* supervisor */
775	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
776
777	/*
778	* Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
779	* page is not present, which is not true in this case.
780	*/
781	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, 1, uErr);
782	if (RT_SUCCESS(rc))
783	{
784	/*
785	* Page was successfully synced, return to guest.
786	*/
787	# ifdef VBOX_STRICT
788	RTGCPHYS GCPhys;
789	uint64_t fPageGst;
790	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, &GCPhys);
791	Assert(RT_SUCCESS(rc) && fPageGst & X86_PTE_RW);
792	LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GCPhys, (uint64_t)fPageGst));
793
794	uint64_t fPageShw;
795	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
796	AssertMsg(RT_SUCCESS(rc) && fPageShw & X86_PTE_RW, ("rc=%Rrc fPageShw=%RX64\n", rc, fPageShw));
797	# endif /* VBOX_STRICT */
798	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
799	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndObs; });
800	return VINF_SUCCESS;
801	}
802
803	/* Check to see if we need to emulate the instruction as X86_CR0_WP has been cleared. */
804	if ( CPUMGetGuestCPL(pVCpu, pRegFrame) == 0
805	&& ((CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG)
806	&& (uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RW \| X86_TRAP_PF_P))
807	{
808	uint64_t fPageGst;
809	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
810	if ( RT_SUCCESS(rc)
811	&& !(fPageGst & X86_PTE_RW))
812	{
813	rc = PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault);
814	if (RT_SUCCESS(rc))
815	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eWPEmulInRZ);
816	else
817	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eWPEmulToR3);
818	return rc;
819	}
820	AssertMsgFailed(("Unexpected r/w page %RGv flag=%x rc=%Rrc\n", pvFault, (uint32_t)fPageGst, rc));
821	}
822	}
823
824	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
825	# ifdef VBOX_STRICT
826	/*
827	* Check for VMM page flags vs. Guest page flags consistency.
828	* Currently only for debug purposes.
829	*/
830	if (RT_SUCCESS(rc))
831	{
832	/* Get guest page flags. */
833	uint64_t fPageGst;
834	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
835	if (RT_SUCCESS(rc))
836	{
837	uint64_t fPageShw;
838	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
839
840	/*
841	* Compare page flags.
842	* Note: we have AVL, A, D bits desynched.
843	*/
844	AssertMsg((fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)),
845	("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64\n", pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst));
846	}
847	else
848	AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc));
849	}
850	else
851	AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc));
852	# endif /* VBOX_STRICT */
853	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
854	}
855	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
856	# endif /* PGM_OUT_OF_SYNC_IN_GC */
857	}
858	else /* GCPhys == NIL_RTGCPHYS */
859	{
860	/*
861	* Page not present in Guest OS or invalid page table address.
862	* This is potential virtual page access handler food.
863	*
864	* For the present we'll say that our access handlers don't
865	* work for this case - we've already discarded the page table
866	* not present case which is identical to this.
867	*
868	* When we perchance find we need this, we will probably have AVL
869	* trees (offset based) to operate on and we can measure their speed
870	* agains mapping a page table and probably rearrange this handling
871	* a bit. (Like, searching virtual ranges before checking the
872	* physical address.)
873	*/
874	}
875	}
876	/* else: !present (guest) */
877
878
879	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
880	/*
881	* Conclusion, this is a guest trap.
882	*/
883	LogFlow(("PGM: Unhandled #PF -> route trap to recompiler!\n"));
884	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eGuestPFUnh);
885	return VINF_EM_RAW_GUEST_TRAP;
886	# else
887	/* present, but not a monitored page; perhaps the guest is probing physical memory */
888	return VINF_EM_RAW_EMULATE_INSTR;
889	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
890
891
892	# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
893
894	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
895	return VERR_INTERNAL_ERROR;
896	# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
897	}
898	#endif /* !IN_RING3 */
899
900
901	/**
902	* Emulation of the invlpg instruction.
903	*
904	*
905	* @returns VBox status code.
906	*
907	* @param pVCpu The VMCPU handle.
908	* @param GCPtrPage Page to invalidate.
909	*
910	* @remark ASSUMES that the guest is updating before invalidating. This order
911	* isn't required by the CPU, so this is speculative and could cause
912	* trouble.
913	*
914	* @todo Flush page or page directory only if necessary!
915	* @todo Add a #define for simply invalidating the page.
916	*/
917	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
918	{
919	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
920	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
921	&& PGM_SHW_TYPE != PGM_TYPE_EPT
922	int rc;
923	PVM pVM = pVCpu->CTX_SUFF(pVM);
924	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
925
926	LogFlow(("InvalidatePage %RGv\n", GCPtrPage));
927	/*
928	* Get the shadow PD entry and skip out if this PD isn't present.
929	* (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
930	*/
931	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
932	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
933	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
934
935	/* Fetch the pgm pool shadow descriptor. */
936	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
937	Assert(pShwPde);
938
939	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
940	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT);
941	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(&pVCpu->pgm.s);
942
943	/* If the shadow PDPE isn't present, then skip the invalidate. */
944	if (!pPdptDst->a[iPdpt].n.u1Present)
945	{
946	Assert(!(pPdptDst->a[iPdpt].u & PGM_PLXFLAGS_MAPPING));
947	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
948	return VINF_SUCCESS;
949	}
950
951	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
952	PPGMPOOLPAGE pShwPde = NULL;
953	PX86PDPAE pPDDst;
954
955	/* Fetch the pgm pool shadow descriptor. */
956	rc = pgmShwGetPaePoolPagePD(&pVCpu->pgm.s, GCPtrPage, &pShwPde);
957	AssertRCSuccessReturn(rc, rc);
958	Assert(pShwPde);
959
960	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGM(&pVM->pgm.s, pShwPde);
961	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
962
963	# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
964	/* PML4 */
965	const unsigned iPml4 = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;
966	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
967	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
968	PX86PDPAE pPDDst;
969	PX86PDPT pPdptDst;
970	PX86PML4E pPml4eDst;
971	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eDst, &pPdptDst, &pPDDst);
972	if (rc != VINF_SUCCESS)
973	{
974	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT \|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
975	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
976	if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
977	PGM_INVL_GUEST_TLBS();
978	return VINF_SUCCESS;
979	}
980	Assert(pPDDst);
981
982	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
983	PX86PDPE pPdpeDst = &pPdptDst->a[iPdpt];
984
985	if (!pPdpeDst->n.u1Present)
986	{
987	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
988	if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
989	PGM_INVL_GUEST_TLBS();
990	return VINF_SUCCESS;
991	}
992
993	# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
994
995	const SHWPDE PdeDst = *pPdeDst;
996	if (!PdeDst.n.u1Present)
997	{
998	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
999	return VINF_SUCCESS;
1000	}
1001
1002	# if defined(IN_RC)
1003	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1004	PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
1005	# endif
1006
1007	/*
1008	* Get the guest PD entry and calc big page.
1009	*/
1010	# if PGM_GST_TYPE == PGM_TYPE_32BIT
1011	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
1012	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
1013	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1014	# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1015	unsigned iPDSrc = 0;
1016	# if PGM_GST_TYPE == PGM_TYPE_PAE
1017	X86PDPE PdpeSrc;
1018	PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, GCPtrPage, &iPDSrc, &PdpeSrc);
1019	# else /* AMD64 */
1020	PX86PML4E pPml4eSrc;
1021	X86PDPE PdpeSrc;
1022	PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
1023	# endif
1024	GSTPDE PdeSrc;
1025
1026	if (pPDSrc)
1027	PdeSrc = pPDSrc->a[iPDSrc];
1028	else
1029	PdeSrc.u = 0;
1030	# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1031
1032	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1033	const bool fIsBigPage = PdeSrc.b.u1Size;
1034	# else
1035	const bool fIsBigPage = PdeSrc.b.u1Size && (CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
1036	# endif
1037
1038	# ifdef IN_RING3
1039	/*
1040	* If a CR3 Sync is pending we may ignore the invalidate page operation
1041	* depending on the kind of sync and if it's a global page or not.
1042	* This doesn't make sense in GC/R0 so we'll skip it entirely there.
1043	*/
1044	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
1045	if ( VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
1046	\|\| ( VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
1047	&& fIsBigPage
1048	&& PdeSrc.b.u1Global
1049	)
1050	)
1051	# else
1052	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
1053	# endif
1054	{
1055	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
1056	return VINF_SUCCESS;
1057	}
1058	# endif /* IN_RING3 */
1059
1060	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1061	/* Fetch the pgm pool shadow descriptor. */
1062	PPGMPOOLPAGE pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
1063	Assert(pShwPdpt);
1064
1065	/* Fetch the pgm pool shadow descriptor. */
1066	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & SHW_PDPE_PG_MASK);
1067	Assert(pShwPde);
1068
1069	Assert(pPml4eDst->n.u1Present && (pPml4eDst->u & SHW_PDPT_MASK));
1070	RTGCPHYS GCPhysPdpt = pPml4eSrc->u & X86_PML4E_PG_MASK;
1071
1072	if ( !pPml4eSrc->n.u1Present
1073	\|\| pShwPdpt->GCPhys != GCPhysPdpt)
1074	{
1075	LogFlow(("InvalidatePage: Out-of-sync PML4E (P/GCPhys) at %RGv GCPhys=%RGp vs %RGp Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
1076	GCPtrPage, pShwPdpt->GCPhys, GCPhysPdpt, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
1077	pgmPoolFreeByPage(pPool, pShwPdpt, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4);
1078	pPml4eDst->u = 0;
1079	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNPs));
1080	PGM_INVL_GUEST_TLBS();
1081	return VINF_SUCCESS;
1082	}
1083	if ( pPml4eSrc->n.u1User != pPml4eDst->n.u1User
1084	\|\| (!pPml4eSrc->n.u1Write && pPml4eDst->n.u1Write))
1085	{
1086	/*
1087	* Mark not present so we can resync the PML4E when it's used.
1088	*/
1089	LogFlow(("InvalidatePage: Out-of-sync PML4E at %RGv Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
1090	GCPtrPage, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
1091	pgmPoolFreeByPage(pPool, pShwPdpt, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4);
1092	pPml4eDst->u = 0;
1093	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1094	PGM_INVL_GUEST_TLBS();
1095	}
1096	else if (!pPml4eSrc->n.u1Accessed)
1097	{
1098	/*
1099	* Mark not present so we can set the accessed bit.
1100	*/
1101	LogFlow(("InvalidatePage: Out-of-sync PML4E (A) at %RGv Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
1102	GCPtrPage, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
1103	pgmPoolFreeByPage(pPool, pShwPdpt, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4);
1104	pPml4eDst->u = 0;
1105	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNAs));
1106	PGM_INVL_GUEST_TLBS();
1107	}
1108
1109	/* Check if the PDPT entry has changed. */
1110	Assert(pPdpeDst->n.u1Present && pPdpeDst->u & SHW_PDPT_MASK);
1111	RTGCPHYS GCPhysPd = PdpeSrc.u & GST_PDPE_PG_MASK;
1112	if ( !PdpeSrc.n.u1Present
1113	\|\| pShwPde->GCPhys != GCPhysPd)
1114	{
1115	LogFlow(("InvalidatePage: Out-of-sync PDPE (P/GCPhys) at %RGv GCPhys=%RGp vs %RGp PdpeSrc=%RX64 PdpeDst=%RX64\n",
1116	GCPtrPage, pShwPde->GCPhys, GCPhysPd, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
1117	pgmPoolFreeByPage(pPool, pShwPde, pShwPdpt->idx, iPdpt);
1118	pPdpeDst->u = 0;
1119	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNPs));
1120	PGM_INVL_GUEST_TLBS();
1121	return VINF_SUCCESS;
1122	}
1123	if ( PdpeSrc.lm.u1User != pPdpeDst->lm.u1User
1124	\|\| (!PdpeSrc.lm.u1Write && pPdpeDst->lm.u1Write))
1125	{
1126	/*
1127	* Mark not present so we can resync the PDPTE when it's used.
1128	*/
1129	LogFlow(("InvalidatePage: Out-of-sync PDPE at %RGv PdpeSrc=%RX64 PdpeDst=%RX64\n",
1130	GCPtrPage, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
1131	pgmPoolFreeByPage(pPool, pShwPde, pShwPdpt->idx, iPdpt);
1132	pPdpeDst->u = 0;
1133	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1134	PGM_INVL_GUEST_TLBS();
1135	}
1136	else if (!PdpeSrc.lm.u1Accessed)
1137	{
1138	/*
1139	* Mark not present so we can set the accessed bit.
1140	*/
1141	LogFlow(("InvalidatePage: Out-of-sync PDPE (A) at %RGv PdpeSrc=%RX64 PdpeDst=%RX64\n",
1142	GCPtrPage, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
1143	pgmPoolFreeByPage(pPool, pShwPde, pShwPdpt->idx, iPdpt);
1144	pPdpeDst->u = 0;
1145	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNAs));
1146	PGM_INVL_GUEST_TLBS();
1147	}
1148	# endif /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
1149
1150	/*
1151	* Deal with the Guest PDE.
1152	*/
1153	rc = VINF_SUCCESS;
1154	if (PdeSrc.n.u1Present)
1155	{
1156	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
1157	{
1158	/*
1159	* Conflict - Let SyncPT deal with it to avoid duplicate code.
1160	*/
1161	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1162	Assert(PGMGetGuestMode(pVCpu) <= PGMMODE_PAE);
1163	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
1164	}
1165	else if ( PdeSrc.n.u1User != PdeDst.n.u1User
1166	\|\| (!PdeSrc.n.u1Write && PdeDst.n.u1Write))
1167	{
1168	/*
1169	* Mark not present so we can resync the PDE when it's used.
1170	*/
1171	LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1172	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1173	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1174	pPdeDst->u = 0;
1175	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1176	PGM_INVL_GUEST_TLBS();
1177	}
1178	else if (!PdeSrc.n.u1Accessed)
1179	{
1180	/*
1181	* Mark not present so we can set the accessed bit.
1182	*/
1183	LogFlow(("InvalidatePage: Out-of-sync (A) at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1184	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1185	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1186	pPdeDst->u = 0;
1187	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNAs));
1188	PGM_INVL_GUEST_TLBS();
1189	}
1190	else if (!fIsBigPage)
1191	{
1192	/*
1193	* 4KB - page.
1194	*/
1195	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1196	RTGCPHYS GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1197	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1198	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1199	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE/2);
1200	# endif
1201	if (pShwPage->GCPhys == GCPhys)
1202	{
1203	# if 0 /* likely cause of a major performance regression; must be SyncPageWorkerTrackDeref then */
1204	const unsigned iPTEDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1205	PSHWPT pPT = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1206	if (pPT->a[iPTEDst].n.u1Present)
1207	{
1208	# ifdef PGMPOOL_WITH_USER_TRACKING
1209	/* This is very unlikely with caching/monitoring enabled. */
1210	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pShwPage, pPT->a[iPTEDst].u & SHW_PTE_PG_MASK);
1211	# endif
1212	pPT->a[iPTEDst].u = 0;
1213	}
1214	# else /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1215	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
1216	if (RT_SUCCESS(rc))
1217	rc = VINF_SUCCESS;
1218	# endif
1219	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePage4KBPages));
1220	PGM_INVL_PG(GCPtrPage);
1221	}
1222	else
1223	{
1224	/*
1225	* The page table address changed.
1226	*/
1227	LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
1228	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1229	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1230	pPdeDst->u = 0;
1231	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1232	PGM_INVL_GUEST_TLBS();
1233	}
1234	}
1235	else
1236	{
1237	/*
1238	* 2/4MB - page.
1239	*/
1240	/* Before freeing the page, check if anything really changed. */
1241	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1242	RTGCPHYS GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
1243	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1244	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1245	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1246	# endif
1247	if ( pShwPage->GCPhys == GCPhys
1248	&& pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1249	{
1250	/* ASSUMES a the given bits are identical for 4M and normal PDEs */
1251	/** @todo PAT */
1252	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
1253	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
1254	&& ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1255	\|\| (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1256	{
1257	LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1258	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip));
1259	# if defined(IN_RC)
1260	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1261	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
1262	# endif
1263	return VINF_SUCCESS;
1264	}
1265	}
1266
1267	/*
1268	* Ok, the page table is present and it's been changed in the guest.
1269	* If we're in host context, we'll just mark it as not present taking the lazy approach.
1270	* We could do this for some flushes in GC too, but we need an algorithm for
1271	* deciding which 4MB pages containing code likely to be executed very soon.
1272	*/
1273	LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1274	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1275	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1276	pPdeDst->u = 0;
1277	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePage4MBPages));
1278	PGM_INVL_BIG_PG(GCPtrPage);
1279	}
1280	}
1281	else
1282	{
1283	/*
1284	* Page directory is not present, mark shadow PDE not present.
1285	*/
1286	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
1287	{
1288	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1289	pPdeDst->u = 0;
1290	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNPs));
1291	PGM_INVL_PG(GCPtrPage);
1292	}
1293	else
1294	{
1295	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1296	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDMappings));
1297	}
1298	}
1299	# if defined(IN_RC)
1300	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1301	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
1302	# endif
1303	return rc;
1304
1305	#else /* guest real and protected mode */
1306	/* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1307	return VINF_SUCCESS;
1308	#endif
1309	}
1310
1311
1312	#ifdef PGMPOOL_WITH_USER_TRACKING
1313	/**
1314	* Update the tracking of shadowed pages.
1315	*
1316	* @param pVCpu The VMCPU handle.
1317	* @param pShwPage The shadow page.
1318	* @param HCPhys The physical page we is being dereferenced.
1319	*/
1320	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys)
1321	{
1322	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1323	PVM pVM = pVCpu->CTX_SUFF(pVM);
1324
1325	STAM_PROFILE_START(&pVM->pgm.s.StatTrackDeref, a);
1326	LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1327
1328	/** @todo If this turns out to be a bottle neck (very likely) two things can be done:
1329	* 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1330	* 2. write protect all shadowed pages. I.e. implement caching.
1331	*/
1332	/*
1333	* Find the guest address.
1334	*/
1335	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
1336	pRam;
1337	pRam = pRam->CTX_SUFF(pNext))
1338	{
1339	unsigned iPage = pRam->cb >> PAGE_SHIFT;
1340	while (iPage-- > 0)
1341	{
1342	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1343	{
1344	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1345	pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage]);
1346	pShwPage->cPresent--;
1347	pPool->cPresent--;
1348	STAM_PROFILE_STOP(&pVM->pgm.s.StatTrackDeref, a);
1349	return;
1350	}
1351	}
1352	}
1353
1354	for (;;)
1355	AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys));
1356	# else /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1357	pShwPage->cPresent--;
1358	pVM->pgm.s.CTX_SUFF(pPool)->cPresent--;
1359	# endif /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1360	}
1361
1362
1363	/**
1364	* Update the tracking of shadowed pages.
1365	*
1366	* @param pVCpu The VMCPU handle.
1367	* @param pShwPage The shadow page.
1368	* @param u16 The top 16-bit of the pPage->HCPhys.
1369	* @param pPage Pointer to the guest page. this will be modified.
1370	* @param iPTDst The index into the shadow table.
1371	*/
1372	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
1373	{
1374	PVM pVM = pVCpu->CTX_SUFF(pVM);
1375	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1376	/*
1377	* Just deal with the simple first time here.
1378	*/
1379	if (!u16)
1380	{
1381	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackVirgin);
1382	u16 = PGMPOOL_TD_MAKE(1, pShwPage->idx);
1383	}
1384	else
1385	u16 = pgmPoolTrackPhysExtAddref(pVM, u16, pShwPage->idx);
1386
1387	/* write back */
1388	Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x iPTDst=%#x\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst));
1389	PGM_PAGE_SET_TRACKING(pPage, u16);
1390
1391	# endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
1392
1393	/* update statistics. */
1394	pVM->pgm.s.CTX_SUFF(pPool)->cPresent++;
1395	pShwPage->cPresent++;
1396	if (pShwPage->iFirstPresent > iPTDst)
1397	pShwPage->iFirstPresent = iPTDst;
1398	}
1399	#endif /* PGMPOOL_WITH_USER_TRACKING */
1400
1401
1402	/**
1403	* Creates a 4K shadow page for a guest page.
1404	*
1405	* For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1406	* physical address. The PdeSrc argument only the flags are used. No page structured
1407	* will be mapped in this function.
1408	*
1409	* @param pVCpu The VMCPU handle.
1410	* @param pPteDst Destination page table entry.
1411	* @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1412	* Can safely assume that only the flags are being used.
1413	* @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1414	* @param pShwPage Pointer to the shadow page.
1415	* @param iPTDst The index into the shadow table.
1416	*
1417	* @remark Not used for 2/4MB pages!
1418	*/
1419	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1420	{
1421	if (PteSrc.n.u1Present)
1422	{
1423	PVM pVM = pVCpu->CTX_SUFF(pVM);
1424
1425	/*
1426	* Find the ram range.
1427	*/
1428	PPGMPAGE pPage;
1429	int rc = pgmPhysGetPageEx(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK, &pPage);
1430	if (RT_SUCCESS(rc))
1431	{
1432	#ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1433	/* Try make the page writable if necessary. */
1434	if ( PteSrc.n.u1Write
1435	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1436	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
1437	{
1438	rc = pgmPhysPageMakeWritableUnlocked(pVM, pPage, PteSrc.u & GST_PTE_PG_MASK);
1439	AssertRC(rc);
1440	}
1441	#endif
1442
1443	/** @todo investiage PWT, PCD and PAT. */
1444	/*
1445	* Make page table entry.
1446	*/
1447	SHWPTE PteDst;
1448	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1449	{
1450	/** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No. */
1451	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1452	{
1453	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1454	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage);
1455	PteDst.n.u1Present = 1;
1456	PteDst.n.u1Execute = 1;
1457	PteDst.n.u1IgnorePAT = 1;
1458	PteDst.n.u3EMT = VMX_EPT_MEMTYPE_WB;
1459	/* PteDst.n.u1Write = 0 && PteDst.n.u1Size = 0 */
1460	#else
1461	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1462	\| PGM_PAGE_GET_HCPHYS(pPage);
1463	#endif
1464	}
1465	else
1466	{
1467	LogFlow(("SyncPageWorker: monitored page (%RHp) -> mark not present\n", PGM_PAGE_GET_HCPHYS(pPage)));
1468	PteDst.u = 0;
1469	}
1470	/** @todo count these two kinds. */
1471	}
1472	else
1473	{
1474	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1475	/*
1476	* If the page or page directory entry is not marked accessed,
1477	* we mark the page not present.
1478	*/
1479	if (!PteSrc.n.u1Accessed \|\| !PdeSrc.n.u1Accessed)
1480	{
1481	LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1482	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,AccessedPage));
1483	PteDst.u = 0;
1484	}
1485	else
1486	/*
1487	* If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1488	* when the page is modified.
1489	*/
1490	if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1491	{
1492	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPage));
1493	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1494	\| PGM_PAGE_GET_HCPHYS(pPage)
1495	\| PGM_PTFLAGS_TRACK_DIRTY;
1496	}
1497	else
1498	#endif
1499	{
1500	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageSkipped));
1501	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1502	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage);
1503	PteDst.n.u1Present = 1;
1504	PteDst.n.u1Write = 1;
1505	PteDst.n.u1Execute = 1;
1506	PteDst.n.u1IgnorePAT = 1;
1507	PteDst.n.u3EMT = VMX_EPT_MEMTYPE_WB;
1508	/* PteDst.n.u1Size = 0 */
1509	#else
1510	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1511	\| PGM_PAGE_GET_HCPHYS(pPage);
1512	#endif
1513	}
1514	}
1515
1516	/*
1517	* Make sure only allocated pages are mapped writable.
1518	*/
1519	if ( PteDst.n.u1Write
1520	&& PteDst.n.u1Present
1521	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1522	{
1523	PteDst.n.u1Write = 0; /** @todo this isn't quite working yet. */
1524	Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", (RTGCPHYS)(PteSrc.u & X86_PTE_PAE_PG_MASK), pPage, iPTDst));
1525	}
1526
1527	#ifdef PGMPOOL_WITH_USER_TRACKING
1528	/*
1529	* Keep user track up to date.
1530	*/
1531	if (PteDst.n.u1Present)
1532	{
1533	if (!pPteDst->n.u1Present)
1534	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1535	else if ((pPteDst->u & SHW_PTE_PG_MASK) != (PteDst.u & SHW_PTE_PG_MASK))
1536	{
1537	Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", (uint64_t)pPteDst->u, (uint64_t)PteDst.u));
1538	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1539	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1540	}
1541	}
1542	else if (pPteDst->n.u1Present)
1543	{
1544	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1545	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1546	}
1547	#endif /* PGMPOOL_WITH_USER_TRACKING */
1548
1549	/*
1550	* Update statistics and commit the entry.
1551	*/
1552	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1553	if (!PteSrc.n.u1Global)
1554	pShwPage->fSeenNonGlobal = true;
1555	#endif
1556	*pPteDst = PteDst;
1557	}
1558	/* else MMIO or invalid page, we must handle them manually in the #PF handler. */
1559	/** @todo count these. */
1560	}
1561	else
1562	{
1563	/*
1564	* Page not-present.
1565	*/
1566	LogFlow(("SyncPageWorker: page not present in Pte\n"));
1567	#ifdef PGMPOOL_WITH_USER_TRACKING
1568	/* Keep user track up to date. */
1569	if (pPteDst->n.u1Present)
1570	{
1571	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1572	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1573	}
1574	#endif /* PGMPOOL_WITH_USER_TRACKING */
1575	pPteDst->u = 0;
1576	/** @todo count these. */
1577	}
1578	}
1579
1580
1581	/**
1582	* Syncs a guest OS page.
1583	*
1584	* There are no conflicts at this point, neither is there any need for
1585	* page table allocations.
1586	*
1587	* @returns VBox status code.
1588	* @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1589	* @param pVCpu The VMCPU handle.
1590	* @param PdeSrc Page directory entry of the guest.
1591	* @param GCPtrPage Guest context page address.
1592	* @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1593	* @param uErr Fault error (X86_TRAP_PF_*).
1594	*/
1595	PGM_BTH_DECL(int, SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr)
1596	{
1597	PVM pVM = pVCpu->CTX_SUFF(pVM);
1598	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1599	LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr));
1600
1601	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
1602	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
1603	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
1604	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
1605	&& PGM_SHW_TYPE != PGM_TYPE_EPT
1606
1607	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
1608	bool fNoExecuteBitValid = !!(CPUMGetGuestEFER(pVCpu) & MSR_K6_EFER_NXE);
1609	# endif
1610
1611	/*
1612	* Assert preconditions.
1613	*/
1614	Assert(PdeSrc.n.u1Present);
1615	Assert(cPages);
1616	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
1617
1618	/*
1619	* Get the shadow PDE, find the shadow page table in the pool.
1620	*/
1621	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1622	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1623	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
1624
1625	/* Fetch the pgm pool shadow descriptor. */
1626	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1627	Assert(pShwPde);
1628
1629	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1630	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1631	PPGMPOOLPAGE pShwPde = NULL;
1632	PX86PDPAE pPDDst;
1633
1634	/* Fetch the pgm pool shadow descriptor. */
1635	int rc = pgmShwGetPaePoolPagePD(&pVCpu->pgm.s, GCPtrPage, &pShwPde);
1636	AssertRCSuccessReturn(rc, rc);
1637	Assert(pShwPde);
1638
1639	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGM(&pVM->pgm.s, pShwPde);
1640	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1641
1642	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1643	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1644	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1645	PX86PDPAE pPDDst;
1646	PX86PDPT pPdptDst;
1647
1648	int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
1649	AssertRCSuccessReturn(rc, rc);
1650	Assert(pPDDst && pPdptDst);
1651	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1652	# endif
1653	SHWPDE PdeDst = *pPdeDst;
1654	AssertMsg(PdeDst.n.u1Present, ("%p=%llx\n", pPdeDst, (uint64_t)PdeDst.u));
1655	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1656
1657	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1658	/* Fetch the pgm pool shadow descriptor. */
1659	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
1660	Assert(pShwPde);
1661	# endif
1662
1663	# if defined(IN_RC)
1664	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1665	PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
1666	# endif
1667
1668	/*
1669	* Check that the page is present and that the shadow PDE isn't out of sync.
1670	*/
1671	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1672	const bool fBigPage = PdeSrc.b.u1Size;
1673	# else
1674	const bool fBigPage = PdeSrc.b.u1Size && (CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
1675	# endif
1676	RTGCPHYS GCPhys;
1677	if (!fBigPage)
1678	{
1679	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1680	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1681	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1682	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE/2);
1683	# endif
1684	}
1685	else
1686	{
1687	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
1688	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1689	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1690	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1691	# endif
1692	}
1693	if ( pShwPage->GCPhys == GCPhys
1694	&& PdeSrc.n.u1Present
1695	&& (PdeSrc.n.u1User == PdeDst.n.u1User)
1696	&& (PdeSrc.n.u1Write == PdeDst.n.u1Write \|\| !PdeDst.n.u1Write)
1697	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
1698	&& (!fNoExecuteBitValid \|\| PdeSrc.n.u1NoExecute == PdeDst.n.u1NoExecute)
1699	# endif
1700	)
1701	{
1702	/*
1703	* Check that the PDE is marked accessed already.
1704	* Since we set the accessed bit before getting here on a #PF, this
1705	* check is only meant for dealing with non-#PF'ing paths.
1706	*/
1707	if (PdeSrc.n.u1Accessed)
1708	{
1709	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1710	if (!fBigPage)
1711	{
1712	/*
1713	* 4KB Page - Map the guest page table.
1714	*/
1715	PGSTPT pPTSrc;
1716	int rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
1717	if (RT_SUCCESS(rc))
1718	{
1719	# ifdef PGM_SYNC_N_PAGES
1720	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
1721	if ( cPages > 1
1722	&& !(uErr & X86_TRAP_PF_P)
1723	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
1724	{
1725	/*
1726	* This code path is currently only taken when the caller is PGMTrap0eHandler
1727	* for non-present pages!
1728	*
1729	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1730	* deal with locality.
1731	*/
1732	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1733	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1734	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1735	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1736	# else
1737	const unsigned offPTSrc = 0;
1738	# endif
1739	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
1740	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1741	iPTDst = 0;
1742	else
1743	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1744	for (; iPTDst < iPTDstEnd; iPTDst++)
1745	{
1746	if (!pPTDst->a[iPTDst].n.u1Present)
1747	{
1748	GSTPTE PteSrc = pPTSrc->a[offPTSrc + iPTDst];
1749	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
1750	NOREF(GCPtrCurPage);
1751	#ifndef IN_RING0
1752	/*
1753	* Assuming kernel code will be marked as supervisor - and not as user level
1754	* and executed using a conforming code selector - And marked as readonly.
1755	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
1756	*/
1757	PPGMPAGE pPage;
1758	if ( ((PdeSrc.u & PteSrc.u) & (X86_PTE_RW \| X86_PTE_US))
1759	\|\| iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK) /* always sync GCPtrPage */
1760	\|\| !CSAMDoesPageNeedScanning(pVM, (RTRCPTR)GCPtrCurPage)
1761	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
1762	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1763	)
1764	#endif /* else: CSAM not active */
1765	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1766	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1767	GCPtrCurPage, PteSrc.n.u1Present,
1768	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1769	PteSrc.n.u1User & PdeSrc.n.u1User,
1770	(uint64_t)PteSrc.u,
1771	(uint64_t)pPTDst->a[iPTDst].u,
1772	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1773	}
1774	}
1775	}
1776	else
1777	# endif /* PGM_SYNC_N_PAGES */
1778	{
1779	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1780	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1781	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1782	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1783	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s\n",
1784	GCPtrPage, PteSrc.n.u1Present,
1785	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1786	PteSrc.n.u1User & PdeSrc.n.u1User,
1787	(uint64_t)PteSrc.u,
1788	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1789	}
1790	}
1791	else /* MMIO or invalid page: emulated in #PF handler. */
1792	{
1793	LogFlow(("PGM_GCPHYS_2_PTR %RGp failed with %Rrc\n", GCPhys, rc));
1794	Assert(!pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK].n.u1Present);
1795	}
1796	}
1797	else
1798	{
1799	/*
1800	* 4/2MB page - lazy syncing shadow 4K pages.
1801	* (There are many causes of getting here, it's no longer only CSAM.)
1802	*/
1803	/* Calculate the GC physical address of this 4KB shadow page. */
1804	RTGCPHYS GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc) \| (GCPtrPage & GST_BIG_PAGE_OFFSET_MASK);
1805	/* Find ram range. */
1806	PPGMPAGE pPage;
1807	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
1808	if (RT_SUCCESS(rc))
1809	{
1810	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1811	/* Try make the page writable if necessary. */
1812	if ( PdeSrc.n.u1Write
1813	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1814	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
1815	{
1816	rc = pgmPhysPageMakeWritableUnlocked(pVM, pPage, GCPhys);
1817	AssertRC(rc);
1818	}
1819	# endif
1820
1821	/*
1822	* Make shadow PTE entry.
1823	*/
1824	SHWPTE PteDst;
1825	PteDst.u = (PdeSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1826	\| PGM_PAGE_GET_HCPHYS(pPage);
1827	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1828	{
1829	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1830	PteDst.n.u1Write = 0;
1831	else
1832	PteDst.u = 0;
1833	}
1834	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1835	# ifdef PGMPOOL_WITH_USER_TRACKING
1836	if (PteDst.n.u1Present && !pPTDst->a[iPTDst].n.u1Present)
1837	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1838	# endif
1839	/* Make sure only allocated pages are mapped writable. */
1840	if ( PteDst.n.u1Write
1841	&& PteDst.n.u1Present
1842	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1843	{
1844	PteDst.n.u1Write = 0; /** @todo this isn't quite working yet... */
1845	Log3(("SyncPage: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, GCPtrPage));
1846	}
1847
1848	pPTDst->a[iPTDst] = PteDst;
1849
1850
1851	/*
1852	* If the page is not flagged as dirty and is writable, then make it read-only
1853	* at PD level, so we can set the dirty bit when the page is modified.
1854	*
1855	* ASSUMES that page access handlers are implemented on page table entry level.
1856	* Thus we will first catch the dirty access and set PDE.D and restart. If
1857	* there is an access handler, we'll trap again and let it work on the problem.
1858	*/
1859	/** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
1860	* As for invlpg, it simply frees the whole shadow PT.
1861	* ...It's possibly because the guest clears it and the guest doesn't really tell us... */
1862	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
1863	{
1864	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageBig));
1865	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
1866	PdeDst.n.u1Write = 0;
1867	}
1868	else
1869	{
1870	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
1871	PdeDst.n.u1Write = PdeSrc.n.u1Write;
1872	}
1873	*pPdeDst = PdeDst;
1874	Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n",
1875	GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
1876	PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1877	}
1878	else
1879	LogFlow(("PGM_GCPHYS_2_PTR %RGp (big) failed with %Rrc\n", GCPhys, rc));
1880	}
1881	# if defined(IN_RC)
1882	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1883	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
1884	# endif
1885	return VINF_SUCCESS;
1886	}
1887	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDNAs));
1888	}
1889	else
1890	{
1891	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSync));
1892	Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
1893	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
1894	}
1895
1896	/*
1897	* Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
1898	* Yea, I'm lazy.
1899	*/
1900	pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
1901
1902	pPdeDst->u = 0;
1903
1904	# if defined(IN_RC)
1905	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1906	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
1907	# endif
1908	PGM_INVL_GUEST_TLBS();
1909	return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
1910
1911	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
1912	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
1913	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
1914	&& !defined(IN_RC)
1915
1916	# ifdef PGM_SYNC_N_PAGES
1917	/*
1918	* Get the shadow PDE, find the shadow page table in the pool.
1919	*/
1920	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1921	X86PDE PdeDst = pgmShwGet32BitPDE(&pVCpu->pgm.s, GCPtrPage);
1922
1923	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1924	X86PDEPAE PdeDst = pgmShwGetPaePDE(&pVCpu->pgm.s, GCPtrPage);
1925
1926	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1927	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
1928	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpt);
1929	PX86PDPAE pPDDst;
1930	X86PDEPAE PdeDst;
1931	PX86PDPT pPdptDst;
1932
1933	int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
1934	AssertRCSuccessReturn(rc, rc);
1935	Assert(pPDDst && pPdptDst);
1936	PdeDst = pPDDst->a[iPDDst];
1937	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
1938	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
1939	PEPTPD pPDDst;
1940	EPTPDE PdeDst;
1941
1942	int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, NULL, &pPDDst);
1943	if (rc != VINF_SUCCESS)
1944	{
1945	AssertRC(rc);
1946	return rc;
1947	}
1948	Assert(pPDDst);
1949	PdeDst = pPDDst->a[iPDDst];
1950	# endif
1951	AssertMsg(PdeDst.n.u1Present, ("%#llx\n", (uint64_t)PdeDst.u));
1952	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1953	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1954
1955	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
1956	if ( cPages > 1
1957	&& !(uErr & X86_TRAP_PF_P)
1958	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
1959	{
1960	/*
1961	* This code path is currently only taken when the caller is PGMTrap0eHandler
1962	* for non-present pages!
1963	*
1964	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1965	* deal with locality.
1966	*/
1967	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1968	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
1969	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1970	iPTDst = 0;
1971	else
1972	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1973	for (; iPTDst < iPTDstEnd; iPTDst++)
1974	{
1975	if (!pPTDst->a[iPTDst].n.u1Present)
1976	{
1977	GSTPTE PteSrc;
1978
1979	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT);
1980
1981	/* Fake the page table entry */
1982	PteSrc.u = GCPtrCurPage;
1983	PteSrc.n.u1Present = 1;
1984	PteSrc.n.u1Dirty = 1;
1985	PteSrc.n.u1Accessed = 1;
1986	PteSrc.n.u1Write = 1;
1987	PteSrc.n.u1User = 1;
1988
1989	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1990
1991	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1992	GCPtrCurPage, PteSrc.n.u1Present,
1993	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1994	PteSrc.n.u1User & PdeSrc.n.u1User,
1995	(uint64_t)PteSrc.u,
1996	(uint64_t)pPTDst->a[iPTDst].u,
1997	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1998
1999	if (RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)))
2000	break;
2001	}
2002	else
2003	Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n", (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT), iPTDst, pPTDst->a[iPTDst].u));
2004	}
2005	}
2006	else
2007	# endif /* PGM_SYNC_N_PAGES */
2008	{
2009	GSTPTE PteSrc;
2010	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2011	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT);
2012
2013	/* Fake the page table entry */
2014	PteSrc.u = GCPtrCurPage;
2015	PteSrc.n.u1Present = 1;
2016	PteSrc.n.u1Dirty = 1;
2017	PteSrc.n.u1Accessed = 1;
2018	PteSrc.n.u1Write = 1;
2019	PteSrc.n.u1User = 1;
2020	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2021
2022	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}PteDst=%08llx%s\n",
2023	GCPtrPage, PteSrc.n.u1Present,
2024	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2025	PteSrc.n.u1User & PdeSrc.n.u1User,
2026	(uint64_t)PteSrc.u,
2027	(uint64_t)pPTDst->a[iPTDst].u,
2028	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2029	}
2030	return VINF_SUCCESS;
2031
2032	#else
2033	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2034	return VERR_INTERNAL_ERROR;
2035	#endif
2036	}
2037
2038
2039	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2040	/**
2041	* Investigate page fault and handle write protection page faults caused by
2042	* dirty bit tracking.
2043	*
2044	* @returns VBox status code.
2045	* @param pVCpu The VMCPU handle.
2046	* @param uErr Page fault error code.
2047	* @param pPdeDst Shadow page directory entry.
2048	* @param pPdeSrc Guest page directory entry.
2049	* @param GCPtrPage Guest context page address.
2050	*/
2051	PGM_BTH_DECL(int, CheckPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCPTR GCPtrPage)
2052	{
2053	bool fWriteProtect = !!(CPUMGetGuestCR0(pVCpu) & X86_CR0_WP);
2054	bool fUserLevelFault = !!(uErr & X86_TRAP_PF_US);
2055	bool fWriteFault = !!(uErr & X86_TRAP_PF_RW);
2056	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2057	bool fBigPagesSupported = true;
2058	# else
2059	bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
2060	# endif
2061	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2062	bool fNoExecuteBitValid = !!(CPUMGetGuestEFER(pVCpu) & MSR_K6_EFER_NXE);
2063	# endif
2064	unsigned uPageFaultLevel;
2065	int rc;
2066	PVM pVM = pVCpu->CTX_SUFF(pVM);
2067	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2068
2069	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2070	LogFlow(("CheckPageFault: GCPtrPage=%RGv uErr=%#x PdeSrc=%08x\n", GCPtrPage, uErr, pPdeSrc->u));
2071
2072	# if PGM_GST_TYPE == PGM_TYPE_PAE \
2073	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
2074
2075	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2076	PX86PML4E pPml4eSrc;
2077	PX86PDPE pPdpeSrc;
2078
2079	pPdpeSrc = pgmGstGetLongModePDPTPtr(&pVCpu->pgm.s, GCPtrPage, &pPml4eSrc);
2080	Assert(pPml4eSrc);
2081
2082	/*
2083	* Real page fault? (PML4E level)
2084	*/
2085	if ( (uErr & X86_TRAP_PF_RSVD)
2086	\|\| !pPml4eSrc->n.u1Present
2087	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPml4eSrc->n.u1NoExecute)
2088	\|\| (fWriteFault && !pPml4eSrc->n.u1Write && (fUserLevelFault \|\| fWriteProtect))
2089	\|\| (fUserLevelFault && !pPml4eSrc->n.u1User)
2090	)
2091	{
2092	uPageFaultLevel = 0;
2093	goto l_UpperLevelPageFault;
2094	}
2095	Assert(pPdpeSrc);
2096
2097	# else /* PAE */
2098	PX86PDPE pPdpeSrc = pgmGstGetPaePDPEPtr(&pVCpu->pgm.s, GCPtrPage);
2099	# endif /* PAE */
2100
2101	/*
2102	* Real page fault? (PDPE level)
2103	*/
2104	if ( (uErr & X86_TRAP_PF_RSVD)
2105	\|\| !pPdpeSrc->n.u1Present
2106	# if PGM_GST_TYPE == PGM_TYPE_AMD64 /* NX, r/w, u/s bits in the PDPE are long mode only */
2107	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPdpeSrc->lm.u1NoExecute)
2108	\|\| (fWriteFault && !pPdpeSrc->lm.u1Write && (fUserLevelFault \|\| fWriteProtect))
2109	\|\| (fUserLevelFault && !pPdpeSrc->lm.u1User)
2110	# endif
2111	)
2112	{
2113	uPageFaultLevel = 1;
2114	goto l_UpperLevelPageFault;
2115	}
2116	# endif
2117
2118	/*
2119	* Real page fault? (PDE level)
2120	*/
2121	if ( (uErr & X86_TRAP_PF_RSVD)
2122	\|\| !pPdeSrc->n.u1Present
2123	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2124	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPdeSrc->n.u1NoExecute)
2125	# endif
2126	\|\| (fWriteFault && !pPdeSrc->n.u1Write && (fUserLevelFault \|\| fWriteProtect))
2127	\|\| (fUserLevelFault && !pPdeSrc->n.u1User) )
2128	{
2129	uPageFaultLevel = 2;
2130	goto l_UpperLevelPageFault;
2131	}
2132
2133	/*
2134	* First check the easy case where the page directory has been marked read-only to track
2135	* the dirty bit of an emulated BIG page
2136	*/
2137	if (pPdeSrc->b.u1Size && fBigPagesSupported)
2138	{
2139	/* Mark guest page directory as accessed */
2140	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2141	pPml4eSrc->n.u1Accessed = 1;
2142	pPdpeSrc->lm.u1Accessed = 1;
2143	# endif
2144	pPdeSrc->b.u1Accessed = 1;
2145
2146	/*
2147	* Only write protection page faults are relevant here.
2148	*/
2149	if (fWriteFault)
2150	{
2151	/* Mark guest page directory as dirty (BIG page only). */
2152	pPdeSrc->b.u1Dirty = 1;
2153
2154	if (pPdeDst->n.u1Present && (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY))
2155	{
2156	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageTrap));
2157
2158	Assert(pPdeSrc->b.u1Write);
2159
2160	pPdeDst->n.u1Write = 1;
2161	pPdeDst->n.u1Accessed = 1;
2162	pPdeDst->au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2163	PGM_INVL_BIG_PG(GCPtrPage);
2164	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2165	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
2166	}
2167	}
2168	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2169	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2170	}
2171	/* else: 4KB page table */
2172
2173	/*
2174	* Map the guest page table.
2175	*/
2176	PGSTPT pPTSrc;
2177	rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
2178	if (RT_SUCCESS(rc))
2179	{
2180	/*
2181	* Real page fault?
2182	*/
2183	PGSTPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2184	const GSTPTE PteSrc = *pPteSrc;
2185	if ( !PteSrc.n.u1Present
2186	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2187	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && PteSrc.n.u1NoExecute)
2188	# endif
2189	\|\| (fWriteFault && !PteSrc.n.u1Write && (fUserLevelFault \|\| fWriteProtect))
2190	\|\| (fUserLevelFault && !PteSrc.n.u1User)
2191	)
2192	{
2193	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyTrackRealPF));
2194	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2195	LogFlow(("CheckPageFault: real page fault at %RGv PteSrc.u=%08x (2)\n", GCPtrPage, PteSrc.u));
2196
2197	/* Check the present bit as the shadow tables can cause different error codes by being out of sync.
2198	* See the 2nd case above as well.
2199	*/
2200	if (pPdeSrc->n.u1Present && pPteSrc->n.u1Present)
2201	TRPMSetErrorCode(pVCpu, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
2202
2203	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2204	return VINF_EM_RAW_GUEST_TRAP;
2205	}
2206	LogFlow(("CheckPageFault: page fault at %RGv PteSrc.u=%08x\n", GCPtrPage, PteSrc.u));
2207
2208	/*
2209	* Set the accessed bits in the page directory and the page table.
2210	*/
2211	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2212	pPml4eSrc->n.u1Accessed = 1;
2213	pPdpeSrc->lm.u1Accessed = 1;
2214	# endif
2215	pPdeSrc->n.u1Accessed = 1;
2216	pPteSrc->n.u1Accessed = 1;
2217
2218	/*
2219	* Only write protection page faults are relevant here.
2220	*/
2221	if (fWriteFault)
2222	{
2223	/* Write access, so mark guest entry as dirty. */
2224	# ifdef VBOX_WITH_STATISTICS
2225	if (!pPteSrc->n.u1Dirty)
2226	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtiedPage));
2227	else
2228	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageAlreadyDirty));
2229	# endif
2230
2231	pPteSrc->n.u1Dirty = 1;
2232
2233	if (pPdeDst->n.u1Present)
2234	{
2235	#ifndef IN_RING0
2236	/* Bail out here as pgmPoolGetPageByHCPhys will return NULL and we'll crash below.
2237	* Our individual shadow handlers will provide more information and force a fatal exit.
2238	*/
2239	if (MMHyperIsInsideArea(pVM, (RTGCPTR)GCPtrPage))
2240	{
2241	LogRel(("CheckPageFault: write to hypervisor region %RGv\n", GCPtrPage));
2242	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2243	return VINF_SUCCESS;
2244	}
2245	#endif
2246	/*
2247	* Map shadow page table.
2248	*/
2249	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2250	if (pShwPage)
2251	{
2252	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2253	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2254	if ( pPteDst->n.u1Present /** @todo Optimize accessed bit emulation? */
2255	&& (pPteDst->u & PGM_PTFLAGS_TRACK_DIRTY))
2256	{
2257	LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage));
2258	# ifdef VBOX_STRICT
2259	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, pPteSrc->u & GST_PTE_PG_MASK);
2260	if (pPage)
2261	AssertMsg(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage),
2262	("Unexpected dirty bit tracking on monitored page %RGv (phys %RGp)!!!!!!\n", GCPtrPage, pPteSrc->u & X86_PTE_PAE_PG_MASK));
2263	# endif
2264	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageTrap));
2265
2266	Assert(pPteSrc->n.u1Write);
2267
2268	pPteDst->n.u1Write = 1;
2269	pPteDst->n.u1Dirty = 1;
2270	pPteDst->n.u1Accessed = 1;
2271	pPteDst->au32[0] &= ~PGM_PTFLAGS_TRACK_DIRTY;
2272	PGM_INVL_PG(GCPtrPage);
2273
2274	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2275	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
2276	}
2277	}
2278	else
2279	AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
2280	}
2281	}
2282	/** @todo Optimize accessed bit emulation? */
2283	# ifdef VBOX_STRICT
2284	/*
2285	* Sanity check.
2286	*/
2287	else if ( !pPteSrc->n.u1Dirty
2288	&& (pPdeSrc->n.u1Write & pPteSrc->n.u1Write)
2289	&& pPdeDst->n.u1Present)
2290	{
2291	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2292	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2293	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2294	if ( pPteDst->n.u1Present
2295	&& pPteDst->n.u1Write)
2296	LogFlow(("Writable present page %RGv not marked for dirty bit tracking!!!\n", GCPtrPage));
2297	}
2298	# endif /* VBOX_STRICT */
2299	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2300	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2301	}
2302	AssertRC(rc);
2303	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2304	return rc;
2305
2306
2307	l_UpperLevelPageFault:
2308	/*
2309	* Pagefault detected while checking the PML4E, PDPE or PDE.
2310	* Single exit handler to get rid of duplicate code paths.
2311	*/
2312	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyTrackRealPF));
2313	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2314	Log(("CheckPageFault: real page fault at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2315
2316	if (
2317	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2318	pPml4eSrc->n.u1Present &&
2319	# endif
2320	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
2321	pPdpeSrc->n.u1Present &&
2322	# endif
2323	pPdeSrc->n.u1Present)
2324	{
2325	/* Check the present bit as the shadow tables can cause different error codes by being out of sync. */
2326	if (pPdeSrc->b.u1Size && fBigPagesSupported)
2327	{
2328	TRPMSetErrorCode(pVCpu, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
2329	}
2330	else
2331	{
2332	/*
2333	* Map the guest page table.
2334	*/
2335	PGSTPT pPTSrc;
2336	rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
2337	if (RT_SUCCESS(rc))
2338	{
2339	PGSTPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2340	const GSTPTE PteSrc = *pPteSrc;
2341	if (pPteSrc->n.u1Present)
2342	TRPMSetErrorCode(pVCpu, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
2343	}
2344	AssertRC(rc);
2345	}
2346	}
2347	return VINF_EM_RAW_GUEST_TRAP;
2348	}
2349	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
2350
2351
2352	/**
2353	* Sync a shadow page table.
2354	*
2355	* The shadow page table is not present. This includes the case where
2356	* there is a conflict with a mapping.
2357	*
2358	* @returns VBox status code.
2359	* @param pVCpu The VMCPU handle.
2360	* @param iPD Page directory index.
2361	* @param pPDSrc Source page directory (i.e. Guest OS page directory).
2362	* Assume this is a temporary mapping.
2363	* @param GCPtrPage GC Pointer of the page that caused the fault
2364	*/
2365	PGM_BTH_DECL(int, SyncPT)(PVMCPU pVCpu, unsigned iPDSrc, PGSTPD pPDSrc, RTGCPTR GCPtrPage)
2366	{
2367	PVM pVM = pVCpu->CTX_SUFF(pVM);
2368	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2369
2370	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2371	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPtPD[iPDSrc]);
2372	LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage));
2373
2374	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2375	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
2376	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
2377	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2378	&& PGM_SHW_TYPE != PGM_TYPE_EPT
2379
2380	int rc = VINF_SUCCESS;
2381
2382	/*
2383	* Validate input a little bit.
2384	*/
2385	AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
2386	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2387	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2388	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
2389
2390	/* Fetch the pgm pool shadow descriptor. */
2391	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2392	Assert(pShwPde);
2393
2394	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2395	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2396	PPGMPOOLPAGE pShwPde = NULL;
2397	PX86PDPAE pPDDst;
2398	PSHWPDE pPdeDst;
2399
2400	/* Fetch the pgm pool shadow descriptor. */
2401	rc = pgmShwGetPaePoolPagePD(&pVCpu->pgm.s, GCPtrPage, &pShwPde);
2402	AssertRCSuccessReturn(rc, rc);
2403	Assert(pShwPde);
2404
2405	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGM(&pVM->pgm.s, pShwPde);
2406	pPdeDst = &pPDDst->a[iPDDst];
2407
2408	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2409	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2410	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2411	PX86PDPAE pPDDst;
2412	PX86PDPT pPdptDst;
2413	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2414	AssertRCSuccessReturn(rc, rc);
2415	Assert(pPDDst);
2416	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2417	# endif
2418	SHWPDE PdeDst = *pPdeDst;
2419
2420	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2421	/* Fetch the pgm pool shadow descriptor. */
2422	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2423	Assert(pShwPde);
2424	# endif
2425
2426	# ifndef PGM_WITHOUT_MAPPINGS
2427	/*
2428	* Check for conflicts.
2429	* GC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
2430	* HC: Simply resolve the conflict.
2431	*/
2432	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2433	{
2434	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
2435	# ifndef IN_RING3
2436	Log(("SyncPT: Conflict at %RGv\n", GCPtrPage));
2437	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2438	return VERR_ADDRESS_CONFLICT;
2439	# else
2440	PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
2441	Assert(pMapping);
2442	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2443	int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2444	# elif PGM_GST_TYPE == PGM_TYPE_PAE
2445	int rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2446	# else
2447	AssertFailed(); /* can't happen for amd64 */
2448	# endif
2449	if (RT_FAILURE(rc))
2450	{
2451	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2452	return rc;
2453	}
2454	PdeDst = *pPdeDst;
2455	# endif
2456	}
2457	# else /* PGM_WITHOUT_MAPPINGS */
2458	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
2459	# endif /* PGM_WITHOUT_MAPPINGS */
2460	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2461
2462	# if defined(IN_RC)
2463	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
2464	PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
2465	# endif
2466
2467	/*
2468	* Sync page directory entry.
2469	*/
2470	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2471	if (PdeSrc.n.u1Present)
2472	{
2473	/*
2474	* Allocate & map the page table.
2475	*/
2476	PSHWPT pPTDst;
2477	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2478	const bool fPageTable = !PdeSrc.b.u1Size;
2479	# else
2480	const bool fPageTable = !PdeSrc.b.u1Size \|\| !(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
2481	# endif
2482	PPGMPOOLPAGE pShwPage;
2483	RTGCPHYS GCPhys;
2484	if (fPageTable)
2485	{
2486	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
2487	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2488	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2489	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE / 2);
2490	# endif
2491	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
2492	}
2493	else
2494	{
2495	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
2496	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2497	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2498	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
2499	# endif
2500	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, pShwPde->idx, iPDDst, &pShwPage);
2501	}
2502	if (rc == VINF_SUCCESS)
2503	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2504	else if (rc == VINF_PGM_CACHED_PAGE)
2505	{
2506	/*
2507	* The PT was cached, just hook it up.
2508	*/
2509	if (fPageTable)
2510	PdeDst.u = pShwPage->Core.Key
2511	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2512	else
2513	{
2514	PdeDst.u = pShwPage->Core.Key
2515	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2516	/* (see explanation and assumptions further down.) */
2517	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
2518	{
2519	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageBig));
2520	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2521	PdeDst.b.u1Write = 0;
2522	}
2523	}
2524	*pPdeDst = PdeDst;
2525	# if defined(IN_RC)
2526	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2527	# endif
2528	return VINF_SUCCESS;
2529	}
2530	else if (rc == VERR_PGM_POOL_FLUSHED)
2531	{
2532	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2533	# if defined(IN_RC)
2534	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2535	# endif
2536	return VINF_PGM_SYNC_CR3;
2537	}
2538	else
2539	AssertMsgFailedReturn(("rc=%Rrc\n", rc), VERR_INTERNAL_ERROR);
2540	PdeDst.u &= X86_PDE_AVL_MASK;
2541	PdeDst.u \|= pShwPage->Core.Key;
2542
2543	/*
2544	* Page directory has been accessed (this is a fault situation, remember).
2545	*/
2546	pPDSrc->a[iPDSrc].n.u1Accessed = 1;
2547	if (fPageTable)
2548	{
2549	/*
2550	* Page table - 4KB.
2551	*
2552	* Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
2553	*/
2554	Log2(("SyncPT: 4K %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
2555	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
2556	PGSTPT pPTSrc;
2557	rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
2558	if (RT_SUCCESS(rc))
2559	{
2560	/*
2561	* Start by syncing the page directory entry so CSAM's TLB trick works.
2562	*/
2563	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| X86_PDE_AVL_MASK))
2564	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2565	*pPdeDst = PdeDst;
2566	# if defined(IN_RC)
2567	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2568	# endif
2569
2570	/*
2571	* Directory/page user or supervisor privilege: (same goes for read/write)
2572	*
2573	* Directory Page Combined
2574	* U/S U/S U/S
2575	* 0 0 0
2576	* 0 1 0
2577	* 1 0 0
2578	* 1 1 1
2579	*
2580	* Simple AND operation. Table listed for completeness.
2581	*
2582	*/
2583	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT4K));
2584	# ifdef PGM_SYNC_N_PAGES
2585	unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2586	unsigned iPTDst = iPTBase;
2587	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2588	if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
2589	iPTDst = 0;
2590	else
2591	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2592	# else /* !PGM_SYNC_N_PAGES */
2593	unsigned iPTDst = 0;
2594	const unsigned iPTDstEnd = RT_ELEMENTS(pPTDst->a);
2595	# endif /* !PGM_SYNC_N_PAGES */
2596	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2597	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2598	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2599	# else
2600	const unsigned offPTSrc = 0;
2601	# endif
2602	for (; iPTDst < iPTDstEnd; iPTDst++)
2603	{
2604	const unsigned iPTSrc = iPTDst + offPTSrc;
2605	const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2606
2607	if (PteSrc.n.u1Present) /* we've already cleared it above */
2608	{
2609	# ifndef IN_RING0
2610	/*
2611	* Assuming kernel code will be marked as supervisor - and not as user level
2612	* and executed using a conforming code selector - And marked as readonly.
2613	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
2614	*/
2615	PPGMPAGE pPage;
2616	if ( ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW \| X86_PTE_US))
2617	\|\| !CSAMDoesPageNeedScanning(pVM, (RTRCPTR)((iPDSrc << GST_PD_SHIFT) \| (iPTSrc << PAGE_SHIFT)))
2618	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
2619	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2620	)
2621	# endif
2622	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2623	Log2(("SyncPT: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
2624	(RTGCPTR)((iPDSrc << GST_PD_SHIFT) \| (iPTSrc << PAGE_SHIFT)),
2625	PteSrc.n.u1Present,
2626	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2627	PteSrc.n.u1User & PdeSrc.n.u1User,
2628	(uint64_t)PteSrc.u,
2629	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : "", pPTDst->a[iPTDst].u, iPTSrc, PdeSrc.au32[0],
2630	(RTGCPHYS)((PdeSrc.u & GST_PDE_PG_MASK) + iPTSrc*sizeof(PteSrc)) ));
2631	}
2632	} /* for PTEs */
2633	}
2634	}
2635	else
2636	{
2637	/*
2638	* Big page - 2/4MB.
2639	*
2640	* We'll walk the ram range list in parallel and optimize lookups.
2641	* We will only sync on shadow page table at a time.
2642	*/
2643	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT4M));
2644
2645	/**
2646	* @todo It might be more efficient to sync only a part of the 4MB page (similar to what we do for 4kb PDs).
2647	*/
2648
2649	/*
2650	* Start by syncing the page directory entry.
2651	*/
2652	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
2653	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2654
2655	/*
2656	* If the page is not flagged as dirty and is writable, then make it read-only
2657	* at PD level, so we can set the dirty bit when the page is modified.
2658	*
2659	* ASSUMES that page access handlers are implemented on page table entry level.
2660	* Thus we will first catch the dirty access and set PDE.D and restart. If
2661	* there is an access handler, we'll trap again and let it work on the problem.
2662	*/
2663	/** @todo move the above stuff to a section in the PGM documentation. */
2664	Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
2665	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
2666	{
2667	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageBig));
2668	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2669	PdeDst.b.u1Write = 0;
2670	}
2671	*pPdeDst = PdeDst;
2672	# if defined(IN_RC)
2673	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2674	# endif
2675
2676	/*
2677	* Fill the shadow page table.
2678	*/
2679	/* Get address and flags from the source PDE. */
2680	SHWPTE PteDstBase;
2681	PteDstBase.u = PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT);
2682
2683	/* Loop thru the entries in the shadow PT. */
2684	const RTGCPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
2685	Log2(("SyncPT: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n",
2686	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
2687	GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2688	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
2689	unsigned iPTDst = 0;
2690	while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2691	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2692	{
2693	/* Advance ram range list. */
2694	while (pRam && GCPhys > pRam->GCPhysLast)
2695	pRam = pRam->CTX_SUFF(pNext);
2696	if (pRam && GCPhys >= pRam->GCPhys)
2697	{
2698	unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2699	do
2700	{
2701	/* Make shadow PTE. */
2702	PPGMPAGE pPage = &pRam->aPages[iHCPage];
2703	SHWPTE PteDst;
2704
2705	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2706	/* Try make the page writable if necessary. */
2707	if ( PteDstBase.n.u1Write
2708	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2709	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
2710	{
2711	rc = pgmPhysPageMakeWritableUnlocked(pVM, pPage, GCPhys);
2712	AssertRCReturn(rc, rc);
2713	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2714	break;
2715	}
2716	# endif
2717
2718	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2719	{
2720	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
2721	{
2722	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) \| PteDstBase.u;
2723	PteDst.n.u1Write = 0;
2724	}
2725	else
2726	PteDst.u = 0;
2727	}
2728	# ifndef IN_RING0
2729	/*
2730	* Assuming kernel code will be marked as supervisor and not as user level and executed
2731	* using a conforming code selector. Don't check for readonly, as that implies the whole
2732	* 4MB can be code or readonly data. Linux enables write access for its large pages.
2733	*/
2734	else if ( !PdeSrc.n.u1User
2735	&& CSAMDoesPageNeedScanning(pVM, (RTRCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))))
2736	PteDst.u = 0;
2737	# endif
2738	else
2739	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) \| PteDstBase.u;
2740
2741	/* Only map writable pages writable. */
2742	if ( PteDst.n.u1Write
2743	&& PteDst.n.u1Present
2744	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2745	{
2746	PteDst.n.u1Write = 0; /** @todo this isn't quite working yet... */
2747	Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))));
2748	}
2749
2750	# ifdef PGMPOOL_WITH_USER_TRACKING
2751	if (PteDst.n.u1Present)
2752	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2753	# endif
2754	/* commit it */
2755	pPTDst->a[iPTDst] = PteDst;
2756	Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
2757	(RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT)), PteDst.n.u1Present, PteDst.n.u1Write, PteDst.n.u1User, (uint64_t)PteDst.u,
2758	PteDst.u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2759
2760	/* advance */
2761	GCPhys += PAGE_SIZE;
2762	iHCPage++;
2763	iPTDst++;
2764	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2765	&& GCPhys <= pRam->GCPhysLast);
2766	}
2767	else if (pRam)
2768	{
2769	Log(("Invalid pages at %RGp\n", GCPhys));
2770	do
2771	{
2772	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2773	GCPhys += PAGE_SIZE;
2774	iPTDst++;
2775	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2776	&& GCPhys < pRam->GCPhys);
2777	}
2778	else
2779	{
2780	Log(("Invalid pages at %RGp (2)\n", GCPhys));
2781	for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
2782	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2783	}
2784	} /* while more PTEs */
2785	} /* 4KB / 4MB */
2786	}
2787	else
2788	AssertRelease(!PdeDst.n.u1Present);
2789
2790	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2791	if (RT_FAILURE(rc))
2792	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPTFailed));
2793	return rc;
2794
2795	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2796	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2797	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2798	&& !defined(IN_RC)
2799
2800	/*
2801	* Validate input a little bit.
2802	*/
2803	int rc = VINF_SUCCESS;
2804	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2805	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2806	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
2807
2808	/* Fetch the pgm pool shadow descriptor. */
2809	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2810	Assert(pShwPde);
2811
2812	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2813	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2814	PPGMPOOLPAGE pShwPde;
2815	PX86PDPAE pPDDst;
2816	PSHWPDE pPdeDst;
2817
2818	/* Fetch the pgm pool shadow descriptor. */
2819	rc = pgmShwGetPaePoolPagePD(&pVCpu->pgm.s, GCPtrPage, &pShwPde);
2820	AssertRCSuccessReturn(rc, rc);
2821	Assert(pShwPde);
2822
2823	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGM(&pVM->pgm.s, pShwPde);
2824	pPdeDst = &pPDDst->a[iPDDst];
2825
2826	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2827	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2828	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2829	PX86PDPAE pPDDst;
2830	PX86PDPT pPdptDst;
2831	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2832	AssertRCSuccessReturn(rc, rc);
2833	Assert(pPDDst);
2834	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2835
2836	/* Fetch the pgm pool shadow descriptor. */
2837	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2838	Assert(pShwPde);
2839
2840	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2841	const unsigned iPdpt = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
2842	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2843	PEPTPD pPDDst;
2844	PEPTPDPT pPdptDst;
2845
2846	rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, &pPdptDst, &pPDDst);
2847	if (rc != VINF_SUCCESS)
2848	{
2849	AssertRC(rc);
2850	return rc;
2851	}
2852	Assert(pPDDst);
2853	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2854
2855	/* Fetch the pgm pool shadow descriptor. */
2856	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & EPT_PDPTE_PG_MASK);
2857	Assert(pShwPde);
2858	# endif
2859	SHWPDE PdeDst = *pPdeDst;
2860
2861	Assert(!(PdeDst.u & PGM_PDFLAGS_MAPPING));
2862	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2863
2864	GSTPDE PdeSrc;
2865	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2866	PdeSrc.n.u1Present = 1;
2867	PdeSrc.n.u1Write = 1;
2868	PdeSrc.n.u1Accessed = 1;
2869	PdeSrc.n.u1User = 1;
2870
2871	/*
2872	* Allocate & map the page table.
2873	*/
2874	PSHWPT pPTDst;
2875	PPGMPOOLPAGE pShwPage;
2876	RTGCPHYS GCPhys;
2877
2878	/* Virtual address = physical address */
2879	GCPhys = GCPtrPage & X86_PAGE_4K_BASE_MASK;
2880	rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
2881
2882	if ( rc == VINF_SUCCESS
2883	\|\| rc == VINF_PGM_CACHED_PAGE)
2884	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2885	else
2886	AssertMsgFailedReturn(("rc=%Rrc\n", rc), VERR_INTERNAL_ERROR);
2887
2888	PdeDst.u &= X86_PDE_AVL_MASK;
2889	PdeDst.u \|= pShwPage->Core.Key;
2890	PdeDst.n.u1Present = 1;
2891	PdeDst.n.u1Write = 1;
2892	# if PGM_SHW_TYPE == PGM_TYPE_EPT
2893	PdeDst.n.u1Execute = 1;
2894	# else
2895	PdeDst.n.u1User = 1;
2896	PdeDst.n.u1Accessed = 1;
2897	# endif
2898	*pPdeDst = PdeDst;
2899
2900	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, PGM_SYNC_NR_PAGES, 0 /* page not present */);
2901	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2902	return rc;
2903
2904	#else
2905	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
2906	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2907	return VERR_INTERNAL_ERROR;
2908	#endif
2909	}
2910
2911
2912
2913	/**
2914	* Prefetch a page/set of pages.
2915	*
2916	* Typically used to sync commonly used pages before entering raw mode
2917	* after a CR3 reload.
2918	*
2919	* @returns VBox status code.
2920	* @param pVCpu The VMCPU handle.
2921	* @param GCPtrPage Page to invalidate.
2922	*/
2923	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
2924	{
2925	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
2926	&& PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
2927	/*
2928	* Check that all Guest levels thru the PDE are present, getting the
2929	* PD and PDE in the processes.
2930	*/
2931	int rc = VINF_SUCCESS;
2932	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2933	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2934	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
2935	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
2936	# elif PGM_GST_TYPE == PGM_TYPE_PAE
2937	unsigned iPDSrc;
2938	X86PDPE PdpeSrc;
2939	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, GCPtrPage, &iPDSrc, &PdpeSrc);
2940	if (!pPDSrc)
2941	return VINF_SUCCESS; /* not present */
2942	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
2943	unsigned iPDSrc;
2944	PX86PML4E pPml4eSrc;
2945	X86PDPE PdpeSrc;
2946	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
2947	if (!pPDSrc)
2948	return VINF_SUCCESS; /* not present */
2949	# endif
2950	const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2951	# else
2952	PGSTPD pPDSrc = NULL;
2953	const unsigned iPDSrc = 0;
2954	GSTPDE PdeSrc;
2955
2956	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2957	PdeSrc.n.u1Present = 1;
2958	PdeSrc.n.u1Write = 1;
2959	PdeSrc.n.u1Accessed = 1;
2960	PdeSrc.n.u1User = 1;
2961	# endif
2962
2963	if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
2964	{
2965	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2966	const X86PDE PdeDst = pgmShwGet32BitPDE(&pVCpu->pgm.s, GCPtrPage);
2967	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2968	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2969	PX86PDPAE pPDDst;
2970	X86PDEPAE PdeDst;
2971	# if PGM_GST_TYPE != PGM_TYPE_PAE
2972	X86PDPE PdpeSrc;
2973
2974	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
2975	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
2976	# endif
2977	int rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, &PdpeSrc, &pPDDst);
2978	if (rc != VINF_SUCCESS)
2979	{
2980	AssertRC(rc);
2981	return rc;
2982	}
2983	Assert(pPDDst);
2984	PdeDst = pPDDst->a[iPDDst];
2985
2986	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2987	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2988	PX86PDPAE pPDDst;
2989	X86PDEPAE PdeDst;
2990
2991	# if PGM_GST_TYPE == PGM_TYPE_PROT
2992	/* AMD-V nested paging */
2993	X86PML4E Pml4eSrc;
2994	X86PDPE PdpeSrc;
2995	PX86PML4E pPml4eSrc = &Pml4eSrc;
2996
2997	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
2998	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_NX \| X86_PML4E_A;
2999	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_NX \| X86_PDPE_A;
3000	# endif
3001
3002	int rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc, &PdpeSrc, &pPDDst);
3003	if (rc != VINF_SUCCESS)
3004	{
3005	AssertRC(rc);
3006	return rc;
3007	}
3008	Assert(pPDDst);
3009	PdeDst = pPDDst->a[iPDDst];
3010	# endif
3011	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
3012	{
3013	if (!PdeDst.n.u1Present)
3014	/** r=bird: This guy will set the A bit on the PDE, probably harmless. */
3015	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3016	else
3017	{
3018	/** @note We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
3019	* R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
3020	* makes no sense to prefetch more than one page.
3021	*/
3022	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3023	if (RT_SUCCESS(rc))
3024	rc = VINF_SUCCESS;
3025	}
3026	}
3027	}
3028	return rc;
3029
3030	#elif PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3031	return VINF_SUCCESS; /* ignore */
3032	#endif
3033	}
3034
3035
3036
3037
3038	/**
3039	* Syncs a page during a PGMVerifyAccess() call.
3040	*
3041	* @returns VBox status code (informational included).
3042	* @param pVCpu The VMCPU handle.
3043	* @param GCPtrPage The address of the page to sync.
3044	* @param fPage The effective guest page flags.
3045	* @param uErr The trap error code.
3046	*/
3047	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fPage, unsigned uErr)
3048	{
3049	PVM pVM = pVCpu->CTX_SUFF(pVM);
3050
3051	LogFlow(("VerifyAccessSyncPage: GCPtrPage=%RGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
3052
3053	Assert(!HWACCMIsNestedPagingActive(pVM));
3054	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_TYPE_AMD64) \
3055	&& PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
3056
3057	# ifndef IN_RING0
3058	if (!(fPage & X86_PTE_US))
3059	{
3060	/*
3061	* Mark this page as safe.
3062	*/
3063	/** @todo not correct for pages that contain both code and data!! */
3064	Log(("CSAMMarkPage %RGv; scanned=%d\n", GCPtrPage, true));
3065	CSAMMarkPage(pVM, (RTRCPTR)GCPtrPage, true);
3066	}
3067	# endif
3068
3069	/*
3070	* Get guest PD and index.
3071	*/
3072	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3073	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3074	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
3075	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
3076	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3077	unsigned iPDSrc = 0;
3078	X86PDPE PdpeSrc;
3079	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, GCPtrPage, &iPDSrc, &PdpeSrc);
3080
3081	if (pPDSrc)
3082	{
3083	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3084	return VINF_EM_RAW_GUEST_TRAP;
3085	}
3086	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3087	unsigned iPDSrc;
3088	PX86PML4E pPml4eSrc;
3089	X86PDPE PdpeSrc;
3090	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3091	if (!pPDSrc)
3092	{
3093	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3094	return VINF_EM_RAW_GUEST_TRAP;
3095	}
3096	# endif
3097	# else
3098	PGSTPD pPDSrc = NULL;
3099	const unsigned iPDSrc = 0;
3100	# endif
3101	int rc = VINF_SUCCESS;
3102
3103	/*
3104	* First check if the shadow pd is present.
3105	*/
3106	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3107	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
3108	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3109	PX86PDEPAE pPdeDst;
3110	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3111	PX86PDPAE pPDDst;
3112	# if PGM_GST_TYPE != PGM_TYPE_PAE
3113	X86PDPE PdpeSrc;
3114
3115	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3116	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3117	# endif
3118	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, &PdpeSrc, &pPDDst);
3119	if (rc != VINF_SUCCESS)
3120	{
3121	AssertRC(rc);
3122	return rc;
3123	}
3124	Assert(pPDDst);
3125	pPdeDst = &pPDDst->a[iPDDst];
3126
3127	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3128	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3129	PX86PDPAE pPDDst;
3130	PX86PDEPAE pPdeDst;
3131
3132	# if PGM_GST_TYPE == PGM_TYPE_PROT
3133	/* AMD-V nested paging */
3134	X86PML4E Pml4eSrc;
3135	X86PDPE PdpeSrc;
3136	PX86PML4E pPml4eSrc = &Pml4eSrc;
3137
3138	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3139	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_NX \| X86_PML4E_A;
3140	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_NX \| X86_PDPE_A;
3141	# endif
3142
3143	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc, &PdpeSrc, &pPDDst);
3144	if (rc != VINF_SUCCESS)
3145	{
3146	AssertRC(rc);
3147	return rc;
3148	}
3149	Assert(pPDDst);
3150	pPdeDst = &pPDDst->a[iPDDst];
3151	# endif
3152
3153	# if defined(IN_RC)
3154	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
3155	PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
3156	# endif
3157
3158	if (!pPdeDst->n.u1Present)
3159	{
3160	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3161	AssertRC(rc);
3162	if (rc != VINF_SUCCESS)
3163	{
3164	# if defined(IN_RC)
3165	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
3166	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
3167	# endif
3168	return rc;
3169	}
3170	}
3171
3172	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3173	/* Check for dirty bit fault */
3174	rc = PGM_BTH_NAME(CheckPageFault)(pVCpu, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
3175	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
3176	Log(("PGMVerifyAccess: success (dirty)\n"));
3177	else
3178	{
3179	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3180	#else
3181	{
3182	GSTPDE PdeSrc;
3183	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
3184	PdeSrc.n.u1Present = 1;
3185	PdeSrc.n.u1Write = 1;
3186	PdeSrc.n.u1Accessed = 1;
3187	PdeSrc.n.u1User = 1;
3188
3189	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
3190	Assert(rc != VINF_EM_RAW_GUEST_TRAP);
3191	if (uErr & X86_TRAP_PF_US)
3192	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncUser));
3193	else /* supervisor */
3194	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
3195
3196	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3197	if (RT_SUCCESS(rc))
3198	{
3199	/* Page was successfully synced */
3200	Log2(("PGMVerifyAccess: success (sync)\n"));
3201	rc = VINF_SUCCESS;
3202	}
3203	else
3204	{
3205	Log(("PGMVerifyAccess: access violation for %RGv rc=%d\n", GCPtrPage, rc));
3206	rc = VINF_EM_RAW_GUEST_TRAP;
3207	}
3208	}
3209	# if defined(IN_RC)
3210	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
3211	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
3212	# endif
3213	return rc;
3214
3215	#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
3216
3217	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
3218	return VERR_INTERNAL_ERROR;
3219	#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
3220	}
3221
3222
3223	#if PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64
3224	# if PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
3225	/**
3226	* Figures out which kind of shadow page this guest PDE warrants.
3227	*
3228	* @returns Shadow page kind.
3229	* @param pPdeSrc The guest PDE in question.
3230	* @param cr4 The current guest cr4 value.
3231	*/
3232	DECLINLINE(PGMPOOLKIND) PGM_BTH_NAME(CalcPageKind)(const GSTPDE *pPdeSrc, uint32_t cr4)
3233	{
3234	# if PMG_GST_TYPE == PGM_TYPE_AMD64
3235	if (!pPdeSrc->n.u1Size)
3236	# else
3237	if (!pPdeSrc->n.u1Size \|\| !(cr4 & X86_CR4_PSE))
3238	# endif
3239	return BTH_PGMPOOLKIND_PT_FOR_PT;
3240	//switch (pPdeSrc->u & (X86_PDE4M_RW \| X86_PDE4M_US /\| X86_PDE4M_PAE_NX/))
3241	//{
3242	// case 0:
3243	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RO;
3244	// case X86_PDE4M_RW:
3245	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW;
3246	// case X86_PDE4M_US:
3247	// return BTH_PGMPOOLKIND_PT_FOR_BIG_US;
3248	// case X86_PDE4M_RW \| X86_PDE4M_US:
3249	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US;
3250	# if 0
3251	// case X86_PDE4M_PAE_NX:
3252	// return BTH_PGMPOOLKIND_PT_FOR_BIG_NX;
3253	// case X86_PDE4M_RW \| X86_PDE4M_PAE_NX:
3254	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_NX;
3255	// case X86_PDE4M_US \| X86_PDE4M_PAE_NX:
3256	// return BTH_PGMPOOLKIND_PT_FOR_BIG_US_NX;
3257	// case X86_PDE4M_RW \| X86_PDE4M_US \| X86_PDE4M_PAE_NX:
3258	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US_NX;
3259	# endif
3260	return BTH_PGMPOOLKIND_PT_FOR_BIG;
3261	//}
3262	}
3263	# endif
3264	#endif
3265
3266	#undef MY_STAM_COUNTER_INC
3267	#define MY_STAM_COUNTER_INC(a) do { } while (0)
3268
3269
3270	/**
3271	* Syncs the paging hierarchy starting at CR3.
3272	*
3273	* @returns VBox status code, no specials.
3274	* @param pVCpu The VMCPU handle.
3275	* @param cr0 Guest context CR0 register
3276	* @param cr3 Guest context CR3 register
3277	* @param cr4 Guest context CR4 register
3278	* @param fGlobal Including global page directories or not
3279	*/
3280	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
3281	{
3282	PVM pVM = pVCpu->CTX_SUFF(pVM);
3283
3284	if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
3285	fGlobal = true; /* Change this CR3 reload to be a global one. */
3286
3287	LogFlow(("SyncCR3 %d\n", fGlobal));
3288
3289	#if PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
3290	/*
3291	* Update page access handlers.
3292	* The virtual are always flushed, while the physical are only on demand.
3293	* WARNING: We are incorrectly not doing global flushing on Virtual Handler updates. We'll
3294	* have to look into that later because it will have a bad influence on the performance.
3295	* @note SvL: There's no need for that. Just invalidate the virtual range(s).
3296	* bird: Yes, but that won't work for aliases.
3297	*/
3298	/** @todo this MUST go away. See #1557. */
3299	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncCR3Handlers), h);
3300	PGM_GST_NAME(HandlerVirtualUpdate)(pVM, cr4);
3301	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncCR3Handlers), h);
3302	#endif
3303
3304	#if PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3305	/*
3306	* Nested / EPT - almost no work.
3307	*/
3308	/** @todo check if this is really necessary; the call does it as well... */
3309	HWACCMFlushTLB(pVM);
3310	return VINF_SUCCESS;
3311
3312	#elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3313	/*
3314	* AMD64 (Shw & Gst) - No need to check all paging levels; we zero
3315	* out the shadow parts when the guest modifies its tables.
3316	*/
3317	return VINF_SUCCESS;
3318
3319	#else /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3320
3321	# ifdef PGM_WITHOUT_MAPPINGS
3322	Assert(pVM->pgm.s.fMappingsFixed);
3323	return VINF_SUCCESS;
3324	# else
3325	/* Nothing to do when mappings are fixed. */
3326	if (pVM->pgm.s.fMappingsFixed)
3327	return VINF_SUCCESS;
3328
3329	int rc = PGMMapResolveConflicts(pVM);
3330	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
3331	if (rc == VINF_PGM_SYNC_CR3)
3332	{
3333	LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
3334	return VINF_PGM_SYNC_CR3;
3335	}
3336	# endif
3337	return VINF_SUCCESS;
3338	#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3339	}
3340
3341
3342
3343
3344	#ifdef VBOX_STRICT
3345	#ifdef IN_RC
3346	# undef AssertMsgFailed
3347	# define AssertMsgFailed Log
3348	#endif
3349	#ifdef IN_RING3
3350	# include <VBox/dbgf.h>
3351
3352	/**
3353	* Dumps a page table hierarchy use only physical addresses and cr4/lm flags.
3354	*
3355	* @returns VBox status code (VINF_SUCCESS).
3356	* @param cr3 The root of the hierarchy.
3357	* @param crr The cr4, only PAE and PSE is currently used.
3358	* @param fLongMode Set if long mode, false if not long mode.
3359	* @param cMaxDepth Number of levels to dump.
3360	* @param pHlp Pointer to the output functions.
3361	*/
3362	__BEGIN_DECLS
3363	VMMR3DECL(int) PGMR3DumpHierarchyHC(PVM pVM, uint32_t cr3, uint32_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp);
3364	__END_DECLS
3365
3366	#endif
3367
3368	/**
3369	* Checks that the shadow page table is in sync with the guest one.
3370	*
3371	* @returns The number of errors.
3372	* @param pVM The virtual machine.
3373	* @param pVCpu The VMCPU handle.
3374	* @param cr3 Guest context CR3 register
3375	* @param cr4 Guest context CR4 register
3376	* @param GCPtr Where to start. Defaults to 0.
3377	* @param cb How much to check. Defaults to everything.
3378	*/
3379	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)
3380	{
3381	#if PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3382	return 0;
3383	#else
3384	unsigned cErrors = 0;
3385	PVM pVM = pVCpu->CTX_SUFF(pVM);
3386	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3387
3388	#if PGM_GST_TYPE == PGM_TYPE_PAE
3389	/** @todo currently broken; crashes below somewhere */
3390	AssertFailed();
3391	#endif
3392
3393	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
3394	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3395	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
3396
3397	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3398	bool fBigPagesSupported = true;
3399	# else
3400	bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
3401	# endif
3402	PPGMCPU pPGM = &pVCpu->pgm.s;
3403	RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
3404	RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
3405	# ifndef IN_RING0
3406	RTHCPHYS HCPhys; /* general usage. */
3407	# endif
3408	int rc;
3409
3410	/*
3411	* Check that the Guest CR3 and all its mappings are correct.
3412	*/
3413	AssertMsgReturn(pPGM->GCPhysCR3 == (cr3 & GST_CR3_PAGE_MASK),
3414	("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
3415	false);
3416	# if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64
3417	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3418	rc = PGMShwGetPage(pVCpu, (RTGCPTR)pPGM->pGst32BitPdRC, NULL, &HCPhysShw);
3419	# else
3420	rc = PGMShwGetPage(pVCpu, (RTGCPTR)pPGM->pGstPaePdptRC, NULL, &HCPhysShw);
3421	# endif
3422	AssertRCReturn(rc, 1);
3423	HCPhys = NIL_RTHCPHYS;
3424	rc = pgmRamGCPhys2HCPhys(&pVM->pgm.s, cr3 & GST_CR3_PAGE_MASK, &HCPhys);
3425	AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
3426	# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
3427	pgmGstGet32bitPDPtr(pPGM);
3428	RTGCPHYS GCPhys;
3429	rc = PGMR3DbgR3Ptr2GCPhys(pVM, pPGM->pGst32BitPdR3, &GCPhys);
3430	AssertRCReturn(rc, 1);
3431	AssertMsgReturn((cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
3432	# endif
3433	# endif /* !IN_RING0 */
3434
3435	/*
3436	* Get and check the Shadow CR3.
3437	*/
3438	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3439	unsigned cPDEs = X86_PG_ENTRIES;
3440	unsigned cIncrement = X86_PG_ENTRIES * PAGE_SIZE;
3441	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3442	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3443	unsigned cPDEs = X86_PG_PAE_ENTRIES * 4; /* treat it as a 2048 entry table. */
3444	# else
3445	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3446	# endif
3447	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3448	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3449	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3450	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3451	# endif
3452	if (cb != ~(RTGCPTR)0)
3453	cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
3454
3455	/** @todo call the other two PGMAssert() functions. /
3456
3457	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3458	unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3459
3460	for (; iPml4 < X86_PG_PAE_ENTRIES; iPml4++)
3461	{
3462	PPGMPOOLPAGE pShwPdpt = NULL;
3463	PX86PML4E pPml4eSrc;
3464	PX86PML4E pPml4eDst;
3465	RTGCPHYS GCPhysPdptSrc;
3466
3467	pPml4eSrc = pgmGstGetLongModePML4EPtr(&pVCpu->pgm.s, iPml4);
3468	pPml4eDst = pgmShwGetLongModePML4EPtr(&pVCpu->pgm.s, iPml4);
3469
3470	/* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
3471	if (!pPml4eDst->n.u1Present)
3472	{
3473	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3474	continue;
3475	}
3476
3477	pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
3478	GCPhysPdptSrc = pPml4eSrc->u & X86_PML4E_PG_MASK_FULL;
3479
3480	if (pPml4eSrc->n.u1Present != pPml4eDst->n.u1Present)
3481	{
3482	AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3483	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3484	cErrors++;
3485	continue;
3486	}
3487
3488	if (GCPhysPdptSrc != pShwPdpt->GCPhys)
3489	{
3490	AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
3491	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3492	cErrors++;
3493	continue;
3494	}
3495
3496	if ( pPml4eDst->n.u1User != pPml4eSrc->n.u1User
3497	\|\| pPml4eDst->n.u1Write != pPml4eSrc->n.u1Write
3498	\|\| pPml4eDst->n.u1NoExecute != pPml4eSrc->n.u1NoExecute)
3499	{
3500	AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3501	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3502	cErrors++;
3503	continue;
3504	}
3505	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3506	{
3507	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3508
3509	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
3510	/*
3511	* Check the PDPTEs too.
3512	*/
3513	unsigned iPdpt = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
3514
3515	for (;iPdpt <= SHW_PDPT_MASK; iPdpt++)
3516	{
3517	unsigned iPDSrc;
3518	PPGMPOOLPAGE pShwPde = NULL;
3519	PX86PDPE pPdpeDst;
3520	RTGCPHYS GCPhysPdeSrc;
3521	# if PGM_GST_TYPE == PGM_TYPE_PAE
3522	X86PDPE PdpeSrc;
3523	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, GCPtr, &iPDSrc, &PdpeSrc);
3524	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(&pVCpu->pgm.s);
3525	# else
3526	PX86PML4E pPml4eSrc;
3527	X86PDPE PdpeSrc;
3528	PX86PDPT pPdptDst;
3529	PX86PDPAE pPDDst;
3530	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, GCPtr, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3531
3532	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtr, NULL, &pPdptDst, &pPDDst);
3533	if (rc != VINF_SUCCESS)
3534	{
3535	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
3536	GCPtr += 512 * _2M;
3537	continue; /* next PDPTE */
3538	}
3539	Assert(pPDDst);
3540	# endif
3541	Assert(iPDSrc == 0);
3542
3543	pPdpeDst = &pPdptDst->a[iPdpt];
3544
3545	if (!pPdpeDst->n.u1Present)
3546	{
3547	GCPtr += 512 * _2M;
3548	continue; /* next PDPTE */
3549	}
3550
3551	pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
3552	GCPhysPdeSrc = PdpeSrc.u & X86_PDPE_PG_MASK;
3553
3554	if (pPdpeDst->n.u1Present != PdpeSrc.n.u1Present)
3555	{
3556	AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3557	GCPtr += 512 * _2M;
3558	cErrors++;
3559	continue;
3560	}
3561
3562	if (GCPhysPdeSrc != pShwPde->GCPhys)
3563	{
3564	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3565	AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3566	# else
3567	AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3568	# endif
3569	GCPtr += 512 * _2M;
3570	cErrors++;
3571	continue;
3572	}
3573
3574	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3575	if ( pPdpeDst->lm.u1User != PdpeSrc.lm.u1User
3576	\|\| pPdpeDst->lm.u1Write != PdpeSrc.lm.u1Write
3577	\|\| pPdpeDst->lm.u1NoExecute != PdpeSrc.lm.u1NoExecute)
3578	{
3579	AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3580	GCPtr += 512 * _2M;
3581	cErrors++;
3582	continue;
3583	}
3584	# endif
3585
3586	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3587	{
3588	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3589	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3590	GSTPD const *pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
3591	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3592	PCX86PD pPDDst = pgmShwGet32BitPDPtr(&pVCpu->pgm.s);
3593	# endif
3594	# endif /* PGM_GST_TYPE == PGM_TYPE_32BIT */
3595	/*
3596	* Iterate the shadow page directory.
3597	*/
3598	GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
3599	unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
3600
3601	for (;
3602	iPDDst < cPDEs;
3603	iPDDst++, GCPtr += cIncrement)
3604	{
3605	# if PGM_SHW_TYPE == PGM_TYPE_PAE
3606	const SHWPDE PdeDst = *pgmShwGetPaePDEPtr(pPGM, GCPtr);
3607	# else
3608	const SHWPDE PdeDst = pPDDst->a[iPDDst];
3609	# endif
3610	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
3611	{
3612	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
3613	if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
3614	{
3615	AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
3616	cErrors++;
3617	continue;
3618	}
3619	}
3620	else if ( (PdeDst.u & X86_PDE_P)
3621	\|\| ((PdeDst.u & (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY))
3622	)
3623	{
3624	HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
3625	PPGMPOOLPAGE pPoolPage = pgmPoolGetPage(pPool, HCPhysShw);
3626	if (!pPoolPage)
3627	{
3628	AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n",
3629	HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
3630	cErrors++;
3631	continue;
3632	}
3633	const SHWPT pPTDst = (const SHWPT )PGMPOOL_PAGE_2_PTR(pVM, pPoolPage);
3634
3635	if (PdeDst.u & (X86_PDE4M_PWT \| X86_PDE4M_PCD))
3636	{
3637	AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
3638	GCPtr, (uint64_t)PdeDst.u));
3639	cErrors++;
3640	}
3641
3642	if (PdeDst.u & (X86_PDE4M_G \| X86_PDE4M_D))
3643	{
3644	AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n",
3645	GCPtr, (uint64_t)PdeDst.u));
3646	cErrors++;
3647	}
3648
3649	const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
3650	if (!PdeSrc.n.u1Present)
3651	{
3652	AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
3653	GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
3654	cErrors++;
3655	continue;
3656	}
3657
3658	if ( !PdeSrc.b.u1Size
3659	\|\| !fBigPagesSupported)
3660	{
3661	GCPhysGst = PdeSrc.u & GST_PDE_PG_MASK;
3662	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3663	GCPhysGst \|= (iPDDst & 1) * (PAGE_SIZE / 2);
3664	# endif
3665	}
3666	else
3667	{
3668	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3669	if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
3670	{
3671	AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n",
3672	GCPtr, (uint64_t)PdeSrc.u));
3673	cErrors++;
3674	continue;
3675	}
3676	# endif
3677	GCPhysGst = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
3678	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3679	GCPhysGst \|= GCPtr & RT_BIT(X86_PAGE_2M_SHIFT);
3680	# endif
3681	}
3682
3683	if ( pPoolPage->enmKind
3684	!= (!PdeSrc.b.u1Size \|\| !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
3685	{
3686	AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n",
3687	pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
3688	cErrors++;
3689	}
3690
3691	PPGMPAGE pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
3692	if (!pPhysPage)
3693	{
3694	AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3695	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3696	cErrors++;
3697	continue;
3698	}
3699
3700	if (GCPhysGst != pPoolPage->GCPhys)
3701	{
3702	AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n",
3703	GCPhysGst, pPoolPage->GCPhys, GCPtr));
3704	cErrors++;
3705	continue;
3706	}
3707
3708	if ( !PdeSrc.b.u1Size
3709	\|\| !fBigPagesSupported)
3710	{
3711	/*
3712	* Page Table.
3713	*/
3714	const GSTPT *pPTSrc;
3715	rc = PGM_GCPHYS_2_PTR(pVM, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1), &pPTSrc);
3716	if (RT_FAILURE(rc))
3717	{
3718	AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3719	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3720	cErrors++;
3721	continue;
3722	}
3723	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/))
3724	!= (PdeDst.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/)))
3725	{
3726	/// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
3727	// (This problem will go away when/if we shadow multiple CR3s.)
3728	AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3729	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3730	cErrors++;
3731	continue;
3732	}
3733	if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3734	{
3735	AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n",
3736	GCPtr, (uint64_t)PdeDst.u));
3737	cErrors++;
3738	continue;
3739	}
3740
3741	/* iterate the page table. */
3742	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3743	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3744	const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
3745	# else
3746	const unsigned offPTSrc = 0;
3747	# endif
3748	for (unsigned iPT = 0, off = 0;
3749	iPT < RT_ELEMENTS(pPTDst->a);
3750	iPT++, off += PAGE_SIZE)
3751	{
3752	const SHWPTE PteDst = pPTDst->a[iPT];
3753
3754	/* skip not-present entries. */
3755	if (!(PteDst.u & (X86_PTE_P \| PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
3756	continue;
3757	Assert(PteDst.n.u1Present);
3758
3759	const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
3760	if (!PteSrc.n.u1Present)
3761	{
3762	# ifdef IN_RING3
3763	PGMAssertHandlerAndFlagsInSync(pVM);
3764	PGMR3DumpHierarchyGC(pVM, cr3, cr4, (PdeSrc.u & GST_PDE_PG_MASK));
3765	# endif
3766	AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
3767	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u, pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
3768	(PdeSrc.u & GST_PDE_PG_MASK) + (iPT + offPTSrc)*sizeof(PteSrc)));
3769	cErrors++;
3770	continue;
3771	}
3772
3773	uint64_t fIgnoreFlags = GST_PTE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_G \| X86_PTE_D \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
3774	# if 1 /** @todo sync accessed bit properly... */
3775	fIgnoreFlags \|= X86_PTE_A;
3776	# endif
3777
3778	/* match the physical addresses */
3779	HCPhysShw = PteDst.u & SHW_PTE_PG_MASK;
3780	GCPhysGst = PteSrc.u & GST_PTE_PG_MASK;
3781
3782	# ifdef IN_RING3
3783	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3784	if (RT_FAILURE(rc))
3785	{
3786	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
3787	{
3788	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3789	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3790	cErrors++;
3791	continue;
3792	}
3793	}
3794	else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
3795	{
3796	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3797	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3798	cErrors++;
3799	continue;
3800	}
3801	# endif
3802
3803	pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
3804	if (!pPhysPage)
3805	{
3806	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
3807	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
3808	{
3809	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3810	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3811	cErrors++;
3812	continue;
3813	}
3814	# endif
3815	if (PteDst.n.u1Write)
3816	{
3817	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3818	GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3819	cErrors++;
3820	}
3821	fIgnoreFlags \|= X86_PTE_RW;
3822	}
3823	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
3824	{
3825	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3826	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3827	cErrors++;
3828	continue;
3829	}
3830
3831	/* flags */
3832	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
3833	{
3834	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
3835	{
3836	if (PteDst.n.u1Write)
3837	{
3838	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
3839	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3840	cErrors++;
3841	continue;
3842	}
3843	fIgnoreFlags \|= X86_PTE_RW;
3844	}
3845	else
3846	{
3847	if (PteDst.n.u1Present)
3848	{
3849	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
3850	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3851	cErrors++;
3852	continue;
3853	}
3854	fIgnoreFlags \|= X86_PTE_P;
3855	}
3856	}
3857	else
3858	{
3859	if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
3860	{
3861	if (PteDst.n.u1Write)
3862	{
3863	AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
3864	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3865	cErrors++;
3866	continue;
3867	}
3868	if (!(PteDst.u & PGM_PTFLAGS_TRACK_DIRTY))
3869	{
3870	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3871	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3872	cErrors++;
3873	continue;
3874	}
3875	if (PteDst.n.u1Dirty)
3876	{
3877	AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3878	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3879	cErrors++;
3880	}
3881	# if 0 /** @todo sync access bit properly... */
3882	if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
3883	{
3884	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
3885	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3886	cErrors++;
3887	}
3888	fIgnoreFlags \|= X86_PTE_RW;
3889	# else
3890	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
3891	# endif
3892	}
3893	else if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
3894	{
3895	/* access bit emulation (not implemented). */
3896	if (PteSrc.n.u1Accessed \|\| PteDst.n.u1Present)
3897	{
3898	AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
3899	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3900	cErrors++;
3901	continue;
3902	}
3903	if (!PteDst.n.u1Accessed)
3904	{
3905	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
3906	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3907	cErrors++;
3908	}
3909	fIgnoreFlags \|= X86_PTE_P;
3910	}
3911	# ifdef DEBUG_sandervl
3912	fIgnoreFlags \|= X86_PTE_D \| X86_PTE_A;
3913	# endif
3914	}
3915
3916	if ( (PteSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
3917	&& (PteSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags)
3918	)
3919	{
3920	AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
3921	GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
3922	fIgnoreFlags, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3923	cErrors++;
3924	continue;
3925	}
3926	} /* foreach PTE */
3927	}
3928	else
3929	{
3930	/*
3931	* Big Page.
3932	*/
3933	uint64_t fIgnoreFlags = X86_PDE_AVL_MASK \| GST_PDE_PG_MASK \| X86_PDE4M_G \| X86_PDE4M_D \| X86_PDE4M_PS \| X86_PDE4M_PWT \| X86_PDE4M_PCD;
3934	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
3935	{
3936	if (PdeDst.n.u1Write)
3937	{
3938	AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3939	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3940	cErrors++;
3941	continue;
3942	}
3943	if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
3944	{
3945	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3946	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3947	cErrors++;
3948	continue;
3949	}
3950	# if 0 /** @todo sync access bit properly... */
3951	if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
3952	{
3953	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
3954	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3955	cErrors++;
3956	}
3957	fIgnoreFlags \|= X86_PTE_RW;
3958	# else
3959	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
3960	# endif
3961	}
3962	else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3963	{
3964	/* access bit emulation (not implemented). */
3965	if (PdeSrc.b.u1Accessed \|\| PdeDst.n.u1Present)
3966	{
3967	AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3968	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3969	cErrors++;
3970	continue;
3971	}
3972	if (!PdeDst.n.u1Accessed)
3973	{
3974	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3975	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3976	cErrors++;
3977	}
3978	fIgnoreFlags \|= X86_PTE_P;
3979	}
3980
3981	if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
3982	{
3983	AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
3984	GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
3985	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3986	cErrors++;
3987	}
3988
3989	/* iterate the page table. */
3990	for (unsigned iPT = 0, off = 0;
3991	iPT < RT_ELEMENTS(pPTDst->a);
3992	iPT++, off += PAGE_SIZE, GCPhysGst += PAGE_SIZE)
3993	{
3994	const SHWPTE PteDst = pPTDst->a[iPT];
3995
3996	if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
3997	{
3998	AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
3999	GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4000	cErrors++;
4001	}
4002
4003	/* skip not-present entries. */
4004	if (!PteDst.n.u1Present) /** @todo deal with ALL handlers and CSAM !P pages! */
4005	continue;
4006
4007	fIgnoreFlags = X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT \| X86_PTE_D \| X86_PTE_A \| X86_PTE_G \| X86_PTE_PAE_NX;
4008
4009	/* match the physical addresses */
4010	HCPhysShw = PteDst.u & X86_PTE_PAE_PG_MASK;
4011
4012	# ifdef IN_RING3
4013	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4014	if (RT_FAILURE(rc))
4015	{
4016	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4017	{
4018	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4019	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4020	cErrors++;
4021	}
4022	}
4023	else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
4024	{
4025	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4026	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4027	cErrors++;
4028	continue;
4029	}
4030	# endif
4031	pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
4032	if (!pPhysPage)
4033	{
4034	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4035	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4036	{
4037	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4038	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4039	cErrors++;
4040	continue;
4041	}
4042	# endif
4043	if (PteDst.n.u1Write)
4044	{
4045	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4046	GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4047	cErrors++;
4048	}
4049	fIgnoreFlags \|= X86_PTE_RW;
4050	}
4051	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4052	{
4053	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4054	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4055	cErrors++;
4056	continue;
4057	}
4058
4059	/* flags */
4060	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4061	{
4062	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4063	{
4064	if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
4065	{
4066	if (PteDst.n.u1Write)
4067	{
4068	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4069	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4070	cErrors++;
4071	continue;
4072	}
4073	fIgnoreFlags \|= X86_PTE_RW;
4074	}
4075	}
4076	else
4077	{
4078	if (PteDst.n.u1Present)
4079	{
4080	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4081	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4082	cErrors++;
4083	continue;
4084	}
4085	fIgnoreFlags \|= X86_PTE_P;
4086	}
4087	}
4088
4089	if ( (PdeSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
4090	&& (PdeSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags) /* lazy phys handler dereg. */
4091	)
4092	{
4093	AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
4094	GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
4095	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4096	cErrors++;
4097	continue;
4098	}
4099	} /* for each PTE */
4100	}
4101	}
4102	/* not present */
4103
4104	} /* for each PDE */
4105
4106	} /* for each PDPTE */
4107
4108	} /* for each PML4E */
4109
4110	# ifdef DEBUG
4111	if (cErrors)
4112	LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
4113	# endif
4114
4115	#endif /* GST == 32BIT, PAE or AMD64 */
4116	return cErrors;
4117
4118	#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT */
4119	}
4120	#endif /* VBOX_STRICT */
4121
4122
4123	/**
4124	* Sets up the CR3 for shadow paging
4125	*
4126	* @returns Strict VBox status code.
4127	* @retval VINF_SUCCESS.
4128	*
4129	* @param pVCpu The VMCPU handle.
4130	* @param GCPhysCR3 The physical address in the CR3 register.
4131	*/
4132	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3)
4133	{
4134	PVM pVM = pVCpu->CTX_SUFF(pVM);
4135
4136	/* Update guest paging info. */
4137	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
4138	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
4139	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
4140
4141	LogFlow(("MapCR3: %RGp\n", GCPhysCR3));
4142
4143	/*
4144	* Map the page CR3 points at.
4145	*/
4146	RTHCPTR HCPtrGuestCR3;
4147	RTHCPHYS HCPhysGuestCR3;
4148	pgmLock(pVM);
4149	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysCR3);
4150	AssertReturn(pPage, VERR_INTERNAL_ERROR_2);
4151	HCPhysGuestCR3 = PGM_PAGE_GET_HCPHYS(pPage);
4152	/** @todo this needs some reworking wrt. locking. */
4153	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4154	HCPtrGuestCR3 = NIL_RTHCPTR;
4155	int rc = VINF_SUCCESS;
4156	# else
4157	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhysCR3 & GST_CR3_PAGE_MASK, (void **)&HCPtrGuestCR3);
4158	# endif
4159	pgmUnlock(pVM);
4160	if (RT_SUCCESS(rc))
4161	{
4162	rc = PGMMap(pVM, (RTGCPTR)pVM->pgm.s.GCPtrCR3Mapping, HCPhysGuestCR3, PAGE_SIZE, 0);
4163	if (RT_SUCCESS(rc))
4164	{
4165	# ifdef IN_RC
4166	PGM_INVL_PG(pVM->pgm.s.GCPtrCR3Mapping);
4167	# endif
4168	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4169	pVCpu->pgm.s.pGst32BitPdR3 = (R3PTRTYPE(PX86PD))HCPtrGuestCR3;
4170	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4171	pVCpu->pgm.s.pGst32BitPdR0 = (R0PTRTYPE(PX86PD))HCPtrGuestCR3;
4172	# endif
4173	pVCpu->pgm.s.pGst32BitPdRC = (RCPTRTYPE(PX86PD))pVM->pgm.s.GCPtrCR3Mapping;
4174
4175	# elif PGM_GST_TYPE == PGM_TYPE_PAE
4176	unsigned off = GCPhysCR3 & GST_CR3_PAGE_MASK & PAGE_OFFSET_MASK;
4177	pVCpu->pgm.s.pGstPaePdptR3 = (R3PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4178	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4179	pVCpu->pgm.s.pGstPaePdptR0 = (R0PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4180	# endif
4181	pVCpu->pgm.s.pGstPaePdptRC = (RCPTRTYPE(PX86PDPT))((RCPTRTYPE(uint8_t *))pVM->pgm.s.GCPtrCR3Mapping + off);
4182	Log(("Cached mapping %RRv\n", pVCpu->pgm.s.pGstPaePdptRC));
4183
4184	/*
4185	* Map the 4 PDs too.
4186	*/
4187	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(&pVCpu->pgm.s);
4188	RTGCPTR GCPtr = pVM->pgm.s.GCPtrCR3Mapping + PAGE_SIZE;
4189	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++, GCPtr += PAGE_SIZE)
4190	{
4191	if (pGuestPDPT->a[i].n.u1Present)
4192	{
4193	RTHCPTR HCPtr;
4194	RTHCPHYS HCPhys;
4195	RTGCPHYS GCPhys = pGuestPDPT->a[i].u & X86_PDPE_PG_MASK;
4196	pgmLock(pVM);
4197	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
4198	AssertReturn(pPage, VERR_INTERNAL_ERROR_2);
4199	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
4200	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4201	HCPtr = NIL_RTHCPTR;
4202	int rc2 = VINF_SUCCESS;
4203	# else
4204	int rc2 = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, (void **)&HCPtr);
4205	# endif
4206	pgmUnlock(pVM);
4207	if (RT_SUCCESS(rc2))
4208	{
4209	rc = PGMMap(pVM, GCPtr, HCPhys, PAGE_SIZE, 0);
4210	AssertRCReturn(rc, rc);
4211
4212	pVCpu->pgm.s.apGstPaePDsR3[i] = (R3PTRTYPE(PX86PDPAE))HCPtr;
4213	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4214	pVCpu->pgm.s.apGstPaePDsR0[i] = (R0PTRTYPE(PX86PDPAE))HCPtr;
4215	# endif
4216	pVCpu->pgm.s.apGstPaePDsRC[i] = (RCPTRTYPE(PX86PDPAE))GCPtr;
4217	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
4218	PGM_INVL_PG(GCPtr); /** @todo This ends up calling HWACCMInvalidatePage, is that correct? */
4219	continue;
4220	}
4221	AssertMsgFailed(("pgmR3Gst32BitMapCR3: rc2=%d GCPhys=%RGp i=%d\n", rc2, GCPhys, i));
4222	}
4223
4224	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4225	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4226	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4227	# endif
4228	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4229	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4230	PGM_INVL_PG(GCPtr); /** @todo this shouldn't be necessary? */
4231	}
4232
4233	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
4234	pVCpu->pgm.s.pGstAmd64Pml4R3 = (R3PTRTYPE(PX86PML4))HCPtrGuestCR3;
4235	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4236	pVCpu->pgm.s.pGstAmd64Pml4R0 = (R0PTRTYPE(PX86PML4))HCPtrGuestCR3;
4237	# endif
4238	# endif
4239	}
4240	else
4241	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4242	}
4243	else
4244	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4245
4246	#else /* prot/real stub */
4247	int rc = VINF_SUCCESS;
4248	#endif
4249
4250	/* Update shadow paging info for guest modes with paging (32, pae, 64). */
4251	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4252	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4253	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64) \
4254	&& ( PGM_GST_TYPE != PGM_TYPE_REAL \
4255	&& PGM_GST_TYPE != PGM_TYPE_PROT))
4256
4257	Assert(!HWACCMIsNestedPagingActive(pVM));
4258
4259	/*
4260	* Update the shadow root page as well since that's not fixed.
4261	*/
4262	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4263	PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
4264	uint32_t iOldShwUserTable = pVCpu->pgm.s.iShwUserTable;
4265	uint32_t iOldShwUser = pVCpu->pgm.s.iShwUser;
4266	PPGMPOOLPAGE pNewShwPageCR3;
4267
4268	Assert(!(GCPhysCR3 >> (PAGE_SHIFT + 32)));
4269	rc = pgmPoolAlloc(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, SHW_POOL_ROOT_IDX, GCPhysCR3 >> PAGE_SHIFT, &pNewShwPageCR3);
4270	AssertFatalRC(rc);
4271	rc = VINF_SUCCESS;
4272
4273	/* Mark the page as locked; disallow flushing. */
4274	pgmPoolLockPage(pPool, pNewShwPageCR3);
4275
4276	# ifdef IN_RC
4277	/* NOTE: We can't deal with jumps to ring 3 here as we're now in an inconsistent state! */
4278	bool fLog = VMMGCLogDisable(pVM);
4279	pgmLock(pVM);
4280	# endif
4281
4282	pVCpu->pgm.s.iShwUser = SHW_POOL_ROOT_IDX;
4283	pVCpu->pgm.s.iShwUserTable = GCPhysCR3 >> PAGE_SHIFT;
4284	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3) = pNewShwPageCR3;
4285	# ifdef IN_RING0
4286	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4287	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4288	# elif defined(IN_RC)
4289	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4290	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4291	# else
4292	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4293	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4294	# endif
4295
4296	# ifndef PGM_WITHOUT_MAPPINGS
4297	/*
4298	* Apply all hypervisor mappings to the new CR3.
4299	* Note that SyncCR3 will be executed in case CR3 is changed in a guest paging mode; this will
4300	* make sure we check for conflicts in the new CR3 root.
4301	*/
4302	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4303	Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL) \|\| VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
4304	# endif
4305	rc = pgmMapActivateCR3(pVM, pNewShwPageCR3);
4306	AssertRCReturn(rc, rc);
4307	# endif
4308
4309	/* Set the current hypervisor CR3. */
4310	CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
4311	SELMShadowCR3Changed(pVM, pVCpu);
4312
4313	# ifdef IN_RC
4314	pgmUnlock(pVM);
4315	VMMGCLogRestore(pVM, fLog);
4316	# endif
4317
4318	/* Clean up the old CR3 root. */
4319	if (pOldShwPageCR3)
4320	{
4321	Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
4322	# ifndef PGM_WITHOUT_MAPPINGS
4323	/* Remove the hypervisor mappings from the shadow page table. */
4324	pgmMapDeactivateCR3(pVM, pOldShwPageCR3);
4325	# endif
4326	/* Mark the page as unlocked; allow flushing again. */
4327	pgmPoolUnlockPage(pPool, pOldShwPageCR3);
4328
4329	pgmPoolFreeByPage(pPool, pOldShwPageCR3, iOldShwUser, iOldShwUserTable);
4330	}
4331
4332	# endif
4333
4334	return rc;
4335	}
4336
4337	/**
4338	* Unmaps the shadow CR3.
4339	*
4340	* @returns VBox status, no specials.
4341	* @param pVCpu The VMCPU handle.
4342	*/
4343	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu)
4344	{
4345	LogFlow(("UnmapCR3\n"));
4346
4347	int rc = VINF_SUCCESS;
4348	PVM pVM = pVCpu->CTX_SUFF(pVM);
4349
4350	/*
4351	* Update guest paging info.
4352	*/
4353	#if PGM_GST_TYPE == PGM_TYPE_32BIT
4354	pVCpu->pgm.s.pGst32BitPdR3 = 0;
4355	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4356	pVCpu->pgm.s.pGst32BitPdR0 = 0;
4357	# endif
4358	pVCpu->pgm.s.pGst32BitPdRC = 0;
4359
4360	#elif PGM_GST_TYPE == PGM_TYPE_PAE
4361	pVCpu->pgm.s.pGstPaePdptR3 = 0;
4362	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4363	pVCpu->pgm.s.pGstPaePdptR0 = 0;
4364	# endif
4365	pVCpu->pgm.s.pGstPaePdptRC = 0;
4366	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4367	{
4368	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4369	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4370	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4371	# endif
4372	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4373	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4374	}
4375
4376	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
4377	pVCpu->pgm.s.pGstAmd64Pml4R3 = 0;
4378	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4379	pVCpu->pgm.s.pGstAmd64Pml4R0 = 0;
4380	# endif
4381
4382	#else /* prot/real mode stub */
4383	/* nothing to do */
4384	#endif
4385
4386	#if !defined(IN_RC) /* In RC we rely on MapCR3 to do the shadow part for us at a safe time */
4387	/*
4388	* Update shadow paging info.
4389	*/
4390	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4391	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4392	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64))
4393
4394	# if PGM_GST_TYPE != PGM_TYPE_REAL
4395	Assert(!HWACCMIsNestedPagingActive(pVM));
4396	# endif
4397
4398	# ifndef PGM_WITHOUT_MAPPINGS
4399	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4400	/* Remove the hypervisor mappings from the shadow page table. */
4401	pgmMapDeactivateCR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4402	# endif
4403
4404	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4405	{
4406	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4407
4408	Assert(pVCpu->pgm.s.iShwUser != PGMPOOL_IDX_NESTED_ROOT);
4409
4410	/* Mark the page as unlocked; allow flushing again. */
4411	pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4412
4413	pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), pVCpu->pgm.s.iShwUser, pVCpu->pgm.s.iShwUserTable);
4414	pVCpu->pgm.s.pShwPageCR3R3 = 0;
4415	pVCpu->pgm.s.pShwPageCR3R0 = 0;
4416	pVCpu->pgm.s.pShwPageCR3RC = 0;
4417	pVCpu->pgm.s.iShwUser = 0;
4418	pVCpu->pgm.s.iShwUserTable = 0;
4419	}
4420	# endif
4421	#endif /* !IN_RC*/
4422
4423	return rc;
4424	}
4425

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 19668

Download in other formats: